ruby-protocol-buffers 0.8.4

data/lib/protocol_buffers/compiler/file_descriptor_to_d.rb

@@ -0,0 +1,195 @@
+require 'protocol_buffers/compiler/descriptor.pb'
+require 'stringio'
+
+class FileDescriptorToD < Struct.new(:descriptor)
+
+  include FieldDescriptorProto::Type
+  include FieldDescriptorProto::Label
+
+  def initialize(descriptor)
+    super
+    @module = descriptor.package_
+    @ns = []
+  end
+
+  def module_name
+    @module
+  end
+
+  def class_name(klass)
+    klass
+  end
+
+  def write(io)
+    @io = io
+
+    @io.write <<HEADER
+// Generated by the protocol buffer compiler. DO NOT EDIT!
+
+static import std.string;
+import protocol_buffers.message;
+
+HEADER
+
+    descriptor.message_type.each do |message|
+      dump_message(message)
+    end
+  end
+
+  protected
+
+  def dump_message(message)
+    in_namespace("final class", class_name(message.name), " : Message") do
+
+      line %{static const #{class_name(message.name)} default_instance;}
+      line %{string typeName() { return "#{class_name(message.name)}"; }}
+
+      line %{// nested messages} unless message.nested_type.empty?
+      message.nested_type.each { |inner| dump_message(inner) }
+
+      line %{// nested enums} unless message.enum_type.empty?
+      message.enum_type.each { |inner| dump_enum(inner) }
+
+      singular_fields = message.field.find_all { |f|
+        f.label == LABEL_OPTIONAL || f.label == LABEL_REQUIRED }
+      repeated_fields = message.field.find_all { |f| f.label == LABEL_REPEATED }
+
+      line
+
+      singular_fields.each { |f| define_field(f, false) }
+      repeated_fields.each { |f| define_field(f, true) }
+
+      line %{enum __tags { #{message.field.map { |f| "#{f.name} = #{f.number}" }.join(", ")} }}
+
+      unless singular_fields.empty?
+        in_namespace("private", "") do
+          line "ubyte[(#{singular_fields.size}/32)+1] _tags;"
+          line %{void _set_bit(uint idx)   { _tags[idx / 32] |= (1u << (idx % 32)); }}
+          line %{void _clear_bit(uint idx) { _tags[idx / 32] &= ~(1u << (idx % 32)); }}
+          line %{bool _check_bit(uint idx) { return (_tags[idx / 32] & (1u << (idx % 32))) != 0; }}
+        end
+        line %{enum __bits { #{m=0; singular_fields.map { |f| "#{f.name} = #{m+=1}" }.join(", ")} }} unless singular_fields.empty?
+      end
+
+      in_namespace("void clear()", "") do
+        message.field.each { |f| line %{clear_#{f.name}();} }
+      end
+
+      in_namespace("bool mergeFromString(string s)", "") do
+        body = <<-BODY
+        uint i;
+        while(i < s.length) {
+          long nextTag = decodeVarint(s[i++]);
+          long tag = nextTag >> 3;
+          uint wire_type = nextTag & 0b111;
+          switch(tag) {
+          }
+        }
+        return true;
+        BODY
+
+        body.each_line { |l| line(l.sub("        ", '').chomp) }
+      end
+
+      line
+    end
+  end
+
+  def field_typename(field)
+    (TYPE_MAPPING[field.type] || field.type_name).sub(/^.#{module_name}/, '')
+  end
+
+  def define_field(field, repeated)
+    typename = field_typename(field)
+    if repeated
+      line "private #{typename}[] _#{field.name};"
+      line "#{typename}[] #{field.name}() { return _#{field.name}; }"
+    else
+      line "private #{typename} _#{field.name};"
+      line "#{typename} #{field.name}() { return _#{field.name}; }"
+    end
+
+    if !repeated
+      line "static const #{typename} default_#{field.name} = #{default_value(field)};"
+      line "bool has_#{field.name}() { return _check_bit(__bits.#{field.name}); }"
+      line "void #{field.name}(#{typename} val) { _set_bit(__bits.#{field.name}); _#{field.name} = val; }"
+    end
+
+    if repeated
+      line "void clear_#{field.name}() { _#{field.name} = []; }"
+    else
+      line "void clear_#{field.name}() { _clear_bit(__bits.#{field.name}); _#{field.name} = default_#{field.name}; }"
+    end
+    line
+  end
+
+  def dump_enum(enum)
+    in_namespace("enum", enum.name) do
+      enum.value.each do |value|
+        line %{#{value.name} = #{value.number},}
+      end
+    end
+  end
+
+  TYPE_MAPPING = {
+    TYPE_DOUBLE => "double",
+    TYPE_FLOAT => "float",
+    TYPE_INT64 => "long",
+    TYPE_UINT64 => "ulong",
+    TYPE_INT32 => "int",
+    TYPE_FIXED64 => "long",
+    TYPE_FIXED32 => "int",
+    TYPE_BOOL => "bool",
+    TYPE_STRING => "string",
+    TYPE_BYTES => "ubyte[]",
+    TYPE_UINT32 => "uint",
+    TYPE_SFIXED32 => "int",
+    TYPE_SFIXED64 => "long",
+    TYPE_SINT32 => "int",
+    TYPE_SINT64 => "long",
+  }
+
+  def default_value(field)
+    typename = field_typename(field)
+
+    if field.default_value && field.default_value != ""
+      case field.type
+      when TYPE_STRING, TYPE_BYTES
+        %{"#{field.default_value}"}
+      when TYPE_BOOL
+        field.default_value || "false"
+      when TYPE_ENUM
+        %{#{typename}.#{field.default_value}}
+      else
+        field.default_value || "0"
+      end
+    else
+        %{(#{typename}).init}
+    end
+  end
+
+  def in_namespace(type, name, rest = "")
+    if !name
+      yield
+    else
+      line "#{type} #{capfirst(name)}#{rest} {"
+      @ns.push name
+      yield
+      @ns.pop
+      line "}"
+    end
+  end
+
+  def capfirst(s)
+    "#{s[0,1].capitalize}#{s[1..-1]}" if s
+  end
+
+  def line(str = nil)
+    if str
+      @ns.size.times { @io.write("  ") }
+      @io.write(str)
+    end
+    @io.write("\n")
+  end
+
+end

data/lib/protocol_buffers/compiler/file_descriptor_to_ruby.rb

@@ -0,0 +1,173 @@
+require 'protocol_buffers/compiler/descriptor.pb'
+require 'stringio'
+
+class FileDescriptorToRuby < Struct.new(:descriptor)
+
+  include FieldDescriptorProto::Type
+  include FieldDescriptorProto::Label
+
+  def initialize(descriptor)
+    super
+    @package = capfirst(descriptor.package_)
+    @ns = []
+  end
+
+  def write(io)
+    @io = io
+
+    @io.write <<HEADER
+#!/usr/bin/env ruby
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+
+require 'protocol_buffers'
+
+HEADER
+    descriptor.dependency.each do |dep|
+      path = File.basename(dep, ".proto") + ".pb"
+      @io.write("begin; require '#{path}'; rescue LoadError; end\n")
+    end
+    @io.write("\n") unless descriptor.dependency.empty?
+
+    # in_namespace correctly handles the case where @package.nil?
+    unless @package.empty?
+      @io.write("# Reload support\nObject.__send__(:remove_const, :#{@package}) if defined?(#{@package})\n\n")
+    end
+
+    in_namespace("module", @package) do
+      declare(descriptor.message_type, descriptor.enum_type)
+
+      descriptor.message_type.each do |message|
+        dump_message(message)
+      end
+
+      descriptor.enum_type.each do |enum|
+        dump_enum(enum)
+      end
+    end
+
+  end
+
+  protected
+
+  def declare(messages, enums)
+    return if messages.empty? && enums.empty?
+
+    line %{# forward declarations}
+    messages.each do |message|
+      line %{class #{name([@package, message.name])} < ::ProtocolBuffers::Message; end}
+    end
+    enums.each do |enum|
+      line %{module #{name([@package, enum.name])}; end}
+    end
+    line
+  end
+
+  def line(str = nil)
+    if str
+      @ns.size.times { @io.write("  ") }
+      @io.write(str)
+    end
+    @io.write("\n")
+  end
+
+  def in_namespace(type, name, rest = "")
+    if !name || name == ""
+      yield
+    else
+      line "#{type} #{capfirst(name)}#{rest}"
+      @ns.push name
+      yield
+      @ns.pop
+      line "end"
+    end
+  end
+
+  def name(parts)
+    ns = @ns.dup
+    (parts.shift; ns.shift) while !parts.empty? && parts.first == ns.first
+    parts.map { |p| capfirst(p) }.join("::")
+  end
+
+  LABEL_MAPPING = {
+    LABEL_OPTIONAL => "optional",
+    LABEL_REQUIRED => "required",
+    LABEL_REPEATED => "repeated",
+  }
+
+  TYPE_MAPPING = {
+    TYPE_DOUBLE => ":double",
+    TYPE_FLOAT => ":float",
+    TYPE_INT64 => ":int64",
+    TYPE_UINT64 => ":uint64",
+    TYPE_INT32 => ":int32",
+    TYPE_FIXED64 => ":fixed64",
+    TYPE_FIXED32 => ":fixed32",
+    TYPE_BOOL => ":bool",
+    TYPE_STRING => ":string",
+    TYPE_BYTES => ":bytes",
+    TYPE_UINT32 => ":uint32",
+    TYPE_SFIXED32 => ":sfixed32",
+    TYPE_SFIXED64 => ":sfixed64",
+    TYPE_SINT32 => ":sint32",
+    TYPE_SINT64 => ":sint64",
+  }
+
+  def dump_message(message)
+    in_namespace("class", message.name, " < ::ProtocolBuffers::Message") do
+      declare(message.nested_type, message.enum_type)
+
+      line %{# nested messages} unless message.nested_type.empty?
+      message.nested_type.each { |inner| dump_message(inner) }
+
+      line %{# nested enums} unless message.enum_type.empty?
+      message.enum_type.each { |inner| dump_enum(inner) }
+
+      message.field.each do |field|
+        typename = field_typename(field)
+        fieldline = %{#{LABEL_MAPPING[field.label]} #{typename}, :#{field.name}, #{field.number}}
+        if field.default_value && field.default_value != ""
+          fieldline << %{, :default => #{default_value(field)}}
+        end
+        line fieldline
+      end
+
+      line
+      line "gen_methods! # new fields ignored after this point"
+    end
+    line
+  end
+
+  def dump_enum(enum)
+    in_namespace("module", enum.name) do
+      line %{include ::ProtocolBuffers::Enum}
+      enum.value.each do |value|
+        line %{#{capfirst(value.name)} = #{value.number}}
+      end
+    end
+    line
+  end
+
+  def field_typename(field)
+    TYPE_MAPPING[field.type] || field.type_name.split(".").map { |t| capfirst(t) }.join("::")
+  end
+
+  # TODO: this probably doesn't work for all default values, expand
+  def default_value(field)
+    case field.type
+    when TYPE_STRING, TYPE_BYTES
+      %{"#{field.default_value}"}
+    when TYPE_BOOL
+      field.default_value
+    when TYPE_ENUM
+      typename = field_typename(field)
+      %{#{typename}::#{field.default_value}}
+    else
+      field.default_value
+    end
+  end
+
+  def capfirst(s)
+    "#{s[0,1].capitalize}#{s[1..-1]}" if s
+  end
+
+end

data/lib/protocol_buffers/limited_io.rb

@@ -0,0 +1,33 @@
+class LimitedIO < Struct.new(:parent, :limit)
+
+  def read(length = nil, buffer = nil)
+    length = length || limit
+    length = limit if length > limit
+    self.limit -= length
+    # seems silly to check for buffer, but some IO#read methods implemented in C
+    # barf if you pass nil, rather than treat it as an argument that wasn't
+    # passed at all.
+    if buffer
+      parent.read(length, buffer)
+    else
+      parent.read(length)
+    end
+  end
+
+  def eof?
+    limit == 0 || parent.eof?
+  end
+
+  def getbyte
+    return nil if limit == 0
+    self.limit -= 1
+    parent.getbyte
+  end
+
+  def getc
+    return nil if limit == 0
+    self.limit -= 1
+    parent.getc
+  end
+
+end

data/lib/protocol_buffers/runtime/decoder.rb

@@ -0,0 +1,65 @@
+require 'protocol_buffers/limited_io'
+
+module ProtocolBuffers
+
+  class DecodeError < StandardError; end
+
+  module Decoder # :nodoc: all
+    def self.decode(io, message)
+      fields = message.fields
+
+      until io.eof?
+        tag_int = Varint.decode(io)
+        tag = tag_int >> 3
+        wire_type = tag_int & 0b111
+        field = fields[tag]
+
+        if field && wire_type != field.wire_type
+          raise(DecodeError, "incorrect wire type for tag: #{field.tag}")
+        end
+
+        # replacing const lookups with hard-coded ints removed an entire 10%
+        # from an earlier decoding benchmark. these values can't change without
+        # breaking the protocol anyway, so we decided it was worth it.
+        case wire_type
+        when 0 # VARINT
+          value = Varint.decode(io)
+        when 1 # FIXED64
+          value = io.read(8)
+        when 2 # LENGTH_DELIMITED
+          length = Varint.decode(io)
+          value = LimitedIO.new(io, length)
+        when 5 # FIXED32
+          value = io.read(4)
+        when 3, 4 # deprecated START_GROUP/END_GROUP types
+          raise(DecodeError, "groups are deprecated and unsupported")
+        else
+          raise(DecodeError, "unknown wire type: #{wire_type}")
+        end
+
+        if field
+          deserialized = field.deserialize(value)
+          # merge_field handles repeated field logic
+          message.merge_field(tag, deserialized, field)
+        else
+          # ignore unknown fields, pass them on when re-serializing this message
+
+          # special handling -- if it's a LENGTH_DELIMITED field, we need to
+          # actually read the IO so that extra bytes aren't left on the wire
+          value = value.read if wire_type == 2 # LENGTH_DELIMITED
+
+          message.remember_unknown_field(tag_int, value)
+        end
+      end
+
+      unless message.valid?
+        raise(DecodeError, "invalid message")
+      end
+
+      return message
+    rescue TypeError, ArgumentError
+      raise(DecodeError, "error parsing message")
+    end
+  end
+
+end