ruby-protocol-buffers 1.3.0 → 1.3.1

data/lib/protocol_buffers/compiler/file_descriptor_to_ruby.rb

@@ -8,7 +8,7 @@ class FileDescriptorToRuby < Struct.new(:descriptor)
 
   def initialize(descriptor)
     super
-    @package_modules = descriptor.package_ ? descriptor.package_.split('.') : []
+    @package_modules = descriptor.package ? descriptor.package.split('.') : []
     @ns = []
   end
 
@@ -126,6 +126,12 @@ HEADER
         if field.default_value && field.default_value != ""
           fieldline << %{, :default => #{default_value(field)}}
         end
+
+        # Dont need to check for 'repeated' attribute or type, protoc will take care of this.
+        if field.options.packed
+          fieldline << %{, :packed => true }
+        end
+
         line fieldline
       end
     end

data/lib/protocol_buffers/runtime/decoder.rb

@@ -14,7 +14,7 @@ module ProtocolBuffers
         wire_type = tag_int & 0b111
         field = fields[tag]
 
-        if field && wire_type != field.wire_type
+        if field && ( !( field.packed? || wire_type == field.wire_type ) || ( field.packed? && wire_type != 2 ) )
           raise(DecodeError, "incorrect wire type for tag: #{field.tag}, expected #{field.wire_type} but got #{wire_type}\n#{field.inspect}")
         end
 
@@ -39,7 +39,23 @@ module ProtocolBuffers
 
         if field
           begin
-            deserialized = field.deserialize(value)
+            if field.packed?
+              deserialized = []
+              until value.eof?
+
+                decoded = case field.wire_type
+                  when 0 # VARINT
+                    Varint.decode(value)
+                  when 1 # FIXED64
+                    value.read(8)
+                  when 5 # FIXED32
+                    value.read(4)
+                  end
+                deserialized << field.deserialize(decoded)
+              end
+            else
+              deserialized = field.deserialize(value)
+            end
             # merge_field handles repeated field logic
             message.merge_field(tag, deserialized, field)
           rescue ArgumentError

data/lib/protocol_buffers/runtime/encoder.rb

@@ -20,8 +20,20 @@ module ProtocolBuffers
         tag = (field.tag << 3) | wire_type
 
         if field.repeated?
-          value.each { |i| serialize_field(io, tag, wire_type,
-                                           field.serialize(i)) }
+          next if value.size == 0
+
+          if field.packed?
+            # encode packed field in a LENGTH_DELIMITED wire
+            wire_type = 2
+            tag = (field.tag << 3) | wire_type
+            buf = StringIO.new
+            value.each { |i| serialize_field_value(buf, field.wire_type, field.serialize(i)) }
+            Varint.encode(io, tag)
+            Varint.encode(io, buf.size)
+            io.write(buf.string)
+          else
+            value.each { |i| serialize_field(io, tag, wire_type, field.serialize(i)) }
+          end
         else
           serialize_field(io, tag, wire_type, field.serialize(value))
         end
@@ -36,7 +48,10 @@ module ProtocolBuffers
     def self.serialize_field(io, tag, wire_type, serialized)
       # write the tag
       Varint.encode(io, tag)
+      self.serialize_field_value(io, wire_type, serialized)
+    end
 
+    def self.serialize_field_value(io, wire_type, serialized)
       # see comment in decoder.rb about magic numbers
       case wire_type
       when 0 # VARINT

data/lib/protocol_buffers/runtime/field.rb

@@ -97,6 +97,7 @@ module ProtocolBuffers
     attr_reader :otype, :name, :tag
 
     def repeated?; otype == :repeated end
+    def packed?; repeated? && @opts[:packed] end
 
     def self.create(sender, otype, type, name, tag, opts = {})
       if type.is_a?(Symbol)
@@ -294,16 +295,26 @@ module ProtocolBuffers
 
       def check_value(value)
         if HAS_ENCODING
-          value.valid_encoding? || raise(ArgumentError, "string value is not valid utf-8")
+          value.dup.force_encoding(Encoding::UTF_8).valid_encoding? || raise(ArgumentError, "string value is not valid utf-8")
+        end
+      end
+
+      def serialize(value)
+        check_value(value)
+        if HAS_ENCODING
+          value.dup.force_encoding(Encoding::UTF_8)
+        else
+          value
         end
       end
 
       def deserialize(value)
         read_value = value.read.to_s
         if HAS_ENCODING
-          read_value.force_encoding("utf-8")
+          read_value.force_encoding(Encoding::UTF_8)
+        else
+          read_value
         end
-        read_value
       end
     end
 

data/lib/protocol_buffers/runtime/varint.rb

@@ -1,41 +1,41 @@
 module ProtocolBuffers
 
-  module Varint # :nodoc:
-    # encode/decode methods defined in ext/varint.c
-
-    unless self.respond_to?(:encode)
-      def self.encode(io, int_val)
-        if int_val < 0
-          # negative varints are always encoded with the full 10 bytes
-          int_val = int_val & 0xffffffff_ffffffff
-        end
-        loop do
-          byte = int_val & 0b0111_1111
-          int_val >>= 7
-          if int_val == 0
-            io << byte.chr
-            break
-          else
-            io << (byte | 0b1000_0000).chr
-          end
+  module VarintPure # :nodoc:
+    def encode(io, int_val)
+      if int_val < 0
+        # negative varints are always encoded with the full 10 bytes
+        int_val = int_val & 0xffffffff_ffffffff
+      end
+      loop do
+        byte = int_val & 0b0111_1111
+        int_val >>= 7
+        if int_val == 0
+          io << byte.chr
+          break
+        else
+          io << (byte | 0b1000_0000).chr
         end
       end
     end
 
-    unless self.respond_to?(:decode)
-      def self.decode(io)
-        int_val = 0
-        shift = 0
-        loop do
-          raise(DecodeError, "too many bytes when decoding varint") if shift >= 64
-          byte = io.getc.ord
-          int_val |= (byte & 0b0111_1111) << shift
-          shift += 7
-          return int_val if (byte & 0b1000_0000) == 0
-        end
+    def decode(io)
+      int_val = 0
+      shift = 0
+      loop do
+        raise(DecodeError, "too many bytes when decoding varint") if shift >= 64
+        # this works in 1.8.7 and 1.9.x, but for different reasons
+        # in 1.8.7 io.getc returns the int representation of the char, and .ord is a noop. String#ord doesn't exist.
+        # in 1.9.3 io.getc returns a String, and .ord returns the int representation of the first char.
+        # this fails in 1.8.6, which is no longer supported by ruby-protocol-buffers.
+        byte = io.getc.ord
+        int_val |= (byte & 0b0111_1111) << shift
+        shift += 7
+        return int_val if (byte & 0b1000_0000) == 0
       end
     end
+  end
 
+  class Varint
     def self.encodeZigZag32(int_val)
       (int_val << 1) ^ (int_val >> 31)
     end
@@ -47,16 +47,22 @@ module ProtocolBuffers
     def self.decodeZigZag32(int_val)
       (int_val >> 1) ^ -(int_val & 1)
     end
-    class << self; alias_method :decodeZigZag64, :decodeZigZag32 end
 
+    class << self
+      alias_method :decodeZigZag64, :decodeZigZag32
+    end
   end
 
 end
 
-# fix for 1.8 <-> 1.9 compat
-unless 'a'.respond_to?(:ord)
-  class String; def ord; self[0]; end; end
-end
-unless (1).respond_to?(:ord)
-  class Fixnum; def ord; self; end; end
+# optionally load C extension
+begin
+  require 'varint/varint'
+  class ProtocolBuffers::Varint
+    extend ::Varint
+  end
+rescue LoadError
+  class ProtocolBuffers::Varint
+    extend ::ProtocolBuffers::VarintPure
+  end
 end

data/lib/protocol_buffers/version.rb

@@ -0,0 +1,3 @@
+module ProtocolBuffers
+  VERSION = "1.3.1"
+end

data/ruby-protocol-buffers.gemspec

@@ -1,27 +1,28 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "protocol_buffers"
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "protocol_buffers/version"
 
-Gem::Specification.new do |s|
-  s.name        = "ruby-protocol-buffers"
-  s.version     = ProtocolBuffers::VERSION
-  s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Brian Palmer", "Rob Marable", "Paulo Luis Franchini Casaretto"]
-  s.email       = ["brian@codekitchen.net"]
-  s.homepage    = "https://github.com/mozy/ruby-protocol-buffers"
-  s.summary     = %{Ruby compiler and runtime for the google protocol buffers library.}
+Gem::Specification.new do |gem|
+  gem.name          = "ruby-protocol-buffers"
+  gem.version       = ProtocolBuffers::VERSION
+  gem.authors       = ["Brian Palmer", "Benedikt Böhm", "Rob Marable", "Paulo Luis Franchini Casaretto"]
+  gem.email         = ["brian@codekitchen.net", "bb@xnull.de"]
+  gem.summary       = %{Ruby compiler and runtime for the google protocol buffers library.}
+  gem.homepage      = "https://github.com/mozy/ruby-protocol-buffers"
 
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ["lib"]
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
 
-  s.extra_rdoc_files << "Changelog.md"
+  gem.extra_rdoc_files << "Changelog.md"
 
-  s.add_development_dependency "autotest-standalone"
-  s.add_development_dependency "autotest-growl"
-  s.add_development_dependency "rake"
-  s.add_development_dependency "rcov"
-  s.add_development_dependency "rspec", "~> 2.5"
-  s.add_development_dependency "yard"
+  gem.add_development_dependency "autotest-standalone"
+  gem.add_development_dependency "autotest-growl"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rake-compiler"
+  gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "rspec", "~> 2.5"
+  gem.add_development_dependency "yard"
 end

data/spec/compiler_spec.rb

@@ -8,6 +8,10 @@ describe ProtocolBuffers, "compiler" do
 
   test_files = Dir[File.join(File.dirname(__FILE__), "proto_files", "*.proto")]
 
+  before do
+    pending "need protoc installed" unless has_compiler?
+  end
+
   test_files.each do |file|
     it "can compile #{File.basename(file)}" do
       proc do
@@ -16,4 +20,22 @@ describe ProtocolBuffers, "compiler" do
     end
   end
 
+  it "can compile and instantiate a message in a package with under_scores" do
+    Object.send(:remove_const, :UnderScore) if defined?(UnderScore)
+
+    ProtocolBuffers::Compiler.compile_and_load(
+      File.join(File.dirname(__FILE__), "proto_files", "under_score_package.proto"))
+
+    proc do
+      under_test = UnderScore::UnderTest.new
+    end.should_not raise_error()
+  end
+
+  it "should compile and correctly translate" do
+    ProtocolBuffers::Compiler.compile_and_load(
+      File.join(File.dirname(__FILE__), "proto_files", "simple.proto"))
+    ProtocolBuffers::Compiler.compile_and_load(
+      File.join(File.dirname(__FILE__), "proto_files", "featureful.proto"))
+  end
+
 end

data/spec/fields_spec.rb

@@ -1,3 +1,5 @@
+# encoding: binary
+
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 require 'stringio'
@@ -36,25 +38,61 @@ describe ProtocolBuffers, "fields" do
     proc { int64.check_value(int64.deserialize(val2)) }.should_not raise_error
   end
 
-  it "verifies UTF-8 for string fields" do
+  context "UTF-8 encoding of length-delimited fields" do
     if RUBY_VERSION < "1.9"
       pending "UTF-8 validation only happens in ruby 1.9+"
     else
-      good = proc { StringIO.new("hello") }
-      bad  = proc { StringIO.new("\xff\xff") }
-
-      s1 = mkfield(:StringField)
-      s1.deserialize(good[]).should == "hello"
-      s1.deserialize(good[]).encoding.should == Encoding.find('utf-8')
-      proc { s1.check_valid(s1.deserialize(good[])) }.should_not raise_error()
-      s1.deserialize(bad[]).encoding.should == Encoding.find('utf-8')
-      proc { s1.check_valid(s1.deserialize(bad[])) }.should raise_error(ArgumentError)
-
-      b1 = mkfield(:BytesField)
-      b1.deserialize(good[]).should == "hello"
-      b1.deserialize(good[]).encoding.should == Encoding.find("us-ascii")
-      b1.deserialize(bad[]).encoding.should == Encoding.find("binary")
-      proc { b1.check_valid(b1.deserialize(bad[])) }.should_not raise_error()
+
+      before :each do
+        @good_utf   = "\xc2\xa1hola\x21"
+        @bad_utf    = "\xc2"
+        @good_ascii = "!hola!".force_encoding("us-ascii")
+
+        @good_utf_io   = proc { StringIO.new(@good_utf) }
+        @bad_utf_io    = proc { StringIO.new(@bad_utf) }
+        @good_ascii_io = proc { StringIO.new(@good_ascii) }
+
+        @s = mkfield(:StringField)
+        @b = mkfield(:BytesField)
+      end
+
+      context "string fields" do
+
+        it "forces UTF-8 on serializing" do
+          @s.serialize(@good_utf).encoding.should == Encoding::UTF_8
+          proc { @s.check_valid(@s.serialize(@good_utf)) }.should_not raise_error()
+
+          @s.serialize(@good_ascii).encoding.should == Encoding::UTF_8
+          proc { @s.check_valid(@s.serialize(@good_ascii)) }.should_not raise_error()
+
+          proc { @s.serialize(@bad_utf) }.should raise_error(ArgumentError)
+        end
+
+        it "forces UTF-8 on deserializing" do
+          @s.deserialize(@good_utf_io[]).encoding.should == Encoding::UTF_8
+          proc { @s.check_valid(@s.deserialize(@good_utf_io[])) }.should_not raise_error()
+
+          @s.deserialize(@good_ascii_io[]).encoding.should == Encoding::UTF_8
+          proc { @s.check_valid(@s.deserialize(@good_ascii_io[])) }.should_not raise_error()
+
+          @s.deserialize(@bad_utf_io[]).encoding.should == Encoding::UTF_8
+          proc { @s.check_valid(@s.deserialize(@bad_utf_io[])) }.should raise_error(ArgumentError)
+        end
+      end
+
+      context "byte fields" do
+
+        it "does not force UTF-8 on deserializing" do
+          @b.deserialize(@good_utf_io[]).encoding.should == Encoding::BINARY
+          proc { @b.check_valid(@b.deserialize(@good_utf_io[])) }.should_not raise_error()
+
+          @b.deserialize(@good_ascii_io[]).encoding.should == Encoding.find("us-ascii")
+          proc { @b.check_valid(@b.deserialize(@good_ascii_io[])) }.should_not raise_error()
+
+          @b.deserialize(@bad_utf_io[]).encoding.should == Encoding::BINARY
+          proc { @b.check_valid(@b.deserialize(@bad_utf_io[])) }.should_not raise_error()
+        end
+      end
     end
   end
 

data/spec/proto_files/featureful.pb.rb

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+
+require 'protocol_buffers'
+
+module Featureful
+  # forward declarations
+  class A < ::ProtocolBuffers::Message; end
+  class B < ::ProtocolBuffers::Message; end
+  class ABitOfEverything < ::ProtocolBuffers::Message; end
+
+  # enums
+  module MainPayloads
+    include ::ProtocolBuffers::Enum
+    P1 = 2
+    P2 = 3
+    P3 = 4
+  end
+
+  class A < ::ProtocolBuffers::Message
+    # forward declarations
+    class Sub < ::ProtocolBuffers::Message; end
+
+    # nested messages
+    class Sub < ::ProtocolBuffers::Message
+      # forward declarations
+      class SubSub < ::ProtocolBuffers::Message; end
+
+      # enums
+      module Payloads
+        include ::ProtocolBuffers::Enum
+        P1 = 0
+        P2 = 1
+      end
+
+      # nested messages
+      class SubSub < ::ProtocolBuffers::Message
+        optional :string, :subsub_payload, 1
+      end
+
+      optional :string, :payload, 1
+      required ::Featureful::A::Sub::Payloads, :payload_type, 2, :default => ::Featureful::A::Sub::Payloads::P1
+      optional ::Featureful::A::Sub::SubSub, :subsub1, 3
+    end
+
+    repeated :int32, :i1, 1
+    optional :int32, :i2, 2
+    required :int32, :i3, 3
+    repeated ::Featureful::A::Sub, :sub1, 4
+    optional ::Featureful::A::Sub, :sub2, 5
+    required ::Featureful::A::Sub, :sub3, 6
+  end
+
+  class B < ::ProtocolBuffers::Message
+    repeated ::Featureful::A, :a, 1
+  end
+
+  class ABitOfEverything < ::ProtocolBuffers::Message
+    optional :double, :double_field, 1
+    optional :float, :float_field, 2
+    optional :int32, :int32_field, 3
+    optional :int64, :int64_field, 4, :default => 15
+    optional :uint32, :uint32_field, 5
+    optional :uint64, :uint64_field, 6
+    optional :sint32, :sint32_field, 7
+    optional :sint64, :sint64_field, 8
+    optional :fixed32, :fixed32_field, 9
+    optional :fixed64, :fixed64_field, 10
+    optional :sfixed32, :sfixed32_field, 11
+    optional :sfixed64, :sfixed64_field, 12
+    optional :bool, :bool_field, 13, :default => false
+    optional :string, :string_field, 14, :default => "zomgkittenz"
+    optional :bytes, :bytes_field, 15
+  end
+
+end