ruby_protobuf 0.3.2 → 0.3.3

data/History.txt

@@ -1,6 +1,11 @@
-=== 0.3.2 / 2008-10-01
+=== 0.3.3 / 2009-07-10
 
-* 0.3.2 fixed a lot of bugs. Thanks code.monkey.steve. (rev.177)
+* 0.3.3 (rev.218)
+  * Support C-like comment.
+  * Optimize for speed.
+  * Bug fixes.
+* 0.3.2 (rev.185)
+  * Bug fixes.
 * 0.3.1 fixed bugs concerning RPC (rev.146)
 * 0.3.0 RPC is available (rev.124)
   * RPC is available

data/Manifest.txt

@@ -0,0 +1,74 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+script/mk_parser
+bin/rprotoc
+examples/addressbook.proto
+examples/addressbook.pb.rb
+examples/reading_a_message.rb
+examples/writing_a_message.rb
+lib/protobuf/common/wire_type.rb
+lib/protobuf/compiler/compiler.rb
+lib/protobuf/compiler/nodes.rb
+lib/protobuf/compiler/proto.y
+lib/protobuf/compiler/proto2.ebnf
+lib/protobuf/compiler/proto_parser.rb
+lib/protobuf/compiler/template/rpc_bin.erb
+lib/protobuf/compiler/template/rpc_client.erb
+lib/protobuf/compiler/template/rpc_service.erb
+lib/protobuf/compiler/visitors.rb
+lib/protobuf/descriptor/descriptor.proto
+lib/protobuf/descriptor/descriptor.rb
+lib/protobuf/descriptor/descriptor_builder.rb
+lib/protobuf/descriptor/descriptor_proto.rb
+lib/protobuf/descriptor/enum_descriptor.rb
+lib/protobuf/descriptor/field_descriptor.rb
+lib/protobuf/descriptor/file_descriptor.rb
+lib/protobuf/message/decoder.rb
+lib/protobuf/message/encoder.rb
+lib/protobuf/message/enum.rb
+lib/protobuf/message/extend.rb
+lib/protobuf/message/field.rb
+lib/protobuf/message/message.rb
+lib/protobuf/message/protoable.rb
+lib/protobuf/message/service.rb
+lib/protobuf/rpc/client.rb
+lib/protobuf/rpc/handler.rb
+lib/protobuf/rpc/server.rb
+lib/ruby_protobuf.rb
+test/addressbook.rb
+test/addressbook_base.rb
+test/addressbook_ext.rb
+test/check_unbuild.rb
+test/collision.rb
+test/data/data.bin
+test/data/data_source.py
+test/data/types.bin
+test/data/types_source.py
+test/data/unk.png
+test/ext_collision.rb
+test/ext_range.rb
+test/merge.rb
+test/nested.rb
+test/proto/addressbook.proto
+test/proto/addressbook_base.proto
+test/proto/addressbook_ext.proto
+test/proto/collision.proto
+test/proto/ext_collision.proto
+test/proto/ext_range.proto
+test/proto/merge.proto
+test/proto/nested.proto
+test/proto/rpc.proto
+test/proto/types.proto
+test/test_addressbook.rb
+test/test_compiler.rb
+test/test_descriptor.rb
+test/test_extension.rb
+test/test_message.rb
+test/test_parse.rb
+test/test_ruby_protobuf.rb
+test/test_serialize.rb
+test/test_standard_message.rb
+test/test_types.rb
+test/types.rb

data/README.txt

@@ -14,7 +14,7 @@ Protocol Buffers for Ruby.
 
 == SYNOPSIS:
 
-  rprotoc test/addressbook.proto
+  rprotoc examples/addressbook.proto
 
 == REQUIREMENTS:
 

data/Rakefile

@@ -5,14 +5,14 @@ require 'rubygems'
 require 'hoe'
 require 'ruby_protobuf'
 
-Hoe.new('ruby_protobuf', RubyProtobuf::VERSION) do |p|
+Hoe.spec('ruby_protobuf') do |p|
+  p.version = RubyProtobuf::VERSION
   p.rubyforge_name = 'ruby-protobuf'
-  p.author = 'ANDO Yasushi'
-  p.email = 'andyjpn@gmail.com'
+  p.developer('MATSUYAMA Kengo', 'macksx@gmail.com')
   p.summary = 'Protocol Buffers for Ruby'
   p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
-  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.url = 'http://code.google.com/p/ruby-protobuf'
   p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
 end
 
-# vim: syntax=Ruby
+# vim: syntax=ruby

data/bin/rprotoc

@@ -1,21 +1,36 @@
 #!/usr/bin/env ruby
-$: << "#{File.dirname(__FILE__)}/../lib"
 require 'optparse'
+if File.exist? "#{File.dirname(__FILE__)}/../lib"
+  $: << "#{File.dirname(__FILE__)}/../lib"
+else
+  require 'rubygems'
+  gem 'ruby_protobuf'
+end
 require 'ruby_protobuf'
-#require 'protobuf/compiler/compiler_old'
 require 'protobuf/compiler/compiler'
 
 
-COMMAND_LINE = "#{$0} #{ARGV.join(' ')} PROTO_FILE"
-OPT = {}
-opts = OptionParser.new
-opts.on('-p', '--proto_path <PATH>', 'Specify the directory in which to search for imports. The current directory is default.'){|v| OPT[:proto_path] = v}
-opts.on('-o', '--out <OUT_DIR>', 'Specify the directory in which Ruby source file is generated. The current directory is default.'){|v| OPT[:out] = v}
+options = {}
+opts = OptionParser.new("#{$0} [OPTIONS] PROTO_FILE")
+opts.on('-p', '--proto_path <PATH>', 'Specify the directory in which to search for imports. The current directory is default.'){|v| options[:proto_path] = v}
+opts.on('-o', '--out <OUT_DIR>', 'Specify the directory in which Ruby source file is generated. The current directory is default.'){|v| options[:out] = v}
 opts.on_tail('-v', '--version', 'Show version.'){puts(opts.ver); exit}
 opts.on_tail('-h', '--help', 'Show this message.'){puts(opts.help); exit}
 
 ::Version = RubyProtobuf::VERSION
-opts.order! ARGV
-PROTO_FILE = ARGV.shift
 
-Protobuf::Compiler.compile(PROTO_FILE, (OPT[:proto_path] or '.'), (OPT[:out] or '.'))
+begin
+  opts.order! ARGV
+rescue OptionParser::ParseError
+  $stderr.puts $!.to_s
+  exit 1
+end
+
+proto_file = ARGV.shift
+
+unless proto_file
+  puts opts
+  exit
+end
+
+Protobuf::Compiler.compile(proto_file, (options[:proto_path] or '.'), (options[:out] or '.'))

data/{test/proto → examples}/addressbook.pb.rb

@@ -1,5 +1,5 @@
 ### Generated by rprotoc. DO NOT EDIT!
-### <proto file: test/proto/addressbook.proto>
+### <proto file: examples/addressbook.proto>
 # package tutorial;
 # 
 # message Person {
@@ -19,13 +19,6 @@
 #   }
 # 
 #   repeated PhoneNumber phone = 4;
-#   optional uint32 age = 5 [default = 20];
-# 
-#   extensions 100 to 200;
-# }
-# 
-# extend Person {
-#   optional int32 age = 100;
 # }
 # 
 # message AddressBook {
@@ -55,12 +48,6 @@ module Tutorial
       optional :PhoneType, :type, 2, :default => :HOME
     end
     repeated :PhoneNumber, :phone, 4
-    optional :uint32, :age, 5, :default => 20
-    extensions 100..200
-  end
-  class Person < ::Protobuf::Message
-    defined_in __FILE__
-    optional :int32, :age, 100, :extension => true
   end
   class AddressBook < ::Protobuf::Message
     defined_in __FILE__

data/examples/addressbook.proto

@@ -0,0 +1,24 @@
+package tutorial;
+
+message Person {
+  required string name = 1;
+  required int32 id = 2;
+  optional string email = 3;
+
+  enum PhoneType {
+    MOBILE = 0;
+    HOME = 1;
+    WORK = 2;
+  }
+
+  message PhoneNumber {
+    required string number = 1;
+    optional PhoneType type = 2 [default = HOME];
+  }
+
+  repeated PhoneNumber phone = 4;
+}
+
+message AddressBook {
+  repeated Person person = 1;
+}

data/examples/reading_a_message.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+require 'addressbook.pb'
+
+def list_people(address_book)
+  address_book.person.each do |person|
+    puts "Person ID: #{person.id}"
+    puts "  Name: #{person.name}"
+    puts "  E-mail: #{person.email}" unless person.email.empty?
+    person.phone.each do |phone_number|
+      print(case phone_number.type
+            when Tutorial::Person::PhoneType::MOBILE
+              '  Mobile phone #: '
+            when Tutorial::Person::PhoneType::HOME
+              '  Home phone #: '
+            when Tutorial::Person::PhoneType::WORK
+              '  Work phone #: '
+            end)
+      puts phone_number.number
+    end
+  end
+end
+
+unless ARGV.size == 1
+  puts "Usage: #{$0} ADDRESS_BOOK_FILE"
+  exit
+end
+
+address_book = Tutorial::AddressBook.new
+address_book.parse_from_file ARGV[0]
+
+list_people address_book

data/examples/writing_a_message.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require 'addressbook.pb'
+
+def prompt_for_address(person)
+  print 'Enter person ID number: '
+  person.id = STDIN.gets.strip.to_i
+  print 'Enter name: '
+  person.name = STDIN.gets.strip
+  print 'Enter email address (blank for none): '
+  email = STDIN.gets.strip
+  person.email = email unless email.empty?
+
+  loop do
+    print 'Enter a phone number (or leave blank to finish): '
+    break if (number = STDIN.gets.strip).empty?
+
+    person.phone << Tutorial::Person::PhoneNumber.new
+    person.phone.last.number = number
+
+    print 'Is this a mobile, home, or work phone? '
+    person.phone.last.type = 
+      case type = STDIN.gets.strip
+      when 'mobile'
+        Tutorial::Person::PhoneType::MOBILE
+      when 'home'
+        Tutorial::Person::PhoneType::HOME
+      when 'work'
+        Tutorial::Person::PhoneType::WORK
+      else
+        puts 'Unknown phone type; leaving as default value.'
+        nil
+      end
+  end
+end
+
+unless ARGV.size == 1
+  puts "Usage: #{$0} ADDRESS_BOOK_FILE" 
+  exit
+end
+
+address_book = Tutorial::AddressBook.new
+address_book.parse_from_file ARGV[0] if File.exist? ARGV[0]
+address_book.person << Tutorial::Person.new
+prompt_for_address address_book.person.last
+address_book.serialize_to_file ARGV[0]

data/lib/protobuf/compiler/compiler.rb

@@ -15,12 +15,7 @@ module Protobuf
     end
 
     def create_message(proto_file, proto_dir='.', out_dir='.', file_create=true)
-      out_dir.sub! %r{/$}, ''
-      proto_dir.sub! %r{/$}, ''
-      rb_file = 
-        if proto_file =~ %r{^/} 
-        then "#{out_dir}/#{proto_file.split('/').last.sub(/\.proto$/, '.pb.rb')}"  
-        else "#{out_dir}/#{proto_file.sub(/\.proto$/, '.pb.rb')}" end
+      rb_file = File.join(out_dir, File.basename(proto_file).sub(/\.[^\0]*\z/, '') + '.pb.rb')
       proto_path = validate_existence proto_file, proto_dir
 
       message_visitor = Protobuf::Visitor::CreateMessageVisitor.new proto_file, proto_dir, out_dir
@@ -31,8 +26,7 @@ module Protobuf
     end
 
     def create_rpc(proto_file, proto_dir='.', out_dir='.', file_create=true)
-      message_file = "#{out_dir}/#{proto_file.sub(/\.proto$/, '.pb.rb')}"
-      out_dir = "#{out_dir}/#{File.dirname proto_file}"
+      message_file = File.join(out_dir, File.basename(proto_file).sub(/\.[^\0]*\z/, '') + '.pb.rb')
       proto_path = validate_existence proto_file, proto_dir
 
       rpc_visitor = Protobuf::Visitor::CreateRpcVisitor.new
@@ -44,11 +38,15 @@ module Protobuf
 
     def validate_existence(path, base_dir)
       if File.exist? path
-      elsif File.exist?(path = "#{base_dir or '.'}/#{path}")
+        path
       else
-        raise ArgumentError.new("File does not exist: #{path}")
+        newpath = File.join(base_dir, path)
+        if File.exist? newpath
+          newpath
+        else
+          raise ArgumentError.new("File does not exist: #{path}")
+        end
       end
-      path
     end
   end
 end

data/lib/protobuf/compiler/nodes.rb

@@ -106,10 +106,14 @@ require 'protobuf/message/extend'
       end
 
       def accept_message_visitor(visitor)
-        visitor.write "class #{@name} < ::Protobuf::Message"
+        class_name = @name.to_s
+        class_name.gsub!(/\A[a-z]/) {|c| c.upcase}
+        visitor.write "class #{class_name} < ::Protobuf::Message"
         visitor.in_context self.class do 
           define_in_the_file visitor
-          @children.each {|child| child.accept_message_visitor visitor}
+          #@children.each {|child| child.accept_message_visitor visitor}
+          @children.each {|child| next if child == ';'; child.accept_message_visitor visitor}
+          # TODO: `next if child == ';';' is a monky patching. There must be a parser error.
         end
         visitor.write "end"
       end
@@ -129,7 +133,8 @@ require 'protobuf/message/extend'
       end
 
       def accept_message_visitor(visitor)
-        visitor.write "class #{@name} < ::Protobuf::Message"
+        name = @name.is_a?(Array) ? @name.join : name.to_s
+        visitor.write "class #{name} < ::Protobuf::Message"
         visitor.in_context self.class do 
           define_in_the_file visitor
           @children.each {|child| child.accept_message_visitor visitor}
@@ -246,7 +251,10 @@ require 'protobuf/message/extend'
         if visitor.context.first == Protobuf::Node::ExtendNode
           opts += ', :extension => true'
         end
-        type = (Array === @type and 1 < @type.size) ?  "'#{@type.join '::'}'" : @type 
+        type = if @type.is_a?(Array)
+               then (@type.size > 1) ? "'#{@type.join '::'}'" : @type[0]
+               else @type
+               end
         visitor.write "#{@label} :#{type}, :#{@name}, #{@value}#{opts}"
       end
 
@@ -254,7 +262,7 @@ require 'protobuf/message/extend'
         descriptor = Google::Protobuf::FieldDescriptorProto.new :name => @name.to_s, :number => @value
         descriptor.label = Google::Protobuf::FieldDescriptorProto::Label.const_get "LABEL_#{@label.to_s.upcase}"
         descriptor.type = Google::Protobuf::FieldDescriptorProto::Type.const_get "TYPE_#{@type.to_s.upcase}" if predefined_type?
-        descriptor.type_name = @type.to_s
+        descriptor.type_name = @type.is_a?(Array) ? @type.join : @type.to_s
         @opts.each do |key, val|
           case key.to_sym
           when :default
@@ -275,7 +283,7 @@ require 'protobuf/message/extend'
       end
 
       def accept_message_visitor(visitor)
-        visitor.write "extensions #{@range.to_s}"
+        visitor.write "extensions #{@range.first.to_s}"
       end
 
       def accept_descriptor_visitor(visitor)
@@ -303,7 +311,7 @@ require 'protobuf/message/extend'
         if @high.nil?
           @low.to_s
         elsif @high == :max
-          "#{@low}..Protobuf::Extend::MAX"
+          "#{@low}..::Protobuf::Extend::MAX"
         else
           "#{@low}..#{@high}"
         end

data/lib/protobuf/compiler/proto.y

@@ -1,5 +1,4 @@
 class Protobuf::ProtoParser
-  #options no_result_var
 rule
   proto : proto_item
           { result = Protobuf::Node::ProtoNode.new val }
@@ -105,11 +104,13 @@ rule
   group : label 'group' CAMEL_IDENT '=' integer_literal message_body
           { result = Protobuf::Node::GroupNode.new val[0], val[2], val[4], val[5] }
 
-  field : label type IDENT '=' integer_literal ';'
+  field : label type field_name '=' integer_literal ';'
           { result = Protobuf::Node::FieldNode.new val[0], val[1], val[2], val[4] }
-        | label type IDENT '=' integer_literal '[' field_option_list ']' ';'
+        | label type field_name '=' integer_literal '[' field_option_list ']' ';'
           { result = Protobuf::Node::FieldNode.new val[0], val[1], val[2], val[4], val[6] }
 
+  field_name : IDENT | "required" | "optional" | "repeated" | "import" | "package" | "option" | "message" | "extend" | "enum" | "service" | "rpc" | "returns" | "group" | "default" | "extensions" | "to" | "max" | "double" | "float" | "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" | "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string" | "bytes"
+
   field_option_list : field_option
                       { result = val }
                     | field_option_list ',' field_option
@@ -160,42 +161,61 @@ end
 
 ---- inner
 
+  require 'strscan'
+
   def parse(f)
-    @q = []
-    f.each do |line|
-      until line.empty? do
-        case line
-        when /\A\s+/, /\A\/\/.*/
-          ;
-        when /\A(required|optional|repeated|import|package|option|message|extend|enum|service|rpc|returns|group|default|extensions|to|max|double|float|int32|int64|uint32|uint64|sint32|sint64|fixed32|fixed64|sfixed32|sfixed64|bool|string|bytes)\b/
-          @q.push [$&, $&.to_sym]
-        when /\A[1-9]\d*(?!\.)/, /\A0(?![.xX0-9])/
-          @q.push [:DEC_INTEGER, $&.to_i]
-        when /\A0[xX]([A-Fa-f0-9])+/
-          @q.push [:HEX_INTEGER, $&.to_i(0)]
-        when /\A0[0-7]+/
-          @q.push [:OCT_INTEGER, $&.to_i(0)]
-        when /\A\d+(\.\d+)?([Ee][\+-]?\d+)?/
-          @q.push [:FLOAT_LITERAL, $&.to_f]
-        when /\A(true|false)/
-          @q.push [:BOOLEAN_LITERAL, $& == 'true']
-        when /\A"(?:[^"\\]+|\\.)*"/, /\A'(?:[^'\\]+|\\.)*'/
-          @q.push [:STRING_LITERAL, eval($&)]
-        when /\A[a-zA-Z_][\w_]*/
-          @q.push [:IDENT, $&.to_sym]
-        when /\A[A-Z][\w_]*/
-          @q.push [:CAMEL_IDENT, $&.to_sym]
-        when /\A./
-          @q.push [$&, $&]
-        else
-          raise ArgumentError.new(line) 
-        end
-        line = $'
+    @scanner = StringScanner.new(f.read)
+    yyparse(self, :scan)
+  end
+
+  def scan_debug
+    scan do |token, value|
+      p [token, value]
+      yield [token, value]
+    end
+  end
+
+  def scan
+    until @scanner.eos?
+      case
+      when match(/\s+/, /\/\/.*/)
+        # skip
+      when match(/\/\*/)
+        # C-like comment
+        raise 'EOF inside block comment' until @scanner.scan_until(/\*\//)
+      when match(/(?:required|optional|repeated|import|package|option|message|extend|enum|service|rpc|returns|group|default|extensions|to|max|double|float|int32|int64|uint32|uint64|sint32|sint64|fixed32|fixed64|sfixed32|sfixed64|bool|string|bytes)\b/)
+        yield [@token, @token.to_sym]
+      when match(/[+-]?\d*\.\d+([Ee][\+-]?\d+)?/)
+        yield [:FLOAT_LITERAL, @token.to_f]
+      when match(/[+-]?[1-9]\d*(?!\.)/, /0(?![.xX0-9])/)
+        yield [:DEC_INTEGER, @token.to_i]
+      when match(/0[xX]([A-Fa-f0-9])+/)
+        yield [:HEX_INTEGER, @token.to_i(0)]
+      when match(/0[0-7]+/)
+        yield [:OCT_INTEGER, @token.to_i(0)]
+      when match(/(true|false)\b/)
+        yield [:BOOLEAN_LITERAL, @token == 'true']
+      when match(/"(?:[^"\\]+|\\.)*"/, /'(?:[^'\\]+|\\.)*'/)
+        yield [:STRING_LITERAL, eval(@token)]
+      when match(/[a-zA-Z_][\w_]*/)
+        yield [:IDENT, @token.to_sym]
+      when match(/[A-Z][\w_]*/)
+        yield [:CAMEL_IDENT, @token.to_sym]
+      when match(/./)
+        yield [@token, @token]
+      else
+        raise "parse error around #{@scanner.string[@scanner.pos, 32].inspect}"
       end
     end
-    do_parse
+    yield [false, nil]
   end
 
-  def next_token
-    @q.shift
+  def match(*regular_expressions)
+    regular_expressions.each do |re|
+      if @scanner.scan(re)
+        @token = @scanner[0]
+        return true
+      end
+    end
+    false
   end