ruby_protobuf 0.0.1

data/History.txt

@@ -0,0 +1,5 @@
+=== 0.0.1 / 2008-07-26
+
+* 1 major enhancement
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,32 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/rprotoc
+bin/ruby_protobuf
+lib/protobuf/compiler.rb
+lib/protobuf/decoder.rb
+lib/protobuf/encoder.rb
+lib/protobuf/enum.rb
+lib/protobuf/extend.rb
+lib/protobuf/field.rb
+lib/protobuf/message.rb
+lib/protobuf/parser.y
+lib/protobuf/service.rb
+lib/protobuf/wire_type.rb
+lib/ruby_protobuf.rb
+test/addressbook.proto
+test/addressbook.rb
+test/data/data.bin
+test/data/data_source.py
+test/data/types.bin
+test/data/types_source.py
+test/test_addressbook.rb
+test/test_compiler.rb
+test/test_message.rb
+test/test_parse.rb
+test/test_ruby_protobuf.rb
+test/test_serialize.rb
+test/test_types.rb
+test/types.proto
+test/types.rb

data/README.txt

@@ -0,0 +1,50 @@
+= RubyProtobuf
+
+* http://code.google.com/p/ruby-protobuf
+
+== DESCRIPTION:
+
+Protocol Buffers for Ruby.
+
+== FEATURES/PROBLEMS:
+
+* Compile .proto file to ruby script
+* Parse the binary wire format for protocol buffer
+* Serialize data to the binary wire format for protocol buffer
+
+== SYNOPSIS:
+
+  bin/rprotoc -I lib test/addressbook.proto
+
+== REQUIREMENTS:
+
+* All you need are included.
+
+== INSTALL:
+
+* sudo gem install ruby-protobuf
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 FIX
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,18 @@
+# -*- ruby -*-
+
+$:.push(File.dirname(__FILE__) + '/lib')
+require 'rubygems'
+require 'hoe'
+require 'ruby_protobuf'
+
+Hoe.new('ruby_protobuf', RubyProtobuf::VERSION) do |p|
+  p.rubyforge_name = 'ruby-protobuf'
+  p.author = 'ANDO Yasushi'
+  p.email = 'andyjpn@gmail.com'
+  p.summary = 'Protocol Buffers for Ruby'
+  p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
+  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+end
+
+# vim: syntax=Ruby

data/bin/rprotoc

@@ -0,0 +1,21 @@
+#!/usr/local/bin/ruby
+$: << "#{File.dirname(__FILE__)}/../lib"
+require 'optparse'
+require 'ruby_protobuf'
+
+
+COMMAND_LINE = "#{$0} #{ARGV.join(' ')} PROTO_FILE"
+OPT = {}
+opts = OptionParser.new
+#opts.on('-p', '--proto_path <PATH>', 'Specify the directory in which to search for imports. The current directory is default.'){|v| OPT[:proto_path] = v}
+opts.on('-o', '--out <OUT_DIR>', 'Specify the directory in which Ruby source file is generated. The current directory is default.'){|v| OPT[:out] = v}
+opts.on_tail('-v', '--version', 'Show version.'){puts(opts.ver); exit}
+opts.on_tail('-h', '--help', 'Show this message.'){puts(opts.help); exit}
+
+::Version = RubyProtobuf::VERSION
+opts.order! ARGV
+PROTO_FILE = ARGV.shift
+
+RubyProtobuf.new.start PROTO_FILE, OPT
+
+

data/lib/protobuf/compiler.rb

@@ -0,0 +1,90 @@
+# This is a quite temporary implementation.
+# I'll create a compiler class using Racc.
+
+module Protobuf
+  class Compiler
+    INDENT_UNIT = '  '
+
+    def self.compile(filename)
+      self.new.compile filename
+    end
+
+    def initialize
+      @indent_level = 0
+      @ret = <<-eos
+require 'protobuf/message'
+require 'protobuf/enum'
+require 'protobuf/service'
+require 'protobuf/extend'
+
+      eos
+    end
+
+    def indent
+      INDENT_UNIT * @indent_level
+    end
+
+    def puts_with_indent(string)
+      #puts "#{indent}#{string}"
+      @ret += "#{indent}#{string}\n"
+    end
+    alias putswi puts_with_indent
+
+    def compile(filename)
+      File.open filename, 'r' do |file|
+        file.each_line do |line|
+          line.sub! /^(.*)\/\/.*/, '\1'
+          line.strip!
+          case line
+          when /^package\s+(\w+(\.\w+)?)\s*;$/
+            $1.split('.').each do |path|
+              putswi "module #{path.capitalize}"
+              @indent_level += 1
+            end
+          when /^message\s+(\w+)\s*\{$/
+            putswi "class #{$1} < Protobuf::Message"
+            @indent_level += 1
+          when /^(required|optional|repeated)\s+(\w+(\.\w+)?)\s+(\w+)\s*=\s*(\d+)\s*(\[\s*default\s*=\s*(\w+)\s*\])?\s*;$/
+            rule, type, name, tag, default = $1, $2, $4, $5, $7
+            if default
+              default = default =~ /\d+(\.\d+)/ \
+                ? ", {:default => #{default}}" \
+                : ", {:default => :#{default}}"
+            end
+            putswi "#{rule} :#{type}, :#{name}, #{tag}#{default}"
+          when /^enum\s+(\w+)\s*\{$/
+            putswi "class #{$1} < Protobuf::Enum"
+            @indent_level += 1
+          when /^(\w+)\s*=\s*(\w+)\s*;$/
+            putswi "#{$1} = #{$2}"
+          when /^extensions\s+(\w+)\s+to\s+(\w+)\s*;/
+            low, high = $1, $2
+            low = 'Protobuf::Extend.MIN' if low == 'min'
+            high = 'Protobuf::Extend.MAX' if high == 'max'
+            putswi "extensions #{min}..#{max}"
+          when /^extend\s+(\w+)\s*\{/
+            putswi "class #{$1} < Protobuf::Extend"
+            @indent_level += 1
+          when /^service\s+(\w+)\s*\{/
+            putswi "class #{$1} < Protobuf::Service"
+            @indent_level += 1
+          when /^rpc\s+(\w+)\s+\(\s*(\w+)\s*\)\s+returns\s+\(\s*(\w+)\s*\)\s*;/
+            putswi "rpc :#{$1} => :#{$2}, :#{$3} => :#{$4}"
+          when /^option\s+(\w+)\s*=\s*(.+)\s*;/
+            putswi "Protobuf::OPTIONS[:#{$1}] = :#{$2}"
+          when /^\}$/
+            @indent_level -= 1
+            putswi "end"
+          when ''
+            putswi ''
+          end
+        end
+        while 0 < @indent_level 
+          @indent_level -= 1
+          putswi "end"
+        end
+      end
+      @ret
+    end
+  end
+end

data/lib/protobuf/decoder.rb

@@ -0,0 +1,87 @@
+require 'protobuf/wire_type'
+
+module Protobuf
+  class InvalidWireType < StandardError; end
+
+  class Decoder
+    class <<self
+      def decode(stream, message)
+        self.new(stream, message).decode
+      end
+    end
+
+    def initialize(stream=nil, message=nil)
+      @stream, @message = stream, message
+    end
+
+    def decode(stream=@stream, message=@message)
+      until stream.eof?
+        tag, wire_type = read_key stream
+        bytes =
+          case wire_type
+          when WireType::VARINT
+            read_varint stream
+          when WireType::FIXED64
+            read_fixed64 stream
+          when WireType::LENGTH_DELIMITED
+            read_length_delimited stream
+          when WireType::START_GROUP
+            read_start_group stream
+          when WireType::END_GROUP
+            read_end_group stream
+          when WireType::FIXED32
+            read_fixed32 stream
+          else
+            raise InvalidWireType.new(wire_type)
+          end
+        message.set_field tag, bytes
+      end
+      message
+    end
+
+    protected
+
+    def read_key(stream)
+      bytes = read_varint stream
+      #TODO 1 < bytes.size のときエラー
+      wire_type = bytes[0] & 0b00000111
+      tag = bytes[0] >> 3 # TODO
+      [tag, wire_type]
+    end
+
+    def read_varint(stream)
+      bytes = []
+      begin
+        byte = stream.readchar
+        bytes << (byte & 0b01111111)
+      end while byte >> 7 == 1
+      bytes
+    end
+
+    def read_fixed64(stream)
+      stream.read(8)
+    end
+
+    def read_length_delimited(stream)
+      bytes = read_varint stream
+      value_length = 0
+      bytes.each_with_index do |byte, index|
+        value_length |= byte << (7 * index)
+      end
+      value = stream.read value_length
+      value.unpack('c*')
+    end
+
+    def read_start_group(stream)
+      raise NotImplementedError.new('Group is duplecated.')
+    end
+ 
+    def read_end_group(stream)
+      raise NotImplementedError.new('Group is duplecated.')
+    end
+
+    def read_fixed32(stream)
+      stream.read(4)
+    end
+  end
+end

data/lib/protobuf/encoder.rb

@@ -0,0 +1,36 @@
+require 'protobuf/wire_type'
+
+module Protobuf
+  class Encoder
+    class <<self
+      def encode(stream, message)
+        self.new(stream, message).encode
+      end
+    end
+
+    def initialize(stream=nil, message=nil)
+      @stream, @message = stream, message
+    end
+
+    def encode(stream=@stream, message=@message)
+      message.each_field do |field, value|
+        key = (field.tag << 3) | field.wire_type
+        key_bytes = Protobuf::Field::Int.get_bytes key
+        #stream.write key_bytes.pack('C*')
+        stream.write key_bytes
+
+        if field.repeated?
+          value.each do |val|
+            bytes = field.get val
+            #stream.write bytes.pack('C*')
+            stream.write bytes
+          end
+        else
+          bytes = field.get value
+          #stream.write bytes.pack('C*')
+          stream.write bytes
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/enum.rb

@@ -0,0 +1,13 @@
+module Protobuf
+  class Enum
+    def self.get_name_by_tag(tag)
+      constants.find do |name|
+        class_eval(name) == tag
+      end
+    end
+
+    def self.valid_tag?(tag)
+      not get_name_by_tag(tag).nil?
+    end
+  end
+end

data/lib/protobuf/extend.rb

@@ -0,0 +1,8 @@
+require 'protobuf/message'
+
+module Protobuf
+  class Extend < Message
+    MIN = -1.0/0
+    MAX = 1.0/0
+  end
+end

data/lib/protobuf/field.rb

@@ -0,0 +1,547 @@
+require 'protobuf/wire_type'
+
+module Protobuf
+  module Field
+    def self.build(message_class, rule, type, name, tag, opts={})
+      Base.build message_class, rule, type, name, tag, opts
+    end
+
+    class InvalidRuleError < StandardError; end
+
+    class Base
+      class <<self
+        def build(message_class, rule, type, name, tag, opts={})
+          field_class = nil
+          begin
+            field_class = eval "Protobuf::Field::#{type.to_s.capitalize}Field"
+          rescue NameError
+            type = typename_to_class message_class, type
+            field_class =
+              if type.superclass == Protobuf::Enum
+                Protobuf::Field::Enum
+              elsif type.superclass == Protobuf::Message
+                Protobuf::Field::Message
+              else
+                raise $!
+              end
+          end
+          field_class.new message_class, rule, type, name, tag, opts
+        end
+
+        def typename_to_class(message_class, type)
+          modules = message_class.to_s.split('::')
+          while
+            begin
+              type = eval((modules | [type.to_s]).join('::'))
+              break
+            rescue NameError
+              modules.empty? ? raise($!) : modules.pop
+            end
+          end
+          type
+        end
+      end
+
+      attr_accessor :message_class, :rule, :type, :name, :tag, :default
+
+      def initialize(message_class, rule, type, name, tag, opts={})
+        @message_class, @rule, @type, @name, @tag, @default = 
+          message_class, rule, type, name, tag, opts[:default]
+        @error_message = 'Type invalid'
+      end
+
+      def default_value
+        case rule
+        when :repeated
+          FieldArray.new self
+        when :required, :optional
+          typed_default_value default
+        else
+          raise InvalidRuleError
+        end
+      end
+
+      def typed_default_value(default=nil)
+        default
+      end
+
+      def define_accessor(message_instance)
+        message_instance.instance_variable_set "@#{name}", default_value
+        define_getter message_instance
+        define_setter message_instance unless rule == :repeated
+      end
+
+      def define_getter(message_instance)
+        message_instance.instance_eval %Q{
+          def #{name}
+            @#{name}
+          end
+        }
+      end
+
+      def define_setter(message_instance)
+        message_instance.instance_eval %Q{
+          def #{name}=(val)
+            field = get_field_by_name #{name.inspect}
+            if field.acceptable? val
+              @#{name} = val
+            end
+          end
+        }
+      end
+
+      def set(message_instance, bytes)
+        if repeated?
+          set_array message_instance, bytes
+        else
+          set_bytes message_instance, bytes
+        end
+      end
+
+      def set_array(message_instance, bytes)
+        raise NotImplementedError
+      end
+
+      def set_bytes(message_instance, bytes)
+        raise NotImplementedError
+      end
+
+      def get(value)
+        get_bytes value
+      end
+
+      def get_bytes(value)
+        raise NotImplementedError
+      end
+
+      def merge(message_instance, bytes)
+        if repeated?
+          merge_array method_instance, bytes
+        else
+          merge_value method_instance, bytes
+        end
+      end
+
+      def merge_array(message_instance, bytes)
+        raise NotImplementedError
+      end
+
+      def merge_value(message_instance, bytes)
+        raise NotImplementedError
+      end
+
+      def repeated?; rule == :repeated end
+      def required?; rule == :required end
+      def optional?; rule == :optional end
+
+      def max; self.class.max end
+      def min; self.class.min end
+
+      def acceptable?(val)
+        true
+      end
+
+      def error_message
+        @error_message
+      end
+
+      def to_s
+        "#{rule} #{type} #{name} = #{tag} #{default ? "[default=#{default}]" : ''}"
+      end
+    end
+
+    class FieldArray < Array
+      def initialize(field)
+        @field = field
+      end
+
+      def []=(nth, val)
+        if @field.acceptable? val
+          super
+        end
+      end
+
+      def <<(val)
+        if @field.acceptable? val
+          super
+        end
+      end
+
+      def push(val)
+        if @field.acceptable? val
+          super
+        end
+      end
+
+      def unshift(val)
+        if @field.acceptable? val
+          super
+        end
+      end
+
+      def to_s
+        "[#{@field.name}]"
+      end
+    end
+
+    class StringField < Base
+      def wire_type
+        Protobuf::WireType::LENGTH_DELIMITED
+      end
+
+      def typed_default_value(default=nil)
+        default or ''
+      end
+
+      def acceptable?(val)
+        raise TypeError unless val.instance_of? String
+        true
+      end
+
+      def set_bytes(method_instance, bytes)
+        method_instance.send("#{name}=", bytes.pack('U*'))
+      end
+
+      def get_bytes(value)
+        bytes = value.unpack('U*')
+        string_size = Int.get_bytes bytes.size
+        #(string_size + bytes).pack('C*')
+        string_size + bytes.pack('C*')
+      end
+    end
+    
+    class BytesField < Base
+      def wire_type
+        Protobuf::WireType::VARINT
+      end
+
+      def typed_default_value(default=nil)
+        default or []
+      end
+
+      def set_bytes(method_instance, bytes)
+        method_instance.send("#{name}=", bytes)
+      end
+
+      def get_bytes(value)
+        string_size = Int.get_bytes value.size
+        string_size + value.pack('C*')
+      end
+    end
+
+    class Int < Base
+      def wire_type
+        Protobuf::WireType::VARINT
+      end
+
+      def typed_default_value(default=nil)
+        default or 0
+      end
+ 
+      def set_bytes(method_instance, bytes)
+        # TODO should refactor using pack('w*')
+        value = 0
+        bytes.each_with_index do |byte, index|
+          value |= byte << (7 * index)
+        end
+        method_instance.send("#{name}=", value)
+      end
+
+      def self.get_bytes(value)
+        # TODO should refactor using unpack('w*')
+        #return [value].pack('w*').unpack('C*')
+        return [0].pack('C') if value == 0
+        bytes = []
+        until value == 0
+          byte = 0
+          7.times do |i|
+            byte |= (value & 1) << i
+            value >>= 1
+          end
+          byte |= 0b10000000
+          bytes << byte
+        end
+        #bytes[0] &= 0b01111111
+        #bytes
+        bytes[bytes.size - 1] &= 0b01111111
+        bytes.pack('C*')
+      end
+
+      def get_bytes(value)
+        self.class.get_bytes value
+      end
+
+      def acceptable?(val)
+        raise TypeError.new(val.class.name) unless val.is_a? Integer
+        raise RangeError.new(val) if val < min or max < val
+        true
+      end
+    end
+    
+    class Int32Field < Int
+      def self.max; 1.0/0.0 end
+      def self.min; -1.0/0.0 end
+    end
+    
+    class Int64Field < Int
+      def self.max; 1.0/0.0 end
+      def self.min; -1.0/0.0 end
+    end
+    
+    class Uint32Field < Int
+      def self.max; 1.0/0.0 end
+      def self.min; 0 end
+    end
+    
+    class Uint64Field < Int
+      def self.max; 1.0/0.0 end
+      def self.min; 0 end
+    end
+    
+    class Sint32Field < Int
+      def self.max; 1.0/0.0 end
+      def self.min; -1.0/0.0 end
+ 
+      def set_bytes(method_instance, bytes)
+        # TODO use only bit-operations
+        byte = bytes.first
+        value = 
+          if byte % 2 == 0
+            byte / 2
+          else
+            -(byte + 1) / 2
+          end
+        method_instance.send("#{name}=", value)
+      end
+
+      def get_bytes(value)
+        #(value << 1) ^ (value >> 31)
+        [(value << 1) ^ (value >> 31)].pack('C*')
+      end
+    end
+    
+    class Sint64Field < Int
+      def self.max; 1.0/0.0 end
+      def self.min; -1.0/0.0 end
+ 
+      def set_bytes(method_instance, bytes)
+        # TODO use only bit-operations
+        byte = bytes.first
+        value = 
+          if byte % 2 == 0
+            byte / 2
+          else
+            -(byte + 1) / 2
+          end
+        method_instance.send("#{name}=", value)
+      end
+
+      def get_bytes(value)
+        #(value << 1) ^ (value >> 63)
+        [(value << 1) ^ (value >> 63)].pack('C*')
+      end
+    end
+    
+    class DoubleField < Int
+      def wire_type
+        Protobuf::WireType::FIXED64
+      end
+ 
+      #TODO
+      def self.max
+        '0x7fefffffffffffff'.unpack('d').first
+      end
+
+      #TODO
+      def self.min
+        -(2**(64/2) - 1)
+      end
+ 
+      def set_bytes(method_instance, bytes)
+        method_instance.send("#{name}=", bytes.unpack('E').first)
+      end
+
+      def get_bytes(value)
+        [value].pack('E')
+      end
+
+      def acceptable?(val)
+        raise TypeError.new(val.class.name) unless val.is_a? Numeric
+        raise RangeError.new(val) if val < min or max < val
+        true
+      end
+    end
+    
+    class FloatField < Int
+      def wire_type
+        Protobuf::WireType::FIXED32
+      end
+ 
+      #TODO
+      def self.max
+        '0x7fefffffffffffff'.unpack('e').first
+      end
+
+      #TODO
+      def self.min
+        -(2**(32/2) - 1)
+      end
+ 
+      def set_bytes(method_instance, bytes)
+        method_instance.send("#{name}=", bytes.unpack('e').first)
+      end
+
+      def get_bytes(value)
+        [value].pack('e')
+      end
+ 
+      def acceptable?(val)
+        raise TypeError unless val.is_a? Numeric
+        raise RangeError if val < min or max < val
+        true
+      end
+   end
+    
+    class Fixed32Field < Int
+      def wire_type
+        Protobuf::WireType::FIXED32
+      end
+
+      def self.max
+        2**32
+      end
+
+      def self.min
+        0
+      end
+ 
+      def set_bytes(method_instance, bytes)
+        method_instance.send("#{name}=", bytes.unpack('L').first)
+      end
+
+      def get_bytes(value)
+        [value].pack('L')
+      end
+    end
+    
+    class Fixed64Field < Int
+      def wire_type
+        Protobuf::WireType::FIXED64
+      end
+
+      def self.max
+        2**64
+      end
+
+      def self.min
+        0
+      end
+ 
+      def set_bytes(method_instance, bytes)
+        method_instance.send("#{name}=", bytes.unpack('l').first)
+      end
+
+      def get_bytes(value)
+        [value].pack('Q')
+      end
+    end
+    
+    class Sfinxed32Field < Int
+      def wire_type
+        Protobuf::WireType::FIXED32
+      end
+
+      def self.max
+        2**(32/2)
+      end
+
+      def self.min
+        -(2**(32/2) - 1)
+      end
+    end
+    
+    class Sfixed64Field < Int
+      def wire_type
+        Protobuf::WireType::FIXED64
+      end
+
+      def self.max
+        2**(64/2)
+      end
+
+      def self.min
+        -(2**(64/2) - 1)
+      end
+    end
+    
+    class BoolField < Int
+      def typed_default_value(default=nil)
+        default or false
+      end
+
+      def acceptable?(val)
+        raise TypeError unless [TrueClass, FalseClass].include? val.class
+        true
+      end
+ 
+      def set_bytes(method_instance, bytes)
+        method_instance.send("#{name}=", bytes.first == 1)
+      end
+
+      def get_bytes(value)
+        [value ? 1 : 0].pack('C')
+      end
+    end
+    
+    class Message < Base
+      def wire_type
+        Protobuf::WireType::LENGTH_DELIMITED
+      end
+
+      def typed_default_value(default=nil)
+        if default.is_a? Symbol
+          type.module_eval default.to_s
+        else
+          default
+        end
+      end
+
+      def acceptable?(val)
+        raise TypeError unless val.instance_of? type
+        true
+      end
+ 
+      def set_bytes(method_instance, bytes)
+        message = type.new
+        #message.parse_from bytes
+        message.parse_from_string bytes.pack('U*') # TODO
+        method_instance.send("#{name}=", message)
+      end
+ 
+      def set_array(method_instance, bytes)
+        message = type.new
+        #message.parse_from bytes
+        message.parse_from_string bytes.pack('U*')
+        arr = method_instance.send name
+        arr << message
+      end
+
+      def get_bytes(value)
+        stringio = StringIO.new ''
+        value.serialize_to stringio
+        bytes = stringio.string.unpack 'C*'
+        string_size = Int.get_bytes bytes.size
+        #(string_size + bytes).pack('C*')
+        #bytes + string_size
+        string_size + bytes.pack('C*')
+      end
+    end
+
+    class Enum < Int
+      def acceptable?(val)
+        raise TypeError unless type.valid_tag? val
+        true
+      end
+    end
+  end
+end