chrisk-protopuffs 0.2.0

data/LICENSE.txt

@@ -0,0 +1,18 @@
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= Protopuffs!
+
+A new implementation of Protocol Buffers in Ruby.

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:major: 0
+:minor: 2
+:patch: 0

data/lib/protopuffs/message/base.rb

@@ -0,0 +1,90 @@
+module Protopuffs
+  module Message
+
+    class Base
+
+      class << self
+        attr_reader :fields
+
+        def define_message_class(name, fields)
+          name = name.capitalize
+          self.check_fields_for_errors(name, fields)
+          Message.send(:remove_const, name) if Message.const_defined?(name)
+          klass = Message.const_set(name, Class.new(self))
+          klass.instance_variable_set(:@fields, fields)
+          fields.each do |field|
+            klass.send(:attr_accessor, field.identifier)
+          end
+          klass
+        end
+
+        def check_fields_for_errors(name, fields)
+          tags = fields.map { |field| field.tag }
+          if tags.uniq.size != fields.size
+            raise ParseError, "A tag was reused in the descriptor for message #{name}. Please check that each tag is unique."
+          end
+
+          if tags.any? { |tag| tag < 1 || tag > 536_870_911 }
+            raise ParseError, "A tag is out of range in the descriptor for message #{name}. Please check that each tag is in the range 1 <= x <= 536,870,911."
+          end
+
+          if tags.any? { |tag| (19000..19999).include?(tag) }
+            raise ParseError, "A tag is invalid in the descriptor for message #{name}. Please check that each tag is not in the reserved range 19,000 <= x <= 19,999."
+          end
+        end
+      end
+
+      attr_reader :buffer
+
+      def initialize
+        if self.class == Base
+          raise "#{self.class} should not be instantiated directly. Use the factory #{self.class}.define_message_class instead."
+        end
+        @buffer = StringIO.new
+      end
+
+      def to_wire_format
+        self.class.fields.each do |field|
+          value = send(field.identifier)
+          @buffer.write field.to_wire_format_with_value(value) unless value.nil?
+        end
+        @buffer.string
+      end
+
+      def from_wire_format(buffer)
+        @buffer = buffer
+        until @buffer.eof?
+          tag, value_bytes = MessageField.shift_tag_and_value_bytes(@buffer)
+          field = self.class.fields.find { |field| field.tag == tag }
+          next if field.nil?
+
+          value = field.decode(value_bytes)
+          if field.repeated? && send(field.identifier).nil?
+            send("#{field.identifier}=", [value])
+          elsif field.repeated?
+            send(field.identifier) << value
+          else
+            send("#{field.identifier}=", value)
+          end
+        end
+        set_values_for_missing_optional_fields
+      end
+
+      def set_values_for_missing_optional_fields
+        self.class.fields.select { |field| field.optional? }.each do |field|
+          send("#{field.identifier}=", field.default) if send(field.identifier).nil?
+        end
+      end
+
+      def ==(other)
+        return false if self.class != other.class
+        self.class.fields.each do |field|
+          return false if send(field.identifier) != other.send(field.identifier)
+        end
+        true
+      end
+
+    end
+
+  end
+end

data/lib/protopuffs/message/field.rb

@@ -0,0 +1,226 @@
+module Protopuffs
+
+  class MessageField
+    attr_reader :modifier, :type, :identifier, :tag, :default
+
+    def initialize(modifier, type, identifier, tag, default = nil)
+      @modifier = modifier
+      @type = type
+      @identifier = identifier
+      @tag = tag
+      @default = default
+
+      set_default_for_type if optional? && @default.nil?
+    end
+
+    def set_default_for_type
+      if numeric?
+        @default = 0
+      elsif @type == "string"
+        @default = ""
+      elsif @type == "bool"
+        @default = false
+      end
+    end
+
+    def wire_type
+      case @type
+      when "int32", "int64", "uint32", "uint64", "bool" then WireType::VARINT
+      when "double", "fixed64"                          then WireType::FIXED64
+      when "string", "bytes"                            then WireType::LENGTH_DELIMITED
+      when "float", "fixed32"                           then WireType::FIXED32
+      else WireType::LENGTH_DELIMITED # embedded messages
+      end
+    end
+
+    def key
+      (@tag << 3) | wire_type
+    end
+
+    def repeated?
+      @modifier == "repeated"
+    end
+
+    def optional?
+      @modifier == "optional"
+    end
+
+    def numeric?
+      %w(double float int32 int64 uint32 unit64 sint32 sint64 fixed32 fixed64
+         sfixed32 sfixed64).include?(@type)
+    end
+
+    def embedded_message?
+      wire_type == WireType::LENGTH_DELIMITED && @type != "string" && @type != "bytes"
+    end
+
+    def to_wire_format_with_value(value)
+      field_encoder = lambda { |val| self.class.varint_encode(key) + encode(val) }
+
+      if repeated?
+        value.map(&field_encoder).join
+      else
+        field_encoder.call(value)
+      end
+    end
+
+    def encode(value)
+      case wire_type
+      when WireType::VARINT
+        value = (value ? 1 : 0) if @type == "bool"
+        value_bytes = self.class.varint_encode(value)
+      when WireType::LENGTH_DELIMITED
+        if value.respond_to?(:to_wire_format)
+          embedded_bytes = value.to_wire_format
+          value_bytes = self.class.varint_encode(embedded_bytes.size) + embedded_bytes
+        else
+          value_bytes = self.class.varint_encode(value.size)
+          value_bytes += self.class.string_encode(value) if @type == "string"
+          value_bytes += value if @type == "bytes"
+        end
+      when WireType::FIXED32
+        value_bytes = self.class.float_encode(value) if @type == "float"
+        value_bytes = self.class.fixed32_encode(value) if @type == "fixed32"
+      when WireType::FIXED64
+        value_bytes = self.class.double_encode(value) if @type == "double"
+        value_bytes = self.class.fixed64_encode(value) if @type == "fixed64"
+      end
+      value_bytes
+    end
+
+
+    def self.varint_encode(value)
+      return [0].pack('C') if value.zero?
+      bytes = []
+      until value.zero?
+        byte = 0
+        7.times do |i|
+          byte |= (value & 1) << i
+          value >>= 1
+        end
+        byte |= 0b10000000
+        bytes << byte
+      end
+      bytes[-1] &= 0b01111111
+      bytes.pack('C*')
+    end
+
+    def self.string_encode(value)
+      value.unpack('U*').pack('C*')
+    end
+
+    def self.float_encode(value)
+      [value].pack('e')
+    end
+
+    def self.double_encode(value)
+      [value].pack('E')
+    end
+
+    def self.fixed64_encode(value)
+      [value].pack('Q')
+    end
+
+    def self.fixed32_encode(value)
+      [value].pack('V')
+    end
+
+    # note: returns two values
+    def self.shift_tag_and_value_bytes(buffer)
+      bits = 0
+      bytes = shift_varint(buffer)
+      bytes.each_with_index do |byte, index|
+        byte &= 0b01111111
+        bits |= byte << (7 * index)
+      end
+      wire_type = bits & 0b00000111
+      tag = bits >> 3
+
+      case wire_type
+      when WireType::VARINT
+        value_bytes = shift_varint(buffer)
+      when WireType::LENGTH_DELIMITED
+        value_bytes = shift_length_delimited(buffer)
+      when WireType::FIXED32
+        value_bytes = shift_fixed32(buffer)
+      when WireType::FIXED64
+        value_bytes = shift_fixed64(buffer)
+      end
+
+      [tag, value_bytes]
+    end
+
+    def self.shift_varint(buffer)
+      bytes = []
+      begin
+        byte = buffer.readchar
+        bytes << (byte & 0b01111111)
+      end while byte >> 7 == 1
+      bytes
+    end
+
+    def self.shift_length_delimited(buffer)
+      bytes = shift_varint(buffer)
+      value_length = self.varint_decode(bytes)
+      buffer.read(value_length)
+    end
+
+    def self.shift_fixed32(buffer)
+      buffer.read(4)
+    end
+
+    def self.shift_fixed64(buffer)
+      buffer.read(8)
+    end
+
+    def decode(value_bytes)
+      case wire_type
+      when WireType::VARINT
+        value = self.class.varint_decode(value_bytes)
+        if @type == "bool"
+          value = true  if value == 1
+          value = false if value == 0
+        end
+      when WireType::LENGTH_DELIMITED
+        if embedded_message?
+          value = Message.const_get(@type).new
+          value.from_wire_format(StringIO.new(value_bytes))
+        else
+          value = value_bytes
+        end
+      when WireType::FIXED32
+        value = self.class.float_decode(value_bytes) if @type == "float"
+        value = self.class.fixed32_decode(value_bytes) if @type == "fixed32"
+      when WireType::FIXED64
+        value = self.class.double_decode(value_bytes) if @type == "double"
+        value = self.class.fixed64_decode(value_bytes) if @type == "fixed64"
+      end
+      value
+    end
+
+    def self.varint_decode(bytes)
+      value = 0
+      bytes.each_with_index do |byte, index|
+        value |= byte << (7 * index)
+      end
+      value
+    end
+
+    def self.float_decode(bytes)
+      bytes.unpack('e').first
+    end
+
+    def self.double_decode(bytes)
+      bytes.unpack('E').first
+    end
+
+    def self.fixed64_decode(bytes)
+      bytes.unpack('Q').first
+    end
+
+    def self.fixed32_decode(bytes)
+      bytes.unpack('V').first
+    end
+  end
+
+end

data/lib/protopuffs/message/wire_type.rb

@@ -0,0 +1,10 @@
+module Protopuffs
+  class WireType
+    VARINT = 0
+    FIXED64 = 1
+    LENGTH_DELIMITED = 2
+    START_GROUP = 3
+    END_GROUP = 4
+    FIXED32 = 5
+  end
+end

data/lib/protopuffs/parser/parser.rb

@@ -0,0 +1,24 @@
+module Protopuffs
+
+  # The Protopuffs::Parser module holds wrapper classes that parse input
+  # strings using Treetop. They also set off the building of an abstract
+  # syntax tree from Treetop's parse tree.
+  module Parser
+
+    class ProtocolBufferDescriptor
+      def initialize
+        Treetop.load "lib/protopuffs/parser/protocol_buffer"
+        @parser = Protopuffs::ProtocolBufferParser.new
+      end
+
+      def parse(input)
+        parse_tree = @parser.parse(input)
+        parse_tree.build
+        parse_tree
+      rescue
+        raise ParseError
+      end
+    end
+
+  end
+end

data/lib/protopuffs/parser/protocol_buffer.treetop

@@ -0,0 +1,124 @@
+module Protopuffs
+  grammar ProtocolBuffer
+
+    rule proto
+      s? proto_entries s? {
+        def messages
+          proto_entries.entries.select { |p| p.respond_to?(:message?) }
+        end
+
+        def build
+          messages.each { |message| message.build }
+        end
+      }
+    end
+
+    rule proto_entries
+      proto_entry rest:(s? proto_entry)* {
+        def entries
+          rest.elements.inject([proto_entry]) { |a, s_and_pe| a << s_and_pe.proto_entry }
+        end
+      }
+    end
+
+    rule proto_entry
+      message {
+        def message?
+          true
+        end
+      }
+    end
+
+    rule message
+      "message" s identifier s "{" s? body:message_body? s? "}" {
+        def name
+          identifier.text_value
+        end
+        
+        def build
+          Message::Base.define_message_class(name, body.empty? ? [] : body.build)
+        end
+      }
+    end
+
+    rule message_body
+      field rest:(s? field)* {
+        def fields
+          rest.elements.inject([field]) { |a, s_and_field| a << s_and_field.field }
+        end
+
+        def build
+          fields.map { |field| field.build }
+        end
+      }
+    end
+
+    rule field
+      modifier s type s identifier s "=" s integer s? option:field_option? ";" {
+        def default
+          option.empty? ? nil : option.literal.obj
+        end
+
+        def build
+          MessageField.new(modifier.text_value, type.text_value, identifier.text_value,
+                           integer.text_value.to_i, default)
+        end
+      }
+    end
+
+    rule field_option
+      "[" s? "default" s? "=" s? literal s? "]"
+    end
+
+    rule modifier
+      "required" / "optional" / "repeated"
+    end
+
+    rule type
+      "double" / "float" / "int32" / "int64" / "uint32" / "uint64" /
+      "sint32" / "sint64" / "fixed32" / "fixed64" / "sfixed32" /
+      "sfixed64" / "bool" / "string" / "bytes" / identifier
+    end
+
+    rule literal
+      integer / string
+    end
+
+    rule integer
+      [\d]+ {
+        def obj
+          text_value.to_i
+        end
+      }
+    end
+
+    rule string
+      quote chars:[\w]* quote {
+        def obj
+          chars.text_value
+        end
+      }
+    end
+
+    rule quote
+      ["']
+    end
+
+    rule identifier
+      [A-Za-z_] [\w_]*
+    end
+
+    rule s
+      (whitespace / comment_to_eol)+
+    end
+
+    rule comment_to_eol
+      '//' (!"\n" .)*
+    end
+
+    rule whitespace
+      [ \t\n\r]
+    end
+
+  end
+end

data/lib/protopuffs.rb

@@ -0,0 +1,27 @@
+require "protopuffs/message/base"
+require "protopuffs/message/field"
+require "protopuffs/message/wire_type"
+require "protopuffs/parser/parser"
+
+
+module Protopuffs
+  class ParseError < StandardError; end
+
+  def self.proto_load_path
+    @proto_load_path ||= []
+  end
+
+  def self.proto_load_path=(paths)
+    @proto_load_path = paths
+  end
+
+  def self.load_message_classes
+    parser = Protopuffs::Parser::ProtocolBufferDescriptor.new
+
+    @proto_load_path.each do |path|
+      Dir.glob(File.join(path, "**/*.proto")) do |descriptor_path|
+        parser.parse(File.read(descriptor_path))
+      end
+    end
+  end
+end

data/test/abstract_syntax_tree_test.rb

@@ -0,0 +1,142 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class AbstractSyntaxTreeTest < Test::Unit::TestCase
+
+  context "a protocol buffer descriptor" do
+    setup do
+      @parser = Protopuffs::Parser::ProtocolBufferDescriptor.new
+    end
+
+    context "with an empty Person message" do
+      setup do
+        @descriptor = "message Person {}"
+      end
+
+      should "create one MessageDescriptor" do
+        Protopuffs::Message::Base.expects(:define_message_class).once.with("Person", [])
+        @parser.parse(@descriptor)
+      end
+
+      should "not create any MessageFields" do
+        Protopuffs::MessageField.expects(:new).never
+        @parser.parse(@descriptor)
+      end
+    end
+
+
+    context "with two empty messages Apple and Orange" do
+      setup do
+        @descriptor = <<-proto
+          message Apple {}
+          message Orange {}
+        proto
+      end
+
+      should "create two MessageDescriptors" do
+        Protopuffs::Message::Base.expects(:define_message_class).once.with("Apple", []).in_sequence
+        Protopuffs::Message::Base.expects(:define_message_class).once.with("Orange", []).in_sequence
+        @parser.parse(@descriptor)
+      end
+
+      should "not create any MessageFields" do
+        Protopuffs::MessageField.expects(:new).never
+        @parser.parse(@descriptor)
+      end
+    end
+
+
+    context "with a Person message including a name field" do
+      setup do
+        @descriptor = <<-proto
+          message Person {
+            required string name = 1;
+          }
+        proto
+      end
+
+      should "create one 'Person' MessageDescriptor with one field" do
+        Protopuffs::Message::Base.expects(:define_message_class).once.with("Person", responds_with(:size, 1))
+        @parser.parse(@descriptor)
+      end
+
+      should "create one MessageField with correct options" do
+        Protopuffs::MessageField.expects(:new).once.with("required", "string", "name", 1, nil).returns(stub(:tag => 1, :identifier => "name"))
+        @parser.parse(@descriptor)
+      end
+    end
+
+
+    context "with a Person message including three fields" do
+      setup do
+        @descriptor = <<-proto
+          message Person {
+            required string name = 1;
+            required int32 id = 2;
+            optional string email = 3;
+          }
+        proto
+      end
+
+      should "create one 'Person' MessageDescriptor with three fields" do
+        Protopuffs::Message::Base.expects(:define_message_class).once.with("Person", responds_with(:size, 3))
+        @parser.parse(@descriptor)
+      end
+
+      should "create three MessageFields with correct options" do
+        Protopuffs::MessageField.expects(:new).once.with("required", "string", "name", 1, nil).in_sequence.returns(stub(:tag => 1, :identifier => "name"))
+        Protopuffs::MessageField.expects(:new).once.with("required", "int32", "id", 2, nil).in_sequence.returns(stub(:tag => 2, :identifier => "id"))
+        Protopuffs::MessageField.expects(:new).once.with("optional", "string", "email", 3, nil).in_sequence.returns(stub(:tag => 3, :identifier => "email"))
+        @parser.parse(@descriptor)
+      end
+    end
+
+
+    context "with a message including two fields with defaults and one without" do
+      setup do
+        @descriptor = <<-proto
+          message Person {
+            required string name = 1;
+            optional string language = 2 [default = "en"];
+            optional int32 account_code = 3 [default = 0];
+          }
+        proto
+      end
+
+      should "create one 'Person' MessageDescriptor with three fields" do
+        Protopuffs::Message::Base.expects(:define_message_class).once.with("Person", responds_with(:size, 3))
+        @parser.parse(@descriptor)
+      end
+
+      should "create three MessageFields with correct options" do
+        Protopuffs::MessageField.expects(:new).once.with("required", "string", "name", 1, nil).in_sequence.returns(stub(:tag => 1, :identifier => "name"))
+        Protopuffs::MessageField.expects(:new).once.with("optional", "string", "language", 2, "en").in_sequence.returns(stub(:tag => 2, :identifier => "language"))
+        Protopuffs::MessageField.expects(:new).once.with("optional", "int32", "account_code", 3, 0).in_sequence.returns(stub(:tag => 3, :identifier => "account_code"))
+        @parser.parse(@descriptor)
+      end
+    end
+
+
+    context "with a message including a user-typed field" do
+      setup do
+        @descriptor = <<-proto
+          message Person {
+            required string name = 1;
+            repeated Address addresses = 2;
+          }
+        proto
+      end
+
+      should "create one 'Person' MessageDescriptor with two fields" do
+        Protopuffs::Message::Base.expects(:define_message_class).once.with("Person", responds_with(:size, 2))
+        @parser.parse(@descriptor)
+      end
+
+      should "create two MessageFields with correct options" do
+        Protopuffs::MessageField.expects(:new).once.with("required", "string", "name", 1, nil).in_sequence.returns(stub(:tag => 1, :identifier => "name"))
+        Protopuffs::MessageField.expects(:new).once.with("repeated", "Address", "addresses", 2, nil).in_sequence.returns(stub(:tag => 2, :identifier => "addresses"))
+        @parser.parse(@descriptor)
+      end
+    end
+  end
+
+end

data/test/fixtures/proto/person.proto

@@ -0,0 +1,5 @@
+message Person {
+  required string name = 1;
+  required int32 id = 2;
+  optional string email = 3;
+}