stark 0.6.1 → 0.7.0

data/History.txt

@@ -1,3 +1,13 @@
+=== 0.7.0 / 2013-03-04
+
+* 5 bugfixes:
+
+  * Add proper support for include
+  * Add proper support for namespace
+  * Fix lists in structs
+  * Fix reading enums in structs
+  * Flesh out more converters, get struct-in-struct working properly
+
 === 0.6.1 / 2013-03-02
 
 * 1 bugfix:

data/Manifest.txt

@@ -19,9 +19,11 @@ lib/stark/struct.rb
 lib/stark/thrift.kpeg
 stark.gemspec
 test/ThriftSpec.thrift
+test/blah.thrift
 test/gen-rb/profile_constants.rb
 test/gen-rb/profile_types.rb
 test/gen-rb/user_storage.rb
+test/include_blah.thrift
 test/leg.rb
 test/legacy_profile/profile_constants.rb
 test/legacy_profile/profile_types.rb
@@ -29,4 +31,5 @@ test/legacy_profile/user_storage.rb
 test/profile.thrift
 test/test_client.rb
 test/test_parser.rb
+test/test_ruby.rb
 test/test_server.rb

data/README.txt

@@ -13,9 +13,13 @@ Optimized thrift bindings for ruby
 
 == SYNOPSIS:
 
-  $ stark service.thrift > service.rb
+  $ stark service.thrift service.rb
   require 'service'
 
+  OR
+
+  Stark.materialize "service.thrift"
+
   Use Service::Client and Service::Processor like the default thrift
   docs describe them.
 

data/Rakefile

@@ -17,6 +17,4 @@ task :parser do
   sh "kpeg -o lib/stark/raw_parser.rb -s -f lib/stark/thrift.kpeg"
 end
 
-# task :test => :parser
-
 # vim: syntax=ruby

data/bin/stark

@@ -4,14 +4,20 @@ require 'rubygems'
 require 'stark/parser'
 require 'stark/ruby'
 
-data = File.read ARGV.shift
+file = ARGV.shift
+unless file
+  puts "stark blah.thrift <blah.rb>"
+  exit 1
+end
 
-tg = Stark::Parser.new data
+ast = Stark::Parser.ast File.read(file)
 
-unless tg.parse
-  tg.raise_error
+if out = ARGV.shift
+  out = File.open out, "w"
+else
+  out = STDOUT
 end
 
-ruby = Stark::Ruby.new
+ruby = Stark::Ruby.new out
 
-tg.result.each { |i| i.accept ruby }
+ruby.run ast

data/lib/stark/converters.rb

@@ -2,6 +2,38 @@ require 'thrift'
 
 module Stark
   module Converters
+    module BYTE
+      module_function
+
+      def type
+        Thrift::Types::BYTE
+      end
+
+      def read(ip)
+        ip.read_byte
+      end
+
+      def write(op, value)
+        op.write_byte value
+      end
+    end
+
+    module I16
+      module_function
+
+      def type
+        Thrift::Types::I16
+      end
+
+      def read(ip)
+        ip.read_i16
+      end
+
+      def write(op, value)
+        op.write_i16 value
+      end
+    end
+
     module I32
       module_function
 
@@ -18,6 +50,54 @@ module Stark
       end
     end
 
+    module I64
+      module_function
+
+      def type
+        Thrift::Types::I64
+      end
+
+      def read(ip)
+        ip.read_i64
+      end
+
+      def write(op, value)
+        op.write_i64 value
+      end
+    end
+
+    module BOOL
+      module_function
+
+      def type
+        Thrift::Types::BOOL
+      end
+
+      def read(ip)
+        ip.read_bool
+      end
+
+      def write(op, value)
+        op.write_bool value
+      end
+    end
+
+    module DOUBLE
+      module_function
+
+      def type
+        Thrift::Types::DOUBLE
+      end
+
+      def read(ip)
+        ip.read_double
+      end
+
+      def write(op, value)
+        op.write_double value
+      end
+    end
+
     module STRING
       module_function
 
@@ -33,5 +113,76 @@ module Stark
         op.write_string value
       end
     end
+
+    class Struct
+      def initialize(cls)
+        @class = cls
+      end
+
+      def type
+        Thrift::Types::STRUCT
+      end
+
+      def read(ip)
+        obj = @class.new
+        obj.read ip
+      end
+
+      def write(op, value)
+        value.write op
+      end
+    end
+
+    class Enum
+      def initialize(cls)
+        @class = cls
+      end
+
+      def type
+        Thrift::Types::I32
+      end
+
+      def read(ip)
+        @class[ip.read_i32]
+      end
+
+      def write(op, value)
+        op.write_i32 @class[value]
+      end
+    end
+
+    class List
+      def initialize(value)
+        @value = value
+      end
+
+      def type
+        Thrift::Types::LIST
+      end
+
+      def read(ip)
+        vt, size = ip.read_list_begin
+
+        if vt != @value.type
+          raise TypeError, "List expected to be type: #{@value.type}"
+        end
+
+        v = @value
+
+        Array.new(size) { v.read(ip) }
+      end
+
+      def write(op, value)
+        value = Array(value)
+
+        op.write_list_begin @value.type, value.size
+
+        c = @value
+        value.each { |v| c.write op, v }
+
+        op.write_list_end
+      end
+
+    end
   end
 end

data/lib/stark/parser.rb

@@ -2,3 +2,31 @@ module Stark
 end
 
 require 'stark/raw_parser'
+
+class Stark::Parser
+  def self.expand(ast)
+    out = []
+
+    ast.each do |elem|
+      case elem
+      when AST::Include
+        data = File.read elem.path
+        out += expand(Stark::Parser.ast(data))
+      else
+        out << elem
+      end
+    end
+
+    out
+  end
+
+  def ast
+    raise_error unless parse
+    Stark::Parser.expand result
+  end
+
+  def self.ast(arg)
+    parser = Stark::Parser.new arg
+    parser.ast
+  end
+end

data/lib/stark/raw_parser.rb

@@ -1609,7 +1609,7 @@ class Stark::Parser
     return _tmp
   end
 
-  # Element = (Comment | Header bsp | Definition bsp)
+  # Element = (Comment | obsp Header bsp | Definition bsp)
   def _Element
 
     _save = self.pos
@@ -1620,6 +1620,11 @@ class Stark::Parser
 
       _save1 = self.pos
       while true # sequence
+        _tmp = apply(:_obsp)
+        unless _tmp
+          self.pos = _save1
+          break
+        end
         _tmp = apply(:_Header)
         unless _tmp
           self.pos = _save1
@@ -3983,7 +3988,7 @@ class Stark::Parser
   Rules[:_CaptureDocText] = rule_info("CaptureDocText", "{}")
   Rules[:_DestroyDocText] = rule_info("DestroyDocText", "{}")
   Rules[:_HeaderList] = rule_info("HeaderList", "(HeaderList Header | Header)")
-  Rules[:_Element] = rule_info("Element", "(Comment | Header bsp | Definition bsp)")
+  Rules[:_Element] = rule_info("Element", "(Comment | obsp Header bsp | Definition bsp)")
   Rules[:_Header] = rule_info("Header", "(Include | Namespace)")
   Rules[:_Namespace] = rule_info("Namespace", "(\"namespace\" - tok_identifier:l - tok_identifier:n {namespace(l,n)} | \"namespace\" - \"*\" - tok_identifier:n {namespace(nil,n)})")
   Rules[:_Include] = rule_info("Include", "\"include\" - tok_literal:f {include(f)}")

data/lib/stark/ruby.rb

@@ -21,8 +21,22 @@ module Stark
       o "require 'stark/exception'"
     end
 
+    def run(ast)
+      ast.each { |a| a.accept self }
+      close
+    end
+
+    def close
+      if @namespace
+        outdent
+        o "end"
+      end
+    end
+
     def process_namespace(ns)
       @namespace = ns.namespace if ns.lang == "rb"
+      o "module #{ns.namespace}"
+      indent
     end
 
     def process_include(inc)
@@ -42,6 +56,20 @@ module Stark
       @enums[enum.name] = enum
     end
 
+    def converter(t)
+      if t.kind_of? Stark::Parser::AST::List
+        "Stark::Converters::List.new(#{converter(t.value)})"
+      elsif BUILTINS.include? t.downcase
+        "Stark::Converters::#{t.upcase}"
+      elsif desc = @structs[t]
+        "Stark::Converters::Struct.new(#{t})"
+      elsif desc = @enums[t]
+        "Stark::Converters::Enum.new(Enum_#{t})"
+      else
+        raise "Blah"
+      end
+    end
+
     def process_struct(str)
       @structs[str.name] = str
 
@@ -50,13 +78,12 @@ module Stark
       o "Fields = {"
       indent
 
-      str.fields.each do |f|
-        c = "Stark::Converters::#{f.type.upcase}"
-
-        o "#{f.index} => Stark::Field.new(#{f.index}, '#{f.name}', #{c}),"
+      fields = str.fields.map do |f|
+        c = converter f.type
+        "#{f.index} => Stark::Field.new(#{f.index}, '#{f.name}', #{c})"
       end
 
-      o ":count => #{str.fields.size}"
+      o "   #{fields.join(', ')}"
 
       outdent
       o "}"
@@ -70,6 +97,8 @@ module Stark
 
     end
 
+    BUILTINS = %w!bool byte i16 i32 i64 double string!
+
     def process_exception(str)
       @exceptions[str.name] = str
 
@@ -81,13 +110,12 @@ module Stark
       o "Fields = {"
       indent
 
-      str.fields.each do |f|
-        c = "Stark::Converters::#{f.type.upcase}"
-
-        o "#{f.index} => Stark::Field.new(#{f.index}, '#{f.name}', #{c}),"
+      fields = str.fields.map do |f|
+        c = converter f.type
+        "#{f.index} => Stark::Field.new(#{f.index}, '#{f.name}', #{c})"
       end
 
-      o ":count => #{str.fields.size}"
+      o "   #{fields.join(', ')}"
 
       outdent
       o "}"
@@ -187,19 +215,52 @@ module Stark
       WriteFunc[t] || raise("unknown type - #{t}")
     end
 
+    def write_field(ft, name, idx)
+      if desc = @structs[ft]
+        o "op.write_field_begin '#{name}', ::Thrift::Types::STRUCT, #{idx}"
+        output_struct desc, name
+        o "op.write_field_end"
+      elsif desc = @enums[ft]
+        o "op.write_field_begin '#{name}', ::Thrift::Types::I32, #{idx}"
+        o "op.write_i32 Enum_#{desc.name}[#{name}.to_sym]"
+
+        o "op.write_field_end"
+      elsif ft.kind_of? Stark::Parser::AST::Map
+        o "#{name} = hash_cast #{name}"
+        o "op.write_field_begin '#{name}', ::Thrift::Types::MAP, #{idx}"
+        o "op.write_map_begin(#{wire_type(ft.key)}, #{wire_type(ft.value)}, #{name}.size)"
+
+        o "#{name}.each do |k,v|"
+        indent
+        o "op.#{write_func(ft.key)} k"
+        o "op.#{write_func(ft.value)} v"
+        outdent
+        o "end"
+
+        o "op.write_map_end"
+        o "op.write_field_end"
+      elsif ft.kind_of? Stark::Parser::AST::List
+        o "#{name} = Array(#{name})"
+        o "op.write_field_begin '#{name}', ::Thrift::Types::LIST, #{idx}"
+        o "op.write_list_begin(#{wire_type(ft.value)}, #{name}.size)"
+        o "#{name}.each { |v| op.#{write_func(ft.value)}(v) }"
+        o "op.write_list_end"
+
+        o "op.write_field_end"
+      elsif ft != "void"
+        o "op.write_field_begin '#{name}', #{type(ft)}, #{idx}"
+        o "op.#{write_func(ft)} #{name}"
+        o "op.write_field_end"
+      end
+
+    end
+
     def output_struct(desc, obj)
       o "op.write_struct_begin '#{desc.name}'"
 
       desc.fields.each do |f|
-        if desc = @structs[f.type]
-          o "op.write_field_begin '#{f.name}', ::Thrift::Types::STRUCT, #{f.index}"
-          output_struct desc, f.name
-          o "op.write_field_end"
-        else
-          o "op.write_field_begin '#{f.name}', #{type(f.type)}, #{f.index}"
-          o "op.#{write_func(f.type)} #{obj}.#{f.name}"
-          o "op.write_field_end"
-        end
+        o "#{f.name} = #{obj}.#{f.name}"
+        write_field f.type, f.name, f.index
       end
 
       o "op.write_field_stop"
@@ -310,42 +371,7 @@ module Stark
 
         ft = func.return_type
 
-        if desc = @structs[ft]
-          o "op.write_field_begin 'result', ::Thrift::Types::STRUCT, 0"
-          output_struct desc, "result"
-          o "op.write_field_end"
-        elsif desc = @enums[ft]
-          o "op.write_field_begin 'result', ::Thrift::Types::I32, 0"
-          o "op.write_i32 Enum_#{desc.name}[result.to_sym]"
-
-          o "op.write_field_end"
-        elsif ft.kind_of? Stark::Parser::AST::Map
-          o "result = hash_cast result"
-          o "op.write_field_begin 'result', ::Thrift::Types::MAP, 0"
-          o "op.write_map_begin(#{wire_type(ft.key)}, #{wire_type(ft.value)}, result.size)"
-
-          o "result.each do |k,v|"
-          indent
-          o "op.#{write_func(ft.key)} k"
-          o "op.#{write_func(ft.value)} v"
-          outdent
-          o "end"
-
-          o "op.write_map_end"
-          o "op.write_field_end"
-        elsif ft.kind_of? Stark::Parser::AST::List
-          o "result = Array(result)"
-          o "op.write_field_begin 'result', ::Thrift::Types::LIST, 0"
-          o "op.write_list_begin(#{wire_type(ft.value)}, result.size)"
-          o "result.each { |v| op.#{write_func(ft.value)}(v) }"
-          o "op.write_list_end"
-
-          o "op.write_field_end"
-        elsif ft != "void"
-          o "op.write_field_begin 'result', #{type(ft)}, 0"
-          o "op.#{write_func(ft)} result"
-          o "op.write_field_end"
-        end
+        write_field ft, 'result', 0
 
         o "op.write_field_stop"
         o "op.write_struct_end"
@@ -393,42 +419,7 @@ module Stark
         o "op.write_struct_begin \"#{func.name}_args\""
 
         Array(func.arguments).each do |arg|
-          if desc = @structs[arg.type]
-            o "op.write_field_begin '#{arg.name}', ::Thrift::Types::STRUCT, #{arg.index}"
-            output_struct desc, arg.name
-            o "op.write_field_end"
-          elsif desc = @enums[arg.type]
-            o "op.write_field_begin '#{arg.name}', ::Thrift::Types::I32, #{arg.index}"
-            o "op.write_i32 Enum_#{desc.name}[#{arg.name}.to_sym]"
-
-            o "op.write_field_end"
-          elsif arg.type.kind_of? Stark::Parser::AST::Map
-            o "#{arg.name} = hash_cast #{arg.name}"
-            o "op.write_field_begin '#{arg.name}', ::Thrift::Types::MAP, #{arg.index}"
-            o "op.write_map_begin(#{wire_type(arg.type.key)}, #{wire_type(arg.type.value)}, #{arg.name}.size)"
-
-            o "#{arg.name}.each do |k,v|"
-            indent
-            o "op.#{write_func(arg.type.key)} k"
-            o "op.#{write_func(arg.type.value)} v"
-            outdent
-            o "end"
-
-            o "op.write_map_end"
-            o "op.write_field_end"
-          elsif arg.type.kind_of? Stark::Parser::AST::List
-            o "#{arg.name} = Array(#{arg.name})"
-            o "op.write_field_begin '#{arg.name}', ::Thrift::Types::LIST, #{arg.index}"
-            o "op.write_list_begin(#{wire_type(arg.type.value)}, #{arg.name}.size)"
-            o "#{arg.name}.each { |v| op.#{write_func(arg.type.value)}(v) }"
-            o "op.write_list_end"
-
-            o "op.write_field_end"
-          elsif arg.type != "void"
-            o "op.write_field_begin '#{arg.name}', #{type(arg.type)}, #{arg.index}"
-            o "op.#{write_func(arg.type)} #{arg.name}"
-            o "op.write_field_end"
-          end
+          write_field arg.type, arg.name, arg.index
         end
 
         o "op.write_field_stop"

data/lib/stark/struct.rb

@@ -15,6 +15,23 @@ module Stark
       end
     end
 
+    def read(ip)
+      ip.read_struct_begin
+
+      while true
+        _, ftype, fid = ip.read_field_begin
+        break if ftype == ::Thrift::Types::STOP
+
+        set_from_index ftype, fid, ip
+
+        ip.read_field_end
+      end
+
+      ip.read_struct_end
+
+      self
+    end
+
     def write_fields(op)
       self.class::Fields.each do |idx, field|
         next if idx == :count

data/lib/stark/thrift.kpeg

@@ -184,7 +184,7 @@ HeaderList = HeaderList Header
            | Header
 
 Element = Comment
-        | Header bsp
+        | obsp Header bsp
         | Definition bsp
 
 Header = Include | Namespace

data/lib/stark.rb

@@ -1,7 +1,7 @@
 require 'thrift'
 
 module Stark
-  VERSION = '0.6.1'
+  VERSION = '0.7.0'
 
   def self.pipe_transport
     cr, cw = IO.pipe
@@ -18,19 +18,16 @@ module Stark
     require 'stark/ruby'
     require 'stringio'
 
-    data = File.read file
-
-    tg = Stark::Parser.new data
-
-    unless tg.parse
-      tg.raise_error
-    end
+    ast = Stark::Parser.ast File.read(file)
 
     stream = StringIO.new
 
     ruby = Stark::Ruby.new stream
+    ruby.run ast
 
-    tg.result.each { |i| i.accept ruby }
+    if ENV['STARK_DEBUG']
+      puts stream.string
+    end
 
     namespace.module_eval stream.string
   end

data/test/blah.thrift

@@ -0,0 +1,5 @@
+enum Blah {
+  FOO
+  BAR
+  BAZ
+}