grammar 0.8 → 0.8.1

data/benchmark/json.benchmark.rb

@@ -6,25 +6,45 @@ require 'rubygems'
 
 class JSON_Generator
     attr_accessor(:array, :object)
+    def convert(obj)
+        obj.inspect.gsub(/=>/, ':').gsub(/\bnil\b/, 'null')
+    end
     def initialize
         @string1 = '1'
+        @string1s = @string1.inspect
         @string2 = '2'
+        @string2s = @string2.inspect
         @array1 = ['abc : 123', {}, false]
-        @array2 = [1.234e-5, {@string2 => nil}, '']
+        @array2 = [1.234e-5, {@string2 => nil}, ""]
         @object1 = {@string1 => '\\\n\t', @string2 => true}
         @object2 = {@string2 => -789, @string2 => []}
+        @array1s = @array1.map { |obj| convert(obj) }
+        @array2s = @array2.map { |obj| convert(obj) }
+        @object1s = @object1.map { |k,v| "#{convert(k)}:#{convert(v)}" }
+        @object2s = @object2.map { |k,v| "#{convert(k)}:#{convert(v)}" }
     end
     def grow
         @array1,
         @array2,
-        @object1,
-        @object2 =
+        @object2,
+        @object1 =
         @array1.reverse << @object2,
         @array2.reverse.concat(@array1),
-        @object1.merge({@string1 => @array2}),
-        @object2.merge(@object1)
+        @object2.merge(@object1),
+        @object1.merge!({@string1 => @array2})
+        @array1s,
+        @array2s,
+        @object2s,
+        @object1s = 
+        @array1s.reverse << "{#{@object2s.join(', ')}}",
+        @array2s.reverse.concat(@array1s),
+        @object2s.concat(@object1s),
+        @object1s << "#{@string1s}:[#{@array2s.join(', ')}]"
         @array2
     end
+    def to_s
+        "[#{@array2s.join(', ')}]"
+    end
 end
 
 def filesize(filename)
@@ -52,9 +72,6 @@ def benchmark(label='', filename=nil)
     load_path = $LOAD_PATH.clone
     loaded_features = $LOADED_FEATURES.clone
     !ARGV[0] or Regexp.new(ARGV[0])=~label or return
-    make = yield
-    limit = (ARGV[1]||1).to_f
-    max_tries = 4
     unless label.empty?
         STDERR.print(label)
         if filename
@@ -63,6 +80,9 @@ def benchmark(label='', filename=nil)
             STDERR.print(" -")
         end
     end
+    run = yield
+    limit = (ARGV[1]||1).to_f
+    max_tries = 4
     bandwidth = nil
     generator = JSON_Generator.new
     tree0 = nil
@@ -70,39 +90,38 @@ def benchmark(label='', filename=nil)
     tree = nil
     tries = 0
     done = false
+    t0 = nil
+    n = 1
     begin
         GC.start
-        tms = Benchmark.measure {
-            if tries>0
-                tries += 1
-            else
+        if tries>0
+            tries += 1
+        elsif t0 && t0>limit
+            n *= 2
+        else
+            tms = Benchmark.measure {
                 tree0 = generator.grow
-            end
-            s = tree0.inspect
-            s.gsub!(/=>/, ':')
-            s.gsub!(/nil/, 'null')
-        }
-        if !make
-            return if tms.utime+tms.stime>limit
-            next
+                s = generator.to_s
+            }
+            t0 = tms.utime+tms.stime
         end
         #puts s
-        run = make[s]
         GC.start
         GC.start
-        tms = Benchmark.measure { tree = run[] }
+        tms = Benchmark.measure { n.times { tree = run[s] } }
         GC.start
-        raise("#{tree.inspect}!=#{tree0.inspect}") if tree!=tree0
+        raise("#{tree.inspect}!=#{tree0.inspect}") if tree && tree!=tree0
         tree = nil
         t = tms.utime+tms.stime
-        bandwidth = s.length/t
-        s = nil
-        GC.start
+        bandwidth = (n*s.length)/t
         if tries.zero?
             if t>limit
                 tries = 1
+                GC.start
             else
-                next #if t<(limit/3)
+                s = nil unless t0>limit
+                GC.start
+                next
             end
         end
         if bandwidth>=1e9
@@ -122,11 +141,11 @@ def benchmark(label='', filename=nil)
     end until tries>=max_tries
     STDERR.print("\n")
     return bandwidth
-rescue => error
-    p error
+rescue ScriptError,StandardError => error
+    puts(" #{error}")
 ensure
-    #Gem.clear_paths
-    #Gem.loaded_specs.clear
+    begin;Gem.clear_paths;rescue Exception;end
+    begin;Gem.loaded_specs.clear;rescue Exception;end
     self.class.constants.each { |constant|
         next if constants[constant]
         #puts("removing #{constant}")
@@ -142,32 +161,47 @@ require 'stringio'
 json_pure = '/var/lib/gems/1.8/gems/json-1.1.2/lib/json/pure/parser.rb'
 json_ext = '/var/lib/gems/1.8/gems/json-1.1.2/ext/json/ext/parser/parser.rl'
 
-benchmark('') {}
+benchmark('String#index') {
+    lambda { |s|
+        s.index(?\0)
+        nil
+    }
+}
+
+benchmark('String#each_byte') {
+    lambda { |s|
+        s.each_byte{}
+        nil
+    }
+}
 
-benchmark('json/ext', json_ext) {
+benchmark('StringIO#getc') {
+    lambda { |s|
+        i = StringIO.new(s)
+        while i.getc;end
+        nil
+    }
+}
+
+benchmark('Ragel/C(json/ext)', json_ext) {
     gem 'json'
     require 'json/ext'
     lambda { |s|
-        JSON::Ext::Parser.new(s).method(:parse)
+        JSON::Ext::Parser.new(s).parse
     }
 }
 
 benchmark('Grammar/Ruby2CExt', 'json.grammar.rb') {
     $LOAD_PATH << '../lib'
-    require '../lib/grammar'
+    require 'grammar'
     grammar = Grammar.new.instance_eval(IO.read('json.grammar.rb'))
     Kernel.module_eval { alias_method(:_warn, :warn) }
     Kernel.module_eval { def warn(*);end }
     require 'grammar/ruby2cext'
     Kernel.module_eval { alias_method(:warn, :_warn) }
-    grammar_ruby2cext = Grammar::Ruby2CExt.compile(grammar)
+    grammar_ruby2cext = Grammar::Ruby2CExt.compile(grammar, :getc)
     lambda { |s|
-        i = StringIO.new(s).method(:getc)
-        out = []
-        lambda {
-            grammar_ruby2cext[out, i]
-            out[0]
-        }
+        grammar_ruby2cext[out=[], StringIO.new(s)] && out[0]
     }
 }
 
@@ -175,10 +209,7 @@ benchmark('hand/RE', 'json.re.rb') {
     load 'json.re.rb'
     re = JSON.new
     lambda { |s|
-        i = StringScanner.new(s)
-        lambda {
-            re.parse(i)
-        }
+        re.parse(StringScanner.new(s))
     }
 }
 
@@ -186,26 +217,19 @@ benchmark('hand/LL1', 'json.ll1.rb') {
     load 'json.ll1.rb'
     ll1 = JSON.new
     lambda { |s|
-        i = StringIO.new(s)
-        lambda {
-            ll1.parse(i)
-        }
+        ll1.parse(StringIO.new(s))
     }
 }
 
 benchmark('Grammar/Ruby', 'json.grammar.rb') {
     $LOAD_PATH << '../lib'
-    require '../lib/grammar'
+    require 'grammar'
     grammar = Grammar.new.instance_eval(IO.read('json.grammar.rb'))
     require 'grammar/ruby'
-    grammar_ruby = Grammar::Ruby.compile(grammar)
+    grammar_ruby = Grammar::Ruby.compile(grammar, :getc)
     lambda { |s|
-        i = StringIO.new(s).method(:getc)
-        out = []
-        lambda {
-            grammar_ruby[out, i]
-            out[0]
-        }
+        grammar_ruby[out=[], StringIO.new(s)] && out[0]
+        out[0]
     }
 }
 
@@ -222,10 +246,7 @@ benchmark('RACC/ext', 'json.y') {
     }
     racc = JSON.new
     lambda { |s|
-        i = StringScanner.new(s)
-        lambda {
-            racc.parse(i)
-        }
+        racc.parse(StringScanner.new(s))
     }
 }
 
@@ -233,7 +254,7 @@ benchmark('json/pure', json_pure) {
     gem 'json'
     require 'json/pure'
     lambda { |s|
-        JSON::Pure::Parser.new(s).method(:parse)
+        JSON::Pure::Parser.new(s).parse
     }
 }
 
@@ -242,19 +263,14 @@ benchmark('Grammar/RubyCall', 'json.grammar.rb') {
     require '../lib/grammar'
     grammar = Grammar.new.instance_eval(IO.read('json.grammar.rb'))
     require 'grammar/rubycall'
-    grammar_ruby = Grammar::RubyCall.compile(grammar)
+    grammar_ruby = Grammar::RubyCall.compile(grammar, :getc)
     lambda { |s|
-        i = StringIO.new(s).method(:getc)
-        out = []
-        lambda {
-            grammar_ruby[out, i]
-            out[0]
-        }
+        grammar_ruby[out=[], StringIO.new(s)] && out[0]
     }
 }
 
 benchmark('Grammar/0.5', 'json.grammar0_5.rb') {
-    gem 'grammar'
+    gem 'grammar', '0.5'
     require 'grammar'
     require 'duck'
     require 'json.grammar0_5'
@@ -262,11 +278,7 @@ benchmark('Grammar/0.5', 'json.grammar0_5.rb') {
     require 'cursor/io'
     grammar0_5 = JSON.new
     lambda { |s|
-        i = Cursor::IO.new(StringIO.new(s))
-        lambda {
-            grammar0_5.scan(i, out=[])
-            out[0]
-        }
+        grammar0_5.scan(Cursor::IO.new(StringIO.new(s)), out=[]) && out[0]
     }
 }
 
@@ -283,10 +295,7 @@ benchmark('RACC/ruby', 'json.y') {
     }
     racc = JSON.new
     lambda { |s|
-        i = StringScanner.new(s)
-        lambda {
-            racc.parse(i)
-        }
+        racc.parse(StringScanner.new(s))
     }
 }
 
@@ -296,9 +305,7 @@ benchmark('Treetop', 'json.treetop') {
     Treetop.load('json.treetop')
     treetop = JSONParser.new
     lambda { |s|
-        lambda {
-            treetop.parse(s).obj
-        }
+        treetop.parse(s).obj
     }
 }
 
@@ -307,9 +314,7 @@ benchmark('GhostWheel', 'json.ghostwheel') {
     require 'ghost_wheel'
     ghostwheel = GhostWheel.build_parser(IO.read("json.ghostwheel"))
     lambda { |s|
-        lambda {
-            ghostwheel.parse(s)
-        }
+        ghostwheel.parse(s)
     }
 }
 
@@ -320,12 +325,7 @@ benchmark('Grammar/Ruby0', 'json.grammar.rb') {
     require 'grammar/ruby0'
     grammar_ruby0 = Grammar::Ruby0.compile(grammar)
     lambda { |s|
-        i = StringIO.new(s).method(:getc)
-        out = []
-        lambda {
-            grammar_ruby0[out, i]
-            out[0]
-        }
+        grammar_ruby0[out=[], StringIO.new(s).method(:getc)] && out[0]
     }
 }
 
@@ -337,18 +337,8 @@ benchmark('Peggy', 'json.peggy.rb') {
     peggy = JSON.new
     lambda { |s|
         peggy.source_text = s
-        lambda {
-            peggy.parse?(:value)
-            peggy.to_ruby
-        }
-    }
-}
-
-benchmark('json/ext', json_ext) {
-    gem 'json'
-    require 'json/ext'
-    lambda { |s|
-        JSON::Ext::Parser.new(s).method(:parse)
+        peggy.parse?(:value)
+        peggy.to_ruby
     }
 }
 

data/benchmark/json.racc.rb

@@ -0,0 +1,268 @@
+#
+# DO NOT MODIFY!!!!
+# This file is automatically generated by racc 1.4.5
+# from racc grammer file "json.y".
+#
+
+require 'racc/parser'
+
+
+class JSON < Racc::Parser
+
+module_eval <<'..end json.y modeval..idf8edf52a5d', 'json.y', 26
+
+  def parse(input)
+    @input = input
+    yyparse self, :scan
+  end
+
+  private
+
+  def scan
+    input = @input
+    while true
+      if input.scan(/\s+/)
+      elsif input.scan(/"/)
+        s = ""
+        while true
+          if text=input.scan(/[^\\"]+/)
+            s.concat(text)
+          elsif input.scan(/\\/)
+            case (ch=input.getch[0])
+            when ?b ; s << ?\b
+            when ?f ; s << ?\f
+            when ?n ; s << ?\n
+            when ?r ; s << ?\r
+            when ?t ; s << ?\t
+            when ?u
+                text = input.scan(/[0-9a-fA-F]{4}/) or raise("expected hex*4")
+                s << text.to_i(16)
+            else    
+                s << ch
+            end
+          else
+            break
+          end
+        end
+        input.scan(/"/) or raise("expected quote")
+        yield :STRING, s
+      elsif text=input.scan(/-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?\s*/)
+        yield :NUMBER, eval(text)
+      elsif text=input.scan(/true/)
+        yield :BOOLEAN, true
+      elsif text=input.scan(/false/)
+        yield :BOOLEAN, false
+      elsif text=input.scan(/null/)
+        yield :BOOLEAN, nil
+      elsif text=input.scan(/./)
+        yield text, text
+      else
+        break
+      end
+    end
+    yield false, '$'
+  end
+
+..end json.y modeval..idf8edf52a5d
+
+##### racc 1.4.5 generates ###
+
+racc_reduce_table = [
+ 0, 0, :racc_error,
+ 1, 12, :_reduce_none,
+ 1, 12, :_reduce_none,
+ 1, 12, :_reduce_none,
+ 1, 12, :_reduce_none,
+ 1, 12, :_reduce_none,
+ 3, 13, :_reduce_6,
+ 2, 13, :_reduce_7,
+ 3, 15, :_reduce_8,
+ 5, 15, :_reduce_9,
+ 3, 14, :_reduce_10,
+ 2, 14, :_reduce_11,
+ 1, 16, :_reduce_12,
+ 3, 16, :_reduce_13 ]
+
+racc_reduce_n = 14
+
+racc_shift_n = 27
+
+racc_action_table = [
+     3,     4,     6,     7,    18,    23,    19,     8,    15,     3,
+     4,     6,     7,    20,    16,    21,     8,     3,     4,     6,
+     7,     9,    25,    17,     8,     3,     4,     6,     7,   nil,
+   nil,   nil,     8,     3,     4,     6,     7,    10,   nil,   nil,
+     8,    12 ]
+
+racc_action_check = [
+     8,     8,     8,     8,    11,    19,    11,     8,     8,    17,
+    17,    17,    17,    14,     9,    14,    17,    20,    20,    20,
+    20,     1,    23,    10,    20,     0,     0,     0,     0,   nil,
+   nil,   nil,     0,    25,    25,    25,    25,     7,   nil,   nil,
+    25,     7 ]
+
+racc_action_pointer = [
+    23,    21,   nil,   nil,   nil,   nil,   nil,    35,    -2,    14,
+    16,    -2,   nil,   nil,     5,   nil,   nil,     7,   nil,     3,
+    15,   nil,   nil,    15,   nil,    31,   nil ]
+
+racc_action_default = [
+   -14,   -14,    -2,    -1,    -4,    -3,    -5,   -14,   -14,   -14,
+   -14,   -14,    -7,   -12,   -14,   -11,    27,   -14,    -6,   -14,
+   -14,   -10,    -8,   -14,   -13,   -14,    -9 ]
+
+racc_goto_table = [
+     1,    11,    14,   nil,   nil,   nil,   nil,   nil,    13,   nil,
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,    22,   nil,   nil,
+    24,   nil,   nil,   nil,   nil,    26 ]
+
+racc_goto_check = [
+     1,     4,     5,   nil,   nil,   nil,   nil,   nil,     1,   nil,
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,     1,   nil,   nil,
+     1,   nil,   nil,   nil,   nil,     1 ]
+
+racc_goto_pointer = [
+   nil,     0,   nil,   nil,    -6,    -6 ]
+
+racc_goto_default = [
+   nil,   nil,     2,     5,   nil,   nil ]
+
+racc_token_table = {
+ false => 0,
+ Object.new => 1,
+ :STRING => 2,
+ :BOOLEAN => 3,
+ :NUMBER => 4,
+ "{" => 5,
+ "}" => 6,
+ ":" => 7,
+ "," => 8,
+ "[" => 9,
+ "]" => 10 }
+
+racc_use_result_var = false
+
+racc_nt_base = 11
+
+Racc_arg = [
+ racc_action_table,
+ racc_action_check,
+ racc_action_default,
+ racc_action_pointer,
+ racc_goto_table,
+ racc_goto_check,
+ racc_goto_default,
+ racc_goto_pointer,
+ racc_nt_base,
+ racc_reduce_table,
+ racc_token_table,
+ racc_shift_n,
+ racc_reduce_n,
+ racc_use_result_var ]
+
+Racc_token_to_s_table = [
+'$end',
+'error',
+'STRING',
+'BOOLEAN',
+'NUMBER',
+'"{"',
+'"}"',
+'":"',
+'","',
+'"["',
+'"]"',
+'$start',
+'value',
+'object',
+'array',
+'members',
+'elements']
+
+Racc_debug_parser = false
+
+##### racc system variables end #####
+
+ # reduce 0 omitted
+
+ # reduce 1 omitted
+
+ # reduce 2 omitted
+
+ # reduce 3 omitted
+
+ # reduce 4 omitted
+
+ # reduce 5 omitted
+
+module_eval <<'.,.,', 'json.y', 10
+  def _reduce_6( val, _values)
+ val[1]
+  end
+.,.,
+
+module_eval <<'.,.,', 'json.y', 11
+  def _reduce_7( val, _values)
+ {}
+  end
+.,.,
+
+module_eval <<'.,.,', 'json.y', 13
+  def _reduce_8( val, _values)
+ {val[0] => val[2]}
+  end
+.,.,
+
+module_eval <<'.,.,', 'json.y', 14
+  def _reduce_9( val, _values)
+ val[0][val[2]] = val[4]; val[0]
+  end
+.,.,
+
+module_eval <<'.,.,', 'json.y', 16
+  def _reduce_10( val, _values)
+ val[1]
+  end
+.,.,
+
+module_eval <<'.,.,', 'json.y', 17
+  def _reduce_11( val, _values)
+ []
+  end
+.,.,
+
+module_eval <<'.,.,', 'json.y', 19
+  def _reduce_12( val, _values)
+ val
+  end
+.,.,
+
+module_eval <<'.,.,', 'json.y', 20
+  def _reduce_13( val, _values)
+ val[0] << val[2]
+  end
+.,.,
+
+ def _reduce_none( val, _values)
+  val[0]
+ end
+
+end   # class JSON
+
+
+if $0 == __FILE__
+  src = <<EOS
+{
+  "name" : null,
+  "id" : 12.3
+}
+EOS
+  require 'strscan'
+  puts 'Parsing (String):'
+  print src
+  puts
+  puts 'Result (Ruby Object):'
+  p JSON_RACC.new.parse(StringScanner.new(src))
+end
+
+

data/lib/grammar/ruby.rb

@@ -6,8 +6,8 @@ class Ruby
     # compile a grammar to a parser.  The parser can be called with
     # an input (#[] to get next element) and an output (#<< appends
     # to it).
-    def self.compile(gram)
-        code = new.dump(gram)
+    def self.compile(gram, input_method=:[])
+        code = new(input_method).dump(gram)
         #p(code)
         #code.lined
         eval(code.to_s).new
@@ -71,15 +71,19 @@ class Ruby
         @output << item
     end    
     
-    def initialize
+    def initialize(input_method=:[])
         @followed = 0
         @consumed = 0
         @output0 = Code::Primary.new("o")
         @output = @output0
+        @output_init = nil
         @input0 = Code::Primary.new("i")
         @input = @input0
+        @input_method = input_method
         @lookahead0 = Code::Primary.new("la")
         @lookahead = @lookahead0
+        @lookahead_get = Code::Self.at(Code[0])
+        @lookahead_set = Code::Self[Code[0]]
         @fail0 = Code::Primary.new("f")
         @fail = @fail0
         @failure_hard = lambda { |i, la, *expected|
@@ -99,7 +103,7 @@ class Ruby
         }
         @method_definitions = []
         @methods = {}
-        @consume = generate_consume
+        @consume = true
         @void = nil
         @scope = nil
     end
@@ -110,14 +114,6 @@ class Ruby
         @assignments << var._assign(code)
     end
     
-    def generate_consume
-        @consume = Code::SharedSteps.new(
-            (@output ? [@output << @lookahead] : []) <<
-            @lookahead._assign(@input[]) <<
-            Code::True
-        )
-    end
-
     class CodeProxy
         include Enumerable
         def initialize(code)
@@ -167,24 +163,25 @@ class Ruby
         @scope = "call"
         top = gram[self]
         @scope = scope
-        CodeProxy.new(Code[Class].new { ||
+        CodeProxy.new(Code[Class].new(Code[Array]) { ||
             steps = []
             @assignments.each { |assignment|
                 steps << assignment
             }
+            steps << Code::Implicit.super(Code[1])
             steps = [Code::Steps.new(steps)._def(:initialize)] unless
                 steps.empty?
             @method_definitions.each { |definition|
                 steps << definition
             }
-            steps << top._ensure(
+            steps << top._and {
                 Code::Self.block_given?._and{
                     Code::Implicit.yield(@lookahead)
-                }
-            )._def(:call,
+                }._step(Code::True)
+            }._def(:call,
                 @output,
                 @input,
-                @lookahead._assign(@input[]),
+                @lookahead._assign(@input.method_missing(@input_method)),
                 @fail._assign(Code::Self.lambda { |i, la, *exp|
                     exp = exp[0]
                     Code::Implicit.raise(
@@ -213,9 +210,23 @@ class Ruby
         end
     end
     def consume(matching, expected=nil)
-        matching._and{@consume||Code::True}._or{failure(expected)}
+        if @consume
+            matching = matching._and {
+                consume = @lookahead._assign(
+                    @input.method_missing(@input_method)
+                )
+                if @output
+                    consume = ((@output_init ? @output._assign(@output_init) : 
+                                               @output) << @lookahead).
+                              _step(consume)
+                end
+                consume._step(Code::True)
+            }
+        end
+        matching._or{failure(expected)}
     ensure
         if @consume
+            @output_init = nil
             @failure = @failure_hard
             @consumed += 1
         end
@@ -261,14 +272,33 @@ class Ruby
     def alternation(gram1) # :yield: engine
         failure = @failure
         @failure = nil
-        begin
-            gram1[self]
-        ensure
-            failure1 = @failure
-        end._or{
-            @failure = failure
-            yield(self)
-        }
+        if output_init = @output_init
+            begin
+                gram1[self]._and {
+                    @output_init ? @output._assign(@output_init) : Code::True
+                }
+            ensure
+                failure1 = @failure
+                output_init1 = @output_init
+            end._or{
+                @failure = failure
+                @output_init = output_init
+                code = yield(self)
+                if !output_init1 and @output_init
+                    code = @output._assign(@output_init)._step(code)
+                end
+                code
+            }
+        else
+            begin
+                gram1[self]
+            ensure
+                failure1 = @failure
+            end._or{
+                @failure = failure
+                yield(self)
+            }
+        end
     ensure
         @failure = failure1 if failure1
     end
@@ -382,10 +412,12 @@ class Ruby
                 @fail = @fail0
                 @failure = nil
                 @expected = []
-                generate_consume
+                @consume = true
                 code = inner ? recurse(inner, true, &outer) : outer[self]
                 @method_definitions[i] =
-                    code._ensure(Code::Implicit.yield(@lookahead))._def(
+                    code._and {
+                        @lookahead_set._assign(@lookahead)._step(Code::True)
+                    }._def(
                         name,
                         @output,
                         @input,
@@ -412,9 +444,8 @@ class Ruby
             }.new
         ))
         call = Code::Self.method_missing(name, output, @input,
-                                         @lookahead, @fail) {
-            |la|
-            @lookahead._assign(la)
+                                         @lookahead, @fail)._and {
+            @lookahead._assign(@lookahead_get)._step(Code::True)
         }
         if expected.empty?
             Code::Always.new(call)
@@ -496,17 +527,24 @@ class Ruby
         Code::Local.new { |buf|
             begin
                 @output = buf
-                @consume = generate_consume
-                @output._assign(self[buf0,true])._step(
+                @consume = true
+                output_init = self[buf0,true]
+                if @failure
+                    @output._assign(output_init)._step(gram[self])
+                else
+                    @output_init = output_init
                     gram[self]
-                )
+                end
             ensure
                 @followed -= 1 if block_given?
                 @output = output
                 @consume = consume
             end._and {
                 !block_given? ? Code::True :
-                Code::Always.new(block.arity==1 ? yield(buf) : yield(buf, self))
+                (@output_init ? (@output._assign(@output_init)) : Code::True).
+                _step(Code::Always.new(
+                    block.arity==1 ? yield(buf) : yield(buf, self)
+                ))
             }
         }
     end
@@ -518,7 +556,7 @@ class Ruby
         @followed += 1 if block_given?
         begin
             @output = nil
-            @consume = generate_consume
+            @consume = true
             gram[self]
         ensure
             @followed -= 1 if block_given?
@@ -547,7 +585,7 @@ class Ruby
                             buf.slice!(Code[0], buf.size),
                             begin
                                 @output = buf
-                                @consume = generate_consume
+                                @consume = true
                                 @expected = []
                                 tokenizer[self]
                             ensure
@@ -561,7 +599,7 @@ class Ruby
                     begin
                         @input = input1
                         @lookahead = lookahead1
-                        @consume = generate_consume
+                        @consume = true
                         @expected = []
                         parser[self]
                     ensure

data/lib/grammar/ruby2cext.rb

@@ -1,13 +1,14 @@
 
 require 'grammar/ruby'
 require 'rubygems'
+gem 'ruby2cext'
 require 'ruby2cext/eval2c'
 
 class Grammar
 class Ruby2CExt < Ruby
     
-    def self.compile(gram)
-        code = new.dump(gram)
+    def self.compile(gram, input_method=:[])
+        code = new(input_method).dump(gram)
         #p(code)
         #code.lined
         Ruby2CExtension::Eval2C.new.compile_to_proc(code.to_s).call.new

data/lib/grammar/ruby2cext_hack.rb

@@ -0,0 +1,319 @@
+
+require 'grammar/ruby'
+require 'rubygems'
+gem 'ruby2cext'
+require 'ruby2cext/compiler'
+require 'digest/sha1'
+
+class Grammar
+class Ruby2CExt < Ruby
+    
+    def self.compile(gram, input_method=:[])
+        code = new(input_method).dump(gram)
+        ruby_code = code.to_s
+        #puts(code)
+        path = File.join(File.expand_path(
+            ENV["HOME"] || ENV["USERPROFILE"] || ENV["HOMEPATH"] || "."),
+            ".ruby2cext")
+        prefix = 'grammar'
+        name = "#{prefix}_#{Digest::SHA1.hexdigest(ruby_code)}"
+        gvar_name = "$__#{name}__"
+        compiler = Ruby2CExtension::Compiler.new(name)
+        compiler.add_plugins({:optimizations => :all})
+        compiler.add_rb_file("#{gvar_name} = #{ruby_code}", name)
+        c_code = compiler.to_c_code
+        
+        ws = /(?:\s|\/\*.*?\*\/)/m
+        sym2name = []
+        c_code.scan(
+            /^\s*sym\s*\[\s*(\d+)\s*\]\s*\=\s*rb_intern\s*\(\s*\"([^\"]*)\"\s*\)/
+        ) { |i,ruby|
+            sym2name[i.to_i] = ruby;
+        }
+        
+        #c_code.gsub!(/\b
+        #    switch\s*\(TYPE\s*\(argv\[0\]\)\)\s*\{\s*
+        #    case\s+T_FIXNUM:\s*
+        #        (res\s*=.*)\s*
+        #        break;\s*
+        #    default:\s*
+        #        res\s*=.*\s*
+        #    \}
+        #/x, '\1')
+        
+        #c_code.gsub!(/\b
+        #  if\s*\(\!RTEST\s*\(Qtrue\)\)\s*
+        #      break;
+        #/x, ';')
+        #c_code.gsub!(/^\b(VALUE )?while_res = Qnil;/, ';')
+        #c_code.gsub!(/\bres = while_res;/, ';')
+        
+        ##n = 0
+        ##c_code.scan(/VALUE argv\[(\d+)\]/) { |i|
+        ##    i = i[0].to_i
+        ##    n = i if i>n
+        ##}
+        ##c_code.gsub!(/^\s*VALUE argv\[\d+\]\;\n/, '')
+        ##c_code.gsub!(/\bVALUE (recv\s*\=)/, '\1')
+        ##c_code.gsub!(/^(\s*)(VALUE res;\n)/,
+        ##    "\\1\\2\\1VALUE recv;\nVALUE argv[#{n}];\n")
+        ruby2c = Hash.new { |h,k|
+            lambda { |header, assign, call, footer, *argv|
+                call.gsub!(/\brecv\b/, 'self')
+                "#{header}#{assign}#{call}#{footer}"
+            }
+        }
+        ruby2c['at'] = lambda { |header, assign, call, footer, *argv|
+            "res = RARRAY(self)->ptr[NUM2LONG(#{argv[0]})];"
+        }
+        ruby2c['[]='] = lambda { |header, assign, call, footer, *argv|
+            "res = RARRAY(self)->ptr[NUM2LONG(#{argv[0]})] = #{argv[1]};"
+        }
+        c_code.scan(
+            /^\s*rb_define_method\s*\(s_class\,\s*\"([^\"]*)\"\,\s*(\w+)\,\s*\-1\)/
+        ) { |ruby,c|
+            ruby2c[ruby] = lambda { |header, assign, call, footer, *argv|
+                "#{header}#{assign}res = #{c}(argc, argv, self);#{footer}"
+            }
+        }
+        c_code.gsub!(/
+            (\{\s*)
+                VALUE\s+recv\s*\=\s*self\s*\;\s*
+                (
+                const\s+int\s+argc\s*\=\s*\d+\s*\;\s*
+                VALUE\s+argv\s*\[\s*\d+\s*\]\s*\;\s*
+                (?:(?>\/\*.*?\*\/)\s*argv\s*\[\s*\d+\s*\]\s*\=\s*(?>[^\;]+\;)\s*)*
+                )
+                (
+                    (?:res\s*\=\s*)?rb_funcall2\s*\(\s*
+                        recv\s*\,\s*
+                        sym\s*\[\s*(\d+)\s*\]\s*(?>\/\*.*?\*\/)\s*\,\s*
+                        argc\s*\,\s*
+                        argv\s*\)\s*\;
+                    (?:\s*res\s*=\s*argv\s*\[\s*argc\s*\-\s*1\s*\]\s*\;)?
+                )
+            (\s*\})
+        /xm) { |match|
+            header = $1
+            recv = $2
+            assign = $2
+            call = $3
+            symnum = $4.to_i
+            footer = $5
+            ruby = sym2name[symnum]
+            c = ruby2c[ruby]
+            argv = []
+            assign.scan(/argv#{ws}*\[#{ws}*(\d+)#{ws}*\]#{ws}*\=#{ws}*([^\;]+)/) {
+                |i,arg|
+                argv[i.to_i] = arg
+            }
+            c[header, assign, call, footer, *argv]
+        }
+        
+        c_code.gsub!(/(?=\bstatic\s+BUILTINOPT_FP\s+builtinopt_method_lookup)/) {
+            %q{
+                #include <stdarg.h>
+                typedef struct {
+                    BUILTINOPT_FP fp;
+                    void* data;
+                } ruby_method;
+                static VALUE normal_call0(VALUE recv, ruby_method* m) {
+                    return rb_funcall3(recv, (ID)(m->data), 0, 0);
+                }
+                static VALUE remember_call0(VALUE recv, ruby_method* m) {
+                    VALUE klass = CLASS_OF(recv);
+                    NODE* body;
+                    while (!st_lookup(RCLASS(klass)->m_tbl, (ID)(m->data),
+                                      (st_data_t *)&body))
+                    {
+                        klass = RCLASS(klass)->super;
+                        if (!klass) {
+                            m->fp = &normal_call0;
+                            return rb_funcall3(recv, (ID)(m->data), 0, 0);
+                        }
+                    }
+                    body = body->nd_body;
+                    if (nd_type(body)==NODE_FBODY) body = body->nd_head;
+                    if (nd_type(body)==NODE_CFUNC && body->nd_argc==0) {
+                        m->fp = body->nd_cfnc;
+                        return (*(m->fp))(recv, m);
+                    } else {
+                        m->fp = &normal_call0;
+                        return rb_funcall3(recv, (ID)(m->data), 0, 0);
+                    }
+                }
+                static ruby_method next_input;
+            }
+        }
+        
+        input_sym_num = nil
+        c_code.gsub!(/\b
+            rb_funcall3\s*\(\s*
+                lvar\s*\[\s*1\s*\]\s*,\s*
+                sym\s*\[\s*(\d+)\s*\]\s*(?>\/\*.*?\*\/)\s*,\s*
+                0\s*,\s*
+                0\s*\)
+        /xm) { |match|
+            input_sym_num = $1.to_i unless input_sym_num
+            (input_sym_num!=$1.to_i) ? match :
+            "(*next_input.fp)(lvar[1], &next_input)"
+        }
+        if input_sym_num
+            #c_code.gsub!(/(\bVALUE\s+lvar\s*\[\s*\d+\s*\]\s*\;)(\s*)/) {
+            #    "#{$1}#{$2}ruby_method next_input;#{$2}"
+            #}
+            c_code.gsub!(/\bif\s*\(\s*meth_argc\s*>\s*2\s*\)/) { |match|
+            #c_code.gsub!(/\blvar\s*\[\s*1\s*\]\s*\=/) { |match|
+                "next_input.fp = &remember_call0; " +
+                "next_input.data = (void*)sym[#{input_sym_num}]; " +
+                match
+            }
+        end
+        
+        builtinopt_method = {}
+        c_code.scan(/
+            \bbuiltinopt_method_tbl\s*\[\s*(\d+)\s*\]\s*\=\s*
+                builtinopt_method_lookup\s*\(\s*
+                    (\w+)\s*\,\s*
+                    (\w+)\s*\,\s*
+                    sym\s*\[\s*(\d+)\s*\]\s*(?>\/\*.*?\*\/)\s*\,\s*
+                    (-?\d+)\s*\)\s*\;
+        /xm) { |i,klass,origin,sym,arity|
+            builtinopt_method[[klass,sym2name[sym.to_i]]] = [i.to_i,arity.to_i]
+        }
+        
+        #begin
+        #    lvars = []
+        #    done = true
+        #    c_code.gsub!(/
+        #        \blvar\s*\[\s*(\d+)\s*\]\s*\=\s*(\S\w*)
+        #        |
+        #        \{\s*
+        #            VALUE\s+recv\s*\=\s*
+        #              ((?:\(\s*)?lvar\s*\[\s*(\d+)\s*\]
+        #              (?:\s*\=\s*(\S\w*)[^;]*?\))?)\s*\;\s*
+        #            VALUE\s+argv\s*\[\s*1\s*\]\s*\;\s*
+        #            (?>\/\*.*?\*\/)\s*
+        #            (
+        #            (?>\/\*.*?\*\/)\s*
+        #            \{[^}]*\}\s*
+        #            )?
+        #            argv\s*\[\s*0\s*\]\s*\=\s*([^;]*?)\s*\;\s*
+        #            res\s*\=\s*builtinopt(\w+)__1\s*\(\s*
+        #                recv\s*\,\s*
+        #                argv\s*
+        #            \)\s*\;\s*
+        #        \}
+        #    /xm) { |match|
+        #        if $1
+        #            lvars[$1.to_i] = $2
+        #            match
+        #        else
+        #            recv = $4.to_i
+        #            lvars[recv] = $5 if $5
+        #            klass = case lvars[recv]
+        #            when "rb_ary_new"
+        #                "rb_cArray"
+        #            when "rb_str_new3"
+        #                "rb_cString"
+        #            else
+        #                nil
+        #            end
+        #            recv = $3
+        #            arg0 = $6||""
+        #            arg = $7
+        #            meth = case $8
+        #            when "op_lshift"
+        #                "<<"
+        #            when /meth_(.*)/
+        #                $1
+        #            end
+        #            builtin,arity = builtinopt_method[[klass, meth]]
+        #            if builtin && arity==-1
+        #                done = false
+        #                arg0 +
+        #                "{ " \
+        #                "VALUE argv[1]; " \
+        #                "argv[0] = #{arg}; " \
+        #                "res = (*(builtinopt_method_tbl[#{builtin}]))(1, #{recv}, argv);" \
+        #                "}"
+        #            elsif builtin && arity==1
+        #                done = false
+        #                arg0 +
+        #                "res = (*(builtinopt_method_tbl[#{builtin}]))(#{recv}, #{arg});"
+        #            else
+        #                match
+        #            end
+        #        end
+        #    }
+        #end until done
+        #c_code.gsub!(/
+        #    (\blvar\s*\[\s*(\d+)\s*\]\s*\=\s*)(
+        #        builtinopt(\w+)__0\s*\(\s*
+        #            lvar\s*\[\s*(\d+)\s*\]\s*
+        #        \)
+        #        |
+        #        \S\w*
+        #    )
+        #/xm) { |match|
+        #    if !$4
+        #        lvars[$2.to_i] = $3
+        #        match
+        #    else
+        #        recv = $5.to_i
+        #        klass = case lvars[recv]
+        #        when "rb_ary_new"
+        #            "rb_cArray"
+        #        when "rb_str_new3"
+        #            "rb_cString"
+        #        else
+        #            nil
+        #        end
+        #        header = $1
+        #        recv = "lvar[#{recv}]"
+        #        meth = case $4
+        #        when "op_lshift"
+        #            "<<"
+        #        when /meth_(.*)/
+        #            $1
+        #        end
+        #        builtin,arity = builtinopt_method[[klass, meth]]
+        #        if builtin && arity==-1
+        #            done = false
+        #            header +
+        #            "(*(builtinopt_method_tbl[#{builtin}]))(0, #{recv}, 0)"
+        #        elsif builtin && arity==0
+        #            done = false
+        #            header +
+        #            "(*(builtinopt_method_tbl[#{builtin}]))(#{recv})"
+        #        else
+        #            match
+        #        end
+        #    end
+        #}
+        
+        c_filename = File.join(path, "#{name}.c")
+        File.open(c_filename, "w") { |f| f.puts(c_code) }
+        #system("indent -fc1 -st #{c_filename}")
+        #p ::Config::CONFIG
+        
+        dl_filename = File.join(path,
+            "#{name}.#{Ruby2CExtension::Compiler::DLEXT}")
+        cflags = ::Config::CONFIG['CFLAGS']
+        #::Config::CONFIG['CFLAGS'] = cflags.gsub(/-g /,'') + ' -O3'
+        begin
+            Ruby2CExtension::Compiler.compile_c_file_to_dllib(c_filename)
+        ensure
+            ::Config::CONFIG['CFLAGS'] = cflags
+        end == dl_filename or
+            raise(Ruby2CExtension::Ruby2CExtError::Bug,
+                "unexpected return value from compile_c_file_to_dllib")
+        
+        require(dl_filename)
+        eval(gvar_name).new
+    end
+
+end
+end
+
+

data/lib/grammar/rubycall.rb

@@ -5,8 +5,8 @@ require 'rubygems'
 class Grammar
 class RubyCall < Ruby
     
-    def self.compile(gram)
-        code = new.dump(gram)
+    def self.compile(gram, input_method=:[])
+        code = new(input_method).dump(gram)
         #p(code)
         #code.lined
         eval(code.to_s).new

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: grammar
 version: !ruby/object:Gem::Version 
-  version: "0.8"
-date: 2008-09-05 00:00:00 -05:00
+  version: 0.8.1
+date: 2008-09-17 00:00:00 -05:00
 summary: BNF-like grammar specified directly in ruby
 require_paths: 
 - lib
@@ -34,6 +34,7 @@ files:
 - lib/grammar/ruby0.rb
 - lib/grammar/ruby.rb
 - lib/grammar/rubycall.rb
+- lib/grammar/ruby2cext_hack.rb
 - lib/grammar/ruby/code.rb
 - test/molecules.rb
 - test/atoms.rb
@@ -46,6 +47,7 @@ files:
 - test/test_demo.rb
 - test/basic.rb
 - benchmark/json.benchmark.rb
+- benchmark/json.racc.rb
 - benchmark/json.grammar.rb
 - benchmark/json.grammar0_5.rb
 - benchmark/json.ll1.rb