rattler 0.3.0 → 0.4.0

data/lib/rattler/back_end/optimizer/simplify_token_match.rb

@@ -0,0 +1,38 @@
+#
+# = rattler/back_end/optimizer/simplify_token_match.rb
+#
+# Author:: Jason Arhart
+# Documentation:: Author
+#
+require 'rattler'
+
+module Rattler::BackEnd::Optimizer
+  #
+  # A token wrapping a terminal parser is redundant.
+  #
+  # @author Jason Arhart
+  #
+  class SimplifyTokenMatch < Optimization
+
+    include Rattler::Parsers
+
+    protected
+
+    def _applies_to?(parser, context)
+      parser.is_a? Token and
+      terminal? parser.child
+    end
+
+    def _apply(parser, context)
+      parser.child
+    end
+
+    def terminal?(parser)
+      case parser
+      when Match, Token, BackReference then true
+      else false
+      end
+    end
+
+  end
+end

data/lib/rattler/back_end/parser_generator.rb

@@ -14,6 +14,7 @@ module Rattler::BackEnd
   #
   module ParserGenerator
 
+    autoload :RuleSetGenerator, 'rattler/back_end/parser_generator/rule_set_generator'
     autoload :RuleGenerator, 'rattler/back_end/parser_generator/rule_generator'
     autoload :ExprGenerator, 'rattler/back_end/parser_generator/expr_generator'
     autoload :GeneratorHelper, 'rattler/back_end/parser_generator/generator_helper'
@@ -39,12 +40,15 @@ module Rattler::BackEnd
     autoload :TokenGenerator, 'rattler/back_end/parser_generator/token_generator'
     autoload :SkipGenerator, 'rattler/back_end/parser_generator/skip_generator'
     autoload :LabelGenerator, 'rattler/back_end/parser_generator/label_generator'
+    autoload :BackReferenceGenerator, 'rattler/back_end/parser_generator/back_reference_generator'
     autoload :FailGenerator, 'rattler/back_end/parser_generator/fail_generator'
     autoload :RepeatGenerating, 'rattler/back_end/parser_generator/repeat_generating'
     autoload :ListGenerating, 'rattler/back_end/parser_generator/list_generating'
     autoload :PredicatePropogating, 'rattler/back_end/parser_generator/predicate_propogating'
     autoload :TokenPropogating, 'rattler/back_end/parser_generator/token_propogating'
     autoload :SkipPropogating, 'rattler/back_end/parser_generator/skip_propogating'
+    autoload :GroupMatchGenerator, 'rattler/back_end/parser_generator/group_match_generator'
+    autoload :GroupMatch, 'rattler/back_end/parser_generator/group_match'
     autoload :GEN_METHOD_NAMES, 'rattler/back_end/parser_generator/gen_method_names'
 
     # Generate parsing code for a parser model using a ruby generator +g+.
@@ -52,50 +56,53 @@ module Rattler::BackEnd
     # @overload generate(g, grammar)
     #   @param [RubyGenerator] g the ruby generator to use
     #   @param [Rattler::Grammar::Grammar] grammar the grammar model
-    #   @return nil
+    #   @return g
     #
     # @overload generate(g, rules)
     #   @param [RubyGenerator] g the ruby generator to use
     #   @param [Rules] rules the parse rules
-    #   @return nil
+    #   @return g
     #
     # @overload generate(g, rule)
     #   @param [RubyGenerator] g the ruby generator to use
     #   @param [Rule] rule the parse rule
-    #   @return nil
+    #   @return g
     #
     # @overload generate(g, parser)
     #   @param [RubyGenerator] g the ruby generator to use
     #   @param [Parser] parser the parser model
-    #   @return nil
+    #   @return g
     #
-    def self.generate(g, parser)
-      RuleGenerator.new(g).generate(parser)
-      nil
+    def self.generate(g, parser, opts={})
+      unless opts[:no_optimize]
+        parser = ::Rattler::BackEnd::Optimizer.optimize(parser, opts)
+      end
+      RuleSetGenerator.new(g).generate(parser, opts)
+      g
     end
 
     # Generate parsing code for +parser+ using a new {RubyGenerator} with the
     # given options and return the generated code.
     #
-    # @overload code_for(grammar, options)
+    # @overload code_for(grammar, opts)
     #   @param [Rattler::Grammar::Grammar] grammar the grammar model
     #   @return [String] the generated code
     #
-    # @overload code_for(rules, options)
+    # @overload code_for(rules, opts)
     #   @param [Rules] rules the parse rules
     #   @return [String] the generated code
     #
-    # @overload code_for(rule, options)
+    # @overload code_for(rule, opts)
     #   @param [Rule] rule the parse rule
     #   @return [String] the generated code
     #
-    # @overload code_for(parser, options)
+    # @overload code_for(parser, opts)
     #   @param [Parser] parser the parser model
     #   @return [String] the generated code
     #
-    def self.code_for(parser, options={})
-      ::Rattler::BackEnd::RubyGenerator.code(options) {|g| generate(g, parser) }
+    def self.code_for(parser, opts={})
+      ::Rattler::BackEnd::RubyGenerator.code(opts) {|g| generate g, parser, opts }
     end
 
   end
-end
+end

data/lib/rattler/back_end/parser_generator/apply_generator.rb

@@ -1,87 +1,87 @@
 require 'rattler/back_end/parser_generator'
 
 module Rattler::BackEnd::ParserGenerator
-  
+
   # @private
   class ApplyGenerator < ExprGenerator #:nodoc:
-    
-    def gen_basic(apply)
+
+    def gen_basic(apply, scope={})
       @g << "match(:#{apply.rule_name})"
     end
-    
-    def gen_assert_nested(apply)
+
+    def gen_assert_nested(apply, scope={})
       atomic_block { gen_assert_top_level apply }
     end
-    
-    def gen_assert_top_level(apply)
+
+    def gen_assert_top_level(apply, scope={})
       lookahead do
         @g.surround("#{result_name} = (", ')') { gen_skip_top_level apply }
         @g.newline
       end
       @g << result_name
     end
-    
-    def gen_disallow_nested(apply)
+
+    def gen_disallow_nested(apply, scope={})
       atomic_block { gen_disallow_top_level apply }
     end
-    
-    def gen_disallow_top_level(apply)
+
+    def gen_disallow_top_level(apply, scope={})
       lookahead do
         @g.surround("#{result_name} = !", '') { gen_basic_nested apply }
         @g.newline
       end
       @g << result_name
     end
-    
-    def gen_dispatch_action_nested(apply, target, method_name)
-      atomic_expr { gen_dispatch_action_top_level apply, target, method_name }
+
+    def gen_dispatch_action_nested(apply, code, scope={})
+      atomic_expr { gen_dispatch_action_top_level apply, code }
     end
-    
-    def gen_dispatch_action_top_level(apply, target, method_name)
+
+    def gen_dispatch_action_top_level(apply, code, scope={})
       @g.surround("(#{result_name} = ", ')') { gen_basic apply }
-      @g << ' && ' << dispatch_action_result(target, method_name)
+      @g << ' && ' << code.bind(scope, "[#{result_name}]")
     end
-    
-    def gen_direct_action_nested(apply, action)
-      atomic_expr { gen_direct_action_top_level apply, action }
+
+    def gen_direct_action_nested(apply, code, scope={})
+      atomic_expr { gen_direct_action_top_level apply, code }
     end
-    
-    def gen_direct_action_top_level(apply, action)
+
+    def gen_direct_action_top_level(apply, code, scope={})
       @g.surround("(#{result_name} = ", ')') { gen_basic apply }
-      @g << ' && ' << direct_action_result(action)
+      @g << ' && (' << code.bind(scope, [result_name]) << ')'
     end
-    
-    def gen_skip_nested(apply)
+
+    def gen_skip_nested(apply, scope={})
       atomic_expr { gen_skip_top_level apply }
     end
-    
-    def gen_skip_top_level(apply)
+
+    def gen_skip_top_level(apply, scope={})
       gen_intermediate_skip apply
       @g << ' && true'
     end
-    
-    def gen_intermediate_skip(apply)
+
+    def gen_intermediate_skip(apply, scope={})
       gen_basic apply
     end
-    
+
   end
-  
+
   # @private
   class NestedApplyGenerator < ApplyGenerator #:nodoc:
     include Nested
   end
-  
+
   def ApplyGenerator.nested(*args)
     NestedApplyGenerator.new(*args)
   end
-  
+
   # @private
   class TopLevelApplyGenerator < ApplyGenerator #:nodoc:
     include TopLevel
   end
-  
+
   def ApplyGenerator.top_level(*args)
     TopLevelApplyGenerator.new(*args)
   end
-  
+
 end

data/lib/rattler/back_end/parser_generator/assert_generator.rb

@@ -5,67 +5,66 @@ module Rattler::BackEnd::ParserGenerator
   # @private
   class AssertGenerator < ExprGenerator #:nodoc:
 
-    def gen_basic(assert)
-      generate assert.child, :assert
+    def gen_basic(assert, scope={})
+      generate assert.child, :assert, scope
     end
 
-    def gen_assert(assert)
-      gen_basic assert
+    def gen_assert(assert, scope={})
+      gen_basic assert, scope
     end
 
-    def gen_disallow(assert)
+    def gen_disallow(assert, scope={})
       @g << 'false'
     end
 
-    def gen_skip_nested(assert)
-      gen_basic_nested assert
+    def gen_skip_nested(assert, scope={})
+      gen_basic_nested assert, scope
     end
 
-    def gen_skip_top_level(assert)
-      gen_basic_top_level assert
+    def gen_skip_top_level(assert, scope={})
+      gen_basic_top_level assert, scope
     end
 
-    def gen_dispatch_action_nested(assert, target, method_name)
-      atomic_block { gen_dispatch_action_top_level assert, target, method_name }
+    def gen_dispatch_action_nested(assert, code, scope={})
+      atomic_block { gen_dispatch_action_top_level assert, code, scope }
     end
 
-    def gen_dispatch_action_top_level(assert, target, method_name)
-      gen_action assert,
-        dispatch_action_result(target, method_name, :array_expr => '[]')
+    def gen_dispatch_action_top_level(assert, code, scope={})
+      gen_action assert, code.bind(scope, '[]'), scope
     end
 
-    def gen_direct_action_nested(assert, code)
-      atomic_block { gen_direct_action_top_level assert, code }
+    def gen_direct_action_nested(assert, code, scope={})
+      atomic_block { gen_direct_action_top_level assert, code, scope }
     end
 
-    def gen_direct_action_top_level(assert, code)
-      gen_action assert, direct_action_result(code)
+    def gen_direct_action_top_level(assert, code, scope={})
+      gen_action assert, "(#{code.bind scope, []})", scope
     end
 
-    def gen_token_nested(assert)
-      atomic_block { gen_token_top_level assert }
+    def gen_token_nested(assert, scope={})
+      atomic_block { gen_token_top_level assert, scope }
     end
 
-    def gen_token_top_level(assert)
-      gen_action assert, "''"
+    def gen_token_top_level(assert, scope={})
+      gen_action assert, "''", scope
     end
 
-    def gen_intermediate(assert)
-      generate assert.child, :intermediate_assert
+    def gen_intermediate(assert, scope={})
+      generate assert.child, :intermediate_assert, scope
     end
 
-    def gen_intermediate_assert(assert)
-      gen_intermediate assert
+    def gen_intermediate_assert(assert, scope={})
+      gen_intermediate assert, scope
     end
 
-    def gen_intermediate_skip(assert)
-      gen_intermediate assert
+    def gen_intermediate_skip(assert, scope={})
+      gen_intermediate assert, scope
     end
 
     private
 
-    def gen_action(assert, result_code)
-      gen_intermediate assert
+    def gen_action(assert, result_code, scope={})
+      gen_intermediate assert, scope
       (@g << ' &&').newline << result_code
     end
 

data/lib/rattler/back_end/parser_generator/back_reference_generator.rb

@@ -0,0 +1,93 @@
+require 'rattler/back_end/parser_generator'
+
+module Rattler::BackEnd::ParserGenerator
+
+  # @private
+  class BackReferenceGenerator < ExprGenerator #:nodoc:
+
+    def gen_basic(ref, scope={})
+      @g << "@scanner.scan(#{ref.re_expr(scope)})"
+    end
+
+    def gen_assert_nested(ref, scope={})
+      atomic_expr { gen_assert_top_level ref, scope }
+    end
+
+    def gen_assert_top_level(ref, scope={})
+      gen_intermediate_assert ref, scope
+      @g << ' && true'
+    end
+
+    def gen_disallow_nested(ref, scope={})
+      atomic_expr { gen_disallow_top_level ref, scope }
+    end
+
+    def gen_disallow_top_level(ref, scope={})
+      gen_intermediate_disallow ref, scope
+      @g << ' && true'
+    end
+
+    def gen_dispatch_action_nested(ref, code, scope={})
+      atomic_block { gen_dispatch_action_top_level ref, code, scope }
+    end
+
+    def gen_dispatch_action_top_level(ref, code, scope={})
+      @g.surround("(#{result_name} = ", ')') { gen_basic ref, scope }
+      (@g << ' &&').newline << code.bind(scope, "[#{result_name}]")
+    end
+
+    def gen_direct_action_nested(ref, code, scope={})
+      atomic_block { gen_direct_action_top_level ref, code, scope }
+    end
+
+    def gen_direct_action_top_level(ref, code, scope={})
+      @g.surround("(#{result_name} = ", ')') { gen_basic ref, scope }
+      (@g << ' &&').newline << '(' << code.bind(scope, [result_name]) << ')'
+    end
+
+    def gen_token(ref, scope={})
+      gen_basic ref, scope
+    end
+
+    def gen_skip_nested(ref, scope={})
+      atomic_expr { gen_skip_top_level ref, scope }
+    end
+
+    def gen_skip_top_level(ref, scope={})
+      gen_intermediate_skip ref, scope
+      @g << ' && true'
+    end
+
+    def gen_intermediate_assert(ref, scope={})
+      @g << "@scanner.skip(/(?=#{ref.re_source(scope)})/)"
+    end
+
+    def gen_intermediate_disallow(ref, scope={})
+      @g << "@scanner.skip(/(?!#{ref.re_source(scope)})/)"
+    end
+
+    def gen_intermediate_skip(ref, scope={})
+      @g << "@scanner.skip(#{ref.re_expr(scope)})"
+    end
+
+  end
+
+  # @private
+  class NestedBackReferenceGenerator < BackReferenceGenerator #:nodoc:
+    include Nested
+  end
+
+  def BackReferenceGenerator.nested(*args)
+    NestedBackReferenceGenerator.new(*args)
+  end
+
+  # @private
+  class TopLevelBackReferenceGenerator < BackReferenceGenerator #:nodoc:
+    include TopLevel
+  end
+
+  def BackReferenceGenerator.top_level(*args)
+    TopLevelBackReferenceGenerator.new(*args)
+  end
+
+end

data/lib/rattler/back_end/parser_generator/choice_generator.rb

@@ -6,100 +6,84 @@ module Rattler::BackEnd::ParserGenerator
   class ChoiceGenerator < ExprGenerator #:nodoc:
     include NestedSubGenerating
 
-    def gen_basic_nested(choice)
-      atomic_block { gen_basic_top_level choice }
+    def gen_basic_nested(choice, scope={})
+      atomic_block { gen_basic_top_level choice, scope }
     end
 
-    def gen_basic_top_level(choice)
-      @g.intersperse_nl(choice, ' ||') {|_| generate _ }
+    def gen_basic_top_level(choice, scope={})
+      @g.intersperse_nl(choice, ' ||') {|_| generate _, :basic, scope }
     end
 
-    def gen_assert_nested(choice)
-      atomic_expr { gen_assert_top_level choice }
+    def gen_assert_nested(choice, scope={})
+      atomic_expr { gen_assert_top_level choice, scope }
     end
 
-    def gen_assert_top_level(choice)
-      gen_intermediate_assert choice
+    def gen_assert_top_level(choice, scope={})
+      gen_intermediate_assert choice, scope
       @g << ' && true'
     end
 
-    def gen_disallow(choice)
+    def gen_disallow(choice, scope={})
       @g << '!'
-      gen_intermediate_assert choice
+      gen_intermediate_assert choice, scope
     end
 
-    def gen_dispatch_action_nested(choice, target, method_name)
-      atomic_block { gen_dispatch_action_top_level choice, target, method_name }
+    def gen_dispatch_action_nested(choice, code, scope={})
+      atomic_block { gen_dispatch_action_top_level choice, code, scope }
     end
 
-    def gen_dispatch_action_top_level(choice, target, method_name)
-      gen_action_code choice do |labeled|
-        dispatch_action_result(target, method_name, :labeled => labeled)
-      end
+    def gen_dispatch_action_top_level(choice, code, scope={})
+      gen_action_code(choice) { code.bind scope, "[#{result_name}]" }
     end
 
-    def gen_direct_action_nested(choice, action)
-      atomic_block { gen_direct_action_top_level choice, action }
+    def gen_direct_action_nested(choice, code, scope={})
+      atomic_block { gen_direct_action_top_level choice, code }
     end
 
-    def gen_direct_action_top_level(choice, action)
-      gen_action_code choice do |labeled|
-        direct_action_result(action, :labeled => labeled)
-      end
+    def gen_direct_action_top_level(choice, code, scope={})
+      gen_action_code(choice) { "(#{code.bind scope, [result_name]})" }
     end
 
-    def gen_token_nested(choice)
-      atomic_block { gen_token_top_level choice }
+    def gen_token_nested(choice, scope={})
+      atomic_block { gen_token_top_level choice, scope }
     end
 
-    def gen_token_top_level(choice)
-      @g.intersperse_nl(choice, ' ||') {|_| generate _, :token }
+    def gen_token_top_level(choice, scope={})
+      @g.intersperse_nl(choice, ' ||') {|_| generate _, :token, scope }
     end
 
-    def gen_skip_nested(choice)
-      @g.surround('(', ')') { gen_skip_top_level choice }
+    def gen_skip_nested(choice, scope={})
+      @g.surround('(', ')') { gen_skip_top_level choice, scope }
     end
 
-    def gen_skip_top_level(choice)
-      gen_intermediate_skip choice
+    def gen_skip_top_level(choice, scope={})
+      gen_intermediate_skip choice, scope
       @g << ' && true'
     end
 
-    def gen_intermediate_assert(choice)
+    def gen_intermediate_assert(choice, scope={})
       atomic_block do
         @g.intersperse_nl(choice, ' ||') do |_|
-          generate _, :intermediate_assert
+          generate _, :intermediate_assert, scope
         end
       end
     end
 
-    def gen_intermediate_skip(choice)
+    def gen_intermediate_skip(choice, scope={})
       atomic_block do
         @g.intersperse_nl(choice, ' ||') do |_|
-          generate _, :intermediate_skip
+          generate _, :intermediate_skip, scope
         end
       end
     end
 
     private
 
-    def gen_action_code(choice)
-      labeled = choice.any? {|_| _.labeled? } ? labeled_name : nil
+    def gen_action_code(choice, scope={})
       @g.block("(#{result_name} = begin", 'end)') do
-        (@g << "#{labeled} = nil").newline if labeled
-        @g.intersperse_nl(choice, ' ||') do |child|
-          if child.labeled?
-            atomic_expr do
-              @g.surround("(#{result_name} = ", ')') { generate child }
-              @g << " && (#{labeled} = {:#{child.label} => #{result_name}})"
-              @g << " && #{result_name}"
-            end
-          else
-            generate child
-          end
-        end
+        @g.intersperse_nl(choice, ' ||') {|child| generate child }
       end << ' && '
-      @g << yield(labeled)
+      @g << yield
     end
 
     def labeled_name