rattler 0.3.0 → 0.4.0

data/features/grammar/nonterminal.feature

@@ -22,7 +22,7 @@ Feature: Nonterminals
       """
       expr  <-  as "b"
       as    <-  "a" as
-              | "a"
+              / "a"
       """
     When I parse "aaab"
     Then the parse result should be [["a", ["a", "a"]], "b"]

data/features/grammar/ordered_choice.feature

@@ -1,6 +1,6 @@
 Feature: Ordered Choice Expressions
   
-  An ordered choice expression is a series of sub-expressions separated by "|"
+  An ordered choice expression is a series of sub-expressions separated by "/"
   and it means to try each sub-expression in order until one matches, and fail
   if none of the sub-expressions match.
   
@@ -11,7 +11,7 @@ Feature: Ordered Choice Expressions
   Scenario Outline: Parsing
     Given a grammar with:
       """
-      expr <- "A" | "B"
+      expr <- "A" / "B"
       """
     When I parse <input>
     Then the parse result should be <result>

data/features/grammar/skip_operator.feature

@@ -9,7 +9,7 @@ Feature: Skip Operator
   Scenario: Sequence with skipped sub-expressions
     Given a grammar with:
       """
-      sum <- ~"(" /\d+/ ~"+" /\d+/ ~")"
+      sum <- ~"(" @DIGIT+ ~"+" @DIGIT+ ~")"
       """
     When I parse "(23+45)"
     Then the parse result should be ["23", "45"]

data/features/grammar/symantic_action.feature

@@ -15,7 +15,7 @@ Feature: Symantic Actions
   Scenario: Single token
     Given a grammar with:
       """
-      integer <- /\d+/ {|s| s.to_i }
+      integer <- @DIGIT+ {|s| s.to_i }
       """
     When I parse "42"
     Then the parse result should be 42
@@ -24,7 +24,7 @@ Feature: Symantic Actions
     Given a grammar with:
       """
       %whitespace SPACE*
-      product <- /\d+/ /\d+/ {|a,b| a.to_i * b.to_i }
+      product <- @DIGIT+ @DIGIT+ {|a,b| a.to_i * b.to_i }
       """
     When I parse "3 16"
     Then the parse result should be 48
@@ -32,7 +32,7 @@ Feature: Symantic Actions
   Scenario: Sequence with non-capturing expressions
     Given a grammar with:
       """
-      sum <- ~"(" /\d+/ ~"+" /\d+/ ~")" {|a,b| a.to_i + b.to_i }
+      sum <- ~"(" @DIGIT+ ~"+" @DIGIT+ ~")" {|a,b| a.to_i + b.to_i }
       """
     When I parse "(23+17)"
     Then the parse result should be 40
@@ -40,7 +40,7 @@ Feature: Symantic Actions
   Scenario: Sequence with labeled expressions
     Given a grammar with:
       """
-      sum <- "(" left:/\d+/ "+" right:/\d+/ ")" { left.to_i + right.to_i }
+      sum <- "(" left:@DIGIT+ "+" right:@DIGIT+ ")" { left.to_i + right.to_i }
       """
     When I parse "(17+29)"
     Then the parse result should be 46
@@ -48,7 +48,7 @@ Feature: Symantic Actions
   Scenario: Single token using "_"
     Given a grammar with:
       """
-      integer <- /\d+/ { _.to_i }
+      integer <- @DIGIT+ { _.to_i }
       """
     When I parse "23"
     Then the parse result should be 23
@@ -57,7 +57,7 @@ Feature: Symantic Actions
     Given a grammar with:
       """
       %whitespace SPACE*
-      ints <- /\d+/ /\d+/ { _.reverse }
+      ints <- @DIGIT+ @DIGIT+ { _.reverse }
       """
     When I parse "3 16"
     Then the parse result should be ["16", "3"]
@@ -66,7 +66,7 @@ Feature: Symantic Actions
     Given a grammar with:
       """
       %whitespace SPACE*
-      ints <- /\d+/ /\d+/ {|_| _.to_i }
+      ints <- @DIGIT+ @DIGIT+ {|_| _.to_i }
       """
     When I parse "3 16"
     Then the parse result should be 3

data/features/grammar/whitespace.feature

@@ -14,7 +14,7 @@ Feature: Whitespace
     Given a grammar with:
       """
       %whitespace SPACE* {
-        word <- /\w+/
+        word <- @WORD+
       }
       """
     When I parse "  foo"
@@ -24,7 +24,7 @@ Feature: Whitespace
     Given a grammar with:
       """
       %whitespace SPACE*
-      word <- /\w+/
+      word <- @WORD+
       """
     When I parse "  foo"
     Then the parse result should be "foo"

data/features/step_definitions/grammar_steps.rb

@@ -1,8 +1,8 @@
-Given /grammar with:/ do |source|
+Given /grammar with\:/ do |source|
   @parser_class = Rattler.compile_parser(source)
 end
 
-Given /class definition:/ do |source|
+Given /class definition\:/ do |source|
   eval(source, TOPLEVEL_BINDING)
 end
 

data/lib/rattler/back_end.rb

@@ -17,5 +17,6 @@ module Rattler
     autoload :Compiler, 'rattler/back_end/compiler'
     autoload :ParserGenerator, 'rattler/back_end/parser_generator'
     autoload :RubyGenerator, 'rattler/back_end/ruby_generator'
+    autoload :Optimizer, 'rattler/back_end/optimizer'
   end
 end

data/lib/rattler/back_end/compiler.rb

@@ -13,15 +13,16 @@ module Rattler::BackEnd
   # @author Jason Arhart
   #
   class Compiler
-    
+
+    include Rattler::Grammar
     include Rattler::Parsers
-    
+
     # Compile rules or grammar source into a new parser subclass of +base+.
     #
     # @overload compile_parser(base, rules)
     #   Compile +rules+ into a new parser subclass of +base+.
     #   @param [Class] base the base class for the new parser class
-    #   @param [Rattler::Parser::Rules] rules the rules to compile
+    #   @param [Rattler::Parser::RuleSet] rules the rules to compile
     #   @return [Class] a new parser class
     #
     # @overload compile_parser(base, rule)
@@ -40,33 +41,33 @@ module Rattler::BackEnd
       parser_class = Class.new(base)
       compile(parser_class, rules_or_grammar)
     end
-    
+
     # Compile rules or grammar source into match methods in the module +mod+.
     def self.compile(mod, rules_or_grammar)
       self.new(mod).compile(rules_or_grammar)
     end
-    
+
     # Compile +rules+ into match methods in the module +mod+.
     def self.compile_rules(mod, rules)
       self.new(mod).compile_rules(rules)
     end
-    
+
     # Compile grammar +source+ into a match method in the module +mod+.
     def compile_grammar(mod, source)
       self.new(mod).compiler_grammar(source)
     end
-    
+
     # Create a new compiler that compiles rules into match methods in the
     # given module.
     def initialize(mod)
       @mod = mod
     end
-    
+
     # Compile the rules or grammar source into match methods in the module.
     #
     # @overload compile(rules)
     #   Compile +rules+ into match methods in the module.
-    #   @param [Rattler::Parser::Rules] rules the rules to compile
+    #   @param [Rattler::Parser::RuleSet] rules the rules to compile
     #   @return [Module] the module
     #
     # @overload compile(rule)
@@ -81,17 +82,16 @@ module Rattler::BackEnd
     #
     def compile(_)
       case _
-      when Rules then compile_rules(_)
-      when Rule then compile_rules(_)
+      when Grammar, RuleSet, Rule then compile_rules(_)
       else compile_grammar(_.to_s)
       end
     end
-    
+
     # Compile +rules+ into match methods in the module.
     #
     # @overload compile_rules(rules)
     #   Compile +rules+ into match methods in the module.
-    #   @param [Rattler::Parser::Rules] rules the rules to compile
+    #   @param [Rattler::Parser::RuleSet] rules the rules to compile
     #   @return [Module] the module
     #
     # @overload compile_rules(rule)
@@ -100,10 +100,9 @@ module Rattler::BackEnd
     #   @return [Module] the module
     #
     def compile_rules(rules)
-      assert_kind_of([Rules, Rule], rules)
-      compile_model(rules.optimized)
+      compile_model(rules)
     end
-    
+
     # Compile +grammar+ into match methods in the module.
     # @param [String] grammar the grammar source to compile
     # @return [Module] the module
@@ -111,19 +110,19 @@ module Rattler::BackEnd
       result = Rattler::Grammar.parse!(source)
       compile_model(result.rules)
     end
-    
+
     private
-    
+
     def assert_kind_of(kinds, model) #:nodoc:
       unless kinds.any? {|kind| model.kind_of?(kind) }
         raise TypeError, "Expected #{model.inspect} to be a kind of #{klass}", caller
       end
     end
-    
+
     def compile_model(model) #:nodoc:
       @mod.module_eval ParserGenerator.code_for(model)
       @mod
     end
-    
+
   end
 end

data/lib/rattler/back_end/optimizer.rb

@@ -0,0 +1,100 @@
+#
+# = rattler/back_end/optimizer.rb
+#
+# Author:: Jason Arhart
+# Documentation:: Author
+#
+require 'rattler'
+
+module Rattler::BackEnd
+  #
+  # The +Optimizer+ transforms parser models into equivalent models that can
+  # result in more efficient parsing code. This is primarily achieved by
+  # converting regular expressions into equivalent Regexp patterns, thus
+  # reducing object instantiation and method dispatch and having StringScanner
+  # do as much of the parsing work as possible.
+  #
+  # @author Jason Arhart
+  #
+  module Optimizer
+
+    class << self
+      def optimizations
+        @optimizations ||=
+          OptimizeChildren >>
+          InlineRegularRules >>
+          SimplifyRedundantRepeat >>
+          RemoveMeaninglessWrapper >>
+          SimplifyTokenMatch >>
+          FlattenSequence >>
+          FlattenChoice >>
+          ReduceRepeatMatch >>
+          JoinPredicateMatch >>
+          JoinPredicateOrMatch >>
+          JoinMatchSequence >>
+          JoinMatchChoice
+      end
+
+      def optimize(model, opts={})
+        case model
+        when ::Rattler::Grammar::Grammar then optimize_grammar model, opts
+        when ::Rattler::Parsers::RuleSet then optimize_rule_set model, opts
+        when ::Rattler::Parsers::Rule then optimize_rule model, default_context(opts)
+        else optimize_expr model, default_context(opts)
+        end
+      end
+
+      private
+
+      def default_context(opts)
+        OptimizationContext[opts.merge :type => :capturing]
+      end
+
+      def optimize_grammar(grammar, opts)
+        grammar.with_rules optimize_rule_set(grammar.rules, opts)
+      end
+
+      def optimize_rule_set(rule_set, opts)
+        context = default_context(opts).with(:rules => rule_set)
+        rule_set = rule_set.map_rules {|_| optimize_rule _, context }
+        context = context.with(:rules => rule_set)
+        rule_set.select_rules {|_| context.relavent? _ }
+      end
+
+      def optimize_rule(rule, context)
+        rule.with_expr optimizations.apply(rule.expr, context)
+      end
+
+      def optimize_expr(expr, context)
+        optimizations.apply rule.expr, context
+      end
+
+    end
+
+    autoload :OptimizationContext, 'rattler/back_end/optimizer/optimization_context'
+    autoload :Optimization, 'rattler/back_end/optimizer/optimization'
+    autoload :OptimizationSequence, 'rattler/back_end/optimizer/optimization_sequence'
+    autoload :OptimizeChildren, 'rattler/back_end/optimizer/optimize_children'
+    autoload :InlineRegularRules, 'rattler/back_end/optimizer/inline_regular_rules'
+    autoload :SimplifyRedundantRepeat, 'rattler/back_end/optimizer/simplify_redundant_repeat'
+    autoload :RemoveMeaninglessWrapper, 'rattler/back_end/optimizer/remove_meaningless_wrapper'
+    autoload :SimplifyTokenMatch, 'rattler/back_end/optimizer/simplify_token_match'
+    autoload :FlattenSequence, 'rattler/back_end/optimizer/flatten_sequence'
+    autoload :FlattenChoice, 'rattler/back_end/optimizer/flatten_choice'
+    autoload :ReduceRepeatMatch, 'rattler/back_end/optimizer/reduce_repeat_match'
+    autoload :JoinPredicateMatch, 'rattler/back_end/optimizer/join_predicate_match'
+    autoload :JoinPredicateBareMatch, 'rattler/back_end/optimizer/join_predicate_bare_match'
+    autoload :JoinPredicateNestedMatch, 'rattler/back_end/optimizer/join_predicate_nested_match'
+    autoload :JoinPredicateOrMatch, 'rattler/back_end/optimizer/join_predicate_or_match'
+    autoload :JoinPredicateOrBareMatch, 'rattler/back_end/optimizer/join_predicate_or_bare_match'
+    autoload :JoinPredicateOrNestedMatch, 'rattler/back_end/optimizer/join_predicate_or_nested_match'
+    autoload :JoinMatchSequence, 'rattler/back_end/optimizer/join_match_sequence'
+    autoload :JoinMatchCapturingSequence, 'rattler/back_end/optimizer/join_match_capturing_sequence'
+    autoload :JoinMatchMatchingSequence, 'rattler/back_end/optimizer/join_match_matching_sequence'
+    autoload :JoinMatchChoice, 'rattler/back_end/optimizer/join_match_choice'
+    autoload :MatchJoining, 'rattler/back_end/optimizer/match_joining'
+    autoload :Flattening, 'rattler/back_end/optimizer/flattening'
+    autoload :CompositeReducing, 'rattler/back_end/optimizer/composite_reducing'
+
+  end
+end

data/lib/rattler/back_end/optimizer/composite_reducing.rb

@@ -0,0 +1,18 @@
+require 'rattler'
+
+module Rattler::BackEnd::Optimizer
+  # @private
+  module CompositeReducing #:nodoc:
+
+    protected
+
+    def finish_reduce(parser, new_children)
+      if new_children.size == 1
+        new_children[0]
+      else
+        parser.with_children new_children
+      end
+    end
+
+  end
+end

data/lib/rattler/back_end/optimizer/flatten_choice.rb

@@ -0,0 +1,31 @@
+#
+# = rattler/back_end/optimizer/flatten_choice.rb
+#
+# Author:: Jason Arhart
+# Documentation:: Author
+#
+require 'rattler'
+
+module Rattler::BackEnd::Optimizer
+  #
+  # Nested choice expressions can be flattened without affecting how they parse.
+  #
+  # @author Jason Arhart
+  #
+  class FlattenChoice < Optimization
+    include Flattening
+
+    include Rattler::Parsers
+
+    protected
+
+    def _applies_to?(parser, context)
+      parser.is_a?(Choice) and super
+    end
+
+    def eligible_child?(child)
+      child.is_a? Choice
+    end
+
+  end
+end

data/lib/rattler/back_end/optimizer/flatten_sequence.rb

@@ -0,0 +1,59 @@
+#
+# = rattler/back_end/optimizer/flatten_sequence.rb
+#
+# Author:: Jason Arhart
+# Documentation:: Author
+#
+require 'rattler'
+
+module Rattler::BackEnd::Optimizer
+
+  #
+  # Nested sequence expressions can be flattened without affecting how they
+  # match.
+  #
+  # @author Jason Arhart
+  #
+  class FlattenMatchingSequence < Optimization
+    include Flattening
+    include Rattler::Parsers
+
+    protected
+
+    def _applies_to?(parser, context)
+      context.matching? and
+      parser.is_a?(Sequence) and
+      super
+    end
+
+    def eligible_child?(child)
+      child.is_a? Sequence
+    end
+  end
+
+  #
+  # Nested sequence expressions can be flattened without affecting how they
+  # parse if the nested sequence expressions are not multi-capturing.
+  #
+  # @author Jason Arhart
+  #
+  class FlattenCapturingSequence < Optimization
+    include Flattening
+    include Rattler::Parsers
+
+    protected
+
+    def _applies_to?(parser, context)
+      context.capturing? and
+      parser.is_a?(Sequence) and
+      super
+    end
+
+    def eligible_child?(child)
+      child.is_a?(Sequence) and child.capture_count <= 1
+    end
+  end
+
+  FlattenSequence = FlattenMatchingSequence >> FlattenCapturingSequence
+
+end