rattler 0.5.0 → 0.6.0

data/lib/rattler/compiler/optimizer.rb

@@ -0,0 +1,77 @@
+require 'rattler/compiler'
+
+module Rattler::Compiler
+
+  # The +Optimizer+ transforms parser models into equivalent models that can
+  # result in more efficient parsing code. This is primarily achieved by
+  # converting regular expressions into equivalent Regexp patterns, thus
+  # reducing object instantiation and method dispatch and having StringScanner
+  # do as much of the parsing work as possible.
+  module Optimizer
+
+    class << self
+
+      include Rattler::Parsers
+
+      # @param [Rattler::Parsers::Grammar, Rattler::Parsers::RuleSet, Rattler::Parsers::Rule, Rattler::Parsers::Parser]
+      #   model the model to be optimized
+      # @param [Hash] opts options for the optimizer
+      # @return an optimized parser model
+      def optimize(model, opts={})
+        case model
+        when Grammar then optimize_grammar model, opts
+        when RuleSet then optimize_rule_set model, opts
+        else optimizations.apply model, default_context(opts)
+        end
+      end
+
+      private
+
+      def default_context(opts)
+        OptimizationContext[opts.merge :type => :capturing]
+      end
+
+      def optimize_grammar(grammar, opts)
+        grammar.with_rules optimize_rule_set(grammar.rules, opts)
+      end
+
+      def optimize_rule_set(rule_set, opts)
+        context = default_context(opts).with(:rules => rule_set)
+        rule_set = rule_set.map_rules {|_| optimizations.apply _, context }
+        context = context.with(:rules => rule_set)
+        rule_set.select_rules {|_| context.relavent? _ }
+      end
+
+    end
+
+    autoload :OptimizationContext, 'rattler/compiler/optimizer/optimization_context'
+    autoload :Optimization, 'rattler/compiler/optimizer/optimization'
+    autoload :OptimizationSequence, 'rattler/compiler/optimizer/optimization_sequence'
+    autoload :Optimizations, 'rattler/compiler/optimizer/optimizations'
+    autoload :OptimizeChildren, 'rattler/compiler/optimizer/optimize_children'
+    autoload :InlineRegularRules, 'rattler/compiler/optimizer/inline_regular_rules'
+    autoload :SimplifyRedundantRepeat, 'rattler/compiler/optimizer/simplify_redundant_repeat'
+    autoload :RemoveMeaninglessWrapper, 'rattler/compiler/optimizer/remove_meaningless_wrapper'
+    autoload :SimplifyTokenMatch, 'rattler/compiler/optimizer/simplify_token_match'
+    autoload :FlattenSequence, 'rattler/compiler/optimizer/flatten_sequence'
+    autoload :FlattenChoice, 'rattler/compiler/optimizer/flatten_choice'
+    autoload :ReduceRepeatMatch, 'rattler/compiler/optimizer/reduce_repeat_match'
+    autoload :JoinPredicateMatch, 'rattler/compiler/optimizer/join_predicate_match'
+    autoload :JoinPredicateBareMatch, 'rattler/compiler/optimizer/join_predicate_bare_match'
+    autoload :JoinPredicateNestedMatch, 'rattler/compiler/optimizer/join_predicate_nested_match'
+    autoload :JoinPredicateOrMatch, 'rattler/compiler/optimizer/join_predicate_or_match'
+    autoload :JoinPredicateOrBareMatch, 'rattler/compiler/optimizer/join_predicate_or_bare_match'
+    autoload :JoinPredicateOrNestedMatch, 'rattler/compiler/optimizer/join_predicate_or_nested_match'
+    autoload :JoinMatchSequence, 'rattler/compiler/optimizer/join_match_sequence'
+    autoload :JoinMatchCapturingSequence, 'rattler/compiler/optimizer/join_match_capturing_sequence'
+    autoload :JoinMatchMatchingSequence, 'rattler/compiler/optimizer/join_match_matching_sequence'
+    autoload :JoinMatchChoice, 'rattler/compiler/optimizer/join_match_choice'
+    autoload :MatchJoining, 'rattler/compiler/optimizer/match_joining'
+    autoload :Flattening, 'rattler/compiler/optimizer/flattening'
+    autoload :CompositeReducing, 'rattler/compiler/optimizer/composite_reducing'
+
+  end
+
+  Optimizer.extend Optimizer::Optimizations
+
+end

data/lib/rattler/{back_end → compiler}/optimizer/composite_reducing.rb

@@ -1,6 +1,6 @@
-require 'rattler'
+require 'rattler/compiler/optimizer'
 
-module Rattler::BackEnd::Optimizer
+module Rattler::Compiler::Optimizer
   # @private
   module CompositeReducing #:nodoc:
 

data/lib/rattler/{back_end → compiler}/optimizer/flatten_choice.rb

@@ -1,17 +1,8 @@
-#
-# = rattler/back_end/optimizer/flatten_choice.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
 
-module Rattler::BackEnd::Optimizer
-  #
   # Nested choice expressions can be flattened without affecting how they parse.
-  #
-  # @author Jason Arhart
-  #
   class FlattenChoice < Optimization
     include Flattening
 

data/lib/rattler/{back_end → compiler}/optimizer/flatten_sequence.rb

@@ -1,19 +1,9 @@
-#
-# = rattler/back_end/optimizer/flatten_sequence.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
+require 'rattler/compiler/optimizer'
 
-module Rattler::BackEnd::Optimizer
+module Rattler::Compiler::Optimizer
 
-  #
   # Nested sequence expressions can be flattened without affecting how they
   # match.
-  #
-  # @author Jason Arhart
-  #
   class FlattenMatchingSequence < Optimization
     include Flattening
     include Rattler::Parsers
@@ -31,12 +21,8 @@ module Rattler::BackEnd::Optimizer
     end
   end
 
-  #
   # Nested sequence expressions can be flattened without affecting how they
   # parse if the nested sequence expressions are not multi-capturing.
-  #
-  # @author Jason Arhart
-  #
   class FlattenCapturingSequence < Optimization
     include Flattening
     include Rattler::Parsers
@@ -54,6 +40,8 @@ module Rattler::BackEnd::Optimizer
     end
   end
 
+  # Nested sequence expressions can be flattened without affecting how they
+  # parse given certain conditions
   FlattenSequence = FlattenMatchingSequence >> FlattenCapturingSequence
 
 end

data/lib/rattler/{back_end → compiler}/optimizer/flattening.rb

@@ -1,6 +1,6 @@
-require 'rattler'
+require 'rattler/compiler/optimizer'
 
-module Rattler::BackEnd::Optimizer
+module Rattler::Compiler::Optimizer
   # @private
   module Flattening #:nodoc:
 

data/lib/rattler/compiler/optimizer/inline_regular_rules.rb

@@ -0,0 +1,24 @@
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
+
+  # References to regular parse rules can be inlined without affecting how they
+  # parse, assuming the referenced rule does not change. This optimization is
+  # only applied if the referenced rule is regular and marked for inlining.
+  class InlineRegularRules < Optimization
+
+    include Rattler::Parsers
+
+    protected
+
+    def _applies_to?(parser, context)
+      parser.is_a? Apply and
+      context.inlineable? parser.rule_name
+    end
+
+    def _apply(parser, context)
+      context.rules[parser.rule_name].expr
+    end
+
+  end
+end

data/lib/rattler/{back_end → compiler}/optimizer/join_match_capturing_sequence.rb

@@ -1,29 +1,21 @@
-#
-# = rattler/back_end/optimizer/join_match_capturing_sequence.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
-
-module Rattler::BackEnd::Optimizer
-  #
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
+
   # Sequences of Regexp matches can be joined into a single Regexp match using
   # capturing groups if necessary.
-  #
-  # @author Jason Arhart
-  #
   class JoinMatchCapturingSequence < Optimization
     include MatchJoining
 
     include Rattler::Parsers
-    include Rattler::BackEnd::ParserGenerator
+    include Rattler::Compiler::ParserGenerator
 
     protected
 
     def _applies_to?(parser, context)
       context.capturing? and
       parser.is_a?(Sequence) and
+      not disqualifying_captures?(parser) and
       any_neighbors?(parser) {|_| eligible_child? _ }
     end
 
@@ -33,6 +25,16 @@ module Rattler::BackEnd::Optimizer
       (child.is_a? Skip and child.child.is_a? Match)
     end
 
+    def disqualifying_captures?(parser)
+      parser.any? {|_| _.capturing? and eligible_child? _ } and
+      parser.any? {|_| capture_incompatible? _ }
+    end
+
+    def capture_incompatible?(child)
+      (child.capturing? and not eligible_child? child) or
+      child.semantic?
+    end
+
     def create_patterns(parsers)
       num_groups = 0
       patterns = parsers.map do |parser|

data/lib/rattler/{back_end → compiler}/optimizer/join_match_choice.rb

@@ -1,18 +1,9 @@
-#
-# = rattler/back_end/optimizer/join_match_choice.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
-
-module Rattler::BackEnd::Optimizer
-  #
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
+
   # A choice of Regexp matches can be joined into a single Regexp match without
   # affecting how it parses.
-  #
-  # @author Jason Arhart
-  #
   class JoinMatchChoice < Optimization
     include MatchJoining
 

data/lib/rattler/{back_end → compiler}/optimizer/join_match_matching_sequence.rb

@@ -1,18 +1,9 @@
-#
-# = rattler/back_end/optimizer/join_match_matching_sequence.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
-
-module Rattler::BackEnd::Optimizer
-  #
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
+
   # Sequences of Regexp matches can be joined into a single Regexp without
   # affecting how they match.
-  #
-  # @author Jason Arhart
-  #
   class JoinMatchMatchingSequence < Optimization
     include MatchJoining
 

data/lib/rattler/compiler/optimizer/join_match_sequence.rb

@@ -0,0 +1,7 @@
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
+  # Sequences of Regexp matches can be joined into a single Regexp match using
+  # capturing groups if necessary.
+  JoinMatchSequence = JoinMatchMatchingSequence >> JoinMatchCapturingSequence
+end

data/lib/rattler/{back_end → compiler}/optimizer/join_predicate_bare_match.rb

@@ -1,18 +1,9 @@
-#
-# = rattler/back_end/optimizer/join_predicate_bare_match.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
 
-module Rattler::BackEnd::Optimizer
-  #
   # A predicate and an adjacent Regexp match in a Sequence can be joined into a
   # single Regexp match.
-  #
-  # @author Jason Arhart
-  #
   class JoinPredicateBareMatch < Optimization
     include CompositeReducing
 

data/lib/rattler/compiler/optimizer/join_predicate_match.rb

@@ -0,0 +1,7 @@
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
+  # A predicate and an adjacent Regexp match in a Sequence can be joined into a
+  # single Regexp match.
+  JoinPredicateMatch = JoinPredicateBareMatch >> JoinPredicateNestedMatch
+end

data/lib/rattler/{back_end → compiler}/optimizer/join_predicate_nested_match.rb

@@ -1,18 +1,9 @@
-#
-# = rattler/back_end/optimizer/join_predicate_nested_match.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
-
-module Rattler::BackEnd::Optimizer
-  #
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
+
   # A predicate and an adjacent skip of a Regexp match in a Sequence can be
   # joined into a single skip of a Regexp match.
-  #
-  # @author Jason Arhart
-  #
   class JoinPredicateNestedMatch < JoinPredicateBareMatch
     include CompositeReducing
 

data/lib/rattler/{back_end → compiler}/optimizer/join_predicate_or_bare_match.rb

@@ -1,18 +1,9 @@
-#
-# = rattler/back_end/optimizer/join_predicate_or_bare_match.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
 
-module Rattler::BackEnd::Optimizer
-  #
   # A predicate and an adjacent Regexp match in a Choice can be joined into a
   # single Regexp match.
-  #
-  # @author Jason Arhart
-  #
   class JoinPredicateOrBareMatch < Optimization
     include CompositeReducing
 

data/lib/rattler/compiler/optimizer/join_predicate_or_match.rb

@@ -0,0 +1,7 @@
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
+  # A predicate and an adjacent Regexp match in a Choice can be joined into a
+  # single Regexp match.
+  JoinPredicateOrMatch = JoinPredicateOrBareMatch >> JoinPredicateOrNestedMatch
+end

data/lib/rattler/{back_end → compiler}/optimizer/join_predicate_or_nested_match.rb

@@ -1,18 +1,9 @@
-#
-# = rattler/back_end/optimizer/join_predicate_or_nested_match.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
-
-module Rattler::BackEnd::Optimizer
-  #
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
+
   # A predicate and an adjacent skip of a Regexp match in a Choice can be
   # joined into a single skip of a Regexp match.
-  #
-  # @author Jason Arhart
-  #
   class JoinPredicateOrNestedMatch < JoinPredicateOrBareMatch
     include CompositeReducing
 

data/lib/rattler/{back_end → compiler}/optimizer/match_joining.rb

@@ -1,6 +1,6 @@
-require 'rattler'
+require 'rattler/compiler/optimizer'
 
-module Rattler::BackEnd::Optimizer
+module Rattler::Compiler::Optimizer
   # @private
   module MatchJoining #:nodoc:
 

data/lib/rattler/{back_end → compiler}/optimizer/optimization.rb

@@ -1,39 +1,30 @@
-#
-# = rattler/back_end/optimizer/optimization.rb
-#
-# Author:: Jason Arhart
-# Documentation:: Author
-#
-require 'rattler'
-require 'singleton'
+require 'rattler/compiler/optimizer'
+
+module Rattler::Compiler::Optimizer
 
-module Rattler::BackEnd::Optimizer
-  #
   # An +Optimization+ represents a simple transformation of a parser model into
   # an equivalent model that can result in more efficient parsing code.
   # Subclasses override <tt>#_applies_to?</tt> and <tt>#_apply</tt> to define
   # an optimization.
-  #
-  # @author Jason Arhart
-  #
   class Optimization
 
     class <<self
+      # @return a lazy singleton instance
       def instance
         @instance ||= self.new
       end
 
-      # @see Rattler::BackEnd::Optimizer::Optimization#>>
+      # (see Rattler::Compiler::Optimizer::Optimization#>>)
       def >>(other)
         instance >> (other.respond_to?(:instance) ? other.instance : other)
       end
 
-      # @see Rattler::BackEnd::Optimizer::Optimization#match
+      # (see Rattler::Compiler::Optimizer::Optimization#apply)
       def apply(*args)
         instance.apply(*args)
       end
 
-      # @see Rattler::BackEnd::Optimizer::Optimization#applies_to?
+      # (see Rattler::Compiler::Optimizer::Optimization#applies_to?)
       def applies_to?(*args)
         instance.applies_to?(*args)
       end
@@ -47,10 +38,10 @@ module Rattler::BackEnd::Optimizer
     # i.e. the new optimzation applies this optimzation, then applies +other+
     # to the result.
     #
-    # @param [Rattler::BackEnd::Optimizer::Optimization] other the optimization
+    # @param [Rattler::Compiler::Optimizer::Optimization] other the optimization
     #   to apply after this one
     #
-    # @return [Rattler::BackEnd::Optimizer::Optimization] a new optimzation
+    # @return [Rattler::Compiler::Optimizer::Optimization] a new optimzation
     #   that sequences this optimzation and +other+
     def >>(other)
       OptimizationSequence.new(self, other)
@@ -60,35 +51,22 @@ module Rattler::BackEnd::Optimizer
     # <tt>#applies_to?(parser, context)</tt> is +true+.
     #
     # @param [Rattler::Parsers::Parser] parser the parser to be optimized
-    # @param [Rattler::BackEnd::Optimizer::OptimizationContext] context
+    # @param [Rattler::Compiler::Optimizer::OptimizationContext] context
     #
     # @return [Rattler::Parsers::Parser] the optimized parser
     def apply(parser, context)
       applies_to?(parser, context) ? _apply(parser, context) : parser
     end
 
-    # Return +true+ if this optimzation applies to +parser+ in +context+.
-    #
     # @param [Rattler::Parsers::Parser] parser a parser model
-    # @param [Rattler::BackEnd::Optimizer::OptimizationContext] context
+    # @param [Rattler::Compiler::Optimizer::OptimizationContext] context
     #
-    # @return [Boolean] +true+ if this optimzation applies to +parser+ in
-    #   +context+
+    # @return +true+ if this optimzation applies to +parser+ in +context+
     def applies_to?(parser, context)
       @applies_to_cache[context].fetch(parser) do
         @applies_to_cache[context][parser] = _applies_to?(parser, context)
       end
     end
 
-    private
-
-    def optimize(parser, context)
-      optimizations.apply(parser, context)
-    end
-
-    def optimizations
-      ::Rattler::BackEnd::Optimizer.optimizations
-    end
-
   end
 end