rattler 0.2.0 → 0.2.1

data/lib/rattler/grammar/grammar_parser.rb

@@ -8,7 +8,7 @@ require 'rattler/grammar'
 
 module Rattler::Grammar
   # @private
-  class GrammarParser < Rattler::Runtime::WDMParser #:nodoc:
+  class GrammarParser < Rattler::Runtime::ExtendedPackratParser #:nodoc:
     
     include Metagrammar
     include Rattler::Parsers

data/lib/rattler/grammar/metagrammar.rb

@@ -53,7 +53,18 @@ module Rattler
       # @private
       def match_requires #:nodoc:
         a0 = []
-        while r = match(:require_decl)
+        while r = begin
+          p0 = @scanner.pos
+          begin
+            @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?>(?>require)(?>(?![[:alnum:]_])))/) &&
+            (r0_0 = match(:literal)) &&
+            match(:eol) &&
+            r0_0
+          end || begin
+            @scanner.pos = p0
+            false
+          end
+        end
           a0 << r
         end
         ({:requires => select_captures(a0)})
@@ -61,91 +72,70 @@ module Rattler
       
       # @private
       def match_module_decl #:nodoc:
-        match(:parser_decl) ||
-        match(:grammar_decl)
-      end
-      
-      # @private
-      def match_includes #:nodoc:
-        a0 = []
-        while r = match(:include_decl)
-          a0 << r
-        end
-        ({:includes => select_captures(a0)})
-      end
-      
-      # @private
-      def match_require_decl #:nodoc:
-        p0 = @scanner.pos
         begin
-          @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?>(?>require)(?>(?![[:alnum:]_])))/) &&
-          (r0_0 = match(:literal)) &&
-          match(:eol) &&
-          r0_0
-        end || begin
-          @scanner.pos = p0
-          false
-        end
-      end
-      
-      # @private
-      def match_parser_decl #:nodoc:
-        p0 = @scanner.pos
+          p0 = @scanner.pos
+          begin
+            @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?>(?>parser)(?>(?![[:alnum:]_])))/) &&
+            (r0_0 = match(:constant)) &&
+            (r0_1 = ((r = begin
+              p1 = @scanner.pos
+              begin
+                @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?><)/) &&
+                match(:constant)
+              end || begin
+                @scanner.pos = p1
+                false
+              end
+            end) ? [r] : [])) &&
+            match(:eol) &&
+            (parser_decl r0_0, r0_1)
+          end || begin
+            @scanner.pos = p0
+            false
+          end
+        end ||
         begin
-          @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?>(?>parser)(?>(?![[:alnum:]_])))/) &&
-          (r0_0 = match(:constant)) &&
-          (r0_1 = ((r = begin
-            p1 = @scanner.pos
-            begin
-              @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?><)/) &&
-              match(:constant)
-            end || begin
-              @scanner.pos = p1
-              false
-            end
-          end) ? [r] : [])) &&
-          match(:eol) &&
-          (parser_decl r0_0, r0_1)
-        end || begin
-          @scanner.pos = p0
-          false
+          p0 = @scanner.pos
+          begin
+            @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?>(?>grammar)(?>(?![[:alnum:]_])))/) &&
+            (r0_0 = match(:constant)) &&
+            match(:eol) &&
+            ({:grammar_name => r0_0})
+          end || begin
+            @scanner.pos = p0
+            false
+          end
         end
       end
       
       # @private
-      def match_grammar_decl #:nodoc:
-        p0 = @scanner.pos
-        begin
-          @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?>(?>grammar)(?>(?![[:alnum:]_])))/) &&
-          (r0_0 = match(:constant)) &&
-          match(:eol) &&
-          ({:grammar_name => r0_0})
-        end || begin
-          @scanner.pos = p0
-          false
+      def match_includes #:nodoc:
+        a0 = []
+        while r = begin
+          p0 = @scanner.pos
+          begin
+            @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?>(?>include)(?>(?![[:alnum:]_])))/) &&
+            (r0_0 = match(:constant)) &&
+            match(:eol) &&
+            r0_0
+          end || begin
+            @scanner.pos = p0
+            false
+          end
         end
-      end
-      
-      # @private
-      def match_include_decl #:nodoc:
-        p0 = @scanner.pos
-        begin
-          @scanner.skip(/(?>(?>(?>[[:space:]])+|(?>\#)(?>(?>[^\n])*))*)(?>(?>include)(?>(?![[:alnum:]_])))/) &&
-          (r0_0 = match(:constant)) &&
-          match(:eol) &&
-          r0_0
-        end || begin
-          @scanner.pos = p0
-          false
+          a0 << r
         end
+        ({:includes => select_captures(a0)})
       end
       
       # @private
       def match_rules #:nodoc:
         a0 = []
-        while r = match(:directive) ||
-        match(:rule) ||
-        match(:block_close)
+        while r = begin
+          match(:directive) ||
+          match(:rule) ||
+          match(:block_close)
+        end
           a0 << r
         end
         Rules.parsed(select_captures(a0)) unless a0.empty?

data/lib/rattler/grammar/rattler.rtlr

@@ -10,24 +10,16 @@ include Rattler::Parsers
   
   heading         <-  requires module_decl? includes                            {|r,m,i| heading r, m, i }
   
-  requires        <-  require_decl*                                             {|r| {:requires => r} }
+  requires        <-  (~`require` literal ~eol)*                                {|r| {:requires => r} }
   
-  module_decl     <-  parser_decl | grammar_decl
+  module_decl     <-  ~`parser` constant (~'<' constant)? ~eol                  {|p,b| parser_decl p, b }
+                    | ~`grammar` constant ~eol                                  {|g| {:grammar_name => g} }
   
-  includes        <-  include_decl*                                             {|i| {:includes => i} }
-  
-  require_decl    <-  ~`require` literal ~eol
-  
-  parser_decl     <-  ~`parser` constant (~'<' constant)? ~eol                  {|p,b| parser_decl p, b }
-  
-  grammar_decl    <-  ~`grammar` constant ~eol                                  {|g| {:grammar_name => g} }
-  
-  include_decl    <-  ~`include` constant ~eol
+  includes        <-  (~`include` constant ~eol)*                               {|i| {:includes => i} }
   
   rules           <-  (directive | rule | block_close)+                         <Rules>
   
-  directive       <-  ws_directive
-                    | wc_directive
+  directive       <-  ws_directive | wc_directive
   
   ws_directive    <-  ws_decl ~'{'                                              {|e| start_ws e }
                     | ws_decl                                                   {|e| set_ws e }

data/lib/rattler/parsers/combining.rb

@@ -1,3 +1,10 @@
+#
+# = rattler/parsers/combining.rb
+#
+# Author:: Jason Arhart
+# Documentation:: Author
+#
+
 module Rattler::Parsers
   # @private
   module Combining #:nodoc:

data/lib/rattler/parsers/match_joining.rb

@@ -1,3 +1,10 @@
+#
+# = rattler/parsers/match_joining.rb
+#
+# Author:: Jason Arhart
+# Documentation:: Author
+#
+
 require 'rattler/parsers'
 
 module Rattler

data/lib/rattler/parsers/predicate.rb

@@ -1,3 +1,10 @@
+#
+# = rattler/parsers/predicate.rb
+#
+# Author:: Jason Arhart
+# Documentation:: Author
+#
+
 require 'rattler/parsers'
 
 module Rattler::Parsers

data/lib/rattler/parsers/sequence.rb

@@ -1,5 +1,5 @@
 #
-# = rattler/parsers/choice.rb
+# = rattler/parsers/sequence.rb
 #
 # Author:: Jason Arhart
 # Documentation:: Author

data/lib/rattler/parsers/token.rb

@@ -9,7 +9,7 @@ require 'rattler/parsers'
 
 module Rattler::Parsers
   #
-  # Token decorates a parser to return the entire matched string
+  # +Token+ decorates a parser to return the entire matched string
   #
   # @author Jason Arhart
   #

data/lib/rattler/runner.rb

@@ -1,3 +1,9 @@
+#
+# = rattler/runner.rb
+#
+# Author:: Jason Arhart
+# Documentation:: Author
+#
 require 'rattler'
 
 require 'optparse'

data/lib/rattler/runtime.rb

@@ -16,7 +16,7 @@ module Rattler
     autoload :Parser, 'rattler/runtime/parser'
     autoload :RecursiveDescentParser, 'rattler/runtime/recursive_descent_parser'
     autoload :PackratParser, 'rattler/runtime/packrat_parser'
-    autoload :WDMParser, 'rattler/runtime/wdm_parser'
+    autoload :ExtendedPackratParser, 'rattler/runtime/extended_packrat_parser'
     autoload :ParseNode, 'rattler/runtime/parse_node'
     autoload :ParserHelper, 'rattler/runtime/parser_helper'
     autoload :ParseFailure, 'rattler/runtime/parse_failure'

data/lib/rattler/runtime/extended_packrat_parser.rb

@@ -0,0 +1,62 @@
+#
+# = rattler/runtime/extended_packrat_parser.rb
+#
+# Author:: Jason Arhart
+# Documentation:: Author
+#
+
+require 'rattler/runtime'
+
+module Rattler::Runtime
+  #
+  # +ExtendedPackratParser+ implements the algorithm described by Alessandro
+  # Warth, James R. Douglass, and Todd Millstein for extending packrat parsing
+  # to support left-recursive grammars. It currently only implements the first
+  # part to support direct left recursion.
+  #
+  # @author Jason Arhart
+  #
+  class ExtendedPackratParser < PackratParser
+
+    private
+    
+    # @private
+    def apply!(rule_name, key, start_pos) #:nodoc:
+      lr = LR.new
+      m = @memo[key] = MemoEntry.new(lr, start_pos, nil, nil)
+      result = eval_rule rule_name
+      memorize m, result
+      result = grow_lr(rule_name, start_pos, m) if result and lr.detected
+      result
+    end
+
+    def recall(m)
+      if (result = m.result).is_a? LR
+        result.detected = true
+        false
+      else
+        super
+      end
+    end
+
+    # @private
+    def grow_lr(rule_name, start_pos, m) #:nodoc:
+      loop do
+        @scanner.pos = start_pos
+        result = eval_rule(rule_name)
+        return recall(m) if !result or @scanner.pos <= m.end_pos
+        memorize m, result
+      end
+    end
+    
+    # @private
+    class LR
+      def initialize(detected = false)
+        @detected = detected
+      end
+      attr_accessor :detected
+    end
+
+  end
+
+end

data/lib/rattler/runtime/packrat_parser.rb

@@ -1,5 +1,5 @@
 #
-# = rattler/runtime/recursive_descent_parser.rb
+# = rattler/runtime/packrat_parser.rb
 #
 # Author:: Jason Arhart
 # Documentation:: Author
@@ -24,10 +24,13 @@ module Rattler::Runtime
     #
     def initialize(source, options={})
       super
-      @result_memo = {}
-      @pos_memo = {}
+      @memo = {}
     end
     
+    # @private
+    alias_method :eval_rule, :apply
+    private :eval_rule
+    
     protected
     
     # Apply a rule by dispatching to the method associated with the given rule
@@ -40,18 +43,49 @@ module Rattler::Runtime
     # @return (see RecursiveDescentParser#apply)
     #
     def apply(rule_name)
-      key = [rule_name, @scanner.pos]
-      if @result_memo.has_key? key
-        result = @result_memo[key]
-        @scanner.pos = @pos_memo[key]
+      start_pos = @scanner.pos
+      key = [rule_name, start_pos]
+      if @memo.has_key? key
+        recall @memo[key]
       else
-        @result_memo[key] = false
-        @pos_memo[key] = @scanner.pos
-        @result_memo[key] = result = super
-        @pos_memo[key] = @scanner.pos
+        apply! rule_name, key, start_pos
       end
-      result
     end
     
+    private
+
+    # @private
+    def apply!(rule_name, key, start_pos) #:nodoc:
+      m = @memo[key] = MemoEntry.new(false, start_pos, start_pos, 'left-recursion detected')
+      memorize m, eval_rule(rule_name)
+    end
+
+    # @private
+    def memorize(m, result) #:nodoc:
+      m.end_pos = @scanner.pos
+      m.failure_pos = @failure_pos
+      m.failure_msg = @failure_msg
+      m.result = result
+    end
+
+    # @private
+    def recall(m) #:nodoc:
+      @scanner.pos = m.end_pos
+      @failure_pos = m.failure_pos
+      @failure_msg = m.failure_msg
+      m.result
+    end
+
+    # @private
+    class MemoEntry
+      def initialize(result, end_pos, failure_pos, failure_msg)
+        @result = result
+        @end_pos = end_pos
+        @failure_pos = failure_pos
+        @failure_msg = failure_msg
+      end
+      attr_accessor :result, :end_pos, :failure_pos, :failure_msg
+    end
+
   end
 end

data/lib/rattler/runtime/parse_failure.rb

@@ -1,5 +1,5 @@
 #
-# = rattler/runtime/parse_error.rb
+# = rattler/runtime/parse_failure.rb
 #
 # Author:: Jason Arhart
 # Documentation:: Author

data/lib/rattler/util/parser_spec_helper.rb

@@ -131,6 +131,7 @@ MESSAGE
       def initialize(parser)
         @parser = parser
       end
+      attr_reader :parser
       def from(pos)
         @parser.pos = pos
         self

data/spec/rattler/back_end/parser_generator/one_or_more_generator_spec.rb

@@ -9,6 +9,10 @@ describe Rattler::BackEnd::ParserGenerator::OneOrMoreGenerator do
   let(:one_or_more) { OneOrMore[Match[/w+/]] }
   
   describe '#gen_basic' do
+
+    let :one_or_more do
+      OneOrMore[Choice[Match[/[[:alpha:]]/], Match[/[[:digit:]]/]]]
+    end
     
     context 'when nested' do
       it 'generates nested one-or-more matching code' do
@@ -16,7 +20,10 @@ describe Rattler::BackEnd::ParserGenerator::OneOrMoreGenerator do
           should == (<<-CODE).strip
 begin
   a = []
-  while r = @scanner.scan(/w+/)
+  while r = begin
+    @scanner.scan(/[[:alpha:]]/) ||
+    @scanner.scan(/[[:digit:]]/)
+  end
     a << r
   end
   a unless a.empty?
@@ -30,7 +37,10 @@ end
         top_level_code {|g| g.gen_basic one_or_more }.
           should == (<<-CODE).strip
 a = []
-while r = @scanner.scan(/w+/)
+while r = begin
+  @scanner.scan(/[[:alpha:]]/) ||
+  @scanner.scan(/[[:digit:]]/)
+end
   a << r
 end
 a unless a.empty?
@@ -49,7 +59,7 @@ a unless a.empty?
     end
     
     context 'when top-level' do
-      it 'generates top level one-or-more positive lookahead code' do
+      it 'generates nested one-or-more positive lookahead code' do
         top_level_code {|g| g.gen_assert one_or_more }.
           should == '@scanner.skip(/(?=w+)/) && true'
       end
@@ -66,7 +76,7 @@ a unless a.empty?
     end
     
     context 'when top-level' do
-      it 'generates top level one-or-more negative lookahead code' do
+      it 'generates nested one-or-more negative lookahead code' do
         top_level_code {|g| g.gen_disallow one_or_more }.
           should == '@scanner.skip(/(?!w+)/) && true'
       end