dhaka 2.1.0 → 2.2.0

data/lib/lexer/specification.rb

@@ -30,7 +30,17 @@ module Dhaka
   # 'z's instead of a word token.
   #  
   # The patterns are <i>not</i> Ruby regular expressions - a lot of operators featured in Ruby's regular expression engine are not yet supported.
-  # See http://dhaka.rubyforge.org/regex_grammar.html for the current syntax.
+  # See http://dhaka.rubyforge.org/regex_grammar.html for the current syntax. Patterns may be specified using Ruby regular expression literals 
+  # as well as string literals. 
+  # 
+  # There are a few things to keep in mind with regard to the regular expression implementation:
+  # * The greediest matching expression always wins. Precedences are only used when the same set of characters matches
+  #   multiple expressions.
+  # * All quantifiers are greedy. There is as yet no support for non-greedy modifiers.
+  # * The lookahead operator "/" can behave in counter-intuitive ways in situations where the pre-lookahead-operator expression and the 
+  #   post-lookahead-operator expression have characters in common. For example the expression "(ab)+/abcd", when applied to the input
+  #   "abababcd" will yield "ababab" as the match instead of "abab". A good thumb rule is that the pre-lookahead expression is greedy.   
+
   
   class LexerSpecification
     class << self
@@ -38,10 +48,26 @@ module Dhaka
       # it creates a LexerRun object that provides the context for +blk+ to be evaluated in. Methods available in this block
       # are LexerRun#current_lexeme and LexerRun#create_token.
       def for_pattern(pattern, &blk)
-        items[pattern] = LexerSpecificationItem.new(pattern, priority, blk)
+        source = case pattern
+                   when String : pattern
+                   when Regexp : pattern.source
+                 end
+        items[source] = LexerSpecificationItem.new(source, priority, blk)
         self.priority += 1
       end
-
+      
+      # Use this to automatically handle escaping for regular expression metacharacters. For example, 
+      #   for_symbol('+') { ... }
+      # translates to:
+      #   for_pattern('\+') { ... }
+      def for_symbol(symbol, &blk)
+        if LexerSupport::OPERATOR_CHARACTERS.include?(symbol)
+          for_pattern("\\#{symbol}", &blk)
+        else
+          for_pattern(symbol, &blk)
+        end
+      end
+      
       private
         def inherited(specification)
           class << specification

data/lib/lexer/state.rb

@@ -1,15 +1,22 @@
 module Dhaka
   module LexerSupport
+    
     class State
-      attr_reader :transitions, :pattern
-      def initialize state_machine, pattern
-        @state_machine = state_machine
-        @pattern          = pattern
-        @transitions      = {}
+      attr_reader :transitions, :checkpoint_actions, :action
+      def initialize(state_machine, action=nil)
+        @state_machine      = state_machine
+        @transitions        = {}
+        @checkpoint_actions = []
+        @action             = action
       end
-
+      
       def accepting?
-        pattern
+        @action
+      end
+      
+      def process lexer_run
+        checkpoint_actions.each {|action| action.call(lexer_run)}
+        action.call(lexer_run) if accepting?
       end
       
       def for_characters *characters, &blk
@@ -19,13 +26,28 @@ module Dhaka
         end
       end
       
+      def add_checkpoint(pattern)
+        checkpoint_actions << LexerSupport::CheckpointAction.new(pattern)
+      end
+      
+      def accept(pattern)
+        @action = AcceptAction.new(pattern)
+      end
+      
+      def accept_with_lookahead(pattern)
+        @action = LookaheadAcceptAction.new(pattern)
+      end
+      
       def recognize pattern
         @pattern = pattern
       end
       
       def compile_to_ruby_source
         result  = "  at_state(#{object_id}) {\n"
-        result << "    recognize(#{pattern.inspect})\n" if accepting?
+        result << "    #{action.compile_to_ruby_source}\n" if action
+        checkpoint_actions.each do |checkpoint_action|
+          result << "    #{checkpoint_action.compile_to_ruby_source}\n"
+        end
         transition_keys_by_destination_state = Hash.new {|hash, key| hash[key] = []}
         transitions.each do |key, dest_state|
           transition_keys_by_destination_state[dest_state.object_id] << key
@@ -40,6 +62,7 @@ module Dhaka
         result
       end
     end
+    
   end
 end
-  
+  

data/lib/lexer/state_machine.rb

@@ -24,10 +24,10 @@ module Dhaka
           g.edge(start, @start_state)
           @states.values.each do |state|
             state_attributes = {}
-            state_attributes.merge!(:shape => :doublecircle, :label => state.pattern) if state.accepting?
+            state_attributes.merge!(:shape => :doublecircle, :label => state.action.to_dot) if state.accepting?
             g.node(state, state_attributes)
             state.transitions.each do |transition_key, dest_state|
-              g.edge(state, dest_state, :label => transition_key)
+              g.edge(state, dest_state, :label => transition_key.inspect)
             end
           end
         end.to_dot

data/lib/parser/channel.rb

@@ -10,9 +10,9 @@ module Dhaka
     end
 
     def propagate cargo
-      diff = cargo - end_item.lookaheadset
-      end_item.lookaheadset.merge(diff)
-      !diff.empty?
+      initial_size = end_item.lookaheadset.size
+      end_item.lookaheadset.merge(cargo)
+      (end_item.lookaheadset.size - initial_size) > 0
     end
 
     def to_s
@@ -20,7 +20,7 @@ module Dhaka
     end
 
     def eql? other
-      start_item.eql?(other.start_item) and end_item.eql?(other.end_item)
+      start_item.eql?(other.start_item) && end_item.eql?(other.end_item)
     end
     
     def hash

data/lib/parser/parser.rb

@@ -19,17 +19,19 @@ module Dhaka
       @logger         = logger || default_logger
       @transitions    = Hash.new {|hash, state| hash[state] = {}}
       @grammar        = grammar
-      @channels       = []
+      @channels       = Hash.new {|hash, start_item| hash[start_item] = []}
       @states = Hash.new do |hash, kernel|
-          channels, closure = grammar.closure(kernel)
-          @channels.concat channels.to_a
+          closure, channels = grammar.closure(kernel)
+          channels.each do |start_item, channel_set|
+            @channels[start_item].concat channel_set.to_a
+          end
           new_state    = ParserState.new(self, closure)
           hash[kernel] = new_state
           @logger.debug("Created #{new_state.unique_name}.")
           new_state.transition_items.each do |symbol, items|
             destination_kernel = ItemSet.new(items.collect{|item| item.next_item})
             destination_state  = hash[destination_kernel]
-            items.each { |item| @channels << grammar.passive_channel(item, destination_state.items[item.next_item]) }
+            items.each {|item| @channels[item] << grammar.passive_channel(item, destination_state.items[item.next_item])}
             @transitions[new_state][symbol] = destination_state
           end
           new_state
@@ -87,8 +89,8 @@ module Dhaka
       start_items = ItemSet.new(start_productions.collect {|production| Item.new(production, 0)})
       start_items.each {|start_item| start_item.lookaheadset << grammar.end_symbol}
       @start_state = @states[start_items]
-      @logger.debug("Pumping #{@channels.size} channels...")
-      pump_channels
+      @logger.debug("Pumping #{@channels.keys.size} dirty items...")
+      pump_channels @channels.keys
       @logger.debug("Generating shift actions...")
       generate_shift_actions
       @logger.debug("Generating reduce actions...")
@@ -128,14 +130,17 @@ module Dhaka
       end
     end
   
-    def pump_channels
+    def pump_channels dirty_items
       loop do
-        unstable_count = 0
-        @channels.each do |channel| 
-          unstable_count += 1 if channel.pump
+        new_dirty_items = Set.new
+        dirty_items.each do |dirty_item| 
+          @channels[dirty_item].each do |channel| 
+            new_dirty_items << channel.end_item if channel.pump
+          end
         end
-        break if unstable_count.zero?
-        @logger.debug("#{unstable_count} unstable channels...")
+        break if new_dirty_items.empty?
+        @logger.debug("#{new_dirty_items.size} dirty items...")
+        dirty_items = new_dirty_items
       end
     end
   end

data/lib/parser/parser_state.rb

@@ -62,7 +62,9 @@ module Dhaka
 
   class ItemSet < Set #:nodoc:
     def hash
-      collect{|item| item.hash}.inject{|result, hashcode| result ^ hashcode}
+      result = 5381
+      each { |item| result ^= item.hash }
+      result
     end
     
     def eql? other

data/test/chittagong/chittagong_lexer.rb

@@ -2,108 +2,108 @@ class ChittagongLexer < Dhaka::CompiledLexer
 
   self.specification = ChittagongLexerSpecification
 
-  start_with 25917940
+  start_with 21969380
 
-  at_state(25901470) {
-    recognize("\\)")
+  at_state(21968700) {
+    accept("\n")
   }
 
-  at_state(25901730) {
-    recognize(",")
+  at_state(21958430) {
+    accept("-")
   }
 
-  at_state(25902250) {
-    recognize("\\/")
+  at_state(21967500) {
+    accept("<")
   }
 
-  at_state(25911660) {
-    recognize("==")
+  at_state(21957330) {
+    accept("\\(")
   }
 
-  at_state(25910420) {
-    recognize("\\w+")
-    for_characters("K", "V", "k", "v", "W", "A", "L", "w", "l", "a", "b", "M", "B", "X", "m", "x", "c", "Y", "C", "N", "y", "n", "O", "D", "Z", "o", "z", "d", "p", "e", "E", "P", "f", "Q", "F", "q", "G", "R", "r", "g", "S", "H", "s", "h", "I", "T", "i", "t", "U", "J", "u", "j") { switch_to 25910420 }
+  at_state(21957550) {
+    accept("\\/")
   }
 
-  at_state(25912350) {
-    recognize("!")
+  at_state(21967250) {
+    accept("\\*")
   }
 
-  at_state(25916370) {
-    for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 25915990 }
+  at_state(21968360) {
+    accept("=")
+    for_characters("=") { switch_to 21968040 }
   }
 
-  at_state(25916950) {
-    recognize("\\d*(\\.\\d+)?")
-    for_characters(".") { switch_to 25916370 }
-    for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 25916950 }
+  at_state(21969380) {
+    accept("\\d*(\\.\\d+)?")
+    for_characters("<") { switch_to 21967500 }
+    for_characters(")") { switch_to 21961890 }
+    for_characters(" ") { switch_to 21967770 }
+    for_characters("\n") { switch_to 21968700 }
+    for_characters("=") { switch_to 21968360 }
+    for_characters("8", "9", "0", "1", "2", "3", "4", "5", "6", "7") { switch_to 21961100 }
+    for_characters("J", "o", "p", "K", "q", "L", "r", "M", "s", "N", "t", "O", "a", "u", "P", "b", "Q", "c", "v", "R", "d", "w", "S", "e", "x", "T", "f", "y", "A", "U", "g", "z", "B", "h", "C", "V", "i", "D", "W", "j", "E", "X", "F", "Y", "k", "G", "Z", "l", "H", "m", "I", "n") { switch_to 21966980 }
+    for_characters("*") { switch_to 21967250 }
+    for_characters(",") { switch_to 21961360 }
+    for_characters("!") { switch_to 21958210 }
+    for_characters("/") { switch_to 21957550 }
+    for_characters(".") { switch_to 21960630 }
+    for_characters("-") { switch_to 21958430 }
+    for_characters(">") { switch_to 21961620 }
+    for_characters("+") { switch_to 21957990 }
+    for_characters("(") { switch_to 21957330 }
+    for_characters("^") { switch_to 21957770 }
   }
 
-  at_state(25911030) {
-    recognize("<")
+  at_state(21961360) {
+    accept(",")
   }
 
-  at_state(25902770) {
-    recognize(" ")
+  at_state(21961890) {
+    accept("\\)")
   }
 
-  at_state(25917260) {
-    recognize("\\(")
+  at_state(21966980) {
+    accept("\\w+")
+    for_characters("v", "K", "V", "k", "L", "W", "w", "l", "A", "a", "B", "X", "x", "b", "M", "m", "C", "c", "n", "y", "N", "Y", "d", "o", "z", "O", "Z", "D", "e", "p", "E", "P", "f", "q", "F", "Q", "R", "g", "r", "G", "h", "s", "H", "S", "t", "I", "T", "i", "U", "j", "u", "J") { switch_to 21966980 }
   }
 
-  at_state(25912660) {
-    recognize("-")
+  at_state(21957990) {
+    accept("\\+")
   }
 
-  at_state(25901210) {
-    recognize("\\^")
+  at_state(21960630) {
+    for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 21960320 }
   }
 
-  at_state(25912040) {
-    recognize("=")
-    for_characters("=") { switch_to 25911660 }
+  at_state(21961620) {
+    accept(">")
   }
 
-  at_state(25902510) {
-    recognize("\n")
+  at_state(21967770) {
+    accept(" ")
   }
 
-  at_state(25910730) {
-    recognize("\\*")
+  at_state(21968040) {
+    accept("==")
   }
 
-  at_state(25911340) {
-    recognize(">")
+  at_state(21960320) {
+    accept("\\d*(\\.\\d+)?")
+    for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 21960320 }
   }
 
-  at_state(25901990) {
-    recognize("\\+")
+  at_state(21957770) {
+    accept("\\^")
   }
 
-  at_state(25915990) {
-    recognize("\\d*(\\.\\d+)?")
-    for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 25915990 }
+  at_state(21958210) {
+    accept("!")
   }
 
-  at_state(25917940) {
-    recognize("\\d*(\\.\\d+)?")
-    for_characters("^") { switch_to 25901210 }
-    for_characters("/") { switch_to 25902250 }
-    for_characters("=") { switch_to 25912040 }
-    for_characters("(") { switch_to 25917260 }
-    for_characters(")") { switch_to 25901470 }
-    for_characters("\n") { switch_to 25902510 }
-    for_characters(".") { switch_to 25916370 }
-    for_characters("!") { switch_to 25912350 }
-    for_characters("<") { switch_to 25911030 }
-    for_characters(",") { switch_to 25901730 }
-    for_characters("J", "o", "p", "K", "q", "L", "r", "M", "s", "N", "t", "O", "a", "u", "P", "b", "Q", "c", "v", "R", "d", "w", "S", "e", "x", "T", "f", "y", "A", "U", "g", "z", "B", "h", "C", "V", "i", "D", "W", "j", "E", "X", "F", "Y", "k", "G", "Z", "l", "H", "m", "I", "n") { switch_to 25910420 }
-    for_characters(" ") { switch_to 25902770 }
-    for_characters("-") { switch_to 25912660 }
-    for_characters(">") { switch_to 25911340 }
-    for_characters("+") { switch_to 25901990 }
-    for_characters("8", "9", "0", "1", "2", "3", "4", "5", "6", "7") { switch_to 25916950 }
-    for_characters("*") { switch_to 25910730 }
+  at_state(21961100) {
+    accept("\\d*(\\.\\d+)?")
+    for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 21961100 }
+    for_characters(".") { switch_to 21960630 }
   }
 
 end

data/test/chittagong/chittagong_lexer.rb.rej

@@ -0,0 +1,189 @@
+***************
+*** 5
+-   start_with 21824080
+--- 5 -----
++   start_with 22347760
+***************
+*** 7,8
+-   at_state(21819770) {
+-     for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 21819470 }
+--- 7,8 -----
++   at_state(22321370) {
++     accept("\\)")
+***************
+*** 11,12
+-   at_state(21821860) {
+-     recognize("\\+")
+--- 11,12 -----
++   at_state(22339000) {
++     for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 22337320 }
+***************
+*** 15,17
+-   at_state(21817320) {
+-     recognize("\\w+")
+-     for_characters("k", "V", "K", "v", "L", "A", "a", "w", "l", "W", "M", "B", "X", "x", "m", "b", "y", "n", "c", "N", "Y", "C", "d", "Z", "O", "D", "z", "o", "E", "p", "e", "P", "Q", "F", "q", "f", "r", "g", "R", "G", "h", "S", "H", "s", "T", "I", "t", "i", "J", "u", "j", "U") { switch_to 21817320 }
+--- 15,16 -----
++   at_state(22342900) {
++     accept("==")
+***************
+*** 20,21
+-   at_state(21820580) {
+-     recognize("!")
+--- 19,37 -----
++   at_state(22347760) {
++     accept("\\d*(\\.\\d+)?")
++     for_characters(")") { switch_to 22321370 }
++     for_characters("^") { switch_to 22291450 }
++     for_characters("+") { switch_to 22322410 }
++     for_characters("(") { switch_to 22319550 }
++     for_characters(" ") { switch_to 22289950 }
++     for_characters(">") { switch_to 22293250 }
++     for_characters("8", "9", "0", "1", "2", "3", "4", "5", "6", "7") { switch_to 22341260 }
++     for_characters("\n") { switch_to 22294340 }
++     for_characters(".") { switch_to 22339000 }
++     for_characters(",") { switch_to 22320520 }
++     for_characters("!") { switch_to 22325520 }
++     for_characters("*") { switch_to 22292410 }
++     for_characters("J", "o", "p", "K", "q", "L", "r", "M", "s", "N", "t", "O", "a", "u", "P", "b", "Q", "c", "v", "R", "d", "w", "S", "e", "x", "T", "f", "y", "A", "U", "g", "z", "B", "h", "C", "V", "i", "D", "W", "j", "E", "X", "F", "Y", "k", "G", "Z", "l", "H", "m", "I", "n") { switch_to 22317930 }
++     for_characters("/") { switch_to 22288930 }
++     for_characters("-") { switch_to 22326990 }
++     for_characters("=") { switch_to 22344860 }
++     for_characters("<") { switch_to 22324070 }
+***************
+*** 24,25
+-   at_state(21821040) {
+-     recognize("\\^")
+--- 40,41 -----
++   at_state(22325520) {
++     accept("!")
+***************
+*** 28,29
+-   at_state(21822840) {
+-     recognize(",")
+--- 44,46 -----
++   at_state(22317930) {
++     accept("\\w+")
++     for_characters("K", "V", "k", "v", "a", "A", "L", "W", "l", "w", "b", "B", "M", "X", "m", "x", "C", "N", "Y", "c", "n", "y", "D", "O", "Z", "d", "o", "z", "E", "P", "e", "p", "F", "Q", "f", "q", "G", "R", "g", "r", "H", "S", "h", "s", "I", "T", "i", "t", "j", "u", "J", "U") { switch_to 22317930 }
+***************
+*** 32,33
+-   at_state(21822110) {
+-     recognize(">")
+--- 49,50 -----
++   at_state(22320520) {
++     accept(",")
+***************
+*** 36,54
+-   at_state(21824080) {
+-     recognize("\\d*(\\.\\d+)?")
+-     for_characters(".") { switch_to 21819770 }
+-     for_characters("+") { switch_to 21821860 }
+-     for_characters("/") { switch_to 21820370 }
+-     for_characters("*") { switch_to 21823340 }
+-     for_characters(",") { switch_to 21822840 }
+-     for_characters("!") { switch_to 21820580 }
+-     for_characters("=") { switch_to 21821620 }
+-     for_characters(" ") { switch_to 21822330 }
+-     for_characters("\n") { switch_to 21823100 }
+-     for_characters(">") { switch_to 21822110 }
+-     for_characters("(") { switch_to 21820790 }
+-     for_characters("<") { switch_to 21823590 }
+-     for_characters("J", "o", "p", "K", "q", "L", "r", "M", "s", "N", "t", "O", "a", "u", "P", "b", "Q", "c", "v", "R", "d", "w", "S", "e", "x", "T", "f", "y", "A", "U", "g", "z", "B", "h", "C", "V", "i", "D", "W", "j", "E", "X", "F", "Y", "k", "G", "Z", "l", "H", "m", "I", "n") { switch_to 21817320 }
+-     for_characters(")") { switch_to 21817530 }
+-     for_characters("-") { switch_to 21822580 }
+-     for_characters("^") { switch_to 21821040 }
+-     for_characters("8", "9", "0", "1", "2", "3", "4", "5", "6", "7") { switch_to 21820160 }
+--- 53,54 -----
++   at_state(22322410) {
++     accept("\\+")
+***************
+*** 57,58
+-   at_state(21817530) {
+-     recognize("\\)")
+--- 57,59 -----
++   at_state(22344860) {
++     accept("=")
++     for_characters("=") { switch_to 22342900 }
+***************
+*** 61,62
+-   at_state(21822580) {
+-     recognize("-")
+--- 62,63 -----
++   at_state(22324070) {
++     accept("<")
+***************
+*** 65,66
+-   at_state(21823340) {
+-     recognize("\\*")
+--- 66,67 -----
++   at_state(22288930) {
++     accept("\\/")
+***************
+*** 69,70
+-   at_state(21820370) {
+-     recognize("\\/")
+--- 70,71 -----
++   at_state(22289950) {
++     accept(" ")
+***************
+*** 73,74
+-   at_state(21823100) {
+-     recognize("\n")
+--- 74,75 -----
++   at_state(22291450) {
++     accept("\\^")
+***************
+*** 77,80
+-   at_state(21820160) {
+-     recognize("\\d*(\\.\\d+)?")
+-     for_characters(".") { switch_to 21819770 }
+-     for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 21820160 }
+--- 78,79 -----
++   at_state(22292410) {
++     accept("\\*")
+***************
+*** 83,84
+-   at_state(21820790) {
+-     recognize("\\(")
+--- 82,83 -----
++   at_state(22294340) {
++     accept("\n")
+***************
+*** 87,88
+-   at_state(21821310) {
+-     recognize("==")
+--- 86,88 -----
++   at_state(22337320) {
++     accept("\\d*(\\.\\d+)?")
++     for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 22337320 }
+***************
+*** 91,92
+-   at_state(21822330) {
+-     recognize(" ")
+--- 91,94 -----
++   at_state(22341260) {
++     accept("\\d*(\\.\\d+)?")
++     for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 22341260 }
++     for_characters(".") { switch_to 22339000 }
+***************
+*** 95,96
+-   at_state(21823590) {
+-     recognize("<")
+--- 97,98 -----
++   at_state(22293250) {
++     accept(">")
+***************
+*** 99,101
+-   at_state(21819470) {
+-     recognize("\\d*(\\.\\d+)?")
+-     for_characters("6", "7", "8", "9", "0", "1", "2", "3", "4", "5") { switch_to 21819470 }
+--- 101,102 -----
++   at_state(22319550) {
++     accept("\\(")
+***************
+*** 104,106
+-   at_state(21821620) {
+-     recognize("=")
+-     for_characters("=") { switch_to 21821310 }
+--- 105,106 -----
++   at_state(22326990) {
++     accept("-")