kanocc 0.1.0 → 0.2.0

data/examples/ruby_quiz_78.rb

@@ -31,6 +31,7 @@ require "kanocc"
 # PackageList ::= Package
 #               | PackageList Package
 
+# A forward declaration 
 class PackageList < Kanocc::Nonterminal
 end
 
@@ -43,23 +44,23 @@ class Package < Kanocc::Nonterminal
   rule('{', PackageList , '}') { @val = "{#{@rhs[1].val}}"} 
   rule('[', PackageList , ']') { @val = "[#{@rhs[1].val}]"} 
   # Some error-correcting rules 
-  rule(PackageList, ')') {@val = "(#{@rhs[0].val})"}; prec -2
-  rule('(', PackageList) {@val = "(#{@rhs[1].val})"}; prec -2
-  rule(PackageList, '}') {@val = "{#{@rhs[0].val}}"}; prec -2
-  rule('{', PackageList) {@val = "{#{@rhs[1].val}}"}; prec -2
-  rule(PackageList, ']') {@val = "[#{@rhs[0].val}]"}; prec -2
-  rule('[', PackageList) {@val = "[#{@rhs[1].val}]"}; prec -2
+  rule(PackageList, ')') {@val = "(#{@rhs[0].val})"}; precedence -2
+  rule('(', PackageList) {@val = "(#{@rhs[1].val})"}; precedence -2
+  rule(PackageList, '}') {@val = "{#{@rhs[0].val}}"}; precedence -2
+  rule('{', PackageList) {@val = "{#{@rhs[1].val}}"}; precedence -2
+  rule(PackageList, ']') {@val = "[#{@rhs[0].val}]"}; precedence -2
+  rule('[', PackageList) {@val = "[#{@rhs[1].val}]"}; precedence -2
 end
 
-class PackageList 
+class PackageList < Kanocc::Nonterminal
   attr_reader :val
   rule(om(Package)){ @val = @rhs[0].elements.map{|p| p.val}.join("") }
 end
 
 # Set up a parser 
-packageChecker = Kanocc::Kanocc.new(Package)
+packageChecker = Kanocc::Kanocc.new(PackageList)
 
 # And go
-puts "[(B)] becomes " + packageChecker.parse('[(B)]').val
-puts "[[B] becomes " + packageChecker.parse('[[B]').val
-puts "[(B)]](B){{(B)] becomes " + packageChecker.parse("[(B)]](B){{(B)]").val
+puts "[(B)] .............becomes........ " + packageChecker.parse('[(B)]').val
+puts "[[B]  .............becomes........ " + packageChecker.parse('[[B]').val
+puts "[(B)]](B){{(B)] ...becomes........ " + packageChecker.parse("[(B)]](B){{(B)]").val

data/lib/kanocc.rb

@@ -46,7 +46,7 @@ require 'logger'
 #
 #    NUM a sequence of digits
 #
-# In Kanocc yout could do it like this:
+# In Kanocc you could do it like this:
 #
 #    require "kanocc"
 #    
@@ -80,52 +80,46 @@ require 'logger'
 #
 module Kanocc
   class Kanocc
-    attr_accessor :scanner, :parser, :logger
+    attr_accessor :parser, :logger
     
     # Creates a new instance of Kannocc, with the given start symbol.
     # From the start_symbol, Kanocc will deduce the grammar and the 
     # grammarsymbols
     # 
     def initialize(start_symbol)
-      @start_symbol = start_symbol 
+      @start_symbol = start_symbol
       @logger = Logger.new(STDOUT)
       @logger.datetime_format = "" 
-      @logger.level = Logger::WARN 
-      @scanner = Scanner.new(:logger => @logger)
-      @parser = EarleyParser.new(self, :logger => @logger)
+      @logger.level = Logger::WARN
+      @scanner = Scanner.new
+      @scanner.set_recognized(*find_tokens(@start_symbol))
+      @parser = EarleyParser.new(self, @logger)
     end
     
     def logger=(logger)
       @logger = logger || logger.new(STDOUT)
-      @parser.logger = @logger if parser.respond_to?(:logger)
-      @scanner.logger = @logger if scanner.respond_to?(:logger)
-    end
-  
-    def parser=(parser)
-      @parser = parser
       @parser.logger = @logger if parser.respond_to?(:logger=)
     end
-    
-    def scanner=(scanner)
-      @scanner = scanner
-      @scanner.logger = @logger if scanner.respond_to?(:logger=)
-    end
-    
+  
+        
     # Consume input. Kanocc will parse input according to the rules given, and
     # - if parsing succeeds - return an instance of the grammars start symbol.
     # Input may be a String or an IO object.
     def parse(input)
+      if input.is_a?(IO) 
+        @input = input.readlines.join("")
+      elsif input.is_a?(String) 
+        @input = input
+      else
+        raise "Input must be a string or an IO object"
+      end 
       raise "Start symbol not defined" unless @start_symbol
-      tell_parser_start_symbol(@start_symbol) 
-      @parser.prepare 
+      @input = input
+      @scanner.input = input
+      @parser.start_symbol = @start_symbol 
       @stack = []
-      @inputPos = 0 
-      @scanner.each_token(input) do |token_match|
-        @logger.info "got #{token_match.inspect} from scanner"
-        @inputPos += 1
-        @parser.consume(token_match)
-      end
-      @parser.eof
+      @parser.parse(@scanner)
+      @logger.info("Stack: " + @stack.inspect)
       @stack[0][0]
     end
    
@@ -160,8 +154,8 @@ module Kanocc
     def report_reduction(rule) 
       @logger.info "Reducing by " + rule.inspect
       raise "Fatal: stack too short!" if @stack.length < rule.rhs.length
-      nonterminal = rule.lhs.new      
-      stack_part = @stack.slice!(-rule.rhs.length, rule.rhs.length)   
+      nonterminal = rule.lhs.new
+      stack_part = @stack.slice!(-rule.rhs.length, rule.rhs.length)
       if rule.rhs.length > 0
         start_pos, end_pos = stack_part[0][1], stack_part[-1][2]
       elsif @stack.length > 0
@@ -170,7 +164,7 @@ module Kanocc
         start_pos, end_pos = 0,0
       end 
       if rule.method
-	rhs = Rhs.new(stack_part.map{|a| a[0]}, start_pos, end_pos)
+	rhs = Rhs.new(stack_part.map{|a| a[0]}, start_pos, end_pos, @input)
         old_rhs = nonterminal.instance_variable_get('@rhs')
         nonterminal.instance_variable_set('@rhs', rhs)
         nonterminal.send(rule.method)
@@ -181,81 +175,47 @@ module Kanocc
       show_stack
     end
     
-    def calculate_start_and_end_pos(rule)
-          end
-
-    def evaluate_semantics_and_pop(rule, nonterminal)
-   end    
-   
     # The parser must call this method when it consumes a token
-    # As argument it should give the consumed token and the positions 
-    # in the input string corresponding to the token. Positions should be given
-    # as the position of the first character of the token and the position of the 
-    # first character after the token.
-    def report_token(tokenmatch, element)
-      @logger.info("Pushing token: " + element.inspect)
-      match = tokenmatch[:matches].find do |m| 
-	m[:token] == element || m[:literal] == element
-      end 
-       
-      if match[:token]  
-        token = match[:token].new
-        token.m = match[:regexp].match(tokenmatch[:string])
-        token.send(match[:method_name]) if match[:method_name] 
+    # As argument it should give the LexicalMatch and the matched terminal.
+    def report_token(lexical_match, terminal)
+      start_pos = lexical_match.start_pos
+      length = lexical_match.length
+      stringpart = @input.slice(start_pos, length)
+      if terminal.class == Class # It's a token
+	instance = terminal.new
+	regexp = lexical_match.regexp(terminal)
+	instance.m = regexp.match(stringpart)
+        if method = terminal.method(regexp)
+	  instance.send(method)
+	end
       else # It's a string literal
-        token = match[:literal]
+	instance = terminal
       end
-      
-      start_pos = tokenmatch[:start_pos]
-      end_pos = start_pos + tokenmatch[:length]
-      token_with_pos = [token, start_pos, end_pos]
-      
-      @stack.push(token_with_pos)
-      show_stack 
-    end
-        
-    
-    def tell_parser_start_symbol(start_symbol)
-      @parser.startsymbol = start_symbol
-      bag_of_terminals = {}
-      find_tokens(start_symbol, bag_of_terminals)
-      @logger.debug "tokens = " + bag_of_terminals.keys.inspect 
-      strings = bag_of_terminals.keys.find_all{|ter| ter.is_a? String} 
-      @logger.info("Literals: " + strings.inspect)
-      tokens = bag_of_terminals.keys.find_all{|ter| ter.is_a? Class and ter.ancestors.member?(Token)}
-      @logger.info("Tokens: " + tokens.inspect)
-      @scanner.set_recognized(*(strings + tokens))
 
-      # Show rules
-      @logger.info("Rules:")
-      nonterminals = [start_symbol]
-      nonterminals.each do |nonterminal|
-        nonterminal.rules.each do |rule|
-          @logger.info("  " + rule.inspect)
-	  rule.rhs.each do |gs|
-	    if gs.is_a? Class and gs.ancestors.member?(Nonterminal) and not nonterminals.member?(gs)
-	      nonterminals.push(gs)
-	    end
-	  end
-	end
-      end
+      @stack.push([instance, start_pos, start_pos + length])
+      show_stack
     end
-    
-    def find_tokens(nonterminal, collectedTokens,  visited_nonterminals = {})
+       
+    def find_tokens(nonterminal)   
+      collected_tokens = {}
+      find_tokens_helper(nonterminal, collected_tokens)
+      collected_tokens.keys
+    end
+    def find_tokens_helper(nonterminal, collected_tokens,  visited_nonterminals = {})
       unless visited_nonterminals[nonterminal]
         visited_nonterminals[nonterminal] = true
         nonterminal.rules.each do |r| 
           r.rhs.each do |gs|
             if gs.is_a?(Class) and gs.ancestors.member?(Nonterminal)
-              find_tokens(gs, collectedTokens, visited_nonterminals)
+              find_tokens_helper(gs, collected_tokens, visited_nonterminals)
             else
-              collectedTokens[gs] = true 
+              collected_tokens[gs] = true 
             end
           end
         end
       end
     end
-        
+
     # For debugging
     def show_stack
       @logger.info("Stack: #{@stack.inspect}") if @logger
@@ -274,30 +234,41 @@ module Kanocc
         gs.inspect
       end
     end
-  
+
   end
     
   class Rhs < Array
-    attr_accessor :start_pos, :end_pos
-    def initialize(arr, start_pos, end_pos)
-      @start_pos, @end_pos = start_pos, end_pos
+    attr_reader :start_pos, :end_pos
+    def initialize(arr, start_pos, end_pos, input)
+      @start_pos, @end_pos, @input = start_pos, end_pos, input
       super(arr)
     end
 
+    def text
+      @input.slice(start_pos, end_pos - start_pos)
+    end
+    
     def inspect
       return "#{super.inspect}, #{start_pos.inspect}, #{end_pos.inspect}"
     end
   end
-
-  class ParseException < Exception 
-    attr_accessor :inputPos, :inputSymbol, :expected 
-    def initialize(inputPos, inputSymbol, expected)
-      @inputPos, @inputSymbol, @expected = inputPos, inputSymbol, expected
-    end
-  end
   
   class KanoccException < Exception
   end
-end
-
 
+  class ParseException < KanoccException
+    attr_reader :expectedTerminals, :offendingInput, :pos
+    def initialize(expected_terminals, offending_input, pos)
+      @expected_terminals, @offending_input, @pos =
+	expected_terminals, offending_input, pos
+      error_msg = "\n  Could not consume input: #{offending_input} at #{pos}"
+      if expected_terminals.size > 0
+        error_msg += " - expected " +
+                     "#{expected_terminals.map {|t| t.inspect}.join(" or ")}"
+      else
+        error_msg += " - no input could be consumed at this point."
+      end
+      super(error_msg)
+    end
+  end
+end

data/lib/kanocc/earley.rb

@@ -1,4 +1,4 @@
-#  
+## 
 #  Copyright 2008 Christian Surlykke
 #
 #  This file is part of Kanocc.
@@ -16,8 +16,12 @@
 #  version 3 along with Kanocc.  If not, see <http://www.gnu.org/licenses/>.
 #
 require 'kanocc/grammar_rule'
+require 'kanocc/nonterminal'
 require 'kanocc/token'
 require 'logger'
+
+#require 'rubygems'
+
 module Kanocc
   #
   # Parser for Kanocc based on Earleys algorithm. For a description see:
@@ -33,290 +37,331 @@ module Kanocc
   # Christian Surlykke 2007.
   #
   class EarleyParser
-    attr_accessor :kanocc, :logger
+    attr_accessor :kanocc, :logger 
     
     ErrorRule = GrammarRule.new(Error, [], nil)
     
-    def initialize(kanocc, options = {})
+    def initialize(kanocc, logger)
       @kanocc = kanocc
-      @logger = options[:logger] || Logger.new
+      @logger = logger
+    end
+    
+    def start_symbol=(start_symbol)   
+      @start_symbol = Class.new(StartSymbol) do
+        def self.to_s
+         "S'"
+        end
+        rule(start_symbol)
+      end 
     end
+    
 
-    #
-    # Sets up the parser, creating itemlist 0.
-    #
-    def startsymbol=(startSymbol)
-      @start_symbol = startSymbol
-      @itemLists = [ItemList.new(nil, 0)]
-      @inputPos = 0
-      @recoveryPoints = []
-      @itemLists[0].add_all(@start_symbol.rules.map{|rule| Item.new(rule, 0)})
-      predict_and_complete(0)
+    def parse(scanner)
+      @scanner = scanner
+      prepare
+      
+     while (@scanner.next_match!) do
+        @inputPos += 1
+        @input_symbols.push(scanner.current_match)
+        @items.prepare_for_n(@inputPos)
+        # scan, predict and complete until no more can be added
+
+        scan
+
+        predict_and_complete(@inputPos)
+
+        if @logger
+          @logger.info("\nItems at #{@inputPos}:\n" +
+                       @input_symbols[@inputPos].inspect + "\n" +
+                       @items.items_at_n(@inputPos).map{|item| " " + item.inspect}.join("\n") + "\n")
+        end
+
+        handle_error if @items.number_at_n(@inputPos) == 0
+      end
+
+      reduce
     end
     
     def prepare
-      @itemLists = @itemLists[0..0]
+      @items = ItemSet.new
       @inputPos = 0
-      if @recoveryPoints.size > 0 and @recoveryPoints[0] == 0
-        @recoveryPoints = [0]
-      else
-        @recoveryPoints = []
+      @input_symbols = [nil]
+      @recoveryPoints = []
+      @start_symbol.rules.each do |rule|
+        @items.add(rule, 0, 0, 0, -1)
+      end
+      predict_and_complete(0)
+      if @logger
+        @logger.info("\nItems at 0:\n" +
+                     @items.items_at_n(0).map{|item| " " + item.inspect}.join("\n") + "\n")
       end
-      @logger.info("Itemlist 0:\n" + @itemLists[0].inspect) unless not @logger
     end
+    
+    # Scan: At position n, for each terminal a in current match, and each item
+    # of form [A -> x*ay, i, n-1], add [A -> xa*y, i, n]
+    def scan
 
-   
-    def scan(token_match) 
-      token_match[:matches].each do |match| 
-        if match[:token] 
-	  symbol = match[:token]
-        else
-          symbol = match[:literal]
+      @scanner.current_match.terminals.each do |terminal|
+        @items.items_n_and_symbol_after_dot(@inputPos -1, terminal).each do |item|
+           @items.add(item.rule, item.dot + 1, item.j, @inputPos,  @inputPos - 1)
         end
-	items = @itemLists[@inputPos - 1].find_matching(symbol)
-	@itemLists[@inputPos].add_all(items.map{|item| item.move})
       end
+
     end
-    
-    def predict_and_complete(pos)
-      item_list = @itemLists[pos] 
-      prev_size = 0      
-      while prev_size < item_list.size do 
-        prev_size = item_list.size	
-	item_list.each do |item|
-	  if item.rule.rhs.length <= item.dot
-            # complete 
-	    item_list.add_all(@itemLists[item.j].find_matching(item.rule.lhs).map{|item| item.move})
-          elsif (nont = item.rule.rhs[item.dot]).respond_to?(:rules)  
+
+
+    # Predict: For any item of form [A -> a*Bb, j, n] and for all rules of form
+    # B -> c, add [B -> *c, n, n].
+    #
+    # Complete: Given an item of form [A->X*, j, n], find all items of form
+    # [B -> a*Ab, i, j], and add [B -> aA*b, i, n].
+    #
+    # Predict and complete until nothing further can be added.
+    def predict_and_complete(pos, show=false)
+      prev_size = 0
+      while true do
+        break if prev_size >= @items.number_at_n(pos)
+        prev_size = @items.number_at_n(pos)
+        @items.items_at_n(pos).each do |item|
+          if item.dot >= item.rule.rhs.length
+            # complete
+            @items.items_n_and_symbol_after_dot(item.j, item.rule.lhs).each do |previtem|
+              @items.add(previtem.rule, previtem.dot + 1, previtem.j, pos, item.j)
+            end
+          elsif item.rule.rhs[item.dot].respond_to?(:rules)
             # predict
-	    item_list.add_all(nont.rules.map {|rule| Item.new(rule, @inputPos)})
-	  end
+            item.rule.rhs[item.dot].rules.each do |rule|
+              @items.add(rule, 0, pos, pos, -1)
+            end
+          end
         end
-      end 
+      end
     end
     
-    def add_recovery_points(pos)
-      if @recoveryPoints[-1] != pos
-	@itemLists[pos].each do |item| 
-	  if Error == item.rule.rhs[item.dot]
-	    @recoveryPoints.push(pos)
-	    break
-	  end
+    def handle_error
+      if j = find_error_items()
+        @items.add(ErrorRule, 0, j, @inputPos - 1, -1)
+        predict_and_complete(@inputPos - 1, true)
+        if @logger
+	  @logger.info("Items at #{@inputPos - 1} after error handling:\n" + 
+	               @items.items_at_n(@inputPos - 1).map {|item| item.inspect}.join("\n"))
 	end
+	scan
+        predict_and_complete(@inputPos)
+        if @logger
+          @logger.info("Items at #{@inputPos} after error handling:\n" +
+                       @items.items_at_n(@inputPos).map {|item| item.inspect}.join("\n"))
+        end
+      else
+	expected_terminals =
+	  @items.items_at_n(@inputPos - 1).map { |item| item.rule.rhs[item.dot]}.find_all do |gs|
+	    gs.is_a? String or (gs.is_a? Class and gs.ancestors.include?(Token))
+	  end.uniq
+
+	pos, length = @scanner.current_match.start_pos, @scanner.current_match.length
+	offending_input = @scanner.input[pos, length].inspect
+	raise ParseException.new(expected_terminals, offending_input, pos)
       end
     end
-   
-    #
-    # Consume and parse next input symbol
-    #
-    def consume(token_match) 
-      @inputPos += 1
-      @itemLists.push(ItemList.new(token_match, @inputPos))
-  
-      # scan, predict and complete until no more can be added
-      scan(token_match)
-      
-      if @itemLists[@inputPos].size == 0
-        @logger.debug("Found no items matching #{token_match} in itemlist #{@inputPos - 1}")
-        @logger.debug("@recoveryPoints = " + @recoveryPoints.inspect)	
-        for i in 1..@recoveryPoints.length do 
-          if @recoveryPoints[-i] < @inputPos
-            @itemLists[@inputPos - 1].add(Item.new(ErrorRule, @recoveryPoints[-i]))
-            predict_and_complete(@inputPos - 1) 
-	    scan(token_match) 
-	    break if @itemLists[@inputPos].size > 0 
-          end
+
+    def find_error_items
+      for n in (@inputPos - 1).downto(0) do
+        if @items.items_n_and_symbol_after_dot(n, Error).size > 0
+          return n
         end
       end
-      predict_and_complete(@inputPos) 
-      add_recovery_points(@inputPos)
-      @logger.info("Itemlist #{@inputPos}:\n" + @itemLists[@inputPos].inspect) if @logger
+      return nil
     end
-   
-    
 
-
-    #
-    # Signal to the parser that end of input is reached
-    #
-    def eof
-      top_item = find_full_items(@start_symbol, @inputPos).find_all {|item| item.j == 0}.max
-      if top_item 
-        translate(top_item, @inputPos)
+    def reduce
+      item = @items.items_at_n(@inputPos).find do |item|
+        @start_symbol == item.rule.lhs and item.dot == 1
+      end
+      if item
+        # There is at most one of those
+        make_parse(item, @inputPos, 0)
       else
         raise(KanoccException, "It didn't parse")
       end
     end
-    
-    def translate(element, pos)
-      @logger.debug("translate: " + element.inspect + " on " + pos.inspect)   
-      if element.class == Item
-        translate_helper(element, pos) 
-        @kanocc.report_reduction(element.rule)
-      else  # Its a token or a string
-	@kanocc.report_token(@itemLists[pos].inputSymbol, element)
+
+    # FIXME Generates stack overflow when files are large.
+    #  15000-2000 inputsymbols with the calculator syntax.
+    # Should be rewritten to something non-recursive
+    def make_parse(item, pos, prev_pos)
+      return if item.dot <= 0
+
+      prev_item = @items.find(item.rule, item.dot - 1, item.j, prev_pos)
+      prev_prev_pos = prev_item.rule.derives_right ? prev_item.prev_pos_min : prev_item.prev_pos_max
+      
+      if is_nonterminal?(item.symbol_before_dot)
+        subitem, sub_prev_pos = pick_subitem(item.symbol_before_dot, pos, prev_pos)
+        make_parse(prev_item, prev_pos, prev_prev_pos)
+        make_parse(subitem, pos, sub_prev_pos)
+        @kanocc.report_reduction(subitem.rule)
+      else
+        make_parse(prev_item, prev_pos, prev_prev_pos)
+        symbol = item.symbol_before_dot
+        @kanocc.report_token(@input_symbols[pos], symbol)
       end
     end
-    
-    def translate_helper(item, pos)
-      @logger.debug("translateHelper: " + item.inspect + " on " + pos.inspect) 
-      return if item.dot == 0 
-      if item.rule.rhs[item.dot - 1].respond_to?("rules")
-        # Assume item is of form [A --> aB*c, k] in itemlist i
-        # Must then find item of form [B --> x*, j] in itemlist i so 
-        # that there exists item of form [A --> a*Bc, k] on itemlist j
-        
-        # First: Items of form [B --> x*, j] on list i 
-        candidates = find_full_items(item.rule.rhs[item.dot - 1], pos)
-        
-        # Then: Those for which item of form [A --> a*Bc, k] exists
-        # on list j
-        candidates = candidates.find_all {|subItem|
-          @itemLists[subItem.j].find_item(item.rule, item.dot - 1, item.j)
-        }
-        
-        # Precedence: We pick the posibility with the higest precedence
-        sub_item = candidates.max
-        prev_item = @itemLists[sub_item.j].find_item(item.rule, item.dot - 1, item.j)
-        prev_list = sub_item.j
+
+    def pick_subitem(nonterminal, pos, prev_pos)
+      #debugger
+      items = @items.full_items_by_lhs_j_and_n(nonterminal, prev_pos, pos)
+
+      raise "pick_subitem could not find any items" if items.size <= 0
+      items = find_highest(items) {|item| precedence(item)}
+
+      derives_right = all_derives_right(items)
+      if derives_right
+        items = find_highest(items) {|item| -item.prev_pos_min}
       else
-        prev_item = @itemLists[pos - 1].find_item(item.rule, item.dot - 1, item.j)
-        prev_list = pos - 1
-        sub_item = item.rule.rhs[item.dot - 1]
+        items = find_highest(items){|item| item.prev_pos_max}
       end
-      translate_helper(prev_item, prev_list)
-      translate(sub_item, pos)
+
+      return items[0], derives_right ? items[0].prev_pos_min : items[0].prev_pos_max
     end
 
-    
-    
-    def find_full_items(nonterminal, inputPos)
-      @itemLists[inputPos].find_all do |item|
-        item.rule.lhs == nonterminal and item.dot >= item.rule.rhs.length
+    def find_highest(items, &expr)
+      collect = []
+      top_val = nil;
+      items.each do |item|
+        val = expr.call(item)
+        if top_val == nil or top_val < val
+          collect = [item]
+          top_val = val
+        elsif top_val == val
+          collect << item
+        end
       end
+      return collect
     end
-  end 
-  
-  class ItemList
-    attr_reader :inputSymbol
-    attr_accessor :items
-    
-    def initialize(inputSymbol, inputPos)
-      @inputPos = inputPos
-      @inputSymbol = inputSymbol
-      @items = Hash.new 
-    end
-    
-    def copy
-      res = clone
-      res.items = @items.clone
-      return res
-    end
-    
-    def size
-      return @items.size
-    end
-    
-    def find_all(&b)
-      return @items.keys.find_all(&b)
-    end
-    
-    def find_item(rule, dot, j)
-      return @items.keys.find{ |item| 
-        item.rule == rule and 
-        item.dot == dot and
-        item.j == j
-      }
+
+    def precedence(item)
+      item.rule.precedence || 0
     end
-    
-    def each_matching(inputSymbol)
-      find_matching(inputSymbol).each do |item| 
-        yield(item)
+      
+    def all_derives_right(items)
+      items.each do |item|
+        return false unless item.rule.derives_right
       end
+      return true
     end
-    
-    def find_matching(inputSymbol)
-      @items.keys.find_all do |item| 
-        inputSymbol === item.symbol_after_dot or inputSymbol == item.symbol_after_dot
-      end
+
+    def is_nonterminal?(symbol)
+      symbol.respond_to?(:rules)
     end
+  end   
     
-    def contains(item)
-      return @items[item]
+  class Item
+    attr_reader :rule, :dot, :j, :n
+    attr_accessor :prev_pos_min, :prev_pos_max
+
+    def initialize(rule, dot, j, n,  prev_pos_min = 0, prev_pos_max = 0)
+      @rule = rule
+      @dot = dot
+      @j = j
+      @n = n
+      @prev_pos_min = prev_pos_min
+      @prev_pos_max = prev_pos_max
     end
-    
-    def add(item)
-      @items.store(item, true)
+
+    def symbol_after_dot
+      return @dot < @rule.rhs.size  ? @rule.rhs[@dot] : nil
     end
     
-    def add_all(items)
-      items.each {|item| @items.store(item, true)}
+    def symbol_before_dot
+      return @dot > 0 ? @rule.rhs[@dot - 1] : nil
     end
 
-    def each
-      @items.keys.each do |item|
-        yield item
+    def set_prev_pos(new_prev_pos)
+      if new_prev_pos < @prev_pos_min
+        @prev_pos_min = new_prev_pos
+      elsif new_prev_pos > @prev_pos_max
+        @prev_pos_max = new_prev_pos
       end
     end
     
     def inspect
-      return "[" + @inputSymbol.inspect + "\n " + 
-                   @items.keys.map{|item| item.inspect}.join("\n  ") + "]\n" 
+      return "[" + 
+      @rule.lhs.inspect + " --> " + 
+       (@rule.rhs.slice(0, dot) + [Dot.instance] +
+      @rule.rhs.slice(dot, @rule.rhs.length - dot)).map{|symbol| symbol.inspect}.join(" ") + 
+            " ; " + @j.inspect + ", " + @n.inspect + "]"
     end
   end
   
-  
-  class Item
-    attr_reader :rule, :j, :dot
-    @@items = Hash.new
-    
-    def Item.new(rule, j, dot = 0)
-      unless (item = @@items[[rule,j,dot]])
-        item = super(rule, j, dot)
-        @@items.store([rule, j, dot], item)
+
+  class ItemSet
+    # FIXME Optimize all this
+
+    def initialize
+      @item_lists = []
+      @items_n_and_symbol_after_dot = {}
+      @items_rule_dot_j_n = {}
+    end
+
+    def prepare_for_n(n)
+      @item_lists[n] = []
+    end
+
+    def add(rule, dot, j, n, prev_pos)
+      if item = @items_rule_dot_j_n[[rule,dot,j,n]]
+        item.set_prev_pos(prev_pos)
+      else
+        item = Item.new(rule, dot, j, n, prev_pos, prev_pos)
+        @items_rule_dot_j_n[[rule,dot,j,n]] = item
+        @item_lists[item.n] = [] unless @item_lists[item.n]
+        @item_lists[item.n] << item
+
+        if item.symbol_after_dot
+          unless @items_n_and_symbol_after_dot[[item.n, item.symbol_after_dot]]
+            @items_n_and_symbol_after_dot[[item.n, item.symbol_after_dot]] = []
+          end
+          @items_n_and_symbol_after_dot[[item.n, item.symbol_after_dot]] << item
+        end
       end
-      return item
     end
-    
-    def symbol_after_dot
-      return @dot < @rule.rhs.size  ? @rule.rhs[@dot] : nil
+
+    def find(rule, dot, j, n)
+      @items_rule_dot_j_n[[rule, dot, j,n]]
     end
-    
-    def initialize(rule, j, dot = 0)
-      @rule = rule
-      @j = j
-      @dot = dot
+
+    def find_all_by_n(n)
+      @item_lists[n].clone
     end
-    
-    def move
-      return Item.new(@rule, @j, @dot + 1)
+
+    def number_at_n(n)
+      @item_lists[n].length
     end
-    
-    def inspect
-      return "[" + 
-      @rule.lhs.inspect + " --> " + 
-       (@rule.rhs.slice(0, dot) + 
-      [Dot.new] + 
-      @rule.rhs.slice(dot, @rule.rhs.length - dot)).map{|symbol| symbol.inspect}.join(" ") + 
-              " ; " + @j.to_s + "]"
+
+    def items_n_and_symbol_after_dot(n, symbol)
+      return @items_n_and_symbol_after_dot[[n, symbol]] || []
     end
-  
-    def <=>(other)
-      res = @rule.prec <=> other.rule.prec;
-      if res == 0 and @rule.operator_prec and other.rule.operator_prec 
-         res = other.rule.operator_prec <=> @rule.operator_prec
-      end
-      if res == 0
-        res = @j <=> other.j 
+
+    def full_items_by_lhs_j_and_n(lhs, j, n)
+      @item_lists[n].find_all do |item|
+        item.dot >= item.rule.rhs.size and
+        item.j == j and
+        item.rule.lhs == lhs
       end
-      return res
     end
+
+    def items_at_n(n)
+      return @item_lists[n].clone
+    end
+
   end
-  
+
   # Just for Item inspect
   class Dot
+    def Dot.instance
+      @@instance
+    end
     def inspect
       return "*"
     end
+    @@instance = Dot.new
   end
-end
+end