Kanocc 0.1.0

data/examples/calculator.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+#require "rubygems"
+$:.unshift("lib")
+require "kanocc.rb"
+require "logger"
+#require "breakpoint"
+
+# Example use of Kanocc for a small calculator program.
+# It implements the grammar:
+#
+# Program ::=
+#           | Program Expr '\n'R
+# Expr    ::= Expr '+' Expr 
+#           | Expr '-' Expr 
+#           | Expr '*' Expr 
+#           | Expr '/' Expr 
+#           | '(' Expr ')' 
+#           | Number 
+#
+# With the lexical grammar:
+#
+# Number ::= \d+, '(', ')', '+', '-', '*', '/' '\n'
+
+
+# ==========  Define a lexical grammar =============
+class Number < Kanocc::Token
+  attr_reader :val
+  setPattern(/\d+/) {@val = eval @m[0]}
+end
+
+# ==========  Define a grammar =====================
+class Expr < Kanocc::Nonterminal
+  attr_reader :val
+  
+  rule(Expr, "+", Expr)  {@val = @rhs[0].val + @rhs[2].val}
+  rule(Expr, "-", Expr)  {@val = @rhs[0].val - @rhs[2].val}
+  rule(Expr, "*", Expr)  {@val = @rhs[0].val * @rhs[2].val}
+  rule(Expr, "/", Expr)  {@val = @rhs[0].val / @rhs[2].val}
+  rule("(", Expr, ")")   {@val = @rhs[1].val}
+  rule(Number)           {@val = @rhs[0].val}
+  
+  setOperatorPrecedence ['*', '/'], 2
+end
+
+class Line < Kanocc::Nonterminal
+  rule(Expr, "\n")   { p @rhs[0].val}
+  rule(Kanocc::Error, "\n") do 
+    puts "Sorry - didn't understand: #{$source[startPos, endPos-startPos].inspect}"
+  end
+end
+
+class Program < Kanocc::Nonterminal
+  rule(Program, Line)
+  rule()
+end
+
+# Make a parser, give it 'Program' as the grammars startsymbol and run
+
+parser = Kanocc::Kanocc.new(Program)
+#parser.logger.level = Logger::DEBUG
+$source = <<-EOF
+  2 * 3
+  3 - 3 +
+  7 - 2 - 1
+  3 * 2 + 4
+  4 + 3 * 3
+EOF
+
+parser.parse($source)

data/lib/kanocc.rb

@@ -0,0 +1,255 @@
+# 
+#  Kanocc - Kanocc ain't no compiler-compiler
+#
+require 'kanocc/token'
+require 'kanocc/nonterminal'
+require 'kanocc/scanner'
+require 'kanocc/earley'
+require 'logger'
+
+# = Kanocc - Kanocc ain't no compiler-compiler
+#
+# Kanocc is a ruby-framework for parsing and translating.
+# Emphasis is on easy, 'scripty' use, and seamless integration with ruby. Performance has been
+# a secondary concern.
+# In it's default configuration, Kanocc uses it's own lexical scanner and a parser 
+# based on Earley's algorithm to allow handling of any context-free grammer. It is possible, 
+# however, to plug in other lexical scanners or parsers. See ##FIXMEREF.
+# 
+# A simple example.
+#
+# Reading and evaluating reverse polish notated expressions. Consider this grammar:
+#
+#    E ::= E E '+' 
+#        | E E '-'
+#        | E E '*'
+#        | E E '/'
+#        | NUM
+#
+#    NUM a sequence of digits
+#
+# In Kanocc yout could do it like this:
+#
+#    require "kanocc"
+#    
+#    # ==========  Define a lexical grammar =============
+#    class NUM < Kanocc::Token
+#      attr_reader :val
+#      setPattern(/\d+/) { @val = @m[0].to_i}
+#    end
+#    
+#    # ==========  Define a grammar ===================== 
+#    class E < Kanocc::Nonterminal
+#      attr_reader :val 
+#      rule(E, E, "+") { @val = @rhs[0].val + @rhs[1].val} 
+#      rule(E, E, "-") { @val = @rhs[0].val - @rhs[1].val}  
+#      rule(E, E, "*") { @val = @rhs[0].val * @rhs[1].val} 
+#      rule(E, E, "/") { @val = @rhs[0].val / @rhs[1].val} 
+#      rule(NUM) { @val = @rhs[0].val }
+#    end
+#    
+#    # ==========  Set up a parser ======================
+#    myParser = Kanocc::Kanocc.new(E)
+#    
+#    # ==========  And try it out =======================
+#    puts "3 4 + 2 - = #{myParser.parse("3 4 + 2 -").val}"
+#    
+# and you'd get:
+#   
+#    3 4 + 2 - = 5
+#
+# For more examples, please refer to the documentation: ##FIXMEREF
+#
+module Kanocc
+  class Kanocc
+    attr_accessor :scanner, :parser, :logger
+    
+    # Creates a new instance of Kannocc, with the given start symbol.
+    # From the startsymbol, Kanocc will deduce the grammar and the 
+    # grammarsymbols
+    # 
+    def initialize(startSymbol)
+      @startSymbol = startSymbol 
+      @logger = Logger.new(STDOUT)
+      @logger.datetime_format = "" 
+      @logger.level = Logger::WARN 
+      @scanner = Scanner.new(:logger => @logger)
+      @parser = EarleyParser.new(self, :logger => @logger)
+    end
+    
+    def logger=(logger)
+      @logger = logger || logger.new(STDOUT)
+      @parser.logger = @logger if parser.respond_to?(:logger)
+      @scanner.logger = @logger if scanner.respond_to?(:logger)
+    end
+  
+    def parser=(parser)
+      @parser = parser
+      @parser.logger = @logger if parser.respond_to?(:logger=)
+    end
+    
+    def scanner=(scanner)
+      @scanner = scanner
+      @scanner.logger = @logger if scanner.respond_to?(:logger=)
+    end
+    
+    # Consume input. Kanocc will parse input according to the rules given, and
+    # - if parsing succeeds - return an instance of the grammars start symbol.
+    # Input may be a String or an IO object.
+    def parse(input)
+      raise "Start symbol not defined" unless @startSymbol
+      tellParserStartSymbol(@startSymbol) 
+      @parser.prepare 
+      @stack = []
+      @inputPos = 0 
+      @scanner.eachToken(input) do |tokens, startPos, endPos|
+        @logger.info "got #{show(tokens)} from scanner at #{startPos}, #{endPos}"
+        @logger.debug "Consume " + tokens.inspect if @logger
+        @inputPos += 1
+        @parser.consume(tokens, startPos, endPos)
+      end
+      @parser.eof
+      @stack[0]
+    end
+   
+    def parseFile(file)
+      if file.is_a? String # Then we assume it's a path	
+	file = File.open(File.expand_path(file))
+	openedFile = true
+      end
+      input = file.read
+      file.close if openedFile
+      parse(input)
+    end
+    
+    # Define whitespace. By default, Kanocc will recogninze anything that matches 
+    # /\s/ as whitespace. 
+    # whitespace takes a variable number of arguments, each of which must be a 
+    # regular expression.
+    def setWhitespace(*ws)
+      @scanner.setWhitespace(*ws)
+    end
+    
+    # Define which tokens Kanocc should recognize. If this method is not called
+    # Kanocc will scan for those tokens that are mentioned in the grammar.
+    # tokens= takes a variable number of arguments. Each argument must either be
+    # a string or a class which is a subclass of Kanocc::Token
+    def setTokens(*tokens)
+      @scanner.setRecognized(*tokens)
+    end
+    
+    # The parser must call this method when it have decided upon a reduction.
+    # As arguments it should give the rule, by which to reduce. 
+    def reportReduction(rule, startPos, endPos) 
+      @logger.info "Reducing by " + rule.inspect
+      nonterminal = rule.lhs.new      
+      nonterminal.startPos = startPos
+      nonterminal.endPos = endPos
+      rightHandSide = @stack.slice!(-rule.rhs.length, rule.rhs.length)
+      rightHandSide = rightHandSide.map {|e| e.is_a?(List) ? e.elements : e} unless nonterminal.is_a? List
+      if rule.method
+        oldRhs = nonterminal.instance_variable_get('@rhs')
+        nonterminal.instance_variable_set('@rhs', rightHandSide)
+        nonterminal.send(rule.method)
+        nonterminal.instance_variable_set('@rhs', oldRhs)
+      end
+      @stack.push(nonterminal)
+      showStack
+    end
+    
+   
+    # The parser must call this method when it consumes a token
+    # As argument it should give the consumed token and the positions 
+    # in the input string corresponding to the token. Positions should be given
+    # as the position of the first character of the token and the position of the 
+    # first character after the token.
+    def reportToken(token)
+      @logger.info("Pushing token: " + token.inspect)
+      @stack.push(token)
+      if token.respond_to?("__recognize__") 
+        token.__recognize__ 
+      end
+      showStack 
+    end
+        
+    
+    def tellParserStartSymbol(startSymbol)
+      @parser.startSymbol = startSymbol
+      bagOfTerminals = {}
+      findTokens(startSymbol, bagOfTerminals)
+      @logger.debug "tokens = " + bagOfTerminals.keys.inspect 
+      strings = bagOfTerminals.keys.find_all{|ter| ter.is_a? String} 
+      @logger.info("Literals: " + strings.inspect)
+      tokens = bagOfTerminals.keys.find_all{|ter| ter.is_a? Class and ter.ancestors.member?(Token)}
+      @logger.info("Tokens: " + tokens.inspect)
+      @scanner.setRecognized(*(strings + tokens))
+
+      # Show rules
+      @logger.info("Rules:")
+      nonterminals = [startSymbol]
+      nonterminals.each do |nonterminal|
+        nonterminal.rules.each do |rule|
+          @logger.info("  " + rule.inspect)
+	  rule.rhs.each do |gs|
+	    if gs.is_a? Class and gs.ancestors.member?(Nonterminal) and not nonterminals.member?(gs)
+	      nonterminals.push(gs)
+	    end
+	  end
+	end
+      end
+    end
+    
+    def findTokens(nonterminal, collectedTokens,  visitedNonterminals = {})
+      unless visitedNonterminals[nonterminal]
+        visitedNonterminals[nonterminal] = true
+        nonterminal.rules.each do |r| 
+          r.rhs.each do |gs|
+            if gs.is_a?(Class) and gs.ancestors.member?(Nonterminal)
+              findTokens(gs, collectedTokens, visitedNonterminals)
+            else
+              collectedTokens[gs] = true 
+            end
+          end
+        end
+      end
+    end
+    
+    def operatorPrecedence(rule)
+      if operator = rule.operator
+        rule.lhs.operatorPrecedence(operator) || 0
+      else
+        0
+      end
+    end
+    
+    # For debugging
+    def showStack
+      @logger.info("Stack: [" + @stack.map {|gs| show(gs)}.join(", ") + "]" ) if @logger
+    end
+    
+    def show(gs)
+      if gs.is_a?(Nonterminal) or gs.is_a?(Token)
+        gs.class.to_s; 
+      elsif gs.is_a?(String)
+        gs.inspect; 
+      end
+    end
+    
+  
+  end
+  
+  class ParseException < Exception 
+    attr_accessor :inputPos, :inputSymbol, :expected 
+    def initialize(inputPos, inputSymbol, expected)
+      @inputPos, @inputSymbol, @expected = inputPos, inputSymbol, expected
+    end
+  end
+  
+  class KanoccException < Exception
+  end
+
+  
+
+end
+
+

data/lib/kanocc/earley.rb

@@ -0,0 +1,309 @@
+require 'kanocc/grammar_rule'
+require 'kanocc/token'
+require 'logger'
+module Kanocc
+  #
+  # Parser for Kanocc based on Earleys algorithm. For a description see:
+  # Alfred V. Aho, Jeffrey D. Ullman, The Theory of Parsing, Translation and  Compiling, 
+  # or try a web search engine of your choice with 'Earley parsing'
+  #
+  # Earley's parser will parse according to any zcontext-free grammar using O(n*n*n) time 
+  # and O(n*n) space, n being the length of input. If the grammar is unambigous time/space 
+  # complexity is O(n*n)/O(n*n).
+  # As of yet (version 0.1) the implementation is surely not optimal, 
+  # so time/space complexity is probably worse.
+  #
+  # Christian Surlykke 2007.
+  #
+  class EarleyParser
+    attr_accessor :kanocc, :logger
+    
+    ErrorRule = GrammarRule.new(Error, [], nil)
+    
+    def initialize(kanocc, options = {})
+      @kanocc = kanocc
+      @logger = options[:logger] || Logger.new
+    end
+    #
+    # Sets up the parser, creating itemlist 0.
+    #
+    def startSymbol=(startSymbol)
+      @startSymbol = startSymbol
+      @itemLists = [ItemList.new(nil, 0, 0)]
+      @inputPos = 0
+      @recoveryPoints = []
+      @itemLists[0].addAll(@startSymbol.rules.map{|rule| Item.new(rule, 0)})
+      predictAndComplete(0)
+    end
+    
+    def prepare
+      @itemLists = @itemLists[0..0]
+      @inputPos = 0
+      if @recoveryPoints.size > 0 and @recoveryPoints[0] == 0
+        @recoveryPoints = [0]
+      else
+        @recoveryPoints = []
+      end
+      @logger.info("Itemlist 0:\n" + @itemLists[0].inspect) unless not @logger
+    end
+
+    def scan(terminals) 
+      terminals.each do |terminal| 
+        @itemLists[@inputPos].addAll(@itemLists[@inputPos - 1].findMatching(terminal).map{|item| item.move})
+      end
+    end
+    
+    def predictAndComplete(pos)
+      itemList = @itemLists[pos] 
+      prevSize = 0      
+      while prevSize < itemList.size do 
+        prevSize = itemList.size	
+	itemList.each do |item|
+	  if item.rule.rhs.length <= item.dot
+            # complete 
+	    itemList.addAll(@itemLists[item.j].findMatching(item.rule.lhs).map{|item| item.move})
+          elsif (nont = item.rule.rhs[item.dot]).respond_to?(:rules)  
+            # predict
+	    itemList.addAll(nont.rules.map {|rule| Item.new(rule, @inputPos)})
+	  end
+        end
+      end 
+    end
+    
+    def addRecoveryPoints(pos)
+      if @recoveryPoints[-1] != pos
+	@itemLists[pos].each do |item| 
+	  if Error == item.rule.rhs[item.dot]
+	    @recoveryPoints.push(pos)
+	    break
+	  end
+	end
+      end
+    end
+
+    #
+    # Consume and parse next input symbol
+    #
+    def consume(inputSymbols, startPos, endPos) 
+      @inputPos += 1
+      @itemLists.push(ItemList.new(inputSymbols, @inputPos, endPos))
+  
+      # scan, predict and complete until no more can be added
+      scan(inputSymbols)
+      
+      if @itemLists[@inputPos].size == 0
+        @logger.debug("Found no items matching #{inputSymbols} in itemlist #{@inputPos - 1}")
+        @logger.debug("@recoveryPoints = " + @recoveryPoints.inspect)	
+        for i in 1..@recoveryPoints.length do 
+          if @recoveryPoints[-i] < @inputPos
+            @itemLists[@inputPos - 1].add(Item.new(ErrorRule, @recoveryPoints[-i]))
+            predictAndComplete(@inputPos - 1) 
+	    scan(inputSymbols) 
+	    break if @itemLists[@inputPos].size > 0 
+          end
+        end
+      end
+      predictAndComplete(@inputPos) 
+      addRecoveryPoints(@inputPos)
+      @logger.info("Itemlist #{@inputPos}:\n" + @itemLists[@inputPos].inspect) if @logger
+    end
+   
+    
+    #
+    # Signal to the parser that end of input is reached
+    #
+    def eof
+      @logger.debug "--- Parsing done, translating ---"
+      topItem = findFullItems(@startSymbol, @inputPos).find_all {|item| item.j == 0}.min
+      if topItem 
+        translate(topItem, @inputPos)
+      else
+        raise(KanoccException, "It didn't parse")
+      end
+    end
+    
+    def translate(element, pos)
+      @logger.debug("translate: " + element.inspect + ", pos = " + pos.inspect)   
+      if element.class == Item
+        translateHelper(element, pos) 
+        @kanocc.reportReduction(element.rule, 
+                                @itemLists[element.j].textPos, 
+                                @itemLists[pos].textPos)
+      elsif element.class == Class # Its a token class
+	@kanocc.reportToken(@itemLists[pos].inputSymbol.find {|sym| sym.is_a? element})
+      else # Its a string instance
+        @logger.debug @itemLists[pos].inspect
+        @kanocc.reportToken(element)
+      end
+    end
+    
+    def translateHelper(item, pos)
+      @logger.debug("translateHelper: " + item.inspect) 
+      return if item.dot == 0 
+      if item.rule.rhs[item.dot - 1].respond_to?("rules")
+        # Assume item is of form [A --> aB�c, k] in itemlist i
+        # Must then find item of form [B --> x�, j] in itemlist i so 
+        # that there exists item of form [A --> a�Bc, k] on itemlist j
+        #
+        # First: Items of form [B --> x�, j] on list i 
+        candidates = findFullItems(item.rule.rhs[item.dot - 1], pos)
+        # Then: Those for which item of form [A --> a�Bc, k] exists
+        # on list j
+        candidates = candidates.find_all {|subItem|
+          @itemLists[subItem.j].findItem(item.rule, item.dot - 1, item.j)
+        }
+        ##### 
+        # Precedence handling is somewhat problematic in Earley parsing. 
+        # We now have to choose amongst possibly several candidates
+        #
+        # Last: Pick the one with the rule with the _lowest_ precedence
+        # (We are finding reductions top-down, but will evaluate bottom-up, hence
+        # this will make the rule with the _highest_ precedence evaluate first.
+        
+        
+        subItem = candidates.min
+        prevItem = @itemLists[subItem.j].findItem(item.rule, item.dot - 1, item.j)
+        prevList = subItem.j
+      else
+        prevItem = @itemLists[pos - 1].findItem(item.rule, item.dot - 1, item.j)
+        prevList = pos - 1
+        subItem = item.rule.rhs[item.dot - 1]
+      end
+      translateHelper(prevItem, prevList)
+      translate(subItem, pos)
+    end
+        
+    def findFullItems(nonterminal, inputPos)
+      @itemLists[inputPos].find_all do |item|
+        item.rule.lhs == nonterminal and item.dot >= item.rule.rhs.length
+      end
+    end
+    
+    def operatorPrecedence(rule)
+      - (@kanocc.operatorPrecedence(rule))
+    end
+  end 
+  
+  class ItemList
+    attr_reader :inputSymbol, :textPos
+    attr_accessor :items
+    
+    def initialize(inputSymbol, inputPos, textPos)
+      @inputPos = inputPos
+      @inputSymbol = inputSymbol
+      @textPos = textPos
+      @items = Hash.new 
+    end
+    
+    def copy
+      res = clone
+      res.items = @items.clone
+      return res
+    end
+    
+    def size
+      return @items.size
+    end
+    
+    def find_all(&b)
+      return @items.keys.find_all(&b)
+    end
+    
+    def findItem(rule, dot, j)
+      return @items.keys.find{ |item| 
+        item.rule == rule and 
+        item.dot == dot and
+        item.j == j
+      }
+    end
+    
+    def eachMatching(inputSymbol)
+      findMatching(inputSymbol).each do |item| 
+        yield(item)
+      end
+    end
+    
+    def findMatching(inputSymbol)
+      @items.keys.find_all do |item| 
+        inputSymbol === item.symbolAfterDot or inputSymbol == item.symbolAfterDot
+      end
+    end
+    
+    def contains(item)
+      return @items[item]
+    end
+    
+    def add(item)
+      @items.store(item, true)
+    end
+    
+    def addAll(items)
+      items.each {|item| @items.store(item, true)}
+    end
+
+    def each
+      @items.keys.each do |item|
+        yield item
+      end
+    end
+    
+    def inspect
+      return "[" + @inputSymbol.inspect + "\n " + 
+                   @textPos.to_s + "\n " +
+                   @items.keys.map{|item| item.inspect}.join("\n  ") + "]\n" 
+    end
+  end
+  
+  
+  class Item
+    attr_reader :rule, :j, :dot
+    @@items = Hash.new
+    
+    def Item.new(rule, j, dot = 0)
+      unless (item = @@items[[rule,j,dot]])
+        item = super(rule, j, dot)
+        @@items.store([rule, j, dot], item)
+      end
+      return item
+    end
+    
+    def symbolAfterDot
+      return @dot < @rule.rhs.size  ? @rule.rhs[@dot] : nil
+    end
+    
+    def initialize(rule, j, dot = 0)
+      @rule = rule
+      @j = j
+      @dot = dot
+    end
+    
+    def move
+      return Item.new(@rule, @j, @dot + 1)
+    end
+    
+    def inspect
+      return "[" + 
+      @rule.lhs.inspect + " --> " + 
+       (@rule.rhs.slice(0, dot) + 
+      [Dot.new] + 
+      @rule.rhs.slice(dot, @rule.rhs.length - dot)).map{|symbol| symbol.inspect}.join(" ") + 
+              " ; " + @j.to_s + "]"
+    end
+  
+    def <=>(other)
+      tmp = (@rule.prec <=> other.rule.prec)
+      if tmp == 0
+        return other.j <=> @j
+      else 
+        return tmp
+      end
+    end
+  end
+  
+  # Just for Item inspect
+  class Dot
+    def inspect
+      return "�"
+    end
+  end
+end

data/lib/kanocc/grammar_rule.rb

@@ -0,0 +1,36 @@
+module Kanocc
+  class GrammarRule
+    attr_reader :lhs, :rhs, :method, :argPositions
+    attr_accessor :prec 
+
+    def initialize(lhs, rhs, method)
+      @lhs = lhs
+      @rhs = rhs
+      @method = method
+      @logger.debug("#{lhs} --> #{rhs.map {|gs| gs.is_a?(Symbol) ? gs.to_s : gs}.join}, #prec = #{@prec}, method = #{method}") unless not @logger
+    end
+  
+    def operator
+      rhs.find {|s| s.is_a?(String) or s.is_a?(Token)}  
+    end
+    
+    def prec=(newPrec)
+      @prec = newPrec
+    end
+    
+    # The precedence of a rule is defined as:
+    # The given precedence
+    # or (if that's not defined) the precedence of the leftmost operator (token)
+    # or (if that's not defined) 0.
+    def prec
+      @prec or
+      ((o = operator) and (@lhs.operatorPrecedence(o))) or
+      0
+    end
+    
+    def inspect 
+      return lhs.inspect + " ::= " + rhs.map{|gs| gs.inspect}.join(" ")
+    end
+
+  end
+end  

data/lib/kanocc/nonterminal.rb

@@ -0,0 +1,158 @@
+require 'kanocc/grammar_rule'
+module Kanocc
+  class Nonterminal
+    attr_accessor :startPos, :endPos
+    @@rules = Hash.new
+    @@lastRule = Hash.new
+    @@derivesRight = Hash.new
+    @@operatorPrecedence = Hash.new
+    @@methodNames = Hash.new
+    
+    Left = 1
+    Right = 2
+        
+    def Nonterminal.derivesRight
+      @@derivesRight[self] = true
+    end
+    
+    def Nonterminal.derivesRight?
+      return @@derivesRight[self]
+    end
+   
+    def Nonterminal.setOperatorPrecedence(operator, precedence) 
+      raise "Precedence must be an integer" unless precedence.class == Fixnum
+      @@operatorPrecedence[self] ||= Hash.new 
+      if is_an_operator?(operator)
+        @@operatorPrecedence[self][operator] = precedence
+      elsif is_an_array_of_operators(operator)
+        operator.each {|o| @@operatorPrecedence[self][o] = precedence}
+      else
+        raise "Operator must be a string, a token or an array of those"  
+      end 
+    end
+ 
+    def Nonterminal.operatorPrecedence(operator)
+      (@@operatorPrecedence[self] and @@operatorPrecedence[self][operator]) or 0
+    end
+
+    def Nonterminal.is_an_array_of_operators(arr) 
+       arr.is_a?(Array) and
+       arr.collect{|o| is_an_operator?(o)}.inject {|b1, b2| b1 and b2 }
+    end
+    
+    def Nonterminal.is_an_operator?(operator)
+        operator.is_a?(String) or operator.is_a?(Token) 
+    end
+
+    def Nonterminal.rules
+      rules = @@rules[self] 
+      return rules ? rules : []
+    end
+    
+    def Nonterminal.addRule(rule)
+      @@rules[self] ||= []
+      @@rules[self].push(rule)
+      @@lastRule[self] = rule
+    end
+    
+    def Nonterminal.is_a_grammarsymbol?(x) 
+      x.is_a?(String) or (x.respond_to?("is_a_kanocc_grammarsymbol?") and x.is_a_kanocc_grammarsymbol?)
+    end
+
+    def Nonterminal.is_a_kanocc_grammarsymbol? 
+      return true 
+    end
+ 
+    def Nonterminal.rule(*rhs, &block)
+      for pos in 0..rhs.length - 1 do
+        unless is_a_grammarsymbol?(rhs[pos])
+          raise "Problem with rule: #{rhs.inspect}, element:#{pos.to_s} - #{rhs[pos].inspect}\nElements of a rule must be Strings, Tokens or Nonterminals"
+        end
+      end
+           
+      if block_given?
+        methodName = generateMethodName(*rhs) 
+        define_method(methodName.to_sym, &block)
+        addRule(GrammarRule.new(self, rhs, methodName.to_sym))
+      else
+        addRule(GrammarRule.new(self, rhs, nil))
+      end
+    end
+  
+    def Nonterminal.zm(symbols, sep = nil)
+      listClass = newListClass 
+      listClass.rule() {@elements = []}
+      listClass.rule(om(symbols, sep)) {@elements = @rhs[0].elements}
+      return listClass
+    end
+    
+    def Nonterminal.om(symbols, sep = nil)
+      symbols = [symbols] unless symbols.is_a? Array
+      listClass = newListClass
+      listClass.rule(*symbols) {@elements = @rhs}
+      if sep
+        listClass.rule(listClass, sep, *symbols) {@elements = @rhs[0].elements + @rhs[2..@rhs.length]}
+      else
+        listClass.rule(listClass, *symbols) {@elements = @rhs[0].elements + @rhs[1..@rhs.length]}
+      end
+      return listClass
+    end
+    
+    @@listClassNumber = 0
+ 
+    def Nonterminal.newListClass
+      listClass = Class.new(List)
+      @@listClassNumber += 1
+      def listClass.inspect
+        return "anonList_#{@@listClassNumber}"
+      end
+      return listClass
+    end
+
+    def Nonterminal.generateMethodName(*args)
+      methodName = self.name + " --> " + args.map {|a| a.inspect}.join(' ')
+      @@methodNames[self] ||= []
+      i = 1
+      while @@methodNames[self].member?(methodName) do 
+        methodName += ' ';
+      end
+      @@methodNames[self].push(methodName)
+      return methodName
+    end
+  
+    def Nonterminal.prec(p) 
+      raise "Call to prec not preceded by rule" unless @@lastRule[self]
+      @@lastRule[self].prec = p
+    end
+    
+    def Nonterminal.showMethodNames
+      @@methodNames[self].each{|mn| puts mn.inspect} if @@methodNames[self]
+    end
+  end
+  
+  
+  class List < Nonterminal
+    attr_reader :elements
+    
+        protected
+    # Assumes @rhs[0] is a Kanocc::List and that rhs.length > 1
+    def collect(stripSeparator = false)
+      puts "collect with stripSeparator = #{stripSeparator}"
+      @elements = @rhs[0].elements
+      if stripSeparator
+        @elements = @elements + @rhs[2..@rhs.length]
+      else
+        @elements = @elements + @rhs[1..@rhs.length]
+      end
+      puts "@elements: " + @elements.inspect
+    end
+  end
+
+  class Error < Nonterminal
+    attr_reader :text
+    def initialize
+      super
+      @text = "FIXME"
+    end
+  end
+end

data/lib/kanocc/scanner.rb

@@ -0,0 +1,152 @@
+require 'stringio'
+require 'strscan'
+require 'logger'
+
+module Kanocc
+  class Scanner
+    attr_accessor :logger
+    def initialize(init = {})
+      if init[:logger]
+        @logger = init[:logger] 
+      else
+        @logger = Logger.new(STDOUT)
+        @logger.level = Logger::WARN
+      end
+      @wsRegs = [/\s/]
+      @recognizables = []
+      @regexps = []
+    end
+    
+    def setWhitespace(*wsRegs)
+      @wsRegs = []
+      wsRegs.each do |wsReg| 
+        unless wsReg.is_a?(Regexp)
+          raise "setWhitespace must be given a list of Regexp's" 
+        end
+        @wsRegs << r
+      end
+    end
+    
+    def setRecognized(*rec)
+      @recognizables = []
+      @regexps = []
+      rec.each do |r| 
+        @recognizables << r
+        if r.class == Class
+	  @regexps << r.pattern
+        else
+          @regexps << Regexp.compile(Regexp.escape(r))
+        end
+      end
+    end
+    
+    def eachToken(input)
+      if input.is_a?(IO) 
+        @input = input.readlines.join("")
+      elsif input.is_a?(String) 
+        @input = input
+      else
+        raise "Input must be a string or an IO object"
+      end 
+      @stringScanner = StringScanner.new(@input)
+      pos = @stringScanner.pos 
+      while tokens = nextToken do
+        @logger.debug("Yielding with #{tokens}, #{pos}, #{@stringScanner.pos}")
+        yield(tokens, pos, @stringScanner.pos)
+	pos = @stringScanner.pos
+      end
+    end
+  
+    private
+    
+    def nextToken
+       
+      while true do 
+        if @stringScanner.pos >= @input.length
+          return nil 
+	end
+	tokens = matchToken
+        
+	if tokens.size > 0 
+          @logger.debug("nextToken returning #{tokens}")
+          return tokens
+        elsif trimWhitespace
+          # Now we've stripped some whitespace, so we go
+          # back and try to match a token again
+          next
+        else
+          # We've not been able to recognize a token or whitespace, 
+          # so we emit the first character of the remaining input as a string literal.
+          # With this behavior, lexical scanning cannot fail.
+          res = [@stringScanner.scan(/./m)]
+          @logger.debug("nextToken returning #{res.inspect}")
+          return res 
+        end
+      end
+    end
+    
+    def matchToken
+      regPoss = findMatchingReg(@regexps) 
+      @logger.debug("matchToken, regPoss = #{regPoss.inspect}");  
+      tokens = []
+      str = nil
+      regPoss.each do |i|
+        logger.debug("@recognizables[#{i}] = #{@recognizables[i].inspect}") 
+        str = @stringScanner.scan(@regexps[i]) unless str 
+	if @recognizables[i].class == Class
+	  @logger.debug("Its a class")
+	  token = @recognizables[i].new(str)
+	  token.m = token.match(str) # To create a proper match object
+	  @logger.debug("token: " + token.inspect) 
+	  tokens << token 
+	  @logger.debug("tokens: " + tokens.inspect)
+	else
+	  tokens << str
+        end
+      end
+      @logger.debug("matchToken returning: " + tokens.inspect)
+      return tokens  
+    end
+    
+    def trimWhitespace
+      wsPoss = findMatchingReg(@wsRegs)
+      if  wsPoss.size > 0
+	@stringScanner.skip(@wsRegs[wsPoss[0]])
+        return true
+      else
+	return false
+      end
+    end
+        
+    def findMatchingReg(arrayOfRegs)
+      @logger.debug("findMatchingReg: arrayOfRegs = #{arrayOfRegs}")
+      maxLength = 0
+      regPoss = []
+      for i in 0..arrayOfRegs.size-1 do 
+	len = @stringScanner.match?(arrayOfRegs[i]) || 0	
+	if len > maxLength
+	  regPoss = [i]
+	  maxLength = len
+	elsif len == maxLength and len > 0
+	  regPoss << i
+	end
+      end
+      return regPoss
+    end
+  end
+end
+
+
+############################################
+#                Testing
+#require 'Token'
+#
+#class Number < Token
+#  setPattern(/\d+/)
+#end
+#
+#scanner = KanoccScanner.new
+#scanner.setRecognized(Number, "Exit")
+#scanner.setWhitespace(/[ \t]/)
+#
+#scanner.eachTokenDo{|token|  print token.inspect, "\n"}

data/lib/kanocc/token.rb

@@ -0,0 +1,40 @@
+module Kanocc
+  class Token < Regexp
+    attr_reader :str
+    attr_accessor :m
+    
+    @@patterns = Hash.new
+  
+    def initialize(str)
+      @str = str
+      super(@@patterns[self.class])
+    end
+      
+    def ===(klass)
+      self.class == klass
+    end
+    
+    def Token.setPattern(reg, &block)
+      @@patterns[self] = reg
+      if block_given?
+        define_method(:__recognize__, &block)
+      end
+    end
+  
+    def Token.pattern
+      return @@patterns[self]
+    end
+
+    def is_a_kanocc_token?
+      return true
+    end
+
+    def Token.is_a_kanocc_grammarsymbol?
+      return true
+    end
+
+    def inspect
+      self.class.name + "[" + @str + "]" 
+    end
+  end
+end

data/lib/todo

@@ -0,0 +1,3 @@
+Better handling of blocks
+LR Parsers
+Scanner.eachToken method

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification 
+name: Kanocc
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Christian Surlykke
+autorequire: kanocc
+bindir: bin
+cert_chain: []
+
+date: 2008-04-12 00:00:00 +02:00
+default_executable: 
+dependencies: []
+
+description: 
+email: ""
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/todo
+- lib/kanocc.rb
+- lib/kanocc
+- lib/kanocc/token.rb
+- lib/kanocc/grammar_rule.rb
+- lib/kanocc/nonterminal.rb
+- lib/kanocc/scanner.rb
+- lib/kanocc/earley.rb
+- examples/calculator.rb
+has_rdoc: false
+homepage: ""
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: Kanocc - Kanocc ain't no compiler-compiler. A framework for syntax directed translation
+test_files: []
+