foreverman-dhaka 2.2.1

data/Rakefile

@@ -0,0 +1,64 @@
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'rake/testtask'
+require 'rubygems'
+
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_files.include('README', 'lib/**/*.rb')
+  rdoc.rdoc_files.exclude("dhaka/lib/lexer/regex_parser.rb")
+  rdoc.main = "README"
+  rdoc.rdoc_dir = '../doc'
+end
+
+spec = Gem::Specification.new do |s|
+  s.name = "dhaka"
+  s.author = "Mushfeq Khan"
+  s.email = "mushfeq dot khan at gmail dot com"
+  s.version = ENV['VERSION'] || "0.0.0"
+  s.platform = Gem::Platform::RUBY
+  s.summary = "An LALR1 parser generator written in Ruby"
+  s.files = Dir.glob("{lib,test}/**/*").select {|file| file.include?('.rb') || file.include?('.txt')} + ['Rakefile']
+  s.require_path = 'lib'
+  s.autorequire = 'dhaka'
+  s.has_rdoc = true
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+	pkg.package_dir = "../gems"
+end
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/**/*test.rb']
+  t.verbose = true
+end
+
+task :generate_regex_parser do
+  require 'lib/dhaka'
+  File.open('lib/dhaka/lexer/regex_parser.rb', 'w') do |file| 
+    file << Dhaka::Parser.new(Dhaka::LexerSupport::RegexGrammar).compile_to_ruby_source_as('Dhaka::LexerSupport::RegexParser')
+  end
+end
+
+task :gem => [:test, :generate_regex_parser]
+
+task :default => :test
+
+task :test => [:generate_chittagong_parser, :generate_chittagong_lexer]
+
+task :generate_chittagong_parser do
+  require 'lib/dhaka'
+  require 'test/chittagong/chittagong_grammar'
+  require 'test/fake_logger'
+  File.open('test/chittagong/chittagong_parser.rb', 'w') do |file| 
+    file << Dhaka::Parser.new(ChittagongGrammar, FakeLogger.new).compile_to_ruby_source_as(:ChittagongParser)
+  end
+end
+
+task :generate_chittagong_lexer do
+  require 'lib/dhaka'
+  require 'test/chittagong/chittagong_lexer_specification'
+  File.open('test/chittagong/chittagong_lexer.rb', 'w') do |file| 
+    file << Dhaka::Lexer.new(ChittagongLexerSpecification).compile_to_ruby_source_as(:ChittagongLexer)
+  end
+end

data/lib/dhaka.rb

@@ -0,0 +1,62 @@
+#--
+# Copyright (c) 2006, 2007 Mushfeq Khan
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'set'
+require 'logger'
+require 'delegate'
+
+%w[
+dot/dot
+grammar/grammar_symbol
+grammar/production
+grammar/closure_hash
+grammar/grammar
+grammar/precedence
+parser/parse_tree
+parser/parse_result
+parser/item
+parser/channel
+parser/parser_methods
+parser/parser_state
+parser/conflict
+parser/token
+parser/action
+parser/parser_run
+parser/parser
+parser/compiled_parser
+tokenizer/tokenizer
+evaluator/evaluator
+lexer/accept_actions
+lexer/alphabet
+lexer/regex_grammar
+lexer/regex_tokenizer
+lexer/regex_parser
+lexer/state_machine
+lexer/dfa
+lexer/state
+lexer/specification
+lexer/lexeme
+lexer/lexer_run
+lexer/lexer
+lexer/compiled_lexer
+].each {|path| require File.join(File.dirname(__FILE__), 'dhaka/' + path)}

data/lib/dhaka/dot/dot.rb

@@ -0,0 +1,29 @@
+module Dhaka
+  module Dot #:nodoc:
+    class Digraph #:nodoc:
+      def initialize(node_attributes = {})
+        @result = ["digraph x {"]
+        @result << %(node #{dotify_hash(node_attributes)})
+        yield(self)
+        @result << '}'
+      end
+
+      def node(obj, attributes = {})
+        @result << "#{obj.object_id} #{dotify_hash(attributes)}"
+      end
+
+      def edge(src, dest, attributes = {})
+        @result << "#{src.object_id} -> #{dest.object_id} #{dotify_hash(attributes)}"
+      end
+
+      def dotify_hash hash
+        sorted_key_value_pairs = hash.collect {|key, value| [key.to_s, value.to_s]}.sort
+        hash.empty? ? "" : '[' + sorted_key_value_pairs.collect {|key, value| "#{key}=#{value.to_s.inspect}"}.join(' ') + ']'
+      end
+
+      def to_dot
+        @result.join("\n")
+      end
+    end
+  end
+end

data/lib/dhaka/evaluator/evaluator.rb

@@ -0,0 +1,133 @@
+module Dhaka
+  # Abstract base class for evaluators.
+  #
+  # Defining an evaluator is an easy way to perform syntax-directed evaluation without having to generate an abstract
+  # syntax tree representation of the input. 
+  #
+  # An evaluation rule for a given production named +bar+ is defined by calling +for_bar+ with
+  # a block that performs the evaluation. For detailed examples, see the evaluators in the 
+  # test suite.
+  # 
+  # The following is an evaluator for arithmetic expressions. When a parse tree node is encountered that
+  # corresponds to the production named +addition+, the block passed to +for_addition+ is invoked. The +evaluate+
+  # method is then recursively called on the child nodes, in this case the operands to the addition operation. The
+  # result is obtained by adding the evaluation results of the child nodes.
+  # 
+  #    class ArithmeticPrecedenceEvaluator < Dhaka::Evaluator
+  #
+  #      self.grammar = ArithmeticPrecedenceGrammar
+  #
+  #      define_evaluation_rules do
+  #
+  #        for_subtraction do 
+  #          evaluate(child_nodes[0]) - evaluate(child_nodes[2])
+  #        end
+  #
+  #        for_addition do
+  #          evaluate(child_nodes[0]) + evaluate(child_nodes[2])
+  #        end
+  #
+  #        for_division do
+  #          evaluate(child_nodes[0]).to_f/evaluate(child_nodes[2]) 
+  #        end
+  #
+  #        for_multiplication do
+  #          evaluate(child_nodes[0]) * evaluate(child_nodes[2])
+  #        end
+  #
+  #        for_literal do
+  #          child_nodes[0].token.value.to_i
+  #        end
+  #
+  #        for_parenthetized_expression do
+  #          evaluate(child_nodes[1])
+  #        end
+  #
+  #        for_negated_expression do
+  #          -evaluate(child_nodes[1])
+  #        end
+  #
+  #        for_power do
+  #          evaluate(child_nodes[0])**evaluate(child_nodes[2])
+  #        end
+  #
+  #      end
+  #
+  #    end
+
+  class Evaluator < SimpleDelegator
+    class << self
+      # Define evaluation rules within a block passed to this method. The evaluator will define
+      # default evaluation rules for pass-through productions (i.e. productions with expansions
+      # consisting of exactly one grammar symbol). The default evaluation rule for such productions
+      # is to simply return the result of calling +evaluate+ on the unique child node. Setting the
+      # <tt>:raise_error</tt> option to true tells the evaluator to throw an exception if you neglect
+      # to define a rule for a non-pass-through production (one where the expansion consists of
+      # multiple symbols), listing all the productions that absolutely need to be defined before you
+      # can continue.
+      def define_evaluation_rules(options = {})
+        yield
+        check_definitions(options)
+      end
+      
+      private
+
+      def check_definitions(options)
+        filter = lambda {|productions| productions.map {|production| production.name} - actions}
+        pass_through_productions_without_rules = filter[grammar.productions.select {|production| production.expansion.size == 1}]
+        pass_through_productions_without_rules.each do |rule_name|
+          send(:define_method, rule_name) do
+            evaluate(child_nodes.first)
+          end
+        end
+        non_trivial_productions_with_rules_undefined = filter[grammar.productions.select {|production| production.expansion.size != 1}]
+        raise EvaluatorDefinitionError.new(non_trivial_productions_with_rules_undefined) unless non_trivial_productions_with_rules_undefined.empty? || !options[:raise_error]
+      end
+
+      def inherited(evaluator)
+        class << evaluator
+          attr_accessor :grammar, :actions
+        end
+        evaluator.actions = []
+      end
+    
+      def method_missing(method_name, *args, &blk)
+        name = method_name.to_s
+        if name =~ /^for_(.+)$/
+          rule_name = $1
+          raise "Attempted to define evaluation rule for non-existent production '#{rule_name}'" unless grammar.production_named(rule_name)
+          actions   << rule_name
+          send(:define_method, rule_name, &blk)
+        else
+          super
+        end
+      end
+    end
+
+    # Evaluate a parse tree node.
+    def evaluate node
+      @node_stack ||= []
+      @node_stack << node
+      __setobj__(@node_stack.last)
+      result      = send(node.production.name)
+      @node_stack.pop
+      __setobj__(@node_stack.last)
+      result
+    end
+
+    def initialize
+      
+    end
+  end
+
+  class EvaluatorDefinitionError < StandardError #:nodoc:
+    def initialize(non_trivial_productions_with_rules_undefined)
+      @non_trivial_productions_with_rules_undefined = non_trivial_productions_with_rules_undefined
+    end
+  
+    def to_s
+      result = "The following non-trivial productions do not have any evaluation rules defined:\n" 
+      result << @non_trivial_productions_with_rules_undefined.join("\n")
+    end
+  end
+end

data/lib/dhaka/grammar/closure_hash.rb

@@ -0,0 +1,15 @@
+module Dhaka
+  # A subclass of Hash with a dirty flag
+  class ClosureHash < Hash #:nodoc:
+    attr_accessor :dirty
+
+    def initialize
+      super
+      @dirty = false
+    end
+
+    def load_set(set)
+      set.each {|item| self[item] = item}
+    end
+  end
+end

data/lib/dhaka/grammar/grammar.rb

@@ -0,0 +1,236 @@
+module Dhaka
+  
+  # Reserved name for the start symbol for all grammars. 
+  START_SYMBOL_NAME = "_Start_"
+  END_SYMBOL_NAME   = "_End_" #:nodoc:
+  
+  # Productions for specific grammar symbols are defined in the context of this class.
+  class ProductionBuilder
+    # +symbol+ is the grammar symbol that productions are being defined for.
+    def initialize(grammar, symbol)
+      @grammar = grammar
+      @symbol  = symbol
+    end
+  
+    # Creates a new production for +symbol+ with an expansion of +expansion+. The options hash can include
+    # a directive <tt>:prec</tt>, the value of which is a grammar symbol name. The precedence of the production is then
+    # set to the precedence of the grammar symbol corresponding to that name. 
+    #
+    # See the arithmetic precedence grammar in the test suites for an example.
+    def method_missing(production_name, expansion, options = {}, &blk)
+      expansion_symbols = (expansion.respond_to?('lines') ? expansion.lines : expansion).collect {|name| @grammar.symbols[name]}
+      production_args   = [@symbol, expansion_symbols, production_name.to_s, blk, @grammar.production_index]
+      if precedence_symbol_name = options[:prec] 
+        production_args << @grammar.symbol_for_name(precedence_symbol_name).precedence
+      end
+      
+      production = Production.new(*production_args)
+      @grammar.production_index += 1
+      
+      @symbol.nullable = true if expansion_symbols.empty?
+      @grammar.productions_by_symbol[production.symbol] << production
+      raise "Duplicate production named #{production.name}" if @grammar.productions_by_name[production.name]
+      @grammar.productions_by_name[production.name] = production
+    end
+  end
+
+  # The precedence builder defines three methods, +left+, +right+ and +nonassoc+. These accept arrays of grammar
+  # symbols all of which have the same precedence level and associativity.
+  #
+  # See the arithmetic precedence grammar in the test suites for an example of how this works.
+  class PrecedenceBuilder
+    def initialize(grammar) #:nodoc:
+      @grammar          = grammar
+      @precedence_level = 0
+    end
+    
+    [:left, :right, :nonassoc].each do |associativity| 
+      define_method(associativity) do |symbols|
+        assign_precedences associativity, symbols
+      end
+    end
+
+    private
+    def assign_precedences(associativity, symbol_names)
+      symbol_names.each do |symbol_name|
+        symbol            = @grammar.symbols[symbol_name]
+        symbol.precedence = Precedence.new(@precedence_level, associativity)
+      end
+      @precedence_level += 1
+    end
+  end
+
+  # Abstract base class for grammar specifications.
+  # 
+  # The following is a grammar specification for simple arithmetic. Precedences are specified as in Yacc -
+  # in ascending order of binding strength, with equal-strength symbols on the same level. 
+  # Production rules are specified for each symbol by specifying the name of the production (used when
+  # encoding the Evaluator) and the expansion for that particular production. For example, the production named 
+  # +addition+ expands the symbol <tt>'E'</tt> to the list of symbols <tt>['E', '+', 'E']</tt>.
+  #
+  #  class ArithmeticPrecedenceGrammar < Dhaka::Grammar
+  #    precedences do
+  #      left ['+', '-']
+  #      left ['*', '/']
+  #      nonassoc ['^']
+  #    end
+  #
+  #    for_symbol(Dhaka::START_SYMBOL_NAME) do
+  #      expression ['E']
+  #    end
+  #
+  #    for_symbol('E') do
+  #      addition ['E', '+', 'E']
+  #      subtraction ['E', '-', 'E']
+  #      multiplication ['E', '*', 'E']
+  #      division ['E', '/', 'E']
+  #      power ['E', '^', 'E']
+  #      literal ['n']
+  #      parenthetized_expression ['(', 'E', ')']
+  #      negated_expression ['-', 'E'], :prec => '*'
+  #    end
+  #  end
+  #
+  # In the above grammar, the symbols <tt>+</tt> and <tt>-</tt> are declared as being +left+-associative, meaning that 
+  # 1 + 2 + 3 is parsed as (1 + 2) + 3 as opposed to 1 + (2 + 3) (+right+-associativity). The symbol <tt>^</tt> is declared
+  # +nonassoc+ which means that expressions such as 2 ^ 3 ^ 4 are not allowed (non-associative). <tt>+</tt> and <tt>-</tt> are listed 
+  # before <tt>^</tt> which means that they bind lower, and an expression such as 2 + 3 ^ 5 will be always be parsed as
+  # 2 + (3 ^ 5) and not (2 + 3) ^ 5.
+  class Grammar
+    class << self
+      # Used for defining the Production-s for the symbol with name +symbol+. The block +blk+ is 
+      # evaluated in the context of a ProductionBuilder.
+      def for_symbol symbol, &blk
+        symbol              = symbols[symbol]
+        symbol.non_terminal = true
+        ProductionBuilder.new(self, symbol).instance_eval(&blk)
+      end
+
+      # Used for defining the precedences and associativities of symbols. The block +blk+ is
+      # evaluated in the context of a PrecedenceBuilder.
+      def precedences &blk
+        PrecedenceBuilder.new(self).instance_eval(&blk)
+      end
+
+      # Returns the grammar symbol identified by +name+
+      def symbol_for_name(name)
+        if symbols.has_key? name
+          symbols[name]
+        else
+          raise "No symbol with name #{name} found"
+        end
+      end
+      
+      # Returns a list of all the Production-s in this grammar.
+      def productions 
+        productions_by_name.values
+      end
+      
+      def productions_for_symbol(symbol) #:nodoc:
+        productions_by_symbol[symbol]
+      end
+      
+      def closure(kernel) #:nodoc:
+        channels = Hash.new {|hash, start_item| hash[start_item] = Set.new}
+        result = compute_closure(kernel) do |hash, item|    
+          if item.next_symbol and item.next_symbol.non_terminal
+            productions_by_symbol[item.next_symbol].each do |production|
+              new_channel = spontaneous_channel(item, hash[Item.new(production, 0)])
+              channels[item] << new_channel
+            end
+          end
+        end
+        [result, channels]
+      end
+      
+      def passive_channel(start_item, end_item) #:nodoc:
+        PassiveChannel.new(self, start_item, end_item)
+      end
+      
+      def first(given_symbol) #:nodoc:
+        cached_result = __first_cache[given_symbol] 
+        return cached_result if cached_result
+        result = compute_closure([given_symbol]) do |hash, symbol|
+              productions_by_symbol[symbol].each do |production| 
+                symbol_index = 0
+                while next_symbol = production.expansion[symbol_index]
+                  hash[next_symbol]
+                  break unless next_symbol.nullable
+                  symbol_index += 1
+                end
+              end if symbol.non_terminal
+          end.values.select {|symbol| symbol.terminal}.to_set
+        __first_cache[given_symbol] = result
+        result
+      end
+      
+      # Returns the Production identified by +name+.
+      def production_named(name)
+        productions_by_name[name]
+      end
+      
+      # Returns the set of terminal symbols in the grammar.
+      def terminal_symbols
+        symbols.values.select {|symbol| symbol.terminal}
+      end
+      
+      # Returns the set of non-terminal symbols in the grammar.
+      def non_terminal_symbols
+        symbols.values.select {|symbol| symbol.non_terminal}
+      end
+      
+      # Export the grammar to a BNF-like format
+      def to_bnf
+        result = []
+        last_symbol = nil
+        productions.sort.each do |production|
+          if production.symbol != last_symbol
+            result << ""
+            result << "#{production.symbol.name.inspect} :"
+            last_symbol = production.symbol
+          end
+          result << "  | #{production.expansion.collect{|symbol| symbol.name.inspect}.join(' ')}"
+        end
+        result.join("\n")
+      end
+      
+      private 
+      
+        def inherited(grammar)
+          class << grammar
+            attr_accessor :symbols, :productions_by_symbol, :productions_by_name, :start_symbol, :end_symbol, :__first_cache, :production_index
+          end
+          grammar.symbols               = Hash.new {|hash, name| hash[name] = GrammarSymbol.new(name)}
+          grammar.productions_by_symbol = Hash.new {|hash, name| hash[name] = Set.new([])}
+          grammar.productions_by_name   = {}
+          grammar.end_symbol            = grammar.symbols[END_SYMBOL_NAME]
+          grammar.start_symbol          = grammar.symbols[START_SYMBOL_NAME]
+          grammar.__first_cache         = {}
+          grammar.production_index      = 0
+        end
+  
+        def spontaneous_channel(start_item, end_item)
+          SpontaneousChannel.new(self, start_item, end_item)
+        end
+
+        def compute_closure(initial)
+          closure_hash = ClosureHash.new do |hash, item|
+            hash.dirty = true
+            hash[item] = item
+          end
+          
+          closure_hash.load_set(initial)
+          
+          loop do
+            closure_hash.keys.each do |element|
+              yield closure_hash, element
+            end
+            break unless closure_hash.dirty
+            closure_hash.dirty = false
+          end
+          closure_hash
+        end
+      end
+  end
+
+end