dhaka 2.0.0 → 2.0.1

data/lib/parser/parser.rb

@@ -1,7 +1,3 @@
-#!/usr/bin/env ruby
-require 'set'
-require 'logger'
-
 module Dhaka
   # The parser generator. To generate a parser from a grammar specification +ArithmeticPrecedenceGrammar+, one would
   # write:
@@ -12,31 +8,26 @@ module Dhaka
   # which returns a string of Ruby code.
   class Parser
     include ParserMethods
-    attr_reader :grammar, :start_state
+    attr_reader :grammar
   
     # Creates a new parser from the given grammar. Messages are logged by default to STDOUT
     # and the log level is WARN. Shift-reduce conflicts are reported at WARN and reduce-reduce conflicts
     # at ERROR. You may pass in your own logger. Logging at DEBUG shows a lot of progress output.
     def initialize(grammar, logger = nil)
-      if logger
-        @logger = logger
-      else
-        @logger = Logger.new(STDOUT)
-        @logger.level = Logger::WARN
-      end
+      @logger      = logger || default_logger
       @transitions = Hash.new {|hash, state| hash[state] = {}}
-      @grammar = grammar
-      @channels = []
+      @grammar     = grammar
+      @channels    = []
       @states = Hash.new do |hash, kernel|
-          channels, closure = @grammar.closure(kernel)
-          @channels += channels.to_a
-          new_state = ParserState.new(self, closure)
+          channels, closure = grammar.closure(kernel)
+          @channels.concat channels.to_a
+          new_state    = ParserState.new(self, closure)
           hash[kernel] = new_state
-          @logger.debug("Created #{new_state}.")
+          @logger.debug("Created #{new_state.unique_name}.")
           new_state.transition_items.each do |symbol, items|
             destination_kernel = ItemSet.new(items.collect{|item| item.next_item})
-            destination_state = hash[destination_kernel]
-            items.each { |item| @channels << @grammar.passive_channel(item, destination_state.items[item.next_item]) }
+            destination_state  = hash[destination_kernel]
+            items.each { |item| @channels << grammar.passive_channel(item, destination_state.items[item.next_item]) }
             @transitions[new_state][symbol] = destination_state
           end
           new_state
@@ -47,7 +38,7 @@ module Dhaka
     # Returns the Ruby source of the generated parser compiled as +parser_class_name+. This can be written out to a file.
     def compile_to_ruby_source_as parser_class_name
       result = "class #{parser_class_name} < Dhaka::CompiledParser\n\n"
-      result << "  self.grammar = #{@grammar.name}\n\n"
+      result << "  self.grammar = #{grammar.name}\n\n"
       result << "  start_with #{start_state.id}\n\n"
       states.each do |state|
         result << "#{state.compile_to_ruby_source}\n\n"
@@ -60,29 +51,40 @@ module Dhaka
     # options hash, lookaheads are not written out to the parser states, which is helpful when there are dozens
     # of lookahead symbols for every item in every state.
     def to_dot(options = {})
-      result = ["digraph x {", "node [fontsize=\"10\" shape=box size=\"5\"]"]
-      result += states.collect { |state| state.to_dot(options) }
-      states.each { |state| 
-          @transitions[state].each { |symbol, dest_state| 
-              result << "#{state.dot_name} -> #{dest_state.dot_name} [label=\"#{symbol.name}\"]"
-             }
-         }
+      result = ["digraph x {", %(node [fontsize="10" shape=box size="5"])]
+      result.concat states.collect { |state| state.to_dot(options) }
+      states.each do |state| 
+        @transitions[state].each do |symbol, dest_state| 
+          result << %(#{state.unique_name} -> #{dest_state.unique_name} [label="#{symbol.name}"])
+        end
+      end
       result << ['}']
       result.join("\n")
     end
+
+    def inspect
+      "<Dhaka::Parser grammar : #{grammar}>"
+    end
   
-    private :start_state
     private
-    
+      attr_reader :start_state
+      
     def states
       @states.values
     end
+    
+    def default_logger
+      logger           = Logger.new(STDOUT)
+      logger.level     = Logger::WARN
+      logger.formatter = ParserLogOutputFormatter.new
+      logger
+    end
 
     def initialize_states
-      start_productions = @grammar.productions_for_symbol(@grammar.start_symbol)
-      raise NoStartProductionsError.new(@grammar) if start_productions.empty?
+      start_productions = grammar.productions_for_symbol(grammar.start_symbol)
+      raise NoStartProductionsError.new(grammar) if start_productions.empty?
       start_items = ItemSet.new(start_productions.collect {|production| Item.new(production, 0)})
-      start_items.each {|start_item| start_item.lookaheadset << @grammar.end_symbol}
+      start_items.each {|start_item| start_item.lookaheadset << grammar.end_symbol}
       @start_state = @states[start_items]
       @logger.debug("Pumping #{@channels.size} channels...")
       pump_channels
@@ -94,9 +96,9 @@ module Dhaka
 
     def generate_shift_actions
       @states.values.each do |state|
-        @transitions[state].keys.each { |symbol|
-            state.actions[symbol.name] = ShiftAction.new(@transitions[state][symbol])
-           }
+        @transitions[state].keys.each do |symbol|
+          state.actions[symbol.name] = ShiftAction.new(@transitions[state][symbol])
+        end
       end
     end
 
@@ -113,9 +115,11 @@ module Dhaka
         new_action = ReduceAction.new(item.production) 
         if existing_action = state.actions[lookahead.name]
           if ReduceAction === existing_action
-            @logger.error(build_conflict_message(state, lookahead, new_action).join("\n"))
+            message = ReduceReduceConflict.new(state, lookahead, new_action).resolve
+            @logger.error(message)
           else
-            resolve_conflict state, lookahead, new_action
+            message = ShiftReduceConflict.new(state, lookahead, new_action).resolve
+            @logger.warn(message)
           end
         else
           state.actions[lookahead.name] = new_action
@@ -123,56 +127,16 @@ module Dhaka
       end
     end
   
-  
-    def resolve_conflict state, lookahead, new_action
-      message = build_conflict_message(state, lookahead, new_action)
-      shift_precedence = lookahead.precedence
-      reduce_precedence = new_action.production.precedence
-      if (shift_precedence && reduce_precedence)
-        if (shift_precedence > reduce_precedence)
-          message << "Resolving with precedence. Choosing shift over reduce."
-        elsif (shift_precedence < reduce_precedence)
-          message << "Resolving with precedence. Choosing reduce over shift."
-          state.actions[lookahead.name] = new_action
-        else
-          case shift_precedence.associativity
-            when :left
-              message << "Resolving with left associativity. Choosing reduce over shift."
-              state.actions[lookahead.name] = new_action
-            when :right
-              message << "Resolving with right associativity. Choosing shift over reduce."
-            when :nonassoc
-              message << "Resolving with non-associativity. Eliminating action."
-              state.actions.delete(lookahead.name)
-          end
-        end
-      else
-        message << "No precedence rule. Choosing shift over reduce."
-      end
-      @logger.warn(message.join("\n"))
-    end
-  
-    def build_conflict_message state, lookahead, new_action
-      message = ["Parser Conflict at State:"] + state.items.values.collect{|it| it.to_s(:hide_lookaheads => true)}
-      message << "Existing: #{state.actions[lookahead.name]}"
-      message << "New: #{new_action}"
-      message << "Lookahead: #{lookahead}"
-      message
-    end
-  
     def pump_channels
-      while true
+      loop do
         unstable_count = 0
         @channels.each do |channel| 
-          if channel.pump
-            unstable_count += 1
-          end 
+          unstable_count += 1 if channel.pump
         end
-        break if unstable_count == 0
+        break if unstable_count.zero?
         @logger.debug("#{unstable_count} unstable channels...")
       end
     end
-  
   end
 
   # Raised when trying to create a Parser for a grammar that has no productions for the start symbol
@@ -184,5 +148,12 @@ module Dhaka
       "No start productions defined for #{@grammar.name}"
     end
   end
+
+  class ParserLogOutputFormatter < Logger::Formatter #:nodoc:
+    def call(severity, time, progname, msg)
+      "\n%s -- %s: %s\n" % [ severity, progname, msg2str(msg)]
+    end
+  end
+
 end
 

data/lib/parser/parser_run.rb

@@ -2,9 +2,9 @@ module Dhaka
   class ParserRun #:nodoc:
 
     def initialize(grammar, start_state, token_stream)
-      @grammar = grammar
-      @node_stack = []
-      @state_stack = [start_state]
+      @grammar      = grammar
+      @node_stack   = []
+      @state_stack  = [start_state]
       @token_stream = token_stream
       @symbol_queue = []
     end
@@ -17,7 +17,7 @@ module Dhaka
         return error if error
         node_stack << ParseTreeLeafNode.new(@current_token)
       end
-      ParseSuccessResult.new(node_stack[0])
+      ParseSuccessResult.new(node_stack.first)
     end
 
     private
@@ -26,9 +26,9 @@ module Dhaka
 
     def execute_actions
       while symbol_name = @symbol_queue.pop
-        action = state_stack[-1].actions[symbol_name]
+        action = state_stack.last.actions[symbol_name]
         return ParseErrorResult.new(@current_token) unless action
-        self.instance_eval(&action.action_code)
+        instance_eval(&action.action_code)
       end
       nil
     end

data/lib/parser/parser_state.rb

@@ -1,43 +1,40 @@
-#!/usr/bin/env ruby
-require 'set'
 module Dhaka
   class ParserState #:nodoc:
-
     attr_accessor :items, :actions, :id
   
     @@state_id = 0
   
     def self.next_state_id
-      result = @@state_id
+      result      = @@state_id
       @@state_id += 1
       result
     end
   
     def initialize(parser, items, id=nil)
-      @parser = parser
-      @items = items
+      @parser  = parser
+      @items   = items
       @actions = {}
-      @id =  id ? id : ParserState.next_state_id
+      @id      = id || ParserState.next_state_id
     end
 
     def transition_items
       result = Hash.new {|h, k| h[k] = ItemSet.new()}
-      for item in @items.values
-        (result[item.next_symbol] << item) if item.next_symbol
+      items.values.each do |item|
+        result[item.next_symbol] << item if item.next_symbol
       end
       result
     end
 
-    def dot_name
-      self.to_s
+    def unique_name
+      "State#{id}"
     end
 
     def to_dot(options = {})
-      label = self.items.values.collect{|item| item.to_s(options)}.join('\n')
-      "#{dot_name} [label=\"#{label}\"]"
+      %(#{unique_name} [label="#{items.values.collect{|item| item.to_s(options)}.join('\n')}"])
     end
+
     def compile_to_ruby_source
-      result = "  at_state(#{@id}) {\n"
+      result = "  at_state(#{id}) {\n"
       actions.each do |symbol_name, action|
         result << "    for_symbol('#{symbol_name}') { #{action.compile_to_ruby_source} }\n"
       end
@@ -49,18 +46,19 @@ module Dhaka
       actions[symbol_name] = @parser.instance_eval(&blk)
     end
   
-    def to_s
-      "State#{id}"
+    def to_s(options = {})
+      items.values.collect{|item| item.to_s(options)}.join("\n")
     end
   
   end
 
   class ItemSet < Set #:nodoc:
     def hash
-      self.collect{|item| item.hash}.inject{|result, hashcode| result ^ hashcode}
+      collect{|item| item.hash}.inject{|result, hashcode| result ^ hashcode}
     end
+    
     def eql? other
       self == other
     end
   end
-end
+end

data/lib/parser/token.rb

@@ -6,15 +6,17 @@ module Dhaka
   class Token
     attr_accessor :symbol_name, :value, :input_position
     def initialize(symbol_name, value, input_position)
-      @symbol_name = symbol_name
-      @value = value
+      @symbol_name    = symbol_name
+      @value          = value
       @input_position = input_position
     end
+    
     def to_s #:nodoc:
-      "#{symbol_name}"
+      value ? "#{symbol_name} : #{value}" : "#{symbol_name}"
     end
+    
     def == other
-      (symbol_name == other.symbol_name) && (value == other.value)
+      symbol_name == other.symbol_name && value == other.value
     end
   end
 end

data/lib/tokenizer/tokenizer.rb

@@ -1,5 +1,4 @@
 module Dhaka
-  
   # Reserved constant used to identify the idle state of the tokenizer.
   TOKENIZER_IDLE_STATE = :idle_state 
 
@@ -7,17 +6,18 @@ module Dhaka
   # be passed in directly to the parser.
   class TokenizerSuccessResult
     include Enumerable
+    
     def initialize(tokens)
       @tokens = tokens
     end
+    
     # Returns false.
     def has_error?
       false
     end
-    def each
-      @tokens.each do |token|
-        yield token
-      end
+    
+    def each(&block)
+      @tokens.each(&block)
     end
   end  
 
@@ -25,9 +25,11 @@ module Dhaka
   class TokenizerErrorResult
     # The index of the character that caused the error.
     attr_reader :unexpected_char_index
+    
     def initialize(unexpected_char_index)
       @unexpected_char_index = unexpected_char_index
     end
+    
     # Returns true.
     def has_error?
       true
@@ -101,34 +103,41 @@ module Dhaka
   #        switch_to Dhaka::TOKENIZER_IDLE_STATE
   #      end
   #      for_characters digits do
-  #        curr_token.value += curr_char
+  #        curr_token.value << curr_char
   #        advance
   #      end
   #    end
   #
   #  end
-    
-  
   class Tokenizer
+    class << self
+      # Define the action for the state named +state_name+.
+      def for_state(state_name, &blk)
+        states[state_name].instance_eval(&blk)
+      end
     
-    # Define the action for the state named +state_name+.
-    def self.for_state(state_name, &blk)
-      states[state_name].instance_eval(&blk)
-    end
-    
-    # Tokenizes a string +input+ and returns a TokenizerErrorResult on failure or a TokenizerSuccessResult on sucess.
-    def self.tokenize(input)
-      self.new(input).run
+      # Tokenizes a string +input+ and returns a TokenizerErrorResult on failure or a TokenizerSuccessResult on sucess.
+      def tokenize(input)
+        new(input).run
+      end
+      
+      private
+        def inherited(tokenizer)
+          class << tokenizer
+            attr_accessor :states, :grammar
+          end
+          tokenizer.states = Hash.new {|hash, key| hash[key] = TokenizerState.new}
+        end
     end
     
     # The tokens shifted so far.
     attr_reader :tokens
     
     def initialize(input) #:nodoc:
-      @input = input
-      @current_state = self.class.states[TOKENIZER_IDLE_STATE]
+      @input           = input
+      @current_state   = self.class.states[TOKENIZER_IDLE_STATE]
       @curr_char_index = 0
-      @tokens = []
+      @tokens          = []
     end
   
     # The character currently being processed.
@@ -140,10 +149,14 @@ module Dhaka
     def advance
       @curr_char_index += 1
     end
+
+    def inspect
+      "<Dhaka::Tokenizer grammar : #{grammar}>"
+    end
     
     # The token currently on top of the stack. 
     def curr_token
-      tokens[-1]
+      tokens.last
     end
     
     # Push a new token on to the stack with symbol corresponding to +symbol_name+ and a value of +value+.
@@ -164,17 +177,7 @@ module Dhaka
         instance_eval(&blk)
       end
       tokens << Dhaka::Token.new(Dhaka::END_SYMBOL_NAME, nil, nil)
-      return TokenizerSuccessResult.new(tokens)
+      TokenizerSuccessResult.new(tokens)
     end
-
-    private
-    def self.inherited(tokenizer)
-      class << tokenizer
-        attr_accessor :states, :grammar
-      end
-      tokenizer.states = Hash.new {|hash, key| hash[key] = TokenizerState.new}
-    end
-
   end
-
 end