antlr3 1.2.3

data/lib/antlr3/debug/trace-event-listener.rb

@@ -0,0 +1,92 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+module ANTLR3
+module Debug
+=begin rdoc ANTLR3::Debug::EventListener
+
+A listener that simply records text representations of the events.
+Useful for debugging the debugging facility ;)
+Subclasses can override the record() method (which defaults to printing to
+stdout) to record the events in a different way.
+
+=end
+class TraceEventListener
+  include EventListener
+  
+  def initialize(adaptor = nil, device = $stderr)
+    super()
+    @device = device
+    @adaptor = adaptor ||= ANTLR3::AST::CommonTreeAdaptor.new
+  end
+  
+  def record(event_message, *interpolation_arguments)
+    event_message = event_message.to_s << "\n"
+    @device.printf(event_message, *interpolation_arguments)
+  end
+  
+  def enter_alternative( alt_number )
+    record "(%s): number=%s", __method__, alt_number
+  end
+  
+  def enter_rule( grammar_file_name, rule_name )
+    record "(%s): rule=%s", __method__, rule_name
+  end
+  
+  def exit_rule( grammar_file_name, rule_name )
+    record "(%s): rule=%s", __method__, rule_name
+  end
+  
+  def enter_subrule( decision_number )
+    record "(%s): decision=%s", __method__, decision_number
+  end
+  
+  def exit_subrule( decision_number )
+    record "(%s): decision=%s", __method__, decision_number
+  end
+  
+  def location( line, position )
+    record '(%s): line=%s position=%s', __method__, line, position
+  end
+  
+  def consume_node( tree )
+    record '(%s) unique_id=%s text=%p type=%s[%s]', __method__, @adaptor.unique_id(tree),
+           @adaptor.text_of(tree), @adaptor.type_name( tree ), @adaptor.type_of(tree)
+  end
+  
+  def look(i, tree)
+    record '(%s): k=%s unique_id=%s text=%p type=%s[%s]', __method__, i, @adaptor.unique_id( tree ),
+            @adaptor.text_of( tree ), @adaptor.type_name( tree ), @adaptor.type_of( tree )
+  end
+  
+  def flat_node( tree )
+    record '(%s): unique_id=%s', __method__, @adaptor.unique_id( tree )
+  end
+  
+  def create_node(tree, token = nil)
+    unless token
+      record '(%s): unique_id=%s text=%p type=%s[%s]', __method__, @adaptor.unique_id( tree ),
+            @adaptor.text_of( tree ), @adaptor.type_name( tree ), @adaptor.type_of( tree )
+    else
+      record '(%s): unique_id=%s type=%s[%s]', __method__, @adaptor.unique_id( tree ),
+              @adaptor.type_of( tree ), @adaptor.type_name( tree ), @adaptor.type_of( tree )
+    end
+  end
+  
+  def become_root( new_root, old_root )
+    record '(%s): old_root_id=%s new_root_id=%s', __method__, @adaptor.unique_id( new_root ),
+            @adaptor.unique_id( old_root )
+  end
+  
+  def add_child( root, child )
+    record '(%s): root_id=%s child_id=%s', __method__, @adaptor.unique_id( root ),
+            @adaptor.unique_id( child )
+  end
+  
+  def set_token_boundaries( tree, token_start_index, token_stop_index )
+    record '(%s): unique_id=%s index_range=%s..%s', __method__, @adaptor.unique_id( tree ),
+            token_start_index, token_stop_index
+  end
+end # class TraceEventListener
+end # module Debug
+end # module ANTLR3

data/lib/antlr3/dfa.rb

@@ -0,0 +1,247 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+=begin LICENSE
+
+[The "BSD licence"]
+Copyright (c) 2009 Kyle Yetter
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+=end
+
+module ANTLR3
+
+=begin rdoc ANTLR3::DFA
+
+DFA is a class that implements a finite state machine that chooses between
+alternatives in a rule based upon lookahead symbols from an input stream.
+
+Deterministic Finite Automata (DFA) are finite state machines that are capable
+of recognizing <i>regular languages</i>. For more background on the subject,
+check out http://en.wikipedia.org/wiki/Deterministic_finite-state_machine or
+check out general ANTLR documentation at http://www.antlr.org
+
+ANTLR implements most of its decision logic directly using code branching
+structures in methods. However, for certain types of decisions, ANTLR will
+generate a special DFA class definition to implement a decision.
+
+Conceptually, these state machines are defined by a number of states, each state
+represented by an integer indexed upward from zero. State number +0+ is the
+<i>start state</i> of the machine; every prediction will begin in state +0+. At
+each step, the machine examines the next symbol on the input stream, checks the
+value against the transition parameters associated with the current state, and
+either moves to a new state number to repeat the process or decides that the
+machine cannot transition any further. If the machine cannot transition any
+further and the current state is defined as an <i>accept state</i>, an
+alternative has been chosen successfully and the prediction procedure ends. If
+the current state is not an <i>accept state</i>, the prediction has failed and
+there is <i>no viable alternative</i>.
+
+In generated code, ANTLR defines DFA states using seven parameters, each defined
+as a member of seven seperate array constants -- +MIN+, +MAX+, +EOT+, +EOF+,
++SPECIAL+, +ACCEPT+, and +TRANSITION+. The parameters that characterize state
++s+ are defined by the value of these lists at index +s+.
+
+MIN[s]::
+  The smallest value of the next input symbol that has 
+  a transition for state +s+
+MAX[s]::
+  The largest value of the next input symbol that has 
+  a transition for state +s+
+TRANSITION[s]::
+  A list that defines the next state number based upon
+  the current input symbol.
+EOT[s]::
+  If positive, it specifies a state transition in 
+  situations where a non-matching input symbol does
+  not indicate failure.
+SPECIAL[s]::
+  If positive, it indicates that the prediction 
+  algorithm must defer to a special code block 
+  to determine the next state. The value is used
+  by the special state code to determine the next
+  state.
+ACCEPT[s]::
+  If positive and there are no possible state
+  transitions, this is the alternative number
+  that has been predicted
+EOF[s]::
+  If positive and the input stream has been exhausted,
+  this is the alternative number that has been predicted.
+
+For more information on the prediction algorithm, check out the #predict method
+below.
+
+=end
+
+class DFA
+  include Constants
+  include Error
+  
+  attr_reader :recognizer, :decision_number, :eot, :eof, :min, :max,
+              :accept, :special, :transition, :special_block
+  
+  class << self
+    attr_reader :decision
+    
+    def unpack(*data)
+      data.empty? and return [].freeze
+      
+      n = data.length / 2
+      size = 0
+      n.times { |i| size += data[2*i] }
+      if size > 1024
+        values = Hash.new(0)
+        data.each_slice(2) do |count, value|
+          values[value] += count
+        end
+        default = values.keys.max_by { |v| values[v] }
+        
+        unpacked = Hash.new(default)
+        position = 0
+        data.each_slice(2) do |count, value|
+          unless value == default
+            count.times { |i| unpacked[position + i] = value }
+          end
+          position += count
+        end
+      else
+        unpacked = []
+        data.each_slice(2) do |count, value|
+          unpacked.concat Array.new(count, value)
+        end
+      end
+      
+      return unpacked.freeze
+    end
+  end
+  
+  def initialize(recognizer, decision_number = nil, eot = nil, eof = nil,
+                 min = nil, max = nil, accept = nil, special = nil,
+                 transition = nil, &special_block)
+    @recognizer = recognizer
+    @decision_number = decision_number || self.class.decision
+    @eot = eot || self.class::EOT
+    @eof = eof || self.class::EOF
+    @min = min || self.class::MIN
+    @max = max || self.class::MAX
+    @accept = accept || self.class::ACCEPT
+    @special = special || self.class::SPECIAL
+    @transition = transition || self.class::TRANSITION
+    @special_block = special_block
+  rescue NameError => e
+    raise unless e.message =~ /uninitialized constant/
+    constant = e.name
+    message = Util.tidy( <<-END )
+    | No #{constant} information provided.
+    | DFA cannot be instantiated without providing state array information.
+    | When DFAs are generated by ANTLR, this information should already be
+    | provided in the DFA subclass constants.
+    END
+  end
+  
+  def predict(input)
+    mark = input.mark
+    state = 0
+    
+    50000.times do
+      special_state = @special[state]
+      if special_state >= 0
+        state = @special_block.call(special_state)
+        if state == -1
+          no_viable_alternative(state, input)
+          return 0
+        end
+        input.consume
+        next
+      end
+      @accept[state] >= 1 and return @accept[state]
+      
+      # look for a normal char transition
+      
+      c = input.peek
+      # the @min and @max arrays contain the bounds of the character (or token type)
+      # ranges for the transition decisions
+      if c.between?(@min[state], @max[state])
+        # c - @min[state] is the position of the character within the range
+        # so for a range like ?a..?z, a match of ?a would be 0,
+        # ?c would be 2, and ?z would be 25
+        next_state = @transition[state][c - @min[state]]
+        if next_state < 0
+          if @eot[state] >= 0
+            state = @eot[state]
+            input.consume
+            next
+          end
+          no_viable_alternative(state, input)
+          return 0
+        end
+        
+        state = next_state
+        input.consume()
+        next
+      end
+      if @eot[state] >= 0
+        state = @eot[state]
+        input.consume()
+        next
+      end
+      (c == EOF && @eof[state] >= 0) and return @accept[@eof[state]]
+      no_viable_alternative(state, input)
+      return 0
+    end
+    
+    ANTLR3.bug!( Util.tidy(<<-END) )
+    | DFA BANG!
+    |   The prediction loop has exceeded a maximum limit of 50000 iterations
+    | ----
+    | decision: #@decision_number
+    | description: #{description}
+    END
+  ensure
+    input.rewind(mark)
+  end
+
+  def no_viable_alternative(state, input)
+    raise(BacktrackingFailed) if @recognizer.state.backtracking > 0
+    except = NoViableAlternative.new(description, @decision_number, state, input)
+    error(except)
+    raise(except)
+  end
+  
+  def error(except)
+    # overridable debugging hook
+  end
+  
+  def special_state_transition(state, input)
+    return -1
+  end
+  
+  def description
+    return "n/a"
+  end
+end
+end

data/lib/antlr3/dot.rb

@@ -0,0 +1,174 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+=begin LICENSE
+
+[The "BSD licence"]
+Copyright (c) 2009 Kyle Yetter
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+=end
+
+require 'antlr3'
+require 'erb'
+
+module ANTLR3
+
+=begin rdoc ANTLR3::DOT
+
+An extra utility for generating graphviz DOT file representations of ANTLR
+Abstract Syntax Tree nodes.
+
+This module has been directly ported to Ruby from the ANTLR Python runtime
+library.
+
+=end
+
+module DOT
+  class Context
+    def []=(var, value)
+      instance_variable_set(:"@#{var}", value)
+    end
+    def [](var)
+      instance_variable_get(:"@#{var}")
+    end
+    
+    def initialize(template, vars = {})
+      @__template__ = template
+      vars.each do |var, value|
+        self[var] = value
+      end
+    end
+    
+    def to_s
+      @__template__.result(binding)
+    end
+  end
+  class TreeGenerator
+    TREE_TEMPLATE = ERB.new( Util.tidy(<<-END) )
+    | digraph {
+    |   ordering=out;
+    |   ranksep=.4;
+    |   node [shape=plaintext, fixedsize=true, fontsize=11, fontname="Courier",
+    |         width=.25, height=.25];
+    |   edge [arrowsize=.5];
+    |   <%= @nodes.join("\n  ") %>
+    |   <%= @edges.join("\n  ") %>
+    | }
+    END
+    
+    NODE_TEMPLATE = ERB.new( Util.tidy(<<-END) )
+    | <%= @name %> [label="<%= @text %>"];
+    END
+    
+    EDGE_TEMPLATE = ERB.new( Util.tidy(<<-END) )
+    | <%= @parent %> -> <%= @child %>; // "<%= @parent_text %>" -> "<%= @child_text %>"
+    END
+    
+    def self.generate(tree, adaptor = nil, tree_template = TREE_TEMPLATE,
+                      edge_template = EDGE_TEMPLATE)
+      new.to_dot(tree, adaptor, tree_template, edge_template)
+    end
+    
+    def initialize
+      @node_number = 0
+      @node_to_number_map = Hash.new do |map, node|
+        map[node] = @node_number
+        @node_number += 1
+        @node_number - 1
+      end
+    end
+    
+    def to_dot(tree, adaptor = nil, tree_template = TREE_TEMPLATE,
+               edge_template = EDGE_TEMPLATE)
+      adaptor ||= AST::CommonTreeAdaptor.new
+      @node_number = 0
+      tree_template = Context.new(tree_template, :nodes => [], :edges => [])
+      define_nodes( tree, adaptor, tree_template )
+      
+      @node_number = 0
+      define_edges( tree, adaptor, tree_template, edge_template )
+      return tree_template.to_s
+    end
+    
+    def define_nodes( tree, adaptor, tree_template, known_nodes = nil)
+      known_nodes ||= Set.new
+      tree.nil? and return
+      n = adaptor.child_count( tree )
+      n == 0 and return
+      number = node_number( tree )
+      unless known_nodes.include?( number )
+        parent_node_template = node_template_for(adaptor, child)
+        tree_template[:nodes] << parent_node_template
+        known_nodes.add( number )
+      end
+      
+      n.times do |index|
+        child = adaptor.child_of( tree, index )
+        number = @node_to_number_map[ child ]
+        unless known_nodes.include?(number)
+          node_template = node_template_for( adaptor, child )
+          tree_template[:nodes] << node_template
+          known_nodes.add( number )
+        end
+        
+        define_nodes(child, adaptor, tree_template, edge_template)
+      end
+    end
+    
+    def define_edges( tree, adaptor, tree_template, edge_template )
+      tree.nil? or return
+      
+      n = adaptor.child_count( tree )
+      n == 0 and return
+      
+      parent_name = 'n%i' % @node_to_number_map[ tree ]
+      parent_text = adaptor.text_of( tree )
+      n.times do |index|
+        child = adaptor.child_of( tree, index )
+        child_text = adaptor.text_of( child )
+        child_name = 'n%i' % @node_to_number_map[ tree ]
+        edge_template = Context.new(edge_template,
+          :parent => parent_name, :child => child_name,
+          :parent_text => parent_text, :child_text => child_text
+        )
+        tree_template[:edges] << edge_template
+        define_edges( child, adaptor, tree_template, edge_template )
+      end
+    end
+    
+    def node_template_for(adaptor, tree)
+      text = adaptor.text_of( tree )
+      node_template = Context.new(NODE_TEMPLATE)
+      unique_name = 'n%i' % @node_to_number_map[ tree ]
+      node_template[:name] = unique_name
+      text and text = text.gsub(/"/, '\\"')
+      node_template[:text] = text
+      return node_template
+    end
+  end
+end
+end