antlr3 1.6.0 → 1.6.3

data/lib/antlr3/profile.rb

@@ -13,13 +13,13 @@ switch.
 module ParserEvents
   include ANTLR3::Debug::ParserEvents
   
-  def initialize(stream, options = {})
-    options[:debug_listener] ||= Profiler.new( self )
+  def initialize( stream, options = {} )
+    options[ :debug_listener ] ||= Profiler.new( self )
     super( stream, options )
   end
   
-  def already_parsed_rule?(rule)
-    @debug_listener.examine_rule_memoization(rule)
+  def already_parsed_rule?( rule )
+    @debug_listener.examine_rule_memoization( rule )
     super
   end
   
@@ -27,8 +27,8 @@ module ParserEvents
     @debug_listener.profile
   end
   
-  def memoize(rule, start_index, success)
-    @debug_listener.memoize(rule, rule_start_index, sucess)
+  def memoize( rule, start_index, success )
+    @debug_listener.memoize( rule, rule_start_index, sucess )
     super
   end
 end
@@ -36,18 +36,18 @@ end
 class DataSet < ::Array
   include ::Math
   def total
-    inject(:+)
+    inject( :+ )
   end
   def average
-    length > 0 ? (total.to_f / length) : 0
+    length > 0 ? ( total.to_f / length ) : 0
   end
   def variance
-    length.zero? and return(0.0)
+    length.zero? and return( 0.0 )
     mean = average
-    inject(0.0) { |t, i| t + (i - mean)**2 } / (length - 1)
+    inject( 0.0 ) { |t, i| t + ( i - mean )**2 } / ( length - 1 )
   end
   def standard_deviation
-    sqrt(variance)
+    sqrt( variance )
   end
 end
 
@@ -55,8 +55,8 @@ end
 
 
 
-unless const_defined?(:Profile)
-  Profile = Struct.new(
+unless const_defined?( :Profile )
+  Profile = Struct.new( 
     :grammar_file, :parser_class, :top_rule,
     :rule_invocations, :guessing_rule_invocations, :rule_invocation_depth,
     :fixed_looks, :cyclic_looks, :syntactic_predicate_looks,
@@ -69,8 +69,8 @@ end
 
 class Profile
   def initialize
-    init_values = Array.new(self.class.members.length, 0)
-    super(*init_values)
+    init_values = Array.new( self.class.members.length, 0 )
+    super( *init_values )
     self.top_rule = self.parser_class = self.grammar_file = nil
     self.fixed_looks = DataSet.new
     self.cyclic_looks = DataSet.new
@@ -91,58 +91,58 @@ class Profile
   
   def generate_report
     report = '+' << '-' * 78 << "+\n"
-    report << '| ' << "ANTLR Rule Profile".center(76) << " |\n"
+    report << '| ' << "ANTLR Rule Profile".center( 76 ) << " |\n"
     report << '+' << '-' * 78 << "+\n"
-    report << "| Generated at #{Time.now}".ljust(78) << " |\n"
-    report << "| Profiled #{parser_class.name}##{top_rule}".ljust(78) << " |\n"
-    report << "| Rule source generated from grammar file #{grammar_file}".ljust(78) << " |\n"
+    report << "| Generated at #{ Time.now }".ljust( 78 ) << " |\n"
+    report << "| Profiled #{ parser_class.name }##{ top_rule }".ljust( 78 ) << " |\n"
+    report << "| Rule source generated from grammar file #{ grammar_file }".ljust( 78 ) << " |\n"
     report << '+' << '-' * 78 << "+\n"
     
-    report << '| ' << "Rule Invocations".center(76) << " |\n"
+    report << '| ' << "Rule Invocations".center( 76 ) << " |\n"
     report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
-    report << "| %-66s | %7i |\n" % ["Total Invocations", rule_invocations]
-    report << "| %-66s | %7i |\n" % ["``Guessing'' Invocations", guessing_rule_invocations]
-    report << "| %-66s | %7i |\n" % ["Deepest Level of Invocation", rule_invocation_depth]
+    report << "| %-66s | %7i |\n" % [ "Total Invocations", rule_invocations ]
+    report << "| %-66s | %7i |\n" % [ "``Guessing'' Invocations", guessing_rule_invocations ]
+    report << "| %-66s | %7i |\n" % [ "Deepest Level of Invocation", rule_invocation_depth ]
     report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
     
-    report << '| ' << "Execution Events".center(76) << " |\n"
+    report << '| ' << "Execution Events".center( 76 ) << " |\n"
     report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
-    report << "| %-66s | %7i |\n" % ["Semantic Predicates Evaluated", semantic_predicates]
-    report << "| %-66s | %7i |\n" % ["Syntactic Predicates Evaluated", syntactic_predicates]
-    report << "| %-66s | %7i |\n" % ["Errors Reported", reported_errors]
+    report << "| %-66s | %7i |\n" % [ "Semantic Predicates Evaluated", semantic_predicates ]
+    report << "| %-66s | %7i |\n" % [ "Syntactic Predicates Evaluated", syntactic_predicates ]
+    report << "| %-66s | %7i |\n" % [ "Errors Reported", reported_errors ]
     report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
     
-    report << '| ' << "Token and Character Data".center(76) << " |\n"
+    report << '| ' << "Token and Character Data".center( 76 ) << " |\n"
     report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
-    report << "| %-66s | %7i |\n" % ["Tokens Consumed", tokens]
-    report << "| %-66s | %7i |\n" % ["Hidden Tokens Consumed", hidden_tokens]
-    report << "| %-66s | %7i |\n" % ["Characters Matched", characters_matched]
-    report << "| %-66s | %7i |\n" % ["Hidden Characters Matched", hidden_characters_matched]
+    report << "| %-66s | %7i |\n" % [ "Tokens Consumed", tokens ]
+    report << "| %-66s | %7i |\n" % [ "Hidden Tokens Consumed", hidden_tokens ]
+    report << "| %-66s | %7i |\n" % [ "Characters Matched", characters_matched ]
+    report << "| %-66s | %7i |\n" % [ "Hidden Characters Matched", hidden_characters_matched ]
     report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
     
-    report << '| ' << "Memoization".center(76) << " |\n"
+    report << '| ' << "Memoization".center( 76 ) << " |\n"
     report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
-    report << "| %-66s | %7i |\n" % ["Cache Entries", memoization_cache_entries]
-    report << "| %-66s | %7i |\n" % ["Cache Hits", memoization_cache_hits]
-    report << "| %-66s | %7i |\n" % ["Cache Misses", memoization_cache_misses]
+    report << "| %-66s | %7i |\n" % [ "Cache Entries", memoization_cache_entries ]
+    report << "| %-66s | %7i |\n" % [ "Cache Hits", memoization_cache_hits ]
+    report << "| %-66s | %7i |\n" % [ "Cache Misses", memoization_cache_misses ]
     report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
     
-    [
-      ['Fixed Lookahead (k)', fixed_looks],
-      ['Arbitrary Lookahead (k)', cyclic_looks],
-      ['Backtracking (Syntactic Predicate)', syntactic_predicate_looks]
+    [ 
+      [ 'Fixed Lookahead (k)', fixed_looks ],
+      [ 'Arbitrary Lookahead (k)', cyclic_looks ],
+      [ 'Backtracking (Syntactic Predicate)', syntactic_predicate_looks ]
     ].each do |name, set|
       mean, stdev = '%4.2f' % set.average, '%4.2f' % set.standard_deviation
-      report << '| ' << "#{name} Decisions".center(76) << " |\n"
+      report << '| ' << "#{ name } Decisions".center( 76 ) << " |\n"
       report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
-      report << "| %-66s | %7i |\n" % ["Count", set.length]
-      report << "| %-66s | %7i |\n" % ["Minimum k", set.min]
-      report << "| %-66s | %7i |\n" % ["Maximum k", set.max]
-      report << "| %-66s | %7s |\n" % ["Average k", mean]
-      report << "| %-66s | %7s |\n" % ["Standard Deviation of k", stdev]
+      report << "| %-66s | %7i |\n" % [ "Count", set.length ]
+      report << "| %-66s | %7i |\n" % [ "Minimum k", set.min ]
+      report << "| %-66s | %7i |\n" % [ "Maximum k", set.max ]
+      report << "| %-66s | %7s |\n" % [ "Average k", mean ]
+      report << "| %-66s | %7s |\n" % [ "Standard Deviation of k", stdev ]
       report << '+' << '-' * 68 << '+' << '-' * 9 << "+\n"
     end
-    return(report)
+    return( report )
   end
 end
 
@@ -156,7 +156,8 @@ builds a simple report to present the various statistics.
 
 =end
 class Profiler
-  include ANTLR3::Debug::EventListener
+  include Debug::EventListener
+  include Constants
   
   PROTOCOL_VERSION = 2
   
@@ -176,7 +177,7 @@ class Profiler
   
   attr_accessor :output
   
-  def initialize(parser = nil, output = nil)
+  def initialize( parser = nil, output = nil )
     @parser = parser
     @profile = nil
     @rule_level = 0
@@ -196,7 +197,7 @@ class Profiler
     @look_stack = []
   end
   
-  def enter_rule(grammar_file_name, rule_name)
+  def enter_rule( grammar_file_name, rule_name )
     if @rule_level.zero?
       commence
       @profile.grammar_file = grammar_file_name
@@ -209,13 +210,13 @@ class Profiler
       @profile.rule_invocation_depth = @rule_level
   end
   
-  def exit_rule(grammar_file_name, rule_name)
+  def exit_rule( grammar_file_name, rule_name )
     @rule_level -= 1
   end
 
-  def examine_rule_memoization(rule)
-    stop_index = parser.rule_memoization(rule, @parser.input.index)
-    if stop_index == BaseRecognizer::MEMO_RULE_UNKNOWN
+  def examine_rule_memoization( rule )
+    stop_index = parser.rule_memoization( rule, @parser.input.index )
+    if stop_index == MEMO_RULE_UNKNOWN
       @profile.memoization_cache_misses += 1
       @profile.guessing_rule_invocations += 1
     else
@@ -223,18 +224,18 @@ class Profiler
     end
   end
   
-  def memoize(rule, start_index, success)
+  def memoize( rule, start_index, success )
     @profile.memoization_cache_entries += 1
   end
   
   
-  def enter_decision(decision_number)
+  def enter_decision( decision_number )
     @decision_level += 1
     starting_look_index = @parser.token_stream.index
     @look_stack << starting_look_index
   end
 
-  def exit_decision(decision_number)
+  def exit_decision( decision_number )
     @look_stack.pop
     @decision_level -= 1
     if @parser.cyclic_decision? then
@@ -246,39 +247,39 @@ class Profiler
     @decision_look = 0    
   end
   
-  def consume_token(token)
+  def consume_token( token )
     @last_token = token
   end
 
   def in_decision?
-    return(@decision_level > 0)
+    return( @decision_level > 0 )
   end
   
-  def consume_hidden_token(token)
+  def consume_hidden_token( token )
     @last_token = token
   end
 
-  def look(i, token)
+  def look( i, token )
     in_decision? or return
     starting_index = look_stack.last
     input = @parser.token_stream
     this_ref_index = input.index
-    num_hidden = input.tokens(starting_index, this_ref_index).count { |t| t.hidden? }
+    num_hidden = input.tokens( starting_index, this_ref_index ).count { |t| t.hidden? }
     depth = i + this_ref_index - starting_index - num_hidden
     if depth > @decision_look
       @decision_look = depth
     end
   end
   
-  def end_backtrack(level, successful)
+  def end_backtrack( level, successful )
     @profile.syntactic_predicate_looks << @decision_look
   end
   
-  def recognition_exception(error)
+  def recognition_exception( error )
     @profile.reported_errors += 1
   end
   
-  def semantic_predicate(result, predicate)
+  def semantic_predicate( result, predicate )
     in_decision? and @profile.semantic_predicates += 1
   end
   
@@ -287,10 +288,10 @@ class Profiler
     hidden_tokens = input.select { |token| token.hidden? }
     @profile.hidden_tokens = hidden_tokens.length
     @profile.tokens = input.tokens.length
-    @profile.hidden_characters_matched = hidden_tokens.inject(0) do |count, token|
+    @profile.hidden_characters_matched = hidden_tokens.inject( 0 ) do |count, token|
       count + token.text.length rescue count
     end
-    @profile.characters_matched = (@last_token || input.tokens.last).stop + 1
+    @profile.characters_matched = ( @last_token || input.tokens.last ).stop + 1
     write_report
   end
   
@@ -299,17 +300,17 @@ class Profiler
     @output << @profile.generate_report unless @output.nil?
   rescue NoMethodError => error
     if error.name.to_s == '<<'
-      warn(<<-END.strip! % [__FILE__, __LINE__, @output])
+      warn( <<-END.strip! % [ __FILE__, __LINE__, @output ] )
         [%s @ %s]: failed to write report to %p as it does not respond to :<<
       END
     else raise
     end
   rescue IOError => error
-    $stderr.puts( Util.tidy(<<-END) % [__FILE__, __LINE__, @output, error.class, error.message])
+    $stderr.puts( Util.tidy( <<-END ) % [ __FILE__, __LINE__, @output, error.class, error.message ] )
     | [%s @ %s]: failed to write profile report to %p due to an IO Error:
     |   %s: %s
     END
-    $stderr.puts(error.backtrace.map { |call| "  - #{call}" }.join("\n"))
+    $stderr.puts( error.backtrace.map { |call| "  - #{ call }" }.join( "\n" ) )
   end
   
   def report

data/lib/antlr3/recognizers.rb

@@ -4,7 +4,7 @@
 =begin LICENSE
 
 [The "BSD licence"]
-Copyright (c) 2009 Kyle Yetter
+Copyright (c) 2009-2010 Kyle Yetter
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -33,9 +33,9 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 =end
 
 module ANTLR3
-unless const_defined?(:RecognizerSharedState)
+unless const_defined?( :RecognizerSharedState )
 
-RecognizerSharedState = Struct.new(
+RecognizerSharedState = Struct.new( 
   :following,
   :error_recovery,
   :last_error_index,
@@ -89,9 +89,10 @@ text::
   the text of the target token
 
 =end
+
 class RecognizerSharedState
   def initialize
-    super([], false, -1, 0, nil, 0, nil, -1)
+    super( [], false, -1, 0, nil, 0, nil, -1 )
     # ^-- same as this --v 
     # self.following = []
     # self.error_recovery = false
@@ -123,18 +124,18 @@ end
 end
 
 
-=begin rdoc ANTLR3::BaseRecognizer
+=begin rdoc ANTLR3::Recognizer
 
-= BaseRecognizer
+= Recognizer
 
-As the base class of all ANTLR-generated recognizers, BaseRecognizer provides
+As the base class of all ANTLR-generated recognizers, Recognizer provides
 much of the shared functionality and structure used in the recognition process.
 For all effective purposes, the class and its immediate subclasses Lexer,
 Parser, and TreeParser are abstract classes. They can be instantiated, but
 they're pretty useless on their own. Instead, to make useful code, you write an
 ANTLR grammar and ANTLR will generate classes which inherit from one of the
 recognizer base classes, providing the implementation of the grammar rules
-itself. this group of classes to implement necessary tasks. BaseRecognizer
+itself. this group of classes to implement necessary tasks. Recognizer
 defines methods related to:
 
 * token and character matching
@@ -145,17 +146,13 @@ defines methods related to:
 * simple rule tracing and debugging
 
 =end
-class BaseRecognizer
+
+class Recognizer
   include Constants
   include Error
   include TokenFactory
   extend ClassMacros
   
-  MEMO_RULE_FAILED = -2
-  MEMO_RULE_UNKNOWN = -1
-  DEFAULT_TOKEN_CHANNEL = DEFAULT_CHANNEL
-  HIDDEN = HIDDEN_CHANNEL
-  
   @rules = {}
   
   # inherited class methods and hooks
@@ -178,7 +175,7 @@ class BaseRecognizer
       @antlr_version_string = antlr_version.freeze
       @library_version = Util.parse_version( library_version )
       if @antlr_version_string =~ /^(\d+)\.(\d+)(?:\.(\d+)(?:b(\d+))?)?(.*)$/
-        @antlr_version = [$1, $2, $3, $4].map! { |str| str.to_i }
+        @antlr_version = [ $1, $2, $3, $4 ].map! { |str| str.to_i }
         timestamp = $5.strip
         #@antlr_release_time = $5.empty? ? nil : Time.parse($5)
       else
@@ -195,11 +192,11 @@ class BaseRecognizer
     # require additional custom members will have a rule-return
     # name constant that just points to the generic return
     # value. 
-    def define_return_scope(*members)
+    def define_return_scope( *members )
       if members.empty? then generic_return_scope
       else
         members += return_scope_members
-        Struct.new(*members)
+        Struct.new( *members )
       end
     end
     
@@ -209,15 +206,15 @@ class BaseRecognizer
     # this method to add an extra +:tree+ field to
     # all rule return structures.
     def return_scope_members
-      [:start, :stop]
+      [ :start, :stop ]
     end
     
     # sets up and returns the generic rule return
     # scope for a recognizer
     def generic_return_scope
       @generic_return_scope ||= begin
-        struct = Struct.new(*return_scope_members)
-        const_set(:Return, struct)
+        struct = Struct.new( *return_scope_members )
+        const_set( :Return, struct )
       end
     end
     
@@ -245,7 +242,7 @@ class BaseRecognizer
     
     def imports( *grammar_names )
       for grammar in grammar_names
-        imported_grammars.add?(grammar.to_sym) and
+        imported_grammars.add?( grammar.to_sym ) and
           attr_reader( Util.snake_case( grammar ) )
       end
       return imported_grammars
@@ -306,10 +303,10 @@ class BaseRecognizer
   # See the main recognizer subclasses for more specific
   # information about creating recognizer objects like
   # lexers and parsers.
-  def initialize(options = {})
-    @state  = options[:state] || RecognizerSharedState.new
-    @error_output = options.fetch(:error_output, $stderr)
-    defined?(@input) or @input = nil
+  def initialize( options = {} )
+    @state  = options[ :state ] || RecognizerSharedState.new
+    @error_output = options.fetch( :error_output, $stderr )
+    defined?( @input ) or @input = nil
     initialize_dfas
   end
   
@@ -331,15 +328,15 @@ class BaseRecognizer
   # the symbol doesn't match, attempt to use the follow-set
   # data provided by +follow+ to recover from the mismatched
   # token. 
-  def match(type, follow)
-    matched_symbol = current_input_symbol
+  def match( type, follow )
+    matched_symbol = current_symbol
     if @input.peek == type
       @input.consume
       @state.error_recovery = false
       return matched_symbol
     end
-    raise(BacktrackingFailed) if @state.backtracking > 0
-    matched_symbol = recover_from_mismatched_token(type, follow)
+    raise( BacktrackingFailed ) if @state.backtracking > 0
+    matched_symbol = recover_from_mismatched_token( type, follow )
     return matched_symbol
   end
   
@@ -359,7 +356,7 @@ class BaseRecognizer
   # hook for carrying out the error reporting process. The
   # default implementation calls +display_recognition_error+
   # to display the error info on $stderr. 
-  def report_error(e = $!)
+  def report_error( e = $! )
     @state.error_recovery and return
     @state.syntax_errors += 1
     @state.error_recovery = true
@@ -371,87 +368,95 @@ class BaseRecognizer
   # message text using +error_header+ and +error_message+,
   # and calls +emit_error_message+ to write the error
   # message out to some source
-  def display_recognition_error(e = $!)
-    header = error_header(e)
-    message = error_message(e)
-    emit_error_message("#{header} #{message}")
+  def display_recognition_error( e = $! )
+    header = error_header( e )
+    message = error_message( e )
+    emit_error_message( "#{ header } #{ message }" )
   end
   
   # used to construct an appropriate error message
   # based on the specific type of error and the
   # error's attributes
-  def error_message(e = $!)
+  def error_message( e = $! )
     case e
-    when Error::UnwantedToken
-      token_name = token_name(e.expecting)
-      "extraneous input #{token_error_display(e.unexpected_token)} expecting #{token_name}"
-    when Error::MissingToken
-      token_name = token_name(e.expecting)
-      "missing #{token_name} at #{token_error_display(e.symbol)}"
-    when Error::MismatchedToken
-      token_name = token_name(e.expecting)
-      "mismatched input #{token_error_display(e.symbol)} expecting #{token_name}"
-    when Error::MismatchedTreeNode
-      token_name = token_name(e.expecting)
-      "mismatched tree node: #{e.symbol} expecting #{token_name}"
-    when Error::NoViableAlternative
-      "no viable alternative at input " << token_error_display(e.symbol)
-    when Error::MismatchedSet
+    when UnwantedToken
+      token_name = token_name( e.expecting )
+      "extraneous input #{ token_error_display( e.unexpected_token ) } expecting #{ token_name }"
+    when MissingToken
+      token_name = token_name( e.expecting )
+      "missing #{ token_name } at #{ token_error_display( e.symbol ) }"
+    when MismatchedToken
+      token_name = token_name( e.expecting )
+      "mismatched input #{ token_error_display( e.symbol ) } expecting #{ token_name }"
+    when MismatchedTreeNode
+      token_name = token_name( e.expecting )
+      "mismatched tree node: #{ e.symbol } expecting #{ token_name }"
+    when NoViableAlternative
+      "no viable alternative at input " << token_error_display( e.symbol )
+    when MismatchedSet
       "mismatched input %s expecting set %s" %
-        [token_error_display(e.symbol), e.expecting.inspect]
-    when Error::MismatchedNotSet
+        [ token_error_display( e.symbol ), e.expecting.inspect ]
+    when MismatchedNotSet
       "mismatched input %s expecting set %s" %
-        [token_error_display(e.symbol), e.expecting.inspect]
-    when Error::FailedPredicate
-      "rule %s failed predicate: { %s }?" % [e.rule_name, e.predicate_text]
+        [ token_error_display( e.symbol ), e.expecting.inspect ]
+    when FailedPredicate
+      "rule %s failed predicate: { %s }?" % [ e.rule_name, e.predicate_text ]
     else e.message
     end
   end
   
+  # 
   # used to add a tag to the error message that indicates
   # the location of the input stream when the error
   # occurred
-  def error_header(e = $!)
+  # 
+  def error_header( e = $! )
     e.location
   end
   
+  # 
   # formats a token object appropriately for inspection
   # within an error message
-  def token_error_display(token)
-    unless text = token.text
-      if token.type == EOF then text = '<EOF>'
-      elsif name = token_name(token.type) rescue false
-        text = "<#{name}>"
-      elsif token.respond_to?(:name) then text = "<#{token.name}>"
-      else "<#{token.type}>"
-      end
+  # 
+  def token_error_display( token )
+    unless text = token.text || ( token.source_text rescue nil )
+      text =
+        case
+        when token.type == EOF then '<EOF>'
+        when name = token_name( token.type ) rescue nil then "<#{ name }>"
+        when token.respond_to?( :name ) then "<#{ token.name }>"
+        else "<#{ token.type }>"
+        end
     end
     return text.inspect
   end
   
+  # 
   # Write the error report data out to some source. By default,
   # the error message is written to $stderr
-  def emit_error_message(message)
-    @error_output.puts(message) if @error_output
+  # 
+  def emit_error_message( message )
+    @error_output.puts( message ) if @error_output
   end
   
   ##############################################################################################
   ###################################### Error Recovery ########################################
   ##############################################################################################
-  def recover(error = $!)
+  
+  def recover( error = $! )
     @state.last_error_index == @input.index and @input.consume
     @state.last_error_index = @input.index
     
     follow_set = compute_error_recovery_set
     
-    resync { consume_until(follow_set) }
+    resync { consume_until( follow_set ) }
   end
   
   def resync
     begin_resync
-    value = yield
+    return( yield )
+  ensure
     end_resync
-    return(value)
   end
   
   # overridable hook method that is executed at the start of the
@@ -504,9 +509,9 @@ class BaseRecognizer
   # that rule is pushed on a stack.  Here are the various "local"
   # follow sets:
   # 
-  #   FOLLOW(b1_in_a) = FIRST(']') = ']'
-  #   FOLLOW(b2_in_a) = FIRST(')') = ')'
-  #   FOLLOW(c_in_b) = FIRST('^') = '^'
+  #   FOLLOW( b1_in_a ) = FIRST( ']' ) = ']'
+  #   FOLLOW( b2_in_a ) = FIRST( ')' ) = ')'
+  #   FOLLOW( c_in_b ) = FIRST( '^' ) = '^'
   # 
   # Upon erroneous input "[]", the call chain is
   # 
@@ -515,7 +520,7 @@ class BaseRecognizer
   # and, hence, the follow context stack is:
   # 
   #   depth  local follow set     after call to rule
-  #     0         \<EOF>                   a (from main())
+  #     0         \<EOF>                   a (from main( ) )
   #     1          ']'                     b
   #     3          '^'                     c
   # 
@@ -563,44 +568,56 @@ class BaseRecognizer
   # Like Grosch I implemented local FOLLOW sets that are combined
   # at run-time upon error to avoid overhead during parsing.
   def compute_error_recovery_set
-    combine_follows(false)
+    combine_follows( false )
   end
-
-  def recover_from_mismatched_token(type, follow)
-    if mismatch_is_unwanted_token?(type)
-      err = UnwantedToken(type)
+  
+  def recover_from_mismatched_token( type, follow )
+    if mismatch_is_unwanted_token?( type )
+      err = UnwantedToken( type )
       
-      begin_resync
-      @input.consume
-      end_resync
+      resync do 
+        @input.consume
+      end
       
-      report_error(err)
+      report_error( err )
       
-      matched_symbol = current_input_symbol
+      matched_symbol = current_symbol
       @input.consume
       return matched_symbol
     end
     
-    if mismatch_is_missing_token?(follow)
-      inserted = missing_symbol(err, type, follow)
-      err = MissingToken(type, inserted)
+    if mismatch_is_missing_token?( follow )
+      inserted = missing_symbol( err, type, follow )
+      err = MissingToken( type, inserted )
       
-      report_error(err)
+      report_error( err )
       return inserted
     end
     
-    err = MismatchedToken(type)
-    raise err
+    raise MismatchedToken( type )
   end
   
-  def recover_from_mismatched_set(e, follow)
-    if mismatch_is_missing_token?(follow)
-      report_error(e)
-      return missing_symbol(e, INVALID_TOKEN_TYPE, follow)
+  def recover_from_mismatched_set( e, follow )
+    if mismatch_is_missing_token?( follow )
+      report_error( e )
+      return missing_symbol( e, INVALID_TOKEN_TYPE, follow )
     end
     raise e
   end
   
+  def recover_from_mismatched_element( e, follow )
+    follow.nil? and return false
+    if follow.include?( EOR_TOKEN_TYPE )
+      viable_tokens = compute_context_sensitive_rule_follow
+      follow = ( follow | viable_tokens ) - Set[ EOR_TOKEN_TYPE ]
+    end
+    if follow.include?( @input.peek )
+      report_error( e )
+      return true
+    end
+    return false
+  end
+  
   # Conjure up a missing token during error recovery.
   # 
   # The recognizer attempts to recover from single missing
@@ -619,41 +636,32 @@ class BaseRecognizer
   # a CommonToken of the appropriate type. The text will be the token.
   # If you change what tokens must be created by the lexer,
   # override this method to create the appropriate tokens.
-  def missing_symbol(error, expected_token_type, follow)
+  def missing_symbol( error, expected_token_type, follow )
     return nil
   end
   
-  def recover_from_mismatched_element(e, follow)
-    follow.nil? and return false
-    if follow.include?(EOR_TOKEN_TYPE)
-      viable_tokens = compute_context_sensitive_rule_follow()
-      follow = (follow | viable_tokens) - Set.new([EOR_TOKEN_TYPE])
-    end
-    if follow.include?(@input.peek)
-      report_error(e)
-      return true
-    end
-    return false
-  end
-  
-  def mismatch_is_unwanted_token?(type)
-    @input.peek(2) == type
+  def mismatch_is_unwanted_token?( type )
+    @input.peek( 2 ) == type
   end
   
-  def mismatch_is_missing_token?(follow)
+  def mismatch_is_missing_token?( follow )
     follow.nil? and return false
-    if follow.include?(EOR_TOKEN_TYPE)
+    if follow.include?( EOR_TOKEN_TYPE )
       viable_tokens = compute_context_sensitive_rule_follow
       follow = follow | viable_tokens
       
-      follow.delete(EOR_TOKEN_TYPE) unless @state.following.empty?
+      follow.delete( EOR_TOKEN_TYPE ) unless @state.following.empty?
     end
-    if follow.include?(@input.peek) or follow.include?(EOR_TOKEN_TYPE)
+    if follow.include?( @input.peek ) or follow.include?( EOR_TOKEN_TYPE )
       return true
     end
     return false
   end
   
+  def syntax_errors?
+    ( error_count = @state.syntax_errors ) > 0 and return( error_count )
+  end
+  
   # factor out what to do upon token mismatch so
   # tree parsers can behave differently.
   #
@@ -666,7 +674,8 @@ class BaseRecognizer
     @state.syntax_errors
   end
   
-  # Compute the context-sensitive FOLLOW set for current rule.
+  # 
+  # Compute the context-sensitive +FOLLOW+ set for current rule.
   # This is set of token types that can follow a specific rule
   # reference given a specific call chain.  You get the set of
   # viable tokens that can possibly come next (look depth 1)
@@ -717,17 +726,18 @@ class BaseRecognizer
   # the viable next token set, then you know there is most likely
   # a missing token in the input stream.  "Insert" one by just not
   # throwing an exception.
+  # 
   def compute_context_sensitive_rule_follow
-    combine_follows(true)
+    combine_follows true
   end
-
-  def combine_follows(exact)
+  
+  def combine_follows( exact )
     follow_set = Set.new
     @state.following.each_with_index.reverse_each do |local_follow_set, index|
       follow_set |= local_follow_set
       if exact
-        if local_follow_set.include?(EOR_TOKEN_TYPE)
-          follow_set.delete(EOR_TOKEN_TYPE) if index > 0
+        if local_follow_set.include?( EOR_TOKEN_TYPE )
+          follow_set.delete( EOR_TOKEN_TYPE ) if index > 0
         else
           break
         end
@@ -736,6 +746,7 @@ class BaseRecognizer
     return follow_set
   end
   
+  # 
   # Match needs to return the current input symbol, which gets put
   # into the label for the associated token ref; e.g., x=ID.  Token
   # and tree parsers need to return different objects. Rather than test
@@ -744,28 +755,39 @@ class BaseRecognizer
   # input symbol is.
   # 
   # This is ignored for lexers.
-  def current_input_symbol
+  # 
+  def current_symbol
     @input.look
   end
   
-  # Consume tokens until one matches the given token or token set
   # 
-  # tokenTypes can be a single token type or a set of token types
-  def consume_until(token_types)
-    token_types.is_a?(Set) or token_types = Set.new(token_types.to_a)
+  # Consume input symbols until one matches a type within types
+  # 
+  # types can be a single symbol type or a set of symbol types
+  # 
+  def consume_until( types )
+    types.is_a?( Set ) or types = Set[ *types ]
     type = @input.peek
-    until type == EOF or token_types.include?(type)
+    until type == EOF or types.include?( type )
       @input.consume
       type = @input.peek
     end
-    return(type)
+    return( type )
+  end
+  
+  # 
+  # Returns true if the recognizer is currently in a decision for which
+  # backtracking has been enabled
+  # 
+  def backtracking?
+    @state.backtracking > 0
   end
   
   def backtracking_level
     @state.backtracking
   end
   
-  def backtracking_level=(n)
+  def backtracking_level=( n )
     @state.backtracking = n
   end
   
@@ -779,20 +801,21 @@ class BaseRecognizer
       end
     return success
   ensure
-    @input.rewind(start)
+    @input.rewind( start )
     @state.backtracking -= 1
   end
   
-  def syntactic_predicate?(name)
-    backtrack { send(name) }
+  def syntactic_predicate?( name )
+    backtrack { send name }
   end
   
   alias backtracking backtracking_level
   alias backtracking= backtracking_level=
   
   def rule_memoization( rule, start_index )
-    @state.rule_memory[ rule ] ||= Hash.new( MEMO_RULE_UNKNOWN )
-    @state.rule_memory[ rule ][ start_index ]
+    @state.rule_memory.fetch( rule ) do
+      @state.rule_memory[ rule ] = Hash.new( MEMO_RULE_UNKNOWN )
+    end[ start_index ]
   end
   
   def already_parsed_rule?( rule )
@@ -807,40 +830,45 @@ class BaseRecognizer
     return true
   end
   
-  def memoize(rule, start_index, success)
-    stop_index = success ? (@input.index - 1) : MEMO_RULE_FAILED
-    memo = @state.rule_memory[rule] and memo[start_index] = stop_index
+  def memoize( rule, start_index, success )
+    stop_index = success ? @input.index - 1 : MEMO_RULE_FAILED
+    memo = @state.rule_memory[ rule ] and memo[ start_index ] = stop_index
   end
   
-  def trace_in(rule_name, rule_index, input_symbol)
+  def trace_in( rule_name, rule_index, input_symbol )
     @error_output.printf( "--> enter %s on %s", rule_name, input_symbol )
-    @state.backtracking > 0 and @error_output.printf(
+    @state.backtracking > 0 and @error_output.printf( 
       " (in backtracking mode: depth = %s)", @state.backtracking
     )
-    @error_output.print("\n")
+    @error_output.print( "\n" )
   end
   
-  def trace_out(rule_name, rule_index, input_symbol)
-    @error_output.printf("<-- exit %s on %s", rule_name, input_symbol)
-    @state.backtracking > 0 and @error_output.printf(
+  def trace_out( rule_name, rule_index, input_symbol )
+    @error_output.printf( "<-- exit %s on %s", rule_name, input_symbol )
+    @state.backtracking > 0 and @error_output.printf( 
       " (in backtracking mode: depth = %s)", @state.backtracking
     )
-    @error_output.print("\n")
+    @error_output.print( "\n" )
   end
   
-  private
+private
   
   def initialize_dfas
     # do nothing
   end
 end
 
+
+# constant alias for compatibility with older versions of the
+# runtime library
+BaseRecognizer = Recognizer
+
 =begin rdoc ANTLR3::Lexer
 
 = Lexer
 
 Lexer is the default superclass of all lexers generated by ANTLR. The class
-tailors the core functionality provided by BaseRecognizer to the task of
+tailors the core functionality provided by Recognizer to the task of
 matching patterns in the text input and breaking the input into tokens.
 
 == About Lexers
@@ -899,19 +927,19 @@ demonstrates the typical setup for using ANTLR parsers and lexers in Ruby.
   
   source = "some hypothetical source code"
   input = ANTLR3::StringStream.new(source, :file => 'blah-de-blah.hyp')
-  lexer = Hypothetical::Lexer.new(input)
-  tokens = ANTLR3::CommonTokenStream.new(lexer)
-  parser = Hypothetical::Parser.new(tokens)
+  lexer = Hypothetical::Lexer.new( input )
+  tokens = ANTLR3::CommonTokenStream.new( lexer )
+  parser = Hypothetical::Parser.new( tokens )
   
   # if you're using the standard streams, ANTLR3::StringStream and
   # ANTLR3::CommonTokenStream, you can write the same process 
   # shown above more succinctly:
   
   lexer  = Hypothetical::Lexer.new("some hypothetical source code", :file => 'blah-de-blah.hyp')
-  parser = Hypothetical::Parser.new(lexer)
+  parser = Hypothetical::Parser.new( lexer )
 
 =end
-class Lexer < BaseRecognizer
+class Lexer < Recognizer
   include TokenSource
   @token_class = CommonToken
   
@@ -919,36 +947,31 @@ class Lexer < BaseRecognizer
     @default_rule ||= :token!
   end
   
-  def self.main(argv = ARGV, options = {})
-    if argv.is_a?(::Hash) then argv, options = ARGV, argv end
-    main = ANTLR3::Main::LexerMain.new(self, options)
-    block_given? ? yield(main) : main.execute(argv)
+  def self.main( argv = ARGV, options = {} )
+    if argv.is_a?( ::Hash ) then argv, options = ARGV, argv end
+    main = ANTLR3::Main::LexerMain.new( self, options )
+    block_given? ? yield( main ) : main.execute( argv )
   end
   
   def self.associated_parser
     @associated_parser ||= begin
       @grammar_home and @grammar_home::Parser
     rescue NameError
-      grammar_name = @grammar_home.name.split("::").last
+      grammar_name = @grammar_home.name.split( "::" ).last
       begin
-        require "#{grammar_name}Parser"
+        require "#{ grammar_name }Parser"
         @grammar_home::Parser
       rescue LoadError, NameError
       end
     end
   end
 
-  def initialize(input, options = {})
+  def initialize( input, options = {} )
     super( options )
-    @input =
-      case input
-      when ::String then StringStream.new(input, options)
-      when ::IO then FileStream.new(input, options)
-      else input
-      end
+    @input = cast_input( input, options )
   end
   
-  def current_input_symbol
+  def current_symbol
     nil
   end
   
@@ -965,16 +988,16 @@ class Lexer < BaseRecognizer
         token!
         
         case token = @state.token
-        when nil then return(emit())
+        when nil then return( emit )
         when SKIP_TOKEN then next
         else
           return token
         end
       rescue NoViableAlternative => re
-        report_error(re)
-        recover(re)
+        report_error( re )
+        recover( re )
       rescue Error::RecognitionError => re
-        report_error(re)
+        report_error( re )
       end
     end
   end
@@ -989,7 +1012,7 @@ class Lexer < BaseRecognizer
     self.to_a
   end
   
-  def char_stream=(input)
+  def char_stream=( input )
     @input = nil
     reset()
     @input = input
@@ -1005,14 +1028,14 @@ class Lexer < BaseRecognizer
     return token
   end
   
-  def match(expected)
+  def match( expected )
     case expected
     when String
       expected.each_byte do |char|
         unless @input.peek == char
           @state.backtracking > 0 and raise BacktrackingFailed
-          error = MismatchedToken(char)
-          recover(error)
+          error = MismatchedToken( char )
+          recover( error )
           raise error
         end
         @input.consume()
@@ -1020,8 +1043,8 @@ class Lexer < BaseRecognizer
     else # single integer character
       unless @input.peek == expected
         @state.backtracking > 0 and raise BacktrackingFailed
-        error = MismatchedToken(expected)
-        recover(error)
+        error = MismatchedToken( expected )
+        recover( error )
         raise error
       end
       @input.consume
@@ -1033,14 +1056,14 @@ class Lexer < BaseRecognizer
     @input.consume
   end
   
-  def match_range(min, max)
+  def match_range( min, max )
     char = @input.peek
-    if char.between?(min, max) then @input.consume
+    if char.between?( min, max ) then @input.consume
     else
       @state.backtracking > 0 and raise BacktrackingFailed
-      error = MismatchedRange(min.chr, max.chr)
-      recover(error)
-      raise(error)
+      error = MismatchedRange( min.chr, max.chr )
+      recover( error )
+      raise( error )
     end
     return true
   end
@@ -1059,40 +1082,40 @@ class Lexer < BaseRecognizer
   
   def text
     @state.text and return @state.text
-    @input.substring(@state.token_start_position, character_index - 1)
+    @input.substring( @state.token_start_position, character_index - 1 )
   end
   
-  def text=(text)
+  def text=( text )
     @state.text = text
   end
   
-  def report_error(e)
-    display_recognition_error(e)
+  def report_error( e )
+    display_recognition_error( e )
   end
   
-  def error_message(e)
-    char = character_error_display(e.symbol) rescue nil
+  def error_message( e )
+    char = character_error_display( e.symbol ) rescue nil
     case e
     when Error::MismatchedToken
-      expecting = character_error_display(e.expecting)
-      "mismatched character #{char}; expecting #{expecting}"
+      expecting = character_error_display( e.expecting )
+      "mismatched character #{ char }; expecting #{ expecting }"
     when Error::NoViableAlternative
-      "no viable alternative at character #{char}"
+      "no viable alternative at character #{ char }"
     when Error::EarlyExit
-      "required (...)+ loop did not match anything at character #{char}"
+      "required ( ... )+ loop did not match anything at character #{ char }"
     when Error::MismatchedNotSet
-      "mismatched character %s; expecting set %p" % [char, e.expecting]
+      "mismatched character %s; expecting set %p" % [ char, e.expecting ]
     when Error::MismatchedSet
-      "mismatched character %s; expecting set %p" % [char, e.expecting]
+      "mismatched character %s; expecting set %p" % [ char, e.expecting ]
     when Error::MismatchedRange
-      a = character_error_display(e.min)
-      b = character_error_display(e.max)
-      "mismatched character %s; expecting set %s..%s" % [char, a, b]
+      a = character_error_display( e.min )
+      b = character_error_display( e.max )
+      "mismatched character %s; expecting set %s..%s" % [ char, a, b ]
     else super
     end
   end
   
-  def character_error_display(char)
+  def character_error_display( char )
     case char
     when EOF then '<EOF>'
     when Integer then char.chr.inspect
@@ -1100,29 +1123,39 @@ class Lexer < BaseRecognizer
     end
   end
   
-  def recover(re)
+  def recover( re )
     @input.consume
   end
   
+  alias input= char_stream=
   
 private
   
-  def trace_in(rule_name, rule_index)
+  def cast_input( input, options )
+    case input
+    when CharacterStream then input
+    when ::String then StringStream.new( input, options )
+    when ::IO     then FileStream.new( input, options )
+    else input
+    end
+  end
+  
+  def trace_in( rule_name, rule_index )
     if symbol = @input.look and symbol != EOF then symbol = symbol.inspect
     else symbol = '<EOF>' end
-    input_symbol = "#{symbol} @ line #{line} / col #{column}"
-    super(rule_name, rule_index, input_symbol)
+    input_symbol = "#{ symbol } @ line #{ line } / col #{ column }"
+    super( rule_name, rule_index, input_symbol )
   end
   
-  def trace_out(rule_name, rule_index)
+  def trace_out( rule_name, rule_index )
     if symbol = @input.look and symbol != EOF then symbol = symbol.inspect
     else symbol = '<EOF>' end
-    input_symbol = "#{symbol} @ line #{line} / col #{column}"
-    super(rule_name, rule_index, input_symbol)
+    input_symbol = "#{ symbol } @ line #{ line } / col #{ column }"
+    super( rule_name, rule_index, input_symbol )
   end
   
-  def create_token(&b)
-    if block_given? then super(&b)
+  def create_token( &b )
+    if block_given? then super( &b )
     else
       super do |t|
         t.input = @input
@@ -1144,7 +1177,7 @@ end
 = Parser
 
 Parser is the default base class of ANTLR-generated parser classes. The class
-tailors the functionality provided by BaseRecognizer to the task of parsing.
+tailors the functionality provided by Recognizer to the task of parsing.
 
 == About Parsing
 
@@ -1171,56 +1204,56 @@ otherwise within the grammar options. The generated code will provide a method
 for each parser rule defined in the ANTLR grammar, as well as any other
 customized member attributes and methods specified in the source grammar.
 
-This class does not override much of the functionality in BaseRecognizer, and
-thus the API closely mirrors BaseRecognizer.
+This class does not override much of the functionality in Recognizer, and
+thus the API closely mirrors Recognizer.
 
 =end
-class Parser < BaseRecognizer
-  def self.main(argv = ARGV, options = {})
-    if argv.is_a?(::Hash) then argv, options = ARGV, argv end
-    main = ANTLR3::Main::ParserMain.new(self, options)
-    block_given? ? yield(main) : main.execute(argv)
+class Parser < Recognizer
+  def self.main( argv = ARGV, options = {} )
+    if argv.is_a?( ::Hash ) then argv, options = ARGV, argv end
+    main = ANTLR3::Main::ParserMain.new( self, options )
+    block_given? ? yield( main ) : main.execute( argv )
   end
   
   def self.associated_lexer
     @associated_lexer ||= begin
       @grammar_home and @grammar_home::Lexer
     rescue NameError
-      grammar_name = @grammar_home.name.split("::").last
+      grammar_name = @grammar_home.name.split( "::" ).last
       begin
-        require "#{grammar_name}Lexer"
+        require "#{ grammar_name }Lexer"
         @grammar_home::Lexer
       rescue LoadError, NameError
       end
     end
   end
   
+  
   def initialize( input, options = {} )
     super( options )
     @input = nil
     reset
-    input = cast_input( input, options ) unless TokenStream === input
-    @input = input
+    @input = cast_input( input, options )
   end
   
-  def missing_symbol(error, expected_type, follow)
+  def missing_symbol( error, expected_type, follow )
     current = @input.look
-    current = @input.look(-1) if current == ANTLR3::EOF_TOKEN
+    current = @input.look( -1 ) if current == ANTLR3::EOF_TOKEN
     t =
       case
       when current && current != ANTLR3::EOF_TOKEN then current.clone
       when @input.token_class then @input.token_class.new
-      else (create_token rescue CommonToken.new)
+      else ( create_token rescue CommonToken.new )
       end
     
     t.type = expected_type
-    name = t.name.gsub(/(^<)|(>$)/,'')
-    t.text = "<missing #{name}>"
+    name = t.name.gsub( /(^<)|(>$)/,'' )
+    t.text = "<missing #{ name }>"
     t.channel = DEFAULT_CHANNEL
-    return(t)
+    return( t )
   end
   
-  def token_stream=(input)
+  def token_stream=( input )
     @input = nil
     reset
     @input = input
@@ -1231,18 +1264,20 @@ class Parser < BaseRecognizer
     @input.source_name
   end
   
+  
 private
   
-  def trace_in(rule_name, rule_index)
+  def trace_in( rule_name, rule_index )
     super( rule_name, rule_index, @input.look.inspect )
   end
   
-  def trace_out(rule_name, rule_index)
+  def trace_out( rule_name, rule_index )
     super( rule_name, rule_index, @input.look.inspect )
   end
   
   def cast_input( input, options )
     case input
+    when TokenStream then input
     when TokenSource then CommonTokenStream.new( input, options )
     when IO, String, CharacterStream
       if lexer_class = self.class.associated_lexer
@@ -1257,7 +1292,7 @@ private
     else
       # assume it's a stream if it at least implements peek and consume
       unless input.respond_to?( :peek ) and input.respond_to?( :consume )
-        raise ArgumentError, Util.tidy(<<-END, true)
+        raise ArgumentError, Util.tidy( <<-END, true )
         | #{ self.class } requires a token stream as input, but
         | #{ input.inspect } was provided
         END