antlr3 1.8.0 → 1.8.2

data/lib/antlr3/profile.rb

@@ -1,6 +1,37 @@
 #!/usr/bin/ruby
 # encoding: utf-8
 
+=begin LICENSE
+
+[The "BSD licence"]
+Copyright (c) 2009-2010 Kyle Yetter
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+=end
+
 module ANTLR3
 module Profile
 =begin rdoc ANTLR3::Profile::ParserEvents
@@ -52,9 +83,6 @@ class DataSet < ::Array
 end
 
 
-
-
-
 unless const_defined?( :Profile )
   Profile = Struct.new( 
     :grammar_file, :parser_class, :top_rule,
@@ -231,7 +259,7 @@ class Profiler
   
   def enter_decision( decision_number )
     @decision_level += 1
-    starting_look_index = @parser.token_stream.index
+    starting_look_index = @parser.input.index
     @look_stack << starting_look_index
   end
 
@@ -262,7 +290,7 @@ class Profiler
   def look( i, token )
     in_decision? or return
     starting_index = look_stack.last
-    input = @parser.token_stream
+    input = @parser.input
     this_ref_index = input.index
     num_hidden = input.tokens( starting_index, this_ref_index ).count { |t| t.hidden? }
     depth = i + this_ref_index - starting_index - num_hidden
@@ -284,7 +312,7 @@ class Profiler
   end
   
   def terminate
-    input = @parser.token_stream
+    input = @parser.input
     hidden_tokens = input.select { |token| token.hidden? }
     @profile.hidden_tokens = hidden_tokens.length
     @profile.tokens = input.tokens.length

data/lib/antlr3/recognizers.rb

@@ -124,6 +124,52 @@ end
 
 end # unless const_defined?( :RecognizerSharedState )
 
+=begin rdoc ANTLR3::Recognizer
+
+= Scope
+
+Scope is used to represent instances of ANTLR's various attribute scopes.
+It is identical to Ruby's built-in Struct class, but it takes string
+attribute declarations from the ANTLR grammar as parameters, and overrides
+the #initialize method to set the default values if any are present in
+the scope declaration.
+
+  Block = Scope.new( "name", "depth = 0", "variables = {}" )
+  Block.new                    # => #<struct Block name=nil, depth=0, variables={}>
+  Block.new( "function" )      # => #<struct Block name="function", depth=0, variables={}>
+  Block.new( 'a', 1, :x => 3 ) # => #<struct Block name="a", depth=1, variables={ :x => 3 }>
+
+=end
+
+class Scope < ::Struct
+  def self.new( *declarations, &body )
+    names = []
+    defaults = {}
+    for decl in declarations
+      name, default = decl.to_s.split( /\s*=\s*/, 2 )
+      names << ( name = name.to_sym )
+      default and defaults[ name ] = default
+    end
+    super( *names ) do
+      
+      # If no defaults, leave the initialize method the same as
+      # the struct's default initialize for speed. Otherwise,
+      # overwrite the initialize to populate with default values.
+      unless defaults.empty?
+        parameters = names.map do | name |
+          "#{ name } = " << defaults.fetch( name, 'nil' )
+        end.join( ', ' )
+        class_eval( <<-END )
+          def initialize( #{ parameters } )
+            super( #{ names.join( ', ' ) } )
+          end
+        END
+      end
+      
+      body and class_eval( &body )
+    end
+  end
+end
 
 =begin rdoc ANTLR3::Recognizer
 
@@ -158,7 +204,6 @@ class Recognizer
   
   # inherited class methods and hooks
   class << self
-    
     attr_reader :grammar_file_name,
                 :antlr_version,
                 :antlr_version_string,
@@ -262,6 +307,10 @@ class Recognizer
       return false
     end
     
+    def Scope( *declarations, &body )
+      Scope.new( *declarations, &body )
+    end
+    
     def token_class
       @token_class ||= begin
         self::Token            rescue

data/lib/antlr3/streams.rb

@@ -173,46 +173,51 @@ module Stream
   abstract :consume
   
   ##
-  # :method: peek(k=1)
+  # :method: peek( k = 1 )
   # used to quickly retreive the object of interest to a recognizer at lookahead
   # position specified by <tt>k</tt> (such as integer value of a character or an
   # integer token type)
   abstract :peek
   
   ##
-  # :method: look(k=1)
+  # :method: look( k = 1 )
   # used to retreive the full object of interest at lookahead position specified
   # by <tt>k</tt> (such as a character string or a token structure)
   abstract :look
   
   ##
   # :method: mark
-  # TODO: document
+  # saves the current position for the purposes of backtracking and
+  # returns a value to pass to #rewind at a later time
   abstract :mark
   
   ##
   # :method: index
-  # TODO: document
+  # returns the current position of the stream
   abstract :index
   
   ##
-  # :method: rewind(marker=last_marker)
-  # TODO: document
+  # :method: rewind( marker = last_marker )
+  # restores the stream position using the state information previously saved
+  # by the given marker
   abstract :rewind
   
   ##
-  # :method: release(marker = last_marker)
-  # TODO: document
+  # :method: release( marker = last_marker )
+  # clears the saved state information associated with the given marker value
   abstract :release
   
   ##
-  # :method: seek(position)
-  # TODO: document
+  # :method: seek( position )
+  # move the stream to the given absolute index given by +position+
   abstract :seek
   
-  # TODO: document
+  ##
+  # the total number of symbols in the stream
   attr_reader :size
-  # TODO: document
+  
+  ##
+  # indicates an identifying name for the stream -- usually the file path of the input
   attr_accessor :source_name
 end
 
@@ -252,7 +257,6 @@ module CharacterStream
   include Stream
   extend ClassMacros
   include Constants
-  #EOF = -1
   
   ##
   # :method: substring(start,stop)
@@ -328,8 +332,8 @@ module TokenStream
   abstract :to_s
   
   ##
-  # :method: at
-  # TODO: document
+  # :method: at( i )
+  # return the stream symbol at index +i+
   abstract :at
 end
 

data/lib/antlr3/streams/rewrite.rb

@@ -42,6 +42,10 @@ TokenRewriteStream is a specialized form of CommonTokenStream that provides simp
 
 class TokenRewriteStream < CommonTokenStream
 
+  unless defined?( RewriteOperation )
+    RewriteOperation = Struct.new( :stream, :location, :text )
+  end
+
 =begin rdoc ANTLR3::TokenRewriteStream::RewriteOperation
 
 RewiteOperation objects represent some particular editing command that should
@@ -59,29 +63,34 @@ define specific implementations of stream edits.
 
 =end
 
-unless defined?( RewriteOperation )
-  RewriteOperation = Struct.new( :stream, :location, :text )
-end
-
-class RewriteOperation
-  extend ClassMacros
-  @operation_name = ''
-  class << self
-    # TODO: document
-    attr_reader :operation_name
-  end
-  abstract :execute
-  
-  # TODO: document
-  def name
-    self.class.operation_name
+  class RewriteOperation
+    extend ClassMacros
+    @operation_name = ''
+    
+    class << self
+      ##
+      # the printable name of operations represented by the class -- used for inspection
+      attr_reader :operation_name
+    end
+    
+    ##
+    # :method: execute( buffer )
+    # run the rewrite operation represented by this object and append the output to +buffer+
+    abstract :execute
+    
+    ##
+    # return the name of this operation as set by its class
+    def name
+      self.class.operation_name
+    end
+    
+    ##
+    # return a compact, readable representation of this operation
+    def inspect
+      return "(%s @ %p : %p)" % [ name, location, text ]
+    end
   end
   
-  # TODO: document
-  def inspect
-    return "(%s @ %p : %p)" % [ name, location, text ]
-  end
-end
 
 =begin rdoc ANTLR3::TokenRewriteStream::InsertBefore
 
@@ -91,21 +100,21 @@ add string <tt>op.text</tt> to the rewrite output immediately before adding the
 text content of the token at index <tt>op.index</tt>
 
 =end
-
-class InsertBefore < RewriteOperation
-  @operation_name = 'insert-before'.freeze
   
-  alias index  location
-  alias index= location=
-  
-  def execute( buffer )
-    buffer << text.to_s
-    token = stream[ location ]
-    buffer << token.text.to_s if token
-    return location + 1
+  class InsertBefore < RewriteOperation
+    @operation_name = 'insert-before'.freeze
+    
+    alias index  location
+    alias index= location=
+    
+    def execute( buffer )
+      buffer << text.to_s
+      token = stream[ location ]
+      buffer << token.text.to_s if token
+      return location + 1
+    end
   end
-end
-
+  
 =begin rdoc ANTLR3::TokenRewriteStream::Replace
 
 Represents rewrite operation:
@@ -114,34 +123,36 @@ add text <tt>op.text</tt> to the rewrite buffer in lieu of the text of tokens
 indexed within the range <tt>op.index .. op.last_index</tt>
 
 =end
-
-class Replace < RewriteOperation
-  @operation_name = 'replace'.freeze
-  def initialize( stream, location, text )
-    super( stream, nil, text )
-    self.location = location
-  end
   
-  def location=( val )
-    case val
-    when Range then super( val )
-    else
-      val = val.to_i
-      super( val..val )
+  class Replace < RewriteOperation
+    
+    @operation_name = 'replace'.freeze
+    
+    def initialize( stream, location, text )
+      super( stream, nil, text )
+      self.location = location
     end
+    
+    def location=( val )
+      case val
+      when Range then super( val )
+      else
+        val = val.to_i
+        super( val..val )
+      end
+    end
+    
+    def execute( buffer )
+      buffer << text.to_s unless text.nil?
+      return( location.end + 1 )
+    end
+    
+    def index
+      location.first
+    end
+    
   end
   
-  def execute( buffer )
-    buffer << text.to_s unless text.nil?
-    return( location.end + 1 )
-  end
-  
-  def index
-    location.first
-  end
-  
-end
-
 =begin rdoc ANTLR3::TokenRewriteStream::Delete
 
 Represents rewrite operation:
@@ -150,203 +161,204 @@ skip over the tokens indexed within the range <tt>op.index .. op.last_index</tt>
 and do not add any text to the rewrite buffer
 
 =end
-
-class Delete < Replace
-  @operation_name = 'delete'.freeze
-  def initialize( stream, location )
-    super( stream, location, nil )
-  end
-end
-
-class RewriteProgram
-  def initialize( stream, name = nil )
-    @stream = stream
-    @name = name
-    @operations = []
-  end
   
-  def replace( *range_arguments )
-    range, text = cast_range( range_arguments, 1 )
+  class Delete < Replace
+    @operation_name = 'delete'.freeze
     
-    op = Replace.new( @stream, range, text )
-    @operations << op
-    return op
-  end
-  
-  def insert_before( index, text )
-    index = index.to_i
-    index < 0 and index += @stream.length
-    op = InsertBefore.new( @stream, index, text )
-    @operations << op
-    return op
-  end
-  
-  def insert_after( index, text )
-    index = index.to_i
-    index < 0 and index += @stream.length
-    op = InsertBefore.new( @stream, index + 1, text )
-    @operations << op
-    return op
+    def initialize( stream, location )
+      super( stream, location, nil )
+    end
   end
   
-  def delete( *range_arguments )
-    range, = cast_range( range_arguments )
-    op = Delete.new( @stream, range )
-    @operations << op
-    return op
-  end
-
-  def reduce
-    operations = @operations.reverse
-    reduced = []
+  class RewriteProgram
+    def initialize( stream, name = nil )
+      @stream = stream
+      @name = name
+      @operations = []
+    end
     
-    until operations.empty?
-      operation = operations.shift
-      location = operation.location
+    def replace( *range_arguments )
+      range, text = cast_range( range_arguments, 1 )
       
-      case operation
-      when Replace
-        operations.delete_if do |prior_operation|
-          prior_location = prior_operation.location
-          
-          case prior_operation
-          when InsertBefore
-            location.include?( prior_location )
-          when Replace
-            if location.covers?( prior_location )
-              true
-            elsif location.overlaps?( prior_location )
-              conflict!( operation, prior_operation )
+      op = Replace.new( @stream, range, text )
+      @operations << op
+      return op
+    end
+    
+    def insert_before( index, text )
+      index = index.to_i
+      index < 0 and index += @stream.length
+      op = InsertBefore.new( @stream, index, text )
+      @operations << op
+      return op
+    end
+    
+    def insert_after( index, text )
+      index = index.to_i
+      index < 0 and index += @stream.length
+      op = InsertBefore.new( @stream, index + 1, text )
+      @operations << op
+      return op
+    end
+    
+    def delete( *range_arguments )
+      range, = cast_range( range_arguments )
+      op = Delete.new( @stream, range )
+      @operations << op
+      return op
+    end
+  
+    def reduce
+      operations = @operations.reverse
+      reduced = []
+      
+      until operations.empty?
+        operation = operations.shift
+        location = operation.location
+        
+        case operation
+        when Replace
+          operations.delete_if do |prior_operation|
+            prior_location = prior_operation.location
+            
+            case prior_operation
+            when InsertBefore
+              location.include?( prior_location )
+            when Replace
+              if location.covers?( prior_location )
+                true
+              elsif location.overlaps?( prior_location )
+                conflict!( operation, prior_operation )
+              end
             end
           end
-        end
-      when InsertBefore
-        operations.delete_if do |prior_operation|
-          prior_location = prior_operation.location
-          
-          case prior_operation
-          when InsertBefore
-            if prior_location == location
-              operation.text += prior_operation.text
-              true
-            end
-          when Replace
-            if location == prior_location.first
-              prior_operation.text = operation.text << prior_operation.text.to_s
-              operation = nil
-              break( false )
-            elsif prior_location.include?( location )
-              conflict!( operation, prior_operation )
+        when InsertBefore
+          operations.delete_if do |prior_operation|
+            prior_location = prior_operation.location
+            
+            case prior_operation
+            when InsertBefore
+              if prior_location == location
+                operation.text += prior_operation.text
+                true
+              end
+            when Replace
+              if location == prior_location.first
+                prior_operation.text = operation.text << prior_operation.text.to_s
+                operation = nil
+                break( false )
+              elsif prior_location.include?( location )
+                conflict!( operation, prior_operation )
+              end
             end
           end
         end
+        
+        reduced.unshift( operation ) if operation
       end
       
-      reduced.unshift( operation ) if operation
+      @operations.replace( reduced )
+      
+      @operations.inject( {} ) do |map, operation|
+        other_operaiton = map[ operation.index ] and
+          ANTLR3.bug!( Util.tidy( <<-END ) % [ self.class, operation, other_operaiton ] )
+          | %s#reduce! should have left only one operation per index,
+          | but %p conflicts with %p
+          END
+        map[ operation.index ] = operation
+        map
+      end
     end
     
-    @operations.replace( reduced )
-    
-    @operations.inject( {} ) do |map, operation|
-      other_operaiton = map[ operation.index ] and
-        ANTLR3.bug!( Util.tidy( <<-END ) % [ self.class, operation, other_operaiton ] )
-        | %s#reduce! should have left only one operation per index,
-        | but %p conflicts with %p
-        END
-      map[ operation.index ] = operation
-      map
-    end
-  end
-  
-  def execute( *range_arguments )
-    if range_arguments.empty?
-      range = 0 ... @stream.length
-    else
-      range, = cast_range( range_arguments )
+    def execute( *range_arguments )
+      if range_arguments.empty?
+        range = 0 ... @stream.length
+      else
+        range, = cast_range( range_arguments )
+      end
+      
+      output = ''
+      
+      tokens = @stream.tokens
+      
+      operations = reduce
+      
+      cursor = range.first
+      while range.include?( cursor )
+        if operation = operations.delete( cursor )
+          cursor = operation.execute( output )
+        else
+          token = tokens[ cursor ]
+          output << token.text if token
+          cursor += 1
+        end
+      end
+      if operation = operations.delete( cursor ) and
+         operation.is_a?( InsertBefore )
+        # catch edge 'insert-after' operations
+        operation.execute( output )
+      end
+      
+      return output
     end
     
-    output = ''
+    def clear
+      @operations.clear
+    end
     
-    tokens = @stream.tokens
+    def undo( number_of_operations = 1 )
+      @operations.pop( number_of_operations )
+    end
     
-    operations = reduce
+    def conflict!( current, previous )
+      message = 'operation %p overlaps with previous operation %p' % [ current, previous ]
+      raise( RangeError, message, caller )
+    end
     
-    cursor = range.first
-    while range.include?( cursor )
-      if operation = operations.delete( cursor )
-        cursor = operation.execute( output )
-      else
-        token = tokens[ cursor ]
-        output << token.text if token
-        cursor += 1
+    def cast_range( args, extra = 0 )
+      single, pair = extra + 1, extra + 2
+      case check_arguments( args, single, pair )
+      when single
+        loc = args.shift
+        
+        if loc.is_a?( Range )
+          first, last = loc.first.to_i, loc.last.to_i
+          loc.exlude_end? and last -= 1
+          return cast_range( args.unshift( first, last ), extra )
+        else
+          loc = loc.to_i
+          return cast_range( args.unshift( loc, loc ), extra )
+        end
+      when pair
+        first, last = args.shift( 2 ).map! { |arg| arg.to_i }
+        if first < 0 and last < 0
+          first += @stream.length
+          last += @stream.length
+        else
+          last < 0 and last += @stream.length
+          first = first.at_least( 0 )
+        end
+        return( args.unshift( first .. last ) )
       end
     end
-    if operation = operations.delete( cursor ) and
-       operation.is_a?( InsertBefore )
-      # catch edge 'insert-after' operations
-      operation.execute( output )
-    end
     
-    return output
-  end
-  
-  def clear
-    @operations.clear
-  end
-  
-  def undo( number_of_operations = 1 )
-    @operations.pop( number_of_operations )
-  end
-  
-  def conflict!( current, previous )
-    message = 'operation %p overlaps with previous operation %p' % [ current, previous ]
-    raise( RangeError, message, caller )
-  end
-  
-  def cast_range( args, extra = 0 )
-    single, pair = extra + 1, extra + 2
-    case check_arguments( args, single, pair )
-    when single
-      loc = args.shift
-      
-      if loc.is_a?( Range )
-        first, last = loc.first.to_i, loc.last.to_i
-        loc.exlude_end? and last -= 1
-        return cast_range( args.unshift( first, last ), extra )
-      else
-        loc = loc.to_i
-        return cast_range( args.unshift( loc, loc ), extra )
+    def check_arguments( args, min, max )
+      n = args.length
+      if n < min
+        raise ArgumentError,
+          "wrong number of arguments (#{ args.length } for #{ min })",
+          caller
+      elsif n > max
+        raise ArgumentError,
+          "wrong number of arguments (#{ args.length } for #{ max })",
+          caller
+      else return n
       end
-    when pair
-      first, last = args.shift( 2 ).map! { |arg| arg.to_i }
-      if first < 0 and last < 0
-        first += @stream.length
-        last += @stream.length
-      else
-        last < 0 and last += @stream.length
-        first = first.at_least( 0 )
-      end
-      return( args.unshift( first .. last ) )
-    end
-  end
-  
-  def check_arguments( args, min, max )
-    n = args.length
-    if n < min
-      raise ArgumentError,
-        "wrong number of arguments (#{ args.length } for #{ min })",
-        caller
-    elsif n > max
-      raise ArgumentError,
-        "wrong number of arguments (#{ args.length } for #{ max })",
-        caller
-    else return n
     end
+    
+    private :conflict!, :cast_range, :check_arguments
   end
-  
-  private :conflict!, :cast_range, :check_arguments
-end
-  
+    
   attr_reader :programs
 
   def initialize( token_source, options = {} )