antlr3 1.8.5 → 1.8.8

data/Manifest.txt

@@ -1,167 +1,169 @@
 bin/antlr4ruby
-lib/antlr3/main.rb
-lib/antlr3/recognizers.rb
-lib/antlr3/modes/ast-builder.rb
-lib/antlr3/modes/filter.rb
-lib/antlr3/streams/rewrite.rb
-lib/antlr3/streams/interactive.rb
-lib/antlr3/debug.rb
-lib/antlr3/dot.rb
-lib/antlr3/streams.rb
-lib/antlr3/debug/record-event-listener.rb
-lib/antlr3/debug/rule-tracer.rb
-lib/antlr3/debug/socket.rb
-lib/antlr3/debug/trace-event-listener.rb
-lib/antlr3/debug/event-hub.rb
-lib/antlr3/version.rb
-lib/antlr3/error.rb
-lib/antlr3/tree.rb
+lib/antlr3.rb
+lib/antlr3/constants.rb
+lib/antlr3/test/core-extensions.rb
+lib/antlr3/test/grammar.rb
 lib/antlr3/test/functional.rb
 lib/antlr3/test/call-stack.rb
-lib/antlr3/test/grammar.rb
-lib/antlr3/test/core-extensions.rb
-lib/antlr3/template.rb
-lib/antlr3/dfa.rb
-lib/antlr3/profile.rb
-lib/antlr3/template/parameter.rb
+lib/antlr3/debug/socket.rb
+lib/antlr3/debug/event-hub.rb
+lib/antlr3/debug/trace-event-listener.rb
+lib/antlr3/debug/rule-tracer.rb
+lib/antlr3/debug/record-event-listener.rb
+lib/antlr3/template/group-file-lexer.rb
 lib/antlr3/template/group-file.rb
+lib/antlr3/template/parameter.rb
 lib/antlr3/template/group-file-parser.rb
-lib/antlr3/template/group-file-lexer.rb
-lib/antlr3/tree/visitor.rb
-lib/antlr3/tree/debug.rb
+lib/antlr3/template.rb
 lib/antlr3/tree/wizard.rb
+lib/antlr3/tree/debug.rb
+lib/antlr3/tree/visitor.rb
+lib/antlr3/modes/ast-builder.rb
+lib/antlr3/modes/filter.rb
+lib/antlr3/debug.rb
+lib/antlr3/main.rb
+lib/antlr3/util.rb
 lib/antlr3/token.rb
+lib/antlr3/dfa.rb
+lib/antlr3/streams.rb
+lib/antlr3/error.rb
+lib/antlr3/version.rb
+lib/antlr3/recognizers.rb
+lib/antlr3/dot.rb
+lib/antlr3/profile.rb
+lib/antlr3/tree.rb
+lib/antlr3/streams/interactive.rb
+lib/antlr3/streams/rewrite.rb
+lib/antlr3/streams/unicode.rb
 lib/antlr3/task.rb
-lib/antlr3/constants.rb
-lib/antlr3/util.rb
-lib/antlr3.rb
+test/unit/test-streams.rb
+test/unit/test-template.rb
+test/unit/test-unicode-stream.rb
 test/unit/test-dfa.rb
+test/unit/test-exceptions.rb
+test/unit/test-scheme.rb
+test/unit/test-tree-wizard.rb
+test/unit/sample-input/teststreams.input2
 test/unit/sample-input/file-stream-1
 test/unit/sample-input/template-group
-test/unit/sample-input/teststreams.input2
-test/unit/test-tree-wizard.rb
-test/unit/test-streams.rb
-test/unit/test-recognizers.rb
 test/unit/test-trees.rb
-test/unit/test-exceptions.rb
+test/unit/test-recognizers.rb
 test/unit/test-scope.rb
-test/unit/test-scheme.rb
-test/unit/test-template.rb
-test/functional/delegation/import.rb
-test/functional/lexer/xml.rb
+test/functional/main/main-scripts.rb
+test/functional/template-output/template-output.rb
+test/functional/tree-parser/basic.rb
 test/functional/lexer/properties.rb
-test/functional/lexer/nuances.rb
-test/functional/lexer/basic.rb
 test/functional/lexer/filter-mode.rb
 test/functional/lexer/syn-pred.rb
-test/functional/debugging/rule-tracing.rb
-test/functional/debugging/profile-mode.rb
-test/functional/debugging/debug-mode.rb
-test/functional/ast-output/hetero-nodes.rb
-test/functional/ast-output/construction.rb
-test/functional/ast-output/rewrites.rb
-test/functional/ast-output/tree-rewrite.rb
-test/functional/ast-output/auto-ast.rb
-test/functional/main/main-scripts.rb
-test/functional/tree-parser/basic.rb
+test/functional/lexer/basic.rb
+test/functional/lexer/nuances.rb
+test/functional/lexer/xml.rb
+test/functional/delegation/import.rb
+test/functional/parser/scopes.rb
+test/functional/parser/properties.rb
+test/functional/parser/basic.rb
 test/functional/parser/predicates.rb
+test/functional/parser/ll-star.rb
 test/functional/parser/backtracking.rb
-test/functional/parser/properties.rb
+test/functional/parser/calc.rb
 test/functional/parser/nuances.rb
-test/functional/parser/rule-methods.rb
-test/functional/parser/basic.rb
 test/functional/parser/actions.rb
-test/functional/parser/calc.rb
-test/functional/parser/ll-star.rb
-test/functional/parser/scopes.rb
-test/functional/template-output/template-output.rb
-test/functional/token-rewrite/via-parser.rb
+test/functional/parser/rule-methods.rb
+test/functional/debugging/debug-mode.rb
+test/functional/debugging/profile-mode.rb
+test/functional/debugging/rule-tracing.rb
 test/functional/token-rewrite/basic.rb
+test/functional/token-rewrite/via-parser.rb
+test/functional/ast-output/tree-rewrite.rb
+test/functional/ast-output/construction.rb
+test/functional/ast-output/rewrites.rb
+test/functional/ast-output/hetero-nodes.rb
+test/functional/ast-output/auto-ast.rb
+templates/Ruby.stg
 templates/ASTDbg.stg
 templates/AST.stg
-templates/Ruby.stg
 templates/ST.stg
 templates/Dbg.stg
-templates/ASTParser.stg
 templates/Support.stg
 templates/ASTTreeParser.stg
+templates/ASTParser.stg
 samples/CPP.g
 samples/ANTLRv3Grammar.g
 samples/JavaScript.g
-samples/standard/island-grammar/island.rb
-samples/standard/island-grammar/Simple.g
-samples/standard/island-grammar/Javadoc.g
-samples/standard/island-grammar/input
-samples/standard/island-grammar/output
-samples/standard/tweak/tweak.rb
-samples/standard/tweak/input
-samples/standard/tweak/output
-samples/standard/tweak/Tweak.g
-samples/standard/scopes/input
+samples/standard/fuzzy/FuzzyJava.g
+samples/standard/fuzzy/output
+samples/standard/fuzzy/input
+samples/standard/fuzzy/fuzzy.rb
+samples/standard/fuzzy/fuzzy.py
+samples/standard/scopes/scopes.rb
 samples/standard/scopes/output
+samples/standard/scopes/input
 samples/standard/scopes/SymbolTable.g
-samples/standard/scopes/scopes.rb
+samples/standard/LL-star/LLStar.g
+samples/standard/LL-star/ll-star.rb
+samples/standard/LL-star/output
+samples/standard/LL-star/input
 samples/standard/dynamic-scope/DynamicScopes.g
 samples/standard/dynamic-scope/dynamic-scopes.rb
-samples/standard/dynamic-scope/input
 samples/standard/dynamic-scope/output
-samples/standard/hoisted-predicates/hoisted-predicates.rb
-samples/standard/hoisted-predicates/HoistedPredicates.g
-samples/standard/hoisted-predicates/input
-samples/standard/hoisted-predicates/output
-samples/standard/fuzzy/fuzzy.rb
-samples/standard/fuzzy/input
-samples/standard/fuzzy/output
-samples/standard/fuzzy/FuzzyJava.g
-samples/standard/fuzzy/fuzzy.py
-samples/standard/treeparser/LangDumpDecl.g
-samples/standard/treeparser/Lang.g
-samples/standard/treeparser/treeparser.rb
-samples/standard/treeparser/input
-samples/standard/treeparser/output
-samples/standard/LL-star/LLStar.g
-samples/standard/LL-star/input
-samples/standard/LL-star/output
-samples/standard/LL-star/ll-star.rb
-samples/standard/java/Java.g
+samples/standard/dynamic-scope/input
 samples/standard/java/java.rb
-samples/standard/java/input
+samples/standard/java/Java.g
 samples/standard/java/output
-samples/standard/rakefile
-samples/standard/xml/README
-samples/standard/xml/xml.rb
-samples/standard/xml/XML.g
-samples/standard/xml/input
-samples/standard/xml/output
-samples/standard/calc/Calculator.g
-samples/standard/calc/Calculator.py
-samples/standard/calc/Calculator.rb
+samples/standard/java/input
 samples/standard/C/C.g
-samples/standard/C/c.rb
 samples/standard/C/C__testrig.st
-samples/standard/C/input
-samples/standard/C/output
+samples/standard/C/c.rb
 samples/standard/C/C.tokens
-samples/standard/cminus/bytecode.group
-samples/standard/cminus/CMinus.g
-samples/standard/cminus/cminus.rb
-samples/standard/cminus/input
+samples/standard/C/output
+samples/standard/C/input
+samples/standard/python/PythonTokenSource.rb
+samples/standard/python/Python.g
+samples/standard/python/output
+samples/standard/python/input
+samples/standard/python/python.rb
+samples/standard/treeparser/Lang.g
+samples/standard/treeparser/LangDumpDecl.g
+samples/standard/treeparser/treeparser.rb
+samples/standard/treeparser/output
+samples/standard/treeparser/input
+samples/standard/cminus/java.group
 samples/standard/cminus/python.group
 samples/standard/cminus/output
-samples/standard/cminus/java.group
-samples/standard/simplecTreeParser/SimpleC.g
+samples/standard/cminus/input
+samples/standard/cminus/CMinus.g
+samples/standard/cminus/bytecode.group
+samples/standard/cminus/cminus.rb
+samples/standard/calc/Calculator.rb
+samples/standard/calc/Calculator.py
+samples/standard/calc/Calculator.g
+samples/standard/xml/README
+samples/standard/xml/output
+samples/standard/xml/input
+samples/standard/xml/XML.g
+samples/standard/xml/xml.rb
+samples/standard/rakefile
+samples/standard/simplecTreeParser/simplec.rb
 samples/standard/simplecTreeParser/SimpleCWalker.g
-samples/standard/simplecTreeParser/input
 samples/standard/simplecTreeParser/output
-samples/standard/simplecTreeParser/simplec.rb
-samples/standard/python/PythonTokenSource.rb
-samples/standard/python/python.rb
-samples/standard/python/Python.g
-samples/standard/python/input
-samples/standard/python/output
-README.txt
+samples/standard/simplecTreeParser/input
+samples/standard/simplecTreeParser/SimpleC.g
+samples/standard/island-grammar/Simple.g
+samples/standard/island-grammar/output
+samples/standard/island-grammar/input
+samples/standard/island-grammar/Javadoc.g
+samples/standard/island-grammar/island.rb
+samples/standard/tweak/tweak.rb
+samples/standard/tweak/output
+samples/standard/tweak/input
+samples/standard/tweak/Tweak.g
+samples/standard/hoisted-predicates/HoistedPredicates.g
+samples/standard/hoisted-predicates/hoisted-predicates.rb
+samples/standard/hoisted-predicates/output
+samples/standard/hoisted-predicates/input
 ANTLR-LICENSE.txt
 History.txt
+README.txt
 java/antlr-full-3.2.1.jar
 java/RubyTarget.java
 rakefile

data/lib/antlr3/debug/socket.rb

@@ -48,7 +48,8 @@ class EventSocketProxy
         port, host = Socket.unpack_sockaddr_in( addr )
         log!( "Accepted connection from %s:%s\n", host, port )
         
-        @connection.setsockopt( Socket::SOL_TCP, Socket::TCP_NODELAY, 1 )
+        proto_number = ( Socket::SOL_TCP rescue 6 )
+        @connection.setsockopt( proto_number, Socket::TCP_NODELAY, 1 )
         
         write( 'ANTLR %s', PROTOCOL_VERSION )
         write( 'grammar %p', @grammar_file_name )

data/lib/antlr3/dot.rb

@@ -67,8 +67,9 @@ module DOT
       @__template__.result( binding )
     end
   end
+  
   class TreeGenerator
-    TREE_TEMPLATE = ERB.new( Util.tidy( <<-END ) )
+    TREE_TEMPLATE = ERB.new( Util.tidy( <<-END ).chomp )
     | digraph {
     |   ordering=out;
     |   ranksep=.4;
@@ -80,11 +81,11 @@ module DOT
     | }
     END
     
-    NODE_TEMPLATE = ERB.new( Util.tidy( <<-END ) )
+    NODE_TEMPLATE = ERB.new( Util.tidy( <<-END ).chomp )
     | <%= @name %> [label="<%= @text %>"];
     END
     
-    EDGE_TEMPLATE = ERB.new( Util.tidy( <<-END ) )
+    EDGE_TEMPLATE = ERB.new( Util.tidy( <<-END ).chomp )
     | <%= @parent %> -> <%= @child %>; // "<%= @parent_text %>" -> "<%= @child_text %>"
     END
     
@@ -94,22 +95,19 @@ module DOT
     end
     
     def initialize
-      @node_number = 0
-      @node_to_number_map = Hash.new do |map, node|
-        map[ node ] = @node_number
-        @node_number += 1
-        @node_number - 1
+      @node_registry = Hash.new do |map, id|
+        map[ id ] = map.length
       end
     end
     
     def to_dot( tree, adaptor = nil, tree_template = TREE_TEMPLATE,
                edge_template = EDGE_TEMPLATE )
       adaptor ||= AST::CommonTreeAdaptor.new
-      @node_number = 0
+      @node_registry.clear
       tree_template = Context.new( tree_template, :nodes => [], :edges => [] )
       define_nodes( tree, adaptor, tree_template )
       
-      @node_number = 0
+      @node_registry.clear
       define_edges( tree, adaptor, tree_template, edge_template )
       return tree_template.to_s
     end
@@ -117,45 +115,42 @@ module DOT
     def define_nodes( tree, adaptor, tree_template, known_nodes = nil )
       known_nodes ||= Set.new
       tree.nil? and return
-      n = adaptor.child_count( tree )
-      n == 0 and return
-      number = node_number( tree )
+      
+      adaptor.empty?( tree ) and return
+      
+      number = @node_registry[ tree.__id__ ]
       unless known_nodes.include?( number )
-        parent_node_template = node_template_for( adaptor, child )
+        parent_node_template = node_template_for( adaptor, tree )
         tree_template[ :nodes ] << parent_node_template
         known_nodes.add( number )
       end
       
-      n.times do |index|
-        child = adaptor.child_of( tree, index )
-        number = @node_to_number_map[ child ]
+      adaptor.each_child( tree ) do | child |
+        number = @node_registry[ child.__id__ ]
         unless known_nodes.include?( number )
           node_template = node_template_for( adaptor, child )
           tree_template[ :nodes ] << node_template
           known_nodes.add( number )
         end
-        
-        define_nodes( child, adaptor, tree_template, edge_template )
+        define_nodes( child, adaptor, tree_template, known_nodes )
       end
     end
     
     def define_edges( tree, adaptor, tree_template, edge_template )
-      tree.nil? or return
+      tree.nil? or adaptor.empty?( tree ) and return
       
-      n = adaptor.child_count( tree )
-      n == 0 and return
-      
-      parent_name = 'n%i' % @node_to_number_map[ tree ]
+      parent_name = 'n%i' % @node_registry[ tree.__id__ ]
       parent_text = adaptor.text_of( tree )
-      n.times do |index|
-        child = adaptor.child_of( tree, index )
+      
+      adaptor.each_child( tree ) do | child |
         child_text = adaptor.text_of( child )
-        child_name = 'n%i' % @node_to_number_map[ tree ]
-        edge_template = Context.new( edge_template,
+        child_name = 'n%i' % @node_registry[ child.__id__ ]
+        
+        edge = Context.new( edge_template,
           :parent => parent_name, :child => child_name,
           :parent_text => parent_text, :child_text => child_text
         )
-        tree_template[ :edges ] << edge_template
+        tree_template[ :edges ] << edge
         define_edges( child, adaptor, tree_template, edge_template )
       end
     end
@@ -163,7 +158,7 @@ module DOT
     def node_template_for( adaptor, tree )
       text = adaptor.text_of( tree )
       node_template = Context.new( NODE_TEMPLATE )
-      unique_name = 'n%i' % @node_to_number_map[ tree ]
+      unique_name = 'n%i' % @node_registry[ tree.__id__ ]
       node_template[ :name ] = unique_name
       text and text = text.gsub( /"/, '\\"' )
       node_template[ :text ] = text

data/lib/antlr3/streams/unicode.rb

@@ -0,0 +1,80 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+=begin      ::about::
+author:     Kyle Yetter <kcy5b@yahoo.com>
+created on: October 22, 2010
+purpose:    (program | library | utility script | ?)
+summary:    
+loads:      files required by this
+autoloads:  autoload entries in this (e.g. YAML(yaml))
+=end
+
+module ANTLR3
+
+class UnicodeStream < StringStream
+  PACK_MASK = 'U*'.freeze
+  
+  # creates a new StringStream object where +data+ is the string data to stream.
+  # accepts the following options in a symbol-to-value hash:
+  #
+  # [:file or :name] the (file) name to associate with the stream; default: <tt>'(string)'</tt>
+  # [:line] the initial line number; default: +1+
+  # [:column] the initial column number; default: +0+
+  # 
+  def initialize( data, options = {} )    # for 1.8
+    @string = data.to_s
+    @string.equal?( data ) and @string = @string.clone
+    @string.freeze
+    @data = @string.unpack( PACK_MASK )
+    @position = options.fetch :position, 0
+    @line = options.fetch :line, 1
+    @column = options.fetch :column, 0
+    @markers = []
+    @name ||= options[ :file ] || options[ :name ] # || '(string)'
+    mark
+  end
+  
+  #
+  # identical to #peek, except it returns the character value as a String
+  # 
+  def look( k = 1 )                        # for 1.8
+    k == 0 and return nil
+    k += 1 if k < 0
+    
+    index = @position + k - 1
+    index.between?( 0, @data.length - 1) or return nil
+    
+    @data[ index, 1 ].pack( PACK_MASK )
+  end
+  
+  #
+  # return a substring around the stream cursor at a distance +k+
+  # if <tt>k >= 0</tt>, return the next k characters
+  # if <tt>k < 0</tt>, return the previous <tt>|k|</tt> characters
+  # 
+  def through( k )
+    if k >= 0 then @data[ @position, k ].pack( PACK_MASK ) else
+      start = ( @position + k ).at_least( 0 ) # start cannot be negative or index will wrap around
+      @data[ start ... @position ].pack( PACK_MASK )
+    end
+  end
+  
+  #
+  # return the string slice between position +start+ and +stop+
+  # 
+  def substring( start, stop )
+    @data[ start, stop - start + 1 ].pack( PACK_MASK )
+  end
+  
+  
+  #
+  # identical to String#[]
+  # 
+  def []( start, *args )
+    @data[ start, *args ].pack( PACK_MASK )
+  end
+  
+end
+
+end

data/lib/antlr3/version.rb

@@ -20,7 +20,7 @@ module ANTLR3
   # 
   MAJOR_VERSION = 1
   MINOR_VERSION = 8
-  PATCH_VERSION = 5
+  PATCH_VERSION = 8
   VERSION = [ MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION ]
   VERSION_STRING = VERSION.join( '.' ).freeze
   

data/rakefile

@@ -15,45 +15,46 @@ Hoe.spec "antlr3" do
   )
   
   self.test_globs = [
+    "test/unit/test-streams.rb",
+    "test/unit/test-template.rb",
+    "test/unit/test-unicode-stream.rb",
     "test/unit/test-dfa.rb",
+    "test/unit/test-exceptions.rb",
+    "test/unit/test-scheme.rb",
     "test/unit/test-tree-wizard.rb",
-    "test/unit/test-streams.rb",
-    "test/unit/test-recognizers.rb",
     "test/unit/test-trees.rb",
-    "test/unit/test-exceptions.rb",
+    "test/unit/test-recognizers.rb",
     "test/unit/test-scope.rb",
-    "test/unit/test-scheme.rb",
-    "test/unit/test-template.rb",
-    "test/functional/delegation/import.rb",
-    "test/functional/lexer/xml.rb",
+    "test/functional/main/main-scripts.rb",
+    "test/functional/template-output/template-output.rb",
+    "test/functional/tree-parser/basic.rb",
     "test/functional/lexer/properties.rb",
-    "test/functional/lexer/nuances.rb",
-    "test/functional/lexer/basic.rb",
     "test/functional/lexer/filter-mode.rb",
     "test/functional/lexer/syn-pred.rb",
-    "test/functional/debugging/rule-tracing.rb",
-    "test/functional/debugging/profile-mode.rb",
-    "test/functional/debugging/debug-mode.rb",
-    "test/functional/ast-output/hetero-nodes.rb",
-    "test/functional/ast-output/construction.rb",
-    "test/functional/ast-output/rewrites.rb",
-    "test/functional/ast-output/tree-rewrite.rb",
-    "test/functional/ast-output/auto-ast.rb",
-    "test/functional/main/main-scripts.rb",
-    "test/functional/tree-parser/basic.rb",
+    "test/functional/lexer/basic.rb",
+    "test/functional/lexer/nuances.rb",
+    "test/functional/lexer/xml.rb",
+    "test/functional/delegation/import.rb",
+    "test/functional/parser/scopes.rb",
+    "test/functional/parser/properties.rb",
+    "test/functional/parser/basic.rb",
     "test/functional/parser/predicates.rb",
+    "test/functional/parser/ll-star.rb",
     "test/functional/parser/backtracking.rb",
-    "test/functional/parser/properties.rb",
+    "test/functional/parser/calc.rb",
     "test/functional/parser/nuances.rb",
-    "test/functional/parser/rule-methods.rb",
-    "test/functional/parser/basic.rb",
     "test/functional/parser/actions.rb",
-    "test/functional/parser/calc.rb",
-    "test/functional/parser/ll-star.rb",
-    "test/functional/parser/scopes.rb",
-    "test/functional/template-output/template-output.rb",
+    "test/functional/parser/rule-methods.rb",
+    "test/functional/debugging/debug-mode.rb",
+    "test/functional/debugging/profile-mode.rb",
+    "test/functional/debugging/rule-tracing.rb",
+    "test/functional/token-rewrite/basic.rb",
     "test/functional/token-rewrite/via-parser.rb",
-    "test/functional/token-rewrite/basic.rb"
+    "test/functional/ast-output/tree-rewrite.rb",
+    "test/functional/ast-output/construction.rb",
+    "test/functional/ast-output/rewrites.rb",
+    "test/functional/ast-output/hetero-nodes.rb",
+    "test/functional/ast-output/auto-ast.rb"
   ]
   
 end

data/templates/Ruby.stg

@@ -1459,4 +1459,4 @@ placeAction(scope, name) ::= <<
 <endif>
 >>
 
-runtimeLibraryVersion() ::= "1.8.5"
+runtimeLibraryVersion() ::= "1.8.8"

data/test/unit/test-streams.rb

@@ -199,8 +199,6 @@ class TestStringStream < Test::Unit::TestCase
 end
 
 class TestFileStream < Test::Unit::TestCase
-  
-  
   def test_no_encoding
     
     path = File.join(File.dirname(__FILE__), 'sample-input/file-stream-1')
@@ -224,20 +222,11 @@ class TestFileStream < Test::Unit::TestCase
     @stream.peek(1).should == ?a.ord
   end
   
-  def test_encoded
-    
-  end
 end
 
-class TestInputStream < Test::Unit::TestCase
-  def test_no_encoding
-    
-  end
-  
-  def test_encoded
-    
-  end
-end
+
+
+
 
 class TestCommonTokenStream < Test::Unit::TestCase
   class MockSource
@@ -418,43 +407,4 @@ class TestCommonTokenStream < Test::Unit::TestCase
 end
 
 
-__END__
-teststreams.py                                | LN  | STATUS
-----------------------------------------------+-----+--------------
-class TestStringStream(unittest.TestCase)     | 009 | [x]
-  def testSize(self)                          | 012 | [x]
-  def testIndex(self)                         | 020 | [x]
-  def testConsume(self)                       | 028 | [x]
-  def testReset(self)                         | 079 | [x]
-  def testLA(self)                            | 094 | [x]
-  def testSubstring(self)                     | 111 | [x]
-  def testSeekForward(self)                   | 122 | [x]
-  def testMark(self)                          | 150 | [x]
-  def testReleaseLast(self)                   | 167 | [x]
-  def testReleaseNested(self)                 | 186 | [x]
-  def testRewindLast(self)                    | 204 | [x]
-  def testRewindNested(self)                  | 223 | [x]
-class TestFileStream(unittest.TestCase)       | 245 | [o]
-  def testNoEncoding(self)                    | 249 | [x]
-  def testEncoded(self)                       | 272 | [ ]
-class TestInputStream(unittest.TestCase)      | 296 | [ ]
-  def testNoEncoding(self)                    | 299 | [ ]
-  def testEncoded(self)                       | 322 | [ ]
-class TestCommonTokenStream(unittest.TestCase)| 345 | [ ]
-  def setUp(self)                             | 348 | [x]
-  def testInit(self)                          | 369 | [x]
-  def testSetTokenSource(self)                | 376 | [x]
-  def testLTEmptySource(self)                 | 385 | [x]
-  def testLT1(self)                           | 394 | [x]
-  def testLT1WithHidden(self)                 | 407 | [x]
-  def testLT2BeyondEnd(self)                  | 424 | [x]
-  def testLTNegative(self)                    | 442 | [x]
-  def testLB1(self)                           | 461 | [x]
-  def testLTZero(self)                        | 479 | [x]
-  def testLBBeyondBegin(self)                 | 496 | [x]
-  def testFillBuffer(self)                    | 523 | [x]
-  def testConsume(self)                       | 551 | [x]
-  def testSeek(self)                          | 579 | [x]
-  def testMarkRewind(self)                    | 604 | [x]
-  def testToString(self)                      | 631 | [x]
 

data/test/unit/test-unicode-stream.rb

@@ -0,0 +1,186 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3'
+require 'antlr3/streams/unicode'
+
+require 'test/unit'
+require 'spec'
+
+
+include ANTLR3
+
+class TestUnicodeStream < Test::Unit::TestCase
+  def setup
+    #   «    a   »    \n  9   8   °    \n
+    # [ 171, 97, 187, 10, 57, 56, 176, 10]
+    @stream = UnicodeStream.new( "«a»\n98°\n" )
+  end
+  
+  def test_size
+    @stream.size.should == 8
+  end
+  
+  def test_index
+    @stream.index.should == 0
+  end
+  
+  def test_consume
+    @stream.consume # «
+    @stream.index.should == 1
+    @stream.column.should == 1
+    @stream.line.should == 1
+    
+    @stream.consume  # a
+    @stream.index.should == 2
+    @stream.column.should == 2
+    @stream.line.should == 1
+    
+    @stream.consume # »
+    @stream.index.should == 3
+    @stream.column.should == 3
+    @stream.line.should == 1
+    
+    @stream.consume #  \n
+    @stream.index.should == 4
+    @stream.column.should == 0
+    @stream.line.should == 2
+    
+    @stream.consume # 9
+    @stream.index.should == 5
+    @stream.column.should == 1
+    @stream.line.should == 2
+    
+    @stream.consume # 8
+    @stream.index.should == 6
+    @stream.column.should == 2
+    @stream.line.should == 2
+    
+    @stream.consume # °
+    @stream.index.should == 7
+    @stream.column.should == 3
+    @stream.line.should == 2
+    
+    @stream.consume # \n
+    @stream.index.should == 8
+    @stream.column.should == 0
+    @stream.line.should == 3
+    
+    @stream.consume # EOF
+    @stream.index.should == 8
+    @stream.column.should == 0
+    @stream.line.should == 3
+    
+    @stream.consume # EOF
+    @stream.index.should == 8
+    @stream.column.should == 0
+    @stream.line.should == 3
+  end
+  
+  def test_reset
+    2.times { @stream.consume }
+    @stream.reset
+    @stream.index.should == 0
+    @stream.line.should == 1
+    @stream.column.should == 0
+    @stream.peek(1).should == 171 # code point for «
+  end
+  
+  def test_look
+    @stream.look(1).should == '«'
+    @stream.look(2).should == 'a'
+    @stream.look(3).should == "»"
+    @stream.peek(1).should == 171
+    @stream.peek(2).should == 97
+    @stream.peek(3).should == 187
+    
+    6.times { @stream.consume }
+    @stream.look(1).should == '°'
+    @stream.look(2).should == "\n"
+    @stream.look(3).should be_nil
+    @stream.peek(1).should == 176
+    @stream.peek(2).should == 10
+    @stream.peek(3).should == EOF
+  end
+  
+  def test_substring
+    @stream.substring(0,0).should == '«'
+    @stream.substring(0,1).should == '«a'
+    @stream.substring(0,10).should == "«a»\n98°\n"
+    @stream.substring(3,6).should == "\n98°"
+  end
+  
+  def test_seek_forward
+    @stream.seek(4)
+    @stream.index.should == 4
+    @stream.line.should == 2
+    @stream.column.should == 0
+    @stream.peek(1).should == 57
+  end
+  
+  def test_mark
+    @stream.seek(4)
+    marker = @stream.mark
+    marker.should == 1
+    
+    2.times { @stream.consume }
+    marker = @stream.mark
+    
+    marker.should == 2
+  end
+  
+  def test_release_last
+    @stream.seek(4)
+    marker1 = @stream.mark
+    
+    2.times { @stream.consume }
+    marker2 = @stream.mark
+    
+    @stream.release
+    @stream.mark_depth.should == 2
+    @stream.release
+    @stream.mark_depth.should == 1
+  end
+  
+  def test_release_nested
+    @stream.seek(4)
+    marker1 = @stream.mark()
+    
+    @stream.consume()
+    marker2 = @stream.mark()
+    
+    @stream.consume()
+    marker3 = @stream.mark()
+    
+    @stream.release(marker2)
+    @stream.mark_depth.should == 2
+
+  end
+  
+  def test_rewind_last
+    @stream.seek(4)
+
+    marker = @stream.mark
+    @stream.consume
+    @stream.consume
+
+    @stream.rewind
+    @stream.mark_depth.should == 1
+    @stream.index.should == 4
+    @stream.line.should == 2
+    @stream.column.should == 0
+    @stream.peek(1).should == 57
+    
+  end
+  
+  def test_through
+    @stream.through( 2 ).should == '«a'
+    @stream.through( -2 ).should == ''
+    @stream.seek( 4 )
+    @stream.through( 0 ).should == ''
+    @stream.through( 1 ).should == '9'
+    @stream.through( -2 ).should == "»\n"
+    @stream.through( 5 ).should == "98°\n"
+  end
+end
+

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: antlr3
 version: !ruby/object:Gem::Version 
-  hash: 61
   prerelease: false
   segments: 
   - 1
   - 8
-  - 5
-  version: 1.8.5
+  - 8
+  version: 1.8.8
 platform: ruby
 authors: 
 - Kyle Yetter
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-22 00:00:00 -04:00
+date: 2011-01-10 00:00:00 -05:00
 default_executable: 
 dependencies: []
 
@@ -62,175 +61,177 @@ extra_rdoc_files: []
 
 files: 
 - bin/antlr4ruby
-- lib/antlr3/main.rb
-- lib/antlr3/recognizers.rb
-- lib/antlr3/modes/ast-builder.rb
-- lib/antlr3/modes/filter.rb
-- lib/antlr3/streams/rewrite.rb
-- lib/antlr3/streams/interactive.rb
-- lib/antlr3/debug.rb
-- lib/antlr3/dot.rb
-- lib/antlr3/streams.rb
-- lib/antlr3/debug/record-event-listener.rb
-- lib/antlr3/debug/rule-tracer.rb
-- lib/antlr3/debug/socket.rb
-- lib/antlr3/debug/trace-event-listener.rb
-- lib/antlr3/debug/event-hub.rb
-- lib/antlr3/version.rb
-- lib/antlr3/error.rb
-- lib/antlr3/tree.rb
+- lib/antlr3.rb
+- lib/antlr3/constants.rb
+- lib/antlr3/test/core-extensions.rb
+- lib/antlr3/test/grammar.rb
 - lib/antlr3/test/functional.rb
 - lib/antlr3/test/call-stack.rb
-- lib/antlr3/test/grammar.rb
-- lib/antlr3/test/core-extensions.rb
-- lib/antlr3/template.rb
-- lib/antlr3/dfa.rb
-- lib/antlr3/profile.rb
-- lib/antlr3/template/parameter.rb
+- lib/antlr3/debug/socket.rb
+- lib/antlr3/debug/event-hub.rb
+- lib/antlr3/debug/trace-event-listener.rb
+- lib/antlr3/debug/rule-tracer.rb
+- lib/antlr3/debug/record-event-listener.rb
+- lib/antlr3/template/group-file-lexer.rb
 - lib/antlr3/template/group-file.rb
+- lib/antlr3/template/parameter.rb
 - lib/antlr3/template/group-file-parser.rb
-- lib/antlr3/template/group-file-lexer.rb
-- lib/antlr3/tree/visitor.rb
-- lib/antlr3/tree/debug.rb
+- lib/antlr3/template.rb
 - lib/antlr3/tree/wizard.rb
+- lib/antlr3/tree/debug.rb
+- lib/antlr3/tree/visitor.rb
+- lib/antlr3/modes/ast-builder.rb
+- lib/antlr3/modes/filter.rb
+- lib/antlr3/debug.rb
+- lib/antlr3/main.rb
+- lib/antlr3/util.rb
 - lib/antlr3/token.rb
+- lib/antlr3/dfa.rb
+- lib/antlr3/streams.rb
+- lib/antlr3/error.rb
+- lib/antlr3/version.rb
+- lib/antlr3/recognizers.rb
+- lib/antlr3/dot.rb
+- lib/antlr3/profile.rb
+- lib/antlr3/tree.rb
+- lib/antlr3/streams/interactive.rb
+- lib/antlr3/streams/rewrite.rb
+- lib/antlr3/streams/unicode.rb
 - lib/antlr3/task.rb
-- lib/antlr3/constants.rb
-- lib/antlr3/util.rb
-- lib/antlr3.rb
+- test/unit/test-streams.rb
+- test/unit/test-template.rb
+- test/unit/test-unicode-stream.rb
 - test/unit/test-dfa.rb
+- test/unit/test-exceptions.rb
+- test/unit/test-scheme.rb
+- test/unit/test-tree-wizard.rb
+- test/unit/sample-input/teststreams.input2
 - test/unit/sample-input/file-stream-1
 - test/unit/sample-input/template-group
-- test/unit/sample-input/teststreams.input2
-- test/unit/test-tree-wizard.rb
-- test/unit/test-streams.rb
-- test/unit/test-recognizers.rb
 - test/unit/test-trees.rb
-- test/unit/test-exceptions.rb
+- test/unit/test-recognizers.rb
 - test/unit/test-scope.rb
-- test/unit/test-scheme.rb
-- test/unit/test-template.rb
-- test/functional/delegation/import.rb
-- test/functional/lexer/xml.rb
+- test/functional/main/main-scripts.rb
+- test/functional/template-output/template-output.rb
+- test/functional/tree-parser/basic.rb
 - test/functional/lexer/properties.rb
-- test/functional/lexer/nuances.rb
-- test/functional/lexer/basic.rb
 - test/functional/lexer/filter-mode.rb
 - test/functional/lexer/syn-pred.rb
-- test/functional/debugging/rule-tracing.rb
-- test/functional/debugging/profile-mode.rb
-- test/functional/debugging/debug-mode.rb
-- test/functional/ast-output/hetero-nodes.rb
-- test/functional/ast-output/construction.rb
-- test/functional/ast-output/rewrites.rb
-- test/functional/ast-output/tree-rewrite.rb
-- test/functional/ast-output/auto-ast.rb
-- test/functional/main/main-scripts.rb
-- test/functional/tree-parser/basic.rb
+- test/functional/lexer/basic.rb
+- test/functional/lexer/nuances.rb
+- test/functional/lexer/xml.rb
+- test/functional/delegation/import.rb
+- test/functional/parser/scopes.rb
+- test/functional/parser/properties.rb
+- test/functional/parser/basic.rb
 - test/functional/parser/predicates.rb
+- test/functional/parser/ll-star.rb
 - test/functional/parser/backtracking.rb
-- test/functional/parser/properties.rb
+- test/functional/parser/calc.rb
 - test/functional/parser/nuances.rb
-- test/functional/parser/rule-methods.rb
-- test/functional/parser/basic.rb
 - test/functional/parser/actions.rb
-- test/functional/parser/calc.rb
-- test/functional/parser/ll-star.rb
-- test/functional/parser/scopes.rb
-- test/functional/template-output/template-output.rb
-- test/functional/token-rewrite/via-parser.rb
+- test/functional/parser/rule-methods.rb
+- test/functional/debugging/debug-mode.rb
+- test/functional/debugging/profile-mode.rb
+- test/functional/debugging/rule-tracing.rb
 - test/functional/token-rewrite/basic.rb
+- test/functional/token-rewrite/via-parser.rb
+- test/functional/ast-output/tree-rewrite.rb
+- test/functional/ast-output/construction.rb
+- test/functional/ast-output/rewrites.rb
+- test/functional/ast-output/hetero-nodes.rb
+- test/functional/ast-output/auto-ast.rb
+- templates/Ruby.stg
 - templates/ASTDbg.stg
 - templates/AST.stg
-- templates/Ruby.stg
 - templates/ST.stg
 - templates/Dbg.stg
-- templates/ASTParser.stg
 - templates/Support.stg
 - templates/ASTTreeParser.stg
+- templates/ASTParser.stg
 - samples/CPP.g
 - samples/ANTLRv3Grammar.g
 - samples/JavaScript.g
-- samples/standard/island-grammar/island.rb
-- samples/standard/island-grammar/Simple.g
-- samples/standard/island-grammar/Javadoc.g
-- samples/standard/island-grammar/input
-- samples/standard/island-grammar/output
-- samples/standard/tweak/tweak.rb
-- samples/standard/tweak/input
-- samples/standard/tweak/output
-- samples/standard/tweak/Tweak.g
-- samples/standard/scopes/input
+- samples/standard/fuzzy/FuzzyJava.g
+- samples/standard/fuzzy/output
+- samples/standard/fuzzy/input
+- samples/standard/fuzzy/fuzzy.rb
+- samples/standard/fuzzy/fuzzy.py
+- samples/standard/scopes/scopes.rb
 - samples/standard/scopes/output
+- samples/standard/scopes/input
 - samples/standard/scopes/SymbolTable.g
-- samples/standard/scopes/scopes.rb
+- samples/standard/LL-star/LLStar.g
+- samples/standard/LL-star/ll-star.rb
+- samples/standard/LL-star/output
+- samples/standard/LL-star/input
 - samples/standard/dynamic-scope/DynamicScopes.g
 - samples/standard/dynamic-scope/dynamic-scopes.rb
-- samples/standard/dynamic-scope/input
 - samples/standard/dynamic-scope/output
-- samples/standard/hoisted-predicates/hoisted-predicates.rb
-- samples/standard/hoisted-predicates/HoistedPredicates.g
-- samples/standard/hoisted-predicates/input
-- samples/standard/hoisted-predicates/output
-- samples/standard/fuzzy/fuzzy.rb
-- samples/standard/fuzzy/input
-- samples/standard/fuzzy/output
-- samples/standard/fuzzy/FuzzyJava.g
-- samples/standard/fuzzy/fuzzy.py
-- samples/standard/treeparser/LangDumpDecl.g
-- samples/standard/treeparser/Lang.g
-- samples/standard/treeparser/treeparser.rb
-- samples/standard/treeparser/input
-- samples/standard/treeparser/output
-- samples/standard/LL-star/LLStar.g
-- samples/standard/LL-star/input
-- samples/standard/LL-star/output
-- samples/standard/LL-star/ll-star.rb
-- samples/standard/java/Java.g
+- samples/standard/dynamic-scope/input
 - samples/standard/java/java.rb
-- samples/standard/java/input
+- samples/standard/java/Java.g
 - samples/standard/java/output
-- samples/standard/rakefile
-- samples/standard/xml/README
-- samples/standard/xml/xml.rb
-- samples/standard/xml/XML.g
-- samples/standard/xml/input
-- samples/standard/xml/output
-- samples/standard/calc/Calculator.g
-- samples/standard/calc/Calculator.py
-- samples/standard/calc/Calculator.rb
+- samples/standard/java/input
 - samples/standard/C/C.g
-- samples/standard/C/c.rb
 - samples/standard/C/C__testrig.st
-- samples/standard/C/input
-- samples/standard/C/output
+- samples/standard/C/c.rb
 - samples/standard/C/C.tokens
-- samples/standard/cminus/bytecode.group
-- samples/standard/cminus/CMinus.g
-- samples/standard/cminus/cminus.rb
-- samples/standard/cminus/input
+- samples/standard/C/output
+- samples/standard/C/input
+- samples/standard/python/PythonTokenSource.rb
+- samples/standard/python/Python.g
+- samples/standard/python/output
+- samples/standard/python/input
+- samples/standard/python/python.rb
+- samples/standard/treeparser/Lang.g
+- samples/standard/treeparser/LangDumpDecl.g
+- samples/standard/treeparser/treeparser.rb
+- samples/standard/treeparser/output
+- samples/standard/treeparser/input
+- samples/standard/cminus/java.group
 - samples/standard/cminus/python.group
 - samples/standard/cminus/output
-- samples/standard/cminus/java.group
-- samples/standard/simplecTreeParser/SimpleC.g
+- samples/standard/cminus/input
+- samples/standard/cminus/CMinus.g
+- samples/standard/cminus/bytecode.group
+- samples/standard/cminus/cminus.rb
+- samples/standard/calc/Calculator.rb
+- samples/standard/calc/Calculator.py
+- samples/standard/calc/Calculator.g
+- samples/standard/xml/README
+- samples/standard/xml/output
+- samples/standard/xml/input
+- samples/standard/xml/XML.g
+- samples/standard/xml/xml.rb
+- samples/standard/rakefile
+- samples/standard/simplecTreeParser/simplec.rb
 - samples/standard/simplecTreeParser/SimpleCWalker.g
-- samples/standard/simplecTreeParser/input
 - samples/standard/simplecTreeParser/output
-- samples/standard/simplecTreeParser/simplec.rb
-- samples/standard/python/PythonTokenSource.rb
-- samples/standard/python/python.rb
-- samples/standard/python/Python.g
-- samples/standard/python/input
-- samples/standard/python/output
-- README.txt
+- samples/standard/simplecTreeParser/input
+- samples/standard/simplecTreeParser/SimpleC.g
+- samples/standard/island-grammar/Simple.g
+- samples/standard/island-grammar/output
+- samples/standard/island-grammar/input
+- samples/standard/island-grammar/Javadoc.g
+- samples/standard/island-grammar/island.rb
+- samples/standard/tweak/tweak.rb
+- samples/standard/tweak/output
+- samples/standard/tweak/input
+- samples/standard/tweak/Tweak.g
+- samples/standard/hoisted-predicates/HoistedPredicates.g
+- samples/standard/hoisted-predicates/hoisted-predicates.rb
+- samples/standard/hoisted-predicates/output
+- samples/standard/hoisted-predicates/input
 - ANTLR-LICENSE.txt
 - History.txt
+- README.txt
 - java/antlr-full-3.2.1.jar
 - java/RubyTarget.java
 - rakefile
 - Manifest.txt
 has_rdoc: true
-homepage: http://www.ohboyohboyohboy.org/ruby-antlr
+homepage: http://antlr.ohboyohboyohboy.org/
 licenses: []
 
 post_install_message: 
@@ -239,69 +240,66 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 57
       segments: 
       - 1
       - 8
       - 7
       version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: 
 - java
 rubyforge_project: antlr3
-rubygems_version: 1.3.7
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Fully-featured ruby parser generation for ANTLR version 3.
 test_files: 
+- test/unit/test-streams.rb
+- test/unit/test-template.rb
+- test/unit/test-unicode-stream.rb
 - test/unit/test-dfa.rb
+- test/unit/test-exceptions.rb
+- test/unit/test-scheme.rb
 - test/unit/test-tree-wizard.rb
-- test/unit/test-streams.rb
-- test/unit/test-recognizers.rb
 - test/unit/test-trees.rb
-- test/unit/test-exceptions.rb
+- test/unit/test-recognizers.rb
 - test/unit/test-scope.rb
-- test/unit/test-scheme.rb
-- test/unit/test-template.rb
-- test/functional/delegation/import.rb
-- test/functional/lexer/xml.rb
+- test/functional/main/main-scripts.rb
+- test/functional/template-output/template-output.rb
+- test/functional/tree-parser/basic.rb
 - test/functional/lexer/properties.rb
-- test/functional/lexer/nuances.rb
-- test/functional/lexer/basic.rb
 - test/functional/lexer/filter-mode.rb
 - test/functional/lexer/syn-pred.rb
-- test/functional/debugging/rule-tracing.rb
-- test/functional/debugging/profile-mode.rb
-- test/functional/debugging/debug-mode.rb
-- test/functional/ast-output/hetero-nodes.rb
-- test/functional/ast-output/construction.rb
-- test/functional/ast-output/rewrites.rb
-- test/functional/ast-output/tree-rewrite.rb
-- test/functional/ast-output/auto-ast.rb
-- test/functional/main/main-scripts.rb
-- test/functional/tree-parser/basic.rb
+- test/functional/lexer/basic.rb
+- test/functional/lexer/nuances.rb
+- test/functional/lexer/xml.rb
+- test/functional/delegation/import.rb
+- test/functional/parser/scopes.rb
+- test/functional/parser/properties.rb
+- test/functional/parser/basic.rb
 - test/functional/parser/predicates.rb
+- test/functional/parser/ll-star.rb
 - test/functional/parser/backtracking.rb
-- test/functional/parser/properties.rb
+- test/functional/parser/calc.rb
 - test/functional/parser/nuances.rb
-- test/functional/parser/rule-methods.rb
-- test/functional/parser/basic.rb
 - test/functional/parser/actions.rb
-- test/functional/parser/calc.rb
-- test/functional/parser/ll-star.rb
-- test/functional/parser/scopes.rb
-- test/functional/template-output/template-output.rb
-- test/functional/token-rewrite/via-parser.rb
+- test/functional/parser/rule-methods.rb
+- test/functional/debugging/debug-mode.rb
+- test/functional/debugging/profile-mode.rb
+- test/functional/debugging/rule-tracing.rb
 - test/functional/token-rewrite/basic.rb
+- test/functional/token-rewrite/via-parser.rb
+- test/functional/ast-output/tree-rewrite.rb
+- test/functional/ast-output/construction.rb
+- test/functional/ast-output/rewrites.rb
+- test/functional/ast-output/hetero-nodes.rb
+- test/functional/ast-output/auto-ast.rb