antlr3 1.2.3

data/test/unit/sample-input/file-stream-1

@@ -0,0 +1,2 @@
+foo
+bar

data/test/unit/sample-input/teststreams.input2

@@ -0,0 +1,2 @@
+foo
+bär

data/test/unit/test-dfa.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3'
+require 'test/unit'
+require 'spec'
+
+class DFASubclass < ANTLR3::DFA
+  EOT = [1, 2].freeze
+  EOF = [3, 4].freeze
+  MAX = [5, 6].freeze
+  MIN = [7, 8].freeze
+  ACCEPT = [9, 10, 11].freeze
+  SPECIAL = [12].freeze
+  TRANSITION = [
+    [13, 14, 15, 16].freeze,
+    [].freeze
+  ].freeze
+end
+
+class TestDFA < Test::Unit::TestCase
+  def test_init
+    dfa = DFASubclass.new(nil, 1)
+    dfa.eot.should == DFASubclass::EOT
+    dfa.eof.should == DFASubclass::EOF
+    dfa.max.should == DFASubclass::MAX
+    dfa.min.should == DFASubclass::MIN
+    dfa.accept.should == DFASubclass::ACCEPT
+    dfa.special.should == DFASubclass::SPECIAL
+    dfa.transition.should == DFASubclass::TRANSITION
+  end
+  
+  def test_unpack
+    packed = [
+      1, 3, 1, 4, 2, -1, 1, 5, 18, -1, 1, 2,
+      25, -1, 1, 6, 6, -1, 26, 6, 4, -1, 1, 6,
+      1, -1, 26, 6
+    ]
+    unpacked = [
+      3, 4, -1, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+     -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+     -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+      6, -1, -1, -1, -1, -1, -1, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
+      6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, -1, -1, -1, -1, 6, -1,
+      6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
+      6, 6, 6, 6, 6
+    ]
+    
+    ANTLR3::DFA.unpack(*packed).should == unpacked
+  end
+  
+end

data/test/unit/test-exceptions.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3'
+require 'test/unit'
+require 'spec'
+
+include ANTLR3::Error
+#
+#class TestRecognitionError < Test::Unit::TestCase
+#  def test_init_none
+#    RecognitionError.new()
+#  end
+#end
+#
+#class TestEarlyExit < Test::Unit::TestCase
+#  def test_init_none
+#    EarlyExit.new
+#  end
+#end
+#
+#class TestMismatchedNotSet  < Test::Unit::TestCase
+#  def test_init_none
+#    MismatchedNotSet.new
+#  end
+#end
+#
+#class TestMismatchedRange < Test::Unit::TestCase
+#  def test_init_none
+#    MismatchedSet.new
+#  end
+#end
+#
+#class TestMismatchedToken < Test::Unit::TestCase
+#  def test_init_none
+#    MismatchedToken.new
+#  end
+#end
+#
+#class TestNoViableAlternative < Test::Unit::TestCase
+#  def test_init_none
+#    NoViableAlternative.new
+#  end
+#end

data/test/unit/test-recognizers.rb

@@ -0,0 +1,55 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3'
+require 'test/unit'
+require 'spec'
+
+include ANTLR3
+
+class TestTokenSource < Test::Unit::TestCase
+  TrivialToken = Struct.new(:type) do
+    include Token
+  end
+  class TestSource
+    include TokenSource
+    def initialize
+      @tokens = (1..4).map { |i| TrivialToken[i] }
+      @tokens << TrivialToken[EOF]
+    end
+    
+    def next_token
+      @tokens.shift
+    end
+  end
+  
+  def test_iterator_interface
+    src = TestSource.new
+    tokens = []
+    src.each do |token|
+      tokens << token.type
+    end
+    tokens.should == [1,2,3,4]
+  end
+  
+end
+
+class TestLexer < Test::Unit::TestCase
+  class TLexer < Lexer
+    @antlr_version = ANTLR3::ANTLR_VERSION.dup
+  end
+  def test_init
+    stream = StringStream.new('foo')
+    TLexer.new(stream)
+  end
+end
+
+__END__
+testrecognizers.py                           | LN | STATUS
+---------------------------------------------+----+--------------
+class TestBaseRecognizer(unittest.TestCase)  | 07 | [x]
+    def testGetRuleInvocationStack(self)     | 10 | [x]
+class TestTokenSource(unittest.TestCase)     | 20 | [x]
+    def testIteratorInterface(self)          | 24 | [x]
+class TestLexer(unittest.TestCase)           | 54 | [x]
+    def testInit(self)                       | 56 | [x]

data/test/unit/test-scheme.rb

@@ -0,0 +1,62 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+require 'antlr3'
+require 'test/unit'
+require 'spec'
+
+include ANTLR3
+
+describe TokenScheme do
+  before do
+    @ts = TokenScheme.new do
+      define_tokens(:A => 4, :B => 5, :T__6 => 6)
+      register_names('A', 'B', "'+'")
+    end
+    @a_class = Class.new do
+      class << self
+        attr_accessor :token_scheme
+      end
+    end
+    @a_class.send(:include, @ts)
+    
+    @an_instance = @a_class.new
+  end
+  
+  example "token schemes define tokens as constants" do
+    @ts::A.should == 4
+    @ts::B.should == 5
+    @ts::T__6.should == 6
+    @ts::EOF.should == -1
+  end
+  
+  example "token schemes track human-friendly token names" do
+    @ts::TOKEN_NAMES.should == {
+      0 => "<invalid>", -1 => "<EOF>", 1 => "<EOR>",
+      2 => "<DOWN>", 3 => "<UP>", 4 => "A",
+      5 => "B", 6 => "'+'"
+    }
+    @ts.token_name(5).should == 'B'
+    @ts.token_name(6).should == "'+'"
+    @ts.token_name(-1).should == '<EOF>'
+    @ts.token_name(7).should == '<UNKNOWN: 7>'
+  end
+  
+  
+  example 'class-level results of including a token scheme' do
+    #@a_class.token_scheme.should == @ts
+    
+    @a_class::A.should == 4
+    @a_class::B.should == 5
+    @a_class::T__6.should == 6
+    @a_class::EOF.should == -1
+    
+    @a_class.send(:token_names).should == {
+      0 => "<invalid>", -1 => "<EOF>", 1 => "<EOR>",
+      2 => "<DOWN>", 3 => "<UP>", 4 => "A",
+      5 => "B", 6 => "'+'"
+    }
+  end
+  
+  example 'instance-level results of including a token scheme' do
+  end
+end

data/test/unit/test-streams.rb

@@ -0,0 +1,459 @@
+#!/usr/bin/ruby
+# encoding: utf-8
+
+require 'antlr3'
+require 'test/unit'
+require 'spec'
+
+
+include ANTLR3
+
+class TestStringStream < Test::Unit::TestCase
+  def setup
+    @stream = StringStream.new("oh\nhey!\n")
+  end
+  
+  def test_size
+    @stream.size.should == 8
+  end
+  
+  def test_index
+    @stream.index.should == 0
+  end
+  
+  def test_consume
+    @stream.consume # o
+    @stream.index.should == 1
+    @stream.column.should == 1
+    @stream.line.should == 1
+    
+    @stream.consume  # h
+    @stream.index.should == 2
+    @stream.column.should == 2
+    @stream.line.should == 1
+    
+    @stream.consume # \n
+    @stream.index.should == 3
+    @stream.column.should == 0
+    @stream.line.should == 2
+    
+    @stream.consume #  h
+    @stream.index.should == 4
+    @stream.column.should == 1
+    @stream.line.should == 2
+    
+    @stream.consume # e
+    @stream.index.should == 5
+    @stream.column.should == 2
+    @stream.line.should == 2
+    
+    @stream.consume # y
+    @stream.index.should == 6
+    @stream.column.should == 3
+    @stream.line.should == 2
+    
+    @stream.consume # !
+    @stream.index.should == 7
+    @stream.column.should == 4
+    @stream.line.should == 2
+    
+    @stream.consume # \n
+    @stream.index.should == 8
+    @stream.column.should == 0
+    @stream.line.should == 3
+    
+    @stream.consume # EOF
+    @stream.index.should == 8
+    @stream.column.should == 0
+    @stream.line.should == 3
+    
+    @stream.consume # EOF
+    @stream.index.should == 8
+    @stream.column.should == 0
+    @stream.line.should == 3
+  end
+  
+  def test_reset
+    2.times { @stream.consume }
+    @stream.reset
+    @stream.index.should == 0
+    @stream.line.should == 1
+    @stream.column.should == 0
+    @stream.peek(1).should == ?o
+  end
+  
+  def test_look
+    @stream.look(1).should == 'o'
+    @stream.look(2).should == 'h'
+    @stream.look(3).should == "\n"
+    @stream.peek(1).should == ?o
+    @stream.peek(2).should == ?h
+    @stream.peek(3).should == ?\n
+    
+    6.times { @stream.consume }
+    @stream.look(1).should == '!'
+    @stream.look(2).should == "\n"
+    @stream.look(3).should be_nil
+    @stream.peek(1).should == ?!
+    @stream.peek(2).should == ?\n
+    @stream.peek(3).should == EOF
+  end
+  
+  def test_substring
+    @stream.substring(0,0).should == 'o'
+    @stream.substring(0,1).should == 'oh'
+    @stream.substring(0,8).should == "oh\nhey!\n"
+    @stream.substring(3,6).should == "hey!"
+  end
+  
+  def test_seek_forward
+    @stream.seek(3)
+    @stream.index.should == 3
+    @stream.line.should == 2
+    @stream.column.should == 0
+    @stream.peek(1).should == ?h
+  end
+  
+  def test_mark
+    @stream.seek(4)
+    marker = @stream.mark
+    marker.should == 1
+    
+    2.times { @stream.consume }
+    marker = @stream.mark
+    
+    marker.should == 2
+  end
+  
+  def test_release_last
+    @stream.seek(4)
+    marker1 = @stream.mark
+    
+    2.times { @stream.consume }
+    marker2 = @stream.mark
+    
+    @stream.release
+    @stream.mark_depth.should == 2
+    @stream.release
+    @stream.mark_depth.should == 1
+  end
+  
+  def test_release_nested
+    @stream.seek(4)
+    marker1 = @stream.mark()
+    
+    @stream.consume()
+    marker2 = @stream.mark()
+    
+    @stream.consume()
+    marker3 = @stream.mark()
+    
+    @stream.release(marker2)
+    @stream.mark_depth.should == 2
+
+  end
+  
+  def test_rewind_last
+    @stream.seek(4)
+
+    marker = @stream.mark
+    @stream.consume
+    @stream.consume
+
+    @stream.rewind
+    @stream.mark_depth.should == 1
+    @stream.index.should == 4
+    @stream.line.should == 2
+    @stream.column.should == 1
+    @stream.peek(1).should == ?e
+    
+  end
+
+  def test_through
+    @stream.through( 2 ).should == 'oh'
+    @stream.through( -2 ).should == ''
+    @stream.seek( 5 )
+    @stream.through( 0 ).should == ''
+    @stream.through( 1 ).should == 'y'
+    @stream.through( -2 ).should == 'he'
+    @stream.through( 5 ).should == "y!\n"
+  end
+  
+  def test_rewind_nested
+    @stream.seek(4)
+    marker1 = @stream.mark()
+    
+    @stream.consume
+    marker2 = @stream.mark
+    
+    @stream.consume
+    marker3 = @stream.mark
+    
+    @stream.rewind(marker2)
+    @stream.mark_depth.should == 2
+    @stream.index().should == 5
+    @stream.line.should == 2
+    @stream.column.should == 2
+    @stream.peek(1).should == ?y    
+  end
+end
+
+class TestFileStream < Test::Unit::TestCase
+  
+  
+  def test_no_encoding
+    path = File.join(File.dirname(__FILE__), 'sample-input/file-stream-1')
+    @stream = FileStream.new(path)
+    
+    @stream.seek(4)
+    marker1 = @stream.mark()
+    
+    @stream.consume()
+    marker2 = @stream.mark()
+    
+    @stream.consume()
+    marker3 = @stream.mark()
+    
+    @stream.rewind(marker2)
+    @stream.index().should == 5
+    @stream.line.should == 2
+    @stream.column.should == 1
+    @stream.mark_depth.should == 2
+    @stream.look(1).should == 'a'
+    @stream.peek(1).should == ?a
+  end
+  
+  def test_encoded
+    
+  end
+end
+
+class TestInputStream < Test::Unit::TestCase
+  def test_no_encoding
+    
+  end
+  
+  def test_encoded
+    
+  end
+end
+
+class TestCommonTokenStream < Test::Unit::TestCase
+  class MockSource
+    include ANTLR3::TokenSource
+    attr_accessor :tokens
+    def initialize
+      @tokens = []
+    end
+    def next_token
+      @tokens.shift
+    end
+  end
+  
+  # vvvvvvvv tests vvvvvvvvv
+  def test_init
+    @source = MockSource.new
+    @stream = CommonTokenStream.new(@source)
+    @stream.position.should == 0
+  end
+  
+  def test_rebuild
+    @source1 = MockSource.new
+    @source2 = MockSource.new
+    @source2.tokens << new_token(10, :channel => ANTLR3::HIDDEN) << new_token(11)
+    @stream = CommonTokenStream.new(@source1)
+    
+    @stream.position.should == 0
+    @stream.tokens.should be_empty
+    
+    @stream.rebuild(@source2)
+    @stream.token_source.should == @source2
+    @stream.position.should == 1
+    @stream.tokens.should have(2).things
+  end
+  
+  def test_look_empty_source
+    @source = MockSource.new
+    @stream = CommonTokenStream.new(@source)
+    @stream.look.should == ANTLR3::EOF_TOKEN
+  end
+  
+  def test_look1
+    @source = MockSource.new
+    @source.tokens << new_token(12)
+    @stream = CommonTokenStream.new(@source)
+    @stream.look(1).type.should == 12
+  end
+  
+  def test_look1_with_hidden
+    # FIX
+    @source = MockSource.new
+    @source.tokens << new_token(12, :channel => ANTLR3::HIDDEN_CHANNEL) <<
+      new_token(13)
+    @stream = CommonTokenStream.new(@source)
+    @stream.look(1).type.should == 13
+  end
+  
+  def test_look2_beyond_end
+    @source = MockSource.new
+    @source.tokens << new_token(12) <<
+      new_token(13, :channel => ANTLR3::HIDDEN_CHANNEL)
+    
+    @stream = CommonTokenStream.new(@source)
+    @stream.look(2).type.should == EOF
+  end
+  
+  def test_look_negative
+    @source = MockSource.new
+    @source.tokens << new_token(12) << new_token(13)
+    @stream = CommonTokenStream.new(@source)
+    @stream.consume
+    
+    @stream.look(-1).type.should == 12
+  end
+  
+  def test_lb1
+    @source = MockSource.new
+    @source.tokens << new_token(12) << new_token(13)
+    @stream = CommonTokenStream.new(@source)
+    
+    @stream.consume
+    @stream.look(-1).type.should == 12
+  end
+  
+  def test_look_zero
+    # FIX
+    @source = MockSource.new
+    @source.tokens << new_token(12) << new_token(13)
+    @stream = CommonTokenStream.new(@source)
+    @stream.look(0).should be_nil
+  end
+  
+  def test_lb_beyond_begin
+    @source = MockSource.new
+    @source.tokens << new_token(10) <<
+      new_token(11, :channel => HIDDEN_CHANNEL) <<
+      new_token(12, :channel => HIDDEN_CHANNEL) <<
+      new_token(13)
+    @stream = CommonTokenStream.new(@source)
+    
+    @stream.look(-1).should be_nil
+    2.times { @stream.consume }
+    @stream.look(-3).should be_nil
+  end
+  
+  def test_fill_buffer
+    @source = MockSource.new
+    @source.tokens << new_token(12) << new_token(13) <<  new_token(14) << new_token(EOF)
+    @stream = CommonTokenStream.new(@source)
+    
+    @stream.instance_variable_get(:@tokens).length.should == 3
+    @stream.tokens[0].type.should == 12
+    @stream.tokens[1].type.should == 13
+    @stream.tokens[2].type.should == 14
+  end
+  
+  def test_consume
+    @source = MockSource.new
+    @source.tokens << new_token(12) << new_token(13) << new_token(EOF)
+    @stream = CommonTokenStream.new(@source)
+    @stream.peek.should == 12
+    @stream.consume
+    @stream.peek.should == 13
+    @stream.consume
+    @stream.peek.should == EOF
+    @stream.consume
+    @stream.peek.should == EOF
+  end
+  
+  def test_seek
+    @source = MockSource.new
+    @source.tokens << new_token(12) << new_token(13) << new_token(EOF)
+    @stream = CommonTokenStream.new(@source)
+    
+    @stream.peek(1).should == 12
+    @stream.seek(2).peek.should == EOF
+    @stream.seek(0).peek.should == 12
+    @stream.seek(-3).position.should == 0
+    @stream.seek(10).position.should == 2
+  end
+  
+  def test_mark_rewind
+    @source = MockSource.new
+    @source.tokens << new_token(12) << new_token(13) << new_token(EOF)
+    @stream = CommonTokenStream.new(@source)
+    @stream.consume
+    marker = @stream.mark
+    @stream.consume
+    @stream.rewind(marker)
+    @stream.peek(1).should == 13
+  end
+  
+  def test_to_string
+    @source = MockSource.new
+    @source.tokens << new_token(12, 'foo') <<
+      new_token(13, 'bar') << new_token(14, 'gnurz') <<
+      new_token(15, 'blarz')
+    @stream = CommonTokenStream.new(@source)
+    @stream.to_s.should == "foobargnurzblarz"
+    @stream.to_s(1,2).should == 'bargnurz'
+    @stream.to_s(@stream[1], @stream[-2]).should == 'bargnurz'
+  end
+
+  def new_token(type, opts = {})
+    fields = {}
+    case type
+    when Hash then fields.update(type)
+    else
+      fields[:type] = type
+    end
+    case opts
+    when Hash then fields.update(opts)
+    when String then fields[:text] = opts
+    end
+    CommonToken.create(fields)
+  end
+  
+end
+
+
+__END__
+teststreams.py                                | LN  | STATUS
+----------------------------------------------+-----+--------------
+class TestStringStream(unittest.TestCase)     | 009 | [x]
+  def testSize(self)                          | 012 | [x]
+  def testIndex(self)                         | 020 | [x]
+  def testConsume(self)                       | 028 | [x]
+  def testReset(self)                         | 079 | [x]
+  def testLA(self)                            | 094 | [x]
+  def testSubstring(self)                     | 111 | [x]
+  def testSeekForward(self)                   | 122 | [x]
+  def testMark(self)                          | 150 | [x]
+  def testReleaseLast(self)                   | 167 | [x]
+  def testReleaseNested(self)                 | 186 | [x]
+  def testRewindLast(self)                    | 204 | [x]
+  def testRewindNested(self)                  | 223 | [x]
+class TestFileStream(unittest.TestCase)       | 245 | [o]
+  def testNoEncoding(self)                    | 249 | [x]
+  def testEncoded(self)                       | 272 | [ ]
+class TestInputStream(unittest.TestCase)      | 296 | [ ]
+  def testNoEncoding(self)                    | 299 | [ ]
+  def testEncoded(self)                       | 322 | [ ]
+class TestCommonTokenStream(unittest.TestCase)| 345 | [ ]
+  def setUp(self)                             | 348 | [x]
+  def testInit(self)                          | 369 | [x]
+  def testSetTokenSource(self)                | 376 | [x]
+  def testLTEmptySource(self)                 | 385 | [x]
+  def testLT1(self)                           | 394 | [x]
+  def testLT1WithHidden(self)                 | 407 | [x]
+  def testLT2BeyondEnd(self)                  | 424 | [x]
+  def testLTNegative(self)                    | 442 | [x]
+  def testLB1(self)                           | 461 | [x]
+  def testLTZero(self)                        | 479 | [x]
+  def testLBBeyondBegin(self)                 | 496 | [x]
+  def testFillBuffer(self)                    | 523 | [x]
+  def testConsume(self)                       | 551 | [x]
+  def testSeek(self)                          | 579 | [x]
+  def testMarkRewind(self)                    | 604 | [x]
+  def testToString(self)                      | 631 | [x]
+