antlr4 0.9.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e8c6c33daacd490242fa9315d0cb2ff929f6997b
+  data.tar.gz: edfe99a049c875a02ad10bd9bd4f610df729513b
+SHA512:
+  metadata.gz: 1751abe7b1fb45ff15f21ee131da1fb7eac7cb53a22dc51bb977b6c8039f29b178aa1edd235d747a01e9267616644285c1cadcba96d096ccd5e0a642df08a809
+  data.tar.gz: 9b33692abff06abd0ddb9ca7a970c7d3d29ecd810a2a303e83ff79214d6e8ca4ad91126bccb5e14768add6271d6f0125fd2eb4336a6ee16b2cdd3e342bdc156d

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014, Chad Slaughter <chad.slaughter@gmail.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,46 @@
+This is the Ruby runtime and target for Antlr4. 
+
+This is based on the Python3 & Java runtime & targets.
+It includes the Java code for antlr to use *language = Ruby ;* and
+the runtime gem for the ruby code to run. There is a simplistic 
+_bin/antlr-testrig_ for quickly loading a Lexer/Parser pair and
+running them on an input file. 
+
+Usage
+-----
+To use the Ruby language codegen for Antlrv4 do the following.
+
+1. Create the jar file.
+    ```% make antlr4-ruby.jar ```
+
+2. Put the jar file in your java class path 
+
+3. Use either the _options_ section  or the _-Dlanguage=_ switch
+
+```
+  options {  language = Ruby ; }
+```
+
+```
+java org.antlr.v4.Tool -Dlanguage=Ruby input.g4
+```
+
+Build gem for use by Ruby code. It is placed in _pkg_.
+```
+rake build 
+```
+
+You can then install with Bundler or Rubygems directly.
+
+
+Missing Features
+----------------
+* Ruby runtime is not in a proper module and library setup
+* The Ruby test for integration with antlr4 are missing
+* Proper attribution of all code
+
+
+### Fixed Bugs ###
+* Ruby MRI encoding causes fails with some generated ATNs.
+ This was fixed by using \x instead of \u for ATN encoding.
+

data/lib/antlr4.rb

@@ -0,0 +1,262 @@
+require 'double_key_map'
+require 'java_symbols'
+require 'uuid'
+
+require 'stringio'
+require 'set'
+require 'forwardable'
+
+require 'antlr4/base'
+require 'antlr4/version'
+
+require 'antlr4/Token'
+require 'antlr4/InputStream'
+require 'antlr4/FileStream'
+require 'antlr4/TokenStream'
+
+require 'antlr4/error'
+require 'antlr4/error/ErrorListener'
+require 'antlr4/error/DiagnosticErrorListener'
+require 'antlr4/error/ErrorStrategy'
+
+require 'antlr4/BufferedTokenStream'
+require 'antlr4/CommonTokenStream'
+
+require 'antlr4/IntervalSet'
+
+require 'antlr4/tree/Trees'
+require 'antlr4/tree/Tree'
+require 'antlr4/tree/TokenTagToken'
+require 'antlr4/tree/RuleTagToken'
+require 'antlr4/tree/ParseTreeMatch'
+require 'antlr4/tree/ParseTreePatternMatcher'
+# 
+require 'antlr4/Recognizer'
+require 'antlr4/TokenSource'
+require 'antlr4/ListTokenSource'
+require 'antlr4/TokenFactory'
+require 'antlr4/CommonTokenFactory'
+
+require 'antlr4/RuleContext'
+require 'antlr4/ParserRuleContext'
+require 'antlr4/PredictionContext'
+
+require 'antlr4/LL1Analyzer'
+
+require 'antlr4/dfa/DFA'
+require 'antlr4/dfa/DFAState'
+require 'antlr4/dfa/DFASerializer'
+
+require 'antlr4/atn/ATNType'
+require 'antlr4/atn/ATNState'
+require 'antlr4/atn/ATN'
+require 'antlr4/atn/ATNConfig'
+require 'antlr4/atn/ATNConfigSet'
+require 'antlr4/atn/Transition'
+require 'antlr4/atn/ATNSimulator'
+require 'antlr4/atn/SemanticContext'
+require 'antlr4/atn/LexerAction'
+require 'antlr4/atn/LexerActionExecutor'
+require 'antlr4/atn/PredictionMode'
+require 'antlr4/atn/ATNDeserializationOptions'
+require 'antlr4/atn/LexerATNSimulator'
+require 'antlr4/atn/ParserATNSimulator'
+require 'antlr4/atn/ATNDeserializer'
+
+require 'antlr4/Parser'
+require 'antlr4/TraceListener'
+require 'antlr4/ParserInterpreter'
+require 'antlr4/Lexer'
+
+
+__END__
+CommonTokenFactory.rb:3:require 'Token'
+CommonTokenFactory.rb:5:require 'TokenFactory'
+
+FileStream.rb:7:require 'InputStream'
+InputStream.rb:6:require 'Token'
+IntervalSet.rb:3:require 'Token'
+IntervalSet.rb:4:require 'set'
+IntervalSet.rb:19:require 'forwardable'
+LL1Analyzer.rb:3:require 'Token'
+LL1Analyzer.rb:11:require 'set'
+Lexer.rb:15:require 'CommonTokenFactory'
+Lexer.rb:16:require 'atn/LexerATNSimulator'
+Lexer.rb:17:require 'InputStream'
+Lexer.rb:18:require 'Recognizer'
+Lexer.rb:19:require 'error'
+Lexer.rb:20:require 'Token'
+Lexer.rb:21:require 'TokenSource'
+Lexer.rb:23:require 'java_symbols' 
+Lexer.rb:111:            raise IllegalStateException.new("nextToken requires a non-null input stream.")
+ListTokenSource.rb:13:require 'CommonTokenFactory'
+ListTokenSource.rb:14:require 'TokenSource'
+ListTokenSource.rb:15:require 'Token'
+Parser.rb:1:require 'TokenStream'
+Parser.rb:2:require 'TokenFactory'
+Parser.rb:3:require 'error'
+Parser.rb:4:require 'error/ErrorStrategy'
+Parser.rb:5:require 'InputStream'
+Parser.rb:6:require 'Recognizer'
+Parser.rb:7:require 'RuleContext'
+Parser.rb:8:require 'ParserRuleContext'
+Parser.rb:9:require 'Token'
+Parser.rb:10:require 'Lexer'
+Parser.rb:11:require 'tree/ParseTreePatternMatcher'
+Parser.rb:12:require 'tree/Tree'
+Parser.rb:19:require 'java_symbols'
+ParserInterpreter.rb:27:require 'TokenStream'
+ParserInterpreter.rb:28:require 'Parser'
+ParserInterpreter.rb:29:require 'ParserRuleContext'
+ParserInterpreter.rb:30:require 'Token'
+ParserInterpreter.rb:31:require 'error'
+ParserInterpreter.rb:33:require 'set'
+ParserRuleContext.rb:24:require 'RuleContext'
+ParserRuleContext.rb:25:require 'Token'
+ParserRuleContext.rb:26:require 'tree/Tree'
+PredictionContext.rb:6:require 'RuleContext'
+PredictionContext.rb:7:require 'double_key_map'
+Recognizer.rb:4:require 'RuleContext'
+Recognizer.rb:5:require 'Token'
+Recognizer.rb:6:require 'error/ErrorListener'
+Recognizer.rb:7:require 'error'
+RuleContext.rb:22:require 'stringio'
+RuleContext.rb:25:require 'tree/Tree'
+RuleContext.rb:26:require 'tree/Trees'
+TokenSource.rb:2:require 'Recognizer'
+atn/ATN.rb:1:require 'IntervalSet'
+atn/ATN.rb:2:require 'RuleContext'
+atn/ATN.rb:4:require 'Token'
+atn/ATN.rb:5:require 'atn/ATNType'
+atn/ATN.rb:6:require 'atn/ATNState' 
+atn/ATN.rb:8:require 'java_symbols'
+atn/ATN.rb:50:        require 'LL1Analyzer'
+atn/ATNConfig.rb:13:require 'PredictionContext'
+atn/ATNConfig.rb:14:require 'atn/ATNState'
+atn/ATNConfig.rb:15:#require 'atn/LexerActionExecutor'
+atn/ATNConfig.rb:16:#require 'atn/SemanticContext'
+atn/ATNConfigSet.rb:5:require 'stringio'
+atn/ATNConfigSet.rb:6:require 'PredictionContext'
+atn/ATNConfigSet.rb:7:require 'atn/ATN'
+atn/ATNConfigSet.rb:8:require 'atn/ATNConfig'
+atn/ATNConfigSet.rb:9:#require 'atn/SemanticContext'
+atn/ATNConfigSet.rb:10:require 'error'
+atn/ATNConfigSet.rb:12:require 'forwardable'
+atn/ATNDeserializer.rb:2:require 'stringio'
+atn/ATNDeserializer.rb:3:require 'Token'
+atn/ATNDeserializer.rb:4:require 'atn/ATN'
+atn/ATNDeserializer.rb:5:require 'atn/ATNType'
+atn/ATNDeserializer.rb:6:require 'atn/ATNState'
+atn/ATNDeserializer.rb:7:require 'atn/Transition'
+atn/ATNDeserializer.rb:8:require 'atn/LexerAction'
+atn/ATNDeserializer.rb:9:require 'atn/ATNDeserializationOptions'
+atn/ATNDeserializer.rb:11:require 'uuid'
+atn/ATNSimulator.rb:6:require 'PredictionContext'
+atn/ATNSimulator.rb:7:require 'atn/ATN'
+atn/ATNSimulator.rb:8:require 'atn/ATNConfigSet'
+atn/ATNSimulator.rb:9:require 'dfa/DFAState'
+atn/ATNSimulator.rb:31:    #  For the Java grammar on java.*, it dropped the memory requirements
+atn/LexerATNSimulator.rb:16:require 'Lexer'
+atn/LexerATNSimulator.rb:17:require 'PredictionContext'
+atn/LexerATNSimulator.rb:18:require 'InputStream'
+atn/LexerATNSimulator.rb:19:require 'Token'
+atn/LexerATNSimulator.rb:20:require 'atn/ATN'
+atn/LexerATNSimulator.rb:21:require 'atn/ATNConfig'
+atn/LexerATNSimulator.rb:22:require 'atn/ATNSimulator'
+atn/LexerATNSimulator.rb:23:require 'atn/ATNConfigSet'
+atn/LexerATNSimulator.rb:24:require 'atn/ATNState'
+atn/LexerATNSimulator.rb:25:require 'atn/LexerActionExecutor'
+atn/LexerATNSimulator.rb:26:require 'atn/Transition'
+atn/LexerATNSimulator.rb:27:require 'dfa/DFAState'
+atn/LexerATNSimulator.rb:28:require 'error'
+atn/LexerATNSimulator.rb:30:require 'java_symbols'
+atn/LexerAction.rb:4:require 'java_symbols'
+atn/LexerAction.rb:273:# <p>This action is not serialized as part of the ATN, and is only required for
+atn/LexerAction.rb:283:    # <p>Note: This class is only required for lexer actions for which
+atn/LexerActionExecutor.rb:8:require 'InputStream'
+atn/LexerActionExecutor.rb:10:require 'atn/LexerAction' 
+atn/LexerActionExecutor.rb:112:        requiresSeek = false
+atn/LexerActionExecutor.rb:120:                    requiresSeek = (startIndex + offset) != stopIndex
+atn/LexerActionExecutor.rb:123:                    requiresSeek = false
+atn/LexerActionExecutor.rb:128:            input.seek(stopIndex) if requiresSeek
+atn/ParserATNSimulator.rb:49:# than interpreting and much more complicated. Also required a huge amount of
+atn/ParserATNSimulator.rb:225:# both SLL and LL parsing. Erroneous input will therefore require 2 passes over
+atn/ParserATNSimulator.rb:228:require 'dfa/DFA'
+atn/ParserATNSimulator.rb:229:require 'PredictionContext'
+atn/ParserATNSimulator.rb:230:require 'TokenStream'
+atn/ParserATNSimulator.rb:231:require 'Parser'
+atn/ParserATNSimulator.rb:232:require 'ParserRuleContext'
+atn/ParserATNSimulator.rb:233:require 'RuleContext'
+atn/ParserATNSimulator.rb:234:require 'Token'
+atn/ParserATNSimulator.rb:235:require 'atn/ATN'
+atn/ParserATNSimulator.rb:236:require 'atn/ATNConfig'
+atn/ParserATNSimulator.rb:237:require 'atn/ATNConfigSet'
+atn/ParserATNSimulator.rb:238:require 'atn/ATNSimulator'
+atn/ParserATNSimulator.rb:239:require 'atn/ATNState'
+atn/ParserATNSimulator.rb:240:require 'atn/PredictionMode'
+atn/ParserATNSimulator.rb:241:require 'atn/SemanticContext'
+atn/ParserATNSimulator.rb:242:require 'atn/Transition'
+atn/ParserATNSimulator.rb:243:require 'dfa/DFAState'
+atn/ParserATNSimulator.rb:244:require 'error'
+atn/ParserATNSimulator.rb:442:            if cD.requiresFullContext and self.predictionMode != PredictionMode.SLL
+atn/ParserATNSimulator.rb:560:            cD.requiresFullContext = true
+atn/ParserATNSimulator.rb:716:        # For full-context reach operations, separate handling is required to
+atn/ParserATNSimulator.rb:799:            # required.
+atn/PredictionMode.rb:11:require 'atn/ATN'
+atn/PredictionMode.rb:12:require 'atn/ATNConfig'
+atn/PredictionMode.rb:13:require 'atn/ATNConfigSet'
+atn/PredictionMode.rb:14:require 'atn/ATNState'
+atn/PredictionMode.rb:15:require 'atn/SemanticContext'
+atn/PredictionMode.rb:16:require 'java_symbols'
+atn/PredictionMode.rb:34:    # that the particular combination of grammar and input requires the more
+atn/PredictionMode.rb:287:    # <p>No special consideration for semantic predicates is required because
+atn/SemanticContext.rb:11:require 'Recognizer'
+atn/SemanticContext.rb:12:require 'RuleContext'
+atn/Transition.rb:15:require 'IntervalSet'
+atn/Transition.rb:16:require 'Token'
+atn/Transition.rb:18:require 'atn/SemanticContext'
+atn/Transition.rb:20:require 'java_symbols'
+base.rb:1:require 'Token' 
+base.rb:2:require 'FileStream'
+base.rb:3:require 'TokenStream' 
+base.rb:4:require 'BufferedTokenStream'
+base.rb:5:require 'CommonTokenStream'
+base.rb:6:require 'Lexer'
+base.rb:7:require 'Parser'
+base.rb:8:require 'dfa/DFA'
+base.rb:9:require 'atn/ATN'
+base.rb:10:require 'atn/ATNDeserializer'
+base.rb:11:require 'atn/LexerATNSimulator'
+base.rb:12:require 'atn/ParserATNSimulator' 
+base.rb:13:require 'atn/PredictionMode'
+base.rb:14:require 'PredictionContext'
+base.rb:15:require 'ParserRuleContext'
+base.rb:16:require 'tree/Tree' # import ParseTreeListener, ParseTreeVisitor, ParseTreeWalker, TerminalNode, ErrorNode, RuleNode
+base.rb:17:require 'error'    #  Errors import RecognitionException, IllegalStateException, NoViableAltException
+base.rb:18:require 'error/ErrorStrategy' # import BailErrorStrategy
+base.rb:19:require 'error/DiagnosticErrorListener' # import DiagnosticErrorListener
+base.rb:21:require 'java_symbols'
+dfa/DFA.rb:9:require 'dfa/DFASerializer'
+dfa/DFA.rb:102:                precedenceState.requiresFullContext = false
+dfa/DFASerializer.rb:3:require 'stringio'
+dfa/DFASerializer.rb:58:        s_requireContext = nil
+dfa/DFASerializer.rb:59:        s_requireContext = "^" if s.requiresFullContext 
+dfa/DFASerializer.rb:60:        baseStateStr = "s#{s.stateNumber}#{s_requireContext}"
+dfa/DFAState.rb:2:require 'stringio'
+dfa/DFAState.rb:45:    attr_accessor :lexerActionExecutor, :requiresFullContext, :predicates 
+dfa/DFAState.rb:55:        #  {@link #requiresFullContext}.
+dfa/DFAState.rb:62:        self.requiresFullContext = false
+dfa/DFAState.rb:65:        #  {@link #requiresFullContext} is {@code false} since full context prediction evaluates predicates
+dfa/DFAState.rb:69:        #  <p>We only use these for non-{@link #requiresFullContext} but conflicting states. That
+error/DiagnosticErrorListener.rb:52:require 'stringio'
+error/DiagnosticErrorListener.rb:53:require 'set'
+error/DiagnosticErrorListener.rb:56:require 'error/ErrorListener'
+error/ErrorStrategy.rb:32:# require 'IntervalSet' #from antlr4.IntervalSet import IntervalSet
+error/ErrorStrategy.rb:34:#require 'antlr4/Token' #from antlr4.Token import Token
+error/ErrorStrategy.rb:35:#require 'atn.ATNState' #from antlr4.atn.ATNState import ATNState
+error/ErrorStrategy.rb:323:    # This method is called to report a syntax error which requires the removal
+error/ErrorStrategy.rb:350:    # This method is called to report a syntax error which requires the
+error/ErrorStrategy.rb:640:    #  return normally.  Rule b would not find the required '^' though.
+xpath/XPath.rb:59:require 'TokenStream'
+xpath/XPath.rb:60:require 'CommonTokenStream'
+xpath/XPath.rb:61:require 'java_symbols'

data/lib/antlr4/BufferedTokenStream.rb

@@ -0,0 +1,306 @@
+
+class BufferedTokenStream < TokenStream
+
+    attr_accessor :tokenSource, :tokens, :index,:fetchedEOF 
+    def initialize(_tokenSource)
+        # The {@link TokenSource} from which tokens for this stream are fetched.
+        @tokenSource = _tokenSource
+        # A collection of all tokens fetched from the token source. The list is
+        # considered a complete view of the input once {@link #fetchedEOF} is set
+        # to {@code true}.
+        self.tokens = Array.new
+
+        # The index into {@link #tokens} of the current token (next token to
+        # {@link #consume}). {@link #tokens}{@code [}{@link #p}{@code ]} should be
+        # {@link #LT LT(1)}.
+        #
+        # <p>This field is set to -1 when the stream is first constructed or when
+        # {@link #setTokenSource} is called, indicating that the first token has
+        # not yet been fetched from the token source. For additional information,
+        # see the documentation of {@link IntStream} for a description of
+        # Initializing Methods.</p>
+        self.index = -1
+
+        # Indicates whether the {@link Token#EOF} token has been fetched from
+        # {@link #tokenSource} and added to {@link #tokens}. This field improves
+        # performance for the following cases
+        #
+        # <ul>
+        # <li>{@link #consume}: The lookahead check in {@link #consume} to prevent
+        # consuming the EOF symbol is optimized by checking the values of
+        # {@link #fetchedEOF} and {@link #p} instead of calling {@link #LA}.</li>
+        # <li>{@link #fetch}: The check to prevent adding multiple EOF symbols into
+        # {@link #tokens} is trivial with this field.</li>
+        # <ul>
+        self.fetchedEOF = false
+    end
+
+    def mark
+        return 0
+    end
+    
+    def release(marker)
+        # no resources to release
+    end
+    
+    def reset()
+        self.seek(0)
+    end
+    def seek( index)
+        self.lazyInit()
+        self.index = self.adjustSeekIndex(index)
+    end
+    def get(index)
+        self.lazyInit()
+        return self.tokens[index]
+    end
+    def consume()
+        skipEofCheck = false
+        if self.index >= 0 then
+            if self.fetchedEOF then
+                # the last token in tokens is EOF. skip check if p indexes any
+                # fetched token except the last.
+                skipEofCheck = self.index < self.tokens.length - 1
+            else
+               # no EOF token in tokens. skip check if p indexes a fetched token.
+                skipEofCheck = self.index < self.tokens.length
+            end
+        else
+            # not yet initialized
+            skipEofCheck = false
+        end
+        if not skipEofCheck and self.LA(1) == Token::EOF then
+            raise IllegalStateException.new("cannot consume EOF")
+        end
+        if self.sync(self.index + 1) then
+            self.index = self.adjustSeekIndex(self.index + 1)
+        end
+    end
+    # Make sure index {@code i} in tokens has a token.
+    #
+    # @return {@code true} if a token is located at index {@code i}, otherwise
+    #    {@code false}.
+    # @see #get(int i)
+    #/
+    def sync(i)
+        #assert i >= 0
+        n = i - self.tokens.length + 1 # how many more elements we need?
+        if n > 0 then
+            fetched = self.fetch(n)
+            return fetched >= n
+        end
+        return true
+    end
+    # Add {@code n} elements to buffer.
+    #
+    # @return The actual number of elements added to the buffer.
+    #/
+    def fetch(n)
+        return 0 if self.fetchedEOF
+        1.upto(n) do |i| # for i in 0..n-1 do
+            t = self.tokenSource.nextToken()
+            t.tokenIndex = self.tokens.length
+            self.tokens.push(t)
+            if t.type==Token::EOF then
+                self.fetchedEOF = true
+                return i  #  i + 1
+            end
+        end
+        return n
+    end
+
+    # Get all tokens from start..stop inclusively#/
+    def getTokens(start, stop, types=nil)
+        if start<0 or stop<0 then
+            return  nil
+        end
+        self.lazyInit()
+        subset = Array.new
+        if stop >= self.tokens.length
+            stop = self.tokens.length-1
+        end
+        for i in start..stop-1 do
+            t = self.tokens[i]
+            if t.type==Token::EOF
+                break
+            end
+            if (types.nil? or types.member?(t.type)) then
+                subset.push(t)
+            end
+        end
+        return subset
+    end
+    def LA(i)
+        return self.LT(i).type
+    end
+    def LB(k)
+        return nil if (self.index-k) < 0
+        return self.tokens[self.index-k]
+    end
+    def LT(k)
+        self.lazyInit()
+        return nil if k==0
+        return self.LB(-k) if k < 0
+        i = self.index + k - 1
+        self.sync(i)
+        if i >= self.tokens.length then # return EOF token
+            # EOF must be last token
+            return self.tokens[self.tokens.length-1]
+        end
+        return self.tokens[i]
+    end
+    # Allowed derived classes to modify the behavior of operations which change
+    # the current stream position by adjusting the target token index of a seek
+    # operation. The default implementation simply returns {@code i}. If an
+    # exception is thrown in this method, the current stream index should not be
+    # changed.
+    #
+    # <p>For example, {@link CommonTokenStream} overrides this method to ensure that
+    # the seek target is always an on-channel token.</p>
+    #
+    # @param i The target token index.
+    # @return The adjusted target token index.
+
+    def adjustSeekIndex(i)
+        return i
+    end
+
+    def lazyInit
+        if self.index == -1 then
+            self.setup()
+        end
+    end
+
+    def setup()
+        self.sync(0)
+        self.index = self.adjustSeekIndex(0)
+    end
+
+    # Reset this token stream by setting its token source.#/
+    def setTokenSource(tokenSource)
+        self.tokenSource = tokenSource
+        self.tokens = []
+        self.index = -1
+    end
+
+
+
+    # Given a starting index, return the index of the next token on channel.
+    #  Return i if tokens[i] is on channel.  Return -1 if there are no tokens
+    #  on channel between i and EOF.
+    #/
+    def nextTokenOnChannel(i, channel)
+        self.sync(i)
+        return -1 if i>=self.tokens.length 
+        token = self.tokens[i]
+        while token.channel!=self.channel do
+            return -1 if token.type==Token::EOF
+            i = i + 1
+            self.sync(i)
+            token = self.tokens[i]
+        end
+        return i
+    end
+    # Given a starting index, return the index of the previous token on channel.
+    #  Return i if tokens[i] is on channel. Return -1 if there are no tokens
+    #  on channel between i and 0.
+    def previousTokenOnChannel(i, channel)
+        while i>=0 and self.tokens[i].channel!=channel do
+            i = i - 1
+        end
+        return i
+    end
+    # Collect all tokens on specified channel to the right of
+    #  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL or
+    #  EOF. If channel is -1, find any non default channel token.
+    def getHiddenTokensToRight(tokenIndex, channel=-1)
+        self.lazyInit()
+        if self.tokenIndex<0 or tokenIndex>=self.tokens.length then
+            raise Exception.new("#{tokenIndex} not in 0..#{self.tokens.length-1}")
+        end
+        nextOnChannel = self.nextTokenOnChannel(tokenIndex + 1, Lexer::DEFAULT_TOKEN_CHANNEL)
+        from_ = tokenIndex+1
+        # if none onchannel to right, nextOnChannel=-1 so set to = last token
+        if nextOnChannel==-1 
+            to = self.tokens.length-1
+        else 
+            to = nextOnChannel
+        end
+        return self.filterForChannel(from_, to, channel)
+    end
+
+    # Collect all tokens on specified channel to the left of
+    #  the current token up until we see a token on DEFAULT_TOKEN_CHANNEL.
+    #  If channel is -1, find any non default channel token.
+    def getHiddenTokensToLeft(tokenIndex, channel=-1)
+        self.lazyInit()
+        if tokenIndex<0 or tokenIndex>=self.tokens.length
+            raise Exception.new("#{tokenIndex} not in 0..#{self.tokens.length-1}")
+        end
+        prevOnChannel = self.previousTokenOnChannel(tokenIndex - 1, Lexer::DEFAULT_TOKEN_CHANNEL)
+        return nil if prevOnChannel == tokenIndex - 1
+        
+        # if none on channel to left, prevOnChannel=-1 then from=0
+        from_ = prevOnChannel+1
+        to = tokenIndex-1
+        return self.filterForChannel(from_, to, channel)
+    end
+
+    def filterForChannel(left, right, channel)
+        hidden = []
+        for i in left..right do
+            t = self.tokens[i]
+            if channel==-1 then
+                if t.channel!= Lexer::DEFAULT_TOKEN_CHANNEL
+                    hidden.push(t)
+                end
+            elsif t.channel==channel then
+                    hidden.push(t)
+            end
+        end
+        return nil if hidden.length==0 
+        return hidden
+    end
+
+    def getSourceName
+        return self.tokenSource.getSourceName()
+    end
+
+    # Get the text of all tokens in this buffer.#/
+    def getText(interval=nil)
+        self.lazyInit()
+        self.fill()
+        if interval.nil?
+            interval = [0, self.tokens.length-1]
+        end
+        start = interval[0]
+        if start.kind_of? Token
+            start = start.tokenIndex
+        end
+        stop = interval[1]
+        if stop.kind_of? Token
+            stop = stop.tokenIndex
+        end
+        if start.nil? or stop.nil? or start<0 or stop<0
+            return ""
+        end
+        if stop >= self.tokens.length
+            stop = self.tokens.length-1
+        end
+        StringIO.open  do |buf|
+            for i in start..stop do
+                t = self.tokens[i]
+                break if t.type==Token::EOF
+                buf.write(t.text)
+            end
+            return buf.string()
+        end
+    end
+    # Get all tokens from lexer until EOF#/
+    def fill
+        self.lazyInit()
+        while fetch(1000)==1000 do
+            nil
+        end
+    end
+end