antlr4 0.9.2

data/lib/antlr4/LL1Analyzer.rb

@@ -0,0 +1,177 @@
+
+class LL1Analyzer 
+    #  Special value added to the lookahead sets to indicate that we hit
+    #  a predicate during analysis if {@code seeThruPreds==false}.
+    HIT_PRED = Token::INVALID_TYPE
+
+    attr_accessor :atn
+    def initialize(atn)
+        @atn = atn
+    end
+
+    #*
+    # Calculates the SLL(1) expected lookahead set for each outgoing transition
+    # of an {@link ATNState}. The returned array has one element for each
+    # outgoing transition in {@code s}. If the closure from transition
+    # <em>i</em> leads to a semantic predicate before matching a symbol, the
+    # element at index <em>i</em> of the result will be {@code null}.
+    #
+    # @param s the ATN state
+    # @return the expected symbols for each outgoing transition of {@code s}.
+    #/
+    def getDecisionLookahead(s)
+        return nil if s.nil? 
+
+        count = s.transitions.length()
+        look = Array.new
+        for alt in 0..count-1
+            look[alt] = Set.new()
+            lookBusy = Set.new()
+            seeThruPreds = false # fail to get lookahead upon pred
+            self._LOOK(s.transition(alt).target, nil, PredictionContext.EMPTY, \
+                  look[alt], lookBusy, Set.new(), seeThruPreds, false)
+            # Wipe out lookahead for this alternative if we found nothing
+            # or we had a predicate when we !seeThruPreds
+            if look[alt].length==0 or look[alt].member? LL1Analyzer::HIT_PRED then
+                look[alt] = nil
+            end
+        end
+        return look
+    end
+
+    #*
+    # Compute set of tokens that can follow {@code s} in the ATN in the
+    # specified {@code ctx}.
+    #
+    # <p>If {@code ctx} is {@code null} and the end of the rule containing
+    # {@code s} is reached, {@link Token#EPSILON} is added to the result set.
+    # If {@code ctx} is not {@code null} and the end of the outermost rule is
+    # reached, {@link Token#EOF} is added to the result set.</p>
+    #
+    # @param s the ATN state
+    # @param stopState the ATN state to stop at. This can be a
+    # {@link BlockEndState} to detect epsilon paths through a closure.
+    # @param ctx the complete parser context, or {@code null} if the context
+    # should be ignored
+    #
+    # @return The set of tokens that can follow {@code s} in the ATN in the
+    # specified {@code ctx}.
+    #/
+    def LOOK(s, stopState=nil, ctx=nil )
+        r = IntervalSet.new()
+        seeThruPreds = true # ignore preds; get all lookahead
+        if not ctx.nil? then
+            lookContext = PredictionContextFromRuleContext.new(s.atn, ctx) 
+        else 
+            lookContext = nil
+        end
+        # lookContext = PredictionContextFromRuleContext(s.atn, ctx) if ctx is not None else None
+        self._LOOK(s, stopState, lookContext, r, Set.new(), Set.new(), seeThruPreds, true)
+        return r
+    end
+
+    #*
+    # Compute set of tokens that can follow {@code s} in the ATN in the
+    # specified {@code ctx}.
+    #
+    # <p>If {@code ctx} is {@code null} and {@code stopState} or the end of the
+    # rule containing {@code s} is reached, {@link Token#EPSILON} is added to
+    # the result set. If {@code ctx} is not {@code null} and {@code addEOF} is
+    # {@code true} and {@code stopState} or the end of the outermost rule is
+    # reached, {@link Token#EOF} is added to the result set.</p>
+    #
+    # @param s the ATN state.
+    # @param stopState the ATN state to stop at. This can be a
+    # {@link BlockEndState} to detect epsilon paths through a closure.
+    # @param ctx The outer context, or {@code null} if the outer context should
+    # not be used.
+    # @param look The result lookahead set.
+    # @param lookBusy A set used for preventing epsilon closures in the ATN
+    # from causing a stack overflow. Outside code should pass
+    # {@code new HashSet<ATNConfig>} for this argument.
+    # @param calledRuleStack A set used for preventing left recursion in the
+    # ATN from causing a stack overflow. Outside code should pass
+    # {@code new BitSet()} for this argument.
+    # @param seeThruPreds {@code true} to true semantic predicates as
+    # implicitly {@code true} and "see through them", otherwise {@code false}
+    # to treat semantic predicates as opaque and add {@link #HIT_PRED} to the
+    # result if one is encountered.
+    # @param addEOF Add {@link Token#EOF} to the result if the end of the
+    # outermost context is reached. This parameter has no effect if {@code ctx}
+    # is {@code null}.
+    #/
+    def _LOOK(s, stopState, ctx, look, lookBusy, \
+                     calledRuleStack, seeThruPreds, addEOF)
+        c = ATNConfig.new(s, 0, ctx)
+
+        return if lookBusy.member? c
+      
+        lookBusy.add(c)
+
+        if s == stopState then
+            if ctx.nil?  then
+                look.addOne(Token::EPSILON)
+                return
+            elsif ctx.isEmpty() and addEOF
+                look.addOne(Token::EOF)
+                return
+            end
+        end
+
+        if s.kind_of? RuleStopState then
+            if ctx.nil? then 
+                look.addOne(Token::EPSILON)
+                return
+            elsif ctx.isEmpty() and addEOF
+                look.addOne(Token::EOF)
+                return
+            end
+            if PredictionContext.EMPTY != ctx
+                # run thru all possible stack tops in ctx
+                for i in 0..ctx.length-1  do
+                    returnState = self.atn.states[ctx.getReturnState(i)]
+                    removed = calledRuleStack.member? returnState.ruleIndex 
+                    begin
+                        calledRuleStack.delete(returnState.ruleIndex)
+                        self._LOOK(returnState, stopState, ctx.getParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
+                    ensure 
+                        calledRuleStack.add(returnState.ruleIndex) if removed
+                    end
+                end
+                return
+            end
+        end
+        for t in s.transitions do 
+            if t.class == RuleTransition then
+                next if calledRuleStack.member? t.target.ruleIndex 
+
+                newContext = SingletonPredictionContext.create(ctx, t.followState.stateNumber)
+
+                begin
+                    calledRuleStack.add(t.target.ruleIndex)
+                    self._LOOK(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
+                ensure  
+                    calledRuleStack.remove(t.target.ruleIndex)
+                end
+            elsif t.kind_of? AbstractPredicateTransition then
+                if seeThruPreds
+                    self._LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
+                else
+                    look.addOne(LL1Analyzer::HIT_PRED)
+                end
+            elsif t.isEpsilon
+                self._LOOK(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
+            elsif t.class == WildcardTransition
+                look.addRange( Token::MIN_USER_TOKEN_TYPE..self.atn.maxTokenType )
+            else
+                set = t.label
+                if not set.nil? then
+                    if t.kind_of? NotSetTransition then
+                        set = set.complement IntervalSet.of(Token::MIN_USER_TOKEN_TYPE, self.atn.maxTokenType)
+                    end
+                    look.addSet(set)
+                end
+            end
+        end
+    end
+end

data/lib/antlr4/Lexer.rb

@@ -0,0 +1,335 @@
+# A lexer is recognizer that draws input symbols from a character stream.
+#  lexer grammars result in a subclass of self object. A Lexer object
+#  uses simplified match() and error recovery mechanisms in the interest
+#  of speed.
+
+class Lexer < TokenSource
+
+    DEFAULT_MODE = 0
+    MORE = -2
+    SKIP = -3
+
+    DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL
+    HIDDEN = Token::HIDDEN_CHANNEL
+    MIN_CHAR_VALUE = "\u0000"
+    MAX_CHAR_VALUE = "\uFFFE"
+
+    attr_accessor :input, :factory, :tokenFactorySourcePair #, :interp
+    attr_accessor :token, :tokenStartCharIndex, :tokenStartLine, :tokenStartColumn 
+    attr_accessor :hitEOF, :channel,:type, :modeStack, :mode, :text
+
+    def initialize(_input)
+        super()
+        @input = _input
+        @factory = CommonTokenFactory.DEFAULT
+        @tokenFactorySourcePair = [self, _input]
+
+        @interp = nil # child classes must populate this
+        
+        # The goal of all lexer rules/methods is to create a token object.
+        #  self is an instance variable as multiple rules may collaborate to
+        #  create a single token.  nextToken will return self object after
+        #  matching lexer rule(s).  If you subclass to allow multiple token
+        #  emissions, then set self to the last token to be matched or
+        #  something nonnull so that the auto token emit mechanism will not
+        #  emit another token.
+        @token = nil
+
+        # What character index in the stream did the current token start at?
+        #  Needed, for example, to get the text for current token.  Set at
+        #  the start of nextToken.
+        @tokenStartCharIndex = -1
+
+        # The line on which the first character of the token resides#/
+        @tokenStartLine = -1
+
+        # The character position of first character within the line#/
+        @tokenStartColumn = -1
+
+        # Once we see EOF on char stream, next token will be EOF.
+        #  If you have DONE : EOF ; then you see DONE EOF.
+        @hitEOF = false
+
+        # The channel number for the current token#/
+        @channel = Token::DEFAULT_CHANNEL
+
+        # The token type for the current token#/
+        @type = Token::INVALID_TYPE
+
+        @modeStack = Array.new
+        @mode = Lexer::DEFAULT_MODE
+
+        # You can set the text for the current token to override what is in
+        #  the input char buffer.  Use setText() or can set self instance var.
+        #/
+        @text = nil
+    end
+
+    def reset
+        # wack Lexer state variables
+        if not self.input.nil? then 
+            self.input.seek(0) # rewind the input
+        end
+        self.token = nil
+        self.type = Token::INVALID_TYPE
+        self.channel = Token::DEFAULT_CHANNEL
+        self.tokenStartCharIndex = -1
+        self.tokenStartColumn = -1
+        self.tokenStartLine = -1
+        self.text = nil
+
+        self.hitEOF = false
+        self.mode = Lexer::DEFAULT_MODE
+        self.modeStack = Array.new
+
+        self.interp.reset()
+    end
+
+    # Return a token from self source; i.e., match a token on the char
+    #  stream.
+    def nextToken
+        if self.input.nil? 
+            raise IllegalStateException.new("nextToken requires a non-null input stream.")
+        end
+
+        # Mark start location in char stream so unbuffered streams are
+        # guaranteed at least have text of current token
+        tokenStartMarker = self.input.mark()
+        begin
+            while true do 
+                if self.hitEOF then
+                    self.emitEOF()
+                    return self.token
+                end
+                self.token = nil
+                self.channel = Token::DEFAULT_CHANNEL
+                self.tokenStartCharIndex = self.input.index
+                self.tokenStartColumn = self.interp.column
+                self.tokenStartLine = self.interp.line
+                self.text = nil
+                continueOuter = false
+                while true do 
+                    self.type = Token::INVALID_TYPE
+                    ttype = Lexer::SKIP
+                    begin
+                        ttype = self.interp.match(self.input, self.mode)
+                    rescue LexerNoViableAltException => e
+                        self.notifyListeners(e)		# report error
+                        self.recover(e)
+                    end
+                    if self.input.LA(1)==Token::EOF then
+                        self.hitEOF = true
+                    end
+                    if self.type == Token::INVALID_TYPE
+                        self.type = ttype
+                  
+                    end
+                    if self.type == Lexer::SKIP
+                        continueOuter = true
+                        break
+                    end
+                    if self.type!= Lexer::MORE
+                        break
+                    end
+                end
+                next if continueOuter
+                self.emit() if self.token.nil?
+                return self.token
+            end
+        ensure  
+            # make sure we release marker after match or
+            # unbuffered char stream will keep buffering
+            self.input.release(tokenStartMarker)
+        end
+    end
+
+    # Instruct the lexer to skip creating a token for current lexer rule
+    #  and look for another token.  nextToken() knows to keep looking when
+    #  a lexer rule finishes with token set to SKIP_TOKEN.  Recall that
+    #  if token==null at end of any token rule, it creates one for you
+    #  and emits it.
+    #/
+    def skip
+        self.type = Lexer::SKIP
+    end
+    def more
+        self.type = Lexer::MORE
+    end
+    def pushMode(m)
+        if self.interp.debug then
+            puts "pushMode #{m}"
+        end
+        self.modeStack.push(self.mode)
+        self.mode = m
+    end
+    def popMode
+        if self.modeStack.empty? then
+            raise Exception.new("Empty Stack")
+        end
+        if self.interp.debug then
+            puts  "popMode back to #{self.modeStack.slice(0,self.modeStack.length-1)}"
+        end
+        self.mode = self.modeStack.pop() 
+        return self.mode
+    end
+
+    # Set the char stream and reset the lexer#/
+    def inputStream
+        return self.input
+    end
+
+    def inputStream=(input)
+        self.input = nil
+        self.tokenFactorySourcePair = [self, nil]
+        self.reset()
+        self.input = input
+        self.tokenFactorySourcePair = [self, self.input]
+    end
+
+    def sourceName
+        return self.input.sourceName
+    end
+
+    # By default does not support multiple emits per nextToken invocation
+    #  for efficiency reasons.  Subclass and override self method, nextToken,
+    #  and getToken (to push tokens into a list and pull from that list
+    #  rather than a single variable as self implementation does).
+    #/
+    def emitToken(token)
+        self.token = token
+    end
+
+    # The standard method called to automatically emit a token at the
+    #  outermost lexical rule.  The token object should point into the
+    #  char buffer start..stop.  If there is a text override in 'text',
+    #  use that to set the token's text.  Override self method to emit
+    #  custom Token objects or provide a new factory.
+    #/
+    def emit
+        t = self.factory.create(self.tokenFactorySourcePair, self.type, self.text, self.channel, self.tokenStartCharIndex,
+                                 self.getCharIndex()-1, self.tokenStartLine, self.tokenStartColumn)
+        self.emitToken(t)
+        return t
+    end
+
+    def emitEOF()
+        cpos = self.column
+        # The character position for EOF is one beyond the position of
+        # the previous token's last character
+        if not self.token.nil? then
+            n = self.token.stop - self.token.start + 1
+            cpos = self.token.column + n
+        end
+        eof = self.factory.create(self.tokenFactorySourcePair, Token::EOF, nil, Token::DEFAULT_CHANNEL, self.input.index,
+                                   self.input.index-1, self.line, cpos)
+        self.emitToken(eof)
+        return eof
+    end
+
+    def line
+        return self.interp.line
+    end
+
+    def line=(line)
+        self.interp.line = line
+    end
+
+    def column
+        return self.interp.column
+    end
+
+    def column=(column)
+        self.interp.column = column
+    end
+
+    # What is the index of the current character of lookahead?#/
+    def getCharIndex()
+        return self.input.index
+    end
+
+    # Return the text matched so far for the current token or any
+    #  text override.
+    def text
+        if not @text.nil? then
+            @text
+        else
+            self.interp.getText(self.input)
+        end
+    end
+
+    # Set the complete text of self token; it wipes any previous
+    #  changes to the text.
+    def text=(txt)
+        @text = txt
+    end
+
+    # Return a list of all Token objects in input char stream.
+    #  Forces load of all tokens. Does not include EOF token.
+    #/
+    def getAllTokens
+        tokens = Array.new
+        t = self.nextToken()
+        while t.type!=Token::EOF do
+            tokens.push(t)
+            t = self.nextToken()
+        end
+        return tokens
+    end
+    def notifyListeners(e) # :LexerNoViableAltException):
+        start = self.tokenStartCharIndex
+        stop = self.input.index
+        text = self.input.getText(start, stop)
+        msg = "token recognition error at: '#{self.getErrorDisplay(text) }'"
+        listener = self.getErrorListenerDispatch()
+        listener.syntaxError(self, nil, self.tokenStartLine, self.tokenStartColumn, msg, e)
+    end
+
+    def getErrorDisplay(s)
+        StringIO.open  do |buf|
+            s.chars.each{|c| buf.write(self.getErrorDisplayForChar(c)) }
+            return buf.string()
+        end
+    end
+    def getErrorDisplayForChar(c)
+        begin
+          cc = c[0].ord 
+        rescue ArgumentError
+          cc = "\ufffd".ord
+        end
+        if cc==Token::EOF then
+            return "<EOF>"
+        elsif c == "\n"
+            return "\\n"
+        elsif c=="\t"
+            return "\\t"
+        elsif c=="\r"
+            return "\\r"
+        else
+            return c
+        end
+    end
+    def getCharErrorDisplay(c)
+        return "'" + self.getErrorDisplayForChar(c) + "'"
+    end
+
+    # Lexers can normally match any char in it's vocabulary after matching
+    #  a token, so do the easy thing and just kill a character and hope
+    #  it all works out.  You can instead use the rule invocation stack
+    #  to do sophisticated error recovery if you are in a fragment rule.
+    #/
+    def recover(re) # :RecognitionException):
+        if self.input.LA(1) != Token::EOF then
+            if re.kind_of?  LexerNoViableAltException then
+                    # skip a char and try again
+                    self.interp.consume(self.input)
+            else
+                # TODO: Do we lose character or line position information?
+                self.input.consume()
+            end
+        end
+    end
+    def getRuleNames
+        self.ruleNames
+    end
+end
+