antlr4 0.9.2

data/lib/antlr4/TokenFactory.rb

@@ -0,0 +1,3 @@
+class TokenFactory
+end
+

data/lib/antlr4/TokenSource.rb

@@ -0,0 +1,4 @@
+
+class TokenSource < Recognizer
+end
+

data/lib/antlr4/TokenStream.rb

@@ -0,0 +1,3 @@
+class TokenStream
+end
+

data/lib/antlr4/TraceListener.rb

@@ -0,0 +1,23 @@
+class TraceListener < ParseTreeListener
+    
+    attr :parser
+    def initialize(parser=nil)
+        super()
+        if parser then
+          @parser = parser 
+        end
+    end
+    def enterEveryRule(ctx)
+        puts "enter   #{parser.ruleNames[ctx.ruleIndex]}, LT(1)=#{parser.input.LT(1).text.to_s}"
+    end
+
+    def visitTerminal(node)
+        puts "consume #{node.symbol} rule #{parser.ruleNames[parser.ctx.ruleIndex]}"
+    end
+    def visitErrorNode(node)
+    end
+
+    def exitEveryRule(ctx)
+        puts "exit    #{parser.ruleNames[ctx.ruleIndex]}, LT(1)=#{parser.input.LT(1).text}"
+    end
+end

data/lib/antlr4/atn/ATN.rb

@@ -0,0 +1,133 @@
+
+class ATN
+    INVALID_ALT_NUMBER = 0
+    def self.INVALID_ALT_NUMBER 
+      ATN::INVALID_ALT_NUMBER 
+    end
+    # Used for runtime deserialization of ATNs from strings#/
+    attr_accessor :grammarType, :maxTokenType, :states, :decisionToState
+    attr_accessor :ruleToStartState, :ruleToStopState, :modeNameToStartState 
+    attr_accessor :ruleToTokenType, :lexerActions, :modeToStartState 
+    def initialize(grammarType, maxTokenType)
+        # The type of the ATN.
+        @grammarType = grammarType
+        # The maximum value for any symbol recognized by a transition in the ATN.
+        @maxTokenType = maxTokenType
+        @states = Array.new
+        # Each subrule/rule is a decision point and we must track them so we
+        #  can go back later and build DFA predictors for them.  This includes
+        #  all the rules, subrules, optional blocks, ()+, ()* etc...
+        @decisionToState = Array.new
+        # Maps from rule index to starting state number.
+        @ruleToStartState = Array.new
+        # Maps from rule index to stop state number.
+        @ruleToStopState = nil
+        @modeNameToStartState = Hash.new
+        # For lexer ATNs, this maps the rule index to the resulting token type.
+        # For parser ATNs, this maps the rule index to the generated bypass token
+        # type if the
+        # {@link ATNDeserializationOptions#isGenerateRuleBypassTransitions}
+        # deserialization option was specified; otherwise, this is {@code null}.
+        @ruleToTokenType = nil
+        # For lexer ATNs, this is an array of {@link LexerAction} objects which may
+        # be referenced by action transitions in the ATN.
+        @lexerActions = nil
+        @modeToStartState = Array.new
+    end
+    # Compute the set of valid tokens that can occur starting in state {@code s}.
+    #  If {@code ctx} is null, the set of tokens will not include what can follow
+    #  the rule surrounding {@code s}. In other words, the set will be
+    #  restricted to tokens reachable staying within {@code s}'s rule.
+    def nextTokensInContext(s, ctx)
+        require 'antlr4/LL1Analyzer'
+        anal = LL1Analyzer.new(self)
+        return anal.LOOK(s, ctx=ctx)
+    end
+
+    # Compute the set of valid tokens that can occur starting in {@code s} and
+    # staying in same rule. {@link Token#EPSILON} is in set if we reach end of
+    # rule.
+    def nextTokensNoContext(s)
+        if not s.nextTokenWithinRule.nil? 
+          return s.nextTokenWithinRule 
+        end
+        s.nextTokenWithinRule = self.nextTokensInContext(s,nil )
+        s.nextTokenWithinRule.readonly = true
+        return s.nextTokenWithinRule
+    end
+    def nextTokens(s, ctx= nil) 
+        if ctx.nil? 
+            return self.nextTokensNoContext(s)
+        else
+            return self.nextTokensInContext(s, ctx)
+        end
+    end
+    def addState(state)
+        if not state.nil? 
+            state.atn = self
+            state.stateNumber = self.states.length()
+        end
+        self.states.push(state)
+    end
+
+    def removeState(state)
+        self.states[state.stateNumber] = nil 
+    end
+    def defineDecisionState(s)
+        self.decisionToState.push(s)
+        s.decision = self.decisionToState.length()-1
+        return s.decision
+    end
+    def getDecisionState(decision)
+        if self.decisionToState.length==0
+            return  nil
+        else
+            return self.decisionToState[decision]
+        end
+    end
+    # Computes the set of input symbols which could follow ATN state number
+    # {@code stateNumber} in the specified full {@code context}. This method
+    # considers the complete parser context, but does not evaluate semantic
+    # predicates (i.e. all predicates encountered during the calculation are
+    # assumed true). If a path in the ATN exists from the starting state to the
+    # {@link RuleStopState} of the outermost context without matching any
+    # symbols, {@link Token#EOF} is added to the returned set.
+    #
+    # <p>If {@code context} is {@code null}, it is treated as
+    # {@link ParserRuleContext#EMPTY}.</p>
+    #
+    # @param stateNumber the ATN state number
+    # @param context the full parse context
+    # @return The set of potentially valid input symbols which could follow the
+    # specified state in the specified context.
+    # @throws IllegalArgumentException if the ATN does not contain a state with
+    # number {@code stateNumber}
+    #/
+    def getExpectedTokens(stateNumber, ctx)
+        if stateNumber < 0 or stateNumber >= self.states.length
+            raise Exception.new("Invalid state number: #{stateNumber}.")
+        end
+        s = self.states[stateNumber]
+        following = self.nextTokens(s)
+        if not following.member? Token::EPSILON 
+            return following
+        end
+        expected = IntervalSet.new()
+        expected.addSet(following)
+        #puts expected.to_s
+        expected.remove(Token::EPSILON)
+        #puts expected.to_s
+        while ( ctx and ctx.invokingState >= 0 and following.member?(Token::EPSILON) ) do
+            invokingState = self.states[ctx.invokingState]
+            rt = invokingState.transitions[0]
+            following = self.nextTokens(rt.followState)
+            expected.addSet(following)
+            expected.remove(Token::EPSILON)
+            ctx = ctx.parentCtx
+        end
+        if following.member?(Token::EPSILON) then
+            expected.addOne(Token::EOF)
+        end
+        return expected
+    end
+end

data/lib/antlr4/atn/ATNConfig.rb

@@ -0,0 +1,146 @@
+# A tuple: (ATN state, predicted alt, syntactic, semantic context).
+#  The syntactic context is a graph-structured stack node whose
+#  path(s) to the root is the rule invocation(s)
+#  chain used to arrive at the state.  The semantic context is
+#  the tree of semantic predicates encountered before reaching
+#  an ATN state.
+
+class ATNConfig
+
+    def self.createConfigState(config, state)
+        new(state,nil,nil,nil,config)
+    end
+    attr_accessor :reachesIntoOuterContext
+    attr_reader   :state, :alt, :context, :semanticContext
+    attr_reader   :hashcode
+    def initialize(state=nil, alt=nil, context=nil, semantic=nil, config=nil)
+        if config  then
+            state = config.state if state.nil?
+            alt = config.alt if alt.nil?
+            context = config.context if context.nil? 
+            semantic = config.semanticContext if semantic.nil?
+            semantic = SemanticContext.NONE if semantic.nil?
+            @reachesIntoOuterContext = config.reachesIntoOuterContext
+        else
+        # We cannot execute predicates dependent upon local context unless
+        # we know for sure we are in the correct context. Because there is
+        # no way to do this efficiently, we simply cannot evaluate
+        # dependent predicates unless we are in the rule that initially
+        # invokes the ATN simulator.
+        #
+        # closure() tracks the depth of how far we dip into the
+        # outer context: depth > 0.  Note that it may not be totally
+        # accurate depth since I don't ever decrement. TODO: make it a boolean then
+            @reachesIntoOuterContext = 0 
+        end
+
+        #if not isinstance(state, ATNState):
+        #    pass
+        # The ATN state associated with this configuration#/
+        @state = state
+        # What alt (or lexer rule) is predicted by this configuration#/
+        @alt = alt
+        # The stack of invoking states leading to the rule/states associated
+        #  with this config.  We track only those contexts pushed during
+        #  execution of the ATN simulator.
+        @context = context
+        @semanticContext = semantic
+        mk_hashcode 
+    end
+
+    # An ATN configuration is equal to another if both have
+    #  the same state, they predict the same alternative, and
+    #  syntactic/semantic contexts are the same.
+    def eql?(other)
+        self == other
+    end
+    def ==(other)
+        self.equal?(other) or  \
+        other.kind_of?(ATNConfig) and \
+        (@state.stateNumber==other.state.stateNumber      and  
+              @alt==other.alt and (@context==other.context) and 
+              @semanticContext==other.semanticContext )
+    end
+    def context=(o)
+      @context = o
+      mk_hashcode
+    end
+
+    def mk_hashcode
+        @hashcode = "#{@state.stateNumber}/#{@alt}/#{@context}/#{@semanticContext}".hash
+    end
+    def hash
+        @hashcode 
+    end
+    def toString(recog=nil, showAlt=true)
+        to_s(recog,showAlt)
+    end
+    def to_s(recog=nil, showAlt=true) 
+        StringIO.open  do |buf|
+            buf.write('(')
+            buf.write(self.state.to_s)
+            if showAlt then
+              buf.write(",")
+              buf.write(self.alt.to_s)
+            end
+            if self.context then
+                buf.write(",[")
+                buf.write(self.context.to_s)
+                buf.write("]")
+            end
+            if self.semanticContext and (! self.semanticContext.equal?  SemanticContext.NONE) then
+                buf.write(",")
+                buf.write(self.semanticContext.to_s)
+            end
+            if self.reachesIntoOuterContext>0 then
+                buf.write(",up=")
+                buf.write(self.reachesIntoOuterContext.to_s)
+            end
+            buf.write(')')
+            return buf.string()
+        end
+    end
+end
+# need a forward declaration
+
+class LexerATNConfig < ATNConfig
+
+    attr_accessor :passedThroughNonGreedyDecision, :lexerActionExecutor 
+    def initialize(state, alt=nil, context=nil, semantic=SemanticContext.NONE, _lexerActionExecutor=nil, config=nil)
+        super(state, alt, context, semantic, config)
+        if config then 
+            _lexerActionExecutor = config.lexerActionExecutor if _lexerActionExecutor.nil? 
+            @passedThroughNonGreedyDecision = self.checkNonGreedyDecision(config, state)
+        else
+           @passedThroughNonGreedyDecision = false 
+        end
+        # This is the backing field for {@link #getLexerActionExecutor}.
+        @lexerActionExecutor = _lexerActionExecutor
+    end
+
+    def hash
+        b = 0
+        b = 1 if self.passedThroughNonGreedyDecision 
+        [self.state.stateNumber, self.alt, self.context, 
+          self.semanticContext, b, self.lexerActionExecutor
+        ].map(&:to_s).join('').hash
+    end
+    def eql?(other)
+        self == other
+    end
+    def ==(other)
+        return true if self.equal? other
+        return false unless other.kind_of? LexerATNConfig
+        if self.passedThroughNonGreedyDecision != other.passedThroughNonGreedyDecision
+            return false
+        end
+        if self.lexerActionExecutor == other.lexerActionExecutor
+              super(other)
+        else
+            false
+        end
+    end
+    def checkNonGreedyDecision(source, target)
+        source.passedThroughNonGreedyDecision or (target.kind_of?(DecisionState) and target.nonGreedy )
+    end
+end

data/lib/antlr4/atn/ATNConfigSet.rb

@@ -0,0 +1,215 @@
+# Specialized {@link Set}{@code <}{@link ATNConfig}{@code >} that can track
+# info about the set, with support for combining similar configurations using a
+# graph-structured stack.
+
+class ATNConfigSet
+    # The reason that we need this is because we don't want the hash map to use
+    # the standard hash code and equals. We need all configurations with the same
+    # {@code (s,i,_,semctx)} to be equal. Unfortunately, this key effectively doubles
+    # the number of objects associated with ATNConfigs. The other solution is to
+    # use a hash table that lets us specify the equals/hashcode operation.
+    extend Forwardable
+
+    def_delegators :@configs, :size, :each, :map, :length
+
+
+    attr_accessor :configLookup, :fullCtx, :readonly, :configs, :uniqueAlt
+    attr_accessor :conflictingAlts, :hasSemanticContext, :dipsIntoOuterContext
+    attr_accessor :cachedHashCode
+
+    include PredictionContextFunctions
+    def initialize(_fullCtx=true)
+        # All configs but hashed by (s, i, _, pi) not including context. Wiped out
+        # when we go readonly as this set becomes a DFA state.
+        self.configLookup = Set.new()
+        # Indicates that this configuration set is part of a full context
+        #  LL prediction. It will be used to determine how to merge $. With SLL
+        #  it's a wildcard whereas it is not for LL context merge.
+        self.fullCtx = _fullCtx
+        # Indicates that the set of configurations is read-only. Do not
+        #  allow any code to manipulate the set; DFA states will point at
+        #  the sets and they must not change. This does not protect the other
+        #  fields; in particular, conflictingAlts is set after
+        #  we've made this readonly.
+        self.readonly = false
+        # Track the elements as they are added to the set; supports get(i)#/
+        self.configs = Array.new
+        # TODO: these fields make me pretty uncomfortable but nice to pack up info together, saves recomputation
+        # TODO: can we track conflicts as they are added to save scanning configs later?
+        self.uniqueAlt = 0
+        self.conflictingAlts = nil
+
+        # Used in parser and lexer. In lexer, it indicates we hit a pred
+        # while computing a closure operation.  Don't make a DFA state from this.
+        self.hasSemanticContext = false
+        self.dipsIntoOuterContext = false
+        self.cachedHashCode = -1
+    end
+    # Adding a new config means merging contexts with existing configs for
+    # {@code (s, i, pi, _)}, where {@code s} is the
+    # {@link ATNConfig#state}, {@code i} is the {@link ATNConfig#alt}, and
+    # {@code pi} is the {@link ATNConfig#semanticContext}. We use
+    # {@code (s,i,pi)} as key.
+    #
+    # <p>This method updates {@link #dipsIntoOuterContext} and
+    # {@link #hasSemanticContext} when necessary.</p>
+    #/
+    def add(config, mergeCache=nil)
+        raise Exception.new("This set is readonly") if self.readonly
+        if ! config.semanticContext.equal? SemanticContext.NONE
+            self.hasSemanticContext = true
+        end
+        if config.reachesIntoOuterContext > 0
+            self.dipsIntoOuterContext = true
+        end
+        existing = self.getOrAdd(config)
+        if existing.equal? config
+            self.cachedHashCode = -1
+            self.configs.push(config)  # track order here
+            return true
+        end
+        # a previous (s,i,pi,_), merge with it and save result
+        rootIsWildcard =  self.fullCtx.nil?
+        merged = merge(existing.context, config.context, rootIsWildcard, mergeCache)
+        # no need to check for existing.context, config.context in cache
+        # since only way to create new graphs is "call rule" and here. We
+        # cache at both places.
+        existing.reachesIntoOuterContext = [existing.reachesIntoOuterContext, config.reachesIntoOuterContext].max
+        existing.context = merged # replace context; no need to alt mapping
+        return true
+    end
+    def getOrAdd(config)
+        if self.configLookup.member?(config) then
+            self.configLookup.grep(config).first
+        else
+            self.configLookup.add(config)
+            config
+        end
+    end
+    def getStates
+        states = Set.new()
+        self.configs.each {|c| states.add(c.state) }
+        return states
+    end
+    def getPredicates
+        preds = Array.new
+        self.configs.each{|c|
+            if not c.semanticContext.equal?(SemanticContext.NONE) then
+                preds.push(c.semanticContext)
+            end
+        }
+        return preds
+    end
+    def get(i)
+        return self.configs[i]
+    end
+
+    def optimizeConfigs(interpreter)
+        raise IllegalStateException.new("This set is readonly") if self.readonly
+      
+        return if self.configLookup.empty?
+        self.configs.each {|config|
+            config.context = interpreter.getCachedContext(config.context)
+        }
+    end
+
+    def addAll(coll)
+        coll.each {|c| self.add(c) }
+        return false
+    end
+    def eql?(other)
+      self == other
+    end
+    def ==(other)
+        self.equal? other or other.kind_of? ATNConfigSet and \
+            self.configs and \
+            self.configs==other.configs and \
+            self.fullCtx == other.fullCtx and \
+            self.uniqueAlt == other.uniqueAlt and \
+            self.conflictingAlts == other.conflictingAlts and \
+            self.hasSemanticContext == other.hasSemanticContext and \
+            self.dipsIntoOuterContext == other.dipsIntoOuterContext
+    end
+    def hash
+        if self.readonly
+            if self.cachedHashCode == -1
+                self.cachedHashCode = self.hashConfigs()
+            end
+            return self.cachedHashCode
+        end
+        return self.hashConfigs()
+    end
+    def hashConfigs
+        StringIO.open  do |buf|
+          self.configs.each { |cfg| 
+              buf.write(cfg.to_s)
+          }
+          return buf.string().hash
+        end
+    end
+
+#    def length
+#        return self.configs.length
+#    end
+
+    def empty?
+        self.configs.empty?
+    end
+    def isEmpty
+        return self.configs.empty?
+    end
+
+#    def __contains__(self, item)
+#        if self.configLookup is None
+#            raise UnsupportedOperationException("This method is not implemented for readonly sets.")
+#        return item in self.configLookup
+#    end
+    def containsFast( obj)
+        if self.configLookup.nil? 
+            raise UnsupportedOperationException.new("This method is not implemented for readonly sets.")
+        end
+        return self.configLookup.containsFast(obj)
+    end
+
+    def clear
+        raise IllegalStateException.new("This set is readonly") if self.readonly
+        self.configs.clear()
+        self.cachedHashCode = -1
+        self.configLookup.clear()
+    end
+    def setReadonly(readonly)
+        self.readonly = readonly
+        self.configLookup = nil # can't mod, no need for lookup cache
+    end
+    def to_s
+        StringIO.open  do |buf|
+            buf.write("[")
+            buf.write( @configs.map {|x| x.to_s}.join(', ') )
+            buf.write(']')  # @#{@configs.length}")
+            if self.hasSemanticContext
+                buf.write(",hasSemanticContext=")
+                buf.write(self.hasSemanticContext.to_s)
+            end
+            if self.uniqueAlt!=ATN::INVALID_ALT_NUMBER
+                buf.write(",uniqueAlt=")
+                buf.write(self.uniqueAlt.to_s())
+            end
+            if self.conflictingAlts then
+                buf.write(",conflictingAlts=")
+                buf.write(self.conflictingAlts.to_s)
+            end
+            if self.dipsIntoOuterContext
+                buf.write(",dipsIntoOuterContext")
+            end
+            return buf.string()
+        end
+    end
+end
+
+class OrderedATNConfigSet < ATNConfigSet
+
+    def initialize()
+        super()
+    end
+end
+