antlr4-runtime 0.1.0

data/lib/antlr4/runtime/lexer.rb

@@ -0,0 +1,257 @@
+require 'antlr4/runtime/recognizer'
+require 'antlr4/runtime/token'
+require 'antlr4/runtime/common_token_factory'
+require 'antlr4/runtime/lexer_no_viable_alt_exception'
+
+module Antlr4::Runtime
+
+  class Lexer < Recognizer
+    DEFAULT_MODE = 0
+    MORE = -2
+    SKIP = -3
+
+    DEFAULT_TOKEN_CHANNEL = Token::DEFAULT_CHANNEL
+    HIDDEN = Token::HIDDEN_CHANNEL
+    MIN_CHAR_VALUE = 0x0000
+    MAX_CHAR_VALUE = 0x10FFFF
+
+    attr_accessor :_input
+    attr_accessor :token
+    attr_accessor :_token_start_char_index
+    attr_accessor :_token_start_line
+    attr_accessor :_token_start_char_position_in_line
+    attr_accessor :_hit_eof
+    attr_accessor :_channel
+    attr_accessor :_type
+    attr_accessor :_mode_stack
+    attr_accessor :_mode
+    attr_accessor :_text
+
+    def reset
+      # wack Lexer state variables
+      unless @_input.nil?
+        @_input.seek(0) # rewind the input
+      end
+      @_token = nil
+      @_type = Token::INVALID_TYPE
+      @_channel = Token::DEFAULT_CHANNEL
+      @_token_start_char_index = -1
+      @_token_start_char_position_in_line = -1
+      @_token_start_line = -1
+      @_text = nil
+
+      @_hit_eof = false
+      @_mode = DEFAULT_MODE
+      @_mode_stack.clear
+
+      @_interp.reset unless @_interp.nil?
+    end
+
+    def initialize(input = nil)
+      super()
+      unless input.nil?
+        @_input = input
+        @_token_factory_source_pair = OpenStruct.new
+        @_token_factory_source_pair.a = self
+        @_token_factory_source_pair.b = input
+      end
+      @_mode_stack = []
+      reset
+      @_factory = CommonTokenFactory.instance
+    end
+
+    def next_token
+      if @_input.nil?
+        raise IllegalStateException, 'nextToken requires a non-nil input stream.'
+      end
+
+      # Mark start location in char stream so unbuffered streams are
+      # guaranteed at least have text of current token
+      token_start_marker = @_input.mark
+      begin
+        repeat_outer = true
+        repeat_outer = next_token_inner while repeat_outer
+        return @_token
+      ensure # make sure we release marker after match or
+        # unbuffered char stream will keep buffering
+        @_input.release(token_start_marker)
+      end
+    end
+
+    def next_token_inner
+      loop do
+        if @_hit_eof
+          emit_eof
+          return false
+        end
+
+        @_token = nil
+        @_channel = Token::DEFAULT_CHANNEL
+        @_token_start_char_index = @_input.index
+        @_token_start_char_position_in_line = @_interp.char_position_in_line
+        @_token_start_line = @_interp.line
+        @_text = nil
+        loop do
+          @_type = Token::INVALID_TYPE
+
+          begin
+            ttype = @_interp.match(@_input, @_mode)
+          rescue LexerNoViableAltException => e
+            notify_listeners(e) # report error
+            recover1(e)
+            ttype = SKIP
+          end
+          @_hit_eof = true if @_input.la(1) == IntStream::EOF
+          @_type = ttype if @_type == Token::INVALID_TYPE
+          return true if @_type == SKIP
+          break if @_type != MORE
+        end
+
+        emit if @_token.nil?
+        return false
+      end
+    end
+
+    def skip
+      @_type = SKIP
+    end
+
+    def more
+      @_type = MORE
+    end
+
+    def mode(m)
+      @_mode = m
+    end
+
+    def push_mode(m)
+      puts('pushMode ' + m) if LexerATNSimulator.debug
+      @_mode_stack.push(@_mode)
+      mode(m)
+    end
+
+    def pop_mode
+      raise EmptyStackException if @_mode_stack.empty?
+
+      puts('popMode back to ' + @_mode_stack[-1]) if LexerATNSimulator.debug
+      mode(@_mode_stack.pop)
+      @_mode
+    end
+
+    def input_stream(input)
+      @_input = nil
+      @_token_factory_source_pair = OpenStruct.new
+      @_token_factory_source_pair.a = self
+      @_token_factory_source_pair.b = @_input
+      reset
+      @_input = input
+      @_token_factory_source_pair.a = self
+      @_token_factory_source_pair.b = @_input
+    end
+
+    def source_name
+      @_input.get_source_name
+    end
+
+    def emit(token = nil)
+      if !token.nil?
+        @_token = token
+      else
+        @_token = @_factory.create(@_token_factory_source_pair, @_type, @_text, @_channel, @_token_start_char_index, char_index - 1, @_token_start_line, @_token_start_char_position_in_line)
+      end
+    end
+
+    def emit_eof
+      cpos = char_position_in_line
+      eof = @_factory.create(@_token_factory_source_pair, Token::EOF, nil, Token::DEFAULT_CHANNEL, @_input.index, @_input.index - 1, line, cpos)
+      emit(eof)
+      eof
+    end
+
+    def line
+      @_interp.line
+    end
+
+    def char_position_in_line
+      @_interp.char_position_in_line
+    end
+
+    def set_line(line)
+      @_interp.set_line(line)
+    end
+
+    def set_char_position_in_line(char_position_in_line)
+      @_interp.set_char_position_in_line(char_position_in_line)
+    end
+
+    def char_index
+      @_input.index
+    end
+
+    def text
+      return @_text unless @_text.nil?
+
+      @_interp.text(@_input)
+    end
+
+    def all_tokens
+      tokens = []
+      t = next_token
+      while t.type != Token::EOF
+        tokens << t
+        t = next_token
+      end
+      tokens
+    end
+
+    def recover1(_e)
+      if @_input.la(1) != IntStream::EOF
+        # skip a char and begin again
+        @_interp.consume(@_input)
+      end
+    end
+
+    def notify_listeners(e)
+      text = @_input.text(Interval.of(@_token_start_char_index, @_input.index))
+      msg = "token recognition error at: '" + error_display(text) + "'"
+
+      listener = error_listener_dispatch
+      listener.syntax_error(self, nil, @_token_start_line, @_token_start_char_position_in_line, msg, e)
+    end
+
+    def error_display(s)
+      buf = ''
+      s.chars.each do |c|
+        buf << error_display_char(c)
+      end
+      buf
+    end
+
+    def error_display_char(c)
+      s = ''
+      s << c
+      case c
+      when Token::EOF
+        s = '<EOF>'
+      when '\n'
+        s = '\\n'
+      when '\t'
+        s = '\\t'
+      when '\r'
+        s = '\\r'
+      else
+        # type code here
+      end
+      s
+    end
+
+    def char_error_display(c)
+      s = error_display_char(c)
+      "'" + s + "'"
+    end
+
+    def recover2(_re)
+      @_input.consume
+    end
+  end
+end

data/lib/antlr4/runtime/lexer_action.rb

@@ -0,0 +1,12 @@
+module Antlr4::Runtime
+  class LexerAction
+    def action_type
+    end
+
+    def position_dependent?
+    end
+
+    def execute(lexer)
+    end
+  end
+end

data/lib/antlr4/runtime/lexer_action_executor.rb

@@ -0,0 +1,75 @@
+require 'antlr4/runtime/lexer_indexed_custom_action'
+
+module Antlr4::Runtime
+
+  class LexerActionExecutor
+    attr_reader :lexer_actions
+    attr_reader :hash_code
+
+    def initialize(lexer_actions)
+      @lexer_actions = lexer_actions
+      @hash_code = MurmurHash.hash_objs(lexer_actions)
+    end
+
+    def self.append(lexer_action_executor, lexer_action)
+      return LexerActionExecutor.new([lexer_action]) if lexer_action_executor.nil?
+
+      lexer_actions = lexer_action_executor.lexer_actions.dup
+      lexer_actions << lexer_action
+      LexerActionExecutor.new(lexer_actions)
+    end
+
+    def fix_offset_before_match(offset)
+      updated_lexer_actions = nil
+      i = 0
+      while i < @lexer_actions.length
+        if @lexer_actions[i].position_dependent? && !(@lexer_actions[i].is_a? LexerIndexedCustomAction)
+          updated_lexer_actions = @lexer_actions.dup if updated_lexer_actions.nil?
+
+          updated_lexer_actions[i] = LexerIndexedCustomAction.new(offset, @lexer_actions[i])
+        end
+        i += 1
+      end
+
+      return self if updated_lexer_actions.nil?
+
+      LexerActionExecutor.new(updated_lexer_actions)
+    end
+
+    def execute(lexer, input, start_index)
+      requires_seek = false
+      stop_index = input.index
+      begin
+        i = 0
+        while i < @lexer_actions.length
+          lexerAction = @lexer_actions[i]
+          if lexerAction.is_a? LexerIndexedCustomAction
+            offset = lexerAction.getOffset
+            input.seek(start_index + offset)
+            requires_seek = ((start_index + offset) != stop_index)
+          else
+            if lexerAction.position_dependent?
+              input.seek(stop_index)
+              requires_seek = false
+            end
+
+            lexerAction.execute(lexer)
+          end
+          i += 1
+        end
+      ensure
+        input.seek(stop_index) if requires_seek
+      end
+    end
+
+    def eql?(obj)
+      if obj == self
+        return true
+      else
+        return false unless obj.is_a? LexerActionExecutor
+      end
+
+      @hash_code == obj.hash_code && (@lexer_actions == obj._a)
+    end
+  end
+end

data/lib/antlr4/runtime/lexer_action_type.rb

@@ -0,0 +1,12 @@
+module Antlr4::Runtime
+  class LexerActionType
+    CHANNEL = 0
+    CUSTOM = 1
+    MODE = 2
+    MORE = 3
+    POP_MODE = 4
+    PUSH_MODE = 5
+    SKIP = 6
+    TYPE = 7
+  end
+end

data/lib/antlr4/runtime/lexer_atn_config.rb

@@ -0,0 +1,50 @@
+require 'antlr4/runtime/atn_config'
+require 'antlr4/runtime/semantic_context'
+
+module Antlr4::Runtime
+
+  class LexerATNConfig < ATNConfig
+    attr_accessor :passed_through_non_greedy_decision
+    attr_accessor :lexer_action_executor
+
+    def initialize
+      super
+      @passed_through_non_greedy_decision = false
+      @lexer_action_executor = nil
+    end
+
+    def lexer_atn_config1(state, alt, context)
+      atn_config2(state, alt, context, SemanticContext::NONE)
+      @passed_through_non_greedy_decision = false
+      @lexer_action_executor = nil
+    end
+
+    def lexer_atn_config2(state, alt, context, lexer_action_executor)
+      atn_config7(state, alt, context, SemanticContext::NONE)
+      @lexer_action_executor = lexer_action_executor
+      @passed_through_non_greedy_decision = false
+    end
+
+    def lexer_atn_config3(c, state)
+      atn_config7(c, state, c.context, c.semantic_context)
+      @lexer_action_executor = c.lexer_action_executor
+      @passed_through_non_greedy_decision = check_non_greedy_decision(c, state)
+    end
+
+    def lexer_atn_config4(c, state, lexer_action_executor)
+      atn_config7(c, state, c.context, c.semantic_context)
+      @lexer_action_executor = lexer_action_executor
+      @passed_through_non_greedy_decision = check_non_greedy_decision(c, state)
+    end
+
+    def lexer_atn_config5(c, state, context)
+      atn_config7(c, state, context, c.semantic_context)
+      @lexer_action_executor = c.lexer_action_executor
+      @passed_through_non_greedy_decision = check_non_greedy_decision(c, state)
+    end
+
+    def check_non_greedy_decision(source, target)
+      source.passed_through_non_greedy_decision || target.is_a?(DecisionState) && target.non_greedy
+    end
+  end
+end

data/lib/antlr4/runtime/lexer_atn_simulator.rb

@@ -0,0 +1,522 @@
+require 'antlr4/runtime/atn_simulator'
+require 'antlr4/runtime/lexer_atn_config'
+require 'antlr4/runtime/ordered_atn_config_set'
+require 'antlr4/runtime/lexer_action_executor'
+
+module Antlr4::Runtime
+
+  class LexerATNSimulator < ATNSimulator
+    MIN_DFA_EDGE = 0
+    MAX_DFA_EDGE = 127 # forces unicode to stay in ATN
+
+
+    class << self
+      attr_reader :debug
+      attr_reader :empty
+      @@empty = EmptyPredictionContext.new(Integer::MAX)
+    end
+
+    class SimState
+      attr_accessor :index
+      attr_accessor :line
+      attr_accessor :char_pos
+      attr_accessor :dfa_state
+
+      def reset
+        @index = -1
+        @line = 0
+        @char_pos = -1
+        @dfa_state = nil
+      end
+    end
+
+    attr_reader :char_position_in_line
+    attr_reader :line
+
+    def initialize(recog, atn, decision_to_dfa, shared_context_cache)
+      super(atn, shared_context_cache)
+      @@debug = false
+      @dfa_debug = false
+
+      @decision_to_dfa = decision_to_dfa
+      @recog = recog
+      @start_index = -1
+      @line = 1
+      @char_position_in_line = 0
+      @mode = Lexer::DEFAULT_MODE
+      @prev_accept = SimState.new
+      @match_calls = 0
+    end
+
+    def copy_state(simulator)
+      @char_position_in_line = simulator.char_position_in_line
+      @line = simulator.line
+      @mode = simulator.mode
+      @start_index = simulator.start_index
+    end
+
+    def match(input, mode)
+      @match_calls += 1
+      @mode = mode
+      mark = input.mark
+
+      begin
+        @start_index = input.index
+        @prev_accept.reset
+        dfa = @decision_to_dfa[mode]
+        if dfa.s0.nil?
+          return match_atn(input)
+        else
+          return exec_atn(input, dfa.s0)
+        end
+      ensure
+        input.release(mark)
+      end
+    end
+
+    def reset
+      @prev_accept.reset
+      @start_index = -1
+      @line = 1
+      @char_position_in_line = 0
+      @mode = Lexer.DEFAULT_MODE
+    end
+
+    def clear_dfa
+      d = 0
+      while d < @decision_to_dfa.length
+        @decision_to_dfa[d] = DFA.new(atn.decision_state(d), d)
+        d += 1
+      end
+    end
+
+    def match_atn(input)
+      start_state = atn.mode_to_start_state[@mode]
+
+      printf format("matchATN mode %d start: %s\n", @mode, start_state) if @@debug
+
+      old_mode = @mode
+
+      s0_closure = compute_start_state(input, start_state)
+      suppress_edge = s0_closure.has_semantic_context
+      s0_closure.has_semantic_context = false
+
+      next_state = add_dfa_state(s0_closure)
+      @decision_to_dfa[@mode].s0 = next_state unless suppress_edge
+
+      predict = exec_atn(input, next_state)
+
+      if @@debug
+        printf format("DFA after matchATN: %s\n", @decision_to_dfa[old_mode].to_lexer_string)
+      end
+
+      predict
+    end
+
+    def exec_atn(input, ds0)
+      printf format("start state closure=%s\n", ds0.configs) if @@debug
+
+      if ds0.is_accept_state
+        # allow zero-length tokens
+        capture_sim_state(@prev_accept, input, ds0)
+      end
+
+      t = input.la(1)
+
+      s = ds0 # s is current/from DFA state
+
+      loop do # while more work
+        printf format("execATN loop starting closure: %s\n", s.configs) if @@debug
+
+        # As we move src->trg, src->trg, we keep track of the previous trg to
+        # avoid looking up the DFA state again, which is expensive.
+        # If the previous target was already part of the DFA, we might
+        # be able to avoid doing a reach operation upon t. If s!=nil,
+        # it means that semantic predicates didn't prevent us from
+        # creating a DFA state. Once we know s!=nil, we check to see if
+        # the DFA state has an edge already for t. If so, we can just reuse
+        # it's configuration set there's no point in re-computing it.
+        # This is kind of like doing DFA simulation within the ATN
+        # simulation because DFA simulation is really just a way to avoid
+        # computing reach/closure sets. Technically, once we know that
+        # we have a previously added DFA state, we could jump over to
+        # the DFA simulator. But, that would mean popping back and forth
+        # a lot and making things more complicated algorithmically.
+        # This optimization makes a lot of sense for loops within DFA.
+        # A character will take us back to an existing DFA state
+        # that already has lots of edges out of it. e.g., .* in comments.
+        target = existing_target_state(s, t)
+        target = compute_target_state(input, s, t) if target.nil?
+
+        break if target == @@error
+
+        # If this is a consumable input element, make sure to consume before
+        # capturing the accept state so the input index, line, and char
+        # position accurately reflect the state of the interpreter at the
+        # end of the token.
+        consume(input) if t != IntStream::EOF
+
+        if target.is_accept_state
+          capture_sim_state(@prev_accept, input, target)
+          break if t == IntStream::EOF
+        end
+
+        t = input.la(1)
+        s = target # flip current DFA target becomes new src/from state
+      end
+
+      fail_or_accept(@prev_accept, input, s.configs, t)
+    end
+
+    def existing_target_state(s, t)
+      return nil if s.edges.nil? || t < MIN_DFA_EDGE || t > MAX_DFA_EDGE
+
+      target = s.edges[t - MIN_DFA_EDGE]
+      if @@debug && !target.nil?
+        puts 'reuse state ' + s.state_number.to_s + ' edge to ' + target.state_number.to_s
+      end
+
+      target
+    end
+
+    def compute_target_state(input, s, t)
+      reach = OrderedATNConfigSet.new
+
+      # if we don't find an existing DFA state
+      # Fill reach starting from closure, following t transitions
+      reachable_config_set(input, s.configs, reach, t)
+
+      if reach.empty? # we got nowhere on t from s
+        unless reach.has_semantic_context
+          # we got nowhere on t, don't throw out this knowledge it'd
+          # cause a failover from DFA later.
+          add_dfa_edge_dfastate_dfastate(s, t, @@error)
+        end
+
+        # stop when we can't match any more char
+        return @@error
+      end
+
+      # Add an edge from s to target DFA found/created for reach
+      add_dfa_edge_dfastate_atnconfigset(s, t, reach)
+    end
+
+    def fail_or_accept(prev_accept, input, _reach, t)
+      if !prev_accept.dfa_state.nil?
+        lexer_action_executor = prev_accept.dfa_state.lexer_action_executor
+        accept(input, lexer_action_executor, @start_index, prev_accept.index, prev_accept.line, prev_accept.char_pos)
+        prev_accept.dfa_state.prediction
+      else # if no accept and EOF is first char, return EOF
+        return Token::EOF if t == IntStream::EOF && input.index == @start_index
+
+        raise LexerNoViableAltException, @recog
+      end
+    end
+
+    def reachable_config_set(input, closure, reach, t) # this is used to skip processing for configs which have a lower priority
+      # than a config that already reached an accept state for the same rule
+      skip_alt = ATN::INVALID_ALT_NUMBER
+      i = 0
+      while i < closure.configs.length
+        c = closure.configs[i]
+        current_alt_reached_accept_state = (c.alt == skip_alt)
+        if current_alt_reached_accept_state && c.passed_through_non_greedy_decision
+          i += 1
+          next
+        end
+
+        if @@debug
+          printf format("testing %s at %s\n", token_name(t), c.to_s2(@recog, true))
+        end
+
+        n = c.state.number_of_transitions
+        ti = 0
+        while ti < n # for each transition
+          trans = c.state.transition(ti)
+          target = reachable_target(trans, t)
+          unless target.nil?
+            lexer_action_executor = c.lexer_action_executor
+            unless lexer_action_executor.nil?
+              lexer_action_executor = lexer_action_executor.fix_offset_before_match(input.index - start_index)
+            end
+
+            treat_eof_as_epsilon = (t == CharStream::EOF)
+            cfg = LexerATNConfig.new
+            cfg.lexer_atn_config4(c, target, lexer_action_executor)
+            if closure(input, cfg, reach, current_alt_reached_accept_state, true, treat_eof_as_epsilon)
+              # any remaining configs for this alt have a lower priority than
+              # the one that just reached an accept state.
+              skip_alt = c.alt
+              break
+            end
+          end
+          ti += 1
+        end
+        i += 1
+      end
+    end
+
+    def accept(input, lexer_action_executor, start_index, index, line, char_pos)
+      printf format("ACTION %s\n", lexer_action_executor) if @@debug
+
+      # seek to after last char in token
+      input.seek(index)
+      @line = line
+      @char_position_in_line = char_pos
+
+      if !lexer_action_executor.nil? && !@recog.nil?
+        lexer_action_executor.execute(@recog, input, start_index)
+      end
+    end
+
+    def reachable_target(trans, t)
+      if trans.matches(t, Lexer::MIN_CHAR_VALUE, Lexer::MAX_CHAR_VALUE)
+        return trans.target
+      end
+
+      nil
+    end
+
+    def compute_start_state(input, p)
+      initial_context = @@empty
+      configs = ATNConfigSet.new
+      i = 0
+      while i < p.number_of_transitions
+        target = p.transition(i).target
+        c = LexerATNConfig.new
+        c.lexer_atn_config1(target, i + 1, initial_context)
+        closure(input, c, configs, false, false, false)
+        i += 1
+      end
+      configs
+    end
+
+    def closure(input, config, configs, current_alt_reached_accept_state, speculative, treat_eof_as_epsilon)
+      if config.state.is_a? RuleStopState
+        if @@debug
+          if !@recog.nil?
+            printf format("closure at %s rule stop %s\n", @recog.rule_names[config.state.rule_index], config)
+          else
+            printf format("closure at rule stop %s\n", config)
+          end
+        end
+
+        if config.context.nil? || config.context.empty_path?
+          if config.context.nil? || config.context.empty?
+            configs.add(config)
+            return true
+          else
+            configs.add(LexerATNConfig.create_from_config2(config, config.state, EmptyPredictionContext::EMPTY))
+            current_alt_reached_accept_state = true
+          end
+        end
+
+        if !config.context.nil? && !config.context.empty?
+          i = 0
+          while i < config.context.size
+            if config.context.get_return_state(i) != PredictionContext::EMPTY_RETURN_STATE
+              new_context = config.context.get_parent(i) # "pop" return state
+              return_state = atn.states[config.context.get_return_state(i)]
+              c = LexerATNConfig.new
+              c.lexer_atn_config5(config, return_state, new_context)
+              current_alt_reached_accept_state = closure(input, c, configs, current_alt_reached_accept_state, speculative, treat_eof_as_epsilon)
+            end
+            i += 1
+          end
+        end
+
+        return current_alt_reached_accept_state
+      end
+
+      # optimization
+      unless config.state.only_has_epsilon_transitions
+        if !current_alt_reached_accept_state || !config.passed_through_non_greedy_decision
+          configs.add(config)
+        end
+      end
+
+      p = config.state
+      i = 0
+      while i < p.number_of_transitions
+        t = p.transition(i)
+        c = epsilon_target(input, config, t, configs, speculative, treat_eof_as_epsilon)
+        unless c.nil?
+          current_alt_reached_accept_state = closure(input, c, configs, current_alt_reached_accept_state, speculative, treat_eof_as_epsilon)
+        end
+        i += 1
+      end
+
+      current_alt_reached_accept_state
+    end
+
+    # side-effect: can alter configs.hasSemanticContext
+
+    def epsilon_target(input, config, t, configs, speculative, treat_eof_as_epsilon)
+      c = nil
+      case t.serialization_type
+      when Transition::RULE
+        rule_transition = t
+        new_context = SingletonPredictionContext.new(config.context, rule_transition.follow_state.state_number)
+        c = LexerATNConfig.new
+        c.lexer_atn_config5(config, t.target, new_context)
+
+      when Transition::PRECEDENCE
+
+        raise UnsupportedOperationException, 'Precedence predicates are not supported in lexers.'
+
+      when Transition::PREDICATE
+        pt = t
+        puts('EVAL rule ' + pt.rule_index + ':' + pt.pred_index) if @@debug
+        configs.has_semantic_context = true
+        if evaluate_predicate(input, pt.rule_index, pt.pred_index, speculative)
+          c = LexerATNConfig.create_from_config(config, t.target)
+        end
+
+      when Transition::ACTION
+
+        if config.context.nil? || config.context.empty_path?
+          # execute actions anywhere in the start rule for a token.
+          #
+          # TODO: if the entry rule is invoked recursively, some
+          # actions may be executed during the recursive call. The
+          # problem can appear when hasEmptyPath() is true but
+          # isEmpty() is false. In this case, the config needs to be
+          # split into two contexts - one with just the empty path
+          # and another with everything but the empty path.
+          # Unfortunately, the current algorithm does not allow
+          # getEpsilonTarget to return two configurations, so
+          # additional modifications are needed before we can support
+          # the split operation.
+          lexer_action_executor = LexerActionExecutor.append(config.lexer_action_executor, @atn._a[t.action_index])
+          c = LexerATNConfig.new
+          c.lexer_atn_config4(config, t.target, lexer_action_executor)
+        else # ignore actions in referenced rules
+          c = LexerATNConfig.new
+          c.lexer_atn_config3(config, t.target)
+        end
+
+      when Transition::EPSILON
+        c = LexerATNConfig.new
+        c.lexer_atn_config3(config, t.target)
+      when Transition::ATOM, Transition::RANGE, Transition::SET
+        if treat_eof_as_epsilon
+          if t.matches(CharStream.EOF, Lexer.MIN_CHAR_VALUE, Lexer.MAX_CHAR_VALUE)
+            c = LexerATNConfig.create_from_config(config, t.target)
+          end
+        end
+
+      else
+        # empty
+      end
+
+      c
+    end
+
+    def evaluate_predicate(input, rule_index, pred_index, speculative) # assume true if no recognizer was provided
+      return true if @recog.nil?
+
+      return @recog.sempred(nil, rule_index, pred_index) unless speculative
+
+      saved_char_position_in_line = @char_position_in_line
+      saved_line = @line
+      index = input.index
+      marker = input.mark
+      begin
+        consume(input)
+        return @recog.sempred(nil, rule_index, pred_index)
+      ensure
+        @char_position_in_line = saved_char_position_in_line
+        @line = saved_line
+        input.seek(index)
+        input.release(marker)
+      end
+    end
+
+    def capture_sim_state(settings, input, dfa_state)
+      settings.index = input.index
+      settings.line = @line
+      settings.char_pos = @char_position_in_line
+      settings.dfa_state = dfa_state
+    end
+
+    def add_dfa_edge_dfastate_atnconfigset(from, t, q)
+      suppress_edge = q.has_semantic_context
+      q.has_semantic_context = false
+
+      to = add_dfa_state(q)
+
+      return to if suppress_edge
+
+      add_dfa_edge_dfastate_dfastate(from, t, to)
+      to
+    end
+
+    def add_dfa_edge_dfastate_dfastate(p, t, q)
+      if t < MIN_DFA_EDGE || t > MAX_DFA_EDGE
+        # Only track edges within the DFA bounds
+        return
+      end
+
+      if @@debug
+        message = 'EDGE ' << p.to_s << ' -> ' << q.to_s << ' upon ' << token_name(t)
+        puts(message)
+      end
+
+      if p.edges.nil?
+        #  make room for tokens 1..n and -1 masquerading as index 0
+        p.edges = []
+      end
+      p.edges[t - MIN_DFA_EDGE] = q # connect
+    end
+
+    def add_dfa_state(configs)
+      proposed = DFAState.new(configs)
+      first_config_with_rule_stop_state = configs.find_first_rule_stop_state
+
+      unless first_config_with_rule_stop_state.nil?
+        proposed.is_accept_state = true
+        proposed.lexer_action_executor = first_config_with_rule_stop_state.lexer_action_executor
+        proposed.prediction = atn.rule_to_token_type[first_config_with_rule_stop_state.state.rule_index]
+      end
+
+      dfa = @decision_to_dfa[@mode]
+
+      existing = dfa.states[proposed]
+      return existing unless existing.nil?
+
+      new_state = proposed
+
+      new_state.state_number = dfa.states.size
+      configs.readonly = true
+      new_state.configs = configs
+      dfa.states[new_state] = new_state
+      new_state
+    end
+
+    def dfa(mode)
+      @decision_to_dfa[mode]
+    end
+
+    def text(input) # index is first lookahead char, don' t include.
+      input.text(Interval.of(start_index, input.index - 1))
+    end
+
+    def consume(input)
+      cur_char = input.la(1)
+      if cur_char == 10 # newline
+        @line += 1
+        @char_position_in_line = 0
+      else
+        @char_position_in_line += 1
+      end
+      input.consume
+    end
+
+    def token_name(t)
+      return 'EOF' if t == -1
+
+      # if ( atn.g!=nil ) return atn.g.getTokenDisplayName(t)
+      "'" + t.to_s + "'"
+    end
+  end
+end