antlr4-runtime 0.1.0

data/lib/antlr4/runtime/char_stream.rb

@@ -0,0 +1,6 @@
+require 'antlr4/runtime/int_stream'
+
+module Antlr4::Runtime
+  class CharStream < IntStream
+  end
+end

data/lib/antlr4/runtime/char_streams.rb

@@ -0,0 +1,12 @@
+require 'antlr4/runtime/code_point_char_stream'
+
+module Antlr4::Runtime
+
+  class CharStreams
+    DEFAULT_BUFFER_SIZE = 4096
+
+    def self.from_string(s, source_name)
+      CodePointCharStream.new(0, s.length, source_name, s.bytes)
+    end
+  end
+end

data/lib/antlr4/runtime/chunk.rb

@@ -0,0 +1,4 @@
+module Antlr4::Runtime
+  class Chunk
+  end
+end

data/lib/antlr4/runtime/code_point_char_stream.rb

@@ -0,0 +1,83 @@
+require 'antlr4/runtime/char_stream'
+
+module Antlr4::Runtime
+
+  class CodePointCharStream < CharStream
+    def initialize(position, remaining, name, byte_array)
+      @size = remaining
+      @name = name
+      @position = position
+      @byte_array = byte_array
+    end
+
+    def text(interval)
+      start_idx = [interval.a, @size].min
+      len = [interval.b - interval.a + 1, @size - start_idx].min
+
+      # We know the maximum code point in byte_array is U+00FF,
+      # so we can treat this as if it were ISO-8859-1, aka Latin-1,
+      # which shares the same code points up to 0xFF.
+      chars = @byte_array.slice(start_idx, len)
+      result = ''
+      chars.each do |c|
+        result << c
+      end
+      result
+    end
+
+    def la(i)
+      case Integer.signum(i)
+      when -1
+        offset = @position + i
+        return IntStream::EOF if offset < 0
+
+        return @byte_array[offset] & 0xFF
+      when 0
+        # Undefined
+        return 0
+      when 1
+        offset = @position + i - 1
+        return IntStream::EOF if offset >= @size
+
+        return @byte_array[offset] & 0xFF
+      else
+        # type code here
+      end
+      raise UnsupportedOperationException, 'Not reached'
+    end
+
+    def internal_storage
+      @byte_array
+    end
+
+    def consume
+      raise IllegalStateException, 'cannot consume EOF' if (@size - @position).zero?
+
+      @position += 1
+    end
+
+    def index
+      @position
+    end
+
+    attr_reader :size
+
+    def mark
+      -1
+    end
+
+    def release(marker)
+      ;
+    end
+
+    def seek(index)
+      @position = index
+    end
+
+    def source_name
+      return UNKNOWN_SOURCE_NAME if @name.nil? || @name.empty?
+
+      @name
+    end
+  end
+end

data/lib/antlr4/runtime/common_token.rb

@@ -0,0 +1,125 @@
+module Antlr4::Runtime
+
+  class CommonToken
+
+    class << self
+      @@EMPTY_SOURCE = OpenStruct.new
+    end
+
+    attr_accessor :type
+    attr_accessor :line
+    attr_accessor :char_position_in_line
+    attr_accessor :channel
+    attr_accessor :source
+    attr_accessor :index
+    attr_accessor :start
+    attr_accessor :stop
+    attr_accessor :_text
+
+    def initialize(type = nil)
+      @char_position_in_line = -1
+      @channel = Token::DEFAULT_CHANNEL
+      @index = -1
+      @type = type
+      @source = @@EMPTY_SOURCE
+      @_text = nil
+    end
+
+    def self.create1(source, type, channel, start, stop)
+      result = CommonToken.new(type)
+      result.source = source
+      result.channel = channel
+      result.start = start
+      result.stop = stop
+      unless source.a.nil?
+        result.line = source.a.line
+        result.char_position_in_line = source.a.char_position_in_line
+      end
+      result
+    end
+
+    def self.create2(type, text)
+      result = CommonToken.new(type)
+      result._text = text
+      result
+    end
+
+    def create3(old_token)
+      result = CommonToken.new(old_token.type)
+
+      result.line = old_token.line
+      result.index = old_token.token_index
+      result.char_position_in_line = old_token.char_position_in_line
+      result.channel = old_token.channel
+      result.start = old_token.start_index
+      result.stop = old_token.stop_index
+
+      if old_token.is_a? CommonToken
+        result._text = old_token.text
+        result.source = old_token.source
+      else
+        result._text = old_token.text
+        result.source = OpenStruct.new
+        result.source.a = old_token.token_source
+        result.source.b = old_token.input_stream
+      end
+      result
+    end
+
+    def input_stream
+      @source.b
+    end
+
+    def text
+      return @_text unless @_text.nil?
+
+      input = input_stream
+      return nil if input.nil?
+
+      n = input.size
+      if @start < n && @stop < n
+        input.text(Interval.of(@start, @stop))
+      else
+        '<EOF>'
+      end
+    end
+
+    def to_s_recog(r = nil)
+      channel_str = ''
+      channel_str = ',channel=' + @channel.to_s if @channel > 0
+      txt = @_text
+      if !txt.nil?
+        txt = txt.sub("\n", '\\n')
+        txt = txt.sub("\r", '\\r')
+        txt = txt.sub("\t", '\\t')
+      else
+        txt = '<no text>'
+      end
+
+      type_string = type.to_s
+      type_string = r.get_vocabulary.display_name(@type) unless r.nil?
+      '[@' << token_index.to_s << ',' << @start.to_s << ':' << @stop.to_s << "='" << txt << "',<" << type_string << '>' << channel_str << ',' << @line.to_s << ':' << char_position_in_line.to_s << ']'
+    end
+
+    def to_s
+      '[@ ' << @start.to_s << ':' << @stop.to_s << ',' << @line.to_s << ':' << ']'
+    end
+
+    def to_s_old
+      channel_str = ''
+      channel_str = ',channel=' + @channel.to_s if @channel > 0
+      txt = @_text
+      if !txt.nil?
+        txt = txt.sub("\n", '\\n')
+        txt = txt.sub("\r", '\\r')
+        txt = txt.sub("\t", '\\t')
+      else
+        txt = '<no text>'
+      end
+
+      type_string = type.to_s
+
+      '[@' << token_index.to_s << ',' << @start.to_s << ':' << @stop.to_s << "='" << txt << "',<" << type_string << '>' << channel_str << ',' << @line.to_s << ':' << char_position_in_line.to_s << ']'
+    end
+  end
+end

data/lib/antlr4/runtime/common_token_factory.rb

@@ -0,0 +1,30 @@
+require 'antlr4/runtime/common_token'
+
+module Antlr4::Runtime
+
+  class CommonTokenFactory
+    include Singleton
+
+    def initialize(copy_text = false)
+      @copy_text = false
+      @copy_text = copy_text
+    end
+
+    def create(source, type, text, channel, start, stop, line, char_position_in_line)
+      t = CommonToken.create1(source, type, channel, start, stop)
+      t.line = line
+      t.char_position_in_line = char_position_in_line
+      if !text.nil?
+        t._text = text
+      elsif @copy_text && !source.b.nil?
+        t.set_text(source.b.text(Interval.of(start, stop)))
+      end
+
+      t
+    end
+
+    def create_simple(type, text)
+      CommonToken.create2(type, text)
+    end
+  end
+end

data/lib/antlr4/runtime/common_token_stream.rb

@@ -0,0 +1,63 @@
+require 'antlr4/runtime/buffered_token_stream'
+
+module Antlr4::Runtime
+
+  class CommonTokenStream < BufferedTokenStream
+    def initialize(token_source, channel = nil)
+      super(token_source)
+      @channel = Token::DEFAULT_CHANNEL
+      @channel = channel unless channel.nil?
+    end
+
+    def adjust_seek_index(i)
+      next_token_on_channel(i, @channel)
+    end
+
+    def lb(k)
+      return nil if k.zero? || (@ptr - k) < 0
+
+      i = @ptr
+      n = 1
+      # find k good tokens looking backwards
+      while n <= k && i > 0
+        # skip off-channel tokens
+        i = previous_token_on_channel(i - 1, @channel)
+        n += 1
+      end
+      return nil if i < 0
+
+      @tokens[i]
+    end
+
+    def lt(k)
+      lazy_init
+      return nil if k == 0
+      return lb(-k) if k < 0
+
+      i = @ptr
+      n = 1 # we know tokens[p] is a good one
+      # find k good tokens
+      while n < k
+        # skip off-channel tokens, but make sure to not look past EOF
+        i = next_token_on_channel(i + 1, @channel) if sync(i + 1)
+        n += 1
+      end
+      #    if ( i>range ) range = i
+      @tokens[i]
+    end
+
+    def number_of_on_channel_tokens
+      n = 0
+      fill
+      i = 0
+      while i < @tokens.size
+        t = @tokens.get(i)
+        n += 1 if t.channel == @channel
+        break if t.type == Token::EOF
+
+        i += 1
+      end
+      n
+    end
+  end
+end

data/lib/antlr4/runtime/console_error_listener.rb

@@ -0,0 +1,12 @@
+require 'antlr4/runtime/base_error_listener'
+
+module Antlr4::Runtime
+
+  class ConsoleErrorListener < BaseErrorListener
+    include Singleton
+
+    def syntaxError(_recognizer, _offending_symbol, line, char_position_in_line, msg, _e)
+      STDERR.puts 'line ' << line.to_s << ':' << char_position_in_line.to_s << ' ' << msg.to_s << ''
+    end
+  end
+end

data/lib/antlr4/runtime/context_sensitivity_info.rb

@@ -0,0 +1,7 @@
+module Antlr4::Runtime
+  class ContextSensitivityInfo < DecisionEventInfo
+    def initialize(decision, configs, input, start_index, stop_index)
+      super(decision, configs, input, start_index, stop_index, true)
+    end
+  end
+end

data/lib/antlr4/runtime/decision_event_info.rb

@@ -0,0 +1,19 @@
+module Antlr4::Runtime
+  class DecisionEventInfo
+    attr_accessor :decision
+    attr_accessor :configs
+    attr_accessor :input
+    attr_accessor :start_index
+    attr_accessor :stop_index
+    attr_accessor :full_ctx
+
+    def initialize(decision, configs, input, start_index, stop_index, full_ctx)
+      @decision = decision
+      @full_ctx = full_ctx
+      @stop_index = stop_index
+      @input = input
+      @start_index = start_index
+      @configs = configs
+    end
+  end
+end

data/lib/antlr4/runtime/decision_info.rb

@@ -0,0 +1,36 @@
+module Antlr4::Runtime
+  class DecisionInfo
+    attr_accessor :decision
+    attr_accessor :invocations
+    attr_accessor :timeInPrediction
+    attr_accessor :SLL_TotalLook
+    attr_accessor :SLL_MinLook
+    attr_accessor :SLL_MaxLook
+    attr_accessor :SLL_MaxLookEvent
+    attr_accessor :LL_TotalLook
+    attr_accessor :LL_MinLook
+    attr_accessor :LL_MaxLook
+    attr_accessor :LL_MaxLookEvent
+    attr_accessor :context_sensitivities
+    attr_accessor :errors
+    attr_accessor :ambiguities
+    attr_accessor :predicate_evals
+    attr_accessor :SLL_ATNTransitions
+    attr_accessor :SLL_DFATransitions
+    attr_accessor :LL_Fallback
+    attr_accessor :LL_ATNTransitions
+    attr_accessor :LL_DFATransitions
+
+    def initialize(decision)
+      @context_sensitivities = []
+      @errors = []
+      @ambiguities = []
+      @predicate_evals = []
+      @decision = decision
+    end
+
+    def to_s
+      '' + 'decision=' + @decision + ', contextSensitivities=' + @context_sensitivities.size + ', errors=' + @errors.size + ', ambiguities=' + @ambiguities.size + ', SLL_lookahead=' + SLL_TotalLook + ', SLL_ATNTransitions=' + SLL_ATNTransitions + ', SLL_DFATransitions=' + SLL_DFATransitions + ', LL_Fallback=' + LL_Fallback + ', LL_lookahead=' + LL_TotalLook + ', LL_ATNTransitions=' + LL_ATNTransitions + 'end'
+    end
+  end
+end

data/lib/antlr4/runtime/decision_state.rb

@@ -0,0 +1,15 @@
+require 'antlr4/runtime/atn_state'
+
+module Antlr4::Runtime
+
+  class DecisionState < ATNState
+    attr_accessor :decision
+    attr_accessor :non_greedy
+
+    def initialize
+      super
+      @decision = -1
+      @non_greedy = false
+    end
+  end
+end

data/lib/antlr4/runtime/default_error_strategy.rb

@@ -0,0 +1,314 @@
+require 'antlr4/runtime/antlr_error_strategy'
+require 'antlr4/runtime/input_mismatch_exception'
+require 'antlr4/runtime/no_viable_alt_exception'
+
+module Antlr4::Runtime
+
+  class DefaultErrorStrategy < ANTLRErrorStrategy
+    def initialize
+      @error_recovery_mode = false
+      @last_error_index = -1
+      @last_error_states = nil
+      @next_tokens_context = nil
+      @next_tokens_state = nil
+    end
+
+    def reset(recognizer)
+      end_error_condition(recognizer)
+    end
+
+    def begin_error_condition(_recognizer)
+      @error_recovery_mode = true
+    end
+
+    def error_recovery_mode?(_recognizer)
+      @error_recovery_mode
+    end
+
+    def end_error_condition(_recognizer)
+      @error_recovery_mode = false
+      @last_error_states = nil
+      @last_error_index = -1
+    end
+
+    def report_match(recognizer)
+      end_error_condition(recognizer)
+    end
+
+    def report_error(recognizer, e)
+      # if we've already reported an error and have not matched a token
+      # yet successfully, don't report any errors.
+      if error_recovery_mode?(recognizer)
+        #      System.err.print("[SPURIOUS] ")
+        return # don't report spurious errors
+      end
+
+      begin_error_condition(recognizer)
+      if e.is_a? NoViableAltException
+        report_no_viable_alternative(recognizer, e)
+      elsif e.is_a? InputMismatchException
+        report_input_mismatch(recognizer, e)
+      elsif e.is_a? FailedPredicateException
+        report_failed_predicate(recognizer, e)
+      else
+        STDERR.puts 'unknown recognition error type: ' + e.getClass.getName
+        recognizer.notify_error_listeners(e.getOffendingToken, e.getMessage, e)
+      end
+    end
+
+    def recover(recognizer, _e)
+      if @last_error_index == recognizer._input.index && !@last_error_states.nil? && @last_error_states.include?(recognizer._state_number)
+        # uh oh, another error at same token index and previously-visited
+        # state in ATN must be a case where lt(1) is in the recovery
+        # token set so nothing got consumed. Consume a single token
+        # at least to prevent an infinite loop this is a failsafe.
+        #      System.err.println("seen error condition before index="+
+        #                 lastErrorIndex+", states="+last_error_states)
+        #      System.err.println("FAILSAFE consumes "+recognizer.getTokenNames()[recognizer.getInputStream().la(1)])
+        recognizer.consume
+      end
+      @last_error_index = recognizer._input.index
+      @last_error_states = IntervalSet.new if @last_error_states.nil?
+      @last_error_states.add(recognizer._state_number)
+      followSet = error_recovery_set(recognizer)
+      consume_until(recognizer, followSet)
+    end
+
+    def sync(recognizer)
+      s = recognizer._interp.atn.states[recognizer._state_number]
+      #    System.err.println("sync @ "+s.stateNumber+"="+s.getClass().getSimpleName())
+      # If already recovering, don't try to sync
+      return if error_recovery_mode?(recognizer)
+
+      tokens = recognizer._input
+      la = tokens.la(1)
+
+      # try cheaper subset first might get lucky. seems to shave a wee bit off
+      next_tokens = recognizer.atn.next_tokens(s)
+      if next_tokens.contains(la)
+        # We are sure the token matches
+        @next_tokens_context = nil
+        @next_tokens_state = ATNState.invalid_state_number
+        return
+      end
+
+      if next_tokens.contains(Token::EPSILON)
+        if @next_tokens_context.nil?
+          # It's possible the next token won't match information tracked
+          # by sync is restricted for performance.
+          @next_tokens_context = recognizer._ctx
+          @next_tokens_state = recognizer._state_number
+        end
+        return
+      end
+
+      case s.state_type
+      when ATNState::BLOCK_START, ATNState::STAR_BLOCK_START, ATNState::PLUS_BLOCK_START, ATNState::STAR_LOOP_ENTRY
+        # report error and recover if possible
+        return unless single_token_deletion(recognizer).nil?
+
+        exc = InputMismatchException.create(recognizer)
+        raise exc
+
+      when ATNState::PLUS_LOOP_BACK, ATNState::STAR_LOOP_BACK
+
+        #      System.err.println("at loop back: "+s.getClass().getSimpleName())
+        report_unwanted_token(recognizer)
+        expecting = recognizer.expected_tokens
+        what_follows_loop_iteration_or_rule = expecting.or(error_recovery_set(recognizer))
+        consume_until(recognizer, what_follows_loop_iteration_or_rule)
+
+      else # do nothing if we can't identify the exact kind of ATN state end
+      end
+    end
+
+    def report_no_viable_alternative(recognizer, e)
+      tokens = recognizer._input
+      input = if !tokens.nil?
+                if e.start_token.type == Token::EOF
+                  '<EOF>'
+                else
+                  tokens.text4(e.start_token, e.offending_token)
+                end
+              else
+                '<unknown input>'
+              end
+      msg = 'no viable alternative at input ' + escape_ws_and_quote(input)
+      recognizer.notify_error_listeners(e.offending_token, msg, e)
+    end
+
+    def report_input_mismatch(recognizer, e)
+      msg = 'mismatched input ' + token_error_display(e.offending_token) + ' expecting ' + e.expected_tokens.to_string_from_vocabulary(recognizer.get_vocabulary)
+      recognizer.notify_error_listeners(e.offending_token, msg, e)
+    end
+
+    def report_failed_predicate(recognizer, e)
+      rule_name = recognizer.rule_names[recognizer._ctx.rule_index]
+      msg = 'rule ' + rule_name + ' ' + e.getMessage
+      recognizer.notify_error_listeners(e.getOffendingToken, msg, e)
+    end
+
+    def report_unwanted_token(recognizer)
+      return if error_recovery_mode?(recognizer)
+
+      begin_error_condition(recognizer)
+
+      t = recognizer.current_token
+      token_name = token_error_display(t)
+      expecting = expected_tokens(recognizer)
+      msg = 'extraneous input ' + token_name + ' expecting ' + expecting.to_string_from_vocabulary(recognizer.get_vocabulary)
+      recognizer.notify_error_listeners(t, msg, nil)
+    end
+
+    def report_missing_token(recognizer)
+      return if error_recovery_mode?(recognizer)
+
+      begin_error_condition(recognizer)
+
+      t = recognizer.current_token
+      expecting = expected_tokens(recognizer)
+      msg = 'missing ' + expecting.to_string_from_vocabulary(recognizer.get_vocabulary) + ' at ' + token_error_display(t)
+
+      recognizer.notify_error_listeners(t, msg, nil)
+    end
+
+    def recover_in_line(recognizer)
+      # SINGLE TOKEN DELETION
+      matched_symbol = single_token_deletion(recognizer)
+      unless matched_symbol.nil?
+        # we have deleted the extra token.
+        # now, move past ttype token as if all were ok
+        recognizer.consume
+        return matched_symbol
+      end
+
+      # SINGLE TOKEN INSERTION
+      return get_missing_symbol(recognizer) if single_token_insertion(recognizer)
+
+      # even that didn't work must throw the exception
+      exc = InputMismatchException.new
+      exc.recognizer = recognizer
+      if @next_tokens_context.nil?
+        raise exc
+      else
+        exc.offending_token = recognizer._input.lt(1)
+        exc.offending_state = @next_tokens_state
+        exc.context = @next_tokens_context
+        raise exc
+      end
+    end
+
+    def single_token_insertion(recognizer)
+      current_symbol_type = recognizer._input.la(1)
+      # if current token is consistent with what could come after current
+      # ATN state, then we know we're missing a token error recovery
+      # is free to conjure up and insert the missing token
+      current_state = recognizer._interp.atn.states[recognizer._state_number]
+      next_t = current_state.transition(0).target
+      atn = recognizer._interp.atn
+      expecting_at_ll2 = atn.next_tokens_ctx(next_t, recognizer._ctx)
+      #    System.out.println("lt(2) set="+expecting_at_ll2.to_s(recognizer.getTokenNames()))
+      if expecting_at_ll2.contains(current_symbol_type)
+        report_missing_token(recognizer)
+        return true
+      end
+      false
+    end
+
+    def single_token_deletion(recognizer)
+      next_token_type = recognizer._input.la(2)
+      expecting = expected_tokens(recognizer)
+      if expecting.contains(next_token_type)
+        report_unwanted_token(recognizer)
+
+        recognizer.consume # simply delete extra token
+        # we want to return the token we're actually matching
+        matched_symbol = recognizer.current_token
+        report_match(recognizer) # we know current token is correct
+        return matched_symbol
+      end
+      nil
+    end
+
+    def get_missing_symbol(recognizer)
+      current_symbol = recognizer.current_token
+      expecting = expected_tokens(recognizer)
+      expected_token_type = Token::INVALID_TYPE
+      unless expecting.is_nil
+        expected_token_type = expecting.min_element # get any element
+      end
+      if expected_token_type == Token::EOF
+        token_text = '<missing EOF>'
+      else
+        token_text = '<missing ' + recognizer.get_vocabulary.display_name(expected_token_type) + '>'
+      end
+      current = current_symbol
+      look_back = recognizer._input.lt(-1)
+      current = look_back if current.type == Token::EOF && !look_back.nil?
+
+      pair = OpenStruct.new
+      pair.a = current.source
+      pair.b = current.source._input
+      recognizer.token_factory.create(pair, expected_token_type, token_text, Token::DEFAULT_CHANNEL, -1, -1, current.line, current.char_position_in_line)
+    end
+
+    def expected_tokens(recognizer)
+      recognizer.expected_tokens
+    end
+
+    def token_error_display(t)
+      return '<no token>' if t.nil?
+
+      s = symbol_text(t)
+      if s.nil?
+        s = if symbol_type(t) == Token::EOF
+              '<EOF>'
+            else
+              '<' << symbol_type(t).to_s << '>'
+            end
+      end
+
+      escape_ws_and_quote(s)
+    end
+
+    def symbol_text(symbol)
+      return symbol.text if symbol.respond_to? :text
+      symbol.to_s
+    end
+
+    def symbol_type(symbol)
+      symbol.type
+    end
+
+    def escape_ws_and_quote(s) #    if ( s==nil ) return s
+      s = s.sub("\n", '\\n')
+      s = s.sub("\r", '\\r')
+      s = s.sub("\t", '\\t')
+      "'" + s + "'"
+    end
+
+    def error_recovery_set(recognizer)
+      atn = recognizer._interp.atn
+      ctx = recognizer._ctx
+      recover_set = IntervalSet.new
+      while !ctx.nil? && ctx.invoking_state >= 0
+        # compute what follows who invoked us
+        invoking_state = atn.states[ctx.invoking_state]
+        rt = invoking_state.transition(0)
+        follow = atn.next_tokens(rt.follow_state)
+        recover_set.add_all(follow)
+        ctx = ctx.parent
+      end
+      recover_set.remove(Token::EPSILON)
+      recover_set
+    end
+
+    def consume_until(recognizer, set)
+      ttype = recognizer._input.la(1)
+      while ttype != Token::EOF && !set.contains(ttype)
+        recognizer.consume
+        ttype = recognizer._input.la(1)
+      end
+    end
+  end
+end