ibex 0.1.0 → 0.2.0

data/lib/ibex/error_messages/sentence_search.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require_relative "../lalr/conflict_search_limits"
+
+module Ibex
+  module ErrorMessages
+    # Finds deterministic shortest token sentences that reach each syntax
+    # error state without running semantic actions.
+    class SentenceSearch
+      DEFAULT_MAX_TOKENS = LALR::ConflictSearchLimits::DEFAULT_MAX_TOKENS #: Integer
+      DEFAULT_MAX_CONFIGURATIONS = LALR::ConflictSearchLimits::DEFAULT_MAX_CONFIGURATIONS #: Integer
+
+      Witness = Struct.new(
+        :entry, #: String
+        :tokens, #: Array[String]
+        :state, #: Integer
+        keyword_init: true
+      )
+
+      # @rbs @automaton: IR::Automaton
+      # @rbs @grammar: IR::Grammar
+      # @rbs @max_tokens: Integer
+      # @rbs @max_configurations: Integer
+      # @rbs @candidates: Array[IR::GrammarSymbol]
+      # @rbs @input_candidates: Array[IR::GrammarSymbol]
+      # @rbs @explored: Integer
+
+      # @rbs (IR::Automaton automaton, ?max_tokens: Integer, ?max_configurations: Integer) -> void
+      def initialize(automaton, max_tokens: DEFAULT_MAX_TOKENS,
+                     max_configurations: DEFAULT_MAX_CONFIGURATIONS)
+        LALR::ConflictSearchLimits.validate!(
+          max_tokens: max_tokens, max_configurations: max_configurations
+        )
+        @automaton = automaton
+        @grammar = automaton.grammar
+        @max_tokens = max_tokens
+        @max_configurations = max_configurations
+        @candidates = grammar_terminals
+        @input_candidates = @candidates.reject { |symbol| symbol.id.zero? }
+        @explored = 0
+      end
+
+      # @rbs () -> Hash[Integer, Witness]
+      def all
+        targets = ErrorMessages.error_states(@automaton).to_h { |state| [state.id, true] }
+        witnesses = {} #: Hash[Integer, Witness]
+        @automaton.entry_states.each do |entry, initial_state|
+          search_entry(entry, initial_state, targets, witnesses)
+          break if witnesses.length == targets.length
+        end
+        witnesses.sort.to_h.freeze
+      end
+
+      # @rbs (Array[String] tokens, ?entry: String?) -> Integer?
+      def state_for(tokens, entry: nil)
+        return nil if tokens.empty?
+
+        start = entry || @grammar.start
+        initial = @automaton.entry_states[start]
+        return nil unless initial
+
+        states = [initial]
+        tokens.each_with_index do |name, index|
+          status, value = consume_sentence_token(states, name, index == tokens.length - 1)
+          return value if status == :error && value.is_a?(Integer)
+          return nil unless status == :continue && value.is_a?(Array)
+
+          states = value
+        end
+        nil
+      end
+
+      private
+
+      # @rbs (Array[Integer] states, String name, bool final) ->
+      #   [Symbol, Array[Integer] | Integer | nil]
+      def consume_sentence_token(states, name, final)
+        symbol = sentence_symbol(name, final)
+        return [:invalid, nil] unless symbol
+
+        status, value = advance(states, symbol.id)
+        return [:error, value] if status == :error && final && value.is_a?(Integer)
+        return [:continue, value] if status == :shift && value.is_a?(Array)
+
+        [:invalid, nil]
+      end
+
+      # @rbs (String name, bool final) -> IR::GrammarSymbol?
+      def sentence_symbol(name, final)
+        symbol = @grammar.symbol(name)
+        return nil unless symbol&.terminal?
+        return nil if symbol.name == "error" || (symbol.id.zero? && !final)
+
+        symbol
+      end
+
+      # @rbs () -> Array[IR::GrammarSymbol]
+      def grammar_terminals
+        @grammar.terminals
+                .reject { |symbol| symbol.name == "error" }
+                .sort_by { |symbol| [symbol.id.zero? ? 1 : 0, symbol.name] }
+      end
+
+      # @rbs (String entry, Integer initial_state, Hash[Integer, bool] targets,
+      #   Hash[Integer, Witness] witnesses) -> void
+      def search_entry(entry, initial_state, targets, witnesses)
+        queue = [[[initial_state], Array.new(0)]] #: Array[[Array[Integer], Array[String]]]
+        visited = { [initial_state] => true }
+        until queue.empty? || witnesses.length == targets.length
+          states, prefix = queue.shift
+          count_configuration!
+          collect_errors(entry, states, prefix, targets, witnesses)
+          enqueue_shifts(queue, visited, states, prefix) if prefix.length < @max_tokens - 1
+        end
+      end
+
+      # @rbs (String entry, Array[Integer] states, Array[String] prefix, Hash[Integer, bool] targets,
+      #   Hash[Integer, Witness] witnesses) -> void
+      def collect_errors(entry, states, prefix, targets, witnesses)
+        @candidates.each do |symbol|
+          status, state = advance(states, symbol.id)
+          next unless status == :error && state.is_a?(Integer)
+
+          error_state = state #: Integer
+          next unless targets[error_state] && !witnesses[error_state]
+
+          witnesses[error_state] = Witness.new(
+            entry: entry, tokens: (prefix + [symbol.name]).freeze, state: error_state
+          ).freeze
+        end
+      end
+
+      # @rbs (Array[[Array[Integer], Array[String]]] queue, Hash[Array[Integer], bool] visited,
+      #   Array[Integer] states, Array[String] prefix) -> void
+      def enqueue_shifts(queue, visited, states, prefix)
+        @input_candidates.each do |symbol|
+          status, shifted = advance(states, symbol.id)
+          next unless status == :shift && shifted.is_a?(Array)
+
+          shifted_states = shifted #: Array[Integer]
+          next if visited[shifted_states]
+
+          visited[shifted_states] = true
+          queue << [shifted_states, prefix + [symbol.name]]
+        end
+      end
+
+      # @rbs (Array[Integer] initial, Integer token_id) -> [Symbol, Array[Integer] | Integer | nil]
+      def advance(initial, token_id)
+        states = initial
+        visited = {} #: Hash[Array[Integer], bool]
+        loop do
+          return [:invalid, nil] if visited[states]
+
+          visited[states] = true
+          state = @automaton.states.fetch(states.last)
+          action = state.actions[token_id] || state.default_action
+          return [:error, state.id] if action.nil? || action[:type] == :error
+
+          case action[:type]
+          when :shift
+            shift = action #: IR::shift_action
+            return [:shift, (states + [shift[:state]]).freeze]
+          when :reduce
+            reduce = action #: IR::reduce_action
+            states = reduce_stack(states, reduce[:production])
+            return [:invalid, nil] unless states
+          when :accept
+            return [:accept, nil]
+          else
+            return [:invalid, nil]
+          end
+        end
+      end
+
+      # @rbs (Array[Integer] states, Integer production_id) -> Array[Integer]?
+      def reduce_stack(states, production_id)
+        production = @grammar.productions.fetch(production_id)
+        return nil if production.rhs.length >= states.length
+
+        remaining = states.take(states.length - production.rhs.length)
+        target = @automaton.states.fetch(remaining.last).gotos[production.lhs]
+        target && (remaining + [target]).freeze
+      end
+
+      # @rbs () -> void
+      def count_configuration!
+        @explored += 1
+        return if @explored <= @max_configurations
+
+        raise Ibex::Error,
+              "(messages):1:1: error-sentence search exceeded #{@max_configurations} configurations"
+      end
+    end
+  end
+end

data/lib/ibex/error_messages/update.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require_relative "renderer"
+
+module Ibex
+  module ErrorMessages
+    # @rbs (IR::Automaton automaton, ?existing: Document?, ?max_tokens: Integer,
+    #   ?max_configurations: Integer) -> String
+    def render(automaton, existing: nil, max_tokens: SentenceSearch::DEFAULT_MAX_TOKENS,
+               max_configurations: SentenceSearch::DEFAULT_MAX_CONFIGURATIONS)
+      update(
+        automaton, existing: existing, max_tokens: max_tokens, max_configurations: max_configurations
+      ).source
+    end
+    module_function :render
+
+    # @rbs (IR::Automaton automaton, ?existing: Document?, ?max_tokens: Integer,
+    #   ?max_configurations: Integer) -> Update
+    def update(automaton, existing: nil, max_tokens: SentenceSearch::DEFAULT_MAX_TOKENS,
+               max_configurations: SentenceSearch::DEFAULT_MAX_CONFIGURATIONS)
+      document = existing || Document.new(entries: [])
+      search = SentenceSearch.new(
+        automaton, max_tokens: max_tokens, max_configurations: max_configurations
+      )
+      witnesses = search.all
+      classifications = { uncovered: [], unreachable: [], moved: [] } #: Hash[Symbol, Array[String]]
+      next_id = next_error_id(document)
+      entries, covered, next_id = update_existing_entries(
+        document, search, witnesses, classifications, next_id
+      )
+      add_uncovered_entries(entries, covered, witnesses, classifications[:uncovered], next_id)
+      entries.sort_by! { |entry| entry_sort_key(entry) }
+      Update.new(
+        source: render_entries(automaton, entries),
+        uncovered: classifications[:uncovered],
+        unreachable: classifications[:unreachable],
+        moved: classifications[:moved]
+      )
+    end
+    module_function :update
+
+    # @rbs (Array[Entry] entries, Hash[Integer, bool] covered,
+    #   Hash[Integer, SentenceSearch::Witness] witnesses, Array[String] uncovered, Integer next_id) -> void
+    def add_uncovered_entries(entries, covered, witnesses, uncovered, next_id)
+      witnesses.each do |state, witness|
+        next if covered[state]
+
+        error_id = format_error_id(next_id)
+        next_id += 1
+        entries << Entry.new(
+          state: state, status: :active, message: nil, line: 1,
+          sentence: witness.tokens, error_id: error_id, entry: witness.entry
+        )
+        uncovered << "#{error_id} state #{state}: #{witness.tokens.join(' ')}"
+      end
+    end
+
+    # @rbs (Document document, SentenceSearch search, Hash[Integer, SentenceSearch::Witness] witnesses,
+    #   Hash[Symbol, Array[String]] classifications, Integer next_id) ->
+    #   [Array[Entry], Hash[Integer, bool], Integer]
+    def update_existing_entries(document, search, witnesses, classifications, next_id)
+      return migrate_legacy_entries(document, witnesses, classifications, next_id) if document.version == 1
+
+      entries = [] #: Array[Entry]
+      covered = {} #: Hash[Integer, bool]
+      document.entries.each do |entry|
+        if entry.sentence
+          next_id = update_sentence_entry(
+            entry, search, entries, covered, classifications, next_id
+          )
+        else
+          entries << with_error_id(entry, next_id)
+          next_id += 1 unless entry.error_id
+        end
+      end
+      [entries, covered, next_id]
+    end
+
+    # @rbs (Entry entry, SentenceSearch search, Array[Entry] entries, Hash[Integer, bool] covered,
+    #   Hash[Symbol, Array[String]] classifications, Integer next_id) -> Integer
+    def update_sentence_entry(entry, search, entries, covered, classifications, next_id)
+      error_id = entry.error_id || format_error_id(next_id)
+      next_id += 1 unless entry.error_id
+      sentence = entry.sentence || raise(Ibex::Error, "missing error sentence")
+      state = search.state_for(sentence, entry: entry.entry)
+      unless state
+        entries << Entry.new(
+          state: entry.state, status: :unreachable, message: entry.message, line: entry.line,
+          sentence: sentence, error_id: error_id, entry: entry.entry
+        )
+        classifications[:unreachable] << "#{error_id}: #{sentence.join(' ')}"
+        return next_id
+      end
+
+      classifications[:moved] << "#{error_id}: state #{entry.state} -> #{state}" if entry.state && entry.state != state
+      covered[state] = true
+      entries << Entry.new(
+        state: state, status: :active, message: entry.message, line: entry.line,
+        sentence: sentence, error_id: error_id, entry: entry.entry
+      )
+      next_id
+    end
+
+    # @rbs (Document document, Hash[Integer, SentenceSearch::Witness] witnesses,
+    #   Hash[Symbol, Array[String]] classifications, Integer next_id) ->
+    #   [Array[Entry], Hash[Integer, bool], Integer]
+    def migrate_legacy_entries(document, witnesses, classifications, next_id)
+      by_state = document.entries.to_h { |entry| [entry.state, entry] }
+      entries = [] #: Array[Entry]
+      covered = {} #: Hash[Integer, bool]
+      witnesses.each do |state, witness|
+        previous = by_state.delete(state)
+        error_id = format_error_id(next_id)
+        next_id += 1
+        entries << Entry.new(
+          state: state, status: :active, message: previous&.message, line: previous&.line || 1,
+          sentence: witness.tokens, error_id: error_id, entry: witness.entry
+        )
+        covered[state] = true
+        classifications[:uncovered] << "#{error_id} state #{state}: #{witness.tokens.join(' ')}" unless previous
+      end
+      by_state.values.sort_by { |entry| entry.state || -1 }.each do |entry|
+        error_id = format_error_id(next_id)
+        next_id += 1
+        entries << Entry.new(
+          state: entry.state, status: :removed, message: entry.message, line: entry.line, error_id: error_id
+        )
+        classifications[:unreachable] << "#{error_id}: legacy state #{entry.state}"
+      end
+      [entries, covered, next_id]
+    end
+
+    # @rbs (Entry entry, Integer next_id) -> Entry
+    def with_error_id(entry, next_id)
+      Entry.new(
+        state: entry.state, status: entry.status, message: entry.message, line: entry.line,
+        sentence: entry.sentence, error_id: entry.error_id || format_error_id(next_id), entry: entry.entry
+      )
+    end
+
+    # @rbs (Document document) -> Integer
+    def next_error_id(document)
+      maximum = document.entries.filter_map do |entry|
+        match = entry.error_id&.match(/\AE([0-9]{4,})\z/)
+        match && Integer(match[1] || "0", 10)
+      end.max
+      (maximum || 0) + 1
+    end
+
+    # @rbs (Integer number) -> String
+    def format_error_id(number)
+      format("E%04d", number)
+    end
+
+    # @rbs (Entry entry) -> [Integer, Integer, String]
+    def entry_sort_key(entry)
+      rank = { active: 0, unreachable: 1, removed: 2 }.fetch(entry.status)
+      [rank, entry.state || (1 << 62), entry.error_id || ""]
+    end
+
+    module_function :add_uncovered_entries, :update_existing_entries, :update_sentence_entry, :migrate_legacy_entries,
+                    :with_error_id, :next_error_id, :format_error_id, :entry_sort_key
+
+    class << self
+      private :add_uncovered_entries, :update_existing_entries, :update_sentence_entry, :migrate_legacy_entries,
+              :with_error_id, :next_error_id, :format_error_id, :entry_sort_key
+    end
+  end
+end

data/lib/ibex/error_messages.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "error"
+
+module Ibex
+  # Parses, validates, and deterministically updates example-keyed syntax
+  # error messages.
+  module ErrorMessages
+    HEADER_V1 = "# ibex-messages v1"
+    HEADER = "# ibex-messages v2"
+
+    # One active or archived error-sentence entry with its source line.
+    class Entry
+      attr_reader :state #: Integer?
+      attr_reader :status #: :active | :unreachable | :removed
+      attr_reader :message #: String?
+      attr_reader :line #: Integer
+      attr_reader :sentence #: Array[String]?
+      attr_reader :error_id #: String?
+      attr_reader :entry #: String?
+
+      # @rbs (state: Integer?, status: :active | :unreachable | :removed, message: String?, line: Integer,
+      #   ?sentence: Array[String]?, ?error_id: String?, ?entry: String?) -> void
+      def initialize(state:, status:, message:, line:, sentence: nil, error_id: nil, entry: nil)
+        @state = state
+        @status = status
+        @message = message&.freeze
+        @line = line
+        @sentence = sentence&.dup&.freeze
+        @error_id = error_id&.freeze
+        @entry = entry&.freeze
+        freeze
+      end
+    end
+
+    # Immutable parsed messages document.
+    class Document
+      attr_reader :version #: Integer
+      attr_reader :entries #: Array[Entry]
+
+      # @rbs (entries: Array[Entry], ?version: Integer) -> void
+      def initialize(entries:, version: 2)
+        @version = version
+        @entries = entries.freeze
+        freeze
+      end
+    end
+
+    # Deterministic update output and review classifications.
+    class Update
+      attr_reader :source #: String
+      attr_reader :uncovered #: Array[String]
+      attr_reader :unreachable #: Array[String]
+      attr_reader :moved #: Array[String]
+
+      # @rbs (source: String, uncovered: Array[String], unreachable: Array[String], moved: Array[String]) -> void
+      def initialize(source:, uncovered:, unreachable:, moved:)
+        @source = source.freeze
+        @uncovered = uncovered.freeze
+        @unreachable = unreachable.freeze
+        @moved = moved.freeze
+        freeze
+      end
+    end
+
+    require_relative "error_messages/parser"
+    require_relative "error_messages/sentence_search"
+
+    # @rbs (String source, file: String) -> Document
+    def parse(source, file:)
+      Parser.new(source, file: file).parse
+    end
+    module_function :parse
+
+    # @rbs (String path) -> Document
+    def load(path)
+      parse(File.binread(path), file: path)
+    end
+    module_function :load
+
+    # @rbs (IR::Automaton automaton) -> Array[IR::AutomatonState]
+    def error_states(automaton)
+      terminals = ordinary_terminals(automaton)
+      automaton.states.select do |state|
+        terminals.any? { |terminal| error_action?(state.actions[terminal.id] || state.default_action) }
+      end.sort_by(&:id)
+    end
+    module_function :error_states
+
+    # @rbs (Document document, IR::Automaton automaton, file: String) -> Hash[Integer, String]
+    def messages_for(document, automaton, file:)
+      records_for(document, automaton, file: file).transform_values { |record| record.fetch(:message) }.freeze
+    end
+    module_function :messages_for
+
+    # @rbs (Document document, IR::Automaton automaton, file: String) ->
+    #   Hash[Integer, { id: String, message: String }]
+    def records_for(document, automaton, file:)
+      return legacy_records_for(document, automaton, file: file) if document.version == 1
+
+      search = SentenceSearch.new(automaton)
+      records = {} #: Hash[Integer, { id: String, message: String }]
+      document.entries.each do |entry|
+        next unless entry.status == :active && entry.message
+
+        sentence = entry.sentence || raise(Ibex::Error, "missing active error sentence")
+        state = search.state_for(sentence, entry: entry.entry)
+        unless state
+          fail_at(file, entry.line, 1,
+                  "error sentence no longer reaches a syntax error; run `ibex errors --update`")
+        end
+        if records[state]
+          fail_at(file, entry.line, 1,
+                  "multiple messages reach error state #{state}; run `ibex errors --update`")
+        end
+
+        error_id = entry.error_id || raise(Ibex::Error, "missing error id")
+        records[state] = { id: error_id, message: entry.message }
+      end
+      records.sort.to_h.freeze
+    end
+    module_function :records_for
+
+    # @rbs (Document document, IR::Automaton automaton, file: String) ->
+    #   Hash[Integer, { id: String, message: String }]
+    def legacy_records_for(document, automaton, file:)
+      valid = error_states(automaton).to_h { |state| [state.id, true] }
+      records = {} #: Hash[Integer, { id: String, message: String }]
+      document.entries.each do |entry|
+        next if entry.status == :removed || !entry.message
+
+        state = entry.state || raise(Ibex::Error, "missing legacy error state")
+        unless valid[state]
+          fail_at(file, entry.line, 1,
+                  "unknown error state #{state} for current automaton; run `ibex errors --update`")
+        end
+        records[state] = { id: format_error_id(state + 1), message: entry.message }
+      end
+      records.sort.to_h.freeze
+    end
+
+    # @rbs (IR::Automaton automaton) -> Array[IR::GrammarSymbol]
+    def ordinary_terminals(automaton)
+      automaton.grammar.terminals.reject { |terminal| terminal.name == "error" }
+    end
+
+    # @rbs (IR::parser_action? action) -> bool
+    def error_action?(action)
+      action.nil? || action[:type].to_sym == :error
+    end
+
+    # @rbs (String file, Integer line, Integer column, String message) -> bot
+    def fail_at(file, line, column, message)
+      raise Ibex::Error, "#{file}:#{line}:#{column}: #{message}"
+    end
+    module_function :legacy_records_for, :ordinary_terminals, :error_action?, :fail_at
+
+    class << self
+      private :legacy_records_for, :ordinary_terminals, :error_action?, :fail_at
+    end
+  end
+end
+
+require_relative "error_messages/update"