ibex 0.1.0 → 0.3.0

data/lib/ibex/lalr/lr0_collection.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+# steep:ignore:start
+
+require "set"
+
+module Ibex
+  module LALR
+    # Constructs only the LR(0) core collection.  Keeping this separate from
+    # lookahead propagation makes the direct IELR phases reusable and gives
+    # them a stable, canonical-free input representation.
+    class LR0Collection
+      AUGMENTED_PRODUCTION = -1 #: Integer
+
+      # @rbs @grammar: IR::Grammar
+      # @rbs @productions_by_lhs: Hash[Integer, Array[IR::Production]]
+      # @rbs @starts: Array[String]
+      # @rbs @augmented_rhs: Hash[Integer, Array[Integer]]
+      # @rbs @production_rhs: Array[Array[Integer]]
+      # @rbs @item_key_stride: Integer
+
+      # @rbs (IR::Grammar grammar, ?starts: Array[String]?) -> void
+      def initialize(grammar, starts: nil)
+        @grammar = grammar
+        @starts = (starts || grammar.starts).dup
+        raise ArgumentError, "starts must be a nonempty subset of grammar starts" if
+          @starts.empty? || (@starts - grammar.starts).any?
+
+        @productions_by_lhs = grammar.productions.group_by(&:lhs)
+        @production_rhs = grammar.productions.map(&:rhs).freeze
+        @augmented_production_ids = @starts.map { |name| AUGMENTED_PRODUCTION - grammar.starts.index(name) }
+        @augmented_rhs = @starts.each_with_index.to_h do |name, index|
+          symbol = grammar.symbol(name) || raise(Ibex::Error, "missing start symbol #{name}")
+          [@augmented_production_ids.fetch(index), [symbol.id].freeze]
+        end.freeze
+        @item_key_stride = [*@production_rhs, *@augmented_rhs.values].map(&:length).max.to_i + 1
+      end
+
+      # @rbs () -> [Array[core_set], transitions]
+      def build
+        states = @starts.map { |name| closure(Set[[augmented_production(@starts.index(name)), 0]]) }
+        transitions = [] #: transitions
+        indexes = {}
+        states.each_with_index { |items, index| indexes[item_key(items)] = index }
+        cursor = 0
+        while cursor < states.length
+          transitions[cursor] = {}
+          shifted_kernels(states.fetch(cursor)).keys.sort.each do |symbol_id|
+            target = closure(shifted_kernels(states.fetch(cursor)).fetch(symbol_id))
+            target_id = indexes[item_key(target)] ||= begin
+              states << target
+              states.length - 1
+            end
+            transitions.fetch(cursor)[symbol_id] = target_id
+          end
+          cursor += 1
+        end
+        [states, transitions]
+      end
+
+      # @rbs (Integer index) -> Integer
+      def augmented_production(index)
+        @augmented_production_ids.fetch(index)
+      end
+
+      # @rbs (Integer production_id) -> Array[Integer]
+      def rhs_for(production_id)
+        return @augmented_rhs.fetch(production_id) if production_id.negative?
+
+        @production_rhs.fetch(production_id)
+      end
+
+      # @rbs (Integer production_id) -> Integer
+      def lhs_for(production_id)
+        return rhs_for(production_id).fetch(0) if production_id.negative?
+
+        @grammar.productions.fetch(production_id).lhs
+      end
+
+      # @rbs (core_set items) -> Hash[Integer, core_set]
+      def shifted_kernels(items)
+        kernels = Hash.new { |hash, key| hash[key] = Set.new } #: Hash[Integer, core_set]
+        items.each do |production_id, dot|
+          symbol_id = rhs_for(production_id)[dot]
+          kernels[symbol_id] << [production_id, dot + 1].freeze if symbol_id
+        end
+        kernels
+      end
+
+      private
+
+      # @rbs (core_set seed) -> core_set
+      def closure(seed)
+        items = seed.dup
+        queue = seed.to_a
+        cursor = 0
+        while cursor < queue.length
+          production_id, dot = queue.fetch(cursor)
+          cursor += 1
+          symbol = @grammar.symbol_by_id(rhs_for(production_id)[dot])
+          next unless symbol&.nonterminal?
+
+          @productions_by_lhs.fetch(symbol.id, []).each do |production|
+            item = [production.id, 0].freeze
+            queue << item if items.add?(item)
+          end
+        end
+        items
+      end
+
+      # @rbs (core_set items) -> Array[Integer]
+      def item_key(items)
+        items.map do |production_id, dot|
+          ((production_id + @grammar.starts.length) * @item_key_stride) + dot
+        end.sort
+      end
+    end
+  end
+end
+# steep:ignore:end

data/lib/ibex/lalr/on_error_reductions.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LALR
+    # Delays syntax-error detection by filling only otherwise erroneous table
+    # cells with one uniquely highest-priority declared reduction.
+    module OnErrorReductions
+      # @rbs (IR::Grammar grammar, Array[IR::AutomatonState] states) -> Array[IR::AutomatonState]
+      def apply(grammar, states)
+        priorities = reduction_priorities(grammar)
+        return states if priorities.empty?
+
+        terminal_ids = grammar.terminals.reject { |terminal| terminal.name == "error" }.map(&:id)
+        states.map { |state| apply_state(grammar, state, priorities, terminal_ids) }
+      end
+      module_function :apply
+
+      # @rbs (IR::Grammar grammar) -> Hash[Integer, Integer]
+      def reduction_priorities(grammar)
+        priorities = {} #: Hash[Integer, Integer]
+        grammar.recovery[:on_error_reduce].each_with_index do |names, priority|
+          names.each do |name|
+            symbol = grammar.symbol(name) || raise(Ibex::Error, "missing on-error reduction symbol #{name}")
+            priorities[symbol.id] = priority
+          end
+        end
+        priorities
+      end
+
+      # @rbs (IR::Grammar grammar, IR::AutomatonState state, Hash[Integer, Integer] priorities,
+      #   Array[Integer] terminal_ids) -> IR::AutomatonState
+      def apply_state(grammar, state, priorities, terminal_ids)
+        production_id = selected_production(grammar, state, priorities)
+        return state unless production_id
+
+        reduction = { type: :reduce, production: production_id } #: IR::reduce_action
+        actions = state.actions.dup
+        terminal_ids.each do |token_id|
+          action = actions[token_id]
+          actions[token_id] = reduction if action.nil? || action[:type] == :error
+        end
+        IR::AutomatonState.new(
+          id: state.id, items: state.items, transitions: state.transitions,
+          actions: actions, gotos: state.gotos, default_action: state.default_action,
+          conflicts: state.conflicts
+        )
+      end
+
+      # @rbs (IR::Grammar grammar, IR::AutomatonState state, Hash[Integer, Integer] priorities) -> Integer?
+      def selected_production(grammar, state, priorities)
+        candidates = state.items.filter_map do |item|
+          next if item.production.negative?
+
+          production = grammar.productions.fetch(item.production)
+          next unless item.dot == production.rhs.length
+
+          priority = priorities[production.lhs]
+          [priority, production.id] if priority
+        end.uniq
+        highest = candidates.filter_map(&:first).max
+        return unless highest
+
+        productions = candidates.filter_map { |priority, production| production if priority == highest }.uniq
+        productions.one? ? productions.fetch(0) : nil
+      end
+
+      module_function :reduction_priorities, :apply_state, :selected_production
+
+      class << self
+        private :reduction_priorities, :apply_state, :selected_production
+      end
+    end
+  end
+end

data/lib/ibex/lalr/unreachable_states.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+# steep:ignore:start
+
+module Ibex
+  module LALR
+    # Optional post-resolution compaction.  It is intentionally off by
+    # default because state numbers are part of diagnostics and golden output.
+    module UnreachableStates
+      module_function
+
+      # @rbs (Array[IR::AutomatonState] states, Array[Integer] start_states) ->
+      #   [Array[IR::AutomatonState], Hash[Integer, Integer]]
+      def remove(states, start_states)
+        reachable = reachable_states(states, start_states)
+        mapping = reachable.each_with_index.to_h { |old_id, new_id| [old_id, new_id] }
+        return [states, mapping] if reachable.length == states.length
+
+        compacted = reachable.map do |old_id|
+          state = states.fetch(old_id)
+          transitions = remap_edges(state.transitions, mapping)
+          gotos = remap_edges(state.gotos, mapping)
+          actions = state.actions.transform_values { |action| remap_action(action, mapping) }
+          default_action = remap_action(state.default_action, mapping)
+          conflicts = state.conflicts.map { |conflict| remap_conflict(conflict, mapping) }
+          IR::AutomatonState.new(
+            id: mapping.fetch(old_id), items: state.items, transitions: transitions,
+            actions: actions, gotos: gotos, default_action: default_action, conflicts: conflicts
+          )
+        end
+        [compacted, mapping]
+      end
+
+      # @rbs (Array[IR::AutomatonState], Array[Integer]) -> Array[Integer]
+      def reachable_states(states, start_states)
+        visited = {}
+        queue = start_states.dup
+        until queue.empty?
+          state_id = queue.shift
+          next if visited[state_id]
+
+          visited[state_id] = true
+          state = states.fetch(state_id)
+          state.gotos.each_value { |target| queue << target }
+          state.actions.each_value do |action|
+            queue << action[:state] if action[:type] == :shift
+          end
+        end
+        visited.keys.sort
+      end
+
+      # @rbs (Hash[Integer, Integer], Hash[Integer, Integer]) -> Hash[Integer, Integer]
+      def remap_edges(edges, mapping)
+        edges.each_with_object({}) do |(symbol_id, target), result|
+          result[symbol_id] = mapping[target] if mapping.key?(target)
+        end
+      end
+
+      # @rbs (IR::parser_action?, Hash[Integer, Integer]) -> IR::parser_action?
+      def remap_action(action, mapping)
+        return nil unless action
+        return action unless action[:type] == :shift
+        return { type: :shift, state: mapping.fetch(action[:state]) } if mapping.key?(action[:state])
+
+        { type: :error }
+      end
+
+      # @rbs (IR::conflict, Hash[Integer, Integer]) -> IR::conflict
+      def remap_conflict(conflict, mapping)
+        result = conflict.dup
+        if result[:type] == :shift_reduce && mapping.key?(result[:shift_to])
+          result[:shift_to] = mapping.fetch(result[:shift_to])
+        end
+        result
+      end
+    end
+  end
+end
+# steep:ignore:end

data/lib/ibex/lalr.rb

@@ -1,7 +1,23 @@
 # frozen_string_literal: true
 
 require_relative "lalr/conflict"
+require_relative "lalr/on_error_reductions"
 require_relative "lalr/default_reductions"
+require_relative "lalr/build_metrics"
+require_relative "lalr/lr0_collection"
+require_relative "lalr/goto_follows"
+require_relative "lalr/lookahead_propagation"
+require_relative "lalr/unreachable_states"
+require_relative "lalr/ielr/bits"
+require_relative "lalr/ielr/inadequacy"
+require_relative "lalr/ielr/item_lookaheads"
+require_relative "lalr/ielr/split_stability"
+require_relative "lalr/ielr/annotator"
+require_relative "lalr/ielr/split_state"
+require_relative "lalr/ielr/state_splitter"
+require_relative "lalr/ielr/pipeline"
+require_relative "lalr/direct_lookaheads"
+require_relative "lalr/ielr_partition"
 require_relative "lalr/builder"
 require_relative "lalr/conflict_search_limits"
 require_relative "lalr/conflict_search"
@@ -12,10 +28,33 @@ module Ibex
     # @rbs!
     #   type lr_item = [Integer, Integer, Integer]
     #   type item_core = [Integer, Integer]
+    #   type core_set = Set[item_core]
     #   type item_set = Set[lr_item]
     #   type packed_items = Hash[item_core, Set[Integer]]
     #   type transitions = Array[Hash[Integer, Integer]]
-    #   type derivation_node = Hash[Symbol, untyped]
+    #   type build_collection = {
+    #     construction_states: Integer,
+    #     canonical_states: Integer?,
+    #     strategy: Symbol,
+    #     lr0_states: Integer?,
+    #     lr0_items: Integer?,
+    #     canonical_items: Integer?,
+    #     propagation_edges: Integer?,
+    #     ielr_initial_partitions: Integer?,
+    #     ielr_final_partitions: Integer?,
+    #     ielr_annotations: Integer?,
+    #     ielr_annotated_states: Integer?,
+    #     ielr_inadequacies: Integer?,
+    #     ielr_split_stable_discarded: Integer?,
+    #     ielr_lalr_states: Integer?,
+    #     ielr_split_states: Integer?,
+    #     ielr_unreachable_removed: Integer?,
+    #     ielr_remergeable_candidates: Integer?
+    #   }
+    #   type conflict_fingerprint = [Symbol, String, Array[Integer]]
+    #   type lookahead_node = [Integer, Integer, Integer]
+    #   type state_partition = Array[Integer]
+    #   type derivation_node = Hash[Symbol, Object?]
     #   type search_status = :conflict | :shifted | :accepted
     #   type search_entry = [search_status, ConflictSearch::Configuration]
     #   type search_result = {
@@ -23,5 +62,24 @@ module Ibex
     #     lookahead_index: Integer,
     #     interpretations: Array[IR::interpretation]
     #   }
+    #   type search_bounds = { max_tokens: Integer, max_configurations: Integer }
+    #   type search_outcome = {
+    #     status: :found | :not_found | :exhausted,
+    #     result: search_result?,
+    #     explored: Integer,
+    #     exhausted: bool,
+    #     bounds: search_bounds
+    #   }
+    #   type search_counterexample = {
+    #     state: Integer,
+    #     type: Symbol,
+    #     symbol_path: Array[String],
+    #     sentence: Array[String],
+    #     lookahead_index: Integer,
+    #     unifying: bool,
+    #     inconclusive: bool,
+    #     search: search_outcome,
+    #     interpretations: Array[IR::interpretation]
+    #   }
   end
 end

data/lib/ibex/location.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  # Immutable source range suitable for lexer tokens and parser diagnostics.
+  #
+  # Coordinates are one-based. Byte offsets are optional, zero-based, and
+  # half-open. Keeping both forms lets Unicode-aware lexers report human
+  # columns without losing an exact source slice.
+  class Location
+    attr_reader :file #: String?
+    attr_reader :line #: Integer
+    attr_reader :column #: Integer
+    attr_reader :end_line #: Integer
+    attr_reader :end_column #: Integer
+    attr_reader :start_byte #: Integer?
+    attr_reader :end_byte #: Integer?
+    attr_reader :source_line #: String?
+
+    # @rbs (?file: String?, line: Integer, column: Integer, ?end_line: Integer?,
+    #   ?end_column: Integer?, ?start_byte: Integer?, ?end_byte: Integer?, ?source_line: String?) -> void
+    def initialize(line:, column:, file: nil, end_line: nil, end_column: nil,
+                   start_byte: nil, end_byte: nil, source_line: nil)
+      @file = file&.dup&.freeze
+      @line = positive_coordinate(:line, line)
+      @column = positive_coordinate(:column, column)
+      @end_line = positive_coordinate(:end_line, end_line || line)
+      @end_column = positive_coordinate(:end_column, end_column || column)
+      @start_byte = optional_offset(:start_byte, start_byte)
+      @end_byte = optional_offset(:end_byte, end_byte)
+      @source_line = source_line&.dup&.freeze
+      validate_order!
+      freeze
+    end
+
+    # Return the smallest range covering both locations.
+    # @rbs (Location other) -> Location
+    def join(other)
+      raise ArgumentError, "location files differ" unless file == other.file
+
+      first = before_or_equal?(self, other) ? self : other
+      ending = before_or_equal_end?(self, other) ? other : self
+      self.class.new(
+        file: file, line: first.line, column: first.column,
+        end_line: ending.end_line, end_column: ending.end_column,
+        start_byte: joined_start_byte(other), end_byte: joined_end_byte(other),
+        source_line: first.source_line
+      )
+    end
+
+    # Fold a nonempty collection into one covering range.
+    # @rbs (Enumerable[Location] locations) -> Location
+    def self.join(locations)
+      values = locations.to_a
+      first = values.shift || raise(ArgumentError, "locations must not be empty")
+      values.reduce(first) { |combined, location| combined.join(location) }
+    end
+
+    # @rbs () -> bool
+    def empty?
+      @line == @end_line && @column == @end_column &&
+        (@start_byte.nil? || @end_byte.nil? || @start_byte == @end_byte)
+    end
+
+    # @rbs () -> Hash[Symbol, String | Integer]
+    def to_h
+      {
+        file: @file, line: @line, column: @column,
+        end_line: @end_line, end_column: @end_column,
+        start_byte: @start_byte, end_byte: @end_byte
+      }.compact
+    end
+
+    private
+
+    # @rbs (Symbol name, Object? value) -> Integer
+    def positive_coordinate(name, value)
+      return value if value.is_a?(Integer) && value.positive?
+
+      raise ArgumentError, "#{name} must be a positive Integer"
+    end
+
+    # @rbs (Symbol name, Object? value) -> Integer?
+    def optional_offset(name, value)
+      return if value.nil?
+      return value if value.is_a?(Integer) && value >= 0
+
+      raise ArgumentError, "#{name} must be a nonnegative Integer or nil"
+    end
+
+    # @rbs () -> void
+    def validate_order!
+      if @end_line < @line || (@end_line == @line && @end_column < @column)
+        raise ArgumentError, "end position precedes start position"
+      end
+
+      start_byte = @start_byte
+      end_byte = @end_byte
+      return unless start_byte && end_byte && end_byte < start_byte
+
+      raise ArgumentError, "end_byte precedes start_byte"
+    end
+
+    # @rbs (Location left, Location right) -> bool
+    def before_or_equal?(left, right)
+      left.line < right.line || (left.line == right.line && left.column <= right.column)
+    end
+
+    # @rbs (Location left, Location right) -> bool
+    def before_or_equal_end?(left, right)
+      left.end_line < right.end_line ||
+        (left.end_line == right.end_line && left.end_column <= right.end_column)
+    end
+
+    # @rbs (Location other) -> Integer?
+    def joined_start_byte(other)
+      return unless @start_byte && other.start_byte
+
+      [@start_byte, other.start_byte].min
+    end
+
+    # @rbs (Location other) -> Integer?
+    def joined_end_byte(other)
+      return unless @end_byte && other.end_byte
+
+      [@end_byte, other.end_byte].max
+    end
+  end
+end

data/lib/ibex/lsp/document_handlers.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Applies full-text LSP document notifications to the overlay-backed store.
+    module DocumentHandlers
+      include RequestSupport
+
+      private
+
+      # @rbs (lsp_value? raw_params) -> nil
+      def did_open(raw_params)
+        require_running!
+        params = params_hash(raw_params)
+        document = hash_member(params, "textDocument")
+        publications = store.open(
+          string_member(document, "uri"), integer_member(document, "version"), string_member(document, "text")
+        )
+        publish(publications)
+        nil
+      end
+
+      # @rbs (lsp_value? raw_params) -> nil
+      def did_change(raw_params)
+        require_running!
+        params = params_hash(raw_params)
+        document = hash_member(params, "textDocument")
+        changes = params["contentChanges"]
+        first_change = changes.is_a?(Array) ? changes.first : nil
+        unless changes.is_a?(Array) && changes.one? && first_change.is_a?(Hash) &&
+               !first_change.key?("range") && !first_change.key?("rangeLength")
+          raise ProtocolError.new("full sync requires exactly one range-free content change", code: -32_602)
+        end
+
+        change = changes.fetch(0) #: lsp_object
+        source = string_member(change, "text")
+        publications = store.change(
+          string_member(document, "uri"), integer_member(document, "version"), source
+        )
+        publish(publications)
+        nil
+      end
+
+      # @rbs (lsp_value? raw_params) -> nil
+      def did_save(raw_params)
+        require_running!
+        params = params_hash(raw_params)
+        document = hash_member(params, "textDocument")
+        source = params["text"]
+        unless source.nil? || source.is_a?(String)
+          raise ProtocolError.new("saved document text must be a string", code: -32_602)
+        end
+
+        publish(store.save(string_member(document, "uri"), source: source))
+        nil
+      end
+
+      # @rbs (lsp_value? raw_params) -> nil
+      def did_close(raw_params)
+        require_running!
+        params = params_hash(raw_params)
+        document = hash_member(params, "textDocument")
+        publish(store.close(string_member(document, "uri")))
+        nil
+      end
+    end
+  end
+end