ibex-runtime 0.2.0

data/lib/ibex/runtime/cst/incremental/blender.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Runtime
+    module CST
+      # A Green nonterminal that can replace its old LR shift/reduce sequence.
+      class ReusableSubtree
+        attr_reader :green #: GreenNode
+        attr_reader :lhs #: Integer
+        attr_reader :left_state #: Integer
+        attr_reader :left_states #: Array[Integer?]
+        attr_reader :previous_trailing #: Array[GreenTrivia]
+
+        # @rbs (green: GreenNode, lhs: Integer, left_state: Integer, left_states: Array[Integer?],
+        #   previous_trailing: Array[GreenTrivia]) -> void
+        def initialize(green:, lhs:, left_state:, left_states:, previous_trailing:)
+          @green = green
+          @lhs = lhs
+          @left_state = left_state
+          @left_states = left_states.dup.freeze
+          @previous_trailing = previous_trailing.dup.freeze
+          freeze
+        end
+      end
+
+      # A state-aware token source that substitutes validated old subtrees.
+      class Blender
+        class Candidate
+          attr_reader :entry #: ReusableSubtree
+          attr_reader :token_count #: Integer
+
+          # @rbs (entry: ReusableSubtree, token_count: Integer) -> void
+          def initialize(entry:, token_count:)
+            @entry = entry
+            @token_count = token_count
+            freeze
+          end
+        end
+
+        UNSAFE_FLAGS = Flags::CONTAINS_ERROR | Flags::CONTAINS_MISSING | Flags::CONTAINS_SKIPPED #: Integer
+
+        attr_reader :reused_descendants #: Integer
+        attr_reader :decomposed_nodes #: Integer
+        attr_reader :fallback_reason #: Symbol?
+
+        # @rbs @old_root: SyntaxNode
+        # @rbs @parse_memo: ParseMemo
+        # @rbs @lexed: LexedSyntax
+        # @rbs @edits: Array[TextEdit]
+        # @rbs @max_decomposed_nodes: Integer
+        # @rbs @enabled: bool
+        # @rbs @candidates: Hash[Integer, Array[Candidate]]
+        # @rbs @new_offsets: Hash[Integer, Integer]
+        # @rbs @old_tokens: Array[GreenToken]
+        # @rbs @token_index: Integer
+
+        # @rbs (old_root: SyntaxNode, parse_memo: ParseMemo, lexed: LexedSyntax, edits: Array[TextEdit],
+        #   max_decomposed_nodes: Integer, ?enabled: bool) -> void
+        def initialize(old_root:, parse_memo:, lexed:, edits:, max_decomposed_nodes:, enabled: true)
+          @old_root = old_root
+          @parse_memo = parse_memo
+          @lexed = lexed
+          @edits = TextEdit.normalize(edits)
+          @max_decomposed_nodes = max_decomposed_nodes
+          @enabled = enabled
+          @candidates = {} #: Hash[Integer, Array[Candidate]]
+          @new_offsets = offset_index(@lexed.memo.offsets)
+          @old_tokens = @old_root.tokens.map(&:green)
+          @token_index = 0
+          @reused_descendants = 0
+          @decomposed_nodes = 0
+          @fallback_reason = nil #: Symbol?
+          build_candidates if @enabled && reusable_root?
+        end
+
+        # Return the next token or a whole nonterminal valid in the current LR state.
+        # @rbs (Integer state) -> (Array[untyped] | ReusableSubtree | false)
+        def next_for_state(state)
+          candidates = @candidates[@token_index]
+          candidate = candidates&.find { |item| item.entry.left_state == state }
+          if candidate
+            @token_index += candidate.token_count
+            @reused_descendants += candidate.entry.green.descendant_count
+            return candidate.entry
+          end
+
+          token = @lexed.raw_tokens[@token_index]
+          return false unless token
+
+          @token_index += 1
+          token
+        end
+
+        # @rbs () -> Integer
+        def token_count = @lexed.memo.tokens.length
+
+        private
+
+        # @rbs () -> bool
+        def reusable_root?
+          @old_root.green.flags.nobits?(UNSAFE_FLAGS) &&
+            @parse_memo.left_states.length == @old_root.green.descendant_count
+        end
+
+        # @rbs () -> void
+        def build_candidates
+          walk(@old_root.green, offset: 0, preorder: 0, token_index: 0)
+          @candidates.each_value do |values|
+            values.sort_by! { |candidate| -candidate.entry.green.descendant_count }
+          end
+        rescue ResourceLimitError
+          @candidates.clear
+          @fallback_reason = :decomposition_budget
+        end
+
+        # @rbs (GreenNode | GreenToken element, offset: Integer, preorder: Integer, token_index: Integer) ->
+        #   [Integer, Integer, Integer]
+        def walk(element, offset:, preorder:, token_index:)
+          next_preorder = preorder + 1
+          return [next_preorder, offset + element.full_width, token_index + 1] if element.is_a?(GreenToken)
+
+          consume_decomposition_budget!
+          start_token = token_index
+          element.children.each do |child|
+            next_preorder, offset, token_index = walk(
+              child, offset: offset, preorder: next_preorder, token_index: token_index
+            )
+          end
+          add_candidate(element, preorder, offset - element.full_width, start_token, token_index)
+          [next_preorder, offset, token_index]
+        end
+
+        # @rbs (GreenNode node, Integer preorder, Integer old_offset, Integer start_token, Integer end_token) -> void
+        def add_candidate(node, preorder, old_offset, start_token, end_token)
+          return unless reusable_node?(node, old_offset)
+
+          new_offset = translated_offset(old_offset)
+          new_index = @new_offsets[new_offset]
+          token_count = end_token - start_token
+          return unless new_index && token_count.positive?
+          return unless matching_tokens?(start_token, new_index, token_count)
+          return unless matching_follow_token?(end_token, new_index + token_count)
+
+          left_state = @parse_memo.left_state(preorder)
+          return unless left_state
+
+          entry = ReusableSubtree.new(
+            green: node,
+            lhs: @old_root.kinds.nonterminal_of(node.kind),
+            left_state: left_state,
+            left_states: @parse_memo.slice(preorder, node),
+            previous_trailing: previous_trailing(new_index)
+          )
+          empty = [] #: Array[Candidate]
+          (@candidates[new_index] ||= empty) << Candidate.new(entry: entry, token_count: token_count)
+        end
+
+        # @rbs (GreenNode node, Integer old_offset) -> bool
+        def reusable_node?(node, old_offset)
+          node.full_width.positive? &&
+            node.flags.nobits?(UNSAFE_FLAGS) &&
+            @old_root.kinds.nonterminal?(node.kind) &&
+            outside_damage?(old_offset, old_offset + node.full_width)
+        end
+
+        # @rbs (Integer start_offset, Integer finish_offset) -> bool
+        def outside_damage?(start_offset, finish_offset)
+          @edits.none? do |edit|
+            edit_finish = edit.start + edit.delete_length
+            if edit.delete_length.zero?
+              edit.start >= start_offset && edit.start < finish_offset
+            else
+              edit.start < finish_offset && edit_finish > start_offset
+            end
+          end
+        end
+
+        # @rbs (Integer old_offset) -> Integer
+        def translated_offset(old_offset)
+          delta = @edits.sum do |edit|
+            edit.start + edit.delete_length <= old_offset ? edit.insert_text.bytesize - edit.delete_length : 0
+          end
+          old_offset + delta
+        end
+
+        # @rbs (Integer old_index, Integer new_index, Integer count) -> bool
+        def matching_tokens?(old_index, new_index, count)
+          count.times.all? do |relative|
+            @old_tokens.fetch(old_index + relative).equal?(@lexed.memo.tokens[new_index + relative])
+          end
+        end
+
+        # @rbs (Integer old_index, Integer new_index) -> bool
+        def matching_follow_token?(old_index, new_index)
+          old_follow = @old_tokens[old_index]
+          new_follow = @lexed.memo.tokens[new_index]
+          old_follow && new_follow ? old_follow.equal?(new_follow) : false
+        end
+
+        # @rbs (Integer token_index) -> Array[GreenTrivia]
+        def previous_trailing(token_index)
+          location = @lexed.raw_tokens.fetch(token_index).fetch(2)
+          return [] unless location.is_a?(Hash)
+
+          value = location[:cst_previous_trailing]
+          value.is_a?(Array) ? value.grep(GreenTrivia) : []
+        end
+
+        # @rbs (Array[Integer] offsets) -> Hash[Integer, Integer]
+        def offset_index(offsets)
+          result = {} #: Hash[Integer, Integer]
+          offsets.each_with_index { |offset, index| result[offset] ||= index }
+          result
+        end
+
+        # @rbs () -> void
+        def consume_decomposition_budget!
+          @decomposed_nodes += 1
+          return if @decomposed_nodes <= @max_decomposed_nodes
+
+          raise ResourceLimitError.new(
+            resource: :incremental_decomposed_nodes,
+            limit: @max_decomposed_nodes,
+            observed: @decomposed_nodes,
+            state: nil,
+            location: nil
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ibex/runtime/cst/incremental/lexed_syntax.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Runtime
+    module CST
+      # One complete generated-lexer pass, including the explicit EOF token.
+      class LexedSyntax
+        attr_reader :raw_tokens #: Array[Array[untyped]]
+        attr_reader :memo #: TokenMemo
+
+        # @rbs (raw_tokens: Array[Array[untyped]], memo: TokenMemo) -> void
+        def initialize(raw_tokens:, memo:)
+          raise ArgumentError, "raw and Green token counts differ" unless raw_tokens.length == memo.tokens.length
+
+          @raw_tokens = raw_tokens.map { |token| token.dup.freeze }.freeze
+          @memo = memo
+          freeze
+        end
+
+        # @rbs (TokenMemo replacement) -> LexedSyntax
+        def with_memo(replacement)
+          LexedSyntax.new(raw_tokens: @raw_tokens, memo: replacement)
+        end
+      end
+    end
+  end
+end

data/lib/ibex/runtime/cst/incremental/parse_memo.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Runtime
+    module CST
+      # Parser-state metadata parallel to Green preorder occurrences.
+      class ParseMemo
+        VERSION = 1 #: Integer
+        ENTRY_BYTES = 8 #: Integer
+
+        attr_reader :left_states #: Array[Integer?]
+        attr_reader :grammar_digest #: String
+        attr_reader :state_count #: Integer
+        attr_reader :production_count #: Integer
+
+        # @rbs (left_states: Array[Integer?], grammar_digest: String, state_count: Integer,
+        #   production_count: Integer) -> void
+        def initialize(left_states:, grammar_digest:, state_count:, production_count:)
+          unless left_states.all? { |state| state.nil? || state.between?(0, state_count - 1) }
+            raise ArgumentError, "parse memo contains an invalid parser state"
+          end
+
+          @left_states = left_states.dup.freeze
+          @grammar_digest = grammar_digest.dup.freeze
+          @state_count = state_count
+          @production_count = production_count
+          freeze
+        end
+
+        # @rbs (Integer preorder_index) -> Integer?
+        def left_state(preorder_index) = @left_states.fetch(preorder_index)
+
+        # @rbs (Integer preorder_index, GreenNode | GreenToken element) -> Array[Integer?]
+        def slice(preorder_index, element)
+          value = @left_states.slice(preorder_index, element.descendant_count)
+          return value if value && value.length == element.descendant_count
+
+          raise IndexError, "parse memo subtree range is outside the preorder state array"
+        end
+
+        # @rbs (Hash[Symbol, untyped] tables) -> bool
+        def compatible?(tables)
+          @grammar_digest == tables[:grammar_digest] &&
+            @state_count == tables[:state_count] &&
+            @production_count == tables[:production_count]
+        end
+
+        # @rbs () -> Integer
+        def estimated_bytes = @left_states.length * ENTRY_BYTES
+
+        # @rbs () -> Hash[String, untyped]
+        def to_h
+          { "version" => VERSION, "left_states" => @left_states }.freeze
+        end
+
+        # Lightweight construction tree flattened once when a parse completes.
+        class Entry
+          empty_children = [] # @type var empty_children: Array[Entry]
+          EMPTY_CHILDREN = empty_children.freeze #: Array[Entry]
+
+          # @rbs @state: Integer?
+          # @rbs @children: Array[Entry]
+          # @rbs @segment: Array[Integer?]?
+
+          # @rbs (Integer? state, ?children: Array[Entry], ?segment: Array[Integer?]?) -> void
+          def initialize(state, children: EMPTY_CHILDREN, segment: nil)
+            @state = state
+            @children = if children.empty?
+                          EMPTY_CHILDREN
+                        elsif children.frozen?
+                          children
+                        else
+                          children.dup.freeze
+                        end
+            @segment = segment&.then { |values| values.frozen? ? values : values.dup.freeze }
+            freeze
+          end
+
+          # @rbs (Array[Integer?] output) -> void
+          def append_to(output)
+            segment = @segment
+            if segment
+              output.concat(segment)
+              return
+            end
+
+            output << @state
+            @children.each { |child| child.append_to(output) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ibex/runtime/cst/incremental/relexer.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Runtime
+    module CST
+      # Result of comparing one fresh lexical pass with the prior token memo.
+      class RelexResult
+        attr_reader :memo #: TokenMemo
+        attr_reader :reused_count #: Integer
+        attr_reader :scanned_count #: Integer
+        attr_reader :resynchronized_at #: Integer?
+
+        # @rbs (memo: TokenMemo, reused_count: Integer, scanned_count: Integer,
+        #   resynchronized_at: Integer?) -> void
+        def initialize(memo:, reused_count:, scanned_count:, resynchronized_at:)
+          @memo = memo
+          @reused_count = reused_count
+          @scanned_count = scanned_count
+          @resynchronized_at = resynchronized_at
+          freeze
+        end
+
+        # @rbs () -> Float
+        def reused_ratio
+          return 0.0 if @memo.tokens.empty?
+
+          @reused_count.fdiv(@memo.tokens.length)
+        end
+      end
+
+      # Sound Stage-A comparison: scan from the beginning and reuse only exact
+      # token/state/boundary matches, then recognize a shifted suffix boundary.
+      class Relexer
+        # @rbs (TokenMemo previous, TokenMemo current, Array[TextEdit] edits) -> RelexResult
+        def self.reconcile(previous, current, edits)
+          normalized = TextEdit.normalize(edits)
+          return RelexResult.new(memo: current, reused_count: 0, scanned_count: 0, resynchronized_at: nil) if
+            normalized.empty?
+
+          damage, delta, resync_boundary = edit_window(normalized)
+          old_by_offset = offset_index(previous)
+          tokens = current.tokens.dup
+          reused, scanned, resynchronized_at = reuse_tokens(
+            previous, current, tokens, old_by_offset, damage, delta, resync_boundary
+          )
+          memo = TokenMemo.new(tokens: tokens, offsets: current.offsets, states: current.states)
+          RelexResult.new(
+            memo: memo,
+            reused_count: reused,
+            scanned_count: scanned,
+            resynchronized_at: resynchronized_at
+          )
+        end
+
+        # @rbs (Array[TextEdit] edits) -> [Integer, Integer, Integer]
+        def self.edit_window(edits)
+          delta = edits.sum { |edit| edit.insert_text.bytesize - edit.delete_length }
+          prefix_delta = edits.take(edits.length - 1).sum do |edit|
+            edit.insert_text.bytesize - edit.delete_length
+          end
+          last_edit = edits.last
+          [edits.first.start, delta, last_edit.start + prefix_delta + last_edit.insert_text.bytesize]
+        end
+
+        # @rbs (TokenMemo memo) -> Hash[Integer, Integer]
+        def self.offset_index(memo)
+          result = {} #: Hash[Integer, Integer]
+          memo.offsets.each_with_index { |offset, index| result[offset] = index }
+          result
+        end
+
+        # @rbs (TokenMemo previous, TokenMemo current, Array[GreenToken] tokens,
+        #   Hash[Integer, Integer] old_by_offset, Integer damage, Integer delta, Integer resync_boundary) ->
+        #   [Integer, Integer, Integer?]
+        def self.reuse_tokens(previous, current, tokens, old_by_offset, damage, delta, resync_boundary)
+          reused = 0
+          scanned = 0
+          current.tokens.each_index do |index|
+            new_offset = current.offsets.fetch(index)
+            suffix = reusable_suffix(
+              previous, current, index, new_offset, delta, resync_boundary, old_by_offset
+            )
+            if suffix
+              old_index, length = suffix
+              copy_suffix(tokens, previous.tokens, index, old_index, length)
+              return [reused + length, scanned, index]
+            end
+
+            old_index = index if new_offset < damage
+            scanned += 1
+            next unless old_index
+            next unless reusable_token?(previous, old_index, current, index)
+
+            tokens[index] = previous.tokens.fetch(old_index)
+            reused += 1
+          end
+          [reused, scanned, nil]
+        end
+
+        # @rbs (TokenMemo previous, TokenMemo current, Integer index, Integer new_offset, Integer delta,
+        #   Integer boundary, Hash[Integer, Integer] old_by_offset) -> [Integer, Integer]?
+        def self.reusable_suffix(previous, current, index, new_offset, delta, boundary, old_by_offset)
+          return if new_offset < boundary
+
+          old_index = old_by_offset[new_offset - delta]
+          return unless old_index
+
+          length = current.tokens.length - index
+          return unless length == previous.tokens.length - old_index
+          return unless previous.states.fetch(old_index) == current.states.fetch(index)
+
+          [old_index, length]
+        end
+
+        # @rbs (Array[GreenToken] target, Array[GreenToken] source, Integer target_index,
+        #   Integer source_index, Integer length) -> void
+        def self.copy_suffix(target, source, target_index, source_index, length)
+          length.times do |relative|
+            target[target_index + relative] = source.fetch(source_index + relative)
+          end
+        end
+
+        # @rbs (TokenMemo previous, Integer old_index, TokenMemo current, Integer new_index) -> bool
+        def self.reusable_token?(previous, old_index, current, new_index)
+          previous.states.fetch(old_index) == current.states.fetch(new_index) &&
+            previous.tokens.fetch(old_index) == current.tokens.fetch(new_index)
+        end
+        private_class_method :edit_window, :offset_index, :reuse_tokens, :reusable_suffix, :copy_suffix,
+                             :reusable_token?
+      end
+    end
+  end
+end