ibex 0.2.0 → 0.4.0

data/lib/ibex/table_artifact/builder.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    # Projects validated Automaton IR into an action-free executable sidecar.
+    class Builder
+      # @rbs!
+      #   type json_value = String | Integer | Float | bool | nil | Array[json_value] | Hash[String, json_value]
+      #   type json_hash = Hash[String, untyped]
+      #   type action_rows = Array[Hash[Integer, IR::runtime_action]]
+      #   type action_table = action_rows | Tables::CompactActions
+      #   type goto_rows = Array[Hash[Integer, Integer]]
+      #   type goto_table = goto_rows | Tables::Compact
+
+      include CSTProjection
+
+      REPRESENTATIONS = %i[plain compact].freeze #: Array[Symbol]
+
+      # @rbs (IR::Automaton automaton, ?representation: Symbol | String, ?cst_trivia: Symbol | String?,
+      #   ?omit_action_call: bool?) -> void
+      def initialize(automaton, representation: :compact, cst_trivia: nil, omit_action_call: nil)
+        @automaton = automaton
+        @grammar = automaton.grammar
+        @representation = representation.to_sym
+        @cst_trivia = effective_cst_trivia(cst_trivia)
+        @omit_action_call = omit_action_call.nil? ? @grammar.options.fetch(:omit_action_call) : omit_action_call
+        return if REPRESENTATIONS.include?(@representation)
+
+        raise ArgumentError, "representation must be :plain or :compact"
+      end
+
+      # @rbs () -> Document
+      def build
+        source_identity = {
+          "grammar_digest" => @automaton.grammar_digest,
+          "automaton_digest" => automaton_digest
+        }
+        payload = build_payload(source_identity)
+        cost = build_cost(payload)
+        identity = source_identity.merge(
+          "payload_digest" => Serializer.digest(payload),
+          "cst_metadata_digest" => Serializer.digest(payload.fetch("cst")),
+          "recovery_metadata_digest" => Serializer.digest(payload.fetch("recovery"))
+        )
+        Document.new(
+          "artifact_type" => ARTIFACT_TYPE,
+          "schema_version" => SCHEMA_VERSION,
+          "identity" => identity,
+          "payload" => payload,
+          "cost" => cost
+        )
+      end
+
+      private
+
+      # @rbs (Hash[String, String] source_identity) -> json_hash
+      def build_payload(source_identity)
+        {
+          "table_format" => table_format,
+          "source" => source_identity,
+          "state_count" => @automaton.states.length,
+          "symbols" => symbols,
+          "tokens" => tokens,
+          "entry_states" => entry_states,
+          "tables" => tables,
+          "productions" => productions,
+          "cst" => cst,
+          "recovery" => recovery,
+          "semantic_actions" => semantic_actions
+        }
+      end
+
+      # @rbs () -> String
+      def automaton_digest
+        "sha256:#{Digest::SHA256.hexdigest(IR::Serialize.dump(@automaton))}"
+      end
+
+      # @rbs () -> json_hash
+      def table_format
+        {
+          "family" => "ibex-runtime-parser-table",
+          "version" => Runtime::PARSER_TABLE_FORMAT_VERSION,
+          "representation" => @representation.to_s,
+          "action_encoding" => @representation == :compact ? "signed-row-displacement-v1" : "signed-sparse-rows-v1",
+          "goto_encoding" => @representation == :compact ? "row-displacement-v1" : "sparse-rows-v1"
+        }
+      end
+
+      # @rbs () -> Array[json_hash]
+      def symbols
+        @grammar.symbols.sort_by(&:id).map do |symbol|
+          {
+            "id" => symbol.id,
+            "name" => symbol.name,
+            "kind" => symbol.kind.to_s,
+            "display_name" => symbol.display_name
+          }
+        end
+      end
+
+      # @rbs () -> Array[json_hash]
+      def tokens
+        @grammar.terminals.sort_by(&:id).map do |symbol|
+          { "id" => symbol.id, "name" => symbol.name, "display_name" => symbol.display_name }
+        end
+      end
+
+      # @rbs () -> Array[json_hash]
+      def entry_states
+        @automaton.entry_states.map { |name, state| { "name" => name, "state" => state } }
+      end
+
+      # @rbs () -> json_hash
+      def tables
+        table_set = Tables.build(@automaton, format: @representation) #: Tables::TableSet
+        {
+          "actions" => action_table(table_set.actions),
+          "gotos" => goto_table(table_set.gotos),
+          "default_actions" => table_set.default_actions.map { |action| Tables::CompactActions.pack(action) }
+        }
+      end
+
+      # @rbs (action_table table) -> json_hash
+      def action_table(table)
+        return compact_action_table(table) if table.is_a?(Tables::CompactActions)
+
+        {
+          "encoding" => "signed-sparse-rows-v1",
+          "rows" => table.map do |row|
+            row.sort.map { |token_id, action| { "token_id" => token_id, "code" => Tables::CompactActions.pack(action) } }
+          end
+        }
+      end
+
+      # @rbs (Tables::CompactActions table) -> json_hash
+      def compact_action_table(table)
+        {
+          "encoding" => "signed-row-displacement-v1",
+          "row_count" => table.row_count,
+          "column_count" => table.column_count,
+          "offsets" => table.offsets,
+          "codes" => table.codes,
+          "checks" => table.checks
+        }
+      end
+
+      # @rbs (goto_table table) -> json_hash
+      def goto_table(table)
+        return compact_goto_table(table) if table.is_a?(Tables::Compact)
+
+        {
+          "encoding" => "sparse-rows-v1",
+          "rows" => table.map do |row|
+            row.sort.map { |symbol_id, state| { "symbol_id" => symbol_id, "state" => state } }
+          end
+        }
+      end
+
+      # @rbs (Tables::Compact table) -> json_hash
+      def compact_goto_table(table)
+        {
+          "encoding" => "row-displacement-v1",
+          "row_count" => table.row_count,
+          "dense_width" => table.dense_width,
+          "offsets" => table.offsets,
+          "values" => table.values,
+          "checks" => table.checks
+        }
+      end
+
+      # @rbs () -> Array[json_hash]
+      def productions
+        @grammar.productions.sort_by(&:id).map do |production|
+          {
+            "id" => production.id,
+            "lhs" => production.lhs,
+            "rhs" => production.rhs,
+            "rhs_length" => production.rhs.length,
+            "action_slot" => action_slot?(production) ? production.id : nil
+          }
+        end
+      end
+
+      # @rbs (IR::Production production) -> bool
+      def action_slot?(production)
+        !!(production.node || production.action || !@omit_action_call)
+      end
+
+      # @rbs () -> json_hash
+      def semantic_actions
+        {
+          "binding" => "opaque-wrapper-production-id-v1",
+          "verified" => false,
+          "slots" => @grammar.productions.select { |production| action_slot?(production) }.map(&:id)
+        }
+      end
+
+      # @rbs () -> json_hash
+      def recovery
+        sync_tokens = @grammar.recovery.fetch(:sync_tokens) #: Array[String]
+        on_error_reduce = @grammar.recovery.fetch(:on_error_reduce) #: Array[Array[String]]
+        {
+          "sync_token_ids" => sync_tokens.map { |name| @grammar.symbol(name).id },
+          "on_error_reduce_symbol_ids" => on_error_reduce.map do |group|
+            group.map { |name| @grammar.symbol(name).id }
+          end
+        }
+      end
+
+      # @rbs (json_hash payload) -> json_hash
+      def build_cost(payload)
+        tables = payload.fetch("tables") #: json_hash
+        action_table = tables.fetch("actions") #: json_hash
+        goto_table = tables.fetch("gotos") #: json_hash
+        {
+          "canonical_payload_bytes" => Serializer.compact(payload).bytesize,
+          "action_cells" => occupied_cells(action_table, "codes"),
+          "goto_cells" => occupied_cells(goto_table, "values"),
+          "lookup_cost" => lookup_cost,
+          "recognition_cost" => recognition_cost,
+          "measurement" => "not-measured",
+          "bounded_by_max_steps" => true
+        }
+      end
+
+      # @rbs (json_hash table, String values_key) -> Integer
+      def occupied_cells(table, values_key)
+        if table.key?("rows")
+          rows = table.fetch("rows") #: Array[json_value]
+          return rows.sum { |row| row.is_a?(Array) ? row.length : 0 }
+        end
+
+        values = table.fetch(values_key) #: Array[json_value]
+        values.compact.length
+      end
+
+      # @rbs () -> String
+      def lookup_cost
+        return "O(1) row-displacement probe" if @representation == :compact
+
+        "O(log row width) binary search"
+      end
+
+      # @rbs () -> String
+      def recognition_cost
+        prefix = if @representation == :compact
+                   "O(input tokens + reductions)"
+                 else
+                   "O((input tokens + reductions) * log maximum row width)"
+                 end
+        "#{prefix}; lexer, recovery search, and semantic actions excluded"
+      end
+
+      # @rbs (Symbol | String? requested) -> Symbol
+      def effective_cst_trivia(requested)
+        contract_entry = @grammar.parser_contract&.cst_trivia
+        persisted = contract_entry.value if contract_entry&.explicit
+        if requested && persisted && requested.to_sym != persisted
+          raise ArgumentError, "cst_trivia conflicts with the persisted parser contract"
+        end
+
+        (requested || persisted || :leading).to_sym
+      end
+    end
+  end
+end

data/lib/ibex/table_artifact/cst_projection.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    # Converts code-generation CST metadata into closed JSON records.
+    module CSTProjection
+      # @rbs!
+      #   private def cst: () -> Hash[String, Builder::json_value]?
+      #   private def normalize_cst: (Codegen::CSTMetadata::metadata metadata) -> Hash[String, Builder::json_value]
+      #   private def normalize_cst_kinds: (Codegen::CSTMetadata::kinds kinds) -> Hash[String, Builder::json_value]
+      #   private def named_kinds: (Hash[String, Integer] values) -> Array[Hash[String, String | Integer]]
+      #   private def normalize_cst_slot: (Integer production_id,
+      #     Codegen::CSTMetadata::slot slot) -> Hash[String, Builder::json_value]
+
+      private
+
+      # @rbs skip
+      def cst
+        return unless @grammar.options[:cst]
+
+        normalize_cst(Codegen::CSTMetadata.new(@grammar, trivia_policy: @cst_trivia).build)
+      end
+
+      # @rbs skip
+      def normalize_cst(metadata)
+        kinds = metadata.fetch(:kinds)
+        {
+          "version" => metadata.fetch(:version),
+          "trivia_policy" => metadata.fetch(:trivia_policy).to_s,
+          "kinds" => normalize_cst_kinds(kinds),
+          "slots" => metadata.fetch(:slots).sort.map { |production_id, slot| normalize_cst_slot(production_id, slot) }
+        }
+      end
+
+      # @rbs skip
+      def normalize_cst_kinds(kinds)
+        {
+          "names" => kinds.fetch(:names),
+          "terminal_range" => kinds.fetch(:terminal_range),
+          "nonterminal_range" => kinds.fetch(:nonterminal_range),
+          "named" => named_kinds(kinds.fetch(:named)),
+          "named_nonterminals" => kinds.fetch(:named_nonterminals).sort.map do |kind_id, symbol_id|
+            { "kind_id" => kind_id, "symbol_id" => symbol_id }
+          end,
+          "trivia" => named_kinds(kinds.fetch(:trivia)),
+          "synthetic" => named_kinds(kinds.fetch(:synthetic))
+        }
+      end
+
+      # @rbs skip
+      def named_kinds(values)
+        values.sort_by { |_name, id| id }.map { |name, id| { "name" => name, "id" => id } }
+      end
+
+      # @rbs skip
+      def normalize_cst_slot(production_id, slot)
+        fields = slot.fetch(:fields).map do |name, value|
+          value = { index: value, extraction: nil } if value.is_a?(Integer)
+          { "name" => name, "index" => value.fetch(:index), "extraction" => value[:extraction]&.to_s }
+        end
+        {
+          "production_id" => production_id,
+          "node_kind_id" => slot.fetch(:node_kind),
+          "node_name" => slot.fetch(:node_name),
+          "fields" => fields
+        }
+      end
+    end
+  end
+end

data/lib/ibex/table_artifact/document.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    # An immutable, validated table artifact document.
+    class Document
+      # @rbs!
+      #   type json_value = String | Integer | Float | bool | nil | Array[json_value] | Hash[String, json_value]
+
+      attr_reader :data #: Hash[String, json_value]
+
+      # @rbs (Hash[String, json_value] data) -> void
+      def initialize(data)
+        Validator.validate!(data)
+        @data = Serializer.deep_freeze(data)
+      end
+
+      # @rbs () -> String
+      def dump
+        Serializer.dump(@data)
+      end
+
+      # @rbs () -> Hash[String, json_value]
+      def payload
+        @data.fetch("payload") #: Hash[String, json_value]
+      end
+
+      # @rbs () -> Hash[String, String]
+      def identity
+        @data.fetch("identity") #: Hash[String, String]
+      end
+
+      # @rbs () -> Hash[String, json_value]
+      def cost
+        @data.fetch("cost") #: Hash[String, json_value]
+      end
+    end
+  end
+end

data/lib/ibex/table_artifact/executor.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    # Recognition-only driver over internal token ids; it never loads wrapper code.
+    class Executor
+      class Result
+        attr_reader :status #: Symbol
+        attr_reader :steps #: Integer
+        attr_reader :consumed_tokens #: Integer
+        attr_reader :reason #: String?
+
+        # @rbs (status: Symbol, steps: Integer, consumed_tokens: Integer, reason: String?) -> void
+        def initialize(status:, steps:, consumed_tokens:, reason:)
+          @status = status
+          @steps = steps
+          @consumed_tokens = consumed_tokens
+          @reason = reason
+          freeze
+        end
+
+        # @rbs () -> bool
+        def accepted? = status == :accepted
+        # @rbs () -> bool
+        def rejected? = status == :rejected
+        # @rbs () -> bool
+        def exhausted? = status == :exhausted
+      end
+
+      # @rbs (Document | Hash[String, ValidationSupport::json_value] document) -> void
+      def initialize(document)
+        @payload = document.is_a?(Document) ? document.payload : Document.new(document).payload
+        @tables = @payload.fetch("tables") #: Hash[String, ValidationSupport::json_value]
+        @productions = @payload.fetch("productions") #: Array[Hash[String, ValidationSupport::json_value]]
+        tokens = @payload.fetch("tokens") #: Array[Hash[String, ValidationSupport::json_value]]
+        @terminal_ids = tokens.map do |token|
+          token.fetch("id") #: Integer
+        end
+      end
+
+      # @rbs (Array[Integer] token_ids, ?entry: String | Symbol?, ?max_steps: Integer) -> Result
+      def recognize(token_ids, entry: nil, max_steps: 1_000_000)
+        raise ArgumentError, "max_steps must be positive" unless max_steps.is_a?(Integer) && max_steps.positive?
+
+        input = validate_input(token_ids)
+        input << 0 unless input.last&.zero?
+        stack = [entry_state(entry)]
+        execute(input, stack, max_steps)
+      end
+
+      private
+
+      # @rbs (Array[Integer] input, Array[Integer] stack, Integer max_steps) -> Result
+      def execute(input, stack, max_steps)
+        cursor = 0
+        max_steps.times do |step|
+          return result(:rejected, step + 1, cursor, "parser shifted past end of input") if cursor >= input.length
+
+          code = action(stack.last, input.fetch(cursor))
+          immediate = immediate_result(code, input.fetch(cursor), step, cursor)
+          return immediate if immediate
+
+          code ||= -1
+          if code.positive?
+            stack << (code - 1)
+            cursor += 1
+            next
+          end
+
+          reason = reduce_stack(stack, code)
+          return result(:rejected, step + 1, cursor, reason) if reason
+        end
+        result(:exhausted, max_steps, cursor, "max_steps exceeded")
+      end
+
+      # @rbs (Integer? code, Integer token_id, Integer step, Integer cursor) -> Result?
+      def immediate_result(code, token_id, step, cursor)
+        return result(:rejected, step + 1, cursor, "no parser action") if code.nil? || code == -1
+        return unless code.zero?
+        return result(:rejected, step + 1, cursor, "accept action before end of input") unless token_id.zero?
+
+        result(:accepted, step + 1, cursor, nil)
+      end
+
+      # @rbs (Array[Integer] token_ids) -> Array[Integer]
+      def validate_input(token_ids)
+        raise ArgumentError, "token_ids must be an array" unless token_ids.is_a?(Array)
+
+        token_ids.map.with_index do |token_id, index|
+          raise ArgumentError, "token_ids[#{index}] must be an internal terminal id" unless
+            token_id.is_a?(Integer) && @terminal_ids.include?(token_id)
+          raise ArgumentError, "$eof may only appear at the end" if token_id.zero? && index != token_ids.length - 1
+
+          token_id
+        end
+      end
+
+      # @rbs (String | Symbol? requested) -> Integer
+      def entry_state(requested)
+        entries = @payload.fetch("entry_states") #: Array[Hash[String, ValidationSupport::json_value]]
+        default_state = entries.first.fetch("state") #: Integer
+        return default_state unless requested
+
+        entry = entries.find { |candidate| candidate.fetch("name") == requested.to_s }
+        raise ArgumentError, "unknown parser entry #{requested.inspect}" unless entry
+
+        entry.fetch("state") #: Integer
+      end
+
+      # @rbs (Integer row, Integer column) -> Integer?
+      def action(row, column)
+        table = @tables.fetch("actions") #: Hash[String, ValidationSupport::json_value]
+        code = if table.fetch("encoding") == "signed-sparse-rows-v1"
+                 rows = table.fetch("rows") #: Array[Array[Hash[String, ValidationSupport::json_value]]]
+                 cells = rows.fetch(row)
+                 cell = cells.bsearch do |candidate|
+                   token_id = candidate.fetch("token_id") #: Integer
+                   token_id >= column
+                 end
+                 if cell
+                   token_id = cell.fetch("token_id") #: Integer
+                   if token_id == column
+                     cell.fetch("code") #: Integer
+                   end
+                 end
+               else
+                 displacement_lookup(table, row, column, "codes")
+               end
+        defaults = @tables.fetch("default_actions") #: Array[Integer?]
+        code.nil? ? defaults.fetch(row) : code
+      end
+
+      # @rbs (Integer row, Integer column) -> Integer?
+      def goto_state(row, column)
+        table = @tables.fetch("gotos") #: Hash[String, ValidationSupport::json_value]
+        if table.fetch("encoding") == "sparse-rows-v1"
+          rows = table.fetch("rows") #: Array[Array[Hash[String, ValidationSupport::json_value]]]
+          cells = rows.fetch(row)
+          cell = cells.bsearch do |candidate|
+            symbol_id = candidate.fetch("symbol_id") #: Integer
+            symbol_id >= column
+          end
+          if cell
+            symbol_id = cell.fetch("symbol_id") #: Integer
+            if symbol_id == column
+              cell.fetch("state") #: Integer
+            end
+          end
+        else
+          displacement_lookup(table, row, column, "values")
+        end
+      end
+
+      # @rbs (Hash[String, ValidationSupport::json_value] table, Integer row, Integer column,
+      #   String value_key) -> Integer?
+      def displacement_lookup(table, row, column, value_key)
+        offsets = table.fetch("offsets") #: Array[Integer]
+        index = offsets.fetch(row) + column
+        checks = table.fetch("checks") #: Array[Integer?]
+        return if index.negative? || index >= checks.length || checks[index] != row
+
+        values = table.fetch(value_key) #: Array[Integer?]
+        values.fetch(index)
+      end
+
+      # @rbs (Array[Integer] stack, Integer code) -> String?
+      def reduce_stack(stack, code)
+        production = @productions.fetch(-2 - code)
+        length = production.fetch("rhs_length") #: Integer
+        return "reduction underflow" if length >= stack.length
+
+        stack.pop(length)
+        lhs = production.fetch("lhs") #: Integer
+        state = goto_state(stack.last, lhs)
+        return "missing goto after reduction" unless state
+
+        stack << state
+        nil
+      end
+
+      # @rbs (Symbol status, Integer steps, Integer consumed, String? reason) -> Result
+      def result(status, steps, consumed, reason)
+        Result.new(status: status, steps: steps, consumed_tokens: consumed, reason: reason)
+      end
+    end
+  end
+end

data/lib/ibex/table_artifact/serializer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    # Canonical JSON ordering and digest primitives shared by writer and loader.
+    module Serializer
+      # @rbs!
+      #   type json_value = String | Integer | Float | bool | nil | Array[json_value] | Hash[json_value, json_value]
+
+      class << self
+        # @rbs (json_value value) -> json_value
+        def canonical(value)
+          case value
+          when Hash
+            value.keys.sort.to_h { |key| [key, canonical(value.fetch(key))] }
+          when Array
+            value.map { |item| canonical(item) }
+          when String
+            normalized = value.dup.force_encoding(Encoding::UTF_8)
+            normalized.valid_encoding? ? normalized : normalized.scrub
+          else
+            value
+          end
+        end
+
+        # @rbs (json_value value) -> String
+        def compact(value)
+          JSON.generate(canonical(value))
+        end
+
+        # @rbs (json_value value) -> String
+        def dump(value)
+          "#{JSON.pretty_generate(canonical(value))}\n"
+        end
+
+        # @rbs (json_value value) -> String
+        def digest(value)
+          "sha256:#{Digest::SHA256.hexdigest(compact(value))}"
+        end
+
+        # @rbs (json_value value) -> json_value
+        def deep_freeze(value)
+          case value
+          when Hash
+            value.each do |key, item|
+              deep_freeze(key)
+              deep_freeze(item)
+            end
+          when Array
+            value.each { |item| deep_freeze(item) }
+          end
+          value.freeze
+        end
+      end
+    end
+  end
+end