ibex 0.1.0 → 0.3.0

data/lib/ibex/table_artifact/document.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    # An immutable, validated table artifact document.
+    class Document
+      # @rbs!
+      #   type json_value = String | Integer | Float | bool | nil | Array[json_value] | Hash[String, json_value]
+
+      attr_reader :data #: Hash[String, json_value]
+
+      # @rbs (Hash[String, json_value] data) -> void
+      def initialize(data)
+        Validator.validate!(data)
+        @data = Serializer.deep_freeze(data)
+      end
+
+      # @rbs () -> String
+      def dump
+        Serializer.dump(@data)
+      end
+
+      # @rbs () -> Hash[String, json_value]
+      def payload
+        @data.fetch("payload") #: Hash[String, json_value]
+      end
+
+      # @rbs () -> Hash[String, String]
+      def identity
+        @data.fetch("identity") #: Hash[String, String]
+      end
+
+      # @rbs () -> Hash[String, json_value]
+      def cost
+        @data.fetch("cost") #: Hash[String, json_value]
+      end
+    end
+  end
+end

data/lib/ibex/table_artifact/executor.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    # Recognition-only driver over internal token ids; it never loads wrapper code.
+    class Executor
+      class Result
+        attr_reader :status #: Symbol
+        attr_reader :steps #: Integer
+        attr_reader :consumed_tokens #: Integer
+        attr_reader :reason #: String?
+
+        # @rbs (status: Symbol, steps: Integer, consumed_tokens: Integer, reason: String?) -> void
+        def initialize(status:, steps:, consumed_tokens:, reason:)
+          @status = status
+          @steps = steps
+          @consumed_tokens = consumed_tokens
+          @reason = reason
+          freeze
+        end
+
+        # @rbs () -> bool
+        def accepted? = status == :accepted
+        # @rbs () -> bool
+        def rejected? = status == :rejected
+        # @rbs () -> bool
+        def exhausted? = status == :exhausted
+      end
+
+      # @rbs (Document | Hash[String, ValidationSupport::json_value] document) -> void
+      def initialize(document)
+        @payload = document.is_a?(Document) ? document.payload : Document.new(document).payload
+        @tables = @payload.fetch("tables") #: Hash[String, ValidationSupport::json_value]
+        @productions = @payload.fetch("productions") #: Array[Hash[String, ValidationSupport::json_value]]
+        tokens = @payload.fetch("tokens") #: Array[Hash[String, ValidationSupport::json_value]]
+        @terminal_ids = tokens.map do |token|
+          token.fetch("id") #: Integer
+        end
+      end
+
+      # @rbs (Array[Integer] token_ids, ?entry: String | Symbol?, ?max_steps: Integer) -> Result
+      def recognize(token_ids, entry: nil, max_steps: 1_000_000)
+        raise ArgumentError, "max_steps must be positive" unless max_steps.is_a?(Integer) && max_steps.positive?
+
+        input = validate_input(token_ids)
+        input << 0 unless input.last&.zero?
+        stack = [entry_state(entry)]
+        execute(input, stack, max_steps)
+      end
+
+      private
+
+      # @rbs (Array[Integer] input, Array[Integer] stack, Integer max_steps) -> Result
+      def execute(input, stack, max_steps)
+        cursor = 0
+        max_steps.times do |step|
+          return result(:rejected, step + 1, cursor, "parser shifted past end of input") if cursor >= input.length
+
+          code = action(stack.last, input.fetch(cursor))
+          immediate = immediate_result(code, input.fetch(cursor), step, cursor)
+          return immediate if immediate
+
+          code ||= -1
+          if code.positive?
+            stack << (code - 1)
+            cursor += 1
+            next
+          end
+
+          reason = reduce_stack(stack, code)
+          return result(:rejected, step + 1, cursor, reason) if reason
+        end
+        result(:exhausted, max_steps, cursor, "max_steps exceeded")
+      end
+
+      # @rbs (Integer? code, Integer token_id, Integer step, Integer cursor) -> Result?
+      def immediate_result(code, token_id, step, cursor)
+        return result(:rejected, step + 1, cursor, "no parser action") if code.nil? || code == -1
+        return unless code.zero?
+        return result(:rejected, step + 1, cursor, "accept action before end of input") unless token_id.zero?
+
+        result(:accepted, step + 1, cursor, nil)
+      end
+
+      # @rbs (Array[Integer] token_ids) -> Array[Integer]
+      def validate_input(token_ids)
+        raise ArgumentError, "token_ids must be an array" unless token_ids.is_a?(Array)
+
+        token_ids.map.with_index do |token_id, index|
+          raise ArgumentError, "token_ids[#{index}] must be an internal terminal id" unless
+            token_id.is_a?(Integer) && @terminal_ids.include?(token_id)
+          raise ArgumentError, "$eof may only appear at the end" if token_id.zero? && index != token_ids.length - 1
+
+          token_id
+        end
+      end
+
+      # @rbs (String | Symbol? requested) -> Integer
+      def entry_state(requested)
+        entries = @payload.fetch("entry_states") #: Array[Hash[String, ValidationSupport::json_value]]
+        default_state = entries.first.fetch("state") #: Integer
+        return default_state unless requested
+
+        entry = entries.find { |candidate| candidate.fetch("name") == requested.to_s }
+        raise ArgumentError, "unknown parser entry #{requested.inspect}" unless entry
+
+        entry.fetch("state") #: Integer
+      end
+
+      # @rbs (Integer row, Integer column) -> Integer?
+      def action(row, column)
+        table = @tables.fetch("actions") #: Hash[String, ValidationSupport::json_value]
+        code = if table.fetch("encoding") == "signed-sparse-rows-v1"
+                 rows = table.fetch("rows") #: Array[Array[Hash[String, ValidationSupport::json_value]]]
+                 cells = rows.fetch(row)
+                 cell = cells.bsearch do |candidate|
+                   token_id = candidate.fetch("token_id") #: Integer
+                   token_id >= column
+                 end
+                 if cell
+                   token_id = cell.fetch("token_id") #: Integer
+                   if token_id == column
+                     cell.fetch("code") #: Integer
+                   end
+                 end
+               else
+                 displacement_lookup(table, row, column, "codes")
+               end
+        defaults = @tables.fetch("default_actions") #: Array[Integer?]
+        code.nil? ? defaults.fetch(row) : code
+      end
+
+      # @rbs (Integer row, Integer column) -> Integer?
+      def goto_state(row, column)
+        table = @tables.fetch("gotos") #: Hash[String, ValidationSupport::json_value]
+        if table.fetch("encoding") == "sparse-rows-v1"
+          rows = table.fetch("rows") #: Array[Array[Hash[String, ValidationSupport::json_value]]]
+          cells = rows.fetch(row)
+          cell = cells.bsearch do |candidate|
+            symbol_id = candidate.fetch("symbol_id") #: Integer
+            symbol_id >= column
+          end
+          if cell
+            symbol_id = cell.fetch("symbol_id") #: Integer
+            if symbol_id == column
+              cell.fetch("state") #: Integer
+            end
+          end
+        else
+          displacement_lookup(table, row, column, "values")
+        end
+      end
+
+      # @rbs (Hash[String, ValidationSupport::json_value] table, Integer row, Integer column,
+      #   String value_key) -> Integer?
+      def displacement_lookup(table, row, column, value_key)
+        offsets = table.fetch("offsets") #: Array[Integer]
+        index = offsets.fetch(row) + column
+        checks = table.fetch("checks") #: Array[Integer?]
+        return if index.negative? || index >= checks.length || checks[index] != row
+
+        values = table.fetch(value_key) #: Array[Integer?]
+        values.fetch(index)
+      end
+
+      # @rbs (Array[Integer] stack, Integer code) -> String?
+      def reduce_stack(stack, code)
+        production = @productions.fetch(-2 - code)
+        length = production.fetch("rhs_length") #: Integer
+        return "reduction underflow" if length >= stack.length
+
+        stack.pop(length)
+        lhs = production.fetch("lhs") #: Integer
+        state = goto_state(stack.last, lhs)
+        return "missing goto after reduction" unless state
+
+        stack << state
+        nil
+      end
+
+      # @rbs (Symbol status, Integer steps, Integer consumed, String? reason) -> Result
+      def result(status, steps, consumed, reason)
+        Result.new(status: status, steps: steps, consumed_tokens: consumed, reason: reason)
+      end
+    end
+  end
+end

data/lib/ibex/table_artifact/serializer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    # Canonical JSON ordering and digest primitives shared by writer and loader.
+    module Serializer
+      # @rbs!
+      #   type json_value = String | Integer | Float | bool | nil | Array[json_value] | Hash[json_value, json_value]
+
+      class << self
+        # @rbs (json_value value) -> json_value
+        def canonical(value)
+          case value
+          when Hash
+            value.keys.sort.to_h { |key| [key, canonical(value.fetch(key))] }
+          when Array
+            value.map { |item| canonical(item) }
+          when String
+            normalized = value.dup.force_encoding(Encoding::UTF_8)
+            normalized.valid_encoding? ? normalized : normalized.scrub
+          else
+            value
+          end
+        end
+
+        # @rbs (json_value value) -> String
+        def compact(value)
+          JSON.generate(canonical(value))
+        end
+
+        # @rbs (json_value value) -> String
+        def dump(value)
+          "#{JSON.pretty_generate(canonical(value))}\n"
+        end
+
+        # @rbs (json_value value) -> String
+        def digest(value)
+          "sha256:#{Digest::SHA256.hexdigest(compact(value))}"
+        end
+
+        # @rbs (json_value value) -> json_value
+        def deep_freeze(value)
+          case value
+          when Hash
+            value.each do |key, item|
+              deep_freeze(key)
+              deep_freeze(item)
+            end
+          when Array
+            value.each { |item| deep_freeze(item) }
+          end
+          value.freeze
+        end
+      end
+    end
+  end
+end

data/lib/ibex/table_artifact/validator/metadata.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    class MetadataValidator
+      include ValidationSupport
+
+      # @rbs (Hash[String, ValidationSupport::json_value] payload, symbols: Array[Hash[String, ValidationSupport::json_value]],
+      #   productions: Array[Hash[String, ValidationSupport::json_value]]) -> void
+      def initialize(payload, symbols:, productions:)
+        @payload = payload
+        @symbols = symbols
+        @productions = productions
+        @terminal_ids = ids_for_kind(symbols, "terminal")
+        @nonterminal_ids = ids_for_kind(symbols, "nonterminal")
+      end
+
+      # @rbs () -> void
+      def validate!
+        validate_semantic_actions(@payload.fetch("semantic_actions"))
+        validate_recovery(@payload.fetch("recovery"))
+        validate_cst(@payload.fetch("cst"))
+      end
+
+      private
+
+      # @rbs (Array[Hash[String, ValidationSupport::json_value]] symbols, String kind) -> Set[Integer]
+      def ids_for_kind(symbols, kind)
+        symbols.filter_map do |symbol|
+          next unless symbol.fetch("kind") == kind
+
+          symbol.fetch("id") #: Integer
+        end.to_set
+      end
+
+      # @rbs (ValidationSupport::json_value data) -> void
+      def validate_semantic_actions(data)
+        path = "$.payload.semantic_actions"
+        data = record(data, path, %w[binding verified slots])
+        enum(data.fetch("binding"), "#{path}.binding", ["opaque-wrapper-production-id-v1"])
+        invalid("#{path}.verified", "must be false") unless boolean(data.fetch("verified"), "#{path}.verified") == false
+        slots = integer_ids(data.fetch("slots"), "#{path}.slots")
+        expected = @productions.filter_map { |production| production.fetch("action_slot") }
+        invalid("#{path}.slots", "must match production action slots") unless slots == expected
+      end
+
+      # @rbs (ValidationSupport::json_value data) -> void
+      def validate_recovery(data)
+        path = "$.payload.recovery"
+        data = record(data, path, %w[sync_token_ids on_error_reduce_symbol_ids])
+        sync_ids = integer_ids(data.fetch("sync_token_ids"), "#{path}.sync_token_ids", sorted: false)
+        sync_ids.each do |id|
+          invalid("#{path}.sync_token_ids", "must reference terminals") unless @terminal_ids.include?(id)
+        end
+        groups = array(data.fetch("on_error_reduce_symbol_ids"), "#{path}.on_error_reduce_symbol_ids")
+        groups.each_with_index do |group, index|
+          ids = integer_ids(group, "#{path}.on_error_reduce_symbol_ids[#{index}]", sorted: false)
+          invalid("#{path}.on_error_reduce_symbol_ids[#{index}]", "must not be empty") if ids.empty?
+          ids.each do |id|
+            invalid("#{path}.on_error_reduce_symbol_ids[#{index}]", "must reference nonterminals") unless
+              @nonterminal_ids.include?(id)
+          end
+        end
+      end
+
+      # @rbs (ValidationSupport::json_value data) -> void
+      def validate_cst(data)
+        return if data.nil?
+
+        path = "$.payload.cst"
+        data = record(data, path, %w[version trivia_policy kinds slots])
+        invalid("#{path}.version", "must be 1") unless integer(data.fetch("version"), "#{path}.version") == 1
+        enum(data.fetch("trivia_policy"), "#{path}.trivia_policy", %w[leading balanced drop])
+        kind_count = validate_cst_kinds(data.fetch("kinds"), "#{path}.kinds")
+        validate_cst_slots(data.fetch("slots"), "#{path}.slots", kind_count)
+      end
+
+      # @rbs (ValidationSupport::json_value data, String path) -> Integer
+      def validate_cst_kinds(data, path)
+        keys = %w[names terminal_range nonterminal_range named named_nonterminals trivia synthetic]
+        data = record(data, path, keys)
+        names = array(data.fetch("names"), "#{path}.names") #: Array[String]
+        names.each_with_index { |name, index| string(name, "#{path}.names[#{index}]") }
+        validate_range(data.fetch("terminal_range"), "#{path}.terminal_range", names.length)
+        validate_range(data.fetch("nonterminal_range"), "#{path}.nonterminal_range", names.length)
+        %w[named trivia synthetic].each { |key| validate_named_kinds(data.fetch(key), "#{path}.#{key}", names) }
+        validate_named_nonterminals(data.fetch("named_nonterminals"), "#{path}.named_nonterminals", names.length)
+        names.length
+      end
+
+      # @rbs (ValidationSupport::json_value data, String path, Integer limit) -> void
+      def validate_range(data, path, limit)
+        values = array(data, path)
+        invalid(path, "must contain two bounds") unless values.length == 2
+        start = integer(values.fetch(0), "#{path}[0]", minimum: 0)
+        finish = integer(values.fetch(1), "#{path}[1]", minimum: 0)
+        invalid(path, "must be an ordered half-open range") unless finish.between?(start, limit)
+      end
+
+      # @rbs (ValidationSupport::json_value data, String path, Array[String] names) -> void
+      def validate_named_kinds(data, path, names)
+        entries = array(data, path)
+        ids = entries.map.with_index do |entry, index|
+          entry_path = "#{path}[#{index}]"
+          entry = record(entry, entry_path, %w[name id])
+          name = string(entry.fetch("name"), "#{entry_path}.name")
+          id = integer(entry.fetch("id"), "#{entry_path}.id", minimum: 0)
+          invalid(entry_path, "does not match the kind name table") unless names[id] == name
+          id
+        end
+        sorted_unique!(ids, path)
+      end
+
+      # @rbs (ValidationSupport::json_value data, String path, Integer kind_count) -> void
+      def validate_named_nonterminals(data, path, kind_count)
+        pairs = array(data, path).map.with_index do |entry, index|
+          entry_path = "#{path}[#{index}]"
+          entry = record(entry, entry_path, %w[kind_id symbol_id])
+          kind_id = integer(entry.fetch("kind_id"), "#{entry_path}.kind_id", minimum: 0)
+          invalid("#{entry_path}.kind_id", "references a missing kind") unless kind_id < kind_count
+          symbol_id = integer(entry.fetch("symbol_id"), "#{entry_path}.symbol_id", minimum: 0)
+          invalid("#{entry_path}.symbol_id", "must reference a nonterminal") unless @nonterminal_ids.include?(symbol_id)
+          [kind_id, symbol_id]
+        end
+        return if pairs == pairs.sort && pairs.map(&:first).uniq.length == pairs.length
+
+        invalid(path,
+                "must be sorted with unique kind ids")
+      end
+
+      # @rbs (ValidationSupport::json_value data, String path, Integer kind_count) -> void
+      def validate_cst_slots(data, path, kind_count)
+        production_ids = array(data, path).map.with_index do |slot, index|
+          slot_path = "#{path}[#{index}]"
+          slot = record(slot, slot_path, %w[production_id node_kind_id node_name fields])
+          production_id = integer(slot.fetch("production_id"), "#{slot_path}.production_id", minimum: 0)
+          unless production_id < @productions.length
+            invalid("#{slot_path}.production_id",
+                    "references a missing production")
+          end
+          node_kind = integer(slot.fetch("node_kind_id"), "#{slot_path}.node_kind_id", minimum: 0)
+          invalid("#{slot_path}.node_kind_id", "references a missing kind") unless node_kind < kind_count
+          string(slot.fetch("node_name"), "#{slot_path}.node_name")
+          validate_fields(slot.fetch("fields"), "#{slot_path}.fields", @productions.fetch(production_id))
+          production_id
+        end
+        sorted_unique!(production_ids, path)
+      end
+
+      # @rbs (ValidationSupport::json_value data, String path,
+      #   Hash[String, ValidationSupport::json_value] production) -> void
+      def validate_fields(data, path, production)
+        names = array(data, path).map.with_index do |field, index|
+          field_path = "#{path}[#{index}]"
+          field = record(field, field_path, %w[name index extraction])
+          name = string(field.fetch("name"), "#{field_path}.name")
+          field_index = integer(field.fetch("index"), "#{field_path}.index", minimum: 0)
+          rhs_length = production.fetch("rhs_length") #: Integer
+          invalid("#{field_path}.index", "exceeds production rhs") unless field_index < rhs_length
+          extraction = field.fetch("extraction")
+          enum(extraction, "#{field_path}.extraction", %w[repetition separated_list]) unless extraction.nil?
+          name
+        end
+        invalid(path, "must have unique field names") unless names.uniq.length == names.length
+      end
+
+      # @rbs (ValidationSupport::json_value data, String path, ?sorted: bool) -> Array[Integer]
+      def integer_ids(data, path, sorted: true)
+        ids = array(data, path).map.with_index { |id, index| integer(id, "#{path}[#{index}]", minimum: 0) }
+        if sorted
+          sorted_unique!(ids, path)
+        else
+          invalid(path, "must contain unique ids") unless ids.uniq.length == ids.length
+        end
+        ids
+      end
+    end
+  end
+end

data/lib/ibex/table_artifact/validator/support.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Ibex
+  module TableArtifact
+    module ValidationSupport
+      # @rbs!
+      #   type json_value = String | Integer | Float | bool | nil | Array[json_value] | Hash[String, json_value]
+
+      DIGEST = /\Asha256:[0-9a-f]{64}\z/ #: Regexp
+
+      private
+
+      # @rbs (json_value value, String path, Array[String] keys) -> Hash[String, json_value]
+      def record(value, path, keys)
+        invalid(path, "must be an object") unless value.is_a?(Hash)
+        invalid(path, "must use string keys") unless value.keys.all?(String)
+
+        missing = keys - value.keys
+        extra = value.keys - keys
+        invalid(path, "is missing #{missing.join(', ')}") unless missing.empty?
+        invalid(path, "contains unknown fields #{extra.join(', ')}") unless extra.empty?
+        value #: Hash[String, json_value]
+      end
+
+      # @rbs (json_value value, String path) -> Array[json_value]
+      def array(value, path)
+        invalid(path, "must be an array") unless value.is_a?(Array)
+        value #: Array[json_value]
+      end
+
+      # @rbs (json_value value, String path, ?allow_empty: bool) -> String
+      def string(value, path, allow_empty: false)
+        invalid(path, "must be a string") unless value.is_a?(String)
+        invalid(path, "must not be empty") if !allow_empty && value.empty?
+        value #: String
+      end
+
+      # @rbs (json_value value, String path, ?minimum: Integer?) -> Integer
+      def integer(value, path, minimum: nil)
+        invalid(path, "must be an integer") unless value.is_a?(Integer)
+        invalid(path, "must be at least #{minimum}") if minimum && value < minimum
+        value #: Integer
+      end
+
+      # @rbs (json_value value, String path) -> bool
+      def boolean(value, path)
+        invalid(path, "must be true or false") unless [true, false].include?(value)
+        value #: bool
+      end
+
+      # @rbs [T] (T value, String path, Array[T] values) -> T
+      def enum(value, path, values)
+        invalid(path, "must be one of #{values.join(', ')}") unless values.include?(value)
+        value
+      end
+
+      # @rbs (json_value value, String path) -> String
+      def digest(value, path)
+        invalid(path, "must be a sha256 digest") unless value.is_a?(String) && DIGEST.match?(value)
+        value
+      end
+
+      # @rbs (json_value value, String path) -> String?
+      def nullable_string(value, path)
+        string(value, path, allow_empty: true) unless value.nil?
+        value #: String?
+      end
+
+      # @rbs (json_value value, String path, ?minimum: Integer?) -> Integer?
+      def nullable_integer(value, path, minimum: nil)
+        integer(value, path, minimum: minimum) unless value.nil?
+        value #: Integer?
+      end
+
+      # @rbs (Array[json_value] values, String path) -> void
+      def sorted_unique!(values, path)
+        invalid(path, "must be sorted and unique") unless values == values.sort.uniq
+      end
+
+      # @rbs (String path, String message) -> bot
+      def invalid(path, message)
+        raise ValidationError, "#{path} #{message}"
+      end
+    end
+  end
+end