ibex 0.1.0 → 0.3.0

data/lib/ibex/ir/validator/lexer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Ibex
+  module IR
+    module Validator
+      # Structural validation for the independently versioned Lexer IR.
+      class LexerDocument < Base
+        ROOT_REQUIRED = %w[
+          ibex_ir schema_version initial_state states rules warnings source_provenance
+        ].freeze #: Array[String]
+        RULE_REQUIRED = %w[
+          id state kind token pattern pattern_kind options action loc
+        ].freeze #: Array[String]
+
+        # @rbs (json_object data, ?path: String) -> void
+        def initialize(data, path: "$")
+          super()
+          @data = data
+          @path = path
+        end
+
+        # @rbs () -> self
+        def validate
+          record(@data, @path, ROOT_REQUIRED)
+          literal(@data["ibex_ir"], "#{@path}.ibex_ir", "lexer")
+          literal(@data["schema_version"], "#{@path}.schema_version", LEXER_SCHEMA_VERSION)
+          literal(@data["initial_state"], "#{@path}.initial_state", "INITIAL")
+          validate_states
+          validate_rules
+          validate_warnings
+          validate_source_provenance
+          self
+        end
+
+        private
+
+        # @rbs () -> void
+        def validate_states
+          seen = {} #: Hash[String, bool]
+          values = array(@data["states"], "#{@path}.states")
+          invalid("#{@path}.states", "must start with INITIAL") unless values.first == "INITIAL"
+          values.each_with_index do |value, index|
+            name = nonempty_string(value, "#{@path}.states[#{index}]")
+            invalid("#{@path}.states[#{index}]", "duplicates state #{name.inspect}") if seen[name]
+            seen[name] = true
+          end
+          @states = seen
+        end
+
+        # @rbs () -> void
+        def validate_rules # rubocop:disable Metrics/AbcSize
+          array(@data["rules"], "#{@path}.rules").each_with_index do |value, index|
+            path = "#{@path}.rules[#{index}]"
+            rule = record(value, path, RULE_REQUIRED)
+            literal(rule["id"], "#{path}.id", index)
+            state = nonempty_string(rule["state"], "#{path}.state")
+            invalid("#{path}.state", "references missing state #{state.inspect}") unless @states[state]
+            kind = enum(rule["kind"], "#{path}.kind", %w[token skip on])
+            nullable_string(rule["token"], "#{path}.token")
+            invalid("#{path}.token", "is required for token rules") if kind == "token" && rule["token"].nil?
+            invalid("#{path}.token", "must be null for #{kind} rules") if kind != "token" && rule["token"]
+            nonempty_string(rule["pattern"], "#{path}.pattern")
+            enum(rule["pattern_kind"], "#{path}.pattern_kind", %w[regexp literal])
+            options = string(rule["options"], "#{path}.options")
+            invalid("#{path}.options", "may contain only i, m, and x") unless options.match?(/\A[imx]*\z/)
+            validate_pattern(rule, path, options)
+            nullable_string(rule["action"], "#{path}.action")
+            invalid("#{path}.action", "is required for on rules") if kind == "on" && rule["action"].nil?
+            location(rule["loc"], "#{path}.loc", nullable: false)
+          end
+        end
+
+        # @rbs (json_object rule, String path, String options) -> void
+        def validate_pattern(rule, path, options)
+          flags = 0
+          flags |= Regexp::IGNORECASE if options.include?("i")
+          flags |= Regexp::MULTILINE if options.include?("m")
+          flags |= Regexp::EXTENDED if options.include?("x")
+          regexp = Regexp.new("\\A(?:#{rule.fetch('pattern')})", flags)
+          invalid("#{path}.pattern", "must not match an empty string") if regexp.match?("")
+        rescue RegexpError => e
+          invalid("#{path}.pattern", "is not a valid regular expression: #{e.message}")
+        end
+
+        # @rbs () -> void
+        def validate_warnings
+          array(@data["warnings"], "#{@path}.warnings").each_with_index do |value, index|
+            path = "#{@path}.warnings[#{index}]"
+            warning = record(value, path, %w[type rule loc])
+            literal(warning["type"], "#{path}.type", "redos")
+            nonnegative_integer(warning["rule"], "#{path}.rule")
+            location(warning["loc"], "#{path}.loc", nullable: false)
+          end
+        end
+
+        # @rbs () -> void
+        def validate_source_provenance
+          value = @data["source_provenance"]
+          return if value.nil?
+
+          record(value, "#{@path}.source_provenance", %w[file root byte_span])
+          nullable_string(value["file"], "#{@path}.source_provenance.file")
+          nullable_string(value["root"], "#{@path}.source_provenance.root")
+          return if value["byte_span"].nil?
+
+          span = record(value["byte_span"], "#{@path}.source_provenance.byte_span", %w[start end])
+          nonnegative_integer(span["start"], "#{@path}.source_provenance.byte_span.start")
+          nonnegative_integer(span["end"], "#{@path}.source_provenance.byte_span.end")
+        end
+      end
+    end
+  end
+end

data/lib/ibex/ir/validator.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "json"
+require "digest"
+require_relative "validator/base"
+require_relative "validator/grammar"
+require_relative "validator/automaton"
+require_relative "validator/lexer"
+
+module Ibex
+  module IR
+    # Validates a serialized public IR document before constructing immutable IR objects.
+    module Validator
+      POSITION = "(ir):1:1"
+
+      # @rbs (String source) -> (Grammar | Automaton | Lexer)
+      def validate(source)
+        data = JSON.parse(source)
+        raise Ibex::Error, "#{POSITION}: $ must be an object" unless data.is_a?(Hash)
+
+        type = data.fetch("ibex_ir") { raise Ibex::Error, "#{POSITION}: missing ibex_ir discriminator" }
+        version = data["schema_version"]
+        supported = type == "lexer" ? SUPPORTED_LEXER_SCHEMA_VERSIONS : SUPPORTED_SCHEMA_VERSIONS
+        unless supported.include?(version)
+          expected = supported.join(", ")
+          raise Ibex::Error,
+                "#{POSITION}: unsupported schema_version #{version.inspect}; expected the current format (#{expected})"
+        end
+
+        case type
+        when "grammar" then GrammarDocument.new(data, version: version).validate
+        when "automaton" then AutomatonDocument.new(data, version: version).validate
+        when "lexer" then LexerDocument.new(data).validate
+        else raise Ibex::Error, "#{POSITION}: unsupported IR type #{type.inspect}"
+        end
+        value = Serialize.load(source)
+        validate_automaton_digest(value) if value.is_a?(Automaton)
+        value
+      rescue JSON::ParserError => e
+        raise Ibex::Error, "#{POSITION}: invalid JSON: #{e.message}"
+      rescue Ibex::Error
+        raise
+      rescue KeyError, NoMethodError, TypeError, ArgumentError => e
+        raise Ibex::Error, "#{POSITION}: invalid IR structure: #{e.message}"
+      end
+      module_function :validate
+
+      # @rbs (Automaton automaton) -> void
+      def validate_automaton_digest(automaton)
+        expected = "sha256:#{Digest::SHA256.hexdigest(Serialize.dump(automaton.grammar))}"
+        return if automaton.grammar_digest == expected
+
+        raise Ibex::Error,
+              "#{POSITION}: $.grammar_digest does not match the embedded grammar; expected #{expected.inspect}"
+      end
+      module_function :validate_automaton_digest
+
+      class << self
+        private :validate_automaton_digest
+      end
+    end
+  end
+end

data/lib/ibex/ir.rb

@@ -1,18 +1,67 @@
 # frozen_string_literal: true
 
 require_relative "ir/grammar_ir"
+require_relative "ir/lexer_ir"
 require_relative "ir/automaton_ir"
 require_relative "ir/serialize"
+require_relative "ir/validator"
 
 module Ibex
   module IR
     # Shared static shapes used across analysis, automaton construction, and code generation.
     # @rbs!
+    #   type serialized_value = untyped
     #   type location = { file: String, line: Integer, column: Integer }
     #   type precedence = { associativity: Symbol, level: Integer }
     #   type named_ref = { name: String, index: Integer }
+    #   type byte_span = { start: Integer, end: Integer }
+    #   type source_provenance = { file: String?, root: String?, byte_span: byte_span? }
+    #   type parser_parameter = { name: String, semantic_type: String? }
+    #   type value_printer = { symbol: String, code: String, loc: location }
+    #   type recovery_policy = {
+    #     sync_tokens: Array[String],
+    #     on_error_reduce: Array[Array[String]]
+    #   }
+    #   type grammar_test = { expectation: :accept | :reject, source: String, loc: location }
+    #   type node_annotation = { name: String, fields: Array[String], loc: location }
+    #   type lexer_warning = { type: :redos, rule: Integer, loc: location }
+    #   type lexer_rule_document_value = Integer | String | Symbol | nil | location
+    #   type lexer_document_value = String | Integer | Array[String] |
+    #     Array[Hash[Symbol, lexer_rule_document_value]] | Array[lexer_warning] | source_provenance?
+    #   type parameter_expansion = { rule: String, arguments: Array[String] }
+    #   type inline_expansion = { rule: String }
+    #   type production_expansion = {
+    #     parameter: parameter_expansion?,
+    #     inline: inline_expansion?,
+    #     include_chain: Array[source_provenance]
+    #   }
+    #   type action_fragment = { kind: :rule | :inline, source: source_provenance? }
+    #   type action_composition_step = {
+    #     kind: :rule | :inline,
+    #     rule: String?,
+    #     code: String?,
+    #     loc: location,
+    #     named_refs: Array[named_ref],
+    #     context_length: Integer,
+    #     inputs: Array[Integer],
+    #     stack_inputs: Array[Integer],
+    #     lookahead: Integer?,
+    #     result_var: bool,
+    #     result_type: String?
+    #   }
+    #   type action_composition_plan = {
+    #     version: Integer,
+    #     physical: Integer,
+    #     steps: Array[action_composition_step]
+    #   }
+    #   type action_composition = {
+    #     strategy: String,
+    #     fragments: Array[action_fragment],
+    #     ?plan: action_composition_plan
+    #   }
     #   type user_code_chunks = Hash[String, Array[UserCodeChunk]]
-    #   type grammar_options = { result_var: bool, omit_action_call: bool }
+    #   type grammar_options = { result_var: bool, omit_action_call: bool, ?cst: bool }
+    #   type grammar_mode = :default | :extended
     #   type grammar_warning = {
     #     type: Symbol,
     #     ?symbol: String,
@@ -28,18 +77,35 @@ module Ibex
     #   type runtime_action = [:shift, Integer] | [:reduce, Integer] | [:accept] | [:error]
     #   type conflict_resolution = { by: Symbol, chose: Symbol | Integer, ?associativity: Symbol }
     #   type shift_reduce_conflict =
-    #     { type: :shift_reduce, symbol: String, shift_to: Integer, reduce: Integer, resolution: conflict_resolution }
+    #     { type: :shift_reduce, symbol: String, shift_to: Integer, reduce: Integer,
+    #       resolution: conflict_resolution, ?midrule_origins: Array[location],
+    #       ?entries: Array[String], ?composite: bool }
     #   type reduce_reduce_conflict =
-    #     { type: :reduce_reduce, symbol: String, reductions: Array[Integer], resolution: conflict_resolution }
+    #     { type: :reduce_reduce, symbol: String, reductions: Array[Integer],
+    #       resolution: conflict_resolution, ?midrule_origins: Array[location],
+    #       ?entries: Array[String], ?composite: bool }
     #   type conflict = shift_reduce_conflict | reduce_reduce_conflict
     #   type conflict_summary = {
     #     sr: Integer,
     #     resolved_sr: Integer,
     #     rr: Integer,
     #     expected_sr: Integer,
-    #     expectation_met: bool
+    #     expectation_met: bool,
+    #     ?expected_rr: Integer,
+    #     ?rr_expectation_met: bool
+    #   }
+    #   type derivation_node = {
+    #     symbol: String,
+    #     ?token: String,
+    #     ?production: Integer,
+    #     ?children: Array[derivation_node]
+    #   }
+    #   type interpretation = {
+    #     kind: Symbol,
+    #     tree: derivation_node?,
+    #     ?state: Integer,
+    #     ?production: Integer
     #   }
-    #   type interpretation = Hash[Symbol, untyped]
     #   type counterexample = {
     #     state: Integer,
     #     type: Symbol,

data/lib/ibex/lalr/build_metrics.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module LALR
+    # Optional measurements are kept explicit for schema stability.
+    # rubocop:disable Metrics/ParameterLists
+    # Immutable structural measurements from one automaton build. These counts
+    # are intended for diagnostics and benchmarks, not parser behavior.
+    class BuildMetrics
+      attr_reader :construction_states #: Integer
+      attr_reader :canonical_states #: Integer?
+      attr_reader :final_states #: Integer
+      attr_reader :strategy #: Symbol
+      attr_reader :lr0_states #: Integer?
+      attr_reader :lr0_items #: Integer?
+      attr_reader :canonical_items #: Integer?
+      attr_reader :final_items #: Integer?
+      attr_reader :final_lookahead_items #: Integer?
+      attr_reader :propagation_edges #: Integer?
+      attr_reader :ielr_initial_partitions #: Integer?
+      attr_reader :ielr_final_partitions #: Integer?
+      attr_reader :ielr_annotations #: Integer?
+      attr_reader :ielr_annotated_states #: Integer?
+      attr_reader :ielr_inadequacies #: Integer?
+      attr_reader :ielr_split_stable_discarded #: Integer?
+      attr_reader :ielr_lalr_states #: Integer?
+      attr_reader :ielr_split_states #: Integer?
+      attr_reader :ielr_unreachable_removed #: Integer?
+      attr_reader :ielr_remergeable_candidates #: Integer?
+
+      # @rbs (construction_states: Integer, canonical_states: Integer?, final_states: Integer, strategy: Symbol,
+      #   ?lr0_states: Integer?, ?lr0_items: Integer?, ?canonical_items: Integer?, ?final_items: Integer?,
+      #   ?final_lookahead_items: Integer?, ?propagation_edges: Integer?, ?ielr_initial_partitions: Integer?,
+      #   ?ielr_final_partitions: Integer?, ?ielr_annotations: Integer?, ?ielr_annotated_states: Integer?,
+      #   ?ielr_inadequacies: Integer?, ?ielr_split_stable_discarded: Integer?, ?ielr_lalr_states: Integer?,
+      #   ?ielr_split_states: Integer?, ?ielr_unreachable_removed: Integer?,
+      #   ?ielr_remergeable_candidates: Integer?) -> void
+      def initialize(construction_states:, canonical_states:, final_states:, strategy:, lr0_states: nil, lr0_items: nil,
+                     canonical_items: nil, final_items: nil, final_lookahead_items: nil, propagation_edges: nil,
+                     ielr_initial_partitions: nil, ielr_final_partitions: nil, ielr_annotations: nil,
+                     ielr_annotated_states: nil, ielr_inadequacies: nil, ielr_split_stable_discarded: nil,
+                     ielr_lalr_states: nil, ielr_split_states: nil, ielr_unreachable_removed: nil,
+                     ielr_remergeable_candidates: nil)
+        @construction_states = construction_states
+        @canonical_states = canonical_states
+        @final_states = final_states
+        @strategy = strategy
+        @lr0_states = lr0_states
+        @lr0_items = lr0_items
+        @canonical_items = canonical_items
+        @final_items = final_items
+        @final_lookahead_items = final_lookahead_items
+        @propagation_edges = propagation_edges
+        @ielr_initial_partitions = ielr_initial_partitions
+        @ielr_final_partitions = ielr_final_partitions
+        @ielr_annotations = ielr_annotations
+        @ielr_annotated_states = ielr_annotated_states
+        @ielr_inadequacies = ielr_inadequacies
+        @ielr_split_stable_discarded = ielr_split_stable_discarded
+        @ielr_lalr_states = ielr_lalr_states
+        @ielr_split_states = ielr_split_states
+        @ielr_unreachable_removed = ielr_unreachable_removed
+        @ielr_remergeable_candidates = ielr_remergeable_candidates
+        freeze
+      end
+    end
+  end
+end
+
+# rubocop:enable Metrics/ParameterLists