ibex 0.1.0 → 0.2.0

data/lib/ibex/coverage/runtime_event_validator.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Coverage
+    # Dependency-free structural validation for runtime-event schema version 1.
+    module RuntimeEventValidator
+      TOKEN_KEYS = %w[location state token token_id value].freeze #: Array[String]
+      LOCATION_KEYS = %w[column end_column end_line file line].freeze #: Array[String]
+      VALIDATORS = {
+        "start" => :valid_start?,
+        "shift" => :valid_shift?,
+        "reduce" => :valid_reduce?,
+        "error" => :valid_error?,
+        "recover" => :valid_recover?,
+        "discard" => :valid_discard?,
+        "accept" => :valid_accept?,
+        "reject" => :valid_reject?
+      }.freeze #: Hash[String, Symbol]
+
+      class << self
+        # @rbs (Hash[String, untyped] document) -> bool
+        def valid?(document)
+          data = document["data"]
+          return false unless data.is_a?(Hash) && data.keys.all?(String)
+
+          validator = VALIDATORS[document["event"]]
+          validator ? send(validator, data) : false
+        end
+
+        private
+
+        # @rbs (Hash[String, untyped] data) -> bool
+        def valid_start?(data)
+          return false unless exact_keys?(
+            data,
+            %w[driver grammar_digest initial_state production_count state_count table_format_version]
+          )
+
+          integer_fields = %w[initial_state table_format_version]
+          optional_integer_fields = %w[state_count production_count]
+          %w[pull push].include?(data["driver"]) &&
+            integer_fields.all? { |key| integer?(data[key]) } &&
+            optional_integer_fields.all? { |key| optional_integer?(data[key]) } &&
+            optional_string?(data["grammar_digest"])
+        end
+
+        # @rbs (Hash[String, untyped] data) -> bool
+        def valid_shift?(data)
+          exact_keys?(data, TOKEN_KEYS + %w[from_state]) &&
+            valid_token_fields?(data) &&
+            integer?(data["from_state"])
+        end
+
+        # @rbs (Hash[String, untyped] data) -> bool
+        def valid_reduce?(data)
+          keys = %w[goto_state lhs location post_state pre_state production_id result rhs_length]
+          exact_keys?(data, keys) &&
+            %w[goto_state lhs post_state pre_state production_id].all? { |key| integer?(data[key]) } &&
+            integer?(data["rhs_length"]) && data["rhs_length"] >= 0 &&
+            summary?(data["result"]) && location?(data["location"])
+        end
+
+        # @rbs (Hash[String, untyped] data, reasons: Array[String]) -> bool
+        def valid_token_event?(data, reasons:)
+          exact_keys?(data, TOKEN_KEYS + %w[reason]) &&
+            valid_token_fields?(data) &&
+            reasons.include?(data["reason"])
+        end
+
+        # @rbs (Hash[String, untyped] data) -> bool
+        def valid_error?(data)
+          valid_token_event?(data, reasons: %w[syntax semantic])
+        end
+
+        # @rbs (Hash[String, untyped] data) -> bool
+        def valid_discard?(data)
+          valid_token_event?(data, reasons: %w[recovery])
+        end
+
+        # @rbs (Hash[String, untyped] data) -> bool
+        def valid_reject?(data)
+          valid_token_event?(data, reasons: %w[eof_during_recovery no_recovery_state])
+        end
+
+        # @rbs (Hash[String, untyped] data) -> bool
+        def valid_recover?(data)
+          exact_keys?(data, TOKEN_KEYS + %w[from_state reason]) &&
+            valid_token_fields?(data) &&
+            integer?(data["from_state"]) &&
+            %w[syntax semantic].include?(data["reason"])
+        end
+
+        # @rbs (Hash[String, untyped] data) -> bool
+        def valid_accept?(data)
+          exact_keys?(data, %w[reason result state]) &&
+            integer?(data["state"]) &&
+            summary?(data["result"]) &&
+            %w[table semantic].include?(data["reason"])
+        end
+
+        # @rbs (Hash[String, untyped] data) -> bool
+        def valid_token_fields?(data)
+          integer?(data["state"]) &&
+            optional_integer?(data["token_id"]) &&
+            summary?(data["token"]) &&
+            summary?(data["value"]) &&
+            location?(data["location"])
+        end
+
+        # @rbs (untyped value) -> bool
+        def summary?(value)
+          case value
+          when nil, true, false, Integer then true
+          when Float then value.finite?
+          when String then value.length <= 256
+          else composite_summary?(value)
+          end
+        end
+
+        # @rbs (untyped value) -> bool
+        def composite_summary?(value)
+          return value.length <= 17 && value.all? { |entry| summary?(entry) } if value.is_a?(Array)
+          return false unless value.is_a?(Hash)
+
+          value.keys.all?(String) && value.length <= 17 && value.values.all? { |entry| summary?(entry) }
+        end
+
+        # @rbs (untyped value) -> bool
+        def location?(value)
+          return true if value.nil?
+          return false unless value.is_a?(Hash) && value.keys.all?(String)
+          return value["unavailable"].is_a?(String) if value.keys == ["unavailable"]
+
+          (value.keys - LOCATION_KEYS).empty? && value.values.all? { |entry| summary?(entry) }
+        end
+
+        # @rbs (Hash[String, untyped] data, Array[String] keys) -> bool
+        def exact_keys?(data, keys)
+          data.keys.sort == keys.sort
+        end
+
+        # @rbs (untyped value) -> bool
+        def integer?(value)
+          value.is_a?(Integer)
+        end
+
+        # @rbs (untyped value) -> bool
+        def optional_integer?(value)
+          value.nil? || integer?(value)
+        end
+
+        # @rbs (untyped value) -> bool
+        def optional_string?(value)
+          value.nil? || value.is_a?(String)
+        end
+      end
+    end
+  end
+end

data/lib/ibex/coverage.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "coverage/runtime_event_validator"
+require_relative "coverage/event_stream"
+require_relative "coverage/report"
+require_relative "coverage/collector"
+
+module Ibex
+  # Deterministic state and production coverage derived from runtime event streams.
+  module Coverage
+  end
+end

data/lib/ibex/error_messages/parser.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require_relative "parser_v2"
+
+module Ibex
+  module ErrorMessages
+    # Strict line-oriented parser for the ibex-messages v1 and v2 formats.
+    class Parser
+      include ParserV2
+
+      ESCAPES = { "\\" => "\\", "n" => "\n", "t" => "\t", "r" => "\r" }.freeze
+
+      # @rbs @file: String
+      # @rbs @lines: Array[String]
+
+      # @rbs (String source, file: String) -> void
+      def initialize(source, file:)
+        @file = file
+        text = source.dup.force_encoding(Encoding::UTF_8)
+        fail_at(1, 1, "messages file must be valid UTF-8") unless text.valid_encoding?
+
+        @lines = text.lines(chomp: true)
+      end
+
+      # @rbs () -> Document
+      def parse
+        version = validate_header
+        return parse_v2 if version == 2
+
+        parse_v1
+      end
+
+      private
+
+      # @rbs () -> Document
+      def parse_v1
+        entries = [] #: Array[Entry]
+        declarations = {} #: Hash[Integer, Integer]
+        index = 1
+        while index < @lines.length
+          line = @lines.fetch(index)
+          if ignorable?(line)
+            index += 1
+            next
+          end
+
+          match = line.strip.match(/\A(state|removed)\s+([0-9]+)\z/)
+          fail_at(index + 1, column(line), top_level_expectation) unless match
+          state_text = match[2] || raise(Ibex::Error, "missing state number")
+          state = Integer(state_text, 10)
+          reject_duplicate(declarations, state, index, line)
+          declarations[state] = index + 1
+          entry, index = parse_entry(index, state, match[1] == "state" ? :active : :removed)
+          entries << entry
+        end
+        Document.new(version: 1, entries: entries)
+      end
+
+      # @rbs () -> Integer
+      def validate_header
+        header = @lines.first&.delete_prefix("\uFEFF")
+        return 1 if header == HEADER_V1
+        return 2 if header == HEADER
+
+        fail_at(1, 1, "expected #{HEADER.inspect} or #{HEADER_V1.inspect}")
+      end
+
+      # @rbs (Hash[Integer, Integer] declarations, Integer state, Integer index, String line) -> void
+      def reject_duplicate(declarations, state, index, line)
+        return unless declarations.key?(state)
+
+        fail_at(index + 1, column(line),
+                "duplicate state #{state}; first declared at line #{declarations.fetch(state)}")
+      end
+
+      # @rbs (Integer opening_index, Integer state, :active | :removed status) -> [Entry, Integer]
+      def parse_entry(opening_index, state, status)
+        message_lines = [] #: Array[String]
+        index = opening_index + 1
+        while index < @lines.length
+          line = @lines.fetch(index)
+          return close_entry(message_lines, opening_index, index, state, status) if line.strip == "end"
+
+          read_entry_line(message_lines, line, index)
+          index += 1
+        end
+        label = status == :active ? "state" : "removed"
+        fail_at(opening_index + 1, 1, "unterminated #{label} #{state} entry")
+      end
+
+      # @rbs (Array[String] message_lines, Integer opening_index, Integer index, Integer state,
+      #   :active | :removed status) -> [Entry, Integer]
+      def close_entry(message_lines, opening_index, index, state, status)
+        message = message_lines.empty? ? nil : message_lines.join("\n")
+        fail_at(opening_index + 1, 1, "message for state #{state} must not be empty") if message&.strip&.empty?
+
+        entry = Entry.new(state: state, status: status, message: message, line: opening_index + 1)
+        [entry, index + 1]
+      end
+
+      # @rbs (Array[String] message_lines, String line, Integer index) -> void
+      def read_entry_line(message_lines, line, index)
+        return if ignorable?(line)
+
+        unless line.start_with?("|")
+          fail_at(index + 1, column(line), "expected a `| ` message line, a comment, a blank line, or `end`")
+        end
+
+        content = line.delete_prefix("|")
+        content = content.delete_prefix(" ")
+        content_column = line.start_with?("| ") ? 3 : 2
+        message_lines << decode_line(content, index + 1, content_column)
+      end
+
+      # @rbs (String content, Integer line, Integer content_column) -> String
+      def decode_line(content, line, content_column)
+        decoded = +""
+        index = 0
+        while index < content.length
+          character = content[index] || raise(Ibex::Error, "missing message character")
+          unless character == "\\"
+            decoded << character
+            index += 1
+            next
+          end
+
+          escaped = content[index + 1]
+          fail_at(line, content_column + index, "trailing backslash in message line") unless escaped
+          replacement = ESCAPES[escaped]
+          fail_at(line, content_column + index, "unknown escape \\#{escaped}") unless replacement
+
+          decoded << replacement
+          index += 2
+        end
+        decoded
+      end
+
+      # @rbs (String line) -> bool
+      def ignorable?(line)
+        line.strip.empty? || line.lstrip.start_with?("#")
+      end
+
+      # @rbs () -> String
+      def top_level_expectation
+        "expected `state N`, `removed N`, a comment, or a blank line"
+      end
+
+      # @rbs (String line) -> Integer
+      def column(line)
+        (line.index(/\S/) || 0) + 1
+      end
+
+      # @rbs (Integer line, Integer column, String message) -> bot
+      def fail_at(line, column, message)
+        raise Ibex::Error, "#{@file}:#{line}:#{column}: #{message}"
+      end
+    end
+  end
+end

data/lib/ibex/error_messages/parser_v2.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module Ibex
+  module ErrorMessages
+    # Sentence-keyed ibex-messages v2 parsing, mixed into Parser so both
+    # versions share UTF-8, escape, and positioned-diagnostic helpers.
+    module ParserV2
+      # @rbs!
+      #   private def ignorable?: (String) -> bool
+      #   private def column: (String) -> Integer
+      #   private def fail_at: (Integer, Integer, String) -> bot
+      #   private def decode_line: (String, Integer, Integer) -> String
+
+      private
+
+      # @rbs () -> Document
+      def parse_v2
+        entries = [] #: Array[Entry]
+        identifiers = {} #: Hash[String, Integer]
+        keys = {} #: Hash[String, Integer]
+        index = 1
+        while index < @lines.length
+          line = @lines.fetch(index)
+          if ignorable?(line) && !line.lstrip.start_with?("##")
+            index += 1
+            next
+          end
+
+          entry, index = parse_v2_entry(index)
+          record_v2_entry(entry, entries, identifiers, keys)
+        end
+        Document.new(version: 2, entries: entries)
+      end
+
+      # @rbs (Entry entry, Array[Entry] entries, Hash[String, Integer] identifiers,
+      #   Hash[String, Integer] keys) -> void
+      def record_v2_entry(entry, entries, identifiers, keys)
+        error_id = entry.error_id || raise(Ibex::Error, "missing error id")
+        reject_v2_duplicate(identifiers, error_id, entry.line, "error id")
+        key = entry.sentence ? "#{entry.entry}\0#{entry.sentence.join("\0")}" : "legacy\0#{entry.state}"
+        reject_v2_duplicate(keys, key, entry.line, "sentence")
+        identifiers[error_id] = entry.line
+        keys[key] = entry.line
+        entries << entry
+      end
+
+      # @rbs (Integer opening_index) -> [Entry, Integer]
+      def parse_v2_entry(opening_index)
+        status, sentence, state = parse_v2_opening(opening_index)
+        read_v2_entry_body(opening_index, status, sentence, state)
+      end
+
+      # @rbs (Integer opening_index) ->
+      #   [:active | :unreachable | :removed, Array[String]?, Integer?]
+      def parse_v2_opening(opening_index)
+        opening = @lines.fetch(opening_index)
+        if (legacy = opening.match(/\Alegacy-state:[ \t]*([0-9]+)\z/))
+          return [:removed, nil, Integer(legacy[1] || "0", 10)]
+        end
+
+        if (sentence = opening.match(/\A(sentence|unreachable):[ \t]*(.*)\z/))
+          status = sentence[1] == "unreachable" ? :unreachable : :active #: :unreachable | :active
+          return [status, parse_sentence(sentence[2] || "", opening_index + 1), nil]
+        end
+
+        fail_at(opening_index + 1, column(opening),
+                "expected `sentence: TOKENS`, `unreachable: TOKENS`, or `legacy-state: N`")
+      end
+
+      # @rbs (Integer opening_index, :active | :unreachable | :removed status, Array[String]? sentence,
+      #   Integer? state) -> [Entry, Integer]
+      def read_v2_entry_body(opening_index, status, sentence, state)
+        error_id = nil #: String?
+        entry_name = nil #: String?
+        message_lines = [] #: Array[String]
+        index = opening_index + 1
+        while index < @lines.length
+          line = @lines.fetch(index)
+          if line.strip == "end"
+            return close_v2_entry(
+              opening_index, index, status, sentence, state, error_id, entry_name, message_lines
+            )
+          end
+
+          error_id, state, entry_name = read_v2_metadata(
+            line, index, error_id, state, entry_name, message_lines
+          )
+          index += 1
+        end
+        fail_at(opening_index + 1, 1, "unterminated error-sentence entry")
+      end
+
+      # @rbs (Integer opening_index, Integer index, :active | :unreachable | :removed status,
+      #   Array[String]? sentence, Integer? state, String? error_id, String? entry_name,
+      #   Array[String] message_lines) -> [Entry, Integer]
+      def close_v2_entry(opening_index, index, status, sentence, state, error_id, entry_name, message_lines)
+        fail_at(opening_index + 1, 1, "missing `## E0001` error id") unless error_id
+        message = message_lines.empty? ? nil : message_lines.join("\n")
+        fail_at(opening_index + 1, 1, "message must not be empty") if message&.strip&.empty?
+        entry = Entry.new(
+          state: state, status: status, message: message, line: opening_index + 1,
+          sentence: sentence, error_id: error_id, entry: entry_name
+        )
+        [entry, index + 1]
+      end
+
+      # @rbs (String line, Integer index, String? error_id, Integer? state, String? entry_name,
+      #   Array[String] message_lines) -> [String?, Integer?, String?]
+      def read_v2_metadata(line, index, error_id, state, entry_name, message_lines)
+        stripped = line.strip
+        if (match = stripped.match(/\A##[ \t]+(E[0-9]{4,})\z/))
+          fail_at(index + 1, column(line), "duplicate error id") if error_id
+          error_id = match[1]
+        elsif (match = stripped.match(/\A# state:[ \t]*([0-9]+)\z/))
+          state = Integer(match[1] || "0", 10)
+        elsif (match = stripped.match(/\A# entry:[ \t]*(\S+)\z/))
+          entry_name = match[1]
+        elsif line.start_with?("|")
+          append_v2_message_line(message_lines, line, index)
+        elsif !ignorable?(line)
+          fail_at(index + 1, column(line), "expected metadata, a `| ` message line, a comment, or `end`")
+        end
+        [error_id, state, entry_name]
+      end
+
+      # @rbs (Array[String] message_lines, String line, Integer index) -> void
+      def append_v2_message_line(message_lines, line, index)
+        content = line.delete_prefix("|").delete_prefix(" ")
+        content_column = line.start_with?("| ") ? 3 : 2
+        message_lines << decode_line(content, index + 1, content_column)
+      end
+
+      # @rbs (String source, Integer line) -> Array[String]
+      def parse_sentence(source, line)
+        tokens = [] #: Array[String]
+        index = 0
+        loop do
+          index = skip_sentence_space(source, index)
+          break if index == source.length
+
+          token, index = read_sentence_token(source, index, line)
+          tokens << token
+        end
+        fail_at(line, 1, "error sentence must contain at least one token") if tokens.empty?
+        tokens
+      end
+
+      # @rbs (String source, Integer index) -> Integer
+      def skip_sentence_space(source, index)
+        index += 1 while index < source.length && source[index]&.match?(/\s/)
+        index
+      end
+
+      # @rbs (String source, Integer index, Integer line) -> [String, Integer]
+      def read_sentence_token(source, index, line)
+        start = index
+        quote = source[index]
+        if ["'", '"'].include?(quote)
+          index = quoted_token_end(source, index, quote || "", line)
+        else
+          index += 1 while index < source.length && !source[index]&.match?(/\s/)
+        end
+        token = source[start...index] || raise(Ibex::Error, "missing sentence token")
+        [token, index]
+      end
+
+      # @rbs (String source, Integer index, String quote, Integer line) -> Integer
+      def quoted_token_end(source, index, quote, line)
+        ending = scan_quoted_token(source, index, quote, line)
+        return ending if ending == source.length || source[ending]&.match?(/\s/)
+
+        fail_at(line, ending + 1, "quoted token must be followed by whitespace")
+      end
+
+      # @rbs (String source, Integer index, String quote, Integer line) -> Integer
+      def scan_quoted_token(source, index, quote, line)
+        index += 1
+        while index < source.length
+          character = source[index]
+          if character == "\\"
+            index += 2
+          elsif character == quote
+            return index + 1
+          else
+            index += 1
+          end
+        end
+        fail_at(line, source.length + 1, "unterminated quoted token")
+      end
+
+      # @rbs (Hash[String, Integer] declarations, String key, Integer line, String label) -> void
+      def reject_v2_duplicate(declarations, key, line, label)
+        previous = declarations[key]
+        fail_at(line, 1, "duplicate #{label}; first declared at line #{previous}") if previous
+      end
+    end
+  end
+end

data/lib/ibex/error_messages/renderer.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Ibex
+  module ErrorMessages
+    # @rbs (IR::Automaton automaton, Array[Entry] entries) -> String
+    def render_entries(automaton, entries)
+      lines = [
+        HEADER,
+        "# Generated by `ibex errors --update`; sentences and IDs are stable review keys.",
+        "# Edit only `| ` message lines.",
+        ""
+      ]
+      entries.each { |entry| append_entry(lines, automaton, entry) }
+      "#{lines.join("\n").rstrip}\n"
+    end
+
+    # @rbs (Array[String] lines, IR::Automaton automaton, Entry entry) -> void
+    def append_entry(lines, automaton, entry)
+      lines << entry_heading(entry)
+      lines << "## #{entry.error_id}"
+      lines << "# entry: #{entry.entry}" if entry.entry
+      lines << "# state: #{entry.state}" if entry.state
+      append_expected_tokens(lines, automaton, entry)
+      lines << "# Add one or more `| ` lines to customize this error." unless entry.message
+      entry.message&.split("\n", -1)&.each { |line| lines << "| #{encode_line(line)}" }
+      lines.push("end", "")
+    end
+
+    # @rbs (Entry entry) -> String
+    def entry_heading(entry)
+      return "legacy-state: #{entry.state}" unless entry.sentence
+
+      prefix = entry.status == :active ? "sentence" : "unreachable"
+      "#{prefix}: #{entry.sentence.join(' ')}"
+    end
+
+    # @rbs (Array[String] lines, IR::Automaton automaton, Entry entry) -> void
+    def append_expected_tokens(lines, automaton, entry)
+      return unless entry.status == :active && entry.state
+
+      expected = expected_tokens(automaton, automaton.states.fetch(entry.state))
+      lines << "# expected: #{expected.empty? ? '(none)' : expected.join(', ')}"
+    end
+
+    # @rbs (String line) -> String
+    def encode_line(line)
+      line.gsub("\\", "\\\\").gsub("\t", "\\t").gsub("\r", "\\r")
+    end
+
+    # @rbs (IR::Automaton automaton, IR::AutomatonState state) -> Array[String]
+    def expected_tokens(automaton, state)
+      ordinary_terminals(automaton).filter_map do |terminal|
+        action = state.actions[terminal.id] || state.default_action
+        terminal.display_name || terminal.name unless error_action?(action)
+      end
+    end
+
+    module_function :render_entries, :append_entry, :entry_heading, :append_expected_tokens, :encode_line,
+                    :expected_tokens
+
+    class << self
+      private :render_entries, :append_entry, :entry_heading, :append_expected_tokens, :encode_line, :expected_tokens
+    end
+  end
+end