ibex 0.1.0 → 0.2.0

data/lib/ibex/coverage/collector.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Coverage
+    # Strictly turns one or more complete parse sessions into a coverage report.
+    class Collector
+      # @rbs @grammar_digest: String?
+      # @rbs @table_format_version: Integer?
+      # @rbs @state_count: Integer?
+      # @rbs @production_count: Integer?
+      # @rbs @sessions: Integer
+      # @rbs @event_count: Integer
+      # @rbs @state_hits: Hash[Integer, Integer]
+      # @rbs @production_hits: Hash[Integer, Integer]
+      # @rbs @in_session: bool
+      # @rbs @expected_sequence: Integer
+
+      # @rbs () -> void
+      def initialize
+        @grammar_digest = nil
+        @table_format_version = nil
+        @state_count = nil
+        @production_count = nil
+        @sessions = 0
+        @event_count = 0
+        @state_hits = Hash.new(0)
+        @production_hits = Hash.new(0)
+        @in_session = false
+        @expected_sequence = 1
+      end
+
+      # @rbs (String path) -> Report
+      def self.collect_file(path)
+        collector = new
+        EventStream.each_file(path) do |document, line|
+          collector.consume(document, source: path, line: line)
+        end
+        collector.finish(source: path)
+      end
+
+      # @rbs (Hash[String, untyped] document, source: String, line: Integer) -> void
+      def consume(document, source:, line:)
+        event = document.fetch("event")
+        sequence = document.fetch("sequence")
+        if event == "start"
+          start_session(document.fetch("data"), sequence, source, line)
+        else
+          consume_session_event(event, document.fetch("data"), sequence, source, line)
+        end
+        @event_count += 1
+      end
+
+      # @rbs (source: String) -> Report
+      def finish(source:)
+        invalid(source, 1, "event stream ended before accept or reject") if @in_session
+        invalid(source, 1, "event stream does not contain a parse session") if @sessions.zero?
+        Report.new(
+          grammar_digest: required_metadata(@grammar_digest),
+          table_format_version: required_metadata(@table_format_version),
+          state_count: required_metadata(@state_count),
+          production_count: required_metadata(@production_count),
+          sessions: @sessions,
+          event_count: @event_count,
+          state_hits: @state_hits,
+          production_hits: @production_hits
+        )
+      rescue ArgumentError => e
+        raise Ibex::Error, "#{source}:1:1: #{e.message}"
+      end
+
+      private
+
+      # @rbs (Hash[untyped, untyped] data, untyped sequence, String source, Integer line) -> void
+      def start_session(data, sequence, source, line)
+        invalid(source, line, "new start event before prior session ended") if @in_session
+        invalid(source, line, "start event sequence must be 1") unless sequence == 1
+
+        metadata, initial_state = session_metadata(data, source, line)
+        establish_metadata(metadata, source, line)
+        validate_metadata!(source, line)
+        hit_state(initial_state, source, line)
+        @sessions += 1
+        @in_session = true
+        @expected_sequence = 2
+      end
+
+      # @rbs (String event, Hash[untyped, untyped] data, untyped sequence, String source, Integer line) -> void
+      def consume_session_event(event, data, sequence, source, line)
+        invalid(source, line, "event appears outside a parse session") unless @in_session
+        unless sequence == @expected_sequence
+          invalid(source, line, "expected event sequence #{@expected_sequence}, got #{sequence.inspect}")
+        end
+
+        case event
+        when "shift", "recover" then hit_state(data["state"], source, line)
+        when "reduce"
+          hit_production(data["production_id"], source, line)
+          hit_state(data["goto_state"], source, line)
+        end
+        if %w[accept reject].include?(event)
+          @in_session = false
+          @expected_sequence = 1
+        else
+          @expected_sequence += 1
+        end
+      end
+
+      # @rbs (Hash[untyped, untyped] data, String source, Integer line)
+      #   -> [[String, Integer, Integer, Integer], Integer]
+      def session_metadata(data, source, line)
+        digest = data["grammar_digest"]
+        format = data["table_format_version"]
+        states = data["state_count"]
+        productions = data["production_count"]
+        initial_state = data["initial_state"]
+        unless digest.is_a?(String) && format.is_a?(Integer) && states.is_a?(Integer) &&
+               productions.is_a?(Integer) && initial_state.is_a?(Integer)
+          invalid(source, line, "start event lacks generated parser coverage metadata")
+        end
+        [[digest, format, states, productions], initial_state]
+      end
+
+      # @rbs ([String, Integer, Integer, Integer] metadata, String source, Integer line) -> void
+      def establish_metadata(metadata, source, line)
+        unless @grammar_digest
+          @grammar_digest, @table_format_version, @state_count, @production_count = metadata
+          return
+        end
+
+        expected = [@grammar_digest, @table_format_version, @state_count, @production_count]
+        invalid(source, line, "parse sessions use different parser metadata") unless metadata == expected
+      end
+
+      # @rbs (String source, Integer line) -> void
+      def validate_metadata!(source, line)
+        Report.new(
+          grammar_digest: required_metadata(@grammar_digest),
+          table_format_version: required_metadata(@table_format_version),
+          state_count: required_metadata(@state_count),
+          production_count: required_metadata(@production_count),
+          sessions: 1,
+          event_count: 1,
+          state_hits: {},
+          production_hits: {}
+        )
+      rescue ArgumentError => e
+        invalid(source, line, e.message)
+      end
+
+      # @rbs (untyped id, String source, Integer line) -> void
+      def hit_state(id, source, line)
+        total = required_metadata(@state_count)
+        invalid(source, line, "state id #{id.inspect} is outside 0...#{total}") unless valid_id?(id, total)
+        increment(@state_hits, id, "state", source, line)
+      end
+
+      # @rbs (untyped id, String source, Integer line) -> void
+      def hit_production(id, source, line)
+        total = required_metadata(@production_count)
+        invalid(source, line, "production id #{id.inspect} is outside 0...#{total}") unless valid_id?(id, total)
+        increment(@production_hits, id, "production", source, line)
+      end
+
+      # @rbs (untyped id, Integer total) -> bool
+      def valid_id?(id, total)
+        id.is_a?(Integer) && id >= 0 && id < total
+      end
+
+      # @rbs (Hash[Integer, Integer] hits, Integer id, String kind, String source, Integer line) -> void
+      def increment(hits, id, kind, source, line)
+        count = hits[id] + 1
+        invalid(source, line, "#{kind} hit count exceeds the supported count") if count > Report::MAX_COUNT
+        hits[id] = count
+      end
+
+      # @rbs [T] (T? value) -> T
+      def required_metadata(value)
+        value || raise(ArgumentError, "coverage metadata is unavailable")
+      end
+
+      # @rbs (String source, Integer line, String message) -> bot
+      def invalid(source, line, message)
+        raise Ibex::Error, "#{source}:#{line}:1: #{message}"
+      end
+    end
+  end
+end

data/lib/ibex/coverage/event_stream.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "json"
+require_relative "../error"
+require_relative "../runtime/event"
+require_relative "runtime_event_validator"
+
+module Ibex
+  module Coverage
+    # Bounded reader for the public runtime-event JSON Lines protocol.
+    module EventStream
+      # @rbs!
+      #   interface _CoverageEventInput
+      #     def gets: (Integer) -> String?
+      #   end
+
+      MAX_LINE_BYTES = 1_048_576 #: Integer
+      MAX_NESTING = 32 #: Integer
+      ROOT_KEYS = %w[data event ibex_runtime_event schema_version sequence].freeze #: Array[String]
+      EVENT_TYPES = %w[start shift reduce error recover discard accept reject].freeze #: Array[String]
+
+      class << self
+        # @rbs (String path) { (Hash[String, untyped], Integer) -> void } -> void
+        def each_file(path, &block)
+          File.open(path, "rb") { |input| each_io(input, source: path, &block) }
+        end
+
+        # @rbs (_CoverageEventInput input, source: String)
+        #   { (Hash[String, untyped], Integer) -> void } -> void
+        def each_io(input, source:, &block)
+          line_number = 0
+          while (line = input.gets(MAX_LINE_BYTES + 1))
+            line_number += 1
+            reject_oversized_line(line, source, line_number)
+            document = parse_line(line, source, line_number)
+            validate_envelope(document, source, line_number)
+            block.call(document, line_number)
+          end
+          raise_invalid(source, 1, "event stream is empty") if line_number.zero?
+        end
+
+        private
+
+        # @rbs (String line, String source, Integer line_number) -> void
+        def reject_oversized_line(line, source, line_number)
+          return if line.bytesize <= MAX_LINE_BYTES
+
+          raise_invalid(source, line_number, "event line exceeds #{MAX_LINE_BYTES} bytes")
+        end
+
+        # @rbs (String line, String source, Integer line_number) -> Hash[String, untyped]
+        def parse_line(line, source, line_number)
+          text = line.dup.force_encoding(Encoding::UTF_8)
+          raise_invalid(source, line_number, "event line is not valid UTF-8") unless text.valid_encoding?
+
+          value = JSON.parse(text, max_nesting: MAX_NESTING, allow_nan: false)
+          return value if value.is_a?(Hash) && value.keys.all?(String)
+
+          raise_invalid(source, line_number, "event line must be a JSON object")
+        rescue JSON::ParserError => e
+          raise_invalid(source, line_number, "invalid event JSON: #{e.message}")
+        end
+
+        # @rbs (Hash[String, untyped] document, String source, Integer line_number) -> void
+        def validate_envelope(document, source, line_number)
+          unless document.keys.sort == ROOT_KEYS
+            raise_invalid(source, line_number, "event object has unknown or missing fields")
+          end
+          unless document["ibex_runtime_event"] == Runtime::Event::IDENTIFIER &&
+                 document["schema_version"] == Runtime::Event::SCHEMA_VERSION
+            raise_invalid(source, line_number, "unsupported runtime event schema")
+          end
+          sequence = document["sequence"]
+          raise_invalid(source, line_number, "event sequence must be positive") unless positive_integer?(sequence)
+          unless EVENT_TYPES.include?(document["event"])
+            raise_invalid(source, line_number, "unknown runtime event type")
+          end
+          raise_invalid(source, line_number, "event data must be an object") unless document["data"].is_a?(Hash)
+          return if RuntimeEventValidator.valid?(document)
+
+          raise_invalid(source, line_number, "event data does not match the runtime-event v1 schema")
+        end
+
+        # @rbs (untyped value) -> bool
+        def positive_integer?(value)
+          value.is_a?(Integer) && value.positive?
+        end
+
+        # @rbs (String source, Integer line_number, String message) -> bot
+        def raise_invalid(source, line_number, message)
+          raise Ibex::Error, "#{source}:#{line_number}:1: #{message}"
+        end
+      end
+    end
+  end
+end

data/lib/ibex/coverage/report.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "json"
+require_relative "../error"
+
+module Ibex
+  module Coverage
+    # Versioned, mergeable runtime coverage result.
+    class Report
+      IDENTIFIER = "runtime-coverage" #: String
+      SCHEMA_VERSION = 1 #: Integer
+      MAX_DOCUMENT_BYTES = 16_777_216 #: Integer
+      MAX_TOTAL = 1_000_000 #: Integer
+      MAX_COUNT = 9_223_372_036_854_775_807 #: Integer
+      ROOT_KEYS = %w[events grammar_digest ibex_coverage production_hits schema_version sessions state_hits
+                     table_format_version totals].freeze #: Array[String]
+      TOTAL_KEYS = %w[productions states].freeze #: Array[String]
+      HIT_KEYS = %w[count id].freeze #: Array[String]
+
+      attr_reader :grammar_digest #: String
+      attr_reader :table_format_version #: Integer
+      attr_reader :state_count #: Integer
+      attr_reader :production_count #: Integer
+      attr_reader :sessions #: Integer
+      attr_reader :event_count #: Integer
+      attr_reader :state_hits #: Hash[Integer, Integer]
+      attr_reader :production_hits #: Hash[Integer, Integer]
+
+      # rubocop:disable Layout/LineLength
+      # @rbs (grammar_digest: String, table_format_version: Integer, state_count: Integer, production_count: Integer, sessions: Integer, event_count: Integer, state_hits: Hash[Integer, Integer], production_hits: Hash[Integer, Integer]) -> void
+      def initialize(grammar_digest:, table_format_version:, state_count:, production_count:, sessions:, event_count:,
+                     state_hits:, production_hits:)
+        @grammar_digest = validate_digest(grammar_digest)
+        @table_format_version = positive_integer(table_format_version, "table_format_version")
+        @state_count = total_integer(state_count, "state total", minimum: 1)
+        @production_count = total_integer(production_count, "production total", minimum: 0)
+        @sessions = positive_integer(sessions, "sessions")
+        @event_count = positive_integer(event_count, "events")
+        raise ArgumentError, "events must be at least sessions" if @event_count < @sessions
+
+        @state_hits = validate_hits(state_hits, @state_count, "state").freeze
+        @production_hits = validate_hits(production_hits, @production_count, "production").freeze
+        validate_event_bounds
+        freeze
+      end
+      # rubocop:enable Layout/LineLength
+
+      # @rbs () -> Hash[String, untyped]
+      def to_h
+        {
+          "ibex_coverage" => IDENTIFIER,
+          "schema_version" => SCHEMA_VERSION,
+          "grammar_digest" => @grammar_digest,
+          "table_format_version" => @table_format_version,
+          "totals" => { "states" => @state_count, "productions" => @production_count },
+          "sessions" => @sessions,
+          "events" => @event_count,
+          "state_hits" => hit_documents(@state_hits),
+          "production_hits" => hit_documents(@production_hits)
+        }
+      end
+
+      # @rbs (*untyped) -> String
+      def to_json(*)
+        "#{JSON.pretty_generate(to_h)}\n"
+      end
+
+      # @rbs (String path) -> Report
+      def self.load_file(path)
+        size = File.size(path)
+        if size > MAX_DOCUMENT_BYTES
+          raise Ibex::Error, "#{path}:1:1: coverage document exceeds #{MAX_DOCUMENT_BYTES} bytes"
+        end
+
+        source = File.binread(path)
+        if source.bytesize > MAX_DOCUMENT_BYTES
+          raise Ibex::Error, "#{path}:1:1: coverage document exceeds #{MAX_DOCUMENT_BYTES} bytes"
+        end
+
+        source.force_encoding(Encoding::UTF_8)
+        raise Ibex::Error, "#{path}:1:1: coverage document is not valid UTF-8" unless source.valid_encoding?
+
+        value = JSON.parse(source, max_nesting: 16, allow_nan: false)
+        from_h(value, source: path)
+      rescue JSON::ParserError => e
+        raise Ibex::Error, "#{path}:1:1: invalid coverage JSON: #{e.message}"
+      end
+
+      # @rbs (untyped value, source: String) -> Report
+      def self.from_h(value, source:)
+        document = string_hash(value, source, "coverage document")
+        invalid(source, "coverage object has unknown or missing fields") unless document.keys.sort == ROOT_KEYS
+        valid_schema = document["ibex_coverage"] == IDENTIFIER &&
+                       document["schema_version"] == SCHEMA_VERSION
+        invalid(source, "unsupported coverage schema") unless valid_schema
+        totals = string_hash(document["totals"], source, "coverage totals")
+        invalid(source, "coverage totals have unknown or missing fields") unless totals.keys.sort == TOTAL_KEYS
+
+        state_count = totals["states"]
+        production_count = totals["productions"]
+        new(
+          grammar_digest: document["grammar_digest"],
+          table_format_version: document["table_format_version"],
+          state_count: state_count,
+          production_count: production_count,
+          sessions: document["sessions"],
+          event_count: document["events"],
+          state_hits: parse_hits(document["state_hits"], state_count, source, "state"),
+          production_hits: parse_hits(document["production_hits"], production_count, source, "production")
+        )
+      rescue ArgumentError => e
+        raise Ibex::Error, "#{source}:1:1: #{e.message}"
+      end
+
+      # @rbs (Array[Report] reports) -> Report
+      def self.merge(reports)
+        raise ArgumentError, "at least one coverage report is required" if reports.empty?
+
+        first = reports.first
+        reports.drop(1).each { |report| ensure_compatible(first, report) }
+        new(
+          grammar_digest: first.grammar_digest,
+          table_format_version: first.table_format_version,
+          state_count: first.state_count,
+          production_count: first.production_count,
+          sessions: checked_sum(reports.map(&:sessions), "sessions"),
+          event_count: checked_sum(reports.map(&:event_count), "events"),
+          state_hits: merge_hits(reports.map(&:state_hits), "state"),
+          production_hits: merge_hits(reports.map(&:production_hits), "production")
+        )
+      end
+
+      private
+
+      # @rbs (untyped input) -> String
+      def validate_digest(input)
+        valid = input.is_a?(String) && input.match?(/\Asha256:[0-9a-f]{64}\z/)
+        return input.dup.freeze if valid
+
+        raise ArgumentError, "grammar_digest must be a full lowercase SHA-256 digest"
+      end
+
+      # @rbs (untyped input, String name) -> Integer
+      def positive_integer(input, name)
+        return input if input.is_a?(Integer) && input.positive? && input <= MAX_COUNT
+
+        raise ArgumentError, "#{name} must be a bounded positive integer"
+      end
+
+      # @rbs (untyped input, String name, minimum: Integer) -> Integer
+      def total_integer(input, name, minimum:)
+        return input if input.is_a?(Integer) && input >= minimum && input <= MAX_TOTAL
+
+        raise ArgumentError, "#{name} must be between #{minimum} and #{MAX_TOTAL}"
+      end
+
+      # @rbs (Hash[Integer, Integer] input, Integer total, String kind) -> Hash[Integer, Integer]
+      def validate_hits(input, total, kind)
+        input.to_h do |id, count|
+          unless id.is_a?(Integer) && id >= 0 && id < total
+            raise ArgumentError, "#{kind} hit id #{id.inspect} is outside 0...#{total}"
+          end
+
+          [id, positive_integer(count, "#{kind} hit count")]
+        end.sort.to_h
+      end
+
+      # @rbs (Hash[Integer, Integer] hits) -> Array[Hash[String, Integer]]
+      def hit_documents(hits)
+        hits.map { |id, count| { "id" => id, "count" => count } }
+      end
+
+      # @rbs () -> void
+      def validate_event_bounds
+        raise ArgumentError, "state hit counts exceed events" if @state_hits.values.sum > @event_count
+        return unless @production_hits.values.sum > @event_count
+
+        raise ArgumentError, "production hit counts exceed events"
+      end
+
+      class << self
+        private
+
+        # @rbs (untyped value, String source, String name) -> Hash[String, untyped]
+        def string_hash(value, source, name)
+          return value if value.is_a?(Hash) && value.keys.all?(String)
+
+          invalid(source, "#{name} must be an object")
+        end
+
+        # @rbs (untyped value, untyped total, String source, String kind) -> Hash[Integer, Integer]
+        def parse_hits(value, total, source, kind)
+          invalid(source, "#{kind}_hits must be an array") unless value.is_a?(Array)
+          hits = {} #: Hash[Integer, Integer]
+          previous = -1
+          value.each do |entry|
+            id, count = parse_hit(entry, total, previous, source, kind)
+            hits[id] = count
+            previous = id
+          end
+          hits
+        end
+
+        # @rbs (untyped entry, untyped total, Integer previous, String source, String kind) -> [Integer, Integer]
+        def parse_hit(entry, total, previous, source, kind)
+          hit = string_hash(entry, source, "#{kind} hit")
+          invalid(source, "#{kind} hit has unknown or missing fields") unless hit.keys.sort == HIT_KEYS
+          id = hit["id"]
+          count = hit["count"]
+          unless id.is_a?(Integer) && id > previous
+            invalid(source, "#{kind} hits must have unique, ascending integer ids")
+          end
+          unless total.is_a?(Integer) && id >= 0 && id < total
+            invalid(source, "#{kind} hit id is outside the declared total")
+          end
+          unless count.is_a?(Integer) && count.positive? && count <= MAX_COUNT
+            invalid(source, "#{kind} hit count must be positive")
+          end
+          [id, count]
+        end
+
+        # @rbs (Report expected, Report actual) -> void
+        def ensure_compatible(expected, actual)
+          fields = %i[grammar_digest table_format_version state_count production_count]
+          mismatch = fields.find { |field| expected.public_send(field) != actual.public_send(field) }
+          return unless mismatch
+
+          raise ArgumentError, "coverage reports have different #{mismatch}"
+        end
+
+        # @rbs (Array[Integer] values, String name) -> Integer
+        def checked_sum(values, name)
+          total = values.sum
+          raise ArgumentError, "merged #{name} exceed the supported count" if total > MAX_COUNT
+
+          total
+        end
+
+        # @rbs (Array[Hash[Integer, Integer]] collections, String kind) -> Hash[Integer, Integer]
+        def merge_hits(collections, kind)
+          result = Hash.new(0) #: Hash[Integer, Integer]
+          collections.each do |hits|
+            hits.each do |id, count|
+              result[id] += count
+              raise ArgumentError, "merged #{kind} hit count exceeds the supported count" if result[id] > MAX_COUNT
+            end
+          end
+          result
+        end
+
+        # @rbs (String source, String message) -> bot
+        def invalid(source, message)
+          raise Ibex::Error, "#{source}:1:1: #{message}"
+        end
+      end
+    end
+  end
+end