overseer-testing-protocol 0.1.0

data/lib/overseer/testing_control/conformance/runner.rb

@@ -0,0 +1,783 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'json_schemer'
+require 'securerandom'
+require 'time'
+
+require_relative '../json_subset_matcher'
+
+module Overseer
+  module TestingControl
+    module Conformance
+      class Runner
+        attr_reader :discovery
+
+        class DiscoveryRejected < StandardError; end
+
+        ExpectedIdentity = Data.define(:application_id, :application_version, :environment_id)
+        Dependencies = Data.define(:transport, :clock, :random)
+        DEFAULT_DEPENDENCIES = Dependencies.new(transport: nil, clock: Time, random: SecureRandom)
+        INITIAL_MAX_RESPONSE_BYTES = 16 * 1024 * 1024
+        ERROR_STATUSES = {
+          'invalid-request' => 400,
+          'capability-not-found' => 404,
+          'protocol-version-unsupported' => 409,
+          'reset-requires-restart' => 409,
+          'payload-too-large' => 413,
+          'schema-validation-failed' => 422,
+          'limit-exceeded' => 422,
+          'runtime-unavailable' => 503,
+          'internal-error' => 500
+        }.freeze
+        RAW_EXCEPTION_PATTERN = %r{
+          <!DOCTYPE|<html|\bbacktrace\b|\bstack\s+trace\b|
+          /[A-Za-z0-9_./-]+\.(?:rb|js|py|java|go):\d+
+        }ix
+
+        # rubocop:disable Metrics/ParameterLists
+        def initialize(
+          origin:,
+          base_path:,
+          target_label:,
+          expected_identity:,
+          required_capabilities:,
+          cases:,
+          lifecycle:,
+          redaction_policy:,
+          restart: nil,
+          cancellation: nil,
+          progress: nil,
+          discovery_validator: nil,
+          dependencies: DEFAULT_DEPENDENCIES
+        )
+          @origin = HTTPTransport.origin!(origin)
+          @base_path = base_path
+          @target_label = target_label
+          @expected_identity = expected_identity
+          @required_capabilities = required_capabilities
+          @cases = cases&.fetch('cases', []) || []
+          @lifecycle = lifecycle
+          @redaction_policy = redaction_policy
+          @restart = restart
+          @cancellation = cancellation
+          @progress = progress
+          @discovery_validator = discovery_validator
+          @dependencies = dependencies
+          @transport = dependencies.transport || HTTPTransport.new(origin: @origin, base_path:)
+          @run_id = ProtocolV3.identifier!("conformance-#{dependencies.random.hex(8)}", label: 'conformance run ID')
+          @checks = []
+          @negative_responses = []
+          @response_contract_safe = true
+          @capabilities_document = nil
+          @runtime_data = nil
+          @identity = nil
+          @reset_ready = false
+        end
+        # rubocop:enable Metrics/ParameterLists
+
+        def call
+          started_at = timestamp
+          run_core_checks
+          run_capability_cases
+          completed_at = timestamp
+          report = {
+            'schemaVersion' => '1',
+            'protocolVersion' => ProtocolV3::VERSION,
+            'target' => {
+              'label' => safe_message(@target_label),
+              'origin' => @origin,
+              'sourceVersion' => @expected_identity.application_version
+            },
+            'startedAt' => started_at,
+            'completedAt' => completed_at,
+            'lifecycle' => @lifecycle,
+            'identity' => @identity,
+            'capabilities' => report_capabilities,
+            'checks' => @checks,
+            'aggregateStatus' => @checks.all? { |check| check.fetch('status') == 'passed' } ? 'passed' : 'failed'
+          }
+          Report.validate!(report)
+        end
+
+        private
+
+        def run_core_checks
+          check('core.discovery') do
+            @capabilities_document = successful_exchange(
+              :get,
+              ProtocolV3::PATHS.fetch(:capabilities),
+              kind: :capabilities,
+              max_response_bytes: INITIAL_MAX_RESPONSE_BYTES
+            )
+            'Discovered testing-control protocol v3 through the canonical capabilities endpoint.'
+          end
+          check('core.runtime-identity') { validate_runtime_identity }
+          check('core.capabilities') { validate_capabilities }
+          check('core.unsupported-version') { validate_unsupported_version }
+          check('core.malformed-json') { validate_malformed_json }
+          check('core.request-media-type') { validate_request_media_type }
+          check('core.unknown-arrangement') { validate_unknown_capability(:states, :state_request) }
+          check('core.unknown-observation') { validate_unknown_capability(:probes, :probe_request) }
+          check('core.schema-invalid-request') { validate_schema_invalid_request }
+          @reset_ready = check('core.reset') { perform_reset }
+          check('core.error-safety') { validate_error_safety }
+          check('core.response-contract') { validate_response_contract }
+        end
+
+        def validate_runtime_identity
+          require_capabilities!
+          data = @capabilities_document.fetch('data')
+          @identity = {
+            'applicationId' => data.dig('application', 'id'),
+            'applicationVersion' => data.dig('application', 'version'),
+            'environmentId' => data.dig('environment', 'id')
+          }
+          expected = @expected_identity
+          mismatches = []
+          mismatches << 'application ID' unless @identity.fetch('applicationId') == expected.application_id
+          mismatches << 'application source version' unless
+            @identity.fetch('applicationVersion') == expected.application_version
+          mismatches << 'environment ID' unless @identity.fetch('environmentId') == expected.environment_id
+          raise CheckFailure.new('runtime-identity', "Runtime identity mismatch: #{mismatches.join(', ')}") unless
+            mismatches.empty?
+
+          'Runtime application, source version, and environment identity match the external witness.'
+        end
+
+        def validate_capabilities
+          require_capabilities!
+          ProtocolV3.validate_capabilities!(@capabilities_document, required: @required_capabilities)
+          discovery = Discovery.build(@capabilities_document)
+          validate_discovery!(discovery)
+          @runtime_data = @capabilities_document.fetch('data')
+          @discovery = discovery
+          'Capability advertisement and every product-owned JSON Schema are valid and bounded.'
+        rescue ProtocolV3::Error, Discovery::Error => e
+          raise CheckFailure.new('capability-advertisement', e.message)
+        end
+
+        def validate_discovery!(discovery)
+          @discovery_validator&.call(discovery)
+        rescue StandardError => e
+          raise DiscoveryRejected, e.message
+        end
+
+        def validate_unsupported_version
+          error_exchange(
+            :get,
+            ProtocolV3::PATHS.fetch(:capabilities),
+            code: 'protocol-version-unsupported',
+            version: '2',
+            max_response_bytes: negotiated_max_response_bytes
+          )
+          'A non-v3 request fails explicitly with HTTP 409 and protocol-version-unsupported.'
+        end
+
+        def validate_malformed_json
+          error_exchange(
+            :post,
+            ProtocolV3::PATHS.fetch(:states),
+            code: 'invalid-request',
+            raw_body: '{',
+            max_response_bytes: negotiated_max_response_bytes
+          )
+          'Malformed JSON fails with a bounded structured invalid-request response.'
+        end
+
+        def validate_request_media_type
+          body = request_envelope(
+            next_identifier,
+            data: {
+              'capability' => { 'id' => 'overseer-conformance-unknown', 'version' => '1' },
+              'input' => {}
+            }
+          )
+          error_exchange(
+            :post,
+            ProtocolV3::PATHS.fetch(:states),
+            code: 'invalid-request',
+            body:,
+            content_type: 'text/plain',
+            max_response_bytes: negotiated_max_response_bytes
+          )
+          'A non-JSON request media type fails with a bounded structured invalid-request response.'
+        end
+
+        def validate_unknown_capability(path_key, kind)
+          path = ProtocolV3::PATHS.fetch(path_key)
+          body = request_envelope(
+            next_identifier,
+            data: {
+              'capability' => { 'id' => 'overseer-conformance-unknown', 'version' => '1' },
+              'input' => {}
+            }
+          )
+          ProtocolV3.validate_document!(kind, body)
+          error_exchange(
+            :post,
+            path,
+            code: 'capability-not-found',
+            body:,
+            max_response_bytes: negotiated_max_response_bytes
+          )
+          family = path_key == :states ? 'arrangement' : 'observation'
+          "An unknown #{family} capability fails with a structured capability-not-found response."
+        rescue ProtocolV3::Error => e
+          raise CheckFailure.new('request-envelope', e.message)
+        end
+
+        def validate_schema_invalid_request
+          unless @runtime_data
+            return 'No arrangement is advertised; product-payload schema validation is not applicable.'
+          end
+
+          capability = @runtime_data.fetch('states').first
+          return 'No arrangement is advertised; product-payload schema validation is not applicable.' unless capability
+
+          begin
+            ProtocolV3.validate_value!(capability.fetch('inputSchema'), {}, label: 'conformance invalid input')
+            return 'The first arrangement accepts an empty object; its schema has no safe generic negative value.'
+          rescue ProtocolV3::Error
+            # The request below is deliberately known to be invalid against the advertised schema.
+          end
+          reference = capability.slice('id', 'version')
+          body = request_envelope(next_identifier, data: { 'capability' => reference, 'input' => {} })
+          ProtocolV3.validate_document!(:state_request, body)
+          error_exchange(
+            :post,
+            ProtocolV3::PATHS.fetch(:states),
+            code: 'schema-validation-failed',
+            body:,
+            max_response_bytes: negotiated_max_response_bytes
+          )
+          'A payload known to violate an advertised arrangement schema fails with schema-validation-failed.'
+        rescue ProtocolV3::Error => e
+          raise CheckFailure.new('request-envelope', e.message)
+        end
+
+        # rubocop:disable Metrics/CyclomaticComplexity
+        def perform_reset
+          require_runtime_data!
+          strategy = @runtime_data.dig('reset', 'strategy')
+          if strategy == 'in-process'
+            document = successful_exchange(
+              :put,
+              ProtocolV3::PATHS.fetch(:reset),
+              kind: :reset,
+              max_response_bytes: negotiated_max_response_bytes,
+              idempotency_key: "reset-#{next_identifier}"
+            )
+            data = document.fetch('data')
+            unless data['strategy'] == 'in-process' && data['sinksCleared'] == true
+              raise CheckFailure.new('reset', 'Reset response contradicts the advertised in-process strategy')
+            end
+
+            return 'In-process reset completed and reported cleared fake sinks.'
+          end
+
+          error_exchange(
+            :put,
+            ProtocolV3::PATHS.fetch(:reset),
+            code: 'reset-requires-restart',
+            max_response_bytes: negotiated_max_response_bytes,
+            idempotency_key: "reset-#{next_identifier}"
+          )
+          unless @restart
+            raise CheckFailure.new('runtime-lifecycle',
+                                   'Runtime-restart reset requires a managed runtime owner')
+          end
+
+          replacement = @restart.call
+          apply_replacement!(replacement)
+          document = successful_exchange(
+            :get,
+            ProtocolV3::PATHS.fetch(:capabilities),
+            kind: :capabilities,
+            max_response_bytes: INITIAL_MAX_RESPONSE_BYTES
+          )
+          ProtocolV3.validate_capabilities!(document, required: @required_capabilities)
+          replacement_discovery = Discovery.build(document)
+          unless replacement_discovery.digest == @discovery.digest
+            raise CheckFailure.new('stable-discovery', 'Replacement runtime changed its stable discovery digest')
+          end
+          unless document.dig('data', 'reset', 'strategy') == 'runtime-restart'
+            raise CheckFailure.new('reset', 'Replacement runtime changed its advertised reset strategy')
+          end
+
+          @capabilities_document = document
+          @runtime_data = document.fetch('data')
+          validate_runtime_identity
+          'Runtime-restart reset recreated, re-witnessed, and renegotiated the isolated runtime.'
+        rescue ProtocolV3::Error, Discovery::Error => e
+          raise CheckFailure.new('reset', e.message)
+        rescue CheckFailure
+          raise
+        rescue StandardError => e
+          raise CheckFailure.new('runtime-lifecycle', "Managed runtime restart failed (#{e.class.name})")
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity
+
+        def validate_error_safety
+          raise CheckFailure.new('error-safety', 'No negative protocol response was available for safety validation') if
+            @negative_responses.empty?
+          unless @negative_responses.all? { |item| item.fetch(:structured) && !item.fetch(:raw_exception) }
+            raise CheckFailure.new('error-safety', 'A negative response exposed an invalid or framework-specific body')
+          end
+
+          'Every exercised error was schema-valid, bounded, sanitized, and free of raw framework exception content.'
+        end
+
+        def validate_response_contract
+          unless @response_contract_safe
+            raise CheckFailure.new('response-contract',
+                                   'One or more responses violated media-type, size, timeout, or envelope rules')
+          end
+
+          'All exercised responses used application/json, stayed within byte/time bounds, and matched v3 envelopes.'
+        end
+
+        def run_capability_cases
+          @cases.each do |scenario|
+            scenario_id = scenario.fetch('id')
+            unless @runtime_data && @reset_ready
+              add_check(
+                "scenario.#{scenario_id}.prerequisite",
+                'failed',
+                'reset',
+                'Scenario was not executed because discovery, capability validation, or reset failed.',
+                scenario_id:
+              )
+              next
+            end
+
+            run_capability_case(scenario)
+          end
+        end
+
+        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        def run_capability_case(scenario)
+          scenario_id = scenario.fetch('id')
+          if scenario.fetch('resetBeforeCase')
+            reset_ok = check("scenario.#{scenario_id}.reset", scenario_id:) { perform_reset }
+            return unless reset_ok
+          end
+
+          arrangement = nil
+          arranged = check("scenario.#{scenario_id}.arrangement", scenario_id:) do
+            arrangement = run_arrangement(scenario.fetch('arrangement'))
+            'Arrangement matched its declared protocol-level expectation.'
+          end
+          return unless arranged && arrangement
+          return unless scenario['observation']
+
+          observation = nil
+          observed = check("scenario.#{scenario_id}.observation", scenario_id:) do
+            observation = run_observation(scenario.fetch('observation'), arrangement)
+            'Observation matched its declared protocol-level expectation and advertised result schema.'
+          end
+          return unless observed && observation && scenario['expectedObservation']
+
+          check("scenario.#{scenario_id}.assertion", scenario_id:) do
+            validate_observation_assertion!(observation, scenario.fetch('expectedObservation'))
+            'Observation result matched the trusted declarative assertion.'
+          end
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+        def run_arrangement(operation)
+          reference = operation.fetch('capability')
+          input = operation.fetch('input')
+          expected = operation.fetch('expected')
+          capability = capability_for('states', reference)
+          if capability.nil? && expected.fetch('status') == 'success'
+            raise CheckFailure.new('unknown-capability', 'Trusted arrangement capability was not advertised')
+          end
+
+          validate_case_input!(capability, input, expected:, label: 'arrangement')
+          body = request_envelope(next_identifier, data: { 'capability' => reference, 'input' => input })
+          if expected.fetch('status') == 'error'
+            error_exchange(
+              :post,
+              ProtocolV3::PATHS.fetch(:states),
+              code: expected.fetch('errorCode'),
+              body:,
+              max_response_bytes: negotiated_max_response_bytes
+            )
+            return nil
+          end
+
+          document = successful_exchange(
+            :post,
+            ProtocolV3::PATHS.fetch(:states),
+            kind: :state_response,
+            body:,
+            max_response_bytes: negotiated_max_response_bytes,
+            idempotency_key: arrangement_idempotency_key(capability)
+          )
+          data = document.fetch('data')
+          validate_returned_capability!(data, reference, 'arrangement')
+          ProtocolV3.validate_value!(capability.fetch('outputSchema'), data.fetch('output'),
+                                     label: 'arrangement output')
+          data
+        rescue ProtocolV3::Error => e
+          raise CheckFailure.new('capability-schema', e.message)
+        end
+
+        def run_observation(operation, arrangement)
+          reference = operation.fetch('capability')
+          expected = operation.fetch('expected')
+          input = copy_json(operation.fetch('input'))
+          operation.fetch('inputFromArrangementIdentifiers', {}).each do |input_name, identifier_name|
+            identifiers = arrangement.fetch('identifiers')
+            unless identifiers.key?(identifier_name)
+              raise CheckFailure.new('case-binding', "Arrangement identifier #{identifier_name} is unavailable")
+            end
+
+            input[input_name] = identifiers.fetch(identifier_name)
+          end
+          capability = capability_for('probes', reference)
+          if capability.nil? && expected.fetch('status') == 'success'
+            raise CheckFailure.new('unknown-capability', 'Trusted observation capability was not advertised')
+          end
+
+          validate_case_input!(capability, input, expected:, label: 'observation')
+          body = request_envelope(next_identifier, data: { 'capability' => reference, 'input' => input })
+          if expected.fetch('status') == 'error'
+            error_exchange(
+              :post,
+              ProtocolV3::PATHS.fetch(:probes),
+              code: expected.fetch('errorCode'),
+              body:,
+              max_response_bytes: negotiated_max_response_bytes
+            )
+            return nil
+          end
+
+          document = successful_exchange(
+            :post,
+            ProtocolV3::PATHS.fetch(:probes),
+            kind: :probe_response,
+            body:,
+            max_response_bytes: negotiated_max_response_bytes
+          )
+          data = document.fetch('data')
+          validate_returned_capability!(data, reference, 'observation')
+          ProtocolV3.validate_value!(
+            capability.fetch('outputSchema'),
+            data.fetch('projection'),
+            label: 'observation result'
+          )
+          data.fetch('projection')
+        rescue ProtocolV3::Error => e
+          raise CheckFailure.new('capability-schema', e.message)
+        end
+
+        def validate_case_input!(capability, input, expected:, label:)
+          return unless capability
+
+          valid = JSONSchemer.schema(capability.fetch('inputSchema')).valid?(input)
+          if expected.fetch('status') == 'success' && !valid
+            raise CheckFailure.new('case-definition', "Trusted #{label} input violates its advertised schema")
+          end
+          return unless expected['errorCode'] == 'schema-validation-failed' && valid
+
+          raise CheckFailure.new('case-definition', "Trusted #{label} negative input is valid against its schema")
+        end
+
+        def validate_observation_assertion!(value, assertion)
+          if assertion.key?('subset')
+            result = JsonSubsetMatcher.call(expected: assertion.fetch('subset'), actual: value)
+            return if result.matched?
+
+            raise CheckFailure.new('scenario-assertion', 'Observation result does not contain the expected JSON subset')
+          end
+          errors = JSONSchemer.schema(assertion.fetch('schema')).validate(value).take(3)
+          return if errors.empty?
+
+          raise CheckFailure.new('scenario-assertion', 'Observation result does not satisfy the expected JSON Schema')
+        end
+
+        # rubocop:disable Metrics/ParameterLists
+        def successful_exchange(method, path, kind:, max_response_bytes:, body: nil, idempotency_key: nil)
+          _response, document = exchange(
+            method,
+            path,
+            expected_status: 200,
+            success_kind: kind,
+            max_response_bytes:,
+            body:,
+            idempotency_key:
+          )
+          document
+        end
+
+        def error_exchange(
+          method,
+          path,
+          code:,
+          max_response_bytes:,
+          version: ProtocolV3::VERSION,
+          body: nil,
+          raw_body: nil,
+          content_type: nil,
+          idempotency_key: nil
+        )
+          status = ERROR_STATUSES.fetch(code)
+          _response, document = exchange(
+            method,
+            path,
+            expected_status: status,
+            success_kind: nil,
+            expected_error_code: code,
+            max_response_bytes:,
+            version:,
+            body:,
+            raw_body:,
+            content_type:,
+            idempotency_key:
+          )
+          document
+        end
+        # rubocop:enable Metrics/ParameterLists
+
+        # HTTP conformance deliberately evaluates the whole bounded response contract in one exchange.
+        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
+        def exchange(
+          method,
+          path,
+          expected_status:,
+          success_kind:,
+          max_response_bytes:,
+          expected_error_code: nil,
+          version: ProtocolV3::VERSION,
+          body: nil,
+          raw_body: nil,
+          content_type: nil,
+          idempotency_key: nil
+        )
+          correlation_id = body&.dig('meta', 'correlationId') || next_identifier
+          encoded = raw_body || (JSON.generate(body) if body)
+          response = @transport.request(
+            method:,
+            path:,
+            headers: request_headers(
+              correlation_id:,
+              version:,
+              content: !encoded.nil?,
+              content_type:,
+              idempotency_key:
+            ),
+            body: encoded,
+            max_response_bytes:
+          )
+          raw_exception = response.body.match?(RAW_EXCEPTION_PATTERN)
+          content_type = header_value(response.headers, 'Content-Type')
+          content_type = content_type.split(';', 2).first.strip.downcase if content_type
+          raise CheckFailure.new('response-media-type', 'Response media type must be application/json') unless
+            content_type == 'application/json'
+
+          document = JSON.parse(response.body)
+          ProtocolV3.validate_response_headers!(response.headers, correlation_id:)
+          kind = response.status.between?(200, 299) ? success_kind : :error
+          raise CheckFailure.new('http-status', "Unexpected successful HTTP #{response.status}") unless kind
+
+          ProtocolV3.validate_document!(kind, document)
+          ProtocolV3.validate_meta!(document, run_id: @run_id, correlation_id:)
+          if response.status != expected_status
+            raise CheckFailure.new(
+              'http-status',
+              "Expected HTTP #{expected_status}; received HTTP #{response.status}"
+            )
+          end
+          if expected_error_code && document.dig('errors', 0, 'code') != expected_error_code
+            raise CheckFailure.new(
+              'error-code',
+              "Expected #{expected_error_code}; received #{document.dig('errors', 0, 'code') || 'no error code'}"
+            )
+          end
+          if response.status >= 400
+            @negative_responses << { structured: true, raw_exception: }
+          elsif raw_exception
+            raise CheckFailure.new('error-safety', 'Successful response contains framework exception content')
+          end
+          [response, document]
+        rescue HTTPTransport::TimeoutError => e
+          @response_contract_safe = false
+          raise CheckFailure.new('timeout', e.message)
+        rescue HTTPTransport::ResponseTooLarge => e
+          @response_contract_safe = false
+          raise CheckFailure.new('response-size', e.message)
+        rescue HTTPTransport::Error => e
+          @response_contract_safe = false
+          raise CheckFailure.new('transport', e.message)
+        rescue JSON::ParserError
+          @response_contract_safe = false
+          raise CheckFailure.new('response-json', 'Response body is not valid JSON')
+        rescue ProtocolV3::Error => e
+          @response_contract_safe = false
+          raise CheckFailure.new('response-envelope', e.message)
+        rescue CheckFailure => e
+          @response_contract_safe = false if %w[response-media-type response-envelope
+                                                response-json].include?(e.category)
+          raise
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity
+
+        def request_headers(correlation_id:, version:, content:, content_type: nil, idempotency_key: nil)
+          headers = ProtocolV3::HEADERS
+          {
+            'Accept' => 'application/json',
+            **(content ? { 'Content-Type' => content_type || 'application/json' } : {}),
+            headers.fetch(:version) => version,
+            headers.fetch(:run_id) => @run_id,
+            headers.fetch(:correlation_id) => correlation_id,
+            **(idempotency_key ? { headers.fetch(:idempotency_key) => idempotency_key } : {})
+          }
+        end
+
+        def request_envelope(correlation_id, data:)
+          {
+            'protocol' => { 'name' => ProtocolV3::NAME, 'version' => ProtocolV3::VERSION },
+            'meta' => { 'runId' => @run_id, 'correlationId' => correlation_id },
+            'data' => data
+          }
+        end
+
+        def apply_replacement!(replacement)
+          unless replacement.is_a?(Hash) && replacement[:origin] && replacement[:expected_identity]
+            raise CheckFailure.new('runtime-lifecycle', 'Managed runtime restart returned no replacement witness')
+          end
+
+          @origin = HTTPTransport.origin!(replacement.fetch(:origin))
+          @expected_identity = replacement.fetch(:expected_identity)
+          @transport = HTTPTransport.new(origin: @origin, base_path: @base_path) unless @dependencies.transport
+        end
+
+        def capability_for(family, reference)
+          @runtime_data.fetch(family).find do |item|
+            item['id'] == reference.fetch('id') && item['version'] == reference.fetch('version')
+          end
+        end
+
+        def arrangement_idempotency_key(capability)
+          return unless capability&.fetch('idempotency') == 'required'
+
+          "case-#{next_identifier}"
+        end
+
+        def validate_returned_capability!(data, expected, label)
+          return if data.fetch('capability') == expected
+
+          raise CheckFailure.new('capability-mismatch', "#{label.capitalize} returned a different capability")
+        end
+
+        def report_capabilities
+          return unless @runtime_data
+
+          {
+            'resetStrategy' => @runtime_data.dig('reset', 'strategy'),
+            'arrangements' => @runtime_data.fetch('states').map do |item|
+              report_capability(item, input: 'inputSchema', output: 'outputSchema')
+            end,
+            'observations' => @runtime_data.fetch('probes').map do |item|
+              report_capability(item, input: 'inputSchema', output: 'outputSchema')
+            end,
+            'sinks' => @runtime_data.fetch('sinks').map do |item|
+              report_capability(item, input: 'queryInputSchema', output: 'recordSchema')
+            end
+          }
+        end
+
+        def report_capability(item, input:, output:)
+          {
+            'id' => item.fetch('id'),
+            'version' => item.fetch('version'),
+            'inputSchema' => Redaction.body(item.fetch(input), @redaction_policy),
+            'outputSchema' => Redaction.body(item.fetch(output), @redaction_policy)
+          }
+        end
+
+        def negotiated_max_response_bytes
+          @runtime_data&.dig('limits', 'maxResponseBytes') || INITIAL_MAX_RESPONSE_BYTES
+        end
+
+        def require_capabilities!
+          return if @capabilities_document
+
+          raise CheckFailure.new('prerequisite', 'Capability discovery did not complete')
+        end
+
+        def require_runtime_data!
+          return if @runtime_data
+
+          raise CheckFailure.new('prerequisite', 'Capability validation did not complete')
+        end
+
+        def check(id, scenario_id: nil)
+          raise CheckFailure.new('cancelled', 'Conformance was cancelled') if @cancellation&.call
+
+          @progress&.call(phase: conformance_phase(id))
+          message = yield
+          add_check(id, 'passed', 'protocol', message, scenario_id:)
+          @progress&.call(phase: conformance_phase(id))
+          true
+        rescue DiscoveryRejected
+          raise
+        rescue CheckFailure => e
+          add_check(id, 'failed', e.category, e.message, scenario_id:)
+          false
+        rescue StandardError => e
+          add_check(id, 'failed', 'runner', "Conformance check failed safely (#{e.class.name})", scenario_id:)
+          false
+        end
+
+        def add_check(id, status, category, message, scenario_id: nil)
+          @checks << {
+            'id' => id,
+            'status' => status,
+            'category' => category,
+            'message' => safe_message(message),
+            **(scenario_id ? { 'scenarioId' => scenario_id } : {})
+          }
+        end
+
+        def conformance_phase(id)
+          %w[core.discovery core.runtime-identity core.capabilities].include?(id) ? 'discovery' : 'conformance'
+        end
+
+        def safe_message(value)
+          Redaction.string(value.to_s).byteslice(0, 500).to_s.scrub
+        end
+
+        def next_identifier
+          ProtocolV3.identifier!(@dependencies.random.uuid, label: 'conformance correlation ID')
+        end
+
+        def timestamp
+          @dependencies.clock.now.utc.iso8601(3)
+        end
+
+        def copy_json(value)
+          JSON.parse(JSON.generate(value))
+        end
+
+        def header_value(headers, name)
+          headers.each do |candidate, value|
+            return Array(value).first if candidate.to_s.casecmp?(name)
+          end
+          nil
+        end
+
+        class CheckFailure < StandardError
+          attr_reader :category
+
+          def initialize(category, message)
+            @category = category
+            super(message)
+          end
+        end
+      end
+    end
+  end
+end