overseer-testing-protocol 0.1.0

data/lib/overseer/testing_control/discovery.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'json'
+
+module Overseer
+  module TestingControl
+    # A stable semantic projection of a fully validated v3 capabilities document.
+    module Discovery
+      MAX_CAPABILITIES_PER_FAMILY = 128
+      MAX_BYTES = 256 * 1024
+      Result = Data.define(:snapshot, :digest, :bytes)
+
+      class Error < StandardError; end
+
+      module_function
+
+      def build(document)
+        data = document.fetch('data')
+        %w[states probes sinks].each do |family|
+          raise Error, "Testing-control discovery exceeds #{family} capability boundary" if
+            data.fetch(family).length > MAX_CAPABILITIES_PER_FAMILY
+        end
+
+        snapshot = {
+          'discoveryVersion' => '1',
+          'protocol' => document.fetch('protocol').slice('name', 'version'),
+          'application' => data.fetch('application').slice('id', 'version'),
+          'environment' => data.fetch('environment').slice('kind', 'isolated'),
+          'features' => data.fetch('features').sort,
+          'limits' => data.fetch('limits'),
+          'safety' => data.fetch('safety'),
+          'reset' => data.fetch('reset'),
+          'states' => capabilities(
+            data.fetch('states'),
+            %w[id version title description idempotency inputSchema outputSchema]
+          ),
+          'probes' => capabilities(
+            data.fetch('probes'),
+            %w[id version title description readOnly inputSchema outputSchema]
+          ),
+          'sinks' => capabilities(
+            data.fetch('sinks'),
+            %w[id version title description effectKinds outcomes queryInputSchema recordSchema]
+          )
+        }
+        source = canonical_json(snapshot)
+        raise Error, 'Stable testing-control discovery exceeds its artifact boundary' if source.bytesize > MAX_BYTES
+
+        Result.new(snapshot:, digest: Digest::SHA256.hexdigest(source), bytes: source.bytesize)
+      rescue KeyError => e
+        raise Error, "Validated testing-control discovery is incomplete: #{e.message}"
+      end
+
+      def canonical_json(value)
+        normalized = case value
+                     when Hash
+                       value.keys.sort.to_h { |key| [key, JSON.parse(canonical_json(value.fetch(key)))] }
+                     when Array
+                       value.map { |item| JSON.parse(canonical_json(item)) }
+                     else
+                       value
+                     end
+        JSON.generate(normalized)
+      end
+
+      def capabilities(items, keys)
+        selected_items = items.map do |item|
+          selected = item.slice(*keys)
+          selected['effectKinds'] = selected.fetch('effectKinds').sort if selected.key?('effectKinds')
+          selected['outcomes'] = selected.fetch('outcomes').sort if selected.key?('outcomes')
+          selected
+        end
+        selected_items.sort_by { |item| [item.fetch('id'), item.fetch('version')] }
+      end
+      private_class_method :capabilities
+    end
+  end
+end

data/lib/overseer/testing_control/json_subset_matcher.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Overseer
+  module TestingControl
+    module JsonSubsetMatcher
+      Result = Data.define(:matched, :mismatch) do
+        def matched?
+          matched
+        end
+
+        def to_h
+          {
+            'matched' => matched,
+            **(mismatch ? { 'mismatch' => mismatch } : {})
+          }
+        end
+      end
+
+      class << self
+        def call(expected:, actual:)
+          mismatch = compare(expected, actual, '')
+          Result.new(matched: mismatch.nil?, mismatch: mismatch&.freeze)
+        end
+
+        private
+
+        def compare(expected, actual, path)
+          return compare_object(expected, actual, path) if expected.is_a?(Hash)
+          return compare_array(expected, actual, path) if expected.is_a?(Array)
+          return if expected == actual
+
+          mismatch(
+            reason: json_kind(expected) == json_kind(actual) ? 'value-mismatch' : 'type-mismatch',
+            path:,
+            expected_kind: json_kind(expected),
+            actual_kind: json_kind(actual)
+          )
+        end
+
+        def compare_object(expected, actual, path)
+          return type_mismatch(expected, actual, path) unless actual.is_a?(Hash)
+
+          expected.keys.sort.each do |key|
+            child_path = pointer(path, key)
+            unless actual.key?(key)
+              return mismatch(
+                reason: 'missing-key',
+                path: child_path,
+                expected_kind: json_kind(expected.fetch(key)),
+                actual_kind: 'missing'
+              )
+            end
+
+            finding = compare(expected.fetch(key), actual.fetch(key), child_path)
+            return finding if finding
+          end
+          nil
+        end
+
+        def compare_array(expected, actual, path)
+          return type_mismatch(expected, actual, path) unless actual.is_a?(Array)
+          unless expected.length == actual.length
+            return mismatch(
+              reason: 'array-length-mismatch',
+              path:,
+              expected_kind: 'array',
+              actual_kind: 'array',
+              lengths: {
+                'expectedLength' => expected.length,
+                'actualLength' => actual.length
+              }
+            )
+          end
+
+          expected.each_index do |index|
+            finding = compare(expected.fetch(index), actual.fetch(index), pointer(path, index))
+            return finding if finding
+          end
+          nil
+        end
+
+        def type_mismatch(expected, actual, path)
+          mismatch(
+            reason: 'type-mismatch',
+            path:,
+            expected_kind: json_kind(expected),
+            actual_kind: json_kind(actual)
+          )
+        end
+
+        def mismatch(reason:, path:, expected_kind:, actual_kind:, lengths: {})
+          {
+            'reason' => reason,
+            'path' => path,
+            'expectedKind' => expected_kind,
+            'actualKind' => actual_kind,
+            **lengths
+          }
+        end
+
+        def pointer(path, token)
+          escaped = token.to_s.gsub('~', '~0').gsub('/', '~1')
+          "#{path}/#{escaped}"
+        end
+
+        def json_kind(value)
+          case value
+          when Hash then 'object'
+          when Array then 'array'
+          when String then 'string'
+          when Numeric then 'number'
+          when TrueClass, FalseClass then 'boolean'
+          when NilClass then 'null'
+          else 'unknown'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/overseer/testing_control/protocol_v3.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+require 'json_schemer'
+
+module Overseer
+  module TestingControl
+    module ProtocolV3
+      NAME = 'overseer-testing-control'
+      VERSION = '3'
+      PACKAGE_VERSION = '0.1.0'
+      SCHEMA_SHA256 = '9e0591133fd38c15e96383eae11b202cb27ea8f7ddc98d2a64f2abff326cb924'
+      CAPABILITY_PATTERN = /\A[a-z][a-z0-9-]*(?:\.[a-z][a-z0-9-]*)*\z/
+      CAPABILITY_VERSION_PATTERN = /\A[1-9][0-9]{0,31}\z/
+      JSON_SCHEMA_DIALECT = 'https://json-schema.org/draft/2020-12/schema'
+      IDENTIFIER_PATTERN = /\A[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\z/
+      REQUIRED_FEATURES = %w[
+        correlation-propagation
+        deterministic-replay
+        reset
+        schema-validation
+        sink-clearing
+      ].freeze
+      HEADERS = {
+        version: 'Overseer-Testing-Control-Version',
+        run_id: 'Overseer-Run-Id',
+        correlation_id: 'Overseer-Correlation-Id',
+        step_id: 'Overseer-Step-Id',
+        idempotency_key: 'Overseer-Idempotency-Key'
+      }.freeze
+      PATHS = {
+        capabilities: '/capabilities',
+        reset: '/reset',
+        states: '/states',
+        probes: '/probes',
+        sink_query: '/sinks/query'
+      }.freeze
+      SCHEMA_PATH = File.expand_path(
+        '../../../protocol/testing-control/v3/schema.json',
+        __dir__
+      )
+      OPENAPI_PATH = File.join(File.dirname(SCHEMA_PATH), 'openapi.yaml')
+      DOCUMENT_DEFINITIONS = {
+        capabilities: 'capabilitiesResponse',
+        reset: 'resetResponse',
+        state_request: 'stateRequest',
+        state_response: 'stateResponse',
+        probe_request: 'probeRequest',
+        probe_response: 'probeResponse',
+        sink_query_request: 'sinkQueryRequest',
+        sink_query_response: 'sinkQueryResponse',
+        error: 'errorResponse'
+      }.freeze
+
+      module_function
+
+      def protocol
+        { 'name' => NAME, 'version' => VERSION }
+      end
+
+      def meta(run_id:, correlation_id:, step_id: nil)
+        {
+          'runId' => run_id,
+          'correlationId' => correlation_id,
+          **(step_id ? { 'stepId' => step_id } : {})
+        }
+      end
+
+      def envelope(data:, **identity)
+        { 'protocol' => protocol, 'meta' => meta(**identity), 'data' => data }
+      end
+
+      def error_envelope(errors:, **identity)
+        { 'protocol' => protocol, 'meta' => meta(**identity), 'errors' => errors }
+      end
+
+
+      def validate_schema!(schema, label:, max_bytes:)
+        bytes = JSON.generate(schema).bytesize
+        raise ValidationError, "#{label} schema exceeds #{max_bytes} bytes" if bytes > max_bytes
+        unless schema.is_a?(Hash) && schema['$schema'] == JSON_SCHEMA_DIALECT && schema['type'] == 'object'
+          raise ValidationError, "#{label} must be a Draft 2020-12 schema with an object root"
+        end
+        raise ValidationError, "#{label} schema contains a non-local reference" unless only_local_references?(schema)
+
+        errors = JSONSchemer.validate_schema(schema).take(3)
+        raise ValidationError, "#{label} schema is invalid: #{error_details(errors)}" unless errors.empty?
+
+        JSONSchemer.schema(schema).validate({}).to_a
+        schema
+      rescue JSONSchemer::UnknownRef, JSONSchemer::InvalidRefResolution, JSONSchemer::InvalidRefPointer => e
+        raise ValidationError, "#{label} schema cannot be compiled: #{e.message}"
+      end
+
+
+      def capability_id!(value, label:)
+        return value if value.is_a?(String) && value.bytesize <= 96 && value.match?(CAPABILITY_PATTERN)
+
+        raise ValidationError, "#{label} must be a valid capability identifier"
+      end
+
+      def capability_version!(value, label:)
+        return value if value.is_a?(String) && value.match?(CAPABILITY_VERSION_PATTERN)
+
+        raise ValidationError, "#{label} must be a positive decimal version"
+      end
+
+      def application_version!(value)
+        return value if value.is_a?(String) && value.bytesize.between?(1, 128)
+
+        raise ValidationError, 'application source identity must be between 1 and 128 bytes'
+      end
+
+      def timestamp
+        Time.now.utc.iso8601(3)
+      end
+
+      def stringify(value)
+        case value
+        when Hash
+          value.to_h { |key, child| [key.to_s, stringify(child)] }
+        when Array
+          value.map { |child| stringify(child) }
+        else
+          value
+        end
+      end
+
+
+      def validate_document!(kind, value)
+        errors = document_schemer(kind).validate(value).take(3)
+        return value if errors.empty?
+
+        raise Error, "Invalid protocol v3 #{kind} response: #{error_details(errors)}"
+      end
+
+      def validate_response_headers!(headers, correlation_id:)
+        version = header_value(headers, HEADERS.fetch(:version))
+        correlation = header_value(headers, HEADERS.fetch(:correlation_id))
+        raise Error, 'Protocol v3 response version header must equal 3' unless version == VERSION
+        raise Error, 'Protocol v3 response correlation header is invalid' unless
+          correlation&.match?(IDENTIFIER_PATTERN)
+        raise Error, 'Protocol v3 response correlation does not match the request' unless
+          correlation == correlation_id
+      end
+
+      def validate_meta!(document, run_id:, correlation_id:, step_id: nil)
+        meta = document.fetch('meta')
+        return if meta['runId'] == run_id &&
+                  meta['correlationId'] == correlation_id &&
+                  meta['stepId'] == step_id &&
+                  meta.key?('stepId') == !step_id.nil?
+
+        raise Error, 'Protocol v3 response metadata does not match the request identity'
+      end
+
+      def validate_capabilities!(document, required:)
+        validate_document!(:capabilities, document)
+        data = document.fetch('data')
+        validate_required_features!(data.fetch('features'))
+        validate_capability_family!('state', data.fetch('states'), data.dig('limits', 'maxSchemaBytes')) do |item|
+          [item.fetch('inputSchema'), item.fetch('outputSchema')]
+        end
+        validate_capability_family!('probe', data.fetch('probes'), data.dig('limits', 'maxSchemaBytes')) do |item|
+          [item.fetch('inputSchema'), item.fetch('outputSchema')]
+        end
+        validate_capability_family!('sink', data.fetch('sinks'), data.dig('limits', 'maxSchemaBytes')) do |item|
+          [item.fetch('queryInputSchema'), item.fetch('recordSchema')]
+        end
+        validate_required_capabilities!(data, required)
+        document
+      end
+
+      def validate_value!(schema, value, label:)
+        errors = JSONSchemer.schema(schema).validate(value).take(3)
+        return value if errors.empty?
+
+        raise Error, "#{label} does not satisfy its advertised schema: #{error_details(errors)}"
+      end
+
+      def identifier!(value, label:)
+        return value if value.is_a?(String) && value.match?(IDENTIFIER_PATTERN)
+
+        raise Error, "#{label} must be a valid opaque identifier"
+      end
+
+      def document_schemer(kind)
+        definition = DOCUMENT_DEFINITIONS.fetch(kind)
+        @document_schemers ||= {}
+        @document_schemers[definition] ||= root_schemer.ref("#/$defs/#{definition}")
+      end
+      private_class_method :document_schemer
+
+      def root_schemer
+        @root_schemer ||= JSONSchemer.schema(JSON.parse(File.read(SCHEMA_PATH, encoding: 'UTF-8')))
+      end
+      private_class_method :root_schemer
+
+      def validate_required_features!(features)
+        missing = REQUIRED_FEATURES - features
+        return if missing.empty?
+
+        raise Error, "Protocol v3 capabilities omit required features: #{missing.join(', ')}"
+      end
+      private_class_method :validate_required_features!
+
+      def validate_capability_family!(kind, capabilities, max_schema_bytes)
+        seen = {}
+        capabilities.each do |capability|
+          key = "#{capability.fetch('id')}@#{capability.fetch('version')}"
+          raise Error, "Duplicate #{kind} capability #{key}" if seen[key]
+
+          seen[key] = true
+          yield(capability).each do |schema|
+            validate_advertised_schema!(schema, label: "#{kind} #{key}", max_bytes: max_schema_bytes)
+          end
+        end
+      end
+      private_class_method :validate_capability_family!
+
+      def validate_advertised_schema!(schema, label:, max_bytes:)
+        bytes = JSON.generate(schema).bytesize
+        raise Error, "#{label} schema exceeds maxSchemaBytes #{max_bytes}" if bytes > max_bytes
+        raise Error, "#{label} schema contains a non-local reference" unless only_local_references?(schema)
+
+        errors = JSONSchemer.validate_schema(schema).take(3)
+        raise Error, "#{label} schema is invalid: #{error_details(errors)}" unless errors.empty?
+
+        compile_schema!(schema, label:)
+      end
+      private_class_method :validate_advertised_schema!
+
+      def only_local_references?(value)
+        references(value).all? { |reference| reference.is_a?(String) && reference.start_with?('#') }
+      end
+      private_class_method :only_local_references?
+
+      def references(value)
+        case value
+        when Hash
+          own = value.filter_map do |key, child|
+            child if %w[$ref $dynamicRef].include?(key)
+          end
+          own + value.values.flat_map { |child| references(child) }
+        when Array
+          value.flat_map { |child| references(child) }
+        else
+          []
+        end
+      end
+      private_class_method :references
+
+      def compile_schema!(schema, label:)
+        JSONSchemer.schema(schema).validate({}).to_a
+      rescue StandardError => e
+        raise Error, "#{label} schema cannot be compiled: #{e.message}"
+      end
+      private_class_method :compile_schema!
+
+      def validate_required_capabilities!(data, required)
+        {
+          'state' => ['states', required.fetch('states')],
+          'probe' => ['probes', required.fetch('probes')],
+          'sink' => ['sinks', required.fetch('sinks')]
+        }.each do |kind, (family, expected)|
+          advertised = data.fetch(family)
+          expected.each do |id, version|
+            next if advertised.any? do |capability|
+              capability['id'] == id && capability['version'] == version
+            end
+
+            raise Error, "Required testing #{kind} #{id}@#{version} is unavailable"
+          end
+        end
+      end
+      private_class_method :validate_required_capabilities!
+
+      def header_value(headers, name)
+        headers.each do |candidate, value|
+          return Array(value).first if candidate.to_s.casecmp?(name)
+        end
+        nil
+      end
+      private_class_method :header_value
+
+      def error_details(errors)
+        errors.map do |error|
+          "#{error.fetch('data_pointer', '/')} #{error.fetch('type')}"
+        end.join('; ')
+      end
+      private_class_method :error_details
+
+      class Error < StandardError; end
+      ValidationError = Error
+    end
+  end
+end

data/lib/overseer/testing_control/redaction.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Overseer
+  module TestingControl
+    module Redaction
+      REDACTED = '<redacted>'
+      REDACTED_EMAIL = '<redacted-email>'
+      JWT_PATTERN = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/
+      BEARER_PATTERN = %r{\bBearer\s+[A-Za-z0-9._~+/-]+=*}i
+      EMAIL_PATTERN = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i
+
+      Policy = Data.define(:header_names, :field_patterns, :max_body_bytes, :exact_values) do
+        def initialize(exact_values: [], **values)
+          safe = exact_values.select do |value|
+            value.is_a?(String) && !value.empty?
+          end
+          safe = safe.uniq.sort_by { |value| -value.bytesize }
+          super(**values, exact_values: safe.freeze)
+        end
+
+        def self.from_config(config, exact_values: [])
+          new(
+            header_names: config.fetch('headerNames').map(&:downcase).freeze,
+            field_patterns: config.fetch('fieldPatterns').map(&:downcase).freeze,
+            max_body_bytes: config.fetch('maxBodyBytes'),
+            exact_values:
+          )
+        end
+
+        def with_exact_values(values)
+          self.class.new(
+            header_names:,
+            field_patterns:,
+            max_body_bytes:,
+            exact_values: exact_values + Array(values)
+          )
+        end
+      end
+
+      module_function
+
+      def value(value, policy, key: '')
+        return REDACTED if sensitive_key?(key, policy)
+
+        case value
+        when String
+          string(value, policy)
+        when Array
+          value.map { |item| self.value(item, policy) }
+        when Hash
+          value.to_h do |entry_key, entry_value|
+            [safe_hash_key(entry_key, policy), self.value(entry_value, policy, key: entry_key.to_s)]
+          end
+        else
+          value
+        end
+      end
+
+      def string(value, policy = nil)
+        safe = value
+               .gsub(JWT_PATTERN, REDACTED)
+               .gsub(BEARER_PATTERN, "Bearer #{REDACTED}")
+               .gsub(EMAIL_PATTERN) do |email|
+                 email.downcase.end_with?('.invalid') ? email : REDACTED_EMAIL
+               end
+        Array(policy&.exact_values).reduce(safe) { |text, exact| text.gsub(exact, '<redacted-generated-secret>') }
+      end
+
+      def headers(headers, policy)
+        headers.to_h do |name, value|
+          normalized_name = name.to_s.downcase
+          normalized_value = Array(value).join(', ')
+          safe_name = string(normalized_name, policy)
+          safe_value = if policy.header_names.include?(normalized_name)
+                         REDACTED
+                       else
+                         string(normalized_value, policy)
+                       end
+          [safe_name, safe_value]
+        end
+      end
+
+      def body(value, policy)
+        safe = self.value(value, policy)
+        serialized = JSON.generate(safe)
+        return safe if serialized.bytesize <= policy.max_body_bytes
+
+        {
+          'truncated' => true,
+          'originalBytes' => serialized.bytesize,
+          'retainedBytes' => policy.max_body_bytes,
+          'preview' => string(serialized.byteslice(0, policy.max_body_bytes).to_s.scrub, policy)
+        }
+      end
+
+      def sensitive_key?(key, policy)
+        normalized = key.downcase
+        policy.field_patterns.each { |pattern| return true if normalized.include?(pattern) }
+        false
+      end
+      private_class_method :sensitive_key?
+
+      def safe_hash_key(key, policy)
+        key.is_a?(String) ? string(key, policy) : key
+      end
+      private_class_method :safe_hash_key
+    end
+  end
+end

data/lib/overseer/testing_protocol.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative 'testing_control/protocol_v3'
+require_relative 'testing_control/discovery'
+require_relative 'testing_control/redaction'
+require_relative 'testing_control/conformance/case_file'
+require_relative 'testing_control/conformance/http_transport'
+require_relative 'testing_control/conformance/report'
+require_relative 'testing_control/conformance/runner'