haveapi 0.30.0 → 0.30.2

data/spec/performance/comparator_spec.rb

@@ -0,0 +1,342 @@
+require 'spec_helper'
+require 'tmpdir'
+require 'open3'
+require_relative 'support/samples'
+
+describe HaveAPI::Performance::Comparator do
+  def compare_cli(samples, budgets)
+    Dir.mktmpdir('haveapi-comparator-') do |directory|
+      input = File.join(directory, 'samples.json')
+      policy = File.join(directory, 'budgets.json')
+      output = File.join(directory, 'result')
+      File.write(input, JSON.generate(samples))
+      File.write(policy, budgets)
+      _stdout, _stderr, status = Open3.capture3(RbConfig.ruby, File.expand_path('../../benchmarks/endpoints.rb', __dir__),
+                                                'compare', '--input', input, '--budgets', policy, '--output', output)
+      [status.exitstatus, JSON.parse(File.read(File.join(output, 'report.json')))]
+    end
+  end
+
+  def replace_raw_body(sample, body)
+    output = sample['outputs'].first
+    raw = sample['bodies'].delete(output['body_sha256'])
+    digest = Digest::SHA256.hexdigest(body)
+    sample['bodies'][digest] = raw.merge('body' => body)
+    output.merge!('body_sha256' => digest, 'body_bytes' => body.bytesize)
+  end
+
+  def rebind_source(samples, role, sha)
+    samples["#{role}_sha"] = sha
+    samples['sources'][role]['sha'] = sha
+    samples['observations'].each_value do |observation|
+      observation['source']['sha'] = sha if observation['source']['root'] == samples['sources'][role]['root']
+    end
+  end
+
+  it 'passes controlled AA traces through the actual CLI' do
+    samples, budgets = PerformanceSamples.fixture
+    code, report = compare_cli(samples, budgets)
+    expect(code).to eq(0)
+    expect(report['status']).to eq('pass')
+    expect(report['cases'].first['timing'].keys).to eq(%w[wall_seconds cpu_seconds])
+    expect(report['schema_version']).to eq(2)
+    expect(report['cases'].first['correctness']).to include('status' => 'pass', 'operations_per_sample' => 1)
+    expect(JSON.generate(report)).not_to include('canonical_body', 'body_sha256', 'self_pairs', 'observations', 'loaded_sources')
+  end
+
+  it 'rejects unsupported sample formats without converting historical evidence' do
+    samples, budgets = PerformanceSamples.fixture
+    samples['schema_version'] = 1
+    code, report = compare_cli(samples, budgets)
+    expect(code).to eq(3)
+    expect(report.dig('error', 'message')).to include('Unsupported samples schema; expected 2')
+  end
+
+  it 'blocks an agreed meaningful slowdown for either clock independently' do
+    %w[wall_seconds cpu_seconds].each do |metric|
+      samples, budgets = PerformanceSamples.fixture
+      samples['cases'].first['pairs'].each { |pair| pair['candidate'][metric] = 3.0 }
+      code, report = compare_cli(samples, budgets)
+      expect(code).to eq(1)
+      expect(report['cases'].first['timing'][metric]['regression_agreement']).to eq(5)
+    end
+  end
+
+  it 'retains noisy inconclusive diagnostics without failing readiness' do
+    samples, budgets = PerformanceSamples.fixture
+    samples['cases'].first['self_pairs'].first['candidate']['wall_seconds'] = 6.0
+    code, report = compare_cli(samples, budgets)
+    expect(code).to eq(0)
+    expect(report['status']).to eq('inconclusive')
+  end
+
+  it 'requires four of five quick agreements and six of seven weekly agreements' do
+    %w[quick weekly].each do |profile|
+      samples, budgets = PerformanceSamples.fixture(profile:)
+      agreement = HaveAPI::Performance::PROFILES.fetch(profile).fetch(:agreement)
+      samples['cases'].first['pairs'].first(agreement - 1).each { |pair| pair['candidate']['wall_seconds'] = 3.0 }
+      code, report = compare_cli(samples, budgets)
+      expect(code).to eq(0)
+      expect(report['status']).to eq('inconclusive')
+      samples['cases'].first['pairs'][agreement - 1]['candidate']['wall_seconds'] = 3.0
+      expect(compare_cli(samples, budgets).first).to eq(1)
+    end
+  end
+
+  it 'uses the stricter weekly budget on the same fifteen percent increase' do
+    %w[quick weekly].each do |profile|
+      samples, budgets = PerformanceSamples.fixture(profile:)
+      samples['cases'].first['pairs'].each { |pair| pair['candidate']['wall_seconds'] = 2.30 }
+      expect(compare_cli(samples, budgets).first).to eq(profile == 'quick' ? 0 : 1)
+    end
+  end
+
+  it 'detects a cheap-request twenty-five percent regression using the batch floor' do
+    samples, budgets = PerformanceSamples.fixture
+    item = samples['cases'].first
+    item['iterations'] = 25_000
+    (item['self_pairs'] + item['pairs']).each do |pair|
+      pair.each_value do |sample|
+        next unless sample.is_a?(Hash)
+
+        sample['operations'] = 25_000
+        sample['outputs'].first['count'] = 25_000
+        sample['counters']['operations'] = 25_000
+        sample['wall_seconds'] = sample['cpu_seconds'] = 0.00004
+      end
+    end
+    item['pairs'].each { |pair| pair['candidate']['wall_seconds'] = 0.00005 }
+    expect(compare_cli(samples, budgets).first).to eq(1)
+  end
+
+  it 'validates distinct examples and rejects malformed compact counts, identities and counter summaries' do
+    samples, json = PerformanceSamples.fixture
+    budget = JSON.parse(json)
+    spec = samples['cases'].first
+    sample = Marshal.load(Marshal.dump(spec['pairs'].first['candidate']))
+    sample['operations'] = 4
+    sample['outputs'].first['count'] = 4
+    sample['counters']['operations'] = 4
+    comparator = described_class.new(samples, budget, budget_sha256: samples['budget_sha256'])
+    body = sample['bodies'].values.first['body']
+    allow(HaveAPI::Performance::Identity).to receive(:parse).and_call_original
+    comparator.send(:validate_sample, sample, 4, budget['profiles']['quick'].first, spec)
+    expect(HaveAPI::Performance::Identity).to have_received(:parse).with(body).once
+    %w[http_status envelope_status body_bytes body_sha256 canonical_sha256 comparison counter missing zero negative float total counter_count duplicate unreferenced].each do |fault|
+      broken = Marshal.load(Marshal.dump(sample))
+      output = broken['outputs'].last
+      case fault
+      when 'http_status' then output[fault] = 403
+      when 'envelope_status' then output[fault] = false
+      when 'body_bytes' then output[fault] += 1
+      when 'comparison' then output[fault] = { 'kind' => 'own-webui-href', 'own_version' => '0.30.2' }
+      when 'counter' then broken['counters']['record']['authorizations'] = 0
+      when 'missing' then broken['counters'].delete('record')
+      when 'zero' then output['count'] = 0
+      when 'negative' then output['count'] = -1
+      when 'float' then output['count'] = 4.0
+      when 'total' then output['count'] = 3
+      when 'counter_count' then broken['counters']['operations'] = 4.0
+      when 'duplicate'
+        output['count'] = 2
+        broken['outputs'] << output.dup
+      when 'unreferenced'
+        other_body = '{"status":true}'
+        broken['bodies'][Digest::SHA256.hexdigest(other_body)] = { 'body' => other_body, 'canonical_body' => JSON.parse(other_body) }
+      else output[fault] = '0' * 64
+      end
+      expect { comparator.send(:validate_sample, broken, 4, budget['profiles']['quick'].first, spec) }.to raise_error(StandardError)
+    end
+    # A second sample must independently prove its retained body again.
+    comparator.send(:validate_sample, sample, 4, budget['profiles']['quick'].first, spec)
+    expect(HaveAPI::Performance::Identity).to have_received(:parse).with(body).at_least(:twice)
+  end
+
+  it 'rejects raw/canonical disagreement and duplicate JSON on a reused body and rejects counts above the shared cap' do
+    %w[canonical duplicate cap].each do |fault|
+      samples, json = PerformanceSamples.fixture
+      spec = samples['cases'].first
+      sample = spec['pairs'].first['candidate']
+      if fault == 'canonical'
+        sample['bodies'].values.first['canonical_body']['response']['zero'] = 1
+      elsif fault == 'duplicate'
+        body = sample['bodies'].values.first['body'].sub('"status":true', '"status":false,"status":true')
+        replace_raw_body(sample, body)
+      else
+        spec['iterations'] = 100_001
+      end
+      comparator = described_class.new(samples, JSON.parse(json), budget_sha256: samples['budget_sha256'])
+      expect { comparator.compare }.to raise_error(HaveAPI::Performance::ContractError)
+    end
+  end
+
+  it 'compares response occurrence totals across revisions without retaining request order' do
+    samples, json = PerformanceSamples.fixture
+    spec = samples['cases'].first
+    spec['iterations'] = 3
+    (spec['self_pairs'] + spec['pairs']).each do |pair|
+      %w[baseline candidate].each do |role|
+        sample = pair[role]
+        output = sample['outputs'].first
+        body = sample['bodies'].values.first['body'].sub('"zero":0', '"zero":1')
+        canonical = JSON.parse(body)
+        digest = Digest::SHA256.hexdigest(body)
+        sample['bodies'][digest] = { 'body' => body, 'canonical_body' => canonical }
+        sample['operations'] = 3
+        sample['outputs'] << output.merge('body_sha256' => digest, 'body_bytes' => body.bytesize,
+                                          'canonical_sha256' => HaveAPI::Performance::Identity.digest(canonical), 'count' => 2)
+        sample['counters']['operations'] = 3
+      end
+    end
+    comparator = described_class.new(samples, JSON.parse(json), budget_sha256: samples['budget_sha256'])
+    expect(comparator.compare['status']).to eq('pass')
+    spec['pairs'].last['candidate']['outputs'].reverse!
+    expect(comparator.compare['status']).to eq('pass')
+    spec['pairs'].last['candidate']['outputs'].each { |example| example['count'] = 3 - example['count'] }
+    expect { comparator.compare }.to raise_error(HaveAPI::Performance::ContractError, /Response identity/)
+  end
+
+  it 'rejects retained runaway request batches even if both sources are equally slow' do
+    samples, budgets = PerformanceSamples.fixture
+    item = samples['cases'].first
+    (item['self_pairs'] + item['pairs']).each do |pair|
+      %w[baseline candidate].each { |role| pair[role]['wall_seconds'] = 100.0 }
+    end
+    code, report = compare_cli(samples, budgets)
+    expect(code).to eq(3)
+    expect(report.dig('error', 'message')).to include('Request batch exceeds 30.0s')
+  end
+
+  it 'rejects raw integer/float drift even when forged raw metadata is consistent' do
+    [[0, 0.0], [0.0, 0]].each do |original, changed|
+      samples, budgets = PerformanceSamples.fixture(zero: original)
+      target = samples['cases'].first['pairs'].first['candidate']
+      body = JSON.parse(target['bodies'].values.first['body'])
+      body['response']['zero'] = changed
+      replace_raw_body(target, JSON.generate(body))
+      code, report = compare_cli(samples, budgets)
+      expect(code).to eq(3)
+      expect(report.dig('error', 'message')).to include('Raw/canonical output values differ')
+    end
+  end
+
+  it 'allows raw map-order changes through the actual CLI' do
+    samples, budgets = PerformanceSamples.fixture
+    target = samples['cases'].first['pairs'].first['candidate']
+    body = JSON.parse(target['bodies'].values.first['body'])
+    body['response'] = body['response'].to_a.reverse.to_h
+    replace_raw_body(target, JSON.generate(body.to_a.reverse.to_h))
+    expect(compare_cli(samples, budgets).first).to eq(0)
+  end
+
+  it 'rejects identical source SHAs even with consistently rebound provenance' do
+    samples, budgets = PerformanceSamples.fixture
+    rebind_source(samples, 'candidate', samples['baseline_sha'])
+    code, report = compare_cli(samples, budgets)
+    expect(code).to eq(3)
+    expect(report.dig('error', 'message')).to include('Baseline and candidate must differ')
+  end
+
+  it 'requires the fixed weekly baseline and accepts valid fixed-reference evidence' do
+    samples, budgets = PerformanceSamples.fixture(profile: 'weekly')
+    expect(compare_cli(samples, budgets).first).to eq(0)
+    rebind_source(samples, 'baseline', 'a' * 40)
+    code, report = compare_cli(samples, budgets)
+    expect(code).to eq(3)
+    expect(report.dig('error', 'message')).to include('Weekly baseline must be the fixed reference')
+  end
+
+  it 'requires both the percentage and absolute timing thresholds' do
+    samples, budgets = PerformanceSamples.fixture
+    samples['cases'].first['pairs'].each do |pair|
+      pair['baseline']['wall_seconds'] = 0.0001
+      pair['candidate']['wall_seconds'] = 0.00013
+    end
+    expect(compare_cli(samples, budgets).first).to eq(0)
+  end
+
+  it 'blocks allocation growth and the independent fixed reference ceiling' do
+    samples, budgets = PerformanceSamples.fixture
+    samples['cases'].first['pairs'].each { |pair| pair['candidate']['allocated_objects'] = 140 }
+    expect(compare_cli(samples, budgets).first).to eq(1)
+    samples['cases'].first['pairs'].each do |pair|
+      pair['baseline']['allocated_objects'] = 180
+      pair['candidate']['allocated_objects'] = 180
+    end
+    expect(compare_cli(samples, budgets).first).to eq(1)
+  end
+
+  it 'requires full Rust cases and their relative allocation and authorization contracts' do
+    %w[missing allocations authorization].each do |fault|
+      samples, budgets = PerformanceSamples.fixture
+      case fault
+      when 'missing'
+        samples['inventory'].pop
+        samples['cases'].pop
+      when 'allocations'
+        samples['cases'].last['pairs'].each { |pair| pair['candidate']['allocated_objects'] = 200 }
+      else
+        samples['cases'].last['pairs'].first['candidate']['counters']['record']['authorizations'] -= 1
+      end
+      expect(compare_cli(samples, budgets).first).to eq(fault == 'allocations' ? 1 : 3)
+    end
+  end
+
+  it 'blocks uncalibrated reference budgets' do
+    samples, json = PerformanceSamples.fixture
+    budgets = JSON.parse(json)
+    budgets.delete('calibration')
+    json = JSON.generate(budgets)
+    samples['budget_sha256'] = Digest::SHA256.hexdigest(json)
+    expect(compare_cli(samples, json).first).to eq(3)
+  end
+
+  it 'blocks missing cases, operations, dependency graphs and source provenance' do
+    %w[cases outputs inputs sources].each do |missing|
+      samples, budgets = PerformanceSamples.fixture
+      case missing
+      when 'cases' then samples['cases'].clear
+      when 'outputs' then samples['cases'].first['pairs'].first['candidate']['outputs'].clear
+      else samples.delete(missing)
+      end
+      expect(compare_cli(samples, budgets).first).to eq(3)
+    end
+  end
+
+  it 'rejects missing or changed workload order and runtime policy' do
+    %w[workload_order worker_order runtime_policy].each do |fault|
+      samples, budgets = PerformanceSamples.fixture
+      case fault
+      when 'workload_order' then samples['cases'].first['pairs'][1]['workload_order'] = 'forward'
+      when 'worker_order' then samples['observations']['candidate-reverse']['workload_order'] = 'forward'
+      else samples['observations']['baseline']['runtime_policy']['timezone'] = 'local'
+      end
+      expect(compare_cli(samples, budgets).first).to eq(3)
+    end
+  end
+
+  it 'blocks extra queries, lost authorizations, wrong output and changed dependency graphs' do
+    %w[queries authorizations output environment].each do |fault|
+      samples, budgets = PerformanceSamples.fixture
+      target = samples['cases'].first['pairs'].first['candidate']
+      case fault
+      when 'queries' then target['counters']['record']['queries'] = 1
+      when 'authorizations' then target['counters']['record']['authorizations'] = 0
+      when 'output' then target['outputs'].first['canonical_sha256'] = '0' * 64
+      else samples['observations']['candidate']['environment'] = {}
+      end
+      expect(compare_cli(samples, budgets).first).to eq(3)
+    end
+  end
+
+  it 'preserves scalar types and array order while ignoring map order only' do
+    identity = HaveAPI::Performance::Identity
+    expect(identity.digest(a: 0, b: [false, '', nil])).to eq(identity.digest('b' => [false, '', nil], 'a' => 0))
+    expect(identity.digest(a: 0)).not_to eq(identity.digest(a: 0.0))
+    expect(identity.digest(a: [false, nil])).not_to eq(identity.digest(a: [nil, false]))
+    expect { identity.digest(a: 0, 'a' => 1) }.to raise_error(HaveAPI::Performance::ContractError)
+    ['{"status":false,"status":true}', '{"a":[{"b":1,"b":2}]}'].each do |json|
+      expect { identity.parse(json) }.to raise_error(HaveAPI::Performance::ContractError, /Duplicate/)
+    end
+  end
+end

data/spec/performance/endpoint_contract_spec.rb

@@ -0,0 +1,226 @@
+require 'spec_helper'
+require_relative '../../test_support/performance_endpoint_fixture'
+
+RSpec.describe HaveAPI::PerformanceFixtures::Endpoints do
+  let(:scenarios) { [] }
+
+  def build(id = 'api.basic_get', **dimensions)
+    scenario = described_class.build(id:, dimensions:)
+    scenarios << scenario
+    scenario
+  end
+
+  def headers(user = 'member')
+    { 'HTTP_HOST' => 'example.test', 'HTTP_ACCEPT' => 'application/json',
+      'HTTP_AUTHORIZATION' => "Basic #{["#{user}:#{described_class::PASSWORD}"].pack('m0')}",
+      'CONTENT_TYPE' => 'application/json' }
+  end
+
+  def json_request(request, method, path, body = nil, user: 'member')
+    response = request.request(method, "/performance/v1/#{path}",
+                               headers(user).merge(input: body.nil? ? '' : JSON.generate(body)))
+    [response, JSON.parse(response.body)]
+  end
+
+  after do
+    scenarios.each(&:close)
+  end
+
+  it 'retains the seven quick cases and adds only the accepted weekly shapes' do
+    quick = described_class.cases
+    expect(quick.map { |item| item[:id] }).to eq(described_class::CASE_IDS)
+    expect(quick.last).to eq(id: 'api.list', dimensions: { rows: 32 })
+    expect(quick[0...-1].map { |item| item[:dimensions] }).to all(eq({}))
+    weekly = described_class.cases(profile: 'weekly')
+    expect(weekly.first(quick.length)).to eq(quick)
+    additions = [
+      { id: 'api.list', dimensions: { rows: 1024, user: 'member' } },
+      { id: 'api.list', dimensions: { rows: 1024, user: 'admin' } },
+      { id: 'api.basic_get', dimensions: { user: 'admin' } }
+    ]
+    expect(weekly.drop(quick.length)).to eq(additions)
+    expect(weekly.uniq).to eq(weekly)
+  end
+
+  it 'executes and validates every real quick-case response and counter record' do
+    described_class.cases.each do |item|
+      scenario = build(item[:id], **item[:dimensions])
+      scenario.reset!(iterations: 3)
+      responses = Array.new(3) { scenario.call }
+      responses.each_with_index do |response, index|
+        identity = scenario.validate!(response, index:)
+        expect(identity).to include(http_status: 200, body_bytes: response.body.bytesize,
+                                    body_sha256: Digest::SHA256.hexdigest(response.body))
+        expect(identity[:envelope_status]).to eq(item[:id] != 'api.invalid_post')
+        expect(identity[:canonical_body]).to eq(JSON.parse(response.body))
+        expect(identity[:canonical_sha256]).to eq(HaveAPI::Performance::Identity.digest(JSON.parse(response.body)))
+      end
+      executions = item[:id] == 'api.invalid_post' ? 0 : 1
+      expect(scenario.counters).to eq(operations: Array.new(3) { { queries: 0, authorizations: 1, executions: } })
+    end
+  end
+
+  it 'validates the weekly large-list and administrator shapes' do
+    described_class.cases(profile: 'weekly').drop(described_class::CASE_IDS.length).each do |item|
+      scenario = build(item[:id], **item[:dimensions])
+      scenario.reset!(iterations: 1)
+      identity = scenario.validate!(scenario.call, index: 0)
+      output = identity.fetch(:canonical_body).fetch('response').fetch('probe')
+      records = item[:id] == 'api.list' ? output : [output]
+      expect(records.length).to eq(item[:dimensions].fetch(:rows, 1))
+      records.each do |record|
+        expect(record.has_key?('secret')).to eq(item[:dimensions][:user] == 'admin')
+      end
+      expect(scenario.counters).to eq(operations: [{ queries: 0, authorizations: 1, executions: 1 }])
+    end
+  end
+
+  it 'rejects a corrupted early response even when the final response is valid' do
+    scenario = build
+    scenario.reset!(iterations: 2)
+    first = scenario.call
+    last = scenario.call
+    body = JSON.parse(first.body)
+    body['response']['probe']['count'] = '0'
+    broken = Rack::MockResponse.new(200, first.headers, JSON.generate(body))
+
+    expect { scenario.validate!(last, index: 1) }.not_to raise_error
+    expect { scenario.validate!(broken, index: 0) }.to raise_error(/Unexpected output/)
+    expect { scenario.validate!(Rack::MockResponse.new(403, {}, '{}'), index: 0) }
+      .to raise_error(/HTTP status/)
+    duplicate = Rack::MockResponse.new(200, first.headers, first.body.sub('"status":true', '"status":false,"status":true'))
+    expect { scenario.validate!(duplicate, index: 0) }.to raise_error(HaveAPI::Performance::ContractError, /duplicate/i)
+  end
+
+  it 'preserves typed zero, false, empty, null, nested Custom data and omission' do
+    scenario = build('api.typed_post')
+    scenario.reset!(iterations: 1)
+    response = scenario.call
+    scenario.validate!(response, index: 0)
+    output = JSON.parse(response.body).fetch('response').fetch('probe')
+
+    expect(output).to include('count' => 0, 'ratio' => 1.25, 'enabled' => false, 'label' => '', 'nullable' => nil,
+                              'at' => '2020-01-01T00:00:00+00:00', 'payload' => { 'nested' => [0, false, '', nil] })
+    expect(output).not_to have_key('omitted')
+  end
+
+  it 'uses distinct Basic and token providers with real credential failures' do
+    basic = build('api.basic_get')
+    token = build('api.token_get')
+    [basic, token].each { |scenario| scenario.reset!(iterations: 1) }
+    expect(basic.validate!(basic.call, index: 0)).to eq(token.validate!(token.call, index: 0))
+    request = Rack::MockRequest.new(basic.app)
+    missing = request.get('/performance/v1/probes/secure', 'HTTP_HOST' => 'example.test')
+    bad_basic = request.get('/performance/v1/probes/secure', headers.merge('HTTP_AUTHORIZATION' => 'Basic invalid'))
+    bad_token = request.get('/performance/v1/probes/secure', headers.merge('HTTP_AUTHORIZATION' => '',
+                                                                           'HTTP_X_HAVEAPI_AUTH_TOKEN' => 'invalid'))
+    expect([missing.status, bad_basic.status, bad_token.status]).to eq([401, 401, 401])
+    [missing, bad_basic, bad_token].each { |response| expect(JSON.parse(response.body)['status']).to be(false) }
+    denied, body = json_request(request, 'GET', 'probes/admin')
+    expect(denied.status).to eq(403)
+    expect(body.fetch('status')).to be(false)
+    allowed, body = json_request(request, 'GET', 'probes/admin', user: 'admin')
+    expect(allowed.status).to eq(200)
+    expect(body.dig('response', 'probe', 'secret')).to eq('ADMIN_ONLY_VALUE')
+  end
+
+  it 'executes real CRUD, isolates stores and resets writes outside measured requests' do
+    scenario = build('api.list', rows: 1)
+    other = build('api.list', rows: 1)
+    scenario.reset!(iterations: 1)
+    other.reset!(iterations: 1)
+    request = Rack::MockRequest.new(scenario.app)
+    created, body = json_request(request, 'POST', 'items', { probe: { label: 'created' } })
+    expect(created.status).to eq(200)
+    expect(body.fetch('status')).to be(true)
+    expect(body.dig('response', 'probe')).to include('id' => 2, 'label' => 'created')
+    _, body = json_request(request, 'GET', 'items/2')
+    expect(body.dig('response', 'probe', 'label')).to eq('created')
+    _, body = json_request(request, 'PUT', 'items/2', { probe: { label: 'updated' } })
+    expect(body.dig('response', 'probe', 'label')).to eq('updated')
+    _, body = json_request(request, 'DELETE', 'items/2')
+    expect(body.dig('response', 'probe', 'label')).to eq('updated')
+    missing, body = json_request(request, 'GET', 'items/2')
+    expect(missing.status).to eq(404)
+    expect(body.fetch('status')).to be(false)
+    json_request(request, 'POST', 'items', { probe: { label: 'discarded by reset' } })
+    expect(other.validate!(other.call, index: 0)[:envelope_status]).to be(true)
+    scenario.reset!(iterations: 1)
+    expect(scenario.validate!(scenario.call, index: 0)[:envelope_status]).to be(true)
+  end
+
+  it 'does not mutate timed counter records through direct app requests' do
+    scenario = build
+    scenario.reset!(iterations: 1)
+    response = scenario.call
+    before = Marshal.dump(scenario.counters)
+    request = Rack::MockRequest.new(scenario.app)
+    json_request(request, 'GET', 'probes/secure')
+    json_request(request, 'POST', 'items', { probe: { label: 'untimed' } })
+
+    expect(Marshal.dump(scenario.counters)).to eq(before)
+    expect { scenario.validate!(response, index: 0) }.not_to raise_error
+  end
+
+  it 'rejects malformed bodies, missing routes and invalid typed input through real middleware' do
+    scenario = build
+    request = Rack::MockRequest.new(scenario.app)
+    ['{', '[]', 'null'].each do |input|
+      response = request.post('/performance/v1/probes/typed', headers.merge(input:))
+      expect(response.status).to eq(400)
+      expect(JSON.parse(response.body)['status']).to be(false)
+    end
+    response = request.get('/performance/v1/not-a-route', headers)
+    expect(response.status).to eq(404)
+    expect(JSON.parse(response.body)['status']).to be(false)
+    response = request.post('/performance/v1/probes/typed', headers.merge(input: '{"probe":{"count":"wrong"}}'))
+    expect(response.status).to eq(200)
+    expect(JSON.parse(response.body).fetch('errors')).to have_key('count')
+  end
+
+  it 'alternates privileged and restricted users without exposing hidden fields' do
+    scenario = build
+    request = Rack::MockRequest.new(scenario.app)
+    %w[admin member admin].each do |user|
+      _, body = json_request(request, 'GET', 'probes/secure', user:)
+      output = body.fetch('response').fetch('probe')
+      if user == 'admin'
+        expect(output.fetch('secret')).to eq('ADMIN_ONLY_VALUE')
+      else
+        expect(output).not_to have_key('secret')
+        expect(JSON.generate(body)).not_to include('ADMIN_ONLY_VALUE')
+      end
+    end
+  end
+
+  it 'requires reset and every operation record, and rejects overflow and damaged counters' do
+    scenario = build
+    expect { scenario.call }.to raise_error(/reset!/)
+    expect { scenario.counters }.to raise_error(/Missing operation/)
+    scenario.reset!(iterations: 2)
+    first = scenario.call
+    expect { scenario.counters }.to raise_error(/Missing operation/)
+    scenario.call
+    expect { scenario.call }.to raise_error(/More calls/)
+    expect { scenario.validate!(first, index: -1) }.to raise_error(ArgumentError)
+    expect { scenario.validate!(first, index: 2) }.to raise_error(/not called/)
+    scenario.counters[:operations].first[:queries] = 1
+    expect { scenario.validate!(first, index: 0) }.to raise_error(/counts/)
+    expect { scenario.counters }.to raise_error(/counts/)
+    scenario.reset!(iterations: 1)
+    scenario.validate!(scenario.call, index: 0)
+    scenario.close
+    expect { scenario.call }.to raise_error(/closed/)
+    expect { scenario.reset!(iterations: 1) }.to raise_error(/closed/)
+  end
+
+  it 'fails closed on unknown cases, profiles, dimensions, users and invalid batch sizes' do
+    expect { described_class.cases(profile: 'other') }.to raise_error(ArgumentError)
+    expect { build('api.unknown') }.to raise_error(ArgumentError)
+    expect { build('api.list', rows: 0) }.to raise_error(ArgumentError)
+    expect { build('api.list', rows: '32') }.to raise_error(ArgumentError)
+    expect { build('api.list', user: 'unknown') }.to raise_error(ArgumentError)
+    expect { build('api.list', accidental: true) }.to raise_error(ArgumentError)
+    expect { build.reset!(iterations: 0) }.to raise_error(ArgumentError)
+  end
+end

data/spec/performance/endpoint_scaling_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+require_relative '../../test_support/performance_endpoint_fixture'
+
+RSpec.describe HaveAPI::PerformanceFixtures::Endpoints, '.build' do
+  def run_list(rows)
+    scenario = described_class.build(id: 'api.list', dimensions: { rows: })
+    scenario.reset!(iterations: 2)
+    responses = Array.new(2) { scenario.call }
+    identities = responses.each_with_index.map { |response, index| scenario.validate!(response, index:) }
+    [identities, scenario.counters]
+  ensure
+    scenario&.close
+  end
+
+  def allocated_per_request(rows)
+    scenario = described_class.build(id: 'api.list', dimensions: { rows: })
+    2.times do
+      scenario.reset!(iterations: 10)
+      10.times { scenario.call }
+    end
+    Array.new(3) do
+      scenario.reset!(iterations: 10)
+      responses = Array.new(10)
+      GC.start
+      allocated = GC.stat(:total_allocated_objects)
+      index = 0
+      while index < responses.length
+        responses[index] = scenario.call
+        index += 1
+      end
+      delta = GC.stat(:total_allocated_objects) - allocated
+      responses.each_with_index { |response, operation| scenario.validate!(response, index: operation) }
+      scenario.counters
+      delta / 10.0
+    end.min
+  ensure
+    scenario&.close
+  end
+
+  it 'serializes complete ordered lists at sizes 1, 32 and 128 with fixed action work' do
+    bytes = [1, 32, 128].map do |rows|
+      identities, counters = run_list(rows)
+      expect(identities.first).to eq(identities.last)
+      output = identities.first[:canonical_body].fetch('response').fetch('probe')
+      expect(output.length).to eq(rows)
+      expect(output.map { |row| row.fetch('id') }).to eq((1..rows).to_a)
+      expect(output.last.fetch('label')).to eq("item-#{rows}")
+      output.each { |row| expect(row).not_to have_key('secret') }
+      expect(counters[:operations]).to eq(Array.new(2) { { queries: 0, authorizations: 1, executions: 1 } })
+      identities.first[:body_bytes]
+    end
+    expect(bytes.each_cons(2)).to all(satisfy { |small, large| small < large })
+  end
+
+  it 'keeps output and counters independent across reset batch sizes' do
+    scenario = described_class.build(id: 'api.list', dimensions: { rows: 32 })
+    scenario.reset!(iterations: 1)
+    first = scenario.validate!(scenario.call, index: 0)
+    records = scenario.counters[:operations]
+    scenario.reset!(iterations: 3)
+    responses = Array.new(3) { scenario.call }
+    responses.each_with_index do |response, index|
+      expect(scenario.validate!(response, index:)).to eq(first)
+    end
+    expect(scenario.counters[:operations].length).to eq(3)
+    expect(scenario.counters[:operations].first).not_to equal(records.first)
+    expect(scenario.counters[:operations].map(&:object_id).uniq.length).to eq(3)
+  ensure
+    scenario&.close
+  end
+
+  it 'bounds allocation growth by serialized rows rather than their square' do
+    small, medium, large = [1, 32, 128].map { |rows| allocated_per_request(rows) }
+
+    # Reference Ruby 3.4.11: 640/795/1275 objects per request. Output work
+    # grows with rows; the 96-row increment has a 4x allowance relative to
+    # the 31-row increment, plus 32 objects of fixed VM bookkeeping slack.
+    expect(medium).to be >= small
+    expect(large).to be > medium
+    expect(large - medium).to be <= (4 * (medium - small)) + 32
+  end
+end