pwn 0.5.643 → 0.5.650

data/spec/integration/reinforced_feedback_loop_spec.rb

@@ -44,6 +44,7 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
     stub_const('PWN::AI::Agent::Reward::DPO_DIR',            File.join(@tmp, 'finetune'))
     stub_const('PWN::AI::Agent::Curriculum::CURRICULUM_DIR', File.join(@tmp, 'curriculum'))
     stub_const('PWN::AI::Agent::Curriculum::MODELS_FILE',    File.join(@tmp, 'curriculum', 'models.json'))
+    stub_const('PWN::AI::Agent::Curriculum::KPI_FILE',       File.join(@tmp, 'curriculum_kpi.jsonl'))
     stub_const('PWN::Sessions::SESSIONS_DIR',                File.join(@tmp, 'sessions'))
     stub_const('PWN::Memory::MEMORY_FILE',                   File.join(@tmp, 'memory.json'))
     stub_const('PWN::MemoryIndex::INDEX_FILE',               File.join(@tmp, 'memory.idx')) if defined?(PWN::MemoryIndex)
@@ -314,7 +315,7 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
         score: 0.85, verdict: :solved,
         final: 'use `nmap` (typo: nmpa→nmap)',
         prompt: 'fix nmap typo',
-        trace: 'shell → nmap -sn 10.0.0.0/24'
+        trace: "shell → nmap -sn 10.0.0.0/24\nshell → true\npwn_eval → :ok"
       )
       r = curriculum.practice(limit: 1, prompts_per: 2)
       expect(r[:practiced]).to eq 1
@@ -323,7 +324,13 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
       fixed = mistakes.find(signature: sig)
       expect(fixed[:structured_fix]).to be_a(Hash)
       expect(fixed[:structured_fix][:strategy]).to eq 'curriculum_practice'
-      expect(reward.preferences(source: 'curriculum')).not_to be_empty
+      expect(fixed[:structured_fix][:winning_trace]).to include('shell →')
+      prefs = reward.preferences(source: 'curriculum')
+      expect(prefs).not_to be_empty
+      # P14 — chosen is trajectory, not first-3-lines fix prose
+      expect(prefs.first[:shape].to_s).to eq 'winning_trace'
+      expect(prefs.first[:chosen]).to match(/WINNING_TRACE|shell →/i)
+      expect(prefs.first[:chosen]).not_to eq 'use `nmap` (typo: nmpa→nmap)'
     end
 
     it 'does NOT auto-resolve with a single holdout success (2.4 gate)' do
@@ -366,16 +373,25 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
       @agent_cfg[:counterfactual] = true
       allow(curriculum).to receive(:ensure_persona).and_return(nil)
       allow(curriculum).to receive(:ask_persona).and_return('pwn_eval(code: "PWN::Plugins::NmapIt.scan(...)")')
-      allow(curriculum).to receive(:score_branch) { |o| o[:branch].include?('pwn_eval') ? 0.80 : 0.30 }
+      # P9 — counterfactual scores via score_branch_detailed
+      allow(curriculum).to receive(:score_branch_detailed) do |o|
+        if o[:branch].to_s.include?('pwn_eval')
+          { score: 0.80, mode: :real_dispatch, trace: o[:branch].to_s }
+        else
+          { score: 0.30, mode: :imagined, trace: nil }
+        end
+      end
 
       r = curriculum.counterfactual(request: 'scan 10.0.0.1', name: 'shell',
                                     args: '{"command":"nmpa -sV"}', error: 'nmpa: not found',
                                     hint: 'retry shell with nmap')
       expect(r[:branch]).to eq :b
       expect(r[:content]).to include('pwn_eval')
+      expect(r[:shape]).to eq :real_dispatch
       pref = reward.preferences(source: 'counterfactual').first
       expect(pref[:chosen]).to include('pwn_eval')
       expect(pref[:rejected]).to include('retry shell')
+      expect(pref[:shape].to_s).to eq 'real_dispatch'
     end
   end
 
@@ -415,9 +431,20 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
   describe 'W1 · preference-pair generation' do
     it 'Mistakes.resolve emits a (rejected: failing action, chosen: fix) pair' do
       m = mistakes.record(tool: 'shell', error: 'nmpa: command not found')
-      mistakes.resolve(signature: m[:signature], fix: 'use `nmap`, not `nmpa`')
+      # P21/P25 — W1 pair only lands when structured winning_trace is present
+      mistakes.resolve(
+        signature: m[:signature],
+        fix: 'use `nmap`, not `nmpa`',
+        structured: {
+          strategy: 'correct_binary_name',
+          tool: 'shell',
+          winning_trace: "shell → which nmap && nmap --version\nnmap present"
+        }
+      )
       pair = reward.preferences(source: 'mistakes_resolve').first
+      expect(pair).not_to be_nil
       expect(pair[:chosen]).to include('nmap')
+      expect(pair[:shape].to_s).to eq 'winning_trace'
       expect(PWN::Memory.load.keys.map(&:to_s)).to include("mistake_fix_#{m[:signature]}")
     end
 
@@ -462,17 +489,30 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
       expect(r).not_to have_key(:promoted)
     end
 
-    it 'promotes iff resolved(candidate) > resolved(baseline) on the Mistakes eval set' do
+    it 'promotes under gate v2 when resolved margin + mean + smoke all win' do
       allow(learning).to   receive(:export_finetune).and_return(path: 'sft', rows: 0)
       allow(reward).to     receive(:export_dpo).and_return(path: 'dpo', rows: 0)
       allow(curriculum).to receive(:detect_trainer).and_return(:unsloth)
       allow(curriculum).to receive(:run_trainer).and_return(File.join(@tmp, 'adapter'))
       allow(curriculum).to receive(:ollama_create).and_return('pwn-v1')
-      allow(curriculum).to receive(:build_eval_set).and_return(%w[p1 p2 p3])
-      allow(curriculum).to receive(:ab_gate).and_return(baseline_resolved: 1, candidate_resolved: 3)
+      allow(curriculum).to receive(:build_eval_set).and_return(
+        Array.new(10) { |i| { signature: "s#{i}", prompt: "p#{i}" } }
+      )
+      allow(curriculum).to receive(:ab_gate_v2).and_return(
+        baseline_resolved: 1, candidate_resolved: 4,
+        baseline_mean: 0.5, candidate_mean: 0.8,
+        resolved_win: true, mean_win: true, smoke_ok: true,
+        promote: true, gate_version: 2, evalset_size: 10
+      )
+      # P19 — diet gate is AND-ed with ab_gate_v2; stub a clean W1 diet for this contract
+      allow(curriculum).to receive(:preference_diet_gate).and_return(
+        ok: true, reason: 'diet_ok', total: 40, trajectory_fraction: 0.5, monoculture: false
+      )
       r = curriculum.train_and_gate(dry_run: false, base_model: 'llama3')
       expect(r[:promoted]).to be true
+      expect(r[:weight_loop]).to eq :closed
       expect(r[:gate][:candidate_resolved]).to be > r[:gate][:baseline_resolved]
+      expect(r[:gate][:gate_version]).to eq 2
       expect(File.exist?(curriculum::MODELS_FILE)).to be true
     end
   end
@@ -609,16 +649,171 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
   # P-controls — operational fixes that make the loop actually close
   # ═══════════════════════════════════════════════════════════════════════
 
+  describe 'P9 · W1 pair geometry + write-time source quota' do
+    it 'rejects CORRECTION: flaw-prose chosen sides without force' do
+      r = reward.record_preference(
+        prompt: 'which CVE?', rejected: 'CVE-2099-0001',
+        chosen: 'CORRECTION: cited CVE does not exist', source: :critic
+      )
+      expect(r[:skipped]).to eq :weak_pair_geometry
+      expect(reward.preferences.length).to eq 0
+    end
+
+    it 'accepts revised_answer shaped critic pairs' do
+      r = reward.record_preference(
+        prompt: 'which CVE?', rejected: 'CVE-2099-0001 is critical',
+        chosen: "REVISED ANSWER (addresses: CVE does not exist):\nNo such CVE in NVD; verify before citing.",
+        source: :critic, shape: :revised_answer
+      )
+      expect(r[:skipped]).to be_nil
+      expect(r[:source]).to eq 'critic'
+      expect(r[:shape]).to eq 'revised_answer'
+    end
+
+    it 'write-time quota refuses further mistakes_resolve once over cap' do
+      # P25 requires trajectory shape; P9 still caps source share so traj
+      # pairs cannot monoculture the ledger. Seed resolve-heavy window.
+      12.times do |i|
+        reward.record_preference(
+          prompt: "p#{i}", rejected: "r#{i}" * 5,
+          chosen: "STRATEGY: s\nWINNING_TRACE:\nshell → ok #{i}\n" + ('t' * 40),
+          source: :mistakes_resolve, shape: :winning_trace, force: true
+        )
+      end
+      # one non-resolve so multi-source window, but resolve still >40%
+      reward.record_preference(
+        prompt: 'q', rejected: 'x' * 20,
+        chosen: "REVISED ANSWER:\n#{'y' * 40}",
+        source: :critic, shape: :revised_answer, force: true
+      )
+      r = reward.record_preference(
+        prompt: 'pX', rejected: 'rX' * 5,
+        chosen: "STRATEGY: s\nWINNING_TRACE:\nshell → next\n#{'t' * 40}",
+        source: :mistakes_resolve, shape: :winning_trace
+      )
+      expect(r[:skipped]).to eq :source_quota
+      expect(r[:over_cap]).to be true
+    end
+
+    it 'Curriculum.critic records revised_answer preference not CORRECTION prose' do
+      @agent_cfg[:critic] = true
+      allow(curriculum).to receive(:ensure_persona).and_return(nil)
+      allow(curriculum).to receive(:ask_persona).and_return('FLAW: cited CVE does not exist')
+      allow(curriculum).to receive(:revise_after_flaw).and_return(
+        "REVISED ANSWER (addresses: cited CVE does not exist):\nUse NVD-verified ids only."
+      )
+      v = curriculum.critic(request: 'which CVE?', final: 'CVE-2099-0001 is critical')
+      expect(v[:verdict]).to eq :flaw
+      pref = reward.preferences(source: 'critic').first
+      expect(pref).not_to be_nil
+      expect(pref[:chosen]).to include('REVISED ANSWER')
+      expect(pref[:chosen]).not_to match(/\ACORRECTION:/i)
+      expect(pref[:shape].to_s).to eq 'revised_answer'
+    end
+  end
+
+  describe 'P10 · Reward.warm_sentinel' do
+    it 'fills the R3 window from scored Learning outcomes' do
+      45.times do |i|
+        learning.note_outcome(task: "t#{i}", success: i.even?, score: i.even? ? 0.9 : 0.2,
+                              details: 'x', tags: %w[rspec])
+      end
+      r = reward.warm_sentinel(limit: 80)
+      expect(r[:added]).to be > 0
+      expect(r[:samples]).to be >= r[:added]
+      expect(%i[warmed_full warmed_partial full]).to include(r[:status])
+    end
+  end
+
+  describe 'P11 · ab_gate_v2 promotion contract' do
+    it 'requires resolved margin + mean win + smoke ok' do
+      allow(curriculum).to receive(:replay_on_detailed) do |o|
+        tag = o[:tag].to_s
+        evalset = Array(o[:evalset])
+        if evalset.any? { |e| e[:signature].to_s.start_with?('smoke_') }
+          # smoke equal
+          { resolved: 3, mean_score: 0.9, scores: [0.9, 0.9, 0.9] }
+        elsif tag.include?('cand') || tag == 'cand'
+          { resolved: 8, mean_score: 0.85, scores: [0.85] * 10 }
+        else
+          { resolved: 5, mean_score: 0.70, scores: [0.7] * 10 }
+        end
+      end
+      allow(curriculum).to receive(:smoke_eval_set).and_return(
+        [{ signature: 'smoke_uname', prompt: 'uname' }]
+      )
+      g = curriculum.send(:ab_gate_v2, baseline: 'base', candidate: 'cand',
+                                       evalset: Array.new(10) { |i| { signature: "s#{i}", prompt: "p#{i}" } })
+      expect(g[:gate_version]).to eq 2
+      expect(g[:promote]).to be true
+      expect(g[:resolved_win]).to be true
+      expect(g[:mean_win]).to be true
+      expect(g[:smoke_ok]).to be true
+    end
+
+    it 'refuses promotion on smoke regression' do
+      allow(curriculum).to receive(:replay_on_detailed) do |o|
+        evalset = Array(o[:evalset])
+        if evalset.any? { |e| e[:signature].to_s.start_with?('smoke_') }
+          tag = o[:tag].to_s
+          if tag == 'cand'
+            { resolved: 0, mean_score: 0.1, scores: [0.1] }
+          else
+            { resolved: 3, mean_score: 0.9, scores: [0.9] }
+          end
+        elsif o[:tag].to_s == 'cand'
+          { resolved: 9, mean_score: 0.95, scores: [0.95] * 10 }
+        else
+          { resolved: 5, mean_score: 0.70, scores: [0.7] * 10 }
+        end
+      end
+      allow(curriculum).to receive(:smoke_eval_set).and_return(
+        [{ signature: 'smoke_uname', prompt: 'uname' }]
+      )
+      g = curriculum.send(:ab_gate_v2, baseline: 'base', candidate: 'cand',
+                                       evalset: Array.new(10) { |i| { signature: "s#{i}", prompt: "p#{i}" } })
+      expect(g[:promote]).to be false
+      expect(g[:smoke_ok]).to be false
+    end
+  end
+
+  describe 'P12 · export_finetune quality gate' do
+    it 'drops low-score gold and compresses trajectories' do
+      good = PWN::Sessions.create(title: 'sft-good')
+      bad  = PWN::Sessions.create(title: 'sft-bad')
+      [good, bad].each do |s|
+        PWN::Sessions.append(session_id: s[:id], role: 'user', content: 'scan lab')
+        PWN::Sessions.append(session_id: s[:id], role: 'tool', content: ok_trace)
+        PWN::Sessions.append(session_id: s[:id], role: 'assistant', content: '3 hosts up')
+      end
+      learning.note_outcome(task: 'scan lab', success: true, score: 0.9, session_id: good[:id])
+      learning.note_outcome(task: 'scan lab', success: true, score: 0.2, session_id: bad[:id])
+      info = learning.export_finetune(out: File.join(@tmp, 'sft-p12.jsonl'))
+      expect(info[:samples]).to eq 1
+      expect(info[:min_score]).to eq 0.6
+      expect(info[:compressed]).to be true
+    end
+  end
+
   describe 'P5 · export_dpo source-cap (no monoculture in weights)' do
     it 'downsamples so no single source exceeds DPO_SOURCE_CAP of the corpus' do
       10.times do |i|
-        reward.record_preference(prompt: "p#{i}", rejected: "r#{i}", chosen: "c#{i}", source: :mistakes_resolve)
+        reward.record_preference(
+          prompt: "p#{i}", rejected: "r#{i}", chosen: "WINNING_TRACE:\nshell → ok #{i} " + ('x' * 40),
+          source: :mistakes_resolve, shape: :winning_trace, force: true
+        )
       end
       3.times do |i|
-        reward.record_preference(prompt: "q#{i}", rejected: "x#{i}", chosen: "y#{i}", source: :counterfactual)
+        reward.record_preference(
+          prompt: "q#{i}", rejected: "x#{i}", chosen: "y#{i} " + ('alt ' * 20),
+          source: :counterfactual, shape: :real_dispatch, force: true
+        )
       end
       2.times do |i|
-        reward.record_preference(prompt: "z#{i}", rejected: "a#{i}", chosen: "b#{i}", source: :critic)
+        reward.record_preference(
+          prompt: "z#{i}", rejected: "a#{i}", chosen: "REVISED ANSWER:\n#{'b' * 40}",
+          source: :critic, shape: :revised_answer, force: true
+        )
       end
       info = reward.export_dpo
       expect(info[:balanced]).to be true
@@ -627,8 +822,8 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
       info[:by_source].each_value do |n|
         expect(n.to_f / total).to be <= (reward::DPO_SOURCE_CAP + 0.05)
       end
-      # raw dump still available for diagnostics
-      raw = reward.export_dpo(balance: false, out: File.join(@tmp, 'raw-dpo.jsonl'))
+      # raw dump still available for diagnostics (no scrub, no balance)
+      raw = reward.export_dpo(balance: false, scrub: false, out: File.join(@tmp, 'raw-dpo.jsonl'))
       expect(raw[:pairs]).to eq 15
       expect(raw[:balanced]).to be false
     end
@@ -706,5 +901,604 @@ RSpec.describe 'PWN::AI::Agent reinforced feedback loop', :aggregate_failures do
       expect(out).to eq 'raw'
     end
   end
+
+  describe 'P14 · practice preference geometry' do
+    it 'records winning_trace curriculum pairs instead of fix prose' do
+      mistakes.record(tool: 'shell', error: 'p14-geometry-unique-zz')
+      allow(curriculum).to receive(:reflect_available?).and_return(false)
+      allow(curriculum).to receive(:self_play).and_return(
+        score: 0.9, verdict: :solved,
+        final: "fixed with nmap\nsecond line\nthird",
+        prompt: 'scan the lab safely',
+        trace: "shell → nmap -sn 10.0.0.0/24\nshell → true"
+      )
+      r = curriculum.practice(limit: 1, prompts_per: 2)
+      expect(r[:resolved]).to eq 1
+      pref = reward.preferences(source: 'curriculum').first
+      expect(pref[:shape].to_s).to eq 'winning_trace'
+      expect(pref[:chosen]).to include('WINNING_TRACE')
+      expect(pref[:chosen]).to include('nmap')
+    end
+  end
+
+  describe 'P15 · preference ledger hygiene' do
+    it 'usable_preference? drops CORRECTION prose and short resolve commentary' do
+      expect(reward.usable_preference?(
+               prompt: 'p', rejected: 'long ' * 80, chosen: 'CORRECTION: no', source: 'critic'
+             )).to be false
+      expect(reward.usable_preference?(
+               prompt: 'p', rejected: 'fail',
+               chosen: "STRATEGY: x\nWINNING_TRACE:\nshell → ok\n#{'x' * 80}",
+               source: 'curriculum', shape: 'winning_trace'
+             )).to be true
+    end
+
+    it 'scrub_preferences dry_run reports drops without rewriting' do
+      reward.record_preference(
+        prompt: 'p', rejected: 'bad final answer that is fairly long ' * 5,
+        chosen: 'CORRECTION: short', source: :critic, force: true
+      )
+      reward.record_preference(
+        prompt: 'q', rejected: 'old', chosen: "REVISED ANSWER:\n#{'ok ' * 40}",
+        source: :critic, shape: :revised_answer, force: true
+      )
+      r = reward.scrub_preferences(dry_run: true)
+      expect(r[:before]).to be >= 2
+      expect(r[:dropped]).to be >= 1
+      expect(r[:dry_run]).to be true
+      expect(reward.preferences.length).to eq r[:before]
+    end
+
+    it 'export_dpo scrub drops weak geometry before source cap' do
+      5.times do |i|
+        reward.record_preference(
+          prompt: "p#{i}", rejected: 'long rejected final ' * 30,
+          chosen: 'CORRECTION: nope', source: :mistakes_resolve, force: true
+        )
+      end
+      5.times do |i|
+        reward.record_preference(
+          prompt: "q#{i}", rejected: "r#{i}",
+          chosen: "WINNING_TRACE:\nshell → uname -r\n#{i} " + ('trace ' * 20),
+          source: :counterfactual, shape: :real_dispatch, force: true
+        )
+      end
+      info = reward.export_dpo(out: File.join(@tmp, 'p15-dpo.jsonl'))
+      expect(info[:geometry_dropped]).to be >= 5
+      expect(info[:scrubbed]).to be true
+      lines = File.readlines(info[:path]).map { |l| JSON.parse(l) }
+      expect(lines).not_to be_empty
+      expect(lines.any? { |l| l['chosen'].to_s.start_with?('CORRECTION:') }).to be false
+    end
+
+    it 'preference_balance(scrub: true) reports trajectory_fraction' do
+      reward.record_preference(
+        prompt: 'a', rejected: 'x', chosen: "WINNING_TRACE:\n#{'t' * 100}",
+        source: :curriculum, shape: :winning_trace, force: true
+      )
+      bal = reward.preference_balance(scrub: true)
+      expect(bal[:trajectory_fraction]).to be >= 0.0
+      expect(bal).to have_key(:by_shape)
+      expect(bal[:kept]).to be <= bal[:total]
+    end
+  end
+
+  describe 'P16 · warm_sentinel engages controllers' do
+    it 'fills to capacity and exposes proxy_distrust' do
+      50.times do |i|
+        learning.note_outcome(task: "w#{i}", success: i.even?, score: i.even? ? 0.85 : 0.2,
+                              details: 'x', tags: %w[rspec p16])
+      end
+      r = reward.warm_sentinel(limit: 120)
+      expect(r[:samples]).to be >= r[:added]
+      expect(r).to have_key(:proxy_distrust)
+      expect(%i[warmed_full warmed_partial full]).to include(r[:status])
+    end
+  end
+
+  describe 'P17 · budget-exhaustion curriculum target' do
+    it 'natural_repro_prompts for agent_loop favour short-finish tasks' do
+      prompts = curriculum.send(
+        :natural_repro_prompts,
+        mistake: { tool: 'agent_loop', error: 'iteration budget exhausted', shape: 'handler_error' },
+        count: 4
+      )
+      expect(prompts.length).to eq 4
+      blob = prompts.join(' ').downcase
+      expect(blob).to match(/one shell|at most two|no tools|three iterations|uname|cwd/)
+    end
+
+    it 'practice sorts budget fingerprints ahead of shell noise' do
+      mistakes.record(tool: 'shell', error: 'shell-noise-unique-aaa')
+      3.times { mistakes.record(tool: 'agent_loop', error: 'iteration budget exhausted without a final answer') }
+      allow(curriculum).to receive(:reflect_available?).and_return(false)
+      allow(curriculum).to receive(:practice_skip?).and_return(false)
+      seen = []
+      allow(curriculum).to receive(:generate_reproducers) do |o|
+        seen << o[:mistake][:tool]
+        ['Answer in one shell call: print kernel release with uname -r']
+      end
+      curriculum.practice(limit: 1, prompts_per: 1, dry_run: true)
+      expect(seen.first).to eq 'agent_loop'
+    end
+  end
+
+  describe 'P18 · PRM closes into Registry.rank' do
+    it 'Metrics.record_step_reward shifts prm_advantage and rank scoring' do
+      10.times { metrics.record_step_reward(name: 'shell', reward: 1.0) }
+      10.times { metrics.record_step_reward(name: 'extro_rf_tune', reward: -1.0) }
+      expect(metrics.prm_advantage(name: 'shell')).to be > metrics.prm_advantage(name: 'extro_rf_tune')
+      ranked = registry.rank(query: 'run a shell command on the host')
+      expect(ranked.first.name).to eq 'shell'
+    end
+  end
+
+  describe 'P19 · train_and_gate diet gate blocks promote on weak W1' do
+    it 'preference_diet_gate fails when trajectory_fraction is low' do
+      allow(reward).to receive(:preference_balance).and_return(
+        total: 20, kept: 20, monoculture: true, fractions: { 'mistakes_resolve' => 0.9 },
+        trajectory_fraction: 0.05, by_source: { 'mistakes_resolve' => 18 }
+      )
+      g = curriculum.send(:preference_diet_gate)
+      expect(g[:ok]).to be false
+    end
+
+    it 'preference_diet_gate passes on diverse trajectory diet' do
+      allow(reward).to receive(:preference_balance).and_return(
+        total: 40, kept: 40, monoculture: false,
+        fractions: { 'curriculum' => 0.3, 'critic' => 0.3, 'counterfactual' => 0.25, 'mistakes_resolve' => 0.15 },
+        trajectory_fraction: 0.55, by_source: { 'curriculum' => 12 }
+      )
+      g = curriculum.send(:preference_diet_gate)
+      expect(g[:ok]).to be true
+      expect(g[:reason]).to eq 'diet_ok'
+    end
+  end
+
+  describe 'P20 · judge-blended Metrics scalar' do
+    it 'record_judge shifts effective_rate and advantage under distrust' do
+      allow(reward).to receive(:proxy_distrust).and_return(0.8)
+      20.times { metrics.record(name: 'shell', success: true, duration: 0.1) }
+      20.times { metrics.record(name: 'extro_rf_tune', success: true, duration: 0.1) }
+      10.times { metrics.record_judge(name: 'shell', score: 0.9) }
+      10.times { metrics.record_judge(name: 'extro_rf_tune', score: 0.1) }
+      expect(metrics.judge_rate(name: 'shell')).to be > 0.8
+      expect(metrics.judge_rate(name: 'extro_rf_tune')).to be < 0.2
+      expect(metrics.effective_rate(name: 'shell')).to be > metrics.effective_rate(name: 'extro_rf_tune')
+      expect(metrics.advantage(name: 'shell')).to be > metrics.advantage(name: 'extro_rf_tune')
+    end
+
+    it 'exemplars_for drops low-score success rows' do
+      learning.note_outcome(task: 'p20 high judge shell uname', success: true, score: 0.95,
+                            details: 'ok', session_id: 'sess-p20-hi', tags: %w[rspec p20])
+      learning.note_outcome(task: 'p20 low judge shell uname', success: true, score: 0.2,
+                            details: 'proxy lie', session_id: 'sess-p20-lo', tags: %w[rspec p20])
+      # compress_exemplar needs real sessions — just ensure filter does not raise
+      # and low score alone is excluded from pool scoring path
+      pool = learning.outcomes(limit: 50, success: true)
+      low = pool.select { |r| r[:score].to_f < 0.6 && r[:task].to_s.include?('p20 low') }
+      expect(low).not_to be_empty
+      # unit-check the filter predicate used in exemplars_for
+      kept = pool.reject { |r| r.key?(:score) && r[:score].to_f < 0.6 }
+      expect(kept.any? { |r| r[:task].to_s.include?('p20 low') }).to be false
+      expect(kept.any? { |r| r[:task].to_s.include?('p20 high') }).to be true
+    end
+  end
+
+  describe 'P21/P25 · trajectory-only W1 writers' do
+    it 'record_preference refuses non-trajectory shape without force' do
+      r = reward.record_preference(
+        prompt: 'p', rejected: 'bad', chosen: 'do this instead with more words here',
+        source: :mistakes_resolve, shape: :fix_prose
+      )
+      expect(r).to be_a(Hash)
+      expect(r[:skipped]).to eq :non_trajectory_shape
+    end
+
+    it 'record_preference accepts winning_trace shape' do
+      r = reward.record_preference(
+        prompt: 'p', rejected: 'failing shell args',
+        chosen: "STRATEGY: x\nWINNING_TRACE:\nshell → uname -r\n#{'t' * 40}",
+        source: :mistakes_resolve, shape: :winning_trace
+      )
+      expect(r).to be_a(Hash)
+      expect(r[:skipped]).to be_nil
+      expect(r[:shape].to_s).to eq 'winning_trace'
+    end
+
+    it 'mistakes.resolve does not write fix_prose preference pairs' do
+      m = mistakes.record(tool: 'shell', error: 'p25-no-prose-unique-zz')
+      before = reward.preferences.length
+      mistakes.resolve(signature: m[:signature], fix: 'use a different flag next time')
+      # no structured winning_trace → no new preference row
+      expect(reward.preferences.length).to eq before
+    end
+
+    it 'mistakes.resolve writes winning_trace when structured_fix has trace' do
+      m = mistakes.record(tool: 'shell', error: 'p25-with-trace-unique-yy')
+      before = reward.preferences.length
+      # winning_trace must be ≥40 chars (P21 gate inside Mistakes.resolve)
+      mistakes.resolve(
+        signature: m[:signature],
+        fix: 'run uname -r once',
+        structured: {
+          strategy: 'short_horizon_finish',
+          tool: 'shell',
+          winning_trace: "shell → uname -r\n6.19.14-kali\n(kernel release)"
+        }
+      )
+      expect(reward.preferences.length).to be > before
+      pref = reward.preferences(source: 'mistakes_resolve').find { |p| p[:chosen].to_s.include?('uname') }
+      expect(pref).not_to be_nil
+      expect(pref[:shape].to_s).to eq 'winning_trace'
+    end
+  end
+
+  describe 'P22 · W3 calibration lights up from plan_first' do
+    it 'plan_first parse accepts p(success)= and stashes on Thread' do
+      # Direct unit: emulate stash write path used by plan_first
+      Thread.current[:pwn_plan_predicted] = 0.72
+      expect(Thread.current[:pwn_plan_predicted]).to eq 0.72
+      # calibrate path
+      r = curriculum.calibrate(predicted: 0.72, actual: 1.0, engine: :ollama)
+      expect(r[:brier]).to be_a(Numeric)
+      expect(metrics.calibration(engine: :ollama)[:n]).to be >= 1
+      Thread.current[:pwn_plan_predicted] = nil
+    end
+
+    it 'recover_predicted_from_session reads Thread stash' do
+      Thread.current[:pwn_plan_predicted] = 0.55
+      pred = learning.send(:recover_predicted_from_session, session_id: 'missing')
+      expect(pred).to eq 0.55
+      Thread.current[:pwn_plan_predicted] = nil
+    end
+  end
+
+  describe 'P23 · short-horizon budget practice' do
+    it 'self_play detects short-horizon prompt language' do
+      # natural prompts already short-horizon; ensure generator still emits them
+      prompts = curriculum.send(
+        :natural_repro_prompts,
+        mistake: { tool: 'agent_loop', error: 'iteration budget exhausted' },
+        count: 3
+      )
+      expect(prompts.join(' ')).to match(/one shell|at most two|no tools|three iterations/i)
+    end
+
+    it 'practice refuses resolve when holdouts ok but trace weak on budget target' do
+      mistakes.record(tool: 'agent_loop', error: 'p23-weak-trace-budget-zz')
+      allow(curriculum).to receive(:reflect_available?).and_return(false)
+      allow(curriculum).to receive(:self_play).and_return(
+        score: 0.9, verdict: :solved,
+        final: 'x' * 2000, # long prose, no tool arrow
+        prompt: 'Answer in one shell call: print kernel release with uname -r',
+        trace: '' # empty trace
+      )
+      r = curriculum.practice(limit: 1, prompts_per: 2)
+      row = r[:results]&.first || (r[:practiced] && nil)
+      # practice returns practiced/resolved counts; weak trace should not resolve
+      expect(r[:resolved]).to eq 0
+    end
+  end
+
+  describe 'P24 · critic cost capped under budget hot' do
+    it 'critic(text_only: true) returns without persona tools' do
+      allow(curriculum).to receive(:reflect_available?).and_return(false)
+      r = curriculum.critic(
+        request: 'what is kernel',
+        final: '[pwn-ai] iteration budget exhausted',
+        text_only: true
+      )
+      expect(r[:source].to_s).to include('text_only')
+      expect(%i[pass flaw]).to include(r[:verdict])
+      expect(r[:verdict]).to eq :flaw # heuristic sees budget exhausted
+    end
+
+    it 'critic(text_only: true) does not require critic env flag' do
+      # even when critic disabled, text_only path works (forced by budget hot)
+      allow(curriculum).to receive(:enabled?).and_return(false)
+      allow(curriculum).to receive(:reflect_available?).and_return(false)
+      r = curriculum.critic(request: 'q', final: 'solid answer with facts', text_only: true)
+      expect(r[:verdict]).to eq :pass
+    end
+  end
+
+  # ═══════════════════════════════════════════════════════════════════════
+  # Design-priority STATUS (post P14–P25) — P0 / P1 / P2
+  # documentation/Reinforcement-Learning.md § Design-priority STATUS
+  # ═══════════════════════════════════════════════════════════════════════
+
+  describe 'P0 · W1 generator diversity (TARGET_SOURCE_MIX + generator_mix)' do
+    it 'exposes TARGET_SOURCE_MIX soft targets that sum to 1.0' do
+      mix = reward::TARGET_SOURCE_MIX
+      expect(mix.keys).to include('mistakes_resolve', 'curriculum', 'counterfactual', 'critic', 'user_correction')
+      expect(mix.values.sum).to be_within(0.001).of(1.0)
+      expect(mix['mistakes_resolve']).to be <= 0.40
+    end
+
+    it 'generator_mix flags underfilled sources and returns recommendation' do
+      # seed a few usable traj pairs skewed toward curriculum/critic
+      6.times do |i|
+        reward.record_preference(
+          prompt: "p#{i}",
+          rejected: "bad answer #{i} " + ('x' * 40),
+          chosen: "STRATEGY: use shell\nWINNING_TRACE: shell(uname)\nFINAL: linux #{i}",
+          source: i.even? ? :curriculum : :critic,
+          shape: :winning_trace
+        )
+      end
+      m = reward.generator_mix
+      expect(m[:n]).to be >= 5
+      expect(m[:shares]).to be_a(Hash)
+      expect(m[:targets]).to eq(reward::TARGET_SOURCE_MIX)
+      expect(m[:urgent]).to include('counterfactual') # 0 share << 0.20 target
+      expect(m[:healthy]).to eq(false)
+      expect(m[:recommendation]).to match(/boost:|need_/)
+    end
+
+    it 'write_source_quota reports underfilled against TARGET_SOURCE_MIX' do
+      12.times do |i|
+        reward.record_preference(
+          prompt: "q#{i}",
+          rejected: "nope #{'y' * 40}",
+          chosen: "STRATEGY: a\nWINNING_TRACE: t\nFINAL: ok #{i}",
+          source: :curriculum,
+          shape: :winning_trace
+        )
+      end
+      q = reward.write_source_quota(source: 'counterfactual')
+      expect(q[:underfilled]).to eq(true)
+      expect(q[:target]).to eq(0.20)
+    end
+
+    it 'counterfactual and critic force-run when generator_mix marks them urgent' do
+      allow(reward).to receive(:generator_mix).and_return(
+        urgent: %w[counterfactual critic], suppress: [], healthy: false, n: 20,
+        recommendation: 'boost:counterfactual,critic'
+      )
+      # env flags OFF — mix_need should still open the gate
+      @agent_cfg[:counterfactual] = false
+      @agent_cfg[:critic] = false
+      allow(curriculum).to receive(:in_curriculum?).and_return(false)
+      allow(curriculum).to receive(:reflect_available?).and_return(false)
+
+      # counterfactual: stubs the expensive branch; we only assert it does not
+      # short-circuit on disabled when mix_need is true. If still nil for other
+      # reasons, at least enabled? gate is not the cause.
+      allow(curriculum).to receive(:enabled?).and_call_original
+      # critic text_only always works once gate opens
+      r = curriculum.critic(request: 'q', final: 'a solid factual answer', text_only: true)
+      expect(r[:source].to_s).not_to eq('disabled')
+    end
+  end
+
+  describe 'P0 · Introspect budget (soft/hard ms + stages_skipped)' do
+    it 'defines INTROSPECT_SOFT_MS < INTROSPECT_HARD_MS' do
+      expect(learning::INTROSPECT_SOFT_MS).to be < learning::INTROSPECT_HARD_MS
+      expect(learning::INTROSPECT_SOFT_MS).to be_between(500, 10_000)
+    end
+
+    it 'auto_introspect returns stages_skipped and respects hard budget' do
+      @agent_cfg[:auto_introspect] = true
+      # Force over_hard by collapsing thresholds
+      stub_const('PWN::AI::Agent::Learning::INTROSPECT_SOFT_MS', 0)
+      stub_const('PWN::AI::Agent::Learning::INTROSPECT_HARD_MS', 0)
+      allow(loop_mod).to receive(:budget_exhaustion_hot?).and_return(false)
+
+      sid = 'spec_budget_introspect'
+      dir = File.join(@tmp, 'sessions')
+      FileUtils.mkdir_p(dir)
+      payload = [
+        { role: 'user', content: 'uname -a', timestamp: Time.now.utc.iso8601 },
+        { role: 'assistant', content: 'Linux kali', timestamp: Time.now.utc.iso8601 }
+      ].map { |h| JSON.generate(h) }.join("\n")
+      File.write(File.join(dir, "#{sid}.jsonl"), "#{payload}\n")
+
+      r = learning.auto_introspect(session_id: sid, request: 'uname -a', final: 'Linux kali')
+      expect(r).to be_a(Hash)
+      expect(r).to have_key(:stages_run)
+      expect(r).to have_key(:stages_skipped)
+      expect(r[:stages_skipped]).to include(:prm) # hard skip
+      expect(r[:elapsed_ms]).to be_a(Numeric)
+    end
+  end
+
+  describe 'P1 · Local judge calibration (shrinkage + confidence)' do
+    it 'heuristic judge returns confidence and shrinks extreme local scores' do
+      v = reward.judge(
+        request: 'what is the kernel',
+        final: 'The Linux kernel is a monolithic unix-like kernel used by kali.',
+        trace: [],
+        proxy_ok: true,
+        commit: false
+      )
+      expect(v[:source].to_s).to eq('heuristic')
+      expect(v[:confidence]).to be <= 0.5
+      expect(v).to have_key(:score_raw)
+      # local + no-trace cap pulls highs down
+      expect(v[:score]).to be <= 0.45 if v[:score_raw].to_f >= 0.6
+      expect(v[:rationale].to_s).to match(/P1:local_no_trace_cap|heuristic/)
+    end
+
+    it 'Metrics.effective_rate scales distrust by judge_confidence' do
+      # establish proxy success
+      10.times { metrics.record(name: 'shell', success: true, duration: 0.01) }
+      # high judge scores but low confidence (heuristic)
+      8.times { metrics.record_judge(name: 'shell', score: 0.2, confidence: 0.35) }
+      allow(metrics).to receive(:proxy_trust).and_return(0.3) # distrust 0.7
+      rate = metrics.effective_rate(name: 'shell')
+      conf = metrics.judge_confidence(name: 'shell')
+      expect(conf).to be_within(0.05).of(0.35)
+      # With conf=0.35, eff_d = 0.7*0.35 = 0.245 — still blended, not pure judge
+      # pure judge would be ~0.2; pure proxy ~1.0; blend must sit between
+      expect(rate).to be > 0.2
+      expect(rate).to be < 1.0
+    end
+  end
+
+  describe 'P1 · Practice outer KPI (practice_kpi + repeating_trend)' do
+    it 'practice_kpi snapshots repeating counts and writes KPI_FILE' do
+      mistakes.record(tool: 'agent_loop', error: 'iteration budget exhausted without a final answer')
+      3.times { mistakes.record(tool: 'agent_loop', error: 'iteration budget exhausted without a final answer') }
+      r = curriculum.practice_kpi(results: [{ resolved: true, mean_score: 0.8 }])
+      expect(r[:repeating_n]).to be >= 1
+      expect(r[:budget_repeating_n]).to be >= 1
+      expect(r[:practiced]).to eq 1 # len(results) recorded tonight
+      expect(r[:resolved_tonight]).to eq 1
+      expect(File.exist?(curriculum::KPI_FILE)).to eq(true)
+      expect(r[:trend]).to be_a(Hash)
+    end
+
+    it 'repeating_trend reports baseline then delta across snapshots' do
+      curriculum.practice_kpi(results: [])
+      # bump repeating by recording more
+      4.times { mistakes.record(tool: 'assistant_answer', error: 'critic: [pwn-ai] iteration budget exhausted') }
+      curriculum.practice_kpi(results: [])
+      t = curriculum.repeating_trend
+      expect(t[:samples]).to be >= 2
+      expect(t).to have_key(:delta_repeating)
+      expect(%i[improving flat regressing baseline]).to include(t[:status])
+    end
+  end
+
+  describe 'P2 · PRM sample efficiency (PRM_MIN_N / fleet coverage)' do
+    it 'prm_advantage is 0 until n >= PRM_MIN_N then applies shrinkage' do
+      expect(metrics::PRM_MIN_N).to eq 5
+      expect(metrics::PRM_FULL_N).to eq 20
+      4.times { metrics.record_step_reward(name: 'shell', reward: 1.0) }
+      expect(metrics.prm_n(name: 'shell')).to eq 4
+      expect(metrics.prm_advantage(name: 'shell')).to eq 0.0
+      metrics.record_step_reward(name: 'shell', reward: 1.0)
+      expect(metrics.prm_n(name: 'shell')).to eq 5
+      # single-tool zero-variance → damp*shrink keeps |adv| small but defined
+      adv = metrics.prm_advantage(name: 'shell')
+      expect(adv).to be_a(Numeric)
+    end
+
+    it 'Registry.rank drops PRM delta until ≥3 tools are PRM-ready' do
+      # only 1 tool ready → delta 0; rank still returns by keyword
+      10.times { metrics.record_step_reward(name: 'shell', reward: 1.0) }
+      expect(metrics.prm_n(name: 'shell')).to be >= metrics::PRM_MIN_N
+      ranked = registry.rank(query: 'run a shell command on the host')
+      expect(ranked.map(&:name)).to include('shell')
+      # spin up 3 ready tools — rank path must not raise
+      %w[shell memory_recall sessions_list].each do |n|
+        10.times { metrics.record_step_reward(name: n, reward: 0.5) }
+      end
+      expect do
+        registry.rank(query: 'run a shell command on the host')
+      end.not_to raise_error
+    end
+  end
+
+  describe 'P2 · STATUS table is the flag authority (no archaeology)' do
+    it 'Reinforcement-Learning.md ships a Design-priority STATUS section' do
+      doc_path = File.expand_path('../../documentation/Reinforcement-Learning.md', __dir__)
+      expect(File.exist?(doc_path)).to eq(true), "missing #{doc_path}"
+      doc = File.read(doc_path)
+      expect(doc).to match(/Design-priority STATUS/)
+      expect(doc).to match(/generator_mix/)
+      expect(doc).to match(/INTROSPECT_SOFT_MS/)
+      expect(doc).to match(/practice_kpi/)
+      expect(doc).to match(/PRM_MIN_N/)
+    end
+  end
+
+  # Design-priority ops closure — make STATUS success criteria reachable on-host
+  describe 'P0 ops · nightly diet close + shape backfill + mix in prompt' do
+    it 'infer_shape tags legacy winning_trace-like chosen sides' do
+      row = {
+        source: 'mistakes_resolve',
+        chosen: "shell → {\"success\":true}\npwn_eval → ok\n#{'x' * 80}",
+        rejected: "failed earlier #{'y' * 80}",
+        shape: ''
+      }
+      s = reward.infer_shape(row: row)
+      expect(s).to eq('winning_trace')
+    end
+
+    it 'infer_shape tags long critic revisions as revised_answer' do
+      row = {
+        source: 'critic',
+        chosen: 'A' * 250,
+        rejected: 'B' * 100,
+        shape: nil
+      }
+      expect(reward.infer_shape(row: row)).to eq('revised_answer')
+    end
+
+    it 'scrub_preferences backfills shape on kept rows when dry_run:false path runs over tmp' do
+      # exercise infer_shape + usable path without rewriting the live ledger
+      raw = {
+        source: 'curriculum',
+        prompt: 'finish under 5 tools',
+        rejected: "iteration budget exhausted #{'z' * 80}",
+        chosen: "shell → ok\nmemory_recall → hit\n#{'w' * 100}",
+        shape: ''
+      }
+      expect(reward.usable_preference?(row: raw.merge(shape: 'winning_trace'))).to eq(true)
+      inferred = reward.infer_shape(row: raw)
+      expect(reward::TRAJECTORY_SHAPES).to include(inferred)
+    end
+
+    it 'offline_judge return hash reserves scrub/generator_mix/practice_kpi keys' do
+      # dry structural contract: method source must call scrub + mix + kpi
+      src = begin
+        File.read(curriculum.instance_method(:offline_judge).source_location.first)
+      rescue StandardError
+        File.read('lib/pwn/ai/agent/curriculum.rb')
+      end
+      expect(src).to match(/scrub_preferences/)
+      expect(src).to match(/generator_mix/)
+      expect(src).to match(/practice_kpi/)
+    end
+
+    it 'Metrics.to_context surfaces W1 MIX when generator_mix is unhealthy' do
+      allow(reward).to receive(:generator_mix).and_return(
+        n: 20, healthy: false, trajectory_fraction: 0.1,
+        urgent: %w[critic counterfactual], suppress: %w[mistakes_resolve],
+        recommendation: 'boost:critic,counterfactual'
+      )
+      # ensure at least one metrics row so to_context is non-empty
+      metrics.record(name: 'shell', success: true, duration: 0.01) if metrics.respond_to?(:record)
+      ctx = metrics.to_context(limit: 3)
+      expect(ctx).to match(/W1 MIX:/)
+      expect(ctx).to match(/boost:critic/)
+    end
+  end
+
+  describe 'P0 · Budget exhaust deepen (last-iter / no-CF-hot / exhaust Learning)' do
+    it 'tightens budget-hot max_iters caps to 8 local / 12 remote' do
+      src = File.read(loop_mod.method(:run).source_location.first)
+      # max_iters is private_class_method — read surrounding source
+      expect(src).to match(/ollama \? 8 : 12/)
+      expect(src).to match(/budget_exhaustion_hot\?/)
+    end
+
+    it 'forces tools=nil on last iter and skips counterfactual when budget-hot' do
+      src = File.read(loop_mod.method(:run).source_location.first)
+      expect(src).to match(/FINAL ITERATION/)
+      expect(src).to match(/last_iter \? nil : tools/)
+      # CF gate requires !budget_exhaustion_hot?
+      expect(src).to match(/!budget_exhaustion_hot\?/)
+      expect(src).to match(/Curriculum\.counterfactual/)
+    end
+
+    it 'exhaust path appends session + auto_introspect (not bare string only)' do
+      src = File.read(loop_mod.method(:run).source_location.first)
+      idx = src.index('shape: :budget_exhausted')
+      expect(idx).not_to be_nil
+      tail = src[idx, 800]
+      expect(tail).to match(/append_session/)
+      expect(tail).to match(/auto_introspect/)
+      expect(tail).to match(/final_msg/)
+    end
+
+    it 'STATUS table lists Budget exhaust deepen' do
+      doc = File.read(File.expand_path('../../documentation/Reinforcement-Learning.md', __dir__))
+      expect(doc).to match(/Budget exhaust deepen/)
+      expect(doc).to match(/Last-iter force-final/)
+    end
+  end
 end
 # rubocop:enable Metrics/BlockLength