pwn 0.5.642 → 0.5.647

data/lib/pwn/ai/agent/learning.rb

@@ -168,7 +168,10 @@ module PWN
           # C2 — strict success:true only (excludes HER success:'soft'). Also
           # down-weight any residual hindsight-tagged rows so partial failures
           # never launder into full-strength few-shot exemplars.
+          # P20 — strict success:true AND prefer high judge scores. Drop rows
+          # with explicit low ORM score so proxy-true / judge-low cannot be few-shot.
           pool = outcomes(limit: 500, success: true).reject { |r| r[:session_id].to_s.empty? }
+          pool = pool.reject { |r| r.key?(:score) && r[:score].to_f < 0.6 }
           scored = pool.map do |r|
             sim   = tokens.count { |t| r[:task].to_s.downcase.include?(t) }.to_f / tokens.length
             age_d = (now - Time.parse(r[:timestamp].to_s)) / 86_400.0
@@ -202,26 +205,65 @@ module PWN
         # Modelfile` over the export - the only path to ACTUAL parity with a
         # frontier model, because it changes the weights not just the scaffold.
 
+        # P12 — SFT quality gate (as hard as DPO source-cap): drop HER/soft,
+        # low judge_score, auto-only noise without score, and PRM-compress
+        # trajectories so LoRA is not 5MB of "how we flailed".
+        SFT_MIN_SCORE = 0.6
+        SFT_MAX_TOOL_CHARS = 1_200
+
         public_class_method def self.export_finetune(opts = {})
           fmt       = (opts[:format] || :sharegpt).to_sym
           min_tools = (opts[:min_tools] || 1).to_i
+          min_score = (opts[:min_score] || SFT_MIN_SCORE).to_f
+          compress  = opts.key?(:compress) ? opts[:compress] : true
           FileUtils.mkdir_p(FINETUNE_DIR)
           out = opts[:out] || File.join(FINETUNE_DIR, "pwn-#{Time.now.utc.strftime('%Y%m%d')}.jsonl")
 
-          # 4.1 — exclude HER soft-success from SFT gold (success: 'soft' or tags)
+          # 4.1 / P12 — exclude HER soft-success + low-score + untagged auto flail
           gold = outcomes(limit: 10_000, success: true).reject do |r|
-            r[:success].to_s == 'soft' ||
-              Array(r[:tags]).any? { |t| %w[hindsight her].include?(t.to_s) }
+            tags = Array(r[:tags]).map(&:to_s)
+            soft = r[:success].to_s == 'soft' || tags.intersect?(%w[hindsight her soft])
+            low  = !r[:score].nil? && r[:score].to_f < min_score
+            # require a score when present in corpus era that has scores
+            soft || low
+          end
+          # prefer highest-score outcome per session
+          by_sid = {}
+          gold.each do |r|
+            sid = r[:session_id].to_s
+            next if sid.empty?
+
+            prev = by_sid[sid]
+            by_sid[sid] = r if prev.nil? || r[:score].to_f >= prev[:score].to_f
           end
-          sids = gold.map { |r| r[:session_id] }.compact.uniq
+          sids = by_sid.keys
           rows = 0
+          dropped = { tools: 0, empty: 0, load: 0 }
           File.open(out, 'w') do |f|
             sids.each do |sid|
-              t = PWN::Sessions.load(session_id: sid)
-              next if t.count { |e| e[:role].to_s == 'tool' } < min_tools
+              t = begin
+                PWN::Sessions.load(session_id: sid)
+              rescue StandardError
+                dropped[:load] += 1
+                next
+              end
+              tool_n = t.count { |e| e[:role].to_s == 'tool' }
+              if tool_n < min_tools
+                dropped[:tools] += 1
+                next
+              end
+
+              conv = if compress
+                       compress_finetune_trace(transcript: t, max_tool_chars: SFT_MAX_TOOL_CHARS)
+                     else
+                       t.map { |e| { role: e[:role].to_s, content: e[:content].to_s } }
+                        .reject { |e| e[:role] == 'system' && e[:content].start_with?('Session started') }
+                     end
+              if conv.nil? || conv.empty? || conv.none? { |m| m[:role].to_s == 'assistant' }
+                dropped[:empty] += 1
+                next
+              end
 
-              conv = t.map { |e| { role: e[:role].to_s, content: e[:content].to_s } }
-                      .reject { |e| e[:role] == 'system' && e[:content].start_with?('Session started') }
               line = case fmt
                      when :openai_jsonl then { messages: conv }
                      else { conversations: conv.map { |m| { from: sharegpt_role(role: m[:role]), value: m[:content] } } }
@@ -230,7 +272,11 @@ module PWN
               rows += 1
             end
           end
-          { path: out, format: fmt, sessions: sids.length, samples: rows, bytes: File.size(out) }
+          {
+            path: out, format: fmt, sessions: sids.length, samples: rows,
+            bytes: File.size(out), min_score: min_score, compressed: compress,
+            dropped: dropped
+          }
         end
 
         # Supported Method Parameters::
@@ -317,6 +363,14 @@ module PWN
           # S3 — tool-armed constitutional critic runs BEFORE the reward
           # model so its verdict is evidence, not hindsight.
           # P7 — force critic when W3 says this engine is overconfident
+          # P24 — under budget_exhaustion_hot?, skip tool-armed critic entirely
+          # (or single-shot text) so the critic cannot re-thrash the budget.
+          budget_hot = begin
+            defined?(Loop) && Loop.respond_to?(:budget_exhaustion_hot?, true) &&
+              Loop.send(:budget_exhaustion_hot?)
+          rescue StandardError
+            false
+          end
           force_critic = begin
             eng = (PWN::Env.dig(:ai, :active) if defined?(PWN::Env))
             cal = defined?(Metrics) ? Metrics.calibration(engine: eng) : { n: 0 }
@@ -324,7 +378,15 @@ module PWN
           rescue StandardError
             false
           end
-          crit = if defined?(Curriculum)
+          crit = if budget_hot && defined?(Curriculum)
+                   # P24 — cheap text-only critic: no persona tool swarm
+                   Curriculum.critic(
+                     request: opts[:request],
+                     final: opts[:final],
+                     session_id: session_id,
+                     text_only: true
+                   )
+                 elsif defined?(Curriculum)
                    if force_critic
                      prev = (PWN::Env[:ai][:agent][:critic] if defined?(PWN::Env) && PWN::Env[:ai].is_a?(Hash) && PWN::Env[:ai][:agent].is_a?(Hash))
                      begin
@@ -349,7 +411,14 @@ module PWN
           # user correction on the previous turn.
           pend = Thread.current[:pwn_pending_pref]
           if pend && ok && defined?(Reward)
-            Reward.record_preference(prompt: pend[:prompt], rejected: pend[:rejected], chosen: opts[:final].to_s, source: :user_correction)
+            Reward.record_preference(
+              prompt: pend[:prompt],
+              rejected: pend[:rejected],
+              chosen: opts[:final].to_s,
+              source: :user_correction,
+              shape: :revised_answer,
+              force: true
+            )
             Thread.current[:pwn_pending_pref] = nil
           end
 
@@ -361,13 +430,22 @@ module PWN
             session_id: session_id,
             tags: ['auto', 'loop', v[:verdict].to_s]
           )
+          # P20 — fold episode ORM score into per-tool Metrics so UCB/Thompson
+          # /advantage track judge, not only handler-ok. Attribution: every
+          # tool touched this session gets the episode score (coarse but
+          # closes the bandit onto the north-star scalar).
+          fold_judge_into_metrics(session_id: session_id, score: v[:score])
           # R2 — per-step credit assignment ALWAYS (1.6): failed trajectories
           # are where step credit matters; feed negative steps into counterfactual.
           # C3 — HER soft-relabel on failure only.
           Reward.prm(request: opts[:request], session_id: session_id) if defined?(Reward)
           Curriculum.hindsight(request: opts[:request], final: opts[:final], session_id: session_id) if !ok && defined?(Curriculum)
-          # W3 — calibration: predicted (from plan_first) vs actual
-          Curriculum.calibrate(predicted: opts[:predicted], actual: v[:score], engine: PWN::Env.dig(:ai, :active)) if opts[:predicted] && defined?(Curriculum)
+          # W3 — calibration: predicted (from plan_first) vs actual.
+          # Always try: also recover p(success)= from session PLAN if the
+          # live return value was lost (nil) but plan_first did emit one.
+          predicted = opts[:predicted]
+          predicted = recover_predicted_from_session(session_id: session_id) if predicted.nil?
+          Curriculum.calibrate(predicted: predicted, actual: v[:score], engine: PWN::Env.dig(:ai, :active)) if !predicted.nil? && defined?(Curriculum)
           reflect(session_id: session_id) if ok
           Reward.sentinel if defined?(Reward)
           Extrospection.auto_extrospect(session_id: session_id) if defined?(Extrospection)
@@ -473,6 +551,46 @@ module PWN
         # privates
         # -------------------------------------------------------------
 
+        # P20 — attribute episode judge score to every tool used in session.
+        private_class_method def self.fold_judge_into_metrics(opts = {})
+          return unless defined?(Metrics) && Metrics.respond_to?(:record_judge)
+
+          sid = opts[:session_id]
+          score = opts[:score]
+          return if sid.to_s.empty? || score.nil?
+          return unless defined?(PWN::Sessions)
+
+          names = PWN::Sessions.load(session_id: sid)
+                               .select { |e| e[:role].to_s == 'tool' }
+                               .map { |e| e[:content].to_s[/\A([a-z0-9_]+)\s*→/i, 1] }
+                               .compact
+                               .uniq
+          names.each { |n| Metrics.record_judge(name: n, score: score) }
+        rescue StandardError
+          nil
+        end
+
+        # P22 — recover plan_first p(success)= from session transcript when
+        # the live return floated away (rescue path / degrade).
+        private_class_method def self.recover_predicted_from_session(opts = {})
+          # Prefer live stash from plan_first in this process.
+          stash = Thread.current[:pwn_plan_predicted]
+          return stash.to_f.clamp(0.0, 1.0) unless stash.nil?
+
+          sid = opts[:session_id]
+          return nil if sid.to_s.empty? || !defined?(PWN::Sessions)
+
+          entries = PWN::Sessions.load(session_id: sid)
+          plan = entries.reverse.find do |e|
+            e[:role].to_s == 'assistant' && e[:content].to_s.match?(/\bPLAN:\b|p\(success\)\s*=/i)
+          end
+          return nil unless plan
+
+          plan[:content].to_s[/p\(success\)\s*=\s*([01](?:\.\d+)?)/i, 1]&.to_f
+        rescue StandardError
+          nil
+        end
+
         private_class_method def self.auto_introspect_enabled?
           return false unless defined?(PWN::Env) && PWN::Env.is_a?(Hash)
 
@@ -578,6 +696,34 @@ module PWN
           warn "[pwn-ai/learning] fact_check swallowed: #{e.class}: #{e.message}"
         end
 
+        # P12 — PRM-aware SFT trace: keep user + reward>0 tools (else first N) +
+        # final assistant, truncate tool payloads. Drops system noise.
+        private_class_method def self.compress_finetune_trace(opts = {})
+          t = Array(opts[:transcript])
+          cap = (opts[:max_tool_chars] || SFT_MAX_TOOL_CHARS).to_i
+          fin_idx = t.rindex { |e| e[:role].to_s == 'assistant' && !e[:content].to_s.strip.empty? }
+          return [] if fin_idx.nil?
+
+          user_idx = t[0...fin_idx].rindex { |e| e[:role].to_s == 'user' }
+          return [] if user_idx.nil?
+
+          window = t[user_idx..fin_idx]
+          tools = window.select { |e| e[:role].to_s == 'tool' }
+          rewarded = tools.select { |e| e[:step_reward].to_i.positive? }
+          tools = rewarded unless rewarded.empty?
+          tools = tools.first(8)
+
+          out = []
+          out << { role: 'user', content: t[user_idx][:content].to_s[0, 2_000] }
+          tools.each do |e|
+            out << { role: 'tool', content: e[:content].to_s[0, cap] }
+          end
+          out << { role: 'assistant', content: t[fin_idx][:content].to_s[0, 4_000] }
+          out
+        rescue StandardError
+          []
+        end
+
         private_class_method def self.compress_exemplar(opts = {})
           sid = opts[:session_id]
           cap = opts[:max_msgs] || 6

data/lib/pwn/ai/agent/loop.rb

@@ -51,6 +51,10 @@ module PWN
       module Loop
         DEFAULT_MAX_ITERS    = 777
         ESCALATE_AFTER_FAILS = 4
+        # P17 — when empty_final / known thrash shapes dominate, stop before
+        # burning the full ollama cap so the corpus is not pure terminal failure.
+        BUDGET_HARD_STOP_FAILS = 8
+        BUDGET_EMPTY_FINAL_STOP = 3
 
         ENGINE_MODS = {
           openai: 'PWN::AI::OpenAI',
@@ -77,6 +81,25 @@ module PWN
           { role: 'assistant', content: txt, tool_calls: [] }
         end
 
+        # P17 — true when unresolved agent_loop / assistant_answer budget
+        # fingerprints dominate Mistakes.top (the #1 live failure mode).
+        private_class_method def self.budget_exhaustion_hot?
+          return false unless defined?(Mistakes)
+
+          top = Mistakes.top(limit: 8, unresolved_only: true)
+          return false if top.empty?
+
+          budgetish = top.count do |m|
+            t = m[:tool].to_s
+            e = m[:error].to_s.downcase
+            t == 'agent_loop' || t == 'assistant_answer' ||
+              e.include?('budget exhausted') || e.include?('iteration budget')
+          end
+          budgetish >= 2 || (budgetish >= 1 && top.first[:tool].to_s == 'agent_loop')
+        rescue StandardError
+          false
+        end
+
         private_class_method def self.max_iters
           v = (PWN::Env.dig(:ai, :agent, :max_iters) if defined?(PWN::Env))
           n = v.to_i.positive? ? v.to_i : DEFAULT_MAX_ITERS
@@ -87,6 +110,11 @@ module PWN
           # shrink the tool budget so thrash can't compound on bad plans.
           cal = calibration_state
           n = [n, cal[:max_iters_cap]].min if cal[:overconfident]
+          # P17 — tighter default when agent_loop budget_exhaustion dominates
+          # open mistakes: finish-under-N is the skill gap, not more thrash.
+          if budget_exhaustion_hot?
+            n = [n, active_engine == :ollama ? 12 : 20].min
+          end
           n
         rescue StandardError
           DEFAULT_MAX_ITERS
@@ -247,8 +275,15 @@ module PWN
             rt = Curriculum.red_team_plan(request: opts[:request], plan: plan)
             messages << { role: 'user', content: rt } if rt
           end
-          # W3 — extract predicted p(success) for calibration tracking
-          plan[/p\(success\)\s*=\s*([01](?:\.\d+)?)/i, 1]&.to_f
+          # W3/P22 — extract predicted p(success) for calibration tracking.
+          # Accept p(success)=0.7 | p(success) = .7 | confidence=0.7 on last lines.
+          predicted = plan[/p\(\s*success\s*\)\s*=\s*([01]?(?:\.\d+)?)/i, 1]&.to_f
+          predicted = plan[/\bconfidence\s*=\s*([01]?(?:\.\d+)?)/i, 1]&.to_f if predicted.nil?
+          predicted = predicted.clamp(0.0, 1.0) if predicted
+          # Stash so auto_introspect / recover can always see it even if the
+          # return value is dropped by a caller rescue.
+          Thread.current[:pwn_plan_predicted] = predicted
+          predicted
         rescue StandardError => e
           warn "[pwn-ai/loop] plan_first swallowed: #{e.class}: #{e.message}"
           nil
@@ -498,9 +533,20 @@ module PWN
           append_session(session_id: session_id, role: 'user', content: request)
 
           predicted = nil
+          Thread.current[:pwn_plan_predicted] = nil
           cal_state = calibration_state
           force_plan = cal_state[:force_plan]
-          predicted = plan_first(messages: messages, request: request) if (force_plan || agent_flag(key: :plan_first, default: local)) && !Array(tools).empty?
+          if (force_plan || agent_flag(key: :plan_first, default: local) || budget_exhaustion_hot?) && !Array(tools).empty?
+            predicted = plan_first(messages: messages, request: request)
+            # P22 — prefer explicit return; fall back to thread stash
+            predicted = Thread.current[:pwn_plan_predicted] if predicted.nil?
+          end
+          if budget_exhaustion_hot?
+            messages << {
+              role: 'user',
+              content: '[pwn-ai/p17] Budget-exhaustion is the top open failure on this host. Prefer the SHORTEST plan that finishes the ask (≤3 tool calls). Emit a final answer as soon as you have evidence — do not explore.'
+            }
+          end
           if force_plan && cal_state[:cal]
             messages << {
               role: 'user',
@@ -589,6 +635,31 @@ module PWN
               )
             end
 
+            # P17 — hard stop: empty-final thrash or cumulative fails past cap.
+            # Prefer a short apologetic final over another 10 useless tool dumps
+            # that poison ORM/PRM/DPO with terminal failures.
+            empty_n = turn_fails['empty_final'].to_i
+            fail_n  = turn_fails.values.sum
+            if empty_n >= BUDGET_EMPTY_FINAL_STOP || fail_n >= BUDGET_HARD_STOP_FAILS
+              msg = if empty_n >= BUDGET_EMPTY_FINAL_STOP
+                      '[pwn-ai] stopped: repeated empty finals (budget thrash guard)'
+                    else
+                      '[pwn-ai] stopped: too many in-turn failures (budget thrash guard)'
+                    end
+              if defined?(Mistakes)
+                Mistakes.record(
+                  tool: 'agent_loop',
+                  error: "budget thrash guard fired empty=#{empty_n} fails=#{fail_n} iter=#{i}",
+                  session_id: session_id,
+                  source: :loop,
+                  shape: :budget_thrash
+                )
+              end
+              append_session(session_id: session_id, role: 'assistant', content: msg)
+              Learning.auto_introspect(session_id: session_id, request: request, final: msg, predicted: predicted) if defined?(Learning) && should_auto_introspect?(local: local, turn_fails: turn_fails, iter: i)
+              return msg
+            end
+
             next unless local && !escalated && turn_fails.values.sum >= ESCALATE_AFTER_FAILS
 
             hint = escalate(request: request, turn_fails: turn_fails, session_id: session_id)
@@ -599,7 +670,7 @@ module PWN
             escalated = true
           end
 
-          Mistakes.record(tool: 'agent_loop', error: 'iteration budget exhausted without a final answer', session_id: session_id, source: :loop) if defined?(Mistakes)
+          Mistakes.record(tool: 'agent_loop', error: 'iteration budget exhausted without a final answer', session_id: session_id, source: :loop, shape: :budget_exhausted) if defined?(Mistakes)
           '[pwn-ai] iteration budget exhausted'
         end
 

data/lib/pwn/ai/agent/metrics.rb

@@ -122,6 +122,8 @@ module PWN
               name: name.to_s,
               calls: calls,
               success_rate: rate,
+              judge_rate: judge_rate(name: name),
+              effective_rate: effective_rate(name: name),
               avg_duration: avg,
               last_error: b[:last_error],
               last_at: b[:last_at]
@@ -149,12 +151,19 @@ module PWN
           scope = engine.to_s.empty? ? 'historical' : "engine=#{engine}"
           scope = "#{scope}, proxy_distrust=#{distrust.round(2)}" if distrust.positive?
           lines = rows.map do |r|
+            # P20 — display effective_rate (judge-blended) when available;
+            # fall back to distrust haircut on raw proxy.
             rate = r[:success_rate].to_f
-            # blend toward 0.5 (uninformative) proportional to distrust
-            adj = rate - ((rate - 0.5) * distrust)
+            eff  = r[:effective_rate]
+            adj  = if eff
+                     eff.to_f
+                   else
+                     rate - ((rate - 0.5) * distrust)
+                   end
             err = r[:last_error] ? " last_err=#{r[:last_error][0, 60]}" : ''
-            tag = distrust.positive? ? ' (adj)' : ''
-            "  - #{r[:name]}: calls=#{r[:calls]} success=#{(adj * 100).round(1)}%#{tag} avg=#{r[:avg_duration]}s#{err}"
+            jtag = r[:judge_rate] ? " judge=#{(r[:judge_rate].to_f * 100).round(1)}%" : ''
+            tag = distrust.positive? || r[:judge_rate] ? ' (adj)' : ''
+            "  - #{r[:name]}: calls=#{r[:calls]} success=#{(adj * 100).round(1)}%#{tag}#{jtag} avg=#{r[:avg_duration]}s#{err}"
           end
           warn_line = distrust.positive? ? "WARNING: reward proxy diverges from judge — success rates haircut by distrust=#{distrust.round(2)}; prefer judge-scored exemplars over raw rates.\n" : ''
           "#{warn_line}TOOL EFFECTIVENESS (#{scope}, adapt tool choice accordingly)\n#{lines.join("\n")}\n\n"
@@ -176,7 +185,8 @@ module PWN
           t    = data[name.to_sym] || blank_bucket
           n    = [t[:calls].to_f, 1.0].max
           total = [data.values.sum { |v| v[:calls].to_f }, 1.0].max
-          mean  = t[:ok].to_f / n
+          # P20 — mean from effective_rate (judge-blended when distrust high)
+          mean  = effective_rate(name: name)
           mean + (c * Math.sqrt(Math.log(total) / n))
         rescue StandardError
           1.0
@@ -190,7 +200,21 @@ module PWN
 
         public_class_method def self.thompson(opts = {})
           t = (load[:tools] || {})[opts[:name].to_s.to_sym] || blank_bucket
-          beta_sample(alpha: t[:ok].to_f + 1.0, beta: t[:fail].to_f + 1.0)
+          # P20 — when judge samples exist and distrust > 0, tilt Beta toward
+          # judge_rate so Thompson explore/exploit tracks ORM not handler-ok.
+          ok = t[:ok].to_f
+          fail = t[:fail].to_f
+          jn = Array(t[:judge_window]).length
+          if jn >= 3 && proxy_trust < 0.95
+            jr = judge_rate(name: opts[:name]).to_f
+            # pseudo-counts from judge window, mixed by distrust
+            d = (1.0 - proxy_trust).clamp(0.0, 1.0)
+            jok = jr * jn
+            jfail = (1.0 - jr) * jn
+            ok = (ok * (1.0 - d)) + (jok * d)
+            fail = (fail * (1.0 - d)) + (jfail * d)
+          end
+          beta_sample(alpha: ok + 1.0, beta: fail + 1.0)
         rescue StandardError
           0.5
         end
@@ -205,14 +229,117 @@ module PWN
           t    = data[opts[:name].to_s.to_sym]
           return 0.0 unless t
 
-          global = data.values.sum { |v| v[:ok].to_f } / [data.values.sum { |v| v[:calls].to_f }, 1.0].max
-          win    = Array(t[:window])
-          local  = win.empty? ? (t[:ok].to_f / [t[:calls].to_f, 1.0].max) : (win.sum.to_f / win.length)
+          # P20 — local/global from effective_rate so bandit tracks judge
+          # when the handler-ok proxy is hacked (proxy_distrust high).
+          local = effective_rate(name: opts[:name])
+          rates = data.keys.map { |k| effective_rate(name: k) }
+          global = rates.empty? ? 0.5 : (rates.sum / rates.length)
           (local - global).round(3)
         rescue StandardError
           0.0
         end
 
+        # P18 — rolling mean step_reward for a tool (from Reward.prm annotations
+        # folded into Metrics via record_step_reward). Range ≈ [-1, 1].
+        public_class_method def self.prm_advantage(opts = {})
+          data = load[:tools] || {}
+          t = data[opts[:name].to_s.to_sym]
+          return 0.0 unless t
+
+          win = Array(t[:prm_window])
+          return 0.0 if win.empty?
+
+          mean = win.sum.to_f / win.length
+          # global mean across tools that have prm signal
+          globals = data.values.map { |v| Array(v[:prm_window]) }.reject(&:empty?)
+          gmean = if globals.empty?
+                    0.0
+                  else
+                    all = globals.flatten
+                    all.sum.to_f / all.length
+                  end
+          (mean - gmean).round(3)
+        rescue StandardError
+          0.0
+        end
+
+        # P18 — called by Reward.prm after session annotate so live routing
+        # can bias toward tools that recently advanced goals (+1 step_reward).
+        public_class_method def self.record_step_reward(opts = {})
+          name = opts[:name].to_s
+          return if name.empty?
+
+          rew = opts[:reward].to_f.clamp(-1.0, 1.0)
+          m = load
+          m[:tools] ||= {}
+          t = m[:tools][name.to_sym] ||= blank_bucket
+          t[:prm_window] = (Array(t[:prm_window]) + [rew]).last(40)
+          t[:prm_sum] = t[:prm_window].sum
+          t[:prm_n] = t[:prm_window].length
+          save(metrics: m)
+          t
+        rescue StandardError
+          nil
+        end
+
+        # P20 — fold ORM judge (0..1) into per-tool telemetry so UCB /
+        # Thompson / advantage can prefer judge-grounded rates over the
+        # inflated handler-ok proxy when proxy_distrust is high.
+        public_class_method def self.record_judge(opts = {})
+          name = opts[:name].to_s
+          return if name.empty?
+
+          score = opts[:score].to_f.clamp(0.0, 1.0)
+          m = load
+          m[:tools] ||= {}
+          t = m[:tools][name.to_sym] ||= blank_bucket
+          t[:judge_window] = (Array(t[:judge_window]) + [score]).last(40)
+          t[:judge_sum] = t[:judge_window].sum.to_f
+          t[:judge_n] = t[:judge_window].length
+          save(metrics: m)
+          t
+        rescue StandardError
+          nil
+        end
+
+        # Mean judge score for a tool (nil when no ORM samples yet).
+        public_class_method def self.judge_rate(opts = {})
+          t = (load[:tools] || {})[opts[:name].to_s.to_sym]
+          return nil unless t
+
+          win = Array(t[:judge_window])
+          return nil if win.empty?
+
+          (win.sum.to_f / win.length).round(3)
+        rescue StandardError
+          nil
+        end
+
+        # Blended success rate: when proxy_distrust > 0 and judge samples
+        # exist, mix judge_rate into the handler-ok rate. distrust=1 → pure
+        # judge (or 0.5 if no judge data). distrust=0 → pure proxy.
+        public_class_method def self.effective_rate(opts = {})
+          name = opts[:name].to_s
+          data = load[:tools] || {}
+          t = data[name.to_sym]
+          return 0.5 unless t
+
+          calls = [t[:calls].to_f, 1.0].max
+          proxy = t[:ok].to_f / calls
+          win = Array(t[:window])
+          proxy = win.sum.to_f / win.length if win.length >= 3
+          distrust = 1.0 - proxy_trust
+          jrate = judge_rate(name: name)
+          if distrust > 0.05 && !jrate.nil?
+            # blend: higher distrust → more judge weight
+            ((proxy * (1.0 - distrust)) + (jrate * distrust)).clamp(0.0, 1.0).round(3)
+          else
+            proxy.round(3)
+          end
+        rescue StandardError
+          0.5
+        end
+
         # Supported Method Parameters::
         # cps = PWN::AI::Agent::Metrics.changepoints
         #
@@ -360,7 +487,10 @@ module PWN
               PWN::AI::Agent::Metrics.to_context(limit: 8, engine: :ollama)   # injected by PromptBuilder
               PWN::AI::Agent::Metrics.ucb(name: 'shell')                 # C1 exploration bonus
               PWN::AI::Agent::Metrics.thompson(name: 'shell')            # C1 Beta(ok+1,fail+1) sample
-              PWN::AI::Agent::Metrics.advantage(name: 'shell')           # C1 local − global
+              PWN::AI::Agent::Metrics.advantage(name: 'shell')           # C1 local − global (P20 judge-blended)
+              PWN::AI::Agent::Metrics.record_judge(name: 'shell', score: 0.8) # P20 ORM→metrics
+              PWN::AI::Agent::Metrics.judge_rate(name: 'shell')         # P20 mean ORM
+              PWN::AI::Agent::Metrics.effective_rate(name: 'shell')     # P20 proxy⋈judge
               PWN::AI::Agent::Metrics.changepoints(within_secs: 3600)    # E1 CUSUM regime changes
               PWN::AI::Agent::Metrics.record_calibration(predicted: 0.8, actual: 1.0, brier: 0.04, engine: :ollama)
               PWN::AI::Agent::Metrics.calibration(engine: :ollama)       # W3 Brier / overconfidence

data/lib/pwn/ai/agent/mistakes.rb

@@ -257,15 +257,33 @@ module PWN
               importance: 0.9
             )
           end
-          # W1 — every resolve is a naturally-generated preference pair:
-          # (rejected: the failing action, chosen: the fix).
+          # W1/P9 — every resolve is a preference pair. Prefer structured
+          # winning_trace (+ strategy/tool) over first-line fix prose so DPO
+          # learns tool trajectories, not commentary.
           if defined?(Reward)
-            Reward.record_preference(
-              prompt: "#{store[key][:tool]}: #{store[key][:error]}",
-              rejected: store[key][:snippet].to_s,
-              chosen: fix.strip,
-              source: :mistakes_resolve
-            )
+            sf = store[key][:structured_fix] || {}
+            trace = sf[:winning_trace].to_s.strip
+            strat = [sf[:strategy], sf[:tool], sf[:args_template]].compact.map(&:to_s).reject(&:empty?).join(' | ')
+            # P21/P25 — only write W1 pairs when we have a real winning_trace.
+            # Prose-only resolve still updates Memory lesson + structured_fix;
+            # it must NOT flood DPO with fix commentary (shape: :fix_prose).
+            if trace.length >= 40
+              parts = []
+              parts << "STRATEGY: #{strat}" unless strat.empty?
+              parts << "WINNING_TRACE:\n#{trace[0, 3_500]}"
+              parts << "FIX: #{fix.strip[0, 400]}"
+              chosen = parts.join("\n")
+              rejected = store[key][:snippet].to_s
+              rejected = "FAILING: tool=#{store[key][:tool]} err=#{store[key][:error]}" if rejected.strip.empty?
+              Reward.record_preference(
+                prompt: "#{store[key][:tool]}: #{store[key][:error]}",
+                rejected: rejected,
+                chosen: chosen,
+                source: :mistakes_resolve,
+                shape: :winning_trace,
+                meta: { signature: sig, strategy: sf[:strategy], tool: sf[:tool] }.compact
+              )
+            end
           end
           store[key]
         end

data/lib/pwn/ai/agent/registry.rb

@@ -136,7 +136,7 @@ module PWN
 
           tokens = query.scan(/[a-z0-9_]{3,}/).uniq
           # C1 — advantage-weighted router:
-          #   score = α·keyword_sim + β·advantage + γ·UCB(tool)
+          #   score = α·keyword_sim + β·advantage + γ·UCB(tool) + δ·prm_advantage
           # UCB gives untried / low-N tools an exploration bonus so a single
           # early failure (before its dep was installed) does not blacklist
           # it forever; advantage prefers tools that outperform the fleet.
@@ -145,12 +145,15 @@ module PWN
           trust = defined?(Metrics) && Metrics.respond_to?(:proxy_trust) ? Metrics.proxy_trust : 1.0
           beta  = 0.3 * trust
           gamma = 0.2
+          # P18 — PRM step_reward advantage closes R2 into the controller
+          delta = 0.25 * trust
           scored = entries.map do |e|
             hay   = "#{e.name} #{e.toolset} #{e.schema[:description]} #{Array(e.schema.dig(:parameters, :properties)&.keys).join(' ')}".downcase
             sim   = tokens.count { |t| hay.include?(t) }
             adv   = defined?(Metrics) && Metrics.respond_to?(:advantage) ? Metrics.advantage(name: e.name) : 0.0
             ucb   = defined?(Metrics) && Metrics.respond_to?(:ucb) ? Metrics.ucb(name: e.name) : 0.5
-            [e, sim, (alpha * sim) + (beta * adv) + (gamma * ucb)]
+            prm   = defined?(Metrics) && Metrics.respond_to?(:prm_advantage) ? Metrics.prm_advantage(name: e.name) : 0.0
+            [e, sim, (alpha * sim) + (beta * adv) + (gamma * ucb) + (delta * prm)]
           end
           scored.reject { |_, sim, _| sim.zero? }
                 .sort_by { |_, _, s| -s }