pwn 0.5.643 → 0.5.650

data/lib/pwn/ai/agent/loop.rb

@@ -51,6 +51,10 @@ module PWN
       module Loop
         DEFAULT_MAX_ITERS    = 777
         ESCALATE_AFTER_FAILS = 4
+        # P17 — when empty_final / known thrash shapes dominate, stop before
+        # burning the full ollama cap so the corpus is not pure terminal failure.
+        BUDGET_HARD_STOP_FAILS = 8
+        BUDGET_EMPTY_FINAL_STOP = 3
 
         ENGINE_MODS = {
           openai: 'PWN::AI::OpenAI',
@@ -77,6 +81,25 @@ module PWN
           { role: 'assistant', content: txt, tool_calls: [] }
         end
 
+        # P17 — true when unresolved agent_loop / assistant_answer budget
+        # fingerprints dominate Mistakes.top (the #1 live failure mode).
+        private_class_method def self.budget_exhaustion_hot?
+          return false unless defined?(Mistakes)
+
+          top = Mistakes.top(limit: 8, unresolved_only: true)
+          return false if top.empty?
+
+          budgetish = top.count do |m|
+            t = m[:tool].to_s
+            e = m[:error].to_s.downcase
+            t == 'agent_loop' || t == 'assistant_answer' ||
+              e.include?('budget exhausted') || e.include?('iteration budget')
+          end
+          budgetish >= 2 || (budgetish >= 1 && top.first[:tool].to_s == 'agent_loop')
+        rescue StandardError
+          false
+        end
+
         private_class_method def self.max_iters
           v = (PWN::Env.dig(:ai, :agent, :max_iters) if defined?(PWN::Env))
           n = v.to_i.positive? ? v.to_i : DEFAULT_MAX_ITERS
@@ -87,6 +110,18 @@ module PWN
           # shrink the tool budget so thrash can't compound on bad plans.
           cal = calibration_state
           n = [n, cal[:max_iters_cap]].min if cal[:overconfident]
+          # P17 — tighter default when agent_loop budget_exhaustion dominates
+          # open mistakes: finish-under-N is the skill gap, not more thrash.
+          # Caps are intentionally harsh (8 local / 12 remote): the open
+          # fingerprint is "iteration budget exhausted" ×N; more headroom
+          # only produces more empty terminal failures for ORM/PRM/DPO.
+          if budget_exhaustion_hot?
+            # Remote/overconfident thrash (this host): 12 was still burning full
+            # 12-step tool plans. Cap tighter when calibration also says overconfident.
+            base = active_engine == :ollama ? 8 : 12
+            base = [base, 8].min if cal[:overconfident]
+            n = [n, base].min
+          end
           n
         rescue StandardError
           DEFAULT_MAX_ITERS
@@ -233,22 +268,43 @@ module PWN
         # without leaking to the user.
         private_class_method def self.plan_first(opts = {})
           messages = opts[:messages]
+          # P17 — under budget_exhaustion_hot the plan must be ultra-short:
+          # red_team_plan is a nested agent loop and was the #1 amplifier of
+          # iteration-budget exhaustion on this host (together with CF).
+          hot = begin
+            budget_exhaustion_hot?
+          rescue StandardError
+            false
+          end
+          plan_prompt = if hot
+                          'Before acting: write AT MOST 3 numbered tool calls (name + key args) that finish the ask. Prefer fewer. LAST line: "p(success)=<0.0-1.0>". Reply ONLY with the plan + that line — no tools, no prose.'
+                        else
+                          'Before acting: (1) list the exact tool calls (name + key args) you will make, in order; (2) on the LAST line write "p(success)=<0.0-1.0>". Reply ONLY with the numbered plan + that line — do NOT call any tool yet.'
+                        end
           plan_msg = call_engine(
-            messages: messages + [{ role: 'user',
-                                    content: 'Before acting: (1) list the exact tool calls (name + key args) you will make, in order; (2) on the LAST line write "p(success)=<0.0-1.0>". Reply ONLY with the numbered plan + that line — do NOT call any tool yet.' }],
+            messages: messages + [{ role: 'user', content: plan_prompt }],
             tools: nil
           )
           return nil unless plan_msg && !plan_msg[:content].to_s.strip.empty?
 
           plan = plan_msg[:content].to_s.strip
           messages << { role: 'assistant', content: "PLAN:\n#{plan}" }
-          # S4 — adversarial plan review grounded in THIS host's telemetry
-          if defined?(Curriculum)
+          # S4 — adversarial plan review grounded in THIS host's telemetry.
+          # P17 — never fork red_team when budget fingerprints dominate: it is
+          # another mini agent loop and compounds iteration-budget exhaustion.
+          if defined?(Curriculum) && !hot
             rt = Curriculum.red_team_plan(request: opts[:request], plan: plan)
             messages << { role: 'user', content: rt } if rt
           end
-          # W3 — extract predicted p(success) for calibration tracking
-          plan[/p\(success\)\s*=\s*([01](?:\.\d+)?)/i, 1]&.to_f
+          # W3/P22 — extract predicted p(success) for calibration tracking.
+          # Accept p(success)=0.7 | p(success) = .7 | confidence=0.7 on last lines.
+          predicted = plan[/p\(\s*success\s*\)\s*=\s*([01]?(?:\.\d+)?)/i, 1]&.to_f
+          predicted = plan[/\bconfidence\s*=\s*([01]?(?:\.\d+)?)/i, 1]&.to_f if predicted.nil?
+          predicted = predicted.clamp(0.0, 1.0) if predicted
+          # Stash so auto_introspect / recover can always see it even if the
+          # return value is dropped by a caller rescue.
+          Thread.current[:pwn_plan_predicted] = predicted
+          predicted
         rescue StandardError => e
           warn "[pwn-ai/loop] plan_first swallowed: #{e.class}: #{e.message}"
           nil
@@ -498,9 +554,20 @@ module PWN
           append_session(session_id: session_id, role: 'user', content: request)
 
           predicted = nil
+          Thread.current[:pwn_plan_predicted] = nil
           cal_state = calibration_state
           force_plan = cal_state[:force_plan]
-          predicted = plan_first(messages: messages, request: request) if (force_plan || agent_flag(key: :plan_first, default: local)) && !Array(tools).empty?
+          if (force_plan || agent_flag(key: :plan_first, default: local) || budget_exhaustion_hot?) && !Array(tools).empty?
+            predicted = plan_first(messages: messages, request: request)
+            # P22 — prefer explicit return; fall back to thread stash
+            predicted = Thread.current[:pwn_plan_predicted] if predicted.nil?
+          end
+          if budget_exhaustion_hot?
+            messages << {
+              role: 'user',
+              content: '[pwn-ai/p17] Budget-exhaustion is the top open failure on this host. Prefer the SHORTEST plan that finishes the ask (≤3 tool calls). Emit a final answer as soon as you have evidence — do not explore.'
+            }
+          end
           if force_plan && cal_state[:cal]
             messages << {
               role: 'user',
@@ -517,7 +584,25 @@ module PWN
             # 3.1 — compact history on local so tool dumps don't fill num_ctx
             compact_history!(messages: messages) if local
 
-            msg = call_engine(messages: messages, tools: tools)
+            # P17 — on the final iteration, strip tools and demand a plain-text
+            # answer. Without this the model happily emits one more tool_calls
+            # batch, burns the last slot, and lands on budget_exhausted with
+            # nothing the user (or ORM) can use.
+            # P17 deepen — when budget_hot, force text-only on the LAST TWO
+            # iters so a final tool_calls batch cannot burn the terminal slot.
+            text_only_iters = budget_exhaustion_hot? ? 2 : 1
+            last_iter = (i >= max_iters - text_only_iters)
+            if last_iter
+              tag = i >= max_iters - 1 ? 'FINAL ITERATION' : 'PENULTIMATE — wrap up'
+              messages << {
+                role: 'user',
+                content: "[pwn-ai/p17] #{tag} — do NOT call any more tools. " \
+                         'Write the best answer you can from evidence already in this ' \
+                         'transcript. If blocked, say what failed and the next single step.'
+              }
+            end
+
+            msg = call_engine(messages: messages, tools: last_iter ? nil : tools)
             return '[pwn-ai] engine returned no message' if msg.nil?
 
             calls = Array(msg[:tool_calls])
@@ -567,7 +652,10 @@ module PWN
                 # alt-persona branch, judge both, inject the winner. Real
                 # advantage estimation; (loser, winner) → DPO preference.
                 thresh = defined?(Mistakes) ? Mistakes::REPEAT_THRESHOLD : 3
-                if count >= thresh && !escalated && defined?(Curriculum)
+                # P17 — never fork counterfactual when budget fingerprints dominate:
+                # CF is another mini agent loop and is the #1 amplifier of
+                # iteration-budget exhaustion on this host.
+                if count >= thresh && !escalated && defined?(Curriculum) && !budget_exhaustion_hot?
                   cf = (turn_fails["cf:#{fkey}"] += 1) == 1 ? Curriculum.counterfactual(request: request, name: name, args: args, error: tele[:err] || raw[0, 200], hint: hint) : nil
                   hint = "#{hint}\n[pwn-ai/counterfactual] branch #{cf[:branch]} (score=#{cf[:score].round(2)}): #{cf[:content]}" if cf
                 end
@@ -589,6 +677,31 @@ module PWN
               )
             end
 
+            # P17 — hard stop: empty-final thrash or cumulative fails past cap.
+            # Prefer a short apologetic final over another 10 useless tool dumps
+            # that poison ORM/PRM/DPO with terminal failures.
+            empty_n = turn_fails['empty_final'].to_i
+            fail_n  = turn_fails.values.sum
+            if empty_n >= BUDGET_EMPTY_FINAL_STOP || fail_n >= BUDGET_HARD_STOP_FAILS
+              msg = if empty_n >= BUDGET_EMPTY_FINAL_STOP
+                      '[pwn-ai] stopped: repeated empty finals (budget thrash guard)'
+                    else
+                      '[pwn-ai] stopped: too many in-turn failures (budget thrash guard)'
+                    end
+              if defined?(Mistakes)
+                Mistakes.record(
+                  tool: 'agent_loop',
+                  error: "budget thrash guard fired empty=#{empty_n} fails=#{fail_n} iter=#{i}",
+                  session_id: session_id,
+                  source: :loop,
+                  shape: :budget_thrash
+                )
+              end
+              append_session(session_id: session_id, role: 'assistant', content: msg)
+              Learning.auto_introspect(session_id: session_id, request: request, final: msg, predicted: predicted) if defined?(Learning) && should_auto_introspect?(local: local, turn_fails: turn_fails, iter: i)
+              return msg
+            end
+
             next unless local && !escalated && turn_fails.values.sum >= ESCALATE_AFTER_FAILS
 
             hint = escalate(request: request, turn_fails: turn_fails, session_id: session_id)
@@ -599,8 +712,29 @@ module PWN
             escalated = true
           end
 
-          Mistakes.record(tool: 'agent_loop', error: 'iteration budget exhausted without a final answer', session_id: session_id, source: :loop) if defined?(Mistakes)
-          '[pwn-ai] iteration budget exhausted'
+          # P17 — exhaust path must still feed Learning so ORM/PRM/HER see the
+          # failure (previously we only Mistakes.record'd and returned a bare
+          # string — no session row, no judge, no hindsight).
+          final_msg = '[pwn-ai] iteration budget exhausted'
+          if defined?(Mistakes)
+            Mistakes.record(
+              tool: 'agent_loop',
+              error: 'iteration budget exhausted without a final answer',
+              session_id: session_id,
+              source: :loop,
+              shape: :budget_exhausted
+            )
+          end
+          append_session(session_id: session_id, role: 'assistant', content: final_msg)
+          if defined?(Learning) && should_auto_introspect?(local: local, turn_fails: turn_fails, iter: max_iters)
+            Learning.auto_introspect(
+              session_id: session_id,
+              request: request,
+              final: final_msg,
+              predicted: predicted
+            )
+          end
+          final_msg
         end
 
         # Author(s):: 0day Inc. <support@0dayinc.com>

data/lib/pwn/ai/agent/metrics.rb

@@ -122,6 +122,8 @@ module PWN
               name: name.to_s,
               calls: calls,
               success_rate: rate,
+              judge_rate: judge_rate(name: name),
+              effective_rate: effective_rate(name: name),
               avg_duration: avg,
               last_error: b[:last_error],
               last_at: b[:last_at]
@@ -149,15 +151,38 @@ module PWN
           scope = engine.to_s.empty? ? 'historical' : "engine=#{engine}"
           scope = "#{scope}, proxy_distrust=#{distrust.round(2)}" if distrust.positive?
           lines = rows.map do |r|
+            # P20 — display effective_rate (judge-blended) when available;
+            # fall back to distrust haircut on raw proxy.
             rate = r[:success_rate].to_f
-            # blend toward 0.5 (uninformative) proportional to distrust
-            adj = rate - ((rate - 0.5) * distrust)
+            eff  = r[:effective_rate]
+            adj  = if eff
+                     eff.to_f
+                   else
+                     rate - ((rate - 0.5) * distrust)
+                   end
             err = r[:last_error] ? " last_err=#{r[:last_error][0, 60]}" : ''
-            tag = distrust.positive? ? ' (adj)' : ''
-            "  - #{r[:name]}: calls=#{r[:calls]} success=#{(adj * 100).round(1)}%#{tag} avg=#{r[:avg_duration]}s#{err}"
+            jtag = r[:judge_rate] ? " judge=#{(r[:judge_rate].to_f * 100).round(1)}%" : ''
+            tag = distrust.positive? || r[:judge_rate] ? ' (adj)' : ''
+            "  - #{r[:name]}: calls=#{r[:calls]} success=#{(adj * 100).round(1)}%#{tag}#{jtag} avg=#{r[:avg_duration]}s#{err}"
           end
           warn_line = distrust.positive? ? "WARNING: reward proxy diverges from judge — success rates haircut by distrust=#{distrust.round(2)}; prefer judge-scored exemplars over raw rates.\n" : ''
-          "#{warn_line}TOOL EFFECTIVENESS (#{scope}, adapt tool choice accordingly)\n#{lines.join("\n")}\n\n"
+          # P0 ops — surface W1 generator_mix when diet is unhealthy so the
+          # online controller (and the model) prefer underfilled sources.
+          mix_line = ''
+          if defined?(Reward) && Reward.respond_to?(:generator_mix)
+            begin
+              m = Reward.generator_mix
+              unless m[:healthy]
+                mix_line = "W1 MIX: n=#{m[:n]} traj=#{m[:trajectory_fraction]} " \
+                           "urgent=#{Array(m[:urgent]).join(',')} " \
+                           "suppress=#{Array(m[:suppress]).join(',')} " \
+                           "rec=#{m[:recommendation]}\n"
+              end
+            rescue StandardError
+              mix_line = ''
+            end
+          end
+          "#{warn_line}#{mix_line}TOOL EFFECTIVENESS (#{scope}, adapt tool choice accordingly)\n#{lines.join("\n")}\n\n"
         end
 
         # P4 helper — Registry.rank calls this so β·advantage is scaled down
@@ -176,7 +201,8 @@ module PWN
           t    = data[name.to_sym] || blank_bucket
           n    = [t[:calls].to_f, 1.0].max
           total = [data.values.sum { |v| v[:calls].to_f }, 1.0].max
-          mean  = t[:ok].to_f / n
+          # P20 — mean from effective_rate (judge-blended when distrust high)
+          mean  = effective_rate(name: name)
           mean + (c * Math.sqrt(Math.log(total) / n))
         rescue StandardError
           1.0
@@ -190,7 +216,21 @@ module PWN
 
         public_class_method def self.thompson(opts = {})
           t = (load[:tools] || {})[opts[:name].to_s.to_sym] || blank_bucket
-          beta_sample(alpha: t[:ok].to_f + 1.0, beta: t[:fail].to_f + 1.0)
+          # P20 — when judge samples exist and distrust > 0, tilt Beta toward
+          # judge_rate so Thompson explore/exploit tracks ORM not handler-ok.
+          ok = t[:ok].to_f
+          fail = t[:fail].to_f
+          jn = Array(t[:judge_window]).length
+          if jn >= 3 && proxy_trust < 0.95
+            jr = judge_rate(name: opts[:name]).to_f
+            # pseudo-counts from judge window, mixed by distrust
+            d = (1.0 - proxy_trust).clamp(0.0, 1.0)
+            jok = jr * jn
+            jfail = (1.0 - jr) * jn
+            ok = (ok * (1.0 - d)) + (jok * d)
+            fail = (fail * (1.0 - d)) + (jfail * d)
+          end
+          beta_sample(alpha: ok + 1.0, beta: fail + 1.0)
         rescue StandardError
           0.5
         end
@@ -205,14 +245,160 @@ module PWN
           t    = data[opts[:name].to_s.to_sym]
           return 0.0 unless t
 
-          global = data.values.sum { |v| v[:ok].to_f } / [data.values.sum { |v| v[:calls].to_f }, 1.0].max
-          win    = Array(t[:window])
-          local  = win.empty? ? (t[:ok].to_f / [t[:calls].to_f, 1.0].max) : (win.sum.to_f / win.length)
+          # P20 — local/global from effective_rate so bandit tracks judge
+          # when the handler-ok proxy is hacked (proxy_distrust high).
+          local = effective_rate(name: opts[:name])
+          rates = data.keys.map { |k| effective_rate(name: k) }
+          global = rates.empty? ? 0.5 : (rates.sum / rates.length)
           (local - global).round(3)
         rescue StandardError
           0.0
         end
 
+        # P18/P2 — rolling mean step_reward advantage for a tool.
+        # Sample-efficiency gate: < PRM_MIN_N samples → 0 (no rank noise).
+        # Shrinkage: adv *= min(1, n/PRM_FULL_N) so sparse signal cannot
+        # dominate UCB. Zero-variance windows (all +1 or all -1 from a
+        # single session) damp to 0.5×.
+        PRM_MIN_N  = 5
+        PRM_FULL_N = 20
+
+        public_class_method def self.prm_advantage(opts = {})
+          data = load[:tools] || {}
+          t = data[opts[:name].to_s.to_sym]
+          return 0.0 unless t
+
+          win = Array(t[:prm_window])
+          n = win.length
+          return 0.0 if n < PRM_MIN_N
+
+          mean = win.sum.to_f / n
+          globals = data.values.map { |v| Array(v[:prm_window]) }.select { |w| w.length >= PRM_MIN_N }
+          gmean = if globals.empty?
+                    0.0
+                  else
+                    all = globals.flatten
+                    all.sum.to_f / all.length
+                  end
+          adv = mean - gmean
+          # shrinkage toward 0 until PRM_FULL_N
+          shrink = [n.to_f / PRM_FULL_N, 1.0].min
+          # variance damp: if all equal, halve influence
+          uniq = win.uniq
+          var_damp = uniq.length <= 1 ? 0.5 : 1.0
+          (adv * shrink * var_damp).round(3)
+        rescue StandardError
+          0.0
+        end
+
+        public_class_method def self.prm_n(opts = {})
+          t = (load[:tools] || {})[opts[:name].to_s.to_sym]
+          return 0 unless t
+
+          Array(t[:prm_window]).length
+        rescue StandardError
+          0
+        end
+
+        # P18 — called by Reward.prm after session annotate so live routing
+        # can bias toward tools that recently advanced goals (+1 step_reward).
+        public_class_method def self.record_step_reward(opts = {})
+          name = opts[:name].to_s
+          return if name.empty?
+
+          rew = opts[:reward].to_f.clamp(-1.0, 1.0)
+          m = load
+          m[:tools] ||= {}
+          t = m[:tools][name.to_sym] ||= blank_bucket
+          t[:prm_window] = (Array(t[:prm_window]) + [rew]).last(40)
+          t[:prm_sum] = t[:prm_window].sum
+          t[:prm_n] = t[:prm_window].length
+          save(metrics: m)
+          t
+        rescue StandardError
+          nil
+        end
+
+        # P20 — fold ORM judge (0..1) into per-tool telemetry so UCB /
+        # Thompson / advantage can prefer judge-grounded rates over the
+        # inflated handler-ok proxy when proxy_distrust is high.
+        public_class_method def self.record_judge(opts = {})
+          name = opts[:name].to_s
+          return if name.empty?
+
+          score = opts[:score].to_f.clamp(0.0, 1.0)
+          conf  = opts.key?(:confidence) ? opts[:confidence].to_f.clamp(0.0, 1.0) : 0.7
+          m = load
+          m[:tools] ||= {}
+          t = m[:tools][name.to_sym] ||= blank_bucket
+          t[:judge_window] = (Array(t[:judge_window]) + [score]).last(40)
+          t[:judge_conf_window] = (Array(t[:judge_conf_window]) + [conf]).last(40)
+          t[:judge_sum] = t[:judge_window].sum.to_f
+          t[:judge_n] = t[:judge_window].length
+          save(metrics: m)
+          t
+        rescue StandardError
+          nil
+        end
+
+        # P1 — mean judge confidence for a tool (nil when no samples).
+        public_class_method def self.judge_confidence(opts = {})
+          t = (load[:tools] || {})[opts[:name].to_s.to_sym]
+          return nil unless t
+
+          win = Array(t[:judge_conf_window])
+          return nil if win.empty?
+
+          (win.sum.to_f / win.length).round(3)
+        rescue StandardError
+          nil
+        end
+
+        # Mean judge score for a tool (nil when no ORM samples yet).
+        public_class_method def self.judge_rate(opts = {})
+          t = (load[:tools] || {})[opts[:name].to_s.to_sym]
+          return nil unless t
+
+          win = Array(t[:judge_window])
+          return nil if win.empty?
+
+          (win.sum.to_f / win.length).round(3)
+        rescue StandardError
+          nil
+        end
+
+        # Blended success rate: when proxy_distrust > 0 and judge samples
+        # exist, mix judge_rate into the handler-ok rate. distrust=1 → pure
+        # judge (or 0.5 if no judge data). distrust=0 → pure proxy.
+        public_class_method def self.effective_rate(opts = {})
+          name = opts[:name].to_s
+          data = load[:tools] || {}
+          t = data[name.to_sym]
+          return 0.5 unless t
+
+          calls = [t[:calls].to_f, 1.0].max
+          proxy = t[:ok].to_f / calls
+          win = Array(t[:window])
+          proxy = win.sum.to_f / win.length if win.length >= 3
+          distrust = 1.0 - proxy_trust
+          jrate = judge_rate(name: name)
+          if distrust > 0.05 && !jrate.nil?
+            # P1 — scale judge weight by judge confidence so a thin local
+            # heuristic ORM cannot fully replace proxy when distrust is high.
+            # effective_distrust = distrust * mean(confidence), floor 0.15 when
+            # we do have judge samples so the signal still moves the needle.
+            jconf = judge_confidence(name: name)
+            jconf = 0.7 if jconf.nil?
+            eff_d = (distrust * jconf).clamp(0.0, 1.0)
+            eff_d = [eff_d, 0.15].max if jconf >= 0.3
+            ((proxy * (1.0 - eff_d)) + (jrate * eff_d)).clamp(0.0, 1.0).round(3)
+          else
+            proxy.round(3)
+          end
+        rescue StandardError
+          0.5
+        end
+
         # Supported Method Parameters::
         # cps = PWN::AI::Agent::Metrics.changepoints
         #
@@ -360,7 +546,10 @@ module PWN
               PWN::AI::Agent::Metrics.to_context(limit: 8, engine: :ollama)   # injected by PromptBuilder
               PWN::AI::Agent::Metrics.ucb(name: 'shell')                 # C1 exploration bonus
               PWN::AI::Agent::Metrics.thompson(name: 'shell')            # C1 Beta(ok+1,fail+1) sample
-              PWN::AI::Agent::Metrics.advantage(name: 'shell')           # C1 local − global
+              PWN::AI::Agent::Metrics.advantage(name: 'shell')           # C1 local − global (P20 judge-blended)
+              PWN::AI::Agent::Metrics.record_judge(name: 'shell', score: 0.8) # P20 ORM→metrics
+              PWN::AI::Agent::Metrics.judge_rate(name: 'shell')         # P20 mean ORM
+              PWN::AI::Agent::Metrics.effective_rate(name: 'shell')     # P20 proxy⋈judge
               PWN::AI::Agent::Metrics.changepoints(within_secs: 3600)    # E1 CUSUM regime changes
               PWN::AI::Agent::Metrics.record_calibration(predicted: 0.8, actual: 1.0, brier: 0.04, engine: :ollama)
               PWN::AI::Agent::Metrics.calibration(engine: :ollama)       # W3 Brier / overconfidence

data/lib/pwn/ai/agent/mistakes.rb

@@ -257,15 +257,33 @@ module PWN
               importance: 0.9
             )
           end
-          # W1 — every resolve is a naturally-generated preference pair:
-          # (rejected: the failing action, chosen: the fix).
+          # W1/P9 — every resolve is a preference pair. Prefer structured
+          # winning_trace (+ strategy/tool) over first-line fix prose so DPO
+          # learns tool trajectories, not commentary.
           if defined?(Reward)
-            Reward.record_preference(
-              prompt: "#{store[key][:tool]}: #{store[key][:error]}",
-              rejected: store[key][:snippet].to_s,
-              chosen: fix.strip,
-              source: :mistakes_resolve
-            )
+            sf = store[key][:structured_fix] || {}
+            trace = sf[:winning_trace].to_s.strip
+            strat = [sf[:strategy], sf[:tool], sf[:args_template]].compact.map(&:to_s).reject(&:empty?).join(' | ')
+            # P21/P25 — only write W1 pairs when we have a real winning_trace.
+            # Prose-only resolve still updates Memory lesson + structured_fix;
+            # it must NOT flood DPO with fix commentary (shape: :fix_prose).
+            if trace.length >= 40
+              parts = []
+              parts << "STRATEGY: #{strat}" unless strat.empty?
+              parts << "WINNING_TRACE:\n#{trace[0, 3_500]}"
+              parts << "FIX: #{fix.strip[0, 400]}"
+              chosen = parts.join("\n")
+              rejected = store[key][:snippet].to_s
+              rejected = "FAILING: tool=#{store[key][:tool]} err=#{store[key][:error]}" if rejected.strip.empty?
+              Reward.record_preference(
+                prompt: "#{store[key][:tool]}: #{store[key][:error]}",
+                rejected: rejected,
+                chosen: chosen,
+                source: :mistakes_resolve,
+                shape: :winning_trace,
+                meta: { signature: sig, strategy: sf[:strategy], tool: sf[:tool] }.compact
+              )
+            end
           end
           store[key]
         end

data/lib/pwn/ai/agent/registry.rb

@@ -136,7 +136,7 @@ module PWN
 
           tokens = query.scan(/[a-z0-9_]{3,}/).uniq
           # C1 — advantage-weighted router:
-          #   score = α·keyword_sim + β·advantage + γ·UCB(tool)
+          #   score = α·keyword_sim + β·advantage + γ·UCB(tool) + δ·prm_advantage
           # UCB gives untried / low-N tools an exploration bonus so a single
           # early failure (before its dep was installed) does not blacklist
           # it forever; advantage prefers tools that outperform the fleet.
@@ -145,12 +145,34 @@ module PWN
           trust = defined?(Metrics) && Metrics.respond_to?(:proxy_trust) ? Metrics.proxy_trust : 1.0
           beta  = 0.3 * trust
           gamma = 0.2
+          # P18/P2 — PRM step_reward advantage closes R2 into the controller.
+          # Sample-efficiency: if fewer than 3 tools have prm_n≥PRM_MIN_N,
+          # drop delta to 0 so sparse PRM cannot inject rank variance.
+          # Otherwise scale delta by fleet coverage fraction.
+          prm_ready = 0
+          if defined?(Metrics) && Metrics.respond_to?(:prm_n)
+            prm_ready = entries.count do |e|
+              Metrics.prm_n(name: e.name).to_i >= begin
+                Metrics::PRM_MIN_N
+              rescue StandardError
+                5
+              end
+            end
+          end
+          fleet = [entries.length, 1].max
+          coverage = prm_ready.to_f / fleet
+          delta = if prm_ready < 3
+                    0.0
+                  else
+                    0.25 * trust * [coverage / 0.3, 1.0].min
+                  end
           scored = entries.map do |e|
             hay   = "#{e.name} #{e.toolset} #{e.schema[:description]} #{Array(e.schema.dig(:parameters, :properties)&.keys).join(' ')}".downcase
             sim   = tokens.count { |t| hay.include?(t) }
             adv   = defined?(Metrics) && Metrics.respond_to?(:advantage) ? Metrics.advantage(name: e.name) : 0.0
             ucb   = defined?(Metrics) && Metrics.respond_to?(:ucb) ? Metrics.ucb(name: e.name) : 0.5
-            [e, sim, (alpha * sim) + (beta * adv) + (gamma * ucb)]
+            prm   = defined?(Metrics) && Metrics.respond_to?(:prm_advantage) ? Metrics.prm_advantage(name: e.name) : 0.0
+            [e, sim, (alpha * sim) + (beta * adv) + (gamma * ucb) + (delta * prm)]
           end
           scored.reject { |_, sim, _| sim.zero? }
                 .sort_by { |_, _, s| -s }