pwn 0.5.643 → 0.5.650

data/lib/pwn/ai/agent/reward.rb

@@ -121,6 +121,40 @@ module PWN
           v = llm_judge(request: request, final: final, trace: trace)
           v ||= heuristic_judge(request: request, final: final, trace: trace)
 
+          # P1 — local/heuristic calibration: thin judges must not be treated
+          # as ground truth when proxy_distrust is already high. Two levers:
+          #   (1) low :confidence so Metrics.effective_rate haircuts blend
+          #       weight (distrust × confidence) instead of replacing proxy;
+          #   (2) score-path caps only for decisive failure floors and for
+          #       local no-trace highs (false "solved"). Do NOT pull known
+          #       wrong (0.0 failure-language) toward 0.5, and do NOT deflate
+          #       tool-backed heuristic scores that already cleared the bar —
+          #       confidence handles that in the bandit blend.
+          eng = (PWN::Env.dig(:ai, :active) if defined?(PWN::Env)).to_s.downcase
+          local = eng == 'ollama' || eng.empty?
+          if v[:source].to_s == 'heuristic'
+            v[:confidence] = local ? 0.35 : 0.5
+            raw = v[:score].to_f
+            v[:score_raw] = raw
+            # preserve empty / failure-language / polite floors exactly (raw <= 0.15)
+            # and pass-through non-capped heuristics via the default arm.
+            if local && raw > 0.15 && trace.empty? && raw >= 0.6 && final.length < 400
+              v[:score] = [raw, 0.45].min
+              v[:rationale] = "#{v[:rationale]} | P1:local_no_trace_cap"
+            elsif local && raw > 0.15 && trace.length < 2 && raw >= 0.85
+              # thin-evidence local highs: mild shrink toward 0.5
+              v[:score] = (0.5 + ((raw - 0.5) * 0.7)).round(3).clamp(0.0, 1.0)
+            else
+              v[:score] = raw
+            end
+            v[:verdict] = if v[:score] >= 0.6 then :solved
+                          elsif v[:score] >= 0.3 then :partial
+                          else :wrong
+                          end
+          else
+            v[:confidence] ||= 0.85
+          end
+
           ground = verify_as_reward(final: final)
           unless ground.nil?
             # Ground-truth override: a browser-refuted claim caps score at
@@ -129,13 +163,17 @@ module PWN
             v[:score] = [v[:score], 0.2].min if ground[:verdict] == :refuted
             v[:score] = [v[:score], 0.6].max if ground[:verdict] == :confirmed
             v[:grounded] = ground
+            v[:confidence] = [v[:confidence].to_f, ground[:confidence].to_f].max if ground[:confidence]
           end
 
           v[:success] = v[:score] >= 0.6
-          record_sentinel(proxy: opts[:proxy_ok], judge: v[:score]) if commit
+          v[:engine] = eng
+          # P1 — sentinel stores confidence so distrust math can haircut
+          # heuristic-heavy windows differently from LLM ORM windows.
+          record_sentinel(proxy: opts[:proxy_ok], judge: v[:score], confidence: v[:confidence]) if commit
           v
         rescue StandardError => e
-          { score: 0.5, verdict: :unknown, rationale: "judge error: #{e.class}", success: !final.strip.empty?, error: e.message }
+          { score: 0.5, verdict: :unknown, rationale: "judge error: #{e.class}", success: !final.strip.empty?, error: e.message, confidence: 0.2, source: :error }
         end
 
         # ----------------------------------------------------------------
@@ -297,6 +335,58 @@ module PWN
           { cleared: true, path: SENTINEL_FILE }
         end
 
+        # P10 — backfill the R3 ring from Learning outcomes so offline/local
+        # hosts reach SENTINEL_WINDOW without waiting for live remote
+        # introspect. Only fills empty slots; never flushes a warm window.
+        # Called by Curriculum.offline_judge and safe to cron.
+        public_class_method def self.warm_sentinel(opts = {})
+          s = normalize_sentinel(raw: load_sentinel)
+          have = Array(s[:window]).length
+          return { added: 0, samples: have, status: :full, proxy_distrust: proxy_distrust } if have >= SENTINEL_WINDOW
+          return { added: 0, samples: have, status: :no_learning, proxy_distrust: proxy_distrust } unless defined?(Learning)
+
+          need = SENTINEL_WINDOW - have
+          limit = (opts[:limit] || [need * 4, 200].max).to_i
+          # Prefer scored rows; fall back to success-boolean so local hosts still warm.
+          rows = Learning.outcomes(limit: limit)
+          scored, unscored = rows.partition { |r| !r[:score].nil? }
+          ordered = scored.reverse + unscored.reverse
+          added = 0
+          ordered.each do |r|
+            break if added >= need
+
+            judge = if r[:score]
+                      r[:score].to_f.clamp(0.0, 1.0)
+                    else
+                      case r[:success]
+                      when true, 'true' then 0.75
+                      when 'soft' then 0.55
+                      when false, 'false' then 0.25
+                      else 0.5
+                      end
+                    end
+            proxy = case r[:success]
+                    when true, 'true' then true
+                    when false, 'false', 'soft' then false
+                    else judge >= 0.6
+                    end
+            record_sentinel(proxy: proxy, judge: judge)
+            added += 1
+          end
+          final_n = Array(load_sentinel[:window]).length
+          # Recompute distrust once window is full so controllers can engage.
+          snap = final_n >= SENTINEL_WINDOW ? sentinel : { samples: final_n, status: :insufficient }
+          {
+            added: added,
+            samples: final_n,
+            status: (final_n >= SENTINEL_WINDOW ? :warmed_full : :warmed_partial),
+            proxy_distrust: proxy_distrust,
+            sentinel: snap.is_a?(Hash) ? snap.slice(:samples, :status, :reward_hacked, :proxy_distrust, :proxy, :judge) : nil
+          }
+        rescue StandardError => e
+          { added: 0, error: "#{e.class}: #{e.message}" }
+        end
+
         # ----------------------------------------------------------------
         # R4 — Structured tool-result classifier
         # ----------------------------------------------------------------
@@ -377,6 +467,14 @@ module PWN
           claim = final[Learning::CLAIM_RX] if defined?(Learning)
           return nil if claim.to_s.empty?
 
+          # P26 — drop metric crumbs ("cap 0.2") that match loose patterns
+          if defined?(Learning) && Learning.respond_to?(:checkable_claim?, true)
+            return nil unless Learning.send(:checkable_claim?, claim: claim)
+          elsif claim.match?(/\A(?:cap|share|proxy|judge|success|only|now|gap|score|rate)\b/i) ||
+                (claim.match?(/\d+\.\d+/) && !claim.match?(/\d+\.\d+\.\d+|CVE-/i))
+            return nil
+          end
+
           # 1.5 — sampled E3: always when flag true; never when false;
           # nil/auto → always on frontier, ~10% on local when CLAIM_RX hits.
           flag = agent_flag(key: :verify_as_reward, default: nil)
@@ -408,6 +506,29 @@ module PWN
         #   source: 'optional - :user_correction | :mistakes_resolve | :counterfactual | :critic'
         # )
 
+        # Trajectory-shaped chosen sides that may land DPO without prose flood.
+        TRAJECTORY_SHAPES = %w[winning_trace revised_answer real_dispatch].freeze
+
+        # P9 — write-time source quota (not only export). Prefer diverse online
+        # generators over resolve-prose flood. Window is last WRITE_SOURCE_WINDOW
+        # pairs; a source already above WRITE_SOURCE_CAP is refused unless
+        # force: true (user_correction always forces).
+        WRITE_SOURCE_CAP    = 0.40
+        WRITE_SOURCE_WINDOW = 100
+
+        # P0 — target online generator mix for W1. Gates alone cannot fill
+        # an empty promote; the controller must *prefer underfilled* sources
+        # (counterfactual / critic / curriculum / user_correction) when
+        # resolve already dominates. Shares are soft targets, not hard caps
+        # (hard cap remains WRITE_SOURCE_CAP). Trajectory-only still applies.
+        TARGET_SOURCE_MIX = {
+          'mistakes_resolve' => 0.30,
+          'curriculum' => 0.25,
+          'counterfactual' => 0.20,
+          'critic' => 0.15,
+          'user_correction' => 0.10
+        }.freeze
+
         public_class_method def self.record_preference(opts = {})
           prompt   = opts[:prompt].to_s
           rejected = opts[:rejected].to_s
@@ -415,20 +536,299 @@ module PWN
           return nil if prompt.strip.empty? || chosen.strip.empty? || rejected.strip.empty?
           return nil if chosen.strip == rejected.strip
 
+          # Reject weak pair geometry: CORRECTION: flaw-prose is not a trajectory.
+          return { skipped: :weak_pair_geometry, reason: 'chosen looks like flaw prose, not a revised answer/trace' } if chosen.match?(/\A\s*CORRECTION:\s*/i) && chosen.length < 400 && !opts[:force]
+
+          source = (opts[:source] || :unknown).to_s
+          shape  = opts[:shape].to_s
+          # P25 — require trajectory shape at write time unless force / user_correction.
+          # Stops resolve-prose flood from ever landing in the ledger; export scrub
+          # is defense-in-depth, not the primary gate.
+          traj = TRAJECTORY_SHAPES.include?(shape)
+          # P25 — non-trajectory prose never lands (export scrub is defense-in-depth).
+          # user_correction and explicit force: still allowed for human / migration paths.
+          unless traj || opts[:force] || source == 'user_correction'
+            return {
+              skipped: :non_trajectory_shape,
+              reason: "shape=#{shape.inspect} not in #{TRAJECTORY_SHAPES.join(',')}; pass force:true or a trajectory shape",
+              source: source
+            }
+          end
+          # P9 — write-time source quota still applies to trajectory pairs.
+          # P25 made every auto-written row trajectory-shaped; if traj also
+          # bypassed the quota, resolve monoculture would return via winning_trace
+          # flood. Only user_correction and explicit force:true skip the cap.
+          bypass_quota = opts[:force] || source == 'user_correction'
+          unless bypass_quota
+            quota = write_source_quota(source: source)
+            return quota.merge(skipped: :source_quota) if quota[:over_cap]
+          end
+
           entry = {
             id: Digest::SHA256.hexdigest("#{prompt}|#{rejected}|#{chosen}")[0, 12],
             prompt: prompt[0, 4_000],
             rejected: rejected[0, 4_000],
             chosen: chosen[0, 4_000],
-            source: (opts[:source] || :unknown).to_s,
+            source: source,
             engine: (PWN::Env.dig(:ai, :active) if defined?(PWN::Env)).to_s,
             timestamp: Time.now.utc.iso8601
           }
+          entry[:meta] = opts[:meta] if opts[:meta].is_a?(Hash)
+          entry[:shape] = opts[:shape].to_s if opts[:shape]
           FileUtils.mkdir_p(File.dirname(PREFERENCES_FILE))
           File.open(PREFERENCES_FILE, 'a') { |f| f.puts(JSON.generate(entry)) }
           entry
         end
 
+        # Share of `source` among the newest WRITE_SOURCE_WINDOW prefs.
+        public_class_method def self.write_source_quota(opts = {})
+          source = opts[:source].to_s
+          recent = preferences(limit: WRITE_SOURCE_WINDOW)
+          return { over_cap: false, share: 0.0, n: 0, window: recent.length, underfilled: true } if recent.length < 10
+
+          n = recent.count { |r| r[:source].to_s == source }
+          share = n.to_f / recent.length
+          target = TARGET_SOURCE_MIX[source]
+          {
+            over_cap: share > WRITE_SOURCE_CAP,
+            share: share.round(3),
+            n: n,
+            window: recent.length,
+            source: source,
+            cap: WRITE_SOURCE_CAP,
+            target: target,
+            underfilled: target ? share < (target * 0.5) : share < 0.05,
+            deficit: target ? (target - share).round(3) : nil
+          }
+        rescue StandardError
+          { over_cap: false, share: 0.0, underfilled: true }
+        end
+
+        # P0 — online generator mix report + urgency flags. Controllers
+        # (auto_introspect, practice, counterfactual gate) consult this so
+        # underfilled sources get scheduling priority while over-cap
+        # resolve stops flooding. Returns {by_source, trajectory_fraction,
+        # urgent:[], suppress:[], healthy:}.
+        public_class_method def self.generator_mix(opts = {})
+          limit = opts[:limit] || WRITE_SOURCE_WINDOW
+          rows = preferences(limit: limit)
+          usable = rows.select { |r| usable_preference?(row: r) }
+          by = Hash.new(0)
+          usable.each { |r| by[r[:source].to_s] += 1 }
+          n = usable.length
+          shares = {}
+          TARGET_SOURCE_MIX.each_key { |k| shares[k] = n.zero? ? 0.0 : (by[k].to_f / n).round(3) }
+          by.each_key { |k| shares[k] ||= (by[k].to_f / n).round(3) }
+
+          traj_n = usable.count { |r| TRAJECTORY_SHAPES.include?(r[:shape].to_s) }
+          traj_f = n.zero? ? 0.0 : (traj_n.to_f / n).round(3)
+
+          urgent = []
+          suppress = []
+          TARGET_SOURCE_MIX.each do |src, target|
+            sh = shares[src].to_f
+            urgent << src if sh < (target * 0.5) && n >= 5
+            suppress << src if sh > WRITE_SOURCE_CAP && n >= 10
+          end
+          suppress << 'mistakes_resolve' if shares['mistakes_resolve'].to_f > WRITE_SOURCE_CAP && n >= 10 && !suppress.include?('mistakes_resolve')
+
+          healthy = urgent.empty? && suppress.empty? && traj_f >= 0.5 && n >= 10
+          {
+            n: n,
+            raw_n: rows.length,
+            by_source: by,
+            shares: shares,
+            targets: TARGET_SOURCE_MIX,
+            trajectory_fraction: traj_f,
+            urgent: urgent.uniq,
+            suppress: suppress.uniq,
+            healthy: healthy,
+            recommendation: if healthy
+                              'mix_ok'
+                            elsif n < 10
+                              'need_more_pairs'
+                            elsif traj_f < 0.5
+                              'need_trajectory_shape'
+                            elsif urgent.any?
+                              "boost:#{urgent.join(',')}"
+                            else
+                              "suppress:#{suppress.join(',')}"
+                            end
+          }
+        rescue StandardError => e
+          {
+            n: 0, healthy: false, error: "#{e.class}: #{e.message}",
+            urgent: %w[curriculum counterfactual critic user_correction],
+            suppress: []
+          }
+        end
+
+        # P0 ops — infer trajectory shape for legacy ledger rows that predate
+        # P21/P25 shape tags. Used by scrub_preferences rewrite so generator_mix
+        # trajectory_fraction reflects content, not missing keys.
+        public_class_method def self.infer_shape(opts = {})
+          r = opts.is_a?(Hash) && opts.key?(:row) ? opts[:row] : opts
+          r = r.transform_keys(&:to_sym) if r.respond_to?(:transform_keys)
+          existing = r[:shape].to_s
+          return existing if TRAJECTORY_SHAPES.include?(existing) || existing == 'fix_prose'
+
+          chosen = r[:chosen].to_s
+          source = r[:source].to_s
+          # tool-call / trace markers → winning_trace
+          if chosen.match?(/\b(shell|pwn_eval|memory_|sessions_|reward_|curriculum_|extro_|mistakes_)\b/i) &&
+             (chosen.include?('→') || chosen.include?('tool_call') || chosen.include?('"name"') ||
+              chosen.lines.count { |l| l.strip.start_with?('{') || l.include?('arguments') } >= 1)
+            return 'winning_trace'
+          end
+          # long revised answer from critic / user / CF → revised_answer
+          return 'revised_answer' if chosen.length >= 200 && %w[critic user_correction counterfactual curriculum].include?(source)
+
+          # counterfactual real dispatch tag in meta
+          meta = r[:meta].is_a?(Hash) ? r[:meta] : {}
+          return 'real_dispatch' if meta[:mode].to_s == 'real_dispatch' || meta['mode'].to_s == 'real_dispatch'
+
+          existing.empty? ? nil : existing
+        rescue StandardError
+          nil
+        end
+
+        # P15 — keep only usable preference pairs for balance/export/promote.
+        # Drops CORRECTION-only chosen, resolve rows without trajectory shape,
+        # and chosen≪rejected unless shape is a known trajectory form.
+        public_class_method def self.usable_preference?(opts = {})
+          r = opts.is_a?(Hash) && opts.key?(:row) ? opts[:row] : opts
+          r = r.transform_keys(&:to_sym) if r.respond_to?(:transform_keys)
+          chosen = r[:chosen].to_s
+          rejected = r[:rejected].to_s
+          shape = r[:shape].to_s
+          source = r[:source].to_s
+          return false if chosen.strip.empty? || rejected.strip.empty?
+          return false if chosen.strip == rejected.strip
+          return false if chosen.match?(/\A\s*CORRECTION:\s*/i) && chosen.length < 400
+          return false if shape == 'fix_prose'
+          # P25 — resolve rows must be trajectory-shaped to count as usable
+          return false if source == 'mistakes_resolve' && !TRAJECTORY_SHAPES.include?(shape)
+
+          # chosen ≪ rejected without trajectory shape → commentary, not policy
+          unless TRAJECTORY_SHAPES.include?(shape)
+            return false if rejected.length >= 200 && chosen.length < (rejected.length * 0.25) && chosen.length < 200
+            return false if rejected.length >= 400 && chosen.length < 120
+          end
+          true
+        rescue StandardError
+          false
+        end
+
+        # P15 — one-shot ledger hygiene. Filters in place (rewrite jsonl) or
+        # report-only. Returns {before:, after:, dropped:, by_reason:, path:}.
+        public_class_method def self.scrub_preferences(opts = {})
+          dry = opts.key?(:dry_run) ? opts[:dry_run] : false
+          path = PREFERENCES_FILE
+          return { before: 0, after: 0, dropped: 0, dry_run: dry, path: path } unless File.exist?(path)
+
+          raw = File.readlines(path)
+          kept = []
+          reasons = Hash.new(0)
+          raw.each do |line|
+            begin
+              r = JSON.parse(line, symbolize_names: true)
+            rescue StandardError
+              reasons[:parse_error] += 1
+              next
+            end
+            if usable_preference?(row: r)
+              # P0 ops — backfill shape so trajectory_fraction is meaningful
+              if r[:shape].to_s.empty?
+                inferred = infer_shape(row: r)
+                r = r.merge(shape: inferred) if inferred
+              end
+              kept << r
+            else
+              why = if r[:chosen].to_s.match?(/\A\s*CORRECTION:\s*/i)
+                      :correction_prose
+                    elsif r[:shape].to_s == 'fix_prose'
+                      :fix_prose
+                    elsif r[:chosen].to_s.length < (r[:rejected].to_s.length * 0.25)
+                      :chosen_too_short
+                    else
+                      :weak_geometry
+                    end
+              reasons[why] += 1
+            end
+          end
+          unless dry
+            bak = "#{path}.bak-p15-#{Time.now.utc.strftime('%Y%m%d%H%M%S')}"
+            FileUtils.cp(path, bak)
+            File.open(path, 'w') { |f| kept.each { |r| f.puts(JSON.generate(r)) } }
+          end
+          {
+            before: raw.length,
+            after: kept.length,
+            dropped: raw.length - kept.length,
+            by_reason: reasons,
+            dry_run: dry,
+            path: path,
+            backup: dry ? nil : bak
+          }
+        rescue StandardError => e
+          { error: "#{e.class}: #{e.message}" }
+        end
+
+        # P15/P5 — geometry-aware source mix. scrub:true uses usable_preference?
+        # so operators see the post-hygiene diet (what export_dpo will train on).
+        public_class_method def self.preference_balance(opts = {})
+          limit = opts[:limit] || 10_000
+          scrub = opts.key?(:scrub) ? opts[:scrub] : false
+          rows = preferences(limit: limit)
+          before = rows.length
+          rows = rows.select { |r| usable_preference?(row: r) } if scrub
+          by = Hash.new(0)
+          by_shape = Hash.new(0)
+          rows.each do |r|
+            by[r[:source].to_s] += 1
+            sh = r[:shape].to_s
+            sh = 'unspecified' if sh.empty?
+            by_shape[sh] += 1
+          end
+          total = rows.length
+          frac = by.transform_values { |n| total.zero? ? 0.0 : (n.to_f / total).round(3) }
+          shape_frac = by_shape.transform_values { |n| total.zero? ? 0.0 : (n.to_f / total).round(3) }
+          traj_n = rows.count { |r| TRAJECTORY_SHAPES.include?(r[:shape].to_s) }
+          traj_frac = total.zero? ? 0.0 : (traj_n.to_f / total).round(3)
+          monoculture = total.positive? && (by.values.max.to_f / total) > 0.7
+          mix = begin
+            generator_mix(limit: limit)
+          rescue StandardError
+            nil
+          end
+          {
+            total: before,
+            kept: total,
+            scrubbed: scrub,
+            dropped: before - total,
+            by_source: by,
+            fractions: frac,
+            by_shape: by_shape,
+            by_shape_fraction: shape_frac,
+            trajectory_fraction: traj_frac,
+            monoculture: monoculture,
+            generator_mix: mix,
+            advice: if total < 12
+                      'W1 thin: need more trajectory-shaped pairs before LoRA promote.'
+                    elsif monoculture
+                      'W1 monoculture: run Reward.scrub_preferences; enable :counterfactual/:critic; stop resolve-prose flood.'
+                    elsif traj_frac < 0.30
+                      'W1 geometry weak: <30% trajectory-shaped chosen sides — DPO would teach commentary.'
+                    elsif mix && !mix[:healthy]
+                      "W1 generator mix: #{mix[:recommendation]}"
+                    else
+                      'W1 source mix OK for gated export'
+                    end
+          }
+        rescue StandardError => e
+          { error: "#{e.class}: #{e.message}" }
+        end
+
         # Supported Method Parameters::
         # rows = PWN::AI::Agent::Reward.preferences(limit: 500, source: nil)
 
@@ -463,6 +863,15 @@ module PWN
           FileUtils.mkdir_p(DPO_DIR)
           out = opts[:out] || File.join(DPO_DIR, "pwn-dpo-#{Time.now.utc.strftime('%Y%m%d')}.jsonl")
           rows = preferences(limit: 100_000)
+          # P15 — drop weak geometry before source-cap so resolve prose cannot
+          # dominate the kept set after balance. opt-out with scrub: false.
+          scrub = opts.key?(:scrub) ? opts[:scrub] : true
+          geometry_dropped = 0
+          if scrub
+            usable = rows.select { |r| usable_preference?(row: r) }
+            geometry_dropped = rows.length - usable.length
+            rows = usable
+          end
           # P5 — downsample so no single source exceeds DPO_SOURCE_CAP of the export.
           # opt-out with balance: false (raw dump for diagnostics).
           balance = opts.key?(:balance) ? opts[:balance] : true
@@ -484,8 +893,14 @@ module PWN
           by_src = selected.group_by { |r| r[:source].to_s }.transform_values(&:length)
           {
             path: out, format: fmt, pairs: selected.length, bytes: File.size(out),
-            balanced: balance, dropped: dropped, by_source: by_src,
-            source_cap: balance ? (opts[:source_cap] || DPO_SOURCE_CAP).to_f : nil
+            balanced: balance, dropped: dropped, geometry_dropped: geometry_dropped,
+            scrubbed: scrub, by_source: by_src,
+            source_cap: balance ? (opts[:source_cap] || DPO_SOURCE_CAP).to_f : nil,
+            preference_balance: begin
+              preference_balance(limit: 10_000, scrub: true)
+            rescue StandardError
+              nil
+            end
           }
         end
 
@@ -615,7 +1030,11 @@ module PWN
             j = JSON.parse(l, symbolize_names: true)
             if j[:role].to_s == 'tool'
               ti += 1
-              j[:step_reward] = rewards[ti] if rewards[ti]
+              if rewards[ti]
+                j[:step_reward] = rewards[ti]
+                # P18 — fold step_reward into Metrics so Registry.rank can bias
+                fold_step_reward_to_metrics(content: j[:content], reward: rewards[ti])
+              end
             end
             "#{JSON.generate(j)}\n"
           rescue StandardError
@@ -626,12 +1045,29 @@ module PWN
           nil
         end
 
+        # Extract tool name from "name → …" session content and record PRM.
+        private_class_method def self.fold_step_reward_to_metrics(opts = {})
+          return unless defined?(Metrics) && Metrics.respond_to?(:record_step_reward)
+
+          content = opts[:content].to_s
+          name = content[/\A([a-z_][a-z0-9_]*)\s*→/i, 1] ||
+                 content[/\A([a-z_][a-z0-9_]*)\s*->/i, 1] ||
+                 content[/\A([a-z_][a-z0-9_]*)/, 1]
+          return if name.to_s.empty?
+
+          Metrics.record_step_reward(name: name, reward: opts[:reward])
+        rescue StandardError
+          nil
+        end
+
         private_class_method def self.record_sentinel(opts = {})
           s = normalize_sentinel(raw: load_sentinel)
           # Clamp judge to [0,1] — LLM/heuristic should already, but a bad
           # write must not poison rolling means forever.
           judge = opts[:judge].to_f.clamp(0.0, 1.0)
           entry = { judge: judge, at: Time.now.utc.iso8601 }
+          # P1 — optional per-sample confidence (heuristic < LLM ORM)
+          entry[:confidence] = opts[:confidence].to_f.clamp(0.0, 1.0) unless opts[:confidence].nil?
           # 1.3 — only roll proxy into the window when the caller actually
           # supplied a R4-aligned proxy_ok. Pre-ORM boolean noise no longer
           # dilutes gap_proxy_judge. Proxy is ALWAYS 0.0 or 1.0 when present.
@@ -872,13 +1308,17 @@ module PWN
               PWN::AI::Agent::Reward.prm(request: req, session_id: sid)                    # R2 PRM → per-step credit
               PWN::AI::Agent::Reward.sentinel                                              # R3 reward-hacking detector
               PWN::AI::Agent::Reward.reset_sentinel                                        # wipe corrupt window + distrust
+              PWN::AI::Agent::Reward.warm_sentinel                                         # P10 fill R3 window from Learning outcomes
               PWN::AI::Agent::Reward.semantic_ok(name: 'shell', raw: json, args: args)     # R4 kills phantom exit≠0 mistakes
 
               # Tier 5 — preference pairs → DPO
               PWN::AI::Agent::Reward.record_preference(prompt: p, rejected: r, chosen: c, source: :user_correction)
               PWN::AI::Agent::Reward.preferences(limit: 100)
-              PWN::AI::Agent::Reward.export_dpo(format: :dpo)                              # W1 → ~/.pwn/finetune/pwn-dpo-*.jsonl (≤40%/source)
+              PWN::AI::Agent::Reward.export_dpo(format: :dpo)                              # W1 → ~/.pwn/finetune/pwn-dpo-*.jsonl (≤40%/source, scrubbed)
               PWN::AI::Agent::Reward.export_dpo(format: :dpo, balance: false)              # raw dump (diagnostics)
+              PWN::AI::Agent::Reward.scrub_preferences(dry_run: true)                      # P15 ledger hygiene report
+              PWN::AI::Agent::Reward.scrub_preferences                                      # P15 rewrite jsonl (backup first)
+              PWN::AI::Agent::Reward.preference_balance(scrub: true)                       # P15 geometry-aware mix
 
               # Tier 6 — grounded reward
               PWN::AI::Agent::Reward.verify_as_reward(final: text)                         # E3 browser-verified reward

data/lib/pwn/ai/agent/tools/curriculum.rb

@@ -144,3 +144,26 @@ PWN::AI::Agent::Registry.register(
   check: -> { defined?(PWN::AI::Agent::Curriculum) && PWN::AI::Agent::Curriculum.respond_to?(:preference_balance) },
   handler: ->(args) { PWN::AI::Agent::Curriculum.preference_balance(limit: args[:limit]) }
 )
+
+PWN::AI::Agent::Registry.register(
+  name: 'curriculum_practice_kpi',
+  toolset: 'curriculum',
+  schema: {
+    name: 'curriculum_practice_kpi',
+    description: 'P1 — Outer curriculum KPI: unresolved [REPEATING] count + week-over-week trend (improving|flat|regressing). Snapshots land in ~/.pwn/curriculum_kpi.jsonl from practice().',
+    parameters: {
+      type: 'object',
+      properties: {
+        limit: { type: 'integer', description: 'Trend window snapshots (default 14)' }
+      },
+      required: []
+    }
+  },
+  check: -> { defined?(PWN::AI::Agent::Curriculum) && PWN::AI::Agent::Curriculum.respond_to?(:repeating_trend) },
+  handler: lambda { |args|
+    {
+      trend: PWN::AI::Agent::Curriculum.repeating_trend(limit: args[:limit]),
+      snapshot: PWN::AI::Agent::Curriculum.practice_kpi(results: [])
+    }
+  }
+)

data/lib/pwn/ai/agent/tools/reward.rb

@@ -126,3 +126,89 @@ PWN::AI::Agent::Registry.register(
     )
   }
 )
+
+PWN::AI::Agent::Registry.register(
+  name: 'reward_warm_sentinel',
+  toolset: 'learning',
+  schema: {
+    name: 'reward_warm_sentinel',
+    description: 'P10 — Backfill the R3 sentinel ring buffer from Learning ' \
+                 'outcomes so proxy_distrust can engage on local/offline hosts ' \
+                 'without waiting for live remote introspect. Only fills empty ' \
+                 'slots; never flushes a warm window.',
+    parameters: {
+      type: 'object',
+      properties: {
+        limit: { type: 'integer', default: 120, description: 'Max outcomes to scan.' }
+      },
+      required: []
+    }
+  },
+  check: -> { defined?(PWN::AI::Agent::Reward) && PWN::AI::Agent::Reward.respond_to?(:warm_sentinel) },
+  handler: ->(args) { PWN::AI::Agent::Reward.warm_sentinel(limit: args[:limit] || 120) }
+)
+
+PWN::AI::Agent::Registry.register(
+  name: 'reward_scrub_preferences',
+  toolset: 'learning',
+  schema: {
+    name: 'reward_scrub_preferences',
+    description: 'P15 — One-shot W1 ledger hygiene. Drops CORRECTION:-only chosen, resolve rows without trajectory shape, and chosen≪rejected pairs. dry_run:true reports; false rewrites ~/.pwn/preferences.jsonl (backup first).',
+    parameters: {
+      type: 'object',
+      properties: {
+        dry_run: { type: 'boolean', default: true, description: 'Report only (default true).' }
+      },
+      required: []
+    }
+  },
+  check: -> { defined?(PWN::AI::Agent::Reward) && PWN::AI::Agent::Reward.respond_to?(:scrub_preferences) },
+  handler: lambda { |args|
+    dry = args.key?(:dry_run) ? args[:dry_run] : true
+    PWN::AI::Agent::Reward.scrub_preferences(dry_run: dry)
+  }
+)
+
+PWN::AI::Agent::Registry.register(
+  name: 'reward_preference_balance',
+  toolset: 'learning',
+  schema: {
+    name: 'reward_preference_balance',
+    description: 'P15/P5 — Geometry-aware W1 diversity report. scrub:true scores the post-hygiene diet (trajectory_fraction, by_shape) so promote/export gates see usable pairs only.',
+    parameters: {
+      type: 'object',
+      properties: {
+        limit: { type: 'integer', default: 10_000 },
+        scrub: { type: 'boolean', default: true }
+      },
+      required: []
+    }
+  },
+  check: -> { defined?(PWN::AI::Agent::Reward) && PWN::AI::Agent::Reward.respond_to?(:preference_balance) },
+  handler: lambda { |args|
+    PWN::AI::Agent::Reward.preference_balance(
+      limit: args[:limit] || 10_000,
+      scrub: args.key?(:scrub) ? args[:scrub] : true
+    )
+  }
+)
+
+PWN::AI::Agent::Registry.register(
+  name: 'reward_generator_mix',
+  toolset: 'reward',
+  schema: {
+    name: 'reward_generator_mix',
+    description: 'P0 — Online W1 generator mix vs TARGET_SOURCE_MIX. Returns urgent/suppress sources, trajectory_fraction, recommendation (boost:… / suppress:…). Controllers use this to schedule counterfactual/critic/curriculum over resolve flood.',
+    parameters: {
+      type: 'object',
+      properties: {
+        limit: { type: 'integer', description: 'Window size (default WRITE_SOURCE_WINDOW)' }
+      },
+      required: []
+    }
+  },
+  check: -> { defined?(PWN::AI::Agent::Reward) && PWN::AI::Agent::Reward.respond_to?(:generator_mix) },
+  handler: lambda { |args|
+    PWN::AI::Agent::Reward.generator_mix(limit: args[:limit])
+  }
+)