pwn 0.5.721 → 0.5.723

data/lib/pwn/ai/agent/learning.rb

@@ -26,6 +26,7 @@ module PWN
       # restarts and is shared by every future session.
       module Learning
         LEARNING_FILE      = File.join(Dir.home, '.pwn', 'learning.jsonl')
+        DISPUTED_FILE      = File.join(Dir.home, '.pwn', 'learning', 'disputed.jsonl')
         LESSONS_FILE       = File.join(Dir.home, '.pwn', 'lessons.json')
         FINETUNE_DIR       = File.join(Dir.home, '.pwn', 'finetune')
         # P0 — post-answer introspect must not train "stop early" while
@@ -102,18 +103,20 @@ module PWN
 
           tags = Array(opts[:tags]).map(&:to_s)
           details = opts[:details].to_s[0, OUTCOME_DETAILS_MAX]
-          # Refuse solved-at-fail (and the reverse): when a score is present,
-          # tags / details / boolean success must match verdict_for_score.
-          # Soft HER stays soft so C3 positives are not flipped to true/false.
-          if opts.key?(:score)
-            score = opts[:score].to_f
-            want = verdict_for_score(score: score).to_s
+          decision = if opts[:outcome].is_a?(Hash)
+                       Reward.resolve_outcome(outcome: opts[:outcome])
+                     elsif opts.key?(:score) && success != 'soft'
+                       Reward.resolve_outcome(outcome: { score: opts[:score], source: opts[:judge_source] || :manual, confidence: opts[:confidence] })
+                     end
+          if decision || opts.key?(:score)
+            score = decision ? decision[:score] : opts[:score].to_f
+            want = (decision ? decision[:verdict] : verdict_for_score(score: score)).to_s
             tags = (tags - %w[solved partial wrong unknown]) << want
             details = details.sub(
               /\A(solved|partial|wrong|unknown)\(\d+(?:\.\d+)?\)/i,
-              "#{want}(#{format('%.2f', score)})"
+              "#{want}(#{score.nil? ? 'unknown' : format('%.2f', score)})"
             )
-            success = (score >= 0.6) if [true, false].include?(success)
+            success = decision[:success] if decision
           end
 
           entry = {
@@ -125,18 +128,27 @@ module PWN
             tags: tags,
             timestamp: Time.now.utc.iso8601
           }
-          entry[:score] = opts[:score].to_f if opts.key?(:score)
+          entry[:score] = score if decision || opts.key?(:score)
+          if decision
+            %i[verdict confidence training_score decision_version verification verifier_verdict grounded critic_pass judge_score].each do |key|
+              entry[key] = decision[key] if decision.key?(key)
+            end
+            entry[:status] = 'unverified' if decision[:training_score].nil?
+          end
           src = opts[:judge_source].to_s
+          src = decision[:source].to_s if decision && decision[:source]
           entry[:judge_source] = src unless src.empty?
-          vv = (opts[:verifier_verdict] || opts['verifier_verdict']).to_s
+          vv = (decision ? decision[:verifier_verdict] : opts[:verifier_verdict] || opts['verifier_verdict']).to_s
           entry[:verifier_verdict] = vv unless vv.empty?
           vc = (opts[:verdict_class] || opts['verdict_class']).to_s
           entry[:verdict_class] = vc unless vc.empty?
           entry[:remediation_hint] = opts[:remediation_hint].to_s unless opts[:remediation_hint].to_s.empty?
-          if vv == 'pass' && opts.key?(:score) && opts[:score].to_f < 0.6
-            entry[:status] = 'conflicted'
-            entry[:success] = true
-            success = true
+          entry[:status] = 'conflicted' if opts[:verifier_verdict].to_s == 'pass' && !entry[:verification] && opts[:score].to_f < 0.6
+          check = consistency_check(details: details, success: success, rationale: opts[:rationale])
+          if check == :disputed
+            entry[:status] = 'disputed'
+            disputed_save(entry: entry)
+            return entry
           end
           FileUtils.mkdir_p(File.dirname(LEARNING_FILE))
           File.open(LEARNING_FILE, 'a') { |f| f.puts(JSON.generate(entry)) }
@@ -147,18 +159,33 @@ module PWN
           # are promoted into PWN::Memory[:lesson] so PromptBuilder recall
           # survives across sessions. Without this, the agent re-learns
           # "run rubocop after every patch" every turn (empty memory.json).
-          promote_process_lesson(entry: entry) if defined?(PWN::Memory) && entry[:status].to_s != 'conflicted'
-          if opts.key?(:score) && defined?(Curriculum) && Curriculum.respond_to?(:calibrate)
+          promote_process_lesson(entry: entry) if defined?(PWN::Memory) && !%w[conflicted unverified].include?(entry[:status].to_s)
+          if opts.key?(:score) && (!decision || !decision[:training_score].nil?) && defined?(Curriculum) && Curriculum.respond_to?(:calibrate)
             pred = opts[:predicted]
             pred = Thread.current[:pwn_plan_predicted] if pred.nil?
             pred = opts[:confidence] if pred.nil?
             eng = opts[:engine]
             eng = (PWN::Env.dig(:ai, :active) if defined?(PWN::Env)) if eng.to_s.empty?
-            Curriculum.calibrate(predicted: pred, actual: opts[:score], engine: eng)
+            Curriculum.calibrate(predicted: pred, actual: decision ? decision[:training_score] : opts[:score], engine: eng)
           end
           entry
         end
 
+        public_class_method def self.consistency_check(opts = {})
+          outcome = opts[:outcome]
+          return :ok unless outcome.is_a?(Hash)
+          return :disputed if Reward.resolve_outcome(outcome: outcome)[:success] != opts[:success]
+
+          :ok
+        end
+
+        public_class_method def self.disputed_save(opts = {})
+          entry = opts[:entry] || {}
+          FileUtils.mkdir_p(File.dirname(DISPUTED_FILE))
+          File.open(DISPUTED_FILE, 'a') { |f| f.puts(JSON.generate(entry)) }
+          entry
+        end
+
         # Supported Method Parameters::
         # rows = PWN::AI::Agent::Learning.outcomes(
         #   limit: 'optional - max entries returned newest-first (default 50)',
@@ -178,6 +205,7 @@ module PWN
             nil
           end
           rows.compact!
+          rows.reject! { |r| r[:decision_version] && r[:training_score].nil? } unless want_ok.nil?
           rows.select! { |r| want_ok == true ? r[:success] == true : r[:success] == want_ok } unless want_ok.nil?
           rows.select! { |r| Array(r[:tags]).any? { |t| t.to_s.downcase.include?(tag) } } unless tag.empty?
           rows.reverse.first(limit)
@@ -189,11 +217,13 @@ module PWN
         public_class_method def self.stats
           rows   = outcomes(limit: 10_000)
           total  = rows.length
+          rows = rows.reject { |r| (r[:decision_version] && r[:training_score].nil?) || r[:success].nil? }
+          evaluated = rows.length
           ok     = rows.count { |r| r[:success] == true }
           skills = defined?(PWN::Skills) && PWN::Skills.is_a?(Hash) ? PWN::Skills.keys.length : 0
           mem    = defined?(PWN::Memory) ? PWN::Memory.load.keys.length : 0
-          raw    = total.positive? ? (ok.to_f / total).round(3) : 0.0
-          jmean  = total.positive? ? weighted_judge_mean(rows: rows) : nil
+          raw    = evaluated.positive? ? (ok.to_f / evaluated).round(3) : 0.0
+          jmean  = evaluated.positive? ? weighted_judge_mean(rows: rows) : nil
           distrust = 0.0
           distrust = Reward.proxy_distrust.to_f.clamp(0.0, 1.0) if defined?(Reward) && Reward.respond_to?(:proxy_distrust)
           orm = rows.select { |r| r[:judge_source].to_s != 'heuristic' && r[:source].to_s != 'heuristic' }
@@ -204,8 +234,9 @@ module PWN
           heur_ok = heur.count { |r| r[:success] == true }
           {
             total_outcomes: total,
+            unknown_outcomes: total - evaluated,
             successes: ok,
-            failures: total - ok,
+            failures: rows.count { |r| r[:success] == false },
             success_rate: raw,
             success_rate_orm: orm_n.positive? ? (orm_ok.to_f / orm_n).round(3) : 0.0,
             success_rate_heur: heur_n.positive? ? (heur_ok.to_f / heur_n).round(3) : 0.0,
@@ -251,17 +282,22 @@ module PWN
           return '' if rows.empty? && fails.empty?
 
           fmt = lambda do |r|
+            unknown = r[:status].to_s == 'unverified' || (r[:decision_version] && r[:training_score].nil?)
             flag = case r[:success]
                    when true then '✓'
                    when 'soft', :soft then '∼'
                    else '✗'
                    end
             score = r.key?(:score) ? format('%.2f', r[:score].to_f) : '-'
+            if unknown
+              flag = '?'
+              score = 'unknown'
+            end
             task  = display_task(task: r[:task])
             line  = "  #{flag} [#{score}] #{task} (#{r[:timestamp]})"
             # Surface a one-line cause crumb so the agent can actually learn
             # from failures instead of only seeing that they failed.
-            if r[:success] != true
+            if r[:success] != true && !unknown
               if r[:verdict_class].to_s == ''
                 crumb = cause_crumb(details: r[:details])
                 line += "\n      cause: #{crumb}" unless crumb.empty?
@@ -609,14 +645,10 @@ module PWN
 
           # R1 judge — always attempt (heuristic is cheap; LLM gated inside)
           stages_run << :judge
-          v = Reward.judge(request: opts[:request], final: opts[:final], session_id: session_id, proxy_ok: proxy_ok, predicted: opts[:predicted]) if defined?(Reward)
-          v ||= { score: proxy_ok ? 1.0 : 0.0, success: proxy_ok, verdict: proxy_ok ? :solved : :wrong }
-          v[:score] = [v[:score], 0.3].min if crit[:verdict] == :flaw && v[:score].to_f < 0.6
-          # P29 — critic floor used to leave stale verdict=:solved at score=0.3,
-          # producing learning.jsonl rows tagged "solved" with success=false
-          # (116+ rows). Always resync verdict/success from the final score.
-          v[:verdict] = verdict_for_score(score: v[:score])
-          v[:success] = v[:score].to_f >= 0.6
+          critic_pass = crit[:verdict] == :flaw ? false : nil
+          v = Reward.judge(request: opts[:request], final: opts[:final], session_id: session_id, proxy_ok: proxy_ok, predicted: opts[:predicted], critic_pass: critic_pass) if defined?(Reward)
+          v ||= { score: nil, source: :error, verdict: :unknown, success: nil }
+          v = Reward.resolve_outcome(outcome: v, critic_pass: critic_pass)
           ok = v[:success]
 
           # W1 pending user_correction pair
@@ -664,20 +696,23 @@ module PWN
             task: task_txt,
             success: ok,
             score: v[:score],
+            outcome: v,
             details: "#{v[:verdict]}(#{v[:score].to_f.round(2)}) #{v[:rationale]} | #{opts[:final].to_s[0, 200]}",
             session_id: session_id,
             tags: outcome_tags,
             judge_source: v[:source]
           )
 
-          stages_run << :fold_judge
-          fold_judge_into_metrics(session_id: session_id, score: v[:score], confidence: v[:confidence])
+          unless v[:training_score].nil?
+            stages_run << :fold_judge
+            fold_judge_into_metrics(session_id: session_id, score: v[:training_score], confidence: v[:confidence])
+          end
           # R5 — close the live MDP episode with the ORM terminal reward.
           if defined?(PWN::AI::Agent::Policy) && Policy.respond_to?(:finish)
             stages_run << :policy
             Policy.finish(
               session_id: session_id,
-              score: v[:score],
+              score: v[:training_score],
               confidence: v[:confidence],
               verdict: v[:verdict],
               proxy_ok: ok,
@@ -687,7 +722,7 @@ module PWN
           end
 
           # R2 PRM — skip under hard cap (expensive LLM); keep under soft if heuristic path
-          if over_hard.call || !defined?(Reward) || v[:score].to_f < 0.6
+          if over_hard.call || !defined?(Reward) || v[:training_score].nil? || !ok
             stages_skipped << :prm
           else
             stages_run << :prm
@@ -695,7 +730,7 @@ module PWN
           end
 
           # C3 HER — only on failure; skip hard
-          if !ok && defined?(Curriculum) && !over_hard.call
+          if !ok && !v[:training_score].nil? && defined?(Curriculum) && !over_hard.call
             stages_run << :hindsight
             Curriculum.hindsight(request: opts[:request], final: opts[:final], session_id: session_id)
           else
@@ -711,7 +746,7 @@ module PWN
           # M4.1 — also reflect when the request/final is a process SOP
           # (code hygiene) even if judge score < 0.6, so rubocop/rake
           # lessons still land in PWN::Memory.
-          process_sop = process_sop_text?(text: "#{opts[:request]} #{opts[:final]}")
+          process_sop = !v[:training_score].nil? && process_sop_text?(text: "#{opts[:request]} #{opts[:final]}")
           if (ok || process_sop) && !over_soft.call
             stages_run << :reflect
             reflect(session_id: session_id)
@@ -1105,6 +1140,8 @@ module PWN
           lines = File.readlines(LEARNING_FILE)
           out = lines.map do |l|
             r = JSON.parse(l, symbolize_names: true)
+            next l if r[:decision_version]
+
             score = r.key?(:score) ? r[:score].to_f : nil
             next l if score.nil?
 
@@ -1874,14 +1911,31 @@ module PWN
 
           n = 0
           rows.each do |r|
-            note_outcome(
-              task: "requeue:#{r[:task]}",
-              success: true,
-              score: [r[:score].to_f, 0.7].max,
-              verifier_verdict: :pass,
-              details: 'rescored after verifier precedence',
-              tags: %w[requeue]
+            next if r[:session_id].to_s.empty?
+
+            transcript = PWN::Sessions.load(session_id: r[:session_id])
+            user_idx = transcript.rindex { |entry| entry[:role].to_s == 'user' }
+            next unless user_idx
+
+            request = transcript[user_idx][:content].to_s
+            final = transcript[(user_idx + 1)..].reverse.find { |entry| entry[:role].to_s == 'assistant' }
+            next unless final && (request == r[:task].to_s || display_task(task: request) == r[:task].to_s)
+
+            outcome = Reward.judge(request: request, final: final[:content], session_id: r[:session_id], commit: false)
+            fresh = note_outcome(
+              task: request,
+              session_id: r[:session_id],
+              outcome: outcome,
+              details: outcome[:rationale].to_s,
+              tags: %w[requeue],
+              judge_source: outcome[:source]
             )
+            updated = File.readlines(LEARNING_FILE).map do |line|
+              row = JSON.parse(line, symbolize_names: true)
+              row.merge!(status: 'rejudged', rescore_id: fresh[:id]) if row[:id] == r[:id]
+              "#{JSON.generate(row)}\n"
+            end
+            File.write(LEARNING_FILE, updated.join)
             n += 1
           end
           { rescored: n, dry_run: false }
@@ -1915,14 +1969,16 @@ module PWN
               task: 'required - short description of what was attempted',
               success: 'required - Boolean, did the attempt achieve its goal',
               details: 'optional - free-form notes / error / evidence',
+              rationale: 'optional - judge explanation retained for consistency checks',
               session_id: 'optional - PWN::Sessions id this outcome belongs to',
               tags: 'optional - Array of String labels for later retrieval',
               score: 'optional - score value consumed by #note_outcome',
+              outcome: 'optional - canonical Reward outcome; preserves evidence, verdict and training eligibility',
               judge_source: 'required - judge source value consumed by #note_outcome',
               predicted: 'optional - predicted value consumed by #note_outcome',
               confidence: 'optional - confidence value consumed by #note_outcome',
               engine: 'optional - engine value consumed by #note_outcome',
-              verifier_verdict: 'optional - pass|fail from a deterministic verifier',
+              verifier_verdict: 'optional - legacy diagnostic flag; cannot prove completion',
               verdict_class: 'optional - missing_artifact|wrong_path|unverified_claim|scope_miss|partial_coverage|style_only',
               remediation_hint: 'optional - one-line fix hint'
             )
@@ -1932,7 +1988,7 @@ module PWN
               limit: 'optional - max entries (defaults to 50)'
             )
 
-            # Rescore conflicted outcomes after verifier-precedence lands.
+            # Rejudge conflicted outcomes from the original session; never boost scores blindly.
             #{self}.requeue_conflicted(
               dry_run: 'optional - true to count without writing'
             )
@@ -2081,6 +2137,17 @@ module PWN
               include_demoted: 'optional - include demoted lessons (defaults to false)'
             )
 
+            # Compare structured decisions, never infer verification from prose.
+            #{self}.consistency_check(
+              outcome: 'optional - canonical outcome to compare against success',
+              success: 'required - boolean success flag'
+            )
+
+            # Append a disputed outcome to ~/.pwn/learning/disputed.jsonl.
+            #{self}.disputed_save(
+              entry: 'required - Hash of the disputed learning row'
+            )
+
             # Print the AUTHOR(S) string for this module.
             #{self}.authors
           "

data/lib/pwn/ai/agent/loop.rb

@@ -171,7 +171,7 @@ module PWN
           return unless debug_on?(opts)
           return unless defined?(PWN::Plugins::Log)
 
-          if defined?(TurnFinalizer) && TurnFinalizer.user_path?
+          if opts[:nested] || (defined?(TurnFinalizer) && TurnFinalizer.user_path?)
             debug_progress(msg: 'nested Loop.run skip_roll', debug: opts[:debug])
             return
           end
@@ -704,7 +704,7 @@ module PWN
           request = opts[:request].to_s
           unmet = []
           files = Array(contract[:paths]) + Array(contract[:proofs])
-          files.each { |path| unmet << "file_exists:#{path}" if deliverable_missing?(path: path) }
+          files.each { |path| unmet << "deliverable_missing:#{path}" if deliverable_missing?(path: path) }
           trace_files = session_files(messages: opts[:messages])
           unmet << 'issue_work_proofs' if contract[:issue_work] && Array(contract[:proofs]).empty? && files.empty? && trace_files.empty?
           unmet << 'skills' if declared_skills_missing?(skills: contract[:skills], request: request)
@@ -713,6 +713,7 @@ module PWN
           asked_tech = Array(contract[:techniques]).select { |tech| request.downcase.include?(tech.to_s.downcase) }
           unmet << 'hosts' if evidence_tokens_missing?(tokens: asked_hosts, blob: blob)
           unmet << 'techniques' if evidence_tokens_missing?(tokens: asked_tech, blob: blob)
+          Array(Swarm.honesty_unmet(swarm_id: Thread.current[:pwn_swarm_id])).each { |u| unmet << u } if defined?(Swarm)
           unmet
         rescue StandardError
           []
@@ -1063,6 +1064,9 @@ module PWN
         end
 
         private_class_method def self.active_engine
+          tl = Thread.current[:pwn_swarm_engine].to_s
+          return tl.downcase.to_sym unless tl.empty?
+
           e = (PWN::Env.dig(:ai, :active) if defined?(PWN::Env)).to_s.downcase.to_sym
           e == :'' ? :openai : e
         rescue StandardError
@@ -1117,9 +1121,16 @@ module PWN
           Metrics.record(name: name, success: sem[:semantic_ok], duration: dur, error: sem[:err], engine: opts[:engine]) if defined?(Metrics)
           # R5 — live MDP step. Hygiene reward only; terminal credit is judge.
           if defined?(PWN::AI::Agent::Policy) && Policy.respond_to?(:observe_step)
+            policy_args = begin
+              opts[:args].is_a?(String) ? JSON.parse(opts[:args]) : opts[:args]
+            rescue JSON::ParserError
+              nil
+            end
             Policy.observe_step(
               session_id: opts[:session_id],
               action: name,
+              args: policy_args,
+              result_type: sem[:shape],
               ok: sem[:semantic_ok],
               duration: dur,
               engine: opts[:engine],
@@ -1176,6 +1187,16 @@ module PWN
           false
         end
 
+        private_class_method def self.session_tool_budget(opts = {})
+          name = opts[:name].to_s
+          result = opts[:result].to_s
+          return 16_384 if name.match?(/decompile|binary_triage|artifact_read|exploitdev/)
+          return 16_384 if name == 'pwn_eval' && result.match?(/r2 |objdump|xxd |pdf\b|afl-/)
+          return 16_384 if name == 'shell' && result.match?(/objdump|xxd|radare|r2 |disassembl/)
+
+          1_024
+        end
+
         # E1 — did the environment change under this tool? If Metrics CUSUM
         # tripped for it in the last hour AND Extrospection.drift shows a
         # toolchain/net/repo change, blame the WORLD not the AGENT.
@@ -1384,7 +1405,9 @@ module PWN
           hint = Swarm.ask(
             name: persona.to_s,
             request: "Local agent is stuck on: #{request}\nFailed attempts: #{summary}\n" \
-                     'Give a 3-line corrective hint (which tool, which args, why). Reply with the hint ONLY.'
+                     'Give a 3-line corrective hint (which tool, which args, why). Reply with the hint ONLY.',
+            swarm_id: Thread.current[:pwn_swarm_id],
+            text_only: true
           )
           reply = hint.is_a?(Hash) ? hint[:reply].to_s : hint.to_s
           Mistakes.record(tool: 'escalation', error: "local stuck after #{turn_fails.values.sum} fails; frontier hint requested", session_id: opts[:session_id], source: :loop) if defined?(Mistakes)
@@ -1692,10 +1715,12 @@ module PWN
 
           mod = Object.const_get(mod_name)
           if mod.respond_to?(:chat_with_tools)
-            # xAI/OpenAI reject Hash function.arguments / Hash content (422 map→string).
+            # xAI rejects Hash function.arguments / Hash content (422 map→string).
+            # OpenAI sanitizes inside its provider so native Responses reasoning
+            # survives until the credential-specific transport is selected.
             # Ollama / Open WebUI reject *string* function.arguments (HTTP 400
             # "can't find closing '}' symbol") — opposite of OpenAI wire form.
-            wire_msgs = if %i[grok openai].include?(engine)
+            wire_msgs = if engine == :grok
                           openai_wire_messages(messages: messages)
                         elsif local_engine?(engine: engine)
                           ollama_wire_messages(messages: messages)
@@ -2717,7 +2742,7 @@ module PWN
           loud_debug_tui!(debug: opts[:debug])
           debug_progress(msg: "Loop.run start request=#{request[0, 240]}", debug: opts[:debug])
           ToolGuard.reset_timeout_budget! if defined?(ToolGuard) && ToolGuard.respond_to?(:reset_timeout_budget!)
-          nested = defined?(TurnFinalizer) && TurnFinalizer.user_path?
+          nested = opts[:nested] == true || Thread.current[:pwn_swarm_depth].to_i.positive? || (defined?(TurnFinalizer) && TurnFinalizer.user_path?)
           TurnFinalizer.enter_user_path! if defined?(TurnFinalizer)
           Thread.current[:pwn_loop_nested] = nested
           bound = operator_bound_refusal(from: opts[:from] || opts[:account])
@@ -2932,6 +2957,14 @@ module PWN
 
             t0 = Time.now
             begin
+              # Observations update the policy's context during execution.
+              # Refresh exposure for the next hop without widening its scope
+              # or substituting a generated goal for the original request.
+              if tools_called.positive?
+                tools = Registry.definitions(enabled: opts[:enabled_toolsets], relevance: request, core_only: core_only, intent: intent)
+                no_tools = Array(tools).empty?
+                Thread.current[:pwn_loop_no_tools] = no_tools
+              end
               repair_tool_history!(messages: messages)
               msg = call_engine(messages: messages, tools: tools, ts_state: ts_state)
             rescue StandardError => e
@@ -3126,7 +3159,7 @@ module PWN
               append_session(
                 session_id: session_id,
                 role: 'tool',
-                content: "#{name} → #{result[0, 1_024]}"
+                content: "#{name} → #{result[0, session_tool_budget(name: name, result: result)]}"
               )
             end
 
@@ -3151,6 +3184,11 @@ module PWN
           end
           raise
         rescue StandardError => e
+          if defined?(PWN::AI::HttpRetry) && PWN::AI::HttpRetry.quota_exhausted?(error: e)
+            msg = PWN::AI::HttpRetry.quota_message(error: e)
+            debug_progress(msg: "engine quota: #{msg}")
+            return msg
+          end
           if defined?(PWN::Plugins::Log) && PWN::Plugins::Log.respond_to?(:note_exception!)
             PWN::Plugins::Log.note_exception!(error: e, where: 'Loop.run', which_self: self)
           else
@@ -3250,7 +3288,11 @@ module PWN
               debug: 'optional - debug value consumed by #run',
               from: 'optional - sender account or address to bind as operator',
               account: 'optional - operator account id to bind',
-              force_tools: 'optional - force tools value consumed by #run'
+              force_tools: 'optional - force tools value consumed by #run',
+              nested: 'optional - true for Swarm/child Loop.run (skip RN footer)',
+              core_only: 'optional - restrict to CORE_TOOLS when true',
+              trace: 'optional - enable TracePoint debug for this run',
+              debug_tee: 'optional - IO to tee debug logs'
             )
 
             # Remaining time/token/mutation budget for the current loop.

data/lib/pwn/ai/agent/metrics.rb

@@ -156,6 +156,49 @@ module PWN
               .sort_by { |r| [-r[:calls], -r[:success_rate]] }.first(limit)
         end
 
+        public_class_method def self.snapshot(opts = {})
+          _day = opts[:day]
+          learn = defined?(Learning) ? Learning.stats : {}
+          {
+            success_rate: learn[:success_rate],
+            success_rate_orm: learn[:success_rate_orm],
+            success_rate_heur: learn[:success_rate_heur],
+            brier: (learn.dig(:calibration, :brier) if learn.is_a?(Hash)),
+            proxy_distrust: learn[:proxy_distrust],
+            tools: summary(limit: 8)
+          }
+        end
+
+        public_class_method def self.record_tokens(opts = {})
+          n = opts[:tokens].to_i
+          cost = opts[:cost].to_f
+          model = opts[:model].to_s
+          m = load
+          m[:usage] ||= { tokens: 0, cost: 0.0, calls: 0, by_model: {} }
+          m[:usage][:tokens] += n
+          m[:usage][:cost] += cost
+          m[:usage][:calls] += 1
+          m[:usage][:by_model][model] ||= { tokens: 0, cost: 0.0, calls: 0 }
+          m[:usage][:by_model][model][:tokens] += n
+          m[:usage][:by_model][model][:cost] += cost
+          m[:usage][:by_model][model][:calls] += 1
+          save(metrics: m)
+          m[:usage]
+        end
+
+        public_class_method def self.usage(opts = {})
+          _sid = opts[:session_id]
+          load[:usage] || { tokens: 0, cost: 0.0, calls: 0 }
+        end
+
+        public_class_method def self.routing(opts = {})
+          _n = opts[:n]
+          chain = (PWN::Env.dig(:ai, :routing) if defined?(PWN::Env))
+          Array(chain)
+        rescue StandardError
+          []
+        end
+
         # Supported Method Parameters::
         # ctx = PWN::AI::Agent::Metrics.to_context(
         #   limit: 'optional - cap number of tools included (default 8)',
@@ -544,9 +587,12 @@ module PWN
                     end
           task_ok = nil
           if defined?(Learning) && Learning.respond_to?(:outcomes)
-            rec = Learning.outcomes(limit: 200)
+            rec = Learning.outcomes(limit: 200).reject do |r|
+              r[:success].nil? || r[:status].to_s == 'unverified' || r[:verdict].to_s == 'unknown' ||
+                (r[:decision_version] && r[:training_score].nil?)
+            end
             if rec.any?
-              hits = rec.count { |r| r[:success] == true || r[:score].to_f >= 0.6 }
+              hits = rec.count { |r| r[:success] == true }
               task_ok = (hits.to_f / rec.length).round(3)
             end
           end
@@ -814,6 +860,28 @@ module PWN
             # Run reset and return its result
             #{self}.reset
 
+            # Snapshot success rates per judge source for the LEARNING block.
+            #{self}.snapshot(
+              day: 'optional - reserved day key for rotation'
+            )
+
+            # Record token/cost for a model call.
+            #{self}.record_tokens(
+              tokens: 'optional - integer token count',
+              cost: 'optional - Float USD cost',
+              model: 'optional - model id string'
+            )
+
+            # Return cumulative token/cost usage.
+            #{self}.usage(
+              session_id: 'optional - unused reserved session id'
+            )
+
+            # Return ai.routing fallback chain from pwn.yaml.
+            #{self}.routing(
+              n: 'optional - unused reserved index'
+            )
+
             # Print the AUTHOR(S) string for this module.
             #{self}.authors
           "

data/lib/pwn/ai/agent/mistakes.rb

@@ -127,6 +127,31 @@ module PWN
           'other'
         end
 
+        public_class_method def self.family(opts = {})
+          klass = error_class(opts)
+          case klass
+          when 'docker_registry_auth', 'auth_denied' then 'auth_denied'
+          when 'name_conflict' then 'name_conflict'
+          when 'parse_error' then 'parse_error'
+          when 'missing_path', 'enoent' then 'missing_path'
+          when 'perm_denied', 'socket_perm', 'eacces' then 'permission_capability'
+          when 'net_unreach' then 'network_unreachable'
+          when 'timeout' then 'timeout'
+          else 'other'
+          end
+        end
+
+        FAMILY_FIXES = {
+          'auth_denied' => 'check registry credentials / image name; do NOT ls paths',
+          'name_conflict' => 'rename the resource or remove the existing name; do NOT ls paths',
+          'parse_error' => 'fix template/JSON syntax; do NOT ls paths',
+          'missing_path' => 'Path missing. ls/test -e the parent first, then run. Do not retry the same missing path.',
+          'permission_capability' => 'Raw sockets need CAP_NET_RAW. Do not retry open_sockraw. Use connect-scan or Sock.',
+          'network_unreachable' => 'network unreachable; check routing/VPN, do not ls paths',
+          'timeout' => 'deadline too short; retry same payload with timeout += 180',
+          'other' => 'inspect the exact error class before retrying'
+        }.freeze
+
         # Supported Method Parameters::
         # entry = PWN::AI::Agent::Mistakes.find(
         #   signature: 'optional - exact signature to fetch',
@@ -630,10 +655,38 @@ module PWN
 
           parts = ["seen #{m[:count]}× across #{Array(m[:sessions]).length} session(s), sig=#{m[:signature]}"]
           parts << 'REGRESSED (previous fix did not hold)' if m[:regressed]
-          parts << "KNOWN FIX: #{m[:fix]}" if m[:fix].to_s.strip.length.positive?
+          fam = family(error: opts[:error] || m[:error])
+          fam_fix = FAMILY_FIXES[fam]
+          stored = m[:fix].to_s.strip
+          stored = '' if stored.match?(%r{ls/test -e the parent}) && fam != 'missing_path'
+          stored = '' if m[:hint_confidence].to_f.negative?
+          stored = '' if stored.include?('[UNVERIFIED]')
+          hint = stored.empty? ? fam_fix : stored
+          parts << "KNOWN FIX: #{hint}" unless hint.to_s.empty?
           "[pwn-ai/mistakes] #{parts.join(' | ')}"
         end
 
+        public_class_method def self.note_hint_outcome(opts = {})
+          sig = (opts[:signature] || signature(tool: opts[:tool], error: opts[:error])).to_s
+          return nil if sig.empty?
+
+          store = load
+          m = store[sig.to_sym]
+          return nil unless m
+
+          m[:hint_outcomes] = m[:hint_outcomes].to_i + 1
+          unless opts[:helped]
+            m[:hint_misses] = m[:hint_misses].to_i + 1
+            if m[:hint_misses].to_i >= 3
+              m[:fix] = "[UNVERIFIED] #{m[:fix]}"
+              m[:hint_confidence] = -1.0
+            end
+          end
+          store[sig.to_sym] = m
+          save(store: store)
+          m
+        end
+
         # Supported Method Parameters::
         # bool = PWN::AI::Agent::Mistakes.correction?(request: user_text)
 
@@ -988,6 +1041,19 @@ module PWN
               signature: 'optional - signature value consumed by #effective_count'
             )
 
+            # Map an error string onto a coarse family (auth vs path vs parse).
+            #{self}.family(
+              error: 'required - raw error text'
+            )
+
+            # Record whether an injected KNOWN FIX helped; demote after 3 misses.
+            #{self}.note_hint_outcome(
+              tool: 'optional - tool name',
+              error: 'optional - raw error text',
+              signature: 'optional - explicit signature',
+              helped: 'optional - true when the hint resolved the failure'
+            )
+
             # Print the AUTHOR(S) string for this module.
             #{self}.authors
           "