pwn 0.5.707 → 0.5.709

data/lib/pwn/ai/agent/metrics.rb

@@ -484,7 +484,69 @@ module PWN
         # Supported Method Parameters::
         # PWN::AI::Agent::Metrics.reset
 
+        public_class_method def self.calibration_green?(opts = {})
+          cal = calibration(engine: opts[:engine])
+          return false if cal[:n].to_i < 8
+          return false if cal[:overconfidence].nil?
+
+          cal[:overconfidence].to_f <= 0.08
+        rescue StandardError
+          false
+        end
+
+        public_class_method def self.scale_prediction(opts = {})
+          p = opts[:predicted].to_f.clamp(0.0, 1.0)
+          cal = calibration(engine: opts[:engine])
+          return p.round(3) if cal[:n].to_i < 8 || cal[:overconfidence].nil?
+
+          oc = cal[:overconfidence].to_f
+          return p.round(3) if oc <= 0.02
+
+          temp = (1.0 + (4.0 * oc.clamp(0.0, 1.0))).clamp(1.0, 8.0)
+          q = p.clamp(1.0e-6, 1.0 - 1.0e-6)
+          logit = Math.log(q / (1.0 - q))
+          scaled = 1.0 / (1.0 + Math.exp(-(logit / temp)))
+          scaled.round(3)
+        rescue StandardError
+          opts[:predicted].to_f.clamp(0.0, 1.0)
+        end
+
+        public_class_method def self.scoreboard(opts = {})
+          rows = summary(limit: 50)
+          tool_ok = if rows.empty?
+                      nil
+                    else
+                      w = rows.sum { |r| r[:calls].to_f }
+                      w.positive? ? (rows.sum { |r| r[:success_rate].to_f * r[:calls].to_i } / w).round(3) : nil
+                    end
+          task_ok = nil
+          if defined?(Learning) && Learning.respond_to?(:outcomes)
+            rec = Learning.outcomes(limit: 200)
+            if rec.any?
+              hits = rec.count { |r| r[:success] == true || r[:score].to_f >= 0.6 }
+              task_ok = (hits.to_f / rec.length).round(3)
+            end
+          end
+          cal = calibration(engine: opts[:engine])
+          judge_ok = cal[:mean_actual]
+          if judge_ok.nil? && defined?(Reward) && Reward.respond_to?(:sentinel)
+            s = Reward.sentinel
+            judge_ok = s[:judge] if s.is_a?(Hash)
+          end
+          {
+            tool_ok: tool_ok,
+            task_ok: task_ok,
+            judge_ok: judge_ok,
+            mean_predicted: cal[:mean_predicted],
+            overconfidence: cal[:overconfidence],
+            n: cal[:n].to_i
+          }
+        rescue StandardError
+          { tool_ok: nil, task_ok: nil, judge_ok: nil, n: 0 }
+        end
+
         public_class_method def self.health_line
+          board = scoreboard
           gap = nil
           if defined?(Reward) && Reward.respond_to?(:sentinel)
             s = Reward.sentinel
@@ -497,7 +559,9 @@ module PWN
                   end
           traj = (Reward.generator_mix[:trajectory_fraction] if defined?(Reward) && Reward.respond_to?(:generator_mix))
           parked = (Mistakes.operator_inbox(limit: 50)[:count] if defined?(Mistakes) && Mistakes.respond_to?(:operator_inbox))
-          "HEALTH judge-proxy-gap=#{gap || '-'} repeating=#{trend[:status] || '-'} " \
+          "HEALTH tool_ok=#{board[:tool_ok] || '-'} task_ok=#{board[:task_ok] || '-'} " \
+            "judge_ok=#{board[:judge_ok] || '-'} pred=#{board[:mean_predicted] || '-'} " \
+            "judge-proxy-gap=#{gap || '-'} repeating=#{trend[:status] || '-'} " \
             "w1-traj=#{traj || '-'} parked-needs-human=#{parked || '-'}\n"
         rescue StandardError
           ''
@@ -587,26 +651,139 @@ module PWN
         # Display Usage for this Module
 
         public_class_method def self.help
-          puts <<~USAGE
-            USAGE:
-              PWN::AI::Agent::Metrics.record(name: 'shell', success: true, duration: 0.42, engine: :ollama)
-              PWN::AI::Agent::Metrics.summary(limit: 10, engine: :ollama)
-              PWN::AI::Agent::Metrics.to_context(limit: 8, engine: :ollama)   # injected by PromptBuilder
-              PWN::AI::Agent::Metrics.ucb(name: 'shell')                 # C1 exploration bonus
-              PWN::AI::Agent::Metrics.thompson(name: 'shell')            # C1 Beta(ok+1,fail+1) sample
-              PWN::AI::Agent::Metrics.advantage(name: 'shell')           # C1 local − global (P20 judge-blended)
-              PWN::AI::Agent::Metrics.record_judge(name: 'shell', score: 0.8) # P20 ORM→metrics
-              PWN::AI::Agent::Metrics.judge_rate(name: 'shell')         # P20 mean ORM
-              PWN::AI::Agent::Metrics.effective_rate(name: 'shell')     # P20 proxy⋈judge
-              PWN::AI::Agent::Metrics.changepoints(within_secs: 3600)    # E1 CUSUM regime changes
-              PWN::AI::Agent::Metrics.record_calibration(predicted: 0.8, actual: 1.0, brier: 0.04, engine: :ollama)
-              PWN::AI::Agent::Metrics.calibration(engine: :ollama)       # W3 Brier / overconfidence
-              PWN::AI::Agent::Metrics.reset
-              PWN::AI::Agent::Metrics.load
-              PWN::AI::Agent::Metrics.save(metrics: hash)
-
-              #{self}.authors
-          USAGE
+          puts "USAGE:
+            # Run load and return its result
+            #{self}.load
+
+            # Run save and return its result
+            #{self}.save(
+              metrics: 'required - Hash returned by .load / mutated in place'
+            )
+
+            # Run record and return its result
+            #{self}.record(
+              name: 'required - tool name that was dispatched',
+              success: 'required - Boolean, did the handler complete without error',
+              duration: 'optional - Float seconds the dispatch took',
+              error: 'optional - String error message when success is false',
+              engine: 'optional - Symbol/String AI engine that chose this tool (segments telemetry)'
+            )
+
+            # Run summary and return its result
+            #{self}.summary(
+              limit: 'optional - cap number of tools returned (default 25)',
+              engine: 'optional - only that engine\\s sub-bucket (falls back to global when absent)'
+            )
+
+            # Run to context and return its result
+            #{self}.to_context(
+              limit: 'optional - cap number of tools included (default 8)',
+              engine: 'optional - restrict to one engine\\s telemetry'
+            )
+
+            # P4 helper — Registry.rank calls this so β·advantage is scaled down
+            #{self}.proxy_trust
+
+            # Run ucb and return its result
+            #{self}.ucb(
+              name: 'optional - binary or identifier name',
+              c: 'optional - c value consumed by #ucb'
+            )
+
+            # Run thompson and return its result
+            #{self}.thompson(
+              name: 'optional - binary or identifier name'
+            )
+
+            # Run advantage and return its result
+            #{self}.advantage(
+              name: 'optional - binary or identifier name'
+            )
+
+            # Run prm advantage and return its result
+            #{self}.prm_advantage(
+              name: 'optional - binary or identifier name'
+            )
+
+            # Run prm n and return its result
+            #{self}.prm_n(
+              name: 'optional - binary or identifier name'
+            )
+
+            # P18 — called by Reward.prm after session annotate so live routing
+            #{self}.record_step_reward(
+              name: 'required - binary or identifier name',
+              reward: 'optional - reward value consumed by #record_step_reward'
+            )
+
+            # P20 — fold ORM judge (0..1) into per-tool telemetry so UCB /
+            #{self}.record_judge(
+              name: 'required - binary or identifier name',
+              score: 'optional - score value consumed by #record_judge',
+              confidence: 'optional - confidence value consumed by #record_judge',
+              source: 'optional - source value consumed by #record_judge'
+            )
+
+            # P1 — mean judge confidence for a tool (nil when no samples)
+            #{self}.judge_confidence(
+              name: 'optional - binary or identifier name'
+            )
+
+            # Mean judge score for a tool (nil when no ORM samples yet)
+            #{self}.judge_rate(
+              name: 'optional - binary or identifier name'
+            )
+
+            # Blended success rate: when proxy_distrust > 0 and judge samples
+            #{self}.effective_rate(
+              name: 'optional - binary or identifier name'
+            )
+
+            # Run changepoints and return its result
+            #{self}.changepoints(
+              cause: 'optional - :env_drift and does NOT count toward [REPEATING].',
+              within_secs: 'optional - within secs value consumed by #changepoints'
+            )
+
+            # W3 — plan_first emits p(success); Loop.run calls this with the
+            #{self}.record_calibration(
+              engine: 'optional - engine value consumed by #record_calibration (defaults to :global))',
+              brier: 'optional - brier value consumed by #record_calibration',
+              predicted: 'optional - predicted value consumed by #record_calibration',
+              actual: 'optional - actual value consumed by #record_calibration'
+            )
+
+            # Run calibration and return its result
+            #{self}.calibration(
+              engine: 'required - engine value consumed by #calibration'
+            )
+
+            # Run calibration green and return its result
+            #{self}.calibration_green?(
+              engine: 'optional - engine value consumed by #calibration_green?'
+            )
+
+            # Run scale prediction and return its result
+            #{self}.scale_prediction(
+              predicted: 'optional - predicted value consumed by #scale_prediction',
+              engine: 'optional - engine value consumed by #scale_prediction'
+            )
+
+            # Run scoreboard and return its result
+            #{self}.scoreboard(
+              engine: 'optional - engine value consumed by #scoreboard'
+            )
+
+            # Run health line and return its result
+            #{self}.health_line
+
+            # Run reset and return its result
+            #{self}.reset
+
+            # Print the AUTHOR(S) string for this module.
+            #{self}.authors
+          "
+          constants.sort
         end
       end
     end

data/lib/pwn/ai/agent/mistakes.rb

@@ -504,7 +504,12 @@ module PWN
           # 2.6 — request-conditioned rank (sim × recency × count), same idea
           # as exemplars_for. Stops injecting loudest scar (reward_signal ×13)
           # on every unrelated turn.
-          open = rank_for_request(rows: open_rows, request: request, limit: limit)
+          include_open = opts[:include_open] == true || opts[:full] == true || request.match?(/mistake|known error|repeat/i)
+          open = if include_open
+                   rank_for_request(rows: open_rows, request: request, limit: limit)
+                 else
+                   []
+                 end
           closed = load.values.select { |m| m[:resolved] && m[:fix] }
           closed = rank_for_request(rows: closed, request: request, limit: limit)
           return '' if open.empty? && closed.empty?
@@ -779,7 +784,9 @@ module PWN
         # Strip volatile substrings so semantically-identical failures
         # collapse to one signature and their :count actually climbs.
         private_class_method def self.normalize_error(opts = {})
-          e = opts[:error].to_s.strip.downcase
+          e = opts[:error].to_s.strip
+          klass = e[/\b([A-Z][A-Za-z0-9_]+(?:Error|Exception|Interrupt))\b/, 1].to_s
+          e = e.downcase
           e = e.gsub(/0x[0-9a-f]{4,}/, '0xADDR')
           e = e.gsub(%r{(/[\w.@+-]+)+/?}, '/PATH')
           e = e.gsub(/:\d+:in\b/, ':LINE:in')
@@ -790,7 +797,8 @@ module PWN
           e = e.gsub(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/, 'UUID')
           e = e.gsub(/\bpid\s*\d+\b/, 'pid N')
           e = e.gsub(/\b\d{4,}\b/, 'N')
-          e.gsub(/\s+/, ' ')[0, 300]
+          body = e.gsub(/\s+/, ' ')[0, 180]
+          klass.empty? ? body : "#{klass.downcase}|#{body}"
         end
 
         # Age-weighted count for [REPEATING] threshold — a ×8 signature from
@@ -835,23 +843,132 @@ module PWN
         # Display Usage for this Module
 
         public_class_method def self.help
-          puts <<~USAGE
-            USAGE:
-              PWN::AI::Agent::Mistakes.record(tool: 'shell', error: 'nmpa: command not found', args: '{"command":"nmpa -sV"}')
-              PWN::AI::Agent::Mistakes.top(limit: 10, unresolved_only: true)
-              PWN::AI::Agent::Mistakes.find(signature: 'abc123def456')
-              PWN::AI::Agent::Mistakes.for_tool(tool: 'shell')
-              PWN::AI::Agent::Mistakes.resolve(signature: 'abc123def456', fix: 'binary is spelled `nmap`, not `nmpa`')
-              PWN::AI::Agent::Mistakes.correction_hint(tool: 'shell', error: err)  # inline self-correct
-              PWN::AI::Agent::Mistakes.to_context(limit: 6)                        # injected by PromptBuilder
-              PWN::AI::Agent::Mistakes.correction?(request: "no that's wrong")
-              PWN::AI::Agent::Mistakes.check_user_correction(request: req, session_id: sid)
-              PWN::AI::Agent::Mistakes.signature(tool: 'shell', error: err)
-              PWN::AI::Agent::Mistakes.operator_inbox(limit: 12)
-              PWN::AI::Agent::Mistakes.reset
-
-              #{self}.authors
-          USAGE
+          puts "USAGE:
+            # Run load and return its result
+            #{self}.load
+
+            # Run save and return its result
+            #{self}.save(
+              store: 'optional - store value consumed by #save'
+            )
+
+            # Run signature and return its result
+            #{self}.signature(
+              tool: 'required - tool/component name that failed',
+              error: 'required - raw error text (will be normalised)'
+            )
+
+            # Run find and return its result
+            #{self}.find(
+              signature: 'optional - exact signature to fetch',
+              tool: 'optional - with error:, compute signature and fetch',
+              error: 'optional - raw error text (used with tool:)'
+            )
+
+            # Run for tool and return its result
+            #{self}.for_tool(
+              tool: 'required - tool name',
+              unresolved_only: 'optional - default false'
+            )
+
+            # Run record and return its result
+            #{self}.record(
+              tool: 'required - tool/component that produced the failure',
+              error: 'required - error text / message',
+              args: 'optional - args that triggered it (stored truncated as sample)',
+              session_id: 'optional - PWN::Sessions id',
+              source: 'optional - :tool | :user_correction | :loop | :model | :heuristic (default :tool)',
+              force: 'optional - force value consumed by #record',
+              meta: 'optional - meta value consumed by #record',
+              cause: 'optional - cause value consumed by #record (defaults to :self))',
+              shape: 'optional - shape value consumed by #record',
+              needs_code_change: 'optional - needs code change value consumed by #record'
+            )
+
+            # Run resolve and return its result
+            #{self}.resolve(
+              signature: 'required - mistake signature (from mistakes_list / .top)',
+              fix: 'required - what to do INSTEAD next time',
+              structured: 'optional - structured value consumed by #resolve',
+              clear_needs_code_change: 'optional - clear needs code change value consumed by #resolve'
+            )
+
+            # Run top and return its result
+            #{self}.top(
+              limit: 'optional - max rows (default 10)',
+              unresolved_only: 'optional - default true',
+              practiceable_only: 'optional - practiceable only value consumed by #top'
+            )
+
+            # Run extinguish and return its result
+            #{self}.extinguish!(
+              signature: 'optional - signature value consumed by #extinguish!',
+              shape: 'optional - shape value consumed by #extinguish!',
+              force: 'optional - force value consumed by #extinguish!'
+            )
+
+            # Auto-resolve parked items that already have a known extinguish recipe
+            #{self}.extinguish_parked!(
+              limit: 'optional - limit value consumed by #extinguish_parked!',
+              dry_run: 'optional - dry run value consumed by #extinguish_parked!'
+            )
+
+            # Run park and return its result
+            #{self}.park(
+              signature: 'required - signature value consumed by #park',
+              reason: 'optional - reason value consumed by #park'
+            )
+
+            # Operator inbox: parked / needs_code_change / needs_human scars that
+            #{self}.operator_inbox(
+              limit: 'optional - limit value consumed by #operator_inbox'
+            )
+
+            # Run to context and return its result
+            #{self}.to_context(
+              limit: 'optional - limit value consumed by #to_context (defaults to 6)',
+              request: 'optional - request value consumed by #to_context',
+              include_open: 'optional - include open value consumed by #to_context',
+              full: 'optional - full value consumed by #to_context'
+            )
+
+            # Run correction hint and return its result
+            #{self}.correction_hint(
+              tool: 'required - tool that just failed',
+              error: 'required - raw error it failed with'
+            )
+
+            # Run correction and return its result
+            #{self}.correction?(
+              request: 'optional - request value consumed by #correction?'
+            )
+
+            # Run check user correction and return its result
+            #{self}.check_user_correction(
+              request: 'required - the incoming user message',
+              session_id: 'optional - session to inspect for the previous answer'
+            )
+
+            # Run lean and return its result
+            #{self}.lean!(
+              dry_run: 'optional - Boolean (default false)',
+              max_resolved_kept: 'optional - cap on resolved-with-fix records',
+              resolved_min_age_days: 'optional - age before resolved count=1 may drop'
+            )
+
+            # Run reset and return its result
+            #{self}.reset
+
+            # Age-weighted count for [REPEATING] threshold — a ×8 signature from
+            #{self}.effective_count(
+              mistake: 'optional - mistake value consumed by #effective_count',
+              signature: 'optional - signature value consumed by #effective_count'
+            )
+
+            # Print the AUTHOR(S) string for this module.
+            #{self}.authors
+          "
+          constants.sort
         end
       end
     end

data/lib/pwn/ai/agent/open_goal.rb

@@ -67,15 +67,31 @@ module PWN
         end
 
         public_class_method def self.help
-          puts <<~USAGE
-            USAGE:
-              PWN::AI::Agent::OpenGoal.begin!(request: '...', session_id: '...')
-              PWN::AI::Agent::OpenGoal.current
-              PWN::AI::Agent::OpenGoal.resume?(request: 'continue')
-              PWN::AI::Agent::OpenGoal.clear!
-
-              #{self}.authors
-          USAGE
+          puts "USAGE:
+            # Run goal file and return its result
+            #{self}.goal_file
+
+            # Run current and return its result
+            #{self}.current
+
+            # Run begin and return its result
+            #{self}.begin!(
+              request: 'required - request value consumed by #begin!',
+              session_id: 'optional - session id value consumed by #begin!'
+            )
+
+            # Run clear and return its result
+            #{self}.clear!
+
+            # Run resume and return its result
+            #{self}.resume?(
+              request: 'optional - request value consumed by #resume?'
+            )
+
+            # Print the AUTHOR(S) string for this module.
+            #{self}.authors
+          "
+          constants.sort
         end
       end
     end