pwn 0.5.706 → 0.5.708

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -1
  3. data/documentation/Reinforcement-Learning.md +2 -2
  4. data/documentation/Reporting.md +1 -0
  5. data/etc/default_skills/pwn/ai/agent/curriculum/SKILL.md +1 -0
  6. data/etc/default_skills/pwn/ai/agent/metrics/SKILL.md +4 -0
  7. data/etc/default_skills/pwn/ai/agent/policy/SKILL.md +1 -1
  8. data/etc/default_skills/pwn/ai/agent/reward/SKILL.md +2 -0
  9. data/etc/default_skills/pwn/ai/agent/tool_guard/SKILL.md +2 -0
  10. data/etc/default_skills/pwn/reports/SKILL.md +4 -2
  11. data/etc/default_skills/pwn/reports/csv/SKILL.md +47 -0
  12. data/etc/default_skills/pwn/reports/html/SKILL.md +47 -0
  13. data/etc/default_skills/pwn/reports/json/SKILL.md +47 -0
  14. data/etc/default_skills/pwn/reports/markdown/SKILL.md +47 -0
  15. data/etc/default_skills/pwn/reports/pdf/SKILL.md +47 -0
  16. data/etc/default_skills/pwn/reports/xml/SKILL.md +47 -0
  17. data/lib/pwn/ai/agent/curriculum.rb +73 -27
  18. data/lib/pwn/ai/agent/dispatch.rb +7 -0
  19. data/lib/pwn/ai/agent/learning.rb +22 -14
  20. data/lib/pwn/ai/agent/loop.rb +112 -45
  21. data/lib/pwn/ai/agent/metrics.rb +68 -1
  22. data/lib/pwn/ai/agent/mistakes.rb +11 -3
  23. data/lib/pwn/ai/agent/policy.rb +62 -33
  24. data/lib/pwn/ai/agent/prompt_builder.rb +17 -5
  25. data/lib/pwn/ai/agent/reward.rb +36 -29
  26. data/lib/pwn/ai/agent/tool_guard.rb +19 -0
  27. data/lib/pwn/ai/agent/turn_finalizer.rb +0 -1
  28. data/lib/pwn/config.rb +7 -6
  29. data/lib/pwn/reports/ai_red_team.rb +1 -1
  30. data/lib/pwn/reports/csv.rb +38 -0
  31. data/lib/pwn/reports/fuzz.rb +1 -1
  32. data/lib/pwn/reports/html.rb +58 -0
  33. data/lib/pwn/reports/json.rb +32 -0
  34. data/lib/pwn/reports/markdown.rb +40 -0
  35. data/lib/pwn/reports/pdf.rb +93 -0
  36. data/lib/pwn/reports/phone.rb +1 -1
  37. data/lib/pwn/reports/sast.rb +1 -1
  38. data/lib/pwn/reports/uri_buster.rb +1 -1
  39. data/lib/pwn/reports/xml.rb +44 -0
  40. data/lib/pwn/reports.rb +54 -6
  41. data/lib/pwn/version.rb +1 -1
  42. data/spec/integration/prompt_builder_spec.rb +1 -1
  43. data/spec/integration/reinforced_feedback_loop_spec.rb +27 -12
  44. data/spec/lib/pwn/ai/agent/injection_guard_spec.rb +65 -0
  45. data/spec/lib/pwn/ai/agent/loop_spec.rb +61 -12
  46. data/spec/lib/pwn/ai/agent/metrics_spec.rb +15 -0
  47. data/spec/lib/pwn/ai/agent/mistakes_spec.rb +5 -2
  48. data/spec/lib/pwn/ai/agent/policy_spec.rb +45 -4
  49. data/spec/lib/pwn/ai/agent/reward_spec.rb +72 -0
  50. data/spec/lib/pwn/ai/agent/scoreboard_roadmap_spec.rb +61 -0
  51. data/spec/lib/pwn/reports/csv_spec.rb +19 -0
  52. data/spec/lib/pwn/reports/formats_spec.rb +90 -0
  53. data/spec/lib/pwn/reports/html_spec.rb +19 -0
  54. data/spec/lib/pwn/reports/json_spec.rb +19 -0
  55. data/spec/lib/pwn/reports/markdown_spec.rb +19 -0
  56. data/spec/lib/pwn/reports/pdf_spec.rb +19 -0
  57. data/spec/lib/pwn/reports/xml_spec.rb +19 -0
  58. data/third_party/pwn_rdoc.jsonl +45 -2
  59. metadata +24 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f9233042c1c9518d8cfdcbfe6499eb521cb0348384554e03ab6641abdda253b9
4
- data.tar.gz: 51b81fd36f292bc54e762d4bb47f5641c9b263addf9f37997aaa4c2f6a00c49f
3
+ metadata.gz: 5d0deee1782b9f4be05a64ad407eb94b10e351f79481935e2bbd2fd79ac3855f
4
+ data.tar.gz: b559b0d6ebb5d6b02a51dcbe158f17f70f693a3ed918b2b3498771e5a261ba39
5
5
  SHA512:
6
- metadata.gz: 370d8b9305f7dcd2a692061b6708feca282546109533131d244cd97d207d1b216afac16baf44847a8a098c9847467208dd30e81084729f79ee7e3c16c050ae32
7
- data.tar.gz: 97725299f78baef12d73614b7258ebf5cc3664316c82d2bf53d6a82e3d9e0e0d2542a3143cbc686fc75b46b6883fb8edc0174cfd38ce74b88a0c908ab4864642
6
+ metadata.gz: 6829d1d1b0306785f50881d971fe95c5b5f284e18a60650850c2d98e6fbf48b8688474218a3bd099a59c5c842db20290356901bcccebee271713e1bd31df675b
7
+ data.tar.gz: 9904b5315b4946052e38e9f00b6bc6ff2e4dde07b9827153e0c982738281c093f7e6ad63609ca75091fffff08e50bacf71b3da09869445384d0be42f59190460
data/Gemfile CHANGED
@@ -49,7 +49,7 @@ gem 'jwt', '3.2.0'
49
49
  gem 'libusb', '0.8.0'
50
50
  gem 'luhn', '3.0.0'
51
51
  gem 'mail', '2.9.1'
52
- gem 'mcp', '1.3.0'
52
+ gem 'mcp', '1.4.0'
53
53
  gem 'meshtastic', '0.0.175'
54
54
  gem 'metasm', '1.0.6'
55
55
  gem 'mongo', '2.25.0'
@@ -49,8 +49,8 @@ This is the live numeric controller. It does not replace planning.
49
49
  |---|---|
50
50
  | State | request kind, task family, plan quality, answer completeness, usable-result, last action, fail bin, and engine |
51
51
  | Action | tool name, or `final` |
52
- | Step reward | `Reward.semantic_ok` hygiene (`+0.05` / `-0.20`) |
53
- | Terminal reward | `Reward.judge` score (skipped when the cheap proxy is untrusted and there is no judge) |
52
+ | Step reward | 0; 0.01 per tool after 8 |
53
+ | Terminal reward | `Reward.judge` × confidence (sole large R). `plan_coverage` is a tag, not the score. |
54
54
  | Updates | Q-learning (`alpha=0.15`, `gamma=0.85`) and REINFORCE (`alpha=0.05`). Stored trajectories replay twice on warmup so a short table is not empty advice. |
55
55
  | Budget | Eight finished episodes (live or warmup-credited) unlock greedy suggestions. Until then the prompt omits them. |
56
56
  | Steer | Q-advantage in `Registry.rank` once the episode budget is met; keyword fit and CORE_TOOLS still come first. Suggested actions follow `Registry.preference_order` (`ai.agent.tool_preference`). |
@@ -9,6 +9,7 @@
9
9
 
10
10
  | Module | Consumes | Emits |
11
11
  |---|---|---|
12
+ | `Reports::PDF` / `HTML` / `Markdown` / `XML` / `CSV` / `JSON` | findings Hash | `.pdf` / `.html` / `.md` / `.xml` / `.csv` / `.json` |
12
13
  | `Reports::SAST` | `PWN::SAST::Factory` output | HTML (with `HTMLHeader`/`HTMLFooter`) + JSON |
13
14
  | `Reports::Fuzz` | `PWN::Plugins::Fuzz` crash log | HTML + JSON |
14
15
  | `Reports::URIBuster` | `pwn_www_uri_buster` output | HTML |
@@ -41,6 +41,7 @@ PWN::AI::Agent::Curriculum.practice(opts)
41
41
  - `red_team_plan`
42
42
  - `hindsight`
43
43
  - `train_and_gate`
44
+ - `reclassify_backlog`
44
45
  - `practice_kpi`
45
46
  - `repeating_trend`
46
47
  - `calibrate`
@@ -52,10 +52,14 @@ PWN::AI::Agent::Metrics.load(opts)
52
52
  - `changepoints`
53
53
  - `record_calibration`
54
54
  - `calibration`
55
+ - `calibration_green`
56
+ - `scale_prediction`
57
+ - `scoreboard`
55
58
  - `health_line`
56
59
  - `reset`
57
60
  - `authors`
58
61
  - `help`
62
+ - `calibration_green?`
59
63
 
60
64
  ## Source
61
65
 
@@ -12,7 +12,7 @@ metadata:
12
12
 
13
13
  # PWN::AI::Agent::Policy
14
14
 
15
- PWN::AI::Agent::Policy is the LIVE tabular RL controller that pwn-ai did not have before R5. Everything else in the harness is retrieval-plus-policy: scores are written to disk and re-injected as prose, or exported later for optional LoRA. This module is the missing MDP: state s — discretized (kind, task, plan, completeness, usable, last, fail) action a — tool name, or "final" reward r — step: semantic_ok hygiene; terminal: Reward.judge next s' — state after the tool result Each Loop turn is one episode. Transitions land in ~/.pwn/policy_traj.jsonl. Q(s,a) and REINFORCE logits H(s,a) are updated from those tuples and persisted in ~/.pwn/policy.json. The learned Q values are an ADVISORY term in Registry.rank. They never replace TaskSummarizer planning, plan_first, or CORE_TOOLS. Disable with PWN::Env[:ai][:agent][:policy] = false.
15
+ PWN::AI::Agent::Policy is the LIVE tabular RL controller that pwn-ai did not have before R5. Everything else in the harness is retrieval-plus-policy: scores are written to disk and re-injected as prose, or exported later for optional LoRA. This module is the missing MDP: state s — discretized (kind, task, plan, completeness, usable, last, fail) action a — tool name, or "final" reward r — step: 0 (spam cost −0.01 after 8 tools); terminal: judge × confidence next s' — state after the tool result Each Loop turn is one episode. Transitions land in ~/.pwn/policy_traj.jsonl. Q(s,a) and REINFORCE logits H(s,a) are updated from those tuples and persisted in ~/.pwn/policy.json. The learned Q values are an ADVISORY term in Registry.rank. They never replace TaskSummarizer planning, plan_first, or CORE_TOOLS. Disable with PWN::Env[:ai][:agent][:policy] = false.
16
16
 
17
17
  ## When to use
18
18
 
@@ -34,6 +34,7 @@ PWN::AI::Agent::Reward.judge(opts)
34
34
  ## Public methods
35
35
 
36
36
  - `judge`
37
+ - `promote_to_success`
37
38
  - `prm`
38
39
  - `plan_coverage`
39
40
  - `sentinel`
@@ -58,6 +59,7 @@ PWN::AI::Agent::Reward.judge(opts)
58
59
  - `judge_sample_weight`
59
60
  - `authors`
60
61
  - `help`
62
+ - `promote_to_success?`
61
63
  - `usable_preference?`
62
64
 
63
65
  ## Source
@@ -52,6 +52,7 @@ PWN::AI::Agent::ToolGuard.present(opts)
52
52
  - `timeout_lesson`
53
53
  - `timeout_result`
54
54
  - `timeout_prior_count`
55
+ - `refuse_copied_persist`
55
56
  - `authors`
56
57
  - `help`
57
58
  - `bashism?`
@@ -60,6 +61,7 @@ PWN::AI::Agent::ToolGuard.present(opts)
60
61
  - `present?`
61
62
  - `protect_core_constants!`
62
63
  - `protect_http!`
64
+ - `refuse_copied_persist?`
63
65
  - `reset_timeout_budget!`
64
66
  - `shell_bash?`
65
67
 
@@ -28,11 +28,13 @@ Class methods take `(opts = {})` and read `opts`.
28
28
 
29
29
  ```ruby
30
30
  PWN::Reports.help
31
- PWN::Reports.help(opts)
31
+ PWN::Reports.resolve_path(opts)
32
32
  ```
33
33
 
34
34
  ## Public methods
35
35
 
36
+ - `resolve_path`
37
+ - `report_payload`
36
38
  - `authors`
37
39
  - `help`
38
40
 
@@ -46,5 +48,5 @@ PWN::Reports.help(opts)
46
48
 
47
49
  ## Verification
48
50
 
49
- `PWN::Reports.respond_to?(:authors)` after the
51
+ `PWN::Reports.respond_to?(:resolve_path)` after the
50
52
  module is loaded. Read the source for parameter names.
@@ -0,0 +1,47 @@
1
+ ---
2
+ name: pwn-reports-csv
3
+ description: Drive PWN::Reports::CSV from pwn_eval.
4
+ license: MIT
5
+ allowed-tools: [pwn, pwn_eval]
6
+ metadata:
7
+ bundled: true
8
+ generated: true
9
+ module: PWN::Reports::CSV
10
+ source: pwn/reports/csv.rb
11
+ ---
12
+
13
+ # PWN::Reports::CSV
14
+
15
+ Generic CSV report writer for pentest / findings payloads.
16
+
17
+ ## When to use
18
+
19
+ Call `PWN::Reports::CSV` from `pwn_eval` when the task needs this module.
20
+ Do not reimplement it in shell.
21
+
22
+ ## Methodologies
23
+
24
+ Generated from `pwn/reports/csv.rb`. Prefer the public class methods below.
25
+ Class methods take `(opts = {})` and read `opts`.
26
+
27
+ ## How to call
28
+
29
+ ```ruby
30
+ PWN::Reports::CSV.help
31
+ PWN::Reports::CSV.generate(opts)
32
+ ```
33
+
34
+ ## Public methods
35
+
36
+ - `generate`
37
+ - `authors`
38
+ - `help`
39
+
40
+ ## Source
41
+
42
+ `pwn/reports/csv.rb`
43
+
44
+ ## Verification
45
+
46
+ `PWN::Reports::CSV.respond_to?(:generate)` after the
47
+ module is loaded. Read the source for parameter names.
@@ -0,0 +1,47 @@
1
+ ---
2
+ name: pwn-reports-html
3
+ description: Drive PWN::Reports::HTML from pwn_eval.
4
+ license: MIT
5
+ allowed-tools: [pwn, pwn_eval]
6
+ metadata:
7
+ bundled: true
8
+ generated: true
9
+ module: PWN::Reports::HTML
10
+ source: pwn/reports/html.rb
11
+ ---
12
+
13
+ # PWN::Reports::HTML
14
+
15
+ Generic HTML report writer for pentest / findings payloads.
16
+
17
+ ## When to use
18
+
19
+ Call `PWN::Reports::HTML` from `pwn_eval` when the task needs this module.
20
+ Do not reimplement it in shell.
21
+
22
+ ## Methodologies
23
+
24
+ Generated from `pwn/reports/html.rb`. Prefer the public class methods below.
25
+ Class methods take `(opts = {})` and read `opts`.
26
+
27
+ ## How to call
28
+
29
+ ```ruby
30
+ PWN::Reports::HTML.help
31
+ PWN::Reports::HTML.generate(opts)
32
+ ```
33
+
34
+ ## Public methods
35
+
36
+ - `generate`
37
+ - `authors`
38
+ - `help`
39
+
40
+ ## Source
41
+
42
+ `pwn/reports/html.rb`
43
+
44
+ ## Verification
45
+
46
+ `PWN::Reports::HTML.respond_to?(:generate)` after the
47
+ module is loaded. Read the source for parameter names.
@@ -0,0 +1,47 @@
1
+ ---
2
+ name: pwn-reports-json
3
+ description: Drive PWN::Reports::JSON from pwn_eval.
4
+ license: MIT
5
+ allowed-tools: [pwn, pwn_eval]
6
+ metadata:
7
+ bundled: true
8
+ generated: true
9
+ module: PWN::Reports::JSON
10
+ source: pwn/reports/json.rb
11
+ ---
12
+
13
+ # PWN::Reports::JSON
14
+
15
+ Generic JSON report writer for pentest / findings payloads.
16
+
17
+ ## When to use
18
+
19
+ Call `PWN::Reports::JSON` from `pwn_eval` when the task needs this module.
20
+ Do not reimplement it in shell.
21
+
22
+ ## Methodologies
23
+
24
+ Generated from `pwn/reports/json.rb`. Prefer the public class methods below.
25
+ Class methods take `(opts = {})` and read `opts`.
26
+
27
+ ## How to call
28
+
29
+ ```ruby
30
+ PWN::Reports::JSON.help
31
+ PWN::Reports::JSON.generate(opts)
32
+ ```
33
+
34
+ ## Public methods
35
+
36
+ - `generate`
37
+ - `authors`
38
+ - `help`
39
+
40
+ ## Source
41
+
42
+ `pwn/reports/json.rb`
43
+
44
+ ## Verification
45
+
46
+ `PWN::Reports::JSON.respond_to?(:generate)` after the
47
+ module is loaded. Read the source for parameter names.
@@ -0,0 +1,47 @@
1
+ ---
2
+ name: pwn-reports-markdown
3
+ description: Drive PWN::Reports::Markdown from pwn_eval.
4
+ license: MIT
5
+ allowed-tools: [pwn, pwn_eval]
6
+ metadata:
7
+ bundled: true
8
+ generated: true
9
+ module: PWN::Reports::Markdown
10
+ source: pwn/reports/markdown.rb
11
+ ---
12
+
13
+ # PWN::Reports::Markdown
14
+
15
+ Generic Markdown report writer for pentest / findings payloads.
16
+
17
+ ## When to use
18
+
19
+ Call `PWN::Reports::Markdown` from `pwn_eval` when the task needs this module.
20
+ Do not reimplement it in shell.
21
+
22
+ ## Methodologies
23
+
24
+ Generated from `pwn/reports/markdown.rb`. Prefer the public class methods below.
25
+ Class methods take `(opts = {})` and read `opts`.
26
+
27
+ ## How to call
28
+
29
+ ```ruby
30
+ PWN::Reports::Markdown.help
31
+ PWN::Reports::Markdown.generate(opts)
32
+ ```
33
+
34
+ ## Public methods
35
+
36
+ - `generate`
37
+ - `authors`
38
+ - `help`
39
+
40
+ ## Source
41
+
42
+ `pwn/reports/markdown.rb`
43
+
44
+ ## Verification
45
+
46
+ `PWN::Reports::Markdown.respond_to?(:generate)` after the
47
+ module is loaded. Read the source for parameter names.
@@ -0,0 +1,47 @@
1
+ ---
2
+ name: pwn-reports-pdf
3
+ description: Drive PWN::Reports::PDF from pwn_eval.
4
+ license: MIT
5
+ allowed-tools: [pwn, pwn_eval]
6
+ metadata:
7
+ bundled: true
8
+ generated: true
9
+ module: PWN::Reports::PDF
10
+ source: pwn/reports/pdf.rb
11
+ ---
12
+
13
+ # PWN::Reports::PDF
14
+
15
+ Generic PDF report writer for pentest / findings payloads. Emits a minimal PDF 1.4 document (no wkhtmltopdf).
16
+
17
+ ## When to use
18
+
19
+ Call `PWN::Reports::PDF` from `pwn_eval` when the task needs this module.
20
+ Do not reimplement it in shell.
21
+
22
+ ## Methodologies
23
+
24
+ Generated from `pwn/reports/pdf.rb`. Prefer the public class methods below.
25
+ Class methods take `(opts = {})` and read `opts`.
26
+
27
+ ## How to call
28
+
29
+ ```ruby
30
+ PWN::Reports::PDF.help
31
+ PWN::Reports::PDF.generate(opts)
32
+ ```
33
+
34
+ ## Public methods
35
+
36
+ - `generate`
37
+ - `authors`
38
+ - `help`
39
+
40
+ ## Source
41
+
42
+ `pwn/reports/pdf.rb`
43
+
44
+ ## Verification
45
+
46
+ `PWN::Reports::PDF.respond_to?(:generate)` after the
47
+ module is loaded. Read the source for parameter names.
@@ -0,0 +1,47 @@
1
+ ---
2
+ name: pwn-reports-xml
3
+ description: Drive PWN::Reports::XML from pwn_eval.
4
+ license: MIT
5
+ allowed-tools: [pwn, pwn_eval]
6
+ metadata:
7
+ bundled: true
8
+ generated: true
9
+ module: PWN::Reports::XML
10
+ source: pwn/reports/xml.rb
11
+ ---
12
+
13
+ # PWN::Reports::XML
14
+
15
+ Generic XML report writer for pentest / findings payloads.
16
+
17
+ ## When to use
18
+
19
+ Call `PWN::Reports::XML` from `pwn_eval` when the task needs this module.
20
+ Do not reimplement it in shell.
21
+
22
+ ## Methodologies
23
+
24
+ Generated from `pwn/reports/xml.rb`. Prefer the public class methods below.
25
+ Class methods take `(opts = {})` and read `opts`.
26
+
27
+ ## How to call
28
+
29
+ ```ruby
30
+ PWN::Reports::XML.help
31
+ PWN::Reports::XML.generate(opts)
32
+ ```
33
+
34
+ ## Public methods
35
+
36
+ - `generate`
37
+ - `authors`
38
+ - `help`
39
+
40
+ ## Source
41
+
42
+ `pwn/reports/xml.rb`
43
+
44
+ ## Verification
45
+
46
+ `PWN::Reports::XML.respond_to?(:generate)` after the
47
+ module is loaded. Read the source for parameter names.
@@ -94,15 +94,7 @@ module PWN
94
94
  []
95
95
  end
96
96
  cool = load_cooldown
97
- # P17 prefer budget-exhaustion fingerprints (agent_loop / critic)
98
- # so nightly self-play attacks the #1 live skill gap first.
99
- candidates = candidates.sort_by do |m|
100
- t = m[:tool].to_s
101
- e = m[:error].to_s.downcase
102
- budget = t == 'agent_loop' || t == 'assistant_answer' ||
103
- e.include?('budget exhausted') || e.include?('iteration budget')
104
- [budget ? 0 : 1, -m[:count].to_i]
105
- end
97
+ candidates = candidates.sort_by { |m| -m[:count].to_i }
106
98
  targets = candidates.reject { |m| practice_skip?(mistake: m, cooldown: cool) }.first(limit)
107
99
  results = []
108
100
 
@@ -112,7 +104,7 @@ module PWN
112
104
 
113
105
  prompts = generate_reproducers(mistake: m, count: [per, 2].max)
114
106
  runs = dry_run ? [] : prompts.map { |p| self_play(prompt: p, tag: "practice:#{m[:signature]}") }
115
- solved = runs.select { |r| r[:score].to_f >= 0.7 }
107
+ solved = runs.select { |r| r[:score].to_f >= 0.6 }
116
108
  mean = runs.empty? ? 0.0 : (runs.sum { |r| r[:score].to_f } / runs.length)
117
109
  resolved = false
118
110
  # 2.4 — auto-resolve only with N≥2 holdout successes + store trace
@@ -129,7 +121,12 @@ module PWN
129
121
  trace_ok = winning.length >= 20 && (
130
122
  !budgetish || winning.match?(/→|shell|pwn_eval|tool/i) || best[:final].to_s.length.between?(1, 800)
131
123
  )
132
- unless trace_ok
124
+ poc_ok = if budgetish || %w[agent_loop assistant_answer].include?(m[:tool].to_s)
125
+ practice_poc_ok?(run: best)
126
+ else
127
+ true
128
+ end
129
+ unless trace_ok && poc_ok
133
130
  bump_cooldown!(cooldown: cool, signature: m[:signature], mean: mean) unless dry_run
134
131
  results << {
135
132
  signature: m[:signature], tool: m[:tool], prompts: prompts,
@@ -327,7 +324,9 @@ module PWN
327
324
  scrub = Reward.scrub_preferences(dry_run: false) if Reward.respond_to?(:scrub_preferences)
328
325
  mix = Reward.generator_mix if Reward.respond_to?(:generator_mix)
329
326
  end
330
- kpi = practice_kpi(results: []) if commit && respond_to?(:practice_kpi)
327
+ rec = { reclassified: 0 }
328
+ rec = reclassify_backlog if commit && respond_to?(:reclassify_backlog)
329
+ kpi = practice_kpi(results: [], reclassified_n: rec[:reclassified]) if commit && respond_to?(:practice_kpi)
331
330
  out = {
332
331
  scored: scored.length, mean: mean, since_hours: since_h,
333
332
  results: scored.first(10), sentinel_warm: warm,
@@ -582,7 +581,14 @@ module PWN
582
581
  # even on a box without GPU.
583
582
 
584
583
  public_class_method def self.train_and_gate(opts = {})
585
- dry_run = opts.key?(:dry_run) ? opts[:dry_run] : true
584
+ dry_run = if opts.key?(:dry_run)
585
+ opts[:dry_run]
586
+ else
587
+ mix = defined?(Reward) && Reward.respond_to?(:generator_mix) ? Reward.generator_mix : {}
588
+ healthy = mix.is_a?(Hash) && mix[:healthy] == true
589
+ cal_ok = defined?(Metrics) && Metrics.respond_to?(:calibration_green?) && Metrics.calibration_green?
590
+ !(healthy && cal_ok)
591
+ end
586
592
  FileUtils.mkdir_p(CURRICULUM_DIR)
587
593
  sft = defined?(Learning) ? Learning.export_finetune(format: :sharegpt) : nil
588
594
  dpo = defined?(Reward) ? Reward.export_dpo : nil
@@ -662,6 +668,40 @@ module PWN
662
668
 
663
669
  KPI_FILE = File.join(Dir.home, '.pwn', 'curriculum_kpi.jsonl')
664
670
 
671
+ public_class_method def self.reclassify_backlog(opts = {})
672
+ rows = defined?(Mistakes) ? Mistakes.top(limit: opts[:limit] || 80, unresolved_only: true) : []
673
+ inbox = if defined?(Mistakes) && Mistakes.respond_to?(:operator_inbox)
674
+ Array(Mistakes.operator_inbox(limit: 80)[:items] || Mistakes.operator_inbox(limit: 80)[:rows])
675
+ else
676
+ []
677
+ end
678
+ counts = { fixable_by_agent: 0, needs_code_change: 0, wontfix_obsolete: 0, reclassified: 0 }
679
+ now = Time.now.utc
680
+ (rows + inbox).uniq { |m| m[:signature] }.each do |m|
681
+ next unless m.is_a?(Hash)
682
+
683
+ err = m[:error].to_s
684
+ age_d = begin
685
+ (now - Time.parse(m[:last_seen].to_s)) / 86_400.0
686
+ rescue StandardError
687
+ 0.0
688
+ end
689
+ obsolete = err.match?(/PATH=|utf-8|invalid byte|generator/i) || (age_d > 7 && m[:count].to_i <= 1)
690
+ if obsolete || (m[:parked] && age_d > 7)
691
+ Mistakes.park(signature: m[:signature], reason: 'wontfix_obsolete') if defined?(Mistakes) && Mistakes.respond_to?(:park)
692
+ counts[:wontfix_obsolete] += 1
693
+ counts[:reclassified] += 1
694
+ elsif m[:needs_code_change]
695
+ counts[:needs_code_change] += 1
696
+ else
697
+ counts[:fixable_by_agent] += 1
698
+ end
699
+ end
700
+ counts
701
+ rescue StandardError => e
702
+ { error: "#{e.class}: #{e.message}" }
703
+ end
704
+
665
705
  public_class_method def self.practice_kpi(opts = {})
666
706
  results = Array(opts[:results])
667
707
  top = defined?(Mistakes) ? Mistakes.top(limit: 50, unresolved_only: true) : []
@@ -680,11 +720,12 @@ module PWN
680
720
  budget_repeating_n: budgetish,
681
721
  practiced: results.length,
682
722
  resolved_tonight: results.count { |r| r[:resolved] },
683
- mean_holdout: if results.empty?
684
- nil
685
- else
686
- (results.sum { |r| r[:mean_score].to_f } / results.length).round(3)
687
- end
723
+ reclassified_n: (opts[:reclassified_n] || 0).to_i,
724
+ mean_holdout: begin
725
+ hold = results.map { |r| r[:mean_score] || r[:score] }.compact
726
+ hold = Learning.outcomes(limit: 20).filter_map { |o| o[:score] } if hold.empty? && defined?(Learning)
727
+ hold.empty? ? 0.0 : (hold.sum(&:to_f) / hold.length).round(3)
728
+ end
688
729
  }
689
730
  begin
690
731
  FileUtils.mkdir_p(File.dirname(KPI_FILE))
@@ -895,15 +936,11 @@ module PWN
895
936
  'Return Dir.pwd from pwn_eval'
896
937
  ]
897
938
  when 'agent_loop', 'assistant_answer'
898
- # P17 — dominant live failure: iteration / critic budget exhaustion.
899
- # Practise finishing under a tight tool budget, not shell shapes.
900
939
  [
901
- 'Answer in one shell call: print kernel release with uname -r',
902
- 'In at most two tools, show cwd and ruby version then stop',
903
- 'Give a final answer with no tools: what is 7 times 8?',
904
- 'Finish under three iterations: list /tmp and report file count',
905
- 'Do not explore — one pwn_eval of Dir.pwd and return the path',
906
- 'Short plan then one command: show free disk with df -h /'
940
+ 'Write a working PoC to /tmp/pwn-practice-poc.txt that proves the issue, then stop',
941
+ 'Save a chained-impact PoC under /tmp/pwn-practice-chain.txt with reproduction steps',
942
+ 'Use pwn_eval to call PWN::Reports::JSON.generate with a findings hash to /tmp/pwn-practice-findings.json',
943
+ 'Produce evidence-backed severity in /tmp/pwn-practice-sev.md including a PoC path'
907
944
  ]
908
945
  else
909
946
  if err.include?('budget exhausted') || err.include?('iteration budget') ||
@@ -1000,7 +1037,7 @@ module PWN
1000
1037
  capped = true
1001
1038
  end
1002
1039
  begin
1003
- final = Loop.run(request: prompt, session_id: sid, enabled_toolsets: %w[terminal pwn memory learning])
1040
+ final = Loop.run(request: prompt, session_id: sid)
1004
1041
  ensure
1005
1042
  if capped && defined?(PWN::Env) && PWN::Env[:ai].is_a?(Hash) && PWN::Env[:ai][:agent].is_a?(Hash) && !PWN::Env[:ai][:agent].frozen?
1006
1043
  if prev_max == :__unset__
@@ -1083,6 +1120,15 @@ module PWN
1083
1120
  nil
1084
1121
  end
1085
1122
 
1123
+ private_class_method def self.practice_poc_ok?(opts = {})
1124
+ run = opts[:run] || {}
1125
+ blob = "#{run[:final]}\n#{run[:trace]}\n#{run[:prompt]}"
1126
+ paths = blob.scan(%r{(/tmp/[A-Za-z0-9._+-]+)})
1127
+ paths.flatten.any? { |path| File.file?(path) && File.size(path).positive? }
1128
+ rescue StandardError
1129
+ false
1130
+ end
1131
+
1086
1132
  private_class_method def self.ask_persona(opts = {})
1087
1133
  return nil unless defined?(Swarm)
1088
1134
 
@@ -46,6 +46,13 @@ module PWN
46
46
  args = parse_args(raw: raw, entry: entry)
47
47
  required = Array(entry.schema&.dig(:parameters, :required))
48
48
  args = ToolGuard.coerce_args(args: args, required: required) if defined?(ToolGuard)
49
+ if defined?(ToolGuard) && ToolGuard.respond_to?(:refuse_copied_persist?) &&
50
+ ToolGuard.refuse_copied_persist?(name: entry.name, args: args)
51
+ return JSON.generate(
52
+ success: false,
53
+ error: 'refused: memory_remember/skills_update text copied from last tool output'
54
+ )
55
+ end
49
56
  result = entry.handler.call(args)
50
57
  JSON.generate(success: true, result: result, effect: effect(name: entry.name, args: args))
51
58
  rescue StandardError => e