pwn 0.5.706 → 0.5.708
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/documentation/Reinforcement-Learning.md +2 -2
- data/documentation/Reporting.md +1 -0
- data/etc/default_skills/pwn/ai/agent/curriculum/SKILL.md +1 -0
- data/etc/default_skills/pwn/ai/agent/metrics/SKILL.md +4 -0
- data/etc/default_skills/pwn/ai/agent/policy/SKILL.md +1 -1
- data/etc/default_skills/pwn/ai/agent/reward/SKILL.md +2 -0
- data/etc/default_skills/pwn/ai/agent/tool_guard/SKILL.md +2 -0
- data/etc/default_skills/pwn/reports/SKILL.md +4 -2
- data/etc/default_skills/pwn/reports/csv/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/html/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/json/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/markdown/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/pdf/SKILL.md +47 -0
- data/etc/default_skills/pwn/reports/xml/SKILL.md +47 -0
- data/lib/pwn/ai/agent/curriculum.rb +73 -27
- data/lib/pwn/ai/agent/dispatch.rb +7 -0
- data/lib/pwn/ai/agent/learning.rb +22 -14
- data/lib/pwn/ai/agent/loop.rb +112 -45
- data/lib/pwn/ai/agent/metrics.rb +68 -1
- data/lib/pwn/ai/agent/mistakes.rb +11 -3
- data/lib/pwn/ai/agent/policy.rb +62 -33
- data/lib/pwn/ai/agent/prompt_builder.rb +17 -5
- data/lib/pwn/ai/agent/reward.rb +36 -29
- data/lib/pwn/ai/agent/tool_guard.rb +19 -0
- data/lib/pwn/ai/agent/turn_finalizer.rb +0 -1
- data/lib/pwn/config.rb +7 -6
- data/lib/pwn/reports/ai_red_team.rb +1 -1
- data/lib/pwn/reports/csv.rb +38 -0
- data/lib/pwn/reports/fuzz.rb +1 -1
- data/lib/pwn/reports/html.rb +58 -0
- data/lib/pwn/reports/json.rb +32 -0
- data/lib/pwn/reports/markdown.rb +40 -0
- data/lib/pwn/reports/pdf.rb +93 -0
- data/lib/pwn/reports/phone.rb +1 -1
- data/lib/pwn/reports/sast.rb +1 -1
- data/lib/pwn/reports/uri_buster.rb +1 -1
- data/lib/pwn/reports/xml.rb +44 -0
- data/lib/pwn/reports.rb +54 -6
- data/lib/pwn/version.rb +1 -1
- data/spec/integration/prompt_builder_spec.rb +1 -1
- data/spec/integration/reinforced_feedback_loop_spec.rb +27 -12
- data/spec/lib/pwn/ai/agent/injection_guard_spec.rb +65 -0
- data/spec/lib/pwn/ai/agent/loop_spec.rb +61 -12
- data/spec/lib/pwn/ai/agent/metrics_spec.rb +15 -0
- data/spec/lib/pwn/ai/agent/mistakes_spec.rb +5 -2
- data/spec/lib/pwn/ai/agent/policy_spec.rb +45 -4
- data/spec/lib/pwn/ai/agent/reward_spec.rb +72 -0
- data/spec/lib/pwn/ai/agent/scoreboard_roadmap_spec.rb +61 -0
- data/spec/lib/pwn/reports/csv_spec.rb +19 -0
- data/spec/lib/pwn/reports/formats_spec.rb +90 -0
- data/spec/lib/pwn/reports/html_spec.rb +19 -0
- data/spec/lib/pwn/reports/json_spec.rb +19 -0
- data/spec/lib/pwn/reports/markdown_spec.rb +19 -0
- data/spec/lib/pwn/reports/pdf_spec.rb +19 -0
- data/spec/lib/pwn/reports/xml_spec.rb +19 -0
- data/third_party/pwn_rdoc.jsonl +45 -2
- metadata +24 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5d0deee1782b9f4be05a64ad407eb94b10e351f79481935e2bbd2fd79ac3855f
|
|
4
|
+
data.tar.gz: b559b0d6ebb5d6b02a51dcbe158f17f70f693a3ed918b2b3498771e5a261ba39
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6829d1d1b0306785f50881d971fe95c5b5f284e18a60650850c2d98e6fbf48b8688474218a3bd099a59c5c842db20290356901bcccebee271713e1bd31df675b
|
|
7
|
+
data.tar.gz: 9904b5315b4946052e38e9f00b6bc6ff2e4dde07b9827153e0c982738281c093f7e6ad63609ca75091fffff08e50bacf71b3da09869445384d0be42f59190460
|
data/Gemfile
CHANGED
|
@@ -49,8 +49,8 @@ This is the live numeric controller. It does not replace planning.
|
|
|
49
49
|
|---|---|
|
|
50
50
|
| State | request kind, task family, plan quality, answer completeness, usable-result, last action, fail bin, and engine |
|
|
51
51
|
| Action | tool name, or `final` |
|
|
52
|
-
| Step reward |
|
|
53
|
-
| Terminal reward | `Reward.judge`
|
|
52
|
+
| Step reward | 0; −0.01 per tool after 8 |
|
|
53
|
+
| Terminal reward | `Reward.judge` × confidence (sole large R). `plan_coverage` is a tag, not the score. |
|
|
54
54
|
| Updates | Q-learning (`alpha=0.15`, `gamma=0.85`) and REINFORCE (`alpha=0.05`). Stored trajectories replay twice on warmup so a short table is not empty advice. |
|
|
55
55
|
| Budget | Eight finished episodes (live or warmup-credited) unlock greedy suggestions. Until then the prompt omits them. |
|
|
56
56
|
| Steer | Q-advantage in `Registry.rank` once the episode budget is met; keyword fit and CORE_TOOLS still come first. Suggested actions follow `Registry.preference_order` (`ai.agent.tool_preference`). |
|
data/documentation/Reporting.md
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
| Module | Consumes | Emits |
|
|
11
11
|
|---|---|---|
|
|
12
|
+
| `Reports::PDF` / `HTML` / `Markdown` / `XML` / `CSV` / `JSON` | findings Hash | `.pdf` / `.html` / `.md` / `.xml` / `.csv` / `.json` |
|
|
12
13
|
| `Reports::SAST` | `PWN::SAST::Factory` output | HTML (with `HTMLHeader`/`HTMLFooter`) + JSON |
|
|
13
14
|
| `Reports::Fuzz` | `PWN::Plugins::Fuzz` crash log | HTML + JSON |
|
|
14
15
|
| `Reports::URIBuster` | `pwn_www_uri_buster` output | HTML |
|
|
@@ -52,10 +52,14 @@ PWN::AI::Agent::Metrics.load(opts)
|
|
|
52
52
|
- `changepoints`
|
|
53
53
|
- `record_calibration`
|
|
54
54
|
- `calibration`
|
|
55
|
+
- `calibration_green`
|
|
56
|
+
- `scale_prediction`
|
|
57
|
+
- `scoreboard`
|
|
55
58
|
- `health_line`
|
|
56
59
|
- `reset`
|
|
57
60
|
- `authors`
|
|
58
61
|
- `help`
|
|
62
|
+
- `calibration_green?`
|
|
59
63
|
|
|
60
64
|
## Source
|
|
61
65
|
|
|
@@ -12,7 +12,7 @@ metadata:
|
|
|
12
12
|
|
|
13
13
|
# PWN::AI::Agent::Policy
|
|
14
14
|
|
|
15
|
-
PWN::AI::Agent::Policy is the LIVE tabular RL controller that pwn-ai did not have before R5. Everything else in the harness is retrieval-plus-policy: scores are written to disk and re-injected as prose, or exported later for optional LoRA. This module is the missing MDP: state s — discretized (kind, task, plan, completeness, usable, last, fail) action a — tool name, or "final" reward r — step:
|
|
15
|
+
PWN::AI::Agent::Policy is the LIVE tabular RL controller that pwn-ai did not have before R5. Everything else in the harness is retrieval-plus-policy: scores are written to disk and re-injected as prose, or exported later for optional LoRA. This module is the missing MDP: state s — discretized (kind, task, plan, completeness, usable, last, fail) action a — tool name, or "final" reward r — step: 0 (spam cost −0.01 after 8 tools); terminal: judge × confidence next s' — state after the tool result Each Loop turn is one episode. Transitions land in ~/.pwn/policy_traj.jsonl. Q(s,a) and REINFORCE logits H(s,a) are updated from those tuples and persisted in ~/.pwn/policy.json. The learned Q values are an ADVISORY term in Registry.rank. They never replace TaskSummarizer planning, plan_first, or CORE_TOOLS. Disable with PWN::Env[:ai][:agent][:policy] = false.
|
|
16
16
|
|
|
17
17
|
## When to use
|
|
18
18
|
|
|
@@ -34,6 +34,7 @@ PWN::AI::Agent::Reward.judge(opts)
|
|
|
34
34
|
## Public methods
|
|
35
35
|
|
|
36
36
|
- `judge`
|
|
37
|
+
- `promote_to_success`
|
|
37
38
|
- `prm`
|
|
38
39
|
- `plan_coverage`
|
|
39
40
|
- `sentinel`
|
|
@@ -58,6 +59,7 @@ PWN::AI::Agent::Reward.judge(opts)
|
|
|
58
59
|
- `judge_sample_weight`
|
|
59
60
|
- `authors`
|
|
60
61
|
- `help`
|
|
62
|
+
- `promote_to_success?`
|
|
61
63
|
- `usable_preference?`
|
|
62
64
|
|
|
63
65
|
## Source
|
|
@@ -52,6 +52,7 @@ PWN::AI::Agent::ToolGuard.present(opts)
|
|
|
52
52
|
- `timeout_lesson`
|
|
53
53
|
- `timeout_result`
|
|
54
54
|
- `timeout_prior_count`
|
|
55
|
+
- `refuse_copied_persist`
|
|
55
56
|
- `authors`
|
|
56
57
|
- `help`
|
|
57
58
|
- `bashism?`
|
|
@@ -60,6 +61,7 @@ PWN::AI::Agent::ToolGuard.present(opts)
|
|
|
60
61
|
- `present?`
|
|
61
62
|
- `protect_core_constants!`
|
|
62
63
|
- `protect_http!`
|
|
64
|
+
- `refuse_copied_persist?`
|
|
63
65
|
- `reset_timeout_budget!`
|
|
64
66
|
- `shell_bash?`
|
|
65
67
|
|
|
@@ -28,11 +28,13 @@ Class methods take `(opts = {})` and read `opts`.
|
|
|
28
28
|
|
|
29
29
|
```ruby
|
|
30
30
|
PWN::Reports.help
|
|
31
|
-
PWN::Reports.
|
|
31
|
+
PWN::Reports.resolve_path(opts)
|
|
32
32
|
```
|
|
33
33
|
|
|
34
34
|
## Public methods
|
|
35
35
|
|
|
36
|
+
- `resolve_path`
|
|
37
|
+
- `report_payload`
|
|
36
38
|
- `authors`
|
|
37
39
|
- `help`
|
|
38
40
|
|
|
@@ -46,5 +48,5 @@ PWN::Reports.help(opts)
|
|
|
46
48
|
|
|
47
49
|
## Verification
|
|
48
50
|
|
|
49
|
-
`PWN::Reports.respond_to?(:
|
|
51
|
+
`PWN::Reports.respond_to?(:resolve_path)` after the
|
|
50
52
|
module is loaded. Read the source for parameter names.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-reports-csv
|
|
3
|
+
description: Drive PWN::Reports::CSV from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::Reports::CSV
|
|
10
|
+
source: pwn/reports/csv.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::Reports::CSV
|
|
14
|
+
|
|
15
|
+
Generic CSV report writer for pentest / findings payloads.
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::Reports::CSV` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/reports/csv.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::Reports::CSV.help
|
|
31
|
+
PWN::Reports::CSV.generate(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- `generate`
|
|
37
|
+
- `authors`
|
|
38
|
+
- `help`
|
|
39
|
+
|
|
40
|
+
## Source
|
|
41
|
+
|
|
42
|
+
`pwn/reports/csv.rb`
|
|
43
|
+
|
|
44
|
+
## Verification
|
|
45
|
+
|
|
46
|
+
`PWN::Reports::CSV.respond_to?(:generate)` after the
|
|
47
|
+
module is loaded. Read the source for parameter names.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-reports-html
|
|
3
|
+
description: Drive PWN::Reports::HTML from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::Reports::HTML
|
|
10
|
+
source: pwn/reports/html.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::Reports::HTML
|
|
14
|
+
|
|
15
|
+
Generic HTML report writer for pentest / findings payloads.
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::Reports::HTML` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/reports/html.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::Reports::HTML.help
|
|
31
|
+
PWN::Reports::HTML.generate(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- `generate`
|
|
37
|
+
- `authors`
|
|
38
|
+
- `help`
|
|
39
|
+
|
|
40
|
+
## Source
|
|
41
|
+
|
|
42
|
+
`pwn/reports/html.rb`
|
|
43
|
+
|
|
44
|
+
## Verification
|
|
45
|
+
|
|
46
|
+
`PWN::Reports::HTML.respond_to?(:generate)` after the
|
|
47
|
+
module is loaded. Read the source for parameter names.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-reports-json
|
|
3
|
+
description: Drive PWN::Reports::JSON from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::Reports::JSON
|
|
10
|
+
source: pwn/reports/json.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::Reports::JSON
|
|
14
|
+
|
|
15
|
+
Generic JSON report writer for pentest / findings payloads.
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::Reports::JSON` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/reports/json.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::Reports::JSON.help
|
|
31
|
+
PWN::Reports::JSON.generate(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- `generate`
|
|
37
|
+
- `authors`
|
|
38
|
+
- `help`
|
|
39
|
+
|
|
40
|
+
## Source
|
|
41
|
+
|
|
42
|
+
`pwn/reports/json.rb`
|
|
43
|
+
|
|
44
|
+
## Verification
|
|
45
|
+
|
|
46
|
+
`PWN::Reports::JSON.respond_to?(:generate)` after the
|
|
47
|
+
module is loaded. Read the source for parameter names.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-reports-markdown
|
|
3
|
+
description: Drive PWN::Reports::Markdown from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::Reports::Markdown
|
|
10
|
+
source: pwn/reports/markdown.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::Reports::Markdown
|
|
14
|
+
|
|
15
|
+
Generic Markdown report writer for pentest / findings payloads.
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::Reports::Markdown` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/reports/markdown.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::Reports::Markdown.help
|
|
31
|
+
PWN::Reports::Markdown.generate(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- `generate`
|
|
37
|
+
- `authors`
|
|
38
|
+
- `help`
|
|
39
|
+
|
|
40
|
+
## Source
|
|
41
|
+
|
|
42
|
+
`pwn/reports/markdown.rb`
|
|
43
|
+
|
|
44
|
+
## Verification
|
|
45
|
+
|
|
46
|
+
`PWN::Reports::Markdown.respond_to?(:generate)` after the
|
|
47
|
+
module is loaded. Read the source for parameter names.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-reports-pdf
|
|
3
|
+
description: Drive PWN::Reports::PDF from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::Reports::PDF
|
|
10
|
+
source: pwn/reports/pdf.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::Reports::PDF
|
|
14
|
+
|
|
15
|
+
Generic PDF report writer for pentest / findings payloads. Emits a minimal PDF 1.4 document (no wkhtmltopdf).
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::Reports::PDF` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/reports/pdf.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::Reports::PDF.help
|
|
31
|
+
PWN::Reports::PDF.generate(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- `generate`
|
|
37
|
+
- `authors`
|
|
38
|
+
- `help`
|
|
39
|
+
|
|
40
|
+
## Source
|
|
41
|
+
|
|
42
|
+
`pwn/reports/pdf.rb`
|
|
43
|
+
|
|
44
|
+
## Verification
|
|
45
|
+
|
|
46
|
+
`PWN::Reports::PDF.respond_to?(:generate)` after the
|
|
47
|
+
module is loaded. Read the source for parameter names.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-reports-xml
|
|
3
|
+
description: Drive PWN::Reports::XML from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::Reports::XML
|
|
10
|
+
source: pwn/reports/xml.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::Reports::XML
|
|
14
|
+
|
|
15
|
+
Generic XML report writer for pentest / findings payloads.
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::Reports::XML` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/reports/xml.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::Reports::XML.help
|
|
31
|
+
PWN::Reports::XML.generate(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- `generate`
|
|
37
|
+
- `authors`
|
|
38
|
+
- `help`
|
|
39
|
+
|
|
40
|
+
## Source
|
|
41
|
+
|
|
42
|
+
`pwn/reports/xml.rb`
|
|
43
|
+
|
|
44
|
+
## Verification
|
|
45
|
+
|
|
46
|
+
`PWN::Reports::XML.respond_to?(:generate)` after the
|
|
47
|
+
module is loaded. Read the source for parameter names.
|
|
@@ -94,15 +94,7 @@ module PWN
|
|
|
94
94
|
[]
|
|
95
95
|
end
|
|
96
96
|
cool = load_cooldown
|
|
97
|
-
|
|
98
|
-
# so nightly self-play attacks the #1 live skill gap first.
|
|
99
|
-
candidates = candidates.sort_by do |m|
|
|
100
|
-
t = m[:tool].to_s
|
|
101
|
-
e = m[:error].to_s.downcase
|
|
102
|
-
budget = t == 'agent_loop' || t == 'assistant_answer' ||
|
|
103
|
-
e.include?('budget exhausted') || e.include?('iteration budget')
|
|
104
|
-
[budget ? 0 : 1, -m[:count].to_i]
|
|
105
|
-
end
|
|
97
|
+
candidates = candidates.sort_by { |m| -m[:count].to_i }
|
|
106
98
|
targets = candidates.reject { |m| practice_skip?(mistake: m, cooldown: cool) }.first(limit)
|
|
107
99
|
results = []
|
|
108
100
|
|
|
@@ -112,7 +104,7 @@ module PWN
|
|
|
112
104
|
|
|
113
105
|
prompts = generate_reproducers(mistake: m, count: [per, 2].max)
|
|
114
106
|
runs = dry_run ? [] : prompts.map { |p| self_play(prompt: p, tag: "practice:#{m[:signature]}") }
|
|
115
|
-
solved = runs.select { |r| r[:score].to_f >= 0.
|
|
107
|
+
solved = runs.select { |r| r[:score].to_f >= 0.6 }
|
|
116
108
|
mean = runs.empty? ? 0.0 : (runs.sum { |r| r[:score].to_f } / runs.length)
|
|
117
109
|
resolved = false
|
|
118
110
|
# 2.4 — auto-resolve only with N≥2 holdout successes + store trace
|
|
@@ -129,7 +121,12 @@ module PWN
|
|
|
129
121
|
trace_ok = winning.length >= 20 && (
|
|
130
122
|
!budgetish || winning.match?(/→|shell|pwn_eval|tool/i) || best[:final].to_s.length.between?(1, 800)
|
|
131
123
|
)
|
|
132
|
-
|
|
124
|
+
poc_ok = if budgetish || %w[agent_loop assistant_answer].include?(m[:tool].to_s)
|
|
125
|
+
practice_poc_ok?(run: best)
|
|
126
|
+
else
|
|
127
|
+
true
|
|
128
|
+
end
|
|
129
|
+
unless trace_ok && poc_ok
|
|
133
130
|
bump_cooldown!(cooldown: cool, signature: m[:signature], mean: mean) unless dry_run
|
|
134
131
|
results << {
|
|
135
132
|
signature: m[:signature], tool: m[:tool], prompts: prompts,
|
|
@@ -327,7 +324,9 @@ module PWN
|
|
|
327
324
|
scrub = Reward.scrub_preferences(dry_run: false) if Reward.respond_to?(:scrub_preferences)
|
|
328
325
|
mix = Reward.generator_mix if Reward.respond_to?(:generator_mix)
|
|
329
326
|
end
|
|
330
|
-
|
|
327
|
+
rec = { reclassified: 0 }
|
|
328
|
+
rec = reclassify_backlog if commit && respond_to?(:reclassify_backlog)
|
|
329
|
+
kpi = practice_kpi(results: [], reclassified_n: rec[:reclassified]) if commit && respond_to?(:practice_kpi)
|
|
331
330
|
out = {
|
|
332
331
|
scored: scored.length, mean: mean, since_hours: since_h,
|
|
333
332
|
results: scored.first(10), sentinel_warm: warm,
|
|
@@ -582,7 +581,14 @@ module PWN
|
|
|
582
581
|
# even on a box without GPU.
|
|
583
582
|
|
|
584
583
|
public_class_method def self.train_and_gate(opts = {})
|
|
585
|
-
dry_run = opts.key?(:dry_run)
|
|
584
|
+
dry_run = if opts.key?(:dry_run)
|
|
585
|
+
opts[:dry_run]
|
|
586
|
+
else
|
|
587
|
+
mix = defined?(Reward) && Reward.respond_to?(:generator_mix) ? Reward.generator_mix : {}
|
|
588
|
+
healthy = mix.is_a?(Hash) && mix[:healthy] == true
|
|
589
|
+
cal_ok = defined?(Metrics) && Metrics.respond_to?(:calibration_green?) && Metrics.calibration_green?
|
|
590
|
+
!(healthy && cal_ok)
|
|
591
|
+
end
|
|
586
592
|
FileUtils.mkdir_p(CURRICULUM_DIR)
|
|
587
593
|
sft = defined?(Learning) ? Learning.export_finetune(format: :sharegpt) : nil
|
|
588
594
|
dpo = defined?(Reward) ? Reward.export_dpo : nil
|
|
@@ -662,6 +668,40 @@ module PWN
|
|
|
662
668
|
|
|
663
669
|
KPI_FILE = File.join(Dir.home, '.pwn', 'curriculum_kpi.jsonl')
|
|
664
670
|
|
|
671
|
+
public_class_method def self.reclassify_backlog(opts = {})
|
|
672
|
+
rows = defined?(Mistakes) ? Mistakes.top(limit: opts[:limit] || 80, unresolved_only: true) : []
|
|
673
|
+
inbox = if defined?(Mistakes) && Mistakes.respond_to?(:operator_inbox)
|
|
674
|
+
Array(Mistakes.operator_inbox(limit: 80)[:items] || Mistakes.operator_inbox(limit: 80)[:rows])
|
|
675
|
+
else
|
|
676
|
+
[]
|
|
677
|
+
end
|
|
678
|
+
counts = { fixable_by_agent: 0, needs_code_change: 0, wontfix_obsolete: 0, reclassified: 0 }
|
|
679
|
+
now = Time.now.utc
|
|
680
|
+
(rows + inbox).uniq { |m| m[:signature] }.each do |m|
|
|
681
|
+
next unless m.is_a?(Hash)
|
|
682
|
+
|
|
683
|
+
err = m[:error].to_s
|
|
684
|
+
age_d = begin
|
|
685
|
+
(now - Time.parse(m[:last_seen].to_s)) / 86_400.0
|
|
686
|
+
rescue StandardError
|
|
687
|
+
0.0
|
|
688
|
+
end
|
|
689
|
+
obsolete = err.match?(/PATH=|utf-8|invalid byte|generator/i) || (age_d > 7 && m[:count].to_i <= 1)
|
|
690
|
+
if obsolete || (m[:parked] && age_d > 7)
|
|
691
|
+
Mistakes.park(signature: m[:signature], reason: 'wontfix_obsolete') if defined?(Mistakes) && Mistakes.respond_to?(:park)
|
|
692
|
+
counts[:wontfix_obsolete] += 1
|
|
693
|
+
counts[:reclassified] += 1
|
|
694
|
+
elsif m[:needs_code_change]
|
|
695
|
+
counts[:needs_code_change] += 1
|
|
696
|
+
else
|
|
697
|
+
counts[:fixable_by_agent] += 1
|
|
698
|
+
end
|
|
699
|
+
end
|
|
700
|
+
counts
|
|
701
|
+
rescue StandardError => e
|
|
702
|
+
{ error: "#{e.class}: #{e.message}" }
|
|
703
|
+
end
|
|
704
|
+
|
|
665
705
|
public_class_method def self.practice_kpi(opts = {})
|
|
666
706
|
results = Array(opts[:results])
|
|
667
707
|
top = defined?(Mistakes) ? Mistakes.top(limit: 50, unresolved_only: true) : []
|
|
@@ -680,11 +720,12 @@ module PWN
|
|
|
680
720
|
budget_repeating_n: budgetish,
|
|
681
721
|
practiced: results.length,
|
|
682
722
|
resolved_tonight: results.count { |r| r[:resolved] },
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
723
|
+
reclassified_n: (opts[:reclassified_n] || 0).to_i,
|
|
724
|
+
mean_holdout: begin
|
|
725
|
+
hold = results.map { |r| r[:mean_score] || r[:score] }.compact
|
|
726
|
+
hold = Learning.outcomes(limit: 20).filter_map { |o| o[:score] } if hold.empty? && defined?(Learning)
|
|
727
|
+
hold.empty? ? 0.0 : (hold.sum(&:to_f) / hold.length).round(3)
|
|
728
|
+
end
|
|
688
729
|
}
|
|
689
730
|
begin
|
|
690
731
|
FileUtils.mkdir_p(File.dirname(KPI_FILE))
|
|
@@ -895,15 +936,11 @@ module PWN
|
|
|
895
936
|
'Return Dir.pwd from pwn_eval'
|
|
896
937
|
]
|
|
897
938
|
when 'agent_loop', 'assistant_answer'
|
|
898
|
-
# P17 — dominant live failure: iteration / critic budget exhaustion.
|
|
899
|
-
# Practise finishing under a tight tool budget, not shell shapes.
|
|
900
939
|
[
|
|
901
|
-
'
|
|
902
|
-
'
|
|
903
|
-
'
|
|
904
|
-
'
|
|
905
|
-
'Do not explore — one pwn_eval of Dir.pwd and return the path',
|
|
906
|
-
'Short plan then one command: show free disk with df -h /'
|
|
940
|
+
'Write a working PoC to /tmp/pwn-practice-poc.txt that proves the issue, then stop',
|
|
941
|
+
'Save a chained-impact PoC under /tmp/pwn-practice-chain.txt with reproduction steps',
|
|
942
|
+
'Use pwn_eval to call PWN::Reports::JSON.generate with a findings hash to /tmp/pwn-practice-findings.json',
|
|
943
|
+
'Produce evidence-backed severity in /tmp/pwn-practice-sev.md including a PoC path'
|
|
907
944
|
]
|
|
908
945
|
else
|
|
909
946
|
if err.include?('budget exhausted') || err.include?('iteration budget') ||
|
|
@@ -1000,7 +1037,7 @@ module PWN
|
|
|
1000
1037
|
capped = true
|
|
1001
1038
|
end
|
|
1002
1039
|
begin
|
|
1003
|
-
final = Loop.run(request: prompt, session_id: sid
|
|
1040
|
+
final = Loop.run(request: prompt, session_id: sid)
|
|
1004
1041
|
ensure
|
|
1005
1042
|
if capped && defined?(PWN::Env) && PWN::Env[:ai].is_a?(Hash) && PWN::Env[:ai][:agent].is_a?(Hash) && !PWN::Env[:ai][:agent].frozen?
|
|
1006
1043
|
if prev_max == :__unset__
|
|
@@ -1083,6 +1120,15 @@ module PWN
|
|
|
1083
1120
|
nil
|
|
1084
1121
|
end
|
|
1085
1122
|
|
|
1123
|
+
private_class_method def self.practice_poc_ok?(opts = {})
|
|
1124
|
+
run = opts[:run] || {}
|
|
1125
|
+
blob = "#{run[:final]}\n#{run[:trace]}\n#{run[:prompt]}"
|
|
1126
|
+
paths = blob.scan(%r{(/tmp/[A-Za-z0-9._+-]+)})
|
|
1127
|
+
paths.flatten.any? { |path| File.file?(path) && File.size(path).positive? }
|
|
1128
|
+
rescue StandardError
|
|
1129
|
+
false
|
|
1130
|
+
end
|
|
1131
|
+
|
|
1086
1132
|
private_class_method def self.ask_persona(opts = {})
|
|
1087
1133
|
return nil unless defined?(Swarm)
|
|
1088
1134
|
|
|
@@ -46,6 +46,13 @@ module PWN
|
|
|
46
46
|
args = parse_args(raw: raw, entry: entry)
|
|
47
47
|
required = Array(entry.schema&.dig(:parameters, :required))
|
|
48
48
|
args = ToolGuard.coerce_args(args: args, required: required) if defined?(ToolGuard)
|
|
49
|
+
if defined?(ToolGuard) && ToolGuard.respond_to?(:refuse_copied_persist?) &&
|
|
50
|
+
ToolGuard.refuse_copied_persist?(name: entry.name, args: args)
|
|
51
|
+
return JSON.generate(
|
|
52
|
+
success: false,
|
|
53
|
+
error: 'refused: memory_remember/skills_update text copied from last tool output'
|
|
54
|
+
)
|
|
55
|
+
end
|
|
49
56
|
result = entry.handler.call(args)
|
|
50
57
|
JSON.generate(success: true, result: result, effect: effect(name: entry.name, args: args))
|
|
51
58
|
rescue StandardError => e
|