pwn 0.5.721 → 0.5.723
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -2
- data/bin/pwn_setup +5 -5
- data/documentation/AI-Integration.md +34 -1
- data/documentation/Policy-Benchmark.md +151 -0
- data/documentation/Reinforcement-Learning.md +27 -1
- data/etc/default_skills/pwn/ai/agent/engagement/SKILL.md +1 -0
- data/etc/default_skills/pwn/ai/agent/learning/SKILL.md +2 -0
- data/etc/default_skills/pwn/ai/agent/metrics/SKILL.md +4 -0
- data/etc/default_skills/pwn/ai/agent/mistakes/SKILL.md +2 -0
- data/etc/default_skills/pwn/ai/agent/policy/SKILL.md +1 -0
- data/etc/default_skills/pwn/ai/agent/reward/SKILL.md +2 -0
- data/etc/default_skills/pwn/ai/agent/swarm/SKILL.md +6 -0
- data/etc/default_skills/pwn/ai/agent/tools/capabilities/SKILL.md +45 -0
- data/etc/default_skills/pwn/ai/agent/tools/context/SKILL.md +45 -0
- data/etc/default_skills/pwn/ai/context/SKILL.md +50 -0
- data/etc/default_skills/pwn/ai/http_retry/SKILL.md +7 -0
- data/etc/default_skills/pwn/ai/http_retry/references/urls.md +4 -0
- data/etc/default_skills/pwn/ai/open_ai/SKILL.md +1 -0
- data/etc/default_skills/pwn/ai/open_ai/references/urls.md +1 -0
- data/etc/default_skills/pwn/plugins/exploit_dev/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/findings/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/gdb/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/gdbmi/SKILL.md +55 -0
- data/etc/default_skills/pwn/plugins/ghidra_headless/SKILL.md +49 -0
- data/etc/default_skills/pwn/plugins/jobs/SKILL.md +6 -0
- data/etc/default_skills/pwn/plugins/packet/SKILL.md +3 -0
- data/etc/default_skills/pwn/plugins/preflight_checker/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/radare2/SKILL.md +1 -0
- data/etc/default_skills/pwn/plugins/transparent_browser/SKILL.md +3 -0
- data/etc/default_skills/pwn/reports/engagement/SKILL.md +3 -2
- data/lib/pwn/ai/agent/curriculum.rb +37 -45
- data/lib/pwn/ai/agent/dispatch.rb +9 -0
- data/lib/pwn/ai/agent/engagement.rb +59 -0
- data/lib/pwn/ai/agent/learning.rb +111 -44
- data/lib/pwn/ai/agent/loop.rb +50 -8
- data/lib/pwn/ai/agent/metrics.rb +70 -2
- data/lib/pwn/ai/agent/mistakes.rb +67 -1
- data/lib/pwn/ai/agent/policy.rb +142 -16
- data/lib/pwn/ai/agent/registry.rb +1 -1
- data/lib/pwn/ai/agent/reward.rb +153 -53
- data/lib/pwn/ai/agent/swarm.rb +235 -35
- data/lib/pwn/ai/agent/tool_guard.rb +13 -1
- data/lib/pwn/ai/agent/tools/artifacts.rb +50 -3
- data/lib/pwn/ai/agent/tools/capabilities.rb +19 -0
- data/lib/pwn/ai/agent/tools/context.rb +38 -0
- data/lib/pwn/ai/agent/tools/finding_record.rb +18 -0
- data/lib/pwn/ai/agent/tools/fuzz_campaign.rb +10 -1
- data/lib/pwn/ai/agent/tools/job_run.rb +32 -0
- data/lib/pwn/ai/agent/tools/learning.rb +5 -6
- data/lib/pwn/ai/agent/tools/metrics.rb +16 -0
- data/lib/pwn/ai/agent/tools/pty_session.rb +4 -4
- data/lib/pwn/ai/agent/tools/ruby_eval.rb +6 -5
- data/lib/pwn/ai/agent/tools/shell.rb +10 -1
- data/lib/pwn/ai/agent/tools/skills.rb +30 -0
- data/lib/pwn/ai/agent/tools/swarm.rb +8 -2
- data/lib/pwn/ai/context.rb +193 -0
- data/lib/pwn/ai/http_retry.rb +53 -7
- data/lib/pwn/ai/open_ai.rb +302 -45
- data/lib/pwn/ai.rb +1 -0
- data/lib/pwn/migrate.rb +10 -1
- data/lib/pwn/plugins/artifact_registry.rb +40 -17
- data/lib/pwn/plugins/binary_parser.rb +4 -1
- data/lib/pwn/plugins/exploit_dev.rb +15 -0
- data/lib/pwn/plugins/findings.rb +48 -8
- data/lib/pwn/plugins/gdb.rb +17 -0
- data/lib/pwn/plugins/gdbmi.rb +128 -0
- data/lib/pwn/plugins/ghidra_headless.rb +104 -0
- data/lib/pwn/plugins/jobs.rb +72 -0
- data/lib/pwn/plugins/packet.rb +51 -0
- data/lib/pwn/plugins/preflight_checker.rb +29 -0
- data/lib/pwn/plugins/process_tube.rb +24 -7
- data/lib/pwn/plugins/radare2.rb +14 -2
- data/lib/pwn/plugins/repl.rb +6 -1
- data/lib/pwn/plugins/transparent_browser.rb +64 -0
- data/lib/pwn/plugins.rb +2 -0
- data/lib/pwn/reports/engagement.rb +19 -0
- data/lib/pwn/sessions.rb +3 -1
- data/lib/pwn/version.rb +1 -1
- data/scripts/benchmark_policy.rb +276 -0
- data/spec/documentation/installation_md_spec.rb +18 -4
- data/spec/integration/reinforced_feedback_loop_spec.rb +20 -13
- data/spec/lib/pwn/ai/agent/curriculum_spec.rb +267 -0
- data/spec/lib/pwn/ai/agent/engagement_spec.rb +12 -0
- data/spec/lib/pwn/ai/agent/learning_spec.rb +81 -3
- data/spec/lib/pwn/ai/agent/loop_spec.rb +51 -0
- data/spec/lib/pwn/ai/agent/metrics_spec.rb +44 -0
- data/spec/lib/pwn/ai/agent/mistakes_spec.rb +10 -0
- data/spec/lib/pwn/ai/agent/policy_spec.rb +103 -0
- data/spec/lib/pwn/ai/agent/registry_spec.rb +62 -0
- data/spec/lib/pwn/ai/agent/reward_spec.rb +175 -12
- data/spec/lib/pwn/ai/agent/swarm_spec.rb +121 -1
- data/spec/lib/pwn/ai/agent/tool_guard_spec.rb +6 -0
- data/spec/lib/pwn/ai/agent/tools/capabilities_spec.rb +14 -0
- data/spec/lib/pwn/ai/agent/tools/context_spec.rb +14 -0
- data/spec/lib/pwn/ai/agent/tools/job_run_spec.rb +2 -0
- data/spec/lib/pwn/ai/agent/tools/learning_spec.rb +25 -0
- data/spec/lib/pwn/ai/context_spec.rb +48 -0
- data/spec/lib/pwn/ai/http_retry_spec.rb +27 -0
- data/spec/lib/pwn/ai/open_ai_oauth_transport_spec.rb +245 -0
- data/spec/lib/pwn/ai/open_ai_spec.rb +150 -0
- data/spec/lib/pwn/migrate_spec.rb +24 -0
- data/spec/lib/pwn/plugins/artifact_registry_spec.rb +19 -0
- data/spec/lib/pwn/plugins/findings_spec.rb +2 -0
- data/spec/lib/pwn/plugins/gdbmi_spec.rb +17 -0
- data/spec/lib/pwn/plugins/ghidra_headless_spec.rb +17 -0
- data/third_party/pwn_rdoc.jsonl +86 -2
- metadata +24 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5ba71758dbd8a3d5231af6489af4c3dd2bd25c0c16df3fd3292b14a77abdf851
|
|
4
|
+
data.tar.gz: 31c99dda2f5bac281c8ba2b4cc7b7b4493f6c3f09cbbe6788a1067d177ec5ecc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5f17d3c8729c22425cfef5b71746c82bc03eaec569b2d470a83e0734a72e8f411bf8542bc302091211f5f6dc578e5fb64b5a72f97fdb2d8d2f4b8f9f7601100d
|
|
7
|
+
data.tar.gz: 6359d1601fa3cc2f340c3edd8efbe71578c0d580c1c7da6dcc212cd2ab56011455cb10578e11d7be9c359c406b2da5d33a6c1eedaa46f73a5a2a0e59d12c8199
|
data/Gemfile
CHANGED
|
@@ -49,7 +49,7 @@ gem 'jwt', '3.2.0'
|
|
|
49
49
|
gem 'libusb', '0.8.0'
|
|
50
50
|
gem 'luhn', '3.0.0'
|
|
51
51
|
gem 'mail', '2.9.1'
|
|
52
|
-
gem 'mcp', '1.
|
|
52
|
+
gem 'mcp', '1.5.0'
|
|
53
53
|
gem 'meshtastic', '0.0.175'
|
|
54
54
|
gem 'metasm', '1.0.6'
|
|
55
55
|
gem 'mongo', '2.25.0'
|
|
@@ -77,7 +77,7 @@ gem 'rbvmomi2', '3.10.0'
|
|
|
77
77
|
gem 'rdoc', '7.0.4'
|
|
78
78
|
gem 'rest-client', '2.1.0'
|
|
79
79
|
gem 'rex', '2.0.13'
|
|
80
|
-
gem 'rmagick', '7.1.
|
|
80
|
+
gem 'rmagick', '7.1.4'
|
|
81
81
|
gem 'rqrcode', '3.2.0'
|
|
82
82
|
gem 'rspec', '3.13.2'
|
|
83
83
|
gem 'rtesseract', '3.1.4'
|
data/bin/pwn_setup
CHANGED
|
@@ -25,8 +25,10 @@ pwn_driver = PWN::Driver::Parser.new do |options|
|
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
options.on('-l', '--list-profiles',
|
|
28
|
-
'<Optional - List capability profiles and exit>') do
|
|
29
|
-
|
|
28
|
+
'<Optional - List capability profiles and exit>') do
|
|
29
|
+
# Public metadata, like --help: no environment decryption or bootstrap.
|
|
30
|
+
PWN::Setup.list_profiles
|
|
31
|
+
exit 0
|
|
30
32
|
end
|
|
31
33
|
|
|
32
34
|
options.on('-m', '--migrate',
|
|
@@ -66,9 +68,7 @@ pwn_driver.parse!
|
|
|
66
68
|
begin
|
|
67
69
|
status = 0
|
|
68
70
|
|
|
69
|
-
if opts[:
|
|
70
|
-
PWN::Setup.list_profiles
|
|
71
|
-
elsif opts[:migrate]
|
|
71
|
+
if opts[:migrate]
|
|
72
72
|
r = PWN::Setup.migrate(
|
|
73
73
|
fix: opts[:fix] || opts[:yes],
|
|
74
74
|
dry_run: opts[:dry_run]
|
|
@@ -10,7 +10,7 @@ agent code never cares which model is behind it.
|
|
|
10
10
|
|
|
11
11
|
| Engine | Client | Auth | Notes |
|
|
12
12
|
|---|---|---|---|
|
|
13
|
-
| `openai` | `PWN::AI::OpenAI` | `key:` |
|
|
13
|
+
| `openai` | `PWN::AI::OpenAI` | ChatGPT/Codex `oauth:` or `key:` | OAuth preferred when configured; subscription chat uses the Codex Responses backend, API keys use the Platform API |
|
|
14
14
|
| `anthropic` | `PWN::AI::Anthropic` | `key:` | tool-use native |
|
|
15
15
|
| `grok` | `PWN::AI::Grok` | `key:` **or** `oauth: true` | OAuth = RFC-8628 device-code flow using xAI's public Grok-CLI client id (no secret) - see skill `xai_grok_oauth_device_flow` |
|
|
16
16
|
| `gemini` | `PWN::AI::Gemini` | `key:` | function-calling native |
|
|
@@ -38,6 +38,39 @@ ai:
|
|
|
38
38
|
PWN::Env[:ai][:active] = :ollama
|
|
39
39
|
```
|
|
40
40
|
|
|
41
|
+
## OpenAI OAuth enrollment and persistence
|
|
42
|
+
|
|
43
|
+
To prefer subscription OAuth, set `ai.openai.oauth.enroll: true` through
|
|
44
|
+
`pwn-vault`, then restart `pwn-ai` and follow the device-code consent prompt.
|
|
45
|
+
An existing bearer or refresh token is preferred over `ai.openai.key`.
|
|
46
|
+
The account must have access to the requested model through Codex; OAuth
|
|
47
|
+
does not grant all Platform API permissions or bypass subscription limits.
|
|
48
|
+
|
|
49
|
+
You can also enroll explicitly from a PWN Ruby console:
|
|
50
|
+
|
|
51
|
+
```ruby
|
|
52
|
+
PWN::AI::OpenAI.obtain_oauth_bearer_token; nil
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The trailing `nil` prevents the console from echoing the returned bearer.
|
|
56
|
+
Successful enrollment updates the live environment and calls the private
|
|
57
|
+
`persist_oauth_to_vault` helper, as token refresh does. It uses the configured
|
|
58
|
+
`driver_opts.pwn_env_path` and `pwn_dec_path`, defaulting to
|
|
59
|
+
`~/.pwn/pwn.yaml` and its `.decryptor` file. Existing decryption artifacts are
|
|
60
|
+
required; missing artifacts leave tokens in the session only and produce a
|
|
61
|
+
notice. Tokens are not printed by the enrollment success message.
|
|
62
|
+
|
|
63
|
+
Persistence retains unrelated settings and the existing key/IV, encrypts a
|
|
64
|
+
private temporary file, and replaces the vault only after encryption succeeds.
|
|
65
|
+
The resulting vault has mode `0600`. The original vault remains unchanged on
|
|
66
|
+
a persistence failure; the helper does not create or rotate decryptor secrets.
|
|
67
|
+
|
|
68
|
+
Subscription OAuth requests must not go to the default Platform
|
|
69
|
+
`/v1/responses` endpoint: that mismatch can produce `Missing scopes:
|
|
70
|
+
api.responses.write`. OAuth chat uses
|
|
71
|
+
`https://chatgpt.com/backend-api/codex/responses` instead. An actual Platform
|
|
72
|
+
API key still needs the relevant project and endpoint permissions.
|
|
73
|
+
|
|
41
74
|
## Engine-aware behavior
|
|
42
75
|
|
|
43
76
|
The harness adapts to the *class* of engine, not the model name:
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# Independent local Policy/Registry benchmark
|
|
2
|
+
|
|
3
|
+
This is a **deterministic controller benchmark, not proof of live LLM improvement**.
|
|
4
|
+
It executes real local file tasks and trains the actual tabular
|
|
5
|
+
`PWN::AI::Agent::Policy` implementation, then selects actions through the actual
|
|
6
|
+
`Registry.rank`. No model responses, provider usage, or benchmark gains are
|
|
7
|
+
simulated. The action handlers are explicitly hand-written local algorithms,
|
|
8
|
+
not a simulated language model.
|
|
9
|
+
|
|
10
|
+
## Run from a source checkout
|
|
11
|
+
|
|
12
|
+
```sh
|
|
13
|
+
ruby scripts/benchmark_policy.rb --self-check
|
|
14
|
+
ruby scripts/benchmark_policy.rb --output /tmp/pwn-policy-benchmark.json
|
|
15
|
+
bundle exec rubocop scripts/benchmark_policy.rb
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
The experiment uses Ruby and its standard libraries; it does not require the
|
|
19
|
+
full application boot sequence. Use a fresh Ruby process, not the live agent
|
|
20
|
+
console. JSON is printed to stdout and optionally written to `--output`.
|
|
21
|
+
`--self-check` runs independent scorer tests and both complete experimental arms;
|
|
22
|
+
it prints a short pass message instead of a report. Use the options separately.
|
|
23
|
+
The benchmark exits nonzero if persistence changes during evaluation, splits
|
|
24
|
+
overlap, a negative control succeeds, or a positive control fails. Self-checks
|
|
25
|
+
also require actual training updates in the on arm and none in the off arm.
|
|
26
|
+
They deliberately do **not** require an improvement of a predetermined size.
|
|
27
|
+
|
|
28
|
+
## Relation to existing evaluation
|
|
29
|
+
|
|
30
|
+
`lib/pwn/ai/agent/curriculum.rb` provides live self-play, judge-based practice,
|
|
31
|
+
mistake-derived evaluation prompts, and adapter training/promotion gates. This
|
|
32
|
+
standalone script does not invoke those paths. It provides a small, independently
|
|
33
|
+
scored experiment where the training labels come from actual task achievement,
|
|
34
|
+
not `Reward.judge`, model prose, or a previously assigned reward. It calls
|
|
35
|
+
`Policy.begin_episode`, `observe_step`, and `finish` with those labels during
|
|
36
|
+
training. This is not a test of Reward's verification-record binding, the live
|
|
37
|
+
Loop, Metrics learning, memory retrieval, or adapter training.
|
|
38
|
+
|
|
39
|
+
The new `scripts/` directory keeps this experimental driver outside the installed
|
|
40
|
+
CLI and production module tree; no module manifest or generated skill is needed.
|
|
41
|
+
|
|
42
|
+
## Protocol
|
|
43
|
+
|
|
44
|
+
1. **Isolation.** Create a private `/tmp/pwn-policy-benchmark-*` directory. Clear
|
|
45
|
+
the process environment, set `HOME` and `TMPDIR` to that directory, then load
|
|
46
|
+
only Policy and Registry. Assert their persistence constants resolve below
|
|
47
|
+
the temporary `HOME/.pwn`. Do not load user configuration, credentials,
|
|
48
|
+
registered application tools, providers, or network libraries. Delete the
|
|
49
|
+
entire temporary directory on normal completion or Ruby exception and restore
|
|
50
|
+
the process environment. A force-killed process may leave its temporary files.
|
|
51
|
+
This is isolation for trusted fixed handlers, not a sandbox for untrusted code.
|
|
52
|
+
2. **Paired arms.** Run `off`, then `on`, resetting Policy between them. Both arms
|
|
53
|
+
start empty and execute the identical training schedule. The only learning
|
|
54
|
+
switch is `PWN::Env[:ai][:agent][:policy]`. No Q entries, visits, episode counts,
|
|
55
|
+
or rewards are seeded directly. Other learning modules are not loaded.
|
|
56
|
+
3. **Training only.** There are four training fixtures: two numeric sorting tasks
|
|
57
|
+
and two active-inventory filtering tasks. Six fixed exploration rounds execute
|
|
58
|
+
every one of the three candidate actions for each fixture: 72 real handler
|
|
59
|
+
calls per arm. Every action gets equal exposure; the scheduler does not use
|
|
60
|
+
answer keys to choose actions. Each call writes into a fresh task directory.
|
|
61
|
+
Exact artifact correctness supplies a binary training label to Policy; the
|
|
62
|
+
off arm executes the same work but Policy declines to update.
|
|
63
|
+
4. **Held-out evaluation.** Only after training, materialize eight distinct
|
|
64
|
+
held-out inputs with literal answer keys: four numeric and four inventory
|
|
65
|
+
tasks. Assert no training input appears in evaluation. Task families and
|
|
66
|
+
request wording are intentionally shared with training. The held-out units
|
|
67
|
+
are **input instances, not unseen families, prompts, or tools**. This is
|
|
68
|
+
in-distribution transfer within a small public fixture set, not a blind test
|
|
69
|
+
or a generalization claim about arbitrary operator requests.
|
|
70
|
+
5. **Frozen controller.** Rank the task family's three entries using
|
|
71
|
+
`Registry.rank(query: ..., entries: ..., preference: [])`. There are no
|
|
72
|
+
evaluation `begin_episode`, `observe_step`, or `finish` calls. The policy uses
|
|
73
|
+
Registry's normal fallback state rather than a fabricated state/table. Allow
|
|
74
|
+
at most three attempts, stopping only when the independent artifact checker
|
|
75
|
+
passes. The controller does not get corrective feedback or adapt between
|
|
76
|
+
attempts. Hash all persisted `.pwn` files before evaluation and after controls;
|
|
77
|
+
abort if they changed.
|
|
78
|
+
6. **Separate controls.** For each held-out input, force a claim-only handler and
|
|
79
|
+
a handler that writes a wrong JSON object. Both return convincing
|
|
80
|
+
`PASS: completed successfully...verified` prose and `ok: true`. All 16 forced
|
|
81
|
+
negative controls must fail objective scoring. Also execute the correct
|
|
82
|
+
algorithm on each input; all eight positive controls must pass. Controls are
|
|
83
|
+
reported separately and never trained on or included in evaluation rates.
|
|
84
|
+
|
|
85
|
+
## Real task behavior and independent scoring
|
|
86
|
+
|
|
87
|
+
Numeric candidates perform lexical sorting, numeric sorting, or no artifact
|
|
88
|
+
write. Inventory candidates filter active rows, include every row, or write
|
|
89
|
+
nothing. Handlers receive only input/output paths; the literal expected values
|
|
90
|
+
are not passed to them. All handlers claim success, intentionally making textual
|
|
91
|
+
claims unreliable. The checker ignores their prose and action names. It accepts
|
|
92
|
+
only a regular non-symlink output whose parsed JSON exactly equals the answer key,
|
|
93
|
+
with the source input unchanged. Missing output, invalid JSON, a plausible wrong
|
|
94
|
+
artifact, or an altered input is failure. Self-checks separately test missing,
|
|
95
|
+
wrong, prose-only, symlink, and correct artifacts.
|
|
96
|
+
|
|
97
|
+
Each family has equal keyword-fit descriptions, with preference disabled. The
|
|
98
|
+
unlearned deterministic tie-break favors lexical sort in one family and the
|
|
99
|
+
correct active filter in the other; it is not configured to lose every task.
|
|
100
|
+
Lexical sorting can genuinely solve some numeric fixtures and receives credit
|
|
101
|
+
when it does. The learned controller can change these tied rankings from
|
|
102
|
+
observed outcomes. This construction deliberately isolates the learning-to-router
|
|
103
|
+
connection; it does not measure natural-language tool-selection quality.
|
|
104
|
+
|
|
105
|
+
The answer keys and algorithms live in the same public script but do not call
|
|
106
|
+
one another. Independence here means scoring actual artifact/task achievement
|
|
107
|
+
without trusting action claims or training rewards, not process-level secrecy or
|
|
108
|
+
an external audit. Extending the task set requires reviewing literal answer keys
|
|
109
|
+
and adding both positive and negative controls before interpreting new results.
|
|
110
|
+
|
|
111
|
+
## Report definitions
|
|
112
|
+
|
|
113
|
+
Each arm contains full training/evaluation traces, fixture inputs and answer keys,
|
|
114
|
+
control traces, Policy statistics, persistence fingerprints, and split/freeze
|
|
115
|
+
checks. Source hashes identify the Policy, Registry, and harness revisions used.
|
|
116
|
+
No fixed gains are embedded in the report.
|
|
117
|
+
|
|
118
|
+
- **Completion:** tasks with at least one objectively successful attempt divided
|
|
119
|
+
by evaluated tasks. A failed attempt does not count as task completion.
|
|
120
|
+
- **False-success count/rate:** attempts claiming `ok: true` without achieving the
|
|
121
|
+
task; rate denominator is executed attempts, not tasks. This is false reporting
|
|
122
|
+
by the handler, not acceptance of that report by the independent checker.
|
|
123
|
+
- **Repeated mistakes:** every failed attempt after the first occurrence of the
|
|
124
|
+
same `(family, action, checker failure)` signature within that phase and arm.
|
|
125
|
+
This includes both repeated attempts on one task and recurrence on later
|
|
126
|
+
held-out inputs. It is not the production Mistakes store's count.
|
|
127
|
+
- **Calls:** `tool_calls` counts actual local handler invocations. Evaluation makes
|
|
128
|
+
one Registry ranking decision per attempted handler call. Training invokes
|
|
129
|
+
begin/observe/finish once per training call; the returned update reports are
|
|
130
|
+
preserved. Each separately listed control row is one additional handler call.
|
|
131
|
+
- **Elapsed:** monotonic measured seconds. Row elapsed time measures the handler;
|
|
132
|
+
phase elapsed time also includes setup, ranking/checking, and training updates
|
|
133
|
+
as applicable. Evaluation phase time excludes separately listed controls;
|
|
134
|
+
top-level elapsed includes both arms and controls but not final JSON output or
|
|
135
|
+
temporary-directory teardown. Tiny timings vary with caching and filesystem
|
|
136
|
+
load; fixed arm order is not a timing-performance study.
|
|
137
|
+
- **Cost:** zero LLM calls and zero provider tokens because no provider is invoked.
|
|
138
|
+
Monetary cost is `null` (not estimated), not a claim that local computing is
|
|
139
|
+
economically free. No token-price or electricity estimates are invented.
|
|
140
|
+
|
|
141
|
+
Training completion is descriptive coverage under forced exploration, not a
|
|
142
|
+
learned-policy score. Only the held-out evaluation rates compare the controllers.
|
|
143
|
+
Repeated executions should reproduce actions, counts, fixture hashes, and update
|
|
144
|
+
counts for the same source revisions. Wall times, temporary paths, timestamps,
|
|
145
|
+
and timestamp-bearing persistence hashes are expected to differ.
|
|
146
|
+
|
|
147
|
+
There is deliberately no external-runner plug-in: accepting arbitrary commands
|
|
148
|
+
would undermine the no-network/no-credentials guarantee. A future live-model
|
|
149
|
+
study should use a separately reviewed runner, identical model/tool budgets,
|
|
150
|
+
external objective verifiers, frozen held-out evaluation, and actual provider
|
|
151
|
+
usage records. Do not present this controller experiment as that study.
|
|
@@ -65,6 +65,32 @@ high-return / high-score episodes.
|
|
|
65
65
|
|
|
66
66
|
## Reward signal (`PWN::AI::Agent::Reward`)
|
|
67
67
|
|
|
68
|
+
`Reward.resolve_outcome` is the shared decision used by live learning and
|
|
69
|
+
offline practice. The ledger retains the raw judge score, source, confidence,
|
|
70
|
+
verdict, and verification evidence. Heuristic guesses, evaluator errors, and
|
|
71
|
+
unresolved high-score/critic disagreements are unverified, with a nil
|
|
72
|
+
`training_score`; they do not become policy updates, failures, or supervised
|
|
73
|
+
success examples. Requeuing a disputed result rejudges its original session
|
|
74
|
+
instead of raising its score automatically.
|
|
75
|
+
|
|
76
|
+
Writing “PASS” in an answer, returning exit zero, or confirming one claim does
|
|
77
|
+
not establish completion. A trusted host verifier can call
|
|
78
|
+
`Reward.record_verification` after checking every original-request criterion,
|
|
79
|
+
providing actual boolean check results and evidence. Records are bound to the
|
|
80
|
+
current session request and invalidated by subsequent tool execution. The API
|
|
81
|
+
trusts the host verifier to cover the complete request; it cannot infer missing
|
|
82
|
+
criteria or automatically verify arbitrary tasks. LLM judgments remain fallible.
|
|
83
|
+
|
|
84
|
+
Policy observations also capture allowlisted operation, argument-role/type
|
|
85
|
+
features, and result classification, without retaining raw argument values.
|
|
86
|
+
These condition next-tool ranking only after enough contextual samples exist,
|
|
87
|
+
with backoff to broad tool scores. Ranking remains advisory.
|
|
88
|
+
|
|
89
|
+
For an independently checked learning-on/off experiment, see
|
|
90
|
+
[Policy-Benchmark.md](Policy-Benchmark.md). It exercises real local handlers and
|
|
91
|
+
controller updates on separate training and held-out inputs; it does not prove
|
|
92
|
+
that a live language model performs better.
|
|
93
|
+
|
|
68
94
|
| Method | What it does |
|
|
69
95
|
|--------|--------------|
|
|
70
96
|
| **R1** `.judge` | Cheap LLM outcome score on `(request, final)` -> `{score:0..1, verdict:, rationale:, key_step:, source:}`. Calls the active engine `.chat` with a short timeout (default 12s). `Reflect.on` is used only when `module_reflection` is on. Fallback scores completeness, plan cover, claims, and tool-trace echo. Token overlap is only a small on-topic gate. |
|
|
@@ -147,7 +173,7 @@ This is the live control list. Track the outcomes, not source comments.
|
|
|
147
173
|
|-------|------|
|
|
148
174
|
| **E1** `Metrics.changepoints` + `Loop.attribute_cause` | Env-drift-attributed failures get `cause: :env_drift` and do not inflate `[REPEATING]`. |
|
|
149
175
|
| **E2** `Extrospection.correlate` | Lead-lag style joins ("tool X started failing after toolchain Y changed"). |
|
|
150
|
-
| **E3** `Reward.verify_as_reward` | Browser-backed claim checks
|
|
176
|
+
| **E3** `Reward.verify_as_reward` | Browser-backed claim checks provide diagnostics and refutation caps; one confirmed claim cannot promote whole-request success. |
|
|
151
177
|
|
|
152
178
|
## Config (`PWN::Env[:ai][:agent]`)
|
|
153
179
|
|
|
@@ -37,6 +37,7 @@ PWN::AI::Agent::Mistakes.load(opts)
|
|
|
37
37
|
- `save`
|
|
38
38
|
- `signature`
|
|
39
39
|
- `error_class`
|
|
40
|
+
- `family`
|
|
40
41
|
- `find`
|
|
41
42
|
- `for_tool`
|
|
42
43
|
- `record`
|
|
@@ -48,6 +49,7 @@ PWN::AI::Agent::Mistakes.load(opts)
|
|
|
48
49
|
- `operator_inbox`
|
|
49
50
|
- `to_context`
|
|
50
51
|
- `correction_hint`
|
|
52
|
+
- `note_hint_outcome`
|
|
51
53
|
- `correction`
|
|
52
54
|
- `check_user_correction`
|
|
53
55
|
- `lean`
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-ai-agent-tools-capabilities
|
|
3
|
+
description: Drive PWN::Ai::Agent::Tools::Capabilities from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::Ai::Agent::Tools::Capabilities
|
|
10
|
+
source: pwn/ai/agent/tools/capabilities.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::Ai::Agent::Tools::Capabilities
|
|
14
|
+
|
|
15
|
+
Public API for PWN::Ai::Agent::Tools::Capabilities.
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::Ai::Agent::Tools::Capabilities` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/ai/agent/tools/capabilities.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::Ai::Agent::Tools::Capabilities.help
|
|
31
|
+
PWN::Ai::Agent::Tools::Capabilities.help(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- _(no public class methods parsed)_
|
|
37
|
+
|
|
38
|
+
## Source
|
|
39
|
+
|
|
40
|
+
`pwn/ai/agent/tools/capabilities.rb`
|
|
41
|
+
|
|
42
|
+
## Verification
|
|
43
|
+
|
|
44
|
+
`PWN::Ai::Agent::Tools::Capabilities.respond_to?(:help)` after the
|
|
45
|
+
module is loaded. Read the source for parameter names.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-ai-agent-tools-context
|
|
3
|
+
description: Drive PWN::Ai::Agent::Tools::Context from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::Ai::Agent::Tools::Context
|
|
10
|
+
source: pwn/ai/agent/tools/context.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::Ai::Agent::Tools::Context
|
|
14
|
+
|
|
15
|
+
Public API for PWN::Ai::Agent::Tools::Context.
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::Ai::Agent::Tools::Context` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/ai/agent/tools/context.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::Ai::Agent::Tools::Context.help
|
|
31
|
+
PWN::Ai::Agent::Tools::Context.help(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- _(no public class methods parsed)_
|
|
37
|
+
|
|
38
|
+
## Source
|
|
39
|
+
|
|
40
|
+
`pwn/ai/agent/tools/context.rb`
|
|
41
|
+
|
|
42
|
+
## Verification
|
|
43
|
+
|
|
44
|
+
`PWN::Ai::Agent::Tools::Context.respond_to?(:help)` after the
|
|
45
|
+
module is loaded. Read the source for parameter names.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-ai-context
|
|
3
|
+
description: Drive PWN::AI::Context from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::AI::Context
|
|
10
|
+
source: pwn/ai/context.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::AI::Context
|
|
14
|
+
|
|
15
|
+
Attach files, hexdumps, disassembly, and HTTP transcripts to model context with auto-chunking. Oversize artifacts are summarized inline and persisted under ~/.pwn/artifacts/<session_id>/ with SHA-256.
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::AI::Context` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/ai/context.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::AI::Context.help
|
|
31
|
+
PWN::AI::Context.attach_file(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- `attach_file`
|
|
37
|
+
- `attach_hexdump`
|
|
38
|
+
- `attach_disasm`
|
|
39
|
+
- `attach_http_transcript`
|
|
40
|
+
- `authors`
|
|
41
|
+
- `help`
|
|
42
|
+
|
|
43
|
+
## Source
|
|
44
|
+
|
|
45
|
+
`pwn/ai/context.rb`
|
|
46
|
+
|
|
47
|
+
## Verification
|
|
48
|
+
|
|
49
|
+
`PWN::AI::Context.respond_to?(:attach_file)` after the
|
|
50
|
+
module is loaded. Read the source for parameter names.
|
|
@@ -37,11 +37,18 @@ PWN::AI::HttpRetry.timeout_s(opts)
|
|
|
37
37
|
- `max_attempts`
|
|
38
38
|
- `retryable`
|
|
39
39
|
- `retry_after_s`
|
|
40
|
+
- `quota_exhausted`
|
|
41
|
+
- `quota_message`
|
|
40
42
|
- `report_event`
|
|
41
43
|
- `authors`
|
|
42
44
|
- `help`
|
|
45
|
+
- `quota_exhausted?`
|
|
43
46
|
- `retryable?`
|
|
44
47
|
|
|
48
|
+
## References
|
|
49
|
+
|
|
50
|
+
- `references/urls.md` — URLs from source
|
|
51
|
+
|
|
45
52
|
## Source
|
|
46
53
|
|
|
47
54
|
`pwn/ai/http_retry.rb`
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pwn-plugins-gdbmi
|
|
3
|
+
description: Drive PWN::Plugins::GDBMI from pwn_eval.
|
|
4
|
+
license: MIT
|
|
5
|
+
allowed-tools: [pwn, pwn_eval]
|
|
6
|
+
metadata:
|
|
7
|
+
bundled: true
|
|
8
|
+
generated: true
|
|
9
|
+
module: PWN::Plugins::GDBMI
|
|
10
|
+
source: pwn/plugins/gdbmi.rb
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# PWN::Plugins::GDBMI
|
|
14
|
+
|
|
15
|
+
GDB machine-interface bridge: breakpoints, stepping, registers, memory, backtraces, checksec. Pairs with ProcessTube for interactive sessions.
|
|
16
|
+
|
|
17
|
+
## When to use
|
|
18
|
+
|
|
19
|
+
Call `PWN::Plugins::GDBMI` from `pwn_eval` when the task needs this module.
|
|
20
|
+
Do not reimplement it in shell.
|
|
21
|
+
|
|
22
|
+
## Methodologies
|
|
23
|
+
|
|
24
|
+
Generated from `pwn/plugins/gdbmi.rb`. Prefer the public class methods below.
|
|
25
|
+
Class methods take `(opts = {})` and read `opts`.
|
|
26
|
+
|
|
27
|
+
## How to call
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
PWN::Plugins::GDBMI.help
|
|
31
|
+
PWN::Plugins::GDBMI.required_bins(opts)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Public methods
|
|
35
|
+
|
|
36
|
+
- `required_bins`
|
|
37
|
+
- `open`
|
|
38
|
+
- `break`
|
|
39
|
+
- `step`
|
|
40
|
+
- `registers`
|
|
41
|
+
- `read_memory`
|
|
42
|
+
- `backtrace`
|
|
43
|
+
- `checksec`
|
|
44
|
+
- `mi`
|
|
45
|
+
- `authors`
|
|
46
|
+
- `help`
|
|
47
|
+
|
|
48
|
+
## Source
|
|
49
|
+
|
|
50
|
+
`pwn/plugins/gdbmi.rb`
|
|
51
|
+
|
|
52
|
+
## Verification
|
|
53
|
+
|
|
54
|
+
`PWN::Plugins::GDBMI.respond_to?(:required_bins)` after the
|
|
55
|
+
module is loaded. Read the source for parameter names.
|