pwn 0.5.647 → 0.5.650
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/documentation/Agent-Tool-Registry.md +10 -5
- data/documentation/Home.md +1 -1
- data/documentation/How-PWN-Works.md +1 -1
- data/documentation/Reinforcement-Learning.md +36 -1
- data/documentation/What-is-PWN.md +1 -1
- data/documentation/pwn-ai-Agent.md +1 -1
- data/lib/pwn/ai/agent/curriculum.rb +125 -5
- data/lib/pwn/ai/agent/extrospection.rb +18 -4
- data/lib/pwn/ai/agent/learning.rb +171 -63
- data/lib/pwn/ai/agent/loop.rb +72 -9
- data/lib/pwn/ai/agent/metrics.rb +69 -10
- data/lib/pwn/ai/agent/registry.rb +21 -2
- data/lib/pwn/ai/agent/reward.rb +171 -5
- data/lib/pwn/ai/agent/tools/curriculum.rb +23 -0
- data/lib/pwn/ai/agent/tools/reward.rb +20 -0
- data/lib/pwn/ai/agent/tools/ruby_eval.rb +29 -6
- data/lib/pwn/version.rb +1 -1
- data/spec/integration/reinforced_feedback_loop_spec.rb +299 -0
- data/spec/lib/pwn/ai/agent/tools/ruby_eval_spec.rb +39 -1
- data/third_party/pwn_rdoc.jsonl +9 -2
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 394ee3adf7ad1cbf7d9259288a928bfd210af98deb50e77d08337e2068b1f3ec
|
|
4
|
+
data.tar.gz: 69668afba8bc9e82043e7497a997494e31c7fbc07b5b77f4e32dfa974cc1c0cf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2979343849a38d63ba659b54d606faca9e0de51e855584910a58e63c9282cdc02456f3315941a8fceb479564fb2410fba0a7fefab9672eb3e10b2134389599ee
|
|
7
|
+
data.tar.gz: 7896ea04844a847a9c17f370ed9536be045a92707f952efd6759791f868a7e87ce9e3e761fa7784d1e156ae3f03847e4ee1bf4762e405479dae6afd01f7d2839
|
|
@@ -6,7 +6,7 @@ toolsets; the JSON-Schema for each tool is what the model actually sees.
|
|
|
6
6
|
|
|
7
7
|

|
|
8
8
|
|
|
9
|
-
## Toolsets → Tools (
|
|
9
|
+
## Toolsets → Tools (12 toolsets · 78 tools)
|
|
10
10
|
|
|
11
11
|
| Toolset | Tools | Backed by |
|
|
12
12
|
|---|---|---|
|
|
@@ -15,7 +15,9 @@ toolsets; the JSON-Schema for each tool is what the model actually sees.
|
|
|
15
15
|
| `memory` | `memory_remember` · `memory_recall` · `memory_forget` · `memory_clear` | `PWN::Memory` → `~/.pwn/memory.json` |
|
|
16
16
|
| `skills` | `skill_list` · `skill_view` · `skill_create` · `skill_add_reference` · `skill_delete` · `skill_migrate_legacy` | `~/.pwn/skills/<name>/SKILL.md` (**[agentskills.io](https://agentskills.io) spec**; legacy flat `*.md` auto-migrated) |
|
|
17
17
|
| `sessions` | `sessions_list` · `sessions_view` · `sessions_current` · `sessions_delete` · `sessions_stats` | `PWN::Sessions` → `~/.pwn/sessions/` |
|
|
18
|
-
| `learning` | `learning_note_outcome` · `learning_reflect` · `learning_distill_skill` · `learning_stats` · `learning_outcomes` · `learning_consolidate` · `learning_reset` · `learning_auto_introspect_toggle` · **`mistakes_list`** · **`mistakes_record`** · **`mistakes_resolve`** · **`mistakes_reset`** · **`reward_judge`** · **`reward_prm`** · **`reward_sentinel`** · **`reward_preferences`** · **`reward_export_dpo`** · **`curriculum_practice`** · **`curriculum_train`** · **`curriculum_hindsight`** · **`learning_purge_noise`** | `PWN::AI::Agent::Learning` + `Mistakes` + `Reward` + `Curriculum` → `~/.pwn/learning.jsonl` + `~/.pwn/mistakes.json` + `~/.pwn/preferences.jsonl` + `~/.pwn/curriculum/` + `~/.pwn/finetune/` |
|
|
18
|
+
| `learning` | `learning_note_outcome` · `learning_reflect` · `learning_distill_skill` · `learning_stats` · `learning_outcomes` · `learning_consolidate` · `learning_reset` · `learning_auto_introspect_toggle` · **`mistakes_list`** · **`mistakes_record`** · **`mistakes_resolve`** · **`mistakes_reset`** · **`reward_judge`** · **`reward_prm`** · **`reward_sentinel`** · **`reward_preferences`** · **`reward_export_dpo`** · **`reward_warm_sentinel`** · **`reward_scrub_preferences`** · **`reward_preference_balance`** · **`curriculum_practice`** · **`curriculum_train`** · **`curriculum_hindsight`** · **`curriculum_offline_judge`** · **`curriculum_preference_balance`** · **`learning_purge_noise`** | `PWN::AI::Agent::Learning` + `Mistakes` + `Reward` + `Curriculum` → `~/.pwn/learning.jsonl` + `~/.pwn/mistakes.json` + `~/.pwn/preferences.jsonl` + `~/.pwn/curriculum/` + `~/.pwn/finetune/` |
|
|
19
|
+
| `reward` | **`reward_generator_mix`** | `PWN::AI::Agent::Reward.generator_mix` → W1 online source-mix controller (`preferences.jsonl`) |
|
|
20
|
+
| `curriculum` | **`curriculum_practice_kpi`** | `PWN::AI::Agent::Curriculum.practice_kpi` → `~/.pwn/curriculum_kpi.jsonl` |
|
|
19
21
|
| `metrics` | `metrics_summary` · `metrics_reset` | `PWN::AI::Agent::Metrics` → `~/.pwn/metrics.json` |
|
|
20
22
|
| `extrospection` | `extro_snapshot` · `extro_drift` · `extro_observe` · `extro_observations` · `extro_intel` · **`extro_watch`** · **`extro_verify`** · **`extro_rf_tune`** · **`extro_osint`** · **`extro_serial`** · **`extro_telecomm`** · **`extro_packet`** · **`extro_vision`** · **`extro_voice`** · `extro_correlate` · `extro_stats` · `extro_reset` · `extro_auto_toggle` | `PWN::AI::Agent::Extrospection` (+ Serial/Packet/OCR/Voice/BareSIP/TransparentBrowser/GQRX) → `~/.pwn/extrospection.json` |
|
|
21
23
|
| `cron` | `cron_list` · `cron_create` · `cron_run` · `cron_enable` · `cron_disable` · `cron_remove` | `PWN::Cron` → `~/.pwn/cron/jobs.yml` |
|
|
@@ -24,7 +26,10 @@ toolsets; the JSON-Schema for each tool is what the model actually sees.
|
|
|
24
26
|
The `learning` toolset is deliberately fat: **Mistakes** (negative feedback),
|
|
25
27
|
**Reward** (ORM/PRM/sentinel/DPO ledger) and **Curriculum** (self-play, HER,
|
|
26
28
|
regression-gated LoRA) are all facets of the same self-improvement loop -
|
|
27
|
-
see [Reinforcement Learning](Reinforcement-Learning.md).
|
|
29
|
+
see [Reinforcement Learning](Reinforcement-Learning.md). The thin `reward` and
|
|
30
|
+
`curriculum` toolsets expose controller/KPI surfaces (`reward_generator_mix`,
|
|
31
|
+
`curriculum_practice_kpi`) so personas can grant just those without the full
|
|
32
|
+
learning surface.
|
|
28
33
|
|
|
29
34
|
## Dynamic tool-set slimming (`ai.agent.tool_router`)
|
|
30
35
|
|
|
@@ -45,8 +50,8 @@ CORE_TOOLS = shell · pwn_eval · memory_remember · memory_recall
|
|
|
45
50
|
```ruby
|
|
46
51
|
PWN::AI::Agent::Registry.definitions(relevance: 'nmap sweep 10.0.0.0/8', top_k: 10)
|
|
47
52
|
PWN::AI::Agent::Registry.rank(query: 'run a shell command') # inspect ranking
|
|
48
|
-
PWN::AI::Agent::Registry.toolsets # → the
|
|
49
|
-
PWN::AI::Agent::Registry.all.count # →
|
|
53
|
+
PWN::AI::Agent::Registry.toolsets # → the 12 names above
|
|
54
|
+
PWN::AI::Agent::Registry.all.count # → 78
|
|
50
55
|
```
|
|
51
56
|
|
|
52
57
|
Frontier engines leave `tool_router` off and receive the full set.
|
data/documentation/Home.md
CHANGED
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
| | |
|
|
37
37
|
|---|---|
|
|
38
38
|
| [AI / LLM Integration](AI-Integration.md) | OpenAI · Anthropic · Grok (OAuth) · Gemini · Ollama |
|
|
39
|
-
| [Agent Tool Registry](Agent-Tool-Registry.md) |
|
|
39
|
+
| [Agent Tool Registry](Agent-Tool-Registry.md) | 12 toolsets · **78** LLM-callable tools |
|
|
40
40
|
| [Memory · Skills · Learning](Skills-Memory-Learning.md) | Introspection - the self-improvement loop |
|
|
41
41
|
| [Mistakes](Mistakes.md) | **Negative feedback** - fingerprint failures · do-NOT-repeat · `[REPEATING]`/`[REGRESSED]` · inline self-correction |
|
|
42
42
|
| [Reinforcement Learning](Reinforcement-Learning.md) | **`Reward` + `Curriculum`** - ORM/PRM · P9/P14 trajectory DPO · P15 scrub · P16 warm · P17 budget skill · P18 PRM→rank · P19 diet gate · export-ready LoRA |
|
|
@@ -28,7 +28,7 @@ hardware).
|
|
|
28
28
|
| Module | Role |
|
|
29
29
|
|---|---|
|
|
30
30
|
| `Loop` | plan → dispatch tool_calls → observe → repeat until final answer |
|
|
31
|
-
| `Registry` | JSON-Schema function definitions grouped into
|
|
31
|
+
| `Registry` | JSON-Schema function definitions grouped into 12 **toolsets** · **78 tools** |
|
|
32
32
|
| `Dispatch` / `Result` | execute a tool, capture stdout/value/error/duration |
|
|
33
33
|
| `PromptBuilder` | inject MEMORY / SKILLS / LEARNING / **KNOWN MISTAKES + FIXES** / METRICS / EXTROSPECTION blocks |
|
|
34
34
|
| `Metrics` · `Learning` · `Reflect` | **introspection** - how well am I doing? |
|
|
@@ -175,11 +175,46 @@ and source diversity, not missing tiers:
|
|
|
175
175
|
3. P15 scrub + export geometry filter purge historical prose flood before DPO/LoRA.
|
|
176
176
|
4. R3/W3 warm via `offline_judge` + `warm_sentinel` so controllers are not permanently cold on ollama hosts (P10/P16).
|
|
177
177
|
5. SFT is filtered as hard as DPO; gate v2 needs smoke + mean, not count alone; P19 also requires trajectory diet.
|
|
178
|
-
6. Budget-exhaustion is a first-class curriculum target (P17/P23); PRM step_reward biases `Registry.rank` (P18); critic is text-only when budget-hot (P24).
|
|
178
|
+
6. Budget-exhaustion is a first-class curriculum target (P17/P23); PRM step_reward biases `Registry.rank` (P18); critic is text-only when budget-hot (P24); last-iter force-final + no-CF-when-hot + exhaust-path Learning close the bare-string hole.
|
|
179
179
|
7. ORM judge blends into Metrics/UCB when distrust is high (P20); W3 calibration lights from plan_first `p(success)=` (P22).
|
|
180
180
|
8. Resolve/record_preference are trajectory-only at write time (P21/P25); write-time source quota still caps traj monoculture.
|
|
181
181
|
9. Without unsloth/axolotl **or** a clean preference diet, `train_and_gate` stays `weight_loop: :export_ready` (honest).
|
|
182
182
|
|
|
183
|
+
## Design-priority STATUS (post P14–P25)
|
|
184
|
+
|
|
185
|
+
Collapse of the review-driven priority order into a living table. Stop minting new P-numbers for chores already covered; track these outcomes instead.
|
|
186
|
+
|
|
187
|
+
| Pri | ID | Control | Module(s) | Success criterion |
|
|
188
|
+
|-----|----|---------|-----------|-------------------|
|
|
189
|
+
| **P0** | W1 generator diversity | `Reward::TARGET_SOURCE_MIX` + `generator_mix` + mix-urgent force on critic/counterfactual | `reward.rb`, `curriculum.rb` | `generator_mix.healthy` OR `recommendation` not stuck on `suppress:mistakes_resolve`; trajectory_fraction ≥ 0.5 |
|
|
190
|
+
| **P0** | Introspect budget | `Learning::INTROSPECT_SOFT_MS` / `HARD_MS`; stage skip under soft/hard / `budget_exhaustion_hot?` | `learning.rb` | `auto_introspect` returns `stages_skipped` when over soft; post-answer path cannot re-thrash tool critic |
|
|
191
|
+
| **P1** | Local judge calibration | Heuristic score shrinkage + `confidence`; `Metrics.effective_rate` scales distrust by `judge_confidence` | `reward.rb`, `metrics.rb` | distrust×heuristic no longer fully replaces proxy; local no-trace highs capped |
|
|
192
|
+
| **P1** | Practice outer KPI | `Curriculum.practice_kpi` / `repeating_trend` → `~/.pwn/curriculum_kpi.jsonl` | `curriculum.rb` | week-over-week `delta_repeating` ≤ 0 on budget fingerprints after practice nights |
|
|
193
|
+
| **P2** | PRM sample efficiency | `PRM_MIN_N=5`, shrinkage to `PRM_FULL_N=20`, fleet coverage gate in `Registry.rank` | `metrics.rb`, `registry.rb` | `prm_advantage=0` until n≥5; rank delta=0 until ≥3 tools ready |
|
|
194
|
+
| **P2** | STATUS over flag archaeology | This table | docs | New work cites Pri/ID here, not fresh P26+ comments for the same theme |
|
|
195
|
+
| **ops** | Nightly diet close | `offline_judge` → `scrub_preferences` + `generator_mix` + `practice_kpi` | `curriculum.rb` | Cron path returns `scrub`/`generator_mix`/`practice_kpi`; raw resolve prose does not survive the night |
|
|
196
|
+
| **ops** | Shape backfill | `Reward.infer_shape` + scrub rewrite | `reward.rb` | Legacy shapeless rows get `winning_trace`/`revised_answer` when content warrants; traj_f measurable |
|
|
197
|
+
| **ops** | Mix in prompt | `Metrics.to_context` emits `W1 MIX:` when unhealthy | `metrics.rb` | Unhealthy diet visible every turn without a tool call |
|
|
198
|
+
| **P0** | Budget exhaust deepen | Last-iter force-final (tools=nil); skip CF when `budget_exhaustion_hot?`; tighter caps 8/12; exhaust path `append_session`+`auto_introspect` | `loop.rb` | Exhaust returns a judged final, not a bare string; CF cannot re-enter under hot; last iter cannot tool-call |
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
### Config additions
|
|
202
|
+
|
|
203
|
+
```yaml
|
|
204
|
+
:ai:
|
|
205
|
+
:agent:
|
|
206
|
+
# P0 introspect budget (ms wall-clock inside auto_introspect)
|
|
207
|
+
# INTROSPECT_SOFT_MS / HARD_MS are constants; override only via code/reload today.
|
|
208
|
+
:critic: null # also force-on when generator_mix.urgent includes critic
|
|
209
|
+
:counterfactual: null # also force-on when generator_mix.urgent includes counterfactual
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
### Tools (design-priority)
|
|
213
|
+
|
|
214
|
+
`reward_generator_mix` · `curriculum_practice_kpi` (plus existing reward/curriculum set)
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
183
218
|
**See also:** [Skills, Memory & Learning](Skills-Memory-Learning.md) ·
|
|
184
219
|
[Mistakes](Mistakes.md) · [Cron](Cron.md) · [pwn-ai Agent](pwn-ai-Agent.md)
|
|
185
220
|
|
|
@@ -22,7 +22,7 @@ tool-calling AI agent** on top of it.
|
|
|
22
22
|
| `PWN::FFI::*` | **8** | Native DSP/RF backends: Volk · Liquid · FFTW · RTLSdr · HackRF · AdalmPluto · SoapySDR · Stdio |
|
|
23
23
|
| `PWN::AI::*` | **5** engines | OpenAI, Anthropic, Grok (OAuth device-flow), Gemini, Ollama |
|
|
24
24
|
| `bin/pwn_*` | **53** | Headless CLI drivers for CI/CD |
|
|
25
|
-
| Agent toolsets | **
|
|
25
|
+
| Agent toolsets | **12** · **78 tools** | terminal · pwn · memory · skills · sessions · learning · metrics · extrospection · cron · swarm |
|
|
26
26
|
|
|
27
27
|
## The three ways to use it
|
|
28
28
|
|
|
@@ -81,7 +81,7 @@ $ pwn --ai "run bin/pwn_sast against ./src and push findings to DefectDojo"
|
|
|
81
81
|
|
|
82
82
|
## What the agent can call
|
|
83
83
|
|
|
84
|
-
|
|
84
|
+
12 toolsets · **78 tools** - full table at
|
|
85
85
|
[Agent Tool Registry](Agent-Tool-Registry.md).
|
|
86
86
|
|
|
87
87
|
The two that matter most:
|
|
@@ -193,12 +193,15 @@ module PWN
|
|
|
193
193
|
end
|
|
194
194
|
save_cooldown(cooldown: cool)
|
|
195
195
|
log(event: :practice, data: results)
|
|
196
|
+
kpi = practice_kpi(results: results)
|
|
196
197
|
{
|
|
197
198
|
practiced: results.length,
|
|
198
199
|
resolved: results.count { |r| r[:resolved] },
|
|
199
200
|
skipped_cooldown: cool.count { |_, v| v[:fail_nights].to_i >= COOLDOWN_FAIL_NIGHTS },
|
|
200
201
|
results: results,
|
|
201
|
-
dry_run: dry_run
|
|
202
|
+
dry_run: dry_run,
|
|
203
|
+
kpi: kpi,
|
|
204
|
+
generator_mix: (defined?(Reward) && Reward.respond_to?(:generator_mix) ? Reward.generator_mix : nil)
|
|
202
205
|
}
|
|
203
206
|
rescue StandardError => e
|
|
204
207
|
{ error: "#{e.class}: #{e.message}" }
|
|
@@ -304,11 +307,22 @@ module PWN
|
|
|
304
307
|
mean = scored.empty? ? nil : (scored.sum { |r| r[:score].to_f } / scored.length).round(3)
|
|
305
308
|
# P10 — keep R3 window warm so proxy_distrust can engage on local hosts
|
|
306
309
|
warm = (Reward.warm_sentinel(limit: 120) if commit && defined?(Reward) && Reward.respond_to?(:warm_sentinel))
|
|
310
|
+
# P0 ops — nightly ledger hygiene so generator_mix success criteria can
|
|
311
|
+
# fire: drop prose flood, backfill missing shapes, then snapshot mix+KPI.
|
|
312
|
+
scrub = nil
|
|
313
|
+
mix = nil
|
|
314
|
+
kpi = nil
|
|
315
|
+
if commit && defined?(Reward)
|
|
316
|
+
scrub = Reward.scrub_preferences(dry_run: false) if Reward.respond_to?(:scrub_preferences)
|
|
317
|
+
mix = Reward.generator_mix if Reward.respond_to?(:generator_mix)
|
|
318
|
+
end
|
|
319
|
+
kpi = practice_kpi(results: []) if commit && respond_to?(:practice_kpi)
|
|
307
320
|
out = {
|
|
308
321
|
scored: scored.length, mean: mean, since_hours: since_h,
|
|
309
|
-
results: scored.first(10), sentinel_warm: warm
|
|
322
|
+
results: scored.first(10), sentinel_warm: warm,
|
|
323
|
+
scrub: scrub, generator_mix: mix, practice_kpi: kpi
|
|
310
324
|
}
|
|
311
|
-
log(event: :offline_judge, data: out)
|
|
325
|
+
log(event: :offline_judge, data: out.except(:results))
|
|
312
326
|
out
|
|
313
327
|
rescue StandardError => e
|
|
314
328
|
{ error: "#{e.class}: #{e.message}" }
|
|
@@ -349,7 +363,16 @@ module PWN
|
|
|
349
363
|
end
|
|
350
364
|
|
|
351
365
|
public_class_method def self.counterfactual(opts = {})
|
|
352
|
-
|
|
366
|
+
# P0 — when generator_mix marks counterfactual underfilled, run even
|
|
367
|
+
# if the auto-flag would keep it off (remote-default still respected
|
|
368
|
+
# only when mix is healthy). Still refuse recursion.
|
|
369
|
+
mix_need = begin
|
|
370
|
+
m = defined?(Reward) && Reward.respond_to?(:generator_mix) ? Reward.generator_mix : {}
|
|
371
|
+
Array(m[:urgent]).include?('counterfactual')
|
|
372
|
+
rescue StandardError
|
|
373
|
+
false
|
|
374
|
+
end
|
|
375
|
+
return nil unless enabled?(key: :counterfactual) || mix_need || opts[:force]
|
|
353
376
|
return nil if in_curriculum?
|
|
354
377
|
|
|
355
378
|
request = opts[:request].to_s
|
|
@@ -416,7 +439,13 @@ module PWN
|
|
|
416
439
|
# self-correction becomes DPO signal.
|
|
417
440
|
|
|
418
441
|
public_class_method def self.critic(opts = {})
|
|
419
|
-
|
|
442
|
+
mix_need = begin
|
|
443
|
+
m = defined?(Reward) && Reward.respond_to?(:generator_mix) ? Reward.generator_mix : {}
|
|
444
|
+
Array(m[:urgent]).include?('critic')
|
|
445
|
+
rescue StandardError
|
|
446
|
+
false
|
|
447
|
+
end
|
|
448
|
+
return { verdict: :pass, source: :disabled } unless enabled?(key: :critic) || opts[:text_only] || mix_need || opts[:force]
|
|
420
449
|
return { verdict: :pass, source: :recursion } if in_curriculum?
|
|
421
450
|
|
|
422
451
|
# P24 — text_only: single Reflect shot, no tool-armed persona swarm.
|
|
@@ -473,6 +502,17 @@ module PWN
|
|
|
473
502
|
return nil unless enabled?(key: :red_team_plan)
|
|
474
503
|
return nil if in_curriculum?
|
|
475
504
|
|
|
505
|
+
# P17 — never nest a red-team persona loop when budget_exhaustion
|
|
506
|
+
# fingerprints dominate open mistakes (amplifier of agent_loop ×N).
|
|
507
|
+
begin
|
|
508
|
+
if defined?(Loop) && Loop.respond_to?(:budget_exhaustion_hot?, true) &&
|
|
509
|
+
Loop.send(:budget_exhaustion_hot?)
|
|
510
|
+
return nil
|
|
511
|
+
end
|
|
512
|
+
rescue StandardError
|
|
513
|
+
# fall through
|
|
514
|
+
end
|
|
515
|
+
|
|
476
516
|
ensure_persona(name: RED_TEAM_NAME, role: 'You are pwn-ai\'s adversarial plan reviewer. Given a numbered tool plan and telemetry from THIS host (tool success rates, known mistakes, environment drift), identify the ONE step most likely to fail and say why in ≤2 lines. Cite the metric/mistake/drift. If the plan is sound reply: SOUND.')
|
|
477
517
|
telemetry = build_telemetry
|
|
478
518
|
reply = with_curriculum_guard do
|
|
@@ -602,6 +642,86 @@ module PWN
|
|
|
602
642
|
# Supported Method Parameters::
|
|
603
643
|
# PWN::AI::Agent::Curriculum.calibrate(predicted:, actual:, engine:)
|
|
604
644
|
|
|
645
|
+
# ----------------------------------------------------------------
|
|
646
|
+
# P1 — Outer curriculum KPI: does practice cut live [REPEATING]?
|
|
647
|
+
# ----------------------------------------------------------------
|
|
648
|
+
# Snapshot unresolved repeating counts before/after practice nights
|
|
649
|
+
# into ~/.pwn/curriculum_kpi.jsonl so week-over-week delta is visible
|
|
650
|
+
# without scraping Mistakes by hand. practice() always appends a row.
|
|
651
|
+
|
|
652
|
+
KPI_FILE = File.join(Dir.home, '.pwn', 'curriculum_kpi.jsonl')
|
|
653
|
+
|
|
654
|
+
public_class_method def self.practice_kpi(opts = {})
|
|
655
|
+
results = Array(opts[:results])
|
|
656
|
+
top = defined?(Mistakes) ? Mistakes.top(limit: 50, unresolved_only: true) : []
|
|
657
|
+
repeating = top.select { |m| m[:count].to_i >= 3 }
|
|
658
|
+
budgetish = repeating.count do |m|
|
|
659
|
+
t = m[:tool].to_s
|
|
660
|
+
e = m[:error].to_s.downcase
|
|
661
|
+
t == 'agent_loop' || t == 'assistant_answer' ||
|
|
662
|
+
e.include?('budget') || e.include?('iteration budget')
|
|
663
|
+
end
|
|
664
|
+
row = {
|
|
665
|
+
at: Time.now.utc.iso8601,
|
|
666
|
+
unresolved_total: top.length,
|
|
667
|
+
repeating_n: repeating.length,
|
|
668
|
+
repeating_sum_count: repeating.sum { |m| m[:count].to_i },
|
|
669
|
+
budget_repeating_n: budgetish,
|
|
670
|
+
practiced: results.length,
|
|
671
|
+
resolved_tonight: results.count { |r| r[:resolved] },
|
|
672
|
+
mean_holdout: if results.empty?
|
|
673
|
+
nil
|
|
674
|
+
else
|
|
675
|
+
(results.sum { |r| r[:mean_score].to_f } / results.length).round(3)
|
|
676
|
+
end
|
|
677
|
+
}
|
|
678
|
+
begin
|
|
679
|
+
FileUtils.mkdir_p(File.dirname(KPI_FILE))
|
|
680
|
+
File.open(KPI_FILE, 'a') { |f| f.puts(JSON.generate(row)) }
|
|
681
|
+
rescue StandardError
|
|
682
|
+
nil
|
|
683
|
+
end
|
|
684
|
+
trend = repeating_trend
|
|
685
|
+
row.merge(trend: trend)
|
|
686
|
+
rescue StandardError => e
|
|
687
|
+
{ error: "#{e.class}: #{e.message}" }
|
|
688
|
+
end
|
|
689
|
+
|
|
690
|
+
# Week-over-week (or last-N snapshots) delta on repeating_n.
|
|
691
|
+
# Positive delta_repeating = getting worse; negative = practice working.
|
|
692
|
+
public_class_method def self.repeating_trend(opts = {})
|
|
693
|
+
limit = (opts[:limit] || 14).to_i
|
|
694
|
+
return { samples: 0, delta_repeating: nil, status: :no_data } unless File.exist?(KPI_FILE)
|
|
695
|
+
|
|
696
|
+
rows = File.readlines(KPI_FILE).last(limit).filter_map do |l|
|
|
697
|
+
JSON.parse(l, symbolize_names: true)
|
|
698
|
+
rescue StandardError
|
|
699
|
+
nil
|
|
700
|
+
end
|
|
701
|
+
return { samples: 0, delta_repeating: nil, status: :no_data } if rows.empty?
|
|
702
|
+
return { samples: rows.length, delta_repeating: 0, status: :baseline, latest: rows.last } if rows.length < 2
|
|
703
|
+
|
|
704
|
+
first = rows.first
|
|
705
|
+
last = rows.last
|
|
706
|
+
d_rep = last[:repeating_n].to_i - first[:repeating_n].to_i
|
|
707
|
+
d_budget = last[:budget_repeating_n].to_i - first[:budget_repeating_n].to_i
|
|
708
|
+
status = if d_rep <= -2 then :improving
|
|
709
|
+
elsif d_rep >= 2 then :regressing
|
|
710
|
+
else :flat
|
|
711
|
+
end
|
|
712
|
+
{
|
|
713
|
+
samples: rows.length,
|
|
714
|
+
from: first[:at],
|
|
715
|
+
to: last[:at],
|
|
716
|
+
delta_repeating: d_rep,
|
|
717
|
+
delta_budget_repeating: d_budget,
|
|
718
|
+
latest_repeating_n: last[:repeating_n],
|
|
719
|
+
status: status
|
|
720
|
+
}
|
|
721
|
+
rescue StandardError => e
|
|
722
|
+
{ samples: 0, status: :error, error: "#{e.class}: #{e.message}" }
|
|
723
|
+
end
|
|
724
|
+
|
|
605
725
|
public_class_method def self.calibrate(opts = {})
|
|
606
726
|
p = opts[:predicted].to_f.clamp(0.0, 1.0)
|
|
607
727
|
a = opts[:actual].to_f.clamp(0.0, 1.0)
|
|
@@ -1722,14 +1722,24 @@ module PWN
|
|
|
1722
1722
|
c = opts[:claim].to_s
|
|
1723
1723
|
return :doc if opts[:url] || c =~ URI::DEFAULT_PARSER.make_regexp(%w[http https])
|
|
1724
1724
|
return :cve if c =~ /CVE-\d{4}-\d{4,}/i
|
|
1725
|
-
|
|
1725
|
+
# P26 — full semver (x.y.z) OR "latest X is [v]N.N" product claims.
|
|
1726
|
+
# Bare two-part floats alone stay :generic (metrics like "cap 0.2").
|
|
1727
|
+
return :version if c =~ /\b[A-Za-z][\w.+-]{2,}\s+v?\d+\.\d+\.\d+\b/
|
|
1728
|
+
return :version if c =~ /\blatest\b.+\bis\b\s+v?\d+\.\d+/i
|
|
1726
1729
|
|
|
1727
1730
|
:generic
|
|
1728
1731
|
end
|
|
1729
1732
|
|
|
1730
1733
|
private_class_method def self.commit_verdict(opts = {})
|
|
1731
|
-
claim = opts[:claim]
|
|
1734
|
+
claim = opts[:claim].to_s
|
|
1732
1735
|
ev = Array(opts[:evidence]).first
|
|
1736
|
+
# P26 — never pollute learning.jsonl with :unknown on metric crumbs
|
|
1737
|
+
# or sub-8-char blobs; those are not human-reviewable world claims.
|
|
1738
|
+
junk_unknown = opts[:verdict].to_s == 'unknown' && (
|
|
1739
|
+
claim.length < 12 ||
|
|
1740
|
+
claim.match?(/\A(?:cap|share|proxy|judge|success|only|now|clears|gap|score|rate|mean|brier|overconf|distrust|trajectory_fraction|handler|orm|prm|delta|limit|window|pct|percent|ms|iter|budget|conf|confidence|ruby|python|linux|kernel|host|cwd|e\.g)\b/i) ||
|
|
1741
|
+
(claim.match?(/\d+\.\d+/) && !claim.match?(/CVE-|\d+\.\d+\.\d+/i))
|
|
1742
|
+
)
|
|
1733
1743
|
case opts[:verdict]
|
|
1734
1744
|
when :refuted
|
|
1735
1745
|
Mistakes.record(tool: 'assumption', error: "REFUTED (extro_verify #{opts[:kind]}, conf=#{opts[:confidence].round(2)}): #{claim}", args: ev&.dig(:final_url), source: :model) if defined?(Mistakes)
|
|
@@ -1739,8 +1749,12 @@ module PWN
|
|
|
1739
1749
|
observe(source: 'extro_verify', category: :intel, target: ev&.dig(:final_url), data: claim, tags: %w[verify confirmed], ttl: 30 * 24 * 3600)
|
|
1740
1750
|
:extro_observe
|
|
1741
1751
|
else
|
|
1742
|
-
|
|
1743
|
-
|
|
1752
|
+
if junk_unknown
|
|
1753
|
+
:skipped_junk
|
|
1754
|
+
else
|
|
1755
|
+
Learning.note_outcome(task: "extro_verify: #{claim[0, 120]}", success: false, details: 'verdict :unknown - needs human review', tags: %w[needs_human extro_verify]) if defined?(Learning) && Learning.respond_to?(:note_outcome)
|
|
1756
|
+
:learning_note
|
|
1757
|
+
end
|
|
1744
1758
|
end
|
|
1745
1759
|
rescue StandardError
|
|
1746
1760
|
nil
|