pwn 0.5.670 → 0.5.673

data/documentation/diagrams/dot/agent-tool-registry.dot

@@ -1,6 +1,6 @@
 digraph "PWN_Agent_Tool_Registry" {
   graph [
-    label=<<B>PWN::AI::Agent::Registry - Toolsets exposed to the LLM</B><BR/><FONT POINT-SIZE="11" COLOR="#94a3b8">12 toolsets · 82 callable tools · lib/pwn/ai/agent/tools/*</FONT>>,
+    label=<<B>PWN::AI::Agent::Registry - Toolsets exposed to the LLM</B><BR/><FONT POINT-SIZE="11" COLOR="#94a3b8">13 toolsets · 85 callable tools · lib/pwn/ai/agent/tools/*</FONT>>,
     labelloc=t, fontsize=20, fontname="Helvetica",
     rankdir=LR, splines=spline, nodesep=0.35, ranksep=1.4,
     bgcolor="#0f172a", fontcolor="#e2e8f0", pad=0.6, newrank=true
@@ -26,14 +26,15 @@ digraph "PWN_Agent_Tool_Registry" {
     swarm    [label="swarm",    fillcolor="#6ee7b7"];
     reward   [label="reward",   fillcolor="#6ee7b7"];
     curric   [label="curriculum", fillcolor="#6ee7b7"];
+    policy   [label="policy",   fillcolor="#6ee7b7"];
   }
 
   subgraph cluster_tools {
     label="Tools  (LLM-callable)"; fontcolor="#fde68a"; style=rounded;
     color="#a16207"; bgcolor="#422006"; penwidth=2;
     node [fillcolor="#fcd34d", fontsize=9];
-    t_shell   [label="shell"];
-    t_eval    [label="pwn_eval"];
+    t_shell   [label="shell\n(ToolGuard)"];
+    t_eval    [label="pwn_eval\n(ToolGuard)"];
     t_mem     [label="memory_remember\nrecall · forget · clear\nlean"];
     t_skill   [label="skill_list · view · create\nadd_reference · delete\nmigrate_legacy"];
     t_sess    [label="sessions_list · view\ncurrent · delete · stats\nlean"];
@@ -44,12 +45,13 @@ digraph "PWN_Agent_Tool_Registry" {
     t_swarm   [label="agent_list · spawn · ask\ndebate · broadcast\nswarm_bus · swarm_list"];
     t_reward  [label="reward_generator_mix"];
     t_curric  [label="curriculum_practice_kpi"];
+    t_policy  [label="policy_stats\npolicy_evaluate\npolicy_recommend"];
   }
 
   Registry -> terminal; Registry -> pwn; Registry -> memory;
   Registry -> skills; Registry -> sessions; Registry -> learning;
   Registry -> metrics; Registry -> extro; Registry -> cron; Registry -> swarm;
-  Registry -> reward; Registry -> curric;
+  Registry -> reward; Registry -> curric; Registry -> policy;
 
   terminal -> t_shell  [color="#f59e0b"];
   pwn      -> t_eval   [color="#f59e0b"];
@@ -62,6 +64,7 @@ digraph "PWN_Agent_Tool_Registry" {
   cron     -> t_cron   [color="#f59e0b"];
   swarm    -> t_swarm  [color="#f59e0b"];
   reward   -> t_reward [color="#f59e0b"];
+  policy   -> t_policy [color="#f59e0b"];
   curric   -> t_curric [color="#f59e0b"];
 
   t_metric -> Registry [label="success_rate\n→ router tie-break", style=dashed, color="#fbbf24", constraint=false];

data/documentation/diagrams/dot/memory-skills-detailed.dot

@@ -1,6 +1,6 @@
 digraph "PWN_MemorySkills" {
   graph [
-    label=<<B>Persistent Knowledge - Memory · Skills · Learning · Mistakes · Sessions</B><BR/><FONT POINT-SIZE="11" COLOR="#94a3b8">everything under ~/.pwn/ that shapes future prompts</FONT>>,
+    label=<<B>Persistent Knowledge - Memory · Skills · Learning · Mistakes · Policy · Sessions</B><BR/><FONT POINT-SIZE="11" COLOR="#94a3b8">everything under ~/.pwn/ that shapes future prompts</FONT>>,
     labelloc=t, fontsize=20, fontname="Helvetica",
     rankdir=TB, splines=spline, nodesep=0.55, ranksep=0.95,
     bgcolor="#0f172a", fontcolor="#e2e8f0", pad=0.6, newrank=true
@@ -20,8 +20,9 @@ digraph "PWN_MemorySkills" {
     Mrec   [label="mistakes_record\nmistakes_resolve",           fillcolor="#6ee7b7"];
     Xft    [label="Learning.export_finetune\n(sessions → dataset)", fillcolor="#6ee7b7"];
     Rdpo   [label="Reward.record_preference\nexport_dpo", fillcolor="#6ee7b7"];
+    PolW   [label="Policy.begin / observe / finish\n(Q + REINFORCE)", fillcolor="#6ee7b7"];
   }
-  {rank=same; Rem; Skc; Note; Dist; Mrec; Xft; Rdpo}
+  {rank=same; Rem; Skc; Note; Dist; Mrec; Xft; Rdpo; PolW}
 
   subgraph cluster_files {
     label="~/.pwn/"; fontcolor="#fde68a"; style=rounded;
@@ -35,8 +36,9 @@ digraph "PWN_MemorySkills" {
     Fext [label="extrospection.json\nsnapshot · rf · web · obs", shape=cylinder, fillcolor="#fcd34d"];
     Fftn [label="finetune/*.jsonl\nShareGPT / OpenAI / DPO", shape=cylinder, fillcolor="#fcd34d"];
     Fprf [label="preferences.jsonl\nDPO (prompt,rejected,chosen)", shape=cylinder, fillcolor="#fcd34d"];
+    Fpol [label="policy.json + policy_traj.jsonl\nQ table · REINFORCE · MDP log", shape=cylinder, fillcolor="#fcd34d"];
   }
-  {rank=same; Fmem; Fidx; Fskl; Flrn; Fses; Fmis; Fext; Fftn; Fprf}
+  {rank=same; Fmem; Fidx; Fskl; Flrn; Fses; Fmis; Fext; Fftn; Fprf; Fpol}
 
   subgraph cluster_read {
     label="Read-Side / Injection"; fontcolor="#ddd6fe"; style=rounded;
@@ -48,10 +50,11 @@ digraph "PWN_MemorySkills" {
     Sesv [label="sessions_view\nsessions_current",   fillcolor="#c4b5fd"];
     Mlst [label="mistakes_list\ncorrection_hint",    fillcolor="#c4b5fd"];
     Exem [label="Learning.exemplars_for\n(few-shot trace)",   fillcolor="#c4b5fd"];
+    PolR [label="policy_stats · evaluate\npolicy_recommend",   fillcolor="#c4b5fd"];
   }
-  {rank=same; Rec; Midx; Skv; Out; Sesv; Mlst; Exem}
+  {rank=same; Rec; Midx; Skv; Out; Sesv; Mlst; Exem; PolR}
 
-  Prompt [label="PromptBuilder.build(request:)\nengine-aware .budget\nMEMORY (relevance-ranked) + SKILLS\n+ LEARNING + KNOWN MISTAKES/FIXES\n+ METRICS (per-engine) + EXTRO", fillcolor="#7dd3fc", penwidth=2];
+  Prompt [label="PromptBuilder.build(request:)\nengine-aware .budget\nMEMORY (relevance-ranked) + SKILLS\n+ LEARNING + KNOWN MISTAKES/FIXES\n+ METRICS (per-engine) + POLICY + EXTRO", fillcolor="#7dd3fc", penwidth=2];
 
   Rem  -> Fmem [color="#f59e0b"];
   Skc  -> Fskl [color="#f59e0b"];
@@ -94,6 +97,10 @@ digraph "PWN_MemorySkills" {
   Teach [label="Reflect.on\n(engine: reflect_engine)\nteacher-student", fillcolor="#fda4af", penwidth=2];
   Fses  -> Teach [label="transcript", style=dashed, color="#94a3b8", constraint=false];
   Teach -> Fmem  [label="frontier-authored\n:lesson", color="#fb7185", penwidth=2];
+
+  PolW -> Fpol [color="#f59e0b"];
+  Fpol -> PolR [color="#a78bfa"];
+  PolR -> Prompt [label="POLICY block\nadvisory only", color="#38bdf8"];
   Rdpo -> Fprf [color="#f59e0b"];
   Mrec -> Fprf [label="resolve → pair", style=dashed, color="#fbbf24"];
   Fprf -> Fftn [label="export_dpo", style=dashed, color="#fbbf24", constraint=false];

data/documentation/diagrams/dot/overall-pwn-architecture.dot

@@ -37,9 +37,9 @@ digraph "PWN_Overall_Architecture" {
     label="PWN::AI::Agent"; fontcolor="#ddd6fe";
     style=rounded; color="#6d28d9"; bgcolor="#2e1065"; penwidth=2;
     Loop     [label="Loop\nTaskSummarizer briefs\nplan_first → dispatch\n→ observe → escalate\nbudget-pressure caps",      fillcolor="#c4b5fd"];
-    Registry [label="Registry\n12 toolsets · 82 tools\ntool_router (CORE+topK)",    fillcolor="#c4b5fd"];
+    Registry [label="Registry\n13 toolsets · 85 tools\ntool_router (CORE+topK)",    fillcolor="#c4b5fd"];
     Swarm    [label="Swarm\npersonas · debate · bus",        fillcolor="#c4b5fd"];
-    Prompt   [label="PromptBuilder\nengine-budgeted blocks\nMemoryIndex → relevance-ranked",fillcolor="#c4b5fd"];
+    Prompt   [label="PromptBuilder\nengine-budgeted blocks\nMEMORY · SKILLS · POLICY\nMemoryIndex → relevance-ranked",fillcolor="#c4b5fd"];
   }
   {rank=same; Loop; Registry; Swarm; Prompt}
 
@@ -74,8 +74,9 @@ digraph "PWN_Overall_Architecture" {
     CronF    [label="cron/jobs.yml",      shape=cylinder, fillcolor="#fcd34d"];
     Finetune [label="finetune/*.jsonl", shape=cylinder, fillcolor="#fcd34d"];
     Prefs    [label="preferences.jsonl", shape=cylinder, fillcolor="#fcd34d"];
+    PolicyF  [label="policy.json\npolicy_traj.jsonl", shape=cylinder, fillcolor="#fcd34d"];
   }
-  {rank=same; Memory; MemIdx; Skills; Learn; Metrics; MistF; Extro; Sessions; SwarmB; CronF; Finetune; Prefs}
+  {rank=same; Memory; MemIdx; Skills; Learn; Metrics; MistF; Extro; Sessions; SwarmB; CronF; Finetune; Prefs; PolicyF}
 
   /* ── L0 → L1 ── */
   User -> REPL   [color="#38bdf8"];

data/documentation/diagrams/dot/persistence-filesystem.dot

@@ -21,6 +21,7 @@ digraph "PWN_Persistence" {
   prf  [label="preferences.jsonl\nDPO/KTO/ORPO pairs\nuser_correction · resolve · counterfactual"];
   mis  [label="mistakes.json\nfailure fingerprints · fixes\n[REPEATING] · [REGRESSED]"];
   met  [label="metrics.json\nper-tool · per-engine telemetry\ncalibration"];
+  pol  [label="policy.json + policy_traj.jsonl\nlive Q / REINFORCE · MDP log"];
   rws  [label="reward_sentinel.json\nproxy vs judge vs correction gap"];
   ext  [label="extrospection.json\nsnapshot (host/net/tc/repo/env/\nrf/web/osint/serial/telecomm/\npacket/vision/voice) + prev + obs[]"];
   eart [label="extrospection/{web,packet,voice}/*\nscreenshots · pcaps · wav/txt"];
@@ -34,7 +35,7 @@ digraph "PWN_Persistence" {
   hist [label="~/.pwn_history\nREPL history"];
 
   Root->cfg;  Root->schm; Root->mem;  Root->midx; Root->skl;  Root->lrn;
-  Root->prf;  Root->mis;  Root->met;  Root->rws;  Root->ext;  Root->eart;
+  Root->prf;  Root->mis;  Root->met;  Root->pol;  Root->rws;  Root->ext;  Root->eart;
   Root->ses;  Root->crn;  Root->agt;  Root->swm;  Root->cur;  Root->ftn;
   Root->bkp;  Root->hist;
 }

data/documentation/diagrams/dot/pwn-ai-feedback-learning-loop.dot

@@ -1,6 +1,6 @@
 digraph "PWN_AI_Feedback_Learning_Loop" {
   graph [
-    label=<<B>pwn-ai - Closed Self-Improvement Loop</B><BR/><FONT POINT-SIZE="11" COLOR="#94a3b8">Introspection (self)  ⟷  Extrospection (world)  ·  Mistakes (negative)  ·  Reward + Curriculum (RL)  ·  Local-model scaffolding</FONT>>,
+    label=<<B>pwn-ai - Closed Self-Improvement Loop</B><BR/><FONT POINT-SIZE="11" COLOR="#94a3b8">Introspection (self)  ⟷  Extrospection (world)  ·  Mistakes (negative)  ·  Reward + Curriculum + Policy (RL)  ·  Local-model scaffolding</FONT>>,
     labelloc=t, fontsize=20, fontname="Helvetica",
     rankdir=TB, splines=spline, nodesep=0.6, ranksep=1.0,
     bgcolor="#0f172a", fontcolor="#e2e8f0", pad=0.6, newrank=true, compound=true
@@ -21,7 +21,7 @@ digraph "PWN_AI_Feedback_Learning_Loop" {
     label="PWN::AI::Agent::Loop  ·  budget pressure · TaskSummarizer"; fontcolor="#ddd6fe";
     style=rounded; color="#6d28d9"; bgcolor="#2e1065"; penwidth=2;
     Prompt   [label="PromptBuilder\nbudget · MemoryIndex\nrelevance-ranked ctx", fillcolor="#c4b5fd"];
-    Dispatch [label="Dispatch\ntolerant parse\nrepair_name", fillcolor="#c4b5fd"];
+    Dispatch [label="Dispatch\ntolerant parse\nToolGuard · repair_name", fillcolor="#c4b5fd"];
 Router   [label="Registry\ntool_router\nCORE + top-K",   fillcolor="#c4b5fd"];
 Plan     [label="plan_first\n(pre-pass, local)",         fillcolor="#c4b5fd"];
 TaskSum  [label="TaskSummarizer\nemit_plan! · about_to\ndedup last_brief_fp", fillcolor="#c4b5fd", penwidth=2];
@@ -40,6 +40,7 @@ Escalate [label="escalate\n>=N fails → Swarm hint",        fillcolor="#c4b5fd"
     ReflectT [label="Reflect\nteacher-student\n(reflect_engine)", fillcolor="#6ee7b7", penwidth=2];
     Reward   [label="Reward\njudge · prm · semantic_ok\nsentinel warm\nDPO scrub <=40%/src", fillcolor="#6ee7b7", penwidth=2];
     Curric   [label="Curriculum\npractice · offline_judge\ncritic/CF geometry · gate + diet", fillcolor="#6ee7b7", penwidth=2];
+    Policy   [label="Policy\nlive Q / REINFORCE\nbegin · observe · finish\nadvisory rank only", fillcolor="#6ee7b7", penwidth=2];
   }
   subgraph cluster_extro {
     label="EXTROSPECTION  (world)"; fontcolor="#fde68a";
@@ -49,7 +50,7 @@ Escalate [label="escalate\n>=N fails → Swarm hint",        fillcolor="#c4b5fd"
     Verify    [label="verify (browser)\nfact-check own claims\nrevalidate_memory", fillcolor="#fcd34d", penwidth=2];
     RFTune    [label="rf_tune  📡\ntune GQRX · demod\nRDS → now_playing",           fillcolor="#fcd34d", penwidth=2];
   }
-  {rank=same; Metrics; Learning; Mistakes; ReflectT; Reward; Curric; Snapshot; Observe; Verify; RFTune}
+  {rank=same; Metrics; Learning; Mistakes; ReflectT; Reward; Curric; Policy; Snapshot; Observe; Verify; RFTune}
 
   /* L3 ─ correlate */
   Correlate [label="extro_correlate()\n\"I did it wrong\"  vs  \"the world changed\"",
@@ -68,8 +69,9 @@ Escalate [label="escalate\n>=N fails → Swarm hint",        fillcolor="#c4b5fd"
     Fext [label="extrospection.json", shape=cylinder, fillcolor="#fcd34d"];
     Fftn [label="finetune/*.jsonl",   shape=cylinder, fillcolor="#fcd34d"];
     Fprf [label="preferences.jsonl", shape=cylinder, fillcolor="#fcd34d"];
+    Fpol [label="policy.json\npolicy_traj.jsonl", shape=cylinder, fillcolor="#fcd34d"];
   }
-  {rank=same; Fmet; Flrn; Fmis; Fmem; Fidx; Fskl; Fext; Fftn; Fprf}
+  {rank=same; Fmet; Flrn; Fmis; Fmem; Fidx; Fskl; Fext; Fftn; Fprf; Fpol}
 
   /* L0 → L1 */
   User -> Prompt   [label="task", color="#38bdf8", penwidth=2];
@@ -87,7 +89,9 @@ Escalate [label="escalate\n>=N fails → Swarm hint",        fillcolor="#c4b5fd"
 
   /* L1 → L2 */
   Dispatch -> Metrics  [label="record(engine:)", color="#34d399"];
+  Dispatch -> Policy   [label="observe_step", color="#34d399"];
   Result   -> Learning [label="auto_introspect", color="#34d399"];
+  Result   -> Policy   [label="finish (judge)", color="#34d399"];
   Dispatch -> Mistakes [label="on failure\nrecord()", color="#fb7185", penwidth=2];
   User     -> Mistakes [label="\"that's wrong\"\ncheck_user_correction", color="#fb7185",
                         style=dashed, constraint=false];
@@ -111,6 +115,7 @@ Escalate [label="escalate\n>=N fails → Swarm hint",        fillcolor="#c4b5fd"
   /* L3 → L4 */
   Correlate -> Fmet [style=invis];
   Metrics   -> Fmet [color="#94a3b8"];
+  Policy    -> Fpol [color="#94a3b8"];
   Learning  -> Flrn [color="#94a3b8"];
   Mistakes  -> Fmis [color="#94a3b8"];
   Mistakes  -> Fmem [label="resolve→lesson", color="#94a3b8"];
@@ -141,6 +146,8 @@ Escalate [label="escalate\n>=N fails → Swarm hint",        fillcolor="#c4b5fd"
   Fidx -> Prompt [label="MemoryIndex\ncosine top-K", style=dashed, color="#fbbf24", constraint=false, penwidth=2];
   Flrn -> Plan   [label="exemplars_for\n(few-shot, local)", style=dashed, color="#fbbf24", constraint=false];
   Fmet -> Router [label="success_rate\n→ tie-break", style=dashed, color="#fbbf24", constraint=false];
+  Fpol -> Prompt [label="POLICY block", style=dashed, color="#fbbf24", constraint=false];
+  Fpol -> Router [label="Q-advantage\nvisits≥2", style=dashed, color="#fbbf24", constraint=false];
   Escalate -> LLM [label="Swarm.ask(persona)\nfrontier hint", color="#a78bfa", constraint=false, penwidth=2, style=dashed];
   Escalate -> Mistakes [label="record\n(tool:'escalation')", color="#fb7185", constraint=false, style=dashed];
   Fftn -> LLM  [label="LoRA (weights)\ncron weekly", style=dashed, color="#fb7185", constraint=false, penwidth=2];

data/documentation/diagrams/dot/reinforcement-learning.dot

@@ -19,13 +19,14 @@ digraph "PWN_Reinforcement_Learning" {
   subgraph cluster_reward {
     label="Tier 1 · Reward  (PWN::AI::Agent::Reward)"; fontcolor="#ddd6fe";
     style=rounded; color="#6d28d9"; bgcolor="#2e1065"; penwidth=2;
-    R1 [label="judge (outcome)\nscore · verdict · rationale", fillcolor="#c4b5fd", penwidth=2];
+    R1 [label="judge (cheap LLM ORM)\nscore · verdict · source\nevidence-prior fallback", fillcolor="#c4b5fd", penwidth=2];
     R2 [label="prm (process)\nper-step +1/0/-1\n→ Registry.rank", fillcolor="#c4b5fd"];
     R3 [label="sentinel (ring N=40)\nproxy != judge != (1-corr)\nwarm + proxy distrust", fillcolor="#c4b5fd"];
     R4 [label="semantic_ok\ngrep exit 1 != failure", fillcolor="#c4b5fd"];
+    R5 [label="Policy (live MDP)\nplan · complete · usable\nwarm Q + episode budget", fillcolor="#c4b5fd", penwidth=2];
     E3 [label="verify_as_reward\nextro_verify caps/floors judge", fillcolor="#c4b5fd"];
   }
-  {rank=same; R1; R2; R3; R4; E3}
+  {rank=same; R1; R2; R3; R4; R5; E3}
 
   /* Tier 4 — Curriculum / self-play */
   subgraph cluster_curr {
@@ -55,22 +56,26 @@ digraph "PWN_Reinforcement_Learning" {
     color="#a16207"; bgcolor="#422006";
     Fmis [label="mistakes.json", shape=cylinder, fillcolor="#fcd34d"];
     Fses [label="sessions/*.jsonl\n(step_reward)", shape=cylinder, fillcolor="#fcd34d"];
+    Fpol [label="policy.json +\npolicy_traj.jsonl", shape=cylinder, fillcolor="#fcd34d"];
     Fprf [label="preferences.jsonl", shape=cylinder, fillcolor="#fcd34d"];
     Fcur [label="curriculum/", shape=cylinder, fillcolor="#fcd34d"];
     Fftn [label="finetune/*.jsonl", shape=cylinder, fillcolor="#fcd34d"];
     Frws [label="reward_sentinel.json", shape=cylinder, fillcolor="#fcd34d"];
   }
-  {rank=same; Fmis; Fses; Fprf; Fcur; Fftn; Frws}
+  {rank=same; Fmis; Fses; Fpol; Fprf; Fcur; Fftn; Frws}
 
   Ollama [label="ollama create pwn-vN+1\n(local LoRA adapter)", fillcolor="#7dd3fc"];
 
   /* edges */
   Req  -> Loop [color="#38bdf8", penwidth=2];
   Loop -> R4   [label="Dispatch", color="#a78bfa"];
+  Loop -> R5   [label="(s,a,r,s')", color="#38bdf8", penwidth=2];
   Loop -> R1   [label="final", color="#a78bfa", penwidth=2];
   Loop -> R2   [label="session", color="#a78bfa"];
   Loop -> R3   [color="#a78bfa"];
   R1   -> E3   [label="ground", style=dashed, color="#fbbf24"];
+  R1   -> R5   [label="terminal", color="#a78bfa"];
+  R5   -> Fpol [color="#f59e0b"];
   Loop -> S4   [label="plan_first", color="#34d399"];
   Loop -> S3   [label="final", color="#34d399"];
   Loop -> S2   [label="[REPEATING]", color="#fb7185"];