packet-tracer-skill 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/README.md +31 -3
- package/SKILL.md +36 -0
- package/package.json +1 -1
- package/scripts/generate_pkt.py +253 -8
- package/scripts/session_log.py +325 -0
- package/scripts/usage_ledger.py +230 -218
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,55 @@ All notable changes to this project should be recorded in this file.
|
|
|
4
4
|
|
|
5
5
|
The format is intentionally simple and release-oriented.
|
|
6
6
|
|
|
7
|
+
## [0.3.2] - 2026-09-12
|
|
8
|
+
|
|
9
|
+
### Picking up a half-finished edit session
|
|
10
|
+
|
|
11
|
+
Almost nothing here needs remembering between turns: `--doctor`,
|
|
12
|
+
`--explain-plan` and `--parity-report` all recompute from the install, the
|
|
13
|
+
donor registry and the bridge, so an agent that has lost its earlier context
|
|
14
|
+
re-runs one and is exactly as certain as before. One question had no such
|
|
15
|
+
source -- part-way through `--explain-plan` -> `--edit` -> `--parity-report`,
|
|
16
|
+
*which lab am I working on and what comes next*.
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- `output/session-log.jsonl`, one line per run, and two flags that read it
|
|
21
|
+
back: `--session-state` lists the labs in flight, `--resume <lab.pkt>` says
|
|
22
|
+
where one was left.
|
|
23
|
+
- `--resume` re-hashes the lab and compares it with what the last step
|
|
24
|
+
recorded, and claims a position **only when the two agree**. When they do
|
|
25
|
+
not it says the lab has changed and to re-derive instead -- the same
|
|
26
|
+
refusal-first stance the rest of the skill takes. Verified first that a
|
|
27
|
+
digest can carry that weight: decoding a lab and re-encoding it reproduces
|
|
28
|
+
the file byte for byte.
|
|
29
|
+
- Three constraints, each with a test. No secrets: facts are allow-listed
|
|
30
|
+
rather than filtered, because an edit prompt holds a passphrase in a field
|
|
31
|
+
called `passphrase` and a deny-list would leak whatever secret field is added
|
|
32
|
+
next; the prompt survives only as a non-reversible shape fingerprint. No
|
|
33
|
+
weight: a test regenerates a lab with and without the log and compares the
|
|
34
|
+
decoded content. Ships nowhere: `output/` is gitignored and in no
|
|
35
|
+
`package.json` file list, checked rather than assumed.
|
|
36
|
+
|
|
37
|
+
### Fixed
|
|
38
|
+
|
|
39
|
+
- `PKT_USAGE_LEDGER=on`, the obvious way to switch learning on, made the path
|
|
40
|
+
resolver treat `"on"` as a filename and write the ledger to a file called
|
|
41
|
+
`on` in the working directory. One variable carrying two meanings with only
|
|
42
|
+
one of its two readers knowing the switch words. Found by doing it.
|
|
43
|
+
- `--session-state` and `--resume` were recorded as steps by the same wrapper
|
|
44
|
+
as the work they report on, so each recovery pushed itself in front of the
|
|
45
|
+
real chain. In a bounded log a reader that writes eventually evicts the steps
|
|
46
|
+
it exists to report.
|
|
47
|
+
|
|
48
|
+
### Changed
|
|
49
|
+
|
|
50
|
+
- `main()` is split into `_build_parser`, `_dispatch` and a wrapper that
|
|
51
|
+
records once around the whole dispatch. Fourteen branches with twelve return
|
|
52
|
+
points is not somewhere to repeat a hook.
|
|
53
|
+
|
|
54
|
+
847 passed, 1 skipped. Corpus: 32 of 33 generated, 31 opened, 0 unexpected.
|
|
55
|
+
|
|
7
56
|
## [0.3.1] - 2026-09-06
|
|
8
57
|
|
|
9
58
|
### Wireless labs that actually carry traffic
|
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ This repository is built for one job: take a natural-language network request, b
|
|
|
10
10
|
|
|
11
11
|
It is intended for networking labs where correctness matters more than producing a pretty but unverifiable diagram. The skill can plan, inspect, edit, compare, and explain Packet Tracer scenarios, but it deliberately separates "recognized by the parser", "visible in inventory", "edit-proven", "donor-backed ready", and "generate-ready" support.
|
|
12
12
|
|
|
13
|
-
## What `0.3.
|
|
13
|
+
## What `0.3.2` does
|
|
14
14
|
|
|
15
15
|
In this release a prompt produces a lab Packet Tracer opens.
|
|
16
16
|
|
|
@@ -229,6 +229,34 @@ Launch references:
|
|
|
229
229
|
- [docs/security-edge-deepening-proof.md](docs/security-edge-deepening-proof.md)
|
|
230
230
|
- [docs/packet-tracer-feature-gap-atlas.md](docs/packet-tracer-feature-gap-atlas.md)
|
|
231
231
|
|
|
232
|
+
## Picking Up A Half-Finished Edit Session
|
|
233
|
+
|
|
234
|
+
Almost nothing in this skill needs remembering between turns. `--doctor`,
|
|
235
|
+
`--explain-plan` and `--parity-report` recompute from the install, the donor
|
|
236
|
+
registry and the bridge, so an agent that has lost its earlier context re-runs
|
|
237
|
+
one and is exactly as certain as before.
|
|
238
|
+
|
|
239
|
+
One question has no such source: part-way through
|
|
240
|
+
`--explain-plan` -> `--edit` -> `--parity-report`, *which lab am I working on
|
|
241
|
+
and what comes next*. Each run now appends a line to
|
|
242
|
+
`output/session-log.jsonl`, and two flags read it back:
|
|
243
|
+
|
|
244
|
+
```powershell
|
|
245
|
+
python scripts\generate_pkt.py --session-state
|
|
246
|
+
python scripts\generate_pkt.py --resume output\lab.pkt
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
`--resume` re-hashes the lab and compares it with what the last step recorded.
|
|
250
|
+
A position is reported **only when the two agree**; otherwise it says the lab
|
|
251
|
+
has changed and to re-derive instead. That refusal is the useful answer, and it
|
|
252
|
+
is the same stance the rest of the skill takes when it cannot prove something.
|
|
253
|
+
|
|
254
|
+
The log carries no secrets (facts are allow-listed, and the prompt survives
|
|
255
|
+
only as a non-reversible shape fingerprint), carries no weight (a test
|
|
256
|
+
regenerates a lab with and without it and compares the decoded content), and
|
|
257
|
+
ships nowhere (`output/` is gitignored and in no `package.json` file list).
|
|
258
|
+
`PKT_SESSION_LOG=off` disables it.
|
|
259
|
+
|
|
232
260
|
## Runtime Doctor Contract
|
|
233
261
|
|
|
234
262
|
`--doctor` is a product surface, not a debug afterthought. It reports:
|
|
@@ -598,7 +626,7 @@ See also:
|
|
|
598
626
|
|
|
599
627
|
## Release and Launch State
|
|
600
628
|
|
|
601
|
-
The current line is `packet-tracer-skill@0.3.
|
|
629
|
+
The current line is `packet-tracer-skill@0.3.2`, and it is the first release
|
|
602
630
|
where generation is the headline rather than a deferred promise. `0.2.3`, the
|
|
603
631
|
previous published line, was a capability proof and readiness release that
|
|
604
632
|
deliberately refused broad generation.
|
|
@@ -652,7 +680,7 @@ qurur, mövcud faylı redaktə edir və hər iddiasını ölçü ilə əsasland
|
|
|
652
680
|
fayl Packet Tracer-də açılır və cihazları bir-birini ping edir; alınmayanda
|
|
653
681
|
səbəbini açıq deyir.
|
|
654
682
|
|
|
655
|
-
### `0.3.
|
|
683
|
+
### `0.3.2` nə dəyişdi
|
|
656
684
|
|
|
657
685
|
Bu, promptun Packet Tracer-in açdığı fayla çevrildiyi ilk buraxılışdır.
|
|
658
686
|
|
package/SKILL.md
CHANGED
|
@@ -175,6 +175,42 @@ There is no artificial device limit. The only ceiling is physical: a switch has
|
|
|
175
175
|
the ports it has, and the generator says so plainly when it runs out. Ask for
|
|
176
176
|
more switches, not fewer hosts.
|
|
177
177
|
|
|
178
|
+
### If you lost the thread part-way through an edit chain
|
|
179
|
+
|
|
180
|
+
Almost nothing here needs remembering. `--doctor`, `--explain-plan` and
|
|
181
|
+
`--parity-report` all recompute from the install, the donor registry and the
|
|
182
|
+
bridge, so re-running one costs time and nothing else. Losing the conversation
|
|
183
|
+
does not make the skill less certain about anything it reports.
|
|
184
|
+
|
|
185
|
+
One question has no such source. Part-way through
|
|
186
|
+
`--explain-plan` -> `--edit` -> `--parity-report`, *which lab am I working on
|
|
187
|
+
and what comes next* lived only in the conversation. So each run appends a line
|
|
188
|
+
to `output/session-log.jsonl`, and two flags read it back:
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
python scripts/generate_pkt.py --session-state # which labs are in flight
|
|
192
|
+
python scripts/generate_pkt.py --resume output/lab.pkt # where this one was left
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
`--resume` re-hashes the lab and compares it to what the last step recorded. It
|
|
196
|
+
reports a position **only when the two agree**; when they do not, it says the
|
|
197
|
+
lab has changed and tells you to re-derive instead. Treat "position not
|
|
198
|
+
claimed" as the correct answer, not a failure -- it is the same refusal-first
|
|
199
|
+
stance as everything else here.
|
|
200
|
+
|
|
201
|
+
Three things the log deliberately is not:
|
|
202
|
+
|
|
203
|
+
- **not a secret store.** Facts are allow-listed, never filtered. An edit
|
|
204
|
+
prompt carries a passphrase in an ordinary field, so nothing is written but
|
|
205
|
+
the named, known-safe keys, and the prompt survives only as the same
|
|
206
|
+
non-reversible shape fingerprint the usage ledger uses.
|
|
207
|
+
- **not load-bearing.** Delete it and every result is identical; there is a
|
|
208
|
+
test that regenerates a lab with and without it and compares the content.
|
|
209
|
+
- **not shipped.** It lives under `output/`, which is gitignored and in no
|
|
210
|
+
`package.json` file list.
|
|
211
|
+
|
|
212
|
+
`PKT_SESSION_LOG=off` turns it off; `PKT_SESSION_LOG=<path>` moves it.
|
|
213
|
+
|
|
178
214
|
### A generated lab becomes the next build's donor
|
|
179
215
|
|
|
180
216
|
Donor selection can pick a lab this skill produced, so every repair pass
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "packet-tracer-skill",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.2",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Donor-backed Cisco Packet Tracer 9.x skill for natural-language planning, safe-open generation, editing, parity reporting, and runtime diagnostics across coding hosts.",
|
|
6
6
|
"keywords": [
|
package/scripts/generate_pkt.py
CHANGED
|
@@ -341,7 +341,7 @@ def _local_donor_candidates(
|
|
|
341
341
|
|
|
342
342
|
|
|
343
343
|
def _rank_generation_donors(
|
|
344
|
-
plan: IntentPlan,
|
|
344
|
+
plan: "IntentPlan",
|
|
345
345
|
topology_tags: list[str],
|
|
346
346
|
donor_roots: list[Path] | None = None,
|
|
347
347
|
) -> tuple[list[SampleCandidate], list[SampleCandidate], list[SampleCandidate]]:
|
|
@@ -405,7 +405,7 @@ def _default_import_cache_root() -> Path:
|
|
|
405
405
|
|
|
406
406
|
|
|
407
407
|
def _resolve_remote_sources(
|
|
408
|
-
plan: IntentPlan,
|
|
408
|
+
plan: "IntentPlan",
|
|
409
409
|
reference_roots: list[Path] | None,
|
|
410
410
|
donor_roots: list[Path] | None,
|
|
411
411
|
*,
|
|
@@ -658,7 +658,7 @@ def _candidate_archetype_alignment(
|
|
|
658
658
|
|
|
659
659
|
|
|
660
660
|
def _build_support_reports(
|
|
661
|
-
plan: IntentPlan,
|
|
661
|
+
plan: "IntentPlan",
|
|
662
662
|
*,
|
|
663
663
|
blueprint: dict[str, object] | None = None,
|
|
664
664
|
cisco_ranked: list[SampleCandidate] | None = None,
|
|
@@ -1486,7 +1486,7 @@ def _pool_selected_a_donor(diagnostics: list[dict[str, object]]) -> bool:
|
|
|
1486
1486
|
|
|
1487
1487
|
|
|
1488
1488
|
def _evaluate_donor_prune_candidates(
|
|
1489
|
-
plan: IntentPlan,
|
|
1489
|
+
plan: "IntentPlan",
|
|
1490
1490
|
blueprint: dict[str, object],
|
|
1491
1491
|
donor_candidates: list[SampleCandidate],
|
|
1492
1492
|
) -> tuple[
|
|
@@ -3979,7 +3979,7 @@ def _address_hosts_per_vlan(
|
|
|
3979
3979
|
|
|
3980
3980
|
|
|
3981
3981
|
def _unify_host_segment(
|
|
3982
|
-
plan: IntentPlan,
|
|
3982
|
+
plan: "IntentPlan",
|
|
3983
3983
|
devices: list[dict[str, object]],
|
|
3984
3984
|
links: list[dict[str, object]],
|
|
3985
3985
|
donor_root=None,
|
|
@@ -10624,7 +10624,7 @@ def _build_donor_prune_plan_for_donor(plan: IntentPlan, blueprint: dict[str, obj
|
|
|
10624
10624
|
|
|
10625
10625
|
|
|
10626
10626
|
def _build_donor_prune_plan(
|
|
10627
|
-
plan: IntentPlan,
|
|
10627
|
+
plan: "IntentPlan",
|
|
10628
10628
|
blueprint: dict[str, object],
|
|
10629
10629
|
donor_roots: list[Path] | None = None,
|
|
10630
10630
|
) -> tuple[IntentPlan, DonorArchetypePlan]:
|
|
@@ -11683,6 +11683,14 @@ def generate_from_prompt(
|
|
|
11683
11683
|
donor_archetype=donor_archetype,
|
|
11684
11684
|
outcome=usage_ledger.OUTCOME_GENERATED_UNVERIFIED,
|
|
11685
11685
|
)
|
|
11686
|
+
_record_session_generation(
|
|
11687
|
+
output_path=output_path,
|
|
11688
|
+
plan=raw_plan,
|
|
11689
|
+
blueprint_plan=blueprint_plan,
|
|
11690
|
+
scenario_decision=scenario_generate_decision,
|
|
11691
|
+
donor_archetype=donor_archetype,
|
|
11692
|
+
root=root,
|
|
11693
|
+
)
|
|
11686
11694
|
if blueprint_out_path is not None:
|
|
11687
11695
|
blueprint_out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
11688
11696
|
blueprint_out_path.write_text(json.dumps(blueprint_plan, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
@@ -11691,6 +11699,54 @@ def generate_from_prompt(
|
|
|
11691
11699
|
print(f"Loaded reference-only samples: {len(references)}")
|
|
11692
11700
|
|
|
11693
11701
|
|
|
11702
|
+
def _record_session_generation(
|
|
11703
|
+
*,
|
|
11704
|
+
output_path: Path,
|
|
11705
|
+
plan: "IntentPlan",
|
|
11706
|
+
blueprint_plan: dict,
|
|
11707
|
+
scenario_decision: dict,
|
|
11708
|
+
donor_archetype: object,
|
|
11709
|
+
root: ET.Element,
|
|
11710
|
+
) -> None:
|
|
11711
|
+
"""Note what was built, next to where the ledger notes that it worked.
|
|
11712
|
+
|
|
11713
|
+
Placed here because this is the one point where the answer to "what is this
|
|
11714
|
+
lab" is fully known: the plan that asked for it, the donor it came from and
|
|
11715
|
+
the file that was just written. Recording it anywhere earlier would be
|
|
11716
|
+
recording an intention rather than an outcome.
|
|
11717
|
+
|
|
11718
|
+
The prompt itself is not passed on. What survives is what it parsed to --
|
|
11719
|
+
counts, VLAN numbers, capability names -- which says what was being built
|
|
11720
|
+
without saying who it was for, and carries none of the passphrases an edit
|
|
11721
|
+
prompt can hold.
|
|
11722
|
+
"""
|
|
11723
|
+
try:
|
|
11724
|
+
from session_log import note_detailed, record
|
|
11725
|
+
|
|
11726
|
+
note_detailed()
|
|
11727
|
+
donor = getattr(donor_archetype, "compat_donor", "")
|
|
11728
|
+
record(
|
|
11729
|
+
"generate",
|
|
11730
|
+
artifact=output_path,
|
|
11731
|
+
status="ok",
|
|
11732
|
+
facts={
|
|
11733
|
+
"goal": getattr(plan, "goal", ""),
|
|
11734
|
+
"device_counts": dict(getattr(plan, "device_counts", {}) or {}),
|
|
11735
|
+
"vlan_ids": list(getattr(plan, "vlan_ids", []) or []),
|
|
11736
|
+
"capabilities": list(blueprint_plan.get("required_capabilities") or []),
|
|
11737
|
+
"scenario_family": scenario_decision.get("family"),
|
|
11738
|
+
"readiness_status": scenario_decision.get("readiness_status"),
|
|
11739
|
+
"allow_generate": scenario_decision.get("allow_generate"),
|
|
11740
|
+
"donor": Path(str(donor)).name if donor else "",
|
|
11741
|
+
"device_count": len(root.findall(".//DEVICES/DEVICE")),
|
|
11742
|
+
"link_count": len(root.findall(".//LINKS/LINK")),
|
|
11743
|
+
"next_best_action": scenario_decision.get("what_would_make_it_pass") or "",
|
|
11744
|
+
},
|
|
11745
|
+
)
|
|
11746
|
+
except Exception:
|
|
11747
|
+
return
|
|
11748
|
+
|
|
11749
|
+
|
|
11694
11750
|
def _resolve_edit_link_ports(pkt_path: Path, plan: IntentPlan) -> None:
|
|
11695
11751
|
"""Fill in real, free ports on link edits phrased without them.
|
|
11696
11752
|
|
|
@@ -12559,7 +12615,62 @@ def local_sample_audit(root: Path, audit_out: Path | None = None) -> None:
|
|
|
12559
12615
|
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
12560
12616
|
|
|
12561
12617
|
|
|
12562
|
-
def
|
|
12618
|
+
def _print_resume(artifact: Path) -> None:
|
|
12619
|
+
"""Say where this lab was left, and admit it when the answer cannot be trusted.
|
|
12620
|
+
|
|
12621
|
+
Deliberately not a claim. The log says what the last step was; the lab on
|
|
12622
|
+
disk says what it is now. Only when the two agree is a position reported,
|
|
12623
|
+
because a checkpoint believed without checking would be the same defect
|
|
12624
|
+
this skill spends its passes hunting: one fact derived twice, with nothing
|
|
12625
|
+
comparing the derivations.
|
|
12626
|
+
"""
|
|
12627
|
+
from session_log import resume_report
|
|
12628
|
+
|
|
12629
|
+
report = resume_report(artifact)
|
|
12630
|
+
if not report.get("known"):
|
|
12631
|
+
print(f"No recorded steps for {artifact}.")
|
|
12632
|
+
print("Nothing is lost: run --explain-plan to re-derive the plan, or --coherence-report to see where the lab stands.")
|
|
12633
|
+
return
|
|
12634
|
+
|
|
12635
|
+
print(f"{artifact}")
|
|
12636
|
+
print(f" last step : {report.get('last_command')} ({report.get('last_status')}) at {report.get('at')}")
|
|
12637
|
+
print(f" steps logged: {report.get('steps')}")
|
|
12638
|
+
print(f" {report.get('summary')}")
|
|
12639
|
+
if not report.get("matches_log"):
|
|
12640
|
+
print(" -> re-derive rather than resume: --explain-plan, then --coherence-report")
|
|
12641
|
+
if report.get("next_best_action"):
|
|
12642
|
+
print(f" next : {report['next_best_action']}")
|
|
12643
|
+
facts = report.get("last_facts") or {}
|
|
12644
|
+
for key in ("device_counts", "vlan_ids", "capabilities", "donor", "contradiction_counts"):
|
|
12645
|
+
if facts.get(key):
|
|
12646
|
+
print(f" {key:12}: {facts[key]}")
|
|
12647
|
+
history = report.get("history") or []
|
|
12648
|
+
if len(history) > 1:
|
|
12649
|
+
print(" history :")
|
|
12650
|
+
for step in history:
|
|
12651
|
+
print(f" {step.get('at')} {step.get('command')} ({step.get('status')})")
|
|
12652
|
+
|
|
12653
|
+
|
|
12654
|
+
def _print_session_state() -> None:
|
|
12655
|
+
from session_log import artifacts_seen, latest_for, log_enabled, log_path
|
|
12656
|
+
|
|
12657
|
+
if not log_enabled():
|
|
12658
|
+
print("Session logging is off (PKT_SESSION_LOG).")
|
|
12659
|
+
return
|
|
12660
|
+
seen = artifacts_seen()
|
|
12661
|
+
if not seen:
|
|
12662
|
+
print(f"No session steps recorded yet ({log_path()}).")
|
|
12663
|
+
return
|
|
12664
|
+
print(f"{len(seen)} lab(s) touched, most recent first:")
|
|
12665
|
+
for name in seen[:20]:
|
|
12666
|
+
last = latest_for(name) or {}
|
|
12667
|
+
print(f" {name}")
|
|
12668
|
+
print(f" {last.get('command')} ({last.get('status')}) at {last.get('at')}")
|
|
12669
|
+
print()
|
|
12670
|
+
print("Run --resume <path> for one of them.")
|
|
12671
|
+
|
|
12672
|
+
|
|
12673
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
12563
12674
|
parser = argparse.ArgumentParser(description="Generate or inspect Cisco Packet Tracer 9.0 .pkt files")
|
|
12564
12675
|
parser.add_argument("--blueprint", help="Path to the topology blueprint JSON")
|
|
12565
12676
|
parser.add_argument("--prompt", help="Natural language topology or edit request")
|
|
@@ -12598,7 +12709,19 @@ def main() -> None:
|
|
|
12598
12709
|
parser.add_argument("--matrix-out", help="Optional JSON output path when using --compare-scenarios")
|
|
12599
12710
|
parser.add_argument("--acceptance-json-out", help="Optional JSON output path for explain/compare/parity payloads")
|
|
12600
12711
|
parser.add_argument("--device-family", help="Optional device family filter for --coverage-report")
|
|
12601
|
-
|
|
12712
|
+
parser.add_argument(
|
|
12713
|
+
"--resume",
|
|
12714
|
+
help="Say where a multi-step edit session left this .pkt, and whether the file still matches what was recorded",
|
|
12715
|
+
)
|
|
12716
|
+
parser.add_argument(
|
|
12717
|
+
"--session-state",
|
|
12718
|
+
action="store_true",
|
|
12719
|
+
help="List the labs a session has touched, most recently first",
|
|
12720
|
+
)
|
|
12721
|
+
return parser
|
|
12722
|
+
|
|
12723
|
+
|
|
12724
|
+
def _dispatch(args: argparse.Namespace) -> None:
|
|
12602
12725
|
if args.compat_donor:
|
|
12603
12726
|
os.environ["PACKET_TRACER_COMPAT_DONOR"] = args.compat_donor
|
|
12604
12727
|
reference_roots = [Path(path) for path in (args.reference_root or [])]
|
|
@@ -12657,7 +12780,15 @@ def main() -> None:
|
|
|
12657
12780
|
inventory_out=Path(args.inventory_out) if args.inventory_out else None,
|
|
12658
12781
|
)
|
|
12659
12782
|
return
|
|
12783
|
+
if args.resume:
|
|
12784
|
+
_print_resume(Path(args.resume))
|
|
12785
|
+
return
|
|
12786
|
+
if args.session_state:
|
|
12787
|
+
_print_session_state()
|
|
12788
|
+
return
|
|
12660
12789
|
if args.coherence_report:
|
|
12790
|
+
from collections import Counter
|
|
12791
|
+
|
|
12661
12792
|
from lab_coherence import check_lab_coherence, summarise
|
|
12662
12793
|
|
|
12663
12794
|
root = decode_pkt_to_root(Path(args.coherence_report))
|
|
@@ -12665,6 +12796,23 @@ def main() -> None:
|
|
|
12665
12796
|
print(summarise(findings))
|
|
12666
12797
|
for finding in findings:
|
|
12667
12798
|
print(f" {finding}")
|
|
12799
|
+
try:
|
|
12800
|
+
from session_log import note_detailed, record
|
|
12801
|
+
|
|
12802
|
+
note_detailed()
|
|
12803
|
+
record(
|
|
12804
|
+
"coherence-report",
|
|
12805
|
+
source=args.coherence_report,
|
|
12806
|
+
status="ok" if not findings else "contradictions",
|
|
12807
|
+
facts={
|
|
12808
|
+
"contradiction_counts": dict(Counter(getattr(f, "kind", "") for f in findings)),
|
|
12809
|
+
"next_best_action": (
|
|
12810
|
+
"" if not findings else "resolve the contradictions above, then re-run --coherence-report"
|
|
12811
|
+
),
|
|
12812
|
+
},
|
|
12813
|
+
)
|
|
12814
|
+
except Exception:
|
|
12815
|
+
pass
|
|
12668
12816
|
# A lab that contradicts itself opens perfectly well, so the exit code
|
|
12669
12817
|
# is the only place the answer can be acted on.
|
|
12670
12818
|
raise SystemExit(1 if findings else 0)
|
|
@@ -12734,5 +12882,102 @@ def main() -> None:
|
|
|
12734
12882
|
generate_from_blueprint(Path(args.blueprint), Path(args.output), Path(args.xml_out) if args.xml_out else None)
|
|
12735
12883
|
|
|
12736
12884
|
|
|
12885
|
+
|
|
12886
|
+
def main() -> None:
|
|
12887
|
+
"""Run the command, and leave a note saying it ran.
|
|
12888
|
+
|
|
12889
|
+
The note is for a later agent, not for this one. Everything this skill
|
|
12890
|
+
decides is recomputed from the environment, so a lost conversation costs
|
|
12891
|
+
only the time to run `--doctor` again -- except for one question, which has
|
|
12892
|
+
no source but the conversation: part-way through a chain of
|
|
12893
|
+
`--explain-plan` -> `--edit` -> `--parity-report`, which lab is being
|
|
12894
|
+
worked on and which step is next. `--resume` answers that from the log.
|
|
12895
|
+
|
|
12896
|
+
Wrapped once, around the whole dispatch, rather than added to each of the
|
|
12897
|
+
fourteen branches: a hook repeated fourteen times is a hook that will be
|
|
12898
|
+
forgotten on the fifteenth.
|
|
12899
|
+
"""
|
|
12900
|
+
args = _build_parser().parse_args()
|
|
12901
|
+
status = "ok"
|
|
12902
|
+
try:
|
|
12903
|
+
_dispatch(args)
|
|
12904
|
+
except SystemExit as exit_request:
|
|
12905
|
+
status = "ok" if not exit_request.code else "refused"
|
|
12906
|
+
raise
|
|
12907
|
+
except BaseException:
|
|
12908
|
+
status = "error"
|
|
12909
|
+
raise
|
|
12910
|
+
finally:
|
|
12911
|
+
_record_session_step(args, status)
|
|
12912
|
+
|
|
12913
|
+
|
|
12914
|
+
def _record_session_step(args: argparse.Namespace, status: str) -> None:
|
|
12915
|
+
"""Append one line describing this invocation. Never raises."""
|
|
12916
|
+
try:
|
|
12917
|
+
from session_log import had_detailed, record
|
|
12918
|
+
from usage_ledger import prompt_fingerprint
|
|
12919
|
+
|
|
12920
|
+
# Reading the log is not a step in the work. Recording it pushed two
|
|
12921
|
+
# `other` entries in front of the real chain on every recovery, and in
|
|
12922
|
+
# a bounded log a reader that writes eventually evicts the very steps
|
|
12923
|
+
# it exists to report.
|
|
12924
|
+
if getattr(args, "resume", None) or getattr(args, "session_state", False):
|
|
12925
|
+
return
|
|
12926
|
+
if had_detailed() and status == "ok":
|
|
12927
|
+
return
|
|
12928
|
+
|
|
12929
|
+
command = next(
|
|
12930
|
+
(
|
|
12931
|
+
name
|
|
12932
|
+
for name in (
|
|
12933
|
+
"explain_plan",
|
|
12934
|
+
"parity_report",
|
|
12935
|
+
"coherence_report",
|
|
12936
|
+
"compare_scenarios",
|
|
12937
|
+
"inventory",
|
|
12938
|
+
"validate_open",
|
|
12939
|
+
"validate_open_debug",
|
|
12940
|
+
"decode",
|
|
12941
|
+
"edit",
|
|
12942
|
+
"coverage_report",
|
|
12943
|
+
"feature_gap_report",
|
|
12944
|
+
)
|
|
12945
|
+
if getattr(args, name, None)
|
|
12946
|
+
),
|
|
12947
|
+
"generate" if getattr(args, "prompt", None) else "other",
|
|
12948
|
+
)
|
|
12949
|
+
|
|
12950
|
+
# The prompt is never written down. Its shape is: "3 switch 6 pc" and
|
|
12951
|
+
# "5 switch 2 pc" share a fingerprint, which is the granularity that
|
|
12952
|
+
# answers "what was I building" without recording who it was for.
|
|
12953
|
+
text = ""
|
|
12954
|
+
for candidate in (
|
|
12955
|
+
getattr(args, "prompt", None),
|
|
12956
|
+
getattr(args, "explain_plan", None),
|
|
12957
|
+
getattr(args, "parity_report", None),
|
|
12958
|
+
):
|
|
12959
|
+
if isinstance(candidate, str) and candidate:
|
|
12960
|
+
text = candidate
|
|
12961
|
+
break
|
|
12962
|
+
|
|
12963
|
+
facts = {}
|
|
12964
|
+
if text:
|
|
12965
|
+
facts["prompt_shape"] = prompt_fingerprint(text)
|
|
12966
|
+
|
|
12967
|
+
source = getattr(args, "edit", None) or getattr(args, "coherence_report", None)
|
|
12968
|
+
if not source and isinstance(getattr(args, "validate_open", None), str):
|
|
12969
|
+
source = args.validate_open
|
|
12970
|
+
|
|
12971
|
+
record(
|
|
12972
|
+
command.replace("_", "-"),
|
|
12973
|
+
artifact=getattr(args, "output", None),
|
|
12974
|
+
source=source,
|
|
12975
|
+
status=status,
|
|
12976
|
+
facts=facts,
|
|
12977
|
+
)
|
|
12978
|
+
except Exception:
|
|
12979
|
+
return
|
|
12980
|
+
|
|
12981
|
+
|
|
12737
12982
|
if __name__ == "__main__":
|
|
12738
12983
|
main()
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Where a multi-step edit session had got to, so a compacted agent can ask.
|
|
3
|
+
|
|
4
|
+
Almost nothing in this skill lives in conversation memory: `--doctor`,
|
|
5
|
+
`--parity-report` and `--explain-plan` all recompute from the environment, so
|
|
6
|
+
losing an earlier turn costs nothing but the time to run them again. One thing
|
|
7
|
+
is not recoverable that way. Part-way through a chain of `--explain-plan` ->
|
|
8
|
+
`--edit` -> `--parity-report` calls, the question "which file am I working on,
|
|
9
|
+
and which step of which plan am I on" has no source but the conversation --
|
|
10
|
+
and that is exactly what compaction takes.
|
|
11
|
+
|
|
12
|
+
So each invocation appends a line here, and `--resume` reads it back.
|
|
13
|
+
|
|
14
|
+
The rules are the usage ledger's, for the same reasons:
|
|
15
|
+
|
|
16
|
+
- local only. Written under `output/`, which is gitignored and appears in no
|
|
17
|
+
`package.json` file list, so it reaches neither a repository nor a registry.
|
|
18
|
+
- **nothing is load-bearing.** Deleting this file must change no result
|
|
19
|
+
anywhere. It answers a question; it never feeds a decision.
|
|
20
|
+
- bounded, and a corrupt or unreadable log is ignored rather than fatal.
|
|
21
|
+
- `PKT_SESSION_LOG=off` disables it entirely.
|
|
22
|
+
|
|
23
|
+
And one rule of its own. Facts are recorded by **allow-list**, never by copying
|
|
24
|
+
a payload and stripping what looks sensitive. An edit prompt carries secrets in
|
|
25
|
+
ordinary fields -- `passphrase` on `set_wireless_ssid`, `password` on three
|
|
26
|
+
more operations, `community` on `set_bgp_neighbor` -- so a deny-list would leak
|
|
27
|
+
the first secret field anyone adds after this was written. Operation *names and
|
|
28
|
+
counts* are recorded; operation *values* never are, and neither is the prompt,
|
|
29
|
+
which is kept as the same non-reversible shape fingerprint the ledger uses.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import hashlib
|
|
35
|
+
import json
|
|
36
|
+
import os
|
|
37
|
+
from datetime import datetime, timezone
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
from typing import Any
|
|
40
|
+
|
|
41
|
+
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
|
42
|
+
DEFAULT_LOG_PATH = SKILL_ROOT / "output" / "session-log.jsonl"
|
|
43
|
+
MAX_ENTRIES = 500
|
|
44
|
+
LOG_VERSION = 1
|
|
45
|
+
|
|
46
|
+
# Every fact that may be written. A key absent from here is dropped whatever it
|
|
47
|
+
# holds -- see the module docstring for why this is an allow-list.
|
|
48
|
+
ALLOWED_FACTS = frozenset(
|
|
49
|
+
{
|
|
50
|
+
"goal",
|
|
51
|
+
"prompt_shape",
|
|
52
|
+
"device_counts",
|
|
53
|
+
"vlan_ids",
|
|
54
|
+
"capabilities",
|
|
55
|
+
"scenario_family",
|
|
56
|
+
"readiness_status",
|
|
57
|
+
"allow_generate",
|
|
58
|
+
"donor",
|
|
59
|
+
"target_version",
|
|
60
|
+
"operation_counts",
|
|
61
|
+
"next_best_action",
|
|
62
|
+
"parity_counts",
|
|
63
|
+
"contradiction_counts",
|
|
64
|
+
"device_count",
|
|
65
|
+
"link_count",
|
|
66
|
+
"opened",
|
|
67
|
+
"detail",
|
|
68
|
+
}
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
# Values are summarised, never copied wholesale, so a long list cannot become a
|
|
72
|
+
# long line and a stray string cannot smuggle a secret through a counter.
|
|
73
|
+
_MAX_STRING = 200
|
|
74
|
+
_MAX_ITEMS = 40
|
|
75
|
+
|
|
76
|
+
_OFF_WORDS = {"off", "0", "false", "none"}
|
|
77
|
+
_ON_WORDS = {"on", "1", "true"}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def log_path() -> Path:
|
|
81
|
+
override = (os.getenv("PKT_SESSION_LOG") or "").strip()
|
|
82
|
+
if override and override.lower() not in (_OFF_WORDS | _ON_WORDS):
|
|
83
|
+
return Path(override).expanduser()
|
|
84
|
+
return DEFAULT_LOG_PATH
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def log_enabled() -> bool:
|
|
88
|
+
return (os.getenv("PKT_SESSION_LOG") or "").strip().lower() not in _OFF_WORDS
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# Set when a branch has already written a fuller entry for this process, so the
|
|
92
|
+
# wrapper around the dispatch does not add a thinner duplicate beside it. The
|
|
93
|
+
# wrapper still fires for every command that writes nothing of its own, which
|
|
94
|
+
# is the point of wrapping rather than hooking each branch.
|
|
95
|
+
_DETAILED = False
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def note_detailed() -> None:
|
|
99
|
+
global _DETAILED
|
|
100
|
+
_DETAILED = True
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def had_detailed() -> bool:
|
|
104
|
+
return _DETAILED
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _clean(value: Any) -> Any:
|
|
108
|
+
"""Reduce a value to something small, printable and free of running text."""
|
|
109
|
+
if value is None or isinstance(value, bool) or isinstance(value, int):
|
|
110
|
+
return value
|
|
111
|
+
if isinstance(value, float):
|
|
112
|
+
return round(value, 3)
|
|
113
|
+
if isinstance(value, str):
|
|
114
|
+
return value[:_MAX_STRING]
|
|
115
|
+
if isinstance(value, dict):
|
|
116
|
+
return {str(key)[:_MAX_STRING]: _clean(item) for key, item in list(value.items())[:_MAX_ITEMS]}
|
|
117
|
+
if isinstance(value, (list, tuple, set)):
|
|
118
|
+
return [_clean(item) for item in list(value)[:_MAX_ITEMS]]
|
|
119
|
+
return str(value)[:_MAX_STRING]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def normalise(path: Path | str | None) -> str:
|
|
123
|
+
"""One spelling per file, so one lab is not mistaken for two.
|
|
124
|
+
|
|
125
|
+
The same lab arrived down two routes and was listed twice: the generation
|
|
126
|
+
branch passes a `Path`, which prints with backslashes on Windows, while the
|
|
127
|
+
flag branches pass the string the shell gave, which had forward slashes.
|
|
128
|
+
Resolving both collapses separators and relative prefixes together.
|
|
129
|
+
"""
|
|
130
|
+
if not path:
|
|
131
|
+
return ""
|
|
132
|
+
try:
|
|
133
|
+
return str(Path(path).resolve())
|
|
134
|
+
except (OSError, ValueError):
|
|
135
|
+
return str(path)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def artifact_digest(path: Path | str | None) -> str:
|
|
139
|
+
"""A `.pkt` identifies itself by its bytes.
|
|
140
|
+
|
|
141
|
+
Measured before relying on it: decoding a lab and re-encoding it reproduces
|
|
142
|
+
the file byte for byte, so identical content always hashes identically and
|
|
143
|
+
a changed digest means the lab really changed. That is what lets `--resume`
|
|
144
|
+
check its own answer instead of asserting it.
|
|
145
|
+
"""
|
|
146
|
+
if not path:
|
|
147
|
+
return ""
|
|
148
|
+
candidate = Path(path)
|
|
149
|
+
try:
|
|
150
|
+
if not candidate.is_file():
|
|
151
|
+
return ""
|
|
152
|
+
return hashlib.sha256(candidate.read_bytes()).hexdigest()
|
|
153
|
+
except OSError:
|
|
154
|
+
return ""
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def record(
|
|
158
|
+
command: str,
|
|
159
|
+
*,
|
|
160
|
+
artifact: Path | str | None = None,
|
|
161
|
+
source: Path | str | None = None,
|
|
162
|
+
status: str = "ok",
|
|
163
|
+
facts: dict[str, Any] | None = None,
|
|
164
|
+
) -> None:
|
|
165
|
+
"""Append one step. Never raises: a broken log must not break a build."""
|
|
166
|
+
if not log_enabled():
|
|
167
|
+
return
|
|
168
|
+
try:
|
|
169
|
+
entry: dict[str, Any] = {
|
|
170
|
+
"v": LOG_VERSION,
|
|
171
|
+
"at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
172
|
+
"command": str(command)[:_MAX_STRING],
|
|
173
|
+
"status": str(status)[:_MAX_STRING],
|
|
174
|
+
}
|
|
175
|
+
if artifact:
|
|
176
|
+
entry["artifact"] = normalise(artifact)[:_MAX_STRING]
|
|
177
|
+
entry["artifact_sha256"] = artifact_digest(artifact)
|
|
178
|
+
if source:
|
|
179
|
+
entry["source"] = normalise(source)[:_MAX_STRING]
|
|
180
|
+
entry["source_sha256"] = artifact_digest(source)
|
|
181
|
+
kept = {key: _clean(value) for key, value in (facts or {}).items() if key in ALLOWED_FACTS}
|
|
182
|
+
if kept:
|
|
183
|
+
entry["facts"] = kept
|
|
184
|
+
|
|
185
|
+
path = log_path()
|
|
186
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
187
|
+
# One write of one line, so concurrent runs interleave whole lines and
|
|
188
|
+
# never characters: two agents working at once cannot corrupt a log
|
|
189
|
+
# neither of them is allowed to depend on anyway.
|
|
190
|
+
with path.open("a", encoding="utf-8") as handle:
|
|
191
|
+
handle.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
|
192
|
+
_trim(path)
|
|
193
|
+
except Exception:
|
|
194
|
+
return
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _trim(path: Path) -> None:
|
|
198
|
+
try:
|
|
199
|
+
lines = path.read_text(encoding="utf-8").splitlines()
|
|
200
|
+
if len(lines) <= MAX_ENTRIES:
|
|
201
|
+
return
|
|
202
|
+
path.write_text("\n".join(lines[-MAX_ENTRIES:]) + "\n", encoding="utf-8")
|
|
203
|
+
except Exception:
|
|
204
|
+
return
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def read_entries(path: Path | None = None) -> list[dict[str, Any]]:
|
|
208
|
+
"""Every readable entry, oldest first. An unreadable line is skipped."""
|
|
209
|
+
target = path or log_path()
|
|
210
|
+
try:
|
|
211
|
+
if not target.is_file():
|
|
212
|
+
return []
|
|
213
|
+
text = target.read_text(encoding="utf-8")
|
|
214
|
+
except OSError:
|
|
215
|
+
return []
|
|
216
|
+
entries: list[dict[str, Any]] = []
|
|
217
|
+
for line in text.splitlines():
|
|
218
|
+
line = line.strip()
|
|
219
|
+
if not line:
|
|
220
|
+
continue
|
|
221
|
+
try:
|
|
222
|
+
parsed = json.loads(line)
|
|
223
|
+
except json.JSONDecodeError:
|
|
224
|
+
continue
|
|
225
|
+
if isinstance(parsed, dict):
|
|
226
|
+
entries.append(parsed)
|
|
227
|
+
return entries
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _names_match(entry: dict[str, Any], wanted: Path) -> bool:
|
|
231
|
+
target = normalise(wanted)
|
|
232
|
+
for key in ("artifact", "source"):
|
|
233
|
+
recorded = entry.get(key)
|
|
234
|
+
if not recorded:
|
|
235
|
+
continue
|
|
236
|
+
recorded = str(recorded)
|
|
237
|
+
if normalise(recorded) == target or Path(recorded).name == wanted.name:
|
|
238
|
+
return True
|
|
239
|
+
return False
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def history_for(artifact: Path | str, path: Path | None = None) -> list[dict[str, Any]]:
|
|
243
|
+
"""The steps touching one lab, oldest first."""
|
|
244
|
+
wanted = Path(artifact)
|
|
245
|
+
return [entry for entry in read_entries(path) if _names_match(entry, wanted)]
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def latest_for(artifact: Path | str, path: Path | None = None) -> dict[str, Any] | None:
|
|
249
|
+
steps = history_for(artifact, path)
|
|
250
|
+
return steps[-1] if steps else None
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def artifacts_seen(path: Path | None = None) -> list[str]:
|
|
254
|
+
"""Every lab the log mentions, most recently touched first."""
|
|
255
|
+
order: list[str] = []
|
|
256
|
+
for entry in read_entries(path):
|
|
257
|
+
for key in ("artifact", "source"):
|
|
258
|
+
name = entry.get(key)
|
|
259
|
+
if not name:
|
|
260
|
+
continue
|
|
261
|
+
name = str(name)
|
|
262
|
+
if name in order:
|
|
263
|
+
order.remove(name)
|
|
264
|
+
order.append(name)
|
|
265
|
+
return list(reversed(order))
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def resume_report(artifact: Path | str, path: Path | None = None) -> dict[str, Any]:
|
|
269
|
+
"""Where this lab was left, and whether the log still describes it.
|
|
270
|
+
|
|
271
|
+
The check is the point. A log that simply asserted a position would be the
|
|
272
|
+
defect this repository keeps finding: a fact derived in one place and
|
|
273
|
+
believed in another, with nothing comparing them. So the lab is re-hashed
|
|
274
|
+
and a position is claimed only when the two agree. Otherwise the mismatch
|
|
275
|
+
is the answer -- which is the same refusal-first stance the rest of the
|
|
276
|
+
skill takes when it cannot prove something.
|
|
277
|
+
"""
|
|
278
|
+
wanted = Path(artifact)
|
|
279
|
+
steps = history_for(wanted, path)
|
|
280
|
+
if not steps:
|
|
281
|
+
return {
|
|
282
|
+
"artifact": str(wanted),
|
|
283
|
+
"known": False,
|
|
284
|
+
"matches_log": False,
|
|
285
|
+
"steps": 0,
|
|
286
|
+
"summary": "no recorded steps for this lab",
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
last = steps[-1]
|
|
290
|
+
recorded = str(last.get("artifact_sha256") or last.get("source_sha256") or "")
|
|
291
|
+
on_disk = artifact_digest(wanted)
|
|
292
|
+
exists = wanted.is_file()
|
|
293
|
+
matches = bool(on_disk) and bool(recorded) and on_disk == recorded
|
|
294
|
+
|
|
295
|
+
if not exists:
|
|
296
|
+
summary = "the lab the log describes is not on disk any more"
|
|
297
|
+
elif not recorded:
|
|
298
|
+
summary = "the last step recorded no digest, so the position cannot be confirmed"
|
|
299
|
+
elif matches:
|
|
300
|
+
summary = f"last step was `{last.get('command')}` and the lab still matches it"
|
|
301
|
+
else:
|
|
302
|
+
summary = "the lab has changed since the last recorded step; position not claimed"
|
|
303
|
+
|
|
304
|
+
report: dict[str, Any] = {
|
|
305
|
+
"artifact": str(wanted),
|
|
306
|
+
"known": True,
|
|
307
|
+
"exists": exists,
|
|
308
|
+
"matches_log": matches,
|
|
309
|
+
"steps": len(steps),
|
|
310
|
+
"last_command": last.get("command"),
|
|
311
|
+
"last_status": last.get("status"),
|
|
312
|
+
"at": last.get("at"),
|
|
313
|
+
"summary": summary,
|
|
314
|
+
"history": [
|
|
315
|
+
{"at": step.get("at"), "command": step.get("command"), "status": step.get("status")}
|
|
316
|
+
for step in steps[-10:]
|
|
317
|
+
],
|
|
318
|
+
}
|
|
319
|
+
facts = last.get("facts") or {}
|
|
320
|
+
if isinstance(facts, dict):
|
|
321
|
+
if facts.get("next_best_action"):
|
|
322
|
+
report["next_best_action"] = facts["next_best_action"]
|
|
323
|
+
if facts:
|
|
324
|
+
report["last_facts"] = facts
|
|
325
|
+
return report
|
package/scripts/usage_ledger.py
CHANGED
|
@@ -1,218 +1,230 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""A local, append-only record of what actually worked, so the skill improves with use.
|
|
3
|
-
|
|
4
|
-
Donor selection is expensive and mostly repetitive: the same scenario families
|
|
5
|
-
come back, and the same handful of donors keep winning or keep failing for the
|
|
6
|
-
same reasons. Rediscovering that on every run wastes several seconds per
|
|
7
|
-
candidate and, worse, throws away the only real evidence the skill ever gets
|
|
8
|
-
about which donors survive a Packet Tracer open.
|
|
9
|
-
|
|
10
|
-
The ledger records the outcome of each generation and feeds it back into donor
|
|
11
|
-
ranking on the next run. Nothing is inferred or guessed: an entry is written
|
|
12
|
-
only after a real attempt produced a real result.
|
|
13
|
-
|
|
14
|
-
Privacy and safety rules, deliberately strict:
|
|
15
|
-
|
|
16
|
-
- the ledger is local only. It is written under `output/`, which is gitignored,
|
|
17
|
-
and it is never committed, packaged, or transmitted anywhere.
|
|
18
|
-
- prompts are stored as a normalised fingerprint, not verbatim text, so a lab
|
|
19
|
-
description containing names or addresses does not end up on disk.
|
|
20
|
-
- the file is bounded. Old entries are dropped once the cap is reached.
|
|
21
|
-
- a corrupt or unreadable ledger is ignored, never fatal. Learning is an
|
|
22
|
-
optimisation; the skill must work identically with the ledger deleted.
|
|
23
|
-
"""
|
|
24
|
-
|
|
25
|
-
from __future__ import annotations
|
|
26
|
-
|
|
27
|
-
import hashlib
|
|
28
|
-
import json
|
|
29
|
-
import os
|
|
30
|
-
import re
|
|
31
|
-
from collections import defaultdict
|
|
32
|
-
from dataclasses import dataclass, field
|
|
33
|
-
from datetime import datetime, timezone
|
|
34
|
-
from pathlib import Path
|
|
35
|
-
|
|
36
|
-
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
|
37
|
-
DEFAULT_LEDGER_PATH = SKILL_ROOT / "output" / "usage-ledger.jsonl"
|
|
38
|
-
MAX_ENTRIES = 2000
|
|
39
|
-
LEDGER_VERSION = 1
|
|
40
|
-
|
|
41
|
-
OUTCOME_GENERATED_VERIFIED = "generated_verified"
|
|
42
|
-
OUTCOME_GENERATED_UNVERIFIED = "generated_unverified"
|
|
43
|
-
OUTCOME_REFUSED = "refused"
|
|
44
|
-
OUTCOMES = (OUTCOME_GENERATED_VERIFIED, OUTCOME_GENERATED_UNVERIFIED, OUTCOME_REFUSED)
|
|
45
|
-
|
|
46
|
-
# Outcomes that count as evidence a donor works, best first.
|
|
47
|
-
_SUCCESS_WEIGHT = {
|
|
48
|
-
OUTCOME_GENERATED_VERIFIED: 3,
|
|
49
|
-
OUTCOME_GENERATED_UNVERIFIED: 1,
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
def
|
|
64
|
-
""
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
try:
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
return
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
try:
|
|
142
|
-
|
|
143
|
-
except OSError:
|
|
144
|
-
return
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
)
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""A local, append-only record of what actually worked, so the skill improves with use.
|
|
3
|
+
|
|
4
|
+
Donor selection is expensive and mostly repetitive: the same scenario families
|
|
5
|
+
come back, and the same handful of donors keep winning or keep failing for the
|
|
6
|
+
same reasons. Rediscovering that on every run wastes several seconds per
|
|
7
|
+
candidate and, worse, throws away the only real evidence the skill ever gets
|
|
8
|
+
about which donors survive a Packet Tracer open.
|
|
9
|
+
|
|
10
|
+
The ledger records the outcome of each generation and feeds it back into donor
|
|
11
|
+
ranking on the next run. Nothing is inferred or guessed: an entry is written
|
|
12
|
+
only after a real attempt produced a real result.
|
|
13
|
+
|
|
14
|
+
Privacy and safety rules, deliberately strict:
|
|
15
|
+
|
|
16
|
+
- the ledger is local only. It is written under `output/`, which is gitignored,
|
|
17
|
+
and it is never committed, packaged, or transmitted anywhere.
|
|
18
|
+
- prompts are stored as a normalised fingerprint, not verbatim text, so a lab
|
|
19
|
+
description containing names or addresses does not end up on disk.
|
|
20
|
+
- the file is bounded. Old entries are dropped once the cap is reached.
|
|
21
|
+
- a corrupt or unreadable ledger is ignored, never fatal. Learning is an
|
|
22
|
+
optimisation; the skill must work identically with the ledger deleted.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import hashlib
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
import re
|
|
31
|
+
from collections import defaultdict
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
from datetime import datetime, timezone
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
SKILL_ROOT = Path(__file__).resolve().parents[1]
|
|
37
|
+
DEFAULT_LEDGER_PATH = SKILL_ROOT / "output" / "usage-ledger.jsonl"
|
|
38
|
+
MAX_ENTRIES = 2000
|
|
39
|
+
LEDGER_VERSION = 1
|
|
40
|
+
|
|
41
|
+
OUTCOME_GENERATED_VERIFIED = "generated_verified"
|
|
42
|
+
OUTCOME_GENERATED_UNVERIFIED = "generated_unverified"
|
|
43
|
+
OUTCOME_REFUSED = "refused"
|
|
44
|
+
OUTCOMES = (OUTCOME_GENERATED_VERIFIED, OUTCOME_GENERATED_UNVERIFIED, OUTCOME_REFUSED)
|
|
45
|
+
|
|
46
|
+
# Outcomes that count as evidence a donor works, best first.
|
|
47
|
+
_SUCCESS_WEIGHT = {
|
|
48
|
+
OUTCOME_GENERATED_VERIFIED: 3,
|
|
49
|
+
OUTCOME_GENERATED_UNVERIFIED: 1,
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# One variable carries two meanings -- a switch and a path -- and only one of
|
|
54
|
+
# them was reading the switch words. `PKT_USAGE_LEDGER=on`, the obvious way to
|
|
55
|
+
# turn learning on, made `ledger_path` treat "on" as a filename and write the
|
|
56
|
+
# ledger to a file called `on` in the working directory. Found by doing exactly
|
|
57
|
+
# that while testing something else, which left an untracked `on` in the
|
|
58
|
+
# repository root. Both readers share the vocabulary now.
|
|
59
|
+
_OFF_WORDS = {"off", "0", "false", "none"}
|
|
60
|
+
_ON_WORDS = {"on", "1", "true"}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def ledger_path() -> Path:
|
|
64
|
+
override = (os.getenv("PKT_USAGE_LEDGER") or "").strip()
|
|
65
|
+
if override and override.lower() not in (_OFF_WORDS | _ON_WORDS):
|
|
66
|
+
return Path(override).expanduser()
|
|
67
|
+
return DEFAULT_LEDGER_PATH
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def ledger_enabled() -> bool:
|
|
71
|
+
"""Learning is on by default; `PKT_USAGE_LEDGER=off` disables it entirely."""
|
|
72
|
+
return (os.getenv("PKT_USAGE_LEDGER") or "").strip().lower() not in _OFF_WORDS
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def prompt_fingerprint(prompt: str) -> str:
|
|
76
|
+
"""A stable, non-reversible fingerprint of a prompt's *shape*.
|
|
77
|
+
|
|
78
|
+
Digits are collapsed to `#` and case and spacing are normalised, so
|
|
79
|
+
"3 switch 6 pc" and "5 switch 2 pc" share a fingerprint: they are the same
|
|
80
|
+
kind of request, which is exactly the granularity donor reuse needs. The
|
|
81
|
+
result is hashed so no prompt text is ever written to disk.
|
|
82
|
+
"""
|
|
83
|
+
normalised = re.sub(r"\d+", "#", (prompt or "").strip().lower())
|
|
84
|
+
normalised = re.sub(r"[^\w#]+", " ", normalised).strip()
|
|
85
|
+
return hashlib.sha256(normalised.encode("utf-8")).hexdigest()[:16]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass
|
|
89
|
+
class LedgerEntry:
|
|
90
|
+
scenario_family: str
|
|
91
|
+
donor: str
|
|
92
|
+
outcome: str
|
|
93
|
+
prompt_shape: str = ""
|
|
94
|
+
target_version: str = ""
|
|
95
|
+
donor_version: str = ""
|
|
96
|
+
rejected_donors: list[str] = field(default_factory=list)
|
|
97
|
+
rejection_codes: list[str] = field(default_factory=list)
|
|
98
|
+
recorded_at: str = ""
|
|
99
|
+
|
|
100
|
+
def to_json(self) -> dict[str, object]:
|
|
101
|
+
return {
|
|
102
|
+
"v": LEDGER_VERSION,
|
|
103
|
+
"recorded_at": self.recorded_at or datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
104
|
+
"scenario_family": self.scenario_family,
|
|
105
|
+
"prompt_shape": self.prompt_shape,
|
|
106
|
+
"donor": self.donor,
|
|
107
|
+
"donor_version": self.donor_version,
|
|
108
|
+
"target_version": self.target_version,
|
|
109
|
+
"outcome": self.outcome,
|
|
110
|
+
"rejected_donors": self.rejected_donors[:20],
|
|
111
|
+
"rejection_codes": self.rejection_codes[:20],
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def record(entry: LedgerEntry, path: Path | None = None) -> bool:
|
|
116
|
+
"""Append one outcome. Returns False if the ledger is off or unwritable."""
|
|
117
|
+
if not ledger_enabled():
|
|
118
|
+
return False
|
|
119
|
+
if entry.outcome not in OUTCOMES:
|
|
120
|
+
raise ValueError(f"unknown outcome: {entry.outcome}")
|
|
121
|
+
|
|
122
|
+
target = path or ledger_path()
|
|
123
|
+
try:
|
|
124
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
125
|
+
with target.open("a", encoding="utf-8") as handle:
|
|
126
|
+
handle.write(json.dumps(entry.to_json(), ensure_ascii=False) + "\n")
|
|
127
|
+
except OSError:
|
|
128
|
+
return False
|
|
129
|
+
|
|
130
|
+
_trim(target)
|
|
131
|
+
return True
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _trim(path: Path) -> None:
|
|
135
|
+
try:
|
|
136
|
+
lines = path.read_text(encoding="utf-8").splitlines()
|
|
137
|
+
except OSError:
|
|
138
|
+
return
|
|
139
|
+
if len(lines) <= MAX_ENTRIES:
|
|
140
|
+
return
|
|
141
|
+
try:
|
|
142
|
+
path.write_text("\n".join(lines[-MAX_ENTRIES:]) + "\n", encoding="utf-8")
|
|
143
|
+
except OSError:
|
|
144
|
+
return
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def load_entries(path: Path | None = None) -> list[dict[str, object]]:
|
|
148
|
+
"""Read the ledger. A damaged file yields whatever lines still parse."""
|
|
149
|
+
target = path or ledger_path()
|
|
150
|
+
if not ledger_enabled() or not target.exists():
|
|
151
|
+
return []
|
|
152
|
+
entries: list[dict[str, object]] = []
|
|
153
|
+
try:
|
|
154
|
+
raw_lines = target.read_text(encoding="utf-8").splitlines()
|
|
155
|
+
except OSError:
|
|
156
|
+
return []
|
|
157
|
+
for line in raw_lines:
|
|
158
|
+
line = line.strip()
|
|
159
|
+
if not line:
|
|
160
|
+
continue
|
|
161
|
+
try:
|
|
162
|
+
parsed = json.loads(line)
|
|
163
|
+
except json.JSONDecodeError:
|
|
164
|
+
continue
|
|
165
|
+
if isinstance(parsed, dict) and parsed.get("donor"):
|
|
166
|
+
entries.append(parsed)
|
|
167
|
+
return entries
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def donor_scores(
|
|
171
|
+
scenario_family: str,
|
|
172
|
+
prompt_shape: str = "",
|
|
173
|
+
path: Path | None = None,
|
|
174
|
+
) -> dict[str, int]:
|
|
175
|
+
"""Learned preference per donor, as `relative_path -> score`.
|
|
176
|
+
|
|
177
|
+
Positive means the donor has produced output for this kind of request
|
|
178
|
+
before; negative means it has been rejected. An exact prompt-shape match
|
|
179
|
+
counts double, because it is stronger evidence than family alone.
|
|
180
|
+
"""
|
|
181
|
+
scores: dict[str, int] = defaultdict(int)
|
|
182
|
+
for entry in load_entries(path):
|
|
183
|
+
if str(entry.get("scenario_family") or "") != scenario_family:
|
|
184
|
+
continue
|
|
185
|
+
multiplier = 2 if prompt_shape and entry.get("prompt_shape") == prompt_shape else 1
|
|
186
|
+
|
|
187
|
+
donor = str(entry.get("donor") or "")
|
|
188
|
+
weight = _SUCCESS_WEIGHT.get(str(entry.get("outcome") or ""), 0)
|
|
189
|
+
if donor and weight:
|
|
190
|
+
scores[donor] += weight * multiplier
|
|
191
|
+
|
|
192
|
+
for rejected in entry.get("rejected_donors") or []:
|
|
193
|
+
name = str(rejected)
|
|
194
|
+
if name:
|
|
195
|
+
scores[name] -= multiplier
|
|
196
|
+
return dict(scores)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def summary(path: Path | None = None) -> dict[str, object]:
|
|
200
|
+
"""Human-facing view of what the skill has learned so far."""
|
|
201
|
+
entries = load_entries(path)
|
|
202
|
+
by_outcome: dict[str, int] = defaultdict(int)
|
|
203
|
+
by_family: dict[str, int] = defaultdict(int)
|
|
204
|
+
proven: dict[str, int] = defaultdict(int)
|
|
205
|
+
for entry in entries:
|
|
206
|
+
outcome = str(entry.get("outcome") or "")
|
|
207
|
+
by_outcome[outcome] += 1
|
|
208
|
+
by_family[str(entry.get("scenario_family") or "unknown")] += 1
|
|
209
|
+
if outcome in _SUCCESS_WEIGHT:
|
|
210
|
+
proven[str(entry.get("donor") or "")] += _SUCCESS_WEIGHT[outcome]
|
|
211
|
+
return {
|
|
212
|
+
"ledger_path": str(path or ledger_path()),
|
|
213
|
+
"enabled": ledger_enabled(),
|
|
214
|
+
"entry_count": len(entries),
|
|
215
|
+
"outcomes": dict(by_outcome),
|
|
216
|
+
"scenario_families": dict(by_family),
|
|
217
|
+
"proven_donors": sorted(
|
|
218
|
+
({"donor": donor, "score": score} for donor, score in proven.items() if score > 0),
|
|
219
|
+
key=lambda item: (-int(item["score"]), str(item["donor"])),
|
|
220
|
+
)[:10],
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def main() -> int:
|
|
225
|
+
print(json.dumps(summary(), ensure_ascii=False, indent=2))
|
|
226
|
+
return 0
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
if __name__ == "__main__":
|
|
230
|
+
raise SystemExit(main())
|