cyber-aced 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/.claude-plugin/plugin.json +18 -0
  2. package/.plugin/pins.json +1 -0
  3. package/.plugin/plugin.json +18 -0
  4. package/agents/aced-case-judge.md +165 -0
  5. package/agents/aced-impl-judge.md +55 -0
  6. package/agents/aced-scenario-writer.md +76 -0
  7. package/agents/aced-spec-validator.md +66 -0
  8. package/governances/README.md +52 -0
  9. package/governances/agent-tool-output.md +86 -0
  10. package/governances/cli-resolution.md +102 -0
  11. package/governances/skill-design.md +258 -0
  12. package/governances/skill-repo-structure.md +173 -0
  13. package/package.json +44 -0
  14. package/readme.md +124 -0
  15. package/skills/aced-architect-impl/README.md +9 -0
  16. package/skills/aced-architect-impl/SKILL.md +73 -0
  17. package/skills/aced-builder-impl/README.md +14 -0
  18. package/skills/aced-builder-impl/SKILL.md +46 -0
  19. package/skills/aced-builder-spec/README.md +15 -0
  20. package/skills/aced-builder-spec/SKILL.md +130 -0
  21. package/skills/aced-fit/README.md +21 -0
  22. package/skills/aced-fit/SKILL.md +105 -0
  23. package/skills/aced-impl-producer/SKILL.md +75 -0
  24. package/skills/add-scenario/README.md +25 -0
  25. package/skills/add-scenario/SKILL.md +75 -0
  26. package/skills/bench/README.md +64 -0
  27. package/skills/bench/SKILL.md +199 -0
  28. package/skills/bench/scripts/bench-claude-code.mts +187 -0
  29. package/skills/bench/scripts/bench-compare.mts +369 -0
  30. package/skills/bench/scripts/bench-record.mts +133 -0
  31. package/skills/bench/scripts/bench.mts +930 -0
  32. package/skills/check-freshness/README.md +57 -0
  33. package/skills/check-freshness/SKILL.md +76 -0
  34. package/skills/check-freshness/scripts/check-freshness.mts +399 -0
  35. package/skills/compare/README.md +25 -0
  36. package/skills/compare/SKILL.md +87 -0
  37. package/skills/contribute-skill/README.md +25 -0
  38. package/skills/contribute-skill/SKILL.md +241 -0
  39. package/skills/contribute-skill/scripts/source.mts +188 -0
  40. package/skills/define-agent/README.md +40 -0
  41. package/skills/define-agent/SKILL.md +279 -0
  42. package/skills/define-command/README.md +36 -0
  43. package/skills/define-command/SKILL.md +174 -0
  44. package/skills/define-governance/SKILL.md +151 -0
  45. package/skills/define-skill/README.md +37 -0
  46. package/skills/define-skill/SKILL.md +200 -0
  47. package/skills/extract-situation/README.md +32 -0
  48. package/skills/extract-situation/SKILL.md +54 -0
  49. package/skills/extract-situation/scripts/extract-situation.mts +287 -0
  50. package/skills/improve/README.md +25 -0
  51. package/skills/improve/SKILL.md +100 -0
  52. package/skills/improve-skill/README.md +38 -0
  53. package/skills/improve-skill/SKILL.md +169 -0
  54. package/skills/improve-skill/references/check-definitions.md +215 -0
  55. package/skills/improve-skill/scripts/validate.mts +1033 -0
  56. package/skills/init-aced/README.md +5 -0
  57. package/skills/init-aced/SKILL.md +76 -0
  58. package/skills/init-aced/scripts/ensure-results-ignored.mts +117 -0
  59. package/skills/list-skills/README.md +19 -0
  60. package/skills/list-skills/SKILL.md +55 -0
  61. package/skills/list-skills/scripts/list-skills.mts +210 -0
  62. package/skills/manage/README.md +35 -0
  63. package/skills/manage/SKILL.md +65 -0
  64. package/skills/manage-model-runners/README.md +25 -0
  65. package/skills/manage-model-runners/SKILL.md +113 -0
  66. package/skills/manage-skill-dirs/README.md +17 -0
  67. package/skills/manage-skill-dirs/SKILL.md +54 -0
  68. package/skills/manage-skill-dirs/scripts/manage-skill-dirs.mts +328 -0
  69. package/skills/repair-private-skills/README.md +17 -0
  70. package/skills/repair-private-skills/SKILL.md +59 -0
  71. package/skills/repair-private-skills/scripts/repair-private-skills.mts +279 -0
  72. package/skills/report/README.md +25 -0
  73. package/skills/report/SKILL.md +74 -0
  74. package/skills/run/README.md +25 -0
  75. package/skills/run/SKILL.md +220 -0
  76. package/skills/skillify/README.md +25 -0
  77. package/skills/skillify/SKILL.md +141 -0
@@ -0,0 +1,25 @@
1
+ # add-scenario
2
+
3
+ Add a new scenario to an existing ACED `.feature` suite — from a real failure, a production edge case, or a gap you noticed.
4
+
5
+ ## When to use
6
+
7
+ Use this skill when you already have a suite (`eval.md` + a frozen `.feature`) and want to grow its coverage with one more scenario.
8
+
9
+ Good triggers include:
10
+
11
+ - "Add a test case for this failure"
12
+ - "We need to cover this edge case"
13
+ - Turning a pasted agent transcript that went wrong into a guarded-against scenario
14
+
15
+ Defer to a sibling when the request is about scoring the existing set (`run`), diagnosing why cases fail (`improve`), or there's no suite yet at all (`define-skill` / `define-agent` / `define-governance`, then `sdd:start-mission` to author one).
16
+
17
+ ## What it does
18
+
19
+ Locates the target suite, classifies the new scenario's layer (trigger / behavior / quality) from how you describe the failure, drafts a Gherkin scenario (boolean, or `@rubric` with an inline rubric, or a `@trigger` `Examples` row), confirms it with you, and appends it to the frozen `.feature` (additive — self-clears).
20
+
21
+ ## Install
22
+
23
+ ```bash
24
+ npx skills add cyberuni/cyberplace --skill aced/add-scenario
25
+ ```
@@ -0,0 +1,75 @@
1
+ ---
2
+ name: add-scenario
3
+ description: Use this skill when adding a new scenario to an ACED .feature suite — from a real failure, a production edge case, or a gap the user noticed.
4
+ ---
5
+
6
+ # ACED Add Scenario
7
+
8
+ Add one or more scenarios to an existing frozen `.feature` suite. Adding a scenario is **additive** —
9
+ it widens the contract and cannot break existing impl, so it **self-clears** and the `.feature` stays
10
+ `@frozen` (per `sdd:suite-format-governance`); no re-open is needed.
11
+
12
+ ## Locate the suite
13
+
14
+ Find the target's node in the project spec — `.agents/specs/<project>/…/<node>/` (discovered through
15
+ the SDD spec tree; the node's `eval.md` names the subject) — from user context or ask. Read the
16
+ node's colocated `eval.md` for the `subject` and the `eval:` run policy (`eval.judge.default_threshold`).
17
+
18
+ ## Gather input
19
+
20
+ The user may provide:
21
+ - A description of a scenario that failed in production
22
+ - A pasted agent transcript showing incorrect behavior
23
+ - A description of an edge case not yet covered
24
+ - A "must not do" behavior they want to guard against
25
+
26
+ If a transcript is provided, extract: what the user said, what state the system was in, what the agent did, what it should have done instead.
27
+
28
+ ## Determine the layer
29
+
30
+ | Input type | Layer |
31
+ |---|---|
32
+ | "Agent invoked the skill when it shouldn't have" | trigger |
33
+ | "Agent didn't invoke the skill when it should have" | trigger |
34
+ | "Agent invoked correctly but skipped a step" | behavior |
35
+ | "Agent did the step but produced poor output" | quality |
36
+
37
+ Ask if unclear.
38
+
39
+ If the resolved layer is not enabled in the suite's `eval.md` `eval.layers`, warn that the layer is not enabled and the scenario will not be exercised until it is.
40
+
41
+ ## Scaffold the scenario
42
+
43
+ Draft a Gherkin scenario for the frozen `.feature`, tagged with its layer:
44
+
45
+ - **trigger** → add a row to the `@trigger` `Scenario Outline`'s `Examples` table
46
+ (`| <query> | <should_trigger> |`), or start one if the suite has none.
47
+ - **behavior (deterministic)** → a boolean scenario whose `Then` asserts the observable action (a
48
+ must-not-do guard is a `Then` asserting the agent *does not* do the prohibited action).
49
+ - **behavior / quality (graded)** → a `@rubric` scenario with the rubric **inline**:
50
+
51
+ ```gherkin
52
+ @behavior @rubric
53
+ Scenario: <name — the situation being guarded>
54
+ Given <concrete situation: user message, repo/file state, context>
55
+ When <the agent acts>
56
+ Then the judge evaluates the scenario against the rubric
57
+ """
58
+ dimensions:
59
+ - name: <criterion>
60
+ max: <n>
61
+ threshold: <from eval.judge.default_threshold>
62
+ """
63
+ And the rubric score is at least the threshold
64
+ ```
65
+
66
+ Show the draft to the user and ask for confirmation before writing. Adjust based on feedback.
67
+
68
+ ## Write the scenario
69
+
70
+ Append the scenario to the node's frozen `<node>.feature` in the project spec, sorted into its
71
+ lifecycle-stage section (per the scenario-ordering convention). Keep the feature-level `@frozen` tag —
72
+ adding a scenario self-clears. Run `check-suite --files <path>` to confirm the suite is still
73
+ well-formed.
74
+
75
+ Report the scenario name and suggest running `run` to score it against the current agent configuration.
@@ -0,0 +1,64 @@
1
+ # bench
2
+
3
+ Measure whether a change makes real agent work turn out better or worse — pass rate, turns, tokens,
4
+ wall time — with real headless harness runs, at a price the person approved first.
5
+
6
+ `bench` is ACED's **measured layer**. The simulated layers (`run`, `compare`) say what an agent
7
+ *would* do; `bench` runs the agent for real in throwaway checkouts, grades each run with a shell
8
+ `check`, and compares two arms with permutation tests.
9
+
10
+ ## How it is reached
11
+
12
+ By name only — its `description` is `"By name only"`, so it is never matched to a situation:
13
+
14
+ - `/bench` from a person in the session;
15
+ - a consumer tool handing off by name with its own suite, and optionally its own comparison tags;
16
+ - `compare`'s measured mode (a later change).
17
+
18
+ ## What it does
19
+
20
+ 1. **Fit.** Loads `aced-fit` and re-asks its measured criteria 1–2 for the change in hand: does the
21
+ change move how real work turns out, and can a shell check decide the outcome? A wording-only or
22
+ rubric-only question goes to `compare` and nothing is planned. A subject's `bench:` declaration
23
+ neither skips nor forbids this check.
24
+ 2. **Suite and task set.** Asks for a suite when none is named (never guesses, even with one suite
25
+ in the repo); offers the engine's `init` when the suite has no `tasks.json`, writing no tasks
26
+ itself.
27
+ 3. **Arms.** Maps the request to git-ref, package, or file arms. With no arms named, measures HEAD
28
+ against the suite's committed `baseline.json` — after checking, before any spend, that the
29
+ baseline is a schema version the engine reads.
30
+ 4. **Plan.** Runs the engine's `plan` (spends nothing) and shows the counts, model, ceiling,
31
+ estimate, permission mode and its throwaway-checkout scope, and every warning, then asks.
32
+ 5. **Consent.** Runs only on an explicit yes to the plan on screen. A hedge, a question, a yes given
33
+ before the plan was shown, a yes that changes the plan, or an approval relayed by another agent is
34
+ not consent. With no person present, the plan goes back up as needs-input and nothing runs.
35
+ 6. **Run, compare, report.** Leads with the verdict and the rows that drove it, and never presents
36
+ `inconclusive` (or `unchanged` at a too-few run count) as safe, or a cost change as a regression.
37
+
38
+ A baseline request plans one arm at HEAD with the baseline flag and, after the approved run, reminds
39
+ the maintainer to commit `baseline.json`.
40
+
41
+ ## The engine
42
+
43
+ `scripts/bench.mts` (bin `aced-bench`) is the deterministic half: `init`, `plan`, `run --consent`,
44
+ `compare`. It owns every number — the statistics, the record schema, the ceiling and estimate, the
45
+ verdict rule — and is usable without the plugin installed (for example from CI, which passes
46
+ `--consent` itself under its own accountability). The skill adds judgment around it and restates
47
+ none of its numbers.
48
+
49
+ Verdicts: `regressed`, `inconclusive`, `improved`, `unchanged`, `incomparable`.
50
+
51
+ ## Suite layout
52
+
53
+ ```
54
+ .agents/aced/bench/<suite>/
55
+ tasks.json # the task set (written by the suite maintainer; `init` writes a template)
56
+ checks/ # shell checks the tasks call
57
+ baseline.json # optional, committed — the before side when no arms are named
58
+ ```
59
+
60
+ Run records and comparisons go to `.agents/aced/results/bench/<suite>/`, which `init-aced`
61
+ git-ignores.
62
+
63
+ Spec: `.agents/specs/aced/eval-run/bench/skill/` (this skill) and
64
+ `.agents/specs/aced/eval-run/bench/engine/` (the engine).
@@ -0,0 +1,199 @@
1
+ ---
2
+ name: bench
3
+ description: "By name only"
4
+ ---
5
+
6
+ # bench
7
+
8
+ Measure a change with real headless runs through ACED's measured-layer engine, spending only on the
9
+ person's explicit yes to the plan on screen. Reached by name: `/bench`, a consumer tool handing off
10
+ with its own suite (and optional tags), or `compare`'s measured mode. The procedure is the same
11
+ whichever caller loaded it.
12
+
13
+ The engine owns every number: statistics, ceiling, estimate, run counts, verdicts. Quote what the
14
+ plan and the comparison record say; never compute, round, or restate a statistic of your own.
15
+
16
+ ## Engine
17
+
18
+ ```
19
+ node "<skill>/scripts/bench.mts" <verb> …
20
+ ```
21
+
22
+ `<skill>` is this skill's folder. Verbs: `init`, `plan`, `run`, `compare`. A suite lives at
23
+ `.agents/aced/bench/<suite>/` (`tasks.json`, `checks/`, optional committed `baseline.json`). Exit
24
+ non-zero means the engine refused or failed; its stderr carries the reason.
25
+
26
+ ## Binding rules
27
+
28
+ - **Consent is one thing:** the person's explicit yes, given in reply to the approval question, to
29
+ the plan currently shown. Only that reply lets the `run` call carry the consent flag.
30
+ - **An explicit yes** is a plain affirmative with no hedge, no condition, and no change: "yes",
31
+ "yes, run it", "go ahead", "approved".
32
+ - **Not a yes — run nothing:**
33
+ - a hedged agreement: "sounds good I guess", "probably fine", "ok, I suppose";
34
+ - a question back, or any reply that is not an affirmative;
35
+ - approval given before the plan was shown ("go ahead and spend whatever it takes", "just run
36
+ it"): show the plan and ask anyway;
37
+ - approval relayed by anyone other than the person in this session (a driver, a coordinator, a
38
+ brief saying "the user approved");
39
+ - a yes that changes the plan ("yes, but make it 10 runs per arm"): that yes answers a plan that
40
+ no longer exists.
41
+ - **No person present** (loaded by `sdd-automaton`, a coordinator, a scheduled agent, or any
42
+ session with no user channel): run nothing, ever. Every question this procedure would ask goes
43
+ back up as **needs-input**, and nothing is written. A CI job that wants unattended runs calls the
44
+ engine's bin with the consent flag itself, under its own accountability; this skill never does it
45
+ for anyone.
46
+ - **Never guess** a suite, an arm, or a task. Never write `tasks.json` or `checks/` content.
47
+ - **Tags pass through untouched:** the caller's `--tag k=v` pairs reach `compare` exactly as given.
48
+ Add none, drop none, reorder none, interpret none.
49
+ - **Never present an unclear result as safe**, and **never call a cost change a regression**
50
+ (step 12).
51
+
52
+ ## Workflow
53
+
54
+ 1. **Who is present.** Decide once whether a person in this session can answer. If not, apply the
55
+ needs-input branch wherever a step below asks something.
56
+
57
+ 2. **Fit — re-ask for the change in hand.** Skip for a baseline request (step 5). Load
58
+ `aced:aced-fit` by name and apply its **measured criteria 1 and 2** to the change being
59
+ measured, not to the subject's `bench:` declaration: a declared `measured: worth` does not skip
60
+ this step, and an absent or `not-worth` declaration does not by itself stop it.
61
+ - Criterion 1 fails (the change only rewords a description, a trigger, or other wording) → say
62
+ the simulated layer answers a wording change and name `compare` (ACED's simulated diff).
63
+ - Criterion 2 fails (no shell `check` can decide the outcome, for example "do replies read
64
+ warmer") → say no shell check can decide this outcome, point to the simulated layer, and name
65
+ `compare`.
66
+ - Either failure: plan nothing, return no plan. Same answer with no person present.
67
+
68
+ 3. **Suite.** Use the suite the request or the calling tool names. None named → ask which suite to
69
+ use and plan nothing. You may list the suites found under `.agents/aced/bench/`, but never pick
70
+ one, even when only one exists. No person present → return that question as needs-input.
71
+
72
+ 4. **Task set.** `.agents/aced/bench/<suite>/tasks.json` missing → say the suite has no task set,
73
+ offer to run the engine's `init` for it (a template the suite maintainer then fills in), and plan
74
+ nothing. Write no task of your own. Run `init` only on the person's yes to that offer:
75
+
76
+ ```
77
+ node "<skill>/scripts/bench.mts" init --suite <suite>
78
+ ```
79
+
80
+ No person present → return the init offer as needs-input; run nothing, write nothing.
81
+
82
+ 5. **Arms.** Map the request to `--arm <label>=<subject>` pairs. "X against Y" makes Y `before` and
83
+ X `after`.
84
+
85
+ | The request names | Arms |
86
+ |---|---|
87
+ | two git refs (commits, branches, tags) | `--arm before=git:<Y> --arm after=git:<X>` |
88
+ | two versions of one package or plugin | `--arm before=package:<name>@<older> --arm after=package:<name>@<newer>` |
89
+ | with and without one file that HEAD carries | `--arm without=file:<path>=absent --arm with=file:<path>=ref:HEAD` — never a git-ref arm |
90
+ | one file swapped in from a ref or a path | `--arm before=file:<path>=ref:HEAD --arm after=file:<path>=ref:<ref>` (or `=path:<source>`) — both arms `file`, since the engine never compares two subject kinds |
91
+ | a request to record or refresh the baseline | `--arm baseline=git:HEAD` plus `--baseline` — one arm only |
92
+ | no arms | read `.agents/aced/bench/<suite>/baseline.json` first (below) |
93
+
94
+ **No arms named** — check the baseline now, before any plan or spend:
95
+ - `baseline.json` exists with `schemaVersion: 3` (the only version the engine reads) → one arm,
96
+ `--arm after=git:HEAD`; at step 11 compare it with `--before baseline`.
97
+ - no `baseline.json` → say the suite has no baseline, ask which two arms to compare (or offer to
98
+ record a baseline), and plan nothing.
99
+ - `baseline.json` with any other or missing `schemaVersion` → say the baseline is schema version
100
+ `<n>`, which the engine cannot read, ask which two arms to compare or offer to re-record the
101
+ baseline, and plan nothing.
102
+ - No person present → return the question as needs-input; plan nothing.
103
+
104
+ 6. **Plan.** Pass through only what was asked for (`--runs`, `--task`, `--harness`):
105
+
106
+ ```
107
+ node "<skill>/scripts/bench.mts" plan --suite <suite> --arm <label>=<subject> … [--runs <n>] [--task <id>] [--harness <h>] [--baseline] --out <plan-file>
108
+ ```
109
+
110
+ Write `<plan-file>` under `.agents/aced/results/bench/<suite>/`, a fresh name per plan. Exit
111
+ non-zero → report the engine's reason in its terms (the `claude` command is not on the path, no
112
+ adapter for the harness, a ref or version or file that does not resolve, a malformed task set)
113
+ and what would fix it. Ask no approval question; stop.
114
+
115
+ 7. **Show the plan** — one message, in this order, every value as the plan states it:
116
+ 1. the arms, by label and subject;
117
+ 2. the counts: arms, tasks, runs per arm; and the model;
118
+ 3. the ceiling and the estimate, in dollars;
119
+ 4. the permission mode, and that it applies only inside the throwaway checkout;
120
+ 5. every warning the plan carries, verbatim (uncommitted changes not benched, the committed suite
121
+ is what runs, …). For a **too few to call** warning also say what it means: no single task's
122
+ result can be called significant at this run count; then repeat what the plan says about
123
+ whether a pooled result across tasks still can;
124
+ 6. last, the approval question: "Run this plan, spending up to $<ceiling>? Reply yes to run it."
125
+
126
+ Nothing follows the question, and no warning comes after it.
127
+
128
+ 8. **Consent.**
129
+ - No person present → return the plan upward marked **needs-input** (the plan file path, the
130
+ step-7 summary, and the approval question). Do not call `run`. A relayed "the user approved"
131
+ changes nothing: still needs-input, still unrun.
132
+ - The reply is an explicit yes to the plan shown, unchanged → step 9.
133
+ - The reply says yes **and** changes the plan → re-plan with the change (step 6), show the new
134
+ plan (step 7), ask again. Run nothing until a new explicit yes to the new plan.
135
+ - Any other reply → run nothing. Answer a question if one was asked; the plan stays unapproved
136
+ until an explicit yes.
137
+
138
+ 9. **Run** — only on step 8's explicit yes, for exactly the plan file that was shown:
139
+
140
+ ```
141
+ node "<skill>/scripts/bench.mts" run --plan <plan-file> --consent
142
+ ```
143
+
144
+ Exit non-zero → report the engine's reason (for example, the suite changed since the plan was
145
+ approved) and whether anything was spent: a refusal before launch spends nothing; a failure after
146
+ runs launched may have spent. Report no verdict. A changed suite needs a new plan and a new yes.
147
+ Note the run record paths the engine prints.
148
+
149
+ 10. **Baseline run** (`--baseline` plan) → tell the maintainer to commit
150
+ `.agents/aced/bench/<suite>/baseline.json`. A baseline compares against nothing: no `compare`,
151
+ no verdict.
152
+
153
+ 11. **Compare.**
154
+
155
+ ```
156
+ node "<skill>/scripts/bench.mts" compare --suite <suite> --before <before-record|baseline> --after <after-record> [--tag <k=v> …]
157
+ ```
158
+
159
+ `--before baseline` when step 5 chose the baseline. The `--tag` pairs are exactly the caller's;
160
+ none when the caller passed none. Exit non-zero → report the engine's reason (a record it cannot
161
+ read, a missing baseline), that the run was paid for and where its records are, and no verdict.
162
+
163
+ 12. **Report** — lead with the verdict the comparison record states, then the rows that drove it.
164
+
165
+ | Verdict | Say |
166
+ |---|---|
167
+ | `regressed` | each gated row that moved the wrong way and is significant: metric, task (or pooled), p-value. Then the footer's test count and its count of rows chance alone would make significant, both as the record states them, so one significant row is not read as proof. |
168
+ | `inconclusive` | the result cannot be called and is **not evidence the change is safe**. Name the wrong-way rows behind it, with their p-values. Suggest more runs per arm (a new plan with a higher `--runs`). |
169
+ | `incomparable` | every reason the record lists (different model, harness, task set, …) and what would make the two records comparable. Present no per-task deltas, means, or p-values. |
170
+ | `improved` | each row that improved significantly: metric, task (or pooled), p-value. |
171
+ | `unchanged`, every gated row `tooFew` | the run count was too low to call any change. It is **not evidence the change is safe**; do not say nothing changed. Suggest more runs per arm. |
172
+ | `unchanged`, gated rows not all `tooFew` | no significant change in any gated metric. |
173
+
174
+ **Cost.** A significant `costUsd` (or `cacheReadTokens`) row is reported as a cost change on
175
+ its own line, under any verdict except `incomparable`. It is never a regression and never turns
176
+ the verdict into one; only the engine's verdict says `regressed`.
177
+
178
+ ## Validate
179
+
180
+ Assertions about this file, checked by its author before handoff and by a cold reader after:
181
+
182
+ 1. **Consent only on an explicit yes to the shown plan.** Exactly one fenced command in this file
183
+ carries `--consent`: the `run` command in step 9, and step 9 opens with "only on step 8's
184
+ explicit yes, for exactly the plan file that was shown". *(mechanical: count fenced lines
185
+ containing `--consent`; read step 9's first line)*
186
+ 2. **No consent with no person present.** Step 8's no-person bullet returns the plan as
187
+ needs-input and says not to call `run`, and the Binding rules' no-person bullet says to run
188
+ nothing, ever. *(mechanical: both bullets present with those words)*
189
+ 3. **No unclear result presented as safe.** The step-12 rows for `inconclusive` and for
190
+ `unchanged` with every gated row `tooFew` each contain "not evidence the change is safe".
191
+ *(mechanical: grep those two rows)*
192
+ 4. **No cost change called a regression.** Step 12's **Cost** paragraph says a significant cost row
193
+ is reported as a cost change and is never a regression. *(mechanical: the paragraph contains
194
+ "never a regression")*
195
+
196
+ ## References
197
+
198
+ - `aced:aced-fit` — loaded by name in step 2; its measured criteria 1–2 are the fit test.
199
+ - `README.md` in this folder — the engine's surface, its verdicts, and who calls this skill.
@@ -0,0 +1,187 @@
1
+ // bench-claude-code — the Claude Code adapter: launch headless, parse the transcript into counts.
2
+ //
3
+ // The launch facts (the command, the one-shot flag, how the model, permission mode, and output
4
+ // format are passed, and which flag stops session persistence) come from @cyberuni/agent-harness's
5
+ // headless fact. The spec adds what a measured run must carry beyond those facts: the budget cap,
6
+ // project-only settings and a strict MCP config (the checkout's own, never the operator's), and,
7
+ // for a package arm, the plugin directory. Those are added here.
8
+ //
9
+ // `print` is the only runner: headless `-p` with stream-json output. A run is `capped` whenever it
10
+ // stops without a success result — the budget cap, an error result, the timeout, or the process
11
+ // dying on a signal. A spawn failure (the command is gone) is not a run outcome: it throws.
12
+
13
+ import { spawnSync } from 'node:child_process'
14
+ import { copyFileSync, existsSync, mkdirSync } from 'node:fs'
15
+ import { homedir } from 'node:os'
16
+ import { join } from 'node:path'
17
+ import { headlessCommand, headlessInvocation } from '@cyberuni/agent-harness'
18
+
19
+ export interface LaunchOptions {
20
+ prompt: string
21
+ model: string
22
+ maxBudgetUsd: number
23
+ permissionMode: string
24
+ checkout: string
25
+ timeoutMs: number
26
+ /** A package arm's unpacked plugin. */
27
+ pluginDir?: string
28
+ /** Extra environment, such as a package arm's isolated CLAUDE_CONFIG_DIR. */
29
+ env?: Record<string, string>
30
+ }
31
+
32
+ export interface AgentOutcome {
33
+ inputTokens: number
34
+ outputTokens: number
35
+ cacheReadTokens: number
36
+ cacheCreationTokens: number
37
+ turns: number
38
+ toolCalls: number
39
+ costUsd: number
40
+ capped: boolean
41
+ /** The model the transcript reports, when it reports one. */
42
+ model?: string
43
+ transcript: string
44
+ }
45
+
46
+ export interface Adapter {
47
+ harness: string
48
+ name: string
49
+ runner: string
50
+ executable: string
51
+ launch(options: LaunchOptions): AgentOutcome
52
+ /**
53
+ * A fresh harness config directory inside `runTemp`: the operator's credentials, none of their
54
+ * plugins or settings. Returns the environment that points the harness at it.
55
+ */
56
+ isolatedConfig(runTemp: string): Record<string, string>
57
+ }
58
+
59
+ /** Reads `--output-format stream-json`: tool calls from assistant messages, the rest from `result`. */
60
+ export function parseStreamJson(text: string): Omit<AgentOutcome, 'transcript'> {
61
+ const out: Omit<AgentOutcome, 'transcript'> = {
62
+ inputTokens: 0,
63
+ outputTokens: 0,
64
+ cacheReadTokens: 0,
65
+ cacheCreationTokens: 0,
66
+ turns: 0,
67
+ toolCalls: 0,
68
+ costUsd: 0,
69
+ capped: true,
70
+ }
71
+ for (const line of text.split('\n')) {
72
+ if (!line.startsWith('{')) continue
73
+ let event: {
74
+ type?: string
75
+ subtype?: string
76
+ model?: string
77
+ is_error?: boolean
78
+ num_turns?: number
79
+ total_cost_usd?: number
80
+ usage?: {
81
+ input_tokens?: number
82
+ output_tokens?: number
83
+ cache_read_input_tokens?: number
84
+ cache_creation_input_tokens?: number
85
+ }
86
+ message?: { model?: string; content?: Array<{ type?: string }> }
87
+ }
88
+ try {
89
+ event = JSON.parse(line)
90
+ } catch {
91
+ continue
92
+ }
93
+ if (event.type === 'system' && typeof event.model === 'string' && event.model && out.model === undefined) {
94
+ out.model = event.model
95
+ } else if (event.type === 'assistant') {
96
+ const content = Array.isArray(event.message?.content) ? event.message.content : []
97
+ out.toolCalls += content.filter((c) => c?.type === 'tool_use').length
98
+ if (out.model === undefined && typeof event.message?.model === 'string' && event.message.model)
99
+ out.model = event.message.model
100
+ } else if (event.type === 'result') {
101
+ const u = event.usage ?? {}
102
+ out.inputTokens = u.input_tokens ?? 0
103
+ out.outputTokens = u.output_tokens ?? 0
104
+ out.cacheReadTokens = u.cache_read_input_tokens ?? 0
105
+ out.cacheCreationTokens = u.cache_creation_input_tokens ?? 0
106
+ out.turns = event.num_turns ?? 0
107
+ out.costUsd = event.total_cost_usd ?? 0
108
+ out.capped = event.subtype !== 'success' || event.is_error === true
109
+ }
110
+ }
111
+ return out
112
+ }
113
+
114
+ const HARNESS = 'claude-code'
115
+
116
+ function launchArgs(o: LaunchOptions): string[] {
117
+ const facts = headlessInvocation(HARNESS)
118
+ const cmd = headlessCommand(HARNESS, {
119
+ prompt: o.prompt,
120
+ model: o.model,
121
+ permission: o.permissionMode,
122
+ outputFormat: 'stream-json',
123
+ })
124
+ if (!cmd || facts.supported !== true)
125
+ throw new Error('@cyberuni/agent-harness reports no headless mode for claude-code')
126
+ const args = [...cmd.args]
127
+ // The prompt is placed right after the facts' own (boolean-terminated) flags, so a variadic flag
128
+ // added below can never swallow it.
129
+ const disable = facts.transcript?.disable ?? []
130
+ const extra = [
131
+ '--max-budget-usd',
132
+ String(o.maxBudgetUsd),
133
+ ...disable,
134
+ ...(disable.includes('--no-session-persistence') ? [] : ['--no-session-persistence']),
135
+ '--setting-sources',
136
+ 'project',
137
+ '--strict-mcp-config',
138
+ ]
139
+ if (o.pluginDir) extra.push('--plugin-dir', o.pluginDir)
140
+ if (existsSync(join(o.checkout, '.mcp.json'))) extra.push('--mcp-config', '.mcp.json')
141
+ return [...args, ...extra]
142
+ }
143
+
144
+ export const claudeCode: Adapter = {
145
+ harness: HARNESS,
146
+ name: 'aced.claude-code',
147
+ runner: 'print',
148
+ executable: (() => {
149
+ const facts = headlessInvocation(HARNESS)
150
+ return facts.supported === true ? facts.executable : 'claude'
151
+ })(),
152
+ launch(o) {
153
+ const r = spawnSync(this.executable, launchArgs(o), {
154
+ cwd: o.checkout,
155
+ encoding: 'utf8',
156
+ timeout: o.timeoutMs,
157
+ killSignal: 'SIGKILL',
158
+ maxBuffer: 512 * 1024 * 1024,
159
+ stdio: ['ignore', 'pipe', 'pipe'],
160
+ env: { ...process.env, ...o.env },
161
+ })
162
+ const code = (r.error as NodeJS.ErrnoException | undefined)?.code
163
+ if (r.error && code !== 'ETIMEDOUT' && code !== 'ENOBUFS')
164
+ throw new Error(`could not launch ${this.executable}: ${r.error.message}`)
165
+ const transcript = r.stdout ?? ''
166
+ const parsed = parseStreamJson(transcript)
167
+ // Killed — by the timeout or any other signal — is a stop without a success result.
168
+ if (r.error || r.signal) parsed.capped = true
169
+ return { ...parsed, transcript }
170
+ },
171
+ isolatedConfig(runTemp) {
172
+ const dir = join(runTemp, 'claude-config')
173
+ mkdirSync(dir, { recursive: true })
174
+ const operator = process.env.CLAUDE_CONFIG_DIR || join(homedir(), '.claude')
175
+ const credentials = join(operator, '.credentials.json')
176
+ if (existsSync(credentials)) copyFileSync(credentials, join(dir, '.credentials.json'))
177
+ return { CLAUDE_CONFIG_DIR: dir }
178
+ },
179
+ }
180
+
181
+ const ADAPTERS: Record<string, Adapter> = { [HARNESS]: claudeCode }
182
+
183
+ export function adapterFor(harness: string): Adapter | undefined {
184
+ return Object.hasOwn(ADAPTERS, harness) ? ADAPTERS[harness] : undefined
185
+ }
186
+
187
+ export const ADAPTER_HARNESSES = Object.keys(ADAPTERS)