cyber-aced 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +18 -0
- package/.plugin/pins.json +1 -0
- package/.plugin/plugin.json +18 -0
- package/agents/aced-case-judge.md +165 -0
- package/agents/aced-impl-judge.md +55 -0
- package/agents/aced-scenario-writer.md +76 -0
- package/agents/aced-spec-validator.md +66 -0
- package/governances/README.md +52 -0
- package/governances/agent-tool-output.md +86 -0
- package/governances/cli-resolution.md +102 -0
- package/governances/skill-design.md +258 -0
- package/governances/skill-repo-structure.md +173 -0
- package/package.json +44 -0
- package/readme.md +124 -0
- package/skills/aced-architect-impl/README.md +9 -0
- package/skills/aced-architect-impl/SKILL.md +73 -0
- package/skills/aced-builder-impl/README.md +14 -0
- package/skills/aced-builder-impl/SKILL.md +46 -0
- package/skills/aced-builder-spec/README.md +15 -0
- package/skills/aced-builder-spec/SKILL.md +130 -0
- package/skills/aced-fit/README.md +21 -0
- package/skills/aced-fit/SKILL.md +105 -0
- package/skills/aced-impl-producer/SKILL.md +75 -0
- package/skills/add-scenario/README.md +25 -0
- package/skills/add-scenario/SKILL.md +75 -0
- package/skills/bench/README.md +64 -0
- package/skills/bench/SKILL.md +199 -0
- package/skills/bench/scripts/bench-claude-code.mts +187 -0
- package/skills/bench/scripts/bench-compare.mts +369 -0
- package/skills/bench/scripts/bench-record.mts +133 -0
- package/skills/bench/scripts/bench.mts +930 -0
- package/skills/check-freshness/README.md +57 -0
- package/skills/check-freshness/SKILL.md +76 -0
- package/skills/check-freshness/scripts/check-freshness.mts +399 -0
- package/skills/compare/README.md +25 -0
- package/skills/compare/SKILL.md +87 -0
- package/skills/contribute-skill/README.md +25 -0
- package/skills/contribute-skill/SKILL.md +241 -0
- package/skills/contribute-skill/scripts/source.mts +188 -0
- package/skills/define-agent/README.md +40 -0
- package/skills/define-agent/SKILL.md +279 -0
- package/skills/define-command/README.md +36 -0
- package/skills/define-command/SKILL.md +174 -0
- package/skills/define-governance/SKILL.md +151 -0
- package/skills/define-skill/README.md +37 -0
- package/skills/define-skill/SKILL.md +200 -0
- package/skills/extract-situation/README.md +32 -0
- package/skills/extract-situation/SKILL.md +54 -0
- package/skills/extract-situation/scripts/extract-situation.mts +287 -0
- package/skills/improve/README.md +25 -0
- package/skills/improve/SKILL.md +100 -0
- package/skills/improve-skill/README.md +38 -0
- package/skills/improve-skill/SKILL.md +169 -0
- package/skills/improve-skill/references/check-definitions.md +215 -0
- package/skills/improve-skill/scripts/validate.mts +1033 -0
- package/skills/init-aced/README.md +5 -0
- package/skills/init-aced/SKILL.md +76 -0
- package/skills/init-aced/scripts/ensure-results-ignored.mts +117 -0
- package/skills/list-skills/README.md +19 -0
- package/skills/list-skills/SKILL.md +55 -0
- package/skills/list-skills/scripts/list-skills.mts +210 -0
- package/skills/manage/README.md +35 -0
- package/skills/manage/SKILL.md +65 -0
- package/skills/manage-model-runners/README.md +25 -0
- package/skills/manage-model-runners/SKILL.md +113 -0
- package/skills/manage-skill-dirs/README.md +17 -0
- package/skills/manage-skill-dirs/SKILL.md +54 -0
- package/skills/manage-skill-dirs/scripts/manage-skill-dirs.mts +328 -0
- package/skills/repair-private-skills/README.md +17 -0
- package/skills/repair-private-skills/SKILL.md +59 -0
- package/skills/repair-private-skills/scripts/repair-private-skills.mts +279 -0
- package/skills/report/README.md +25 -0
- package/skills/report/SKILL.md +74 -0
- package/skills/run/README.md +25 -0
- package/skills/run/SKILL.md +220 -0
- package/skills/skillify/README.md +25 -0
- package/skills/skillify/SKILL.md +141 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# add-scenario
|
|
2
|
+
|
|
3
|
+
Add a new scenario to an existing ACED `.feature` suite — from a real failure, a production edge case, or a gap you noticed.
|
|
4
|
+
|
|
5
|
+
## When to use
|
|
6
|
+
|
|
7
|
+
Use this skill when you already have a suite (`eval.md` + a frozen `.feature`) and want to grow its coverage with one more scenario.
|
|
8
|
+
|
|
9
|
+
Good triggers include:
|
|
10
|
+
|
|
11
|
+
- "Add a test case for this failure"
|
|
12
|
+
- "We need to cover this edge case"
|
|
13
|
+
- Turning a pasted agent transcript that went wrong into a guarded-against scenario
|
|
14
|
+
|
|
15
|
+
Defer to a sibling when the request is about scoring the existing set (`run`), diagnosing why cases fail (`improve`), or there's no suite yet at all (`define-skill` / `define-agent` / `define-governance`, then `sdd:start-mission` to author one).
|
|
16
|
+
|
|
17
|
+
## What it does
|
|
18
|
+
|
|
19
|
+
Locates the target suite, classifies the new scenario's layer (trigger / behavior / quality) from how you describe the failure, drafts a Gherkin scenario (boolean, or `@rubric` with an inline rubric, or a `@trigger` `Examples` row), confirms it with you, and appends it to the frozen `.feature` (additive — self-clears).
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
npx skills add cyberuni/cyberplace --skill aced/add-scenario
|
|
25
|
+
```
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: add-scenario
|
|
3
|
+
description: Use this skill when adding a new scenario to an ACED .feature suite — from a real failure, a production edge case, or a gap the user noticed.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# ACED Add Scenario
|
|
7
|
+
|
|
8
|
+
Add one or more scenarios to an existing frozen `.feature` suite. Adding a scenario is **additive** —
|
|
9
|
+
it widens the contract and cannot break existing impl, so it **self-clears** and the `.feature` stays
|
|
10
|
+
`@frozen` (per `sdd:suite-format-governance`); no re-open is needed.
|
|
11
|
+
|
|
12
|
+
## Locate the suite
|
|
13
|
+
|
|
14
|
+
Find the target's node in the project spec — `.agents/specs/<project>/…/<node>/` (discovered through
|
|
15
|
+
the SDD spec tree; the node's `eval.md` names the subject) — from user context or ask. Read the
|
|
16
|
+
node's colocated `eval.md` for the `subject` and the `eval:` run policy (`eval.judge.default_threshold`).
|
|
17
|
+
|
|
18
|
+
## Gather input
|
|
19
|
+
|
|
20
|
+
The user may provide:
|
|
21
|
+
- A description of a scenario that failed in production
|
|
22
|
+
- A pasted agent transcript showing incorrect behavior
|
|
23
|
+
- A description of an edge case not yet covered
|
|
24
|
+
- A "must not do" behavior they want to guard against
|
|
25
|
+
|
|
26
|
+
If a transcript is provided, extract: what the user said, what state the system was in, what the agent did, what it should have done instead.
|
|
27
|
+
|
|
28
|
+
## Determine the layer
|
|
29
|
+
|
|
30
|
+
| Input type | Layer |
|
|
31
|
+
|---|---|
|
|
32
|
+
| "Agent invoked the skill when it shouldn't have" | trigger |
|
|
33
|
+
| "Agent didn't invoke the skill when it should have" | trigger |
|
|
34
|
+
| "Agent invoked correctly but skipped a step" | behavior |
|
|
35
|
+
| "Agent did the step but produced poor output" | quality |
|
|
36
|
+
|
|
37
|
+
Ask if unclear.
|
|
38
|
+
|
|
39
|
+
If the resolved layer is not enabled in the suite's `eval.md` `eval.layers`, warn that the layer is not enabled and the scenario will not be exercised until it is.
|
|
40
|
+
|
|
41
|
+
## Scaffold the scenario
|
|
42
|
+
|
|
43
|
+
Draft a Gherkin scenario for the frozen `.feature`, tagged with its layer:
|
|
44
|
+
|
|
45
|
+
- **trigger** → add a row to the `@trigger` `Scenario Outline`'s `Examples` table
|
|
46
|
+
(`| <query> | <should_trigger> |`), or start one if the suite has none.
|
|
47
|
+
- **behavior (deterministic)** → a boolean scenario whose `Then` asserts the observable action (a
|
|
48
|
+
must-not-do guard is a `Then` asserting the agent *does not* do the prohibited action).
|
|
49
|
+
- **behavior / quality (graded)** → a `@rubric` scenario with the rubric **inline**:
|
|
50
|
+
|
|
51
|
+
```gherkin
|
|
52
|
+
@behavior @rubric
|
|
53
|
+
Scenario: <name — the situation being guarded>
|
|
54
|
+
Given <concrete situation: user message, repo/file state, context>
|
|
55
|
+
When <the agent acts>
|
|
56
|
+
Then the judge evaluates the scenario against the rubric
|
|
57
|
+
"""
|
|
58
|
+
dimensions:
|
|
59
|
+
- name: <criterion>
|
|
60
|
+
max: <n>
|
|
61
|
+
threshold: <from eval.judge.default_threshold>
|
|
62
|
+
"""
|
|
63
|
+
And the rubric score is at least the threshold
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Show the draft to the user and ask for confirmation before writing. Adjust based on feedback.
|
|
67
|
+
|
|
68
|
+
## Write the scenario
|
|
69
|
+
|
|
70
|
+
Append the scenario to the node's frozen `<node>.feature` in the project spec, sorted into its
|
|
71
|
+
lifecycle-stage section (per the scenario-ordering convention). Keep the feature-level `@frozen` tag —
|
|
72
|
+
adding a scenario self-clears. Run `check-suite --files <path>` to confirm the suite is still
|
|
73
|
+
well-formed.
|
|
74
|
+
|
|
75
|
+
Report the scenario name and suggest running `run` to score it against the current agent configuration.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# bench
|
|
2
|
+
|
|
3
|
+
Measure whether a change makes real agent work turn out better or worse — pass rate, turns, tokens,
|
|
4
|
+
wall time — with real headless harness runs, at a price the person approved first.
|
|
5
|
+
|
|
6
|
+
`bench` is ACED's **measured layer**. The simulated layers (`run`, `compare`) say what an agent
|
|
7
|
+
*would* do; `bench` runs the agent for real in throwaway checkouts, grades each run with a shell
|
|
8
|
+
`check`, and compares two arms with permutation tests.
|
|
9
|
+
|
|
10
|
+
## How it is reached
|
|
11
|
+
|
|
12
|
+
By name only — its `description` is `"By name only"`, so it is never matched to a situation:
|
|
13
|
+
|
|
14
|
+
- `/bench` from a person in the session;
|
|
15
|
+
- a consumer tool handing off by name with its own suite, and optionally its own comparison tags;
|
|
16
|
+
- `compare`'s measured mode (a later change).
|
|
17
|
+
|
|
18
|
+
## What it does
|
|
19
|
+
|
|
20
|
+
1. **Fit.** Loads `aced-fit` and re-asks its measured criteria 1–2 for the change in hand: does the
|
|
21
|
+
change move how real work turns out, and can a shell check decide the outcome? A wording-only or
|
|
22
|
+
rubric-only question goes to `compare` and nothing is planned. A subject's `bench:` declaration
|
|
23
|
+
neither skips nor forbids this check.
|
|
24
|
+
2. **Suite and task set.** Asks for a suite when none is named (never guesses, even with one suite
|
|
25
|
+
in the repo); offers the engine's `init` when the suite has no `tasks.json`, writing no tasks
|
|
26
|
+
itself.
|
|
27
|
+
3. **Arms.** Maps the request to git-ref, package, or file arms. With no arms named, measures HEAD
|
|
28
|
+
against the suite's committed `baseline.json` — after checking, before any spend, that the
|
|
29
|
+
baseline is a schema version the engine reads.
|
|
30
|
+
4. **Plan.** Runs the engine's `plan` (spends nothing) and shows the counts, model, ceiling,
|
|
31
|
+
estimate, permission mode and its throwaway-checkout scope, and every warning, then asks.
|
|
32
|
+
5. **Consent.** Runs only on an explicit yes to the plan on screen. A hedge, a question, a yes given
|
|
33
|
+
before the plan was shown, a yes that changes the plan, or an approval relayed by another agent is
|
|
34
|
+
not consent. With no person present, the plan goes back up as needs-input and nothing runs.
|
|
35
|
+
6. **Run, compare, report.** Leads with the verdict and the rows that drove it, and never presents
|
|
36
|
+
`inconclusive` (or `unchanged` at a too-few run count) as safe, or a cost change as a regression.
|
|
37
|
+
|
|
38
|
+
A baseline request plans one arm at HEAD with the baseline flag and, after the approved run, reminds
|
|
39
|
+
the maintainer to commit `baseline.json`.
|
|
40
|
+
|
|
41
|
+
## The engine
|
|
42
|
+
|
|
43
|
+
`scripts/bench.mts` (bin `aced-bench`) is the deterministic half: `init`, `plan`, `run --consent`,
|
|
44
|
+
`compare`. It owns every number — the statistics, the record schema, the ceiling and estimate, the
|
|
45
|
+
verdict rule — and is usable without the plugin installed (for example from CI, which passes
|
|
46
|
+
`--consent` itself under its own accountability). The skill adds judgment around it and restates
|
|
47
|
+
none of its numbers.
|
|
48
|
+
|
|
49
|
+
Verdicts: `regressed`, `inconclusive`, `improved`, `unchanged`, `incomparable`.
|
|
50
|
+
|
|
51
|
+
## Suite layout
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
.agents/aced/bench/<suite>/
|
|
55
|
+
tasks.json # the task set (written by the suite maintainer; `init` writes a template)
|
|
56
|
+
checks/ # shell checks the tasks call
|
|
57
|
+
baseline.json # optional, committed — the before side when no arms are named
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Run records and comparisons go to `.agents/aced/results/bench/<suite>/`, which `init-aced`
|
|
61
|
+
git-ignores.
|
|
62
|
+
|
|
63
|
+
Spec: `.agents/specs/aced/eval-run/bench/skill/` (this skill) and
|
|
64
|
+
`.agents/specs/aced/eval-run/bench/engine/` (the engine).
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: bench
|
|
3
|
+
description: "By name only"
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# bench
|
|
7
|
+
|
|
8
|
+
Measure a change with real headless runs through ACED's measured-layer engine, spending only on the
|
|
9
|
+
person's explicit yes to the plan on screen. Reached by name: `/bench`, a consumer tool handing off
|
|
10
|
+
with its own suite (and optional tags), or `compare`'s measured mode. The procedure is the same
|
|
11
|
+
whichever caller loaded it.
|
|
12
|
+
|
|
13
|
+
The engine owns every number: statistics, ceiling, estimate, run counts, verdicts. Quote what the
|
|
14
|
+
plan and the comparison record say; never compute, round, or restate a statistic of your own.
|
|
15
|
+
|
|
16
|
+
## Engine
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
node "<skill>/scripts/bench.mts" <verb> …
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
`<skill>` is this skill's folder. Verbs: `init`, `plan`, `run`, `compare`. A suite lives at
|
|
23
|
+
`.agents/aced/bench/<suite>/` (`tasks.json`, `checks/`, optional committed `baseline.json`). Exit
|
|
24
|
+
non-zero means the engine refused or failed; its stderr carries the reason.
|
|
25
|
+
|
|
26
|
+
## Binding rules
|
|
27
|
+
|
|
28
|
+
- **Consent is one thing:** the person's explicit yes, given in reply to the approval question, to
|
|
29
|
+
the plan currently shown. Only that reply lets the `run` call carry the consent flag.
|
|
30
|
+
- **An explicit yes** is a plain affirmative with no hedge, no condition, and no change: "yes",
|
|
31
|
+
"yes, run it", "go ahead", "approved".
|
|
32
|
+
- **Not a yes — run nothing:**
|
|
33
|
+
- a hedged agreement: "sounds good I guess", "probably fine", "ok, I suppose";
|
|
34
|
+
- a question back, or any reply that is not an affirmative;
|
|
35
|
+
- approval given before the plan was shown ("go ahead and spend whatever it takes", "just run
|
|
36
|
+
it"): show the plan and ask anyway;
|
|
37
|
+
- approval relayed by anyone other than the person in this session (a driver, a coordinator, a
|
|
38
|
+
brief saying "the user approved");
|
|
39
|
+
- a yes that changes the plan ("yes, but make it 10 runs per arm"): that yes answers a plan that
|
|
40
|
+
no longer exists.
|
|
41
|
+
- **No person present** (loaded by `sdd-automaton`, a coordinator, a scheduled agent, or any
|
|
42
|
+
session with no user channel): run nothing, ever. Every question this procedure would ask goes
|
|
43
|
+
back up as **needs-input**, and nothing is written. A CI job that wants unattended runs calls the
|
|
44
|
+
engine's bin with the consent flag itself, under its own accountability; this skill never does it
|
|
45
|
+
for anyone.
|
|
46
|
+
- **Never guess** a suite, an arm, or a task. Never write `tasks.json` or `checks/` content.
|
|
47
|
+
- **Tags pass through untouched:** the caller's `--tag k=v` pairs reach `compare` exactly as given.
|
|
48
|
+
Add none, drop none, reorder none, interpret none.
|
|
49
|
+
- **Never present an unclear result as safe**, and **never call a cost change a regression**
|
|
50
|
+
(step 12).
|
|
51
|
+
|
|
52
|
+
## Workflow
|
|
53
|
+
|
|
54
|
+
1. **Who is present.** Decide once whether a person in this session can answer. If not, apply the
|
|
55
|
+
needs-input branch wherever a step below asks something.
|
|
56
|
+
|
|
57
|
+
2. **Fit — re-ask for the change in hand.** Skip for a baseline request (step 5). Load
|
|
58
|
+
`aced:aced-fit` by name and apply its **measured criteria 1 and 2** to the change being
|
|
59
|
+
measured, not to the subject's `bench:` declaration: a declared `measured: worth` does not skip
|
|
60
|
+
this step, and an absent or `not-worth` declaration does not by itself stop it.
|
|
61
|
+
- Criterion 1 fails (the change only rewords a description, a trigger, or other wording) → say
|
|
62
|
+
the simulated layer answers a wording change and name `compare` (ACED's simulated diff).
|
|
63
|
+
- Criterion 2 fails (no shell `check` can decide the outcome, for example "do replies read
|
|
64
|
+
warmer") → say no shell check can decide this outcome, point to the simulated layer, and name
|
|
65
|
+
`compare`.
|
|
66
|
+
- Either failure: plan nothing, return no plan. Same answer with no person present.
|
|
67
|
+
|
|
68
|
+
3. **Suite.** Use the suite the request or the calling tool names. None named → ask which suite to
|
|
69
|
+
use and plan nothing. You may list the suites found under `.agents/aced/bench/`, but never pick
|
|
70
|
+
one, even when only one exists. No person present → return that question as needs-input.
|
|
71
|
+
|
|
72
|
+
4. **Task set.** `.agents/aced/bench/<suite>/tasks.json` missing → say the suite has no task set,
|
|
73
|
+
offer to run the engine's `init` for it (a template the suite maintainer then fills in), and plan
|
|
74
|
+
nothing. Write no task of your own. Run `init` only on the person's yes to that offer:
|
|
75
|
+
|
|
76
|
+
```
|
|
77
|
+
node "<skill>/scripts/bench.mts" init --suite <suite>
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
No person present → return the init offer as needs-input; run nothing, write nothing.
|
|
81
|
+
|
|
82
|
+
5. **Arms.** Map the request to `--arm <label>=<subject>` pairs. "X against Y" makes Y `before` and
|
|
83
|
+
X `after`.
|
|
84
|
+
|
|
85
|
+
| The request names | Arms |
|
|
86
|
+
|---|---|
|
|
87
|
+
| two git refs (commits, branches, tags) | `--arm before=git:<Y> --arm after=git:<X>` |
|
|
88
|
+
| two versions of one package or plugin | `--arm before=package:<name>@<older> --arm after=package:<name>@<newer>` |
|
|
89
|
+
| with and without one file that HEAD carries | `--arm without=file:<path>=absent --arm with=file:<path>=ref:HEAD` — never a git-ref arm |
|
|
90
|
+
| one file swapped in from a ref or a path | `--arm before=file:<path>=ref:HEAD --arm after=file:<path>=ref:<ref>` (or `=path:<source>`) — both arms `file`, since the engine never compares two subject kinds |
|
|
91
|
+
| a request to record or refresh the baseline | `--arm baseline=git:HEAD` plus `--baseline` — one arm only |
|
|
92
|
+
| no arms | read `.agents/aced/bench/<suite>/baseline.json` first (below) |
|
|
93
|
+
|
|
94
|
+
**No arms named** — check the baseline now, before any plan or spend:
|
|
95
|
+
- `baseline.json` exists with `schemaVersion: 3` (the only version the engine reads) → one arm,
|
|
96
|
+
`--arm after=git:HEAD`; at step 11 compare it with `--before baseline`.
|
|
97
|
+
- no `baseline.json` → say the suite has no baseline, ask which two arms to compare (or offer to
|
|
98
|
+
record a baseline), and plan nothing.
|
|
99
|
+
- `baseline.json` with any other or missing `schemaVersion` → say the baseline is schema version
|
|
100
|
+
`<n>`, which the engine cannot read, ask which two arms to compare or offer to re-record the
|
|
101
|
+
baseline, and plan nothing.
|
|
102
|
+
- No person present → return the question as needs-input; plan nothing.
|
|
103
|
+
|
|
104
|
+
6. **Plan.** Pass through only what was asked for (`--runs`, `--task`, `--harness`):
|
|
105
|
+
|
|
106
|
+
```
|
|
107
|
+
node "<skill>/scripts/bench.mts" plan --suite <suite> --arm <label>=<subject> … [--runs <n>] [--task <id>] [--harness <h>] [--baseline] --out <plan-file>
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Write `<plan-file>` under `.agents/aced/results/bench/<suite>/`, a fresh name per plan. Exit
|
|
111
|
+
non-zero → report the engine's reason in its terms (the `claude` command is not on the path, no
|
|
112
|
+
adapter for the harness, a ref or version or file that does not resolve, a malformed task set)
|
|
113
|
+
and what would fix it. Ask no approval question; stop.
|
|
114
|
+
|
|
115
|
+
7. **Show the plan** — one message, in this order, every value as the plan states it:
|
|
116
|
+
1. the arms, by label and subject;
|
|
117
|
+
2. the counts: arms, tasks, runs per arm; and the model;
|
|
118
|
+
3. the ceiling and the estimate, in dollars;
|
|
119
|
+
4. the permission mode, and that it applies only inside the throwaway checkout;
|
|
120
|
+
5. every warning the plan carries, verbatim (uncommitted changes not benched, the committed suite
|
|
121
|
+
is what runs, …). For a **too few to call** warning also say what it means: no single task's
|
|
122
|
+
result can be called significant at this run count; then repeat what the plan says about
|
|
123
|
+
whether a pooled result across tasks still can;
|
|
124
|
+
6. last, the approval question: "Run this plan, spending up to $<ceiling>? Reply yes to run it."
|
|
125
|
+
|
|
126
|
+
Nothing follows the question, and no warning comes after it.
|
|
127
|
+
|
|
128
|
+
8. **Consent.**
|
|
129
|
+
- No person present → return the plan upward marked **needs-input** (the plan file path, the
|
|
130
|
+
step-7 summary, and the approval question). Do not call `run`. A relayed "the user approved"
|
|
131
|
+
changes nothing: still needs-input, still unrun.
|
|
132
|
+
- The reply is an explicit yes to the plan shown, unchanged → step 9.
|
|
133
|
+
- The reply says yes **and** changes the plan → re-plan with the change (step 6), show the new
|
|
134
|
+
plan (step 7), ask again. Run nothing until a new explicit yes to the new plan.
|
|
135
|
+
- Any other reply → run nothing. Answer a question if one was asked; the plan stays unapproved
|
|
136
|
+
until an explicit yes.
|
|
137
|
+
|
|
138
|
+
9. **Run** — only on step 8's explicit yes, for exactly the plan file that was shown:
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
node "<skill>/scripts/bench.mts" run --plan <plan-file> --consent
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Exit non-zero → report the engine's reason (for example, the suite changed since the plan was
|
|
145
|
+
approved) and whether anything was spent: a refusal before launch spends nothing; a failure after
|
|
146
|
+
runs launched may have spent. Report no verdict. A changed suite needs a new plan and a new yes.
|
|
147
|
+
Note the run record paths the engine prints.
|
|
148
|
+
|
|
149
|
+
10. **Baseline run** (`--baseline` plan) → tell the maintainer to commit
|
|
150
|
+
`.agents/aced/bench/<suite>/baseline.json`. A baseline compares against nothing: no `compare`,
|
|
151
|
+
no verdict.
|
|
152
|
+
|
|
153
|
+
11. **Compare.**
|
|
154
|
+
|
|
155
|
+
```
|
|
156
|
+
node "<skill>/scripts/bench.mts" compare --suite <suite> --before <before-record|baseline> --after <after-record> [--tag <k=v> …]
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
`--before baseline` when step 5 chose the baseline. The `--tag` pairs are exactly the caller's;
|
|
160
|
+
none when the caller passed none. Exit non-zero → report the engine's reason (a record it cannot
|
|
161
|
+
read, a missing baseline), that the run was paid for and where its records are, and no verdict.
|
|
162
|
+
|
|
163
|
+
12. **Report** — lead with the verdict the comparison record states, then the rows that drove it.
|
|
164
|
+
|
|
165
|
+
| Verdict | Say |
|
|
166
|
+
|---|---|
|
|
167
|
+
| `regressed` | each gated row that moved the wrong way and is significant: metric, task (or pooled), p-value. Then the footer's test count and its count of rows chance alone would make significant, both as the record states them, so one significant row is not read as proof. |
|
|
168
|
+
| `inconclusive` | the result cannot be called and is **not evidence the change is safe**. Name the wrong-way rows behind it, with their p-values. Suggest more runs per arm (a new plan with a higher `--runs`). |
|
|
169
|
+
| `incomparable` | every reason the record lists (different model, harness, task set, …) and what would make the two records comparable. Present no per-task deltas, means, or p-values. |
|
|
170
|
+
| `improved` | each row that improved significantly: metric, task (or pooled), p-value. |
|
|
171
|
+
| `unchanged`, every gated row `tooFew` | the run count was too low to call any change. It is **not evidence the change is safe**; do not say nothing changed. Suggest more runs per arm. |
|
|
172
|
+
| `unchanged`, gated rows not all `tooFew` | no significant change in any gated metric. |
|
|
173
|
+
|
|
174
|
+
**Cost.** A significant `costUsd` (or `cacheReadTokens`) row is reported as a cost change on
|
|
175
|
+
its own line, under any verdict except `incomparable`. It is never a regression and never turns
|
|
176
|
+
the verdict into one; only the engine's verdict says `regressed`.
|
|
177
|
+
|
|
178
|
+
## Validate
|
|
179
|
+
|
|
180
|
+
Assertions about this file, checked by its author before handoff and by a cold reader after:
|
|
181
|
+
|
|
182
|
+
1. **Consent only on an explicit yes to the shown plan.** Exactly one fenced command in this file
|
|
183
|
+
carries `--consent`: the `run` command in step 9, and step 9 opens with "only on step 8's
|
|
184
|
+
explicit yes, for exactly the plan file that was shown". *(mechanical: count fenced lines
|
|
185
|
+
containing `--consent`; read step 9's first line)*
|
|
186
|
+
2. **No consent with no person present.** Step 8's no-person bullet returns the plan as
|
|
187
|
+
needs-input and says not to call `run`, and the Binding rules' no-person bullet says to run
|
|
188
|
+
nothing, ever. *(mechanical: both bullets present with those words)*
|
|
189
|
+
3. **No unclear result presented as safe.** The step-12 rows for `inconclusive` and for
|
|
190
|
+
`unchanged` with every gated row `tooFew` each contain "not evidence the change is safe".
|
|
191
|
+
*(mechanical: grep those two rows)*
|
|
192
|
+
4. **No cost change called a regression.** Step 12's **Cost** paragraph says a significant cost row
|
|
193
|
+
is reported as a cost change and is never a regression. *(mechanical: the paragraph contains
|
|
194
|
+
"never a regression")*
|
|
195
|
+
|
|
196
|
+
## References
|
|
197
|
+
|
|
198
|
+
- `aced:aced-fit` — loaded by name in step 2; its measured criteria 1–2 are the fit test.
|
|
199
|
+
- `README.md` in this folder — the engine's surface, its verdicts, and who calls this skill.
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// bench-claude-code — the Claude Code adapter: launch headless, parse the transcript into counts.
|
|
2
|
+
//
|
|
3
|
+
// The launch facts (the command, the one-shot flag, how the model, permission mode, and output
|
|
4
|
+
// format are passed, and which flag stops session persistence) come from @cyberuni/agent-harness's
|
|
5
|
+
// headless fact. The spec adds what a measured run must carry beyond those facts: the budget cap,
|
|
6
|
+
// project-only settings and a strict MCP config (the checkout's own, never the operator's), and,
|
|
7
|
+
// for a package arm, the plugin directory. Those are added here.
|
|
8
|
+
//
|
|
9
|
+
// `print` is the only runner: headless `-p` with stream-json output. A run is `capped` whenever it
|
|
10
|
+
// stops without a success result — the budget cap, an error result, the timeout, or the process
|
|
11
|
+
// dying on a signal. A spawn failure (the command is gone) is not a run outcome: it throws.
|
|
12
|
+
|
|
13
|
+
import { spawnSync } from 'node:child_process'
|
|
14
|
+
import { copyFileSync, existsSync, mkdirSync } from 'node:fs'
|
|
15
|
+
import { homedir } from 'node:os'
|
|
16
|
+
import { join } from 'node:path'
|
|
17
|
+
import { headlessCommand, headlessInvocation } from '@cyberuni/agent-harness'
|
|
18
|
+
|
|
19
|
+
export interface LaunchOptions {
|
|
20
|
+
prompt: string
|
|
21
|
+
model: string
|
|
22
|
+
maxBudgetUsd: number
|
|
23
|
+
permissionMode: string
|
|
24
|
+
checkout: string
|
|
25
|
+
timeoutMs: number
|
|
26
|
+
/** A package arm's unpacked plugin. */
|
|
27
|
+
pluginDir?: string
|
|
28
|
+
/** Extra environment, such as a package arm's isolated CLAUDE_CONFIG_DIR. */
|
|
29
|
+
env?: Record<string, string>
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface AgentOutcome {
|
|
33
|
+
inputTokens: number
|
|
34
|
+
outputTokens: number
|
|
35
|
+
cacheReadTokens: number
|
|
36
|
+
cacheCreationTokens: number
|
|
37
|
+
turns: number
|
|
38
|
+
toolCalls: number
|
|
39
|
+
costUsd: number
|
|
40
|
+
capped: boolean
|
|
41
|
+
/** The model the transcript reports, when it reports one. */
|
|
42
|
+
model?: string
|
|
43
|
+
transcript: string
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface Adapter {
|
|
47
|
+
harness: string
|
|
48
|
+
name: string
|
|
49
|
+
runner: string
|
|
50
|
+
executable: string
|
|
51
|
+
launch(options: LaunchOptions): AgentOutcome
|
|
52
|
+
/**
|
|
53
|
+
* A fresh harness config directory inside `runTemp`: the operator's credentials, none of their
|
|
54
|
+
* plugins or settings. Returns the environment that points the harness at it.
|
|
55
|
+
*/
|
|
56
|
+
isolatedConfig(runTemp: string): Record<string, string>
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Reads `--output-format stream-json`: tool calls from assistant messages, the rest from `result`. */
|
|
60
|
+
export function parseStreamJson(text: string): Omit<AgentOutcome, 'transcript'> {
|
|
61
|
+
const out: Omit<AgentOutcome, 'transcript'> = {
|
|
62
|
+
inputTokens: 0,
|
|
63
|
+
outputTokens: 0,
|
|
64
|
+
cacheReadTokens: 0,
|
|
65
|
+
cacheCreationTokens: 0,
|
|
66
|
+
turns: 0,
|
|
67
|
+
toolCalls: 0,
|
|
68
|
+
costUsd: 0,
|
|
69
|
+
capped: true,
|
|
70
|
+
}
|
|
71
|
+
for (const line of text.split('\n')) {
|
|
72
|
+
if (!line.startsWith('{')) continue
|
|
73
|
+
let event: {
|
|
74
|
+
type?: string
|
|
75
|
+
subtype?: string
|
|
76
|
+
model?: string
|
|
77
|
+
is_error?: boolean
|
|
78
|
+
num_turns?: number
|
|
79
|
+
total_cost_usd?: number
|
|
80
|
+
usage?: {
|
|
81
|
+
input_tokens?: number
|
|
82
|
+
output_tokens?: number
|
|
83
|
+
cache_read_input_tokens?: number
|
|
84
|
+
cache_creation_input_tokens?: number
|
|
85
|
+
}
|
|
86
|
+
message?: { model?: string; content?: Array<{ type?: string }> }
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
event = JSON.parse(line)
|
|
90
|
+
} catch {
|
|
91
|
+
continue
|
|
92
|
+
}
|
|
93
|
+
if (event.type === 'system' && typeof event.model === 'string' && event.model && out.model === undefined) {
|
|
94
|
+
out.model = event.model
|
|
95
|
+
} else if (event.type === 'assistant') {
|
|
96
|
+
const content = Array.isArray(event.message?.content) ? event.message.content : []
|
|
97
|
+
out.toolCalls += content.filter((c) => c?.type === 'tool_use').length
|
|
98
|
+
if (out.model === undefined && typeof event.message?.model === 'string' && event.message.model)
|
|
99
|
+
out.model = event.message.model
|
|
100
|
+
} else if (event.type === 'result') {
|
|
101
|
+
const u = event.usage ?? {}
|
|
102
|
+
out.inputTokens = u.input_tokens ?? 0
|
|
103
|
+
out.outputTokens = u.output_tokens ?? 0
|
|
104
|
+
out.cacheReadTokens = u.cache_read_input_tokens ?? 0
|
|
105
|
+
out.cacheCreationTokens = u.cache_creation_input_tokens ?? 0
|
|
106
|
+
out.turns = event.num_turns ?? 0
|
|
107
|
+
out.costUsd = event.total_cost_usd ?? 0
|
|
108
|
+
out.capped = event.subtype !== 'success' || event.is_error === true
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return out
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const HARNESS = 'claude-code'
|
|
115
|
+
|
|
116
|
+
function launchArgs(o: LaunchOptions): string[] {
|
|
117
|
+
const facts = headlessInvocation(HARNESS)
|
|
118
|
+
const cmd = headlessCommand(HARNESS, {
|
|
119
|
+
prompt: o.prompt,
|
|
120
|
+
model: o.model,
|
|
121
|
+
permission: o.permissionMode,
|
|
122
|
+
outputFormat: 'stream-json',
|
|
123
|
+
})
|
|
124
|
+
if (!cmd || facts.supported !== true)
|
|
125
|
+
throw new Error('@cyberuni/agent-harness reports no headless mode for claude-code')
|
|
126
|
+
const args = [...cmd.args]
|
|
127
|
+
// The prompt is placed right after the facts' own (boolean-terminated) flags, so a variadic flag
|
|
128
|
+
// added below can never swallow it.
|
|
129
|
+
const disable = facts.transcript?.disable ?? []
|
|
130
|
+
const extra = [
|
|
131
|
+
'--max-budget-usd',
|
|
132
|
+
String(o.maxBudgetUsd),
|
|
133
|
+
...disable,
|
|
134
|
+
...(disable.includes('--no-session-persistence') ? [] : ['--no-session-persistence']),
|
|
135
|
+
'--setting-sources',
|
|
136
|
+
'project',
|
|
137
|
+
'--strict-mcp-config',
|
|
138
|
+
]
|
|
139
|
+
if (o.pluginDir) extra.push('--plugin-dir', o.pluginDir)
|
|
140
|
+
if (existsSync(join(o.checkout, '.mcp.json'))) extra.push('--mcp-config', '.mcp.json')
|
|
141
|
+
return [...args, ...extra]
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export const claudeCode: Adapter = {
|
|
145
|
+
harness: HARNESS,
|
|
146
|
+
name: 'aced.claude-code',
|
|
147
|
+
runner: 'print',
|
|
148
|
+
executable: (() => {
|
|
149
|
+
const facts = headlessInvocation(HARNESS)
|
|
150
|
+
return facts.supported === true ? facts.executable : 'claude'
|
|
151
|
+
})(),
|
|
152
|
+
launch(o) {
|
|
153
|
+
const r = spawnSync(this.executable, launchArgs(o), {
|
|
154
|
+
cwd: o.checkout,
|
|
155
|
+
encoding: 'utf8',
|
|
156
|
+
timeout: o.timeoutMs,
|
|
157
|
+
killSignal: 'SIGKILL',
|
|
158
|
+
maxBuffer: 512 * 1024 * 1024,
|
|
159
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
160
|
+
env: { ...process.env, ...o.env },
|
|
161
|
+
})
|
|
162
|
+
const code = (r.error as NodeJS.ErrnoException | undefined)?.code
|
|
163
|
+
if (r.error && code !== 'ETIMEDOUT' && code !== 'ENOBUFS')
|
|
164
|
+
throw new Error(`could not launch ${this.executable}: ${r.error.message}`)
|
|
165
|
+
const transcript = r.stdout ?? ''
|
|
166
|
+
const parsed = parseStreamJson(transcript)
|
|
167
|
+
// Killed — by the timeout or any other signal — is a stop without a success result.
|
|
168
|
+
if (r.error || r.signal) parsed.capped = true
|
|
169
|
+
return { ...parsed, transcript }
|
|
170
|
+
},
|
|
171
|
+
isolatedConfig(runTemp) {
|
|
172
|
+
const dir = join(runTemp, 'claude-config')
|
|
173
|
+
mkdirSync(dir, { recursive: true })
|
|
174
|
+
const operator = process.env.CLAUDE_CONFIG_DIR || join(homedir(), '.claude')
|
|
175
|
+
const credentials = join(operator, '.credentials.json')
|
|
176
|
+
if (existsSync(credentials)) copyFileSync(credentials, join(dir, '.credentials.json'))
|
|
177
|
+
return { CLAUDE_CONFIG_DIR: dir }
|
|
178
|
+
},
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const ADAPTERS: Record<string, Adapter> = { [HARNESS]: claudeCode }
|
|
182
|
+
|
|
183
|
+
export function adapterFor(harness: string): Adapter | undefined {
|
|
184
|
+
return Object.hasOwn(ADAPTERS, harness) ? ADAPTERS[harness] : undefined
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export const ADAPTER_HARNESSES = Object.keys(ADAPTERS)
|