@zenera/cli 1.1.22 → 1.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -2
- package/dist/archive.d.ts +139 -0
- package/dist/archive.js +376 -0
- package/dist/ask.d.ts +51 -0
- package/dist/ask.js +143 -0
- package/dist/batch.d.ts +77 -0
- package/dist/batch.js +338 -0
- package/dist/command.d.ts +6 -0
- package/dist/commands/export.d.ts +3 -0
- package/dist/commands/export.js +255 -0
- package/dist/commands/import.d.ts +3 -0
- package/dist/commands/import.js +173 -0
- package/dist/commands/index.d.ts +5 -1
- package/dist/commands/index.js +13 -1
- package/dist/commands/inspect.d.ts +11 -0
- package/dist/commands/inspect.js +388 -89
- package/dist/commands/list.js +128 -19
- package/dist/commands/memory.js +453 -16
- package/dist/commands/models.js +14 -2
- package/dist/commands/run.js +139 -34
- package/dist/commands/sandbox.js +72 -1
- package/dist/engine.d.ts +19 -0
- package/dist/engine.js +60 -8
- package/dist/liveness.js +6 -3
- package/dist/main.js +5 -2
- package/dist/podman.d.ts +16 -0
- package/dist/podman.js +115 -2
- package/dist/progress.d.ts +33 -0
- package/dist/progress.js +402 -0
- package/dist/request.d.ts +45 -0
- package/dist/request.js +300 -0
- package/dist/sandbox.js +10 -0
- package/dist/scaffold.d.ts +8 -0
- package/dist/scaffold.js +4 -1
- package/dist/session.d.ts +26 -1
- package/dist/session.js +42 -3
- package/dist/term.d.ts +3 -0
- package/dist/term.js +17 -1
- package/dist/trace.d.ts +84 -0
- package/dist/trace.js +686 -0
- package/dist/validate.js +93 -4
- package/package.json +12 -6
- package/templates/editor/.github/copilot-instructions.md +11 -6
- package/templates/editor/.github/prompts/finetune.prompt.md +251 -0
- package/templates/editor/.github/prompts/spec-sync-project.prompt.md +25 -2
- package/templates/editor/.github/skills/zen-cli/SKILL.md +14 -3
- package/templates/editor/.github/skills/zen-cli/references/check.md +13 -5
- package/templates/editor/.github/skills/zen-cli/references/export.md +107 -0
- package/templates/editor/.github/skills/zen-cli/references/inspect.md +262 -21
- package/templates/editor/.github/skills/zen-cli/references/memory.md +123 -15
- package/templates/editor/.github/skills/zen-cli/references/models.md +1 -1
- package/templates/editor/.github/skills/zen-cli/references/projects.md +12 -5
- package/templates/editor/.github/skills/zen-cli/references/rag.md +6 -0
- package/templates/editor/.github/skills/zen-cli/references/run.md +201 -4
- package/templates/editor/.github/skills/zen-finetune/SKILL.md +1000 -0
- package/templates/editor/.github/skills/zen-finetune/scripts/collect.sh +432 -0
- package/templates/editor/.github/skills/zen-finetune/scripts/sample.sh +199 -0
- package/templates/editor/.github/skills/zen-inspect/SKILL.md +396 -0
- package/templates/editor/.github/skills/zen-memory/SKILL.md +74 -28
- package/templates/editor/.github/skills/zen-memory-warmup/SKILL.md +370 -408
- package/templates/editor/.github/skills/zen-sandbox/SKILL.md +19 -0
- package/templates/editor/.github/skills/zen-sandbox-capacity/SKILL.md +334 -0
- package/templates/editor/.github/skills/zen-sandbox-capacity/references/loadtest.md +181 -0
- package/templates/editor/.github/skills/zen-sandbox-capacity/scripts/preflight_sandbox.sh +457 -0
- package/templates/editor/.github/skills/zen-spec-sync/SKILL.md +6 -5
- package/templates/editor/.vscode/agents.schema.json +1 -1
- package/templates/editor/.vscode/settings.json +1 -0
- package/templates/project/SPECIFICATION.md.tmpl +2 -0
- package/templates/project/agents/files-instructions.md +13 -0
- package/templates/project/agents/fork-instructions.md +27 -0
- package/templates/project/agents/memory-instructions.md +80 -7
- package/templates/project/gitignore +3 -0
- package/templates/project/scripts/_setup.sh +49 -7
package/dist/validate.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { EXA_GROUP, FileSkillProvider, SANDBOX_MOUNT, Sandbox, exaTools, frontmatter, projectRegistry, readProjectConfig, sandboxTools, selectTools, toList,
|
|
1
|
+
import { EXA_GROUP, FileSkillProvider, SANDBOX_MOUNT, Sandbox, exaTools, fileTools, frontmatter, projectRegistry, readProjectConfig, sandboxTools, selectTools, toList, } from '@zenera/neo';
|
|
2
2
|
import { existsSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, } from 'node:fs';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
|
@@ -7,7 +7,7 @@ import { resolveBuild } from "./image.js";
|
|
|
7
7
|
import { SHAPES, envNames, form } from "./keys.js";
|
|
8
8
|
import { probeModels } from "./liveness.js";
|
|
9
9
|
import { BuildError, ensurePodmanReady } from "./podman.js";
|
|
10
|
-
import { staleShared } from "./scaffold.js";
|
|
10
|
+
import { ownVersion, staleShared } from "./scaffold.js";
|
|
11
11
|
// Mirrors the loader's own constants (`packages/neo/src/project/load.ts`).
|
|
12
12
|
// Duplicated rather than exported, because a check that agreed with the loader
|
|
13
13
|
// by construction could not report that the two had diverged.
|
|
@@ -35,10 +35,17 @@ const FILE_TOOLS = [
|
|
|
35
35
|
'find_files',
|
|
36
36
|
'write_file',
|
|
37
37
|
'apply_patch',
|
|
38
|
+
'copy_file',
|
|
38
39
|
'move_file',
|
|
39
40
|
'delete_file',
|
|
40
41
|
];
|
|
41
|
-
const FILE_WRITE_TOOLS = new Set([
|
|
42
|
+
const FILE_WRITE_TOOLS = new Set([
|
|
43
|
+
'write_file',
|
|
44
|
+
'apply_patch',
|
|
45
|
+
'copy_file',
|
|
46
|
+
'move_file',
|
|
47
|
+
'delete_file',
|
|
48
|
+
]);
|
|
42
49
|
function activeFileTools(selectors) {
|
|
43
50
|
const active = new Set();
|
|
44
51
|
if (!selectors?.length) {
|
|
@@ -607,7 +614,7 @@ export function availableTools(root, config) {
|
|
|
607
614
|
// Constructed, not started: a pool creates its container on the first
|
|
608
615
|
// command, so naming one here costs nothing and needs no container engine.
|
|
609
616
|
return [
|
|
610
|
-
...
|
|
617
|
+
...fileTools({
|
|
611
618
|
root,
|
|
612
619
|
mount: config.sandbox?.workdir ?? SANDBOX_MOUNT,
|
|
613
620
|
}),
|
|
@@ -1252,6 +1259,7 @@ function checkSandbox(root, config, record, add) {
|
|
|
1252
1259
|
if (!ok) {
|
|
1253
1260
|
return undefined;
|
|
1254
1261
|
}
|
|
1262
|
+
checkPin(join(root, file), file, add);
|
|
1255
1263
|
try {
|
|
1256
1264
|
return resolveBuild(root, config.sandbox);
|
|
1257
1265
|
}
|
|
@@ -1265,6 +1273,72 @@ function checkSandbox(root, config, record, add) {
|
|
|
1265
1273
|
return undefined;
|
|
1266
1274
|
}
|
|
1267
1275
|
}
|
|
1276
|
+
/** `@zenera/cli`, and the version after it when the line pins one. */
|
|
1277
|
+
const ZENERA_INSTALL = /@zenera\/([a-z0-9-]+)(?:@([^\s\\'"]+))?/g;
|
|
1278
|
+
/**
|
|
1279
|
+
* Whether the Dockerfile's `@zenera/*` installs name this `zen`.
|
|
1280
|
+
*
|
|
1281
|
+
* `zen init` writes the version it was run by and never rewrites the file
|
|
1282
|
+
* again — the scaffold keeps what the project already has — so the pin is fixed
|
|
1283
|
+
* at creation and falls behind the host on the next upgrade. Nothing else
|
|
1284
|
+
* notices: the file parses, the image builds, and the container simply holds a
|
|
1285
|
+
* different `zen` than the session outside it, which is a mismatch that shows
|
|
1286
|
+
* up as a tool behaving unlike its documentation.
|
|
1287
|
+
*
|
|
1288
|
+
* Read rather than run, so it is true on the laptop with no container engine —
|
|
1289
|
+
* the one `--no-sandbox` leaves with no other word on the subject.
|
|
1290
|
+
*/
|
|
1291
|
+
function checkPin(path, rel, add) {
|
|
1292
|
+
let text;
|
|
1293
|
+
try {
|
|
1294
|
+
// Existence is already a finding; an unreadable file is not worth a
|
|
1295
|
+
// second one, and a throw here would cost every check after it.
|
|
1296
|
+
text = readFileSync(path, 'utf8');
|
|
1297
|
+
}
|
|
1298
|
+
catch {
|
|
1299
|
+
return;
|
|
1300
|
+
}
|
|
1301
|
+
const mine = ownVersion();
|
|
1302
|
+
const stale = new Set();
|
|
1303
|
+
const loose = new Set();
|
|
1304
|
+
for (const line of text.split('\n')) {
|
|
1305
|
+
// A comment naming the package is prose about the pin, not the pin.
|
|
1306
|
+
if (/^\s*#/.test(line)) {
|
|
1307
|
+
continue;
|
|
1308
|
+
}
|
|
1309
|
+
for (const match of line.matchAll(ZENERA_INSTALL)) {
|
|
1310
|
+
const name = `@zenera/${match[1]}`;
|
|
1311
|
+
const pinned = match[2];
|
|
1312
|
+
if (pinned === undefined) {
|
|
1313
|
+
loose.add(name);
|
|
1314
|
+
}
|
|
1315
|
+
else if (pinned !== mine) {
|
|
1316
|
+
stale.add(`${name}@${pinned}`);
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
const fix = `pin them to ${mine}, the version \`zen --version\` reports`;
|
|
1321
|
+
if (stale.size > 0) {
|
|
1322
|
+
add({
|
|
1323
|
+
severity: 'warning',
|
|
1324
|
+
code: 'sandbox.pin.stale',
|
|
1325
|
+
where: rel,
|
|
1326
|
+
message: `installs ${[...stale].join(', ')}, but this \`zen\` is ${mine} — the CLI an ` +
|
|
1327
|
+
'agent runs inside the container is not the one that built the session',
|
|
1328
|
+
fix,
|
|
1329
|
+
});
|
|
1330
|
+
}
|
|
1331
|
+
if (loose.size > 0) {
|
|
1332
|
+
add({
|
|
1333
|
+
severity: 'warning',
|
|
1334
|
+
code: 'sandbox.pin.absent',
|
|
1335
|
+
where: rel,
|
|
1336
|
+
message: `installs ${[...loose].join(', ')} unpinned, so the next image build takes ` +
|
|
1337
|
+
'whatever is current and drifts away from the host without saying so',
|
|
1338
|
+
fix,
|
|
1339
|
+
});
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1268
1342
|
/** Written by the smoke test and read back, to prove the mount is real. */
|
|
1269
1343
|
const SMOKE = 'zen-check';
|
|
1270
1344
|
/**
|
|
@@ -1476,6 +1550,21 @@ function checkModels(root, config, keys, add, probe = false) {
|
|
|
1476
1550
|
if (resolved !== name) {
|
|
1477
1551
|
report.ref = resolved;
|
|
1478
1552
|
}
|
|
1553
|
+
// Reported here rather than left to the probe below: a knob for the
|
|
1554
|
+
// wrong vendor is a config mistake that wants naming whether or not
|
|
1555
|
+
// there is a credential to build the model with.
|
|
1556
|
+
if (role === 'model') {
|
|
1557
|
+
for (const stray of registry.knobs(ref)) {
|
|
1558
|
+
add({
|
|
1559
|
+
severity: 'error',
|
|
1560
|
+
code: 'model.knob',
|
|
1561
|
+
where: whereFor(config, role, name, usedBy),
|
|
1562
|
+
message: `\`${stray.knob}\` means nothing to a ${stray.target} model — ` +
|
|
1563
|
+
'it would never reach the request',
|
|
1564
|
+
fix: `this model takes: ${stray.supported.join(', ')}`,
|
|
1565
|
+
});
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1479
1568
|
}
|
|
1480
1569
|
catch (err) {
|
|
1481
1570
|
add({
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zenera/cli",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.24",
|
|
4
4
|
"description": "Command-line front end for @zenera/neo: agentic projects you can run, share and commit.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agents",
|
|
@@ -48,14 +48,20 @@
|
|
|
48
48
|
"prepack": "tsc -b"
|
|
49
49
|
},
|
|
50
50
|
"dependencies": {
|
|
51
|
-
"@inkjs/ui": "^2.0.0",
|
|
52
|
-
"ink": "^7.1.1",
|
|
53
|
-
"react": "^19.2.8",
|
|
54
|
-
"@zenera/neo": "^1.1.22",
|
|
55
51
|
"@anthropic-ai/sdk": "^0.120.0",
|
|
56
52
|
"@google/genai": "^2.18.0",
|
|
53
|
+
"@inkjs/ui": "^2.0.0",
|
|
57
54
|
"@openrouter/sdk": "^1.2.80",
|
|
55
|
+
"@zenera/neo": "^1.1.24",
|
|
58
56
|
"google-auth-library": "^10.0.0",
|
|
59
|
-
"
|
|
57
|
+
"ink": "^7.1.1",
|
|
58
|
+
"openai": "^6.0.0",
|
|
59
|
+
"react": "^19.2.8",
|
|
60
|
+
"yauzl": "^3.4.0",
|
|
61
|
+
"yazl": "^3.3.1"
|
|
62
|
+
},
|
|
63
|
+
"devDependencies": {
|
|
64
|
+
"@types/yauzl": "^3.4.0",
|
|
65
|
+
"@types/yazl": "^3.3.1"
|
|
60
66
|
}
|
|
61
67
|
}
|
|
@@ -47,7 +47,8 @@ visible in `agents.yaml` and that `zen check` does not report.** Before
|
|
|
47
47
|
reviewing or changing a project, list the capabilities it has turned on and load
|
|
48
48
|
the editor skill for each one - `zen-memory` for a `memory:` block,
|
|
49
49
|
`zen-sandbox` for sandbox configuration or shell tools, `zen-rag-schema` or
|
|
50
|
-
`zen-rag-docs` for an index, `zen-
|
|
50
|
+
`zen-rag-docs` for an index, `zen-sandbox-capacity` before any `zen run batch`,
|
|
51
|
+
`zen-cli` always. A capability that is already
|
|
51
52
|
configured and already passing `zen check` is exactly the case that looks
|
|
52
53
|
finished and is not: the obligation lives in the skill, so a review that never
|
|
53
54
|
opens the skill cannot find what is missing. This applies whatever prompted the
|
|
@@ -810,7 +811,7 @@ committed shows up as a skill with no files.
|
|
|
810
811
|
### 3.5 Tools
|
|
811
812
|
|
|
812
813
|
A tool is what an agent can _do_ rather than say. `zen run` provides three
|
|
813
|
-
groups - the
|
|
814
|
+
groups - the file tools (§3.6), the sandbox tools (§3.7) and the web tools
|
|
814
815
|
(§3.8) - and `agents.yaml` decides which agent holds which. Nothing else reaches
|
|
815
816
|
the machine, so `tools:` is the whole permission model: an agent that does not
|
|
816
817
|
name a tool cannot use it, whatever its prompt says.
|
|
@@ -846,6 +847,7 @@ them cannot see a file at all.
|
|
|
846
847
|
| `find_files` | Paths containing a substring, case-insensitive |
|
|
847
848
|
| `write_file` | Creates or overwrites a whole file, making parent directories |
|
|
848
849
|
| `apply_patch` | Edits by surrounding context rather than line numbers; several files atomically |
|
|
850
|
+
| `copy_file` | Copies, bytes never touching the context; the source may be a read-only mount |
|
|
849
851
|
| `move_file` | Moves or renames; refuses to clobber without `overwrite` |
|
|
850
852
|
| `delete_file` | Deletes; a directory needs `recursive` |
|
|
851
853
|
|
|
@@ -876,8 +878,8 @@ agents:
|
|
|
876
878
|
tools: [files:*]
|
|
877
879
|
|
|
878
880
|
- name: reviewer
|
|
879
|
-
# Everything except the
|
|
880
|
-
tools: [files:*, -write_file, -apply_patch, -move_file, -delete_file]
|
|
881
|
+
# Everything except the five that can change something.
|
|
882
|
+
tools: [files:*, -write_file, -apply_patch, -copy_file, -move_file, -delete_file]
|
|
881
883
|
```
|
|
882
884
|
|
|
883
885
|
Selectors apply in the order written, so a `-` line reads as an exception to the
|
|
@@ -1638,7 +1640,9 @@ over one connection.
|
|
|
1638
1640
|
| `includeThoughts` | gemini | Thought summaries; default `true` |
|
|
1639
1641
|
| `routing` / `fallbacks` | openrouter | Upstream provider preferences, and models to fall back to - §7.5 |
|
|
1640
1642
|
|
|
1641
|
-
Knobs that do not apply to the chosen vendor are
|
|
1643
|
+
Knobs that do not apply to the chosen vendor are rejected, not dropped: the
|
|
1644
|
+
schema accepts every one on every model, and building it fails with `does not
|
|
1645
|
+
read "<knob>"`. `zen check` reports the same thing as `model.knob`.
|
|
1642
1646
|
`api:` exists only for the OpenAI protocol - naming it on a Gemini or Anthropic
|
|
1643
1647
|
model is an error.
|
|
1644
1648
|
|
|
@@ -2012,7 +2016,8 @@ candidates; these are the judgements to make about each one)**
|
|
|
2012
2016
|
- [ ] No agent holds `write_file` without `apply_patch`, or `read_file` without
|
|
2013
2017
|
`list_dir` and `find_files`
|
|
2014
2018
|
- [ ] An agent that only reads is not holding `write_file`, `apply_patch`,
|
|
2015
|
-
`move_file` or `delete_file` - subtract them from `files:*`
|
|
2019
|
+
`copy_file`, `move_file` or `delete_file` - subtract them from `files:*`
|
|
2020
|
+
(or `workspace:*`)
|
|
2016
2021
|
- [ ] `sandbox:*` is granted only where a shell is actually needed
|
|
2017
2022
|
- [ ] `sandbox.persist: true`, unless a throwaway rootfs is wanted on purpose
|
|
2018
2023
|
- [ ] The `sandbox:` image carries what the work needs, rather than the prompt
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
---
|
|
2
|
+
description: Fine-tune this project against a training set in two stages - first without memory, tuning the prose; then with memory, tuning recall for speed.
|
|
3
|
+
---
|
|
4
|
+
|
|
5
|
+
Tune this project against real queries. Nothing here touches model weights: the
|
|
6
|
+
parameters are the agent prompts, the skills and the house rules under `agents/`,
|
|
7
|
+
and the evidence is the trajectories the queries produce.
|
|
8
|
+
|
|
9
|
+
**Load the `zen-finetune` skill before anything else**, and load `zen-inspect`
|
|
10
|
+
before you grade, `zen-instructions` before you write a policy file, and
|
|
11
|
+
`zen-sandbox-capacity` if a batch dies at 137 or 124. This prompt is the order of
|
|
12
|
+
the work; the skill is how each step is done, and it holds the rules that are
|
|
13
|
+
easy to get wrong.
|
|
14
|
+
|
|
15
|
+
This prompt is **reentrant**. Tuning is long and gets interrupted - a batch dies,
|
|
16
|
+
a session ends, I stop you mid-round. Every step leaves its result on disk under
|
|
17
|
+
`.finetune/`, so a second invocation does not start over: it reads that state,
|
|
18
|
+
works out which stage and which step the work stopped at, says so, and carries
|
|
19
|
+
on from there unless I tell you otherwise.
|
|
20
|
+
|
|
21
|
+
## The two stages
|
|
22
|
+
|
|
23
|
+
This is two stages, run in order, never interleaved. Say which stage you are in
|
|
24
|
+
at the top of every report.
|
|
25
|
+
|
|
26
|
+
**Stage 1 - without memory.** Every batch runs against an empty memory, so every
|
|
27
|
+
run starts from nothing and the result is attributable to the prose. Here you
|
|
28
|
+
tune agent prompts, `agents/instructions.md`, and the project's skills - creating
|
|
29
|
+
or updating them. The runs still _write_ memory, into their own throwaway graphs,
|
|
30
|
+
and what they write is graded too: the commits have to be generalisable, durable
|
|
31
|
+
facts, not cached live readings. Stage 1 optimises **correctness**.
|
|
32
|
+
|
|
33
|
+
**Stage 2 - with memory.** Only once stage 1 is stable and well behaved. The
|
|
34
|
+
prompts, skills and house rules are now frozen; the merged memory from the last
|
|
35
|
+
good cold batch is copied to every item, which reads it and writes into its own
|
|
36
|
+
copy, and the only file you edit is `agents/memory-policy-instructions.md`.
|
|
37
|
+
Stage 2 optimises **memory use and memory correctness**, and the expected result
|
|
38
|
+
is a dramatic drop in tokens and wall clock through reuse, with no verdict
|
|
39
|
+
changed.
|
|
40
|
+
|
|
41
|
+
Do not start stage 2 while any sample still fails cold. Memory will hide the
|
|
42
|
+
defect and it will come back the day the graph is rebuilt.
|
|
43
|
+
|
|
44
|
+
## Where you are - work this out before anything else
|
|
45
|
+
|
|
46
|
+
Do this every time this prompt runs, including the first - on a project that has
|
|
47
|
+
never been tuned it costs one `ls` and tells you to start at section 0.
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
ls .finetune/dataset.json .finetune/rounds 2>/dev/null
|
|
51
|
+
for r in .finetune/rounds/*/; do
|
|
52
|
+
printf '%s ' "$r"
|
|
53
|
+
for f in cases.json batch/batch.json findings.md changes.md; do
|
|
54
|
+
[ -e "$r$f" ] && printf '%s ' "$f"
|
|
55
|
+
done
|
|
56
|
+
echo
|
|
57
|
+
done
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The rounds are ordered by name, so the last one is the live one. Read its
|
|
61
|
+
`findings.md` heading for the stage - it says `COLD` or `WARM` - and its
|
|
62
|
+
`changes.md` for what was already applied. Then resume by the artefacts the round
|
|
63
|
+
is missing:
|
|
64
|
+
|
|
65
|
+
| State of the last round | Where the work stopped | Resume at |
|
|
66
|
+
| --------------------------------------- | ----------------------------------- | -------------------------------------------------------------------------- |
|
|
67
|
+
| no `.finetune/` at all | nothing has been done | section 0, then 1 |
|
|
68
|
+
| `dataset.json`, no `rounds/` | dataset built, never sampled | section 2 (stage 1) |
|
|
69
|
+
| `cases.json` only | sampled, batch never started | preflight, then run the batch into this same round |
|
|
70
|
+
| `batch/` present, no `batch/batch.json` | the batch was interrupted or killed | the round is **void** - see below, then re-run it |
|
|
71
|
+
| `batch/batch.json`, no `findings.md` | the batch finished, nothing graded | section 3 - `collect.sh oom` first |
|
|
72
|
+
| `findings.md`, no `changes.md` | graded, nothing applied yet | section 4 |
|
|
73
|
+
| `findings.md` and `changes.md` | edits applied, never measured | re-run the same cases into a **new** round: section 5 cold, section 6 warm |
|
|
74
|
+
| the last round's heading says `WARM` | stage 2 is in progress | the same rows, read against section 6 |
|
|
75
|
+
|
|
76
|
+
Two things the table does not see, so check them by hand:
|
|
77
|
+
|
|
78
|
+
- **Uncommitted edits under `agents/` with no `changes.md` to explain them.** An
|
|
79
|
+
apply was interrupted part-way. Read `git status --short agents/` and the diff,
|
|
80
|
+
finish writing `changes.md` from what is actually on disk, then re-run - do not
|
|
81
|
+
layer new edits on top of edits you have not recorded.
|
|
82
|
+
- **`zen check` before you resume anything.** A project left broken mid-edit
|
|
83
|
+
fails every item identically and looks like a catastrophic regression.
|
|
84
|
+
|
|
85
|
+
A round whose batch never wrote `batch.json` produced no evidence. Move it aside
|
|
86
|
+
rather than grading it or deleting it - `mv .finetune/rounds/rN/batch
|
|
87
|
+
.finetune/rounds/rN/batch.partial` - and re-run that round from the same
|
|
88
|
+
`cases.json`. The same applies to a round `collect.sh oom` calls void.
|
|
89
|
+
|
|
90
|
+
Resuming never re-samples and never re-merges. Re-use the first round's
|
|
91
|
+
`cases.json` and, in stage 2, the merged graph already sitting in the stage's
|
|
92
|
+
round directory - an interruption is not a reason to change the thing every
|
|
93
|
+
earlier round was measured against. Re-derive nothing from memory of the last
|
|
94
|
+
session either: `findings.md` and `changes.md` are the record, so read them.
|
|
95
|
+
|
|
96
|
+
Then tell me, in three lines before you do any work: which stage, which round and
|
|
97
|
+
step you are resuming at, and what you are about to run. Continue without waiting
|
|
98
|
+
for me up to the next natural checkpoint - the end of a round's report - because
|
|
99
|
+
that is where this prompt already stops and asks. If I named a stage or a round,
|
|
100
|
+
do that instead of what the state says.
|
|
101
|
+
|
|
102
|
+
**If nothing is left to do** - the last round is WARM, it met the stage-2 exit
|
|
103
|
+
criteria, and no prose or policy edit has landed since - do not start another
|
|
104
|
+
tuning round. Run one confirmation round instead: the same `cases.json`, the same
|
|
105
|
+
merged graph, the same flags, the same machine, into
|
|
106
|
+
`.finetune/rounds/rN-verify/`. Grade it, write `findings.md` with the heading
|
|
107
|
+
`VERIFIED` if no verdict regressed and the saving still holds, and change nothing.
|
|
108
|
+
If something did regress, that is a real finding: report it and re-open the stage
|
|
109
|
+
it belongs to.
|
|
110
|
+
|
|
111
|
+
## 0. Before the first round
|
|
112
|
+
|
|
113
|
+
Name the training set. If I have not given you one, ask - a specification
|
|
114
|
+
section, a file of examples, a transcript, a CSV. Fewer than about eight queries
|
|
115
|
+
is a debugging session, not a tuning loop; say so and read the runs singly
|
|
116
|
+
instead.
|
|
117
|
+
|
|
118
|
+
Then run the mechanical checks and report what they say:
|
|
119
|
+
|
|
120
|
+
```sh
|
|
121
|
+
zen check
|
|
122
|
+
.github/skills/zen-review/scripts/check-paths.sh
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
A project that does not load fails every item for one reason. Do not start a
|
|
126
|
+
round until `zen check` passes.
|
|
127
|
+
|
|
128
|
+
## 1. Dataset
|
|
129
|
+
|
|
130
|
+
Build `.finetune/dataset.json` if it is not there, and read it if it is. Read the
|
|
131
|
+
source yourself and write the structure out - do not write a parser. Rubrics are
|
|
132
|
+
copied **verbatim**, one entry per bullet. Classify only the distinctions the
|
|
133
|
+
source already makes, and rate complexity by the trajectory the query demands,
|
|
134
|
+
not by the length of the sentence. Keep ids stable across re-extractions.
|
|
135
|
+
|
|
136
|
+
## 2. Stage 1: sample, preflight, run cold
|
|
137
|
+
|
|
138
|
+
```sh
|
|
139
|
+
.github/skills/zen-finetune/scripts/sample.sh -n 16 --seed 1 -o .finetune/rounds/r1/cases.json
|
|
140
|
+
.github/skills/zen-sandbox-capacity/scripts/preflight_sandbox.sh --concurrency 8
|
|
141
|
+
zen run batch --input .finetune/rounds/r1/cases.json \
|
|
142
|
+
--batch-dir .finetune/rounds/r1/batch --memory .finetune/empty --concurrency 8
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Every stage-1 round is cold and every round re-uses the first round's
|
|
146
|
+
`cases.json`. A warm graph lets a run succeed by recall instead of by
|
|
147
|
+
instruction, and a re-sampled batch means a moved score has two causes and you
|
|
148
|
+
cannot tell which.
|
|
149
|
+
|
|
150
|
+
## 3. Stage 1: grade
|
|
151
|
+
|
|
152
|
+
`collect.sh oom` first - a round with any exit 137 is void, is not graded, and
|
|
153
|
+
its tokens are compared to nothing. Then read the graphs before the numbers.
|
|
154
|
+
Grade every item against all eight criteria in the skill: rubric compliance,
|
|
155
|
+
optimality, memory hygiene, fork, delegation, tools and skills, grounding,
|
|
156
|
+
failure handling. **Cite a node id for every finding.** A finding you cannot
|
|
157
|
+
point at in the graph did not happen.
|
|
158
|
+
|
|
159
|
+
Grade the memory each item _wrote_ as seriously as the answer it gave. Read
|
|
160
|
+
`<id>/memory/` across the batch: every commit should be an operation, a pointer,
|
|
161
|
+
a shape or a rule that is still true next week. A cached live reading poisons
|
|
162
|
+
every future run, and committing nothing after learning something durable throws
|
|
163
|
+
the round away. Commit policy is fixed here, cold, while the cause is still
|
|
164
|
+
attributable - not in stage 2.
|
|
165
|
+
|
|
166
|
+
Write `.finetune/rounds/rN/findings.md`: the verdict block first - `PROGRESS`,
|
|
167
|
+
`NO CHANGE`, `REGRESSION`, `MIXED` or `VOID`, the `memory:` and `machine:` lines
|
|
168
|
+
as printed, and the `compare` table - then the per-sample grading. The heading
|
|
169
|
+
says `COLD` or `WARM`.
|
|
170
|
+
|
|
171
|
+
## 4. Stage 1: generalise, then apply
|
|
172
|
+
|
|
173
|
+
Collect every finding into one table before writing a word of instruction. The
|
|
174
|
+
unit of change is a pattern across findings, not a finding: three items missing
|
|
175
|
+
the same beat is a rule, one item is a note in `findings.md`. Write the rule one
|
|
176
|
+
level up from the sample that produced it - a rule that names the dataset's own
|
|
177
|
+
endpoints makes the eval pass and the product fail. Prefer mechanical wording,
|
|
178
|
+
and remember that a prohibition with a self-judged exemption is a permission.
|
|
179
|
+
|
|
180
|
+
Route each rule by what kind of thing it is: one agent's behaviour to its prompt,
|
|
181
|
+
a fact or a procedure to a skill - update an existing one or create a new one, a
|
|
182
|
+
standing rule to `agents/instructions.md`, a rule about one capability to a new
|
|
183
|
+
`agents/<topic>-policy-instructions.md`.
|
|
184
|
+
|
|
185
|
+
You may edit prompts, skills, house rules and new policy files. You may not edit
|
|
186
|
+
`agents/memory-instructions.md`, `agents/fork-instructions.md`,
|
|
187
|
+
`agents/tools-instructions.md` or `agents/files-instructions.md` - `zen check
|
|
188
|
+
--fix` overwrites them. You may not change a rubric to make a run pass. Tell me
|
|
189
|
+
before you change `agents.yaml`; that finding is structural.
|
|
190
|
+
|
|
191
|
+
Record every edit and the finding numbers behind it in
|
|
192
|
+
`.finetune/rounds/rN/changes.md`, then run `zen check` again.
|
|
193
|
+
|
|
194
|
+
## 5. Stage 1: loop, and leave the stage
|
|
195
|
+
|
|
196
|
+
Re-run the same cases into a new round directory and compare per sample and on
|
|
197
|
+
the aggregate. The verdict column decides, not the token column. A sample that
|
|
198
|
+
passed and now fails is the most valuable result in the round and is almost
|
|
199
|
+
always the new instruction. If the instruction files grew by a third and the
|
|
200
|
+
score moved by one sample, revert and say so.
|
|
201
|
+
|
|
202
|
+
Report to me after each round and wait: the verdict block, what you changed, and
|
|
203
|
+
what you propose to change next. Stop when a round produces no generalisable
|
|
204
|
+
pattern - that is the remaining failure being the model's rather than the prose's.
|
|
205
|
+
|
|
206
|
+
Stage 1 ends when, on one cold round, **every sample holds** and **what the runs
|
|
207
|
+
wrote to memory is worth keeping**. Tell me both before proposing stage 2.
|
|
208
|
+
|
|
209
|
+
## 6. Stage 2: memory
|
|
210
|
+
|
|
211
|
+
Freeze the prompts, the skills and the house rules. Merge the last good cold
|
|
212
|
+
round's memory into one graph, read `zen memory stats`, and run the same cases
|
|
213
|
+
against it with the same flags the cold round used - `--memory <merged graph>`
|
|
214
|
+
and nothing else. Each item copies that graph, recalls from the copy and writes
|
|
215
|
+
back into the copy, so every item starts from the same memory and the source is
|
|
216
|
+
left untouched. Do not use `--memory-read-only`: it removes the commit tool and
|
|
217
|
+
changes the memory prose, which would make the warm round differ from its cold
|
|
218
|
+
baseline in two ways instead of one.
|
|
219
|
+
|
|
220
|
+
Compare against the cold round the memory came from.
|
|
221
|
+
|
|
222
|
+
The target is reuse: recall should replace the discovery the cold runs repeated
|
|
223
|
+
every time, so the warm batch is dramatically cheaper in tokens, calls and wall
|
|
224
|
+
clock with no verdict changed. A warm round that costs about the same, or more,
|
|
225
|
+
means memory was carried and not used - that is a recall-timing finding, not a
|
|
226
|
+
reason for more memory. A warm round that is much cheaper still needs one
|
|
227
|
+
trajectory opened to confirm the saving is recall replacing _discovery_ and not
|
|
228
|
+
recall replacing a _live reading_.
|
|
229
|
+
|
|
230
|
+
Grade what the warm items committed as well as what they recalled - this is the
|
|
231
|
+
one place it can be seen. A node that duplicates something the same run just
|
|
232
|
+
recalled is waste, and a correction committed without `SUPERSEDES` beside the
|
|
233
|
+
stale node it contradicts is worse than waste.
|
|
234
|
+
|
|
235
|
+
In stage 2 the only file you edit is `agents/memory-policy-instructions.md`. Keep
|
|
236
|
+
the merged graph byte-identical for the whole stage: do not re-merge between
|
|
237
|
+
rounds, and never merge a warm round's own copies back into it - either changes
|
|
238
|
+
the memory and the prose at once. If a finding cannot be written as a recall or
|
|
239
|
+
re-commit rule, it is a stage-1 finding that escaped - tell me, and we re-open
|
|
240
|
+
stage 1.
|
|
241
|
+
|
|
242
|
+
Stage 2 ends when no verdict regressed, the saving is obvious next to the noise
|
|
243
|
+
floor, every large saving is explained, and nothing was committed that memory
|
|
244
|
+
already held.
|
|
245
|
+
|
|
246
|
+
Commit `dataset.json`, `cases.json`, `findings.md` and `changes.md` - they are
|
|
247
|
+
the eval history, and they are what a resumed session reads to find out where it
|
|
248
|
+
got to. Write each of them as its step ends rather than at the end of the round,
|
|
249
|
+
so an interruption never costs more than the step it lands in. Add
|
|
250
|
+
`.finetune/rounds/*/batch/` and `.finetune/empty/` to `.gitignore` - the batch
|
|
251
|
+
directories are large and may hold live API responses.
|
|
@@ -115,7 +115,10 @@ the model to compute or remember. Specifically:
|
|
|
115
115
|
in it to the version of `zen` in hand. Read that version with `zen --version`
|
|
116
116
|
and write it into the `npm install -g` line, so the CLI an agent runs inside
|
|
117
117
|
the container is the same one that built it. An unpinned name drifts away from
|
|
118
|
-
the host on the next image build; a stale pin
|
|
118
|
+
the host on the next image build; a stale pin leaves the container holding an
|
|
119
|
+
older `zen` than the session outside it. `zen check` reports both, as
|
|
120
|
+
`sandbox.pin.absent` and `sandbox.pin.stale`, but will not repair either - the
|
|
121
|
+
Dockerfile is the project's file, so this edit is yours to make.
|
|
119
122
|
|
|
120
123
|
Never delete an **extra** without saying so. Report it, say why the
|
|
121
124
|
specification does not cover it, and leave it unless I tell you to remove it.
|
|
@@ -148,6 +151,18 @@ not a paragraph of instructions for a human.
|
|
|
148
151
|
to a temporary path and moving it into place so an interrupted run never leaves
|
|
149
152
|
a half-built artefact behind. `scripts/_setup.sh --force` redoes everything,
|
|
150
153
|
and reaches each step as `$FORCE=1`.
|
|
154
|
+
- **Every step answers a dry run.** `scripts/_setup.sh --check` reaches each step
|
|
155
|
+
as `$CHECK=1` and means: make the same "already done" test, then do none of the
|
|
156
|
+
work. Exit **3** when the answer is yes, otherwise say what is missing and exit
|
|
157
|
+
**4**, which the runner reports as `needs setup`. This is the one command that
|
|
158
|
+
says whether the project can run without spending the download and the
|
|
159
|
+
embedding to find out, so the line a step prints under `$CHECK` is the whole
|
|
160
|
+
value of it: name the artefact and the command that produces it. Pass a tool's
|
|
161
|
+
own words through rather than writing a second opinion: `zen rag docs ready`
|
|
162
|
+
without `--quiet` already prints what is wrong and the `restore` or `index`
|
|
163
|
+
that fixes it. A check must not assume an earlier step has run: a missing
|
|
164
|
+
prerequisite is a 4 of its own, not a failure, or the first unbuilt thing
|
|
165
|
+
hides everything after it.
|
|
151
166
|
- **A step's "already done" test must name something that survives a clone.**
|
|
152
167
|
Git-ignored output does not. A rag index commits its `manifest.json` and
|
|
153
168
|
ignores `lance/`, so `[ -f "$OUT/manifest.json" ]` is true on a fresh checkout
|
|
@@ -164,6 +179,10 @@ not a paragraph of instructions for a human.
|
|
|
164
179
|
echo "$OUT is already built"
|
|
165
180
|
exit 3
|
|
166
181
|
fi
|
|
182
|
+
if [ "${CHECK:-0}" = 1 ]; then
|
|
183
|
+
zen rag docs ready --dir "$OUT" || true # its reason and its fix
|
|
184
|
+
exit 4
|
|
185
|
+
fi
|
|
167
186
|
if [ "${FORCE:-0}" = 0 ] && [ -f "$OUT/manifest.json" ]; then
|
|
168
187
|
zen rag docs restore --dir "$OUT" # committed index, ignored vectors
|
|
169
188
|
exit 0
|
|
@@ -175,7 +194,8 @@ not a paragraph of instructions for a human.
|
|
|
175
194
|
|
|
176
195
|
- **It must be watchable.** `scripts/_setup.sh` tees each step's output to
|
|
177
196
|
`.tmp/logs/setup-<step>.log`, prints a heartbeat line while a long step runs,
|
|
178
|
-
and finishes with one line per step: `ok`, `skipped`, or
|
|
197
|
+
and finishes with one line per step: `ok`, `skipped`, `needs setup`, or
|
|
198
|
+
`failed`.
|
|
179
199
|
- **Everything transient goes under `.tmp/`** - logs, scratch files, downloads,
|
|
180
200
|
test output, and the temporary path a step writes to before moving its
|
|
181
201
|
artefact into place. Nothing else in the tree is a scratch directory, and
|
|
@@ -379,6 +399,9 @@ background and follow the logs instead of waiting blind:
|
|
|
379
399
|
for the same reason, stop and raise it in `SPECIFICATION-FEEDBACK.md`.
|
|
380
400
|
- Prove re-entrancy: once it has succeeded, run `scripts/_setup.sh` once more and
|
|
381
401
|
check that every step reports `skipped` rather than rebuilding.
|
|
402
|
+
- Prove the dry run: `scripts/_setup.sh --check` must then report every step
|
|
403
|
+
`skipped` and exit 0. `needs setup` for something setup has just built means
|
|
404
|
+
the step's two tests disagree, and one of them is wrong.
|
|
382
405
|
- **`skipped` on a machine that has never built it is a bug, not a pass.** A
|
|
383
406
|
step that tests for a committed file skips on a fresh clone and leaves the
|
|
384
407
|
project unusable. For every index the project builds, check it can actually
|
|
@@ -17,14 +17,17 @@ failed, `2` usage, `3` invalid project, `4` no usable credential, `5` sandbox.
|
|
|
17
17
|
|
|
18
18
|
```
|
|
19
19
|
zen init [dir] [--name <name>] [--model <ref>] [--force]
|
|
20
|
-
zen list [--sessions] [--prune]
|
|
20
|
+
zen list [--sessions] [--runs] [--limit <n>] [--prune]
|
|
21
21
|
zen run [project] [prompt] [options]
|
|
22
|
+
zen run batch --input <file> [--batch-dir <dir>] [--concurrency <n>] [--memory-read-only]
|
|
22
23
|
zen open [project] [--editor <cmd>] [--wait]
|
|
23
24
|
zen key <ls|add|use|check|rm|show|env> [ref] [options]
|
|
24
25
|
zen models <providers|ls|search|show|test|pick> [ref] [options]
|
|
25
26
|
zen check [name|dir] [--project <name|dir>] [--no-sandbox] [--no-models] [--strict] [--quiet]
|
|
26
|
-
zen inspect [run] [--
|
|
27
|
-
zen memory [stats|ls|show|export|forget] [args] [options]
|
|
27
|
+
zen inspect [report|graph|node|ask] [run] [--dir <run dir>] [--part <name>] [--full] [--open]
|
|
28
|
+
zen memory [stats|ls|search|grep|show|export|merge|forget] [args] [options]
|
|
29
|
+
zen export [project] [--out <file>] [--no-vectors] [--no-memory] [--force]
|
|
30
|
+
zen import <file.zip> [dir] [--name <name>] [--force] [--no-register]
|
|
28
31
|
zen sandbox [status|up|pull|clean|disk] [options]
|
|
29
32
|
zen version
|
|
30
33
|
|
|
@@ -44,12 +47,15 @@ guess a flag, and do not read them all.
|
|
|
44
47
|
| Global flags, `--json`, exit codes, environment, where files live | [frame.md](./references/frame.md) |
|
|
45
48
|
| Creating, finding or opening a project | [projects.md](./references/projects.md) |
|
|
46
49
|
| Running: the TUI, one-shot answers, sessions, workspaces, overrides | [run.md](./references/run.md) |
|
|
50
|
+
| Many questions at once: `zen run batch`, its file, where answers go | [run.md](./references/run.md) |
|
|
47
51
|
| Validating: `zen check`, what it can and cannot see | [check.md](./references/check.md) |
|
|
48
52
|
| API keys, providers, the keyring, "no credential" errors | [keys.md](./references/keys.md) |
|
|
49
53
|
| Which models exist, whether one works, recovering from a refusal | [models.md](./references/models.md) |
|
|
50
54
|
| The container shell commands run in, images, `persist` | [sandbox.md](./references/sandbox.md) (and `zen-sandbox` skill) |
|
|
51
55
|
| Run reports, trajectories, what a session directory holds | [inspect.md](./references/inspect.md) |
|
|
56
|
+
| Reading a run yourself: the node graph, opening a node by id | [inspect.md](./references/inspect.md) (and `zen-inspect` skill) |
|
|
52
57
|
| What the agents remember, why something was recalled, forgetting | [memory.md](./references/memory.md) |
|
|
58
|
+
| Sharing a project: archives, what travels, importing one safely | [export.md](./references/export.md) |
|
|
53
59
|
| `zen faker` - a mock API from an OpenAPI/Swagger document | [faker.md](./references/faker.md) |
|
|
54
60
|
| `zen rag` - searching an API description, or a pile of markdown | [rag.md](./references/rag.md) |
|
|
55
61
|
|
|
@@ -60,8 +66,13 @@ zen init scaffold a project here, pick a model
|
|
|
60
66
|
zen check validate everything before spending a turn
|
|
61
67
|
zen run the TUI
|
|
62
68
|
zen run "what changed?" one answer, this directory as the workspace
|
|
69
|
+
zen run batch --input cases.json many questions at once, each isolated
|
|
63
70
|
zen inspect --open what the model was actually given
|
|
71
|
+
zen inspect graph --dir <run dir> the same run as a graph you can read
|
|
72
|
+
zen inspect node n14..n20 --dir <d> the nodes behind the ids, in full
|
|
64
73
|
zen memory export --open the whole memory graph, as a page
|
|
74
|
+
zen export the project as a zip you can send
|
|
75
|
+
zen import a.zip unpack one somebody sent, run nothing in it
|
|
65
76
|
zen key ls --check which credentials still work
|
|
66
77
|
zen models test <ref> whether one model actually answers
|
|
67
78
|
zen models pick --embedding the first embedder that does, on stdout
|
|
@@ -51,6 +51,11 @@ checked too. A word that is neither is a usage error (exit 2), not a report.
|
|
|
51
51
|
exist, and - unless `--no-sandbox` - the image **builds** and one command runs
|
|
52
52
|
in it, against a temporary directory rather than your workspace. No container
|
|
53
53
|
engine at all is a warning, not an error.
|
|
54
|
+
- The `@zenera/*` the Dockerfile installs are pinned to **this** `zen`. The pin
|
|
55
|
+
is written once, by `zen init`, and nothing rewrites it afterwards, so it falls
|
|
56
|
+
behind the host on the next upgrade and the container ends up holding a
|
|
57
|
+
different `zen` than the session outside it. Read rather than run, so this one
|
|
58
|
+
is reported with `--no-sandbox` too.
|
|
54
59
|
|
|
55
60
|
### Findings
|
|
56
61
|
|
|
@@ -66,10 +71,11 @@ fork.unknown skills.missing skill.unloadable / .no-skill-md
|
|
|
66
71
|
skills.no-catalog skills.unreachable skill.unused
|
|
67
72
|
assets.missing assets.overbroad sandbox.dockerfile.missing
|
|
68
73
|
sandbox.build sandbox.smoke sandbox.start / .unchecked
|
|
74
|
+
sandbox.pin.stale sandbox.pin.absent
|
|
69
75
|
provider.invalid model.none model.unresolvable
|
|
70
|
-
model.refused
|
|
71
|
-
embedding.refused
|
|
72
|
-
credential.*
|
|
76
|
+
model.knob model.refused model.unreachable
|
|
77
|
+
model.unusable embedding.refused embedding.unreachable
|
|
78
|
+
embedding.unusable credential.* service.credential
|
|
73
79
|
```
|
|
74
80
|
|
|
75
81
|
The report goes to **stdout** - it is the answer. `--json` gives the same
|
|
@@ -81,8 +87,10 @@ required it.
|
|
|
81
87
|
|
|
82
88
|
### What it cannot catch
|
|
83
89
|
|
|
84
|
-
Combinations that are only rejected by the provider at the first call.
|
|
85
|
-
|
|
90
|
+
Combinations that are only rejected by the provider at the first call. A knob
|
|
91
|
+
written for the wrong vendor is no longer one of them - `model.knob` names it,
|
|
92
|
+
and building the model refuses outright. The known one that remains:
|
|
93
|
+
**OpenAI reasoning and tools only meet on the responses API.** A
|
|
86
94
|
model with `reasoningEffort` and tools on the default chat-completions API is a
|
|
87
95
|
valid configuration that fails at runtime with _"Function tools with
|
|
88
96
|
reasoning_effort are not supported … in /v1/chat/completions"_. Set
|