@zenera/cli 1.1.24 → 1.1.26

package/templates/editor/.github/skills/zen-finetune/scripts/difficult.mjs

@@ -0,0 +1,243 @@
+#!/usr/bin/env node
+// @ts-check
+//
+// Keep the list of difficult cases: the ones that caused the most trouble.
+//
+// A case goes on the list when grading shows it is hard for the project - it
+// stayed wrong after a fix, it broke as a recheck case, it flipped between right
+// and wrong on the same prose, memory made it wrong or no cheaper, or it costs
+// far more than its batch. batch.mjs
+// puts open cases first in every later batch's rechecks, and fills whole batches
+// with them once the selection is used up, until they are fixed; fixed ones stay
+// ahead of the other rechecks, because what was hard once breaks again first.
+//
+// A case is `stuck` once `maxRetries` batches (config.json, default 3) have held
+// it since it was added, without it being fixed: that is the model's ceiling or a
+// rubric problem, and it is reported rather than retried forever.
+//
+// Run it from anywhere; it finds the project root from its own location.
+// See .github/skills/zen-finetune/SKILL.md, "Difficult cases".
+
+import { existsSync, readdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const NAME = 'difficult.mjs';
+
+const REASONS = ['wrong', 'regressed', 'flaky', 'memory', 'costly'];
+
+const USAGE = `Keep the list of difficult cases: the ones that caused the most trouble.
+
+  ${NAME}                                      the list: status, reasons, retries
+  ${NAME} add <id...> --why <reason> --run <run> [--note <text>]
+  ${NAME} fix <id...> --run <run>
+
+  reasons   wrong       still wrong after a run that tried to fix it
+            regressed   a recheck case that failed
+            flaky       right in one run, wrong in a later run of the same cases
+            memory      with memory: wrong, no cheaper, or re-committing what it
+                        recalled - still, after a run that tried to fix it
+            costly      over 2x the batch's median llm calls in its passing run
+
+  status    open        comes first in every later batch until fixed
+            fixed       right in a PASSED run; still preferred as a recheck case
+            stuck       open after maxRetries batches; reported, not retried
+
+  <run> is a run directory name, e.g. batch03-nomem-run2 or batch03-mem-run1`;
+
+process.stdout.on('error', (err) => {
+    if (/** @type {NodeJS.ErrnoException} */ (err).code === 'EPIPE') {
+        process.exit(0);
+    }
+});
+
+process.chdir(resolve(dirname(fileURLToPath(import.meta.url)), '../../../..'));
+
+const FILE = 'finetune/difficult.json';
+const CONFIG = 'finetune/config.json';
+const RUNS = 'finetune/runs';
+const RUN_NAME = /^batch(\d{2})-(nomem|mem)-run(\d+)$/;
+
+/** @param {string} msg @param {number} [code] @returns {never} */
+function die(msg, code = 2) {
+    console.error(`${NAME}: ${msg}`);
+    process.exit(code);
+}
+
+/** @param {string} path @returns {any} */
+function json(path) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    } catch {
+        return undefined;
+    }
+}
+
+/**
+ * @typedef {{ id: string, reasons: string[], added: string, notes: string[],
+ *             fixed?: string }} Entry
+ */
+
+/** @type {{ cases: Entry[] }} */
+const doc = json(FILE) ?? { cases: [] };
+const maxRetries = Number(json(CONFIG)?.maxRetries) || 3;
+
+/**
+ * How many batches have held it since it was added, read off the first
+ * no-memory run of every batch.
+ *
+ * @param {Entry} e
+ * @returns {number}
+ */
+function retries(e) {
+    const since = Number(RUN_NAME.exec(e.added)?.[1] ?? 0);
+    /** @type {string[]} */
+    let names = [];
+    try {
+        names = readdirSync(RUNS);
+    } catch {
+        names = [];
+    }
+    let count = 0;
+    for (const name of names) {
+        const m = RUN_NAME.exec(name);
+        if (!m || m[2] !== 'nomem' || +m[3] !== 1 || +m[1] <= since) {
+            continue;
+        }
+        const ids = (json(join(RUNS, name, 'cases.json'))?.batch ?? []).map(
+            (/** @type {{ id: string }} */ c) => c.id,
+        );
+        if (ids.includes(e.id)) {
+            count++;
+        }
+    }
+    return count;
+}
+
+/** @param {Entry} e @returns {'open' | 'fixed' | 'stuck'} */
+function status(e) {
+    if (e.fixed) {
+        return 'fixed';
+    }
+    return retries(e) >= maxRetries ? 'stuck' : 'open';
+}
+
+function save() {
+    const tmp = `${FILE}.tmp`;
+    writeFileSync(tmp, `${JSON.stringify(doc, null, 2)}\n`);
+    renameSync(tmp, FILE);
+}
+
+// ---------------------------------------------------------------------------
+
+const argv = process.argv.slice(2);
+const command = argv[0] && !argv[0].startsWith('-') ? /** @type {string} */ (argv.shift()) : 'list';
+
+/** @type {string[]} */
+const ids = [];
+let why = '';
+let run = '';
+let note = '';
+while (argv.length > 0) {
+    const arg = /** @type {string} */ (argv.shift());
+    const value = () => argv.shift() ?? die(`${arg} needs a value`);
+    if (arg === '--why') {
+        why = value();
+    } else if (arg === '--run') {
+        run = value();
+    } else if (arg === '--note') {
+        note = value();
+    } else if (arg === '-h' || arg === '--help') {
+        console.log(USAGE);
+        process.exit(0);
+    } else if (arg.startsWith('-')) {
+        die(`unknown flag ${arg}`);
+    } else {
+        ids.push(arg);
+    }
+}
+
+if (command === 'add' || command === 'fix') {
+    if (ids.length === 0) {
+        die(`${command} needs at least one case id`);
+    }
+    if (!RUN_NAME.test(run)) {
+        die('--run must name a run, e.g. batch03-nomem-run2');
+    }
+    if (!existsSync(join(RUNS, run))) {
+        die(`no ${RUNS}/${run}`);
+    }
+
+    if (command === 'add') {
+        if (!REASONS.includes(why)) {
+            die(`--why must be one of: ${REASONS.join(', ')}`);
+        }
+        for (const id of ids) {
+            const held = doc.cases.find((e) => e.id === id);
+            if (held) {
+                held.reasons = [...new Set([...held.reasons, why])];
+                // Back on the list after a fix: it was not fixed, and its retries start again.
+                if (held.fixed) {
+                    delete held.fixed;
+                    held.added = run;
+                }
+                if (note) {
+                    held.notes.push(`${run}: ${note}`);
+                }
+            } else {
+                doc.cases.push({
+                    id,
+                    reasons: [why],
+                    added: run,
+                    notes: note ? [`${run}: ${note}`] : [],
+                });
+            }
+        }
+    } else {
+        for (const id of ids) {
+            const held = doc.cases.find((e) => e.id === id);
+            if (!held) {
+                die(`${id} is not on the list`);
+            }
+            held.fixed = run;
+        }
+    }
+    save();
+    console.error(`${NAME}: ${command === 'add' ? 'added' : 'fixed'} ${ids.join(' ')}`);
+    process.exit(0);
+}
+
+if (command !== 'list') {
+    console.error(`${NAME}: unknown command ${command}`);
+    console.error(USAGE);
+    process.exit(2);
+}
+
+if (doc.cases.length === 0) {
+    console.log('no difficult cases yet');
+    process.exit(0);
+}
+
+const rows = doc.cases.map((e) => [
+    e.id,
+    status(e),
+    e.reasons.join(','),
+    String(retries(e)),
+    e.added,
+    e.fixed ?? '-',
+]);
+const header = ['CASE', 'STATUS', 'WHY', 'RETRIES', 'ADDED', 'FIXED'];
+const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
+for (const row of [header, ...rows]) {
+    console.log(
+        row
+            .map((cell, i) => cell.padEnd(widths[i]))
+            .join('  ')
+            .trimEnd(),
+    );
+}
+const count = (/** @type {string} */ s) => doc.cases.filter((e) => status(e) === s).length;
+console.log('');
+console.log(
+    `${count('open')} open · ${count('fixed')} fixed · ${count('stuck')} stuck (max ${maxRetries} retries)`,
+);

package/templates/editor/.github/skills/zen-finetune/scripts/memory.mjs

@@ -0,0 +1,280 @@
+#!/usr/bin/env node
+// @ts-check
+//
+// Keep the memory the with-memory runs read: the candidate for the batch being
+// tuned, and the last good memory every passed batch has added to.
+//
+// A batch's no-memory runs each start from an empty graph, and every case writes
+// its own. When one of them passes, `candidate <run>` copies the last good
+// memory into a NEW directory and folds what each of that run's cases committed
+// into it with `zen memory merge`, which drops what the graph already held. That
+// is the candidate: the memory the batch's with-memory runs start from, every
+// case from its own copy, so nothing a case commits reaches the candidate or
+// another case.
+//
+// When a with-memory run passes, `checkpoint <run>` makes the candidate the last
+// good memory: it renames the directory and only then repoints
+// finetune/memory/last-good.json at it. `zen memory merge` writes its files one
+// after another, so a merge killed half-way can leave a graph whose files
+// disagree - but only ever a candidate, never the one the pointer names. Stop the
+// tuning at any moment and `memory.mjs path` still names a usable graph.
+//
+// Needs `zen` on PATH. Run it from anywhere; it finds the project root from its
+// own location. See .github/skills/zen-finetune/SKILL.md, "The memory".
+
+import { spawnSync } from 'node:child_process';
+import {
+    cpSync,
+    existsSync,
+    mkdirSync,
+    readdirSync,
+    readFileSync,
+    renameSync,
+    rmSync,
+    writeFileSync,
+} from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const NAME = 'memory.mjs';
+
+const USAGE = `Keep the candidate memory of the batch being tuned, and the last good memory.
+
+  ${NAME}                         the last good memory and the candidate, and what built them
+  ${NAME} path                    the last good memory's directory
+  ${NAME} path --candidate        the candidate's directory, for --memory "$(...)"
+  ${NAME} candidate <run>         a PASSED no-memory run: last good + what its cases committed
+  ${NAME} checkpoint <run>        a PASSED with-memory run: its candidate becomes last good
+
+  <run> is a run directory name, e.g. batch03-nomem-run2 or batch03-mem-run1`;
+
+process.chdir(resolve(dirname(fileURLToPath(import.meta.url)), '../../../..'));
+
+const ROOT = 'finetune/memory';
+const POINTER = `${ROOT}/last-good.json`;
+const CANDIDATE = `${ROOT}/candidate.json`;
+const RUNS = 'finetune/runs';
+const RUN_NAME = /^batch(\d{2})-(nomem|mem)-run(\d+)$/;
+// Older graphs are kept to roll back to; beyond this many they are only disk.
+const KEEP = 3;
+
+/** @param {string} msg @param {number} [code] @returns {never} */
+function die(msg, code = 2) {
+    console.error(`${NAME}: ${msg}`);
+    process.exit(code);
+}
+
+/** @param {string} path @returns {any} */
+function json(path) {
+    try {
+        return JSON.parse(readFileSync(path, 'utf8'));
+    } catch {
+        return undefined;
+    }
+}
+
+/** @param {string} path @param {unknown} value */
+function writeAtomic(path, value) {
+    const tmp = `${path}.tmp`;
+    writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`);
+    renameSync(tmp, path);
+}
+
+/** @param {string} dir @returns {number} */
+const nodesIn = (dir) => Number(json(join(dir, 'manifest.json'))?.nodes) || 0;
+
+/** @param {string} run */
+function passed(run) {
+    const path = join(RUNS, run, 'findings.md');
+    return existsSync(path) && /^## PASSED/m.test(readFileSync(path, 'utf8'));
+}
+
+/**
+ * @typedef {{ dir: string, batches: number[], runs: string[], nodes: number, at: string }} Pointer
+ * @typedef {{ dir: string, batch: number, from: string, nodes: number, at: string }} Candidate
+ */
+
+/** @type {Pointer | undefined} */
+const pointer = json(POINTER);
+/** @type {Candidate | undefined} */
+const candidate = json(CANDIDATE);
+
+const [command = 'status', arg] = process.argv.slice(2);
+
+if (command === '-h' || command === '--help') {
+    console.log(USAGE);
+    process.exit(0);
+}
+
+if (command === 'path') {
+    if (arg === '--candidate') {
+        if (!candidate) {
+            die(
+                'no candidate - build one from a PASSED no-memory run: memory.mjs candidate <run>',
+                1,
+            );
+        }
+        console.log(candidate.dir);
+        process.exit(0);
+    }
+    if (arg !== undefined) {
+        die(`unknown argument ${arg}`);
+    }
+    if (!pointer) {
+        die('no last good memory yet - checkpoint a PASSED with-memory run first', 1);
+    }
+    console.log(pointer.dir);
+    process.exit(0);
+}
+
+if (command === 'status') {
+    if (pointer) {
+        console.log(`last good   ${pointer.dir}`);
+        console.log(`nodes       ${pointer.nodes}`);
+        console.log(`updated     ${pointer.at}`);
+        console.log(`batches     ${pointer.batches.join(', ')}`);
+        console.log(`built from  ${pointer.runs.join(' ')}`);
+    } else {
+        console.log('no last good memory yet');
+    }
+    console.log('');
+    if (candidate) {
+        console.log(`candidate   ${candidate.dir}`);
+        console.log(`batch       ${candidate.batch}, from ${candidate.from}`);
+        console.log(`nodes       ${candidate.nodes}`);
+    } else {
+        console.log('no candidate');
+    }
+    process.exit(0);
+}
+
+if (command !== 'candidate' && command !== 'checkpoint') {
+    console.error(`${NAME}: unknown command ${command}`);
+    console.error(USAGE);
+    process.exit(2);
+}
+
+const m = RUN_NAME.exec(arg ?? '');
+if (!arg || !m) {
+    die(`${command} needs a run, e.g. batch03-${command === 'candidate' ? 'nomem' : 'mem'}-run2`);
+}
+const batch = +m[1];
+const pad = String(batch).padStart(2, '0');
+
+// ---------------------------------------------------------------------------
+// candidate
+// ---------------------------------------------------------------------------
+
+if (command === 'candidate') {
+    if (m[2] !== 'nomem') {
+        die(
+            'a candidate is built from a no-memory run: a with-memory run holds the whole graph again in every case',
+        );
+    }
+    if (!passed(arg)) {
+        die(`${arg} does not say PASSED - only a passing run's memory is kept`);
+    }
+    if (pointer?.batches.includes(batch)) {
+        die(`batch ${batch} is already in the last good memory`);
+    }
+
+    // Only case memories with a manifest: a case that committed nothing has none.
+    const dir = join(RUNS, arg, 'batch');
+    const sources = (existsSync(dir) ? readdirSync(dir, { withFileTypes: true }) : [])
+        .filter((e) => e.isDirectory() && existsSync(join(dir, e.name, 'memory', 'manifest.json')))
+        .map((e) => join(dir, e.name, 'memory'));
+
+    const target = join(ROOT, `candidate-batch${pad}`);
+    rmSync(target, { recursive: true, force: true });
+    mkdirSync(ROOT, { recursive: true });
+    if (pointer && existsSync(pointer.dir)) {
+        cpSync(pointer.dir, target, { recursive: true, filter: (src) => !src.endsWith('/.lock') });
+    } else {
+        mkdirSync(target, { recursive: true });
+    }
+
+    if (sources.length > 0) {
+        const merged = spawnSync('zen', ['memory', 'merge', ...sources, '--dir', target, '--yes'], {
+            stdio: 'inherit',
+        });
+        if (merged.error) {
+            rmSync(target, { recursive: true, force: true });
+            die(`could not run zen: ${merged.error.message}`);
+        }
+        if (merged.status !== 0) {
+            rmSync(target, { recursive: true, force: true });
+            die('zen memory merge failed - no candidate was written', 1);
+        }
+    }
+
+    /** @type {Candidate} */
+    const next = {
+        dir: target,
+        batch,
+        from: arg,
+        nodes: nodesIn(target),
+        at: new Date().toISOString(),
+    };
+    writeAtomic(CANDIDATE, next);
+    const before = pointer ? nodesIn(pointer.dir) : 0;
+    console.error(
+        `${NAME}: candidate for batch ${batch} is ${target} - ${next.nodes} nodes ` +
+            `(${before} from the last good memory, ${sources.length} case memor${sources.length === 1 ? 'y' : 'ies'} from ${arg})`,
+    );
+    if (next.nodes === 0) {
+        console.error(
+            `${NAME}: the candidate is EMPTY - nothing was ever committed, so a with-memory run ` +
+                'measures nothing. That is a commit-policy finding for the no-memory runs.',
+        );
+    }
+    process.exit(0);
+}
+
+// ---------------------------------------------------------------------------
+// checkpoint
+// ---------------------------------------------------------------------------
+
+if (m[2] !== 'mem') {
+    die('checkpoint takes a PASSED with-memory run; a no-memory run builds a candidate first');
+}
+if (!passed(arg)) {
+    die(
+        `${arg} does not say PASSED - the candidate is kept only once its batch passes with memory`,
+    );
+}
+if (pointer?.batches.includes(batch)) {
+    console.error(`${NAME}: batch ${batch} is already in the last good memory`);
+    process.exit(0);
+}
+if (!candidate || candidate.batch !== batch) {
+    die(`no candidate for batch ${batch} - memory.mjs candidate batch${pad}-nomem-run<N>`);
+}
+
+const target = join(ROOT, `after-batch${pad}`);
+if (existsSync(candidate.dir)) {
+    rmSync(target, { recursive: true, force: true });
+    renameSync(candidate.dir, target);
+} else if (!existsSync(target)) {
+    die(`the candidate ${candidate.dir} is gone - build it again from ${candidate.from}`);
+}
+// A checkpoint killed after the rename resumes here: the directory is already in place.
+
+/** @type {Pointer} */
+const next = {
+    dir: target,
+    batches: [...(pointer?.batches ?? []), batch],
+    runs: [...(pointer?.runs ?? []), candidate.from, arg],
+    nodes: nodesIn(target),
+    at: new Date().toISOString(),
+};
+writeAtomic(POINTER, next);
+rmSync(CANDIDATE, { force: true });
+
+const history = next.batches.map((b) => join(ROOT, `after-batch${String(b).padStart(2, '0')}`));
+for (const dir of history.slice(0, -KEEP)) {
+    rmSync(dir, { recursive: true, force: true });
+}
+
+console.error(
+    `${NAME}: last good memory is now ${target} - ${next.nodes} nodes, batches ${next.batches.join(', ')}`,
+);