@zenera/cli 1.1.23 → 1.1.25

package/dist/meta/finetune/retry.d.ts

@@ -0,0 +1,24 @@
+import type { Case } from './dataset/types.ts';
+import type { Tuning } from './tuning.ts';
+export interface RetryPick {
+    /** every difficult case */
+    difficult?: boolean;
+    classes?: string[];
+    /** ids or globs */
+    ids: string[];
+    yes?: boolean;
+}
+export interface RetryPlan {
+    take: Case[];
+    /** matched, but not difficult: why each is left as it is */
+    skip: {
+        id: string;
+        why: string;
+    }[];
+    /** what the cases taken spent last round, as a guide to what another will */
+    tokens: number;
+}
+export declare function planRetry(t: Tuning, pick: RetryPick): RetryPlan;
+/** Moves each case's round aside, under the loop's lock so no tuning is writing into it. */
+export declare function retry(t: Tuning, pick: RetryPick): Promise<RetryPlan>;
+//# sourceMappingURL=retry.d.ts.map

package/dist/meta/finetune/retry.js

@@ -0,0 +1,96 @@
+import { anyOf, claimLock, ownLock, wildcard } from '@zenera/neo';
+import { mkdirSync, renameSync, unlinkSync } from 'node:fs';
+import { hostname } from 'node:os';
+import { join } from 'node:path';
+import { CliError, confirm, EXIT, note, usageError } from "../host.js";
+import { DatasetStore } from "./dataset/store.js";
+import { recover } from "./journal.js";
+import { resultOf, selection } from "./sampler.js";
+import { short, STAGES, total } from "./usage.js";
+export function planRetry(t, pick) {
+    if (!pick.difficult && !pick.classes?.length && pick.ids.length === 0) {
+        throw usageError('retry which cases?', 'name them (ids or globs), or --difficult for all of them, or --class <c>');
+    }
+    const ids = anyOf(pick.ids.map((p) => wildcard(p)));
+    const plan = { take: [], skip: [], tokens: 0 };
+    for (const c of selection(t)) {
+        // Names add to --difficult; a class narrows both.
+        const named = ids?.(c.id) ?? false;
+        if (ids && !named && !pick.difficult) {
+            continue;
+        }
+        if (pick.classes?.length && !pick.classes.includes(c.class ?? '')) {
+            continue;
+        }
+        const r = resultOf(t, c);
+        if (r?.state === 'difficult') {
+            plan.take.push(c);
+            for (const stage of STAGES) {
+                const s = total(r.tokens?.[stage]);
+                plan.tokens += s.input + s.output;
+            }
+            continue;
+        }
+        // Only named cases are worth a word; a class or --difficult says nothing of the rest.
+        if (named) {
+            plan.skip.push({
+                id: c.id,
+                why: !r
+                    ? 'not finished yet - start carries on with it'
+                    : r.state === 'failed'
+                        ? 'failed, not difficult - start resumes it'
+                        : 'completed - change the case to run it again',
+            });
+        }
+    }
+    return plan;
+}
+/** Moves each case's round aside, under the loop's lock so no tuning is writing into it. */
+export async function retry(t, pick) {
+    const plan = planRetry(t, pick);
+    for (const s of plan.skip) {
+        note(`${s.id}: ${s.why}`);
+    }
+    if (plan.take.length === 0) {
+        note('nothing to retry');
+        return plan;
+    }
+    const said = `${plan.take.length} case(s) for another round - about ${short(plan.tokens)} tokens, going by their last round`;
+    if (!pick.yes && !(await confirm(`${said}. Go ahead?`))) {
+        note('left as they were');
+        return { ...plan, take: [] };
+    }
+    mkdirSync(t.dir, { recursive: true });
+    const lock = join(t.dir, 'loop.lock');
+    claimLock(lock, ownLock(), (held) => new CliError(`a tuning is running here (pid ${held.pid})`, EXIT.failed, 'stop it first: zen meta finetune stop'));
+    try {
+        // A step a kill cut off points into the very folders about to move.
+        await recover(t);
+        const store = DatasetStore.open(t.root);
+        for (const c of plan.take) {
+            const round = t.round(c);
+            mkdirSync(t.roundsDir(c), { recursive: true });
+            renameSync(t.caseDir(c), join(t.roundsDir(c), `r${c.rev}.${round}`));
+            t.event({ what: 'retry', case: c.id, rev: c.rev, detail: `round ${round + 1}` });
+            store.note(c.id, {
+                type: 'note',
+                at: new Date().toISOString(),
+                kind: 'observation',
+                caseRev: c.rev,
+                text: `retried: round ${round} was difficult, round ${round + 1} starts on system v${t.system.version()}`,
+                by: { prompt: 'finetune', host: hostname() },
+            });
+        }
+    }
+    finally {
+        try {
+            unlinkSync(lock);
+        }
+        catch {
+            // Taken over as stale; nothing to give back.
+        }
+    }
+    note(`${said}. Run them: zen meta finetune start`);
+    return plan;
+}
+//# sourceMappingURL=retry.js.map

package/dist/meta/finetune/sampler.d.ts

@@ -0,0 +1,18 @@
+import type { Case } from './dataset/types.ts';
+import { type Tuning } from './tuning.ts';
+import type { ByStage } from './usage.ts';
+export interface Result {
+    /** `failed`: something outside the prose broke, so nothing was learned and it runs again */
+    state: 'completed' | 'difficult' | 'failed';
+    phase: 'nomem' | 'mem';
+    reason?: string;
+    caseRev: number;
+    at: string;
+    /** per stage and model, over every try of the case */
+    tokens?: ByStage;
+}
+/** The cases this tuning covers, in the order they are taken. */
+export declare function selection(t: Tuning): Case[];
+export declare function resultOf(t: Tuning, c: Case): Result | undefined;
+export declare function nextCase(t: Tuning): Case | undefined;
+//# sourceMappingURL=sampler.d.ts.map

package/dist/meta/finetune/sampler.js

@@ -0,0 +1,41 @@
+import { anyOf, wildcard } from '@zenera/neo';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { sample } from "./dataset/sample.js";
+import { DatasetStore } from "./dataset/store.js";
+import { readJson } from "./tuning.js";
+/** The cases this tuning covers, in the order they are taken. */
+export function selection(t) {
+    const store = DatasetStore.open(t.root);
+    const cases = store.all();
+    const ids = anyOf((t.config.ids ?? []).map((p) => wildcard(p)));
+    const drawn = sample(cases, {
+        status: 'active',
+        ...(t.config.classes?.length ? { classes: t.config.classes } : {}),
+        ...(t.config.rubric !== undefined ? { rubric: t.config.rubric } : {}),
+        ...(ids ? { ids } : {}),
+    }, { by: ['class'], weights: new Map(), order: ['rubric'], seed: t.config.seed }, () => []);
+    const byId = new Map(cases.map((c) => [c.id, c]));
+    const ordered = drawn.ids.map((id) => byId.get(id));
+    return t.config.limit ? ordered.slice(0, t.config.limit) : ordered;
+}
+export function resultOf(t, c) {
+    const file = join(t.caseDir(c), 'result.json');
+    const r = existsSync(file) ? readJson(file) : undefined;
+    // Written before `failed` existed: a void run or an error was filed as difficult.
+    if (r?.state === 'difficult' && /^(void|error):/.test(r.reason ?? '')) {
+        return { ...r, state: 'failed' };
+    }
+    return r;
+}
+export function nextCase(t) {
+    return selection(t).find((c) => {
+        if (t.claimed.has(c.id)) {
+            return false;
+        }
+        const r = resultOf(t, c);
+        // Once per start: a sandbox that is down would only fail it again at once.
+        return !r || (r.state === 'failed' && !t.tried.has(c.id));
+    });
+}
+//# sourceMappingURL=sampler.js.map

package/dist/meta/finetune/system.d.ts

@@ -0,0 +1,27 @@
+export interface VersionRow {
+    v: number;
+    hash: string;
+    at: string;
+    from: 'start' | 'external' | 'apply';
+    apply?: string;
+}
+export declare class SystemVersions {
+    #private;
+    readonly root: string;
+    readonly dir: string;
+    constructor(root: string, dir: string);
+    hash(): string;
+    rows(): VersionRow[];
+    current(): VersionRow | undefined;
+    version(): number;
+    snapshotDir(v: number): string;
+    /** A new version when the system moved since the last one; the last one otherwise. */
+    record(from: VersionRow['from'], apply?: string): VersionRow;
+    /** Puts version `v` back, exactly: files added since are removed. */
+    restore(v: number): void;
+    /** Forgets the versions an apply recorded, as if it never ran. */
+    drop(apply: string): void;
+    /** `git diff` between two versions' copies; a file list where git is missing. */
+    diff(a: number, b: number): string;
+}
+//# sourceMappingURL=system.d.ts.map

package/dist/meta/finetune/system.js

@@ -0,0 +1,132 @@
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { appendFileSync, copyFileSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, } from 'node:fs';
+import { join } from 'node:path';
+// ---------------------------------------------------------------------------
+// The system under tuning, as numbered versions
+//
+// What tuning changes is `agents.yaml` and everything under `agents/`, so that
+// is the system: hashed as a whole, copied whole at every version, so any
+// version can be diffed against another or put back. A version is recorded
+// when the hash moves - by an apply, or by someone's hand between sessions.
+// ---------------------------------------------------------------------------
+const CONFIG_FILES = ['agents.yaml', 'agents.yml'];
+const AGENTS = 'agents';
+const pad = (n) => String(n).padStart(2, '0');
+export class SystemVersions {
+    root;
+    dir;
+    constructor(root, dir) {
+        this.root = root;
+        this.dir = dir;
+    }
+    #files() {
+        const found = CONFIG_FILES.filter((f) => existsSync(join(this.root, f)));
+        const agents = join(this.root, AGENTS);
+        if (existsSync(agents)) {
+            for (const entry of readdirSync(agents, { recursive: true, withFileTypes: true })) {
+                if (entry.isFile() && entry.name !== '.DS_Store') {
+                    const rel = join(entry.parentPath, entry.name).slice(this.root.length + 1);
+                    found.push(rel.split('\\').join('/'));
+                }
+            }
+        }
+        return found.sort();
+    }
+    hash() {
+        const h = createHash('sha256');
+        for (const rel of this.#files()) {
+            h.update(rel)
+                .update('\0')
+                .update(readFileSync(join(this.root, rel)))
+                .update('\0');
+        }
+        return h.digest('hex').slice(0, 16);
+    }
+    rows() {
+        try {
+            return readFileSync(join(this.dir, 'systems.jsonl'), 'utf8')
+                .split('\n')
+                .filter(Boolean)
+                .map((line) => JSON.parse(line));
+        }
+        catch {
+            return [];
+        }
+    }
+    current() {
+        return this.rows().at(-1);
+    }
+    version() {
+        return this.current()?.v ?? 0;
+    }
+    snapshotDir(v) {
+        return join(this.dir, `v${pad(v)}`);
+    }
+    /** A new version when the system moved since the last one; the last one otherwise. */
+    record(from, apply) {
+        const hash = this.hash();
+        const last = this.current();
+        if (last?.hash === hash) {
+            return last;
+        }
+        const row = {
+            v: (last?.v ?? 0) + 1,
+            hash,
+            at: new Date().toISOString(),
+            from: from === 'start' && last ? 'external' : from,
+            ...(apply ? { apply } : {}),
+        };
+        const to = this.snapshotDir(row.v);
+        rmSync(to, { recursive: true, force: true });
+        mkdirSync(to, { recursive: true });
+        for (const f of CONFIG_FILES) {
+            if (existsSync(join(this.root, f))) {
+                copyFileSync(join(this.root, f), join(to, f));
+            }
+        }
+        if (existsSync(join(this.root, AGENTS))) {
+            cpSync(join(this.root, AGENTS), join(to, AGENTS), { recursive: true });
+        }
+        mkdirSync(this.dir, { recursive: true });
+        appendFileSync(join(this.dir, 'systems.jsonl'), `${JSON.stringify(row)}\n`);
+        return row;
+    }
+    /** Puts version `v` back, exactly: files added since are removed. */
+    restore(v) {
+        const from = this.snapshotDir(v);
+        for (const f of CONFIG_FILES) {
+            rmSync(join(this.root, f), { force: true });
+            if (existsSync(join(from, f))) {
+                copyFileSync(join(from, f), join(this.root, f));
+            }
+        }
+        rmSync(join(this.root, AGENTS), { recursive: true, force: true });
+        if (existsSync(join(from, AGENTS))) {
+            cpSync(join(from, AGENTS), join(this.root, AGENTS), { recursive: true });
+        }
+    }
+    /** Forgets the versions an apply recorded, as if it never ran. */
+    drop(apply) {
+        const rows = this.rows();
+        const keep = rows.filter((r) => r.apply !== apply);
+        if (keep.length === rows.length) {
+            return;
+        }
+        for (const r of rows) {
+            if (r.apply === apply) {
+                rmSync(this.snapshotDir(r.v), { recursive: true, force: true });
+            }
+        }
+        writeFileSync(join(this.dir, 'systems.jsonl'), keep.map((r) => `${JSON.stringify(r)}\n`).join(''));
+    }
+    /** `git diff` between two versions' copies; a file list where git is missing. */
+    diff(a, b) {
+        const res = spawnSync('git', ['diff', '--no-index', '--no-color', '--', `v${pad(a)}`, `v${pad(b)}`], { cwd: this.dir, encoding: 'utf8' });
+        if (res.error || (res.status !== 0 && res.status !== 1)) {
+            return `(git diff unavailable) v${pad(a)} -> v${pad(b)}\n`;
+        }
+        return res.stdout;
+    }
+}
+//# sourceMappingURL=system.js.map

package/dist/meta/finetune/train.d.ts

@@ -0,0 +1,39 @@
+import type { TokenUsage } from '@zenera/neo';
+import type { Case } from './dataset/types.ts';
+import { StepError } from './failure.ts';
+import { type Phase } from './feedback.ts';
+import { type Seat, type Tuning } from './tuning.ts';
+import { type ByModel, type RunMetrics } from './usage.ts';
+export declare function train(t: Tuning, s: Seat, c: Case): Promise<void>;
+export interface Run {
+    /** the run directory under sessions/ */
+    dir: string;
+    /** the memory it ran with and committed into */
+    memory: string;
+    /** its own empty directory, the root of the agent's file tools */
+    workspace?: string;
+    /** the system version it ran on */
+    system: number;
+    report?: string;
+    graph?: string;
+    /** the run's own error, when it failed - still worth analyzing */
+    failed?: string;
+    turns?: number;
+    usage?: TokenUsage;
+    /** per model, from the run's trajectory */
+    tokens?: ByModel;
+    metrics?: RunMetrics;
+    durationMs?: number;
+}
+export declare class VoidRun extends StepError {
+}
+/** The container engine failed: every run after this one would fail the same way. */
+export declare class SandboxDown extends StepError {
+}
+/** The case keeps one analyze session per revision, so each analysis remembers the last. */
+export declare function sessionOf(t: Tuning, c: Case): string | undefined;
+export declare function markCompleted(t: Tuning, c: Case): void;
+export declare function markDifficult(t: Tuning, c: Case, phase: Phase, reason: string): void;
+/** Something outside the prose broke. The case keeps its tries and runs again at the next start. */
+export declare function markFailed(t: Tuning, c: Case, phase: Phase, reason: string): void;
+//# sourceMappingURL=train.d.ts.map