@zenera/cli 1.1.25 → 1.1.27

package/dist/meta/finetune/loop.js

@@ -12,9 +12,10 @@ import { Stopped } from "./tuning.js";
 // The loop: N workers, cases started as they come free, one applier
 //
 // Whenever a worker is free - and no woken case is waiting for one - the next
-// case starts on it and is trained to the end. A parked case gives its worker
-// back, so a lane never sits idle waiting for an apply. A stop never interrupts
-// a step: it is noticed between steps, and the case resumes next time.
+// recheck, or else the next case, starts on it and is trained to the end. A
+// parked case gives its worker back, so a lane never sits idle waiting for an
+// apply. A stop never interrupts a step: it is noticed between steps, and the
+// case resumes next time.
 // ---------------------------------------------------------------------------
 const REPORT_EVERY_MS = 15_000;
 const LIVE_EVERY_MS = 1000;
@@ -24,10 +25,15 @@ export async function runLoop(t) {
     claimLock(lock, ownLock(), (held) => new CliError(`a tuning is already running here (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `stop it with: zen meta finetune stop - or remove ${lock} if it crashed`));
     rmSync(t.stopFile, { force: true });
     const timer = setInterval(() => t.report(), REPORT_EVERY_MS);
-    const live = setInterval(() => publish(t), LIVE_EVERY_MS);
+    const live = setInterval(() => {
+        publish(t);
+        t.report(true);
+    }, LIVE_EVERY_MS);
     try {
         await recover(t);
+        t.rechecks.resume();
         const start = t.system.record('start');
+        await t.improvements.baseline();
         t.event({ what: 'started', detail: `system v${start.v}, ${t.workers.length} workers` });
         publish(t);
         const applier = t.improvements.loop();
@@ -42,6 +48,7 @@ export async function runLoop(t) {
         clearInterval(live);
         t.finished = true;
         publish(t);
+        t.report();
         try {
             unlinkSync(lock);
         }
@@ -55,33 +62,75 @@ export async function runLoop(t) {
 }
 async function dispatch(t) {
     const running = new Set();
+    const start = (seat, work) => {
+        const task = work(seat).finally(() => running.delete(task));
+        running.add(task);
+    };
     while (!t.stopping) {
         if (t.improvements.applying || t.resuming > 0 || !t.freeWorker()) {
             await t.tick();
             continue;
         }
+        // A recheck tests the prose of the day, so it goes before a new case.
+        const taken = t.rechecks.take();
+        if (taken) {
+            const seat = { case: taken.case };
+            t.claimed.add(taken.case.id);
+            await t.seat(seat, false);
+            start(seat, (s) => recheckCase(t, s, taken.recheck));
+            continue;
+        }
         const c = nextCase(t);
         if (!c) {
             t.drained = true;
-            break;
+            // An apply still to come queues rechecks; only when none can is the work done.
+            const more = running.size > 0 ||
+                t.improvements.applying ||
+                t.improvements.pending.length > 0 ||
+                t.rechecks.waiting() > 0;
+            if (!more) {
+                break;
+            }
+            await t.tick();
+            continue;
         }
         const seat = { case: c };
         t.claimed.add(c.id);
         t.tried.add(c.id);
         const before = resultOf(t, c);
         if (before?.state === 'failed') {
-            // Taken again: until it ends anew it is in progress, not failed. Its analyze session
-            // may be what broke (another model, a dead provider), so the next analysis starts fresh.
+            // Taken again: until it ends anew it is in progress, not failed.
             rmSync(join(t.caseDir(c), 'result.json'), { force: true });
-            rmSync(join(t.caseDir(c), 'session'), { force: true });
             t.event({ what: 'resumed', case: c.id, rev: c.rev, detail: before.reason });
         }
         await t.seat(seat, false);
-        const task = trainCase(t, seat).finally(() => running.delete(task));
-        running.add(task);
+        start(seat, (s) => trainCase(t, s));
     }
     await Promise.all(running);
 }
+async function recheckCase(t, s, r) {
+    try {
+        await t.rechecks.run(s, r, s.case);
+    }
+    catch (err) {
+        if (!(err instanceof Stopped)) {
+            t.error(`${s.case.id} · recheck after ${r.apply}`, err.message, err instanceof StepError ? err.log : undefined);
+            if (err instanceof SandboxDown) {
+                // Left running: the next start queues it again.
+                t.sandboxDown(err.message);
+            }
+            else {
+                t.rechecks.void(r, err.message);
+                t.failed(`recheck: ${err.message}`);
+            }
+        }
+    }
+    finally {
+        t.unseat(s);
+        t.claimed.delete(s.case.id);
+        t.report();
+    }
+}
 async function trainCase(t, s) {
     const c = s.case;
     try {

package/dist/meta/finetune/recheck.d.ts

@@ -0,0 +1,58 @@
+import type { Case } from './dataset/types.ts';
+import { type Seat, type Tuning } from './tuning.ts';
+import { type ByStage } from './usage.ts';
+export interface Recheck {
+    case: string;
+    rev: number;
+    /** the apply whose version it runs on */
+    apply: string;
+    state: 'queued' | 'running' | 'passed' | 'regressed' | 'void';
+    at: string;
+    /** each try's verdict */
+    tries?: string[];
+    /** the request a regression sent to the improvement queue */
+    request?: string;
+    /** why it was void, or why a regression sent no request */
+    note?: string;
+}
+/** A case regresses only when every one of these fails: one failure may be the model's noise. */
+export declare const RECHECK_TRIES = 2;
+export declare class Rechecks {
+    #private;
+    constructor(t: Tuning);
+    get file(): string;
+    all(): Recheck[];
+    dir(r: {
+        case: string;
+        apply: string;
+    }): string;
+    /** Each try's folder, oldest first. */
+    tryDirs(r: {
+        case: string;
+        apply: string;
+    }): string[];
+    waiting(): number;
+    /**
+     * After a stop: a recheck cut off part-way runs again, and a regression's request
+     * - kept only in memory until an apply takes it - goes back in the queue.
+     */
+    resume(): void;
+    /**
+     * Queues completed cases to run on the version `apply` made: a case whose
+     * last recheck regressed first - this apply was meant to repair it - then
+     * the rest in an order fixed by the seed and the apply.
+     */
+    queue(apply: string): Recheck[];
+    /** The next queued recheck, marked running, with its case; one whose case changed is dropped. */
+    take(): {
+        recheck: Recheck;
+        case: Case;
+    } | undefined;
+    /** A step broke for a reason outside the prose: the recheck says nothing. */
+    void(r: Recheck, why: string): void;
+    /** Runs a recheck on the seat's worker, to its verdict. */
+    run(s: Seat, r: Recheck, c: Case): Promise<void>;
+    /** Every recheck run and analysis so far. */
+    tokens(): ByStage;
+}
+//# sourceMappingURL=recheck.d.ts.map

package/dist/meta/finetune/recheck.js

@@ -0,0 +1,311 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { writeAnalyzeBrief } from "./analyze-brief.js";
+import { readFeedback } from "./feedback.js";
+import { writeApply } from "./report.js";
+import { resultOf, selection } from "./sampler.js";
+import { keepAnalysis, runOnce } from "./train.js";
+import { readJson } from "./tuning.js";
+import { envelopesIn, mergeStages, spent } from "./usage.js";
+/** A case regresses only when every one of these fails: one failure may be the model's noise. */
+export const RECHECK_TRIES = 2;
+/** Requests one case's regressions may send; past that it is reported, so the tuning ends. */
+const REQUESTS_PER_CASE = 2;
+const pad = (n) => String(n).padStart(2, '0');
+export class Rechecks {
+    #t;
+    constructor(t) {
+        this.#t = t;
+    }
+    get file() {
+        return join(this.#t.dir, 'rechecks.json');
+    }
+    all() {
+        return readJson(this.file) ?? [];
+    }
+    #save(all) {
+        mkdirSync(this.#t.dir, { recursive: true });
+        writeFileSync(this.file, `${JSON.stringify(all, null, 2)}\n`);
+    }
+    #update(r, change) {
+        const all = this.all();
+        const i = all.findIndex((x) => x.case === r.case && x.apply === r.apply);
+        Object.assign(r, change, { at: new Date().toISOString() });
+        if (i >= 0) {
+            all[i] = r;
+            this.#save(all);
+        }
+        if (change.state && change.state !== 'running') {
+            this.#page(r.apply);
+        }
+    }
+    /** The apply's page lists its rechecks; it was written before they ran. */
+    #page(apply) {
+        try {
+            writeApply(this.#t, join(this.#t.improvements.dir, apply));
+        }
+        catch {
+            // A page that failed to render must not stop the tuning.
+        }
+    }
+    dir(r) {
+        return join(this.#t.dir, 'rechecks', r.apply, r.case);
+    }
+    /** Each try's folder, oldest first. */
+    tryDirs(r) {
+        try {
+            return readdirSync(this.dir(r))
+                .filter((n) => /^\d{2}-nomem$/.test(n))
+                .sort()
+                .map((n) => join(this.dir(r), n));
+        }
+        catch {
+            return [];
+        }
+    }
+    waiting() {
+        return this.all().filter((r) => r.state === 'queued' || r.state === 'running').length;
+    }
+    /**
+     * After a stop: a recheck cut off part-way runs again, and a regression's request
+     * - kept only in memory until an apply takes it - goes back in the queue.
+     */
+    resume() {
+        const all = this.all();
+        let changed = false;
+        for (const r of all) {
+            if (r.state === 'running') {
+                r.state = 'queued';
+                changed = true;
+            }
+            if (r.state === 'regressed' &&
+                r.request &&
+                !this.#t.improvements.isApplied(r.request)) {
+                const f = this.#feedback(r);
+                if (f) {
+                    this.#t.improvements.regressed(f);
+                }
+            }
+        }
+        if (changed) {
+            this.#save(all);
+        }
+    }
+    /**
+     * Queues completed cases to run on the version `apply` made: a case whose
+     * last recheck regressed first - this apply was meant to repair it - then
+     * the rest in an order fixed by the seed and the apply.
+     */
+    queue(apply) {
+        const t = this.#t;
+        if (t.config.recheck === 0) {
+            return [];
+        }
+        const all = this.all();
+        const open = new Set(all.filter((r) => r.state === 'queued' || r.state === 'running').map((r) => r.case));
+        const last = new Map(all.map((r) => [r.case, r]));
+        const key = (id) => createHash('sha256').update(`${t.config.seed}:${apply}:${id}`).digest('hex');
+        const rank = (id) => (last.get(id)?.state === 'regressed' ? 0 : 1);
+        const at = new Date().toISOString();
+        const picked = selection(t)
+            .filter((c) => !open.has(c.id) && resultOf(t, c)?.state === 'completed')
+            .sort((a, b) => rank(a.id) - rank(b.id) || key(a.id).localeCompare(key(b.id)))
+            .slice(0, t.config.recheck)
+            .map((c) => ({ case: c.id, rev: c.rev, apply, state: 'queued', at }));
+        if (picked.length > 0) {
+            this.#save([...all, ...picked]);
+            this.#page(apply);
+            t.event({
+                what: 'recheck',
+                detail: `${apply}: ${picked.map((r) => r.case).join(', ')} queued to run again on the edit`,
+            });
+        }
+        return picked;
+    }
+    /** The next queued recheck, marked running, with its case; one whose case changed is dropped. */
+    take() {
+        const t = this.#t;
+        for (;;) {
+            const r = this.all().find((x) => x.state === 'queued' && !t.claimed.has(x.case));
+            if (!r) {
+                return undefined;
+            }
+            const c = selection(t).find((x) => x.id === r.case);
+            if (!c || c.rev !== r.rev || resultOf(t, c)?.state !== 'completed') {
+                this.#update(r, { state: 'void', note: 'the case changed since it was queued' });
+                continue;
+            }
+            this.#update(r, { state: 'running' });
+            return { recheck: r, case: c };
+        }
+    }
+    /** A step broke for a reason outside the prose: the recheck says nothing. */
+    void(r, why) {
+        this.#update(r, { state: 'void', note: why });
+    }
+    /** Runs a recheck on the seat's worker, to its verdict. */
+    async run(s, r, c) {
+        const t = this.#t;
+        const tries = [];
+        let last;
+        for (let n = 1; n <= RECHECK_TRIES; n++) {
+            const dir = join(this.dir(r), `${pad(n)}-nomem`);
+            const one = await this.#try(s, r, c, n, dir);
+            tries.push(one.verdict);
+            if (one.passed) {
+                const state = one.verdict === 'right' ? 'passed' : 'void';
+                this.#update(r, { state, tries, ...(one.note ? { note: one.note } : {}) });
+                t.event({
+                    what: `recheck ${state}`,
+                    case: c.id,
+                    rev: c.rev,
+                    result: tries.join(', '),
+                });
+                return;
+            }
+            last = one.feedback;
+        }
+        const sent = this.all().filter((x) => x.case === c.id && x.request).length;
+        if (last && sent < REQUESTS_PER_CASE) {
+            const request = `${c.id}@recheck-${r.apply}`;
+            const f = {
+                ...last,
+                id: request,
+                recheck: r.apply,
+                dir: join(this.dir(r), `${pad(RECHECK_TRIES)}-nomem`),
+            };
+            this.#update(r, { state: 'regressed', tries, request });
+            t.improvements.regressed(f);
+        }
+        else {
+            this.#update(r, {
+                state: 'regressed',
+                tries,
+                note: `${sent} request(s) sent for this case already - reported, not queued`,
+            });
+        }
+        t.event({
+            what: 'regressed',
+            case: c.id,
+            rev: c.rev,
+            detail: `passed before ${r.apply}, failed ${RECHECK_TRIES} tries on it`,
+            result: tries.join(', '),
+        });
+    }
+    /** A void run or an unreadable analysis counts as a pass: only a graded failure is a regression. */
+    async #try(s, r, c, n, dir) {
+        const t = this.#t;
+        const w = s.worker;
+        mkdirSync(dir, { recursive: true });
+        w.recheck = { apply: r.apply, dir };
+        let why;
+        if (!existsSync(join(dir, 'run.json'))) {
+            await t.improvements.notApplying();
+            t.at(s, 'nomem', n, 'run');
+            t.improvements.runs++;
+            const began = Date.now();
+            try {
+                why = await runOnce(t, c, dir);
+            }
+            finally {
+                t.improvements.runs--;
+            }
+            const ran = why ? undefined : readJson(join(dir, 'run.json'));
+            t.ended(s, {
+                what: why ? 'void' : 'ran',
+                phase: 'nomem',
+                attempt: n,
+                result: `recheck: ${why ?? `${ran?.turns ?? '?'} turns`}`,
+                took: Date.now() - began,
+                tokens: spent(ran?.tokens),
+            });
+        }
+        if (why) {
+            return { verdict: 'void', passed: true, note: why };
+        }
+        const run = readJson(join(dir, 'run.json'));
+        const file = join(dir, 'feedback.json');
+        const known = { case: c.id, caseRev: c.rev, phase: 'nomem', attempt: n };
+        const rubricIds = c.rubric.map((x) => x.id);
+        let read = readFeedback(file, known, rubricIds);
+        if (!('feedback' in read)) {
+            t.checkStopping();
+            t.at(s, 'nomem', n, 'analyze');
+            const began = Date.now();
+            const patch = join(t.improvements.dir, r.apply, 'diff.patch');
+            const session = randomUUID();
+            writeFileSync(join(dir, 'session'), `${session}\n`);
+            const brief = await writeAnalyzeBrief(t, dir, {
+                run,
+                case: c,
+                phase: 'nomem',
+                attempt: n,
+                earlier: [],
+                recheck: patch,
+            });
+            const res = await t.zen([
+                'meta',
+                'run',
+                '--no-refresh',
+                '--json',
+                '--events',
+                join(dir, 'analyze.events.jsonl'),
+                '--session-id',
+                session,
+                '--prompt-file',
+                brief,
+                '/analyze',
+                run.dir,
+                `case=${c.id}`,
+                `rev=${c.rev}`,
+                'phase=nomem',
+                `attempt=${n}`,
+                `feedback=${file}`,
+                `recheck=${patch}`,
+            ], join(dir, 'analyze.log'));
+            keepAnalysis(dir, res.stdout);
+            read = readFeedback(file, known, rubricIds);
+            t.ended(s, {
+                what: 'analyzed',
+                phase: 'nomem',
+                attempt: n,
+                result: `recheck: ${'feedback' in read ? read.feedback.verdict : read.problem}`,
+                took: Date.now() - began,
+                tokens: spent(envelopesIn(dir, 'analyze')),
+            });
+        }
+        if (!('feedback' in read)) {
+            return { verdict: 'unread', passed: true, note: read.problem };
+        }
+        const f = read.feedback;
+        if (f.verdict === 'void') {
+            return { verdict: 'void', passed: true, note: f.infra };
+        }
+        const passed = f.verdict === 'right' && !Object.values(f.rubric).includes('fail');
+        return { verdict: f.verdict, passed, feedback: f };
+    }
+    /** A regression's request, read back from its last try. */
+    #feedback(r) {
+        const c = selection(this.#t).find((x) => x.id === r.case);
+        const dir = join(this.dir(r), `${pad(RECHECK_TRIES)}-nomem`);
+        const read = readFeedback(join(dir, 'feedback.json'), { case: r.case, caseRev: r.rev, phase: 'nomem', attempt: RECHECK_TRIES }, c?.rubric.map((x) => x.id) ?? []);
+        return 'feedback' in read
+            ? { ...read.feedback, id: r.request, recheck: r.apply, dir }
+            : undefined;
+    }
+    /** Every recheck run and analysis so far. */
+    tokens() {
+        const parts = [];
+        for (const r of this.all()) {
+            for (const dir of this.tryDirs(r)) {
+                parts.push({
+                    run: readJson(join(dir, 'run.json'))?.tokens,
+                    analyze: envelopesIn(dir, 'analyze'),
+                });
+            }
+        }
+        return mergeStages(...parts);
+    }
+}
+//# sourceMappingURL=recheck.js.map

package/dist/meta/finetune/report.d.ts

@@ -13,13 +13,18 @@ export interface Attempt {
 /** A table cell: one line, no column breaks. */
 export declare const cell: (text: string | number | undefined) => string;
 export declare function since(ms: number): string;
+/** The analyze session of one try, kept in its folder. */
+export declare function analyzeSession(dir: string): string | undefined;
 export declare function attemptsOf(t: Tuning, c: Case): Attempt[];
 /** Every try of a case, per stage and model. Applies serve many cases at once, so they are not split. */
 export declare function caseTokens(t: Tuning, c: Case): ByStage;
+export declare const rubricScore: (f: Attempt["feedback"]) => string;
 export declare function feedbackPath(t: Tuning, c: {
     id: string;
 }): string;
 export declare function writeFeedback(t: Tuning, c: Case): void;
+/** A fence longer than any run of backticks in `text`, so nothing inside can close it. */
+export declare function fence(text: string): string;
 export declare function patchFiles(patch: string): {
     file: string;
     added: number;
@@ -28,6 +33,8 @@ export declare function patchFiles(patch: string): {
 }[];
 export declare function applyPath(dir: string): string;
 export declare function writeApply(t: Tuning, dir: string): void;
-export declare function writeStatus(t: Tuning): void;
-export declare function markdownPage(title: string, md: string, live: boolean): string;
+/** A live line in running text: cut, and escaped. */
+export declare const liveCell: (text: string | undefined) => string;
+/** `live` reuses the parts read from every case and apply; only what moves by the second is read. */
+export declare function writeStatus(t: Tuning, live?: boolean): void;
 //# sourceMappingURL=report.d.ts.map