@zenera/cli 1.1.24 → 1.1.26

package/dist/meta/finetune/memories.js

@@ -0,0 +1,154 @@
+import { appendFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { DatasetStore } from "./dataset/store.js";
+import { errorIn, sizeOf } from "./failure.js";
+import { writeFeedback } from "./report.js";
+import { readJson } from "./tuning.js";
+function rows(file) {
+    try {
+        return readFileSync(file, 'utf8')
+            .split('\n')
+            .filter(Boolean)
+            .map((line) => JSON.parse(line));
+    }
+    catch {
+        return [];
+    }
+}
+export class Memories {
+    #t;
+    #chain = Promise.resolve();
+    merging;
+    constructor(t) {
+        this.#t = t;
+    }
+    get dir() {
+        return join(this.#t.dir, 'memories');
+    }
+    submissions() {
+        return rows(join(this.dir, 'submissions.jsonl'));
+    }
+    merges() {
+        return rows(join(this.dir, 'merges.jsonl'));
+    }
+    lastMerge() {
+        return this.merges()
+            .filter((m) => m.ok)
+            .at(-1);
+    }
+    unmerged() {
+        const merged = new Set(this.lastMerge()?.includes ?? []);
+        return this.submissions().filter((s) => !s.empty && !merged.has(s.key));
+    }
+    /** Keeps a copy of a passing with-memory run's memory. Once per case revision. */
+    submit(c, attempt, run) {
+        const key = `${c.id}@r${c.rev}`;
+        if (this.submissions().some((s) => s.key === key)) {
+            return;
+        }
+        const dir = join(this.dir, 'submitted', key);
+        const empty = !existsSync(join(run.memory, 'manifest.json'));
+        rmSync(dir, { recursive: true, force: true });
+        if (!empty) {
+            cpSync(run.memory, dir, {
+                recursive: true,
+                filter: (src) => basename(src) !== '.lock',
+            });
+        }
+        const row = {
+            key,
+            case: c.id,
+            rev: c.rev,
+            attempt,
+            run: run.dir,
+            dir,
+            at: new Date().toISOString(),
+            empty,
+        };
+        mkdirSync(this.dir, { recursive: true });
+        appendFileSync(join(this.dir, 'submissions.jsonl'), `${JSON.stringify(row)}\n`);
+        this.#t.event({
+            what: 'memory submitted',
+            case: c.id,
+            rev: c.rev,
+            detail: empty ? 'nothing committed' : `${this.unmerged().length} waiting to merge`,
+        });
+        if (this.unmerged().length >= this.#t.config.mergeEvery) {
+            this.#queue();
+        }
+    }
+    /** Merges whatever is left, and waits for every merge queued. */
+    async finish() {
+        if (this.unmerged().length > 0) {
+            this.#queue();
+        }
+        await this.#chain;
+    }
+    #queue() {
+        this.#chain = this.#chain
+            .then(() => this.#merge())
+            .catch((err) => {
+            this.#t.error('memory merge', err.message);
+        });
+    }
+    async #merge() {
+        const fresh = this.unmerged();
+        if (fresh.length === 0) {
+            return;
+        }
+        const n = this.merges().length + 1;
+        const name = `m${String(n).padStart(2, '0')}`;
+        const target = join(this.dir, 'merged', name);
+        const log = join(this.dir, 'merged', `${name}.log`);
+        await this.#t.journal.step({ kind: 'merge', dir: target, log }, () => this.#mergeInto(fresh, n, name, target, log));
+    }
+    async #mergeInto(fresh, n, name, target, log) {
+        const last = this.lastMerge();
+        rmSync(target, { recursive: true, force: true });
+        mkdirSync(join(this.dir, 'merged'), { recursive: true });
+        const began = Date.now();
+        const from = sizeOf(log);
+        this.merging = { name, since: began };
+        this.#t.event({ what: 'memory merge', detail: `${name}: ${fresh.length} new` });
+        const sources = [...(last ? [last.dir] : []), ...fresh.map((s) => s.dir)];
+        const res = await this.#t
+            .zen(['memory', 'merge', ...sources, '--dir', target, '--yes'], log)
+            .finally(() => {
+            this.merging = undefined;
+        });
+        const manifest = readJson(join(target, 'manifest.json'));
+        const row = {
+            n,
+            name,
+            at: new Date().toISOString(),
+            dir: target,
+            log,
+            includes: [...(last?.includes ?? []), ...fresh.map((s) => s.key)],
+            added: fresh.map((s) => s.key),
+            ok: res.code === 0 && manifest !== undefined,
+            ...(manifest?.nodes !== undefined ? { nodes: manifest.nodes } : {}),
+            ...(manifest?.edges !== undefined ? { edges: manifest.edges } : {}),
+        };
+        appendFileSync(join(this.dir, 'merges.jsonl'), `${JSON.stringify(row)}\n`);
+        if (!row.ok) {
+            const e = errorIn(log, from);
+            this.#t.error(`memory ${name}`, e.message ?? 'the merge wrote no manifest', e.log);
+        }
+        const store = DatasetStore.open(this.#t.root);
+        for (const s of fresh) {
+            const c = store.get(s.case);
+            if (c?.rev === s.rev) {
+                writeFeedback(this.#t, c);
+            }
+        }
+        this.#t.event({
+            what: row.ok ? 'memory merged' : 'memory merge failed',
+            detail: `${name}: ${row.includes.length} case(s)${row.ok ? '' : ' - see the log'}`,
+            result: row.ok
+                ? `${row.includes.length} case(s), ${row.nodes ?? '?'} nodes`
+                : 'see the log',
+            took: Date.now() - began,
+        });
+    }
+}
+//# sourceMappingURL=memories.js.map

package/dist/meta/finetune/recheck.d.ts

@@ -0,0 +1,58 @@
+import type { Case } from './dataset/types.ts';
+import { type Seat, type Tuning } from './tuning.ts';
+import { type ByStage } from './usage.ts';
+export interface Recheck {
+    case: string;
+    rev: number;
+    /** the apply whose version it runs on */
+    apply: string;
+    state: 'queued' | 'running' | 'passed' | 'regressed' | 'void';
+    at: string;
+    /** each try's verdict */
+    tries?: string[];
+    /** the request a regression sent to the improvement queue */
+    request?: string;
+    /** why it was void, or why a regression sent no request */
+    note?: string;
+}
+/** A case regresses only when every one of these fails: one failure may be the model's noise. */
+export declare const RECHECK_TRIES = 2;
+export declare class Rechecks {
+    #private;
+    constructor(t: Tuning);
+    get file(): string;
+    all(): Recheck[];
+    dir(r: {
+        case: string;
+        apply: string;
+    }): string;
+    /** Each try's folder, oldest first. */
+    tryDirs(r: {
+        case: string;
+        apply: string;
+    }): string[];
+    waiting(): number;
+    /**
+     * After a stop: a recheck cut off part-way runs again, and a regression's request
+     * - kept only in memory until an apply takes it - goes back in the queue.
+     */
+    resume(): void;
+    /**
+     * Queues completed cases to run on the version `apply` made: a case whose
+     * last recheck regressed first - this apply was meant to repair it - then
+     * the rest in an order fixed by the seed and the apply.
+     */
+    queue(apply: string): Recheck[];
+    /** The next queued recheck, marked running, with its case; one whose case changed is dropped. */
+    take(): {
+        recheck: Recheck;
+        case: Case;
+    } | undefined;
+    /** A step broke for a reason outside the prose: the recheck says nothing. */
+    void(r: Recheck, why: string): void;
+    /** Runs a recheck on the seat's worker, to its verdict. */
+    run(s: Seat, r: Recheck, c: Case): Promise<void>;
+    /** Every recheck run and analysis so far. */
+    tokens(): ByStage;
+}
+//# sourceMappingURL=recheck.d.ts.map

package/dist/meta/finetune/recheck.js

@@ -0,0 +1,311 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { writeAnalyzeBrief } from "./analyze-brief.js";
+import { readFeedback } from "./feedback.js";
+import { writeApply } from "./report.js";
+import { resultOf, selection } from "./sampler.js";
+import { keepAnalysis, runOnce } from "./train.js";
+import { readJson } from "./tuning.js";
+import { envelopesIn, mergeStages, spent } from "./usage.js";
+/** A case regresses only when every one of these fails: one failure may be the model's noise. */
+export const RECHECK_TRIES = 2;
+/** Requests one case's regressions may send; past that it is reported, so the tuning ends. */
+const REQUESTS_PER_CASE = 2;
+const pad = (n) => String(n).padStart(2, '0');
+export class Rechecks {
+    #t;
+    constructor(t) {
+        this.#t = t;
+    }
+    get file() {
+        return join(this.#t.dir, 'rechecks.json');
+    }
+    all() {
+        return readJson(this.file) ?? [];
+    }
+    #save(all) {
+        mkdirSync(this.#t.dir, { recursive: true });
+        writeFileSync(this.file, `${JSON.stringify(all, null, 2)}\n`);
+    }
+    #update(r, change) {
+        const all = this.all();
+        const i = all.findIndex((x) => x.case === r.case && x.apply === r.apply);
+        Object.assign(r, change, { at: new Date().toISOString() });
+        if (i >= 0) {
+            all[i] = r;
+            this.#save(all);
+        }
+        if (change.state && change.state !== 'running') {
+            this.#page(r.apply);
+        }
+    }
+    /** The apply's page lists its rechecks; it was written before they ran. */
+    #page(apply) {
+        try {
+            writeApply(this.#t, join(this.#t.improvements.dir, apply));
+        }
+        catch {
+            // A page that failed to render must not stop the tuning.
+        }
+    }
+    dir(r) {
+        return join(this.#t.dir, 'rechecks', r.apply, r.case);
+    }
+    /** Each try's folder, oldest first. */
+    tryDirs(r) {
+        try {
+            return readdirSync(this.dir(r))
+                .filter((n) => /^\d{2}-nomem$/.test(n))
+                .sort()
+                .map((n) => join(this.dir(r), n));
+        }
+        catch {
+            return [];
+        }
+    }
+    waiting() {
+        return this.all().filter((r) => r.state === 'queued' || r.state === 'running').length;
+    }
+    /**
+     * After a stop: a recheck cut off part-way runs again, and a regression's request
+     * - kept only in memory until an apply takes it - goes back in the queue.
+     */
+    resume() {
+        const all = this.all();
+        let changed = false;
+        for (const r of all) {
+            if (r.state === 'running') {
+                r.state = 'queued';
+                changed = true;
+            }
+            if (r.state === 'regressed' &&
+                r.request &&
+                !this.#t.improvements.isApplied(r.request)) {
+                const f = this.#feedback(r);
+                if (f) {
+                    this.#t.improvements.regressed(f);
+                }
+            }
+        }
+        if (changed) {
+            this.#save(all);
+        }
+    }
+    /**
+     * Queues completed cases to run on the version `apply` made: a case whose
+     * last recheck regressed first - this apply was meant to repair it - then
+     * the rest in an order fixed by the seed and the apply.
+     */
+    queue(apply) {
+        const t = this.#t;
+        if (t.config.recheck === 0) {
+            return [];
+        }
+        const all = this.all();
+        const open = new Set(all.filter((r) => r.state === 'queued' || r.state === 'running').map((r) => r.case));
+        const last = new Map(all.map((r) => [r.case, r]));
+        const key = (id) => createHash('sha256').update(`${t.config.seed}:${apply}:${id}`).digest('hex');
+        const rank = (id) => (last.get(id)?.state === 'regressed' ? 0 : 1);
+        const at = new Date().toISOString();
+        const picked = selection(t)
+            .filter((c) => !open.has(c.id) && resultOf(t, c)?.state === 'completed')
+            .sort((a, b) => rank(a.id) - rank(b.id) || key(a.id).localeCompare(key(b.id)))
+            .slice(0, t.config.recheck)
+            .map((c) => ({ case: c.id, rev: c.rev, apply, state: 'queued', at }));
+        if (picked.length > 0) {
+            this.#save([...all, ...picked]);
+            this.#page(apply);
+            t.event({
+                what: 'recheck',
+                detail: `${apply}: ${picked.map((r) => r.case).join(', ')} queued to run again on the edit`,
+            });
+        }
+        return picked;
+    }
+    /** The next queued recheck, marked running, with its case; one whose case changed is dropped. */
+    take() {
+        const t = this.#t;
+        for (;;) {
+            const r = this.all().find((x) => x.state === 'queued' && !t.claimed.has(x.case));
+            if (!r) {
+                return undefined;
+            }
+            const c = selection(t).find((x) => x.id === r.case);
+            if (!c || c.rev !== r.rev || resultOf(t, c)?.state !== 'completed') {
+                this.#update(r, { state: 'void', note: 'the case changed since it was queued' });
+                continue;
+            }
+            this.#update(r, { state: 'running' });
+            return { recheck: r, case: c };
+        }
+    }
+    /** A step broke for a reason outside the prose: the recheck says nothing. */
+    void(r, why) {
+        this.#update(r, { state: 'void', note: why });
+    }
+    /** Runs a recheck on the seat's worker, to its verdict. */
+    async run(s, r, c) {
+        const t = this.#t;
+        const tries = [];
+        let last;
+        for (let n = 1; n <= RECHECK_TRIES; n++) {
+            const dir = join(this.dir(r), `${pad(n)}-nomem`);
+            const one = await this.#try(s, r, c, n, dir);
+            tries.push(one.verdict);
+            if (one.passed) {
+                const state = one.verdict === 'right' ? 'passed' : 'void';
+                this.#update(r, { state, tries, ...(one.note ? { note: one.note } : {}) });
+                t.event({
+                    what: `recheck ${state}`,
+                    case: c.id,
+                    rev: c.rev,
+                    result: tries.join(', '),
+                });
+                return;
+            }
+            last = one.feedback;
+        }
+        const sent = this.all().filter((x) => x.case === c.id && x.request).length;
+        if (last && sent < REQUESTS_PER_CASE) {
+            const request = `${c.id}@recheck-${r.apply}`;
+            const f = {
+                ...last,
+                id: request,
+                recheck: r.apply,
+                dir: join(this.dir(r), `${pad(RECHECK_TRIES)}-nomem`),
+            };
+            this.#update(r, { state: 'regressed', tries, request });
+            t.improvements.regressed(f);
+        }
+        else {
+            this.#update(r, {
+                state: 'regressed',
+                tries,
+                note: `${sent} request(s) sent for this case already - reported, not queued`,
+            });
+        }
+        t.event({
+            what: 'regressed',
+            case: c.id,
+            rev: c.rev,
+            detail: `passed before ${r.apply}, failed ${RECHECK_TRIES} tries on it`,
+            result: tries.join(', '),
+        });
+    }
+    /** A void run or an unreadable analysis counts as a pass: only a graded failure is a regression. */
+    async #try(s, r, c, n, dir) {
+        const t = this.#t;
+        const w = s.worker;
+        mkdirSync(dir, { recursive: true });
+        w.recheck = { apply: r.apply, dir };
+        let why;
+        if (!existsSync(join(dir, 'run.json'))) {
+            await t.improvements.notApplying();
+            t.at(s, 'nomem', n, 'run');
+            t.improvements.runs++;
+            const began = Date.now();
+            try {
+                why = await runOnce(t, c, dir);
+            }
+            finally {
+                t.improvements.runs--;
+            }
+            const ran = why ? undefined : readJson(join(dir, 'run.json'));
+            t.ended(s, {
+                what: why ? 'void' : 'ran',
+                phase: 'nomem',
+                attempt: n,
+                result: `recheck: ${why ?? `${ran?.turns ?? '?'} turns`}`,
+                took: Date.now() - began,
+                tokens: spent(ran?.tokens),
+            });
+        }
+        if (why) {
+            return { verdict: 'void', passed: true, note: why };
+        }
+        const run = readJson(join(dir, 'run.json'));
+        const file = join(dir, 'feedback.json');
+        const known = { case: c.id, caseRev: c.rev, phase: 'nomem', attempt: n };
+        const rubricIds = c.rubric.map((x) => x.id);
+        let read = readFeedback(file, known, rubricIds);
+        if (!('feedback' in read)) {
+            t.checkStopping();
+            t.at(s, 'nomem', n, 'analyze');
+            const began = Date.now();
+            const patch = join(t.improvements.dir, r.apply, 'diff.patch');
+            const session = randomUUID();
+            writeFileSync(join(dir, 'session'), `${session}\n`);
+            const brief = await writeAnalyzeBrief(t, dir, {
+                run,
+                case: c,
+                phase: 'nomem',
+                attempt: n,
+                earlier: [],
+                recheck: patch,
+            });
+            const res = await t.zen([
+                'meta',
+                'run',
+                '--no-refresh',
+                '--json',
+                '--events',
+                join(dir, 'analyze.events.jsonl'),
+                '--session-id',
+                session,
+                '--prompt-file',
+                brief,
+                '/analyze',
+                run.dir,
+                `case=${c.id}`,
+                `rev=${c.rev}`,
+                'phase=nomem',
+                `attempt=${n}`,
+                `feedback=${file}`,
+                `recheck=${patch}`,
+            ], join(dir, 'analyze.log'));
+            keepAnalysis(dir, res.stdout);
+            read = readFeedback(file, known, rubricIds);
+            t.ended(s, {
+                what: 'analyzed',
+                phase: 'nomem',
+                attempt: n,
+                result: `recheck: ${'feedback' in read ? read.feedback.verdict : read.problem}`,
+                took: Date.now() - began,
+                tokens: spent(envelopesIn(dir, 'analyze')),
+            });
+        }
+        if (!('feedback' in read)) {
+            return { verdict: 'unread', passed: true, note: read.problem };
+        }
+        const f = read.feedback;
+        if (f.verdict === 'void') {
+            return { verdict: 'void', passed: true, note: f.infra };
+        }
+        const passed = f.verdict === 'right' && !Object.values(f.rubric).includes('fail');
+        return { verdict: f.verdict, passed, feedback: f };
+    }
+    /** A regression's request, read back from its last try. */
+    #feedback(r) {
+        const c = selection(this.#t).find((x) => x.id === r.case);
+        const dir = join(this.dir(r), `${pad(RECHECK_TRIES)}-nomem`);
+        const read = readFeedback(join(dir, 'feedback.json'), { case: r.case, caseRev: r.rev, phase: 'nomem', attempt: RECHECK_TRIES }, c?.rubric.map((x) => x.id) ?? []);
+        return 'feedback' in read
+            ? { ...read.feedback, id: r.request, recheck: r.apply, dir }
+            : undefined;
+    }
+    /** Every recheck run and analysis so far. */
+    tokens() {
+        const parts = [];
+        for (const r of this.all()) {
+            for (const dir of this.tryDirs(r)) {
+                parts.push({
+                    run: readJson(join(dir, 'run.json'))?.tokens,
+                    analyze: envelopesIn(dir, 'analyze'),
+                });
+            }
+        }
+        return mergeStages(...parts);
+    }
+}
+//# sourceMappingURL=recheck.js.map

package/dist/meta/finetune/report.d.ts

@@ -0,0 +1,40 @@
+import { type Case } from './dataset/types.ts';
+import { type Feedback, type Phase } from './feedback.ts';
+import type { Run } from './train.ts';
+import { type Tuning } from './tuning.ts';
+import { type ByStage } from './usage.ts';
+export interface Attempt {
+    phase: Phase;
+    attempt: number;
+    dir: string;
+    run?: Run;
+    feedback?: Feedback;
+}
+/** A table cell: one line, no column breaks. */
+export declare const cell: (text: string | number | undefined) => string;
+export declare function since(ms: number): string;
+/** The analyze session of one try, kept in its folder. */
+export declare function analyzeSession(dir: string): string | undefined;
+export declare function attemptsOf(t: Tuning, c: Case): Attempt[];
+/** Every try of a case, per stage and model. Applies serve many cases at once, so they are not split. */
+export declare function caseTokens(t: Tuning, c: Case): ByStage;
+export declare const rubricScore: (f: Attempt["feedback"]) => string;
+export declare function feedbackPath(t: Tuning, c: {
+    id: string;
+}): string;
+export declare function writeFeedback(t: Tuning, c: Case): void;
+/** A fence longer than any run of backticks in `text`, so nothing inside can close it. */
+export declare function fence(text: string): string;
+export declare function patchFiles(patch: string): {
+    file: string;
+    added: number;
+    removed: number;
+    text: string;
+}[];
+export declare function applyPath(dir: string): string;
+export declare function writeApply(t: Tuning, dir: string): void;
+/** A live line in running text: cut, and escaped. */
+export declare const liveCell: (text: string | undefined) => string;
+/** `live` reuses the parts read from every case and apply; only what moves by the second is read. */
+export declare function writeStatus(t: Tuning, live?: boolean): void;
+//# sourceMappingURL=report.d.ts.map