@zenera/cli 1.1.24 → 1.1.25

package/dist/meta/finetune/live.js

@@ -0,0 +1,226 @@
+import { closeSync, fstatSync, mkdirSync, openSync, readSync, renameSync, writeFileSync, } from 'node:fs';
+import { basename, join } from 'node:path';
+import { modelName, STAGES } from "./usage.js";
+const KEEP_LINES = 200;
+const KEEP_LOG = 50;
+const KEEP_ERRORS = 5;
+const PHASE_WORD = { nomem: 'no memory', mem: 'with memory' };
+/** Not log rows: a start shows in `now` and ends in an event of its own, an error has its own section. */
+const UNLOGGED = new Set(['run', 'analyze', 'apply', 'memory merge', 'idle', 'error']);
+const blank = () => ({ calls: 0, input: 0, output: 0, byModel: {}, lines: [] });
+function fold(a, text) {
+    let e;
+    try {
+        e = JSON.parse(text);
+    }
+    catch {
+        return;
+    }
+    a.lines.push(e);
+    if (a.lines.length > KEEP_LINES) {
+        a.lines.shift();
+    }
+    if (e.type === 'llm') {
+        const model = modelName(e.model);
+        a.model = model;
+        a.calls++;
+        a.input += e.in;
+        a.output += e.out;
+        const m = (a.byModel[model] ??= { calls: 0, input: 0, output: 0 });
+        m.calls++;
+        m.input += e.in;
+        m.output += e.out;
+    }
+    else if (e.type === 'tool' && e.phase === 'start') {
+        a.last = `${e.name} ${e.subject}`.trim();
+    }
+    else if (e.type === 'say') {
+        a.last = e.text;
+    }
+}
+/** Reads only what each events file gained since the last look; a file cleared for a new step starts over. */
+export class Tails {
+    #files = new Map();
+    read(file) {
+        let fd;
+        try {
+            fd = openSync(file, 'r');
+        }
+        catch {
+            this.#files.delete(file);
+            return blank();
+        }
+        try {
+            const st = fstatSync(fd);
+            let f = this.#files.get(file);
+            if (!f || f.ino !== st.ino || st.size < f.offset) {
+                f = { ino: st.ino, offset: 0, rest: '', act: blank() };
+                this.#files.set(file, f);
+            }
+            if (st.size > f.offset) {
+                const buf = Buffer.alloc(st.size - f.offset);
+                readSync(fd, buf, 0, buf.length, f.offset);
+                f.offset = st.size;
+                const parts = (f.rest + buf.toString('utf8')).split('\n');
+                f.rest = parts.pop() ?? '';
+                for (const line of parts) {
+                    fold(f.act, line);
+                }
+            }
+            return f.act;
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    /** Forgets the files no step writes any more. */
+    keep(files) {
+        for (const file of this.#files.keys()) {
+            if (!files.has(file)) {
+                this.#files.delete(file);
+            }
+        }
+    }
+}
+const tails = new WeakMap();
+function stateOf(t) {
+    if (t.finished) {
+        return t.stopping ? 'stopped' : 'finished';
+    }
+    return t.stopping ? 'stopping' : 'running';
+}
+function logRow(e) {
+    const where = e.phase ? `${PHASE_WORD[e.phase]}${e.attempt ? ` try ${e.attempt}` : ''}` : '';
+    return {
+        at: e.at,
+        action: e.what,
+        case: [e.case, where].filter(Boolean).join(' · '),
+        result: e.result ?? e.detail ?? '',
+        ...(e.took !== undefined ? { took: e.took } : {}),
+        ...(e.tokens ? { tokens: e.tokens } : {}),
+    };
+}
+export function snapshot(t) {
+    let tail = tails.get(t);
+    if (!tail) {
+        tail = new Tails();
+        tails.set(t, tail);
+    }
+    const now = [];
+    const files = new Set();
+    const watch = (file) => {
+        files.add(file);
+        return tail.read(file);
+    };
+    const clock = Date.now();
+    for (const w of t.workers) {
+        if (!w.case || (w.step !== 'run' && w.step !== 'analyze')) {
+            continue;
+        }
+        const dir = t.attemptDir(w.case, w.phase ?? 'nomem', w.attempt ?? 1);
+        now.push({
+            key: `w${w.slot}`,
+            who: `worker ${w.slot}`,
+            step: w.step,
+            what: `${w.case.id} · ${PHASE_WORD[w.phase ?? 'nomem']} try ${w.attempt ?? 1}`,
+            sinceMs: clock - w.since,
+            stage: w.step,
+            act: watch(join(dir, w.step === 'run' ? 'events.jsonl' : 'analyze.events.jsonl')),
+        });
+    }
+    const q = t.improvements;
+    if (q.current?.dir) {
+        now.push({
+            key: 'apply',
+            who: 'applier',
+            step: 'apply',
+            what: `apply ${basename(q.current.dir)} · ${q.current.requests} request(s)`,
+            sinceMs: clock - q.current.since,
+            stage: 'apply',
+            act: watch(join(q.current.dir, 'apply.events.jsonl')),
+        });
+    }
+    else if (q.current) {
+        now.push({
+            key: 'apply',
+            who: 'applier',
+            step: 'waiting',
+            what: `${q.pending.length} request(s), ${q.runs} run(s) to finish first`,
+            sinceMs: clock - q.current.since,
+        });
+    }
+    const merge = t.memories.merging;
+    if (merge) {
+        now.push({
+            key: 'merge',
+            who: 'memory',
+            step: 'merge',
+            what: merge.name,
+            sinceMs: clock - merge.since,
+        });
+    }
+    tail.keep(files);
+    const sums = new Map();
+    const add = (stage, model, s) => {
+        const key = `${stage}\u0000${model}`;
+        const row = sums.get(key) ?? { stage, model, calls: 0, input: 0, output: 0 };
+        row.calls += s.calls;
+        row.input += s.input;
+        row.output += s.output;
+        sums.set(key, row);
+    };
+    for (const stage of STAGES) {
+        for (const [model, s] of Object.entries(t.progress?.spent[stage] ?? {})) {
+            add(stage, model, s);
+        }
+    }
+    for (const row of now) {
+        for (const [model, s] of Object.entries(row.act?.byModel ?? {})) {
+            add(row.stage, model, s);
+        }
+    }
+    const tokens = [...sums.values()]
+        .filter((r) => r.calls > 0)
+        .sort((a, b) => STAGES.indexOf(a.stage) - STAGES.indexOf(b.stage) ||
+        b.input + b.output - (a.input + a.output));
+    return {
+        pid: process.pid,
+        project: basename(t.root),
+        state: stateOf(t),
+        at: new Date(clock).toISOString(),
+        elapsedMs: clock - t.startedAt,
+        system: t.system.version(),
+        workers: { busy: t.busy(), all: t.workers.length },
+        total: t.progress?.total ?? 0,
+        counts: t.progress?.counts ?? {},
+        parked: q.pending.length,
+        applyAt: t.config.applyAt,
+        now,
+        tokens,
+        log: t.recent
+            .filter((e) => !UNLOGGED.has(e.what))
+            .slice(0, KEEP_LOG)
+            .map(logRow),
+        errors: t.errors.slice(0, KEEP_ERRORS),
+    };
+}
+/** Takes a snapshot, keeps it in live.json without the raw lines, and hands it to every watcher. */
+export function publish(t) {
+    let snap;
+    try {
+        snap = snapshot(t);
+        const file = join(t.dir, 'live.json');
+        mkdirSync(t.dir, { recursive: true });
+        writeFileSync(`${file}.tmp`, `${JSON.stringify(snap, (k, v) => (k === 'lines' ? undefined : v), 2)}\n`);
+        renameSync(`${file}.tmp`, file);
+    }
+    catch {
+        // A live view that failed to draw must not stop the tuning.
+        return undefined;
+    }
+    for (const watcher of t.watchers) {
+        watcher(snap);
+    }
+    return snap;
+}
+//# sourceMappingURL=live.js.map

package/dist/meta/finetune/loop.d.ts

@@ -0,0 +1,3 @@
+import { type Tuning } from './tuning.ts';
+export declare function runLoop(t: Tuning): Promise<void>;
+//# sourceMappingURL=loop.d.ts.map

package/dist/meta/finetune/loop.js

@@ -0,0 +1,114 @@
+import { claimLock, ownLock } from '@zenera/neo';
+import { mkdirSync, rmSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+import { CliError, EXIT } from "../host.js";
+import { StepError } from "./failure.js";
+import { recover } from "./journal.js";
+import { publish } from "./live.js";
+import { nextCase, resultOf } from "./sampler.js";
+import { markFailed, SandboxDown, train, VoidRun } from "./train.js";
+import { Stopped } from "./tuning.js";
+// ---------------------------------------------------------------------------
+// The loop: N workers, cases started as they come free, one applier
+//
+// Whenever a worker is free - and no woken case is waiting for one - the next
+// case starts on it and is trained to the end. A parked case gives its worker
+// back, so a lane never sits idle waiting for an apply. A stop never interrupts
+// a step: it is noticed between steps, and the case resumes next time.
+// ---------------------------------------------------------------------------
+const REPORT_EVERY_MS = 15_000;
+const LIVE_EVERY_MS = 1000;
+export async function runLoop(t) {
+    mkdirSync(t.dir, { recursive: true });
+    const lock = join(t.dir, 'loop.lock');
+    claimLock(lock, ownLock(), (held) => new CliError(`a tuning is already running here (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `stop it with: zen meta finetune stop - or remove ${lock} if it crashed`));
+    rmSync(t.stopFile, { force: true });
+    const timer = setInterval(() => t.report(), REPORT_EVERY_MS);
+    const live = setInterval(() => publish(t), LIVE_EVERY_MS);
+    try {
+        await recover(t);
+        const start = t.system.record('start');
+        t.event({ what: 'started', detail: `system v${start.v}, ${t.workers.length} workers` });
+        publish(t);
+        const applier = t.improvements.loop();
+        await dispatch(t);
+        t.finished = true;
+        await applier;
+        await t.memories.finish();
+        t.event({ what: t.stopping ? 'stopped' : 'finished' });
+    }
+    finally {
+        clearInterval(timer);
+        clearInterval(live);
+        t.finished = true;
+        publish(t);
+        try {
+            unlinkSync(lock);
+        }
+        catch {
+            // Taken over as stale; nothing to give back.
+        }
+    }
+    if (t.fatal) {
+        throw t.fatal;
+    }
+}
+async function dispatch(t) {
+    const running = new Set();
+    while (!t.stopping) {
+        if (t.improvements.applying || t.resuming > 0 || !t.freeWorker()) {
+            await t.tick();
+            continue;
+        }
+        const c = nextCase(t);
+        if (!c) {
+            t.drained = true;
+            break;
+        }
+        const seat = { case: c };
+        t.claimed.add(c.id);
+        t.tried.add(c.id);
+        const before = resultOf(t, c);
+        if (before?.state === 'failed') {
+            // Taken again: until it ends anew it is in progress, not failed. Its analyze session
+            // may be what broke (another model, a dead provider), so the next analysis starts fresh.
+            rmSync(join(t.caseDir(c), 'result.json'), { force: true });
+            rmSync(join(t.caseDir(c), 'session'), { force: true });
+            t.event({ what: 'resumed', case: c.id, rev: c.rev, detail: before.reason });
+        }
+        await t.seat(seat, false);
+        const task = trainCase(t, seat).finally(() => running.delete(task));
+        running.add(task);
+    }
+    await Promise.all(running);
+}
+async function trainCase(t, s) {
+    const c = s.case;
+    try {
+        await train(t, s, c);
+    }
+    catch (err) {
+        if (!(err instanceof Stopped)) {
+            const why = err instanceof SandboxDown
+                ? `sandbox: ${err.message}`
+                : err instanceof VoidRun
+                    ? `void: ${err.message}`
+                    : `error: ${err.message}`;
+            markFailed(t, c, s.phase ?? 'nomem', why);
+            const attempt = s.worker?.attempt;
+            t.error(`${c.id} · ${s.phase === 'mem' ? 'with memory' : 'no memory'}${attempt ? ` try ${attempt}` : ''}`, err.message, err instanceof StepError ? err.log : undefined);
+            if (err instanceof SandboxDown) {
+                t.sandboxDown(err.message);
+            }
+            else {
+                t.failed(why);
+            }
+        }
+    }
+    finally {
+        t.unseat(s);
+        t.claimed.delete(c.id);
+        t.report();
+    }
+}
+//# sourceMappingURL=loop.js.map

package/dist/meta/finetune/memories.d.ts

@@ -0,0 +1,47 @@
+import type { Case } from './dataset/types.ts';
+import type { Run } from './train.ts';
+import { type Tuning } from './tuning.ts';
+export interface Submission {
+    /** `<case>@r<rev>` */
+    key: string;
+    case: string;
+    rev: number;
+    attempt: number;
+    run: string;
+    dir: string;
+    at: string;
+    /** the run committed nothing: there is no graph to merge */
+    empty: boolean;
+}
+export interface MergeRow {
+    n: number;
+    name: string;
+    at: string;
+    dir: string;
+    log: string;
+    /** every submission in this graph */
+    includes: string[];
+    /** the ones this merge brought in */
+    added: string[];
+    ok: boolean;
+    nodes?: number;
+    edges?: number;
+}
+export declare class Memories {
+    #private;
+    merging: {
+        name: string;
+        since: number;
+    } | undefined;
+    constructor(t: Tuning);
+    get dir(): string;
+    submissions(): Submission[];
+    merges(): MergeRow[];
+    lastMerge(): MergeRow | undefined;
+    unmerged(): Submission[];
+    /** Keeps a copy of a passing with-memory run's memory. Once per case revision. */
+    submit(c: Case, attempt: number, run: Run): void;
+    /** Merges whatever is left, and waits for every merge queued. */
+    finish(): Promise<void>;
+}
+//# sourceMappingURL=memories.d.ts.map

package/dist/meta/finetune/memories.js

@@ -0,0 +1,154 @@
+import { appendFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { DatasetStore } from "./dataset/store.js";
+import { errorIn, sizeOf } from "./failure.js";
+import { writeFeedback } from "./report.js";
+import { readJson } from "./tuning.js";
+function rows(file) {
+    try {
+        return readFileSync(file, 'utf8')
+            .split('\n')
+            .filter(Boolean)
+            .map((line) => JSON.parse(line));
+    }
+    catch {
+        return [];
+    }
+}
+export class Memories {
+    #t;
+    #chain = Promise.resolve();
+    merging;
+    constructor(t) {
+        this.#t = t;
+    }
+    get dir() {
+        return join(this.#t.dir, 'memories');
+    }
+    submissions() {
+        return rows(join(this.dir, 'submissions.jsonl'));
+    }
+    merges() {
+        return rows(join(this.dir, 'merges.jsonl'));
+    }
+    lastMerge() {
+        return this.merges()
+            .filter((m) => m.ok)
+            .at(-1);
+    }
+    unmerged() {
+        const merged = new Set(this.lastMerge()?.includes ?? []);
+        return this.submissions().filter((s) => !s.empty && !merged.has(s.key));
+    }
+    /** Keeps a copy of a passing with-memory run's memory. Once per case revision. */
+    submit(c, attempt, run) {
+        const key = `${c.id}@r${c.rev}`;
+        if (this.submissions().some((s) => s.key === key)) {
+            return;
+        }
+        const dir = join(this.dir, 'submitted', key);
+        const empty = !existsSync(join(run.memory, 'manifest.json'));
+        rmSync(dir, { recursive: true, force: true });
+        if (!empty) {
+            cpSync(run.memory, dir, {
+                recursive: true,
+                filter: (src) => basename(src) !== '.lock',
+            });
+        }
+        const row = {
+            key,
+            case: c.id,
+            rev: c.rev,
+            attempt,
+            run: run.dir,
+            dir,
+            at: new Date().toISOString(),
+            empty,
+        };
+        mkdirSync(this.dir, { recursive: true });
+        appendFileSync(join(this.dir, 'submissions.jsonl'), `${JSON.stringify(row)}\n`);
+        this.#t.event({
+            what: 'memory submitted',
+            case: c.id,
+            rev: c.rev,
+            detail: empty ? 'nothing committed' : `${this.unmerged().length} waiting to merge`,
+        });
+        if (this.unmerged().length >= this.#t.config.mergeEvery) {
+            this.#queue();
+        }
+    }
+    /** Merges whatever is left, and waits for every merge queued. */
+    async finish() {
+        if (this.unmerged().length > 0) {
+            this.#queue();
+        }
+        await this.#chain;
+    }
+    #queue() {
+        this.#chain = this.#chain
+            .then(() => this.#merge())
+            .catch((err) => {
+            this.#t.error('memory merge', err.message);
+        });
+    }
+    async #merge() {
+        const fresh = this.unmerged();
+        if (fresh.length === 0) {
+            return;
+        }
+        const n = this.merges().length + 1;
+        const name = `m${String(n).padStart(2, '0')}`;
+        const target = join(this.dir, 'merged', name);
+        const log = join(this.dir, 'merged', `${name}.log`);
+        await this.#t.journal.step({ kind: 'merge', dir: target, log }, () => this.#mergeInto(fresh, n, name, target, log));
+    }
+    async #mergeInto(fresh, n, name, target, log) {
+        const last = this.lastMerge();
+        rmSync(target, { recursive: true, force: true });
+        mkdirSync(join(this.dir, 'merged'), { recursive: true });
+        const began = Date.now();
+        const from = sizeOf(log);
+        this.merging = { name, since: began };
+        this.#t.event({ what: 'memory merge', detail: `${name}: ${fresh.length} new` });
+        const sources = [...(last ? [last.dir] : []), ...fresh.map((s) => s.dir)];
+        const res = await this.#t
+            .zen(['memory', 'merge', ...sources, '--dir', target, '--yes'], log)
+            .finally(() => {
+            this.merging = undefined;
+        });
+        const manifest = readJson(join(target, 'manifest.json'));
+        const row = {
+            n,
+            name,
+            at: new Date().toISOString(),
+            dir: target,
+            log,
+            includes: [...(last?.includes ?? []), ...fresh.map((s) => s.key)],
+            added: fresh.map((s) => s.key),
+            ok: res.code === 0 && manifest !== undefined,
+            ...(manifest?.nodes !== undefined ? { nodes: manifest.nodes } : {}),
+            ...(manifest?.edges !== undefined ? { edges: manifest.edges } : {}),
+        };
+        appendFileSync(join(this.dir, 'merges.jsonl'), `${JSON.stringify(row)}\n`);
+        if (!row.ok) {
+            const e = errorIn(log, from);
+            this.#t.error(`memory ${name}`, e.message ?? 'the merge wrote no manifest', e.log);
+        }
+        const store = DatasetStore.open(this.#t.root);
+        for (const s of fresh) {
+            const c = store.get(s.case);
+            if (c?.rev === s.rev) {
+                writeFeedback(this.#t, c);
+            }
+        }
+        this.#t.event({
+            what: row.ok ? 'memory merged' : 'memory merge failed',
+            detail: `${name}: ${row.includes.length} case(s)${row.ok ? '' : ' - see the log'}`,
+            result: row.ok
+                ? `${row.includes.length} case(s), ${row.nodes ?? '?'} nodes`
+                : 'see the log',
+            took: Date.now() - began,
+        });
+    }
+}
+//# sourceMappingURL=memories.js.map

package/dist/meta/finetune/report.d.ts

@@ -0,0 +1,33 @@
+import { type Case } from './dataset/types.ts';
+import { type Feedback, type Phase } from './feedback.ts';
+import type { Run } from './train.ts';
+import { type Tuning } from './tuning.ts';
+import { type ByStage } from './usage.ts';
+export interface Attempt {
+    phase: Phase;
+    attempt: number;
+    dir: string;
+    run?: Run;
+    feedback?: Feedback;
+}
+/** A table cell: one line, no column breaks. */
+export declare const cell: (text: string | number | undefined) => string;
+export declare function since(ms: number): string;
+export declare function attemptsOf(t: Tuning, c: Case): Attempt[];
+/** Every try of a case, per stage and model. Applies serve many cases at once, so they are not split. */
+export declare function caseTokens(t: Tuning, c: Case): ByStage;
+export declare function feedbackPath(t: Tuning, c: {
+    id: string;
+}): string;
+export declare function writeFeedback(t: Tuning, c: Case): void;
+export declare function patchFiles(patch: string): {
+    file: string;
+    added: number;
+    removed: number;
+    text: string;
+}[];
+export declare function applyPath(dir: string): string;
+export declare function writeApply(t: Tuning, dir: string): void;
+export declare function writeStatus(t: Tuning): void;
+export declare function markdownPage(title: string, md: string, live: boolean): string;
+//# sourceMappingURL=report.d.ts.map