@zenera/cli 1.1.24 → 1.1.26

package/dist/meta/finetune/live.d.ts

@@ -0,0 +1,84 @@
+import type { EventLine } from '../host.ts';
+import type { StepFailure, Tuning } from './tuning.ts';
+import { type Stage, type Tokens } from './usage.ts';
+export type ModelSpend = Tokens;
+/** One events file, folded as it grows. */
+export interface Activity {
+    model?: string;
+    calls: number;
+    input: number;
+    output: number;
+    byModel: Record<string, ModelSpend>;
+    /** the latest tool call, with what was said just before it, or the latest thing said */
+    last?: string;
+    /** said since the last tool call */
+    said?: string;
+    /** what the agent is doing now: `says: …`, `thinks: …` or `runs: …` */
+    now?: string;
+    /** the last few `now` lines, oldest first */
+    recent: string[];
+    /** the latest lines, oldest first */
+    lines: EventLine[];
+}
+export interface NowRow {
+    /** stable across snapshots: `w1`, `apply`, `merge` */
+    key: string;
+    who: string;
+    step: string;
+    what: string;
+    sinceMs: number;
+    stage?: Stage;
+    act?: Activity;
+    /** what it is doing now, in one line */
+    now?: string;
+    /** the last few such lines, oldest first */
+    recent?: string[];
+}
+export interface TokenRow extends ModelSpend {
+    stage: Stage;
+    model: string;
+}
+export interface LogRow {
+    at: string;
+    action: string;
+    case: string;
+    result: string;
+    took?: number;
+    tokens?: number;
+}
+export interface Snapshot {
+    pid: number;
+    project: string;
+    state: 'running' | 'stopping' | 'stopped' | 'finished';
+    at: string;
+    elapsedMs: number;
+    system: number;
+    workers: {
+        busy: number;
+        all: number;
+    };
+    total: number;
+    counts: Record<string, number>;
+    parked: number;
+    applyAt: number;
+    now: NowRow[];
+    tokens: TokenRow[];
+    log: LogRow[];
+    /** the latest step failures, newest first */
+    errors: StepFailure[];
+}
+/** What the applier runs once the agent is done. */
+export declare const CHECK_NOW = "runs: zen check --no-models --no-sandbox";
+/** `now` lines a step keeps. */
+export declare const KEEP_NOW = 6;
+/** Reads only what each events file gained since the last look; a file cleared for a new step starts over. */
+export declare class Tails {
+    #private;
+    read(file: string): Activity;
+    /** Forgets the files no step writes any more. */
+    keep(files: ReadonlySet<string>): void;
+}
+export declare function snapshot(t: Tuning): Snapshot;
+/** Takes a snapshot, keeps it in live.json without the raw lines, and hands it to every watcher. */
+export declare function publish(t: Tuning): Snapshot | undefined;
+//# sourceMappingURL=live.d.ts.map

package/dist/meta/finetune/live.js

@@ -0,0 +1,272 @@
+import { closeSync, fstatSync, mkdirSync, openSync, readSync, renameSync, writeFileSync, } from 'node:fs';
+import { basename, join } from 'node:path';
+import { modelName, STAGES } from "./usage.js";
+const KEEP_LINES = 200;
+const KEEP_LOG = 50;
+const KEEP_ERRORS = 5;
+const PHASE_WORD = { nomem: 'no memory', mem: 'with memory' };
+/** Not log rows: a start shows in `now` and ends in an event of its own, an error has its own section. */
+const UNLOGGED = new Set(['run', 'analyze', 'apply', 'memory merge', 'idle', 'error']);
+/** Shell tools: their subject is the command itself. */
+const SHELL = new Set(['bash', 'run_command', 'run_command_background']);
+/** What the applier runs once the agent is done. */
+export const CHECK_NOW = 'runs: zen check --no-models --no-sandbox';
+/** `now` lines a step keeps. */
+export const KEEP_NOW = 6;
+const blank = () => ({
+    calls: 0,
+    input: 0,
+    output: 0,
+    byModel: {},
+    recent: [],
+    lines: [],
+});
+function doing(a, line) {
+    a.now = line;
+    a.recent.push(line);
+    if (a.recent.length > KEEP_NOW) {
+        a.recent.shift();
+    }
+}
+function fold(a, text) {
+    let e;
+    try {
+        e = JSON.parse(text);
+    }
+    catch {
+        return;
+    }
+    a.lines.push(e);
+    if (a.lines.length > KEEP_LINES) {
+        a.lines.shift();
+    }
+    if (e.type === 'llm') {
+        const model = modelName(e.model);
+        a.model = model;
+        a.calls++;
+        a.input += e.in;
+        a.output += e.out;
+        const m = (a.byModel[model] ??= { calls: 0, input: 0, cached: 0, output: 0, reasoning: 0 });
+        m.calls++;
+        m.input += e.in;
+        m.cached += e.cached;
+        m.output += e.out;
+        m.reasoning += e.reasoning;
+    }
+    else if (e.type === 'tool' && e.phase === 'start') {
+        const tool = `${e.name} ${e.subject}`.trim();
+        a.last = a.said ? `${a.said} · ${tool}` : tool;
+        a.said = undefined;
+        doing(a, `runs: ${SHELL.has(e.name) && e.subject ? e.subject : tool}`);
+    }
+    else if (e.type === 'say' || e.type === 'think') {
+        // A call's thought comes before what it said, so the words win.
+        a.said = e.text;
+        a.last = e.text;
+        doing(a, `${e.type === 'say' ? 'says' : 'thinks'}: ${e.text}`);
+    }
+}
+/** Reads only what each events file gained since the last look; a file cleared for a new step starts over. */
+export class Tails {
+    #files = new Map();
+    read(file) {
+        let fd;
+        try {
+            fd = openSync(file, 'r');
+        }
+        catch {
+            this.#files.delete(file);
+            return blank();
+        }
+        try {
+            const st = fstatSync(fd);
+            let f = this.#files.get(file);
+            if (!f || f.ino !== st.ino || st.size < f.offset) {
+                f = { ino: st.ino, offset: 0, rest: '', act: blank() };
+                this.#files.set(file, f);
+            }
+            if (st.size > f.offset) {
+                const buf = Buffer.alloc(st.size - f.offset);
+                readSync(fd, buf, 0, buf.length, f.offset);
+                f.offset = st.size;
+                const parts = (f.rest + buf.toString('utf8')).split('\n');
+                f.rest = parts.pop() ?? '';
+                for (const line of parts) {
+                    fold(f.act, line);
+                }
+            }
+            return f.act;
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    /** Forgets the files no step writes any more. */
+    keep(files) {
+        for (const file of this.#files.keys()) {
+            if (!files.has(file)) {
+                this.#files.delete(file);
+            }
+        }
+    }
+}
+const tails = new WeakMap();
+function stateOf(t) {
+    if (t.finished) {
+        return t.stopping ? 'stopped' : 'finished';
+    }
+    return t.stopping ? 'stopping' : 'running';
+}
+function logRow(e) {
+    const where = e.phase ? `${PHASE_WORD[e.phase]}${e.attempt ? ` try ${e.attempt}` : ''}` : '';
+    return {
+        at: e.at,
+        action: e.what,
+        case: [e.case, where].filter(Boolean).join(' · '),
+        result: e.result ?? e.detail ?? '',
+        ...(e.took !== undefined ? { took: e.took } : {}),
+        ...(e.tokens ? { tokens: e.tokens } : {}),
+    };
+}
+export function snapshot(t) {
+    let tail = tails.get(t);
+    if (!tail) {
+        tail = new Tails();
+        tails.set(t, tail);
+    }
+    const now = [];
+    const files = new Set();
+    const watch = (file) => {
+        files.add(file);
+        return tail.read(file);
+    };
+    const clock = Date.now();
+    for (const w of t.workers) {
+        if (!w.case || (w.step !== 'run' && w.step !== 'analyze')) {
+            continue;
+        }
+        const dir = t.workDir({ ...w, case: w.case });
+        const act = watch(join(dir, w.step === 'run' ? 'events.jsonl' : 'analyze.events.jsonl'));
+        now.push({
+            key: `w${w.slot}`,
+            who: `worker ${w.slot}`,
+            step: w.step,
+            what: `${w.case.id} · ${w.recheck ? 'recheck' : PHASE_WORD[w.phase ?? 'nomem']} try ${w.attempt ?? 1}`,
+            sinceMs: clock - w.since,
+            stage: w.step,
+            act,
+            ...(act.now ? { now: act.now, recent: [...act.recent] } : {}),
+        });
+    }
+    const q = t.improvements;
+    if (q.current?.dir) {
+        const checking = q.current.step === 'check';
+        const act = watch(join(q.current.dir, 'apply.events.jsonl'));
+        const line = checking ? CHECK_NOW : act.now;
+        const recent = checking ? [...act.recent, CHECK_NOW].slice(-KEEP_NOW) : [...act.recent];
+        now.push({
+            key: 'apply',
+            who: 'applier',
+            step: checking ? 'check' : 'apply',
+            what: `apply ${basename(q.current.dir)} · ${q.current.requests} request(s)`,
+            sinceMs: clock - q.current.since,
+            stage: 'apply',
+            act,
+            ...(line ? { now: line, recent } : {}),
+        });
+    }
+    else if (q.current) {
+        now.push({
+            key: 'apply',
+            who: 'applier',
+            step: 'waiting',
+            what: `${q.pending.length} request(s), ${q.runs} run(s) to finish first`,
+            sinceMs: clock - q.current.since,
+        });
+    }
+    const merge = t.memories.merging;
+    if (merge) {
+        now.push({
+            key: 'merge',
+            who: 'memory',
+            step: 'merge',
+            what: merge.name,
+            sinceMs: clock - merge.since,
+        });
+    }
+    tail.keep(files);
+    const sums = new Map();
+    const add = (stage, model, s) => {
+        const key = `${stage}\u0000${model}`;
+        const row = sums.get(key) ?? {
+            stage,
+            model,
+            calls: 0,
+            input: 0,
+            cached: 0,
+            output: 0,
+            reasoning: 0,
+        };
+        row.calls += s.calls;
+        row.input += s.input;
+        row.cached += s.cached;
+        row.output += s.output;
+        row.reasoning += s.reasoning;
+        sums.set(key, row);
+    };
+    for (const stage of STAGES) {
+        for (const [model, s] of Object.entries(t.progress?.spent[stage] ?? {})) {
+            add(stage, model, s);
+        }
+    }
+    for (const row of now) {
+        for (const [model, s] of Object.entries(row.act?.byModel ?? {})) {
+            add(row.stage, model, s);
+        }
+    }
+    const tokens = [...sums.values()]
+        .filter((r) => r.calls > 0)
+        .sort((a, b) => STAGES.indexOf(a.stage) - STAGES.indexOf(b.stage) ||
+        b.input + b.output - (a.input + a.output));
+    return {
+        pid: process.pid,
+        project: basename(t.root),
+        state: stateOf(t),
+        at: new Date(clock).toISOString(),
+        elapsedMs: clock - t.startedAt,
+        system: t.system.version(),
+        workers: { busy: t.busy(), all: t.workers.length },
+        total: t.progress?.total ?? 0,
+        counts: t.progress?.counts ?? {},
+        parked: q.pending.length,
+        applyAt: t.config.applyAt,
+        now,
+        tokens,
+        log: t.recent
+            .filter((e) => !UNLOGGED.has(e.what))
+            .slice(0, KEEP_LOG)
+            .map(logRow),
+        errors: t.errors.slice(0, KEEP_ERRORS),
+    };
+}
+/** Takes a snapshot, keeps it in live.json without the raw lines, and hands it to every watcher. */
+export function publish(t) {
+    let snap;
+    try {
+        snap = snapshot(t);
+        const file = join(t.dir, 'live.json');
+        mkdirSync(t.dir, { recursive: true });
+        writeFileSync(`${file}.tmp`, `${JSON.stringify(snap, (k, v) => (k === 'lines' ? undefined : v), 2)}\n`);
+        renameSync(`${file}.tmp`, file);
+    }
+    catch {
+        // A live view that failed to draw must not stop the tuning.
+        return undefined;
+    }
+    t.live = snap;
+    for (const watcher of t.watchers) {
+        watcher(snap);
+    }
+    return snap;
+}
+//# sourceMappingURL=live.js.map

package/dist/meta/finetune/loop.d.ts

@@ -0,0 +1,3 @@
+import { type Tuning } from './tuning.ts';
+export declare function runLoop(t: Tuning): Promise<void>;
+//# sourceMappingURL=loop.d.ts.map

package/dist/meta/finetune/loop.js

@@ -0,0 +1,163 @@
+import { claimLock, ownLock } from '@zenera/neo';
+import { mkdirSync, rmSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+import { CliError, EXIT } from "../host.js";
+import { StepError } from "./failure.js";
+import { recover } from "./journal.js";
+import { publish } from "./live.js";
+import { nextCase, resultOf } from "./sampler.js";
+import { markFailed, SandboxDown, train, VoidRun } from "./train.js";
+import { Stopped } from "./tuning.js";
+// ---------------------------------------------------------------------------
+// The loop: N workers, cases started as they come free, one applier
+//
+// Whenever a worker is free - and no woken case is waiting for one - the next
+// recheck, or else the next case, starts on it and is trained to the end. A
+// parked case gives its worker back, so a lane never sits idle waiting for an
+// apply. A stop never interrupts a step: it is noticed between steps, and the
+// case resumes next time.
+// ---------------------------------------------------------------------------
+const REPORT_EVERY_MS = 15_000;
+const LIVE_EVERY_MS = 1000;
+export async function runLoop(t) {
+    mkdirSync(t.dir, { recursive: true });
+    const lock = join(t.dir, 'loop.lock');
+    claimLock(lock, ownLock(), (held) => new CliError(`a tuning is already running here (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `stop it with: zen meta finetune stop - or remove ${lock} if it crashed`));
+    rmSync(t.stopFile, { force: true });
+    const timer = setInterval(() => t.report(), REPORT_EVERY_MS);
+    const live = setInterval(() => {
+        publish(t);
+        t.report(true);
+    }, LIVE_EVERY_MS);
+    try {
+        await recover(t);
+        t.rechecks.resume();
+        const start = t.system.record('start');
+        await t.improvements.baseline();
+        t.event({ what: 'started', detail: `system v${start.v}, ${t.workers.length} workers` });
+        publish(t);
+        const applier = t.improvements.loop();
+        await dispatch(t);
+        t.finished = true;
+        await applier;
+        await t.memories.finish();
+        t.event({ what: t.stopping ? 'stopped' : 'finished' });
+    }
+    finally {
+        clearInterval(timer);
+        clearInterval(live);
+        t.finished = true;
+        publish(t);
+        t.report();
+        try {
+            unlinkSync(lock);
+        }
+        catch {
+            // Taken over as stale; nothing to give back.
+        }
+    }
+    if (t.fatal) {
+        throw t.fatal;
+    }
+}
+async function dispatch(t) {
+    const running = new Set();
+    const start = (seat, work) => {
+        const task = work(seat).finally(() => running.delete(task));
+        running.add(task);
+    };
+    while (!t.stopping) {
+        if (t.improvements.applying || t.resuming > 0 || !t.freeWorker()) {
+            await t.tick();
+            continue;
+        }
+        // A recheck tests the prose of the day, so it goes before a new case.
+        const taken = t.rechecks.take();
+        if (taken) {
+            const seat = { case: taken.case };
+            t.claimed.add(taken.case.id);
+            await t.seat(seat, false);
+            start(seat, (s) => recheckCase(t, s, taken.recheck));
+            continue;
+        }
+        const c = nextCase(t);
+        if (!c) {
+            t.drained = true;
+            // An apply still to come queues rechecks; only when none can is the work done.
+            const more = running.size > 0 ||
+                t.improvements.applying ||
+                t.improvements.pending.length > 0 ||
+                t.rechecks.waiting() > 0;
+            if (!more) {
+                break;
+            }
+            await t.tick();
+            continue;
+        }
+        const seat = { case: c };
+        t.claimed.add(c.id);
+        t.tried.add(c.id);
+        const before = resultOf(t, c);
+        if (before?.state === 'failed') {
+            // Taken again: until it ends anew it is in progress, not failed.
+            rmSync(join(t.caseDir(c), 'result.json'), { force: true });
+            t.event({ what: 'resumed', case: c.id, rev: c.rev, detail: before.reason });
+        }
+        await t.seat(seat, false);
+        start(seat, (s) => trainCase(t, s));
+    }
+    await Promise.all(running);
+}
+async function recheckCase(t, s, r) {
+    try {
+        await t.rechecks.run(s, r, s.case);
+    }
+    catch (err) {
+        if (!(err instanceof Stopped)) {
+            t.error(`${s.case.id} · recheck after ${r.apply}`, err.message, err instanceof StepError ? err.log : undefined);
+            if (err instanceof SandboxDown) {
+                // Left running: the next start queues it again.
+                t.sandboxDown(err.message);
+            }
+            else {
+                t.rechecks.void(r, err.message);
+                t.failed(`recheck: ${err.message}`);
+            }
+        }
+    }
+    finally {
+        t.unseat(s);
+        t.claimed.delete(s.case.id);
+        t.report();
+    }
+}
+async function trainCase(t, s) {
+    const c = s.case;
+    try {
+        await train(t, s, c);
+    }
+    catch (err) {
+        if (!(err instanceof Stopped)) {
+            const why = err instanceof SandboxDown
+                ? `sandbox: ${err.message}`
+                : err instanceof VoidRun
+                    ? `void: ${err.message}`
+                    : `error: ${err.message}`;
+            markFailed(t, c, s.phase ?? 'nomem', why);
+            const attempt = s.worker?.attempt;
+            t.error(`${c.id} · ${s.phase === 'mem' ? 'with memory' : 'no memory'}${attempt ? ` try ${attempt}` : ''}`, err.message, err instanceof StepError ? err.log : undefined);
+            if (err instanceof SandboxDown) {
+                t.sandboxDown(err.message);
+            }
+            else {
+                t.failed(why);
+            }
+        }
+    }
+    finally {
+        t.unseat(s);
+        t.claimed.delete(c.id);
+        t.report();
+    }
+}
+//# sourceMappingURL=loop.js.map

package/dist/meta/finetune/memories.d.ts

@@ -0,0 +1,47 @@
+import type { Case } from './dataset/types.ts';
+import type { Run } from './train.ts';
+import { type Tuning } from './tuning.ts';
+export interface Submission {
+    /** `<case>@r<rev>` */
+    key: string;
+    case: string;
+    rev: number;
+    attempt: number;
+    run: string;
+    dir: string;
+    at: string;
+    /** the run committed nothing: there is no graph to merge */
+    empty: boolean;
+}
+export interface MergeRow {
+    n: number;
+    name: string;
+    at: string;
+    dir: string;
+    log: string;
+    /** every submission in this graph */
+    includes: string[];
+    /** the ones this merge brought in */
+    added: string[];
+    ok: boolean;
+    nodes?: number;
+    edges?: number;
+}
+export declare class Memories {
+    #private;
+    merging: {
+        name: string;
+        since: number;
+    } | undefined;
+    constructor(t: Tuning);
+    get dir(): string;
+    submissions(): Submission[];
+    merges(): MergeRow[];
+    lastMerge(): MergeRow | undefined;
+    unmerged(): Submission[];
+    /** Keeps a copy of a passing with-memory run's memory. Once per case revision. */
+    submit(c: Case, attempt: number, run: Run): void;
+    /** Merges whatever is left, and waits for every merge queued. */
+    finish(): Promise<void>;
+}
+//# sourceMappingURL=memories.d.ts.map