@zenera/cli 1.1.24 → 1.1.26

package/dist/meta/finetune/dataset/store.d.ts

@@ -0,0 +1,35 @@
+import { type By, type Case, type ChangeRow, type JournalRow, type Manifest, type NoteRow, type RevisionRow } from './types.ts';
+export declare function whoIsThis(): By;
+export interface Commit {
+    rev: number;
+    rows: ChangeRow[];
+    revision: RevisionRow;
+    manifest: Manifest;
+    /** cases whose stored source hash moved with no change worth a revision */
+    refreshed: Case[];
+}
+export declare class DatasetStore {
+    #private;
+    readonly root: string;
+    readonly dir: string;
+    manifest: Manifest;
+    private constructor();
+    /** `write` claims the directory, creating it if it does not exist yet. */
+    static open(root: string, options?: {
+        write?: boolean;
+    }): DatasetStore;
+    exists(): boolean;
+    ids(): string[];
+    get(id: string): Case | undefined;
+    all(): Case[];
+    journal(id: string): JournalRow[];
+    /** The case as it stood at a revision, read back from its journal. */
+    at(id: string, rev: number): Case | undefined;
+    revisions(): RevisionRow[];
+    /** Cases written by an apply that never reached its manifest. */
+    unfinished(): string[];
+    note(id: string, row: NoteRow): void;
+    commit(commit: Commit): void;
+    release(): void;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/meta/finetune/dataset/store.js

@@ -0,0 +1,177 @@
+import { claimLock, ownLock } from '@zenera/neo';
+import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from 'node:fs';
+import { hostname } from 'node:os';
+import { join } from 'node:path';
+import { CliError, EXIT, META_PROMPT_ENV, META_SESSION_ENV } from "../../host.js";
+import { DATASET_DIR, DATASET_VERSION, } from "./types.js";
+// ---------------------------------------------------------------------------
+// The store
+//
+// Plain files, so git can diff them and a person can read them: a case per
+// file, a journal per case, a line per revision, and a manifest written last —
+// the manifest's revision is the commit marker, so an apply killed half-way
+// shows up as cases newer than it.
+//
+// Writes take the directory lock; reads never do. A note is one appended line
+// and takes no lock either, so an `analyze` running beside an apply can still
+// say what it found.
+// ---------------------------------------------------------------------------
+const MANIFEST = 'manifest.json';
+const REVISIONS = 'revisions.jsonl';
+const CASES = 'cases';
+const LOG = 'log';
+const LOCK = '.lock';
+export function whoIsThis() {
+    const session = process.env[META_SESSION_ENV]?.trim();
+    const prompt = process.env[META_PROMPT_ENV]?.trim();
+    return {
+        ...(session ? { session } : {}),
+        ...(prompt ? { prompt } : {}),
+        host: hostname(),
+    };
+}
+function atomic(path, text) {
+    const tmp = `${path}.tmp`;
+    writeFileSync(tmp, text);
+    renameSync(tmp, path);
+}
+function lines(path) {
+    let text;
+    try {
+        text = readFileSync(path, 'utf8');
+    }
+    catch {
+        return [];
+    }
+    return text
+        .split('\n')
+        .filter((line) => line.trim())
+        .flatMap((line) => {
+        try {
+            return [JSON.parse(line)];
+        }
+        catch {
+            return [];
+        }
+    });
+}
+const empty = () => ({ version: DATASET_VERSION, revision: 0, sources: {}, ignored: [] });
+export class DatasetStore {
+    root;
+    dir;
+    manifest;
+    #locked = false;
+    constructor(root) {
+        this.root = root;
+        this.dir = join(root, DATASET_DIR);
+        this.manifest = empty();
+        try {
+            const read = JSON.parse(readFileSync(join(this.dir, MANIFEST), 'utf8'));
+            if (read.version !== DATASET_VERSION) {
+                throw new CliError(`${this.dir} is dataset version ${read.version}; this zen reads ${DATASET_VERSION}`, EXIT.invalid, 'upgrade zen');
+            }
+            this.manifest = { ...empty(), ...read };
+        }
+        catch (err) {
+            if (err instanceof CliError) {
+                throw err;
+            }
+        }
+    }
+    /** `write` claims the directory, creating it if it does not exist yet. */
+    static open(root, options = {}) {
+        const store = new DatasetStore(root);
+        if (options.write) {
+            mkdirSync(join(store.dir, CASES), { recursive: true });
+            mkdirSync(join(store.dir, LOG), { recursive: true });
+            claimLock(join(store.dir, LOCK), ownLock(), (held) => new CliError(`the dataset is being written (pid ${held.pid}, since ${held.startedAt})`, EXIT.failed, `wait for it, or remove ${join(store.dir, LOCK)} if that process crashed`));
+            store.#locked = true;
+        }
+        return store;
+    }
+    exists() {
+        return existsSync(join(this.dir, MANIFEST));
+    }
+    ids() {
+        try {
+            return readdirSync(join(this.dir, CASES))
+                .filter((f) => f.endsWith('.json'))
+                .map((f) => f.slice(0, -'.json'.length))
+                .sort();
+        }
+        catch {
+            return [];
+        }
+    }
+    get(id) {
+        try {
+            return JSON.parse(readFileSync(join(this.dir, CASES, `${id}.json`), 'utf8'));
+        }
+        catch {
+            return undefined;
+        }
+    }
+    all() {
+        return this.ids().flatMap((id) => this.get(id) ?? []);
+    }
+    journal(id) {
+        return lines(join(this.dir, LOG, `${id}.jsonl`));
+    }
+    /** The case as it stood at a revision, read back from its journal. */
+    at(id, rev) {
+        let found;
+        for (const row of this.journal(id)) {
+            if (row.type === 'change' && row.rev <= rev) {
+                found = row.case;
+            }
+        }
+        return found;
+    }
+    revisions() {
+        return lines(join(this.dir, REVISIONS));
+    }
+    /** Cases written by an apply that never reached its manifest. */
+    unfinished() {
+        return this.all()
+            .filter((c) => c.rev > this.manifest.revision)
+            .map((c) => c.id);
+    }
+    note(id, row) {
+        mkdirSync(join(this.dir, LOG), { recursive: true });
+        appendFileSync(join(this.dir, LOG, `${id}.jsonl`), `${JSON.stringify(row)}\n`);
+    }
+    commit(commit) {
+        if (!this.#locked) {
+            throw new Error('commit without the dataset lock');
+        }
+        for (const row of commit.rows) {
+            atomic(join(this.dir, CASES, `${row.case.id}.json`), `${JSON.stringify(row.case, null, 4)}\n`);
+            appendFileSync(join(this.dir, LOG, `${row.case.id}.jsonl`), `${JSON.stringify(row)}\n`);
+        }
+        for (const c of commit.refreshed) {
+            atomic(join(this.dir, CASES, `${c.id}.json`), `${JSON.stringify(c, null, 4)}\n`);
+        }
+        if (commit.rows.length > 0) {
+            appendFileSync(join(this.dir, REVISIONS), `${JSON.stringify(commit.revision)}\n`);
+        }
+        this.manifest = {
+            ...commit.manifest,
+            revision: commit.rows.length > 0 ? commit.rev : commit.manifest.revision,
+            updatedAt: new Date().toISOString(),
+        };
+        atomic(join(this.dir, MANIFEST), `${JSON.stringify(this.manifest, null, 4)}\n`);
+    }
+    release() {
+        if (!this.#locked) {
+            return;
+        }
+        this.#locked = false;
+        try {
+            unlinkSync(join(this.dir, LOCK));
+        }
+        catch {
+            // Already gone, or taken over as stale.
+        }
+    }
+}
+//# sourceMappingURL=store.js.map

package/dist/meta/finetune/dataset/types.d.ts

@@ -0,0 +1,127 @@
+export declare const DATASET_DIR = "dataset";
+export declare const DATASET_VERSION = 1;
+export declare const COMPLEXITIES: readonly ["simple", "medium", "complex"];
+export type Complexity = (typeof COMPLEXITIES)[number];
+export declare const STATUSES: readonly ["active", "retired"];
+export type Status = (typeof STATUSES)[number];
+export declare const NOTE_KINDS: readonly ["graded", "analyze", "difficult", "rubric-suspect", "observation"];
+export type NoteKind = (typeof NOTE_KINDS)[number];
+export declare const VERDICTS: readonly ["right", "wrong", "void"];
+export type Verdict = (typeof VERDICTS)[number];
+/** A case id names a directory under a batch dir, so it is one path segment. */
+export declare const CASE_ID: RegExp;
+export declare const MEDIA_KINDS: readonly ["image", "audio", "video", "file"];
+export type MediaKind = (typeof MEDIA_KINDS)[number];
+export type Part = string | {
+    text: string;
+} | ({
+    [K in MediaKind]?: string;
+} & {
+    mimeType?: string;
+});
+export type Input = string | Part[];
+export interface RubricItem {
+    id: string;
+    text: string;
+}
+/** Where in a source file a case came from. Neither key means the whole file. */
+export interface Anchor {
+    heading?: string[];
+    pointer?: string;
+}
+export interface Source {
+    /** project-relative */
+    file: string;
+    anchor?: Anchor;
+    /** of the section the anchor names, when the case was last applied */
+    sha256?: string;
+}
+export interface Case {
+    id: string;
+    rev: number;
+    status: Status;
+    class?: string;
+    complexity?: Complexity;
+    tags?: string[];
+    input: Input;
+    rubric: RubricItem[];
+    expected?: string;
+    notes?: string;
+    source?: Source;
+    /** fields a re-extraction must not overwrite */
+    overrides?: string[];
+}
+/** A change to these is a different question: earlier results no longer count. */
+export declare const RESTART_FIELDS: readonly ["input", "rubric", "expected"];
+/** Fields a proposal or `update --set` may write. */
+export declare const CONTENT_FIELDS: readonly ["class", "complexity", "tags", "input", "rubric", "expected", "notes", "source"];
+export type ContentField = (typeof CONTENT_FIELDS)[number];
+/** Who did it: read from the environment `zen meta` gives its agent. */
+export interface By {
+    session?: string;
+    prompt?: string;
+    host: string;
+}
+export interface FieldChange {
+    path: string;
+    from?: unknown;
+    to?: unknown;
+}
+export type ChangeOp = 'add' | 'update' | 'retire' | 'restore';
+export interface ChangeRow {
+    type: 'change';
+    rev: number;
+    at: string;
+    op: ChangeOp;
+    changes: FieldChange[];
+    why: string;
+    by: By;
+    /** the case as it stood after this change, so any revision can be read back */
+    case: Case;
+}
+export interface NoteRow {
+    type: 'note';
+    at: string;
+    kind: NoteKind;
+    /** the case revision the note is about */
+    caseRev: number;
+    run?: string;
+    verdict?: Verdict;
+    rubric?: Record<string, Grade>;
+    text: string;
+    by: By;
+}
+/** `invalid`: the rubric line itself is broken - it says how one implementation did it, not what is right. */
+export type Grade = 'pass' | 'fail' | 'invalid';
+export type JournalRow = ChangeRow | NoteRow;
+export interface Ignored {
+    file: string;
+    anchor: Anchor;
+    why: string;
+    rev: number;
+}
+export interface Manifest {
+    version: number;
+    revision: number;
+    updatedAt?: string;
+    /** whole-file hash of every source a case or an ignore names */
+    sources: Record<string, {
+        sha256: string;
+    }>;
+    ignored: Ignored[];
+}
+export interface RevisionRow {
+    rev: number;
+    at: string;
+    why: string;
+    by: By;
+    added: string[];
+    updated: string[];
+    retired: string[];
+    restored: string[];
+    /** cases whose input, rubric or expected changed */
+    restarted: string[];
+}
+export declare const anchorText: (anchor: Anchor | undefined) => string;
+export declare const sameAnchor: (a: Anchor | undefined, b: Anchor | undefined) => boolean;
+//# sourceMappingURL=types.d.ts.map

package/dist/meta/finetune/dataset/types.js

@@ -0,0 +1,39 @@
+// ---------------------------------------------------------------------------
+// A dataset, as stored
+//
+// One file per case holds what the case is now; one append-only journal per
+// case holds how it got there and what was learned about it. Nothing is ever
+// deleted: a removed case is `retired`, so an id is never reused and an old run
+// that names it still resolves.
+// ---------------------------------------------------------------------------
+export const DATASET_DIR = 'dataset';
+export const DATASET_VERSION = 1;
+export const COMPLEXITIES = ['simple', 'medium', 'complex'];
+export const STATUSES = ['active', 'retired'];
+export const NOTE_KINDS = [
+    'graded',
+    'analyze',
+    'difficult',
+    'rubric-suspect',
+    'observation',
+];
+export const VERDICTS = ['right', 'wrong', 'void'];
+/** A case id names a directory under a batch dir, so it is one path segment. */
+export const CASE_ID = /^[A-Za-z0-9_.-]+$/;
+export const MEDIA_KINDS = ['image', 'audio', 'video', 'file'];
+/** A change to these is a different question: earlier results no longer count. */
+export const RESTART_FIELDS = ['input', 'rubric', 'expected'];
+/** Fields a proposal or `update --set` may write. */
+export const CONTENT_FIELDS = [
+    'class',
+    'complexity',
+    'tags',
+    'input',
+    'rubric',
+    'expected',
+    'notes',
+    'source',
+];
+export const anchorText = (anchor) => anchor?.heading ? anchor.heading.join(' > ') : (anchor?.pointer ?? '(whole file)');
+export const sameAnchor = (a, b) => anchorText(a) === anchorText(b);
+//# sourceMappingURL=types.js.map

package/dist/meta/finetune/failure.d.ts

@@ -0,0 +1,15 @@
+/** A step that failed, with the log that says why. */
+export declare class StepError extends Error {
+    readonly log?: string | undefined;
+    constructor(message: string, log?: string | undefined);
+}
+export declare const sizeOf: (file: string) => number;
+/**
+ * What broke, from what a step appended to its log since `from`: the provider's own line before
+ * zen's generic `error ...`, and the meta agent's fuller log when the step names one.
+ */
+export declare function errorIn(log: string, from?: number): {
+    message?: string;
+    log: string;
+};
+//# sourceMappingURL=failure.d.ts.map

package/dist/meta/finetune/failure.js

@@ -0,0 +1,26 @@
+import { existsSync, readFileSync, statSync } from 'node:fs';
+/** A step that failed, with the log that says why. */
+export class StepError extends Error {
+    log;
+    constructor(message, log) {
+        super(message);
+        this.log = log;
+    }
+}
+export const sizeOf = (file) => (existsSync(file) ? statSync(file).size : 0);
+/**
+ * What broke, from what a step appended to its log since `from`: the provider's own line before
+ * zen's generic `error ...`, and the meta agent's fuller log when the step names one.
+ */
+export function errorIn(log, from = 0) {
+    const text = existsSync(log) ? readFileSync(log).subarray(from).toString('utf8') : '';
+    const lines = text
+        .split('\n')
+        .map((l) => l.trim())
+        .filter(Boolean);
+    const provider = lines.findLast((l) => /\bHTTP \d{3}\b|^"\d{3} [^"]+" \(\d{3}\)|not found|quota|rate limit|unavailable|timed out/i.test(l));
+    const exited = lines.findLast((l) => /^error\s/.test(l))?.replace(/^error\s+/, '');
+    const meta = lines.findLast((l) => /^log\s+\//.test(l))?.replace(/^log\s+/, '');
+    return { message: provider ?? exited, log: meta ?? log };
+}
+//# sourceMappingURL=failure.js.map

package/dist/meta/finetune/feedback.d.ts

@@ -0,0 +1,57 @@
+import { type Grade, type Verdict } from './dataset/types.ts';
+export type Phase = 'nomem' | 'mem';
+export interface Improvement {
+    id: string;
+    kind?: string;
+    file?: string;
+    change: string;
+    why?: string;
+    expect?: string;
+}
+export interface Feedback {
+    /** `<case>@<phase>-<attempt>`, `<case>#<round>@...` after a retry; unique across the tuning */
+    id: string;
+    case: string;
+    caseRev: number;
+    /** absent in the first round */
+    round?: number;
+    phase: Phase;
+    attempt: number;
+    verdict: Verdict;
+    rubric: Record<string, Grade>;
+    /** right, and nothing left worth changing */
+    done: boolean;
+    summary: string;
+    improvements: Improvement[];
+    /** what outside the project made the run void: the sandbox, the network, a provider */
+    infra?: string;
+    /** a recheck's: the apply whose version the case failed on, after passing before it */
+    recheck?: string;
+    /** the try's folder, when it is not the case's own attempt folder */
+    dir?: string;
+}
+/** A summary's first paragraph - the verdict, for a table cell or a one-line note. */
+export declare const lead: (summary: string) => string;
+/** A `specification` improvement is for the dataset, never for an apply: prose cannot fix a rubric. */
+export declare const forDataset: (i: Improvement) => boolean;
+/** The improvements an apply can act on. */
+export declare const forProse: (f: {
+    improvements: Improvement[];
+}) => Improvement[];
+export declare const feedbackId: (caseId: string, phase: Phase, attempt: number, round?: number) => string;
+/**
+ * The feedback, or what is wrong with the file - said so a model can fix it.
+ * `rubric` is the case's rubric ids: given, every one must be graded.
+ */
+export declare function readFeedback(file: string, known: {
+    case: string;
+    caseRev: number;
+    round?: number;
+    phase: Phase;
+    attempt: number;
+}, rubricIds?: readonly string[]): {
+    feedback: Feedback;
+} | {
+    problem: string;
+};
+//# sourceMappingURL=feedback.d.ts.map

package/dist/meta/finetune/feedback.js

@@ -0,0 +1,104 @@
+import { readFileSync } from 'node:fs';
+import { VERDICTS } from "./dataset/types.js";
+/** A summary's first paragraph - the verdict, for a table cell or a one-line note. */
+export const lead = (summary) => summary.trim().split(/\n\s*\n/)[0] ?? '';
+/** A `specification` improvement is for the dataset, never for an apply: prose cannot fix a rubric. */
+export const forDataset = (i) => i.kind === 'specification';
+/** The improvements an apply can act on. */
+export const forProse = (f) => f.improvements.filter((i) => !forDataset(i));
+// Round 1 keeps the bare form, so the ids applies already took still match.
+export const feedbackId = (caseId, phase, attempt, round = 1) => `${caseId}${round > 1 ? `#${round}` : ''}@${phase}-${attempt}`;
+const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+/**
+ * The feedback, or what is wrong with the file - said so a model can fix it.
+ * `rubric` is the case's rubric ids: given, every one must be graded.
+ */
+export function readFeedback(file, known, rubricIds = []) {
+    let raw;
+    try {
+        raw = JSON.parse(readFileSync(file, 'utf8'));
+    }
+    catch (err) {
+        return {
+            problem: err.code === 'ENOENT' ? 'missing' : 'not JSON',
+        };
+    }
+    if (!isObject(raw)) {
+        return { problem: 'not a JSON object' };
+    }
+    if (!VERDICTS.includes(raw.verdict)) {
+        return { problem: `"verdict" must be one of ${VERDICTS.join(', ')}` };
+    }
+    if (typeof raw.done !== 'boolean') {
+        return { problem: '"done" must be true or false' };
+    }
+    if (!Array.isArray(raw.improvements)) {
+        return { problem: '"improvements" must be a list (empty when there is nothing to change)' };
+    }
+    const improvements = [];
+    for (const [i, item] of raw.improvements.entries()) {
+        if (!isObject(item) || typeof item.change !== 'string' || !item.change.trim()) {
+            return { problem: `improvements[${i}] needs a "change"` };
+        }
+        const text = (key) => typeof item[key] === 'string' ? item[key] : undefined;
+        improvements.push({
+            id: text('id') ?? `i${i + 1}`,
+            change: item.change,
+            ...(text('kind') ? { kind: text('kind') } : {}),
+            ...(text('file') ? { file: text('file') } : {}),
+            ...(text('why') ? { why: text('why') } : {}),
+            ...(text('expect') ? { expect: text('expect') } : {}),
+        });
+    }
+    const rubric = {};
+    if (isObject(raw.rubric)) {
+        for (const [id, result] of Object.entries(raw.rubric)) {
+            if (result === 'pass' || result === 'fail' || result === 'invalid') {
+                rubric[id] = result;
+            }
+        }
+    }
+    const verdict = raw.verdict;
+    const infra = typeof raw.infra === 'string' ? raw.infra.trim() : '';
+    if (verdict === 'void' && !infra) {
+        return {
+            problem: 'void without "infra" - say what outside the project failed (the sandbox, the ' +
+                'network, a provider, a service the case needs), with node ids',
+        };
+    }
+    const ungraded = rubricIds.filter((id) => !(id in rubric));
+    if (verdict !== 'void' && ungraded.length > 0) {
+        return {
+            problem: `missing grades for rubric lines ${ungraded.join(', ')} - grade every line of ` +
+                `\`zen meta dataset show ${known.case}@${known.caseRev}\` "pass", "fail" or "invalid" under "rubric"`,
+        };
+    }
+    const unexplained = Object.entries(rubric)
+        .filter(([, g]) => g === 'invalid')
+        .map(([id]) => id)
+        .filter((id) => !improvements.some((i) => forDataset(i) && `${i.change} ${i.why ?? ''}`.split(/[^\w-]+/).includes(id)));
+    if (unexplained.length > 0) {
+        return {
+            problem: `rubric lines ${unexplained.join(', ')} are graded "invalid" with no ` +
+                '"specification" improvement naming them - add one per line saying what is wrong ' +
+                'with it and how it should read',
+        };
+    }
+    const failed = Object.values(rubric).includes('fail');
+    // A right run whose only findings are about the dataset has nothing left for the prose.
+    const onlyDataset = improvements.length > 0 && forProse({ improvements }).length === 0;
+    return {
+        feedback: {
+            id: feedbackId(known.case, known.phase, known.attempt, known.round),
+            ...known,
+            verdict,
+            rubric,
+            // `done` on a wrong answer or a failed rubric line is a contradiction; those win.
+            done: (raw.done || onlyDataset) && verdict === 'right' && !failed,
+            summary: typeof raw.summary === 'string' ? raw.summary : '',
+            improvements,
+            ...(verdict === 'void' ? { infra } : {}),
+        },
+    };
+}
+//# sourceMappingURL=feedback.js.map

package/dist/meta/finetune/guard.d.ts

@@ -0,0 +1,29 @@
+import type { Case } from './dataset/types.ts';
+import { type Tuning } from './tuning.ts';
+export interface GuardTry {
+    dir: string;
+    verdict: string;
+    passed: boolean;
+    summary?: string;
+}
+export interface GuardResult {
+    case: string;
+    tries: GuardTry[];
+    broken: boolean;
+}
+/** A guard case as the live view and the status page show it, updated as it goes. */
+export interface GuardSeat {
+    case: string;
+    try: number;
+    step: 'run' | 'analyze' | 'passed' | 'broken';
+    /** the try's directory */
+    dir: string;
+    since: number;
+}
+/** A case is broken only after failing this many tries in a row. */
+export declare const GUARD_TRIES = 2;
+/** Completed cases outside `exclude`, in an order fixed by the seed and the apply. */
+export declare function guardCases(t: Tuning, apply: string, exclude: Set<string>): Case[];
+/** Runs the guard for one attempt of an apply; `patch` is that attempt's diff. */
+export declare function guard(t: Tuning, dir: string, attempt: number, exclude: Set<string>, patch: string): Promise<GuardResult[]>;
+//# sourceMappingURL=guard.d.ts.map