@zenera/cli 1.1.25 → 1.1.26

package/dist/meta/finetune/feedback.js

@@ -1,5 +1,11 @@
 import { readFileSync } from 'node:fs';
 import { VERDICTS } from "./dataset/types.js";
+/** A summary's first paragraph - the verdict, for a table cell or a one-line note. */
+export const lead = (summary) => summary.trim().split(/\n\s*\n/)[0] ?? '';
+/** A `specification` improvement is for the dataset, never for an apply: prose cannot fix a rubric. */
+export const forDataset = (i) => i.kind === 'specification';
+/** The improvements an apply can act on. */
+export const forProse = (f) => f.improvements.filter((i) => !forDataset(i));
 // Round 1 keeps the bare form, so the ids applies already took still match.
 export const feedbackId = (caseId, phase, attempt, round = 1) => `${caseId}${round > 1 ? `#${round}` : ''}@${phase}-${attempt}`;
 const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
@@ -47,7 +53,7 @@ export function readFeedback(file, known, rubricIds = []) {
     const rubric = {};
     if (isObject(raw.rubric)) {
         for (const [id, result] of Object.entries(raw.rubric)) {
-            if (result === 'pass' || result === 'fail') {
+            if (result === 'pass' || result === 'fail' || result === 'invalid') {
                 rubric[id] = result;
             }
         }
@@ -64,10 +70,23 @@ export function readFeedback(file, known, rubricIds = []) {
     if (verdict !== 'void' && ungraded.length > 0) {
         return {
             problem: `missing grades for rubric lines ${ungraded.join(', ')} - grade every line of ` +
-                `\`zen meta dataset show ${known.case}@${known.caseRev}\` "pass" or "fail" under "rubric"`,
+                `\`zen meta dataset show ${known.case}@${known.caseRev}\` "pass", "fail" or "invalid" under "rubric"`,
+        };
+    }
+    const unexplained = Object.entries(rubric)
+        .filter(([, g]) => g === 'invalid')
+        .map(([id]) => id)
+        .filter((id) => !improvements.some((i) => forDataset(i) && `${i.change} ${i.why ?? ''}`.split(/[^\w-]+/).includes(id)));
+    if (unexplained.length > 0) {
+        return {
+            problem: `rubric lines ${unexplained.join(', ')} are graded "invalid" with no ` +
+                '"specification" improvement naming them - add one per line saying what is wrong ' +
+                'with it and how it should read',
         };
     }
     const failed = Object.values(rubric).includes('fail');
+    // A right run whose only findings are about the dataset has nothing left for the prose.
+    const onlyDataset = improvements.length > 0 && forProse({ improvements }).length === 0;
     return {
         feedback: {
             id: feedbackId(known.case, known.phase, known.attempt, known.round),
@@ -75,7 +94,7 @@ export function readFeedback(file, known, rubricIds = []) {
             verdict,
             rubric,
             // `done` on a wrong answer or a failed rubric line is a contradiction; those win.
-            done: raw.done && verdict === 'right' && !failed,
+            done: (raw.done || onlyDataset) && verdict === 'right' && !failed,
             summary: typeof raw.summary === 'string' ? raw.summary : '',
             improvements,
             ...(verdict === 'void' ? { infra } : {}),

package/dist/meta/finetune/guard.d.ts

@@ -0,0 +1,29 @@
+import type { Case } from './dataset/types.ts';
+import { type Tuning } from './tuning.ts';
+export interface GuardTry {
+    dir: string;
+    verdict: string;
+    passed: boolean;
+    summary?: string;
+}
+export interface GuardResult {
+    case: string;
+    tries: GuardTry[];
+    broken: boolean;
+}
+/** A guard case as the live view and the status page show it, updated as it goes. */
+export interface GuardSeat {
+    case: string;
+    try: number;
+    step: 'run' | 'analyze' | 'passed' | 'broken';
+    /** the try's directory */
+    dir: string;
+    since: number;
+}
+/** A case is broken only after failing this many tries in a row. */
+export declare const GUARD_TRIES = 2;
+/** Completed cases outside `exclude`, in an order fixed by the seed and the apply. */
+export declare function guardCases(t: Tuning, apply: string, exclude: Set<string>): Case[];
+/** Runs the guard for one attempt of an apply; `patch` is that attempt's diff. */
+export declare function guard(t: Tuning, dir: string, attempt: number, exclude: Set<string>, patch: string): Promise<GuardResult[]>;
+//# sourceMappingURL=guard.d.ts.map

package/dist/meta/finetune/guard.js

@@ -0,0 +1,104 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import { readFeedback } from "./feedback.js";
+import { resultOf, selection } from "./sampler.js";
+import { keepAnalysis, runOnce } from "./train.js";
+import { readJson } from "./tuning.js";
+/** A case is broken only after failing this many tries in a row. */
+export const GUARD_TRIES = 2;
+const pad = (n) => String(n).padStart(2, '0');
+/** Completed cases outside `exclude`, in an order fixed by the seed and the apply. */
+export function guardCases(t, apply, exclude) {
+    const key = (id) => createHash('sha256').update(`${t.config.seed}:${apply}:${id}`).digest('hex');
+    return selection(t)
+        .filter((c) => !exclude.has(c.id) && resultOf(t, c)?.state === 'completed')
+        .sort((a, b) => key(a.id).localeCompare(key(b.id)))
+        .slice(0, t.config.guard);
+}
+/** Runs the guard for one attempt of an apply; `patch` is that attempt's diff. */
+export async function guard(t, dir, attempt, exclude, patch) {
+    const cases = guardCases(t, basename(dir), exclude);
+    if (cases.length === 0) {
+        return [];
+    }
+    const seats = cases.map((c) => ({
+        case: c.id,
+        try: 1,
+        step: 'run',
+        dir: join(dir, 'guard', `a${attempt}`, c.id),
+        since: Date.now(),
+    }));
+    if (t.improvements.current) {
+        t.improvements.current.guard = seats;
+    }
+    t.event({
+        what: 'guard',
+        detail: `${basename(dir)}: ${cases.map((c) => c.id).join(', ')} run again on the edit`,
+    });
+    const results = await Promise.all(cases.map((c, i) => guardOne(t, join(dir, 'guard', `a${attempt}`, c.id), c, patch, seats[i])));
+    writeFileSync(join(dir, 'guard.json'), `${JSON.stringify(results, null, 2)}\n`);
+    return results;
+}
+async function guardOne(t, dir, c, patch, seat) {
+    const tries = [];
+    for (let n = 1; n <= GUARD_TRIES; n++) {
+        const at = join(dir, `${pad(n)}-nomem`);
+        const one = await tryOne(t, c, at, n, patch, seat);
+        tries.push(one);
+        if (one.passed) {
+            break;
+        }
+    }
+    const broken = tries.length === GUARD_TRIES && tries.every((x) => !x.passed);
+    Object.assign(seat, { step: broken ? 'broken' : 'passed', since: Date.now() });
+    t.event({
+        what: broken ? 'guard broken' : 'guard passed',
+        case: c.id,
+        rev: c.rev,
+        detail: tries.map((x) => x.verdict).join(', '),
+    });
+    return { case: c.id, tries, broken };
+}
+/** A run or an analysis that says nothing about the prose counts as a pass: only a graded failure undoes an apply. */
+async function tryOne(t, c, dir, n, patch, seat) {
+    mkdirSync(dir, { recursive: true });
+    Object.assign(seat, { try: n, step: 'run', dir, since: Date.now() });
+    const why = await runOnce(t, c, dir);
+    if (why) {
+        return { dir, verdict: 'void', passed: true, summary: why };
+    }
+    const run = readJson(join(dir, 'run.json'));
+    const file = join(dir, 'feedback.json');
+    Object.assign(seat, { step: 'analyze', since: Date.now() });
+    const res = await t.zen([
+        'meta',
+        'run',
+        '--no-refresh',
+        '--json',
+        '--events',
+        join(dir, 'analyze.events.jsonl'),
+        '--session-id',
+        randomUUID(),
+        '/analyze',
+        run.dir,
+        `case=${c.id}`,
+        `rev=${c.rev}`,
+        'phase=nomem',
+        `attempt=${n}`,
+        `feedback=${file}`,
+        `guard=${patch}`,
+    ], join(dir, 'analyze.log'));
+    keepAnalysis(dir, res.stdout);
+    const read = readFeedback(file, { case: c.id, caseRev: c.rev, phase: 'nomem', attempt: n }, c.rubric.map((r) => r.id));
+    if (!('feedback' in read)) {
+        return { dir, verdict: 'unread', passed: true, summary: read.problem };
+    }
+    const f = read.feedback;
+    if (f.verdict === 'void') {
+        return { dir, verdict: 'void', passed: true, summary: f.infra };
+    }
+    const passed = f.verdict === 'right' && !Object.values(f.rubric).includes('fail');
+    return { dir, verdict: f.verdict, passed, summary: f.summary };
+}
+//# sourceMappingURL=guard.js.map

package/dist/meta/finetune/improvements.d.ts

@@ -1,4 +1,5 @@
-import type { Feedback } from './feedback.ts';
+import { type Cost } from './cost.ts';
+import { type Feedback, type Improvement } from './feedback.ts';
 import { type Seat, type Tuning } from './tuning.ts';
 export interface AppliedFile {
     ids: string[];
@@ -7,7 +8,33 @@ export interface AppliedFile {
     at: string;
     startedAt?: string;
     skipped?: boolean;
+    /** why the edit was undone: it failed `zen check`, or went over a limit */
+    discarded?: string;
+    /** a consolidation, from before an apply tidied up after itself */
+    kind?: 'consolidate';
+    /** what the agents always read, before and after */
+    bytes?: {
+        before: number;
+        after: number;
+    };
 }
+export interface Decision {
+    decision?: string;
+    note?: string;
+    disposition?: string;
+    /** the skill the improvement created or changed */
+    skill?: string;
+}
+export type Decisions = Record<string, Record<string, Decision>>;
+export interface Deferred {
+    request: string;
+    case: string;
+    apply: string;
+    improvement: Improvement;
+    note?: string;
+}
+/** `<request> <improvement>` for each improvement of inputs.json that decisions.json gives no valid decision. */
+export declare function undecided(dir: string): string[];
 export declare class Improvements {
     #private;
     readonly pending: Feedback[];
@@ -24,6 +51,8 @@ export declare class Improvements {
         name?: string;
         dir?: string;
         requests?: number;
+        /** which gate the edit is at; absent while it waits for runs */
+        step?: 'edit' | 'check';
     } | undefined;
     /** applies started in this process */
     started: number;
@@ -34,12 +63,30 @@ export declare class Improvements {
     reload(): void;
     get dir(): string;
     isApplied(id: string): boolean;
-    /** Parks until an apply has taken this feedback, the worker free meanwhile. Taken already - after a restart - returns at once. */
-    apply(feedback: Feedback, s: Seat): Promise<void>;
+    /**
+     * Every apply's decisions, later ones over earlier, per improvement. An edit
+     * that was undone kept nothing, so what it applied or merged reads `discarded`.
+     */
+    decisions(): Decisions;
+    /** Improvements held back for want of a second case, and not taken up since. */
+    deferred(): Deferred[];
+    /** The skills an apply made or changed for this request: the case's next run should load them. */
+    skillsFor(id: string): string[];
+    /**
+     * Parks until an apply has taken this feedback, the worker free meanwhile. Taken already - after a restart - returns at once.
+     * `structural` when the apply found nothing prose can fix.
+     */
+    apply(feedback: Feedback, s: Seat): Promise<{
+        structural?: string;
+    }>;
+    /** A recheck's request: queued like a parked case's, with no case waiting on it. */
+    regressed(feedback: Feedback): void;
     /** Before a run starts: wait out an apply, or stop. */
     notApplying(): Promise<void>;
     /** The applier: runs until the tuning is finished. */
     loop(): Promise<void>;
+    /** What the agents read when the tuning began - its first version, even for a tuning begun before this was kept. */
+    baseline(): Promise<Cost>;
     /** Requests every apply so far took, for the report. */
     applies(): {
         name: string;