klyro 1.0.5 → 1.0.7

package/dist/context/accounting.js

@@ -4,8 +4,14 @@
  */
 import { totalTokens } from './tokenizer.js';
 import { getModelInfo } from '../providers/model-info.js';
+/**
+ * Single output-reserve used by every budget in the harness (displayed
+ * accounting, runtime enforcement, compaction elide). One constant so the
+ * meter and the enforcer can never disagree.
+ */
+export const RESERVE_OUTPUT_TOKENS = 8000;
 export function accounting(system, messages, opts = {}) {
-    const reserveOutput = opts.reserveOutput ?? 16_000;
+    const reserveOutput = opts.reserveOutput ?? RESERVE_OUTPUT_TOKENS;
     // Window-aware default: the legacy 120k ceiling overflows small-window
     // models (e.g. 8k local models) and wastes large ones — size to the model.
     const cap = opts.cap ?? capForModel(opts.model, reserveOutput);
@@ -21,7 +27,7 @@ export function accounting(system, messages, opts = {}) {
  * tiny windows still function. Unknown models use the registry fallback
  * window (100k); a missing model name keeps the legacy 120k.
  */
-export function capForModel(model, reserveOutput = 16_000) {
+export function capForModel(model, reserveOutput = RESERVE_OUTPUT_TOKENS) {
     if (!model)
         return 120_000;
     const window = getModelInfo(model).contextWindow;

package/dist/context/compaction.d.ts

@@ -2,13 +2,14 @@
  * 8.3 — Auto-compaction: (a) elide → (b) summarize 60% with model.small → (c) keep last N verbatim
  * Trigger at compactAt (80%) or /compact [focus]. Validates summary mentions every checkpointed file else fallback.
  */
-import type { Message } from '../agent/message.js';
+import type { Message, ContentBlock } from '../agent/message.js';
 export interface CompactionResult {
     messages: Message[];
     summary: string;
     dropped: number;
     method: 'elide' | 'summarize' | 'fallback';
 }
+export declare function textBlock(s: string): ContentBlock;
 export declare function compact(messages: Message[], opts: {
     system?: string;
     cap?: number;

package/dist/context/compaction.js

@@ -1,14 +1,37 @@
 import { compressTranscript } from './tokenizer.js';
-import { capForModel } from './accounting.js';
+import { capForModel, RESERVE_OUTPUT_TOKENS } from './accounting.js';
+export function textBlock(s) {
+    return { kind: 'text', text: s };
+}
+/**
+ * Tightened checkpoint validation: every checkpointed file must be mentioned
+ * by a path segment, not a loose basename substring. `src/util.ts` is matched
+ * by "util.ts" only when it names a segment; `foo` never matches `foo.png`
+ * nor a path nested inside it.
+ */
+function mentionsFile(summary, filePath) {
+    const segs = filePath.split('/').filter(Boolean);
+    const tail = segs[segs.length - 1] ?? filePath;
+    // Match the full basename OR any path segment borne by the file.
+    // Prefer exact-path segments (src/util.ts or util.ts) to avoid the
+    // substring false-positive the old `includes(basename)` produced.
+    const quoted = filePath.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const quotedTail = tail.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    return new RegExp(`(^|[\\W_])${quotedTail}([\\W_]|$)`).test(summary) || summary.includes(quoted);
+}
 export async function compact(messages, opts) {
     const cap = opts.cap ?? capForModel(opts.model);
     // (a) elide old tool results
-    const elided = compressTranscript(opts.system, messages, { total: cap, reservedOutput: 16_000 });
+    const elided = compressTranscript(opts.system, messages, { total: cap, reservedOutput: RESERVE_OUTPUT_TOKENS });
     if (opts.checkpointedFiles && elided.dropped > 0) {
-        // (b) try summarize oldest 60% using strict template (mock small model)
-        const n = Math.floor(messages.length * 0.6);
-        const oldest = messages.slice(0, n);
-        const newest = messages.slice(n);
+        // (b) try summarize oldest 60% using strict template (mock small model).
+        // Slice from the ELIDED sequence so the summarize pass operates on what
+        // actually survives elision (previously it sliced the raw `messages`,
+        // silently dropping the elide benefit on the oldest portion).
+        const elidedBest = elided.messages;
+        const n = Math.floor(elidedBest.length * 0.6);
+        const oldest = elidedBest.slice(0, n);
+        const newest = elidedBest.slice(n);
         const template = `Summarize the following ${oldest.length} messages, mentioning every file: ${opts.checkpointedFiles.join(', ')}.\nFocus: ${opts.focus ?? 'general'}\n\n` + oldest.map((m) => JSON.stringify(m.content).slice(0, 500)).join('\n');
         let summary = `Earlier in session: ${oldest.length} turns covering ${opts.checkpointedFiles.join(', ')}`;
         if (opts.summarizeFn) {
@@ -17,24 +40,28 @@ export async function compact(messages, opts) {
             }
             catch { /* fallback */ }
         }
-        // validate: every checkpointed file mentioned
-        const missing = opts.checkpointedFiles.filter((f) => !summary.includes(f.split('/').pop() ?? f));
+        // validate: every checkpointed file mentioned (path-segment match).
+        const missing = opts.checkpointedFiles.filter((f) => !mentionsFile(summary, f));
         if (missing.length === 0) {
-            // (c) keep last N verbatim + summary as first message
-            const summaryMsg = { role: 'user', content: [{ kind: 'text', text: summary }] };
+            // (c) keep last N verbatim + summary as first message (typed block).
+            const summaryMsg = { role: 'user', content: [textBlock(summary)] };
             return { messages: [summaryMsg, ...newest], summary, dropped: elided.dropped, method: 'summarize' };
         }
         // retry once then fallback to elide
         if (opts.summarizeFn) {
             try {
                 const retry = await opts.summarizeFn(template + '\nEnsure to mention: ' + missing.join(', '));
-                if (missing.every((f) => retry.includes(f.split('/').pop() ?? f))) {
-                    const summaryMsg2 = { role: 'user', content: [{ kind: 'text', text: retry }] };
+                if (missing.every((f) => mentionsFile(retry, f))) {
+                    const summaryMsg2 = { role: 'user', content: [textBlock(retry)] };
                     return { messages: [summaryMsg2, ...newest], summary: retry, dropped: elided.dropped, method: 'summarize' };
                 }
             }
             catch { /* fallback */ }
         }
     }
+    // Post-check the elide path too: if it still exceeds cap, keep eliding.
+    if (elided.dropped === 0) {
+        return { messages, summary: '', dropped: 0, method: 'fallback' };
+    }
     return { messages: elided.messages, summary: elided.dropped > 0 ? `Elided ${elided.dropped} observations` : '', dropped: elided.dropped, method: elided.dropped > 0 ? 'elide' : 'fallback' };
 }

package/dist/context/memory.d.ts

@@ -3,6 +3,17 @@ import type { PlanStep } from '../agent/runtime.js';
 export declare const MEMORY_WRITE_LIMIT_CHARS = 8000;
 /** Steady-state cap (~1k tokens ≈ 4k chars) kept via tail slice. */
 export declare const MEMORY_STEADY_STATE_CHARS = 4000;
+/** Archive index filename (recall without an LLM pass). */
+export declare const MEMORY_ARCHIVE_INDEX = "archive-index.json";
+/** Max retained archives. */
+export declare const MEMORY_ARCHIVE_KEEP = 20;
+export interface MemoryArchiveEntry {
+    file: string;
+    ts: number;
+    chars: number;
+    /** First line of the archived head — lets the model decide what to re-read. */
+    preview: string;
+}
 export declare function memoryWrite(cwd: string, content: string): Promise<string>;
 export declare function loadMemory(cwd: string): Promise<string>;
 /** Synchronous read for the system-prompt build path (run on every turn). */

package/dist/context/memory.js

@@ -10,22 +10,63 @@ import * as fs from 'node:fs/promises';
 import { readFileSync } from 'node:fs';
 import * as path from 'node:path';
 import { redact } from '../policy/secret-redactor.js';
+import { resolveAndFollowSymlinks, assertNotSymlink } from '../policy/path-guard.js';
 /** Single-write ceiling — anything larger is rejected, not sliced. */
 export const MEMORY_WRITE_LIMIT_CHARS = 8000;
 /** Steady-state cap (~1k tokens ≈ 4k chars) kept via tail slice. */
 export const MEMORY_STEADY_STATE_CHARS = 4000;
+/** Archive index filename (recall without an LLM pass). */
+export const MEMORY_ARCHIVE_INDEX = 'archive-index.json';
+/** Max retained archives. */
+export const MEMORY_ARCHIVE_KEEP = 20;
 export async function memoryWrite(cwd, content) {
     if (content.length > MEMORY_WRITE_LIMIT_CHARS) {
         throw new Error(`memory budget exceeded: single write is ${content.length} chars (max ${MEMORY_WRITE_LIMIT_CHARS}) — split it into smaller notes`);
     }
+    // B4 — symlink/TOCTOU guard: the memory dir must stay inside cwd even if
+    // `.klyro` or `.klyro/memory` is (or becomes) a symlink. Resolve+follow up
+    // the existing parent chain first (defeats an escaping parent symlink), then
+    // mkdir, then a final realpath + lstat: a swapped-in final symlink is
+    // refused before any write lands outside cwd.
     const dir = path.join(cwd, '.klyro', 'memory');
+    await resolveAndFollowSymlinks(cwd, '.klyro'); // throws if `.klyro` escapes cwd
     await fs.mkdir(dir, { recursive: true });
-    const p = path.join(dir, 'session-notes.md');
+    const { resolved } = await resolveAndFollowSymlinks(cwd, '.klyro/memory');
+    await assertNotSymlink(resolved);
+    const p = path.join(resolved, 'session-notes.md');
     const prev = await fs.readFile(p, 'utf-8').catch(() => '');
     // S4-at-rest: redact before appending — redact() only fires on secret
     // shapes (key/token/password with [:=-]), so normal prose survives.
-    const next = (prev + '\n' + redact(content)).slice(-MEMORY_STEADY_STATE_CHARS); // ≤1k tokens ~4k chars
-    const tmp = path.join(dir, `.session-notes.md.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+    const full = prev + '\n' + redact(content);
+    let next = full;
+    if (full.length > MEMORY_STEADY_STATE_CHARS) {
+        // Rotation, not silent loss: the dropped head is archived to a dated
+        // file (pruned to the latest 20) instead of vanishing, and the archive
+        // index is refreshed so the model knows what exists to re-read.
+        const head = full.slice(0, full.length - MEMORY_STEADY_STATE_CHARS);
+        next = full.slice(-MEMORY_STEADY_STATE_CHARS);
+        try {
+            const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+            const file = `archive-${stamp}.md`;
+            await fs.writeFile(path.join(resolved, file), head, 'utf-8');
+            const entries = (await fs.readdir(resolved)).filter((e) => e.startsWith('archive-') && e.endsWith('.md')).sort();
+            for (const old of entries.slice(0, Math.max(0, entries.length - MEMORY_ARCHIVE_KEEP))) {
+                await fs.unlink(path.join(resolved, old)).catch(() => undefined);
+            }
+            const index = [];
+            for (const e of (await fs.readdir(resolved)).filter((e) => e.startsWith('archive-') && e.endsWith('.md')).sort()) {
+                try {
+                    const content = await fs.readFile(path.join(resolved, e), 'utf-8');
+                    const stat = await fs.stat(path.join(resolved, e));
+                    index.push({ file: e, ts: Math.round(stat.mtimeMs), chars: content.length, preview: content.split('\n')[0]?.slice(0, 160) ?? '' });
+                }
+                catch { /* skip unreadable entries */ }
+            }
+            await fs.writeFile(path.join(resolved, MEMORY_ARCHIVE_INDEX), JSON.stringify(index, null, 2), 'utf-8');
+        }
+        catch { /* archive is best-effort; the live notes still persist */ }
+    }
+    const tmp = path.join(resolved, `.session-notes.md.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
     await fs.writeFile(tmp, next, 'utf-8');
     try {
         const fh = await fs.open(tmp, 'r+');
@@ -66,7 +107,21 @@ export function loadMemorySync(cwd) {
 /** Wrap persisted notes into the injected prompt block; '' when empty. */
 export function memoryBlock(cwd) {
     const notes = loadMemorySync(cwd).trim();
-    return notes ? `\n\n<memory>\n${notes.slice(0, 4000)}\n</memory>` : '';
+    if (!notes)
+        return '';
+    // Recall aid: tell the model which archives exist (with previews) so it
+    // can re-read them via read_file instead of losing rotated knowledge.
+    let archived = '';
+    try {
+        const raw = readFileSync(path.join(cwd, '.klyro', 'memory', MEMORY_ARCHIVE_INDEX), 'utf-8');
+        const entries = JSON.parse(raw);
+        if (Array.isArray(entries) && entries.length > 0) {
+            const lines = entries.slice(-5).map((e) => `  - .klyro/memory/${e.file} (${e.chars} chars): ${e.preview}`);
+            archived = `\nEarlier notes archived (read one with read_file for detail):\n${lines.join('\n')}`;
+        }
+    }
+    catch { /* no index — nothing archived yet */ }
+    return `\n\n<memory>\n${notes.slice(0, 4000)}${archived}\n</memory>`;
 }
 export function shouldRemind(turn, lastRemindTurn) {
     return turn - lastRemindTurn >= 20;

package/dist/eval/harness.d.ts

@@ -7,7 +7,7 @@
  * This is the MVP gate per Devolopment-plan.md / docs/plan.md §10: a
  * reproducible suite of programmatic tasks. Real repo tasks come in v1.0.
  */
-import type { StreamEvent } from '../agent/provider-adapter.js';
+import type { ProviderAdapter, StreamEvent } from '../agent/provider-adapter.js';
 export interface ScriptedTask {
     id: string;
     description: string;
@@ -20,6 +20,13 @@ export interface ScriptedTask {
     expectStatus: 'complete' | 'max_steps' | 'aborted' | 'verify_failed' | 'no_final';
     /** Expected tool-call count. */
     expectToolCalls?: number;
+    /**
+     * Semantic rubric graded by a model judge (see judge.ts). Only runs when
+     * the caller supplies a judge adapter; otherwise recorded as skipped.
+     */
+    judge?: {
+        rubric: string[];
+    };
 }
 export interface TaskResult {
     id: string;
@@ -28,8 +35,17 @@ export interface TaskResult {
     observedStatus?: string;
     observedToolCalls?: number;
     durationMs: number;
+    judge?: {
+        pass: boolean;
+        notes: string;
+        skipped: boolean;
+    };
 }
-export declare function runTask(t: ScriptedTask): Promise<TaskResult>;
+export declare function runTask(t: ScriptedTask, opts?: {
+    judgeAdapter?: ProviderAdapter;
+    judgeModel?: string;
+    workDir?: string;
+}): Promise<TaskResult>;
 export interface HarnessSummary {
     total: number;
     passed: number;
@@ -38,7 +54,10 @@ export interface HarnessSummary {
     results: TaskResult[];
     durationMs: number;
 }
-export declare function runHarness(tasks: ScriptedTask[]): Promise<HarnessSummary>;
+export declare function runHarness(tasks: ScriptedTask[], opts?: {
+    judgeAdapter?: ProviderAdapter;
+    judgeModel?: string;
+}): Promise<HarnessSummary>;
 /** Format a harness summary as a markdown report. */
 export declare function formatReport(summary: HarnessSummary): string;
 /** 5.4 — File-based fixture support: repo|repo.json, task.md, check.sh, meta.json */
@@ -48,14 +67,30 @@ export interface FileFixture {
     checkSh: string;
     meta: Record<string, unknown>;
     repo?: string;
+    /**
+     * Optional canned agent turns (StreamEvent[][], same shape as
+     * ScriptedTask.script). When present the fixture is agent-driven: the
+     * runtime executes the script with real tools in a seeded tmp repo and
+     * check.sh then asserts the filesystem outcomes.
+     */
+    script?: StreamEvent[][];
 }
 export declare function loadFileFixture(dir: string): Promise<FileFixture>;
 export declare function runFileFixture(fixture: FileFixture, opts?: {
     runs?: number;
     parallel?: number;
 }): Promise<TaskResult>;
-/** Hook the harness up to a session + audit log so task runs are durable. */
-export declare function runHarnessWithPersistence(tasks: ScriptedTask[], opts: {
+/**
+ * Agent-driven fixture: seed a tmp repo, run the fixture's canned script
+ * through the REAL runtime + tools, then assert outcomes with check.sh
+ * (plus optional meta judge rubric). This is what makes fixtures more than
+ * static `exit 0` stubs: the agent loop genuinely executes.
+ */
+export declare function runAgentFixture(fixture: FileFixture, opts?: {
+    judgeAdapter?: ProviderAdapter;
+    judgeModel?: string;
+}): Promise<TaskResult>;
+/** Hook the harness up to a session + audit log so task runs are durable. */ export declare function runHarnessWithPersistence(tasks: ScriptedTask[], opts: {
     storeDir: string;
     auditPath: string;
 }): Promise<HarnessSummary>;

package/dist/eval/harness.js

@@ -42,9 +42,13 @@ function scriptedAdapter(script) {
         },
     };
 }
-export async function runTask(t) {
+export async function runTask(t, opts = {}) {
     const start = Date.now();
-    const cwd = path.join(os.tmpdir(), 'klyro-eval-' + t.id + '-' + Math.random().toString(36).slice(2));
+    // workDir lets callers seed a repo and inspect outcomes afterwards
+    // (agent-driven fixtures); otherwise an isolated tmp dir is used and
+    // removed.
+    const owned = !opts.workDir;
+    const cwd = opts.workDir ?? path.join(os.tmpdir(), 'klyro-eval-' + t.id + '-' + Math.random().toString(36).slice(2));
     await fs.mkdir(cwd, { recursive: true });
     const reg = new ToolRegistry()
         .register(readFileTool).register(writeFileTool).register(editFileTool)
@@ -72,6 +76,33 @@ export async function runTask(t) {
         if (verifyFailure) {
             details += ` verifyFailure=${verifyFailure};`;
         }
+        // Model-graded semantic check (opt-in: needs a live judge adapter).
+        let judge;
+        if (t.judge && t.judge.rubric.length > 0) {
+            if (opts.judgeAdapter) {
+                const { runJudge } = await import('./judge.js');
+                const texts = result.transcript
+                    .filter((m) => m.role === 'assistant')
+                    .flatMap((m) => m.content)
+                    .filter((b) => b.kind === 'text')
+                    .map((b) => b.text)
+                    .join('\n');
+                const v = await runJudge(opts.judgeAdapter, opts.judgeModel ?? 'mock-judge', {
+                    task: t.task,
+                    finalText: result.finalText,
+                    toolCalls: result.toolCalls,
+                    extra: `assistant transcript:\n${texts.slice(0, 3000)}`,
+                }, t.judge.rubric);
+                judge = { pass: v.pass, notes: v.notes, skipped: v.skipped };
+                if (!v.pass) {
+                    details += ` judge=fail (${v.notes || 'rubric unmet'});`;
+                    pass = false;
+                }
+            }
+            else {
+                judge = { pass: true, notes: 'no judge adapter — skipped', skipped: true };
+            }
+        }
         return {
             id: t.id,
             status: pass ? 'pass' : 'fail',
@@ -79,20 +110,23 @@ export async function runTask(t) {
             observedStatus,
             observedToolCalls: result.toolCalls,
             durationMs: Date.now() - start,
+            ...(judge ? { judge } : {}),
         };
     }
     finally {
-        try {
-            await fs.rm(cwd, { recursive: true, force: true });
+        if (owned) {
+            try {
+                await fs.rm(cwd, { recursive: true, force: true });
+            }
+            catch { }
         }
-        catch { }
     }
 }
-export async function runHarness(tasks) {
+export async function runHarness(tasks, opts = {}) {
     const start = Date.now();
     const results = [];
     for (const t of tasks)
-        results.push(await runTask(t));
+        results.push(await runTask(t, opts));
     const passed = results.filter((r) => r.status === 'pass').length;
     return {
         total: results.length,
@@ -130,7 +164,14 @@ export async function loadFileFixture(dir) {
         repo = await fs.readFile(path.join(dir, 'repo'), 'utf-8');
     }
     catch { /* ignore */ }
-    return { dir, task: task.trim(), checkSh, meta, repo };
+    let script;
+    try {
+        const raw = JSON.parse(await fs.readFile(path.join(dir, 'script.json'), 'utf-8'));
+        if (Array.isArray(raw))
+            script = raw;
+    }
+    catch { /* no script → static fixture */ }
+    return { dir, task: task.trim(), checkSh, meta, ...(repo ? { repo } : {}), ...(script ? { script } : {}) };
 }
 export async function runFileFixture(fixture, opts = {}) {
     const start = Date.now();
@@ -168,8 +209,60 @@ export async function runFileFixture(fixture, opts = {}) {
     await fs.rm(tmp, { recursive: true, force: true }).catch(() => undefined);
     return result;
 }
-/** Hook the harness up to a session + audit log so task runs are durable. */
-export async function runHarnessWithPersistence(tasks, opts) {
+/**
+ * Agent-driven fixture: seed a tmp repo, run the fixture's canned script
+ * through the REAL runtime + tools, then assert outcomes with check.sh
+ * (plus optional meta judge rubric). This is what makes fixtures more than
+ * static `exit 0` stubs: the agent loop genuinely executes.
+ */
+export async function runAgentFixture(fixture, opts = {}) {
+    const start = Date.now();
+    const id = path.basename(fixture.dir);
+    if (!fixture.script) {
+        return { id, status: 'fail', details: 'agent fixture needs script.json', durationMs: Date.now() - start };
+    }
+    const tmp = path.join(os.tmpdir(), 'klyro-eval-agent-' + Math.random().toString(36).slice(2));
+    await fs.mkdir(tmp, { recursive: true });
+    try {
+        if (fixture.repo) {
+            const src = path.isAbsolute(fixture.repo) ? fixture.repo : path.join(fixture.dir, fixture.repo);
+            await fs.cp(src, tmp, { recursive: true }).catch(() => undefined);
+        }
+        const meta = fixture.meta;
+        const judgeRubric = meta['judge'] && typeof meta['judge'] === 'object'
+            ? meta['judge'].rubric
+            : undefined;
+        const r = await runTask({
+            id,
+            description: fixture.task.slice(0, 120),
+            task: fixture.task,
+            script: fixture.script,
+            expectStatus: (typeof meta['expectStatus'] === 'string' ? meta['expectStatus'] : 'complete'),
+            ...(typeof meta['expectToolCalls'] === 'number' ? { expectToolCalls: meta['expectToolCalls'] } : {}),
+            ...(Array.isArray(judgeRubric) && judgeRubric.every((s) => typeof s === 'string') ? { judge: { rubric: judgeRubric } } : {}),
+        }, { workDir: tmp, ...(opts.judgeAdapter ? { judgeAdapter: opts.judgeAdapter, judgeModel: opts.judgeModel } : {}) });
+        if (r.status === 'fail')
+            return { ...r, durationMs: Date.now() - start };
+        // Structural pass — now assert real filesystem outcomes.
+        const { spawn } = await import('node:child_process');
+        const out = await new Promise((resolve) => {
+            const child = spawn('bash', ['-c', fixture.checkSh], { cwd: tmp, shell: false });
+            let buf = '';
+            child.stdout?.on('data', (b) => { buf += b.toString(); });
+            child.stderr?.on('data', (b) => { buf += b.toString(); });
+            child.on('close', (code) => resolve(`exit=${code ?? -1} ${buf.slice(0, 500)}`));
+            child.on('error', (err) => resolve(`spawn-error: ${String(err)}`));
+        });
+        if (!out.startsWith('exit=0')) {
+            return { ...r, status: 'fail', details: `${r.details} check.sh: ${out}`.trim(), durationMs: Date.now() - start };
+        }
+        return { ...r, durationMs: Date.now() - start };
+    }
+    finally {
+        await fs.rm(tmp, { recursive: true, force: true }).catch(() => undefined);
+    }
+}
+/** Hook the harness up to a session + audit log so task runs are durable. */ export async function runHarnessWithPersistence(tasks, opts) {
     const store = new SessionStore(opts.storeDir);
     const audit = new AuditLog(opts.auditPath);
     const start = Date.now();

package/dist/eval/judge.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Model-graded judge for evals: scores a finished run against a rubric.
+ *
+ * Structural asserts (status, tool counts) catch regressions; the judge
+ * catches semantic failures (wrong file, wrong content, ignored task).
+ * Runs on any ProviderAdapter — in CI, pass a live adapter + judge model;
+ * offline runs skip judging (recorded as `skipped`).
+ */
+import type { ProviderAdapter } from '../agent/provider-adapter.js';
+export interface JudgeSpec {
+    /** Each item is one binary criterion, e.g. "note.txt contains exactly 'hi'". */
+    rubric: string[];
+    /** Model for grading (defaults to the caller's judge model). */
+    model?: string;
+}
+export interface JudgeVerdict {
+    pass: boolean;
+    scores: Record<string, number>;
+    notes: string;
+    skipped: boolean;
+}
+/**
+ * Grade a finished run. Returns `{pass:false}` (never throws) when the
+ * model output is unparsable or the call fails — an inconclusive judge
+ * must not silently pass.
+ */
+export declare function runJudge(adapter: ProviderAdapter, model: string, input: {
+    task: string;
+    finalText: string;
+    toolCalls: number;
+    extra?: string;
+}, rubric: string[]): Promise<JudgeVerdict>;

package/dist/eval/judge.js

@@ -0,0 +1,63 @@
+function extractJson(text) {
+    const start = text.indexOf('{');
+    const end = text.lastIndexOf('}');
+    if (start === -1 || end <= start)
+        return null;
+    try {
+        return JSON.parse(text.slice(start, end + 1));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Grade a finished run. Returns `{pass:false}` (never throws) when the
+ * model output is unparsable or the call fails — an inconclusive judge
+ * must not silently pass.
+ */
+export async function runJudge(adapter, model, input, rubric) {
+    if (rubric.length === 0)
+        return { pass: true, scores: {}, notes: 'empty rubric', skipped: false };
+    const criteria = rubric.map((r, i) => `  c${i + 1}. ${r}`).join('\n');
+    const prompt = [
+        'You are an evaluator grading an AI coding agent run. Score ONLY the criteria below.',
+        `Task: ${input.task}`,
+        `Final answer: ${input.finalText.slice(0, 2000)}`,
+        `Tool calls made: ${input.toolCalls}`,
+        input.extra ? `Run facts:\n${input.extra.slice(0, 2000)}` : '',
+        'Criteria (score each 1 = met, 0 = not met):',
+        criteria,
+        'Respond with ONLY a JSON object: {"scores": {"c1": 1, ...}, "notes": "<one line>"}.',
+    ].filter(Boolean).join('\n\n');
+    let text = '';
+    try {
+        for await (const ev of adapter.stream({
+            model,
+            system: 'You are a strict evaluator. Reply with only the requested JSON.',
+            messages: [{ role: 'user', content: [{ kind: 'text', text: prompt }] }],
+            tools: [],
+        })) {
+            if (ev.kind === 'text_delta')
+                text += ev.text;
+            else if (ev.kind === 'error')
+                return { pass: false, scores: {}, notes: `judge call failed: ${ev.message}`, skipped: false };
+        }
+    }
+    catch (err) {
+        return { pass: false, scores: {}, notes: `judge call threw: ${err instanceof Error ? err.message : String(err)}`, skipped: false };
+    }
+    const parsed = extractJson(text);
+    if (!parsed || typeof parsed.scores !== 'object' || parsed.scores === null) {
+        return { pass: false, scores: {}, notes: 'judge output unparsable', skipped: false };
+    }
+    const scores = {};
+    let pass = true;
+    rubric.forEach((_r, i) => {
+        const v = parsed.scores[`c${i + 1}`];
+        const n = v === 1 || v === '1' || v === true ? 1 : 0;
+        scores[`c${i + 1}`] = n;
+        if (n !== 1)
+            pass = false;
+    });
+    return { pass, scores, notes: typeof parsed.notes === 'string' ? parsed.notes.slice(0, 500) : '', skipped: false };
+}