@xingular/harness 0.1.6 → 1.0.0

package/dist/sdk/refine/suite.js

@@ -0,0 +1,174 @@
+// Evaluating variants on a task set: one execution per version × task, recorded as an
+// answer-free matrix of what each variant delivered. No improver, no selection, no adoption —
+// the only executions here are the given Executor's probe runs. Records live beside a
+// refinement's, under `<data root>/refine/<id>/`, so a sweep over variant versions or package
+// commits reads the same record family (`refine/<id>/matrix.json`).
+//
+// Every row carries its comparability keys (task-prompt hash, executor name, package
+// version, variant fingerprint), so two matrices — taken at different commits or versions —
+// are diffable without re-reading the runs that produced them.
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { now, readJson, safeId, writeJson } from "../core/fs.js";
+import * as store from "../variant/store.js";
+import { PI_SURFACE } from "../variant/surface.js";
+import { writeVariant } from "../variant/variant.js";
+import { assertVariantHasNoGraderVerdicts } from "./evidence.js";
+import { mapLimit } from "./recursion.js";
+export function matrixDir(root, id) {
+    return join(root, 'refine', safeId(id));
+}
+export function readMatrix(root, id) {
+    return readJson(join(matrixDir(root, id), 'matrix.json'));
+}
+let packageVersionMemo = null;
+/** This package's release, whatever shape the module took: source, esbuild bundle, or an
+ * installed copy. Walks up from the module's own URL to the nearest package.json. */
+function packageVersion() {
+    if (packageVersionMemo === null) {
+        packageVersionMemo = 'unknown';
+        try {
+            let dir = fileURLToPath(import.meta.url);
+            for (let up = 0; up < 6; up++) {
+                dir = dirname(dir);
+                const manifest = join(dir, 'package.json');
+                if (existsSync(manifest)) {
+                    const json = JSON.parse(readFileSync(manifest, 'utf8'));
+                    if (json.version)
+                        packageVersionMemo = json.version;
+                    break;
+                }
+            }
+        }
+        catch {
+            // A record without a version is still a record.
+        }
+    }
+    return packageVersionMemo;
+}
+function taskHash(task) {
+    return createHash('sha256').update(task.prompt).digest('hex').slice(0, 16);
+}
+const SEGMENT = /^[A-Za-z0-9_-]{1,128}$/;
+function stage(input) {
+    const labels = [];
+    for (const entry of input.versions) {
+        if (!SEGMENT.test(entry.label))
+            throw new Error(`Version label '${entry.label}' must be a usable record name (A-Z a-z 0-9 _ -).`);
+        labels.push(entry.label);
+    }
+    if (new Set(labels).size !== labels.length)
+        throw new Error('Version labels must be unique.');
+    if (input.tasks.some((task) => !SEGMENT.test(task.id)))
+        throw new Error('Task ids must be usable record names (A-Z a-z 0-9 _ -).');
+    if (new Set(input.tasks.map((task) => task.id)).size !== input.tasks.length)
+        throw new Error('Task ids must be unique.');
+    // Staging and the tripwire happen before any execution: a variant carrying grader output
+    // never reaches an execution, the same guard the refinement's evidence folder applies
+    // (INV-REF-1). A variant that cannot be written becomes an error row, never an abort.
+    return input.versions.map((entry) => {
+        try {
+            assertVariantHasNoGraderVerdicts(entry.variant);
+            return { label: entry.label, variant: entry.variant, fingerprint: store.fingerprint(entry.variant), dir: writeVariant(PI_SURFACE, entry.variant, join(matrixDir(input.root, input.id), 'variants', entry.label)), error: '' };
+        }
+        catch (error) {
+            return { label: entry.label, variant: entry.variant, fingerprint: store.fingerprint(entry.variant), dir: '', error: error instanceof Error ? error.message : String(error) };
+        }
+    });
+}
+/** Runs the full version × task matrix. Bad input throws; execution failures become rows. */
+export async function runMatrix(input) {
+    if (!input.versions.length)
+        throw new Error('A matrix needs at least one harness version.');
+    if (!input.tasks.length)
+        throw new Error('A matrix needs at least one task.');
+    if (input.tasks.some((task) => !task.prompt.trim()))
+        throw new Error('Every task needs a non-empty prompt.');
+    const { root, id, tasks, executor } = input;
+    const executorName = input.executorName ?? executor.constructor?.name ?? 'executor';
+    const emit = (event) => input.onEvent?.(event);
+    const staged = stage(input);
+    const summary = {
+        id,
+        status: 'running',
+        startedAt: now(),
+        endedAt: null,
+        versions: staged.map((entry) => entry.label),
+        tasks: tasks.map((task) => task.id),
+        rows: [],
+        usd: 0,
+        errors: []
+    };
+    const save = () => {
+        summary.usd = Math.round(summary.rows.reduce((sum, row) => sum + row.costUsd, 0) * 1e6) / 1e6;
+        writeJson(join(matrixDir(root, id), 'matrix.json'), summary);
+        emit({ type: 'summary', summary: structuredClone(summary) });
+    };
+    writeJson(join(matrixDir(root, id), 'suite.json'), { id, versions: staged.map(({ label, fingerprint }) => ({ label, fingerprint })), tasks });
+    save();
+    const pairs = staged.flatMap((version) => tasks.map((task) => ({ version, task })));
+    await mapLimit(pairs, input.rolloutWidth ?? 4, async ({ version, task }) => {
+        const runId = `${id}-${version.label}-${task.id}`;
+        let row;
+        if (input.signal?.aborted)
+            row = failedRow(version, task, runId, executorName, 'The matrix was cancelled.');
+        else if (version.error)
+            row = failedRow(version, task, runId, executorName, `the variant could not be staged: ${version.error}`);
+        else {
+            let validation = null;
+            if (executor.validateVariant) {
+                try {
+                    executor.validateVariant(task, version.variant);
+                }
+                catch (error) {
+                    validation = `invalid variant for this task: ${error instanceof Error ? error.message : String(error)}`;
+                }
+            }
+            if (validation)
+                row = failedRow(version, task, runId, executorName, validation);
+            else {
+                emit({ type: 'execution.started', version: version.label, task: task.id, label: runId });
+                let rollout;
+                try {
+                    rollout = await executor.execute(task, version.dir, runId, runId);
+                }
+                catch (error) {
+                    // The Executor contract says execute never throws; this keeps the record honest if one does.
+                    rollout = { label: runId, runId, dir: join(matrixDir(root, id), 'missing', version.label, task.id), ok: false, delivered: false, costUsd: 0, wallMs: 0, error: error instanceof Error ? error.message : String(error) };
+                }
+                emit({ type: 'execution.finished', version: version.label, task: task.id, ok: rollout.ok, delivered: rollout.delivered, error: rollout.error });
+                row = rowOf(version, task, runId, executorName, rollout);
+            }
+        }
+        summary.rows.push(row);
+        save();
+    });
+    for (const error of staged.filter((entry) => entry.error).map((entry) => `${entry.label}: ${entry.error}`))
+        summary.errors.push(error);
+    summary.status = input.signal?.aborted ? 'cancelled' : summary.rows.some((row) => row.ok) ? 'ok' : summary.rows.length ? 'partial' : 'failed';
+    summary.endedAt = now();
+    save();
+    return summary;
+}
+function rowOf(version, task, runId, executorName, rollout) {
+    return {
+        version: version.label,
+        fingerprint: version.fingerprint,
+        task: task.id,
+        runId,
+        dir: rollout.dir,
+        ok: rollout.ok,
+        delivered: rollout.delivered,
+        costUsd: rollout.costUsd,
+        wallMs: rollout.wallMs,
+        error: rollout.error,
+        taskHash: taskHash(task),
+        executor: executorName,
+        packageVersion: packageVersion()
+    };
+}
+function failedRow(version, task, runId, executorName, error) {
+    return rowOf(version, task, runId, executorName, { label: runId, runId, dir: '', ok: false, delivered: false, costUsd: 0, wallMs: 0, error });
+}

package/dist/sdk/refine/trajectory.d.ts

@@ -0,0 +1,9 @@
+import type { RunEvent } from "../sessions/contracts.js";
+export declare function renderEvents(events: readonly RunEvent[], options: {
+    obsTrunc: number;
+    argsTrunc?: number;
+}): string;
+/** The last assistant message of a run. */
+export declare function finalText(events: readonly RunEvent[]): string;
+/** The run's cost so far, from its last usage report. */
+export declare function costOf(events: readonly RunEvent[]): number;

package/dist/sdk/refine/trajectory.js

@@ -0,0 +1,68 @@
+const TRUNC = (n) => `…(+${n} chars — render truncation)`;
+function clip(text, limit) {
+    const clean = text.replace(/\0/g, '');
+    return limit > 0 && clean.length > limit ? `${clean.slice(0, limit)}${TRUNC(clean.length - limit)}` : clean;
+}
+export function renderEvents(events, options) {
+    const out = [];
+    const argsTrunc = options.argsTrunc ?? 2000;
+    for (const event of events) {
+        const payload = (event.payload ?? {});
+        switch (event.type) {
+            case 'message.completed':
+                if (typeof payload.text === 'string' && payload.text.trim())
+                    out.push(`[${payload.role === 'user' ? 'USER' : 'ASSISTANT'}]\n${clip(payload.text, options.obsTrunc)}`);
+                break;
+            case 'tool.started':
+                out.push(`[ACTION] ${String(payload.name ?? 'tool')}${typeof payload.input === 'string' && payload.input ? ` ${clip(payload.input, argsTrunc)}` : ''}`);
+                break;
+            case 'tool.completed': {
+                const text = typeof payload.output === 'string' ? payload.output : '';
+                out.push(`${payload.ok === false ? '[ERROR]' : '[OBSERVATION]'} ${String(payload.name ?? 'tool')}${typeof payload.durationMs === 'number' ? ` (${Math.round(payload.durationMs)} ms)` : ''}\n${clip(text, options.obsTrunc)}${payload.outputTruncated ? '\n(the tool output was already bounded when it was recorded)' : ''}`);
+                break;
+            }
+            case 'retry.started':
+                out.push(`[ERROR] model request failed, retrying (${String(payload.attempt)}/${String(payload.maxAttempts)}): ${String(payload.error ?? '')}`);
+                break;
+            case 'compaction.ended':
+                out.push(`[ACTION] context compacted (${String(payload.reason ?? '')})${typeof payload.tokensBefore === 'number' ? `: ${payload.tokensBefore} → ${String(payload.tokensAfter ?? '?')} tokens` : ''}`);
+                break;
+            case 'approval.resolved':
+                if (payload.decision === 'denied')
+                    out.push('[ERROR] a tool call was refused');
+                break;
+            case 'extension.notice':
+                if (typeof payload.text === 'string')
+                    out.push(`[OBSERVATION] notice: ${clip(payload.text, 2000)}`);
+                break;
+            case 'failed':
+                out.push(`[ERROR] run failed: ${String(payload.message ?? payload.code ?? '')}`);
+                break;
+            case 'status.changed':
+                if (payload.status === 'interrupted')
+                    out.push('[ERROR] run interrupted');
+                break;
+            default:
+                break;
+        }
+    }
+    return out.length ? out.join('\n\n') : '(no trajectory: the run recorded no messages or tool use)';
+}
+/** The last assistant message of a run. */
+export function finalText(events) {
+    for (let i = events.length - 1; i >= 0; i--) {
+        const payload = events[i].payload;
+        if (events[i].type === 'message.completed' && payload?.role === 'assistant' && typeof payload.text === 'string' && payload.text.trim())
+            return payload.text;
+    }
+    return '';
+}
+/** The run's cost so far, from its last usage report. */
+export function costOf(events) {
+    for (let i = events.length - 1; i >= 0; i--) {
+        const payload = events[i].payload;
+        if (events[i].type === 'usage.updated' && typeof payload?.costUsd === 'number')
+            return payload.costUsd;
+    }
+    return 0;
+}

package/dist/sdk/refine/types.d.ts

@@ -0,0 +1,77 @@
+import type { Variant } from "../variant/surface.js";
+import type { WorkPrompts } from "./prompts.js";
+export type RecursionMode = 'parallel' | 'sequential';
+/** One piece of work to refine on. */
+export interface RefineTask {
+    /** Stable id of the task; names its folder and its library entry. */
+    id: string;
+    /** The request, as the agent received it, with any expected answer stripped. */
+    prompt: string;
+    /** The harness scope: whose variant this is and whose library it reads and grows. */
+    scope: string;
+    /** A grouping inside the scope's library. Defaults to the scope. */
+    domain?: string;
+    /** Material the work was done from, served to improvers and the selector as `source/`. */
+    sourceDir?: string | null;
+    /** Paths that must never reach a model (answer keys and the like), as globs. */
+    leakGlobs?: string[];
+    /** The text about the work the prompts interpolate. */
+    prompts?: WorkPrompts;
+    /** Whether a harness here may carry subagents. Default true. */
+    allowSubagents?: boolean;
+}
+/** What the method knows about one finished execution. Never carries a score. */
+export interface Rollout {
+    label: string;
+    runId: string;
+    /** A folder that holds this execution's evidence. */
+    dir: string;
+    ok: boolean;
+    delivered: boolean;
+    costUsd: number;
+    wallMs: number;
+    error: string;
+}
+export interface Executor {
+    /**
+     * Runs one execution of `task` with the harness in `variantDir` (null: the base harness) and
+     * returns what it produced. A failure is returned as `ok: false`, never thrown.
+     */
+    execute(task: RefineTask, variantDir: string | null, label: string, runId: string): Promise<Rollout>;
+    /** The execution in the shared `[ACTION]/[OBSERVATION]/[ERROR]` grammar. */
+    trajectory(rollout: Rollout, options: {
+        obsTrunc: number;
+    }): string;
+    /** filename → readable text, one entry per artifact produced. */
+    deliverableFiles(rollout: Rollout): Record<string, string>;
+    /** Extra validation of a proposed variant for this task; throw with every problem listed. */
+    validateVariant?(task: RefineTask, variant: Variant): void;
+}
+export interface ImproverReply {
+    label: string;
+    text: string;
+    costUsd: number;
+    attempts: number;
+    cached: boolean;
+    /** The harvested folder, when the call asked for one; null when it did not come back. */
+    workspace: string | null;
+}
+export declare class ImproverError extends Error {
+    readonly attempts: number;
+    readonly costUsd: number;
+    constructor(message: string, attempts?: number, costUsd?: number);
+}
+export interface ImproverCall {
+    /** The instruction; evidence stays on disk under `context/`. */
+    mission: string;
+    /** The folder copied into the session as `context/`. */
+    contextDir: string;
+    /** A unique label for the audit trail and cache. */
+    label: string;
+    /** A path inside the session's folder to copy out before it is removed, e.g. `context/harness`. */
+    harvest?: string;
+    signal?: AbortSignal;
+}
+export interface Improver {
+    call(input: ImproverCall): Promise<ImproverReply>;
+}

package/dist/sdk/refine/types.js

@@ -0,0 +1,10 @@
+export class ImproverError extends Error {
+    attempts;
+    costUsd;
+    constructor(message, attempts = 0, costUsd = 0) {
+        super(message);
+        this.attempts = attempts;
+        this.costUsd = costUsd;
+        this.name = 'ImproverError';
+    }
+}

package/dist/sdk/server/methods.d.ts

@@ -0,0 +1,3 @@
+import type { Harness } from "../harness.js";
+import type { Handler } from "./rpc.js";
+export declare function methods(harness: Harness): Record<string, Handler>;