@iowarp/clio-coder 0.3.0 → 0.3.1

package/src/domains/eval/runners/external-command.ts

@@ -0,0 +1,350 @@
+import { spawn } from "node:child_process";
+import { performance } from "node:perf_hooks";
+import {
+	addChaosObservations,
+	chaosMetricEntries,
+	createChaosFold,
+	EMPTY_CHAOS_OBSERVATION,
+	type EvalChaosObservation,
+	parseChaosMarker,
+} from "../metrics/chaos-stream.js";
+import {
+	addFleetLoopObservations,
+	createFleetLoopFold,
+	EMPTY_FLEET_LOOP_OBSERVATION,
+	type EvalFleetLoopObservation,
+	fleetLoopMetricEntries,
+} from "../metrics/fleet-loop-stream.js";
+import {
+	addStreamInvariants,
+	createStreamInvariantFold,
+	EMPTY_STREAM_INVARIANTS,
+	type EvalStreamInvariants,
+	streamInvariantMetrics,
+} from "../metrics/invariants.js";
+import {
+	addTokenStreamUsage,
+	createTokenUsageFold,
+	type EvalTokenStreamUsage,
+	tokenMetricEntries,
+	UNMEASURED_TOKEN_USAGE,
+} from "../metrics/token-stream.js";
+import type { EvalRunnerV2 } from "../schema/suite.js";
+
+export interface EvalRunnerOutput {
+	/** Exact terminal assignment/receipt linkage when this runner dispatched Clio. */
+	assignmentId: string | null;
+	terminalReceiptDigest: string | null;
+	exitCode: number;
+	stdout: string;
+	stderr: string;
+	wallTimeMs: number;
+	metrics: Record<string, number | string | boolean | null>;
+	artifacts: Record<string, string | string[] | null>;
+}
+
+export interface ShellCommandResult {
+	command: string;
+	exitCode: number;
+	stdout: string;
+	/** Compact, complete tool-event JSONL retained independently of artifact truncation. */
+	metricJsonl: string;
+	/** Provider usage folded from the live stream, independent of artifact truncation. */
+	usage: EvalTokenStreamUsage;
+	/** Wire-stream structural invariants folded live, for the same reason. */
+	streamInvariants: EvalStreamInvariants;
+	/** Strict SIGINT harness marker folded live from stdout. */
+	chaos: EvalChaosObservation;
+	fleetLoops: EvalFleetLoopObservation;
+	stderr: string;
+	wallTimeMs: number;
+	timedOut: boolean;
+}
+
+const OUTPUT_LIMIT = 200_000;
+const OUTPUT_HEAD_LIMIT = 20_000;
+const OUTPUT_TRUNCATION_MARKER = "\n[output middle truncated; tail preserved]\n";
+const METRIC_JSONL_LIMIT = 256_000;
+const METRIC_JSONL_LINE_LIMIT = 64_000;
+
+export async function runExternalCommandRunner(
+	runner: EvalRunnerV2,
+	cwd: string,
+	timeoutMs: number,
+	env?: NodeJS.ProcessEnv,
+): Promise<EvalRunnerOutput> {
+	const runnerCommands = runner.commands ?? [];
+	const commands = runnerCommands.length > 0 ? runnerCommands : runner.command === undefined ? [] : [runner.command];
+	let stdout = "";
+	let stderr = "";
+	let wallTimeMs = 0;
+	// An external command is an opaque subprocess: it may be a `clio-coder run --json`
+	// whose usage events cross this stdout, or a harness script that runs Clio
+	// out of sight. Usage observed on the stream is reported; nothing observed
+	// is reported as unmeasured, never as zero cost.
+	let usage = UNMEASURED_TOKEN_USAGE;
+	let streamInvariants = EMPTY_STREAM_INVARIANTS;
+	let chaos = EMPTY_CHAOS_OBSERVATION;
+	let fleetLoops = EMPTY_FLEET_LOOP_OBSERVATION;
+	for (const command of commands) {
+		const result = await runShellCommand(command, cwd, runner.timeoutMs ?? timeoutMs, env);
+		stdout = appendLimited(stdout, result.stdout);
+		stderr = appendLimited(stderr, result.stderr);
+		wallTimeMs += result.wallTimeMs;
+		usage = addTokenStreamUsage(usage, result.usage);
+		streamInvariants = addStreamInvariants(streamInvariants, result.streamInvariants);
+		chaos = addChaosObservations(chaos, result.chaos);
+		fleetLoops = addFleetLoopObservations(fleetLoops, result.fleetLoops);
+		if (result.exitCode !== 0) {
+			return {
+				assignmentId: null,
+				terminalReceiptDigest: null,
+				exitCode: result.exitCode,
+				stdout,
+				stderr,
+				wallTimeMs,
+				metrics: {
+					"latency.wallMs": wallTimeMs,
+					...tokenMetricEntries(usage),
+					...streamInvariantMetrics(streamInvariants),
+					...chaosMetricEntries(chaos),
+					...fleetLoopMetricEntries(fleetLoops),
+					"verifier.exitCode": result.exitCode,
+				},
+				artifacts: chaosArtifacts(chaos),
+			};
+		}
+	}
+	return {
+		assignmentId: null,
+		terminalReceiptDigest: null,
+		exitCode: 0,
+		stdout,
+		stderr,
+		wallTimeMs,
+		metrics: {
+			"latency.wallMs": wallTimeMs,
+			...tokenMetricEntries(usage),
+			...streamInvariantMetrics(streamInvariants),
+			...chaosMetricEntries(chaos),
+			...fleetLoopMetricEntries(fleetLoops),
+			"verifier.exitCode": 0,
+		},
+		artifacts: chaosArtifacts(chaos),
+	};
+}
+
+export function runShellCommand(
+	command: string,
+	cwd: string,
+	timeoutMs: number,
+	env?: NodeJS.ProcessEnv,
+): Promise<ShellCommandResult> {
+	// `wallTimeMs` is published and compared across runs and machines, so it is
+	// measured on the monotonic clock: an NTP correction mid-suite would
+	// otherwise corrupt one row with no marker on it.
+	const started = performance.now();
+	return new Promise((resolve) => {
+		let stdout = "";
+		let stderr = "";
+		const metricCapture = createJsonlMetricCapture();
+		const usageFold = createTokenUsageFold();
+		const streamFold = createStreamInvariantFold();
+		const chaosFold = createChaosFold();
+		const fleetLoopFold = createFleetLoopFold();
+		let timedOut = false;
+		let settled = false;
+		const child = spawn(command, {
+			cwd,
+			shell: true,
+			stdio: ["ignore", "pipe", "pipe"],
+			// The overlay is additive: an eval item pins where Clio writes its
+			// journal, and inherits everything else the operator's shell provides.
+			env: env === undefined ? process.env : { ...process.env, ...env },
+		});
+		child.stdout.setEncoding("utf8");
+		child.stderr.setEncoding("utf8");
+		child.stdout.on("data", (chunk: string) => {
+			stdout = appendLimited(stdout, chunk);
+			metricCapture.push(chunk);
+			usageFold.push(chunk);
+			streamFold.push(chunk);
+			chaosFold.push(chunk);
+			fleetLoopFold.push(chunk);
+		});
+		child.stderr.on("data", (chunk: string) => {
+			stderr = appendLimited(stderr, chunk);
+		});
+		const timer = setTimeout(() => {
+			timedOut = true;
+			child.kill("SIGTERM");
+			setTimeout(() => child.kill("SIGKILL"), 1000);
+		}, timeoutMs);
+		const finish = (exitCode: number): void => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timer);
+			resolve({
+				command,
+				exitCode,
+				stdout,
+				metricJsonl: metricCapture.finish(),
+				usage: usageFold.usage(),
+				streamInvariants: streamFold.invariants(),
+				chaos: chaosFold.observation(),
+				fleetLoops: fleetLoopFold.observation(),
+				stderr,
+				wallTimeMs: Math.round(performance.now() - started),
+				timedOut,
+			});
+		};
+		child.on("error", (error) => {
+			stderr = appendLimited(stderr, error.message);
+			finish(1);
+		});
+		child.on("close", (code) => {
+			finish(typeof code === "number" ? code : timedOut ? 124 : 1);
+		});
+	});
+}
+
+/** Bounded head+tail capture: terminal JSON/receipts must survive verbose runs. */
+export function appendLimited(current: string, chunk: string): string {
+	const next = `${current}${chunk}`;
+	if (next.length <= OUTPUT_LIMIT) return next;
+	const tailLimit = OUTPUT_LIMIT - OUTPUT_HEAD_LIMIT - OUTPUT_TRUNCATION_MARKER.length;
+	return `${next.slice(0, OUTPUT_HEAD_LIMIT)}${OUTPUT_TRUNCATION_MARKER}${next.slice(-tailLimit)}`;
+}
+
+export interface JsonlMetricCapture {
+	push(chunk: string): void;
+	finish(): string;
+}
+
+/**
+ * Preserve the small tool-event facts used by eval metrics even when verbose
+ * streaming deltas force the operator-facing stdout artifact to retain only
+ * its head and tail. Input is line-buffered because child-process chunks may
+ * split JSON arbitrarily; oversized non-metric lines are discarded without
+ * growing memory.
+ */
+export function createJsonlMetricCapture(): JsonlMetricCapture {
+	const lines: string[] = [];
+	let storedBytes = 0;
+	let pending = "";
+	let discardingOversizedLine = false;
+	let finished = false;
+
+	const store = (line: string): void => {
+		let parsed: unknown;
+		try {
+			parsed = JSON.parse(line) as unknown;
+		} catch {
+			return;
+		}
+		if (!isRecord(parsed)) return;
+		const compact = compactMetricEvent(parsed);
+		if (compact === null) return;
+		const encoded = JSON.stringify(compact);
+		if (encoded.length > METRIC_JSONL_LINE_LIMIT) return;
+		lines.push(encoded);
+		storedBytes += encoded.length + 1;
+		while (storedBytes > METRIC_JSONL_LIMIT && lines.length > 1) {
+			const removed = lines.shift();
+			if (removed !== undefined) storedBytes -= removed.length + 1;
+		}
+	};
+
+	const push = (chunk: string): void => {
+		if (finished || chunk.length === 0) return;
+		let remaining = chunk;
+		if (discardingOversizedLine) {
+			const newline = remaining.indexOf("\n");
+			if (newline === -1) return;
+			remaining = remaining.slice(newline + 1);
+			discardingOversizedLine = false;
+		}
+		pending += remaining;
+		for (;;) {
+			const newline = pending.indexOf("\n");
+			if (newline === -1) break;
+			store(pending.slice(0, newline).replace(/\r$/u, ""));
+			pending = pending.slice(newline + 1);
+		}
+		if (pending.length > METRIC_JSONL_LINE_LIMIT) {
+			pending = "";
+			discardingOversizedLine = true;
+		}
+	};
+
+	return {
+		push,
+		finish(): string {
+			if (!finished && !discardingOversizedLine && pending.length > 0) store(pending.replace(/\r$/u, ""));
+			finished = true;
+			pending = "";
+			return lines.join("\n");
+		},
+	};
+}
+
+function compactMetricEvent(event: Record<string, unknown>): Record<string, unknown> | null {
+	const type = event.type;
+	if (type === "clio_soak_chaos") {
+		const marker = parseChaosMarker(event);
+		return marker === null ? null : { type, ...marker };
+	}
+	if (type === "tool_execution_start") {
+		const toolName = stringField(event, "toolName");
+		if (toolName !== "dispatch" && toolName !== "code_nav" && toolName !== "read" && toolName !== "grep") {
+			return null;
+		}
+		return {
+			type,
+			...(stringField(event, "toolCallId") !== undefined ? { toolCallId: stringField(event, "toolCallId") } : {}),
+			toolName,
+			...(toolName === "dispatch" && isRecord(event.args)
+				? { args: event.args }
+				: toolName === "code_nav" && isRecord(event.args)
+					? { args: { mode: event.args.mode } }
+					: {}),
+		};
+	}
+	if (type === "tool_execution_end") {
+		return {
+			type,
+			...(stringField(event, "toolCallId") !== undefined ? { toolCallId: stringField(event, "toolCallId") } : {}),
+			...(stringField(event, "toolName") !== undefined ? { toolName: stringField(event, "toolName") } : {}),
+			...(event.isError === true ? { isError: true } : {}),
+			...(stringField(event, "outcome") !== undefined ? { outcome: stringField(event, "outcome") } : {}),
+		};
+	}
+	if (type !== "clio_tool_finish" || !isRecord(event.payload)) return null;
+	return {
+		type,
+		payload: {
+			...(stringField(event.payload, "tool") !== undefined ? { tool: stringField(event.payload, "tool") } : {}),
+			...(stringField(event.payload, "toolCallId") !== undefined
+				? { toolCallId: stringField(event.payload, "toolCallId") }
+				: stringField(event, "toolCallId") !== undefined
+					? { toolCallId: stringField(event, "toolCallId") }
+					: {}),
+			...(stringField(event.payload, "outcome") !== undefined ? { outcome: stringField(event.payload, "outcome") } : {}),
+		},
+	};
+}
+
+function chaosArtifacts(observation: EvalChaosObservation): Record<string, string> {
+	return observation.markerCount === 1 && observation.marker !== null
+		? { chaosSeed: String(observation.marker.seed) }
+		: {};
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function stringField(record: Record<string, unknown>, field: string): string | undefined {
+	const value = record[field];
+	return typeof value === "string" && value.length > 0 ? value : undefined;
+}

package/src/domains/eval/runtime-paths.ts

@@ -0,0 +1,171 @@
+import { access, readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { EvalRunArtifact, EvalRunRecord } from "./types.js";
+
+interface RunEnvelopeLike {
+	id: string;
+	cwd: string;
+	startedAt: string;
+	endedAt: string | null;
+	receiptPath: string | null;
+	sessionId: string | null;
+}
+
+interface LinkedRunPaths {
+	receipt?: string;
+	sessionLedger?: string;
+}
+
+export async function linkEvalArtifactRuntimePaths(
+	artifact: EvalRunArtifact,
+	stateDir: string,
+	evidencePath?: string,
+): Promise<EvalRunArtifact> {
+	const matches = (await readRunLedger(stateDir)).filter((envelope) => evalRunMatches(artifact, envelope));
+	const linksByCwd = await linksByCwdForRuns(stateDir, matches);
+	const receipts = uniqueStrings(
+		[...linksByCwd.values()].flatMap((links) =>
+			links.flatMap((link) => (link.receipt === undefined ? [] : [link.receipt])),
+		),
+	);
+	const sessionLedgers = uniqueStrings(
+		[...linksByCwd.values()].flatMap((links) =>
+			links.flatMap((link) => (link.sessionLedger === undefined ? [] : [link.sessionLedger])),
+		),
+	);
+	return {
+		...artifact,
+		paths: {
+			...artifact.paths,
+			...(evidencePath === undefined ? {} : { evidence: evidencePath }),
+			receipts,
+			sessionLedgers,
+		},
+		results: artifact.results.map((result) => linkResult(result, linksByCwd.get(result.cwd)?.[0], evidencePath)),
+	};
+}
+
+async function readRunLedger(stateDir: string): Promise<RunEnvelopeLike[]> {
+	try {
+		const parsed = JSON.parse(await readFile(join(stateDir, "runs.json"), "utf8")) as unknown;
+		return Array.isArray(parsed) ? parsed.flatMap(parseRunEnvelopeLike) : [];
+	} catch {
+		return [];
+	}
+}
+
+function parseRunEnvelopeLike(value: unknown): RunEnvelopeLike[] {
+	if (!isRecord(value)) return [];
+	const id = stringField(value, "id");
+	const cwd = stringField(value, "cwd");
+	const startedAt = stringField(value, "startedAt");
+	if (id === undefined || cwd === undefined || startedAt === undefined) return [];
+	return [
+		{
+			id,
+			cwd,
+			startedAt,
+			endedAt: nullableStringField(value, "endedAt"),
+			receiptPath: nullableStringField(value, "receiptPath"),
+			sessionId: nullableStringField(value, "sessionId"),
+		},
+	];
+}
+
+function evalRunMatches(artifact: EvalRunArtifact, envelope: RunEnvelopeLike): boolean {
+	const cwds = new Set(artifact.results.map((result) => result.cwd));
+	if (!cwds.has(envelope.cwd)) return false;
+	const evalStart = Date.parse(artifact.startedAt);
+	const evalEnd = Date.parse(artifact.endedAt);
+	const runStart = Date.parse(envelope.startedAt);
+	const runEnd = envelope.endedAt === null ? runStart : Date.parse(envelope.endedAt);
+	if (![evalStart, evalEnd, runStart, runEnd].every(Number.isFinite)) return false;
+	return runStart <= evalEnd && runEnd >= evalStart;
+}
+
+async function linksByCwdForRuns(
+	stateDir: string,
+	runs: ReadonlyArray<RunEnvelopeLike>,
+): Promise<Map<string, LinkedRunPaths[]>> {
+	const out = new Map<string, LinkedRunPaths[]>();
+	for (const run of [...runs].sort(compareRunEnvelopeLike)) {
+		const receipt = await existingPath(run.receiptPath ?? join(stateDir, "receipts", `${run.id}.json`));
+		const sessionLedger = run.sessionId === null ? undefined : await findSessionLedgerPath(stateDir, run.sessionId);
+		if (receipt === undefined && sessionLedger === undefined) continue;
+		const list = out.get(run.cwd) ?? [];
+		list.push({
+			...(receipt === undefined ? {} : { receipt }),
+			...(sessionLedger === undefined ? {} : { sessionLedger }),
+		});
+		out.set(run.cwd, list);
+	}
+	return out;
+}
+
+function linkResult(
+	result: EvalRunRecord,
+	paths: LinkedRunPaths | undefined,
+	evidencePath: string | undefined,
+): EvalRunRecord {
+	if (paths === undefined && evidencePath === undefined) return result;
+	return {
+		...result,
+		...(paths?.receipt === undefined ? {} : { receiptPath: paths.receipt }),
+		paths: {
+			...(result.paths ?? {}),
+			...(paths?.receipt === undefined ? {} : { receipt: paths.receipt }),
+			...(paths?.sessionLedger === undefined ? {} : { sessionLedger: paths.sessionLedger }),
+			...(evidencePath === undefined ? {} : { evidence: evidencePath }),
+		},
+	};
+}
+
+async function findSessionLedgerPath(stateDir: string, sessionId: string): Promise<string | undefined> {
+	const root = join(stateDir, "sessions");
+	let cwdHashes: string[];
+	try {
+		cwdHashes = await readdir(root);
+	} catch {
+		return undefined;
+	}
+	for (const cwdHash of cwdHashes.sort(compareStrings)) {
+		const path = await existingPath(join(root, cwdHash, sessionId, "current.jsonl"));
+		if (path !== undefined) return path;
+	}
+	return undefined;
+}
+
+async function existingPath(path: string): Promise<string | undefined> {
+	try {
+		await access(path);
+		return path;
+	} catch {
+		return undefined;
+	}
+}
+
+function uniqueStrings(values: ReadonlyArray<string>): string[] {
+	return [...new Set(values)].sort(compareStrings);
+}
+
+function compareRunEnvelopeLike(a: RunEnvelopeLike, b: RunEnvelopeLike): number {
+	return compareStrings(a.startedAt, b.startedAt) || compareStrings(a.id, b.id);
+}
+
+function compareStrings(a: string, b: string): number {
+	return a.localeCompare(b);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function stringField(record: Record<string, unknown>, field: string): string | undefined {
+	const value = record[field];
+	return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function nullableStringField(record: Record<string, unknown>, field: string): string | null {
+	const value = record[field];
+	return typeof value === "string" && value.length > 0 ? value : null;
+}

package/src/domains/eval/schema/artifact.ts

@@ -0,0 +1,75 @@
+import type { EvalClioProvenance, EvalEnvironmentProvenance } from "../types.js";
+
+export interface EvalTokenMetricsV4 {
+	input: number;
+	output: number;
+	total: number;
+	cacheRead: number;
+	cacheWrite: number;
+}
+
+/**
+ * Whole-artifact token accounting. The counts exist only when at least one run
+ * reported provider usage, because a run whose Clio work happened out of the
+ * harness's sight has no count and a numeric zero would claim it cost nothing.
+ * `measuredRuns` of `runs` is the coverage the counts are true for.
+ */
+export type EvalTokenAccountingV4 =
+	| { measured: false; runs: number; measuredRuns: 0 }
+	| ({ measured: true; runs: number; measuredRuns: number } & EvalTokenMetricsV4);
+
+export interface EvalArtifactSummaryV4 {
+	runs: number;
+	passed: number;
+	failed: number;
+	passRate: number;
+	tokens: EvalTokenAccountingV4;
+	wallTimeMs: number;
+}
+
+/** Required explicit linkage; null means this runner did not evaluate a dispatch assignment. */
+export interface EvalArtifactAssignmentReference {
+	assignmentId: string | null;
+	terminalReceiptDigest: string | null;
+}
+
+export interface EvalArtifactResultV4 extends EvalArtifactAssignmentReference {
+	taskId: string;
+	repeatIndex: number;
+	target: {
+		id: string;
+		model: string | null;
+		thinking: string | null;
+	};
+	pass: boolean;
+	failureClass: string | null;
+	metrics: Record<string, number | string | boolean | null>;
+	artifacts: Record<string, string | string[] | null>;
+}
+
+/** The only current eval artifact format. Routing accepts this version only. */
+export interface EvalArtifactV4 {
+	version: 4;
+	evalId: string;
+	suite: {
+		id: string;
+		hash: string;
+	};
+	clio: EvalClioProvenance;
+	environment: EvalEnvironmentProvenance;
+	matrix: {
+		target: string;
+		model: string | null;
+		thinking: string | null;
+	};
+	summary: EvalArtifactSummaryV4;
+	results: EvalArtifactResultV4[];
+}
+
+export const ZERO_TOKEN_METRICS_V4: EvalTokenMetricsV4 = {
+	input: 0,
+	output: 0,
+	total: 0,
+	cacheRead: 0,
+	cacheWrite: 0,
+};