@iowarp/clio-coder 0.3.0 → 0.3.1

package/src/domains/eval/store.ts

@@ -0,0 +1,392 @@
+import { createHash, randomBytes } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { assertSafeId } from "../../core/safe-id.js";
+import { safeResourceWrite } from "../../core/safe-resource-write.js";
+import { redactArtifactForStorage } from "./artifacts/redact.js";
+import { evalHarnessMetricsFromCommands, ZERO_EVAL_HARNESS_METRICS } from "./harness-metrics.js";
+import type {
+	EvalArtifactPaths,
+	EvalClioProvenance,
+	EvalCommandResult,
+	EvalEnvironmentProvenance,
+	EvalFailureClass,
+	EvalFailureClassCount,
+	EvalHarnessMetrics,
+	EvalRunArtifact,
+	EvalRunPaths,
+	EvalRunRecord,
+	EvalSummary,
+} from "./types.js";
+
+const NODE_SIGNALS = new Set<string>([
+	"SIGHUP",
+	"SIGINT",
+	"SIGQUIT",
+	"SIGILL",
+	"SIGTRAP",
+	"SIGABRT",
+	"SIGIOT",
+	"SIGBUS",
+	"SIGFPE",
+	"SIGKILL",
+	"SIGUSR1",
+	"SIGSEGV",
+	"SIGUSR2",
+	"SIGPIPE",
+	"SIGALRM",
+	"SIGTERM",
+	"SIGCHLD",
+	"SIGSTKFLT",
+	"SIGCONT",
+	"SIGSTOP",
+	"SIGTSTP",
+	"SIGTTIN",
+	"SIGTTOU",
+	"SIGURG",
+	"SIGXCPU",
+	"SIGXFSZ",
+	"SIGVTALRM",
+	"SIGPROF",
+	"SIGWINCH",
+	"SIGIO",
+	"SIGPOLL",
+	"SIGPWR",
+	"SIGSYS",
+	"SIGUNUSED",
+]);
+
+export function evalRoot(dataDir: string): string {
+	return join(dataDir, "evals");
+}
+
+export function evalArtifactPath(dataDir: string, evalId: string): string {
+	assertSafeId(evalId, "eval");
+	return join(evalRoot(dataDir), `${evalId}.json`);
+}
+
+/**
+ * The stamp plus task-file hash is not unique: two workers starting the same
+ * task file in the same millisecond built the same id, and `evalArtifactPath`
+ * maps an id straight to one file, so one run clobbered the other. The random
+ * suffix is the repo idiom (state.ts newRunId, reservation-store, capacity-lease).
+ */
+export function createEvalId(startedAt: Date, taskFileHash: string): string {
+	const stamp = startedAt.toISOString().replace(/[-:.]/g, "");
+	return `eval-${stamp}-${taskFileHash.slice(0, 8)}-${randomBytes(6).toString("hex")}`;
+}
+
+export async function writeEvalArtifact(dataDir: string, artifact: EvalRunArtifact): Promise<string> {
+	// safeResourceWrite is sync, but the eval harness writes artifacts at run end
+	// in a CLI batch context, so a synchronous fsync there is fine. Delegating
+	// closes the real bug: the previous bare writeFile had no tmp+rename or fsync,
+	// so a crash mid-write left a torn ${evalId}.json that loadEvalArtifact rejects.
+	const path = evalArtifactPath(dataDir, artifact.evalId);
+	const redacted = redactArtifactForStorage({
+		...artifact,
+		paths: { ...artifact.paths, artifact: path },
+	});
+	safeResourceWrite(path, `${JSON.stringify(redacted, null, 2)}\n`, { encoding: "utf8" });
+	return path;
+}
+
+export async function loadEvalArtifact(dataDir: string, evalId: string): Promise<EvalRunArtifact> {
+	let raw: string;
+	try {
+		raw = await readFile(evalArtifactPath(dataDir, evalId), "utf8");
+	} catch (error) {
+		if (isErrorWithCode(error) && error.code === "ENOENT") throw new Error(`eval artifact not found: ${evalId}`);
+		throw error;
+	}
+	const source = evalArtifactPath(dataDir, evalId);
+	const parsed = parseJson(raw, source);
+	return parseArtifact(parsed, source);
+}
+
+export function taskFileHash(raw: string): string {
+	return createHash("sha256").update(raw, "utf8").digest("hex");
+}
+
+function parseArtifact(value: unknown, source: string): EvalRunArtifact {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	if (value.version !== 1) throw new Error(`${source}.version: expected 1`);
+	const results = readArray(value, source, "results").map((entry, index) =>
+		parseRecord(entry, `${source}.results[${index}]`),
+	);
+	return {
+		version: 1,
+		evalId: readString(value, source, "evalId"),
+		taskFile: readString(value, source, "taskFile"),
+		taskFileHash: readString(value, source, "taskFileHash"),
+		clio: parseClioProvenance(readRequiredArtifactField(value, source, "clio"), `${source}.clio`),
+		environment: parseEnvironmentProvenance(
+			readRequiredArtifactField(value, source, "environment"),
+			`${source}.environment`,
+		),
+		target: readOptionalNullableString(value, source, "target"),
+		model: readOptionalNullableString(value, source, "model"),
+		thinking: readOptionalNullableString(value, source, "thinking"),
+		paths: parseArtifactPaths(readRequiredArtifactField(value, source, "paths"), `${source}.paths`),
+		repeat: readNumber(value, source, "repeat"),
+		startedAt: readString(value, source, "startedAt"),
+		endedAt: readString(value, source, "endedAt"),
+		summary: parseSummary(value.summary, `${source}.summary`),
+		results,
+	};
+}
+
+function parseSummary(value: unknown, source: string): EvalSummary {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	return {
+		runs: readNumber(value, source, "runs"),
+		passed: readNumber(value, source, "passed"),
+		failed: readNumber(value, source, "failed"),
+		passRate: readNumber(value, source, "passRate"),
+		tokens: readNumber(value, source, "tokens"),
+		costUsd: readNumber(value, source, "costUsd"),
+		wallTimeMs: readNumber(value, source, "wallTimeMs"),
+		harness:
+			value.harness === undefined
+				? { ...ZERO_EVAL_HARNESS_METRICS }
+				: parseHarnessMetrics(value.harness, `${source}.harness`),
+		failureClasses: readArray(value, source, "failureClasses").map((entry, index) =>
+			parseFailureClassCount(entry, `${source}.failureClasses[${index}]`),
+		),
+	};
+}
+
+function parseRecord(value: unknown, source: string): EvalRunRecord {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	const failureClass = readOptionalFailureClass(value, source, "failureClass");
+	const commands = readArray(value, source, "commands").map((entry, index) =>
+		parseCommand(entry, `${source}.commands[${index}]`),
+	);
+	const record: EvalRunRecord = {
+		taskId: readString(value, source, "taskId"),
+		runId: readString(value, source, "runId"),
+		repeatIndex: readNumber(value, source, "repeatIndex"),
+		cwd: readString(value, source, "cwd"),
+		prompt: readString(value, source, "prompt"),
+		tags: readStringArray(value, source, "tags"),
+		pass: readBoolean(value, source, "pass"),
+		exitCode: readNumber(value, source, "exitCode"),
+		tokens: readNumber(value, source, "tokens"),
+		costUsd: readNumber(value, source, "costUsd"),
+		wallTimeMs: readNumber(value, source, "wallTimeMs"),
+		harness:
+			value.harness === undefined
+				? evalHarnessMetricsFromCommands(commands)
+				: parseHarnessMetrics(value.harness, `${source}.harness`),
+		commands,
+	};
+	if (failureClass !== undefined) record.failureClass = failureClass;
+	const receiptPath = readOptionalString(value, source, "receiptPath");
+	if (receiptPath !== undefined) record.receiptPath = receiptPath;
+	const evidenceId = readOptionalString(value, source, "evidenceId");
+	if (evidenceId !== undefined) record.evidenceId = evidenceId;
+	const paths = value.paths === undefined ? undefined : parseRunPaths(value.paths, `${source}.paths`);
+	if (paths !== undefined) record.paths = paths;
+	return record;
+}
+
+function parseClioProvenance(value: unknown, source: string): EvalClioProvenance {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	return {
+		version: readString(value, source, "version"),
+		commit: readNullableString(value, source, "commit"),
+		entry: readString(value, source, "entry"),
+	};
+}
+
+function parseEnvironmentProvenance(value: unknown, source: string): EvalEnvironmentProvenance {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	return {
+		platform: readString(value, source, "platform"),
+		node: readString(value, source, "node"),
+	};
+}
+
+function parseArtifactPaths(value: unknown, source: string): EvalArtifactPaths {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	const out: EvalArtifactPaths = {
+		taskFile: readString(value, source, "taskFile"),
+		receipts: readStringArray(value, source, "receipts"),
+		sessionLedgers: readStringArray(value, source, "sessionLedgers"),
+	};
+	const artifact = readOptionalString(value, source, "artifact");
+	if (artifact !== undefined) out.artifact = artifact;
+	const evidence = readOptionalString(value, source, "evidence");
+	if (evidence !== undefined) out.evidence = evidence;
+	return out;
+}
+
+function parseRunPaths(value: unknown, source: string): EvalRunPaths {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	const out: EvalRunPaths = {};
+	const receipt = readOptionalString(value, source, "receipt");
+	if (receipt !== undefined) out.receipt = receipt;
+	const sessionLedger = readOptionalString(value, source, "sessionLedger");
+	if (sessionLedger !== undefined) out.sessionLedger = sessionLedger;
+	const evidence = readOptionalString(value, source, "evidence");
+	if (evidence !== undefined) out.evidence = evidence;
+	return out;
+}
+
+function readRequiredArtifactField(
+	record: Record<string, unknown>,
+	source: string,
+	field: "clio" | "environment" | "paths",
+): unknown {
+	const value = record[field];
+	if (value === undefined) {
+		throw new Error(
+			`eval artifact has an invalid schema: missing required field '${field}' in ${source}. Re-run the evaluation suite to generate a complete artifact.`,
+		);
+	}
+	return value;
+}
+
+function parseHarnessMetrics(value: unknown, source: string): EvalHarnessMetrics {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	return {
+		receiptCount: readNumber(value, source, "receiptCount"),
+		toolCalls: readNumber(value, source, "toolCalls"),
+		retries: readNumber(value, source, "retries"),
+		safetyBlocks: readNumber(value, source, "safetyBlocks"),
+		correctionLatencyMs: readNumber(value, source, "correctionLatencyMs"),
+		validationEvidence: readNumber(value, source, "validationEvidence"),
+	};
+}
+
+function parseCommand(value: unknown, source: string): EvalCommandResult {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	const phase = readString(value, source, "phase");
+	if (phase !== "setup" && phase !== "verifier") throw new Error(`${source}.phase: expected setup or verifier`);
+	return {
+		phase,
+		index: readNumber(value, source, "index"),
+		command: readString(value, source, "command"),
+		exitCode: readNumber(value, source, "exitCode"),
+		signal: readSignal(value.signal, `${source}.signal`),
+		timedOut: readBoolean(value, source, "timedOut"),
+		wallTimeMs: readNumber(value, source, "wallTimeMs"),
+		stdout: readStringAllowEmpty(value, source, "stdout"),
+		stderr: readStringAllowEmpty(value, source, "stderr"),
+	};
+}
+
+function readSignal(value: unknown, source: string): NodeJS.Signals | null {
+	if (value === null) return null;
+	if (typeof value === "string" && NODE_SIGNALS.has(value)) return value as NodeJS.Signals;
+	throw new Error(`${source}: expected signal or null`);
+}
+
+function parseFailureClassCount(value: unknown, source: string): EvalFailureClassCount {
+	if (!isRecord(value)) throw new Error(`${source}: expected object`);
+	return {
+		failureClass: readFailureClass(value, source, "failureClass"),
+		count: readNumber(value, source, "count"),
+	};
+}
+
+function readFailureClass(record: Record<string, unknown>, source: string, field: string): EvalFailureClass {
+	const value = record[field];
+	if (
+		value === "setup_failed" ||
+		value === "verifier_failed" ||
+		value === "timeout" ||
+		value === "cwd_missing" ||
+		value === "command_error"
+	) {
+		return value;
+	}
+	throw new Error(`${source}.${field}: expected failure class`);
+}
+
+function readOptionalFailureClass(
+	record: Record<string, unknown>,
+	source: string,
+	field: string,
+): EvalFailureClass | undefined {
+	if (record[field] === undefined) return undefined;
+	return readFailureClass(record, source, field);
+}
+
+function readString(record: Record<string, unknown>, source: string, field: string): string {
+	const value = record[field];
+	if (typeof value !== "string" || value.length === 0) throw new Error(`${source}.${field}: expected string`);
+	return value;
+}
+
+function readStringAllowEmpty(record: Record<string, unknown>, source: string, field: string): string {
+	const value = record[field];
+	if (typeof value !== "string") throw new Error(`${source}.${field}: expected string`);
+	return value;
+}
+
+function readOptionalString(record: Record<string, unknown>, source: string, field: string): string | undefined {
+	const value = record[field];
+	if (value === undefined) return undefined;
+	if (typeof value !== "string" || value.length === 0) throw new Error(`${source}.${field}: expected string`);
+	return value;
+}
+
+function readNullableString(record: Record<string, unknown>, source: string, field: string): string | null {
+	const value = record[field];
+	if (value === null) return null;
+	if (typeof value !== "string" || value.length === 0) throw new Error(`${source}.${field}: expected string or null`);
+	return value;
+}
+
+function readOptionalNullableString(record: Record<string, unknown>, source: string, field: string): string | null {
+	const value = record[field];
+	if (value === undefined || value === null) return null;
+	if (typeof value !== "string" || value.length === 0) throw new Error(`${source}.${field}: expected string or null`);
+	return value;
+}
+
+function readStringArray(record: Record<string, unknown>, source: string, field: string): string[] {
+	const value = readArray(record, source, field);
+	const out: string[] = [];
+	for (let index = 0; index < value.length; index += 1) {
+		const entry = value[index];
+		if (typeof entry !== "string") throw new Error(`${source}.${field}[${index}]: expected string`);
+		out.push(entry);
+	}
+	return out;
+}
+
+function readNumber(record: Record<string, unknown>, source: string, field: string): number {
+	const value = record[field];
+	if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${source}.${field}: expected number`);
+	return value;
+}
+
+function readBoolean(record: Record<string, unknown>, source: string, field: string): boolean {
+	const value = record[field];
+	if (typeof value !== "boolean") throw new Error(`${source}.${field}: expected boolean`);
+	return value;
+}
+
+function readArray(record: Record<string, unknown>, source: string, field: string): unknown[] {
+	const value = record[field];
+	if (!Array.isArray(value)) throw new Error(`${source}.${field}: expected array`);
+	return value;
+}
+
+function parseJson(raw: string, source: string): unknown {
+	try {
+		return JSON.parse(raw) as unknown;
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		throw new Error(`${source}: invalid JSON: ${message}`);
+	}
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isErrorWithCode(error: unknown): error is Error & { code: string } {
+	return error instanceof Error && "code" in error && typeof (error as { code?: unknown }).code === "string";
+}

package/src/domains/eval/suites/load.ts

@@ -0,0 +1,74 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { parse as parseYaml } from "yaml";
+import type { EvalSuiteV2, LoadedEvalSuiteV2 } from "../schema/suite.js";
+import { type EvalValidationIssue, validateEvalSuiteV2 } from "../schema/validate.js";
+import { loadEvalTaskFile } from "../task-file.js";
+
+export class EvalSuiteFileError extends Error {
+	readonly issues: EvalValidationIssue[];
+
+	constructor(issues: ReadonlyArray<EvalValidationIssue>) {
+		super(`eval suite invalid (${issues.length} ${issues.length === 1 ? "issue" : "issues"})`);
+		this.name = "EvalSuiteFileError";
+		this.issues = [...issues];
+	}
+}
+
+export async function loadEvalSuiteFile(path: string): Promise<LoadedEvalSuiteV2> {
+	const resolved = resolve(path);
+	const raw = await readFile(resolved, "utf8");
+	let parsed: unknown;
+	try {
+		parsed = parseYaml(raw);
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		throw new EvalSuiteFileError([{ path: "$", message: `invalid YAML: ${message}` }]);
+	}
+	const result = validateEvalSuiteV2(parsed);
+	if (!result.valid) throw new EvalSuiteFileError(result.issues);
+	return {
+		path: resolved,
+		baseDir: dirname(resolved),
+		hash: sha256Hex(raw),
+		suite: result.suite,
+	};
+}
+
+export async function loadV1TaskFileAsSuite(path: string, repeatOverride?: number): Promise<LoadedEvalSuiteV2> {
+	const loaded = await loadEvalTaskFile(path);
+	const suite: EvalSuiteV2 = {
+		version: 2,
+		suite: {
+			id: "v1-task-file",
+			title: "v1 task file",
+			visibility: "local",
+			description: "Compatibility adapter for version 1 eval task files.",
+			provenance: { taskFileVersion: 1 },
+		},
+		matrix: {
+			targets: [{ id: "local" }],
+			repeats: repeatOverride ?? 1,
+		},
+		tasks: loaded.taskFile.tasks.map((task) => ({
+			id: task.id,
+			tags: task.tags,
+			workspace: { kind: "local", path: task.cwd },
+			runner: { kind: "external-command", commands: task.setup },
+			verify: { commands: task.verifier },
+			metrics: { collect: ["latency.wallMs", "verifier.exitCode"] },
+			timeoutMs: task.timeoutMs,
+		})),
+	};
+	return {
+		path: loaded.path,
+		baseDir: loaded.baseDir,
+		hash: loaded.contentHash,
+		suite,
+	};
+}
+
+function sha256Hex(content: string): string {
+	return createHash("sha256").update(content, "utf8").digest("hex");
+}

package/src/domains/eval/suites/matrix.ts

@@ -0,0 +1,19 @@
+import type { EvalSuiteTargetV2, EvalSuiteTaskV2, EvalSuiteV2 } from "../schema/suite.js";
+
+export interface EvalMatrixRun {
+	task: EvalSuiteTaskV2;
+	target: EvalSuiteTargetV2;
+	repeatIndex: number;
+}
+
+export function expandEvalMatrix(suite: EvalSuiteV2): EvalMatrixRun[] {
+	const runs: EvalMatrixRun[] = [];
+	for (let repeatIndex = 0; repeatIndex < suite.matrix.repeats; repeatIndex += 1) {
+		for (const target of suite.matrix.targets) {
+			for (const task of suite.tasks) {
+				runs.push({ task, target, repeatIndex });
+			}
+		}
+	}
+	return runs;
+}

package/src/domains/eval/suites/resolve.ts

@@ -0,0 +1,44 @@
+import type { EvalSuiteTargetV2, EvalSuiteV2 } from "../schema/suite.js";
+
+export interface ResolveSuiteOptions {
+	target?: string;
+	model?: string;
+}
+
+export function resolveSuiteForRun(suite: EvalSuiteV2, options: ResolveSuiteOptions): EvalSuiteV2 {
+	const targets = resolveTargets(suite.matrix.targets, options);
+	return {
+		...suite,
+		matrix: {
+			...suite.matrix,
+			targets,
+		},
+	};
+}
+
+export function artifactMatrixIdentity(targets: ReadonlyArray<EvalSuiteTargetV2>): {
+	target: string;
+	model: string | null;
+	thinking: string | null;
+} {
+	if (targets.length === 1) {
+		const target = targets[0];
+		return {
+			target: target?.id ?? "unknown",
+			model: target?.model ?? null,
+			thinking: target?.thinking ?? null,
+		};
+	}
+	return { target: "multiple", model: null, thinking: null };
+}
+
+function resolveTargets(targets: ReadonlyArray<EvalSuiteTargetV2>, options: ResolveSuiteOptions): EvalSuiteTargetV2[] {
+	const filtered =
+		options.target === undefined ? [...targets] : targets.filter((target) => target.id === options.target);
+	const selected =
+		filtered.length > 0 ? filtered : options.target === undefined ? [...targets] : [{ id: options.target }];
+	return selected.map((target) => ({
+		...target,
+		...(options.model === undefined ? {} : { model: options.model }),
+	}));
+}