@possumtech/rummy 2.2.0 → 2.2.1

package/audit/build.js

@@ -1,528 +0,0 @@
-#!/usr/bin/env node
-/**
- * Rebuild per-task audit folders deterministically from rummy.db + task source.
- *
- * Layout per task:
- *   audit/sweep/<task>/
- *     summary.md       (interpretive — preserved if exists)
- *     prompt.md        (instruction.md from harbor cache)
- *     verifier.md      (test_outputs.py source[s] + verifier stdout)
- *     logs.md          (index only, no full bodies)
- *     deliverables.md  (final file contents from <set> entries)
- *     timing.md        (per-turn token + status)
- *     tools.md         (chronological sh/env commands)
- *     searches.md      (chronological search/get)
- *     turns/turn_NNN.md (reasoning + emission + actions + update)
- *
- * Usage: node audit/build.js [<sweep-dir>] [<harbor-cache-dir>]
- *
- * Idempotent — re-run rebuilds all derived files. summary.md preserved.
- */
-import {
-	cpSync,
-	existsSync,
-	mkdirSync,
-	readFileSync,
-	readdirSync,
-	renameSync,
-	rmSync,
-	statSync,
-	writeFileSync,
-} from "node:fs";
-import { homedir } from "node:os";
-import { dirname, join } from "node:path";
-import { DatabaseSync } from "node:sqlite";
-import { fileURLToPath } from "node:url";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = join(__dirname, "..");
-const RESULTS = join(ROOT, "test/tbench/results");
-const HARBOR_CACHE = join(homedir(), ".cache/harbor/tasks");
-const OUT_SWEEP_DIR = join(__dirname, "sweep");
-
-const sweepArg = process.argv[2];
-const harborArg = process.argv[3] ?? HARBOR_CACHE;
-
-const sweepDir = sweepArg
-	? sweepArg.startsWith("/")
-		? sweepArg
-		: join(ROOT, sweepArg)
-	: join(
-			RESULTS,
-			readdirSync(RESULTS)
-				.filter((d) => statSync(join(RESULTS, d)).isDirectory())
-				.sort()
-				.pop(),
-		);
-
-console.error(`[build] sweep:  ${sweepDir}`);
-console.error(`[build] harbor: ${harborArg}`);
-
-// --- helpers ---
-
-function safeRead(path) {
-	try {
-		return readFileSync(path, "utf8");
-	} catch {
-		return null;
-	}
-}
-
-function findTrialDirs(root) {
-	const trials = [];
-	function walk(dir) {
-		for (const name of readdirSync(dir)) {
-			const full = join(dir, name);
-			if (!statSync(full).isDirectory()) continue;
-			if (existsSync(join(full, "agent"))) {
-				trials.push(full);
-				continue;
-			}
-			walk(full);
-		}
-	}
-	walk(root);
-	return trials;
-}
-
-function trialName(dir) {
-	const last = dir.split("/").pop();
-	return last.replace(/__[A-Za-z0-9]+$/, "");
-}
-
-function findHarborTaskDir(taskName) {
-	if (!existsSync(harborArg)) return null;
-	for (const id of readdirSync(harborArg)) {
-		const candidate = join(harborArg, id, taskName);
-		if (existsSync(candidate)) return candidate;
-		// task names may contain dots that get sanitized to dashes
-		const altName = taskName.replace(/-/g, ".");
-		const altCandidate = join(harborArg, id, altName);
-		if (existsSync(altCandidate)) return altCandidate;
-	}
-	return null;
-}
-
-function parseAttrs(s) {
-	if (!s) return {};
-	try {
-		return JSON.parse(s);
-	} catch {
-		return {};
-	}
-}
-
-function turnFromPath(path) {
-	const m = path.match(/^log:\/\/turn_(\d+)\//);
-	return m ? Number(m[1]) : null;
-}
-
-function actionFromPath(path) {
-	const m = path.match(/^log:\/\/turn_\d+\/([^/]+)\//);
-	return m ? m[1] : null;
-}
-
-function indexFromPath(path, prefix) {
-	const m = path.match(new RegExp(`^${prefix}://(\\d+)`));
-	return m ? Number(m[1]) : null;
-}
-
-function pad(n, w = 3) {
-	return String(n).padStart(w, "0");
-}
-
-function fence(s, lang = "") {
-	if (s == null) return "(empty)";
-	const safe = String(s).replace(/```/g, "ʼ``");
-	return `\`\`\`${lang}\n${safe}\n\`\`\``;
-}
-
-function trim80(s) {
-	if (!s) return "";
-	const oneLine = String(s).replace(/\s+/g, " ").trim();
-	return oneLine.length > 80 ? `${oneLine.slice(0, 77)}…` : oneLine;
-}
-
-// --- per-task build ---
-
-function buildTask(trialDir) {
-	const name = trialName(trialDir);
-	const taskDir = join(OUT_SWEEP_DIR, name);
-	const turnsDir = join(taskDir, "turns");
-
-	// Migration: if a flat <task>.md exists, move it into the folder as summary.md
-	const flatMd = join(OUT_SWEEP_DIR, `${name}.md`);
-	const summaryPath = join(taskDir, "summary.md");
-	if (existsSync(flatMd) && !existsSync(summaryPath)) {
-		mkdirSync(taskDir, { recursive: true });
-		renameSync(flatMd, summaryPath);
-	}
-
-	mkdirSync(taskDir, { recursive: true });
-	// Wipe + recreate derived files (preserve summary.md)
-	for (const f of ["prompt.md", "verifier.md", "logs.md", "deliverables.md", "timing.md", "tools.md", "searches.md"]) {
-		const p = join(taskDir, f);
-		if (existsSync(p)) rmSync(p);
-	}
-	if (existsSync(turnsDir)) rmSync(turnsDir, { recursive: true });
-	mkdirSync(turnsDir, { recursive: true });
-
-	const agentDir = join(trialDir, "agent");
-	const dbPath = join(agentDir, "rummy.db");
-	if (!existsSync(dbPath)) {
-		writeFileSync(
-			join(taskDir, "logs.md"),
-			`# logs.md\n\nNo rummy.db found for this trial — likely exfil-failed before drain completed.\n`,
-		);
-		return { name, status: "no-db" };
-	}
-
-	const db = new DatabaseSync(dbPath, { readOnly: true });
-
-	// --- prompt.md: instruction.md from harbor task cache ---
-	const taskSrc = findHarborTaskDir(name);
-	if (taskSrc) {
-		const inst = safeRead(join(taskSrc, "instruction.md"));
-		writeFileSync(
-			join(taskDir, "prompt.md"),
-			`# prompt.md\n\nSource: \`${taskSrc.replace(homedir(), "~")}/instruction.md\`\n\n${inst ? `\n${inst}\n` : "(instruction.md not found)\n"}`,
-		);
-	} else {
-		writeFileSync(
-			join(taskDir, "prompt.md"),
-			`# prompt.md\n\nHarbor cache for task \`${name}\` not found at \`${harborArg}\`.\n`,
-		);
-	}
-
-	// --- verifier.md: test_outputs.py source(s) + verifier stdout ---
-	{
-		const parts = [`# verifier.md\n`];
-		if (taskSrc) {
-			const testsTo = safeRead(join(taskSrc, "tests/test_outputs.py"));
-			parts.push(`## /tests/test_outputs.py (the actual verifier)\n\nSource: \`${taskSrc.replace(homedir(), "~")}/tests/test_outputs.py\`\n\n${fence(testsTo, "python")}\n`);
-			// If task ships a copy to /app, surface it
-			const envTo = safeRead(join(taskSrc, "environment/test_outputs.py"));
-			const envTestsTo = safeRead(join(taskSrc, "environment/tests/test_outputs.py"));
-			const dockerfile = safeRead(join(taskSrc, "environment/Dockerfile"));
-			let appCopyNote = "";
-			if (dockerfile && /COPY\s+(?:tests\/)?test_outputs\.py\s+\/app/.test(dockerfile)) {
-				appCopyNote = "\n\n*Note: this task's Dockerfile copies test_outputs.py to /app — agent has read access during the run by task design.*\n";
-			}
-			parts.push(`## /app/test_outputs.py (visible to agent if shipped)\n${appCopyNote}\n${envTo || envTestsTo ? fence(envTo || envTestsTo, "python") : "(not shipped to /app for this task — only /tests has it)"}\n`);
-		} else {
-			parts.push("(harbor task source not found — verifier source unavailable)\n");
-		}
-		const stdout = safeRead(join(trialDir, "verifier/test-stdout.txt"));
-		parts.push(`## verifier stdout (captured from this run)\n\n${fence(stdout || "(empty)")}\n`);
-		const reward = safeRead(join(trialDir, "verifier/reward.txt"));
-		parts.push(`## reward.txt\n\n\`${(reward || "").trim() || "(empty)"}\`\n`);
-		writeFileSync(join(taskDir, "verifier.md"), parts.join("\n"));
-	}
-
-	// --- pull all relevant entries once ---
-	const allEntries = db
-		.prepare(
-			`SELECT id, scope, path, body, attributes
-       FROM entries
-       ORDER BY id`,
-		)
-		.all();
-	const turnsRows = db
-		.prepare(
-			`SELECT id, sequence, prompt_tokens, cached_tokens, completion_tokens, reasoning_tokens, total_tokens, cost, reasoning_content, created_at
-       FROM turns
-       ORDER BY sequence`,
-		)
-		.all();
-	db.close();
-
-	// Group entries by scheme/turn
-	const byScheme = {};
-	for (const e of allEntries) {
-		const m = e.path.match(/^([a-z][a-z0-9_]*):\/\//);
-		const scheme = m ? m[1] : "(none)";
-		if (!byScheme[scheme]) byScheme[scheme] = [];
-		byScheme[scheme].push(e);
-	}
-
-	const turnEntries = {};
-	for (const e of allEntries) {
-		const t = turnFromPath(e.path);
-		if (t == null) continue;
-		if (!turnEntries[t]) turnEntries[t] = [];
-		turnEntries[t].push(e);
-	}
-
-	const reasoningByN = {};
-	for (const e of byScheme.reasoning ?? []) {
-		const n = indexFromPath(e.path, "reasoning");
-		if (n != null) reasoningByN[n] = e;
-	}
-	const assistantByN = {};
-	for (const e of byScheme.assistant ?? []) {
-		const n = indexFromPath(e.path, "assistant");
-		if (n != null) assistantByN[n] = e;
-	}
-	const modelByN = {};
-	for (const e of byScheme.model ?? []) {
-		const n = indexFromPath(e.path, "model");
-		if (n != null) modelByN[n] = e;
-	}
-
-	// --- logs.md: chronological INDEX only ---
-	{
-		const lines = [
-			`# logs.md\n`,
-			`Chronological index of run entries — paths and brief titles only.\n`,
-			`Full bodies live in \`turns/turn_NNN.md\`. Verifier source in \`verifier.md\`. Files the model wrote in \`deliverables.md\`. Shell commands in \`tools.md\`. Web research in \`searches.md\`.\n`,
-			"",
-			"| id | path | title |",
-			"| --- | --- | --- |",
-		];
-		for (const e of allEntries) {
-			const attrs = parseAttrs(e.attributes);
-			let title = "";
-			const m = e.path.match(/^([a-z][a-z0-9_]*):\/\/(.*)$/);
-			const scheme = m ? m[1] : "(none)";
-			if (scheme === "log") {
-				const action = actionFromPath(e.path);
-				if (action === "update") {
-					title = `[update status=${attrs.status ?? "?"}] ${trim80(e.body)}`;
-				} else if (action === "error") {
-					title = `[error status=${attrs.status ?? "?"}] ${trim80(e.body)}`;
-				} else if (action === "sh" || action === "env") {
-					const cmd = attrs.command ?? attrs.summary ?? "(no command)";
-					title = `[${action}] ${trim80(cmd)}`;
-				} else if (action === "search") {
-					const q = attrs.query ?? attrs.summary ?? "(no query)";
-					title = `[search] ${trim80(q)}`;
-				} else if (action === "get") {
-					title = `[get] ${trim80(attrs.path ?? attrs.url ?? e.path.replace(/^log:\/\/turn_\d+\/get\//, ""))}`;
-				} else if (action === "set") {
-					title = `[set] ${trim80(attrs.path ?? "(no target)")}`;
-				} else if (action === "cp" || action === "mv") {
-					title = `[${action}] ${trim80(`${attrs.from} → ${attrs.to}`)}`;
-				} else if (action === "rm") {
-					title = `[rm] ${trim80(attrs.path ?? "(no target)")}`;
-				} else {
-					title = `[${action ?? "log"}]`;
-				}
-			} else if (scheme === "prompt") {
-				title = `[prompt] ${trim80(e.body)}`;
-			} else if (scheme === "reasoning") {
-				title = `[think] ${trim80(e.body)}`;
-			} else if (scheme === "assistant") {
-				title = `[assistant emission — see turns/]`;
-			} else if (scheme === "system") {
-				title = `[system prompt]`;
-			} else if (scheme === "user") {
-				title = `[user packet (assembled context)]`;
-			} else if (scheme === "model") {
-				title = `[model usage record]`;
-			} else if (scheme === "known") {
-				title = `[known] ${trim80(e.body)}`;
-			} else if (scheme === "unknown") {
-				title = `[unknown] ${trim80(e.body)}`;
-			} else {
-				title = `[${scheme}]`;
-			}
-			lines.push(`| ${e.id} | \`${e.path}\` | ${title.replace(/\|/g, "\\|")} |`);
-		}
-		writeFileSync(join(taskDir, "logs.md"), `${lines.join("\n")}\n`);
-	}
-
-	// --- deliverables.md: final content of each <set> bare-file path ---
-	{
-		// Pick latest body for each unique target file from set log entries
-		const fileBody = new Map();
-		const fileFirst = new Map();
-		for (const e of byScheme.log ?? []) {
-			if (!/^log:\/\/turn_\d+\/set\//.test(e.path)) continue;
-			const attrs = parseAttrs(e.attributes);
-			const target = attrs.path;
-			if (!target) continue;
-			if (target.includes("://")) continue; // scheme writes (known://, etc.) — separate
-			const turn = turnFromPath(e.path);
-			fileBody.set(target, { body: e.body, turn, merge: attrs.merge ?? null });
-			if (!fileFirst.has(target)) fileFirst.set(target, turn);
-		}
-		const lines = [
-			"# deliverables.md\n",
-			"Final state of each file the model wrote, deduplicated to the last version. Direct from `<set>` entries with `attrs.path` set to a bare file path (not a URI scheme).\n",
-		];
-		if (fileBody.size === 0) {
-			lines.push("(no file deliverables — model wrote no bare-path files via `<set>`)\n");
-		} else {
-			const sorted = [...fileBody.entries()].sort((a, b) => a[0].localeCompare(b[0]));
-			for (const [target, info] of sorted) {
-				const lang = target.match(/\.(\w+)$/)?.[1] ?? "";
-				lines.push(`## \`${target}\`\n`);
-				lines.push(`*Last written turn ${info.turn}; first written turn ${fileFirst.get(target)}.*\n`);
-				lines.push(fence(info.body || "(empty body)", lang));
-				if (info.merge) {
-					lines.push(`*This entry also carries an \`attrs.merge\` (SEARCH/REPLACE block applied to the prior file content) — body shown above is what the materializer would write.*\n`);
-				}
-				lines.push("");
-			}
-		}
-		writeFileSync(join(taskDir, "deliverables.md"), lines.join("\n"));
-	}
-
-	// --- timing.md: per-turn token + commands ---
-	{
-		const lines = [
-			"# timing.md\n",
-			"Per-turn telemetry from `turns` table.\n",
-			"",
-			"| turn | prompt | cached | completion | reasoning | total | cost | created_at |",
-			"|---:|---:|---:|---:|---:|---:|---:|---|",
-		];
-		for (const t of turnsRows) {
-			lines.push(
-				`| ${t.sequence} | ${t.prompt_tokens} | ${t.cached_tokens} | ${t.completion_tokens} | ${t.reasoning_tokens} | ${t.total_tokens} | ${t.cost} | ${t.created_at} |`,
-			);
-		}
-		writeFileSync(join(taskDir, "timing.md"), `${lines.join("\n")}\n`);
-	}
-
-	// --- tools.md: chronological sh/env ---
-	{
-		const lines = [
-			"# tools.md\n",
-			"Chronological `<sh>` and `<env>` commands the model emitted, with exit codes (when known).\n",
-			"",
-			"| turn | tool | command | exit |",
-			"|---:|---|---|---|",
-		];
-		// Get exits from run_views.outcome — but that table wasn't fetched; pull from log body which often has 'exit=N'
-		for (const e of byScheme.log ?? []) {
-			const action = actionFromPath(e.path);
-			if (action !== "sh" && action !== "env") continue;
-			// Skip the data:// per-channel children — keep only the proposal/log entry
-			if (/_(\d+)$/.test(e.path)) continue;
-			const turn = turnFromPath(e.path);
-			const attrs = parseAttrs(e.attributes);
-			const cmd = (attrs.command ?? attrs.summary ?? "").replace(/\n/g, " ").slice(0, 200);
-			const exitMatch = (e.body ?? "").match(/exit=(\d+)/);
-			const exit = exitMatch ? exitMatch[1] : "?";
-			lines.push(`| ${turn} | ${action} | \`${cmd.replace(/\|/g, "\\|").replace(/`/g, "ʼ")}\` | ${exit} |`);
-		}
-		writeFileSync(join(taskDir, "tools.md"), `${lines.join("\n")}\n`);
-	}
-
-	// --- searches.md: chronological search/get ---
-	{
-		const lines = [
-			"# searches.md\n",
-			"Chronological `<search>` queries and `<get https://...>` URL fetches.\n",
-			"",
-			"| turn | scheme | target |",
-			"|---:|---|---|",
-		];
-		for (const e of byScheme.log ?? []) {
-			const action = actionFromPath(e.path);
-			if (action !== "search" && action !== "get") continue;
-			if (/_(\d+)$/.test(e.path)) continue;
-			const turn = turnFromPath(e.path);
-			const attrs = parseAttrs(e.attributes);
-			let target;
-			if (action === "search") {
-				target = attrs.query ?? attrs.summary ?? "(no query attribute)";
-			} else {
-				const p = attrs.path ?? "";
-				if (!/^https?:\/\//.test(p)) continue;
-				target = p;
-			}
-			lines.push(`| ${turn} | ${action} | \`${target.replace(/\|/g, "\\|")}\` |`);
-		}
-		// If no search/get were emitted at all, note it explicitly
-		const tableHasRows = lines.length > 5;
-		if (!tableHasRows) {
-			lines.push("| — | — | (model emitted no `<search>` or `<get https://...>` during this run) |");
-		}
-		writeFileSync(join(taskDir, "searches.md"), `${lines.join("\n")}\n`);
-	}
-
-	// --- turns/turn_NNN.md ---
-	{
-		const turnNumbers = new Set([
-			...Object.keys(reasoningByN).map(Number),
-			...Object.keys(assistantByN).map(Number),
-			...Object.keys(turnEntries).map(Number),
-			...turnsRows.map((t) => t.sequence),
-		]);
-		for (const n of [...turnNumbers].sort((a, b) => a - b)) {
-			const turnRow = turnsRows.find((t) => t.sequence === n);
-			const reasoning = reasoningByN[n];
-			const assistant = assistantByN[n];
-			const turnLogs = (turnEntries[n] ?? []).slice().sort((a, b) => a.id - b.id);
-
-			const updateEntry = turnLogs.find((e) => /\/update\//.test(e.path));
-			const updateAttrs = updateEntry ? parseAttrs(updateEntry.attributes) : null;
-
-			const parts = [];
-			parts.push(`# turn ${pad(n)}\n`);
-			if (turnRow) {
-				parts.push(
-					`tokens: prompt=${turnRow.prompt_tokens} cached=${turnRow.cached_tokens} completion=${turnRow.completion_tokens} reasoning=${turnRow.reasoning_tokens} total=${turnRow.total_tokens} cost=${turnRow.cost}`,
-				);
-				parts.push(`created: ${turnRow.created_at}\n`);
-			}
-			if (updateEntry) {
-				parts.push(`update: status=${updateAttrs?.status ?? "?"} body=${trim80(updateEntry.body)}\n`);
-			}
-			parts.push("---\n");
-			parts.push("## reasoning (think)\n");
-			parts.push(reasoning ? fence(reasoning.body) : "(no reasoning entry for this turn)");
-			parts.push("\n## assistant emission\n");
-			parts.push(assistant ? fence(assistant.body) : "(no assistant entry for this turn)");
-			parts.push("\n## actions (chronological log entries)\n");
-			if (turnLogs.length === 0) {
-				parts.push("(none)");
-			} else {
-				for (const e of turnLogs) {
-					const action = actionFromPath(e.path);
-					const attrs = parseAttrs(e.attributes);
-					parts.push(`### \`${e.path}\``);
-					parts.push(`scheme=log action=${action} attrs=${JSON.stringify(attrs)}\n`);
-					parts.push(fence(e.body || "(empty body)"));
-					parts.push("");
-				}
-			}
-			writeFileSync(join(turnsDir, `turn_${pad(n)}.md`), `${parts.join("\n")}\n`);
-		}
-	}
-
-	return { name, status: "ok", turns: turnsRows.length };
-}
-
-// --- main ---
-
-const trials = findTrialDirs(sweepDir);
-console.error(`[build] ${trials.length} trials`);
-
-let okCount = 0;
-let nodbCount = 0;
-for (const trialDir of trials) {
-	const r = buildTask(trialDir);
-	if (r.status === "ok") okCount++;
-	else if (r.status === "no-db") nodbCount++;
-}
-
-// Move any orphan flat .md files (tasks without a trial — shouldn't happen but
-// keep the migration tidy)
-for (const f of readdirSync(OUT_SWEEP_DIR)) {
-	if (f.endsWith(".md") && !["SWEEP_TEMPLATE.md"].includes(f)) {
-		const p = join(OUT_SWEEP_DIR, f);
-		if (statSync(p).isFile()) {
-			const taskName = f.replace(/\.md$/, "");
-			const taskDir = join(OUT_SWEEP_DIR, taskName);
-			mkdirSync(taskDir, { recursive: true });
-			const dest = join(taskDir, "summary.md");
-			if (!existsSync(dest)) renameSync(p, dest);
-		}
-	}
-}
-
-console.error(`[build] ok=${okCount} no-db=${nodbCount}`);
-console.error(`[build] output: ${OUT_SWEEP_DIR}`);