@possumtech/rummy 2.2.0 → 2.2.1

package/audit/extract.js

@@ -1,445 +0,0 @@
-#!/usr/bin/env node
-/**
- * Extract per-task data from a sweep result-dir into:
- * - audit/sweep/<task>.md — one markdown file per task, template-shaped,
- *   with structured fields populated. Forensic "What would have rescued
- *   this run" + smoking-gun tags left for human fill.
- * - audit/SWEEP.md — index/dashboard table across all tasks with
- *   key fields for at-a-glance scanning.
- *
- * Usage: node audit/extract.js [<sweep-dir>]
- */
-import {
-	existsSync,
-	mkdirSync,
-	readFileSync,
-	readdirSync,
-	statSync,
-	writeFileSync,
-} from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-import { DatabaseSync } from "node:sqlite";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = join(__dirname, "..");
-const RESULTS = join(ROOT, "test/tbench/results");
-const OUT_SWEEP_DIR = join(__dirname, "sweep");
-const OUT_INDEX = join(__dirname, "SWEEP.md");
-mkdirSync(OUT_SWEEP_DIR, { recursive: true });
-
-const argDir = process.argv[2];
-const sweepDir = argDir
-	? argDir.startsWith("/")
-		? argDir
-		: join(ROOT, argDir)
-	: join(
-			RESULTS,
-			readdirSync(RESULTS)
-				.filter((d) => statSync(join(RESULTS, d)).isDirectory())
-				.sort()
-				.pop(),
-		);
-
-console.error(`[extract] sweep: ${sweepDir}`);
-
-function findTrialDirs(root) {
-	const trials = [];
-	function walk(dir) {
-		for (const name of readdirSync(dir)) {
-			const full = join(dir, name);
-			if (!statSync(full).isDirectory()) continue;
-			const agent = join(full, "agent");
-			if (existsSync(agent) && statSync(agent).isDirectory()) {
-				trials.push(full);
-				continue;
-			}
-			walk(full);
-		}
-	}
-	walk(root);
-	return trials;
-}
-
-function trialName(dir) {
-	const last = dir.split("/").pop();
-	return last.replace(/__[A-Za-z0-9]+$/, "");
-}
-
-function safeRead(path) {
-	try {
-		return readFileSync(path, "utf8");
-	} catch {
-		return null;
-	}
-}
-
-function parseRunSummary(rummyTxt) {
-	if (!rummyTxt) return null;
-	const m = rummyTxt.match(/__RUMMY_RUN_SUMMARY__\s+(\{.*\})\s*$/m);
-	if (!m) return null;
-	try {
-		return JSON.parse(m[1]);
-	} catch {
-		return null;
-	}
-}
-
-function verdictTrace(rummyTxt) {
-	if (!rummyTxt) return [];
-	const re = /\[LOOP\] \S+ iter=(\d+) verdict: continue=(\w+) status=(\S+) reason=(.*)/g;
-	const out = [];
-	let m;
-	while ((m = re.exec(rummyTxt))) {
-		out.push({
-			iter: Number(m[1]),
-			cont: m[2] === "true",
-			status: m[3],
-			reason: m[4].trim(),
-		});
-	}
-	return out;
-}
-
-function classifyTermination(summary, verdicts, agentDir) {
-	if (!summary) return "exfil-fail";
-	if (summary.status === 500) return "dispatch-crash";
-	if (summary.status === 499) {
-		if (summary.turn >= 99) return "max-loop-turns";
-		const last = verdicts[verdicts.length - 1];
-		if (last && /Abandoned after.*strikes/.test(last.reason)) return "max-strikes";
-		return "abandoned-other";
-	}
-	if (summary.status === 200) return "status-200";
-	return "unknown";
-}
-
-function readDb(dbPath) {
-	if (!existsSync(dbPath)) return null;
-	try {
-		return new DatabaseSync(dbPath, { readOnly: true });
-	} catch {
-		return null;
-	}
-}
-
-function updateSequence(db) {
-	if (!db) return [];
-	const rows = db
-		.prepare(
-			`SELECT path, body, attributes FROM entries WHERE path LIKE 'log://%/update/%' ORDER BY id`,
-		)
-		.all();
-	return rows.map((r) => {
-		const m = r.path.match(/^log:\/\/turn_(\d+)\//);
-		const turn = m ? Number(m[1]) : null;
-		let status = null;
-		try {
-			const attrs = JSON.parse(r.attributes ?? "{}");
-			status = attrs.status ?? null;
-		} catch {}
-		return { turn, status, body: r.body };
-	});
-}
-
-function compressUpdates(updates) {
-	const seq = [];
-	let last = null;
-	let count = 0;
-	for (const u of updates) {
-		const s = u.status;
-		if (s === last) {
-			count++;
-		} else {
-			if (last != null) seq.push(count > 1 ? `${last}×${count}` : `${last}`);
-			last = s;
-			count = 1;
-		}
-	}
-	if (last != null) seq.push(count > 1 ? `${last}×${count}` : `${last}`);
-	return seq.join(" → ");
-}
-
-function lastUpdate(updates) {
-	return updates[updates.length - 1] ?? null;
-}
-
-function categorizeUpdateBody(u) {
-	if (!u || !u.body) return "(absent)";
-	const b = u.body.toLowerCase();
-	if (u.status === 200) {
-		// Look for give-up signals first
-		if (
-			/no.*(?:found|recovered|matching|valid|solution)|couldn'?t|cannot|unable|exhaust|gave?.up|abandon|no.*viable/.test(
-				b,
-			)
-		) {
-			return "gave-up";
-		}
-		if (
-			/(verified|confirmed|passed|complet|success|works?\b|tested ok|matches|correct)/i.test(
-				u.body,
-			)
-		) {
-			return "confirmed-success";
-		}
-		return "asserted-success";
-	}
-	const stage = String(u.status);
-	if (/^1[4-7][4-7]$/.test(stage)) return "stage-transition";
-	if (/^17[0-9]$/.test(stage)) return "deployment-continuation";
-	return `update-${stage}`;
-}
-
-function confirmAuditFromUpdates(updates) {
-	// Did model emit any 177 (Deployment continuation) before reaching 200?
-	let saw177 = false;
-	let saw200 = false;
-	for (const u of updates) {
-		if (u.status === 177) saw177 = true;
-		if (u.status === 200) {
-			saw200 = true;
-			break;
-		}
-	}
-	if (!saw200) return "no-200";
-	if (saw177) return "has-confirm-step";
-	return "skipped-confirm";
-}
-
-function shCommandsBeforeFinalUpdate(db) {
-	if (!db) return [];
-	const rows = db
-		.prepare(
-			`SELECT e.path, e.body, e.attributes
-       FROM entries e
-       WHERE e.path LIKE 'log://%/sh/%' OR e.path LIKE 'log://%/env/%'
-       ORDER BY e.id`,
-		)
-		.all();
-	return rows.map((r) => {
-		let attrs = {};
-		try {
-			attrs = JSON.parse(r.attributes ?? "{}");
-		} catch {}
-		return {
-			path: r.path,
-			cmd: attrs.command ?? attrs.summary ?? "(no command attr)",
-			body: (r.body ?? "").slice(0, 200),
-		};
-	});
-}
-
-function verifierTail(trialDir) {
-	const v = safeRead(join(trialDir, "verifier", "test-stdout.txt"));
-	if (!v) return null;
-	const lines = v.trim().split("\n");
-	return lines.slice(-15).join("\n");
-}
-
-function verifierFailures(verifierText) {
-	if (!verifierText) return [];
-	return verifierText
-		.split("\n")
-		.filter((l) => /^FAILED|^E\s+|AssertionError/.test(l))
-		.slice(0, 10);
-}
-
-function escapeFenced(s) {
-	if (!s) return "";
-	return String(s).replace(/```/g, "ʼ``");
-}
-
-function renderTask(task) {
-	const {
-		name,
-		trialDir,
-		reward,
-		exitCode,
-		summary,
-		verdicts,
-		updates,
-		termination,
-		confirmAudit,
-		final,
-		finalCategory,
-		shCommands,
-		verifier,
-		verifierFailLines,
-	} = task;
-	const status = summary?.status ?? "(none)";
-	const turn = summary?.turn ?? "(none)";
-	const tokens = summary?.tokens ?? {};
-	const updateSeq = compressUpdates(updates);
-	// Last 3 sh/env commands the model ran (likely Confirm candidates)
-	const lastShEnv = shCommands.slice(-3);
-	return `# ${name}
-
-## Identity
-
-- Task slug: \`${name}\`
-- Run alias: \`${summary?.run ?? "(no summary)"}\`
-- Trial dir: \`${trialDir.replace(ROOT + "/", "")}\`
-- Reward: \`${reward ?? "?"}\`
-- Run status: \`${status}\`
-- Turn count: \`${turn}\`
-- Exit code: \`${exitCode ?? "?"}\`
-- Tokens: prompt=${tokens.prompt ?? "?"} cached=${tokens.cached ?? "?"} completion=${tokens.completion ?? "?"} total=${tokens.total ?? "?"}
-
-## Termination mode
-
-\`${termination}\`
-
-${
-	termination === "status-200" && reward === 0
-		? `Sub-tag (best-effort heuristic): \`${
-				finalCategory === "gave-up"
-					? "gave-up-as-200"
-					: confirmAudit === "skipped-confirm"
-						? "skipped-confirm"
-						: "claim-success-fail"
-			}\` — verify by reading body + sh commands below.`
-		: ""
-}
-
-## FCRM walk
-
-Update status sequence: \`${updateSeq || "(no updates)"}\`
-
-- Reached Deployment (177): ${updates.some((u) => u.status === 177) ? "yes" : "no"}
-- Reached Complete (200): ${updates.some((u) => u.status === 200) ? "yes" : "no"}
-- Confirm-step audit: \`${confirmAudit}\`
-
-## Confirm-step audit
-
-Last 3 \`<sh>\`/\`<env>\` commands the model ran (Confirm candidates):
-
-${
-	lastShEnv.length === 0
-		? "(none — model emitted no shell commands)"
-		: lastShEnv
-				.map(
-					(s, i) =>
-						`${i + 1}. \`${s.cmd.slice(0, 200).replace(/\n/g, " ")}\`\n   body: \`${escapeFenced(s.body).replace(/\n/g, " ").slice(0, 160)}\``,
-				)
-				.join("\n")
-}
-
-## Final update body
-
-Status \`${final?.status ?? "(absent)"}\` body:
-
-\`\`\`
-${escapeFenced(final?.body ?? "(absent)").slice(0, 500)}
-\`\`\`
-
-Categorization (heuristic): \`${finalCategory}\`
-
-## Verifier output
-
-${
-	verifier
-		? `\`\`\`
-${escapeFenced(verifier)}
-\`\`\`
-
-Top failures:
-${verifierFailLines.map((l) => `- \`${escapeFenced(l).slice(0, 160)}\``).join("\n") || "- (none parsed)"}`
-		: "(no verifier output captured)"
-}
-
-## What would have rescued this run
-
-> **TODO** — fill manually per \`feedback_forensic_burden_of_proof\`.
-> Either: a concrete harness-side change (prompt phrasing, protocol gap,
-> tool ergonomic, context-assembly fix), OR explicit "model-capability
-> bottleneck, exhausted alternatives" with concrete list of harness
-> rescues already considered and ruled out.
-
-## Smoking-gun candidate tags
-
-> **TODO** — assign cross-task pattern tags after reading the entry.
-> Aggregate in \`audit/SWEEPX.md\`.
-
-## Notes
-
-> **TODO** — observations not captured by the structured fields.
-`;
-}
-
-const trials = findTrialDirs(sweepDir);
-console.error(`[extract] found ${trials.length} trials`);
-
-const tasks = [];
-for (const trialDir of trials) {
-	const name = trialName(trialDir);
-	const agentDir = join(trialDir, "agent");
-	const rewardStr = safeRead(join(trialDir, "verifier", "reward.txt"));
-	const reward = rewardStr != null ? Number(rewardStr.trim()) : null;
-	const exitCode = safeRead(join(agentDir, "rummy_exit_code"))?.trim() ?? null;
-	const rummyTxt = safeRead(join(agentDir, "rummy.txt"));
-	const summary = parseRunSummary(rummyTxt);
-	const verdicts = verdictTrace(rummyTxt);
-	const db = readDb(join(agentDir, "rummy.db"));
-	const updates = updateSequence(db);
-	const final = lastUpdate(updates);
-	const finalCategory = categorizeUpdateBody(final);
-	const confirmAudit = confirmAuditFromUpdates(updates);
-	const shCommands = shCommandsBeforeFinalUpdate(db);
-	const termination = classifyTermination(summary, verdicts, agentDir);
-	const verifier = verifierTail(trialDir);
-	const verifierFailLines = verifierFailures(verifier);
-	if (db) db.close();
-
-	const task = {
-		name,
-		trialDir,
-		reward,
-		exitCode,
-		summary,
-		verdicts,
-		updates,
-		final,
-		finalCategory,
-		confirmAudit,
-		shCommands,
-		termination,
-		verifier,
-		verifierFailLines,
-	};
-	tasks.push(task);
-
-	const md = renderTask(task);
-	writeFileSync(join(OUT_SWEEP_DIR, `${name}.md`), md);
-}
-
-// Index
-tasks.sort((a, b) => a.name.localeCompare(b.name));
-let counts = {};
-for (const t of tasks) counts[t.termination] = (counts[t.termination] ?? 0) + 1;
-
-const indexLines = [];
-indexLines.push("# SWEEP.md\n");
-indexLines.push(
-	`Index for sweep \`${sweepDir.replace(ROOT + "/", "")}\`. Per-task entries in \`audit/sweep/<task>.md\`.\n`,
-);
-indexLines.push("## Termination mode counts\n");
-for (const [k, v] of Object.entries(counts).sort()) {
-	indexLines.push(`- \`${k}\`: ${v}`);
-}
-indexLines.push("");
-indexLines.push("## Per-task table\n");
-indexLines.push(
-	"| Task | Reward | Status | Turns | Term | Confirm | Final | Final body category |",
-);
-indexLines.push("|---|---|---|---|---|---|---|---|");
-for (const t of tasks) {
-	indexLines.push(
-		`| [${t.name}](sweep/${t.name}.md) | ${t.reward ?? "?"} | ${t.summary?.status ?? "—"} | ${t.summary?.turn ?? "—"} | ${t.termination} | ${t.confirmAudit} | ${t.final?.status ?? "—"} | ${t.finalCategory} |`,
-	);
-}
-writeFileSync(OUT_INDEX, `${indexLines.join("\n")}\n`);
-
-console.error(
-	`[extract] wrote ${tasks.length} entries to audit/sweep/, index to audit/SWEEP.md`,
-);

package/audit/sweep/1_build-pov-ray/deliverables.md

@@ -1,19 +0,0 @@
-# deliverables.md
-
-Final state of each file the model wrote, deduplicated to the last version. Direct from `<set>` entries with `attrs.path` set to a bare file path (not a URI scheme).
-
-## `/app/povray-2.2/source/unix.c`
-
-*Last written turn 37; first written turn 28.*
-
-```c
-(empty body)
-```
-
-## `/app/povray-2.2/source/unix.mak`
-
-*Last written turn 37; first written turn 30.*
-
-```mak
-(empty body)
-```