@leostera/evalkit 0.10.0 → 0.11.0

package/dist/dashboard/index.html

@@ -4,8 +4,8 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>EvalKit</title>
-    <script type="module" crossorigin src="/assets/index-DNR4shrM.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-TPYprg75.css">
+    <script type="module" crossorigin src="/assets/index-DgoX2x_M.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-CFJzcw3o.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/{index-db-B1FeXzmS.mjs → index-db-DI7xKMSX.mjs}

@@ -1,4 +1,5 @@
-import { a as upgradeManifest, i as upgradeEvent, o as upgradeStatus, r as upgradeCatalog } from "./upgrade-G-uFPJKg.mjs";
+import { a as upgradeManifest, i as upgradeEvent, o as upgradeStatus, r as upgradeCatalog } from "./upgrade-rMsGzUEJ.mjs";
+import { n as upgradeScoring, t as convertTrajectory } from "./upgrade-transcript-C8DUykLd.mjs";
 import { Hono } from "hono";
 //#region ../share/src/bundle.ts
 /**
@@ -82,40 +83,40 @@ function escapeAttr(value) {
 }
 //#endregion
 //#region ../share/src/legacy.ts
-const decoder = new TextDecoder();
-const encoder = new TextEncoder();
-const parse = (data) => data ? JSON.parse(decoder.decode(data)) : void 0;
-const encode = (value) => encoder.encode(`${JSON.stringify(value)}\n`);
-const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+const decoder$1 = new TextDecoder();
+const encoder$1 = new TextEncoder();
+const parse$1 = (data) => data ? JSON.parse(decoder$1.decode(data)) : void 0;
+const encode$1 = (value) => encoder$1.encode(`${JSON.stringify(value)}\n`);
+const isRecord$1 = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
 /** True for a version 1 run.json: no `snapshotVersion`. */
 function isLegacyRun(runJson) {
-	return isRecord(runJson) && runJson.snapshotVersion === void 0;
+	return isRecord$1(runJson) && runJson.snapshotVersion === void 0;
 }
 function upgradeSnapshotRun(files) {
-	const run = parse(files.get("run.json"));
-	if (!isRecord(run)) throw new Error("run.json must be an object");
-	const listed = parse(files.get("trials.json"));
-	const trials = (isRecord(listed) && Array.isArray(listed.trials) ? listed.trials : []).filter(isRecord);
+	const run = parse$1(files.get("run.json"));
+	if (!isRecord$1(run)) throw new Error("run.json must be an object");
+	const listed = parse$1(files.get("trials.json"));
+	const trials = (isRecord$1(listed) && Array.isArray(listed.trials) ? listed.trials : []).filter(isRecord$1);
 	const size = trials.length;
 	return trials.map((trial) => {
 		const id = size === 1 ? String(run.id) : String(trial.id);
 		const prefix = `trials/${String(trial.id)}/`;
-		const detail = parse(files.get(`${prefix}detail.json`));
-		const trialManifest = isRecord(detail) && isRecord(detail.manifest) ? detail.manifest : {};
-		const trialSummary = isRecord(detail) && isRecord(detail.summary) ? detail.summary : {};
+		const detail = parse$1(files.get(`${prefix}detail.json`));
+		const trialManifest = isRecord$1(detail) && isRecord$1(detail.manifest) ? detail.manifest : {};
+		const trialSummary = isRecord$1(detail) && isRecord$1(detail.summary) ? detail.summary : {};
 		const index = typeof trial.index === "number" ? trial.index : trials.indexOf(trial);
 		const batch = size > 1 ? {
 			id: String(run.id),
 			index,
 			size
 		} : void 0;
-		const scores = Array.isArray(trial.scores) ? trial.scores.filter(isRecord) : [];
-		const metrics = Array.isArray(trial.metrics) ? trial.metrics.filter(isRecord) : [];
-		const datasets = Array.isArray(trialManifest.datasets) ? trialManifest.datasets.filter(isRecord) : [];
+		const scores = Array.isArray(trial.scores) ? trial.scores.filter(isRecord$1) : [];
+		const metrics = Array.isArray(trial.metrics) ? trial.metrics.filter(isRecord$1) : [];
+		const datasets = Array.isArray(trialManifest.datasets) ? trialManifest.datasets.filter(isRecord$1) : [];
 		const status = String(trial.status);
 		const pick = (keys) => Object.fromEntries(keys.flatMap((key) => run[key] === void 0 ? [] : [[key, run[key]]]));
 		const out = /* @__PURE__ */ new Map();
-		out.set("run.json", encode({
+		out.set("run.json", encode$1({
 			snapshotVersion: 2,
 			id,
 			...pick([
@@ -153,7 +154,7 @@ function upgradeSnapshotRun(files) {
 			})) } : {}
 		}));
 		const passed = trialSummary.scoring?.passed;
-		out.set("detail.json", encode({
+		out.set("detail.json", encode$1({
 			scores,
 			...metrics.length ? { metrics } : {},
 			checkpoints: Array.isArray(trial.checkpoints) ? trial.checkpoints : [],
@@ -181,8 +182,8 @@ function upgradeSnapshotRun(files) {
 			const rest = path.slice(prefix.length);
 			if (rest === "detail.json") continue;
 			if (rest === "events.json") {
-				const events = parse(data);
-				out.set(rest, encode({ events: isRecord(events) && Array.isArray(events.events) ? events.events.map(upgradeEvent) : [] }));
+				const events = parse$1(data);
+				out.set(rest, encode$1({ events: isRecord$1(events) && Array.isArray(events.events) ? events.events.map(upgradeEvent) : [] }));
 			} else out.set(rest, data);
 		}
 		return {
@@ -193,12 +194,107 @@ function upgradeSnapshotRun(files) {
 }
 /** A stored or uploaded version 1 catalog, as version 2 stores it. */
 function upgradeCatalogFile(data) {
-	return encode(upgradeCatalog(parse(data)));
+	return encode$1(upgradeCatalog(parse$1(data)));
+}
+//#endregion
+//#region ../share/src/transcripts.ts
+/**
+* Converts one run in snapshot format version 2 (EvalKit 0.9 and 0.10) to
+* version 3 (RFD0009): `events.json`, its trajectory, becomes
+* `transcript.json`, an ATXF transcript with EvalKit's events placed in it;
+* `detail.json` points into it, and token metrics no longer count cache
+* reads, in `detail.json` and `run.json` alike. Pure: the Worker runs it on
+* upload, in `POST /api/v1/upgrade`, and on read for runs not upgraded yet,
+* so all three give the same files.
+*
+* Throws when the run cannot be converted, such as tokens that do not add
+* up; the caller keeps the run as it was.
+*/
+function upgradeV2Run(files) {
+	const run = record$1(parse(files.get("run.json")), "run.json");
+	const detail = record$1(parse(files.get("detail.json")) ?? {}, "detail.json");
+	const events = parse(files.get("events.json"));
+	const trajectory = isRecord(events) && Array.isArray(events.events) ? events.events : [];
+	const manifest = isRecord(detail.manifest) ? detail.manifest : {};
+	const runId = String(run.id);
+	const startedAt = typeof run.startedAt === "string" ? run.startedAt : (/* @__PURE__ */ new Date(0)).toISOString();
+	const aut = isRecord(manifest.aut) ? manifest.aut : void 0;
+	const converted = convertTrajectory(trajectory, {
+		runId,
+		...typeof run.evalId === "string" ? { evalId: run.evalId } : {},
+		startedAt,
+		...aut ? { agent: {
+			...typeof aut.id === "string" ? { id: aut.id } : {},
+			...typeof aut.version === "string" ? { version: aut.version } : {}
+		} } : {}
+	});
+	const upgrade = (scoring) => upgradeScoring(scoring, {
+		runId,
+		startedAt
+	}, converted.recordOfEvent, converted.transcript);
+	const scoring = upgrade({
+		results: Array.isArray(detail.scores) ? detail.scores : [],
+		...Array.isArray(detail.metrics) ? { metrics: detail.metrics } : {},
+		...Array.isArray(detail.checkpoints) ? { checkpoints: detail.checkpoints } : {},
+		passed: false
+	});
+	const summary = isRecord(detail.summary) ? detail.summary : void 0;
+	const upgraded = new Map(files);
+	upgraded.delete("events.json");
+	upgraded.set("transcript.json", encode({
+		transcript: converted.transcript,
+		subagents: {},
+		runEvents: converted.runEvents
+	}));
+	upgraded.set("detail.json", encode({
+		...detail,
+		scores: scoring.results,
+		...scoring.metrics ? { metrics: scoring.metrics } : {},
+		...scoring.checkpoints ? { checkpoints: scoring.checkpoints } : {},
+		manifest: {
+			...manifest,
+			schemaVersion: 7
+		},
+		...summary ? { summary: {
+			...summary,
+			...isRecord(summary.scoring) ? { scoring: upgrade(summary.scoring) } : {}
+		} } : {}
+	}));
+	upgraded.set("run.json", encode({
+		...run,
+		snapshotVersion: 3,
+		...Array.isArray(scoring.metrics) && scoring.metrics.length ? { metrics: runMetrics(scoring.metrics) } : {}
+	}));
+	return upgraded;
+}
+/** True when the stored run is in snapshot format version 2. */
+function isV2Run(runJson) {
+	return isRecord(runJson) && runJson.snapshotVersion === 2;
+}
+/** `run.json`'s metrics, as `evalkit share` derives them from the scoring's. */
+function runMetrics(scoring) {
+	return scoring.filter(isRecord).map((metric) => ({
+		name: metric.name,
+		...metric.unit ? { unit: metric.unit } : {},
+		...metric.better ? { better: metric.better } : {},
+		...metric.builtIn ? { builtIn: true } : {},
+		...metric.value === void 0 ? {} : { value: metric.value }
+	}));
+}
+const decoder = new TextDecoder();
+const encoder = new TextEncoder();
+const parse = (data) => data ? JSON.parse(decoder.decode(data)) : void 0;
+const encode = (value) => encoder.encode(`${JSON.stringify(value)}\n`);
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+function record$1(value, name) {
+	if (!isRecord(value)) throw new Error(`${name} must be an object`);
+	return value;
 }
 //#endregion
 //#region ../share/src/paths.ts
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
-const RUN_JSON = /^(detail|events|artifacts|workspace)\.json$/;
+const RUN_JSON = /^(detail|transcript|artifacts|workspace)\.json$/;
+const V2_RUN_JSON = /^(detail|events|artifacts|workspace)\.json$/;
 const RUN_FILE$1 = /^(?:workspace|attachments)\/(.+)$/;
 const LEGACY_TRIAL_JSON = /^trials\/([^/]+)\/(detail|events|artifacts|workspace)\.json$/;
 const LEGACY_TRIAL_FILE = /^trials\/([^/]+)\/(?:workspace|attachments)\/(.+)$/;
@@ -209,12 +305,13 @@ function safeSegment(segment) {
 	return segment.length > 0 && segment.length <= 255 && segment !== "." && segment !== ".." && !/[\\/\0]/.test(segment);
 }
 /**
-* The only files a run bundle may contain (snapshot format version 2, from
-* `evalkit build`, for one run): its JSON, candidate workspace files and
-* attachments. Never evaluator workspaces.
+* The only files a run bundle may contain (snapshot format version 3, from
+* `evalkit share`, for one run): its JSON, candidate workspace files and
+* attachments. Never evaluator workspaces. Version 2 (EvalKit 0.9 and
+* 0.10) has `events.json` where version 3 has `transcript.json`.
 */
-function allowedRunPath(path) {
-	if (path === "run.json" || path === "catalog.json" || RUN_JSON.test(path)) return true;
+function allowedRunPath(path, version = 3) {
+	if (path === "run.json" || path === "catalog.json" || (version === 3 ? RUN_JSON : V2_RUN_JSON).test(path)) return true;
 	const file = RUN_FILE$1.exec(path);
 	return Boolean(file && file[1].split("/").every(safeSegment));
 }
@@ -255,18 +352,30 @@ function parseUpload(bundle) {
 	}
 	const runJson = byPath.get("run.json");
 	if (!runJson) throw new Rejected("Bundle has no run.json");
-	const legacy = isLegacyRun(parseJson(runJson, "run.json"));
-	const allowed = legacy ? allowedLegacyRunPath : allowedRunPath;
+	const runValue = parseJson(runJson, "run.json");
+	const legacy = isLegacyRun(runValue);
+	const v2 = isV2Run(runValue);
+	const allowed = legacy ? allowedLegacyRunPath : (path) => allowedRunPath(path, v2 ? 2 : 3);
 	for (const path of byPath.keys()) if (!allowed(path)) throw new Rejected(`File not allowed in a run bundle: ${path}`);
 	const storedCatalog = byPath.get("catalog.json");
 	if (storedCatalog) parseCatalog(storedCatalog);
 	const catalog = storedCatalog && legacy ? upgradeCatalogFile(storedCatalog) : storedCatalog;
+	if (v2) return [runUpload(toV3(byPath), catalog)];
 	if (!legacy) return [runUpload(byPath, catalog)];
 	parseRun(runJson, { legacy: true });
 	return upgradeSnapshotRun(byPath).filter(({ files }) => {
 		const status = JSON.parse(new TextDecoder().decode(files.get("run.json"))).status;
 		return status === "passed" || status === "failed" || status === "errored";
-	}).map(({ files }) => runUpload(files, catalog));
+	}).map(({ files }) => runUpload(toV3(files), catalog));
+}
+/** A version 2 run as version 3; an upload that cannot be converted is refused. */
+function toV3(files) {
+	parseRun(files.get("run.json"), { version: 2 });
+	try {
+		return upgradeV2Run(files);
+	} catch (error) {
+		throw new Rejected(`This run, from EvalKit 0.10 or older, could not be converted: ${error instanceof Error ? error.message : String(error)}. Upgrade the CLI and run \`evalkit upgrade\`, then share again.`);
+	}
 }
 function runUpload(files, catalog) {
 	const runJson = files.get("run.json");
@@ -282,7 +391,8 @@ function runUpload(files, catalog) {
 }
 function parseRun(data, options = {}) {
 	const value = record(parseJson(data, "run.json"), "run.json");
-	if (!options.legacy && value.snapshotVersion !== 2) throw new Rejected("run.json must be snapshot format version 2");
+	const version = options.version ?? 3;
+	if (!options.legacy && value.snapshotVersion !== version) throw new Rejected(`run.json must be snapshot format version ${version}`);
 	const source = record(value.source, "run.json source");
 	const id = text(value.id, "id");
 	if (!isUuid(id)) throw new Rejected("run.json id must be a UUID");
@@ -290,7 +400,7 @@ function parseRun(data, options = {}) {
 	if (!isSha(commit)) throw new Rejected("source.commit must be a full 40-character SHA");
 	if (typeof source.dirty !== "boolean") throw new Rejected("source.dirty must be a boolean");
 	const status = value.status;
-	if (status !== "passed" && status !== "failed" && status !== "errored") throw new Rejected(`Only finished runs can be shared (status ${JSON.stringify(status)})`);
+	if (status !== "passed" && status !== "failed" && status !== "errored" && status !== "limited") throw new Rejected(`Only finished runs can be shared (status ${JSON.stringify(status)})`);
 	if (value.score !== void 0 && typeof value.score !== "number") throw new Rejected("score must be a number");
 	const shared = {
 		repo: text(source.repo, "source.repo"),
@@ -374,7 +484,7 @@ const jsonLine = (event) => console.log(JSON.stringify(event));
 *
 *   POST /api/v1/uploads                         one run bundle
 *   POST /api/v1/reindex                         rebuild the index from stored run.json files
-*   POST /api/v1/upgrade                         convert stored runs from before 0.9 (RFD0008)
+*   POST /api/v1/upgrade                         convert stored runs from older EvalKit versions (RFD0008, RFD0009)
 *   GET  /api/v1/repos                           indexed repositories
 *   GET  /api/v1/repos/<repo>/-/commits          commits, newest first
 *   GET  /api/v1/repos/<repo>/-/<sha>/data/…     snapshot data
@@ -450,15 +560,20 @@ function createShareApp(options) {
 		}, created ? 201 : 200);
 	});
 	/**
-	* Converts stored runs from before 0.9 (snapshot format version 1, a run of
-	* trials) into one run per trial, `limit` old runs per request (default
-	* 20), so it fits a Worker's limits. Call it until `remaining` is 0. New
-	* objects and index rows are written before old ones are removed, so an
-	* interrupted upgrade is finished by calling it again.
+	* Converts stored runs from older EvalKit versions, `limit` runs per
+	* request (default 20), so it fits a Worker's limits. Call it until
+	* `remaining` is 0. Runs from before 0.9 (snapshot format version 1, a
+	* run of trials) become one run per trial (RFD0008); runs from 0.9 and
+	* 0.10 (version 2) get an ATXF transcript in place of their trajectory
+	* (RFD0009). New objects and index rows are written before old ones are
+	* removed, and run.json last, so an interrupted upgrade is finished by
+	* calling it again. A run that cannot be converted is left as it was and
+	* listed in `failed`.
 	*/
 	app.post("/api/v1/upgrade", async (context) => {
 		const limit = Math.min(Math.max(Number(context.req.query("limit") ?? 20) || 20, 1), 200);
 		const legacy = [];
+		const v2 = [];
 		for await (const key of files.list("repos/")) {
 			if (!RUN_FILE.test(key)) continue;
 			const data = await readBytes(await files.get(key));
@@ -470,21 +585,66 @@ function createShareApp(options) {
 				continue;
 			}
 			if (isLegacyRun(value)) legacy.push(key);
+			else if (isV2Run(value)) v2.push(key);
 		}
 		let runs = 0;
+		let transcripts = 0;
+		const failed = [];
 		const batch = legacy.slice(0, limit);
 		for (const key of batch) runs += await upgradeStoredRun(key);
+		for (const key of v2.slice(0, limit - batch.length)) try {
+			await upgradeV2StoredRun(key);
+			transcripts += 1;
+		} catch (error) {
+			failed.push({
+				run: key,
+				reason: error instanceof Error ? error.message : String(error)
+			});
+		}
+		const done = batch.length + transcripts + failed.length;
 		const result = {
 			upgraded: batch.length,
 			runs,
-			remaining: legacy.length - batch.length
+			transcripts,
+			failed: failed.length,
+			remaining: legacy.length + v2.length - done
 		};
 		log({
 			event: "share.upgrade",
 			...result
 		});
-		return context.json(result);
+		return context.json({
+			...result,
+			failures: failed
+		});
 	});
+	/** A stored version 2 run, converted to version 3 in place. */
+	async function upgradeV2StoredRun(runJsonKey) {
+		const runBase = runJsonKey.slice(0, -9);
+		const upgraded = upgradeV2Run(await v2Files(runBase));
+		for (const name of ["transcript.json", "detail.json"]) await files.put(`${runBase}/${name}`, upgraded.get(name));
+		const runJson = upgraded.get("run.json");
+		const run = storedRun(runJson);
+		if (run) {
+			await index.forgetRun(run.id);
+			await index.recordRun(run);
+		}
+		await files.delete(`${runBase}/events.json`);
+		await files.put(runJsonKey, runJson);
+	}
+	/** The files `upgradeV2Run` reads, from a stored run. */
+	async function v2Files(runBase) {
+		const stored = /* @__PURE__ */ new Map();
+		for (const name of [
+			"run.json",
+			"detail.json",
+			"events.json"
+		]) {
+			const data = await readBytes(await files.get(`${runBase}/${name}`));
+			if (data) stored.set(name, data);
+		}
+		return stored;
+	}
 	/** One stored version 1 run → version 2 runs; returns how many. */
 	async function upgradeStoredRun(runJsonKey) {
 		const [, repo, sha, oldRunId] = RUN_FILE.exec(runJsonKey);
@@ -501,6 +661,9 @@ function createShareApp(options) {
 		const written = /* @__PURE__ */ new Set();
 		let newRunJson;
 		for (const upgraded of upgradeSnapshotRun(stored)) {
+			try {
+				upgraded.files = upgradeV2Run(upgraded.files);
+			} catch {}
 			const runJson = upgraded.files.get("run.json");
 			const run = storedRun(runJson);
 			if (!run) continue;
@@ -666,9 +829,9 @@ function createShareApp(options) {
 			const catalog = data ? parseCatalog(data) : EMPTY_CATALOG;
 			return context.json(path === "catalog.json" ? { evals: catalog.catalog } : path === "evals.json" ? { evals: catalog.evals } : { suites: catalog.suites }, 200, NO_CACHE);
 		}
-		const runFile = /^runs\/([^/]+)\/((?:detail|events|artifacts|workspace)\.json|(?:workspace|attachments)\/.+)$/.exec(path);
+		const runFile = /^runs\/([^/]+)\/((?:detail|transcript|artifacts|workspace)\.json|(?:workspace|attachments)\/.+)$/.exec(path);
 		if (!runFile || !isUuid(runFile[1])) return context.notFound();
-		const body = await files.get(`${commitBase}/${path}`);
+		const body = runFile[2] === "transcript.json" || runFile[2] === "detail.json" ? await currentFile(`${commitBase}/runs/${runFile[1]}`, runFile[2]) : await files.get(`${commitBase}/${path}`);
 		if (!body) return context.notFound();
 		const json = path.endsWith(".json") && !path.includes("/workspace/") && !path.includes("/attachments/");
 		return new Response(body, { headers: json ? {
@@ -681,14 +844,35 @@ function createShareApp(options) {
 			"cache-control": IMMUTABLE
 		} });
 	}
+	/**
+	* A run's detail or transcript as version 3 has it: a version 2 run not
+	* upgraded yet is converted as it is read, the same way the upgrade does,
+	* so its cached files do not change when it is upgraded.
+	*/
+	async function currentFile(runBase, name) {
+		const runJson = await readBytes(await files.get(`${runBase}/run.json`));
+		let value;
+		try {
+			value = runJson ? JSON.parse(new TextDecoder().decode(runJson)) : void 0;
+		} catch {
+			value = void 0;
+		}
+		if (!isV2Run(value)) return files.get(`${runBase}/${name}`);
+		try {
+			return upgradeV2Run(await v2Files(runBase)).get(name) ?? null;
+		} catch {
+			return name === "detail.json" ? files.get(`${runBase}/${name}`) : null;
+		}
+	}
 	return app;
 }
-/** A stored run.json, or undefined if it is not a shareable run. */
+/** A stored run.json (version 3, or 2 until upgraded), or undefined if it is not a shareable run. */
 function storedRun(data) {
 	if (!data) return void 0;
 	try {
-		return parseRun(data);
+		return parseRun(data, { version: JSON.parse(new TextDecoder().decode(data)).snapshotVersion === 2 ? 2 : 3 });
 	} catch (error) {
+		if (error instanceof SyntaxError) return void 0;
 		if (error instanceof Rejected) return void 0;
 		throw error;
 	}
@@ -738,6 +922,12 @@ const MIGRATIONS = [
 	`CREATE INDEX IF NOT EXISTS runs_by_eval ON runs (repo, eval_id, started_at)`
 ];
 /** Newest first; commits without a commit time order by first upload. */
+/** Statuses a commit counts as failed: a limited run is a failure too. */
+const FAILED = /* @__PURE__ */ new Set([
+	"failed",
+	"errored",
+	"limited"
+]);
 const BY_TIME = `ORDER BY COALESCE(committed_at, first_seen_at) DESC, first_seen_at DESC`;
 var ShareIndexDb = class {
 	sql;
@@ -768,7 +958,7 @@ var ShareIndexDb = class {
          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, run.id, run.repoKey, run.source.commit, run.evalId, run.suiteId ?? null, run.status, run.score ?? null, run.startedAt, run.source.dirty ? 1 : 0, JSON.stringify(run.summary), at);
 			if (!run.source.dirty) this.sql.run(`UPDATE commits SET runs = runs + 1,
              passed = passed + ?, failed = failed + ?
-           WHERE repo = ? AND sha = ?`, run.status === "passed" ? 1 : 0, run.status === "failed" || run.status === "errored" ? 1 : 0, run.repoKey, run.source.commit);
+           WHERE repo = ? AND sha = ?`, run.status === "passed" ? 1 : 0, FAILED.has(run.status) ? 1 : 0, run.repoKey, run.source.commit);
 		});
 	}
 	forgetRun(runId) {
@@ -778,7 +968,7 @@ var ShareIndexDb = class {
 			this.sql.run("DELETE FROM runs WHERE id = ?", runId);
 			if (!row.dirty) this.sql.run(`UPDATE commits SET runs = runs - 1,
              passed = passed - ?, failed = failed - ?
-           WHERE repo = ? AND sha = ?`, row.status === "passed" ? 1 : 0, row.status === "failed" || row.status === "errored" ? 1 : 0, row.repo, row.sha);
+           WHERE repo = ? AND sha = ?`, row.status === "passed" ? 1 : 0, FAILED.has(row.status) ? 1 : 0, row.repo, row.sha);
 		});
 	}
 	repos() {