@tangle-network/agent-bench 0.9.4 → 0.10.0

package/HARNESS.md

@@ -62,27 +62,27 @@ Use `LOOP_ATTEMPTS=N` only when the benchmark's own visible feedback is allowed
 `runBenchmarks()` returns each judged artifact, worker events, and observed usage in `perTask`.
 Retry usage includes every attempt; missing receipts leave the measured subtotal explicitly incomplete.
 Judge failures retain completed worker evidence.
+Each task separates `execution` from `measurement` availability.
+Captured empty output and explicit failed turns remain measured failures when the evaluator runs successfully.
+Read, extraction, and judge failures leave measurement unavailable while retaining observed usage.
+Missing dispatch evidence remains unknown; `ok: false` never establishes permission to retry.
 Errors propagated by `close()` remain in `detail` beside the settled task outcome.
 The current Runtime lineage suppresses sandbox deletion errors, so a returned result does not confirm resource deletion.
 The caller's abort signal stops queued shots and reaches active sandbox turns.
 `modelApiKey` supplies sandbox inference authorization separately from the `routerKey` used for sandbox control.
 
-### Full-fidelity improvement fixture
+### Retained strategy driver
 
 ```bash
 cd bench
 pnpm tsx src/swe-self-improve.mts
 ```
 
-This is the canonical real-task Runtime fixture:
-
-- SWE-bench Verified instances;
-- repository state as the produced artifact;
-- the official Docker judge outside the candidate agent;
-- explicit train, selection, and frozen final-test partitions;
-- Runtime's `improve()` boundary and complete cost receipts.
-
-It proves the integrated execution path can support a real value campaign. A paid powered result still belongs in Discovery Lab.
+This driver uses `runStrategyEvolution` with SWE-bench tasks and a frozen holdout.
+It does not exercise `improve`, and it deletes its temporary run directory on exit.
+It therefore cannot provide retained improvement or lineage evidence.
+Use `examples/improve` for the maintained offline API fixture.
+Use the consuming labs for registered learning campaigns with retained execution and comparison evidence.
 
 ### Offline diagnostics
 

package/dist/index.d.ts

@@ -11,7 +11,7 @@ import { createT2RagBenchAdapter } from "./benchmarks/t2-ragbench.js";
 import { AgentProfile, SandboxClient, sumSandboxUsage } from "@tangle-network/agent-runtime/kernel";
 import { SandboxEvent } from "@tangle-network/sandbox";
 import { AgentCandidateBenchmarkGraderPort, AgentCandidateExecutionClaimStore, AgentCandidateExecutorPort, AgentCandidateExecutorRequest, AgentCandidateExecutorStopRequest, AgentCandidateOutputArtifactPort, AgentCandidateRunFinalization, PreparedAgentCandidateExecution } from "@tangle-network/agent-runtime/candidate-execution";
-import { TraceStore } from "@tangle-network/agent-eval";
+import { RunTerminalOutcome, TraceStore } from "@tangle-network/agent-eval";
 //#region src/resolve-client.d.ts
 interface ResolveBenchClientOptions {
   /** The selector (the `BACKEND` env value): `router` = off-box; anything else = in-box Sandbox. */
@@ -56,6 +56,13 @@ interface BenchShotResult {
   /** Provider observations, including explicit unknown counters. Omitted when the shot reports none. */
   readonly usage?: ReturnType<typeof sumSandboxUsage>;
   readonly events?: readonly SandboxEvent[];
+  /** Observed dispatch and terminal state, independent of artifact quality. */
+  readonly execution?: {
+    readonly phase: 'not-started' | 'started' | 'unknown';
+    readonly terminalOutcome: RunTerminalOutcome;
+  };
+  /** Whether the artifact was captured without a read or extraction failure. */
+  readonly artifactAvailable?: boolean;
 }
 /** Runs one (adapter, task, cell) shot. Defaults to `openSandboxRun`. */
 type BenchShot = (input: {
@@ -122,9 +129,11 @@ interface BenchCellTaskResult {
   readonly rep: number;
   readonly resolved: boolean;
   readonly score: number;
-  /** false = the shot threw or produced no artifact (infra/empty), excluded from the resolve
-   *  denominator so a harness outage can't masquerade as a 0% capability result. */
+  /** Whether execution completed successfully and produced a readable, nonempty artifact. */
   readonly ok: boolean;
+  readonly execution?: BenchShotResult['execution'];
+  /** Available failed attempts remain in comparisons; unavailable measurement is reported separately. */
+  readonly measurement?: 'available' | 'unavailable';
   readonly detail?: string;
   readonly wallMs: number;
   /** Exact bytes given to the benchmark judge, retained even when judging fails. */

package/dist/index.js

@@ -304,11 +304,15 @@ const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, rou
 	};
 	const controller = new AbortController();
 	const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : void 0;
+	const observedEvents = [];
 	const runOptions = {
 		agentRun,
 		signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal,
 		runId: `bench:${adapter.name}:${task.id}:${uniq}`,
-		scenarioId: task.id
+		scenarioId: task.id,
+		onSandboxEvent: (event) => {
+			observedEvents.push(event);
+		}
 	};
 	const boxSetup = adapter.boxSetup;
 	if (boxSetup) runOptions.beforeStart = async ({ box, sessionId }) => {
@@ -327,12 +331,23 @@ const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, rou
 	};
 	try {
 		run = await openSandboxRun(client, runOptions, deliverable);
+		result = {
+			...result,
+			execution: {
+				phase: "unknown",
+				terminalOutcome: "unknown"
+			}
+		};
 		const turn = await run.start(prompt ?? task.prompt);
 		result = {
 			artifact: "",
 			ok: false,
 			usage: sumSandboxUsage(turn.events),
-			events: turn.events
+			events: turn.events,
+			execution: {
+				phase: "started",
+				terminalOutcome: turn.outcome.success ? "succeeded" : turn.outcome.status === "failed" ? "failed" : "incomplete"
+			}
 		};
 		let artifact = (turn.out ?? "").trim();
 		let boxExtractError;
@@ -379,19 +394,29 @@ const openSandboxShot = async ({ adapter, task, cell, prompt, routerBaseUrl, rou
 		result = {
 			artifact,
 			ok: turn.outcome.success && artifact.length > 0 && turn.readError === void 0 && boxExtractError === void 0,
+			execution: result.execution,
+			artifactAvailable: turn.readError === void 0 && boxExtractError === void 0,
 			usage: result.usage,
 			events: turn.events,
 			...detail ? { detail } : {}
 		};
 	} catch (err) {
+		const events = err instanceof SandboxRunAbortError ? err.events : observedEvents;
 		result = {
 			...result,
 			ok: false,
+			artifactAvailable: false,
+			execution: {
+				phase: events.length > 0 ? "started" : "unknown",
+				terminalOutcome: result.execution?.terminalOutcome ?? "unknown"
+			},
 			detail: err instanceof Error ? err.message : String(err),
-			...err instanceof SandboxRunAbortError ? {
-				usage: sumSandboxUsage(err.events),
-				events: err.events
-			} : {}
+			usage: {
+				...sumSandboxUsage(events),
+				tokensKnown: false,
+				usdKnown: false
+			},
+			events
 		};
 	} finally {
 		if (timer) clearTimeout(timer);
@@ -496,6 +521,11 @@ async function loopedShot(input, shot, attempts) {
 		return {
 			artifact: completed.at(-1)?.artifact ?? "",
 			ok: false,
+			execution: pendingShot ? {
+				phase: "unknown",
+				terminalOutcome: "unknown"
+			} : completed.at(-1)?.execution,
+			artifactAvailable: false,
 			usage: combinedUsage(pendingShot ? [...completed, {
 				artifact: "",
 				ok: false
@@ -505,8 +535,10 @@ async function loopedShot(input, shot, attempts) {
 		};
 	}
 	const best = result.rounds.reduce((winner, candidate) => {
-		if (shots.get(candidate.round)?.ok !== true) return winner;
-		if (shots.get(winner.round)?.ok !== true) return candidate;
+		const candidateShot = shots.get(candidate.round);
+		const winnerShot = shots.get(winner.round);
+		const rank = (shot) => shot?.ok ? 2 : shot?.artifactAvailable ? 1 : 0;
+		if (rank(candidateShot) !== rank(winnerShot)) return rank(candidateShot) > rank(winnerShot) ? candidate : winner;
 		const a = scores.get(winner.round);
 		const b = scores.get(candidate.round);
 		if (!a) return candidate;
@@ -519,6 +551,8 @@ async function loopedShot(input, shot, attempts) {
 	return {
 		artifact: best.artifact,
 		ok: shots.get(best.round)?.ok === true && best.artifact.trim().length > 0,
+		execution: shots.get(best.round)?.execution,
+		artifactAvailable: shots.get(best.round)?.artifactAvailable,
 		usage: combinedUsage([...shots.values()]),
 		events: [...shots.values()].flatMap((shot) => shot.events ?? []),
 		detail: JSON.stringify({
@@ -641,6 +675,7 @@ async function runBenchmarks(opts) {
 		const startedAt = Date.now();
 		let result;
 		let out;
+		let invoked = false;
 		try {
 			opts.signal?.throwIfAborted();
 			const shotInput = {
@@ -657,6 +692,7 @@ async function runBenchmarks(opts) {
 				...opts.signal ? { signal: opts.signal } : {},
 				...opts.resolveClient ? { resolveClient: opts.resolveClient } : {}
 			};
+			invoked = true;
 			out = loopAttempts > 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput);
 			const score = await job.adapter.judge(job.task, out.artifact);
 			result = {
@@ -667,6 +703,11 @@ async function runBenchmarks(opts) {
 				resolved: out.ok && score.resolved,
 				score: out.ok ? score.score : 0,
 				ok: out.ok,
+				execution: out.execution ?? {
+					phase: out.ok ? "started" : "unknown",
+					terminalOutcome: out.ok ? "succeeded" : "unknown"
+				},
+				measurement: out.artifactAvailable ?? out.ok ? "available" : "unavailable",
 				...out.detail ?? score.detail ? { detail: combineDetails(out.detail, score.detail) } : {},
 				wallMs: Date.now() - startedAt,
 				artifact: out.artifact,
@@ -682,6 +723,11 @@ async function runBenchmarks(opts) {
 				resolved: false,
 				score: 0,
 				ok: false,
+				execution: out?.execution ?? {
+					phase: !invoked ? "not-started" : out?.ok ? "started" : "unknown",
+					terminalOutcome: out?.ok ? "succeeded" : "unknown"
+				},
+				measurement: "unavailable",
 				detail: err instanceof Error ? err.message.slice(0, 200) : String(err),
 				wallMs: Date.now() - startedAt,
 				...out === void 0 ? {} : { artifact: out.artifact },
@@ -714,7 +760,7 @@ function aggregate(perTask) {
 			scoreSum: 0
 		};
 		e.n += 1;
-		if (!r.ok) e.errored += 1;
+		if ((r.measurement ?? (r.ok ? "available" : "unavailable")) === "unavailable") e.errored += 1;
 		else {
 			if (r.resolved) e.resolved += 1;
 			e.scoreSum += r.score;