@tangle-network/agent-eval 1.0.1 → 1.0.3

package/dist/sandbox-harness-BuacGaVn.js

@@ -0,0 +1,181 @@
+import { n as runBoundedProcess } from "./bounded-process-BZrWXIKW.js";
+import { setTimeout } from "node:timers/promises";
+// src/sandbox-harness.ts
+var SubprocessSandboxDriver = class {
+	options;
+	id = "subprocess";
+	constructor(options = {}) {
+		this.options = options;
+	}
+	async exec(phase, command, config) {
+		const effectiveEnv = {
+			...process.env,
+			...this.options.env ?? {},
+			...config.env ?? {}
+		};
+		const result = await runBoundedProcess({
+			command,
+			shell: true,
+			cwd: config.cwd ?? this.options.cwd,
+			env: effectiveEnv,
+			envMode: "replace",
+			timeoutMs: config.timeoutMs ?? 6e5,
+			maxOutputBytes: config.maxOutputBytes ?? 16777216
+		});
+		const parsed = !result.killedByTimeout && phase === "test" && config.testParser ? config.testParser.parse(result.stdout, result.stderr, result.exitCode) : void 0;
+		return {
+			phase,
+			exitCode: result.exitCode,
+			stdout: result.stdout,
+			stderr: result.runnerError ? result.stderr + result.runnerError : result.stderr,
+			wallMs: result.wallMs,
+			testsTotal: parsed?.testsTotal,
+			testsPassed: parsed?.testsPassed,
+			killedByTimeout: result.killedByTimeout || void 0,
+			outputTruncated: result.outputTruncated || void 0
+		};
+	}
+};
+var SandboxHarness = class {
+	driver;
+	constructor(driver = new SubprocessSandboxDriver()) {
+		this.driver = driver;
+	}
+	async run(config, emitter) {
+		const handle = await emitter.sandbox({
+			name: `sandbox(${this.driver.id})`,
+			image: config.image,
+			command: [
+				config.setupCommand,
+				config.runCommand,
+				config.testCommand
+			].filter(Boolean).join(" && ")
+		});
+		const result = {
+			passed: false,
+			totalWallMs: 0,
+			score: 0
+		};
+		try {
+			for (const phase of [
+				"setup",
+				"run",
+				"test"
+			]) {
+				const command = config[`${phase}Command`];
+				if (!command) continue;
+				const measured = await this.driver.exec(phase, command, config);
+				result[phase] = measured;
+				result.totalWallMs += measured.wallMs;
+				const passed = !measured.killedByTimeout && measured.exitCode === 0;
+				if (phase !== "test" && !passed) {
+					await handle.fail(measured.killedByTimeout ? `${phase} timed out after ${measured.wallMs}ms` : `${phase} failed (exit ${measured.exitCode})`, {
+						exitCode: measured.exitCode,
+						wallMs: result.totalWallMs
+					});
+					return result;
+				}
+			}
+			const test = result.test;
+			result.passed = !test || !test.killedByTimeout && test.exitCode === 0;
+			result.score = test?.testsTotal !== void 0 && test.testsTotal > 0 ? (test.testsPassed ?? 0) / test.testsTotal : result.passed ? 1 : 0;
+			await handle.end({
+				...test ? {
+					exitCode: test.exitCode,
+					testsTotal: test.testsTotal,
+					testsPassed: test.testsPassed,
+					status: result.passed ? "ok" : "error"
+				} : {},
+				wallMs: result.totalWallMs
+			});
+		} catch (err) {
+			await handle.fail(err instanceof Error ? err : String(err));
+			throw err;
+		}
+		return result;
+	}
+};
+/** Campaigns own whole-scenario retries; polling only repeats the declared read. */
+async function runSandboxScenario(spec, options = {}) {
+	if (!(spec.budgetMs > 0 && Number.isSafeInteger(spec.budgetMs))) throw new Error("Scenario budgetMs must be positive and finite");
+	const signal = AbortSignal.any([AbortSignal.timeout(spec.budgetMs), ...options.signal ? [options.signal] : []]);
+	const report = {
+		pass: false,
+		assertions: [],
+		timings: []
+	};
+	const retirements = [];
+	const scope = {
+		signal,
+		check: (...assertions) => {
+			report.assertions.push(...assertions);
+		},
+		defer: (retire) => {
+			retirements.push(retire);
+		},
+		async measure(name, run) {
+			const start = performance.now();
+			let measured = {
+				name,
+				durationMs: 0,
+				success: true
+			};
+			try {
+				if (options.measure) measured = await options.measure(name, run);
+				else {
+					const result = await run();
+					if (result?.success === false) throw new Error(result.error ?? "measured operation returned success=false");
+					measured.durationMs = Math.round(performance.now() - start);
+				}
+			} catch (error) {
+				measured = {
+					name,
+					durationMs: Math.round(performance.now() - start),
+					success: false,
+					error: error instanceof Error ? error.message : String(error)
+				};
+			}
+			report.timings.push(measured);
+			return measured;
+		},
+		async poll(read, ready, budget) {
+			if (!(budget.timeoutMs > 0 && budget.intervalMs > 0)) throw new Error("Polling timeout and interval must be positive");
+			const deadline = performance.now() + budget.timeoutMs;
+			while (performance.now() < deadline) {
+				signal.throwIfAborted();
+				const value = await read();
+				if (ready(value)) return value;
+				await setTimeout(Math.min(budget.intervalMs, Math.max(0, deadline - performance.now())), void 0, { signal });
+			}
+		}
+	};
+	try {
+		signal.throwIfAborted();
+		const state = await spec.setup(scope);
+		for (const step of spec.steps) {
+			signal.throwIfAborted();
+			if (step.when && !step.when(state)) continue;
+			const timing = await scope.measure(step.name, () => step.run(state, scope));
+			scope.check(...step.expect?.(state, timing) ?? []);
+		}
+		signal.throwIfAborted();
+		scope.check(...spec.expectations?.(state) ?? []);
+		report.metadata = spec.metadata?.(state);
+	} catch (error) {
+		report.error = error instanceof Error ? error.message : String(error);
+	} finally {
+		for (const retire of retirements.reverse()) {
+			const measured = await scope.measure("cleanup", retire);
+			if (!measured.success) scope.check({
+				name: "cleanup",
+				status: "fail",
+				message: measured.error
+			});
+		}
+	}
+	report.pass = !report.error && report.timings.every((t) => t.success) && report.assertions.every((a) => a.soft || a.status === "pass");
+	return report;
+}
+export { runSandboxScenario as n, SandboxHarness as t };
+
+//# sourceMappingURL=sandbox-harness-BuacGaVn.js.map

package/dist/sandbox-harness-BuacGaVn.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sandbox-harness-BuacGaVn.js","names":["delay"],"sources":["../src/sandbox-harness.ts"],"sourcesContent":["import { setTimeout as delay } from 'node:timers/promises'\nimport { runBoundedProcess } from './bounded-process'\n\nimport type { TraceEmitter } from './trace/emitter'\nimport type { SandboxSpan } from './trace/schema'\n\nexport interface HarnessConfig {\n  /** Setup command (e.g. \"pnpm install\"). Non-zero exit fails the run. */\n  setupCommand?: string\n  /** Run command (e.g. \"pnpm build\"). */\n  runCommand?: string\n  /** Test command (e.g. \"pnpm test --run\"). Drives the test count + pass count. */\n  testCommand?: string\n  /** Absolute cwd for the subprocess driver. Ignored by docker driver. */\n  cwd?: string\n  /** Max wall-clock per phase in ms. Default 10 minutes. */\n  timeoutMs?: number\n  /** Captured stdout + stderr cap, default 16 MiB; excess is dropped. */\n  maxOutputBytes?: number\n  /** Image for the docker driver. */\n  image?: string\n  /** Extra env vars (validated; shell-escaped). */\n  env?: Record<string, string>\n  /** Parser for the test output — maps stdout/stderr/exit code → pass count. */\n  testParser?: TestOutputParser\n}\n\nexport interface TestOutputParser {\n  id: string\n  parse(\n    stdout: string,\n    stderr: string,\n    exitCode: number,\n  ): { testsTotal: number; testsPassed: number } | undefined\n}\n\nexport interface SandboxResult {\n  phase: 'setup' | 'run' | 'test'\n  exitCode: number\n  stdout: string\n  stderr: string\n  wallMs: number\n  testsTotal?: number\n  testsPassed?: number\n  /** A timed-out phase always fails, even with exitCode 0. Absent/false means natural exit. */\n  killedByTimeout?: boolean\n  /** Captured output exceeded maxOutputBytes. */\n  outputTruncated?: boolean\n}\n\nexport interface SandboxDriver {\n  id: string\n  exec(\n    phase: SandboxResult['phase'],\n    command: string,\n    config: HarnessConfig,\n  ): Promise<SandboxResult>\n}\n\ninterface SubprocessSandboxDriverOptions {\n  cwd?: string\n  env?: Record<string, string>\n}\n\nexport class SubprocessSandboxDriver implements SandboxDriver {\n  id = 'subprocess'\n  constructor(private options: SubprocessSandboxDriverOptions = {}) {}\n\n  async exec(\n    phase: SandboxResult['phase'],\n    command: string,\n    config: HarnessConfig,\n  ): Promise<SandboxResult> {\n    const effectiveEnv = { ...process.env, ...(this.options.env ?? {}), ...(config.env ?? {}) }\n    const result = await runBoundedProcess({\n      command,\n      shell: true,\n      cwd: config.cwd ?? this.options.cwd,\n      env: effectiveEnv,\n      envMode: 'replace',\n      timeoutMs: config.timeoutMs ?? 10 * 60_000,\n      maxOutputBytes: config.maxOutputBytes ?? 16 * 1024 * 1024,\n    })\n    // A timed-out phase never reaches the parser: the output of a killed\n    // process is a fragment, and a fragment that happens to parse would\n    // report a pass for a run that never finished.\n    const parsed =\n      !result.killedByTimeout && phase === 'test' && config.testParser\n        ? config.testParser.parse(result.stdout, result.stderr, result.exitCode)\n        : undefined\n    return {\n      phase,\n      exitCode: result.exitCode,\n      stdout: result.stdout,\n      stderr: result.runnerError ? result.stderr + result.runnerError : result.stderr,\n      wallMs: result.wallMs,\n      testsTotal: parsed?.testsTotal,\n      testsPassed: parsed?.testsPassed,\n      killedByTimeout: result.killedByTimeout || undefined,\n      outputTruncated: result.outputTruncated || undefined,\n    }\n  }\n}\n\n// ── Harness orchestration ────────────────────────────────────────────\n\nexport interface SandboxHarnessResult {\n  passed: boolean\n  setup?: SandboxResult\n  run?: SandboxResult\n  test?: SandboxResult\n  totalWallMs: number\n  /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */\n  score: number\n}\n\nexport class SandboxHarness {\n  constructor(private driver: SandboxDriver = new SubprocessSandboxDriver()) {}\n\n  async run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult> {\n    const handle = await emitter.sandbox({\n      name: `sandbox(${this.driver.id})`,\n      image: config.image,\n      command: [config.setupCommand, config.runCommand, config.testCommand]\n        .filter(Boolean)\n        .join(' && '),\n    })\n    const result: SandboxHarnessResult = { passed: false, totalWallMs: 0, score: 0 }\n    try {\n      for (const phase of ['setup', 'run', 'test'] as const) {\n        const command = config[`${phase}Command`]\n        if (!command) continue\n        const measured = await this.driver.exec(phase, command, config)\n        result[phase] = measured\n        result.totalWallMs += measured.wallMs\n        const passed = !measured.killedByTimeout && measured.exitCode === 0\n        if (phase !== 'test' && !passed) {\n          await handle.fail(\n            measured.killedByTimeout\n              ? `${phase} timed out after ${measured.wallMs}ms`\n              : `${phase} failed (exit ${measured.exitCode})`,\n            {\n              exitCode: measured.exitCode,\n              wallMs: result.totalWallMs,\n            } as Partial<SandboxSpan>,\n          )\n          return result\n        }\n      }\n      const test = result.test\n      result.passed = !test || (!test.killedByTimeout && test.exitCode === 0)\n      result.score =\n        test?.testsTotal !== undefined && test.testsTotal > 0\n          ? (test.testsPassed ?? 0) / test.testsTotal\n          : result.passed\n            ? 1\n            : 0\n      await handle.end({\n        ...(test\n          ? {\n              exitCode: test.exitCode,\n              testsTotal: test.testsTotal,\n              testsPassed: test.testsPassed,\n              status: result.passed ? 'ok' : 'error',\n            }\n          : {}),\n        wallMs: result.totalWallMs,\n      } as Partial<SandboxSpan>)\n    } catch (err) {\n      await handle.fail(err instanceof Error ? err : String(err))\n      throw err\n    }\n    return result\n  }\n}\n\n/** Portable integration report; products may retain extra timing and assertion fields. */\nexport interface HarnessAssertion {\n  name: string\n  status: 'pass' | 'fail' | 'skip'\n  expected?: string\n  actual?: string\n  message?: string\n  soft?: boolean\n}\nexport interface HarnessTiming {\n  name: string\n  durationMs: number\n  success: boolean\n  error?: string\n}\nexport interface HarnessReport {\n  pass: boolean\n  assertions: HarnessAssertion[]\n  timings: HarnessTiming[]\n  error?: string\n  metadata?: Record<string, unknown>\n}\nexport interface HarnessScope {\n  signal: AbortSignal\n  check(...assertions: HarnessAssertion[]): void\n  measure(name: string, run: () => Promise<unknown>): Promise<HarnessTiming>\n  /** Register retirement immediately after allocation; cleanup ignores workload cancellation. */\n  defer(retire: () => Promise<unknown>): void\n  poll<T>(\n    read: () => Promise<T>,\n    ready: (value: T) => boolean,\n    budget: { timeoutMs: number; intervalMs: number },\n  ): Promise<T | undefined>\n}\nexport interface HarnessScenario<T> {\n  setup(scope: HarnessScope): Promise<T>\n  steps: ReadonlyArray<{\n    name: string\n    when?: (state: T) => boolean\n    run(state: T, scope: HarnessScope): Promise<unknown>\n    expect?: (state: T, timing: HarnessTiming) => HarnessAssertion[]\n  }>\n  expectations?: (state: T) => HarnessAssertion[]\n  metadata?: (state: T) => Record<string, unknown>\n  /** Includes setup and steps. Transports must honor scope.signal. */\n  budgetMs: number\n}\n\n/** Campaigns own whole-scenario retries; polling only repeats the declared read. */\nexport async function runSandboxScenario<T>(\n  spec: HarnessScenario<T>,\n  options: {\n    signal?: AbortSignal\n    measure?: HarnessScope['measure']\n  } = {},\n): Promise<HarnessReport> {\n  if (!(spec.budgetMs > 0 && Number.isSafeInteger(spec.budgetMs)))\n    throw new Error('Scenario budgetMs must be positive and finite')\n  const signal = AbortSignal.any([\n    AbortSignal.timeout(spec.budgetMs),\n    ...(options.signal ? [options.signal] : []),\n  ])\n  const report: HarnessReport = { pass: false, assertions: [], timings: [] }\n  const retirements: Array<() => Promise<unknown>> = []\n  const scope: HarnessScope = {\n    signal,\n    check: (...assertions) => {\n      report.assertions.push(...assertions)\n    },\n    defer: (retire) => {\n      retirements.push(retire)\n    },\n    async measure(name, run) {\n      const start = performance.now()\n      let measured: HarnessTiming = { name, durationMs: 0, success: true }\n      try {\n        if (options.measure) measured = await options.measure(name, run)\n        else {\n          const result = (await run()) as { success?: boolean; error?: string } | undefined\n          if (result?.success === false)\n            throw new Error(result.error ?? 'measured operation returned success=false')\n          measured.durationMs = Math.round(performance.now() - start)\n        }\n      } catch (error) {\n        measured = {\n          name,\n          durationMs: Math.round(performance.now() - start),\n          success: false,\n          error: error instanceof Error ? error.message : String(error),\n        }\n      }\n      report.timings.push(measured)\n      return measured\n    },\n    async poll(read, ready, budget) {\n      if (!(budget.timeoutMs > 0 && budget.intervalMs > 0))\n        throw new Error('Polling timeout and interval must be positive')\n      const deadline = performance.now() + budget.timeoutMs\n      while (performance.now() < deadline) {\n        signal.throwIfAborted()\n        const value = await read()\n        if (ready(value)) return value\n        await delay(\n          Math.min(budget.intervalMs, Math.max(0, deadline - performance.now())),\n          undefined,\n          { signal },\n        )\n      }\n      return undefined\n    },\n  }\n  try {\n    signal.throwIfAborted()\n    const state = await spec.setup(scope)\n    for (const step of spec.steps) {\n      signal.throwIfAborted()\n      if (step.when && !step.when(state)) continue\n      const timing = await scope.measure(step.name, () => step.run(state, scope))\n      scope.check(...(step.expect?.(state, timing) ?? []))\n    }\n    signal.throwIfAborted()\n    scope.check(...(spec.expectations?.(state) ?? []))\n    report.metadata = spec.metadata?.(state)\n  } catch (error) {\n    report.error = error instanceof Error ? error.message : String(error)\n  } finally {\n    for (const retire of retirements.reverse()) {\n      const measured = await scope.measure('cleanup', retire)\n      if (!measured.success)\n        scope.check({ name: 'cleanup', status: 'fail', message: measured.error })\n    }\n  }\n  report.pass =\n    !report.error &&\n    report.timings.every((t) => t.success) &&\n    report.assertions.every((a) => a.soft || a.status === 'pass')\n  return report\n}\n"],"mappings":";;;AAgEA,IAAa,0BAAb,MAA8D;CAExC;CADpB,KAAK;CACL,YAAY,UAAkD,CAAC,GAAG;EAA9C,KAAA,UAAA;CAA+C;CAEnE,MAAM,KACJ,OACA,SACA,QACwB;EACxB,MAAM,eAAe;GAAE,GAAG,QAAQ;GAAK,GAAI,KAAK,QAAQ,OAAO,CAAC;GAAI,GAAI,OAAO,OAAO,CAAC;EAAG;EAC1F,MAAM,SAAS,MAAM,kBAAkB;GACrC;GACA,OAAO;GACP,KAAK,OAAO,OAAO,KAAK,QAAQ;GAChC,KAAK;GACL,SAAS;GACT,WAAW,OAAO,aAAa;GAC/B,gBAAgB,OAAO,kBAAkB;EAC3C,CAAC;EAID,MAAM,SACJ,CAAC,OAAO,mBAAmB,UAAU,UAAU,OAAO,aAClD,OAAO,WAAW,MAAM,OAAO,QAAQ,OAAO,QAAQ,OAAO,QAAQ,IACrE,KAAA;EACN,OAAO;GACL;GACA,UAAU,OAAO;GACjB,QAAQ,OAAO;GACf,QAAQ,OAAO,cAAc,OAAO,SAAS,OAAO,cAAc,OAAO;GACzE,QAAQ,OAAO;GACf,YAAY,QAAQ;GACpB,aAAa,QAAQ;GACrB,iBAAiB,OAAO,mBAAmB,KAAA;GAC3C,iBAAiB,OAAO,mBAAmB,KAAA;EAC7C;CACF;AACF;AAcA,IAAa,iBAAb,MAA4B;CACN;CAApB,YAAY,SAAgC,IAAI,wBAAwB,GAAG;EAAvD,KAAA,SAAA;CAAwD;CAE5E,MAAM,IAAI,QAAuB,SAAsD;EACrF,MAAM,SAAS,MAAM,QAAQ,QAAQ;GACnC,MAAM,WAAW,KAAK,OAAO,GAAG;GAChC,OAAO,OAAO;GACd,SAAS;IAAC,OAAO;IAAc,OAAO;IAAY,OAAO;GAAW,CAAC,CAClE,OAAO,OAAO,CAAC,CACf,KAAK,MAAM;EAChB,CAAC;EACD,MAAM,SAA+B;GAAE,QAAQ;GAAO,aAAa;GAAG,OAAO;EAAE;EAC/E,IAAI;GACF,KAAK,MAAM,SAAS;IAAC;IAAS;IAAO;GAAM,GAAY;IACrD,MAAM,UAAU,OAAO,GAAG,MAAM;IAChC,IAAI,CAAC,SAAS;IACd,MAAM,WAAW,MAAM,KAAK,OAAO,KAAK,OAAO,SAAS,MAAM;IAC9D,OAAO,SAAS;IAChB,OAAO,eAAe,SAAS;IAC/B,MAAM,SAAS,CAAC,SAAS,mBAAmB,SAAS,aAAa;IAClE,IAAI,UAAU,UAAU,CAAC,QAAQ;KAC/B,MAAM,OAAO,KACX,SAAS,kBACL,GAAG,MAAM,mBAAmB,SAAS,OAAO,MAC5C,GAAG,MAAM,gBAAgB,SAAS,SAAS,IAC/C;MACE,UAAU,SAAS;MACnB,QAAQ,OAAO;KACjB,CACF;KACA,OAAO;IACT;GACF;GACA,MAAM,OAAO,OAAO;GACpB,OAAO,SAAS,CAAC,QAAS,CAAC,KAAK,mBAAmB,KAAK,aAAa;GACrE,OAAO,QACL,MAAM,eAAe,KAAA,KAAa,KAAK,aAAa,KAC/C,KAAK,eAAe,KAAK,KAAK,aAC/B,OAAO,SACL,IACA;GACR,MAAM,OAAO,IAAI;IACf,GAAI,OACA;KACE,UAAU,KAAK;KACf,YAAY,KAAK;KACjB,aAAa,KAAK;KAClB,QAAQ,OAAO,SAAS,OAAO;IACjC,IACA,CAAC;IACL,QAAQ,OAAO;GACjB,CAAyB;EAC3B,SAAS,KAAK;GACZ,MAAM,OAAO,KAAK,eAAe,QAAQ,MAAM,OAAO,GAAG,CAAC;GAC1D,MAAM;EACR;EACA,OAAO;CACT;AACF;;AAmDA,eAAsB,mBACpB,MACA,UAGI,CAAC,GACmB;CACxB,IAAI,EAAE,KAAK,WAAW,KAAK,OAAO,cAAc,KAAK,QAAQ,IAC3D,MAAM,IAAI,MAAM,+CAA+C;CACjE,MAAM,SAAS,YAAY,IAAI,CAC7B,YAAY,QAAQ,KAAK,QAAQ,GACjC,GAAI,QAAQ,SAAS,CAAC,QAAQ,MAAM,IAAI,CAAC,CAC3C,CAAC;CACD,MAAM,SAAwB;EAAE,MAAM;EAAO,YAAY,CAAC;EAAG,SAAS,CAAC;CAAE;CACzE,MAAM,cAA6C,CAAC;CACpD,MAAM,QAAsB;EAC1B;EACA,QAAQ,GAAG,eAAe;GACxB,OAAO,WAAW,KAAK,GAAG,UAAU;EACtC;EACA,QAAQ,WAAW;GACjB,YAAY,KAAK,MAAM;EACzB;EACA,MAAM,QAAQ,MAAM,KAAK;GACvB,MAAM,QAAQ,YAAY,IAAI;GAC9B,IAAI,WAA0B;IAAE;IAAM,YAAY;IAAG,SAAS;GAAK;GACnE,IAAI;IACF,IAAI,QAAQ,SAAS,WAAW,MAAM,QAAQ,QAAQ,MAAM,GAAG;SAC1D;KACH,MAAM,SAAU,MAAM,IAAI;KAC1B,IAAI,QAAQ,YAAY,OACtB,MAAM,IAAI,MAAM,OAAO,SAAS,2CAA2C;KAC7E,SAAS,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;IAC5D;GACF,SAAS,OAAO;IACd,WAAW;KACT;KACA,YAAY,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;KAChD,SAAS;KACT,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;IAC9D;GACF;GACA,OAAO,QAAQ,KAAK,QAAQ;GAC5B,OAAO;EACT;EACA,MAAM,KAAK,MAAM,OAAO,QAAQ;GAC9B,IAAI,EAAE,OAAO,YAAY,KAAK,OAAO,aAAa,IAChD,MAAM,IAAI,MAAM,+CAA+C;GACjE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO;GAC5C,OAAO,YAAY,IAAI,IAAI,UAAU;IACnC,OAAO,eAAe;IACtB,MAAM,QAAQ,MAAM,KAAK;IACzB,IAAI,MAAM,KAAK,GAAG,OAAO;IACzB,MAAMA,WACJ,KAAK,IAAI,OAAO,YAAY,KAAK,IAAI,GAAG,WAAW,YAAY,IAAI,CAAC,CAAC,GACrE,KAAA,GACA,EAAE,OAAO,CACX;GACF;EAEF;CACF;CACA,IAAI;EACF,OAAO,eAAe;EACtB,MAAM,QAAQ,MAAM,KAAK,MAAM,KAAK;EACpC,KAAK,MAAM,QAAQ,KAAK,OAAO;GAC7B,OAAO,eAAe;GACtB,IAAI,KAAK,QAAQ,CAAC,KAAK,KAAK,KAAK,GAAG;GACpC,MAAM,SAAS,MAAM,MAAM,QAAQ,KAAK,YAAY,KAAK,IAAI,OAAO,KAAK,CAAC;GAC1E,MAAM,MAAM,GAAI,KAAK,SAAS,OAAO,MAAM,KAAK,CAAC,CAAE;EACrD;EACA,OAAO,eAAe;EACtB,MAAM,MAAM,GAAI,KAAK,eAAe,KAAK,KAAK,CAAC,CAAE;EACjD,OAAO,WAAW,KAAK,WAAW,KAAK;CACzC,SAAS,OAAO;EACd,OAAO,QAAQ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;CACtE,UAAU;EACR,KAAK,MAAM,UAAU,YAAY,QAAQ,GAAG;GAC1C,MAAM,WAAW,MAAM,MAAM,QAAQ,WAAW,MAAM;GACtD,IAAI,CAAC,SAAS,SACZ,MAAM,MAAM;IAAE,MAAM;IAAW,QAAQ;IAAQ,SAAS,SAAS;GAAM,CAAC;EAC5E;CACF;CACA,OAAO,OACL,CAAC,OAAO,SACR,OAAO,QAAQ,OAAO,MAAM,EAAE,OAAO,KACrC,OAAO,WAAW,OAAO,MAAM,EAAE,QAAQ,EAAE,WAAW,MAAM;CAC9D,OAAO;AACT"}

package/dist/supervisor-run/index.d.ts

@@ -438,7 +438,17 @@ export declare function supervisorRunRolloutLines(src: SupervisorRunSources, opt
  * count was unavailable are counted separately from cells that measured zero.
  */
 export declare function rollupSupervisorRuns(reports: readonly SupervisorRunReport[]): SupervisorRunRollup;
+/**
+ * Optional Runtime run-dir files, read as bytes.
+ *
+ * `coordination-log.jsonl` is Runtime's JSONL of `{ runId, seq, at, event }`:
+ * questions, steers, answers and peer mail between nodes. The `team-shape`
+ * analyst reads communication and steer edges from it.
+ */
+declare const readFileText: (path: string) => Promise<string>;
 interface RuntimeReaderOptions {
+  /** Read original UTF-8 bytes at logical run paths. Only ENOENT means absence; other failures propagate. */
+  readonly readFile?: typeof readFileText;
   /**
    * Throw on a missing spawn journal instead of returning absent-shaped
    * sources. The default (false) models a journal-less run dir — a

package/dist/supervisor-run/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/supervisor-run/analyze.ts","../../src/supervisor-run/claude-code-reader.ts","../../src/supervisor-run/discovery-rollouts.ts","../../src/supervisor-run/integrity-types.ts","../../src/supervisor-run/integrity.ts","../../src/supervisor-run/patch-stats.ts","../../src/supervisor-run/reader.ts","../../src/supervisor-run/render.ts","../../src/supervisor-run/report-command.ts","../../src/supervisor-run/source-facts.ts","../../src/supervisor-run/rollout-nodes.ts","../../src/supervisor-run/rollup.ts","../../src/supervisor-run/runtime-reader.ts","../../src/supervisor-run/terminal-record.ts"],"mappings":";;;;;;wBAuCgB,4BACd,KAAK,sBACL,qBACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UCgCc;;WAEN;;;;;WAKA;WACA;WACA;WACA;WACA;WACA;WACA;;;wBCpDK,mBAAmB;UAoFlB;EACf;EACA;EACA;EACA;;;;;;;;wBASoB,yBAAyB,iBAAiB,QAAQ;UA8EvD;EACf;EACA;EACA,OAAO;EACP,MAAM;;EAEN;;;wBAIoB,yBACpB,gBACA;EAAW;IACV,QAAQ;qBC7NE;KAED;KAEA;KAKA;UAqCK;WACN;WACA;;UAGM;WACN,MAAM;WACN;WACA,UAAU;WACV;WACA;WACA;WACA,mBAAmB;WACnB;WACA,WAAW,SAAS;;UAGd;WACN,eAAe;WACf;WACA;WACA,MAAM;WACN,iBAAiB;;UAGX;;WAEN;;;;;;wBC9BK,8BACd,OAAO,uBAAuB,mBAC9B,UAAS,gCACR;;wBC5Ca,WAAW,eAAe;;;;;;wBCgBpB,qBACpB,gBAAgB,sBAAsB,uBACrC,QAAQ;UAYM;;WAEN;;WAEA;;;;;;WAMA;;;;;;;;wBAeW,sBAAsB,eAAe;;;;;wBCtC3C,4BAA4B,GAAG;wBA8E/B,4BAA4B,GAAG;wBAsL/B,+BACd,QAAQ,qBACR;;;;;;;;;;;KCzQU;;UAQK;WACN,SAAS;WACT,SAAS;;UCIV;EACR;EACA;EACA;EACA;;EAEA;;;;;;EAMA;;UAGQ;WACC,YAAY,kBAAkB;EACvC,QAAQ;;;;;;EAMR;EACA;;;;;;EAMA;;UA6Ge;;EAEf;EACA;EACA;EACA;;EAEA,MAAM;;EAEN;;EAEA;EACA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA,OAAO;;EAEP;;UAGe;;EAEf;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;EAEA;EACA;;EAEA;;EAEA;EACA;EACA;EACA;EACA,eAAe;EACf,uBAAuB;;UAGR;;EAEf;EACA;;UAGe;;EAEf;;EAEA;EACA;;;;;;UAOe;WACN,2BAA2B;WAC3B;WACA,iBAAiB;WACjB,iBAAiB;WACjB,uBAAuB;WACvB,uBAAuB;WACvB;;;;;;;IAOP;IACA;IACA;IACA;;IAEA;;IAEA;;IAEA;IACA;;IAEA;;IAEA;IACA;;IAEA;;WAEO,YAAY,oBAAoB;;WAEhC,wBAAwB;;WAExB;;;;;;;;;WASA;;WAEA;;;;;;WAMA,0BAA0B,SAAS;;;;;;;;WAQnC;;WAEA,gBAAgB;;WAEhB,OAAO;WACP;WACA;;;;;;WAMA;WACA;;;;;;;;;;;;;;;;;;KAmDC;wBAmdI,oBAAoB,KAAK,uBAAuB;UCryB/C;;WAEN;;WAEA,QAAQ;;WAER;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA;WACA;WACA;WACA;;WAEA;;;;;;;wBAwBK,0BACd,KAAK,sBACL,OAAM,2BACL;;;;;;wBCzFa,qBAAqB,kBAAkB,wBAAwB;UC6rB9D;;;;;;WAMN;;;;;;;;;;;;;;;;;;;wBA0EW,yBACpB,gBACA,OAAM,uBACL,QAAQ;;;;;;wBAwFW,0BAA0B,iBAAiB;;qBC71BpD;;qBAIA;UAEI;;WAEN,OAAO;;WAEP,QAAQ;;;;;WAKR,SAAS;;UAGH;WACN,WAAW;WACX,iBAAiB,SAAS;;;;;WAK1B,WAAW;;WAEX,SAAS,SAAS;;;;;WAKlB;;wBAmBK,mBAAmB,OAAO,sBAAsB"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/supervisor-run/analyze.ts","../../src/supervisor-run/claude-code-reader.ts","../../src/supervisor-run/discovery-rollouts.ts","../../src/supervisor-run/integrity-types.ts","../../src/supervisor-run/integrity.ts","../../src/supervisor-run/patch-stats.ts","../../src/supervisor-run/reader.ts","../../src/supervisor-run/render.ts","../../src/supervisor-run/report-command.ts","../../src/supervisor-run/source-facts.ts","../../src/supervisor-run/rollout-nodes.ts","../../src/supervisor-run/rollup.ts","../../src/supervisor-run/coordination-reader.ts","../../src/supervisor-run/runtime-reader.ts","../../src/supervisor-run/terminal-record.ts"],"mappings":";;;;;;wBAuCgB,4BACd,KAAK,sBACL,qBACC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UCgCc;;WAEN;;;;;WAKA;WACA;WACA;WACA;WACA;WACA;WACA;;;wBCpDK,mBAAmB;UAoFlB;EACf;EACA;EACA;EACA;;;;;;;;wBASoB,yBAAyB,iBAAiB,QAAQ;UA8EvD;EACf;EACA;EACA,OAAO;EACP,MAAM;;EAEN;;;wBAIoB,yBACpB,gBACA;EAAW;IACV,QAAQ;qBC7NE;KAED;KAEA;KAKA;UAqCK;WACN;WACA;;UAGM;WACN,MAAM;WACN;WACA,UAAU;WACV;WACA;WACA;WACA,mBAAmB;WACnB;WACA,WAAW,SAAS;;UAGd;WACN,eAAe;WACf;WACA;WACA,MAAM;WACN,iBAAiB;;UAGX;;WAEN;;;;;;wBC9BK,8BACd,OAAO,uBAAuB,mBAC9B,UAAS,gCACR;;wBC5Ca,WAAW,eAAe;;;;;;wBCgBpB,qBACpB,gBAAgB,sBAAsB,uBACrC,QAAQ;UAYM;;WAEN;;WAEA;;;;;;WAMA;;;;;;;;wBAeW,sBAAsB,eAAe;;;;;wBCtC3C,4BAA4B,GAAG;wBA8E/B,4BAA4B,GAAG;wBAsL/B,+BACd,QAAQ,qBACR;;;;;;;;;;;KCzQU;;UAQK;WACN,SAAS;WACT,SAAS;;UCIV;EACR;EACA;EACA;EACA;;EAEA;;;;;;EAMA;;UAGQ;WACC,YAAY,kBAAkB;EACvC,QAAQ;;;;;;EAMR;EACA;;;;;;EAMA;;UA6Ge;;EAEf;EACA;EACA;EACA;;EAEA,MAAM;;EAEN;;EAEA;EACA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA,OAAO;;EAEP;;UAGe;;EAEf;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;EAEA;EACA;;EAEA;;EAEA;EACA;EACA;EACA;EACA,eAAe;EACf,uBAAuB;;UAGR;;EAEf;EACA;;UAGe;;EAEf;;EAEA;EACA;;;;;;UAOe;WACN,2BAA2B;WAC3B;WACA,iBAAiB;WACjB,iBAAiB;WACjB,uBAAuB;WACvB,uBAAuB;WACvB;;;;;;;IAOP;IACA;IACA;IACA;;IAEA;;IAEA;;IAEA;IACA;;IAEA;;IAEA;IACA;;IAEA;;WAEO,YAAY,oBAAoB;;WAEhC,wBAAwB;;WAExB;;;;;;;;;WASA;;WAEA;;;;;;WAMA,0BAA0B,SAAS;;;;;;;;WAQnC;;WAEA,gBAAgB;;WAEhB,OAAO;WACP;WACA;;;;;;WAMA;WACA;;;;;;;;;;;;;;;;;;KAmDC;wBAmdI,oBAAoB,KAAK,uBAAuB;UCryB/C;;WAEN;;WAEA,QAAQ;;WAER;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA;WACA;WACA;WACA;;WAEA;;;;;;;wBAwBK,0BACd,KAAK,sBACL,OAAM,2BACL;;;;;;wBCzFa,qBAAqB,kBAAkB,wBAAwB;;;;;;;;cCElE,eAAgB,iBAAe;UCgsB3B;;WAEN,kBAAkB;;;;;;WAMlB;;;;;;;;;;;;;;;;;;;wBAqEW,yBACpB,gBACA,OAAM,uBACL,QAAQ;;;;;;wBA0FW,0BAA0B,iBAAiB;;qBCj2BpD;;qBAIA;UAEI;;WAEN,OAAO;;WAEP,QAAQ;;;;;WAKR,SAAS;;UAGH;WACN,WAAW;WACX,iBAAiB,SAAS;;;;;WAK1B,WAAW;;WAEX,SAAS,SAAS;;;;;WAKlB;;wBAmBK,mBAAmB,OAAO,sBAAsB"}

package/dist/supervisor-run/index.js

@@ -1,7 +1,7 @@
 import { t as roundDecimal } from "../round-decimal-ZmjAQPo1.js";
 import { n as SUPERVISOR_RUN_INTEGRITY_SCHEMA, t as analyzeSupervisorRunIntegrity } from "../integrity-B1V6klyO.js";
 import { _ as showMeasured, a as readTerminalRecord, g as isUnavailable, i as RUNTIME_FAILED_STATUS, m as SUPERVISOR_RUN_ROLLUP_SCHEMA, r as NO_TERMINAL_RECORD, t as supervisorRunRolloutLines, u as parseSupervisorTree, v as unavailable } from "../rollout-nodes-BHR9Q5jP.js";
-import { a as findSupervisorRunDirs, c as discoveryRunRolloutLines, d as analyzeSupervisorRunSources, f as parsePatch, i as analyzeSupervisorRun, l as isRuntimeSupervisorRunDir, n as renderSupervisorRunHeadline, o as discoveryPlayCheckReward, r as renderSupervisorRunMarkdown, s as discoveryPursuitOf, t as renderSupervisorRollupMarkdown, u as readRuntimeSupervisorRun } from "../render-aB58Be3N.js";
+import { a as findSupervisorRunDirs, c as discoveryRunRolloutLines, d as analyzeSupervisorRunSources, f as parsePatch, i as analyzeSupervisorRun, l as isRuntimeSupervisorRunDir, n as renderSupervisorRunHeadline, o as discoveryPlayCheckReward, r as renderSupervisorRunMarkdown, s as discoveryPursuitOf, t as renderSupervisorRollupMarkdown, u as readRuntimeSupervisorRun } from "../render-CFrJ7M0o.js";
 // src/supervisor-run/rollup.ts
 /** Aggregate supervisor reports while retaining missing-measurement denominators. */
 /**

package/dist/traces.d.ts

@@ -4,8 +4,8 @@ import { a as RunRecord, c as RunSplitTag, d as RunTokenUsage, u as RunTerminalO
 import { $ as QueryTracesPage, G as ReadSpanSourceResult, H as createBoundedTraceAnalysisStore, J as TraceAnalysisStore, K as SpanSourceReader, Q as ErrorCluster, U as BoundedTraceAnalysisStoreOptions, W as ReadSpanSourceInput, X as DEFAULT_TRACE_ANALYST_BUDGETS, Y as TraceAnalysisStoreContext, Z as DatasetOverview, at as TraceAnalystSpan, ct as TraceAnalystTraceSummary, dt as ViewTraceResult, et as SearchSpanResult, it as TraceAnalystFilters, lt as ViewSpansResult, nt as SpanMatchRecord, ot as TraceAnalystSpanKind, q as TRACE_ANALYSIS_LIMITS, rt as TraceAnalystByteBudgets, st as TraceAnalystSpanStatus, tt as SearchTraceResult, ut as ViewTraceOversized } from "./types-DzCO-S1m.js";
 import { G as InMemoryRawProviderSink, J as ProviderRedactor, K as InMemoryRawProviderSinkOptions, Q as RawProviderSinkFilter, U as FileSystemRawProviderSink, W as FileSystemRawProviderSinkOptions, X as RawProviderEvent, Y as RawProviderDirection, Z as RawProviderSink, q as NoopRawProviderSink } from "./types-0KglnQ9o.js";
 import { a as RunFilter, i as InMemoryTraceStore, n as FileSystemTraceStore, o as SpanFilter, r as FileSystemTraceStoreOptions, s as TraceStore, t as EventFilter } from "./store-CMs5wGXP.js";
-import { d as BuildTraceAnalysisToolsOptions, f as TRACE_ANALYST_TOOL_NAMESPACE, h as buildTraceAnalysisToolDescriptors, m as UNTRUSTED_TRACE_TEXT, p as TraceAnalysisToolDescriptor } from "./engine-CaHaCc3W.js";
 import { a as SpanHandle, i as RunCompleteHookContext, n as CaptureWrite, o as TraceEmitter, r as RunCompleteHook, s as TraceEmitterOptions, t as CaptureErrorHandler } from "./emitter-mAXC5PSN.js";
+import { d as BuildTraceAnalysisToolsOptions, f as TRACE_ANALYST_TOOL_NAMESPACE, h as buildTraceAnalysisToolDescriptors, m as UNTRUSTED_TRACE_TEXT, p as TraceAnalysisToolDescriptor } from "./engine-CaHaCc3W.js";
 import { C as classifyKey, E as normalizeKey, _ as redactText, a as RedactionFinding, c as SafetyCategory, d as ShareSafetyVerdict, f as assessShareSafety, g as redactForShare, h as redact, i as RedactOptions, l as ShareSafetyFinding, m as emptyRedactionReport, o as RedactionProfile, p as combineVerdicts, r as REDACTION_VERSION, s as RedactionReport, t as AssessOptions, u as ShareSafetyStatus, v as shareAllowed, w as detectCredential } from "./redact-jrT0APTU.js";
 import { A as TraceInsightReadiness, B as isTraceInsightEvidence, C as TraceInsightContext, D as TraceInsightPromptInput, E as TraceInsightPanelRole, F as describeTraceInsightScope, G as AnalyzeTracesResult, H as tokenizeDomainWords, I as domainEvidencePattern, K as analyzeTraces, L as inferDomainKeywords, M as TraceInsightTask, N as buildTraceInsightContext, O as TraceInsightQualityGate, P as buildTraceInsightPrompt, R as isTraceInsightArtifact, S as TraceInsightAttribution, T as TraceInsightFinding, U as AnalyzeTracesInput, V as scoreTraceInsightReadiness, W as AnalyzeTracesOptions, _ as ToolSpansToTraceAnalysisStoreOptions, a as captureFetchToRawSink, b as OtlpSpan, c as ExtractedUsage, d as extractUsageFromSse, f as OtlpFileTraceStore, g as spanRecordsToTraceAnalysisStore, h as otlpTextToTraceAnalysisStore, i as CaptureFetchOptions, j as TraceInsightSuite, k as TraceInsightQuestion, l as SseUsageMode, m as SpanRecordsToTraceAnalysisStoreOptions, n as toolSpansToTraceAnalysisStore, o as OtlpFlatLine, p as OtlpFileTraceStoreOptions, r as CaptureFetchContext, s as ExtractUsageFromSseOptions, t as ToolTraceMissingError, u as extractUsage, v as OtlpExport, w as TraceInsightEvidence, x as exportRunAsOtlp, y as OtlpResourceSpans, z as isTraceInsightAttribution } from "./store-tool-spans-lWf6INTH.js";
 import { a as RunIntegrityReport, i as RunIntegrityIssueCode, n as RunIntegrityExpectations, o as assertRunCaptured, r as RunIntegrityIssue, t as RunIntegrityError } from "./integrity-BAdVzT8u.js";

package/dist/wire/index.d.ts

@@ -1,7 +1,7 @@
 import { m as CustomTokenPricing, o as CostLedgerHandle } from "../cost-ledger-Czro9wmo.js";
 import { x as ChatClient } from "../types-0KglnQ9o.js";
 import { s as TraceStore } from "../store-CMs5wGXP.js";
-import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-Ck7K7AhK.js";
+import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-BgomnFOe.js";
 import { z } from "zod";
 import { ServerType } from "@hono/node-server";
 import { Hono } from "hono";
@@ -213,9 +213,9 @@ export declare const FeedbackTrajectorySchema: z.ZodObject<{
       value: z.ZodUnknown;
       reason: z.ZodOptional<z.ZodString>;
       severity: z.ZodOptional<z.ZodEnum<{
+        error: "error";
         critical: "critical";
         info: "info";
-        error: "error";
         warning: "warning";
       }>>;
       createdAt: z.ZodString;
@@ -249,9 +249,9 @@ export declare const FeedbackTrajectorySchema: z.ZodObject<{
     value: z.ZodUnknown;
     reason: z.ZodOptional<z.ZodString>;
     severity: z.ZodOptional<z.ZodEnum<{
+      error: "error";
       critical: "critical";
       info: "info";
-      error: "error";
       warning: "warning";
     }>>;
     createdAt: z.ZodString;

package/docs/concepts.md

@@ -56,6 +56,23 @@ Paired lift, failure clusters, contamination checks, and outcome associations re
 Run records, scenarios, judge scores, statistics, and release decisions belong here because they work without an agent runtime.
 Agent sessions, worker coordination, sandbox execution, and runtime-specific profiles belong in `agent-runtime`.
 
+## Product integration scenarios
+
+`runSandboxScenario()` from `@tangle-network/agent-eval/campaign` runs a
+`HarnessScenario`: setup, named steps, expectations and a time budget.
+The product supplies transport callbacks and keeps its SDK, credentials and
+resource authorization. Steps receive one scope for measurement, assertions,
+condition polling and retirement registration. Register retirement immediately
+after allocation; callbacks use the scope signal for workload cancellation.
+Retirements run in reverse order even after setup or workload failure, and a
+failed retirement remains a failed assertion in the report.
+
+Pass the product's existing `measure` adapter to retain its server timing
+ledgers. A poll returns `undefined` when its budget expires; the scenario must
+assert the condition when exhaustion is a failure. The scope's budget signal
+is cooperative: callbacks must pass it through to their transport. Campaigns
+remain the owner of scenario scheduling, retries, artifacts and run reports.
+
 ## Measuring a complete profile
 
 Use the profile improvement functions from `/contract` when a host owns immutable agent-profile snapshots.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {