@tangle-network/agent-eval 1.0.1 → 1.0.2

package/CHANGELOG.md

@@ -4,6 +4,129 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [1.0.2] — 2026-10-29
+
+- Read archived Runtime bytes without changing logical run identity (#1159)
+- fix(adopt-stack): set up pnpm with action-setup v6.1.0 so pnpm 12 callers install (#1158)
+- fix(adopt-stack): plan and deliver on every manual run (#1157)
+- chore(release): 1.0.1
+- Refresh the live benchmark dependency identity after interface updates
+- Preserve exact execution controls in evaluation-backed analysts
+- ci(release): admit the release version before publishing (#1154)
+- fix(adopt-stack): dedupe the lockfile after locking the planned versions (#1153)
+- wip
+- wip
+- wip
+- Lock per importer, keep npm ranges, deliver after 20 h
+- Adopt compatible releases through the lockfile, once a day
+- docs(readme): drop the unmeasured 30-second promise; link the no-change gate example (#1151)
+- fix(adopt-stack): restart an updater-only proposal that no longer merges (#1147)
+- fix(deps): adopt agent-interface 5.10.2 (#1150)
+- fix(deps): adopt agent-interface 5.10.1 for the Zod advisory (#1149)
+- feat(statistics): export meanConfidenceInterval and studentTCritical, a one-sample Student-t interval on the mean
+- factory: Student t mean interval in agent eval
+- feat(analyst): export the behavioral failure-class mapping, its load-time guard and pinBehavioralFailureClass
+- factory: Failure mapping in agent eval
+- fix(adopt-stack): an untracked file the gate wrote does not change the gated tree (#1145)
+- refactor(trace): OTLP ids, encoding and decoding from agent-trace-contract (#1144)
+- feat(redaction): publish shared detectors with safe substring ranges (#1139)
+- chore(release): 1.0.0 (#1142)
+- test(adopt-stack): explain a refusal by any peer whose latest is an excluded major; post-pin cases use a manifest with no 0.x peer
+- test(adopt-stack): the live-registry plans hold across a package's first 1.0 release
+- deps: repin the analyst benchmark's dependency-lock digest
+- deps: lock agent-core 1.0.0, harness-sessions 1.0.1, agent-interface 5.8.1 and agent-trace-contract 2.2.0
+- deps: floor agent-interface at 5.8.1 and agent-trace-contract at 2.2.0
+- deps: agent-core ^1.0.0 and harness-sessions ^1.0.1
+- chore(release): 0.223.0 (#1140)
+- feat(storyboard): one tool classifier, classifyTool, shared by scenes and extractVisual
+- refactor: use statistics mean helpers instead of private copies (#1137)
+- fix(adopt-stack): restart a branch that outlived another's squash merge (#1136)
+- fix(analyst): repin the benchmark dependency lock after the zod range (#1135)
+- chore(release): 0.222.1 (#1134)
+- fix(deps): depend on zod ^4.6.6 so consumers resolve one copy (#1133)
+- chore(test): retire incomplete Runtime adoption snapshot
+- fix(release): build public exports before integration tests
+- chore(release): 0.222.0
+- feat(campaign): share concurrency for measured scenario adapters
+- fix(supervisor-run): staged play-check reward is stages passed, read from the SCORE line only (#1126)
+- refactor(statistics): delete duplicate numeric summary helpers
+- fix(rollout): import survives a damaged deliverables archive; named ids are not keys (#1125)
+- feat(supervisor-run): rollouts importer for Discovery runs, harness sessions and gate receipts (#1124)
+- feat(statistics): extend series summaries with nearest-rank evidence
+- test(rollout): isolate bundle permission probe from runner identity
+- fix(rollout): license class is provenance and an opt-in filter, not a default exclusion (#1121)
+- fix(rollout): allow nonroot images to read the base bundle
+- feat(rollout): license class on rollout lines and whole-row privacy drop in rollout-release (#1119)
+- test(rollout): inherit configured identity for Git fixtures
+- test(rollout): capture empty tool definitions at the ledger boundary
+- feat(rollout): write verified episodes as Harbor tasks
+- feat(rollout): project captured policy steps into Tinker data
+- adopt-stack: delete the adoption branch when its squash merge lands (#1115)
+- chore(deps)!: require Python 3.11 for patched locks and refresh dependencies
+- feat(rollout): read retained Codex sessions through the shared reader
+- chore(release): 0.221.8
+- test(adopt): retain failed browser child outcomes
+- docs: leave changelog preparation to the release workflow
+- chore(release): 0.221.7
+- feat(multishot): conversation legs report the USD they spent
+- chore(release): 0.221.6
+- docs(traces): bound the incomplete usage coverage claim
+- feat: own judge-panel calibration, multishot conversation legs and served-panel identity
+- test(traces): typecheck OTLP usage integration proof
+- fix(traces): retain incomplete OTLP token usage
+- chore(release): 0.221.5
+- ci: retain private adopter failure evidence (#1106)
+- test(process): isolate process-tree fixture environment
+- fix(adopt): invoke each resolved Playwright CLI (#1096)
+- test(systemone): close disposable provider connections
+- chore(release): 0.221.4
+- test(adoption): inherit the maintained integration deadline
+- fix(adoption): scope guard allowance to disposable Git fixtures
+- fix(analyst): carry root-cause target into failure-mode rubric
+- ci: check locally and in publication paths
+- test(adopt): give real-git fixtures their own commit identity (#1101)
+- chore(release): 0.221.3 (#1099)
+- fix(analyst): repin the benchmark implementation digest after #1097 (#1098)
+- fix(trace-analyst): build OTLP stores from bytes, not one joined string
+- fix(adopt): prepare caller-locked Chromium before gates
+- test(adopt): retain admitted identity in real Git fixtures
+- fix(adopt): retain the sparse caller entrypoint
+- fix(adopt): settle pushed proposal reads against remote refs
+- chore(release): 0.221.2
+- Return plain analyst findings from decision replay
+- chore(release): 0.221.1 (#1091)
+- fix(deps): pin zod 4.6.6 (GHSA-mrhx-c7xc-m3w5) (#1088)
+- Give the disposable adoption fixture its own Git author
+- Fix real adoption drift fixture under the declared package manager
+- Capture supporting trace events in the original native decision
+- test: inherit the integration default for behavioral questions
+- test: set the integration default from the observed loaded tail
+- chore(release): 0.221.0
+- test: own the measured integration deadline in Vitest config
+- test: await observed integration completion across tool paths
+- feat: preserve finding composition and assigned arm populations
+- test: await decision-point HTTP and ledger completion
+- test: await durable readiness in filesystem integration cases
+- test(process): wait for argv child readiness before abort
+- feat(analyst): add bounded calibrated trace question observations
+- fix(analyst): reject surrounding whitespace in caller gold ids
+- test(analyst): reproduce whitespace source-id bypass
+- test(optimizer): retire fake CLI metering harness
+- test(analyst): isolate recorded gold admission scenarios
+- fix(systemone): validate decision boundaries before routing
+- ci: pin the eval publication dispatcher body
+- fix(analyst): reserve excluded source ids across caller gold
+- fix(analyst): validate root provenance before gold admission
+- fix(analyst): preserve actor boundaries in AgentRx calibration gold
+- Preserve manual drafts observed while recording failed adoption (#1076)
+- Recover updater-owned failure drafts while preserving manual holds (#1075)
+- feat(systemone): persist calibrated decision points and outcomes
+- ci: move eval verification and Python publishing to owned runners
+- chore(deps): bump opentelemetry-api in /clients/python (#1068)
+- fix(adopt): quote optional workflow flags
+- fix(adopt): preserve shallow history and caller-owned dependency references
+- fix(adopt): reconcile effective workspace pins and preserve proposal history
+
 ## [1.0.1] — 2026-10-28
 
 - Refresh the live benchmark dependency identity after interface updates

package/dist/cli.js

@@ -9,7 +9,7 @@ import { t as createChatClient } from "./chat-client-D0me8iwq.js";
 import { t as runAnalystBenchmarkCommand } from "./benchmark-command-BeDZB3BQ.js";
 import { i as buildOpenApi, l as handleVersion, o as runRpcBatch, r as startServerAsync, s as runRpcOnce } from "./server-CPKyf3bz.js";
 import { b as editCreditText, f as landscape, g as surfaceTextEdits, l as metaSearch, m as formatLandscape, n as taskMatrix, o as skillManifold, r as formatSkillManifold, s as renderMetaSearchText, t as tree, v as front, y as editCredit } from "./tree-C2N31CoI.js";
-import { c as discoveryRunRolloutLines, i as analyzeSupervisorRun, n as renderSupervisorRunHeadline, r as renderSupervisorRunMarkdown } from "./render-aB58Be3N.js";
+import { c as discoveryRunRolloutLines, i as analyzeSupervisorRun, n as renderSupervisorRunHeadline, r as renderSupervisorRunMarkdown } from "./render-CFrJ7M0o.js";
 import { createHash } from "node:crypto";
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "1.0.1",
+    "version": "1.0.2",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.1.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{render-aB58Be3N.js → render-CFrJ7M0o.js}

@@ -682,16 +682,17 @@ function analyzeSupervisorRunSources(src, now = Date.now) {
 * analyst reads communication and steer edges from it.
 */
 const COORDINATION_FILE = "coordination-log.jsonl";
+const readFileText = (path) => readFile(path, "utf8");
 /** File text, or null when the file does not exist. Any other read error throws. */
-async function readMaybe(path) {
-	return readFile(path, "utf8").catch((error) => {
+async function readMaybe(path, read = readFileText) {
+	return read(path).catch((error) => {
 		if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") return null;
 		throw error;
 	});
 }
 /** The run's coordination log, or null when Runtime wrote none. */
-function readCoordinationLog(runDir) {
-	return readMaybe(join(runDir, COORDINATION_FILE));
+function readCoordinationLog(runDir, read = readFileText) {
+	return readMaybe(join(runDir, COORDINATION_FILE), read);
 }
 // src/supervisor-run/runtime-source-limits.ts
 function spendRecord(value) {
@@ -1015,10 +1016,10 @@ const HARNESS_SESSION_FILE = {
 	opencode: /(^|\/)(\.local\/share\/opencode\/(opencode\.db|storage\/session\/.+\.json|export\/.+\.json)|\.opencode\/sessions\/.+)$/u
 };
 /** Session files the receipt's transcript blob lists, or undefined when it cannot say. */
-async function receiptSessionFiles(file) {
+async function receiptSessionFiles(file, read) {
 	let descriptor;
 	try {
-		descriptor = JSON.parse(await readFile(file, "utf8"));
+		descriptor = JSON.parse(await read(file));
 	} catch {
 		return;
 	}
@@ -1055,7 +1056,7 @@ function nativeTraceSummary(workers, root) {
 	const gaps = [count("partial") > 0 ? `${count("partial")} only a partial copy` : null, count("no-session-file") > 0 ? `${count("no-session-file")} a receipt with no session file` : null].filter((gap) => gap !== null);
 	return `per node — ${count("complete")} of ${workers.length} workers carry a complete native session receipt (workers[].nativeSession)${gaps.length > 0 ? `; ${gaps.join("; ")}` : ""}; root: ${rootText} (result.json rootHarnessTranscript)`;
 }
-async function workspaceCapturesFromOutput(runDir, ref) {
+async function workspaceCapturesFromOutput(runDir, ref, read) {
 	const outRef = typeof ref === "string" ? ref : null;
 	const blobPath = blobFile(runDir, ref);
 	const missing = (projectionGap) => ({
@@ -1070,7 +1071,7 @@ async function workspaceCapturesFromOutput(runDir, ref) {
 	});
 	if (blobPath === null) return [missing("Runtime output has no valid content address")];
 	if (!await isFile(blobPath)) return [missing("Runtime output blob is missing or not a file")];
-	const raw = await readMaybe(blobPath);
+	const raw = await readMaybe(blobPath, read);
 	if (raw === null) return [missing("Runtime output blob is missing")];
 	let output;
 	try {
@@ -1146,7 +1147,7 @@ async function workspaceCapturesFromOutput(runDir, ref) {
 * Blob files are checked on disk: a receipt that names a blob the directory no longer holds is
 * not a retained transcript.
 */
-async function nodeTranscripts(runDir, events, nodeIds, root) {
+async function nodeTranscripts(runDir, events, nodeIds, read, root) {
 	const facts = /* @__PURE__ */ new Map();
 	const entry = (id) => {
 		let found = facts.get(id);
@@ -1193,14 +1194,14 @@ async function nodeTranscripts(runDir, events, nodeIds, root) {
 				retained.push(file);
 				if (turn) retainedTurns += 1;
 			}
-			workspaceCaptures.push(...await workspaceCapturesFromOutput(runDir, ref));
+			workspaceCaptures.push(...await workspaceCapturesFromOutput(runDir, ref, read));
 		}
 		let nativeSession = null;
 		if (fact.receipt !== null) {
 			if (fact.receipt.status === "available") {
 				const file = blobFile(runDir, fact.receipt.transcriptRef);
 				if (file !== null && await isFile(file)) {
-					const sessionFiles = await receiptSessionFiles(file);
+					const sessionFiles = await receiptSessionFiles(file, read);
 					const coverageComplete = fact.receipt.coverageComplete;
 					nativeSession = {
 						status: "available",
@@ -1230,11 +1231,6 @@ async function nodeTranscripts(runDir, events, nodeIds, root) {
 	return out;
 }
 /**
-* Sources for a run dir whose spawn journal does not exist. Mirrors the
-* absent shape the reader returns for a missing store: every
-* journal-dependent metric downstream reads `unavailable`, never 0.
-*/
-/**
 * The run identity a terminal record names when no journal exists yet: the
 * settle record's `tree.root`, else the failure record's `runId`. Both are
 * Runtime's own `runId`, so the report names the run the record is about
@@ -1290,12 +1286,13 @@ function absentRuntimeSupervisorRun(runDir, resultText, failureText, instanceId)
 * verdict.
 */
 async function readRuntimeSupervisorRun(runDir, opts = {}) {
+	const read = opts.readFile ?? readFileText;
 	const journalPath = join(runDir, JOURNAL_FILE);
-	const rawJournal = opts.strict === true ? await readFile(journalPath, "utf8") : await readMaybe(journalPath);
+	const rawJournal = opts.strict === true ? await read(journalPath) : await readMaybe(journalPath, read);
 	const resultPath = join(runDir, RESULT_FILE);
 	const failurePath = join(runDir, FAILURE_FILE);
-	const resultText = await readMaybe(resultPath);
-	const failureText = await readMaybe(failurePath);
+	const resultText = await readMaybe(resultPath, read);
+	const failureText = await readMaybe(failurePath, read);
 	const failure = parseOptionalRecord(failureText, failurePath);
 	assertFailureRecord(failure, failurePath);
 	const result = parseOptionalRecord(resultText, resultPath);
@@ -1304,7 +1301,7 @@ async function readRuntimeSupervisorRun(runDir, opts = {}) {
 	assertResultMatchesJournal(normalized.root, result, resultPath);
 	const childSpawns = normalized.events.filter((event) => event.kind === "spawned" && nonEmptyString(event.id) !== null).filter((event) => event.id !== normalized.root);
 	const workerIds = new Set(childSpawns.map((event) => nonEmptyString(event.id)).filter((id) => id !== null));
-	const transcripts = await nodeTranscripts(runDir, normalized.events, /* @__PURE__ */ new Set([normalized.root, ...workerIds]), {
+	const transcripts = await nodeTranscripts(runDir, normalized.events, /* @__PURE__ */ new Set([normalized.root, ...workerIds]), read, {
 		id: normalized.root,
 		receipt: recordOrNull(result?.rootHarnessTranscript)
 	});
@@ -1346,7 +1343,7 @@ async function readRuntimeSupervisorRun(runDir, opts = {}) {
 		judgeSource: null,
 		patch: null,
 		driverLog: null,
-		coordinationLog: await readCoordinationLog(runDir),
+		coordinationLog: await readCoordinationLog(runDir, read),
 		harnessWorkerTokens: null,
 		harnessMissingReason: "Runtime FileRunContext has no external worker-token join",
 		limits: sourceLimits(normalized.root, normalized.events, workerIds),
@@ -1365,10 +1362,13 @@ async function isRuntimeSupervisorRunDir(runDir) {
 	return false;
 }
 async function isFile(path) {
-	return stat(path).then((entry) => entry.isFile()).catch((error) => {
-		if (typeof error === "object" && error !== null && "code" in error && (error.code === "ENOENT" || error.code === "ENOTDIR")) return false;
+	try {
+		return (await stat(path)).isFile();
+	} catch (error) {
+		const code = error !== null && typeof error === "object" && "code" in error ? error.code : null;
+		if (code === "ENOENT" || code === "ENOTDIR") return false;
 		throw error;
-	});
+	}
 }
 // src/supervisor-run/discovery-rollouts.ts
 /**
@@ -1866,4 +1866,4 @@ function renderSupervisorRollupMarkdown(rollup, title = "Round rollup") {
 }
 export { findSupervisorRunDirs as a, discoveryRunRolloutLines as c, analyzeSupervisorRunSources as d, parsePatch as f, analyzeSupervisorRun as i, isRuntimeSupervisorRunDir as l, renderSupervisorRunHeadline as n, discoveryPlayCheckReward as o, renderSupervisorRunMarkdown as r, discoveryPursuitOf as s, renderSupervisorRollupMarkdown as t, readRuntimeSupervisorRun as u };
 
-//# sourceMappingURL=render-aB58Be3N.js.map
+//# sourceMappingURL=render-CFrJ7M0o.js.map