@tangle-network/agent-eval 0.189.0 → 0.190.1

package/dist/index.js

@@ -1,7 +1,7 @@
 import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
 import { i as JudgeError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
 import { a as hashCanonical, r as canonicalString } from "./canonical-BsZVgBhW.js";
-import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-BOY98IJO.js";
+import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-BXjXJHJT.js";
 import { c as groupRunsByAgentProfileCell, d as validateAgentProfileCell, f as verifyAgentProfileCell, h as manifestContentDigest, i as agentProfileCellKey, m as hashJson, r as agentProfileCellHashMaterial, s as buildAgentProfileCell, t as AGENT_PROFILE_KINDS, u as toAgentProfileJson } from "./agent-profile-cell-DcFW4vOc.js";
 import { t as mulberry32 } from "./random-Dn5fPWkt.js";
 import { a as spearmanR, c as weightedMean, i as ranks, n as partialCredit, o as summarizeNumberSeries, r as pearsonR, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
@@ -21,40 +21,40 @@ import { r as observedSplitScore, t as isRealnessGated } from "./reward-nw2xZGZG
 import { a as parseRunRecordSafe, c as runTaskScore, i as modelHasSnapshot, l as searchCellRunId, n as UNKNOWN_MODEL, o as roundTripRunRecord, r as isRunRecord, s as runCostFloorUsd, t as RunRecordValidationError, u as validateRunRecord } from "./run-record-BRlM99rX.js";
 import { a as summaryTable, n as gainHistogram, r as paretoChart } from "./summary-report-BMvou2oZ.js";
 import { n as contentHash, r as fileVerdictCache, t as canonicalJson } from "./verdict-cache-BQEsVy9n.js";
-import { B as DEFAULT_RED_TEAM_CORPUS, H as redTeamReport, N as dominates, P as paretoFrontier, U as scoreRedTeamOutput, V as redTeamDataset, W as runCanaries, X as runCampaign, a as transientDispatchFailure, i as quotaExhaustedUntil, o as aggregateRunScore, s as clamp01, t as llmJudge } from "./llm-judge-M_l_WXw2.js";
+import { $ as scoreRedTeamOutput, Q as redTeamReport, U as dominates, W as paretoFrontier, X as DEFAULT_RED_TEAM_CORPUS, Z as redTeamDataset, a as transientDispatchFailure, et as runCanaries, i as quotaExhaustedUntil, o as aggregateRunScore, ot as runCampaign, s as clamp01, t as llmJudge } from "./llm-judge-BnxAv7ed.js";
 import { a as resolveModelPricing, i as isModelPriced, n as estimateCost, r as estimateTokens, t as MODEL_PRICING } from "./metrics-DOMpkoeb.js";
 import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-_IbkejCF.js";
-import { D as parseReflectionResponse, E as buildReflectionPrompt, G as summarizeBackendIntegrity, U as BackendIntegrityError, W as assertRealBackend, c as ATTESTATION_ALGORITHM, l as attest, u as verifyAttestation, z as surfaceContentHash } from "./campaign-evidence-CBlGJ3kC.js";
-import { d as redactForShare, f as redactText, n as REDACTION_VERSION, p as shareAllowed, r as assessShareSafety, u as redact } from "./redact-DxwlUAvh.js";
-import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-98sgUaC1.js";
+import { B as surfaceContentHash, D as parseReflectionResponse, E as buildReflectionPrompt, G as assertRealBackend, K as summarizeBackendIntegrity, W as BackendIntegrityError, c as ATTESTATION_ALGORITHM, l as attest, u as verifyAttestation } from "./campaign-evidence-D40-LHIL.js";
+import { d as redactForShare, f as redactText, n as REDACTION_VERSION, p as shareAllowed, r as assessShareSafety, u as redact } from "./redact-B7rt9Zot.js";
+import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-DPizhHfy.js";
 import { n as equivalenceVerdict, t as certificationEvidenceDigest } from "./verdict-DhPoXApk.js";
 import { t as TraceEmitter } from "./emitter-DbMXUIVP.js";
 import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
 import { t as runCounterfactual } from "./counterfactual-DtMWvn7D.js";
-import { H as parseFindingSubject, W as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, h as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, m as DEFAULT_TRACE_ANALYST_BUDGETS, t as createTraceAnalyst, u as createBoundedTraceAnalysisStore } from "./kind-factory-BGwN0nE6.js";
-import { _ as completedAnalystReviewQuality, b as validateAnalystReviewDecisions, c as FAILURE_MODE_KIND_SPEC, g as assertUniqueFindingIds, h as analystRunDigest, i as DEFAULT_TRACE_ANALYST_KINDS, m as analystFindingDigest, n as AnalystRegistry, t as buildDefaultAnalystRegistry, v as readAnalystReview, y as snapshotAnalystRun } from "./default-registry-m2jhbG6Q.js";
+import { G as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, U as parseFindingSubject, h as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, m as DEFAULT_TRACE_ANALYST_BUDGETS, t as createTraceAnalyst, u as createBoundedTraceAnalysisStore } from "./kind-factory-ByZeXDqF.js";
+import { _ as completedAnalystReviewQuality, b as validateAnalystReviewDecisions, c as FAILURE_MODE_KIND_SPEC, g as assertUniqueFindingIds, h as analystRunDigest, i as DEFAULT_TRACE_ANALYST_KINDS, m as analystFindingDigest, n as AnalystRegistry, t as buildDefaultAnalystRegistry, v as readAnalystReview, y as snapshotAnalystRun } from "./default-registry-16I-Bxlf.js";
 import { OUTPUT_VALUE } from "./trace-attributes.js";
 import { r as extractUsageFromSse, t as extractUsage } from "./extract-usage-BrQ8mCLX.js";
-import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-C96CcqeG.js";
-import { D as captureFetchToRawSink, S as exportRunAsOtlp, _ as tokenizeDomainWords, d as inferDomainKeywords, g as scoreTraceInsightReadiness, l as describeTraceInsightScope, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as domainEvidencePattern, v as analyzeTraces } from "./store-tool-spans-B_0mXvYl.js";
+import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-DYh4Bszw.js";
+import { S as exportRunAsOtlp, T as captureFetchToRawSink, _ as tokenizeDomainWords, d as inferDomainKeywords, g as scoreTraceInsightReadiness, l as describeTraceInsightScope, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as domainEvidencePattern, v as analyzeTraces } from "./store-tool-spans-DUiqyHvR.js";
 import { n as assertRunCaptured, t as RunIntegrityError } from "./integrity-BFP4kjGj.js";
 import { n as InMemoryTraceStore, t as FileSystemTraceStore } from "./store-DNe_Uv1Q.js";
 import { t as packageVersion$1 } from "./package-version-D7lQHt_-.js";
 import { a as classifyFailure, i as INFRA_FAILURE_BLAMES, o as classifyFailureReason, r as FAILURE_BLAME, s as failureBlame, t as DEFAULT_FAILURE_REASON_RULES } from "./failure-taxonomy-CjabTATj.js";
 import { n as runBoundedProcess, t as DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS } from "./bounded-process-BBZob7vl.js";
-import { t as runEvalCampaign } from "./eval-campaign-C0Q1Eeo_.js";
+import { t as runEvalCampaign } from "./eval-campaign-CwsBn0Gm.js";
 import "./rollout-B-XBReTC.js";
 import { t as mintRolloutRows } from "./mint-iCD-W43s.js";
 import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-CzK5DarL.js";
-import { S as judgeFamily, _ as assertServedModels, b as CrossFamilyError, d as stripFencedJson, f as ModelSubstitutionError, h as assertServedModel, i as backoffMs, l as isTransientLlmError, m as assertCrossFamilyServed, o as costReceiptFromLlm, p as ServedCrossFamilyError, r as LlmResponseError, s as costReceiptFromLlmError, t as LlmCallError, u as maximumChargeForLlmRequest, v as checkServedModel, x as assertCrossFamily, y as servedModelAcceptable } from "./llm-client-DF9bPNYA.js";
-import { n as paidJsonChat } from "./chat-json-call-JVGbfx95.js";
+import { S as judgeFamily, _ as assertServedModels, b as CrossFamilyError, d as stripFencedJson, f as ModelSubstitutionError, h as assertServedModel, i as backoffMs, l as isTransientLlmError, m as assertCrossFamilyServed, o as costReceiptFromLlm, p as ServedCrossFamilyError, r as LlmResponseError, s as costReceiptFromLlmError, t as LlmCallError, u as maximumChargeForLlmRequest, v as checkServedModel, x as assertCrossFamily, y as servedModelAcceptable } from "./llm-client-KYQBh9C8.js";
+import { n as paidJsonChat } from "./chat-json-call-CXHBvUz8.js";
 import { i as makeProposalFinding, n as computeFindingId, r as makeFinding } from "./types-CW5b90JH.js";
-import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-DLYttKBm.js";
-import { a as diffFindings, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "./semantic-concept-judge-Cm2K2wAo.js";
+import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-Cs5LzJ-f.js";
+import { a as diffFindings, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "./semantic-concept-judge-ClB73cei.js";
 import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
-import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-D0AV4cvB.js";
+import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-84DH0OEH.js";
 import { n as evaluateReleaseConfidence, r as bootstrapCi } from "./release-confidence-XQm_xjhR.js";
-import { t as createChatClient } from "./chat-client-C4dg7Ng_.js";
+import { t as createChatClient } from "./chat-client-B_ItsSBw.js";
 import { createHash } from "node:crypto";
 import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
 import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
@@ -1206,7 +1206,7 @@ function contractSpanKind(span) {
 /**
 * Tool name: `span.toolName` (typed ToolSpan) → the tool-name attributes
 * (`gen_ai.tool.name`, `tool.name`, ..., `toolName`) → `span.name` when the
-* span is a TOOL span (otel-bridge's `ExportableSpan` drops `toolName`).
+* span is a TOOL span.
 */
 function contractSpanToolName(span) {
 	if (typeof span.toolName === "string") return span.toolName;
@@ -1219,12 +1219,23 @@ function spanModel(span) {
 	if (typeof span.model === "string" && span.model.length > 0) return span.model;
 	return firstStringAttr(span.attributes ?? {}, MODEL_ATTR_KEYS);
 }
-/** Input plus output tokens, or `undefined` when either side is unrecorded. */
+/**
+* Input plus output tokens, or `undefined` when either side is unrecorded.
+* Input follows the OTel GenAI definition — prompt tokens plus any cache
+* read and cache write tokens the span records — because a cached call's
+* `INPUT_TOKEN_ATTR_KEYS` value (`llm.token_count.prompt` and its aliases)
+* carries only the uncached remainder, and a cache-heavy call can spend far
+* more on cache reads and writes than on that remainder. `llm.maxTotalTokens`
+* would otherwise pass runs at a small fraction of what they actually cost.
+*/
 function spanTotalTokens(span) {
 	const attributes = span.attributes ?? {};
 	const input = finite(span.inputTokens) ?? firstNumberAttr(attributes, INPUT_TOKEN_ATTR_KEYS);
 	const output = finite(span.outputTokens) ?? firstNumberAttr(attributes, OUTPUT_TOKEN_ATTR_KEYS);
-	return input === void 0 || output === void 0 ? void 0 : input + output;
+	if (input === void 0 || output === void 0) return void 0;
+	const cacheRead = firstNumberAttr(attributes, ["llm.token_count.prompt_cache_hit"]) ?? 0;
+	const cacheWrite = firstNumberAttr(attributes, ["llm.token_count.prompt_cache_write"]) ?? 0;
+	return input + output + cacheRead + cacheWrite;
 }
 const ARGUMENT_ATTR_KEYS = [
 	"gen_ai.tool.call.arguments",
@@ -1254,6 +1265,40 @@ function spanArguments(span) {
 		reason: "the span carries no argument evidence"
 	};
 }
+/** The OTel GenAI attribute listing the tool definitions offered to the
+*  model: an array (or its JSON string) of `{ name }`, `{ function: { name } }`
+*  or plain names. */
+const TOOL_DEFINITIONS_ATTR = "gen_ai.tool.definitions";
+/** The tool names a span records as offered to the model. */
+function spanOfferedTools(span) {
+	const raw = span.attributes?.[TOOL_DEFINITIONS_ATTR];
+	if (raw === void 0 || raw === null) return { recorded: false };
+	const value = parseJsonString(raw);
+	if (!Array.isArray(value)) return {
+		recorded: true,
+		unreadable: `${TOOL_DEFINITIONS_ATTR} is not a list`
+	};
+	const names = [];
+	for (const entry of value) {
+		const name = typeof entry === "string" ? entry : definitionName(entry);
+		if (name === void 0) return {
+			recorded: true,
+			unreadable: `${TOOL_DEFINITIONS_ATTR} has an entry without a tool name`
+		};
+		names.push(name);
+	}
+	return {
+		recorded: true,
+		names
+	};
+}
+function definitionName(entry) {
+	if (entry === null || typeof entry !== "object") return void 0;
+	const record = entry;
+	if (typeof record.name === "string" && record.name.length > 0) return record.name;
+	const fn = record.function;
+	return fn !== null && typeof fn === "object" && typeof fn.name === "string" && fn.name.length > 0 ? fn.name : void 0;
+}
 function parseJsonString(value) {
 	if (typeof value !== "string") return value;
 	try {
@@ -1402,6 +1447,14 @@ function assertJsonPointer(pointer, where) {
 	if (typeof pointer !== "string" || pointer !== "" && !pointer.startsWith("/")) throw new ValidationError(`${where}: pointer must be an RFC 6901 JSON Pointer ('' or '/a/b')`);
 	if (/~(?![01])/.test(pointer)) throw new ValidationError(`${where}: pointer has an invalid escape; use ~0 for ~ and ~1 for /`);
 }
+/** A non-empty list of distinct, non-empty tool names. */
+function assertNames(names, where) {
+	if (!Array.isArray(names) || names.length === 0) throw new ValidationError(`${where} must be a non-empty array of tool names`);
+	for (const name of names) if (typeof name !== "string" || name.length === 0) throw new ValidationError(`${where} must hold non-empty tool names`);
+	const dup = names.find((n, i) => names.indexOf(n) !== i);
+	if (dup !== void 0) throw new ValidationError(`${where} lists "${dup}" twice`);
+	return names;
+}
 function assertArgumentCheck(check, where) {
 	const c = check;
 	if (c === null || typeof c !== "object") throw new ValidationError(`${where}: check must be an object`);
@@ -1442,7 +1495,7 @@ function assertRule(rule, where) {
 			assertCount(rule.max, where);
 			return;
 		case "run":
-			if (rule.requireCompleted === void 0 && rule.allowedStatuses === void 0 && rule.maxDurationMs === void 0) throw new ValidationError(`${where}: run rule checks nothing`);
+			if (!rule.requireCompleted && rule.allowedStatuses === void 0 && rule.maxDurationMs === void 0) throw new ValidationError(`${where}: run rule checks nothing`);
 			if (rule.requireCompleted !== void 0 && typeof rule.requireCompleted !== "boolean") throw new ValidationError(`${where}: requireCompleted must be a boolean`);
 			if (rule.allowedStatuses !== void 0) {
 				if (!Array.isArray(rule.allowedStatuses) || rule.allowedStatuses.length === 0) throw new ValidationError(`${where}: allowedStatuses must be a non-empty array`);
@@ -1456,6 +1509,25 @@ function assertRule(rule, where) {
 			assertArgumentCheck(rule.check, where);
 			if (rule.occurrence !== void 0 && !ARGUMENT_OCCURRENCES.includes(rule.occurrence)) throw new ValidationError(`${where}: occurrence must be one of ${ARGUMENT_OCCURRENCES.join(", ")}`);
 			return;
+		case "toolsOffered":
+			if (rule.declared !== void 0) assertNames(rule.declared, `${where}: declared`);
+			return;
+		case "retrySafe": {
+			const reads = rule.reads === void 0 ? [] : assertNames(rule.reads, `${where}: reads`);
+			if (rule.writes === void 0 && rule.reads === void 0) throw new ValidationError(`${where}: retrySafe needs reads or writes`);
+			if (rule.writes !== void 0) {
+				if (!Array.isArray(rule.writes) || rule.writes.length === 0) throw new ValidationError(`${where}: writes must be a non-empty array`);
+				const writes = assertNames(rule.writes.map((w) => w?.tool), `${where}: writes[].tool`);
+				for (const [i, w] of rule.writes.entries()) {
+					if (w.idempotencyKey === void 0) continue;
+					assertJsonPointer(w.idempotencyKey, `${where}: writes[${i}].idempotencyKey`);
+					if (w.idempotencyKey === "") throw new ValidationError(`${where}: writes[${i}].idempotencyKey must name a member, not the whole arguments`);
+				}
+				const both = writes.find((t) => reads.includes(t));
+				if (both !== void 0) throw new ValidationError(`${where}: "${both}" is both a read and a write`);
+			}
+			return;
+		}
 		default: throw new ValidationError(`${where}: unknown rule kind "${String(rule.kind)}"`);
 	}
 }
@@ -1525,12 +1597,19 @@ function describeRule(rule) {
 		case "tokensAtMost": return `spans matching ${describePredicate(rule.p)} use at most ${rule.max} tokens in total`;
 		case "run": {
 			const parts = [];
-			if (rule.requireCompleted) parts.push("the run reached a terminal status");
+			if (rule.requireCompleted) parts.push("the run completed and recorded its end");
 			if (rule.allowedStatuses) parts.push(`its status is one of ${rule.allowedStatuses.join(", ")}`);
 			if (rule.maxDurationMs !== void 0) parts.push(`it took at most ${rule.maxDurationMs} ms`);
 			return parts.join("; ");
 		}
 		case "argument": return `${rule.occurrence ?? "all"} call(s) matching ${describePredicate(rule.p)} have argument ${rule.pointer || "(whole value)"} that ${describeCheck(rule.check)}`;
+		case "toolsOffered": return `the trace records the tools offered to the model, every tool call is one of them${rule.declared ? `, and every offered tool is one of ${rule.declared.join(", ")}` : ""}`;
+		case "retrySafe": {
+			const parts = ["no tool is called twice with the same arguments"];
+			if (rule.reads) parts.push(`except the reads ${rule.reads.join(", ")}`);
+			for (const w of rule.writes ?? []) parts.push(w.idempotencyKey === void 0 ? `write ${w.tool} may never repeat` : `write ${w.tool} may repeat only with one idempotency key at ${w.idempotencyKey}`);
+			return parts.join("; ");
+		}
 	}
 }
 /** Plain-language statement of everything a contract checks, one rule per line. */
@@ -1660,6 +1739,28 @@ var TraceContractBuilder = class TraceContractBuilder {
 		assertRule(rule, this.where("argument"));
 		return this.add(rule);
 	}
+	/** The trace records the tools offered to the model, every call is one of
+	*  them, and, with `declared`, every offered tool is declared. */
+	toolsOffered(declared, label = "toolsOffered") {
+		const rule = {
+			kind: "toolsOffered",
+			label,
+			...declared === void 0 ? {} : { declared: [...declared] }
+		};
+		assertRule(rule, this.where("toolsOffered"));
+		return this.add(rule);
+	}
+	/** No call repeats a side effect that is not proven safe to repeat. */
+	retrySafe(options, label = "retrySafe") {
+		const rule = {
+			kind: "retrySafe",
+			label,
+			...options.reads === void 0 ? {} : { reads: [...options.reads] },
+			...options.writes === void 0 ? {} : { writes: options.writes.map((w) => ({ ...w })) }
+		};
+		assertRule(rule, this.where("retrySafe"));
+		return this.add(rule);
+	}
 	/** Evaluate only the subtree rooted at the one span matching `root`. */
 	scope(root) {
 		this.scopeRoot = normalizePredicate(root, this.where("scope"));
@@ -1868,9 +1969,9 @@ function checkRun(rule, context) {
 	}];
 	const run = context.run;
 	const out = [];
-	if (rule.requireCompleted && (run.status === void 0 || run.status === "running" || finite(run.endedAt) === void 0)) out.push({
+	if (rule.requireCompleted && (run.status !== "completed" || finite(run.endedAt) === void 0)) out.push({
 		rule: rule.label,
-		detail: `run did not reach a terminal status (status ${run.status ?? "unknown"})`
+		detail: run.status === "completed" ? "run status is completed but no end time was recorded" : `run status is ${run.status ?? "unknown"}, not completed`
 	});
 	if (rule.allowedStatuses && !rule.allowedStatuses.includes(run.status ?? "")) out.push({
 		rule: rule.label,
@@ -1890,6 +1991,188 @@ function checkRun(rule, context) {
 	}
 	return out;
 }
+const listed = (names, max = 6) => names.length <= max ? names.join(", ") : `${names.slice(0, max).join(", ")} and ${names.length - max} more`;
+function toolSpans(spans) {
+	const out = [];
+	spans.forEach((span, i) => {
+		if (contractSpanKind(span) === "TOOL") out.push({
+			span,
+			ref: spanRef(span, i)
+		});
+	});
+	return out;
+}
+/** Declared, offered and called tools agree. */
+function checkToolsOffered(rule, spans) {
+	const out = [];
+	const offered = /* @__PURE__ */ new Set();
+	let recorded = 0;
+	const declared = rule.declared ? new Set(rule.declared) : void 0;
+	spans.forEach((span, i) => {
+		const tools = spanOfferedTools(span);
+		if (!tools.recorded) return;
+		recorded += 1;
+		const ref = spanRef(span, i);
+		if ("unreadable" in tools) {
+			out.push({
+				rule: rule.label,
+				spanId: span.spanId,
+				detail: `span ${ref}: ${tools.unreadable}`
+			});
+			return;
+		}
+		for (const name of tools.names) offered.add(name);
+		const undeclared = declared ? [...new Set(tools.names)].filter((n) => !declared.has(n)) : [];
+		if (undeclared.length > 0) out.push({
+			rule: rule.label,
+			spanId: span.spanId,
+			detail: `span ${ref} offered the model ${undeclared.length} undeclared tool(s): ${listed(undeclared)}`
+		});
+	});
+	if (recorded === 0) return [{
+		rule: rule.label,
+		detail: `no span records the tools offered to the model (${TOOL_DEFINITIONS_ATTR}), so what the harness enforced is unknown`
+	}];
+	for (const { span, ref } of toolSpans(spans)) {
+		const name = contractSpanToolName(span);
+		if (name === void 0) out.push({
+			rule: rule.label,
+			spanId: span.spanId,
+			detail: `span ${ref} has no tool name, so it cannot be matched to an offered tool`
+		});
+		else if (!offered.has(name)) out.push({
+			rule: rule.label,
+			spanId: span.spanId,
+			detail: `span ${ref} called ${name}, which was never offered: the harness did not enforce its tool set`
+		});
+	}
+	return out;
+}
+/** The call's idempotency key, or why it has none. */
+function idempotencyKey(span, pointer) {
+	const evidence = spanArguments(span);
+	if (!evidence.known) return { missing: evidence.reason };
+	const resolved = resolveJsonPointer(evidence.value, pointer);
+	const value = resolved.value;
+	if (resolved.found && typeof value === "string" && value.length > 0) return { key: value };
+	if (resolved.found && typeof value === "number" && Number.isFinite(value)) return { key: String(value) };
+	return { missing: `no idempotency key at ${pointer}` };
+}
+/** No call repeats a side effect that is not proven safe to repeat. */
+function checkRetrySafe(rule, spans) {
+	const reads = new Set(rule.reads ?? []);
+	const writes = new Map((rule.writes ?? []).map((w) => [w.tool, w]));
+	const out = [];
+	const groups = /* @__PURE__ */ new Map();
+	const unseen = /* @__PURE__ */ new Map();
+	for (const call of toolSpans(spans)) {
+		const name = contractSpanToolName(call.span);
+		if (name === void 0) {
+			out.push({
+				rule: rule.label,
+				spanId: call.span.spanId,
+				detail: `span ${call.ref} has no tool name, so whether it repeats a call is unknown`
+			});
+			continue;
+		}
+		if (reads.has(name)) continue;
+		const args = spanArguments(call.span);
+		if (!args.known) {
+			const list = unseen.get(name) ?? [];
+			list.push(call);
+			unseen.set(name, list);
+			continue;
+		}
+		const pointer = writes.get(name)?.idempotencyKey;
+		const key = `${name}\u0000${canonicalString(pointer === void 0 ? args.value : withoutMember(args.value, pointer))}`;
+		const list = groups.get(key) ?? [];
+		list.push(call);
+		groups.set(key, list);
+	}
+	for (const [name, calls] of unseen) {
+		const total = calls.length + [...groups].reduce((n, [k, list]) => k.startsWith(`${name}\u0000`) ? n + list.length : n, 0);
+		if (total > 1) out.push({
+			rule: rule.label,
+			spanId: calls[0].span.spanId,
+			detail: `${name} was called ${total} times and span ${calls[0].ref} has no captured arguments, so whether a call repeats another is unknown`
+		});
+	}
+	for (const [key, calls] of groups) {
+		if (calls.length < 2) continue;
+		const name = key.slice(0, key.indexOf("\0"));
+		const refs = listed(calls.map((c) => c.ref));
+		const write = writes.get(name);
+		if (!write) {
+			out.push({
+				rule: rule.label,
+				spanId: calls[1].span.spanId,
+				detail: `${name} was called ${calls.length} times with the same arguments (spans ${refs}) and its side effect is not declared, so the repeat is not proven safe`
+			});
+			continue;
+		}
+		const pointer = write.idempotencyKey;
+		if (pointer === void 0) {
+			out.push({
+				rule: rule.label,
+				spanId: calls[1].span.spanId,
+				detail: `write ${name} was called ${calls.length} times with the same arguments (spans ${refs}) and declares no idempotency key, so it may have applied more than once`
+			});
+			continue;
+		}
+		const keyed = calls.map((c) => ({
+			c,
+			k: idempotencyKey(c.span, pointer)
+		}));
+		const missing = keyed.find((x) => "missing" in x.k);
+		const keys = new Set(keyed.flatMap((x) => "key" in x.k ? [x.k.key] : []));
+		if (missing) out.push({
+			rule: rule.label,
+			spanId: missing.c.span.spanId,
+			detail: `write ${name} was called ${calls.length} times with the same arguments (spans ${refs}); span ${missing.c.ref} has ${missing.k.missing}, so it may have applied more than once`
+		});
+		else if (keys.size > 1) out.push({
+			rule: rule.label,
+			spanId: calls[1].span.spanId,
+			detail: `write ${name} was called ${calls.length} times with the same arguments (spans ${refs}) under ${keys.size} different idempotency keys, so the target cannot tell they are one operation`
+		});
+	}
+	const flagged = new Set(out.map((v) => v.spanId));
+	for (const call of toolSpans(spans)) {
+		const name = contractSpanToolName(call.span);
+		if (name === void 0 || flagged.has(call.span.spanId)) continue;
+		const write = writes.get(name);
+		if (!write || !isSpanError(call.span)) continue;
+		const pointer = write.idempotencyKey;
+		if (!(pointer !== void 0 && "key" in idempotencyKey(call.span, pointer))) {
+			out.push({
+				rule: rule.label,
+				spanId: call.span.spanId,
+				detail: `write ${name} (span ${call.ref}) errored and ${pointer === void 0 ? "declares no idempotency key" : `has ${idempotencyKey(call.span, pointer).missing}`}; its outcome is unknown, so a retry cannot be proven safe`
+			});
+			flagged.add(call.span.spanId);
+		}
+	}
+	return out;
+}
+/** A copy of `value` without the member `pointer` names (RFC 6901). */
+function withoutMember(value, pointer) {
+	const tokens = pointer.slice(1).split("/").map((t) => t.replace(/~1/g, "/").replace(/~0/g, "~"));
+	const strip = (node, depth) => {
+		const token = tokens[depth];
+		const last = depth === tokens.length - 1;
+		if (Array.isArray(node)) {
+			const index = /^(?:0|[1-9]\d*)$/.test(token) ? Number(token) : -1;
+			if (index < 0 || index >= node.length) return node;
+			return last ? node.filter((_, i) => i !== index) : node.map((v, i) => i === index ? strip(v, depth + 1) : v);
+		}
+		if (node === null || typeof node !== "object" || !Object.hasOwn(node, token)) return node;
+		const copy = { ...node };
+		if (last) delete copy[token];
+		else copy[token] = strip(copy[token], depth + 1);
+		return copy;
+	};
+	return strip(value, 0);
+}
 function checkRule(rule, ctx) {
 	const { spans } = ctx;
 	switch (rule.kind) {
@@ -1947,6 +2230,8 @@ function checkRule(rule, ctx) {
 		}
 		case "run": return checkRun(rule, ctx.run);
 		case "argument": return checkArgument(rule, spans);
+		case "toolsOffered": return checkToolsOffered(rule, spans);
+		case "retrySafe": return checkRetrySafe(rule, spans);
 	}
 }
 function executeRule(rule, ctx) {
@@ -1969,12 +2254,16 @@ function executeRule(rule, ctx) {
 /**
 * A run derived from one span: the `run.status` attribute a run-anchor span
 * carries (store-to-otlp writes it), else failed on an error status, else
-* completed once the span ended.
+* unknown. An ended span with an OK status and no declared `run.status` is
+* NOT inferred as completed — a span exporter can end its root span cleanly
+* without ever learning whether the underlying run actually finished (for
+* example, a trace reader that saw no terminal record). `requireCompleted`
+* and `allowedStatuses` must fail closed on that gap rather than pass it.
 */
 function runOfSpan(span) {
 	const declared = span.attributes?.["run.status"];
 	return {
-		status: typeof declared === "string" && RUN_STATUSES.includes(declared) ? declared : isSpanError(span) ? "failed" : finite(span.endedAt) !== void 0 ? "completed" : "running",
+		status: typeof declared === "string" && RUN_STATUSES.includes(declared) ? declared : isSpanError(span) ? "failed" : void 0,
 		...finite(span.startedAt) === void 0 ? {} : { startedAt: span.startedAt },
 		...finite(span.endedAt) === void 0 ? {} : { endedAt: span.endedAt }
 	};
@@ -2008,8 +2297,8 @@ function statusOf(outcomes) {
 }
 /**
 * Evaluate one contract over a span sequence. Pure and synchronous — works on
-* `Span[]` from a TraceStore, `ExportableSpan[]` from the otel-bridge
-* flattening, or any array satisfying `ContractSpan`. Throws
+* `Span[]` from a TraceStore, `contractSpansFromOtlp` output, or any array
+* satisfying `ContractSpan`. Throws
 * `ValidationError` for a malformed contract; a rule that cannot be evaluated
 * is reported as `error` in the verdict instead.
 */
@@ -2108,7 +2397,9 @@ function rulePredicates(rule) {
 	switch (rule.kind) {
 		case "precedes": return [rule.a, rule.b];
 		case "neverUnless": return [rule.p, rule.prior];
-		case "run": return [];
+		case "run":
+		case "toolsOffered":
+		case "retrySafe": return [];
 		default: return [rule.p];
 	}
 }
@@ -2145,7 +2436,12 @@ function checkTraceContracts(spans, contracts, options = {}) {
 *     "allowed": ["lookup_order", "issue_refund", "send_email"],
 *     "maxCalls": 20,
 *     "requiredOrder": ["lookup_order", "issue_refund"],
-*     "arguments": [{ "tool": "issue_refund", "pointer": "/amount", "type": "number" }]
+*     "arguments": [{ "tool": "issue_refund", "pointer": "/amount", "type": "number" }],
+*     "enforced": true
+*   },
+*   "retries": {
+*     "reads": ["lookup_order"],
+*     "writes": [{ "tool": "issue_refund", "idempotencyKey": "/idempotency_key" }]
 *   },
 *   "llm": { "maxCalls": 10, "maxTotalTokens": 50000, "allowedModels": ["claude-sonnet-4-5"] },
 *   "alternatives": { "anyOf": [{ "id": "cache-hit", "tools": { "required": ["read_cache"] } }] }
@@ -2162,6 +2458,7 @@ const PATH_KEYS = [
 	"run",
 	"tools",
 	"llm",
+	"retries",
 	"rules"
 ];
 const SPEC_KEYS = [
@@ -2185,8 +2482,11 @@ const TOOLS_KEYS = [
 	"maxCallsPerTool",
 	"requiredOrder",
 	"orderMode",
-	"arguments"
+	"arguments",
+	"enforced"
 ];
+const RETRIES_KEYS = ["reads", "writes"];
+const WRITE_KEYS = ["tool", "idempotencyKey"];
 const LLM_KEYS = [
 	"maxCalls",
 	"maxTotalTokens",
@@ -2274,6 +2574,17 @@ const RULE_KEYS = {
 		"pointer",
 		"check",
 		"occurrence"
+	],
+	toolsOffered: [
+		"kind",
+		"label",
+		"declared"
+	],
+	retrySafe: [
+		"kind",
+		"label",
+		"reads",
+		"writes"
 	]
 };
 /**
@@ -2368,7 +2679,18 @@ function compilePath(path, prefix) {
 				...arg.occurrence === void 0 ? {} : { occurrence: arg.occurrence }
 			});
 		});
+		if (tools.enforced) rules.push({
+			kind: "toolsOffered",
+			label: at("tools.enforced"),
+			...tools.allowed === void 0 ? {} : { declared: [...tools.allowed] }
+		});
 	}
+	if (path.retries) rules.push({
+		kind: "retrySafe",
+		label: at("retries"),
+		...path.retries.reads === void 0 ? {} : { reads: [...path.retries.reads] },
+		...path.retries.writes === void 0 ? {} : { writes: path.retries.writes.map((w) => ({ ...w })) }
+	});
 	const llm = path.llm;
 	if (llm) {
 		if (llm.maxCalls !== void 0) rules.push({
@@ -2442,6 +2764,7 @@ function parsePath(path, prefix) {
 	if (path.run !== void 0) {
 		const run = object(path.run, at("run"));
 		strictKeys(run, RUN_KEYS, at("run"));
+		if (run.requireCompleted !== void 0 && run.requireCompleted !== true) throw new ValidationError(`${at("run.requireCompleted")}: must be true`);
 		if (Array.isArray(run.allowedStatuses)) run.allowedStatuses.forEach((status, i) => {
 			oneOfValues(status, RUN_STATUSES, at(`run.allowedStatuses[${i}]`));
 		});
@@ -2463,11 +2786,13 @@ function parsePath(path, prefix) {
 		if (tools.maxCalls !== void 0) count(tools.maxCalls, at("tools.maxCalls"));
 		if (tools.maxCallsPerTool !== void 0) {
 			const perTool = object(tools.maxCallsPerTool, at("tools.maxCallsPerTool"));
+			if (Object.keys(perTool).length === 0) throw new ValidationError(`${at("tools.maxCallsPerTool")}: must not be empty`);
 			for (const [tool, max] of Object.entries(perTool)) {
 				nonEmptyString(tool, at("tools.maxCallsPerTool key"));
 				count(max, at(`tools.maxCallsPerTool.${tool}`));
 			}
 		}
+		if (tools.enforced !== void 0 && tools.enforced !== true) throw new ValidationError(`${at("tools.enforced")}: must be true`);
 		if (tools.orderMode !== void 0) oneOfValues(tools.orderMode, ORDER_MODES, at("tools.orderMode"));
 		if (tools.orderMode !== void 0 && tools.requiredOrder === void 0) throw new ValidationError(`${at("tools.orderMode")}: set only with requiredOrder`);
 		if (tools.arguments !== void 0) nonEmptyArray(tools.arguments, at("tools.arguments")).forEach((arg, i) => {
@@ -2490,9 +2815,25 @@ function parsePath(path, prefix) {
 			}, where);
 		});
 	}
+	if (path.retries !== void 0) {
+		const retries = object(path.retries, at("retries"));
+		strictKeys(retries, RETRIES_KEYS, at("retries"));
+		if (retries.reads !== void 0) toolNames(retries.reads, at("retries.reads"));
+		if (retries.writes !== void 0) nonEmptyArray(retries.writes, at("retries.writes")).forEach((w, i) => {
+			const where = at(`retries.writes[${i}]`);
+			strictKeys(object(w, where), WRITE_KEYS, where);
+			nonEmptyString(w.tool, `${where}.tool`);
+		});
+		assertRule({
+			kind: "retrySafe",
+			label: "retries",
+			...retries
+		}, at("retries"));
+	}
 	if (path.llm !== void 0) {
 		const llm = object(path.llm, at("llm"));
 		strictKeys(llm, LLM_KEYS, at("llm"));
+		if (llm.maxCalls === void 0 && llm.maxTotalTokens === void 0 && llm.allowedModels === void 0) throw new ValidationError(`${at("llm")}: llm rule checks nothing`);
 		if (llm.maxCalls !== void 0) count(llm.maxCalls, at("llm.maxCalls"));
 		if (llm.maxTotalTokens !== void 0) count(llm.maxTotalTokens, at("llm.maxTotalTokens"));
 		if (llm.allowedModels !== void 0) toolNames(llm.allowedModels, at("llm.allowedModels"));
@@ -2645,12 +2986,15 @@ function lintPath(path, prefix, out) {
 		if (forbidden.has(arg.tool)) error("tools.argument-on-forbidden", `tools.arguments[${i}]`, `"${arg.tool}" is forbidden, so its arguments can never be checked`);
 		if (!needed.includes(arg.tool)) warn("tools.argument-without-required", `tools.arguments[${i}]`, `"${arg.tool}" is not required; a run that never calls it fails this check for missing evidence`);
 	}
+	if (t.enforced && !allowed) warn("tools.enforced-without-allowed", "tools.enforced", "without allowed, only calls are checked against the offered tools; add allowed to also check what the harness offered");
+	for (const [i, w] of (path.retries?.writes ?? []).entries()) if (forbidden.has(w.tool)) warn("retries.write-forbidden", `retries.writes[${i}]`, `"${w.tool}" is forbidden, so its retry rule never applies`);
 	const order = t.requiredOrder ?? [];
 	const repeated = order.find((tool, i) => order.indexOf(tool) !== i);
 	if (repeated !== void 0) error("tools.order-repeats", "tools.requiredOrder", `"${repeated}" appears twice, so the order contradicts itself`);
 	if (order.length > 1 && t.orderMode === void 0) warn("tools.order-default-mode", "tools.orderMode", "start-order passes when calls overlap; set finish-before-start if each step must finish first");
 	const run = path.run;
-	if (run?.requireCompleted && run.allowedStatuses?.includes("running")) error("run.running-and-completed", "run", "requireCompleted contradicts allowedStatuses containing \"running\"");
+	if (run?.requireCompleted && run.allowedStatuses && !run.allowedStatuses.includes("completed")) error("run.completed-not-allowed", "run", "requireCompleted needs status completed, which allowedStatuses excludes");
+	else if (run?.requireCompleted && run.allowedStatuses?.some((s) => s !== "completed")) warn("run.statuses-beside-completed", "run.allowedStatuses", "requireCompleted accepts only completed, so the other allowed statuses never apply");
 	if (path.llm?.maxCalls === 0 && (path.llm.maxTotalTokens !== void 0 || path.llm.allowedModels)) warn("llm.no-calls", "llm", "maxCalls is 0, so the token and model limits never apply");
 }
 //#endregion