@tangle-network/agent-eval 0.193.0 → 0.193.2

package/dist/index.d.ts

@@ -34,7 +34,7 @@ import { A as SemanticConceptJudgeInput, C as DspyRlmTraceEngineOptions, D as Co
 import { i as BenchmarkEvaluation } from "./types-qoxUfKQg.js";
 import { a as SpanHandle, n as CaptureWrite, o as TraceEmitter, s as TraceEmitterOptions, t as CaptureErrorHandler } from "./emitter-HcCmFQl_.js";
 import { n as SandboxDriver } from "./sandbox-harness-BlSOu4LX.js";
-import { S as shareAllowed, b as redactForShare, d as ShareSafetyVerdict, f as assessShareSafety, i as RedactOptions, o as RedactionProfile, r as REDACTION_VERSION, s as RedactionReport, u as ShareSafetyStatus, x as redactText, y as redact } from "./redact-CopuMxll.js";
+import { S as shareAllowed, b as redactForShare, d as ShareSafetyVerdict, f as assessShareSafety, i as RedactOptions, o as RedactionProfile, r as REDACTION_VERSION, s as RedactionReport, u as ShareSafetyStatus, x as redactText, y as redact } from "./redact-Cveoq6fJ.js";
 import { a as PairedEvalueStep, c as SequentialDecision, i as PairedEvalueSequence, l as evaluateInterimReleaseConfidence, n as InterimReleaseConfidenceInput, r as PairedEvalueOptions, t as InterimReleaseConfidence, u as pairedEvalueSequence } from "./sequential-BhsrMupG.js";
 import { n as TrajectoryStep, r as buildTrajectory, t as Trajectory } from "./trajectory-Y1fmdlw7.js";
 import { a as runCounterfactual, i as CounterfactualRunner, n as CounterfactualMutation, r as CounterfactualResult, t as CounterfactualContext } from "./counterfactual-CPaWN99-.js";

package/dist/index.js

@@ -1,7 +1,7 @@
 import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
-import { i as JudgeError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
+import { i as JudgeError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
 import { a as hashCanonical, r as canonicalString } from "./canonical-BsZVgBhW.js";
-import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-BqeIaIp2.js";
+import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-B2AhDRnq.js";
 import { c as groupRunsByAgentProfileCell, d as validateAgentProfileCell, f as verifyAgentProfileCell, h as manifestContentDigest, i as agentProfileCellKey, m as hashJson, r as agentProfileCellHashMaterial, s as buildAgentProfileCell, t as AGENT_PROFILE_KINDS, u as toAgentProfileJson } from "./agent-profile-cell-DcFW4vOc.js";
 import { t as mulberry32 } from "./random-Dn5fPWkt.js";
 import { a as spearmanR, c as weightedMean, i as ranks, n as partialCredit, o as summarizeNumberSeries, r as pearsonR, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
@@ -22,12 +22,12 @@ import { r as observedSplitScore, t as isRealnessGated } from "./reward-nw2xZGZG
 import { a as parseRunRecordSafe, c as runTaskScore, i as modelHasSnapshot, l as searchCellRunId, n as UNKNOWN_MODEL, o as roundTripRunRecord, r as isRunRecord, s as runCostFloorUsd, t as RunRecordValidationError, u as validateRunRecord } from "./run-record-BRlM99rX.js";
 import { a as summaryTable, n as gainHistogram, r as paretoChart } from "./summary-report-BMvou2oZ.js";
 import { n as contentHash, r as fileVerdictCache, t as canonicalJson } from "./verdict-cache-BQEsVy9n.js";
-import { $ as redTeamDataset, K as dominates, Q as DEFAULT_RED_TEAM_CORPUS, U as pairedPromotionPower, W as requiredPairsForPairedPromotion, a as transientDispatchFailure, ct as runCampaign, et as redTeamReport, i as quotaExhaustedUntil, nt as runCanaries, o as aggregateRunScore, q as paretoFrontier, s as clamp01, t as llmJudge, tt as scoreRedTeamOutput } from "./llm-judge-ChbWfeWl.js";
+import { $ as redTeamDataset, K as dominates, Q as DEFAULT_RED_TEAM_CORPUS, U as pairedPromotionPower, W as requiredPairsForPairedPromotion, a as transientDispatchFailure, ct as runCampaign, et as redTeamReport, i as quotaExhaustedUntil, nt as runCanaries, o as aggregateRunScore, q as paretoFrontier, s as clamp01, t as llmJudge, tt as scoreRedTeamOutput } from "./llm-judge-Cififl_V.js";
 import { a as resolveModelPricing, i as isModelPriced, n as estimateCost, r as estimateTokens, t as MODEL_PRICING } from "./metrics-DOMpkoeb.js";
 import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-_IbkejCF.js";
 import { B as assertRealBackend, F as surfaceContentHash, T as parseReflectionResponse, V as summarizeBackendIntegrity, c as ATTESTATION_ALGORITHM, l as attest, u as verifyAttestation, w as buildReflectionPrompt, z as BackendIntegrityError } from "./campaign-evidence-DJc60F9C.js";
-import { d as redactForShare, f as redactText, n as REDACTION_VERSION, p as shareAllowed, r as assessShareSafety, u as redact } from "./redact-B7rt9Zot.js";
-import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-DkPbnJEY.js";
+import { d as redactForShare, f as redactText, n as REDACTION_VERSION, p as shareAllowed, r as assessShareSafety, u as redact } from "./redact-Dk1ZdJEm.js";
+import { r as selfImprove, t as defineAgentEval } from "./define-agent-eval-qxAPjMSL.js";
 import { n as equivalenceVerdict, t as certificationEvidenceDigest } from "./verdict-DhPoXApk.js";
 import { t as TraceEmitter } from "./emitter-DbMXUIVP.js";
 import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
@@ -36,26 +36,26 @@ import { G as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, U as parseFindingSubject, h a
 import { _ as completedAnalystReviewQuality, b as validateAnalystReviewDecisions, c as FAILURE_MODE_KIND_SPEC, g as assertUniqueFindingIds, h as analystRunDigest, i as DEFAULT_TRACE_ANALYST_KINDS, m as analystFindingDigest, n as AnalystRegistry, t as buildDefaultAnalystRegistry, v as readAnalystReview, y as snapshotAnalystRun } from "./default-registry-xMLX160E.js";
 import { OUTPUT_VALUE } from "./trace-attributes.js";
 import { r as extractUsageFromSse, t as extractUsage } from "./extract-usage-DFATEuWW.js";
-import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-DYh4Bszw.js";
-import { S as exportRunAsOtlp, T as captureFetchToRawSink, _ as tokenizeDomainWords, d as inferDomainKeywords, g as scoreTraceInsightReadiness, l as describeTraceInsightScope, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as domainEvidencePattern, v as analyzeTraces } from "./store-tool-spans-BcC_5ef-.js";
+import { n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-CXuOhLvT.js";
+import { S as exportRunAsOtlp, T as captureFetchToRawSink, _ as tokenizeDomainWords, d as inferDomainKeywords, g as scoreTraceInsightReadiness, l as describeTraceInsightScope, n as toolSpansToTraceAnalysisStore, o as buildTraceInsightContext, s as buildTraceInsightPrompt, u as domainEvidencePattern, v as analyzeTraces } from "./store-tool-spans-BaGakFc2.js";
 import { n as assertRunCaptured, t as RunIntegrityError } from "./integrity-BFP4kjGj.js";
 import { n as InMemoryTraceStore, t as FileSystemTraceStore } from "./store-DNe_Uv1Q.js";
 import { t as packageVersion$1 } from "./package-version-D7lQHt_-.js";
 import { a as classifyFailure, i as INFRA_FAILURE_BLAMES, o as classifyFailureReason, r as FAILURE_BLAME, s as failureBlame, t as DEFAULT_FAILURE_REASON_RULES } from "./failure-taxonomy-CjabTATj.js";
 import { n as runBoundedProcess, t as DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS } from "./bounded-process-BBZob7vl.js";
-import { t as runEvalCampaign } from "./eval-campaign-CwsBn0Gm.js";
+import { t as runEvalCampaign } from "./eval-campaign-CXT2WYUe.js";
 import "./rollout-B-XBReTC.js";
 import { t as mintRolloutRows } from "./mint-iCD-W43s.js";
 import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-CzK5DarL.js";
-import { S as judgeFamily, _ as assertServedModels, b as CrossFamilyError, d as stripFencedJson, f as ModelSubstitutionError, h as assertServedModel, i as backoffMs, l as isTransientLlmError, m as assertCrossFamilyServed, o as costReceiptFromLlm, p as ServedCrossFamilyError, r as LlmResponseError, s as costReceiptFromLlmError, t as LlmCallError, u as maximumChargeForLlmRequest, v as checkServedModel, x as assertCrossFamily, y as servedModelAcceptable } from "./llm-client-KYQBh9C8.js";
-import { n as paidJsonChat } from "./chat-json-call-CXHBvUz8.js";
+import { S as judgeFamily, _ as assertServedModels, b as CrossFamilyError, d as stripFencedJson, f as ModelSubstitutionError, h as assertServedModel, i as backoffMs, l as isTransientLlmError, m as assertCrossFamilyServed, o as costReceiptFromLlm, p as ServedCrossFamilyError, r as LlmResponseError, s as costReceiptFromLlmError, t as LlmCallError, u as maximumChargeForLlmRequest, v as checkServedModel, x as assertCrossFamily, y as servedModelAcceptable } from "./llm-client-c_bKHTXz.js";
+import { n as paidJsonChat } from "./chat-json-call-DnlDjpco.js";
 import { i as makeProposalFinding, n as computeFindingId, r as makeFinding } from "./types-CW5b90JH.js";
-import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-BX34vPVY.js";
-import { a as diffFindings, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "./semantic-concept-judge-Bg-560H0.js";
+import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-k_QdEBZo.js";
+import { a as diffFindings, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "./semantic-concept-judge-Bh5lvKGg.js";
 import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
 import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-BPowgPm6.js";
 import { n as evaluateReleaseConfidence, r as bootstrapCi } from "./release-confidence-XQm_xjhR.js";
-import { t as createChatClient } from "./chat-client-B_ItsSBw.js";
+import { t as createChatClient } from "./chat-client-CJORSG_J.js";
 import { createHash } from "node:crypto";
 import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
 import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
@@ -1358,7 +1358,7 @@ function assertPredicate(value, where) {
 	]) if (p[field] !== void 0) assertMatcher(p[field], `${where} "${field}"`);
 	if (p.kind !== void 0 && !SPAN_KINDS.includes(p.kind)) throw new ValidationError(`${where}: "kind" must be one of ${SPAN_KINDS.join(", ")}`);
 	if (p.attr !== void 0 && (p.attr === null || typeof p.attr !== "object" || Array.isArray(p.attr))) throw new ValidationError(`${where}: "attr" must be a plain object`);
-	for (const [key, expected] of Object.entries(p.attr ?? {})) if (isSerializedRegex(expected)) assertMatcher(expected, `${where} attr "${key}"`);
+	for (const [key, expected] of Object.entries(p.attr ?? {})) if (expected !== null && typeof expected === "object") assertMatcher(expected, `${where} attr "${key}"`);
 	if (p.not !== void 0) assertPredicate(p.not, `${where} "not"`);
 	if (p.requiresCustom && typeof p.custom !== "function") throw new ValidationError(`${where}: rule was built with a custom predicate function, which does not survive JSON serialization — re-attach \`custom\` after deserializing or drop the rule`);
 	const hasAttr = p.attr !== void 0 && Object.keys(p.attr).length > 0;
@@ -1371,7 +1371,7 @@ function predicateMatches(span, p) {
 	if (p.model !== void 0 && !matchText(spanModel(span), p.model)) return false;
 	if (p.attr !== void 0) for (const [key, expected] of Object.entries(p.attr)) {
 		const actual = span.attributes?.[key];
-		if (isRegexValue(expected)) {
+		if (isRegexValue(expected) || isOneOfMatcher(expected)) {
 			if (!matchText(typeof actual === "string" ? actual : void 0, expected)) return false;
 		} else if (actual !== expected) return false;
 	}
@@ -1813,6 +1813,33 @@ function traceContract(name) {
 function spanRef(span, index) {
 	return span.spanId ?? `#${index}`;
 }
+/**
+* Whether this trace shows any sign that spans of `kind` are actually
+* captured, independent of whether any span happens to match a given rule's
+* predicate. A `never`, `atMost`, `tokensAtMost` or `retrySafe` rule that
+* targets TOOL or LLM spans and finds zero matches cannot otherwise tell "no
+* calls happened" from "calls were not recorded": an exporter that silently
+* drops a span kind makes `forbidden`, `allowed`, `maxCalls` and `retries`
+* checks on that kind pass vacuously. Evidence is: a span of that kind
+* anywhere in the trace, or — for TOOL — a recorded offered-tool set
+* (`gen_ai.tool.definitions`), which proves the capture pipeline ran even on
+* a turn that called no tool.
+*/
+function hasCaptureEvidence(spans, kind) {
+	if (spans.some((span) => contractSpanKind(span) === kind)) return true;
+	if (kind === "TOOL") return spans.some((span) => spanOfferedTools(span).recorded);
+	return false;
+}
+/** Only `TOOL`/`LLM`-scoped predicates have capture evidence to check. */
+function capturedKindOf(p) {
+	return p.kind === "TOOL" || p.kind === "LLM" ? p.kind : void 0;
+}
+function requireCaptureEvidence(spans, p, rule, situation) {
+	const kind = capturedKindOf(p);
+	if (kind === void 0) return;
+	if (hasCaptureEvidence(spans, kind)) return;
+	throw new CaptureIntegrityError(`rule '${rule}': ${situation} and the trace has no ${kind} capture evidence (no ${kind} span` + (kind === "TOOL" ? `, no ${TOOL_DEFINITIONS_ATTR}` : "") + `) — cannot tell whether ${describePredicate(p)} never happened or was never recorded`);
+}
 function indexed(spans, p) {
 	const out = [];
 	spans.forEach((span, i) => {
@@ -2066,7 +2093,9 @@ function checkRetrySafe(rule, spans) {
 	const out = [];
 	const groups = /* @__PURE__ */ new Map();
 	const unseen = /* @__PURE__ */ new Map();
-	for (const call of toolSpans(spans)) {
+	const calls = toolSpans(spans);
+	if (calls.length === 0 && !hasCaptureEvidence(spans, "TOOL")) throw new CaptureIntegrityError(`rule '${rule.label}': retrySafe found 0 TOOL spans and the trace has no TOOL capture evidence (no TOOL span, no ${TOOL_DEFINITIONS_ATTR}) — cannot tell whether no reads or writes happened or the trace lost its TOOL spans`);
+	for (const call of calls) {
 		const name = contractSpanToolName(call.span);
 		if (name === void 0) {
 			out.push({
@@ -2182,11 +2211,15 @@ function checkRule(rule, ctx) {
 			spanId: span.spanId,
 			detail: `span ${spanRef(span, i)} ("${span.name ?? ""}") fails always(${describePredicate(rule.p)})`
 		}]);
-		case "never": return indexed(spans, rule.p).map(({ span, ref }) => ({
-			rule: rule.label,
-			spanId: span.spanId,
-			detail: `span ${ref} ("${span.name ?? ""}") matches never(${describePredicate(rule.p)})`
-		}));
+		case "never": {
+			const matches = indexed(spans, rule.p);
+			if (matches.length === 0) requireCaptureEvidence(spans, rule.p, rule.label, `never(${describePredicate(rule.p)}) matched 0 spans`);
+			return matches.map(({ span, ref }) => ({
+				rule: rule.label,
+				spanId: span.spanId,
+				detail: `span ${ref} ("${span.name ?? ""}") matches never(${describePredicate(rule.p)})`
+			}));
+		}
 		case "eventually": return spans.some((span) => predicateMatches(span, rule.p)) ? [] : [{
 			rule: rule.label,
 			detail: `no span matches eventually(${describePredicate(rule.p)}) over ${spans.length} span(s)`
@@ -2209,6 +2242,7 @@ function checkRule(rule, ctx) {
 		case "neverUnless": return orderViolations(rule.label, indexed(spans, rule.prior), indexed(spans, rule.p), rule.order ?? "start-order", `${describePredicate(rule.p)} without an earlier ${describePredicate(rule.prior)}`);
 		case "atMost": {
 			const matches = indexed(spans, rule.p);
+			if (matches.length === 0) requireCaptureEvidence(spans, rule.p, rule.label, `atMost(${describePredicate(rule.p)}) matched 0 spans`);
 			return matches.length <= rule.max ? [] : [{
 				rule: rule.label,
 				spanId: matches[rule.max]?.span.spanId,
@@ -2217,6 +2251,7 @@ function checkRule(rule, ctx) {
 		}
 		case "tokensAtMost": {
 			const matches = indexed(spans, rule.p);
+			if (matches.length === 0) requireCaptureEvidence(spans, rule.p, rule.label, `tokensAtMost(${describePredicate(rule.p)}) matched 0 spans`);
 			const unknown = matches.find((m) => spanTotalTokens(m.span) === void 0);
 			if (unknown) return [{
 				rule: rule.label,