@tangle-network/agent-eval 0.131.1 → 0.133.0

package/dist/index.js

@@ -3,17 +3,16 @@ import { _ as observeAll, a as jsonlReviewStore, c as scoreFromEvals, d as runAg
 import { a as NotFoundError, c as VerificationError, i as JudgeError, n as CaptureIntegrityError, o as ReplayError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-8YnH8WlF.js";
 import { a as verifyManifest, i as signManifest, n as evaluateHypothesis, r as hashJson, t as canonicalize } from "./pre-registration-DakwTRXk.js";
 import { AGENT_PROFILE_KINDS, AgentProfileCellValidationError, agentProfileCellHashMaterial, agentProfileCellKey, assertRunAgentProfileCell, buildAgentInterfaceProfileCell, buildAgentProfileCell, groupRunsByAgentProfileCell, requireAgentProfileCell, toAgentProfileJson, validateAgentProfileCell, verifyAgentProfileCell } from "./profile-cell.js";
-import { F as makeFinding, I as computeTraceMetrics, L as createChatClient, P as computeFindingId, R as createAnalystAi, a as KNOWLEDGE_GAP_KIND_SPEC, d as renderUpstreamFindings, i as KNOWLEDGE_POISONING_KIND_SPEC, l as createTraceAnalystKind, n as AnalystRegistry, o as IMPROVEMENT_KIND_SPEC, r as DEFAULT_TRACE_ANALYST_KINDS, s as FAILURE_MODE_KIND_SPEC, t as buildDefaultAnalystRegistry, u as renderPriorFindings } from "./default-registry-C-vFCSEc.js";
-import { a as estimateTokens, i as estimateCost, n as MetricsCollector, o as isModelPriced, r as TokenCounter, s as resolveModelPricing, t as MODEL_PRICING } from "./metrics-C9YY1OcL.js";
-import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-DIgQUFZZ.js";
+import { F as makeFinding, I as computeTraceMetrics, L as createChatClient, P as computeFindingId, R as createAnalystAi, a as KNOWLEDGE_GAP_KIND_SPEC, d as renderUpstreamFindings, i as KNOWLEDGE_POISONING_KIND_SPEC, l as createTraceAnalystKind, n as AnalystRegistry, o as IMPROVEMENT_KIND_SPEC, r as DEFAULT_TRACE_ANALYST_KINDS, s as FAILURE_MODE_KIND_SPEC, t as buildDefaultAnalystRegistry, u as renderPriorFindings } from "./default-registry-D3T9XbuY.js";
+import { _ as resolveModelPricing, a as CostLedgerPersistenceError, c as costForTokenPricing, d as MODEL_PRICING, f as MetricsCollector, g as isModelPriced, h as estimateTokens, i as CostLedger, l as costForUsage, m as estimateCost, n as CostCallConflictError, o as CostReceiptCaptureError, p as TokenCounter, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-BrJxbrMy.js";
 import { a as providerFromBaseUrl, i as defaultProviderRedactor, n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-BQd7mzyT.js";
-import { a as assertLlmRoute, c as callLlmJson, d as isTransientLlmError, f as maximumChargeForLlmRequest, i as LlmRouteAssertionError, l as costReceiptFromLlm, m as stripFencedJson, n as LlmClient, o as backoffMs, p as probeLlm, r as LlmResponseError, s as callLlm, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client--GR4JbZE.js";
+import { a as assertLlmRoute, c as callLlmJson, d as isTransientLlmError, f as maximumChargeForLlmRequest, i as LlmRouteAssertionError, l as costReceiptFromLlm, m as stripFencedJson, n as LlmClient, o as backoffMs, p as probeLlm, r as LlmResponseError, s as callLlm, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-ClPW-dWB.js";
 import { INPUT_VALUE, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OUTPUT_VALUE, RUN_COST_ATTR_KEYS, SPAN_KIND_ATTR_KEYS, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, applyLlmSpanOtlpAttributes, asNumber, contextInputTokens, firstNumberAttr } from "./trace-attributes.js";
 import { S as traceSpanKindToOpenInferenceKind, a as SpanNotFoundError, b as classifyOtlpSpanRole, c as DEFAULT_TRACE_ANALYST_BUDGETS, f as extractOtlpAttributes, g as readOtlpStatus, h as projectOtlpFlatLine, i as OtlpFileTraceStore, l as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, m as inferOtlpKind, n as traceAnalystFunctionGroup, o as TraceFileMissingError, p as firstStringAttr, s as TraceNotFoundError, t as buildTraceAnalystTools, u as asString, v as stringField, x as isOtlpModelCall, y as applyToolSpanOtlpAttributes } from "./tools-BmuN627J.js";
 import { n as aggregateRunScore, r as clamp01, t as DEFAULT_RUN_SCORE_WEIGHTS } from "./run-score-iEEAWiBY.js";
 import { n as mapConcurrent, t as Mutex } from "./concurrency-MUjT7VjM.js";
-import { a as RunCritic, d as defaultIsMaterial, f as diffFindings, i as runSemanticConceptJudge, n as SEMANTIC_CONCEPT_JUDGE_VERSION, o as SKILL_USAGE_ANALYST, p as LockedJsonlAppender, r as createSemanticConceptJudge, s as SkillUsageAnalyst, t as DEFAULT_COMPLEXITY_WEIGHTS, u as FindingsStore } from "./semantic-concept-judge-b5m3irbR.js";
-import { At as dominates, B as surfaceContentHash, Bt as JudgeParseError, Ct as llmJudge, D as runCanaries, Dt as fileVerdictCache, Et as contentHash, Ft as BackendIntegrityError, G as DEFAULT_MUTATION_PRIMITIVES, It as assertRealAgentReceipts, K as buildReflectionPrompt, Lt as assertRealBackend, Mt as paretoFrontierWithCrowding, Nt as scalarScore, Ot as inMemoryVerdictCache, Rt as summarizeAgentReceiptIntegrity, St as hashScenarios, Tt as canonicalJson, _t as redTeamReport, bt as Dataset, dt as REFERENCE_EQUIVALENCE_INPUT_LIMITS, ft as REFERENCE_EQUIVALENCE_JUDGE_VERSION, gt as redTeamDataset, ht as DEFAULT_RED_TEAM_CORPUS, jt as paretoFrontier, kt as crowdingDistance, mt as runReferenceEquivalenceJudge, pt as createReferenceEquivalenceJudge, q as parseReflectionResponse, vt as scoreRedTeamOutput, wt as cachedJudge, xt as HoldoutLockedError, yt as toolNamesForRun, zt as summarizeBackendIntegrity } from "./skillopt-optimization-method-eOJL2570.js";
+import { a as RunCritic, d as defaultIsMaterial, f as diffFindings, i as runSemanticConceptJudge, n as SEMANTIC_CONCEPT_JUDGE_VERSION, o as SKILL_USAGE_ANALYST, p as LockedJsonlAppender, r as createSemanticConceptJudge, s as SkillUsageAnalyst, t as DEFAULT_COMPLEXITY_WEIGHTS, u as FindingsStore } from "./semantic-concept-judge-BypLt6Fw.js";
+import { At as dominates, B as surfaceContentHash, Bt as JudgeParseError, Ct as llmJudge, D as runCanaries, Dt as fileVerdictCache, Et as contentHash, Ft as BackendIntegrityError, G as DEFAULT_MUTATION_PRIMITIVES, It as assertRealAgentReceipts, K as buildReflectionPrompt, Lt as assertRealBackend, Mt as paretoFrontierWithCrowding, Nt as scalarScore, Ot as inMemoryVerdictCache, Rt as summarizeAgentReceiptIntegrity, St as hashScenarios, Tt as canonicalJson, _t as redTeamReport, bt as Dataset, dt as REFERENCE_EQUIVALENCE_INPUT_LIMITS, ft as REFERENCE_EQUIVALENCE_JUDGE_VERSION, gt as redTeamDataset, ht as DEFAULT_RED_TEAM_CORPUS, jt as paretoFrontier, kt as crowdingDistance, mt as runReferenceEquivalenceJudge, pt as createReferenceEquivalenceJudge, q as parseReflectionResponse, vt as scoreRedTeamOutput, wt as cachedJudge, xt as HoldoutLockedError, yt as toolNamesForRun, zt as summarizeBackendIntegrity } from "./skillopt-optimization-method-DgN7U9iR.js";
 import { A as spearmanR, B as verbosityBias, C as pairedTTest, D as ranks, E as pearsonR, F as calibrateJudge, I as calibrateJudgeContinuous, L as continuousAgreement, M as weightedMean, N as wilcoxonSignedRank, O as requiredPairedSampleSize, P as wilson, R as positionalBias, S as pairedSignTest, T as passAtK, _ as normalizeScores, a as confidenceInterval, b as pairedMde, c as eProcess, d as interpretCliffs, f as mannWhitneyU, g as mulberry32, h as mcnemarRequiredN, i as cohensD, j as weightedComposite, k as requiredSampleSize, l as holm, m as mcnemarPower, n as bonferroni, o as corpusInterRaterAgreement, p as mcnemar, r as cliffsDelta, s as corpusInterRaterAgreementFromJudgeScores, t as benjaminiHochberg, u as interRaterReliability, v as pairedBootstrap, w as partialCredit, x as pairedRiskDifference, y as pairedCohensDz, z as selfPreference } from "./statistics-CnnxdpOg.js";
 import { n as pairArms, r as pairRunRecords, t as comparePairedArms } from "./paired-arms-D9D0wXj2.js";
 import { n as llmSpanFromProvider, t as TraceEmitter } from "./emitter-CPBAhxum.js";
@@ -28,20 +27,19 @@ import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
 import { t as mintRolloutRows } from "./mint-D5_87M5L.js";
 import { D as showMeasured, E as isUnavailable, S as rollupSupervisorRuns, T as SUPERVISOR_RUN_SCHEMA, _ as claudeCodeSupervisorRunReader, f as renderSupervisorRunHeadline, l as writeSupervisorRunReport, n as analyzeSupervisorRun, p as renderSupervisorRunMarkdown, t as supervisorRunRolloutLines, v as readClaudeCodeSupervisorRun, y as analyzeSupervisorRunSources } from "./supervisor-run-_lnTLM3z.js";
 import { n as TRACE_ANALYST_ACTOR_DESCRIPTION, r as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, t as analyzeTraces } from "./analyst-LsnNpSkm.js";
-import { C as planTraceInsightQuestions, E as traceAnalystOnRunComplete, S as inferDomainKeywords, T as tokenizeDomainWords, _ as buildTraceInsightContext, a as convertTraceStoresToOtlp, b as describeTraceInsightScope, c as otelRunCompleteHook, d as captureFetchToRawSink, f as otlpRowsToRunRecords, g as flattenOtlpExportToNdjson, h as otlpToTraceRunRecords, i as iterateRawCalls, l as OTEL_AGENT_EVAL_SCOPE, m as otlpToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as otlpRowsToTraceRunRecords, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as buildTraceInsightPrompt, w as scoreTraceInsightReadiness, x as domainEvidencePattern, y as defaultTraceInsightPanel } from "./replay-D18-pBAA.js";
-import { B as harnessAxisOf, F as HARNESS_NATIVE_MODEL, G as parseCorrectnessResponse, H as completionVerdict, I as agentProfileHash, K as verifyCompletion, L as agentProfileId, P as CODING_HARNESSES, R as agentProfileModelId, U as createLlmCorrectnessChecker, V as extractProducedState, W as createTokenRecallChecker, z as expandProfileAxes } from "./campaign-CLG6y9jF.js";
+import { C as planTraceInsightQuestions, E as traceAnalystOnRunComplete, S as inferDomainKeywords, T as tokenizeDomainWords, _ as buildTraceInsightContext, a as convertTraceStoresToOtlp, b as describeTraceInsightScope, c as otelRunCompleteHook, d as captureFetchToRawSink, f as otlpRowsToRunRecords, g as flattenOtlpExportToNdjson, h as otlpToTraceRunRecords, i as iterateRawCalls, l as OTEL_AGENT_EVAL_SCOPE, m as otlpToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as otlpRowsToTraceRunRecords, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as buildTraceInsightPrompt, w as scoreTraceInsightReadiness, x as domainEvidencePattern, y as defaultTraceInsightPanel } from "./replay-BMR4TEYY.js";
+import { n as extractUsageFromResponse, r as extractUsageFromSse, t as extractUsage } from "./extract-usage-2j25whHw.js";
+import { B as harnessAxisOf, F as HARNESS_NATIVE_MODEL, G as parseCorrectnessResponse, H as completionVerdict, I as agentProfileHash, K as verifyCompletion, L as agentProfileId, P as CODING_HARNESSES, R as agentProfileModelId, U as createLlmCorrectnessChecker, V as extractProducedState, W as createTokenRecallChecker, z as expandProfileAxes } from "./campaign-CyQMCCEU.js";
 import { a as judgeSpans, c as runsForScenario, i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, r as groupBy, s as runFailureClass, t as aggregateLlm } from "./query-Di7eEQ79.js";
 import { a as checkBehavioralCanary, i as canaryLeakView, o as checkCanaries, r as HoldoutAuditor, s as runBehavioralCanaries, t as analyzeRuns } from "./analyze-runs-BjPn_fOS.js";
 import { a as summaryTable, i as researchReport, n as gainHistogram, r as paretoChart, t as RESEARCH_REPORT_HARD_PAIR_FLOOR } from "./summary-report-BNs5nmXI.js";
 import { a as composeParsers, c as vitestTestParser, i as SubprocessSandboxDriver, n as DockerSandboxDriver, o as jestTestParser, r as SandboxHarness, s as pytestTestParser, t as runTestGradedScenario } from "./test-graded-scenario-BsqWLmPt.js";
 import { a as InMemoryTraceStore, i as FileSystemTraceStore, n as assertRunCaptured, r as throwIfRunIncomplete, t as RunIntegrityError } from "./integrity-BzRbCHzi.js";
-import { n as extractUsageFromResponse, r as extractUsageFromSse, t as extractUsage } from "./extract-usage-BrQ8mCLX.js";
 import { i as redactValue, n as REDACTION_VERSION, r as redactString, t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
 import { a as DEFAULT_RULES, i as computeToolUseMetrics, n as iqr, o as classifyFailure, r as welchsTTest, t as compareToBaseline } from "./baseline-HsBvw_dk.js";
-import { n as projectRuntimeTrajectoryEvidence, t as parseRuntimeTrajectoryHookEvent } from "./runtime-trajectory-1gyaTOoC.js";
 import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
-import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-BDtRiOvE.js";
-import { t as runEvalCampaign } from "./eval-campaign-C2k-m4aY.js";
+import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-DncrVrSr.js";
+import { t as runEvalCampaign } from "./eval-campaign-CYkUME2T.js";
 import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
 import { AxGEPA, AxSignature, ax } from "@ax-llm/ax";
 import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
@@ -4587,6 +4585,96 @@ function clampUnit(value) {
 	return Math.max(0, Math.min(1, value));
 }
 //#endregion
+//#region src/runtime-trajectory.ts
+const DEFAULT_SPLIT_TAG = "search";
+function projectRuntimeTrajectoryEvidence(options) {
+	const diagnostics = [];
+	const runsById = /* @__PURE__ */ new Map();
+	const events = [];
+	let recordWithRuntimeEventsCount = 0;
+	let defaultedSplitCount = 0;
+	for (let recordIndex = 0; recordIndex < options.records.length; recordIndex += 1) {
+		const record = options.records[recordIndex];
+		const key = runtimeTrajectoryRecordKey(record, recordIndex, options.recordIdOf);
+		const splitTag = record.splitTag ?? options.defaultSplitTag ?? DEFAULT_SPLIT_TAG;
+		if (record.splitTag === void 0) defaultedSplitCount += 1;
+		const rawEvents = record.runtimeEvents;
+		if (!Array.isArray(rawEvents)) {
+			diagnostics.push(`${key}: runtimeEvents is not an array; no runtime run join can be extracted`);
+			continue;
+		}
+		if (rawEvents.length === 0) {
+			diagnostics.push(`${key}: no runtimeEvents; no runtime run join can be extracted`);
+			continue;
+		}
+		recordWithRuntimeEventsCount += 1;
+		for (let index = 0; index < rawEvents.length; index += 1) {
+			const event = parseRuntimeTrajectoryHookEvent(rawEvents[index]);
+			if (!event) {
+				diagnostics.push(`${key}: runtimeEvents[${index}] is not a RuntimeHookEvent`);
+				continue;
+			}
+			events.push(event);
+			const scenarioId = event.scenarioId ?? stringOrUndefined(options.scenarioIdOf?.(record, recordIndex)) ?? stringOrUndefined(record.scenarioId);
+			const prior = runsById.get(event.runId);
+			if (!prior) {
+				runsById.set(event.runId, {
+					runId: event.runId,
+					scenarioId,
+					splitTag
+				});
+				continue;
+			}
+			if (prior.scenarioId !== scenarioId || prior.splitTag !== splitTag) diagnostics.push(`${key}: runId ${event.runId} has conflicting scenario/split metadata`);
+		}
+	}
+	const runs = [...runsById.values()];
+	return {
+		runs,
+		events,
+		summary: {
+			recordCount: options.records.length,
+			recordWithRuntimeEventsCount,
+			runtimeRunCount: runs.length,
+			lifecycleEventCount: events.length,
+			defaultedSplitCount
+		},
+		diagnostics
+	};
+}
+function parseRuntimeTrajectoryHookEvent(input) {
+	if (!isRecord(input)) return null;
+	if (typeof input.id !== "string" || input.id.length === 0) return null;
+	if (typeof input.runId !== "string" || input.runId.length === 0) return null;
+	if (typeof input.target !== "string" || input.target.length === 0) return null;
+	if (typeof input.phase !== "string" || input.phase.length === 0) return null;
+	if (typeof input.timestamp !== "number" || !Number.isFinite(input.timestamp)) return null;
+	return {
+		id: input.id,
+		runId: input.runId,
+		scenarioId: stringOrUndefined(input.scenarioId),
+		target: input.target,
+		phase: input.phase,
+		timestamp: input.timestamp,
+		stepIndex: finiteNumberOrUndefined(input.stepIndex),
+		parentId: stringOrUndefined(input.parentId),
+		payload: input.payload,
+		metadata: isRecord(input.metadata) ? { ...input.metadata } : void 0
+	};
+}
+function runtimeTrajectoryRecordKey(record, index, recordIdOf) {
+	return stringOrUndefined(recordIdOf?.(record, index)) ?? stringOrUndefined(record.id) ?? `record[${index}]`;
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function stringOrUndefined(value) {
+	return typeof value === "string" && value.length > 0 ? value : void 0;
+}
+function finiteNumberOrUndefined(value) {
+	return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+//#endregion
 //#region src/cost-tracker.ts
 /**
 * Cost tracker — token + USD accounting per scenario and per run.