@tangle-network/agent-eval 0.184.0 → 0.185.0

package/dist/contract/index.js

@@ -1,25 +1,24 @@
-import { s as ValidationError } from "../errors-Dngq5h35.js";
 import { a as hashCanonical } from "../canonical-BsZVgBhW.js";
 import { r as pairedBootstrap } from "../paired-tests-C8iCsioC.js";
-import { s as decidePairedPromotion } from "../run-record-BhMVsHR1.js";
-import { Z as campaignSplitDigest, it as heldoutSignificance } from "../campaign-evidence-aWEmfROB.js";
-import { a as summarizeExecution, i as analyzeRuns, n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-RTekxhUE.js";
+import { t as decidePairedPromotion } from "../paired-promotion-decision-ClJh_9nK.js";
+import { a as heldoutSignificance } from "../statistical-heldout-CeWT_W7y.js";
+import { n as summarizeExecution, t as analyzeRuns } from "../analyze-runs-sTygaDyD.js";
 import { a as parseRunRecordSafe } from "../run-record-tg_5H2IK.js";
-import { X as runCampaign, Y as runEval, a as transientDispatchFailure, c as compareOptimizationMethods, h as runImprovementLoop, t as llmJudge, z as defaultProductionGate } from "../llm-judge-C5wbADA5.js";
+import { X as runCampaign, Y as runEval, a as transientDispatchFailure, c as compareOptimizationMethods, h as runImprovementLoop, t as llmJudge, z as defaultProductionGate } from "../llm-judge-FWWhEb31.js";
 import { i as isModelPriced, n as estimateCost } from "../metrics-DOMpkoeb.js";
 import { i as CostLedger } from "../cost-ledger-_IbkejCF.js";
+import { Z as campaignSplitDigest } from "../campaign-evidence-BsrCmBSE.js";
 import { f as mapConcurrentRange } from "../journal-BuRLm6ho.js";
 import { C as inMemoryCampaignStorage, S as fsCampaignStorage } from "../external-optimizer-subprocess-DMgmqTB6.js";
 import { i as makeProposalFinding } from "../types-C8VBbiw6.js";
-import { K as classifyOtlpSpanRole, q as isOtlpModelCall } from "../kind-factory-DE088Zhv.js";
-import { t as buildDefaultAnalystRegistry } from "../default-registry-CpLko4jB.js";
-import { LLM_MODEL_ATTR_KEYS, SPAN_KIND_ATTR_KEYS } from "../trace-attributes.js";
+import { n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-D4uumjl3.js";
+import { t as buildDefaultAnalystRegistry } from "../default-registry-xiHrldW8.js";
 import { t as extractUsage } from "../extract-usage-BrQ8mCLX.js";
 import { t as createChatClient } from "../chat-client-DqPqWzVV.js";
 import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-BXlkwMPR.js";
-import { i as summarizeTraceErrors, n as recordAggregateMeasurements, r as summarizeExecutionMeasurements, t as readTaskFailureLabels } from "../task-failure-attributes-BOxjfbhr.js";
-import { a as externalTextOptimizationMethod, c as REFERENCE_EQUIVALENCE_INPUT_LIMITS, d as runReferenceEquivalenceJudge, i as composeGate, l as REFERENCE_EQUIVALENCE_JUDGE_VERSION, n as gepaOptimizationMethod, r as heldOutGate, t as skillOptOptimizationMethod, u as createReferenceEquivalenceJudge } from "../skillopt-optimization-method-B2zxoDk0.js";
-import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-CD-bUeyR.js";
+import { a as externalTextOptimizationMethod, c as REFERENCE_EQUIVALENCE_INPUT_LIMITS, d as runReferenceEquivalenceJudge, i as composeGate, l as REFERENCE_EQUIVALENCE_JUDGE_VERSION, n as gepaOptimizationMethod, r as heldOutGate, t as skillOptOptimizationMethod, u as createReferenceEquivalenceJudge } from "../skillopt-optimization-method-FSk8b5FQ.js";
+import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-DdrYWHDh.js";
+import { t as fromOtelSpans } from "../otel-spans-CQY1pNiN.js";
 import { createHash } from "node:crypto";
 import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, agentProfileImprovementExperimentSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileImprovementRunCellSchema, agentProfileImprovementRunReceiptSchema, agentProfileImprovementSuiteInputsSchema, agentProfileImprovementSuiteSchema, agentProfileImprovementTaskSchema, candidateExecutionEvidenceSchema, canonicalCandidateDigest, canonicalCandidateJson, numbersApproximatelyEqual, omitTopLevelDigest } from "@tangle-network/agent-interface";
 import { createReadStream } from "node:fs";
@@ -3343,274 +3342,6 @@ function fromFeedbackTable(opts) {
 	};
 }
 //#endregion
-//#region src/contract/intake/otel-spans.ts
-/**
-* # `intake/otel-spans` — OTel `TraceSpanEvent[]` → `RunRecord[]`.
-*
-* Turns an existing observability stream into the substrate-canonical
-* `RunRecord` shape so consumers with logs but no eval discipline can
-* call `analyzeRuns()` against their production traffic immediately.
-*
-* Pivot rule: spans are grouped by `tangle.runId` (the same attribute the
-* hosted-tier wire format uses) or, when absent, by `traceId`. One group
-* becomes one `RunRecord`. The root span (no `parentSpanId`) supplies:
-*
-*   - `runId` (the group key)
-*   - `wallMs` from `endTimeUnixNano - startTimeUnixNano`
-*   - `model` from `gen_ai.request.model` / `llm.model` / `tangle.model`
-*   - task failure class and detail from explicit `tangle.task.*` attributes
-*   - cost from `cost.usd` / `gen_ai.usage.cost_usd` / `tangle.cost.usd`
-*   - token usage from model-call input, output, cache-read, and cache-write
-*     attributes without double-counting aggregate parent spans
-*   - task quality from an explicit `scoreForRun` callback or a designated
-*     evaluation attribute on a root / `EVALUATOR` span; `outcome.raw`
-*     collects every numeric attribute without promoting it to task quality.
-*
-* Errored tool, model, and child-agent spans contribute to execution-error
-* counts. Root process, guardrail, evaluator, propagated parent, and unknown
-* errors retain separate counters. Only one failed root can set
-* `RunRecord.terminalOutcome` and `RunRecord.terminalFailureReason`; a child
-* error cannot become a task failure.
-*/
-const TASK_SCORE_ATTR_KEYS = [
-	"gen_ai.evaluation.score.value",
-	"tangle.task.score",
-	"eval.score",
-	"tangle.score"
-];
-const MODEL_KEYS = [
-	"tangle.model",
-	...LLM_MODEL_ATTR_KEYS,
-	"model"
-];
-const PROMPT_HASH_KEYS = ["tangle.prompt_hash", "prompt.hash"];
-const CONFIG_HASH_KEYS = ["tangle.config_hash", "config.hash"];
-function fromOtelSpans(opts) {
-	const { spans, defaultSplit = "holdout", experimentId = "otel-corpus" } = opts;
-	const grouped = groupSpans(spans);
-	const runs = [];
-	for (const [groupKey, groupSpans] of grouped) {
-		const root = findRoot(groupSpans);
-		if (!root) continue;
-		const measurements = summarizeExecutionMeasurements(groupSpans.map((span) => ({
-			id: span.spanId,
-			...span.parentSpanId ? { parentId: span.parentSpanId } : {},
-			attributes: span.attributes,
-			modelCall: isExplicitModelCall(span),
-			aggregate: isExplicitAggregate(span)
-		})));
-		const callSpanIds = new Set(measurements.callSpanIds);
-		const callSpans = groupSpans.filter((span) => callSpanIds.has(span.spanId));
-		const wallMs = unixNanoDurationMs(root.startTimeUnixNano, root.endTimeUnixNano);
-		const model = readAttrString(callSpans, MODEL_KEYS) ?? readAttrString(groupSpans, MODEL_KEYS) ?? "unknown@unknown";
-		const capturedCost = (measurements.cost.complete ? measurements.cost.value : void 0) ?? measurements.aggregate?.costUsd;
-		const costUsd = capturedCost ?? null;
-		const scenarioId = readConsistentScenarioId(groupKey, groupSpans) ?? groupKey;
-		const promptHash = readAttrString(groupSpans, PROMPT_HASH_KEYS) ?? "sha256:unknown";
-		const configHash = readAttrString(groupSpans, CONFIG_HASH_KEYS) ?? "sha256:unknown";
-		const score = resolveTaskScore(groupKey, groupSpans, opts.scoreForRun);
-		const taskFailure = readTaskFailureLabels(groupSpans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span)), `fromOtelSpans: run '${groupKey}'`);
-		const rawNumeric = collectNumericAttrs(groupSpans);
-		const errorSummary = summarizeTraceErrors(groupSpans.map((span) => ({
-			id: spanIdentity(span),
-			...span.parentSpanId ? { parentId: parentIdentity(span) } : {},
-			role: errorRoleForSpan(span),
-			error: span.status?.code === "ERROR",
-			processRoot: !span.parentSpanId && isTerminalRootCandidate(span)
-		})));
-		rawNumeric.error_span_count = errorSummary.total;
-		rawNumeric.execution_error_count = errorSummary.execution;
-		rawNumeric.process_error_count = errorSummary.process;
-		rawNumeric.guardrail_error_count = errorSummary.guardrail;
-		rawNumeric.judge_error_count = errorSummary.evaluation;
-		rawNumeric.propagated_error_count = errorSummary.propagated;
-		rawNumeric.unclassified_error_count = errorSummary.unclassified;
-		rawNumeric.llm_span_count = measurements.modelCallCount;
-		if (measurements.cost.value !== void 0 && !measurements.cost.complete) rawNumeric.partial_observed_cost_usd = measurements.cost.value;
-		recordAggregateMeasurements(rawNumeric, measurements.aggregate);
-		const judgeScores = score !== void 0 ? {
-			perJudge: { "otel-derived": { score } },
-			perDimMean: { score },
-			composite: score
-		} : void 0;
-		const terminalOutcome = terminalOutcomeFromRoots(groupSpans);
-		const failedRoot = terminalOutcome === "failed" ? groupSpans.find((span) => !span.parentSpanId && isTerminalRootCandidate(span) && span.status?.code === "ERROR") : void 0;
-		const outcome = {
-			raw: rawNumeric,
-			...judgeScores ? { judgeScores } : {}
-		};
-		if (score !== void 0) if (defaultSplit === "holdout") outcome.holdoutScore = score;
-		else outcome.searchScore = score;
-		runs.push({
-			runId: groupKey,
-			experimentId,
-			candidateId: root.attributes["tangle.candidateId"] ?? "otel-default",
-			seed: 0,
-			model,
-			promptHash,
-			configHash,
-			commitSha: root.attributes["tangle.commit_sha"] ?? "unknown",
-			wallMs,
-			costUsd,
-			costProvenance: capturedCost === void 0 ? {
-				kind: "uncaptured",
-				usd: null
-			} : {
-				kind: "observed",
-				usd: capturedCost
-			},
-			tokenUsage: measurements.tokenUsage,
-			terminalOutcome,
-			...failedRoot ? { terminalFailureReason: failedRoot.status?.message ?? failedRoot.name } : {},
-			outcome,
-			...taskFailure,
-			splitTag: defaultSplit,
-			scenarioId
-		});
-	}
-	return runs;
-}
-function terminalOutcomeFromRoots(spans) {
-	const roots = spans.filter((span) => !span.parentSpanId && isTerminalRootCandidate(span));
-	if (roots.length !== 1) return "unknown";
-	if (roots[0].status?.code === "OK") return "succeeded";
-	if (roots[0].status?.code === "ERROR") return "failed";
-	return "unknown";
-}
-function isTerminalRootCandidate(span) {
-	const role = errorRoleForSpan(span);
-	if (role === "LLM" || role === "TOOL" || role === "GUARDRAIL" || role === "EVALUATOR") return false;
-	return true;
-}
-function readSpanKind(span) {
-	return readAttrString([span], [...SPAN_KIND_ATTR_KEYS, "span.kind"])?.toUpperCase();
-}
-function errorRoleForSpan(span) {
-	return classifyOtlpSpanRole({
-		kind: readSpanKind(span),
-		name: span.name,
-		attributes: span.attributes
-	});
-}
-function spanIdentity(span) {
-	return `${span.traceId}:${span.spanId}`;
-}
-function parentIdentity(span) {
-	return `${span.traceId}:${span.parentSpanId}`;
-}
-function isExplicitModelCall(span) {
-	return isOtlpModelCall({
-		kind: readSpanKind(span),
-		name: span.name,
-		attributes: span.attributes
-	});
-}
-function isExplicitAggregate(span) {
-	const kind = readSpanKind(span);
-	return kind !== void 0 && kind !== "LLM";
-}
-function groupSpans(spans) {
-	const m = /* @__PURE__ */ new Map();
-	for (const span of spans) {
-		const key = span["tangle.runId"] ?? span.traceId;
-		const list = m.get(key) ?? [];
-		list.push(span);
-		m.set(key, list);
-	}
-	return m;
-}
-function findRoot(group) {
-	const structuralRoots = group.filter((span) => !span.parentSpanId);
-	const terminalRoots = structuralRoots.filter(isTerminalRootCandidate);
-	return orderSpans(terminalRoots.length > 0 ? terminalRoots : structuralRoots.length > 0 ? structuralRoots : group)[0];
-}
-function readAttrString(spans, keys) {
-	for (const span of spans) for (const key of keys) {
-		const v = span.attributes[key];
-		if (typeof v === "string" && v.length > 0) return v;
-	}
-}
-function readConsistentScenarioId(runId, spans) {
-	const values = /* @__PURE__ */ new Set();
-	for (const span of spans) {
-		const topLevel = span["tangle.scenarioId"];
-		if (typeof topLevel === "string" && topLevel.length > 0) values.add(topLevel);
-		const attribute = span.attributes["tangle.scenarioId"];
-		if (typeof attribute === "string" && attribute.length > 0) values.add(attribute);
-	}
-	if (values.size > 1) throw new ValidationError(`fromOtelSpans: conflicting scenario ids for run '${runId}': ${[...values].sort().join(", ")}`);
-	return values.values().next().value;
-}
-function resolveTaskScore(runId, spans, scoreForRun) {
-	const orderedSpans = orderSpans(spans);
-	const sources = [];
-	if (scoreForRun) {
-		const supplied = scoreForRun(runId, orderedSpans);
-		if (supplied !== void 0) {
-			if (typeof supplied !== "number" || !Number.isFinite(supplied)) throw new ValidationError(`fromOtelSpans: scoreForRun returned a non-finite number for run '${runId}'`);
-			sources.push({
-				label: "scoreForRun",
-				value: supplied
-			});
-		}
-	}
-	for (const span of orderedSpans) {
-		const role = errorRoleForSpan(span);
-		if (span.parentSpanId && role !== "EVALUATOR") continue;
-		if (role === "EVALUATOR" && span.status?.code === "ERROR") continue;
-		for (const key of TASK_SCORE_ATTR_KEYS) {
-			if (!Object.hasOwn(span.attributes, key)) continue;
-			sources.push({
-				label: `span '${span.spanId}' attribute '${key}'`,
-				value: parseTaskScoreAttribute(runId, span.spanId, key, span.attributes[key])
-			});
-		}
-	}
-	if (sources.length === 0) return void 0;
-	sources.sort((left, right) => left.label.localeCompare(right.label));
-	const score = sources[0].value;
-	if (sources.some((source) => source.value !== score)) throw new ValidationError(`fromOtelSpans: conflicting task-quality scores for run '${runId}': ${sources.map((source) => `${source.label}=${source.value}`).join(", ")}`);
-	return score;
-}
-function parseTaskScoreAttribute(runId, spanId, key, value) {
-	const source = `span '${spanId}' attribute '${key}'`;
-	if (typeof value === "string") {
-		if (value.trim().length === 0) throw new ValidationError(`fromOtelSpans: ${source} is blank for run '${runId}'; task quality must be finite`);
-		const parsed = Number(value);
-		if (Number.isFinite(parsed)) return parsed;
-	} else if (typeof value === "number" && Number.isFinite(value)) return value;
-	throw new ValidationError(`fromOtelSpans: ${source} is not a finite task-quality score for run '${runId}'`);
-}
-function orderSpans(spans) {
-	return [...spans].sort((left, right) => compareUnixNano(left.startTimeUnixNano, right.startTimeUnixNano) || left.spanId.localeCompare(right.spanId));
-}
-function parseUnixNano(value) {
-	try {
-		return BigInt(value);
-	} catch {
-		throw new ValidationError(`fromOtelSpans: invalid Unix nanosecond timestamp '${value}'`);
-	}
-}
-function compareUnixNano(left, right) {
-	const leftValue = parseUnixNano(left);
-	const rightValue = parseUnixNano(right);
-	return leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0;
-}
-function unixNanoDurationMs(start, end) {
-	const delta = parseUnixNano(end) - parseUnixNano(start);
-	if (delta <= 0n) return 0;
-	const wholeMs = delta / 1000000n;
-	const fractionalMs = delta % 1000000n;
-	const value = Number(wholeMs) + Number(fractionalMs) / 1e6;
-	if (!Number.isSafeInteger(Number(wholeMs))) throw new ValidationError("fromOtelSpans: span duration exceeds the safe millisecond range");
-	return value;
-}
-function collectNumericAttrs(spans) {
-	const raw = {};
-	for (const span of spans) for (const [k, v] of Object.entries(span.attributes)) if (typeof v === "number" && Number.isFinite(v)) raw[k] = v;
-	return raw;
-}
-//#endregion
 export { FileSystemOutcomeStore, InMemoryOutcomeStore, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, SelfImproveRunError, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, campaignSplitDigest, compareOptimizationMethods, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evaluatePairedMeasurements, externalTextOptimizationMethod, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromRunRecordDir, fsCampaignStorage, gepaOptimizationMethod, heldOutGate, inMemoryCampaignStorage, llmJudge, makeProposalFinding, measuredComparisonFromAgentProfileImprovementExperiment, measuredComparisonFromCandidateExperiment, observeCodeAgentSession, observeCodeAgentStore, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, parseCodeAgentJsonlFile, partitionRunsByAuthoringModel, runAgentProfileImprovementExperiment, runCampaign, runCandidateExperiment, runEval, runImprovementLoop, runReferenceEquivalenceJudge, sealAgentProfileImprovementExperiment, sealAgentProfileImprovementSuite, sealAgentProfileImprovementTask, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, sealCandidateExperiment, selfImprove, skillOptOptimizationMethod, streamCodeAgentJsonlFile, summarizeExecution, transientDispatchFailure, verifyAgentProfileImprovementExperimentComparison, verifyCandidateExperiment, verifyCandidateExperimentComparison };
 
 //# sourceMappingURL=index.js.map