@tangle-network/agent-eval 0.135.4 → 0.137.0

package/dist/{default-registry-CHmdy2An.js → default-registry-CLXbRt0f.js}

@@ -1,9 +1,10 @@
-import { i as CostLedger } from "./cost-ledger-DHAjwNj7.js";
-import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, n as LlmClient, s as callLlm, u as costReceiptFromLlmError } from "./llm-client-DHx8pzyJ.js";
+import { i as CostLedger } from "./cost-ledger-D-5_-dhi.js";
+import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, n as LlmClient, s as callLlm, u as costReceiptFromLlmError } from "./llm-client-DKB25jV8.js";
 import { LLM_CONTEXT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_OUTPUT_TOKEN_ATTR_KEYS, TOOL_NAME_ATTR_KEYS } from "./trace-attributes.js";
 import { t as executionTrackByLane } from "./execution-tracks-CpgFPpS5.js";
-import { S as spanEpochMillis, r as runTraceAnalysisLoop, t as buildTraceAnalystTools } from "./tools-D8yTtNSN.js";
+import { O as spanEpochMillis, a as runTraceAnalysisLoop, r as buildTraceAnalystTools } from "./tools-DZk2Jn64.js";
 import { c as makeFinding, o as combineAbortSignals } from "./proposal-findings-DCawte-y.js";
+import { t as analyzeSupervisorRunIntegrity } from "./integrity-CCXTftiL.js";
 import { ai } from "@ax-llm/ax";
 import { z } from "zod";
 import { randomUUID } from "node:crypto";
@@ -134,11 +135,8 @@ function resolveModel(req, defaultModel) {
 /**
 * Deterministic behavioral metrics over OTLP spans — pure arithmetic, no LLM.
 *
-* These are the model-independent multiplier: the four trace-quality signals a
-* tolerant analyzer (e.g. HALO) re-derives per run inside the model — token
-* growth, output decay, tool monoculture, missing self-verification — computed
-* here once, in TypeScript, with zero model judgment. A finding that falls out
-* of arithmetic is trivially model-agnostic and cannot hallucinate the trend.
+* It computes token growth, output decay, tool monoculture, and missing
+* self-verification once in TypeScript with no model judgment.
 *
 * General, not trace-specific: the detectors key off token trajectories and
 * tool usage present in any agentic OTLP trace, not any one benchmark.
@@ -472,18 +470,9 @@ function everyAdjacent(values, predicate) {
 //#endregion
 //#region src/analyst/behavioral-analyst.ts
 /**
-* `behavioralAnalyst` — a DETERMINISTIC analyst (cost.kind = 'deterministic',
-* never calls the LLM). It produces the efficiency/behavioral findings a
-* tolerant agentic analyzer (HALO) re-derives per run inside the model —
-* context bloat, output decay, tool monoculture, missing self-verification —
-* directly from arithmetic over spans (`computeTraceMetrics`).
-*
-* Why it matters: these findings are model-agnostic BY CONSTRUCTION (no model
-* in the loop), so they cannot return 0 on a weak model the way the Ax-RLM
-* does — and they are strictly more reliable than HALO, which spends tokens
-* re-deriving the same numbers and can hallucinate the trend. The agentic
-* RLM kinds remain for SEMANTIC findings that genuinely need a model; this
-* analyst owns the behavioral class.
+* Deterministic behavioral analysis over arithmetic in trace spans.
+* This pass is cheap and repeatable; semantic analysis remains the job of
+* model-backed analysts. Relative quality requires a labeled comparison.
 */
 const RECOMMENDED_ACTION = {
 	"monotonic-input-growth": "Inspect context assembly; if prior history is repeatedly included, summarize completed work before the next model call.",
@@ -491,19 +480,44 @@ const RECOMMENDED_ACTION = {
 	"single-tool-dependency": "Test whether an inspect or verification tool improves outcomes after the repeated call fails or returns no progress.",
 	"no-self-verification": "After state-changing actions, require an observable check before the agent proceeds."
 };
-const ANALYST_ID = "efficiency-behavioral";
+const ANALYST_ID$1 = "efficiency-behavioral";
+const DEFAULT_MAX_TRACES = 1e3;
+const DEFAULT_MAX_EVIDENCE_REFS = 20;
+const TRACE_PAGE_SIZE = 200;
 const AGGREGATE_CLAIM = {
 	"monotonic-input-growth": (observed, analyzed) => `${observed}/${analyzed} analyzed traces showed input tokens grow from zero to nonzero or to at least 3x their initial value across at least 3 serial model calls without a decrease.`,
 	"output-length-decay": (observed, analyzed) => `${observed}/${analyzed} analyzed traces showed output tokens decrease while input tokens increased monotonically across at least 3 serial model calls.`,
 	"single-tool-dependency": (observed, analyzed) => `${observed}/${analyzed} analyzed traces used only one named tool across at least 3 tool calls.`,
 	"no-self-verification": (observed, analyzed) => `${observed}/${analyzed} analyzed traces had at least 3 tool calls without a verification-named tool call.`
 };
+async function listTraceIds(store, maxTraces, signal) {
+	const traceIds = /* @__PURE__ */ new Set();
+	let offset = 0;
+	let expectedTotal;
+	while (true) {
+		signal?.throwIfAborted();
+		const page = await store.queryTraces({
+			limit: TRACE_PAGE_SIZE,
+			offset
+		});
+		if (expectedTotal === void 0) expectedTotal = page.total;
+		if (page.total !== expectedTotal) throw new Error(`behavioralAnalyst: trace count changed during pagination (${expectedTotal} to ${page.total})`);
+		if (page.total > maxTraces) throw new RangeError(`behavioralAnalyst: ${page.total} traces exceed maxTraces=${maxTraces}; filter the store or raise the explicit limit`);
+		for (const trace of page.traces) traceIds.add(trace.trace_id);
+		if (traceIds.size > maxTraces) throw new RangeError(`behavioralAnalyst: more than maxTraces=${maxTraces} unique traces were returned`);
+		if (!page.has_more) break;
+		if (page.traces.length === 0) throw new Error("behavioralAnalyst: trace store returned an empty page with has_more=true");
+		offset += page.traces.length;
+	}
+	if (traceIds.size !== expectedTotal) throw new Error(`behavioralAnalyst: pagination returned ${traceIds.size}/${expectedTotal ?? 0} unique traces`);
+	return [...traceIds].sort();
+}
 /**
 * Map computed signals → structured AnalystFindings. Pure: no LLM, no clock
 * dependence beyond `produced_at` (overridable for deterministic tests).
 */
 function deriveEfficiencyFindings(metrics, opts = {}) {
-	const analystId = opts.analystId ?? ANALYST_ID;
+	const analystId = opts.analystId ?? ANALYST_ID$1;
 	const traceId = metrics.traceId;
 	return metrics.signals.map((sig) => makeFinding({
 		analyst_id: analystId,
@@ -528,18 +542,20 @@ function deriveEfficiencyFindings(metrics, opts = {}) {
 	}));
 }
 /** The deterministic behavioral/efficiency analyst (no LLM, any-model). */
-function behavioralAnalyst() {
+function behavioralAnalyst(options = {}) {
+	const maxTraces = positiveInteger(options.maxTraces ?? DEFAULT_MAX_TRACES, "maxTraces");
+	const maxEvidenceRefsPerFinding = positiveInteger(options.maxEvidenceRefsPerFinding ?? DEFAULT_MAX_EVIDENCE_REFS, "maxEvidenceRefsPerFinding");
 	return {
-		id: ANALYST_ID,
+		id: ANALYST_ID$1,
 		description: "Deterministic behavioral/efficiency findings over OTLP spans — token-growth, output-decay, tool-monoculture, missing self-verification. Zero LLM; model-agnostic by construction.",
 		inputKind: "trace-store",
 		cost: { kind: "deterministic" },
 		version: "2.0.0",
-		async analyze(store) {
-			const overview = await store.getOverview();
-			const analyzedTraceIds = [...new Set(overview.sample_trace_ids)].sort();
+		async analyze(store, context) {
+			const analyzedTraceIds = await listTraceIds(store, maxTraces, context.signal);
 			const findingsById = /* @__PURE__ */ new Map();
 			for (const traceId of analyzedTraceIds) {
+				context.signal?.throwIfAborted();
 				const viewed = await store.viewTrace({ trace_id: traceId });
 				if (viewed.trace_id !== traceId) throw new Error(`behavioralAnalyst: requested trace '${traceId}', received '${viewed.trace_id}'`);
 				if (!viewed.spans) throw new Error(`behavioralAnalyst: trace '${traceId}' is oversized; complete spans are required`);
@@ -550,30 +566,39 @@ function behavioralAnalyst() {
 					if (!current) {
 						findingsById.set(finding.finding_id, {
 							finding,
-							traceIds: [traceId],
+							observedTraceCount: 1,
+							evidenceTraceIds: [traceId],
 							evidence: [...finding.evidence_refs]
 						});
 						continue;
 					}
-					current.traceIds.push(traceId);
-					current.evidence.push(...finding.evidence_refs);
+					current.observedTraceCount += 1;
+					if (current.evidence.length < maxEvidenceRefsPerFinding) {
+						current.evidenceTraceIds.push(traceId);
+						current.evidence.push(...finding.evidence_refs);
+					}
 				}
 			}
-			return [...findingsById.values()].map(({ finding, traceIds, evidence }) => ({
+			return [...findingsById.values()].map(({ finding, observedTraceCount, evidenceTraceIds, evidence }) => ({
 				...finding,
-				claim: AGGREGATE_CLAIM[finding.subject](traceIds.length, analyzedTraceIds.length),
-				rationale: `${traceIds.length}/${analyzedTraceIds.length} analyzed traces exhibited this pattern.`,
+				claim: AGGREGATE_CLAIM[finding.subject](observedTraceCount, analyzedTraceIds.length),
+				rationale: `${observedTraceCount}/${analyzedTraceIds.length} analyzed traces exhibited this pattern.`,
 				evidence_refs: evidence,
 				metadata: {
 					deterministic: true,
-					trace_ids: traceIds,
-					observed_trace_count: traceIds.length,
+					evidence_trace_ids: evidenceTraceIds,
+					omitted_evidence_trace_count: observedTraceCount - evidenceTraceIds.length,
+					observed_trace_count: observedTraceCount,
 					analyzed_trace_count: analyzedTraceIds.length
 				}
 			}));
 		}
 	};
 }
+function positiveInteger(value, name) {
+	if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`behavioralAnalyst: ${name} must be a positive safe integer`);
+	return value;
+}
 //#endregion
 //#region src/analyst/ax-cost-service.ts
 /**
@@ -1719,6 +1744,62 @@ function truncateForContext(s, max) {
 	return `${s.slice(0, max - 1).trimEnd()}…`;
 }
 //#endregion
+//#region src/analyst/kinds/control-integrity.ts
+const ANALYST_ID = "control-integrity";
+function shown(value) {
+	if (value === void 0) return "<absent>";
+	const encoded = JSON.stringify(value);
+	return encoded === void 0 ? String(value) : encoded;
+}
+function evidenceRef(namespace, value) {
+	return {
+		kind: "metric",
+		uri: `supervisor-run://${encodeURIComponent(namespace)}/${value.path}`,
+		excerpt: shown(value.value)
+	};
+}
+/** Translate typed supervisor-run integrity issues into the shared analyst envelope. */
+function emitControlIntegrityFindings(input, producedAt) {
+	const report = analyzeSupervisorRunIntegrity(input, { capturedAt: producedAt });
+	return report.issues.map((issue) => makeFinding({
+		analyst_id: ANALYST_ID,
+		produced_at: producedAt,
+		area: issue.area,
+		severity: issue.severity,
+		subject: `${report.runRef}/${issue.subject}`,
+		claim: issue.claim,
+		rationale: issue.detail,
+		evidence_refs: issue.evidence.map((value) => evidenceRef(report.runRef, value)),
+		recommended_action: issue.recommendedAction,
+		validation_plan: "Re-run this deterministic analyst on the retained SupervisorRunSources or SupervisorRunTree after correcting the producer.",
+		confidence: 1,
+		metadata: {
+			integrity_code: issue.code,
+			integrity_input: report.input,
+			integrity_run_ref: report.runRef,
+			integrity_subject: issue.subject,
+			...issue.metadata
+		}
+	}));
+}
+/** Deterministic Analyst adapter for `SupervisorRunSources | SupervisorRunTree`. */
+var ControlIntegrityAnalyst = class {
+	id = ANALYST_ID;
+	description = "Deterministic supervisor-run integrity checks with explicit unavailable evidence.";
+	inputKind = "custom";
+	cost = {
+		kind: "deterministic",
+		est_usd_per_run: 0
+	};
+	version = "2.0.0";
+	async analyze(input, ctx) {
+		const findings = emitControlIntegrityFindings(input, ctx.tags?.producedAt ?? (/* @__PURE__ */ new Date()).toISOString());
+		ctx.log?.(`control-integrity: ${findings.length} finding(s)`, { input: "nodes" in input ? "SupervisorRunTree" : "SupervisorRunSources" });
+		return findings;
+	}
+};
+const CONTROL_INTEGRITY_ANALYST = new ControlIntegrityAnalyst();
+//#endregion
 //#region src/analyst/tool-groups.ts
 const TOOL_NAMES_BY_GROUP = {
 	all: /* @__PURE__ */ new Set(),
@@ -2497,7 +2578,7 @@ function selectPriorFindings(source, analystId) {
 //#region src/analyst/default-registry.ts
 function buildDefaultAnalystRegistry(opts = {}) {
 	const registry = new AnalystRegistry(opts.registry);
-	if (opts.includeBehavioral !== false) registry.register(behavioralAnalyst());
+	if (opts.includeBehavioral !== false) registry.register(behavioralAnalyst(opts.behavioral));
 	if (opts.ai) {
 		const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS;
 		for (const spec of kinds) registry.register(createTraceAnalystKind(spec, {
@@ -2508,6 +2589,6 @@ function buildDefaultAnalystRegistry(opts = {}) {
 	return registry;
 }
 //#endregion
-export { parseFindingSubject as A, stripCodeFences as C, FindingSubjectStringSchema as D, FINDING_SUBJECT_SYNTAX as E, createChatClient as F, createAnalystAi as I, behavioralAnalyst as M, deriveEfficiencyFindings as N, KIND_EXPECTED_SUBJECTS as O, computeTraceMetrics as P, coerceToFindingRows as S, FINDING_SUBJECT_KINDS as T, RawAnalystEvidenceSchema as _, KNOWLEDGE_GAP_KIND_SPEC as a, parseRawFinding as b, buildTraceToolsForGroup as c, renderUpstreamFindings as d, settleUsageReceiptFromCostLedger as f, RAW_FINDING_SCHEMA_PROMPT as g, ANALYST_SEVERITIES as h, KNOWLEDGE_POISONING_KIND_SPEC as i, renderFindingSubject as j, findingSubjectGrammarPromptFor as k, createTraceAnalystKind as l, structureFindings as m, AnalystRegistry as n, IMPROVEMENT_KIND_SPEC as o, validateUsageSettlementTimeout as p, DEFAULT_TRACE_ANALYST_KINDS as r, FAILURE_MODE_KIND_SPEC as s, buildDefaultAnalystRegistry as t, renderPriorFindings as u, RawAnalystFindingSchema as v, FINDING_SUBJECT_GRAMMAR_PROMPT as w, coerceJson as x, evidenceRefsFromRawFinding as y };
+export { FindingSubjectStringSchema as A, parseRawFinding as C, FINDING_SUBJECT_GRAMMAR_PROMPT as D, stripCodeFences as E, behavioralAnalyst as F, deriveEfficiencyFindings as I, computeTraceMetrics as L, findingSubjectGrammarPromptFor as M, parseFindingSubject as N, FINDING_SUBJECT_KINDS as O, renderFindingSubject as P, createChatClient as R, evidenceRefsFromRawFinding as S, coerceToFindingRows as T, structureFindings as _, KNOWLEDGE_GAP_KIND_SPEC as a, RawAnalystEvidenceSchema as b, buildTraceToolsForGroup as c, emitControlIntegrityFindings as d, createTraceAnalystKind as f, validateUsageSettlementTimeout as g, settleUsageReceiptFromCostLedger as h, KNOWLEDGE_POISONING_KIND_SPEC as i, KIND_EXPECTED_SUBJECTS as j, FINDING_SUBJECT_SYNTAX as k, CONTROL_INTEGRITY_ANALYST as l, renderUpstreamFindings as m, AnalystRegistry as n, IMPROVEMENT_KIND_SPEC as o, renderPriorFindings as p, DEFAULT_TRACE_ANALYST_KINDS as r, FAILURE_MODE_KIND_SPEC as s, buildDefaultAnalystRegistry as t, ControlIntegrityAnalyst as u, ANALYST_SEVERITIES as v, coerceJson as w, RawAnalystFindingSchema as x, RAW_FINDING_SCHEMA_PROMPT as y, createAnalystAi as z };
 
-//# sourceMappingURL=default-registry-CHmdy2An.js.map
+//# sourceMappingURL=default-registry-CLXbRt0f.js.map