@tangle-network/agent-eval 0.109.1 → 0.110.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/analyst/index.d.ts +10 -12
- package/dist/analyst/index.js +8 -11
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DJYpep3L.d.ts → analyze-runs-Dmz6LA9e.d.ts} +4 -4
- package/dist/{baseline-Bbid3WoO.d.ts → baseline-DsNteOgR.d.ts} +32 -2
- package/dist/belief-state/index.d.ts +6 -6
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +7 -8
- package/dist/builder-eval/index.d.ts +4 -4
- package/dist/builder-eval/index.js +1 -2
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/{calibration-BPmzuVPk.d.ts → calibration-Dz8TQV4y.d.ts} +2 -2
- package/dist/campaign/index.d.ts +62 -20
- package/dist/campaign/index.js +9 -8
- package/dist/{chunk-LOJ2QVCE.js → chunk-2IY4ILP4.js} +2 -2
- package/dist/{chunk-LIEJUH2I.js → chunk-6PL5MGDL.js} +9 -9
- package/dist/{chunk-2OGPXHOB.js → chunk-7NX6ZSBG.js} +36 -7
- package/dist/chunk-7NX6ZSBG.js.map +1 -0
- package/dist/{chunk-R6D7NEYJ.js → chunk-GBI5J5DB.js} +81 -11
- package/dist/chunk-GBI5J5DB.js.map +1 -0
- package/dist/{chunk-YEHAEDUD.js → chunk-IMWDSFUM.js} +604 -2
- package/dist/chunk-IMWDSFUM.js.map +1 -0
- package/dist/{chunk-OVPVM4JC.js → chunk-J4AKLZEV.js} +15 -4
- package/dist/{chunk-OVPVM4JC.js.map → chunk-J4AKLZEV.js.map} +1 -1
- package/dist/{chunk-JZXGWLK5.js → chunk-MHNQWM4I.js} +62 -6
- package/dist/chunk-MHNQWM4I.js.map +1 -0
- package/dist/{chunk-QRVS7MX4.js → chunk-OW47B5WA.js} +3 -5
- package/dist/{chunk-QRVS7MX4.js.map → chunk-OW47B5WA.js.map} +1 -1
- package/dist/{chunk-DBDRR6GF.js → chunk-PLOMR3HP.js} +48 -2
- package/dist/chunk-PLOMR3HP.js.map +1 -0
- package/dist/{chunk-GDZAWO2I.js → chunk-QFGTU7MT.js} +2 -2
- package/dist/{chunk-V7HNA47Z.js → chunk-RSVSSZKF.js} +5 -5
- package/dist/{chunk-5PK3626Q.js → chunk-XRGOKCMO.js} +88 -17
- package/dist/chunk-XRGOKCMO.js.map +1 -0
- package/dist/{code-agent-session-rnJKlqmT.d.ts → code-agent-session-yitf9I-F.d.ts} +1 -1
- package/dist/contract/index.d.ts +26 -28
- package/dist/contract/index.js +11 -13
- package/dist/contract/index.js.map +1 -1
- package/dist/{control-B8UthSBL.d.ts → control-U8LBKUES.d.ts} +5 -6
- package/dist/control.d.ts +8 -9
- package/dist/control.js +6 -8
- package/dist/{dataset-DS7ytHZU.d.ts → dataset-NENEzRgk.d.ts} +1 -1
- package/dist/{default-registry-BswHCXnU.d.ts → default-registry-Bcf1uKVI.d.ts} +1 -2
- package/dist/{emitter-C2rqGH_l.d.ts → emitter-BRchAAAx.d.ts} +2 -2
- package/dist/{failure-cluster-DH9Flgcf.d.ts → failure-cluster-C48PiReX.d.ts} +2 -2
- package/dist/feedback-trajectory-pDcz1lQ1.d.ts +348 -0
- package/dist/{gepa-B3x5Ulcv.d.ts → gepa-T8T215nw.d.ts} +149 -6
- package/dist/hosted/index.d.ts +7 -7
- package/dist/{index-pPtfoIJO.d.ts → index-Dc3VLGhp.d.ts} +2 -2
- package/dist/index.d.ts +645 -61
- package/dist/index.js +1282 -190
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-B4xrdwEK.d.ts → insight-report-D4cXFsLt.d.ts} +1 -1
- package/dist/{integrity-DqGZg3st.d.ts → integrity-qemeBAyx.d.ts} +1 -1
- package/dist/{types-D1ytG0Yg.d.ts → kind-factory-20hcaYpf.d.ts} +169 -2
- package/dist/meta-eval/index.d.ts +5 -5
- package/dist/meta-eval/index.js +1 -2
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{multi-layer-verifier-CI4jdX-q.d.ts → multi-layer-verifier-BsqKuLyN.d.ts} +1 -1
- package/dist/multishot/index.d.ts +3 -3
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +6 -7
- package/dist/pipelines/index.js +3 -6
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{policy-edit-DQUXYMDm.d.ts → policy-edit-D2bBDZDf.d.ts} +2 -2
- package/dist/{pre-registration-BUhVPzE7.d.ts → pre-registration-BepVVa6P.d.ts} +3 -3
- package/dist/{provenance-DdDhf6cg.d.ts → provenance-CyxkvEi9.d.ts} +3 -5
- package/dist/{query-0aTmbmQe.d.ts → query-Ck190MOd.d.ts} +2 -2
- package/dist/{release-report-DeJpsBiA.d.ts → release-report-oBfOz8ku.d.ts} +3 -3
- package/dist/reporting.d.ts +8 -8
- package/dist/{researcher-Wc7dx6GM.d.ts → researcher-CaH0CwFC.d.ts} +6 -6
- package/dist/rl.d.ts +568 -15
- package/dist/rl.js +4 -4
- package/dist/{rubric-predictive-validity-DPnyG-CE.d.ts → rubric-predictive-validity-C-fMteAW.d.ts} +1 -1
- package/dist/{run-record-I-Z3JNvO.d.ts → run-record-DksGsfgv.d.ts} +1 -1
- package/dist/{runtime-trajectory-iW9IhV3e.d.ts → runtime-trajectory-h5i0SZUj.d.ts} +1 -1
- package/dist/{schema-m0gsnbt3.d.ts → schema-SGWcK9wa.d.ts} +1 -1
- package/dist/{semantic-concept-judge-BmNZPB_j.d.ts → semantic-concept-judge-D7z6JCLZ.d.ts} +57 -4
- package/dist/{store-BcFXE6LG.d.ts → store-BsVi7ncX.d.ts} +1 -1
- package/dist/storyboard/index.d.ts +1 -1
- package/dist/{summary-report-QMZVe3P-.d.ts → summary-report-Bz-0-t8v.d.ts} +2 -2
- package/dist/{test-graded-scenario-DeODGLra.d.ts → test-graded-scenario-mzYBKspu.d.ts} +3 -3
- package/dist/traces.d.ts +54 -11
- package/dist/traces.js +25 -27
- package/dist/{types-BdIv5dvA.d.ts → types-v--ctu-b.d.ts} +2 -2
- package/dist/wire/index.d.ts +5 -6
- package/docs/improvement-glossary.md +14 -13
- package/package.json +1 -71
- package/dist/adapters/http.d.ts +0 -142
- package/dist/adapters/http.js +0 -203
- package/dist/adapters/http.js.map +0 -1
- package/dist/adapters/langchain.d.ts +0 -95
- package/dist/adapters/langchain.js +0 -34
- package/dist/adapters/langchain.js.map +0 -1
- package/dist/adapters/otel.d.ts +0 -112
- package/dist/adapters/otel.js +0 -110
- package/dist/adapters/otel.js.map +0 -1
- package/dist/chunk-2OGPXHOB.js.map +0 -1
- package/dist/chunk-45EEMHTC.js +0 -35
- package/dist/chunk-45EEMHTC.js.map +0 -1
- package/dist/chunk-5BKGXME7.js +0 -65
- package/dist/chunk-5BKGXME7.js.map +0 -1
- package/dist/chunk-5PK3626Q.js.map +0 -1
- package/dist/chunk-6SK5VFYK.js +0 -100
- package/dist/chunk-6SK5VFYK.js.map +0 -1
- package/dist/chunk-DBDRR6GF.js.map +0 -1
- package/dist/chunk-DJWX3GVS.js +0 -81
- package/dist/chunk-DJWX3GVS.js.map +0 -1
- package/dist/chunk-FOUG2VVS.js +0 -855
- package/dist/chunk-FOUG2VVS.js.map +0 -1
- package/dist/chunk-JZXGWLK5.js.map +0 -1
- package/dist/chunk-K7QEIHHJ.js +0 -613
- package/dist/chunk-K7QEIHHJ.js.map +0 -1
- package/dist/chunk-KKHDIONI.js +0 -414
- package/dist/chunk-KKHDIONI.js.map +0 -1
- package/dist/chunk-KMPRBJK4.js +0 -74
- package/dist/chunk-KMPRBJK4.js.map +0 -1
- package/dist/chunk-Q2JRAWRI.js +0 -196
- package/dist/chunk-Q2JRAWRI.js.map +0 -1
- package/dist/chunk-R6D7NEYJ.js.map +0 -1
- package/dist/chunk-RZTMDUO7.js +0 -49
- package/dist/chunk-RZTMDUO7.js.map +0 -1
- package/dist/chunk-STGVSCDH.js +0 -202
- package/dist/chunk-STGVSCDH.js.map +0 -1
- package/dist/chunk-YEHAEDUD.js.map +0 -1
- package/dist/control-runtime-Acf9CGhw.d.ts +0 -182
- package/dist/corpus-eBVwhCp1.d.ts +0 -560
- package/dist/counterfactual-DlOz8PBx.d.ts +0 -85
- package/dist/diagnose.d.ts +0 -252
- package/dist/diagnose.js +0 -382
- package/dist/diagnose.js.map +0 -1
- package/dist/feedback-trajectory-C9KCo8ag.d.ts +0 -169
- package/dist/governance/index.d.ts +0 -135
- package/dist/governance/index.js +0 -18
- package/dist/governance/index.js.map +0 -1
- package/dist/groundedness/index.d.ts +0 -112
- package/dist/groundedness/index.js +0 -77
- package/dist/groundedness/index.js.map +0 -1
- package/dist/harness-optimizer-mOl9XX_O.d.ts +0 -106
- package/dist/kind-factory-DvIGo_cP.d.ts +0 -171
- package/dist/knowledge/index.d.ts +0 -103
- package/dist/knowledge/index.js +0 -18
- package/dist/knowledge/index.js.map +0 -1
- package/dist/pareto-E-pembql.d.ts +0 -81
- package/dist/perf/index.d.ts +0 -123
- package/dist/perf/index.js +0 -18
- package/dist/perf/index.js.map +0 -1
- package/dist/prm/index.d.ts +0 -104
- package/dist/prm/index.js +0 -265
- package/dist/prm/index.js.map +0 -1
- package/dist/product-benchmark/index.d.ts +0 -247
- package/dist/product-benchmark/index.js +0 -37
- package/dist/product-benchmark/index.js.map +0 -1
- package/dist/red-team-KmmiqBlY.d.ts +0 -63
- package/dist/redact-B40YG2M_.d.ts +0 -45
- package/dist/rubric-Cc6UHvUb.d.ts +0 -73
- package/dist/run-critic-CmMf05uV.d.ts +0 -56
- package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
- package/dist/telemetry/file.d.ts +0 -19
- package/dist/telemetry/file.js +0 -45
- package/dist/telemetry/file.js.map +0 -1
- package/dist/telemetry/index.d.ts +0 -38
- package/dist/telemetry/index.js +0 -130
- package/dist/telemetry/index.js.map +0 -1
- package/dist/testing-C21CHsq2.d.ts +0 -20
- package/dist/testing.d.ts +0 -1
- package/dist/testing.js +0 -8
- package/dist/testing.js.map +0 -1
- package/dist/trajectory-2TkpSEVh.d.ts +0 -33
- package/dist/workflow/index.d.ts +0 -496
- package/dist/workflow/index.js +0 -2178
- package/dist/workflow/index.js.map +0 -1
- /package/dist/{chunk-LOJ2QVCE.js.map → chunk-2IY4ILP4.js.map} +0 -0
- /package/dist/{chunk-LIEJUH2I.js.map → chunk-6PL5MGDL.js.map} +0 -0
- /package/dist/{chunk-GDZAWO2I.js.map → chunk-QFGTU7MT.js.map} +0 -0
- /package/dist/{chunk-V7HNA47Z.js.map → chunk-RSVSSZKF.js.map} +0 -0
|
@@ -1,8 +1,57 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
// src/trace/schema.ts
|
|
2
|
+
var TRACE_SCHEMA_VERSION = "1.0.0";
|
|
3
|
+
var FAILURE_CLASSES = [
|
|
4
|
+
"success",
|
|
5
|
+
"reasoning_error",
|
|
6
|
+
"tool_selection_error",
|
|
7
|
+
"tool_argument_error",
|
|
8
|
+
"tool_recovery_failure",
|
|
9
|
+
"hallucination",
|
|
10
|
+
"instruction_following",
|
|
11
|
+
"safety_refusal_miss",
|
|
12
|
+
"policy_violation",
|
|
13
|
+
"budget_exceeded",
|
|
14
|
+
"format_drift",
|
|
15
|
+
"permission_escalation",
|
|
16
|
+
"pii_leak",
|
|
17
|
+
"cost_overrun",
|
|
18
|
+
"timeout",
|
|
19
|
+
"sandbox_failure",
|
|
20
|
+
"missing_user_data",
|
|
21
|
+
"missing_domain_data",
|
|
22
|
+
"missing_codebase_context",
|
|
23
|
+
"missing_runtime_context",
|
|
24
|
+
"missing_credentials",
|
|
25
|
+
"missing_integration_connection",
|
|
26
|
+
"missing_integration_scope",
|
|
27
|
+
"integration_approval_required",
|
|
28
|
+
"integration_auth_expired",
|
|
29
|
+
"integration_provider_failure",
|
|
30
|
+
"bad_integration_manifest",
|
|
31
|
+
"unsafe_integration_write_denied",
|
|
32
|
+
"stale_external_data",
|
|
33
|
+
"bad_retrieval",
|
|
34
|
+
"insufficient_evidence",
|
|
35
|
+
"contradictory_evidence",
|
|
36
|
+
"ambiguous_user_intent",
|
|
37
|
+
"knowledge_readiness_blocked",
|
|
38
|
+
"unknown"
|
|
39
|
+
];
|
|
40
|
+
function isLlmSpan(s) {
|
|
41
|
+
return s.kind === "llm";
|
|
42
|
+
}
|
|
43
|
+
function isToolSpan(s) {
|
|
44
|
+
return s.kind === "tool";
|
|
45
|
+
}
|
|
46
|
+
function isRetrievalSpan(s) {
|
|
47
|
+
return s.kind === "retrieval";
|
|
48
|
+
}
|
|
49
|
+
function isJudgeSpan(s) {
|
|
50
|
+
return s.kind === "judge";
|
|
51
|
+
}
|
|
52
|
+
function isSandboxSpan(s) {
|
|
53
|
+
return s.kind === "sandbox";
|
|
54
|
+
}
|
|
6
55
|
|
|
7
56
|
// src/trace/query.ts
|
|
8
57
|
async function runsForScenario(store, scenarioId) {
|
|
@@ -67,6 +116,13 @@ function runFailureClass(run) {
|
|
|
67
116
|
}
|
|
68
117
|
|
|
69
118
|
export {
|
|
119
|
+
TRACE_SCHEMA_VERSION,
|
|
120
|
+
FAILURE_CLASSES,
|
|
121
|
+
isLlmSpan,
|
|
122
|
+
isToolSpan,
|
|
123
|
+
isRetrievalSpan,
|
|
124
|
+
isJudgeSpan,
|
|
125
|
+
isSandboxSpan,
|
|
70
126
|
runsForScenario,
|
|
71
127
|
llmSpans,
|
|
72
128
|
toolSpans,
|
|
@@ -76,4 +132,4 @@ export {
|
|
|
76
132
|
aggregateLlm,
|
|
77
133
|
runFailureClass
|
|
78
134
|
};
|
|
79
|
-
//# sourceMappingURL=chunk-
|
|
135
|
+
//# sourceMappingURL=chunk-MHNQWM4I.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/trace/schema.ts","../src/trace/query.ts"],"sourcesContent":["/**\n * TraceSchema v1 — the canonical data model for agent-eval.\n *\n * Every score, every failure class, every pipeline in the framework is\n * a view over this data. Shape it once, live with it.\n *\n * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)\n * but extended with agent-specific span kinds (llm, tool, retrieval,\n * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict\n * entities that OTEL leaves as free-form attributes.\n */\n\nexport const TRACE_SCHEMA_VERSION = '1.0.0'\n\n// ── Run ──────────────────────────────────────────────────────────────\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'aborted'\n\nexport interface BudgetSpec {\n tokens?: number\n wallMs?: number\n calls?: number\n usd?: number\n}\n\nexport interface RunOutcome {\n score?: number\n pass?: boolean\n failureClass?: FailureClass\n notes?: string\n}\n\n/**\n * Layer — optional classification in a nested build workflow.\n * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).\n * `app-build`: sandbox harness that compiled + tested the generated scaffold.\n * `app-runtime`: a run of the generated agent against a domain scenario.\n * `meta`: any meta-eval (judge replay, correlation analysis).\n */\nexport type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom'\n\nexport interface Run {\n runId: string\n /**\n * Stable identifier of the scenario being executed.\n *\n * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts\n * input WITHOUT this field, substituting a sensible default\n * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no\n * curated scenario to anchor to (runtime / operator / meta-eval runs). This\n * keeps the persisted shape unambiguous for downstream filters + aggregations\n * while removing the boilerplate of inventing placeholder ids at the call site.\n */\n scenarioId: string\n variantId?: string\n datasetVersion?: string\n /** Git SHA of agent code at run time. */\n codeSha?: string\n /** Hash of the prompt template + any system prompt. */\n promptSha?: string\n /** Model id + date + system-prompt hash, concatenated. */\n modelFingerprint?: string\n seed?: number\n /** Arbitrary environment markers (shell, docker version, tz). */\n envFingerprint?: Record<string, string>\n /** Version of the redaction rules applied to this run. */\n redactionVersion?: string\n /** Parent run in a nested build workflow. A builder run's children are\n * app-build runs; those children are app-runtime runs. */\n parentRunId?: string\n /** Stable project identifier — groups runs across chats + sessions. */\n projectId?: string\n /** Chat/conversation identifier within a project. */\n chatId?: string\n /** Layer classification — hint for aggregation; not enforced. */\n layer?: RunLayer\n startedAt: number\n endedAt?: number\n status: RunStatus\n outcome?: RunOutcome\n budget?: BudgetSpec\n /** Free-form labels for downstream grouping. */\n tags?: Record<string, string>\n}\n\n// ── Spans (hierarchical work units) ──────────────────────────────────\n\nexport type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom'\n\nexport type SpanStatus = 'ok' | 'error'\n\nexport interface SpanBase {\n spanId: string\n parentSpanId?: string\n runId: string\n kind: SpanKind\n name: string\n startedAt: number\n endedAt?: number\n status?: SpanStatus\n error?: string\n /** Anything not covered by typed fields. Kept deliberately free-form. */\n attributes?: Record<string, unknown>\n}\n\nexport interface Message {\n role: 'system' | 'user' | 'assistant' | 'tool'\n content: string\n tokens?: number\n /** Multi-modal content descriptors; blobs themselves live in Artifacts. */\n images?: Array<{ artifactId?: string; url?: string; mime?: string }>\n}\n\nexport interface LlmSpan extends SpanBase {\n kind: 'llm'\n model: string\n messages: Message[]\n output?: string\n inputTokens?: number\n outputTokens?: number\n cachedTokens?: number\n reasoningTokens?: number\n costUsd?: number\n finishReason?: string\n}\n\nexport interface ToolSpan extends SpanBase {\n kind: 'tool'\n toolName: string\n args: unknown\n result?: unknown\n latencyMs?: number\n}\n\nexport interface RetrievalSpan extends SpanBase {\n kind: 'retrieval'\n query: string\n hits: Array<{ docId: string; score: number; content?: string }>\n}\n\nexport interface JudgeSpan extends SpanBase {\n kind: 'judge'\n judgeId: string\n /** Span this judgment applies to. */\n targetSpanId: string\n dimension: string\n /** Numeric score (free-range; interpretation up to the judge). */\n score: number\n rationale?: string\n evidence?: string\n}\n\nexport interface SandboxSpan extends SpanBase {\n kind: 'sandbox'\n image?: string\n command?: string\n exitCode?: number\n testsTotal?: number\n testsPassed?: number\n stdoutHash?: string\n stderrHash?: string\n /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */\n wallMs?: number\n}\n\nexport interface GenericSpan extends SpanBase {\n kind: 'agent' | 'custom'\n}\n\nexport type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan\n\n// ── Events (point-in-time occurrences within a span) ─────────────────\n\nexport type EventKind =\n | 'log'\n | 'error'\n | 'budget_decrement'\n | 'budget_breach'\n | 'state_mutation'\n | 'policy_violation'\n | 'redaction_applied'\n | 'custom'\n\nexport interface TraceEvent {\n eventId: string\n runId: string\n spanId?: string\n kind: EventKind\n timestamp: number\n payload: Record<string, unknown>\n}\n\n// ── Budget ledger (running token/wall/call/$ accounting) ─────────────\n\nexport interface BudgetLedgerEntry {\n runId: string\n dimension: keyof BudgetSpec\n limit: number\n consumed: number\n remaining: number\n timestamp: number\n breached: boolean\n /** Span that triggered this entry, if any. */\n spanId?: string\n}\n\n// ── Artifacts (blobs addressed by hash) ──────────────────────────────\n\nexport interface Artifact {\n artifactId: string\n runId: string\n spanId?: string\n contentType: string\n sizeBytes: number\n /** sha256 in hex. */\n hash: string\n /** External storage URL (R2, S3, filesystem path). */\n storageUrl?: string\n /** Inline content for small blobs — keep under ~64KB. */\n inlineContent?: string\n}\n\n// ── Failure taxonomy ─────────────────────────────────────────────────\n\nexport type FailureClass =\n | 'success'\n | 'reasoning_error'\n | 'tool_selection_error'\n | 'tool_argument_error'\n | 'tool_recovery_failure'\n | 'hallucination'\n | 'instruction_following'\n | 'safety_refusal_miss'\n | 'policy_violation'\n | 'budget_exceeded'\n | 'format_drift'\n | 'permission_escalation'\n | 'pii_leak'\n | 'cost_overrun'\n | 'timeout'\n | 'sandbox_failure'\n | 'missing_user_data'\n | 'missing_domain_data'\n | 'missing_codebase_context'\n | 'missing_runtime_context'\n | 'missing_credentials'\n | 'missing_integration_connection'\n | 'missing_integration_scope'\n | 'integration_approval_required'\n | 'integration_auth_expired'\n | 'integration_provider_failure'\n | 'bad_integration_manifest'\n | 'unsafe_integration_write_denied'\n | 'stale_external_data'\n | 'bad_retrieval'\n | 'insufficient_evidence'\n | 'contradictory_evidence'\n | 'ambiguous_user_intent'\n | 'knowledge_readiness_blocked'\n | 'unknown'\n\nexport const FAILURE_CLASSES: readonly FailureClass[] = [\n 'success',\n 'reasoning_error',\n 'tool_selection_error',\n 'tool_argument_error',\n 'tool_recovery_failure',\n 'hallucination',\n 'instruction_following',\n 'safety_refusal_miss',\n 'policy_violation',\n 'budget_exceeded',\n 'format_drift',\n 'permission_escalation',\n 'pii_leak',\n 'cost_overrun',\n 'timeout',\n 'sandbox_failure',\n 'missing_user_data',\n 'missing_domain_data',\n 'missing_codebase_context',\n 'missing_runtime_context',\n 'missing_credentials',\n 'missing_integration_connection',\n 'missing_integration_scope',\n 'integration_approval_required',\n 'integration_auth_expired',\n 'integration_provider_failure',\n 'bad_integration_manifest',\n 'unsafe_integration_write_denied',\n 'stale_external_data',\n 'bad_retrieval',\n 'insufficient_evidence',\n 'contradictory_evidence',\n 'ambiguous_user_intent',\n 'knowledge_readiness_blocked',\n 'unknown',\n] as const\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nexport function isLlmSpan(s: Span): s is LlmSpan {\n return s.kind === 'llm'\n}\nexport function isToolSpan(s: Span): s is ToolSpan {\n return s.kind === 'tool'\n}\nexport function isRetrievalSpan(s: Span): s is RetrievalSpan {\n return s.kind === 'retrieval'\n}\nexport function isJudgeSpan(s: Span): s is JudgeSpan {\n return s.kind === 'judge'\n}\nexport function isSandboxSpan(s: Span): s is SandboxSpan {\n return s.kind === 'sandbox'\n}\n","/**\n * Typed query helpers over TraceStore.\n *\n * Not a full SQL engine — a minimal, composable set of operators that\n * cover the canned-pipeline use cases. For ad-hoc analytics, persist to\n * NDJSON and point DuckDB at it; the schema is stable so external SQL\n * tooling works out of the box.\n */\n\nimport type { FailureClass, JudgeSpan, LlmSpan, Run, ToolSpan } from './schema'\nimport { isJudgeSpan, isLlmSpan, isToolSpan } from './schema'\nimport type { TraceStore } from './store'\n\nexport async function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]> {\n return store.listRuns({ scenarioId })\n}\n\nexport async function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]> {\n const spans = await store.spans({ runId, kind: 'llm' })\n return spans.filter(isLlmSpan)\n}\n\nexport async function toolSpans(\n store: TraceStore,\n runId?: string,\n toolName?: string,\n): Promise<ToolSpan[]> {\n const spans = await store.spans({ runId, kind: 'tool', toolName })\n return spans.filter(isToolSpan)\n}\n\n/** Query judge-kind spans from the trace store, optionally scoped to a single run. */\nexport async function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]> {\n const spans = await store.spans({ runId, kind: 'judge' })\n return spans.filter(isJudgeSpan)\n}\n\n/** Group spans by any key selector. */\nexport function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]> {\n const map = new Map<K, T[]>()\n for (const item of items) {\n const k = key(item)\n let bucket = map.get(k)\n if (!bucket) {\n bucket = []\n map.set(k, bucket)\n }\n bucket.push(item)\n }\n return map\n}\n\n/** Hash tool arguments to an orderless-key-stable string for de-duplication. */\nexport function argHash(args: unknown): string {\n return stableStringify(args)\n}\n\nfunction stableStringify(value: unknown): string {\n // Must ALWAYS return a string: JSON.stringify(undefined) — and stringify of\n // functions/symbols — yields the JS value `undefined`, which would make\n // argHash return a non-string and silently break the de-dup keys behind\n // stuck-loop and failure-cluster.\n if (value === undefined) return 'undefined'\n if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'undefined'\n if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`\n // Drop undefined-valued keys to match JSON.stringify semantics, so\n // `{a:1}` and `{a:1,b:undefined}` hash identically.\n const obj = value as Record<string, unknown>\n const keys = Object.keys(obj)\n .filter((k) => obj[k] !== undefined)\n .sort()\n const parts = keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`)\n return `{${parts.join(',')}}`\n}\n\n/** Sum an LLM-span array into aggregate token + cost. */\nexport function aggregateLlm(spans: LlmSpan[]): {\n inputTokens: number\n outputTokens: number\n cachedTokens: number\n reasoningTokens: number\n costUsd: number\n} {\n return spans.reduce(\n (acc, s) => ({\n inputTokens: acc.inputTokens + (s.inputTokens ?? 0),\n outputTokens: acc.outputTokens + (s.outputTokens ?? 0),\n cachedTokens: acc.cachedTokens + (s.cachedTokens ?? 0),\n // reasoningTokens is on LlmSpan but was omitted here — reasoning usage was\n // invisible to any cost/perf analysis reading this aggregate.\n reasoningTokens: acc.reasoningTokens + (s.reasoningTokens ?? 0),\n costUsd: acc.costUsd + (s.costUsd ?? 0),\n }),\n { inputTokens: 0, outputTokens: 0, cachedTokens: 0, reasoningTokens: 0, costUsd: 0 },\n )\n}\n\n/** Pick the outcome's failure class when present, else derive 'success' from run status. */\nexport function runFailureClass(run: Run): FailureClass {\n if (run.outcome?.failureClass) return run.outcome.failureClass\n if (run.status === 'completed' && run.outcome?.pass !== false) return 'success'\n if (run.status === 'aborted') return 'budget_exceeded'\n return 'unknown'\n}\n"],"mappings":";AAYO,IAAM,uBAAuB;AAyP7B,IAAM,kBAA2C;AAAA,EACtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAIO,SAAS,UAAU,GAAuB;AAC/C,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,WAAW,GAAwB;AACjD,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,gBAAgB,GAA6B;AAC3D,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,YAAY,GAAyB;AACnD,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,cAAc,GAA2B;AACvD,SAAO,EAAE,SAAS;AACpB;;;AC9SA,eAAsB,gBAAgB,OAAmB,YAAoC;AAC3F,SAAO,MAAM,SAAS,EAAE,WAAW,CAAC;AACtC;AAEA,eAAsB,SAAS,OAAmB,OAAoC;AACpF,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,MAAM,CAAC;AACtD,SAAO,MAAM,OAAO,SAAS;AAC/B;AAEA,eAAsB,UACpB,OACA,OACA,UACqB;AACrB,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,QAAQ,SAAS,CAAC;AACjE,SAAO,MAAM,OAAO,UAAU;AAChC;AAGA,eAAsB,WAAW,OAAmB,OAAsC;AACxF,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,OAAO,MAAM,QAAQ,CAAC;AACxD,SAAO,MAAM,OAAO,WAAW;AACjC;AAGO,SAAS,QAAsC,OAAY,KAA+B;AAC/F,QAAM,MAAM,oBAAI,IAAY;AAC5B,aAAW,QAAQ,OAAO;AACxB,UAAM,IAAI,IAAI,IAAI;AAClB,QAAI,SAAS,IAAI,IAAI,CAAC;AACtB,QAAI,CAAC,QAAQ;AACX,eAAS,CAAC;AACV,UAAI,IAAI,GAAG,MAAM;AAAA,IACnB;AACA,WAAO,KAAK,IAAI;AAAA,EAClB;AACA,SAAO;AACT;AAGO,SAAS,QAAQ,MAAuB;AAC7C,SAAO,gBAAgB,IAAI;AAC7B;AAEA,SAAS,gBAAgB,OAAwB;AAK/C,MAAI,UAAU,OAAW,QAAO;AAChC,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK,KAAK;AACjF,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,IAAI,MAAM,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AAGzE,QAAM,MAAM;AACZ,QAAM,OAAO,OAAO,KAAK,GAAG,EACzB,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,MAAS,EAClC,KAAK;AACR,QAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAgB,IAAI,CAAC,CAAC,CAAC,EAAE;AAC/E,SAAO,IAAI,MAAM,KAAK,GAAG,CAAC;AAC5B;AAGO,SAAS,aAAa,OAM3B;AACA,SAAO,MAAM;AAAA,IACX,CAAC,KAAK,OAAO;AAAA,MACX,aAAa,IAAI,eAAe,EAAE,eAAe;AAAA,MACjD,cAAc,IAAI,gBAAgB,EAAE,gBAAgB;AAAA,MACpD,cAAc,IAAI,gBAAgB,EAAE,gBAAgB;AAAA;AAAA;AAAA,MAGpD,iBAAiB,IAAI,mBAAmB,EAAE,mBAAmB;AAAA,MAC7D,SAAS,IAAI,WAAW,EAAE,WAAW;AAAA,IACvC;AAAA,IACA,EAAE,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,iBAAiB,GAAG,SAAS,EAAE;AAAA,EACrF;AACF;AAGO,SAAS,gBAAgB,KAAwB;AACtD,MAAI,IAAI,SAAS,aAAc,QAAO,IAAI,QAAQ;AAClD,MAAI,IAAI,WAAW,eAAe,IAAI,SAAS,SAAS,MAAO,QAAO;AACtE,MAAI,IAAI,WAAW,UAAW,QAAO;AACrC,SAAO;AACT;","names":[]}
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
AnalystRegistry,
|
|
3
3
|
DEFAULT_TRACE_ANALYST_KINDS,
|
|
4
|
-
createTraceAnalystKind
|
|
5
|
-
} from "./chunk-2OGPXHOB.js";
|
|
6
|
-
import {
|
|
4
|
+
createTraceAnalystKind,
|
|
7
5
|
makeFinding
|
|
8
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-7NX6ZSBG.js";
|
|
9
7
|
import {
|
|
10
8
|
LLM_INPUT_TOKEN_ATTR_KEYS,
|
|
11
9
|
LLM_OUTPUT_TOKEN_ATTR_KEYS,
|
|
@@ -204,4 +202,4 @@ export {
|
|
|
204
202
|
behavioralAnalyst,
|
|
205
203
|
buildDefaultAnalystRegistry
|
|
206
204
|
};
|
|
207
|
-
//# sourceMappingURL=chunk-
|
|
205
|
+
//# sourceMappingURL=chunk-OW47B5WA.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/default-registry.ts"],"sourcesContent":["/**\n * Deterministic behavioral metrics over OTLP spans — pure arithmetic, no LLM.\n *\n * These are the model-independent multiplier: the four trace-quality signals a\n * tolerant analyzer (e.g. HALO) re-derives per run inside the model — token\n * growth, output decay, tool monoculture, missing self-verification — computed\n * here once, in TypeScript, with zero model judgment. A finding that falls out\n * of arithmetic is trivially model-agnostic and cannot hallucinate the trend.\n *\n * General, not trace-specific: the detectors key off token trajectories and\n * tool usage present in any agentic OTLP trace, not any one benchmark.\n */\n\nimport {\n LLM_INPUT_TOKEN_ATTR_KEYS,\n LLM_OUTPUT_TOKEN_ATTR_KEYS,\n TOOL_NAME_ATTR_KEYS,\n} from '../trace/otlp-attributes'\nimport type { TraceAnalystSpan } from './types'\n\nexport type SuboptimalCode =\n | 'monotonic-input-growth'\n | 'output-length-decay'\n | 'single-tool-dependency'\n | 'no-self-verification'\n\nexport interface SuboptimalSignal {\n code: SuboptimalCode\n severity: 'high' | 'medium' | 'low'\n /** Human-readable claim, with the backing numbers inlined. */\n detail: string\n /** The exact figures the detector fired on — auditable, no model in the loop. */\n evidence: Record<string, number | string | boolean>\n}\n\nexport interface BehavioralMetrics {\n llmCallCount: number\n inputTokenTrajectory: number[]\n outputTokenTrajectory: number[]\n toolHistogram: Record<string, number>\n totalToolCalls: number\n distinctTools: number\n /** distinct/total tool calls; 1.0 when there are no tool calls. */\n toolDiversityRatio: number\n hasSelfVerification: boolean\n signals: SuboptimalSignal[]\n}\n\n/** ≥ this input-token growth ratio across a run, with no compression, fires. */\nconst INPUT_GROWTH_FACTOR = 3\n/** Tool-usage signals need at least this many calls to be meaningful. */\nconst MIN_TOOL_CALLS = 3\n/** Tool names that read or check state count as self-verification, not mutation.\n * Covers the inspect verbs plus the read/search tools real harnesses use to\n * verify (Claude Code Read/Grep/Glob, codex read_file/ls/cat, git status/diff,\n * test/lint). A pure shell tool (Bash/exec_command) is intentionally NOT matched\n * — its name can't tell a `pytest` from an `rm`. */\nconst VERIFY_RE =\n /verif|eval|inspect|check|assert|validat|review|confirm|read|grep|glob|search|view|\\blist\\b|\\bls\\b|\\bcat\\b|\\bfind\\b|diff|status|\\btest|lint|typecheck/i\n\nfunction num(v: unknown): number | null {\n return typeof v === 'number' && Number.isFinite(v) ? v : null\n}\nfunction numAttr(attrs: Record<string, unknown>, keys: readonly string[]): number | null {\n for (const key of keys) {\n const value = num(attrs[key])\n if (value !== null) return value\n }\n return null\n}\nfunction inputTokensOf(s: TraceAnalystSpan): number | null {\n return (\n numAttr(s.attributes, LLM_INPUT_TOKEN_ATTR_KEYS) ?? num(s.attributes['llm.usage.input_tokens'])\n )\n}\nfunction outputTokensOf(s: TraceAnalystSpan): number | null {\n return (\n numAttr(s.attributes, LLM_OUTPUT_TOKEN_ATTR_KEYS) ??\n num(s.attributes['llm.usage.output_tokens'])\n )\n}\nfunction stepOf(s: TraceAnalystSpan): number | null {\n return num(s.attributes.step)\n}\nfunction toolNameOf(s: TraceAnalystSpan): string | null {\n if (s.tool_name) return s.tool_name\n for (const key of TOOL_NAME_ATTR_KEYS) {\n const t = s.attributes[key]\n if (typeof t === 'string' && t.length > 0) return t\n }\n return null\n}\n\n/**\n * Reduce a span list to behavioral metrics + fired suboptimality signals.\n * Pure + deterministic: same spans → same output, on any machine, no model.\n */\nexport function computeTraceMetrics(spans: readonly TraceAnalystSpan[]): BehavioralMetrics {\n // Order by step (when present) then start_time so trajectories reflect run order.\n const ordered = [...spans].sort((a, b) => {\n const sa = stepOf(a)\n const sb = stepOf(b)\n if (sa !== null && sb !== null && sa !== sb) return sa - sb\n return a.start_time.localeCompare(b.start_time)\n })\n\n const inputTokenTrajectory: number[] = []\n const outputTokenTrajectory: number[] = []\n const toolHistogram: Record<string, number> = {}\n let hasSelfVerification = false\n\n for (const s of ordered) {\n const inT = inputTokensOf(s)\n if (inT !== null) inputTokenTrajectory.push(inT)\n const outT = outputTokensOf(s)\n if (outT !== null) outputTokenTrajectory.push(outT)\n const tool = toolNameOf(s)\n if (tool) {\n toolHistogram[tool] = (toolHistogram[tool] ?? 0) + 1\n if (VERIFY_RE.test(tool)) hasSelfVerification = true\n }\n }\n\n const totalToolCalls = Object.values(toolHistogram).reduce((a, b) => a + b, 0)\n const distinctTools = Object.keys(toolHistogram).length\n const toolDiversityRatio = totalToolCalls === 0 ? 1 : distinctTools / totalToolCalls\n\n const signals: SuboptimalSignal[] = []\n\n if (inputTokenTrajectory.length >= 3) {\n const first = inputTokenTrajectory[0]!\n const last = inputTokenTrajectory[inputTokenTrajectory.length - 1]!\n // first === 0 with later growth is an unbounded ratio (0→huge context blowup);\n // treat it as infinite so the signal fires, and report it as such instead of\n // dividing by zero for the displayed factor.\n const growthFromZero = first === 0 && last > 0\n const growth = growthFromZero ? Infinity : first > 0 ? last / first : 0\n if (last > first && growth >= INPUT_GROWTH_FACTOR) {\n const growthLabel = growthFromZero ? '0→nonzero (unbounded)' : `${growth.toFixed(1)}x`\n signals.push({\n code: 'monotonic-input-growth',\n severity: 'high',\n detail: `LLM input tokens grew ${growthLabel} (${first}→${last}) across ${inputTokenTrajectory.length} calls — full history re-sent each step with no compression.`,\n evidence: {\n first,\n last,\n growth_x: growthFromZero ? 'unbounded' : Number(growth.toFixed(2)),\n calls: inputTokenTrajectory.length,\n },\n })\n }\n }\n\n if (outputTokenTrajectory.length >= 3) {\n const first = outputTokenTrajectory[0]!\n const last = outputTokenTrajectory[outputTokenTrajectory.length - 1]!\n if (last < first) {\n signals.push({\n code: 'output-length-decay',\n severity: 'medium',\n detail: `LLM output tokens shrank ${first}→${last} over ${outputTokenTrajectory.length} calls — less planning/reasoning per step as context grows.`,\n evidence: { first, last, calls: outputTokenTrajectory.length },\n })\n }\n }\n\n if (totalToolCalls >= MIN_TOOL_CALLS && distinctTools === 1) {\n const only = Object.keys(toolHistogram)[0]!\n signals.push({\n code: 'single-tool-dependency',\n severity: 'medium',\n detail: `All ${totalToolCalls} tool calls are \\`${only}\\` — no tool diversity and no fallback path.`,\n evidence: { tool: only, calls: totalToolCalls, distinct_tools: 1 },\n })\n }\n\n if (totalToolCalls >= MIN_TOOL_CALLS && !hasSelfVerification) {\n signals.push({\n code: 'no-self-verification',\n severity: 'medium',\n detail: `${totalToolCalls} tool calls and none verify/inspect/check state — the agent never validates its own actions.`,\n evidence: { tool_calls: totalToolCalls, verification_calls: 0 },\n })\n }\n\n return {\n llmCallCount: inputTokenTrajectory.length,\n inputTokenTrajectory,\n outputTokenTrajectory,\n toolHistogram,\n totalToolCalls,\n distinctTools,\n toolDiversityRatio,\n hasSelfVerification,\n signals,\n }\n}\n","/**\n * `behavioralAnalyst` — a DETERMINISTIC analyst (cost.kind = 'deterministic',\n * never calls the LLM). It produces the efficiency/behavioral findings a\n * tolerant agentic analyzer (HALO) re-derives per run inside the model —\n * context bloat, output decay, tool monoculture, missing self-verification —\n * directly from arithmetic over spans (`computeTraceMetrics`).\n *\n * Why it matters: these findings are model-agnostic BY CONSTRUCTION (no model\n * in the loop), so they cannot return 0 on a weak model the way the Ax-RLM\n * does — and they are strictly more reliable than HALO, which spends tokens\n * re-deriving the same numbers and can hallucinate the trend. The agentic\n * RLM kinds remain for SEMANTIC findings that genuinely need a model; this\n * analyst owns the behavioral class.\n */\n\nimport {\n type BehavioralMetrics,\n computeTraceMetrics,\n type SuboptimalCode,\n} from '../trace-analyst/behavioral-metrics'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { TraceAnalystSpan } from '../trace-analyst/types'\nimport { type Analyst, type AnalystFinding, makeFinding } from './types'\n\nconst RECOMMENDED_ACTION: Record<SuboptimalCode, string> = {\n 'monotonic-input-growth':\n 'Add a context-budget instruction: once prior context exceeds a threshold, summarize earlier steps into a short status line instead of re-sending full history.',\n 'output-length-decay':\n 'Require a minimum planning/reasoning budget per step so late steps do not degrade into terse, error-prone commands.',\n 'single-tool-dependency':\n 'Direct the agent to use the full toolset (verify / inspect / alternate actions), not a single execute call, and to plan a fallback when a call returns an unexpected result.',\n 'no-self-verification':\n 'After every state-mutating action, verify the result (eval / inspect / assert) before proceeding.',\n}\n\nconst ANALYST_ID = 'efficiency-behavioral'\n\n/**\n * Map computed signals → structured AnalystFindings. Pure: no LLM, no clock\n * dependence beyond `produced_at` (overridable for deterministic tests).\n */\nexport function deriveEfficiencyFindings(\n metrics: BehavioralMetrics,\n opts: { analystId?: string; producedAt?: string } = {},\n): AnalystFinding[] {\n const analystId = opts.analystId ?? ANALYST_ID\n return metrics.signals.map((sig) =>\n makeFinding({\n analyst_id: analystId,\n area: 'efficiency',\n subject: sig.code, // kebab — passes the cluster grammar; stable key for diffFindings\n claim: sig.detail,\n severity: sig.severity,\n // Deterministic arithmetic over spans, not a model judgment → certain.\n confidence: 1,\n evidence_refs: [\n {\n kind: 'metric',\n uri: `metric://efficiency/${sig.code}`,\n excerpt: JSON.stringify(sig.evidence),\n },\n ],\n recommended_action: RECOMMENDED_ACTION[sig.code],\n metadata: { deterministic: true, evidence: sig.evidence },\n ...(opts.producedAt ? { produced_at: opts.producedAt } : {}),\n }),\n )\n}\n\n/** The deterministic behavioral/efficiency analyst (no LLM, any-model). */\nexport function behavioralAnalyst(): Analyst<TraceAnalysisStore> {\n return {\n id: ANALYST_ID,\n description:\n 'Deterministic behavioral/efficiency findings over OTLP spans — token-growth, output-decay, tool-monoculture, missing self-verification. Zero LLM; model-agnostic by construction.',\n inputKind: 'trace-store',\n cost: { kind: 'deterministic' },\n version: '1.0.0',\n async analyze(store) {\n const overview = await store.getOverview()\n const spans: TraceAnalystSpan[] = []\n for (const traceId of overview.sample_trace_ids) {\n const viewed = await store.viewTrace({ trace_id: traceId })\n if (viewed.spans) spans.push(...viewed.spans)\n }\n return deriveEfficiencyFindings(computeTraceMetrics(spans))\n },\n }\n}\n","/**\n * `buildDefaultAnalystRegistry` — the canonical analyst suite, so consumers\n * stop hand-wiring `new AnalystRegistry()` + per-kind `createTraceAnalystKind`.\n *\n * The deterministic `behavioralAnalyst` is ALWAYS registered (it needs no\n * model and is model-agnostic by construction). The agentic RLM kinds are\n * registered only when an `ai` service is supplied — so a caller with no LLM\n * still gets the full behavioral/efficiency diagnosis, and the substrate's\n * \"any model (including no model)\" guarantee holds at the suite level.\n */\n\nimport type { AxAIService } from '@ax-llm/ax'\nimport { behavioralAnalyst } from './behavioral-analyst'\nimport { createTraceAnalystKind, type TraceAnalystKindSpec } from './kind-factory'\nimport { DEFAULT_TRACE_ANALYST_KINDS } from './kinds'\nimport { AnalystRegistry, type AnalystRegistryOptions } from './registry'\n\nexport interface DefaultAnalystRegistryOptions {\n /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */\n ai?: AxAIService\n /** Model for the agentic kinds (falls back to the ai service default). */\n model?: string\n /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */\n kinds?: readonly TraceAnalystKindSpec[]\n /** Set false to omit the deterministic behavioral analyst (default: include). */\n includeBehavioral?: boolean\n /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */\n registry?: AnalystRegistryOptions\n}\n\nexport function buildDefaultAnalystRegistry(\n opts: DefaultAnalystRegistryOptions = {},\n): AnalystRegistry {\n const registry = new AnalystRegistry(opts.registry)\n if (opts.includeBehavioral !== false) {\n registry.register(behavioralAnalyst())\n }\n if (opts.ai) {\n const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS\n for (const spec of kinds) {\n registry.register(createTraceAnalystKind(spec, { ai: opts.ai, model: opts.model }))\n }\n }\n return registry\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiDA,IAAM,sBAAsB;AAE5B,IAAM,iBAAiB;AAMvB,IAAM,YACJ;AAEF,SAAS,IAAI,GAA2B;AACtC,SAAO,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,IAAI,IAAI;AAC3D;AACA,SAAS,QAAQ,OAAgC,MAAwC;AACvF,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,IAAI,MAAM,GAAG,CAAC;AAC5B,QAAI,UAAU,KAAM,QAAO;AAAA,EAC7B;AACA,SAAO;AACT;AACA,SAAS,cAAc,GAAoC;AACzD,SACE,QAAQ,EAAE,YAAY,yBAAyB,KAAK,IAAI,EAAE,WAAW,wBAAwB,CAAC;AAElG;AACA,SAAS,eAAe,GAAoC;AAC1D,SACE,QAAQ,EAAE,YAAY,0BAA0B,KAChD,IAAI,EAAE,WAAW,yBAAyB,CAAC;AAE/C;AACA,SAAS,OAAO,GAAoC;AAClD,SAAO,IAAI,EAAE,WAAW,IAAI;AAC9B;AACA,SAAS,WAAW,GAAoC;AACtD,MAAI,EAAE,UAAW,QAAO,EAAE;AAC1B,aAAW,OAAO,qBAAqB;AACrC,UAAM,IAAI,EAAE,WAAW,GAAG;AAC1B,QAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,EACpD;AACA,SAAO;AACT;AAMO,SAAS,oBAAoB,OAAuD;AAEzF,QAAM,UAAU,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM;AACxC,UAAM,KAAK,OAAO,CAAC;AACnB,UAAM,KAAK,OAAO,CAAC;AACnB,QAAI,OAAO,QAAQ,OAAO,QAAQ,OAAO,GAAI,QAAO,KAAK;AACzD,WAAO,EAAE,WAAW,cAAc,EAAE,UAAU;AAAA,EAChD,CAAC;AAED,QAAM,uBAAiC,CAAC;AACxC,QAAM,wBAAkC,CAAC;AACzC,QAAM,gBAAwC,CAAC;AAC/C,MAAI,sBAAsB;AAE1B,aAAW,KAAK,SAAS;AACvB,UAAM,MAAM,cAAc,CAAC;AAC3B,QAAI,QAAQ,KAAM,sBAAqB,KAAK,GAAG;AAC/C,UAAM,OAAO,eAAe,CAAC;AAC7B,QAAI,SAAS,KAAM,uBAAsB,KAAK,IAAI;AAClD,UAAM,OAAO,WAAW,CAAC;AACzB,QAAI,MAAM;AACR,oBAAc,IAAI,KAAK,cAAc,IAAI,KAAK,KAAK;AACnD,UAAI,UAAU,KAAK,IAAI,EAAG,uBAAsB;AAAA,IAClD;AAAA,EACF;AAEA,QAAM,iBAAiB,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAC7E,QAAM,gBAAgB,OAAO,KAAK,aAAa,EAAE;AACjD,QAAM,qBAAqB,mBAAmB,IAAI,IAAI,gBAAgB;AAEtE,QAAM,UAA8B,CAAC;AAErC,MAAI,qBAAqB,UAAU,GAAG;AACpC,UAAM,QAAQ,qBAAqB,CAAC;AACpC,UAAM,OAAO,qBAAqB,qBAAqB,SAAS,CAAC;AAIjE,UAAM,iBAAiB,UAAU,KAAK,OAAO;AAC7C,UAAM,SAAS,iBAAiB,WAAW,QAAQ,IAAI,OAAO,QAAQ;AACtE,QAAI,OAAO,SAAS,UAAU,qBAAqB;AACjD,YAAM,cAAc,iBAAiB,+BAA0B,GAAG,OAAO,QAAQ,CAAC,CAAC;AACnF,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ,yBAAyB,WAAW,KAAK,KAAK,SAAI,IAAI,YAAY,qBAAqB,MAAM;AAAA,QACrG,UAAU;AAAA,UACR;AAAA,UACA;AAAA,UACA,UAAU,iBAAiB,cAAc,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,UACjE,OAAO,qBAAqB;AAAA,QAC9B;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,sBAAsB,UAAU,GAAG;AACrC,UAAM,QAAQ,sBAAsB,CAAC;AACrC,UAAM,OAAO,sBAAsB,sBAAsB,SAAS,CAAC;AACnE,QAAI,OAAO,OAAO;AAChB,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ,4BAA4B,KAAK,SAAI,IAAI,SAAS,sBAAsB,MAAM;AAAA,QACtF,UAAU,EAAE,OAAO,MAAM,OAAO,sBAAsB,OAAO;AAAA,MAC/D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,kBAAkB,kBAAkB,kBAAkB,GAAG;AAC3D,UAAM,OAAO,OAAO,KAAK,aAAa,EAAE,CAAC;AACzC,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,OAAO,cAAc,qBAAqB,IAAI;AAAA,MACtD,UAAU,EAAE,MAAM,MAAM,OAAO,gBAAgB,gBAAgB,EAAE;AAAA,IACnE,CAAC;AAAA,EACH;AAEA,MAAI,kBAAkB,kBAAkB,CAAC,qBAAqB;AAC5D,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,GAAG,cAAc;AAAA,MACzB,UAAU,EAAE,YAAY,gBAAgB,oBAAoB,EAAE;AAAA,IAChE,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,cAAc,qBAAqB;AAAA,IACnC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;AC5KA,IAAM,qBAAqD;AAAA,EACzD,0BACE;AAAA,EACF,uBACE;AAAA,EACF,0BACE;AAAA,EACF,wBACE;AACJ;AAEA,IAAM,aAAa;AAMZ,SAAS,yBACd,SACA,OAAoD,CAAC,GACnC;AAClB,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO,QAAQ,QAAQ;AAAA,IAAI,CAAC,QAC1B,YAAY;AAAA,MACV,YAAY;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,IAAI;AAAA;AAAA,MACb,OAAO,IAAI;AAAA,MACX,UAAU,IAAI;AAAA;AAAA,MAEd,YAAY;AAAA,MACZ,eAAe;AAAA,QACb;AAAA,UACE,MAAM;AAAA,UACN,KAAK,uBAAuB,IAAI,IAAI;AAAA,UACpC,SAAS,KAAK,UAAU,IAAI,QAAQ;AAAA,QACtC;AAAA,MACF;AAAA,MACA,oBAAoB,mBAAmB,IAAI,IAAI;AAAA,MAC/C,UAAU,EAAE,eAAe,MAAM,UAAU,IAAI,SAAS;AAAA,MACxD,GAAI,KAAK,aAAa,EAAE,aAAa,KAAK,WAAW,IAAI,CAAC;AAAA,IAC5D,CAAC;AAAA,EACH;AACF;AAGO,SAAS,oBAAiD;AAC/D,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS;AAAA,IACT,MAAM,QAAQ,OAAO;AACnB,YAAM,WAAW,MAAM,MAAM,YAAY;AACzC,YAAM,QAA4B,CAAC;AACnC,iBAAW,WAAW,SAAS,kBAAkB;AAC/C,cAAM,SAAS,MAAM,MAAM,UAAU,EAAE,UAAU,QAAQ,CAAC;AAC1D,YAAI,OAAO,MAAO,OAAM,KAAK,GAAG,OAAO,KAAK;AAAA,MAC9C;AACA,aAAO,yBAAyB,oBAAoB,KAAK,CAAC;AAAA,IAC5D;AAAA,EACF;AACF;;;AC1DO,SAAS,4BACd,OAAsC,CAAC,GACtB;AACjB,QAAM,WAAW,IAAI,gBAAgB,KAAK,QAAQ;AAClD,MAAI,KAAK,sBAAsB,OAAO;AACpC,aAAS,SAAS,kBAAkB,CAAC;AAAA,EACvC;AACA,MAAI,KAAK,IAAI;AACX,UAAM,QAAQ,KAAK,SAAS;AAC5B,eAAW,QAAQ,OAAO;AACxB,eAAS,SAAS,uBAAuB,MAAM,EAAE,IAAI,KAAK,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC;AAAA,IACpF;AAAA,EACF;AACA,SAAO;AACT;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/default-registry.ts"],"sourcesContent":["/**\n * Deterministic behavioral metrics over OTLP spans — pure arithmetic, no LLM.\n *\n * These are the model-independent multiplier: the four trace-quality signals a\n * tolerant analyzer (e.g. HALO) re-derives per run inside the model — token\n * growth, output decay, tool monoculture, missing self-verification — computed\n * here once, in TypeScript, with zero model judgment. A finding that falls out\n * of arithmetic is trivially model-agnostic and cannot hallucinate the trend.\n *\n * General, not trace-specific: the detectors key off token trajectories and\n * tool usage present in any agentic OTLP trace, not any one benchmark.\n */\n\nimport {\n LLM_INPUT_TOKEN_ATTR_KEYS,\n LLM_OUTPUT_TOKEN_ATTR_KEYS,\n TOOL_NAME_ATTR_KEYS,\n} from '../trace/otlp-attributes'\nimport type { TraceAnalystSpan } from './types'\n\nexport type SuboptimalCode =\n | 'monotonic-input-growth'\n | 'output-length-decay'\n | 'single-tool-dependency'\n | 'no-self-verification'\n\nexport interface SuboptimalSignal {\n code: SuboptimalCode\n severity: 'high' | 'medium' | 'low'\n /** Human-readable claim, with the backing numbers inlined. */\n detail: string\n /** The exact figures the detector fired on — auditable, no model in the loop. */\n evidence: Record<string, number | string | boolean>\n}\n\nexport interface BehavioralMetrics {\n llmCallCount: number\n inputTokenTrajectory: number[]\n outputTokenTrajectory: number[]\n toolHistogram: Record<string, number>\n totalToolCalls: number\n distinctTools: number\n /** distinct/total tool calls; 1.0 when there are no tool calls. */\n toolDiversityRatio: number\n hasSelfVerification: boolean\n signals: SuboptimalSignal[]\n}\n\n/** ≥ this input-token growth ratio across a run, with no compression, fires. */\nconst INPUT_GROWTH_FACTOR = 3\n/** Tool-usage signals need at least this many calls to be meaningful. */\nconst MIN_TOOL_CALLS = 3\n/** Tool names that read or check state count as self-verification, not mutation.\n * Covers the inspect verbs plus the read/search tools real harnesses use to\n * verify (Claude Code Read/Grep/Glob, codex read_file/ls/cat, git status/diff,\n * test/lint). A pure shell tool (Bash/exec_command) is intentionally NOT matched\n * — its name can't tell a `pytest` from an `rm`. */\nconst VERIFY_RE =\n /verif|eval|inspect|check|assert|validat|review|confirm|read|grep|glob|search|view|\\blist\\b|\\bls\\b|\\bcat\\b|\\bfind\\b|diff|status|\\btest|lint|typecheck/i\n\nfunction num(v: unknown): number | null {\n return typeof v === 'number' && Number.isFinite(v) ? v : null\n}\nfunction numAttr(attrs: Record<string, unknown>, keys: readonly string[]): number | null {\n for (const key of keys) {\n const value = num(attrs[key])\n if (value !== null) return value\n }\n return null\n}\nfunction inputTokensOf(s: TraceAnalystSpan): number | null {\n return (\n numAttr(s.attributes, LLM_INPUT_TOKEN_ATTR_KEYS) ?? num(s.attributes['llm.usage.input_tokens'])\n )\n}\nfunction outputTokensOf(s: TraceAnalystSpan): number | null {\n return (\n numAttr(s.attributes, LLM_OUTPUT_TOKEN_ATTR_KEYS) ??\n num(s.attributes['llm.usage.output_tokens'])\n )\n}\nfunction stepOf(s: TraceAnalystSpan): number | null {\n return num(s.attributes.step)\n}\nfunction toolNameOf(s: TraceAnalystSpan): string | null {\n if (s.tool_name) return s.tool_name\n for (const key of TOOL_NAME_ATTR_KEYS) {\n const t = s.attributes[key]\n if (typeof t === 'string' && t.length > 0) return t\n }\n return null\n}\n\n/**\n * Reduce a span list to behavioral metrics + fired suboptimality signals.\n * Pure + deterministic: same spans → same output, on any machine, no model.\n */\nexport function computeTraceMetrics(spans: readonly TraceAnalystSpan[]): BehavioralMetrics {\n // Order by step (when present) then start_time so trajectories reflect run order.\n const ordered = [...spans].sort((a, b) => {\n const sa = stepOf(a)\n const sb = stepOf(b)\n if (sa !== null && sb !== null && sa !== sb) return sa - sb\n return a.start_time.localeCompare(b.start_time)\n })\n\n const inputTokenTrajectory: number[] = []\n const outputTokenTrajectory: number[] = []\n const toolHistogram: Record<string, number> = {}\n let hasSelfVerification = false\n\n for (const s of ordered) {\n const inT = inputTokensOf(s)\n if (inT !== null) inputTokenTrajectory.push(inT)\n const outT = outputTokensOf(s)\n if (outT !== null) outputTokenTrajectory.push(outT)\n const tool = toolNameOf(s)\n if (tool) {\n toolHistogram[tool] = (toolHistogram[tool] ?? 0) + 1\n if (VERIFY_RE.test(tool)) hasSelfVerification = true\n }\n }\n\n const totalToolCalls = Object.values(toolHistogram).reduce((a, b) => a + b, 0)\n const distinctTools = Object.keys(toolHistogram).length\n const toolDiversityRatio = totalToolCalls === 0 ? 1 : distinctTools / totalToolCalls\n\n const signals: SuboptimalSignal[] = []\n\n if (inputTokenTrajectory.length >= 3) {\n const first = inputTokenTrajectory[0]!\n const last = inputTokenTrajectory[inputTokenTrajectory.length - 1]!\n // first === 0 with later growth is an unbounded ratio (0→huge context blowup);\n // treat it as infinite so the signal fires, and report it as such instead of\n // dividing by zero for the displayed factor.\n const growthFromZero = first === 0 && last > 0\n const growth = growthFromZero ? Infinity : first > 0 ? last / first : 0\n if (last > first && growth >= INPUT_GROWTH_FACTOR) {\n const growthLabel = growthFromZero ? '0→nonzero (unbounded)' : `${growth.toFixed(1)}x`\n signals.push({\n code: 'monotonic-input-growth',\n severity: 'high',\n detail: `LLM input tokens grew ${growthLabel} (${first}→${last}) across ${inputTokenTrajectory.length} calls — full history re-sent each step with no compression.`,\n evidence: {\n first,\n last,\n growth_x: growthFromZero ? 'unbounded' : Number(growth.toFixed(2)),\n calls: inputTokenTrajectory.length,\n },\n })\n }\n }\n\n if (outputTokenTrajectory.length >= 3) {\n const first = outputTokenTrajectory[0]!\n const last = outputTokenTrajectory[outputTokenTrajectory.length - 1]!\n if (last < first) {\n signals.push({\n code: 'output-length-decay',\n severity: 'medium',\n detail: `LLM output tokens shrank ${first}→${last} over ${outputTokenTrajectory.length} calls — less planning/reasoning per step as context grows.`,\n evidence: { first, last, calls: outputTokenTrajectory.length },\n })\n }\n }\n\n if (totalToolCalls >= MIN_TOOL_CALLS && distinctTools === 1) {\n const only = Object.keys(toolHistogram)[0]!\n signals.push({\n code: 'single-tool-dependency',\n severity: 'medium',\n detail: `All ${totalToolCalls} tool calls are \\`${only}\\` — no tool diversity and no fallback path.`,\n evidence: { tool: only, calls: totalToolCalls, distinct_tools: 1 },\n })\n }\n\n if (totalToolCalls >= MIN_TOOL_CALLS && !hasSelfVerification) {\n signals.push({\n code: 'no-self-verification',\n severity: 'medium',\n detail: `${totalToolCalls} tool calls and none verify/inspect/check state — the agent never validates its own actions.`,\n evidence: { tool_calls: totalToolCalls, verification_calls: 0 },\n })\n }\n\n return {\n llmCallCount: inputTokenTrajectory.length,\n inputTokenTrajectory,\n outputTokenTrajectory,\n toolHistogram,\n totalToolCalls,\n distinctTools,\n toolDiversityRatio,\n hasSelfVerification,\n signals,\n }\n}\n","/**\n * `behavioralAnalyst` — a DETERMINISTIC analyst (cost.kind = 'deterministic',\n * never calls the LLM). It produces the efficiency/behavioral findings a\n * tolerant agentic analyzer (HALO) re-derives per run inside the model —\n * context bloat, output decay, tool monoculture, missing self-verification —\n * directly from arithmetic over spans (`computeTraceMetrics`).\n *\n * Why it matters: these findings are model-agnostic BY CONSTRUCTION (no model\n * in the loop), so they cannot return 0 on a weak model the way the Ax-RLM\n * does — and they are strictly more reliable than HALO, which spends tokens\n * re-deriving the same numbers and can hallucinate the trend. The agentic\n * RLM kinds remain for SEMANTIC findings that genuinely need a model; this\n * analyst owns the behavioral class.\n */\n\nimport {\n type BehavioralMetrics,\n computeTraceMetrics,\n type SuboptimalCode,\n} from '../trace-analyst/behavioral-metrics'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { TraceAnalystSpan } from '../trace-analyst/types'\nimport { type Analyst, type AnalystFinding, makeFinding } from './types'\n\nconst RECOMMENDED_ACTION: Record<SuboptimalCode, string> = {\n 'monotonic-input-growth':\n 'Add a context-budget instruction: once prior context exceeds a threshold, summarize earlier steps into a short status line instead of re-sending full history.',\n 'output-length-decay':\n 'Require a minimum planning/reasoning budget per step so late steps do not degrade into terse, error-prone commands.',\n 'single-tool-dependency':\n 'Direct the agent to use the full toolset (verify / inspect / alternate actions), not a single execute call, and to plan a fallback when a call returns an unexpected result.',\n 'no-self-verification':\n 'After every state-mutating action, verify the result (eval / inspect / assert) before proceeding.',\n}\n\nconst ANALYST_ID = 'efficiency-behavioral'\n\n/**\n * Map computed signals → structured AnalystFindings. Pure: no LLM, no clock\n * dependence beyond `produced_at` (overridable for deterministic tests).\n */\nexport function deriveEfficiencyFindings(\n metrics: BehavioralMetrics,\n opts: { analystId?: string; producedAt?: string } = {},\n): AnalystFinding[] {\n const analystId = opts.analystId ?? ANALYST_ID\n return metrics.signals.map((sig) =>\n makeFinding({\n analyst_id: analystId,\n area: 'efficiency',\n subject: sig.code, // kebab — passes the cluster grammar; stable key for diffFindings\n claim: sig.detail,\n severity: sig.severity,\n // Deterministic arithmetic over spans, not a model judgment → certain.\n confidence: 1,\n evidence_refs: [\n {\n kind: 'metric',\n uri: `metric://efficiency/${sig.code}`,\n excerpt: JSON.stringify(sig.evidence),\n },\n ],\n recommended_action: RECOMMENDED_ACTION[sig.code],\n metadata: { deterministic: true, evidence: sig.evidence },\n ...(opts.producedAt ? { produced_at: opts.producedAt } : {}),\n }),\n )\n}\n\n/** The deterministic behavioral/efficiency analyst (no LLM, any-model). */\nexport function behavioralAnalyst(): Analyst<TraceAnalysisStore> {\n return {\n id: ANALYST_ID,\n description:\n 'Deterministic behavioral/efficiency findings over OTLP spans — token-growth, output-decay, tool-monoculture, missing self-verification. Zero LLM; model-agnostic by construction.',\n inputKind: 'trace-store',\n cost: { kind: 'deterministic' },\n version: '1.0.0',\n async analyze(store) {\n const overview = await store.getOverview()\n const spans: TraceAnalystSpan[] = []\n for (const traceId of overview.sample_trace_ids) {\n const viewed = await store.viewTrace({ trace_id: traceId })\n if (viewed.spans) spans.push(...viewed.spans)\n }\n return deriveEfficiencyFindings(computeTraceMetrics(spans))\n },\n }\n}\n","/**\n * `buildDefaultAnalystRegistry` — the canonical analyst suite, so consumers\n * stop hand-wiring `new AnalystRegistry()` + per-kind `createTraceAnalystKind`.\n *\n * The deterministic `behavioralAnalyst` is ALWAYS registered (it needs no\n * model and is model-agnostic by construction). The agentic RLM kinds are\n * registered only when an `ai` service is supplied — so a caller with no LLM\n * still gets the full behavioral/efficiency diagnosis, and the substrate's\n * \"any model (including no model)\" guarantee holds at the suite level.\n */\n\nimport type { AxAIService } from '@ax-llm/ax'\nimport { behavioralAnalyst } from './behavioral-analyst'\nimport { createTraceAnalystKind, type TraceAnalystKindSpec } from './kind-factory'\nimport { DEFAULT_TRACE_ANALYST_KINDS } from './kinds'\nimport { AnalystRegistry, type AnalystRegistryOptions } from './registry'\n\nexport interface DefaultAnalystRegistryOptions {\n /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */\n ai?: AxAIService\n /** Model for the agentic kinds (falls back to the ai service default). */\n model?: string\n /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */\n kinds?: readonly TraceAnalystKindSpec[]\n /** Set false to omit the deterministic behavioral analyst (default: include). */\n includeBehavioral?: boolean\n /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */\n registry?: AnalystRegistryOptions\n}\n\nexport function buildDefaultAnalystRegistry(\n opts: DefaultAnalystRegistryOptions = {},\n): AnalystRegistry {\n const registry = new AnalystRegistry(opts.registry)\n if (opts.includeBehavioral !== false) {\n registry.register(behavioralAnalyst())\n }\n if (opts.ai) {\n const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS\n for (const spec of kinds) {\n registry.register(createTraceAnalystKind(spec, { ai: opts.ai, model: opts.model }))\n }\n }\n return registry\n}\n"],"mappings":";;;;;;;;;;;;;AAiDA,IAAM,sBAAsB;AAE5B,IAAM,iBAAiB;AAMvB,IAAM,YACJ;AAEF,SAAS,IAAI,GAA2B;AACtC,SAAO,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,IAAI,IAAI;AAC3D;AACA,SAAS,QAAQ,OAAgC,MAAwC;AACvF,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,IAAI,MAAM,GAAG,CAAC;AAC5B,QAAI,UAAU,KAAM,QAAO;AAAA,EAC7B;AACA,SAAO;AACT;AACA,SAAS,cAAc,GAAoC;AACzD,SACE,QAAQ,EAAE,YAAY,yBAAyB,KAAK,IAAI,EAAE,WAAW,wBAAwB,CAAC;AAElG;AACA,SAAS,eAAe,GAAoC;AAC1D,SACE,QAAQ,EAAE,YAAY,0BAA0B,KAChD,IAAI,EAAE,WAAW,yBAAyB,CAAC;AAE/C;AACA,SAAS,OAAO,GAAoC;AAClD,SAAO,IAAI,EAAE,WAAW,IAAI;AAC9B;AACA,SAAS,WAAW,GAAoC;AACtD,MAAI,EAAE,UAAW,QAAO,EAAE;AAC1B,aAAW,OAAO,qBAAqB;AACrC,UAAM,IAAI,EAAE,WAAW,GAAG;AAC1B,QAAI,OAAO,MAAM,YAAY,EAAE,SAAS,EAAG,QAAO;AAAA,EACpD;AACA,SAAO;AACT;AAMO,SAAS,oBAAoB,OAAuD;AAEzF,QAAM,UAAU,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM;AACxC,UAAM,KAAK,OAAO,CAAC;AACnB,UAAM,KAAK,OAAO,CAAC;AACnB,QAAI,OAAO,QAAQ,OAAO,QAAQ,OAAO,GAAI,QAAO,KAAK;AACzD,WAAO,EAAE,WAAW,cAAc,EAAE,UAAU;AAAA,EAChD,CAAC;AAED,QAAM,uBAAiC,CAAC;AACxC,QAAM,wBAAkC,CAAC;AACzC,QAAM,gBAAwC,CAAC;AAC/C,MAAI,sBAAsB;AAE1B,aAAW,KAAK,SAAS;AACvB,UAAM,MAAM,cAAc,CAAC;AAC3B,QAAI,QAAQ,KAAM,sBAAqB,KAAK,GAAG;AAC/C,UAAM,OAAO,eAAe,CAAC;AAC7B,QAAI,SAAS,KAAM,uBAAsB,KAAK,IAAI;AAClD,UAAM,OAAO,WAAW,CAAC;AACzB,QAAI,MAAM;AACR,oBAAc,IAAI,KAAK,cAAc,IAAI,KAAK,KAAK;AACnD,UAAI,UAAU,KAAK,IAAI,EAAG,uBAAsB;AAAA,IAClD;AAAA,EACF;AAEA,QAAM,iBAAiB,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC;AAC7E,QAAM,gBAAgB,OAAO,KAAK,aAAa,EAAE;AACjD,QAAM,qBAAqB,mBAAmB,IAAI,IAAI,gBAAgB;AAEtE,QAAM,UAA8B,CAAC;AAErC,MAAI,qBAAqB,UAAU,GAAG;AACpC,UAAM,QAAQ,qBAAqB,CAAC;AACpC,UAAM,OAAO,qBAAqB,qBAAqB,SAAS,CAAC;AAIjE,UAAM,iBAAiB,UAAU,KAAK,OAAO;AAC7C,UAAM,SAAS,iBAAiB,WAAW,QAAQ,IAAI,OAAO,QAAQ;AACtE,QAAI,OAAO,SAAS,UAAU,qBAAqB;AACjD,YAAM,cAAc,iBAAiB,+BAA0B,GAAG,OAAO,QAAQ,CAAC,CAAC;AACnF,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ,yBAAyB,WAAW,KAAK,KAAK,SAAI,IAAI,YAAY,qBAAqB,MAAM;AAAA,QACrG,UAAU;AAAA,UACR;AAAA,UACA;AAAA,UACA,UAAU,iBAAiB,cAAc,OAAO,OAAO,QAAQ,CAAC,CAAC;AAAA,UACjE,OAAO,qBAAqB;AAAA,QAC9B;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,sBAAsB,UAAU,GAAG;AACrC,UAAM,QAAQ,sBAAsB,CAAC;AACrC,UAAM,OAAO,sBAAsB,sBAAsB,SAAS,CAAC;AACnE,QAAI,OAAO,OAAO;AAChB,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ,4BAA4B,KAAK,SAAI,IAAI,SAAS,sBAAsB,MAAM;AAAA,QACtF,UAAU,EAAE,OAAO,MAAM,OAAO,sBAAsB,OAAO;AAAA,MAC/D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,kBAAkB,kBAAkB,kBAAkB,GAAG;AAC3D,UAAM,OAAO,OAAO,KAAK,aAAa,EAAE,CAAC;AACzC,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,OAAO,cAAc,qBAAqB,IAAI;AAAA,MACtD,UAAU,EAAE,MAAM,MAAM,OAAO,gBAAgB,gBAAgB,EAAE;AAAA,IACnE,CAAC;AAAA,EACH;AAEA,MAAI,kBAAkB,kBAAkB,CAAC,qBAAqB;AAC5D,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,GAAG,cAAc;AAAA,MACzB,UAAU,EAAE,YAAY,gBAAgB,oBAAoB,EAAE;AAAA,IAChE,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,cAAc,qBAAqB;AAAA,IACnC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;;;AC5KA,IAAM,qBAAqD;AAAA,EACzD,0BACE;AAAA,EACF,uBACE;AAAA,EACF,0BACE;AAAA,EACF,wBACE;AACJ;AAEA,IAAM,aAAa;AAMZ,SAAS,yBACd,SACA,OAAoD,CAAC,GACnC;AAClB,QAAM,YAAY,KAAK,aAAa;AACpC,SAAO,QAAQ,QAAQ;AAAA,IAAI,CAAC,QAC1B,YAAY;AAAA,MACV,YAAY;AAAA,MACZ,MAAM;AAAA,MACN,SAAS,IAAI;AAAA;AAAA,MACb,OAAO,IAAI;AAAA,MACX,UAAU,IAAI;AAAA;AAAA,MAEd,YAAY;AAAA,MACZ,eAAe;AAAA,QACb;AAAA,UACE,MAAM;AAAA,UACN,KAAK,uBAAuB,IAAI,IAAI;AAAA,UACpC,SAAS,KAAK,UAAU,IAAI,QAAQ;AAAA,QACtC;AAAA,MACF;AAAA,MACA,oBAAoB,mBAAmB,IAAI,IAAI;AAAA,MAC/C,UAAU,EAAE,eAAe,MAAM,UAAU,IAAI,SAAS;AAAA,MACxD,GAAI,KAAK,aAAa,EAAE,aAAa,KAAK,WAAW,IAAI,CAAC;AAAA,IAC5D,CAAC;AAAA,EACH;AACF;AAGO,SAAS,oBAAiD;AAC/D,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,aACE;AAAA,IACF,WAAW;AAAA,IACX,MAAM,EAAE,MAAM,gBAAgB;AAAA,IAC9B,SAAS;AAAA,IACT,MAAM,QAAQ,OAAO;AACnB,YAAM,WAAW,MAAM,MAAM,YAAY;AACzC,YAAM,QAA4B,CAAC;AACnC,iBAAW,WAAW,SAAS,kBAAkB;AAC/C,cAAM,SAAS,MAAM,MAAM,UAAU,EAAE,UAAU,QAAQ,CAAC;AAC1D,YAAI,OAAO,MAAO,OAAM,KAAK,GAAG,OAAO,KAAK;AAAA,MAC9C;AACA,aAAO,yBAAyB,oBAAoB,KAAK,CAAC;AAAA,IAC5D;AAAA,EACF;AACF;;;AC1DO,SAAS,4BACd,OAAsC,CAAC,GACtB;AACjB,QAAM,WAAW,IAAI,gBAAgB,KAAK,QAAQ;AAClD,MAAI,KAAK,sBAAsB,OAAO;AACpC,aAAS,SAAS,kBAAkB,CAAC;AAAA,EACvC;AACA,MAAI,KAAK,IAAI;AACX,UAAM,QAAQ,KAAK,SAAS;AAC5B,eAAW,QAAQ,OAAO;AACxB,eAAS,SAAS,uBAAuB,MAAM,EAAE,IAAI,KAAK,IAAI,OAAO,KAAK,MAAM,CAAC,CAAC;AAAA,IACpF;AAAA,EACF;AACA,SAAO;AACT;","names":[]}
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
argHash,
|
|
6
6
|
groupBy,
|
|
7
7
|
toolSpans
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-MHNQWM4I.js";
|
|
9
9
|
|
|
10
10
|
// src/failure-taxonomy.ts
|
|
11
11
|
var DEFAULT_RULES = [
|
|
@@ -394,6 +394,51 @@ async function computeToolUseMetrics(store, runId, options = {}) {
|
|
|
394
394
|
};
|
|
395
395
|
}
|
|
396
396
|
|
|
397
|
+
// src/trajectory.ts
|
|
398
|
+
async function buildTrajectory(store, runId) {
|
|
399
|
+
const spans = await store.spans({ runId });
|
|
400
|
+
const events = await store.events({ runId });
|
|
401
|
+
const childrenOf = /* @__PURE__ */ new Map();
|
|
402
|
+
for (const s of spans) {
|
|
403
|
+
const arr = childrenOf.get(s.parentSpanId) ?? [];
|
|
404
|
+
arr.push(s);
|
|
405
|
+
childrenOf.set(s.parentSpanId, arr);
|
|
406
|
+
}
|
|
407
|
+
for (const arr of childrenOf.values()) arr.sort((a, b) => a.startedAt - b.startedAt);
|
|
408
|
+
const eventsBySpan = /* @__PURE__ */ new Map();
|
|
409
|
+
for (const e of events) {
|
|
410
|
+
if (!e.spanId) continue;
|
|
411
|
+
const arr = eventsBySpan.get(e.spanId) ?? [];
|
|
412
|
+
arr.push(e);
|
|
413
|
+
eventsBySpan.set(e.spanId, arr);
|
|
414
|
+
}
|
|
415
|
+
const steps = [];
|
|
416
|
+
const walk = (spanId, depth) => {
|
|
417
|
+
const kids = childrenOf.get(spanId) ?? [];
|
|
418
|
+
for (const child of kids) {
|
|
419
|
+
steps.push({
|
|
420
|
+
index: steps.length,
|
|
421
|
+
span: child,
|
|
422
|
+
depth,
|
|
423
|
+
events: eventsBySpan.get(child.spanId) ?? []
|
|
424
|
+
});
|
|
425
|
+
walk(child.spanId, depth + 1);
|
|
426
|
+
}
|
|
427
|
+
};
|
|
428
|
+
walk(void 0, 0);
|
|
429
|
+
const llmTurns = steps.filter((s) => s.span.kind === "llm").length;
|
|
430
|
+
const toolCalls = steps.filter((s) => s.span.kind === "tool").length;
|
|
431
|
+
const judgeVerdicts = steps.filter((s) => s.span.kind === "judge").length;
|
|
432
|
+
const retrievals = steps.filter((s) => s.span.kind === "retrieval").length;
|
|
433
|
+
let totalDurationMs = 0;
|
|
434
|
+
if (steps.length > 0) {
|
|
435
|
+
const starts = spans.map((s) => s.startedAt);
|
|
436
|
+
const ends = spans.map((s) => s.endedAt ?? s.startedAt);
|
|
437
|
+
totalDurationMs = Math.max(...ends) - Math.min(...starts);
|
|
438
|
+
}
|
|
439
|
+
return { runId, steps, llmTurns, toolCalls, judgeVerdicts, retrievals, totalDurationMs };
|
|
440
|
+
}
|
|
441
|
+
|
|
397
442
|
// src/baseline.ts
|
|
398
443
|
function compareToBaseline(samples, options = {}) {
|
|
399
444
|
const effectThreshold = options.effectThreshold ?? 0.5;
|
|
@@ -548,8 +593,9 @@ export {
|
|
|
548
593
|
DEFAULT_RULES,
|
|
549
594
|
classifyFailure,
|
|
550
595
|
computeToolUseMetrics,
|
|
596
|
+
buildTrajectory,
|
|
551
597
|
compareToBaseline,
|
|
552
598
|
iqr,
|
|
553
599
|
welchsTTest
|
|
554
600
|
};
|
|
555
|
-
//# sourceMappingURL=chunk-
|
|
601
|
+
//# sourceMappingURL=chunk-PLOMR3HP.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/failure-taxonomy.ts","../src/tool-use-metrics.ts","../src/trajectory.ts","../src/baseline.ts"],"sourcesContent":["/**\n * Failure taxonomy — canonical classes + a default classifier.\n *\n * Every failed run should end up in a named class. The classifier here\n * is rule-based (fast, deterministic); an LLM fallback can be added by\n * the consumer for novel cases and trained into the rule base over time.\n *\n * Consumers call `classifyFailure(run, spans, events)` and persist the\n * returned class as `Run.outcome.failureClass`.\n */\n\nimport type { FailureClass, Run, Span, TraceEvent } from './trace/schema'\nimport { FAILURE_CLASSES } from './trace/schema'\n\nexport { FAILURE_CLASSES, type FailureClass }\n\nexport interface FailureContext {\n run: Run\n spans: Span[]\n events: TraceEvent[]\n}\n\nexport interface FailureClassification {\n failureClass: FailureClass\n reason: string\n triggerSpanId?: string\n triggerEventId?: string\n}\n\n/** Ordered rules — first match wins. */\nexport interface FailureRule {\n id: string\n match: (ctx: FailureContext) => {\n failureClass: FailureClass\n reason: string\n triggerSpanId?: string\n triggerEventId?: string\n } | null\n}\n\nexport const DEFAULT_RULES: FailureRule[] = [\n // Outcome already named? Respect it.\n {\n id: 'explicit-outcome',\n match: ({ run }) => {\n const fc = run.outcome?.failureClass\n if (fc && fc !== 'unknown')\n return { failureClass: fc, reason: 'outcome.failureClass set explicitly' }\n return null\n },\n },\n {\n id: 'knowledge-readiness-blocked',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n e.payload.kind === 'readiness_scored' &&\n e.payload.passed === false,\n )\n return event\n ? {\n failureClass: 'knowledge_readiness_blocked',\n reason: 'knowledge readiness report blocked execution',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'bad-integration-manifest',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n ((e.payload.kind === 'integration_manifest_validated' && e.payload.valid === false) ||\n (e.payload.kind === 'integration_invoke_failed' &&\n e.payload.code === 'manifest_invalid')),\n )\n return event\n ? {\n failureClass: 'bad_integration_manifest',\n reason: 'integration manifest validation failed before launch',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'missing-integration-connection',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n e.payload.kind === 'integration_manifest_resolved' &&\n hasResolutionStatus(e.payload, 'missing_connection'),\n )\n return event\n ? {\n failureClass: 'missing_integration_connection',\n reason: 'required integration connection was missing',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'missing-integration-scope',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n ((e.payload.kind === 'integration_manifest_resolved' && hasMissingScopes(e.payload)) ||\n (e.payload.kind === 'integration_invoke_failed' && e.payload.code === 'scope_denied')),\n )\n return event\n ? {\n failureClass: 'missing_integration_scope',\n reason: 'integration grant or connection lacks required scopes',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'integration-approval-required',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n ((e.payload.kind === 'integration_invoke' && e.payload.status === 'approval_required') ||\n (e.payload.kind === 'integration_invoke_failed' &&\n e.payload.code === 'approval_required') ||\n e.payload.kind === 'integration_approval_required'),\n )\n return event\n ? {\n failureClass: 'integration_approval_required',\n reason: 'integration write paused for user approval',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'integration-auth-expired',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n e.payload.kind === 'integration_invoke_failed' &&\n (e.payload.code === 'auth_expired' ||\n e.payload.code === 'connection_not_active' ||\n e.payload.code === 'capability_expired' ||\n e.payload.status === 'expired'),\n )\n return event\n ? {\n failureClass: 'integration_auth_expired',\n reason: 'integration connection or capability expired',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'unsafe-integration-write-denied',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n e.payload.kind === 'integration_invoke_failed' &&\n (e.payload.code === 'unsafe_write_denied' ||\n e.payload.code === 'policy_denied' ||\n e.payload.code === 'action_denied'),\n )\n return event\n ? {\n failureClass: 'unsafe_integration_write_denied',\n reason: 'integration write was denied by policy or capability scope',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'integration-provider-failure',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n e.payload.kind === 'integration_invoke_failed' &&\n ![\n 'scope_denied',\n 'approval_required',\n 'auth_expired',\n 'connection_not_active',\n 'capability_expired',\n 'unsafe_write_denied',\n 'policy_denied',\n 'action_denied',\n 'manifest_invalid',\n ].includes(String(e.payload.code)),\n )\n return event\n ? {\n failureClass: 'integration_provider_failure',\n reason: 'integration provider invocation failed',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'missing-credentials',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n e.payload.kind === 'knowledge_gap' &&\n e.payload.category === 'credential_or_secret',\n )\n return event\n ? {\n failureClass: 'missing_credentials',\n reason: 'required credential or secret was missing',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'bad-retrieval',\n match: ({ run, spans }) => {\n if (run.outcome?.pass !== false) return null\n const retrieval = spans.find(\n (s) =>\n s.kind === 'retrieval' && (s.hits.length === 0 || s.hits.every((hit) => hit.score <= 0)),\n )\n return retrieval\n ? {\n failureClass: 'bad_retrieval',\n reason: 'retrieval returned no useful hits for a failed run',\n triggerSpanId: retrieval.spanId,\n }\n : null\n },\n },\n {\n id: 'insufficient-evidence',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n e.payload.kind === 'knowledge_gap' &&\n e.payload.reason === 'insufficient_evidence',\n )\n return event\n ? {\n failureClass: 'insufficient_evidence',\n reason: 'task proceeded with insufficient supporting evidence',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n {\n id: 'contradictory-evidence',\n match: ({ events }) => {\n const event = events.find(\n (e) =>\n e.kind === 'custom' &&\n e.payload.kind === 'knowledge_gap' &&\n e.payload.reason === 'contradictory_evidence',\n )\n return event\n ? {\n failureClass: 'contradictory_evidence',\n reason: 'supporting evidence contradicted itself',\n triggerEventId: event.eventId,\n }\n : null\n },\n },\n // Budget breach events\n {\n id: 'budget-breach',\n match: ({ events }) => {\n const breach = events.find((e) => e.kind === 'budget_breach')\n return breach\n ? {\n failureClass: 'budget_exceeded',\n reason: `budget breached on ${breach.payload.dimension ?? 'unknown dimension'}`,\n triggerEventId: breach.eventId,\n }\n : null\n },\n },\n // Policy violations\n {\n id: 'policy-violation',\n match: ({ events }) => {\n const e = events.find((x) => x.kind === 'policy_violation')\n return e\n ? {\n failureClass: 'policy_violation',\n reason: 'policy_violation event emitted',\n triggerEventId: e.eventId,\n }\n : null\n },\n },\n // Sandbox non-zero exit code\n {\n id: 'sandbox-failure',\n match: ({ spans }) => {\n const s = spans.find(\n (x) => x.kind === 'sandbox' && typeof x.exitCode === 'number' && x.exitCode !== 0,\n )\n if (!s) return null\n return {\n failureClass: 'sandbox_failure',\n reason: `sandbox exited ${(s as Extract<Span, { kind: 'sandbox' }>).exitCode}`,\n triggerSpanId: s.spanId,\n }\n },\n },\n // Timeout: run aborted by external signal\n {\n id: 'timeout',\n match: ({ run, events }) => {\n if (run.status !== 'aborted') return null\n const hasTimeout = events.some(\n (e) =>\n e.kind === 'error' &&\n String(e.payload.reason ?? '')\n .toLowerCase()\n .includes('timeout'),\n )\n const note = (run.outcome?.notes ?? '').toLowerCase()\n if (hasTimeout || note.includes('timeout') || note.includes('deadline')) {\n return { failureClass: 'timeout', reason: 'timeout signal observed' }\n }\n return null\n },\n },\n // Tool recovery failure: many consecutive tool errors on the same tool\n {\n id: 'tool-recovery-failure',\n match: ({ spans }) => {\n const tools = spans.filter((s) => s.kind === 'tool')\n const byTool = new Map<string, Span[]>()\n for (const t of tools) {\n const name = (t as Extract<Span, { kind: 'tool' }>).toolName\n const arr = byTool.get(name) ?? []\n arr.push(t)\n byTool.set(name, arr)\n }\n for (const [name, arr] of byTool) {\n const errs = arr.filter((s) => s.status === 'error')\n if (errs.length >= 3 && errs.length === arr.length) {\n return {\n failureClass: 'tool_recovery_failure',\n reason: `${errs.length} consecutive errors on tool \"${name}\"`,\n triggerSpanId: errs[errs.length - 1]!.spanId,\n }\n }\n }\n return null\n },\n },\n // Tool selection error: the run failed and agent called zero tools despite having them\n {\n id: 'tool-selection-error',\n match: ({ run, spans }) => {\n if (run.outcome?.pass !== false) return null\n const hasToolsAvailable = spans.some(\n (s) =>\n s.kind === 'agent' &&\n (s.attributes?.toolsAvailable as number | undefined) !== undefined &&\n (s.attributes?.toolsAvailable as number) > 0,\n )\n const tools = spans.filter((s) => s.kind === 'tool')\n if (hasToolsAvailable && tools.length === 0) {\n return {\n failureClass: 'tool_selection_error',\n reason: 'tools were available but none were called',\n }\n }\n return null\n },\n },\n // Format drift: scored by a judge with dimension='format' below threshold\n {\n id: 'format-drift',\n match: ({ spans }) => {\n const judge = spans.find(\n (s) =>\n s.kind === 'judge' &&\n (s as Extract<Span, { kind: 'judge' }>).dimension === 'format' &&\n (s as Extract<Span, { kind: 'judge' }>).score < 0.5,\n )\n return judge\n ? {\n failureClass: 'format_drift',\n reason: 'format judge scored below 0.5',\n triggerSpanId: judge.spanId,\n }\n : null\n },\n },\n]\n\nfunction hasResolutionStatus(payload: Record<string, unknown>, status: string): boolean {\n if (status === 'missing_connection' && stringArray(payload.missingConnections).length > 0)\n return true\n return resolutionItems(payload).some((item) => item.status === status)\n}\n\nfunction hasMissingScopes(payload: Record<string, unknown>): boolean {\n if (stringArray(payload.missingScopes).length > 0) return true\n return resolutionItems(payload).some(\n (item) => Array.isArray(item.missingScopes) && item.missingScopes.length > 0,\n )\n}\n\nfunction resolutionItems(payload: Record<string, unknown>): Array<Record<string, unknown>> {\n return [\n ...records(payload.missing),\n ...records(payload.optionalMissing),\n ...records(payload.ready),\n ]\n}\n\nfunction records(value: unknown): Array<Record<string, unknown>> {\n if (!Array.isArray(value)) return []\n return value.filter(\n (item): item is Record<string, unknown> =>\n Boolean(item) && typeof item === 'object' && !Array.isArray(item),\n )\n}\n\nfunction stringArray(value: unknown): string[] {\n return Array.isArray(value)\n ? value.filter((item): item is string => typeof item === 'string')\n : []\n}\n\n/** Classify the failure mode of a run using an ordered rule list. */\nexport function classifyFailure(\n ctx: FailureContext,\n rules: FailureRule[] = DEFAULT_RULES,\n): FailureClassification {\n if (ctx.run.outcome?.pass !== false && ctx.run.status === 'completed') {\n return { failureClass: 'success', reason: 'run completed with pass=true (or no explicit fail)' }\n }\n for (const rule of rules) {\n const hit = rule.match(ctx)\n if (hit) return hit\n }\n return { failureClass: 'unknown', reason: 'no rule matched; run failed for unclassified reason' }\n}\n","/**\n * Tool-use metrics — derived purely from trace data.\n *\n * No scoring assumptions: consumers supply optional ground-truth tool\n * selections per turn + optional \"information used downstream\" signals.\n * Without those, we still compute descriptive metrics (error rate,\n * retry rate, duplicate-call rate) that are useful on their own.\n */\n\nimport { argHash, groupBy, toolSpans } from './trace/query'\nimport type { Span } from './trace/schema'\nimport type { TraceStore } from './trace/store'\n\nexport interface ToolUseMetrics {\n runId: string\n totalCalls: number\n byTool: Record<string, ToolStats>\n errorRate: number\n /** Ratio of calls with identical (toolName, argHash) already seen earlier in the same run. */\n duplicateRate: number\n /** Ratio of error calls followed by ≥1 retry on same tool. */\n retryRate: number\n /** Optional: of the calls agent made, fraction the evaluator marked as \"correct selection\". */\n selectionAccuracy?: number\n}\n\nexport interface ToolStats {\n calls: number\n errors: number\n avgLatencyMs: number\n duplicates: number\n}\n\nexport interface ToolUseOptions {\n /** Map of spanId → whether the evaluator judged the tool selection correct. Optional. */\n selectionLabels?: Record<string, boolean>\n}\n\nexport async function computeToolUseMetrics(\n store: TraceStore,\n runId: string,\n options: ToolUseOptions = {},\n): Promise<ToolUseMetrics> {\n const tools = await toolSpans(store, runId)\n if (tools.length === 0) {\n return { runId, totalCalls: 0, byTool: {}, errorRate: 0, duplicateRate: 0, retryRate: 0 }\n }\n\n const byTool: Record<string, ToolStats> = {}\n let totalErrors = 0\n let totalDuplicates = 0\n const sortedTools = [...tools].sort((a, b) => a.startedAt - b.startedAt)\n const seenSignatures = new Set<string>()\n\n // duplicate detection + per-tool aggregation\n for (const t of sortedTools) {\n byTool[t.toolName] ??= { calls: 0, errors: 0, avgLatencyMs: 0, duplicates: 0 }\n const stat = byTool[t.toolName]!\n stat.calls += 1\n if (t.status === 'error') {\n stat.errors += 1\n totalErrors += 1\n }\n if (typeof t.latencyMs === 'number') stat.avgLatencyMs += t.latencyMs\n const sig = `${t.toolName}|${argHash(t.args)}`\n if (seenSignatures.has(sig)) {\n stat.duplicates += 1\n totalDuplicates += 1\n }\n seenSignatures.add(sig)\n }\n\n for (const stat of Object.values(byTool)) {\n stat.avgLatencyMs = stat.calls > 0 ? stat.avgLatencyMs / stat.calls : 0\n }\n\n // retry detection: per-tool chronological adjacency where error → next same-tool call\n let retryOpportunities = 0\n let retriesFollowed = 0\n for (const [, arr] of groupBy(sortedTools, (t) => t.toolName)) {\n for (let i = 0; i < arr.length; i++) {\n if (arr[i]!.status !== 'error') continue\n retryOpportunities += 1\n if (arr[i + 1]) retriesFollowed += 1\n }\n }\n const retryRate = retryOpportunities > 0 ? retriesFollowed / retryOpportunities : 0\n\n let selectionAccuracy: number | undefined\n if (options.selectionLabels) {\n const labeled = sortedTools.filter((t) => t.spanId in options.selectionLabels!)\n if (labeled.length > 0) {\n selectionAccuracy =\n labeled.filter((t) => options.selectionLabels![t.spanId]).length / labeled.length\n }\n }\n\n return {\n runId,\n totalCalls: sortedTools.length,\n byTool,\n errorRate: totalErrors / sortedTools.length,\n duplicateRate: totalDuplicates / sortedTools.length,\n retryRate,\n selectionAccuracy,\n }\n}\n\nexport type { Span }\n","/**\n * Trajectory — ordered, structured view over a run's spans.\n *\n * A pure function `buildTrajectory(store, runId) → Trajectory` returns\n * a topologically ordered list of `TrajectoryStep` with parent-child\n * grouping collapsed into a single line-of-agent-work. Separate\n * analyzers (stuck-loop detection, waste ratio) live in\n * `pipelines/` and consume the trajectory.\n */\n\nimport type { Span, TraceEvent } from './trace/schema'\nimport type { TraceStore } from './trace/store'\n\nexport interface TrajectoryStep {\n index: number\n span: Span\n /** Depth in the span tree from the root. 0 = top-level. */\n depth: number\n /** Events attached to this span. */\n events: TraceEvent[]\n}\n\nexport interface Trajectory {\n runId: string\n steps: TrajectoryStep[]\n llmTurns: number\n toolCalls: number\n judgeVerdicts: number\n retrievals: number\n totalDurationMs: number\n}\n\nexport async function buildTrajectory(store: TraceStore, runId: string): Promise<Trajectory> {\n const spans = await store.spans({ runId })\n const events = await store.events({ runId })\n const childrenOf = new Map<string | undefined, Span[]>()\n for (const s of spans) {\n const arr = childrenOf.get(s.parentSpanId) ?? []\n arr.push(s)\n childrenOf.set(s.parentSpanId, arr)\n }\n // Sort children by startedAt so DFS yields chronological order within siblings.\n for (const arr of childrenOf.values()) arr.sort((a, b) => a.startedAt - b.startedAt)\n\n const eventsBySpan = new Map<string, TraceEvent[]>()\n for (const e of events) {\n if (!e.spanId) continue\n const arr = eventsBySpan.get(e.spanId) ?? []\n arr.push(e)\n eventsBySpan.set(e.spanId, arr)\n }\n\n const steps: TrajectoryStep[] = []\n const walk = (spanId: string | undefined, depth: number): void => {\n const kids = childrenOf.get(spanId) ?? []\n for (const child of kids) {\n steps.push({\n index: steps.length,\n span: child,\n depth,\n events: eventsBySpan.get(child.spanId) ?? [],\n })\n walk(child.spanId, depth + 1)\n }\n }\n walk(undefined, 0)\n\n const llmTurns = steps.filter((s) => s.span.kind === 'llm').length\n const toolCalls = steps.filter((s) => s.span.kind === 'tool').length\n const judgeVerdicts = steps.filter((s) => s.span.kind === 'judge').length\n const retrievals = steps.filter((s) => s.span.kind === 'retrieval').length\n\n let totalDurationMs = 0\n if (steps.length > 0) {\n const starts = spans.map((s) => s.startedAt)\n const ends = spans.map((s) => s.endedAt ?? s.startedAt)\n totalDurationMs = Math.max(...ends) - Math.min(...starts)\n }\n\n return { runId, steps, llmTurns, toolCalls, judgeVerdicts, retrievals, totalDurationMs }\n}\n\n// Re-export core types for convenience so consumers don't import from two paths.\nexport type { Span, TraceEvent } from './trace/schema'\nexport type { TraceStore } from './trace/store'\n","/**\n * Baseline regression detection.\n *\n * Lifted from ADC baseline.ts. Every promotion-blocking signal boils down\n * to: \"is this run measurably worse than baseline?\" — with enough\n * statistical rigor to distinguish noise from drift.\n *\n * Uses:\n * - Welch's t-test (unequal variance) for per-metric mean comparison\n * - Cohen's d for effect size magnitude\n * - IQR for stability flag (unstable samples can't be trusted for comparisons)\n *\n * Returns a structured verdict: improved | regressed | stable | unstable.\n */\n\nimport { cohensD } from './statistics'\n\nexport interface MetricSamples {\n /** Stable metric key (e.g. \"overallScore\", \"firstTokenMs\"). */\n metric: string\n /** Whether higher values are better. */\n higherIsBetter: boolean\n baseline: number[]\n candidate: number[]\n}\n\nexport interface MetricVerdict {\n metric: string\n baselineMean: number\n candidateMean: number\n delta: number\n cohensD: number\n welchT: number\n welchDf: number\n welchP: number\n stable: boolean\n /** IQR of the combined samples — used as a rough stability indicator. */\n iqr: number\n verdict: 'improved' | 'regressed' | 'stable' | 'unstable'\n}\n\nexport interface BaselineReport {\n metrics: MetricVerdict[]\n /** True if any critical metric regressed. */\n hasRegression: boolean\n /** True if any metric is unstable (too noisy to judge). */\n hasUnstable: boolean\n}\n\nexport interface BaselineOptions {\n /** Effect size threshold for meaningful delta (default 0.5 — medium effect). */\n effectThreshold?: number\n /** p-value threshold for statistical significance (default 0.05). */\n alpha?: number\n /** IQR/mean ratio above which samples are flagged unstable (default 0.30). */\n unstableCvThreshold?: number\n}\n\n/**\n * Compare candidate samples against baseline per metric. Verdict logic:\n * - unstable: IQR/|mean| > threshold on either set — not enough signal\n * - improved: meaningful effect in the \"better\" direction AND p < alpha\n * - regressed: meaningful effect in the \"worse\" direction AND p < alpha\n * - stable: otherwise (no significant change)\n */\nexport function compareToBaseline(\n samples: MetricSamples[],\n options: BaselineOptions = {},\n): BaselineReport {\n const effectThreshold = options.effectThreshold ?? 0.5\n const alpha = options.alpha ?? 0.05\n const cvThreshold = options.unstableCvThreshold ?? 0.3\n\n const metrics: MetricVerdict[] = samples.map((s) => {\n if (s.baseline.length < 2 || s.candidate.length < 2) {\n throw new Error(`compareToBaseline: need ≥2 samples per side for \"${s.metric}\"`)\n }\n const bMean = mean(s.baseline)\n const cMean = mean(s.candidate)\n const delta = cMean - bMean\n const d = cohensD(s.baseline, s.candidate) // positive = candidate higher\n const { t, df, p } = welchsTTest(s.baseline, s.candidate)\n // Stability is per-side: a comparison is trustworthy only when BOTH\n // samples are internally consistent. Combining the sides would flag\n // large-but-real deltas as \"unstable\" which is exactly what we want\n // to detect.\n const baselineIqr = iqr(s.baseline)\n const candidateIqr = iqr(s.candidate)\n const baselineStable = baselineIqr / Math.max(Math.abs(bMean), 1e-9) <= cvThreshold\n const candidateStable = candidateIqr / Math.max(Math.abs(cMean), 1e-9) <= cvThreshold\n const stable = baselineStable && candidateStable\n const reportedIqr = Math.max(baselineIqr, candidateIqr)\n\n let verdict: MetricVerdict['verdict']\n if (!stable) {\n verdict = 'unstable'\n } else if (p < alpha && Math.abs(d) >= effectThreshold) {\n const candidateIsBetter = s.higherIsBetter ? delta > 0 : delta < 0\n verdict = candidateIsBetter ? 'improved' : 'regressed'\n } else {\n verdict = 'stable'\n }\n\n return {\n metric: s.metric,\n baselineMean: bMean,\n candidateMean: cMean,\n delta,\n cohensD: d,\n welchT: t,\n welchDf: df,\n welchP: p,\n stable,\n iqr: reportedIqr,\n verdict,\n }\n })\n\n return {\n metrics,\n hasRegression: metrics.some((m) => m.verdict === 'regressed'),\n hasUnstable: metrics.some((m) => m.verdict === 'unstable'),\n }\n}\n\nfunction mean(xs: number[]): number {\n return xs.reduce((a, b) => a + b, 0) / xs.length\n}\n\n/** Inter-quartile range; 0 when the sample has no spread. */\nexport function iqr(xs: number[]): number {\n if (xs.length === 0) return 0\n const sorted = [...xs].sort((a, b) => a - b)\n const q = (p: number) => {\n const idx = p * (sorted.length - 1)\n const lo = Math.floor(idx)\n const hi = Math.ceil(idx)\n return sorted[lo]! + (sorted[hi]! - sorted[lo]!) * (idx - lo)\n }\n return q(0.75) - q(0.25)\n}\n\n/**\n * Welch's t-test — unequal-variance two-sample t. Uses the same Student-t\n * CDF as `pairedTTest` (via incomplete beta); falls back to normal tail\n * when df is large.\n */\nexport function welchsTTest(a: number[], b: number[]): { t: number; df: number; p: number } {\n if (a.length < 2 || b.length < 2) return { t: 0, df: 0, p: 1 }\n const mA = mean(a)\n const mB = mean(b)\n const vA = variance(a, mA)\n const vB = variance(b, mB)\n const seSquared = vA / a.length + vB / b.length\n if (seSquared === 0) return { t: mA === mB ? 0 : Infinity, df: 0, p: mA === mB ? 1 : 0 }\n const t = (mB - mA) / Math.sqrt(seSquared)\n const df =\n (seSquared * seSquared) /\n ((vA / a.length) ** 2 / (a.length - 1) + (vB / b.length) ** 2 / (b.length - 1))\n const p = 2 * (1 - studentTCdf(Math.abs(t), df))\n return { t, df, p }\n}\n\nfunction variance(xs: number[], m: number): number {\n return xs.reduce((acc, x) => acc + (x - m) ** 2, 0) / (xs.length - 1)\n}\n\n// Re-used from statistics.ts via small local copy to avoid exporting internals.\nfunction studentTCdf(t: number, df: number): number {\n if (df <= 0) return 0.5\n if (df > 100) return normalCdf(t)\n const x = df / (df + t * t)\n const ib = incompleteBeta(x, df / 2, 0.5)\n return t >= 0 ? 1 - 0.5 * ib : 0.5 * ib\n}\n\nfunction incompleteBeta(x: number, a: number, b: number): number {\n if (x <= 0) return 0\n if (x >= 1) return 1\n const lnBeta = lnGamma(a) + lnGamma(b) - lnGamma(a + b)\n const front = Math.exp(Math.log(x) * a + Math.log(1 - x) * b - lnBeta) / a\n let c = 1\n let d = 1 - ((a + b) * x) / (a + 1)\n if (Math.abs(d) < 1e-30) d = 1e-30\n d = 1 / d\n let f = d\n for (let m = 1; m <= 200; m++) {\n const m2 = 2 * m\n let num = (m * (b - m) * x) / ((a + m2 - 1) * (a + m2))\n d = 1 + num * d\n if (Math.abs(d) < 1e-30) d = 1e-30\n c = 1 + num / c\n if (Math.abs(c) < 1e-30) c = 1e-30\n d = 1 / d\n f *= d * c\n num = -((a + m) * (a + b + m) * x) / ((a + m2) * (a + m2 + 1))\n d = 1 + num * d\n if (Math.abs(d) < 1e-30) d = 1e-30\n c = 1 + num / c\n if (Math.abs(c) < 1e-30) c = 1e-30\n d = 1 / d\n const delta = d * c\n f *= delta\n if (Math.abs(delta - 1) < 3e-7) break\n }\n return front * f\n}\n\nfunction lnGamma(z: number): number {\n const coefs = [\n 0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313,\n -176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6,\n 1.5056327351493116e-7,\n ]\n if (z < 0.5) return Math.log(Math.PI / Math.sin(Math.PI * z)) - lnGamma(1 - z)\n z -= 1\n let x = coefs[0]!\n for (let i = 1; i < 9; i++) x += coefs[i]! / (z + i)\n const t = z + 7.5\n return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x)\n}\n\nfunction normalCdf(x: number): number {\n const a1 = 0.254829592\n const a2 = -0.284496736\n const a3 = 1.421413741\n const a4 = -1.453152027\n const a5 = 1.061405429\n const p = 0.3275911\n const sign = x < 0 ? -1 : 1\n const absX = Math.abs(x)\n const t = 1 / (1 + p * absX)\n const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp((-absX * absX) / 2)\n return 0.5 * (1 + sign * y)\n}\n"],"mappings":";;;;;;;;;;AAwCO,IAAM,gBAA+B;AAAA;AAAA,EAE1C;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,IAAI,MAAM;AAClB,YAAM,KAAK,IAAI,SAAS;AACxB,UAAI,MAAM,OAAO;AACf,eAAO,EAAE,cAAc,IAAI,QAAQ,sCAAsC;AAC3E,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,sBACnB,EAAE,QAAQ,WAAW;AAAA,MACzB;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,aACT,EAAE,QAAQ,SAAS,oCAAoC,EAAE,QAAQ,UAAU,SAC1E,EAAE,QAAQ,SAAS,+BAClB,EAAE,QAAQ,SAAS;AAAA,MAC3B;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mCACnB,oBAAoB,EAAE,SAAS,oBAAoB;AAAA,MACvD;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,aACT,EAAE,QAAQ,SAAS,mCAAmC,iBAAiB,EAAE,OAAO,KAC/E,EAAE,QAAQ,SAAS,+BAA+B,EAAE,QAAQ,SAAS;AAAA,MAC5E;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,aACT,EAAE,QAAQ,SAAS,wBAAwB,EAAE,QAAQ,WAAW,uBAC/D,EAAE,QAAQ,SAAS,+BAClB,EAAE,QAAQ,SAAS,uBACrB,EAAE,QAAQ,SAAS;AAAA,MACzB;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,gCAClB,EAAE,QAAQ,SAAS,kBAClB,EAAE,QAAQ,SAAS,2BACnB,EAAE,QAAQ,SAAS,wBACnB,EAAE,QAAQ,WAAW;AAAA,MAC3B;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,gCAClB,EAAE,QAAQ,SAAS,yBAClB,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,SAAS;AAAA,MACzB;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,+BACnB,CAAC;AAAA,UACC;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF,EAAE,SAAS,OAAO,EAAE,QAAQ,IAAI,CAAC;AAAA,MACrC;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,aAAa;AAAA,MAC3B;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,KAAK,MAAM,MAAM;AACzB,UAAI,IAAI,SAAS,SAAS,MAAO,QAAO;AACxC,YAAM,YAAY,MAAM;AAAA,QACtB,CAAC,MACC,EAAE,SAAS,gBAAgB,EAAE,KAAK,WAAW,KAAK,EAAE,KAAK,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC;AAAA,MAC1F;AACA,aAAO,YACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,eAAe,UAAU;AAAA,MAC3B,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,WAAW;AAAA,MACzB;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,QAAQ,OAAO;AAAA,QACnB,CAAC,MACC,EAAE,SAAS,YACX,EAAE,QAAQ,SAAS,mBACnB,EAAE,QAAQ,WAAW;AAAA,MACzB;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,MAAM;AAAA,MACxB,IACA;AAAA,IACN;AAAA,EACF;AAAA;AAAA,EAEA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,SAAS,OAAO,KAAK,CAAC,MAAM,EAAE,SAAS,eAAe;AAC5D,aAAO,SACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ,sBAAsB,OAAO,QAAQ,aAAa,mBAAmB;AAAA,QAC7E,gBAAgB,OAAO;AAAA,MACzB,IACA;AAAA,IACN;AAAA,EACF;AAAA;AAAA,EAEA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,OAAO,MAAM;AACrB,YAAM,IAAI,OAAO,KAAK,CAAC,MAAM,EAAE,SAAS,kBAAkB;AAC1D,aAAO,IACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,gBAAgB,EAAE;AAAA,MACpB,IACA;AAAA,IACN;AAAA,EACF;AAAA;AAAA,EAEA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,MAAM,MAAM;AACpB,YAAM,IAAI,MAAM;AAAA,QACd,CAAC,MAAM,EAAE,SAAS,aAAa,OAAO,EAAE,aAAa,YAAY,EAAE,aAAa;AAAA,MAClF;AACA,UAAI,CAAC,EAAG,QAAO;AACf,aAAO;AAAA,QACL,cAAc;AAAA,QACd,QAAQ,kBAAmB,EAAyC,QAAQ;AAAA,QAC5E,eAAe,EAAE;AAAA,MACnB;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAEA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,KAAK,OAAO,MAAM;AAC1B,UAAI,IAAI,WAAW,UAAW,QAAO;AACrC,YAAM,aAAa,OAAO;AAAA,QACxB,CAAC,MACC,EAAE,SAAS,WACX,OAAO,EAAE,QAAQ,UAAU,EAAE,EAC1B,YAAY,EACZ,SAAS,SAAS;AAAA,MACzB;AACA,YAAM,QAAQ,IAAI,SAAS,SAAS,IAAI,YAAY;AACpD,UAAI,cAAc,KAAK,SAAS,SAAS,KAAK,KAAK,SAAS,UAAU,GAAG;AACvE,eAAO,EAAE,cAAc,WAAW,QAAQ,0BAA0B;AAAA,MACtE;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA,EAEA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,MAAM,MAAM;AACpB,YAAM,QAAQ,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,MAAM;AACnD,YAAM,SAAS,oBAAI,IAAoB;AACvC,iBAAW,KAAK,OAAO;AACrB,cAAM,OAAQ,EAAsC;AACpD,cAAM,MAAM,OAAO,IAAI,IAAI,KAAK,CAAC;AACjC,YAAI,KAAK,CAAC;AACV,eAAO,IAAI,MAAM,GAAG;AAAA,MACtB;AACA,iBAAW,CAAC,MAAM,GAAG,KAAK,QAAQ;AAChC,cAAM,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,WAAW,OAAO;AACnD,YAAI,KAAK,UAAU,KAAK,KAAK,WAAW,IAAI,QAAQ;AAClD,iBAAO;AAAA,YACL,cAAc;AAAA,YACd,QAAQ,GAAG,KAAK,MAAM,gCAAgC,IAAI;AAAA,YAC1D,eAAe,KAAK,KAAK,SAAS,CAAC,EAAG;AAAA,UACxC;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA,EAEA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,KAAK,MAAM,MAAM;AACzB,UAAI,IAAI,SAAS,SAAS,MAAO,QAAO;AACxC,YAAM,oBAAoB,MAAM;AAAA,QAC9B,CAAC,MACC,EAAE,SAAS,WACV,EAAE,YAAY,mBAA0C,UACxD,EAAE,YAAY,iBAA4B;AAAA,MAC/C;AACA,YAAM,QAAQ,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,MAAM;AACnD,UAAI,qBAAqB,MAAM,WAAW,GAAG;AAC3C,eAAO;AAAA,UACL,cAAc;AAAA,UACd,QAAQ;AAAA,QACV;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA,EAEA;AAAA,IACE,IAAI;AAAA,IACJ,OAAO,CAAC,EAAE,MAAM,MAAM;AACpB,YAAM,QAAQ,MAAM;AAAA,QAClB,CAAC,MACC,EAAE,SAAS,WACV,EAAuC,cAAc,YACrD,EAAuC,QAAQ;AAAA,MACpD;AACA,aAAO,QACH;AAAA,QACE,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,eAAe,MAAM;AAAA,MACvB,IACA;AAAA,IACN;AAAA,EACF;AACF;AAEA,SAAS,oBAAoB,SAAkC,QAAyB;AACtF,MAAI,WAAW,wBAAwB,YAAY,QAAQ,kBAAkB,EAAE,SAAS;AACtF,WAAO;AACT,SAAO,gBAAgB,OAAO,EAAE,KAAK,CAAC,SAAS,KAAK,WAAW,MAAM;AACvE;AAEA,SAAS,iBAAiB,SAA2C;AACnE,MAAI,YAAY,QAAQ,aAAa,EAAE,SAAS,EAAG,QAAO;AAC1D,SAAO,gBAAgB,OAAO,EAAE;AAAA,IAC9B,CAAC,SAAS,MAAM,QAAQ,KAAK,aAAa,KAAK,KAAK,cAAc,SAAS;AAAA,EAC7E;AACF;AAEA,SAAS,gBAAgB,SAAkE;AACzF,SAAO;AAAA,IACL,GAAG,QAAQ,QAAQ,OAAO;AAAA,IAC1B,GAAG,QAAQ,QAAQ,eAAe;AAAA,IAClC,GAAG,QAAQ,QAAQ,KAAK;AAAA,EAC1B;AACF;AAEA,SAAS,QAAQ,OAAgD;AAC/D,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,SAAO,MAAM;AAAA,IACX,CAAC,SACC,QAAQ,IAAI,KAAK,OAAO,SAAS,YAAY,CAAC,MAAM,QAAQ,IAAI;AAAA,EACpE;AACF;AAEA,SAAS,YAAY,OAA0B;AAC7C,SAAO,MAAM,QAAQ,KAAK,IACtB,MAAM,OAAO,CAAC,SAAyB,OAAO,SAAS,QAAQ,IAC/D,CAAC;AACP;AAGO,SAAS,gBACd,KACA,QAAuB,eACA;AACvB,MAAI,IAAI,IAAI,SAAS,SAAS,SAAS,IAAI,IAAI,WAAW,aAAa;AACrE,WAAO,EAAE,cAAc,WAAW,QAAQ,qDAAqD;AAAA,EACjG;AACA,aAAW,QAAQ,OAAO;AACxB,UAAM,MAAM,KAAK,MAAM,GAAG;AAC1B,QAAI,IAAK,QAAO;AAAA,EAClB;AACA,SAAO,EAAE,cAAc,WAAW,QAAQ,sDAAsD;AAClG;;;ACvaA,eAAsB,sBACpB,OACA,OACA,UAA0B,CAAC,GACF;AACzB,QAAM,QAAQ,MAAM,UAAU,OAAO,KAAK;AAC1C,MAAI,MAAM,WAAW,GAAG;AACtB,WAAO,EAAE,OAAO,YAAY,GAAG,QAAQ,CAAC,GAAG,WAAW,GAAG,eAAe,GAAG,WAAW,EAAE;AAAA,EAC1F;AAEA,QAAM,SAAoC,CAAC;AAC3C,MAAI,cAAc;AAClB,MAAI,kBAAkB;AACtB,QAAM,cAAc,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AACvE,QAAM,iBAAiB,oBAAI,IAAY;AAGvC,aAAW,KAAK,aAAa;AAC3B,WAAO,EAAE,QAAQ,MAAM,EAAE,OAAO,GAAG,QAAQ,GAAG,cAAc,GAAG,YAAY,EAAE;AAC7E,UAAM,OAAO,OAAO,EAAE,QAAQ;AAC9B,SAAK,SAAS;AACd,QAAI,EAAE,WAAW,SAAS;AACxB,WAAK,UAAU;AACf,qBAAe;AAAA,IACjB;AACA,QAAI,OAAO,EAAE,cAAc,SAAU,MAAK,gBAAgB,EAAE;AAC5D,UAAM,MAAM,GAAG,EAAE,QAAQ,IAAI,QAAQ,EAAE,IAAI,CAAC;AAC5C,QAAI,eAAe,IAAI,GAAG,GAAG;AAC3B,WAAK,cAAc;AACnB,yBAAmB;AAAA,IACrB;AACA,mBAAe,IAAI,GAAG;AAAA,EACxB;AAEA,aAAW,QAAQ,OAAO,OAAO,MAAM,GAAG;AACxC,SAAK,eAAe,KAAK,QAAQ,IAAI,KAAK,eAAe,KAAK,QAAQ;AAAA,EACxE;AAGA,MAAI,qBAAqB;AACzB,MAAI,kBAAkB;AACtB,aAAW,CAAC,EAAE,GAAG,KAAK,QAAQ,aAAa,CAAC,MAAM,EAAE,QAAQ,GAAG;AAC7D,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,UAAI,IAAI,CAAC,EAAG,WAAW,QAAS;AAChC,4BAAsB;AACtB,UAAI,IAAI,IAAI,CAAC,EAAG,oBAAmB;AAAA,IACrC;AAAA,EACF;AACA,QAAM,YAAY,qBAAqB,IAAI,kBAAkB,qBAAqB;AAElF,MAAI;AACJ,MAAI,QAAQ,iBAAiB;AAC3B,UAAM,UAAU,YAAY,OAAO,CAAC,MAAM,EAAE,UAAU,QAAQ,eAAgB;AAC9E,QAAI,QAAQ,SAAS,GAAG;AACtB,0BACE,QAAQ,OAAO,CAAC,MAAM,QAAQ,gBAAiB,EAAE,MAAM,CAAC,EAAE,SAAS,QAAQ;AAAA,IAC/E;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,YAAY,YAAY;AAAA,IACxB;AAAA,IACA,WAAW,cAAc,YAAY;AAAA,IACrC,eAAe,kBAAkB,YAAY;AAAA,IAC7C;AAAA,IACA;AAAA,EACF;AACF;;;AC1EA,eAAsB,gBAAgB,OAAmB,OAAoC;AAC3F,QAAM,QAAQ,MAAM,MAAM,MAAM,EAAE,MAAM,CAAC;AACzC,QAAM,SAAS,MAAM,MAAM,OAAO,EAAE,MAAM,CAAC;AAC3C,QAAM,aAAa,oBAAI,IAAgC;AACvD,aAAW,KAAK,OAAO;AACrB,UAAM,MAAM,WAAW,IAAI,EAAE,YAAY,KAAK,CAAC;AAC/C,QAAI,KAAK,CAAC;AACV,eAAW,IAAI,EAAE,cAAc,GAAG;AAAA,EACpC;AAEA,aAAW,OAAO,WAAW,OAAO,EAAG,KAAI,KAAK,CAAC,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;AAEnF,QAAM,eAAe,oBAAI,IAA0B;AACnD,aAAW,KAAK,QAAQ;AACtB,QAAI,CAAC,EAAE,OAAQ;AACf,UAAM,MAAM,aAAa,IAAI,EAAE,MAAM,KAAK,CAAC;AAC3C,QAAI,KAAK,CAAC;AACV,iBAAa,IAAI,EAAE,QAAQ,GAAG;AAAA,EAChC;AAEA,QAAM,QAA0B,CAAC;AACjC,QAAM,OAAO,CAAC,QAA4B,UAAwB;AAChE,UAAM,OAAO,WAAW,IAAI,MAAM,KAAK,CAAC;AACxC,eAAW,SAAS,MAAM;AACxB,YAAM,KAAK;AAAA,QACT,OAAO,MAAM;AAAA,QACb,MAAM;AAAA,QACN;AAAA,QACA,QAAQ,aAAa,IAAI,MAAM,MAAM,KAAK,CAAC;AAAA,MAC7C,CAAC;AACD,WAAK,MAAM,QAAQ,QAAQ,CAAC;AAAA,IAC9B;AAAA,EACF;AACA,OAAK,QAAW,CAAC;AAEjB,QAAM,WAAW,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,SAAS,KAAK,EAAE;AAC5D,QAAM,YAAY,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,SAAS,MAAM,EAAE;AAC9D,QAAM,gBAAgB,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,SAAS,OAAO,EAAE;AACnE,QAAM,aAAa,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,SAAS,WAAW,EAAE;AAEpE,MAAI,kBAAkB;AACtB,MAAI,MAAM,SAAS,GAAG;AACpB,UAAM,SAAS,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS;AAC3C,UAAM,OAAO,MAAM,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS;AACtD,sBAAkB,KAAK,IAAI,GAAG,IAAI,IAAI,KAAK,IAAI,GAAG,MAAM;AAAA,EAC1D;AAEA,SAAO,EAAE,OAAO,OAAO,UAAU,WAAW,eAAe,YAAY,gBAAgB;AACzF;;;ACfO,SAAS,kBACd,SACA,UAA2B,CAAC,GACZ;AAChB,QAAM,kBAAkB,QAAQ,mBAAmB;AACnD,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,uBAAuB;AAEnD,QAAM,UAA2B,QAAQ,IAAI,CAAC,MAAM;AAClD,QAAI,EAAE,SAAS,SAAS,KAAK,EAAE,UAAU,SAAS,GAAG;AACnD,YAAM,IAAI,MAAM,yDAAoD,EAAE,MAAM,GAAG;AAAA,IACjF;AACA,UAAM,QAAQ,KAAK,EAAE,QAAQ;AAC7B,UAAM,QAAQ,KAAK,EAAE,SAAS;AAC9B,UAAM,QAAQ,QAAQ;AACtB,UAAM,IAAI,QAAQ,EAAE,UAAU,EAAE,SAAS;AACzC,UAAM,EAAE,GAAG,IAAI,EAAE,IAAI,YAAY,EAAE,UAAU,EAAE,SAAS;AAKxD,UAAM,cAAc,IAAI,EAAE,QAAQ;AAClC,UAAM,eAAe,IAAI,EAAE,SAAS;AACpC,UAAM,iBAAiB,cAAc,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,KAAK;AACxE,UAAM,kBAAkB,eAAe,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,IAAI,KAAK;AAC1E,UAAM,SAAS,kBAAkB;AACjC,UAAM,cAAc,KAAK,IAAI,aAAa,YAAY;AAEtD,QAAI;AACJ,QAAI,CAAC,QAAQ;AACX,gBAAU;AAAA,IACZ,WAAW,IAAI,SAAS,KAAK,IAAI,CAAC,KAAK,iBAAiB;AACtD,YAAM,oBAAoB,EAAE,iBAAiB,QAAQ,IAAI,QAAQ;AACjE,gBAAU,oBAAoB,aAAa;AAAA,IAC7C,OAAO;AACL,gBAAU;AAAA,IACZ;AAEA,WAAO;AAAA,MACL,QAAQ,EAAE;AAAA,MACV,cAAc;AAAA,MACd,eAAe;AAAA,MACf;AAAA,MACA,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,SAAS;AAAA,MACT,QAAQ;AAAA,MACR;AAAA,MACA,KAAK;AAAA,MACL;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO;AAAA,IACL;AAAA,IACA,eAAe,QAAQ,KAAK,CAAC,MAAM,EAAE,YAAY,WAAW;AAAA,IAC5D,aAAa,QAAQ,KAAK,CAAC,MAAM,EAAE,YAAY,UAAU;AAAA,EAC3D;AACF;AAEA,SAAS,KAAK,IAAsB;AAClC,SAAO,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;AAGO,SAAS,IAAI,IAAsB;AACxC,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,QAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,QAAM,IAAI,CAAC,MAAc;AACvB,UAAM,MAAM,KAAK,OAAO,SAAS;AACjC,UAAM,KAAK,KAAK,MAAM,GAAG;AACzB,UAAM,KAAK,KAAK,KAAK,GAAG;AACxB,WAAO,OAAO,EAAE,KAAM,OAAO,EAAE,IAAK,OAAO,EAAE,MAAO,MAAM;AAAA,EAC5D;AACA,SAAO,EAAE,IAAI,IAAI,EAAE,IAAI;AACzB;AAOO,SAAS,YAAY,GAAa,GAAmD;AAC1F,MAAI,EAAE,SAAS,KAAK,EAAE,SAAS,EAAG,QAAO,EAAE,GAAG,GAAG,IAAI,GAAG,GAAG,EAAE;AAC7D,QAAM,KAAK,KAAK,CAAC;AACjB,QAAM,KAAK,KAAK,CAAC;AACjB,QAAM,KAAK,SAAS,GAAG,EAAE;AACzB,QAAM,KAAK,SAAS,GAAG,EAAE;AACzB,QAAM,YAAY,KAAK,EAAE,SAAS,KAAK,EAAE;AACzC,MAAI,cAAc,EAAG,QAAO,EAAE,GAAG,OAAO,KAAK,IAAI,UAAU,IAAI,GAAG,GAAG,OAAO,KAAK,IAAI,EAAE;AACvF,QAAM,KAAK,KAAK,MAAM,KAAK,KAAK,SAAS;AACzC,QAAM,KACH,YAAY,cACX,KAAK,EAAE,WAAW,KAAK,EAAE,SAAS,MAAM,KAAK,EAAE,WAAW,KAAK,EAAE,SAAS;AAC9E,QAAM,IAAI,KAAK,IAAI,YAAY,KAAK,IAAI,CAAC,GAAG,EAAE;AAC9C,SAAO,EAAE,GAAG,IAAI,EAAE;AACpB;AAEA,SAAS,SAAS,IAAc,GAAmB;AACjD,SAAO,GAAG,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,KAAK,GAAG,SAAS;AACrE;AAGA,SAAS,YAAY,GAAW,IAAoB;AAClD,MAAI,MAAM,EAAG,QAAO;AACpB,MAAI,KAAK,IAAK,QAAO,UAAU,CAAC;AAChC,QAAM,IAAI,MAAM,KAAK,IAAI;AACzB,QAAM,KAAK,eAAe,GAAG,KAAK,GAAG,GAAG;AACxC,SAAO,KAAK,IAAI,IAAI,MAAM,KAAK,MAAM;AACvC;AAEA,SAAS,eAAe,GAAW,GAAW,GAAmB;AAC/D,MAAI,KAAK,EAAG,QAAO;AACnB,MAAI,KAAK,EAAG,QAAO;AACnB,QAAM,SAAS,QAAQ,CAAC,IAAI,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC;AACtD,QAAM,QAAQ,KAAK,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,MAAM,IAAI;AACzE,MAAI,IAAI;AACR,MAAI,IAAI,KAAM,IAAI,KAAK,KAAM,IAAI;AACjC,MAAI,KAAK,IAAI,CAAC,IAAI,MAAO,KAAI;AAC7B,MAAI,IAAI;AACR,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,KAAK,KAAK,KAAK;AAC7B,UAAM,KAAK,IAAI;AACf,QAAI,MAAO,KAAK,IAAI,KAAK,MAAO,IAAI,KAAK,MAAM,IAAI;AACnD,QAAI,IAAI,MAAM;AACd,QAAI,KAAK,IAAI,CAAC,IAAI,MAAO,KAAI;AAC7B,QAAI,IAAI,MAAM;AACd,QAAI,KAAK,IAAI,CAAC,IAAI,MAAO,KAAI;AAC7B,QAAI,IAAI;AACR,SAAK,IAAI;AACT,UAAM,GAAG,IAAI,MAAM,IAAI,IAAI,KAAK,OAAO,IAAI,OAAO,IAAI,KAAK;AAC3D,QAAI,IAAI,MAAM;AACd,QAAI,KAAK,IAAI,CAAC,IAAI,MAAO,KAAI;AAC7B,QAAI,IAAI,MAAM;AACd,QAAI,KAAK,IAAI,CAAC,IAAI,MAAO,KAAI;AAC7B,QAAI,IAAI;AACR,UAAM,QAAQ,IAAI;AAClB,SAAK;AACL,QAAI,KAAK,IAAI,QAAQ,CAAC,IAAI,KAAM;AAAA,EAClC;AACA,SAAO,QAAQ;AACjB;AAEA,SAAS,QAAQ,GAAmB;AAClC,QAAM,QAAQ;AAAA,IACZ;AAAA,IAAqB;AAAA,IAAmB;AAAA,IAAqB;AAAA,IAC7D;AAAA,IAAqB;AAAA,IAAoB;AAAA,IAAsB;AAAA,IAC/D;AAAA,EACF;AACA,MAAI,IAAI,IAAK,QAAO,KAAK,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,QAAQ,IAAI,CAAC;AAC7E,OAAK;AACL,MAAI,IAAI,MAAM,CAAC;AACf,WAAS,IAAI,GAAG,IAAI,GAAG,IAAK,MAAK,MAAM,CAAC,KAAM,IAAI;AAClD,QAAM,IAAI,IAAI;AACd,SAAO,MAAM,KAAK,IAAI,IAAI,KAAK,EAAE,KAAK,IAAI,OAAO,KAAK,IAAI,CAAC,IAAI,IAAI,KAAK,IAAI,CAAC;AAC/E;AAEA,SAAS,UAAU,GAAmB;AACpC,QAAM,KAAK;AACX,QAAM,KAAK;AACX,QAAM,KAAK;AACX,QAAM,KAAK;AACX,QAAM,KAAK;AACX,QAAM,IAAI;AACV,QAAM,OAAO,IAAI,IAAI,KAAK;AAC1B,QAAM,OAAO,KAAK,IAAI,CAAC;AACvB,QAAM,IAAI,KAAK,IAAI,IAAI;AACvB,QAAM,IAAI,QAAQ,KAAK,IAAI,MAAM,IAAI,MAAM,IAAI,MAAM,IAAI,MAAM,IAAI,KAAK,IAAK,CAAC,OAAO,OAAQ,CAAC;AAC9F,SAAO,OAAO,IAAI,OAAO;AAC3B;","names":[]}
|
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
} from "./chunk-XMSYF4A7.js";
|
|
16
16
|
import {
|
|
17
17
|
llmSpans
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-MHNQWM4I.js";
|
|
19
19
|
|
|
20
20
|
// src/contamination-guard.ts
|
|
21
21
|
function checkCanaries(output, scenarios) {
|
|
@@ -822,4 +822,4 @@ export {
|
|
|
822
822
|
HoldoutAuditor,
|
|
823
823
|
analyzeRuns
|
|
824
824
|
};
|
|
825
|
-
//# sourceMappingURL=chunk-
|
|
825
|
+
//# sourceMappingURL=chunk-QFGTU7MT.js.map
|
|
@@ -20,10 +20,6 @@ import {
|
|
|
20
20
|
spanEpochMillis,
|
|
21
21
|
traceSpanKindToOpenInferenceKind
|
|
22
22
|
} from "./chunk-LNQEP766.js";
|
|
23
|
-
import {
|
|
24
|
-
defaultProviderRedactor,
|
|
25
|
-
providerFromBaseUrl
|
|
26
|
-
} from "./chunk-PC4UYEBM.js";
|
|
27
23
|
import {
|
|
28
24
|
validateRunRecord
|
|
29
25
|
} from "./chunk-VK6HBGAE.js";
|
|
@@ -31,6 +27,10 @@ import {
|
|
|
31
27
|
canonicalize,
|
|
32
28
|
hashJson
|
|
33
29
|
} from "./chunk-VSMTAMNK.js";
|
|
30
|
+
import {
|
|
31
|
+
defaultProviderRedactor,
|
|
32
|
+
providerFromBaseUrl
|
|
33
|
+
} from "./chunk-PC4UYEBM.js";
|
|
34
34
|
import {
|
|
35
35
|
ReplayError
|
|
36
36
|
} from "./chunk-ONWEPEDO.js";
|
|
@@ -1954,4 +1954,4 @@ export {
|
|
|
1954
1954
|
createReplayFetch,
|
|
1955
1955
|
iterateRawCalls
|
|
1956
1956
|
};
|
|
1957
|
-
//# sourceMappingURL=chunk-
|
|
1957
|
+
//# sourceMappingURL=chunk-RSVSSZKF.js.map
|
|
@@ -1,17 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
LockedJsonlAppender
|
|
3
|
-
} from "./chunk-DJWX3GVS.js";
|
|
4
1
|
import {
|
|
5
2
|
aggregateRunScore,
|
|
6
3
|
clamp01
|
|
7
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-2IY4ILP4.js";
|
|
5
|
+
import {
|
|
6
|
+
computeFindingId
|
|
7
|
+
} from "./chunk-7NX6ZSBG.js";
|
|
8
8
|
import {
|
|
9
9
|
LlmClient,
|
|
10
10
|
callLlmJson
|
|
11
11
|
} from "./chunk-FUCQVFMU.js";
|
|
12
|
-
import {
|
|
13
|
-
computeFindingId
|
|
14
|
-
} from "./chunk-45EEMHTC.js";
|
|
15
12
|
import {
|
|
16
13
|
NotFoundError
|
|
17
14
|
} from "./chunk-ONWEPEDO.js";
|
|
@@ -114,8 +111,80 @@ function resolveModel(req, defaultModel) {
|
|
|
114
111
|
return { ...req, model: defaultModel };
|
|
115
112
|
}
|
|
116
113
|
|
|
114
|
+
// src/concurrency.ts
|
|
115
|
+
var Mutex = class {
|
|
116
|
+
locked = false;
|
|
117
|
+
waiters = [];
|
|
118
|
+
async acquire() {
|
|
119
|
+
if (!this.locked) {
|
|
120
|
+
this.locked = true;
|
|
121
|
+
return () => this.release();
|
|
122
|
+
}
|
|
123
|
+
return new Promise((resolve) => {
|
|
124
|
+
this.waiters.push(() => {
|
|
125
|
+
resolve(() => this.release());
|
|
126
|
+
});
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
release() {
|
|
130
|
+
const next = this.waiters.shift();
|
|
131
|
+
if (next) {
|
|
132
|
+
next();
|
|
133
|
+
} else {
|
|
134
|
+
this.locked = false;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
async runExclusive(fn) {
|
|
138
|
+
const release = await this.acquire();
|
|
139
|
+
try {
|
|
140
|
+
return await fn();
|
|
141
|
+
} finally {
|
|
142
|
+
release();
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/** True iff someone holds the lock right now. Diagnostics only. */
|
|
146
|
+
get isLocked() {
|
|
147
|
+
return this.locked;
|
|
148
|
+
}
|
|
149
|
+
/** Pending waiter count. Diagnostics only. */
|
|
150
|
+
get pending() {
|
|
151
|
+
return this.waiters.length;
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
// src/locked-jsonl-appender.ts
|
|
156
|
+
import { appendFileSync, existsSync, mkdirSync } from "fs";
|
|
157
|
+
import { dirname } from "path";
|
|
158
|
+
var mutexes = /* @__PURE__ */ new Map();
|
|
159
|
+
function getMutex(path) {
|
|
160
|
+
let m = mutexes.get(path);
|
|
161
|
+
if (!m) {
|
|
162
|
+
m = new Mutex();
|
|
163
|
+
mutexes.set(path, m);
|
|
164
|
+
}
|
|
165
|
+
return m;
|
|
166
|
+
}
|
|
167
|
+
var LockedJsonlAppender = class {
|
|
168
|
+
constructor(path) {
|
|
169
|
+
this.path = path;
|
|
170
|
+
this.mutex = getMutex(path);
|
|
171
|
+
if (!existsSync(dirname(path))) {
|
|
172
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
path;
|
|
176
|
+
mutex;
|
|
177
|
+
async append(entry) {
|
|
178
|
+
const line = `${JSON.stringify(entry)}
|
|
179
|
+
`;
|
|
180
|
+
await this.mutex.runExclusive(() => {
|
|
181
|
+
appendFileSync(this.path, line);
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
|
|
117
186
|
// src/analyst/findings-store.ts
|
|
118
|
-
import { existsSync, readFileSync } from "fs";
|
|
187
|
+
import { existsSync as existsSync2, readFileSync } from "fs";
|
|
119
188
|
var FindingsStore = class {
|
|
120
189
|
constructor(path) {
|
|
121
190
|
this.path = path;
|
|
@@ -131,7 +200,7 @@ var FindingsStore = class {
|
|
|
131
200
|
}
|
|
132
201
|
/** Load every persisted finding. Discards malformed trailing lines silently. */
|
|
133
202
|
loadAll() {
|
|
134
|
-
if (!
|
|
203
|
+
if (!existsSync2(this.path)) return [];
|
|
135
204
|
const raw = readFileSync(this.path, "utf8");
|
|
136
205
|
if (!raw) return [];
|
|
137
206
|
const out = [];
|
|
@@ -182,23 +251,23 @@ function diffFindings(previous, current, policy = {}) {
|
|
|
182
251
|
}
|
|
183
252
|
|
|
184
253
|
// src/analyst/kinds/skill-usage.ts
|
|
185
|
-
import { existsSync as
|
|
254
|
+
import { existsSync as existsSync3, readdirSync, readFileSync as readFileSync2, statSync } from "fs";
|
|
186
255
|
import { join } from "path";
|
|
187
256
|
var BLOAT_LINE_THRESHOLD = 300;
|
|
188
257
|
var TANGLE_PRIVATE_RE = /\b(cli-bridge|tangletools|ops-board|drew-gtr-pro|@tangle-network\/|~\/company|tangle\.tools|gtm-agent)\b|\bkimi\b|\btcloud\b/gi;
|
|
189
258
|
var TRIGGER_RE = /triggers?\s*[:-]/i;
|
|
190
259
|
function listSkillDirs(root) {
|
|
191
|
-
if (!
|
|
260
|
+
if (!existsSync3(root)) return [];
|
|
192
261
|
const out = [];
|
|
193
262
|
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
|
194
263
|
if (!entry.isDirectory() && !entry.isSymbolicLink()) continue;
|
|
195
264
|
const skillMd = join(root, entry.name, "SKILL.md");
|
|
196
|
-
if (
|
|
265
|
+
if (existsSync3(skillMd)) out.push({ name: entry.name, path: skillMd });
|
|
197
266
|
}
|
|
198
267
|
return out;
|
|
199
268
|
}
|
|
200
269
|
function walkJsonl(dir, cap) {
|
|
201
|
-
if (!
|
|
270
|
+
if (!existsSync3(dir)) return [];
|
|
202
271
|
const files = [];
|
|
203
272
|
const stack = [dir];
|
|
204
273
|
while (stack.length) {
|
|
@@ -231,7 +300,7 @@ function countArtifacts(roots, name, aliases) {
|
|
|
231
300
|
for (const root of roots) {
|
|
232
301
|
const candidates = [join(root, ".evolve", name), ...aliases.map((a) => join(root, a))];
|
|
233
302
|
for (const dir of candidates) {
|
|
234
|
-
if (!
|
|
303
|
+
if (!existsSync3(dir)) continue;
|
|
235
304
|
try {
|
|
236
305
|
if (statSync(dir).isDirectory()) n += readdirSync(dir).length;
|
|
237
306
|
else n += 1;
|
|
@@ -308,8 +377,8 @@ function buildSkillUsageReport(config) {
|
|
|
308
377
|
config.artifactAliases?.[s.name] ?? []
|
|
309
378
|
),
|
|
310
379
|
tanglePrivateRefs: (body.match(TANGLE_PRIVATE_RE) ?? []).length,
|
|
311
|
-
hasReferencesDir:
|
|
312
|
-
hasEvalsDir:
|
|
380
|
+
hasReferencesDir: existsSync3(join(dir, "references")),
|
|
381
|
+
hasEvalsDir: existsSync3(join(dir, "evals")),
|
|
313
382
|
logsRuns: body.includes("skill-runs.jsonl"),
|
|
314
383
|
hasTriggerPhrases: TRIGGER_RE.test(frontmatterDescription(body) || body.slice(0, 600))
|
|
315
384
|
};
|
|
@@ -777,6 +846,8 @@ function createSemanticConceptJudge(options = {}) {
|
|
|
777
846
|
export {
|
|
778
847
|
createAnalystAi,
|
|
779
848
|
createChatClient,
|
|
849
|
+
Mutex,
|
|
850
|
+
LockedJsonlAppender,
|
|
780
851
|
FindingsStore,
|
|
781
852
|
defaultIsMaterial,
|
|
782
853
|
diffFindings,
|
|
@@ -790,4 +861,4 @@ export {
|
|
|
790
861
|
runSemanticConceptJudge,
|
|
791
862
|
createSemanticConceptJudge
|
|
792
863
|
};
|
|
793
|
-
//# sourceMappingURL=chunk-
|
|
864
|
+
//# sourceMappingURL=chunk-XRGOKCMO.js.map
|