@tangle-network/agent-eval 0.117.0 → 0.118.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/analyst/index.d.ts +2772 -21
  3. package/dist/analyst/index.js +7 -6
  4. package/dist/analyst/index.js.map +1 -1
  5. package/dist/belief-state/index.d.ts +706 -10
  6. package/dist/belief-state/index.js +2 -1
  7. package/dist/belief-state/index.js.map +1 -1
  8. package/dist/benchmarks/index.d.ts +958 -14
  9. package/dist/benchmarks/index.js +12 -10
  10. package/dist/builder-eval/index.d.ts +449 -4
  11. package/dist/builder-eval/index.js +4 -3
  12. package/dist/builder-eval/index.js.map +1 -1
  13. package/dist/campaign/index.d.ts +4276 -74
  14. package/dist/campaign/index.js +12 -10
  15. package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
  16. package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
  17. package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
  18. package/dist/{chunk-YZPO4UHR.js → chunk-DP3BBMYJ.js} +100 -143
  19. package/dist/chunk-DP3BBMYJ.js.map +1 -0
  20. package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
  21. package/dist/chunk-HKUCJ437.js.map +1 -0
  22. package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
  23. package/dist/chunk-KSDQVPLR.js +286 -0
  24. package/dist/chunk-KSDQVPLR.js.map +1 -0
  25. package/dist/chunk-MA6HLL3S.js +65 -0
  26. package/dist/chunk-MA6HLL3S.js.map +1 -0
  27. package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
  28. package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
  29. package/dist/chunk-PXE2VKMX.js +140 -0
  30. package/dist/chunk-PXE2VKMX.js.map +1 -0
  31. package/dist/{chunk-ZUXV7UWZ.js → chunk-Q442S5AS.js} +73 -19
  32. package/dist/{chunk-ZUXV7UWZ.js.map → chunk-Q442S5AS.js.map} +1 -1
  33. package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
  34. package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
  35. package/dist/chunk-QKEGNI5B.js.map +1 -0
  36. package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
  37. package/dist/chunk-S3UZOQ5Y.js.map +1 -0
  38. package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
  39. package/dist/chunk-SVH2ANFD.js.map +1 -0
  40. package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
  41. package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
  42. package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
  43. package/dist/chunk-WDHBCA3M.js +31 -0
  44. package/dist/chunk-WDHBCA3M.js.map +1 -0
  45. package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
  46. package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
  47. package/dist/chunk-ZET2UAYW.js.map +1 -0
  48. package/dist/contract/index.d.ts +4012 -38
  49. package/dist/contract/index.js +56 -29
  50. package/dist/contract/index.js.map +1 -1
  51. package/dist/control.d.ts +1013 -9
  52. package/dist/control.js +4 -3
  53. package/dist/fuzz.d.ts +194 -4
  54. package/dist/fuzz.js +3 -3
  55. package/dist/hosted/index.d.ts +498 -17
  56. package/dist/index.d.ts +10982 -1286
  57. package/dist/index.js +97 -80
  58. package/dist/index.js.map +1 -1
  59. package/dist/matrix/index.d.ts +139 -4
  60. package/dist/meta-eval/index.d.ts +862 -15
  61. package/dist/meta-eval/index.js +2 -1
  62. package/dist/meta-eval/index.js.map +1 -1
  63. package/dist/multishot/index.d.ts +214 -14
  64. package/dist/openapi.json +1 -1
  65. package/dist/pipelines/index.d.ts +392 -7
  66. package/dist/pipelines/index.js +5 -3
  67. package/dist/pipelines/index.js.map +1 -1
  68. package/dist/reporting.d.ts +1277 -17
  69. package/dist/rl.d.ts +2359 -28
  70. package/dist/rl.js +6 -5
  71. package/dist/rl.js.map +1 -1
  72. package/dist/storyboard/index.d.ts +86 -1
  73. package/dist/trace-attributes.d.ts +16 -0
  74. package/dist/trace-attributes.js +32 -0
  75. package/dist/trace-attributes.js.map +1 -0
  76. package/dist/traces.d.ts +1970 -697
  77. package/dist/traces.js +49 -32
  78. package/dist/wire/index.d.ts +655 -9
  79. package/docs/insight-report.md +44 -0
  80. package/package.json +9 -3
  81. package/dist/adversarial-B7loGVVX.d.ts +0 -19
  82. package/dist/analyst-C8HHvfJp.d.ts +0 -88
  83. package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
  84. package/dist/baseline-DKq3gJpP.d.ts +0 -141
  85. package/dist/calibration-C8MTS7cw.d.ts +0 -101
  86. package/dist/chunk-5UF54T55.js.map +0 -1
  87. package/dist/chunk-E4BUPP7Z.js.map +0 -1
  88. package/dist/chunk-FQNLDL4D.js.map +0 -1
  89. package/dist/chunk-LQUTGLOZ.js.map +0 -1
  90. package/dist/chunk-S2F4J57L.js.map +0 -1
  91. package/dist/chunk-YZPO4UHR.js.map +0 -1
  92. package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
  93. package/dist/control-6vuGfmDH.d.ts +0 -258
  94. package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
  95. package/dist/dataset-NENEzRgk.d.ts +0 -115
  96. package/dist/default-registry-DaK8b3fv.d.ts +0 -155
  97. package/dist/emitter-CjD7vUwv.d.ts +0 -122
  98. package/dist/errors-oeQrLqXC.d.ts +0 -74
  99. package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
  100. package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
  101. package/dist/gepa-eESocoDi.d.ts +0 -642
  102. package/dist/index-PdX4VnPA.d.ts +0 -423
  103. package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
  104. package/dist/integrity-DqlBiLyK.d.ts +0 -81
  105. package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
  106. package/dist/kind-factory-ClZmO25A.d.ts +0 -171
  107. package/dist/llm-client-qoDd18Qz.d.ts +0 -289
  108. package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
  109. package/dist/off-policy-DiwuKKg7.d.ts +0 -132
  110. package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
  111. package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
  112. package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
  113. package/dist/provenance-DpjwyseI.d.ts +0 -541
  114. package/dist/query-CF7PG61p.d.ts +0 -35
  115. package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
  116. package/dist/release-report-C8G2i5Xi.d.ts +0 -236
  117. package/dist/researcher-C8XyxQsu.d.ts +0 -387
  118. package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
  119. package/dist/run-record-BDH49H2E.d.ts +0 -360
  120. package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
  121. package/dist/schema-B3Q3l9Z_.d.ts +0 -201
  122. package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
  123. package/dist/sequential-5iSVfzl2.d.ts +0 -139
  124. package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
  125. package/dist/statistics-KUnG73jH.d.ts +0 -494
  126. package/dist/storage-DrX3v_5B.d.ts +0 -50
  127. package/dist/store-C1YxJDEK.d.ts +0 -248
  128. package/dist/store-DGqD0Pyo.d.ts +0 -116
  129. package/dist/summary-report-C5bKFfm-.d.ts +0 -445
  130. package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
  131. package/dist/types-BSw1rOUB.d.ts +0 -634
  132. package/dist/types-BUxNaJ8c.d.ts +0 -108
  133. package/dist/types-BkfcQnxV.d.ts +0 -313
  134. package/dist/verdict-C9MlYujm.d.ts +0 -35
  135. /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
  136. /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
  137. /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
  138. /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
  139. /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
  140. /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
  141. /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
  142. /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
  143. /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
@@ -1,24 +1,1839 @@
1
- import { M as MultiLayerVerifier, V as VerifyOptions, S as Severity } from '../multi-layer-verifier-BsqKuLyN.js';
2
- import { R as RunCritic, S as SemanticConceptJudgeOptions, a as RunTrace, b as SemanticConceptJudgeInput, B as BehavioralMetrics } from '../semantic-concept-judge-CXnPEJbf.js';
3
- export { C as CreateAnalystAiConfig, D as DEFAULT_TRACE_ANALYST_KINDS, c as DiffPolicy, F as FAILURE_MODE_KIND_SPEC, d as FINDING_SUBJECT_GRAMMAR_PROMPT, e as FINDING_SUBJECT_KINDS, f as FINDING_SUBJECT_SYNTAX, g as FindingSubject, h as FindingSubjectKind, i as FindingSubjectStringSchema, j as FindingsDiff, k as FindingsStore, I as IMPROVEMENT_KIND_SPEC, K as KIND_EXPECTED_SUBJECTS, l as KNOWLEDGE_GAP_KIND_SPEC, m as KNOWLEDGE_POISONING_KIND_SPEC, P as PersistedFinding, n as SKILL_USAGE_ANALYST, o as SkillUsageAnalyst, p as SkillUsageRecord, q as SkillUsageReport, r as SkillUsageScanConfig, s as buildSkillUsageReport, t as createAnalystAi, u as defaultIsMaterial, v as diffFindings, w as emitSkillUsageFindings, x as findingSubjectGrammarPromptFor, y as parseFindingSubject, z as renderFindingSubject } from '../semantic-concept-judge-CXnPEJbf.js';
4
- import { b as JudgeFn, a as JudgeInput } from '../types-BkfcQnxV.js';
5
- import { A as Analyst, h as AnalystSeverity, c as AnalystFinding } from '../policy-edit-wG9uFEFm.js';
6
- export { a as AnalystContext, g as AnalystCost, i as AnalystInputKind, j as AnalystRequirements, f as AnalystRunEvent, e as AnalystRunInputs, d as AnalystRunResult, b as AnalystRunSummary, k as ChatCallOpts, C as ChatClient, l as ChatRequest, m as ChatResponse, n as ChatTransport, o as CliBridgeTransportOpts, p as CreateChatClientOpts, D as DirectProviderTransportOpts, E as EvidenceRef, F as FindingToPolicyEditOptions, M as MockTransportOpts, q as POLICY_EDIT_AXES, r as POLICY_EDIT_CANDIDATE_RECORD_SCHEMA, s as POLICY_EDIT_TARGET_SURFACES, t as PolicyEdit, u as PolicyEditAdmission, v as PolicyEditAdmissionOptions, w as PolicyEditAxis, P as PolicyEditCandidateRecord, x as PolicyEditChange, y as PolicyEditExpectedGain, z as PolicyEditGainDirection, B as PolicyEditGainUnit, G as PolicyEditInit, H as PolicyEditRisk, I as PolicyEditSchemaVersion, J as PolicyEditSource, K as PolicyEditTarget, L as PolicyEditTargetSurface, N as PolicyEditValidationError, R as RouterTransportOpts, S as SandboxSdkTransportOpts, O as admitPolicyEdit, Q as applyPolicyEditToSurface, T as computeFindingId, U as computePolicyEditId, V as createChatClient, W as isPolicyEdit, X as makeFinding, Y as makePolicyEdit, Z as makePolicyEditCandidateRecord, _ as policyEditFromFinding, $ as policyEditsFromFindings, a0 as scorePolicyEditReadiness, a1 as validatePolicyEdit, a2 as validatePolicyEditCandidateRecord } from '../policy-edit-wG9uFEFm.js';
7
1
  import { TCloud } from '@tangle-network/tcloud';
8
- import { T as TraceAnalysisStore } from '../store-C1YxJDEK.js';
9
- export { a as AnalystHooks, A as AnalystRegistry, b as AnalystRegistryOptions, B as BudgetPolicy, D as DefaultAnalystRegistryOptions, R as RegistryRunOpts, c as buildDefaultAnalystRegistry } from '../default-registry-DaK8b3fv.js';
10
- export { A as ANALYST_SEVERITIES, C as CreateTraceAnalystKindOpts, R as RAW_FINDING_SCHEMA_PROMPT, a as RawAnalystFinding, b as RawAnalystFindingSchema, c as TraceAnalystGolden, T as TraceAnalystKindSpec, d as createTraceAnalystKind, p as parseRawFinding, r as renderPriorFindings } from '../kind-factory-ClZmO25A.js';
11
- import { C as CostLedger } from '../cost-ledger-DWy3XdJc.js';
12
- import { a as LlmClientOptions } from '../llm-client-qoDd18Qz.js';
13
- import { AxFunction } from '@ax-llm/ax';
14
- import '../verdict-C9MlYujm.js';
15
- import 'zod';
16
- import '../schema-B3Q3l9Z_.js';
17
- import '../store-DGqD0Pyo.js';
18
- import '../run-record-BDH49H2E.js';
19
- import '@tangle-network/agent-interface';
20
- import '../errors-oeQrLqXC.js';
21
- import '../raw-provider-sink-C46HDghv.js';
2
+ import { AxAIService, AxFunction } from '@ax-llm/ax';
3
+ import { z } from 'zod';
4
+
5
+ /**
6
+ * Validator-output verdict substrate primitive for "did this output pass,
7
+ * and how well?"
8
+ *
9
+ * Used by:
10
+ * - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
11
+ * - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
12
+ * Runtime keeps `Validator` because it's coupled to runtime-shaped
13
+ * `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
14
+ * itself is a substrate concept and lives here.
15
+ *
16
+ * Repo layering: agent-eval is the substrate (no upward deps). Both
17
+ * agent-runtime and agent-knowledge consume this type FROM agent-eval —
18
+ * never the other way around. See CLAUDE.md "Repo layering" for the rule.
19
+ */
20
+ /**
21
+ * Minimal verdict shape — `valid` + `score` are required; `scores` +
22
+ * `notes` are optional surface. Validators that need richer shapes
23
+ * parameterise `Validator<Output, MyVerdict>` with their own type.
24
+ *
25
+ * Need structured extras? Extend DefaultVerdict with typed fields — never
26
+ * serialize extras into `notes`.
27
+ */
28
+ interface DefaultVerdict {
29
+ /** Whether the output meets the validator's pass criteria. */
30
+ valid: boolean;
31
+ /** Aggregate score in [0, 1]. Drivers use this for winner selection. */
32
+ score: number;
33
+ /** Per-dimension scores. Free-form; weighted into `score` by the validator. */
34
+ scores?: Record<string, number>;
35
+ /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
36
+ notes?: string;
37
+ }
38
+
39
+ /**
40
+ * Multi-layer verifier — ordered pipeline of verification layers.
41
+ *
42
+ * Different contract from {@link JudgeRunner} (which runs parallel
43
+ * specs against a sandbox). MultiLayerVerifier is a DAG of layers
44
+ * (install → typecheck → build → lint → serve → semantic → …) with
45
+ * dependency-based skip, per-layer findings, soft-fail semantics, and
46
+ * an aggregated `blendedScore` across all passed layers.
47
+ *
48
+ * Use when you want:
49
+ * - ordered stages where a failing upstream stage skips downstream ones
50
+ * - each stage produces rich `findings` (severity + message + evidence)
51
+ * - a single composite score across stages with per-stage weights
52
+ * - soft-fail stages whose failure doesn't abort the pipeline
53
+ *
54
+ * Use {@link JudgeRunner} when you want:
55
+ * - N independent judges running in parallel against the same artifact
56
+ * - no inter-judge dependencies
57
+ * - boolean `passed` per judge + overall
58
+ *
59
+ * Both primitives compose — JudgeRunner can be invoked as a single
60
+ * layer inside a MultiLayerVerifier if that suits the caller.
61
+ */
62
+
63
+ type LayerStatus = 'pass' | 'fail' | 'skipped' | 'error' | 'timeout';
64
+ type Severity = 'critical' | 'major' | 'minor' | 'info';
65
+ interface Finding {
66
+ severity: Severity;
67
+ message: string;
68
+ evidence?: string;
69
+ /** Optional layer name the finding belongs to (set by the verifier if omitted). */
70
+ layer?: string;
71
+ /**
72
+ * Free-form structured payload — used by `multiToolchainLayer` to attach
73
+ * `{ adapter: 'pnpm' }`, by judges to attach evidence pointers, etc.
74
+ * Renderers MAY interrogate; agent-eval primitives never assume shape.
75
+ */
76
+ detail?: Record<string, unknown>;
77
+ }
78
+ interface LayerResult {
79
+ layer: string;
80
+ status: LayerStatus;
81
+ /** 0..1 score, optional — layers that don't produce a numeric score omit. */
82
+ score?: number;
83
+ durationMs: number;
84
+ findings: Finding[];
85
+ /** Short human-readable summary (one line). */
86
+ reason?: string;
87
+ /**
88
+ * Numeric layer-level diagnostics: error counts, warning counts,
89
+ * cyclomatic complexity, total adapter wall-time, etc. Keyed by
90
+ * diagnostic name; null = "diagnostic not applicable / not measured."
91
+ * Renderers that know the keys can display them; ones that don't,
92
+ * ignore. Free-form on purpose — consumers type the value shape in
93
+ * their own namespace.
94
+ */
95
+ diagnostics?: Record<string, number | null>;
96
+ /** Any rich per-layer detail — rendered as-is by consumers that know the layer. */
97
+ detail?: Record<string, unknown>;
98
+ }
99
+ interface VerifyContext<Env = unknown> {
100
+ /** Per-run opaque context the caller provides. Layers destructure what they need. */
101
+ env: Env;
102
+ /** Previously-computed results from layers that already ran. */
103
+ prior: Record<string, LayerResult>;
104
+ /** Signal — if aborted, layers MUST bail within reasonable wall. */
105
+ signal: AbortSignal;
106
+ }
107
+ interface Layer<Env = unknown> {
108
+ name: string;
109
+ /** Stages that must have `status: 'pass'` before this layer runs. */
110
+ dependsOn?: string[];
111
+ /**
112
+ * Weight in the composite `blendedScore`. Default 1.0. Layers with weight 0
113
+ * contribute findings but not score.
114
+ */
115
+ weight?: number;
116
+ /**
117
+ * If true, a `fail` status contributes to `blendedScore` (as 0) instead of
118
+ * being dropped — use for layers whose failure is a real signal. Default:
119
+ * fail drops from numerator + denominator, matching VB's existing semantics.
120
+ */
121
+ failContributesToScore?: boolean;
122
+ /** Optional per-layer wall-cap in ms. Honored by the verifier (AbortSignal). */
123
+ capMs?: number;
124
+ run: (ctx: VerifyContext<Env>) => Promise<LayerResult> | LayerResult;
125
+ }
126
+ interface VerifyOptions<Env = unknown> {
127
+ env: Env;
128
+ /**
129
+ * Overall wall cap. Default: sum of layer capMs, or Infinity if any layer
130
+ * omits a cap. The verifier short-circuits remaining layers on overall cap.
131
+ */
132
+ overallCapMs?: number;
133
+ /** Called with each layer result as it completes. */
134
+ onLayer?: (result: LayerResult) => void;
135
+ }
136
+ /** Extends the substrate verdict spine: `valid` = `allPass` and `score` =
137
+ * `blendedScore` — derived where the report is aggregated, so spine
138
+ * consumers (drivers, gates) read this report without an adapter. */
139
+ interface VerificationReport extends DefaultVerdict {
140
+ layers: LayerResult[];
141
+ passCount: number;
142
+ failCount: number;
143
+ skippedCount: number;
144
+ errorCount: number;
145
+ /** True iff at least one scored layer ran AND every scored layer passed. */
146
+ allPass: boolean;
147
+ /**
148
+ * Weighted mean of `score` across contributing layers. 0 when no layers
149
+ * contributed. See {@link Layer.failContributesToScore} for fail semantics.
150
+ */
151
+ blendedScore: number;
152
+ durationMs: number;
153
+ startedAt: string;
154
+ finishedAt: string;
155
+ }
156
+ /**
157
+ * Ordered DAG of verification layers with dependency-based skipping, per-layer findings, soft-fail semantics, and a blended composite score across all passed layers.
158
+ */
159
+ declare class MultiLayerVerifier<Env = unknown> {
160
+ private readonly layers;
161
+ constructor(layers: Layer<Env>[]);
162
+ run(opts: VerifyOptions<Env>): Promise<VerificationReport>;
163
+ }
164
+
165
+ interface RunScore {
166
+ success: number;
167
+ goalProgress: number;
168
+ repoGroundedness: number;
169
+ driftPenalty: number;
170
+ toolUseQuality: number;
171
+ patchQuality: number;
172
+ testReality: number;
173
+ finalGate: number;
174
+ reviewerBlockers: number;
175
+ costUsd: number;
176
+ wallSeconds: number;
177
+ notes?: string[];
178
+ }
179
+ interface RunScoreWeights {
180
+ success: number;
181
+ goalProgress: number;
182
+ repoGroundedness: number;
183
+ driftPenalty: number;
184
+ toolUseQuality: number;
185
+ patchQuality: number;
186
+ testReality: number;
187
+ finalGate: number;
188
+ reviewerBlockers: number;
189
+ costUsd: number;
190
+ wallSeconds: number;
191
+ }
192
+
193
+ /**
194
+ * Error taxonomy for `@tangle-network/agent-eval`.
195
+ *
196
+ * Every error this package throws as part of its *public contract* extends
197
+ * `AgentEvalError`. Consumers can pattern-match by `instanceof <Subclass>` or
198
+ * by the stable string `code` carried on the base class.
199
+ *
200
+ * The codes are stable across minor versions; new codes can be added, but
201
+ * existing codes never change meaning. New subclasses are non-breaking.
202
+ *
203
+ * Internal invariant guards (`throw new Error('this should never happen')`)
204
+ * remain plain `Error`s on purpose — they're programmer-mistake assertions,
205
+ * not consumer-catchable contract failures.
206
+ */
207
+ type AgentEvalErrorCode = 'validation' | 'not_found' | 'config' | 'capture_integrity' | 'judge' | 'verification' | 'replay' | 'backend_integrity' | 'profile_matrix';
208
+ /**
209
+ * Base class for every contract error this package throws — carries the stable
210
+ * string `code` taxonomy so consumers can `instanceof`-match or switch on `code`.
211
+ */
212
+ declare class AgentEvalError extends Error {
213
+ /** Stable string code. Survives minification; safe to switch on. */
214
+ readonly code: AgentEvalErrorCode;
215
+ constructor(code: AgentEvalErrorCode, message: string, options?: {
216
+ cause?: unknown;
217
+ });
218
+ }
219
+ /** Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). */
220
+ declare class ValidationError extends AgentEvalError {
221
+ constructor(message: string, options?: {
222
+ cause?: unknown;
223
+ });
224
+ }
225
+
226
+ type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
227
+ type AgentProfileJson = string | number | boolean | null | AgentProfileJson[] | {
228
+ [key: string]: AgentProfileJson;
229
+ };
230
+ type AgentProfileDimensionValue = string | number | boolean | null;
231
+ interface AgentProfileSource {
232
+ /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
233
+ kind: string;
234
+ /** sha256 over the canonical source profile object. */
235
+ hash: string;
236
+ }
237
+ interface AgentProfileHarness {
238
+ id: string;
239
+ version?: string;
240
+ hash?: string;
241
+ }
242
+ interface AgentProfileCell {
243
+ schemaVersion: AgentProfileCellSchemaVersion;
244
+ cellId: string;
245
+ profileId: string;
246
+ sourceProfile: AgentProfileSource;
247
+ harness?: AgentProfileHarness;
248
+ model?: string;
249
+ promptHash?: string;
250
+ dimensions?: Record<string, AgentProfileDimensionValue>;
251
+ }
252
+
253
+ type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
254
+ interface BudgetSpec {
255
+ tokens?: number;
256
+ wallMs?: number;
257
+ calls?: number;
258
+ usd?: number;
259
+ }
260
+ interface RunOutcome$1 {
261
+ score?: number;
262
+ pass?: boolean;
263
+ failureClass?: FailureClass;
264
+ notes?: string;
265
+ }
266
+ /**
267
+ * Layer — optional classification in a nested build workflow.
268
+ * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
269
+ * `app-build`: sandbox harness that compiled + tested the generated scaffold.
270
+ * `app-runtime`: a run of the generated agent against a domain scenario.
271
+ * `meta`: any meta-eval (judge replay, correlation analysis).
272
+ */
273
+ type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
274
+ interface Run {
275
+ runId: string;
276
+ /**
277
+ * Stable identifier of the scenario being executed.
278
+ *
279
+ * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts
280
+ * input WITHOUT this field, substituting a sensible default
281
+ * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no
282
+ * curated scenario to anchor to (runtime / operator / meta-eval runs). This
283
+ * keeps the persisted shape unambiguous for downstream filters + aggregations
284
+ * while removing the boilerplate of inventing placeholder ids at the call site.
285
+ */
286
+ scenarioId: string;
287
+ variantId?: string;
288
+ datasetVersion?: string;
289
+ /** Git SHA of agent code at run time. */
290
+ codeSha?: string;
291
+ /** Hash of the prompt template + any system prompt. */
292
+ promptSha?: string;
293
+ /** Model id + date + system-prompt hash, concatenated. */
294
+ modelFingerprint?: string;
295
+ seed?: number;
296
+ /** Arbitrary environment markers (shell, docker version, tz). */
297
+ envFingerprint?: Record<string, string>;
298
+ /** Version of the redaction rules applied to this run. */
299
+ redactionVersion?: string;
300
+ /** Parent run in a nested build workflow. A builder run's children are
301
+ * app-build runs; those children are app-runtime runs. */
302
+ parentRunId?: string;
303
+ /** Stable project identifier — groups runs across chats + sessions. */
304
+ projectId?: string;
305
+ /** Chat/conversation identifier within a project. */
306
+ chatId?: string;
307
+ /** Layer classification — hint for aggregation; not enforced. */
308
+ layer?: RunLayer;
309
+ startedAt: number;
310
+ endedAt?: number;
311
+ status: RunStatus;
312
+ outcome?: RunOutcome$1;
313
+ budget?: BudgetSpec;
314
+ /** Free-form labels for downstream grouping. */
315
+ tags?: Record<string, string>;
316
+ }
317
+ type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
318
+ type SpanStatus = 'ok' | 'error';
319
+ interface SpanBase {
320
+ spanId: string;
321
+ parentSpanId?: string;
322
+ runId: string;
323
+ kind: SpanKind;
324
+ name: string;
325
+ startedAt: number;
326
+ endedAt?: number;
327
+ status?: SpanStatus;
328
+ error?: string;
329
+ /** Anything not covered by typed fields. Kept deliberately free-form. */
330
+ attributes?: Record<string, unknown>;
331
+ }
332
+ interface Message {
333
+ role: 'system' | 'user' | 'assistant' | 'tool';
334
+ content: string;
335
+ tokens?: number;
336
+ /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
337
+ images?: Array<{
338
+ artifactId?: string;
339
+ url?: string;
340
+ mime?: string;
341
+ }>;
342
+ }
343
+ interface LlmSpan extends SpanBase {
344
+ kind: 'llm';
345
+ model: string;
346
+ messages: Message[];
347
+ output?: string;
348
+ inputTokens?: number;
349
+ /** All generated tokens, including the reasoning subset when present. */
350
+ outputTokens?: number;
351
+ cachedTokens?: number;
352
+ cacheWriteTokens?: number;
353
+ /** Reasoning-token subset of `outputTokens`. */
354
+ reasoningTokens?: number;
355
+ costUsd?: number;
356
+ finishReason?: string;
357
+ }
358
+ interface ToolSpan extends SpanBase {
359
+ kind: 'tool';
360
+ toolName: string;
361
+ args: unknown;
362
+ /** False when the source observed the call but did not capture its arguments. */
363
+ argsCaptured?: boolean;
364
+ result?: unknown;
365
+ latencyMs?: number;
366
+ }
367
+ interface RetrievalSpan extends SpanBase {
368
+ kind: 'retrieval';
369
+ query: string;
370
+ hits: Array<{
371
+ docId: string;
372
+ score: number;
373
+ content?: string;
374
+ }>;
375
+ }
376
+ interface JudgeSpan extends SpanBase {
377
+ kind: 'judge';
378
+ judgeId: string;
379
+ /** Span this judgment applies to. */
380
+ targetSpanId: string;
381
+ dimension: string;
382
+ /** Numeric score (free-range; interpretation up to the judge). */
383
+ score: number;
384
+ rationale?: string;
385
+ evidence?: string;
386
+ }
387
+ interface SandboxSpan extends SpanBase {
388
+ kind: 'sandbox';
389
+ image?: string;
390
+ command?: string;
391
+ exitCode?: number;
392
+ testsTotal?: number;
393
+ testsPassed?: number;
394
+ stdoutHash?: string;
395
+ stderrHash?: string;
396
+ /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
397
+ wallMs?: number;
398
+ }
399
+ interface GenericSpan extends SpanBase {
400
+ kind: 'agent' | 'custom';
401
+ }
402
+ type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
403
+ type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
404
+ interface TraceEvent {
405
+ eventId: string;
406
+ runId: string;
407
+ spanId?: string;
408
+ kind: EventKind;
409
+ timestamp: number;
410
+ payload: Record<string, unknown>;
411
+ }
412
+ interface BudgetLedgerEntry {
413
+ runId: string;
414
+ dimension: keyof BudgetSpec;
415
+ limit: number;
416
+ consumed: number;
417
+ remaining: number;
418
+ timestamp: number;
419
+ breached: boolean;
420
+ /** Span that triggered this entry, if any. */
421
+ spanId?: string;
422
+ }
423
+ interface Artifact {
424
+ artifactId: string;
425
+ runId: string;
426
+ spanId?: string;
427
+ contentType: string;
428
+ sizeBytes: number;
429
+ /** sha256 in hex. */
430
+ hash: string;
431
+ /** External storage URL (R2, S3, filesystem path). */
432
+ storageUrl?: string;
433
+ /** Inline content for small blobs — keep under ~64KB. */
434
+ inlineContent?: string;
435
+ }
436
+ type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
437
+
438
+ /**
439
+ * Paper-grade RunRecord schema + runtime validator.
440
+ *
441
+ * Every run that participates in a promotion gate, paper table, or
442
+ * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
443
+ * fields are exactly those the paper "Two Loops, Three Roles" requires
444
+ * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
445
+ * holdout split tag and either a `searchScore` or a `holdoutScore`.
446
+ *
447
+ * This is intentionally NOT a replacement for the rich `Run` /
448
+ * `ProposeReviewReport` / `ScenarioResult` types already in the
449
+ * package. Those are runtime structures with full provenance. A
450
+ * `RunRecord` is the analysis-time projection — the JSON-friendly
451
+ * row you'd put in a parquet file or paste into a notebook.
452
+ *
453
+ * Validate at the boundary:
454
+ *
455
+ * const rec = validateRunRecord(rawJson) // throws on missing
456
+ * const ok = isRunRecord(rawJson) // boolean check
457
+ * const rec = parseRunRecordSafe(rawJson) // { ok, value | error }
458
+ *
459
+ * The validator runs in pure TS — zod is intentionally NOT a
460
+ * dependency. Round-trip tested in `tests/run-record.test.ts`.
461
+ */
462
+
463
+ /** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
464
+ * combined train+test pool that the optimizer is allowed to read. */
465
+ type RunSplitTag = 'search' | 'dev' | 'holdout';
466
+ interface RunTokenUsage {
467
+ input: number;
468
+ /** All generated tokens charged as output, including reasoning tokens. */
469
+ output: number;
470
+ /** Reasoning-token subset of `output`, when the provider reports it. */
471
+ reasoning?: number;
472
+ /** Prompt tokens served from a provider cache. */
473
+ cached?: number;
474
+ /** Prompt tokens written into a provider cache. */
475
+ cacheWrite?: number;
476
+ }
477
+ /**
478
+ * How a run's USD amount was obtained.
479
+ *
480
+ * `costUsd` remains mandatory for wire compatibility. New producers should
481
+ * always populate this discriminated union so a missing bill is never
482
+ * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
483
+ * the legacy `0` sentinel while this field carries the truthful null.
484
+ */
485
+ type RunCostProvenance = {
486
+ kind: 'observed';
487
+ usd: number;
488
+ } | {
489
+ kind: 'estimated';
490
+ usd: number;
491
+ } | {
492
+ kind: 'uncaptured';
493
+ usd: null;
494
+ };
495
+ interface RunJudgeMetadata {
496
+ model: string;
497
+ promptVersion: string;
498
+ /** [0,1] confidence the judge declared. Constant judge confidence
499
+ * across many runs is a fallback signal (see `canary.ts`). */
500
+ confidence: number;
501
+ /** True if the judge degraded to a fallback path (rules-only,
502
+ * prior-call cache, etc.). The canary uses this to alert. */
503
+ fallback: boolean;
504
+ }
505
+ /**
506
+ * Per-judge / per-dimension breakdown for runs scored by an ensemble of
507
+ * judges over a multi-dimensional rubric.
508
+ *
509
+ * The collapsed `outcome.searchScore` / `holdoutScore` carries the
510
+ * composite the gate uses. The full breakdown belongs here so consumers
511
+ * can answer "which judge disagreed?", "which dimension dragged the
512
+ * composite down?", and "did half the panel fail?" without re-running.
513
+ *
514
+ * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and
515
+ * `composite` are convenience projections — derivable but precomputed so
516
+ * downstream IRR primitives (`interRaterReliability`,
517
+ * `corpusInterRaterAgreement`) and reporters don't pay the same
518
+ * aggregation twice.
519
+ *
520
+ * Fail-loud discipline: judges that errored out land in `failedJudges`
521
+ * by id. A missing key in `perJudge` is ambiguous (silent zero vs not
522
+ * run); the explicit list makes a partial-failure recorded as such.
523
+ */
524
+ interface JudgeScoresRecord {
525
+ /** Per-judge per-dimension scores. `{ "kimi-k2.6": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */
526
+ perJudge: Record<string, Record<string, number>>;
527
+ /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
528
+ perDimMean: Record<string, number>;
529
+ /** Composite mean across all dims and judges. Mirrors the score
530
+ * the gate sees on `outcome.searchScore` / `holdoutScore`. */
531
+ composite: number;
532
+ /** Judges that errored or returned an unparseable verdict. Recorded
533
+ * by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
534
+ * not inferred from missing keys in `perJudge`. */
535
+ failedJudges?: string[];
536
+ /** Free-form notes the judges emitted (joined across judges or
537
+ * first-judge only — consumer's choice). */
538
+ notes?: string;
539
+ }
540
+ interface RunOutcome {
541
+ /** Score on the search/optimization split. Optional because a
542
+ * holdout-only evaluation only fills `holdoutScore`. */
543
+ searchScore?: number;
544
+ /** Score on the held-out split. Optional because a search-only run
545
+ * only fills `searchScore`. At least one must be present. */
546
+ holdoutScore?: number;
547
+ /** Bag of any other metric the run produced — judge dimensions,
548
+ * pass/fail counters, latency stats, etc. Numeric only — keeps
549
+ * reporters honest. */
550
+ raw: Record<string, number>;
551
+ /** Per-judge / per-dim breakdown. Consumers writing ensemble
552
+ * judgements populate this; substrate primitives like
553
+ * `interRaterReliability` and `corpusInterRaterAgreement` accept
554
+ * these records as input. Optional — single-judge or scalar-only
555
+ * runs leave it unset. */
556
+ judgeScores?: JudgeScoresRecord;
557
+ /** Authenticity / realness verdict — did the run build the REAL thing on the
558
+ * intended infra, or fake it (see `./authenticity`)? Optional: only domains
559
+ * with an authenticity config populate it. Carried in the corpus so the
560
+ * flywheel / off-policy learning can optimize for real completion, not gamed
561
+ * pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run
562
+ * must not count as a real success regardless of `score`. */
563
+ realness?: {
564
+ score: number;
565
+ gated: boolean;
566
+ reason?: string;
567
+ };
568
+ }
569
+ /**
570
+ * Mandatory paper-grade fields for a single evaluation run. Optional
571
+ * fields are extension points; mandatory fields throw if missing.
572
+ *
573
+ * Hash discipline:
574
+ * - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the
575
+ * model (after any steering bundle merge).
576
+ * - `configHash` is the sha256 of the effective run config (model,
577
+ * temperature, tools, judges, splits). The pair (promptHash,
578
+ * configHash) uniquely identifies an experiment cell.
579
+ *
580
+ * Model snapshot discipline:
581
+ * - `model` MUST encode a snapshot version. Bare aliases like
582
+ * `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.
583
+ * Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.
584
+ */
585
+ interface RunRecord {
586
+ /** UUID for the run. */
587
+ runId: string;
588
+ /** Logical experiment grouping (a treatment vs a baseline within
589
+ * the same sweep should share `experimentId`). */
590
+ experimentId: string;
591
+ /** Stable identifier for the candidate (variant) being run. The
592
+ * promotion gate compares two `candidateId`s on matched items. */
593
+ candidateId: string;
594
+ /** RNG seed for the run. Always recorded — silent re-seeding is
595
+ * the most common cause of non-reproducible numbers. */
596
+ seed: number;
597
+ /** Model identifier WITH snapshot version. */
598
+ model: string;
599
+ /** sha256 of the effective prompt (post-steering). */
600
+ promptHash: string;
601
+ /** sha256 of the effective config. */
602
+ configHash: string;
603
+ /** Git SHA the harness was run from. */
604
+ commitSha: string;
605
+ /** End-to-end wall-clock duration in milliseconds. */
606
+ wallMs: number;
607
+ /** Time spent queued before execution started, if known. */
608
+ queueMs?: number;
609
+ /** Total USD cost. Mandatory — runs without a cost number are
610
+ * unbounded by definition and must not be admitted into the gate.
611
+ * `0` is retained as the compatibility sentinel for an uncaptured amount;
612
+ * inspect `costProvenance` before treating it as observed. */
613
+ costUsd: number;
614
+ /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
615
+ * Optional only so existing serialized RunRecords remain valid. */
616
+ costProvenance?: RunCostProvenance;
617
+ /** Token usage breakdown. */
618
+ tokenUsage: RunTokenUsage;
619
+ /** Judge-side metadata, if a judge was used. */
620
+ judgeMetadata?: RunJudgeMetadata;
621
+ /** Per-split scores + raw bag. */
622
+ outcome: RunOutcome;
623
+ /** Canonical, cross-agent failure class drawn from the shared
624
+ * `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
625
+ * "which failure dominates across the whole fleet" answerable in ONE
626
+ * vocabulary — every agent classifies against the same enum. Producers
627
+ * set it via the substrate classifier; leave unset only when the failure
628
+ * genuinely can't be classified. */
629
+ failureClass?: FailureClass;
630
+ /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
631
+ * (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
632
+ * The within-agent drill-down; `failureClass` is the cross-agent key. */
633
+ failureMode?: string;
634
+ /** Which split this run was drawn from. */
635
+ splitTag: RunSplitTag;
636
+ /**
637
+ * Stable scenario identifier the run was scored against. Optional for
638
+ * backwards compatibility, but **strongly recommended**: every primitive
639
+ * that pairs runs by scenario (preferences, paired stats, BT tournament)
640
+ * keys on this. The campaign artifact populates it canonically; legacy
641
+ * runs without it fall back to inference from `outcome.raw.scenario_id`
642
+ * or `experimentId`.
643
+ */
644
+ scenarioId?: string;
645
+ /**
646
+ * Canonical identity for the agent profile cell that produced this row:
647
+ * profile artifact hash plus optional harness/model/prompt/reporting
648
+ * dimensions. Use `agentProfile.cellId` to group persona sweeps and
649
+ * longitudinal reports by the complete source profile, not by a loose
650
+ * candidate label or opaque config hash.
651
+ */
652
+ agentProfile?: AgentProfileCell;
653
+ }
654
+
655
+ /**
656
+ * RawProviderSink — first-class persistence for the actual HTTP-level
657
+ * request/response bodies of every LLM provider call.
658
+ *
659
+ * Why this is a separate sink from the structured `LlmSpan`:
660
+ *
661
+ * - `LlmSpan` records the *intent* — model name, messages, output text,
662
+ * usage. It's what dashboards read; it's NOT enough for forensics.
663
+ * - When a downstream consumer reports "the verifier used the wrong route"
664
+ * or "tokens look right but reasoning was missing," the only way to
665
+ * answer is the raw HTTP body. Span fields can lie (a proxy can echo
666
+ * a different `model` value than what actually answered); the raw
667
+ * response is ground truth.
668
+ *
669
+ * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the
670
+ * matrix runner / BuilderSession sets it up automatically) and every
671
+ * request, response, and error is recorded — including retries, with the
672
+ * attempt index attached so a flaky call's full event chain is recoverable.
673
+ *
674
+ * Redaction is enforced at sink time. The default redactor strips
675
+ * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any
676
+ * payload field whose key matches `apiKey | api_key | bearer | password |
677
+ * secret | token` (case-insensitive). Override via the sink constructor or
678
+ * the per-call `redactor`. The `redactedFields` array on the persisted
679
+ * event lets a reviewer see what was stripped without exposing the values.
680
+ */
681
+ type RawProviderDirection = 'request' | 'response' | 'error';
682
+ interface RawProviderEvent {
683
+ /** Stable id. Generated by the sink if omitted. */
684
+ eventId: string;
685
+ /** Trace context populated by `LlmClient` when the call is wrapped in a span. */
686
+ runId?: string;
687
+ spanId?: string;
688
+ /**
689
+ * Logical provider name. Free-form so callers can use whatever id matches
690
+ * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When
691
+ * omitted, derived from `baseUrl` in `LlmClientOptions`.
692
+ */
693
+ provider: string;
694
+ model: string;
695
+ /** Endpoint path, e.g. `'/v1/chat/completions'`. */
696
+ endpoint: string;
697
+ /** Base URL used for the call (already-normalised — no trailing slash). */
698
+ baseUrl: string;
699
+ /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */
700
+ attemptIndex: number;
701
+ direction: RawProviderDirection;
702
+ /** Unix ms. */
703
+ timestamp: number;
704
+ /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */
705
+ durationMs?: number;
706
+ statusCode?: number;
707
+ requestHeaders?: Record<string, string>;
708
+ requestBody?: unknown;
709
+ responseHeaders?: Record<string, string>;
710
+ responseBody?: unknown;
711
+ /** Set on `direction: 'error'` events. */
712
+ errorMessage?: string;
713
+ /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */
714
+ redactedFields: string[];
715
+ }
716
+ interface RawProviderSinkFilter {
717
+ runId?: string;
718
+ spanId?: string;
719
+ direction?: RawProviderDirection;
720
+ attemptIndex?: number;
721
+ }
722
+ interface RawProviderSink {
723
+ record(event: RawProviderEvent): Promise<void>;
724
+ /** Optional listing — implementations that durably persist (file, db) should support this. */
725
+ list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
726
+ /** Optional teardown for backed implementations. */
727
+ close?(): Promise<void>;
728
+ }
729
+ type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
730
+
731
+ interface RunFilter {
732
+ scenarioId?: string;
733
+ variantId?: string;
734
+ status?: RunStatus;
735
+ since?: number;
736
+ until?: number;
737
+ tag?: {
738
+ key: string;
739
+ value: string;
740
+ };
741
+ parentRunId?: string;
742
+ projectId?: string;
743
+ chatId?: string;
744
+ layer?: RunLayer;
745
+ }
746
+ interface SpanFilter {
747
+ runId?: string;
748
+ parentSpanId?: string;
749
+ kind?: SpanKind;
750
+ name?: string;
751
+ toolName?: string;
752
+ judgeId?: string;
753
+ since?: number;
754
+ until?: number;
755
+ }
756
+ interface EventFilter {
757
+ runId?: string;
758
+ spanId?: string;
759
+ kind?: EventKind;
760
+ since?: number;
761
+ until?: number;
762
+ }
763
+ interface TraceStore {
764
+ appendRun(run: Run): Promise<void>;
765
+ updateRun(runId: string, patch: Partial<Run>): Promise<void>;
766
+ appendSpan(span: Span): Promise<void>;
767
+ updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
768
+ appendEvent(event: TraceEvent): Promise<void>;
769
+ appendArtifact(artifact: Artifact): Promise<void>;
770
+ appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
771
+ getRun(runId: string): Promise<Run | undefined>;
772
+ listRuns(filter?: RunFilter): Promise<Run[]>;
773
+ spans(filter?: SpanFilter): Promise<Span[]>;
774
+ events(filter?: EventFilter): Promise<TraceEvent[]>;
775
+ budget(runId: string): Promise<BudgetLedgerEntry[]>;
776
+ artifacts(runId: string): Promise<Artifact[]>;
777
+ }
778
+
779
+ interface RunTrace {
780
+ run: Run;
781
+ spans: Span[];
782
+ events: TraceEvent[];
783
+ artifacts: Artifact[];
784
+ budget: BudgetLedgerEntry[];
785
+ }
786
+ interface RunCriticOptions {
787
+ weights?: Partial<RunScoreWeights>;
788
+ driftPatterns?: RegExp[];
789
+ }
790
+ declare class RunCritic {
791
+ private readonly weights?;
792
+ private readonly driftPatterns;
793
+ constructor(options?: RunCriticOptions);
794
+ score(store: TraceStore, runId: string): Promise<RunScore>;
795
+ scoreTrace(trace: RunTrace): RunScore;
796
+ rank(score: RunScore): number;
797
+ private isDrift;
798
+ }
799
+
800
+ type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
801
+ interface CostUsage {
802
+ inputTokens: number;
803
+ outputTokens: number;
804
+ cachedTokens?: number;
805
+ }
806
+ interface CostCallBase {
807
+ callId: string;
808
+ channel: CostChannel;
809
+ phase: string;
810
+ actor: string;
811
+ model: string;
812
+ maximumCostUsd?: number;
813
+ tags?: Record<string, string>;
814
+ timestamp: number;
815
+ }
816
+ interface CostReceipt extends CostCallBase, CostUsage {
817
+ status: 'settled';
818
+ costUsd: number;
819
+ costUnknown: boolean;
820
+ usageUnknown?: boolean;
821
+ pricing?: {
822
+ inputUsdPerThousand: number;
823
+ outputUsdPerThousand: number;
824
+ };
825
+ actualCostUsd?: number;
826
+ error?: string;
827
+ }
828
+ interface CostReceiptInput extends CostUsage {
829
+ model: string;
830
+ actualCostUsd?: number;
831
+ costUnknown?: boolean;
832
+ usageUnknown?: boolean;
833
+ }
834
+ type MaximumCharge = {
835
+ externallyEnforcedMaximumUsd: number;
836
+ } | ({
837
+ model: string;
838
+ } & CostUsage);
839
+ interface RunPaidCallInput<T> {
840
+ callId?: string;
841
+ channel: CostChannel;
842
+ phase: string;
843
+ actor: string;
844
+ /** Used before a provider receipt exists and on failures without one. */
845
+ model?: string;
846
+ tags?: Record<string, string>;
847
+ signal?: AbortSignal;
848
+ /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
849
+ maximumCharge?: MaximumCharge;
850
+ /** `callId` can be forwarded as the provider's idempotency key. */
851
+ execute(signal: AbortSignal, callId: string): Promise<T>;
852
+ receipt(value: T): CostReceiptInput;
853
+ receiptFromError?(error: Error): CostReceiptInput | undefined;
854
+ }
855
+ type PaidCallResult<T> = {
856
+ succeeded: true;
857
+ callId: string;
858
+ value: T;
859
+ receipt: CostReceipt;
860
+ } | {
861
+ succeeded: false;
862
+ callId?: string;
863
+ error: Error;
864
+ receipt?: CostReceipt;
865
+ };
866
+ interface ChannelRollup {
867
+ channel: CostChannel;
868
+ calls: number;
869
+ inputTokens: number;
870
+ outputTokens: number;
871
+ cachedTokens: number;
872
+ costUsd: number;
873
+ unpricedCalls: number;
874
+ unknownUsageCalls: number;
875
+ }
876
+ interface CostLedgerSummary {
877
+ totalCalls: number;
878
+ pendingCalls: number;
879
+ unresolvedCalls: number;
880
+ reservedCostUsd: number;
881
+ inputTokens: number;
882
+ outputTokens: number;
883
+ cachedTokens: number;
884
+ totalCostUsd: number;
885
+ byChannel: ChannelRollup[];
886
+ unpricedModels: string[];
887
+ fullyPriced: boolean;
888
+ usageComplete: boolean;
889
+ accountingComplete: boolean;
890
+ incompleteReasons: string[];
891
+ }
892
+ interface CostLedgerFilter {
893
+ channel?: CostChannel;
894
+ phase?: string;
895
+ tags?: Record<string, string>;
896
+ }
897
+ /** Append-only storage. `append` must atomically reject stale revisions. */
898
+ interface CostLedgerPersistence {
899
+ read(): {
900
+ revision: string;
901
+ events: string;
902
+ };
903
+ append(expectedRevision: string, event: string): string | undefined;
904
+ }
905
+ interface CostLedgerOptions {
906
+ costCeilingUsd?: number;
907
+ persistence?: CostLedgerPersistence;
908
+ /** Import already-settled receipts without admitting new paid work. */
909
+ receipts?: readonly CostReceipt[];
910
+ }
911
+ /** Run-wide paid-call admission, durable call state, receipts, and summaries. */
912
+ declare class CostLedger {
913
+ private readonly records;
914
+ private readonly activeCallIds;
915
+ private readonly lateCallIds;
916
+ private completedTasks;
917
+ private revision;
918
+ private costLimitPersisted;
919
+ readonly costCeilingUsd?: number;
920
+ private readonly persistence?;
921
+ constructor(input?: number | CostLedgerOptions);
922
+ runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
923
+ /** Settle a call left pending by a crashed process after reconciling with the provider. */
924
+ reconcile(callId: string, observed: CostReceiptInput, options?: {
925
+ error?: string;
926
+ }): CostReceipt;
927
+ list(filter?: CostLedgerFilter): CostReceipt[];
928
+ summary(filter?: CostLedgerFilter): CostLedgerSummary;
929
+ markCompleted(count?: number): void;
930
+ costPerCompletedTask(): number | null;
931
+ private execute;
932
+ private captureLateOutcome;
933
+ private commitOutcome;
934
+ private captureFailure;
935
+ private commitReceipt;
936
+ private resolveMaximum;
937
+ private hasIncompleteSettledCall;
938
+ private appendRecord;
939
+ private ensureCostLimitPersisted;
940
+ private appendEvent;
941
+ }
942
+
943
+ /**
944
+ * LLM client with graceful degrade.
945
+ *
946
+ * OpenAI-compatible `/v1/chat/completions` client with:
947
+ * - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
948
+ * - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
949
+ * - Graceful json_schema → json_object degrade on 400 with schema-reject body.
950
+ * - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
951
+ * - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI
952
+ * directly, cli-bridge subscriptions, and any router that speaks the spec.
953
+ *
954
+ * Usage:
955
+ * const { value, result } = await callLlmJson<MyType>(
956
+ * { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },
957
+ * { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
958
+ * )
959
+ *
960
+ * This is THE llm-calling seam for agent-eval primitives that need structured
961
+ * output (semantic concept judge, reviewer directives, critic scores). Primitives
962
+ * that need free-form text use `callLlm` and parse output themselves.
963
+ */
964
+
965
+ interface LlmMessage {
966
+ role: 'system' | 'user' | 'assistant';
967
+ /**
968
+ * Either a plain text content string OR a multimodal content array
969
+ * (text + image_url parts) for vision-capable models.
970
+ */
971
+ content: string | Array<{
972
+ type: 'text';
973
+ text: string;
974
+ } | {
975
+ type: 'image_url';
976
+ image_url: {
977
+ url: string;
978
+ detail?: 'auto' | 'low' | 'high';
979
+ };
980
+ }>;
981
+ }
982
+ interface LlmCallRequest {
983
+ model: string;
984
+ messages: LlmMessage[];
985
+ /** Optional JSON-mode response format (response_format: json_object). */
986
+ jsonMode?: boolean;
987
+ /** Optional structured output via JSON Schema. Falls back to json_object on 400. */
988
+ jsonSchema?: {
989
+ name: string;
990
+ schema: Record<string, unknown>;
991
+ };
992
+ temperature?: number;
993
+ maxTokens?: number;
994
+ /** Per-call timeout, default 300s. */
995
+ timeoutMs?: number;
996
+ }
997
+ interface LlmUsage {
998
+ promptTokens: number;
999
+ completionTokens: number;
1000
+ totalTokens: number;
1001
+ /** False when the provider omitted or malformed prompt/completion usage. */
1002
+ captured?: boolean;
1003
+ /** Proxies populate this when prompt caching is on. */
1004
+ cachedPromptTokens?: number;
1005
+ }
1006
+ interface LlmCallResult {
1007
+ /** The text content of the first choice. Empty string if none. */
1008
+ content: string;
1009
+ usage: LlmUsage;
1010
+ /**
1011
+ * Cost in USD. Pulled from proxy's `_response_cost` field when present;
1012
+ * `null` when neither the proxy nor the caller can derive it.
1013
+ */
1014
+ costUsd: number | null;
1015
+ /** Model name actually used (echoed from response). */
1016
+ model: string;
1017
+ /** Wall-clock duration of the HTTP call (last attempt, if retried). */
1018
+ durationMs: number;
1019
+ /**
1020
+ * `finish_reason` echoed from the first choice (`stop`, `length`,
1021
+ * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.
1022
+ * Exposed so a free-form `callLlm` caller CAN detect a truncated answer
1023
+ * (`length`) instead of treating a cut-off completion as complete. Note:
1024
+ * `callLlm` does not itself reject on it — acting on this signal is the
1025
+ * caller's responsibility (in-repo free-form drivers do not yet enforce it).
1026
+ */
1027
+ finishReason?: string | null;
1028
+ /**
1029
+ * True when `content.trim()` is empty. An empty completion is a silent zero
1030
+ * for free-form `callLlm` callers; this flag is the signal a caller can
1031
+ * inspect to fail loud rather than proceed on an empty string. `callLlm`
1032
+ * surfaces it but does not throw on it.
1033
+ */
1034
+ contentEmpty?: boolean;
1035
+ /** Raw response body. */
1036
+ raw: Record<string, unknown>;
1037
+ }
1038
+ interface LlmClientOptions {
1039
+ /** Base URL (without trailing slash). Must end at the `/v1` prefix. */
1040
+ baseUrl?: string;
1041
+ /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
1042
+ apiKey?: string;
1043
+ bearer?: string;
1044
+ /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */
1045
+ authHeader?: {
1046
+ name: string;
1047
+ value: string;
1048
+ };
1049
+ /** Stable provider idempotency key, reused across retries of this logical call. */
1050
+ idempotencyKey?: string;
1051
+ /** Default timeout in ms. Per-call can override. */
1052
+ defaultTimeoutMs?: number;
1053
+ /**
1054
+ * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to
1055
+ * each attempt's per-attempt timeout controller, so aborting it cancels
1056
+ * the in-flight fetch. A caller abort is FATAL: it is not retried even
1057
+ * though an AbortError otherwise matches the transient patterns.
1058
+ */
1059
+ signal?: AbortSignal;
1060
+ /**
1061
+ * Cross-attempt wall-clock budget in ms, measured from the first attempt.
1062
+ * Before launching each attempt the loop checks the remaining budget and
1063
+ * stops retrying once it is exhausted, rather than waiting the full
1064
+ * per-attempt timeout on every retry. Bounds total time independent of
1065
+ * total attempts × `timeoutMs`.
1066
+ */
1067
+ deadlineMs?: number;
1068
+ /** Total provider attempts. Legacy option name; default 3 (1 initial + 2 retries). */
1069
+ maxRetries?: number;
1070
+ /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
1071
+ fetch?: typeof fetch;
1072
+ /**
1073
+ * Optional raw HTTP capture sink. When provided, every request, response,
1074
+ * and error (across all retry attempts) is recorded to the sink, with auth
1075
+ * headers and credential-shaped body fields redacted by default. This is
1076
+ * the layer-1 forensics primitive: structured `LlmSpan`s record intent,
1077
+ * raw events record what actually crossed the wire.
1078
+ */
1079
+ rawSink?: RawProviderSink;
1080
+ /**
1081
+ * Logical provider id attached to raw events. When omitted, derived from
1082
+ * `baseUrl` via `providerFromBaseUrl`.
1083
+ */
1084
+ provider?: string;
1085
+ /** Trace context attached to raw events; populated by emitter-aware callers. */
1086
+ traceContext?: {
1087
+ runId?: string;
1088
+ spanId?: string;
1089
+ };
1090
+ /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */
1091
+ redactor?: ProviderRedactor;
1092
+ }
1093
+
1094
+ /**
1095
+ * Semantic concept judge — "does the built artifact actually implement
1096
+ * the features the user asked for?"
1097
+ *
1098
+ * Distinct from the domain/code/coherence judges in `judges.ts`:
1099
+ * - those judges score free-form conversational agent outputs along
1100
+ * quality dimensions (accuracy, depth, etc.)
1101
+ * - this judge scores a *built artifact* (served HTML + source files)
1102
+ * against an explicit list of expected concepts, returning per-concept
1103
+ * {present, score 0-10, evidence, severity}.
1104
+ *
1105
+ * The judge is strict about distinguishing (a) a working implementation
1106
+ * from (b) a keyword-present stub. "// TODO: mint button" is NOT present.
1107
+ * Only real, functional, wired-up code counts.
1108
+ *
1109
+ * Use via {@link createSemanticConceptJudge} or directly via
1110
+ * {@link runSemanticConceptJudge}. Soft-fails (available=false) on LLM
1111
+ * or JSON-parse errors so the caller can treat that as "layer skipped"
1112
+ * rather than "layer failed" in a multi-layer pipeline.
1113
+ */
1114
+
1115
+ /**
1116
+ * Implementation complexity class for weighted scoring.
1117
+ *
1118
+ * - `render` (default): the concept is a UI surface that displays static
1119
+ * data — render a list, show a counter, lay out a button. Single-file
1120
+ * work, no external integration.
1121
+ * - `integrate`: the concept requires wiring a real external system —
1122
+ * wallet connect (wagmi + RainbowKit + chain config), payment provider
1123
+ * (Stripe Elements + intent + webhook), an API client with auth.
1124
+ * Multi-file, library-knowledge, runtime correctness matters.
1125
+ * - `compute`: the concept requires algorithmic work — solver, simulator,
1126
+ * constraint propagation, ML inference. Correctness > UI polish.
1127
+ *
1128
+ * Default weights (when applied via `weightConcepts: 'complexity'`):
1129
+ * render=1.0, integrate=2.0, compute=2.5
1130
+ *
1131
+ * Cross-vertical scoring without complexity weighting silently inflates
1132
+ * the rate of UI-heavy verticals (healthcare, fintech dashboards) vs
1133
+ * integration-heavy verticals (DeFi, wallets) — all concepts treated
1134
+ * equally even though the agent does 2-3x the work for `integrate`.
1135
+ */
1136
+ type ConceptComplexity = 'render' | 'integrate' | 'compute';
1137
+ interface ConceptSpec {
1138
+ name: string;
1139
+ /** Short hints that help the judge; not used for matching. */
1140
+ keywords?: string[];
1141
+ /** Optional explicit weight; default 1.0. Overrides complexity-derived weight. */
1142
+ weight?: number;
1143
+ /** Implementation complexity class. Default `render`. */
1144
+ complexity?: ConceptComplexity;
1145
+ }
1146
+ interface SemanticConceptJudgeInput {
1147
+ /** Full natural-language prompt the agent was handed. */
1148
+ userRequest: string;
1149
+ /** Rendered HTML the preview returns (UI artifacts). Optional. */
1150
+ servedHtml?: string;
1151
+ /** Top-level source files from the agent's workdir. */
1152
+ sourceFiles: Array<{
1153
+ path: string;
1154
+ content: string;
1155
+ }>;
1156
+ /** The expected concept list. */
1157
+ expectedConcepts: ConceptSpec[];
1158
+ /** Free-form metadata (id, difficulty) to inject into the prompt. */
1159
+ artifactLabel?: string;
1160
+ artifactDescription?: string;
1161
+ }
1162
+ /**
1163
+ * Score-aggregation strategy. `mean` averages 0-10 scores uniformly.
1164
+ * `complexity` applies the default weight table (render=1, integrate=2,
1165
+ * compute=2.5) unless a concept has an explicit `weight`. `explicit`
1166
+ * honors only `weight` (defaulting to 1 for unspecified).
1167
+ */
1168
+ type ConceptWeightStrategy = 'mean' | 'complexity' | 'explicit';
1169
+ interface SemanticConceptJudgeOptions {
1170
+ /** Model id to call. Default 'claude-sonnet-4-6' via agent-eval defaults. */
1171
+ model?: string;
1172
+ /** Per-call timeout. Default 300s. */
1173
+ timeoutMs?: number;
1174
+ /** Provider-enforced output limit. Default 16000. */
1175
+ maxTokens?: number;
1176
+ /** Pipeline budget for the prompt (source blob truncation). Default 45000. */
1177
+ maxSourceChars?: number;
1178
+ /** Per-file cap before inclusion. Default 20000. */
1179
+ maxPerFileChars?: number;
1180
+ /** HTML cap. Default 30000. */
1181
+ maxHtmlChars?: number;
1182
+ /** LlmClient config (baseUrl, apiKey, authHeader, …). */
1183
+ llm?: LlmClientOptions;
1184
+ costLedger?: CostLedger;
1185
+ costPhase?: string;
1186
+ signal?: AbortSignal;
1187
+ /**
1188
+ * Score aggregation strategy. Default `mean` — uniform average across
1189
+ * concepts. Cross-vertical comparisons should use `complexity` to
1190
+ * neutralize the integrate-vs-render asymmetry.
1191
+ */
1192
+ weightConcepts?: ConceptWeightStrategy;
1193
+ /** Override the default complexity → weight table. */
1194
+ complexityWeights?: Partial<Record<ConceptComplexity, number>>;
1195
+ }
1196
+
1197
+ interface Scenario {
1198
+ id: string;
1199
+ persona: string;
1200
+ label: string;
1201
+ thesis: string;
1202
+ dimensions: string[];
1203
+ turns: Turn[];
1204
+ artifactChecks: ArtifactCheck[];
1205
+ systemPromptAppend?: string;
1206
+ }
1207
+ interface Turn {
1208
+ user: string;
1209
+ expectedBehaviors: string[];
1210
+ adversarial?: boolean;
1211
+ feedbackType?: 'correction' | 'rejection' | 'vague' | 'contradictory' | 'escalation';
1212
+ }
1213
+ interface ArtifactCheck {
1214
+ type: 'vault_file_exists' | 'vault_file_contains' | 'block_extracted' | 'code_valid' | 'generation_produced' | 'tool_created' | string;
1215
+ target: string;
1216
+ contains?: string;
1217
+ minCount?: number;
1218
+ description: string;
1219
+ }
1220
+ interface TurnResult {
1221
+ turnIndex: number;
1222
+ userMessage: string;
1223
+ agentResponse: string;
1224
+ durationMs: number;
1225
+ blocksExtracted: {
1226
+ type: string;
1227
+ title: string;
1228
+ }[];
1229
+ containsCode: boolean;
1230
+ containsToolCall: boolean;
1231
+ }
1232
+ interface JudgeScore {
1233
+ judgeName: string;
1234
+ dimension: string;
1235
+ score: number;
1236
+ reasoning: string;
1237
+ evidence?: string;
1238
+ }
1239
+ interface CollectedArtifacts {
1240
+ vaultFiles: {
1241
+ path: string;
1242
+ content: string;
1243
+ }[];
1244
+ blocksExtracted: {
1245
+ type: string;
1246
+ fields: Record<string, string>;
1247
+ }[];
1248
+ codeBlocks: {
1249
+ language: string;
1250
+ code: string;
1251
+ }[];
1252
+ toolCalls: string[];
1253
+ }
1254
+ interface JudgeInput {
1255
+ scenario: Scenario;
1256
+ turns: TurnResult[];
1257
+ artifacts: CollectedArtifacts;
1258
+ /** Shared ledger for paid built-in judges. Direct calls default to an uncapped ledger. */
1259
+ costLedger?: CostLedger;
1260
+ costPhase?: string;
1261
+ costTags?: Record<string, string>;
1262
+ signal?: AbortSignal;
1263
+ /** Exact maximum provider attempts configured on the supplied TCloud client. */
1264
+ tcloudMaximumAttempts?: number;
1265
+ }
1266
+ type JudgeFn = (tc: TCloud, input: JudgeInput) => Promise<JudgeScore[]>;
1267
+
1268
+ /**
1269
+ * Shared types for the trace-analyst module.
1270
+ *
1271
+ * Wire format. The store interface speaks `OtlpSpanLike` rows — one JSONL
1272
+ * line per span, OTLP-shaped. We do NOT depend on a specific tracing
1273
+ * vendor at the type level. Adapter
1274
+ * layers map upstream shapes onto this interface.
1275
+ *
1276
+ * Design constraint. Every read operation that can return arbitrary
1277
+ * payload must carry a byte budget so the agent's tool result stays
1278
+ * bounded regardless of input trace size. Oversized responses
1279
+ * substitute a deterministic summary instead of bytes — see
1280
+ * `ViewTraceOversized`.
1281
+ */
1282
+ /** OTLP span kind (subset we actually use). */
1283
+ type TraceAnalystSpanKind = 'AGENT' | 'LLM' | 'TOOL' | 'CHAIN' | 'GUARDRAIL' | 'SPAN' | 'UNKNOWN';
1284
+ type TraceAnalystSpanStatus = 'OK' | 'ERROR' | 'UNSET';
1285
+ /** Subset of OTLP span fields the analyst exposes to the agent. The
1286
+ * store's job is to project upstream's full span shape down to this
1287
+ * view — the analyst never sees vendor extensions directly. */
1288
+ interface TraceAnalystSpan {
1289
+ trace_id: string;
1290
+ span_id: string;
1291
+ parent_span_id: string | null;
1292
+ name: string;
1293
+ kind: TraceAnalystSpanKind;
1294
+ start_time: string;
1295
+ end_time: string;
1296
+ duration_ms: number;
1297
+ status: TraceAnalystSpanStatus;
1298
+ status_message?: string;
1299
+ service_name: string | null;
1300
+ agent_name: string | null;
1301
+ model_name: string | null;
1302
+ tool_name: string | null;
1303
+ /** Raw JSON-serialisable attribute map. May contain large strings;
1304
+ * callers must respect the per-attribute byte cap. */
1305
+ attributes: Record<string, unknown>;
1306
+ }
1307
+ interface TraceAnalystTraceSummary {
1308
+ trace_id: string;
1309
+ service_name: string | null;
1310
+ agent_name: string | null;
1311
+ span_count: number;
1312
+ has_errors: boolean;
1313
+ start_time: string;
1314
+ end_time: string;
1315
+ duration_ms: number;
1316
+ raw_jsonl_bytes: number;
1317
+ models: string[];
1318
+ tools: string[];
1319
+ }
1320
+ interface TraceAnalystFilters {
1321
+ /** Restrict to traces that contain at least one error span. */
1322
+ has_errors?: boolean;
1323
+ /** Match if any span's `service.name` is in this list. */
1324
+ service_names?: string[];
1325
+ /** Match if any span's `agent.name` is in this list. */
1326
+ agent_names?: string[];
1327
+ /** Match if any LLM span's `llm.model_name` is in this list. */
1328
+ model_names?: string[];
1329
+ /** Match if any tool span's `tool.name` is in this list. */
1330
+ tool_names?: string[];
1331
+ /** ISO-8601 lower bound on the trace's earliest start time. */
1332
+ start_time_after?: string;
1333
+ /** ISO-8601 upper bound on the trace's earliest start time. */
1334
+ start_time_before?: string;
1335
+ /** Single regex applied to raw JSONL bytes for the trace. Opt-in;
1336
+ * expensive on large datasets. Use the indexed filters above first. */
1337
+ regex_pattern?: string;
1338
+ }
1339
+ /** One distinct error signature across the dataset — the deterministic unit of
1340
+ * failure coverage. Signatures normalize volatile tokens (digits, hex/uuids,
1341
+ * paths, durations) out of the span `status_message` so semantically identical
1342
+ * failures collapse into one cluster. An analyst that accounts for every
1343
+ * cluster has, by construction, covered every distinct failure mode. */
1344
+ interface ErrorCluster {
1345
+ /** Normalized status_message — the cluster key. */
1346
+ signature: string;
1347
+ /** A verbatim, un-normalized exemplar message (for exact-string citation). */
1348
+ status_message_sample: string;
1349
+ /** The span name that most often carries this signature, if any. */
1350
+ span_name: string | null;
1351
+ /** The tool that most often carries this signature, if any. */
1352
+ tool_name: string | null;
1353
+ trace_count: number;
1354
+ span_count: number;
1355
+ /** trace_count / total error traces in the matched set (0..1). */
1356
+ prevalence: number;
1357
+ /** Real trace ids carrying this signature (capped), passable to view/search. */
1358
+ exemplar_trace_ids: string[];
1359
+ /** Real span ids carrying this signature (capped). */
1360
+ exemplar_span_ids: string[];
1361
+ }
1362
+ interface DatasetOverview {
1363
+ total_traces: number;
1364
+ raw_jsonl_bytes: number;
1365
+ services: string[];
1366
+ agents: string[];
1367
+ models: string[];
1368
+ tool_names: string[];
1369
+ /** Up to 20 real trace ids the agent may pass to view/search tools. */
1370
+ sample_trace_ids: string[];
1371
+ errors: {
1372
+ trace_count: number;
1373
+ span_count: number;
1374
+ };
1375
+ /** The COMPLETE deterministic error-signature population, sorted by
1376
+ * trace_count desc. This is the failure-coverage checklist: an analysis is
1377
+ * complete only when every cluster here is accounted for. Empty when the
1378
+ * matched set has no error spans. */
1379
+ error_clusters: ErrorCluster[];
1380
+ time_range: {
1381
+ earliest: string;
1382
+ latest: string;
1383
+ } | null;
1384
+ }
1385
+ interface QueryTracesPage {
1386
+ traces: TraceAnalystTraceSummary[];
1387
+ total: number;
1388
+ has_more: boolean;
1389
+ }
1390
+ /** Full-trace view. When the response would exceed the per-call byte
1391
+ * budget, `oversized` is populated INSTEAD of `spans` so the agent
1392
+ * knows to switch to `searchTrace` / `viewSpans`. */
1393
+ interface ViewTraceResult {
1394
+ trace_id: string;
1395
+ spans?: TraceAnalystSpan[];
1396
+ oversized?: ViewTraceOversized;
1397
+ }
1398
+ interface ViewTraceOversized {
1399
+ span_count: number;
1400
+ /** Names with their counts, sorted desc. Capped at 20 entries. */
1401
+ top_span_names: Array<[string, number]>;
1402
+ /** Largest single span body (bytes after attribute-cap projection). */
1403
+ span_response_bytes_max: number;
1404
+ error_span_count: number;
1405
+ }
1406
+ interface ViewSpansResult {
1407
+ trace_id: string;
1408
+ spans: TraceAnalystSpan[];
1409
+ /** Number of requested span ids that were not found in the trace. */
1410
+ missing_span_ids: string[];
1411
+ /** Number of attribute fields truncated to fit the per-attribute cap. */
1412
+ truncated_attribute_count: number;
1413
+ }
1414
+ interface SpanMatchRecord {
1415
+ trace_id: string;
1416
+ span_id: string;
1417
+ span_name: string;
1418
+ span_kind: TraceAnalystSpanKind;
1419
+ /** JSON pointer-style path to the matched value, e.g.
1420
+ * `attributes."llm.input_messages"[2].content`. */
1421
+ attribute_path: string;
1422
+ matched_text: string;
1423
+ context_before: string;
1424
+ context_after: string;
1425
+ match_offset: number;
1426
+ }
1427
+ interface SearchTraceResult {
1428
+ trace_id: string;
1429
+ hits: SpanMatchRecord[];
1430
+ total_matches: number;
1431
+ has_more: boolean;
1432
+ }
1433
+ interface SearchSpanResult {
1434
+ trace_id: string;
1435
+ span_id: string;
1436
+ hits: SpanMatchRecord[];
1437
+ total_matches: number;
1438
+ has_more: boolean;
1439
+ }
1440
+
1441
+ /**
1442
+ * `TraceAnalysisStore` — read-side interface the trace-analyst calls
1443
+ * through. Six operations, all bounded:
1444
+ *
1445
+ * - `getOverview(filters?)` — dataset rollup + sample trace ids.
1446
+ * - `queryTraces(filters?, limit, offset)` — paginated summaries.
1447
+ * - `countTraces(filters?)` — cheap count without materialisation.
1448
+ * - `viewTrace(trace_id, perAttrCap)` — full span list, oversized → summary.
1449
+ * - `viewSpans(trace_id, span_ids, perAttrCap)` — surgical span fetch.
1450
+ * - `searchTrace(trace_id, regex, max_matches)` — bounded regex hits.
1451
+ * - `searchSpan(trace_id, span_id, regex, max_matches)` — single-span search.
1452
+ *
1453
+ * Multiple implementations ship in the core (`OtlpFileTraceStore`).
1454
+ * Downstream callers can supply their own — e.g. a DuckDB-backed
1455
+ * adapter or an in-memory adapter for tests — by implementing this
1456
+ * interface.
1457
+ *
1458
+ * Filters compose with AND semantics. Empty/undefined fields impose
1459
+ * no constraint. `regex_pattern` is the only opt-in raw-bytes scan —
1460
+ * implementations may skip it via `count`/`overview` when not set.
1461
+ */
1462
+
1463
+ interface TraceAnalysisStore {
1464
+ getOverview(filters?: TraceAnalystFilters): Promise<DatasetOverview>;
1465
+ queryTraces(opts: {
1466
+ filters?: TraceAnalystFilters;
1467
+ limit: number;
1468
+ offset?: number;
1469
+ }): Promise<QueryTracesPage>;
1470
+ countTraces(filters?: TraceAnalystFilters): Promise<number>;
1471
+ viewTrace(opts: {
1472
+ trace_id: string;
1473
+ /** Override per-attribute byte cap. Defaults to discovery budget. */
1474
+ per_attribute_byte_cap?: number;
1475
+ }): Promise<ViewTraceResult>;
1476
+ viewSpans(opts: {
1477
+ trace_id: string;
1478
+ span_ids: readonly string[];
1479
+ /** Override per-attribute byte cap. Defaults to surgical budget. */
1480
+ per_attribute_byte_cap?: number;
1481
+ }): Promise<ViewSpansResult>;
1482
+ searchTrace(opts: {
1483
+ trace_id: string;
1484
+ regex_pattern: string;
1485
+ /** Hard cap on matches returned. Default 50. */
1486
+ max_matches?: number;
1487
+ }): Promise<SearchTraceResult>;
1488
+ searchSpan(opts: {
1489
+ trace_id: string;
1490
+ span_id: string;
1491
+ regex_pattern: string;
1492
+ max_matches?: number;
1493
+ }): Promise<SearchSpanResult>;
1494
+ }
1495
+
1496
+ /**
1497
+ * ChatClient — the single LLM abstraction analysts call.
1498
+ *
1499
+ * agent-eval already ships an `LlmClient` (OpenAI-compatible, retry,
1500
+ * graceful JSON-schema degrade) and judges that talk to `TCloud`. Two
1501
+ * mixed patterns force every analyst author to pick a transport, which
1502
+ * couples analyst code to runtime concerns (cli-bridge vs router vs
1503
+ * sandbox-sdk) it shouldn't know about.
1504
+ *
1505
+ * `ChatClient` is one interface every analyst takes via `AnalystContext.chat`.
1506
+ * The operator decides at the registry boundary which transport binds
1507
+ * to it. Analyst code stays transport-agnostic; swapping production
1508
+ * (sandbox-sdk) for local dev (cli-bridge) or tests (mock) is a one-
1509
+ * line factory call.
1510
+ *
1511
+ * Designed to coexist: existing `LlmClient` callers and existing
1512
+ * `TCloud`-based judges keep working untouched. New analyst code uses
1513
+ * `ChatClient`. When old call sites migrate, they pick up budgeting,
1514
+ * cancellation, and unified telemetry for free.
1515
+ */
1516
+
1517
+ /**
1518
+ * Unified chat interface. Mirrors LlmCallRequest/Result so the OpenAI-
1519
+ * compatible mental model stays. Two methods: a one-shot `chat()` and
1520
+ * an `streamChat()` for future agentic loops (not yet exposed).
1521
+ */
1522
+ interface ChatClient {
1523
+ /** Display name of the bound transport — included in telemetry. */
1524
+ readonly transport: ChatTransport;
1525
+ /** Default model when caller omits — operators bind this per environment. */
1526
+ readonly defaultModel?: string;
1527
+ /** Total provider attempts this transport can make for one chat call. */
1528
+ readonly maximumAttempts?: number;
1529
+ /** Implementations must enforce `req.maxTokens` when it is present. */
1530
+ chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>;
1531
+ }
1532
+ type ChatTransport = 'router' | 'sandbox-sdk' | 'cli-bridge' | 'direct-provider' | 'mock';
1533
+ interface ChatRequest extends Omit<LlmCallRequest, 'model'> {
1534
+ /** Optional — falls back to ChatClient.defaultModel. */
1535
+ model?: string;
1536
+ }
1537
+ type ChatResponse = LlmCallResult;
1538
+ interface ChatCallOpts {
1539
+ /** Cancel the in-flight request. */
1540
+ signal?: AbortSignal;
1541
+ /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */
1542
+ maxCostUsd?: number;
1543
+ /** Correlation tag carried into request headers when the transport allows. */
1544
+ correlationId?: string;
1545
+ /** Stable provider idempotency key for retries/redrives of one paid call. */
1546
+ idempotencyKey?: string;
1547
+ }
1548
+ type CreateChatClientOpts = RouterTransportOpts | CliBridgeTransportOpts | DirectProviderTransportOpts | SandboxSdkTransportOpts | MockTransportOpts;
1549
+ interface BaseTransportOpts {
1550
+ defaultModel?: string;
1551
+ /** Total provider attempts. Required for opaque transports used in capped runs. */
1552
+ maximumAttempts?: number;
1553
+ }
1554
+ interface RouterTransportOpts extends BaseTransportOpts {
1555
+ transport: 'router';
1556
+ baseUrl?: string;
1557
+ apiKey: string;
1558
+ }
1559
+ interface CliBridgeTransportOpts extends BaseTransportOpts {
1560
+ transport: 'cli-bridge';
1561
+ baseUrl?: string;
1562
+ bearer?: string;
1563
+ }
1564
+ interface DirectProviderTransportOpts extends BaseTransportOpts {
1565
+ transport: 'direct-provider';
1566
+ baseUrl: string;
1567
+ apiKey: string;
1568
+ }
1569
+ /**
1570
+ * Sandbox-SDK transport. Provided as a thin pass-through: the caller
1571
+ * supplies a callable that mimics LlmClient.chat() against an already-
1572
+ * configured Sandbox handle. We don't import the SDK here to keep
1573
+ * agent-eval dep-free of @tangle-network/sandbox.
1574
+ */
1575
+ interface SandboxSdkTransportOpts extends BaseTransportOpts {
1576
+ transport: 'sandbox-sdk';
1577
+ chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
1578
+ }
1579
+ /**
1580
+ * Mock transport for tests. The handler receives the request and returns
1581
+ * whatever the test wants. No retries, no JSON-schema degrade.
1582
+ */
1583
+ interface MockTransportOpts extends BaseTransportOpts {
1584
+ transport: 'mock';
1585
+ handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
1586
+ }
1587
+ /**
1588
+ * Build a ChatClient bound to a specific transport. The returned client
1589
+ * is safe to share across analysts in a single registry run.
1590
+ */
1591
+ declare function createChatClient(opts: CreateChatClientOpts): ChatClient;
1592
+
1593
+ /**
1594
+ * Analyst contract — the missing orchestration layer over agent-eval's
1595
+ * existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,
1596
+ * SemanticConceptJudge, JudgeFn, ...).
1597
+ *
1598
+ * Each existing primitive returns its own output shape. The Analyst
1599
+ * contract is the single envelope every primitive lifts into, so a
1600
+ * registry can run N analysts against a run and a single renderer can
1601
+ * compose findings without knowing which analyzer produced them.
1602
+ *
1603
+ * The contract is intentionally domain-agnostic: nothing here knows
1604
+ * about code, voice, RAG, or any particular agent stack. Analysts
1605
+ * declare what INPUT KIND they need (a trace store, an artifact dir,
1606
+ * a RunRecord, a JudgeInput, or `custom`), and the registry routes
1607
+ * the matching input from `AnalystRunInputs`.
1608
+ */
1609
+
1610
+ /**
1611
+ * Unified envelope every analyst emits. Schema-versioned so renderers
1612
+ * and time-series diffs survive future field additions.
1613
+ */
1614
+ interface AnalystFinding {
1615
+ schema_version: '1.0.0';
1616
+ /**
1617
+ * Stable hash over identity-defining fields (analyst_id + canonical
1618
+ * claim + area + optional subject). Two findings from two runs that
1619
+ * "are the same finding" share this id — that's what `diffFindings`
1620
+ * uses to compute appeared/disappeared sets across runs.
1621
+ */
1622
+ finding_id: string;
1623
+ analyst_id: string;
1624
+ produced_at: string;
1625
+ severity: AnalystSeverity;
1626
+ /**
1627
+ * Coarse classification. Renderers group by this. Free-form so
1628
+ * domain-specific analysts can introduce categories without a
1629
+ * schema change ('agent-reasoning', 'verification', 'cost',
1630
+ * 'tool-use', 'safety', 'latency', 'data-quality', ...).
1631
+ */
1632
+ area: string;
1633
+ claim: string;
1634
+ rationale?: string;
1635
+ evidence_refs: EvidenceRef[];
1636
+ recommended_action?: string;
1637
+ validation_plan?: string;
1638
+ /** 0..1 — the analyst's own confidence. Not calibrated across analysts. */
1639
+ confidence: number;
1640
+ /**
1641
+ * Optional subject the finding is about — leaf id, agent id, request
1642
+ * id. Included in finding_id when present so per-subject findings
1643
+ * diff cleanly across runs.
1644
+ */
1645
+ subject?: string;
1646
+ /** FIREWALL provenance (docs/learning-flywheel.md): true iff this finding was
1647
+ * lifted from a JUDGE verdict (an acceptance score), not OBSERVED from the
1648
+ * agent's behavior. A judge-derived finding must NEVER be admitted as a
1649
+ * steering input — that is the held-out judge leaking into the loop. Set at
1650
+ * the lift site (createJudgeAdapter); checked by `assertNoJudgeVerdict`.
1651
+ * Provenance, not evidence presence, is the correct discriminator: an
1652
+ * evidence-less trace-analyst observation legitimately steers, while a judge
1653
+ * verdict that happens to cite an artifact must not. */
1654
+ derived_from_judge?: boolean;
1655
+ /** Analyst-private extras; renderers ignore unless they know the analyst. */
1656
+ metadata?: Record<string, unknown>;
1657
+ }
1658
+ type AnalystSeverity = 'critical' | 'high' | 'medium' | 'low' | 'info';
1659
+ interface EvidenceRef {
1660
+ /**
1661
+ * Where the evidence lives. `span` and `event` refer to OTLP trace
1662
+ * elements; `artifact` to a file inside the run's artifact tree;
1663
+ * `finding` to another AnalystFinding (cross-analyst chaining);
1664
+ * `metric` to a named scalar reading the renderer knows how to read.
1665
+ */
1666
+ kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric';
1667
+ uri: string;
1668
+ excerpt?: string;
1669
+ }
1670
+ /**
1671
+ * The discriminator the registry uses to pass the right input.
1672
+ * `custom` is the escape hatch — analysts that need something else
1673
+ * (e.g. an embedding cache, a partner SDK handle) read it from
1674
+ * `AnalystRunInputs.custom[<analyst id>]`.
1675
+ */
1676
+ type AnalystInputKind = 'trace-store' | 'artifact-dir' | 'run-record' | 'judge-input' | 'custom';
1677
+ interface AnalystCost {
1678
+ /** `deterministic` analysts MUST NOT call the LLM. */
1679
+ kind: 'deterministic' | 'llm';
1680
+ /** Optional declared upper bound; the registry can enforce a budget. */
1681
+ est_usd_per_run?: number;
1682
+ /** Models the analyst expects to use (informational). */
1683
+ models?: string[];
1684
+ }
1685
+ interface AnalystRequirements {
1686
+ /** Min number of shots / samples the analyst needs to produce signal. */
1687
+ min_shots?: number;
1688
+ /** Capabilities the runtime must supply (e.g. ['network', 'gpu']). */
1689
+ capabilities?: string[];
1690
+ }
1691
+ /**
1692
+ * What's passed to every analyst call. The registry resolves which
1693
+ * field the analyst's `inputKind` selects and asserts it's present.
1694
+ */
1695
+ interface AnalystRunInputs {
1696
+ traceStore?: TraceAnalysisStore;
1697
+ artifactDir?: string;
1698
+ runRecord?: RunRecord;
1699
+ judgeInput?: JudgeInput;
1700
+ /** Keyed by analyst id; populated by callers that registered custom analysts. */
1701
+ custom?: Record<string, unknown>;
1702
+ }
1703
+ interface AnalystContext {
1704
+ runId: string;
1705
+ /** Stable correlation id so logs from a single registry.run() share a tag. */
1706
+ correlationId: string;
1707
+ /** Wall-clock deadline (epoch ms). Analysts SHOULD honor for graceful cancel. */
1708
+ deadlineMs?: number;
1709
+ /** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
1710
+ budgetUsd?: number;
1711
+ /**
1712
+ * Shared chat client. Analysts that call an LLM go through this so
1713
+ * the operator picks transport (sandbox-sdk | router | cli-bridge |
1714
+ * direct-provider | mock) at the registry boundary without touching
1715
+ * analyst code.
1716
+ */
1717
+ chat?: ChatClient;
1718
+ /**
1719
+ * Findings from a prior run the operator wants the analyst to see as
1720
+ * retrieval context. Kinds that take advantage of cross-run memory
1721
+ * (failure-mode "I saw this cluster last run", knowledge-gap "the wiki
1722
+ * page I asked for is still missing") render these into the actor's
1723
+ * working set. Filtering is the operator's job: pass the slice that
1724
+ * matches the analyst's id, or pass everything and let the kind
1725
+ * filter. Empty / absent means no cross-run context.
1726
+ */
1727
+ priorFindings?: ReadonlyArray<AnalystFinding>;
1728
+ /** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */
1729
+ tags?: Record<string, string>;
1730
+ /** Logger callback — analysts SHOULD prefer this over console.* for testability. */
1731
+ log?: (msg: string, fields?: Record<string, unknown>) => void;
1732
+ /** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */
1733
+ signal?: AbortSignal;
1734
+ }
1735
+ /**
1736
+ * The minimal contract. Concrete analysts can refine `TInput` so
1737
+ * implementations stay type-safe (e.g. a trace analyst's `TInput` is
1738
+ * `TraceAnalysisStore`); the registry passes the right field from
1739
+ * `AnalystRunInputs` based on `inputKind`.
1740
+ */
1741
+ interface Analyst<TInput = unknown> {
1742
+ /** Stable identifier — appears in finding_id, telemetry, and registry exclusion lists. */
1743
+ readonly id: string;
1744
+ /** Human-readable. One sentence. */
1745
+ readonly description: string;
1746
+ readonly inputKind: AnalystInputKind;
1747
+ readonly cost: AnalystCost;
1748
+ readonly requires?: AnalystRequirements;
1749
+ /** Bump on breaking changes to claim wording or area so old finding_ids don't collide. */
1750
+ readonly version: string;
1751
+ analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>;
1752
+ }
1753
+ /**
1754
+ * Compute the stable finding_id from the identity-defining fields.
1755
+ * Default implementation hashes {analyst_id, area, subject, normalized claim}.
1756
+ * Analysts that emit findings whose claim text varies per run (timestamps,
1757
+ * counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,
1758
+ * or (b) move the variable part into `rationale`/`metadata` and keep the
1759
+ * `claim` static.
1760
+ */
1761
+ declare function computeFindingId(input: {
1762
+ analyst_id: string;
1763
+ area: string;
1764
+ subject?: string;
1765
+ claim: string;
1766
+ /** Override the claim for hashing — use when the displayed claim has run-specific bits. */
1767
+ id_basis?: string;
1768
+ }): string;
1769
+ /**
1770
+ * Convenience factory: produce a fully-formed AnalystFinding with the
1771
+ * id computed automatically. Analyst code stays terse.
1772
+ */
1773
+ declare function makeFinding(init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {
1774
+ id_basis?: string;
1775
+ produced_at?: string;
1776
+ }): AnalystFinding;
1777
+ interface AnalystRunSummary {
1778
+ analyst_id: string;
1779
+ status: 'ok' | 'skipped' | 'failed';
1780
+ /** Why skipped — missing input, budget exceeded, capability unmet. */
1781
+ reason?: string;
1782
+ findings_count: number;
1783
+ latency_ms: number;
1784
+ cost_usd: number;
1785
+ /** When `status='failed'`: the error class + message, never the full stack. */
1786
+ error?: {
1787
+ class: string;
1788
+ message: string;
1789
+ };
1790
+ }
1791
+ interface AnalystRunResult {
1792
+ run_id: string;
1793
+ correlation_id: string;
1794
+ started_at: string;
1795
+ ended_at: string;
1796
+ findings: AnalystFinding[];
1797
+ per_analyst: AnalystRunSummary[];
1798
+ /** Total LLM cost in USD across all analysts in this registry.run(). */
1799
+ total_cost_usd: number;
1800
+ }
1801
+ /**
1802
+ * Events emitted by `AnalystRegistry.runStream(...)` in real time as
1803
+ * the registry executes. UIs subscribe via `for await (const ev of
1804
+ * registry.runStream(...))`; `registry.run(...)` is a thin collector
1805
+ * over the same stream, so the two surfaces share their invariants.
1806
+ *
1807
+ * Per-finding events are intentionally omitted — analyzers are batch
1808
+ * operations (an Ax actor returns the full `findings:json[]` at the
1809
+ * end of the responder), so streaming inside one analyst would only
1810
+ * emit partial JSON consumers can't render. The kind-completion event
1811
+ * is the right granularity; subscribers wanting per-finding rendering
1812
+ * iterate `event.findings` themselves.
1813
+ */
1814
+ type AnalystRunEvent = {
1815
+ type: 'run-started';
1816
+ run_id: string;
1817
+ correlation_id: string;
1818
+ started_at: string;
1819
+ /** The ordered list of analyst ids the registry will run. */
1820
+ analyst_ids: ReadonlyArray<string>;
1821
+ } | {
1822
+ type: 'analyst-skipped';
1823
+ summary: AnalystRunSummary;
1824
+ } | {
1825
+ type: 'analyst-started';
1826
+ analyst_id: string;
1827
+ started_at: string;
1828
+ } | {
1829
+ type: 'analyst-completed';
1830
+ /** `summary.status` is `'ok'` for clean completion or `'failed'` for thrown analysts. */
1831
+ summary: AnalystRunSummary;
1832
+ findings: ReadonlyArray<AnalystFinding>;
1833
+ } | {
1834
+ type: 'run-completed';
1835
+ result: AnalystRunResult;
1836
+ };
22
1837
 
23
1838
  /**
24
1839
  * Adapter factories — lift each existing agent-eval primitive into the
@@ -78,6 +1893,78 @@ interface SemanticConceptJudgeAdapterOpts {
78
1893
  }
79
1894
  declare function createSemanticConceptJudgeAdapter(opts?: SemanticConceptJudgeAdapterOpts): Analyst<SemanticConceptJudgeInput>;
80
1895
 
1896
+ interface CreateAnalystAiConfig {
1897
+ /** OpenAI-compatible API key forwarded as `Authorization: Bearer`.
1898
+ * cli-bridge ignores the value on loopback but Ax requires a non-empty string. */
1899
+ apiKey: string;
1900
+ /** OpenAI-compatible base URL — e.g. `https://router.tangle.tools/v1` or a
1901
+ * cli-bridge loopback. */
1902
+ baseUrl: string;
1903
+ /** Model id forwarded to the analyst actor + responder. */
1904
+ model: string;
1905
+ /** Ax provider name. Defaults to the OpenAI-compatible client. */
1906
+ provider?: 'openai' | 'anthropic';
1907
+ }
1908
+ /**
1909
+ * Construct the `AxAIService` an analyst kind calls through
1910
+ * (`createTraceAnalystKind({ ai })`).
1911
+ *
1912
+ * Ax's `ai()` pins `config.model` to the OpenAI catalog enum, but every
1913
+ * OpenAI-compatible router an analyst points at (router.tangle.tools,
1914
+ * cli-bridge) accepts arbitrary model ids (claude-code/sonnet, openai/gpt-5.4,
1915
+ * …). Consumers were each re-rolling `ai({ name, apiKey, apiURL, config })`
1916
+ * behind an `as (a: any) => any` cast to dodge the enum; this is the one
1917
+ * canonical constructor so they don't have to — and don't take a direct
1918
+ * `@ax-llm/ax` dependency for it.
1919
+ */
1920
+ declare function createAnalystAi(config: CreateAnalystAiConfig): AxAIService;
1921
+
1922
+ /**
1923
+ * Deterministic behavioral metrics over OTLP spans — pure arithmetic, no LLM.
1924
+ *
1925
+ * These are the model-independent multiplier: the four trace-quality signals a
1926
+ * tolerant analyzer (e.g. HALO) re-derives per run inside the model — token
1927
+ * growth, output decay, tool monoculture, missing self-verification — computed
1928
+ * here once, in TypeScript, with zero model judgment. A finding that falls out
1929
+ * of arithmetic is trivially model-agnostic and cannot hallucinate the trend.
1930
+ *
1931
+ * General, not trace-specific: the detectors key off token trajectories and
1932
+ * tool usage present in any agentic OTLP trace, not any one benchmark.
1933
+ */
1934
+
1935
+ type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
1936
+ interface SuboptimalSignal {
1937
+ code: SuboptimalCode;
1938
+ severity: 'high' | 'medium' | 'low';
1939
+ /** Human-readable claim, with the backing numbers inlined. */
1940
+ detail: string;
1941
+ /** The exact figures the detector fired on — auditable, no model in the loop. */
1942
+ evidence: Record<string, number | string | boolean>;
1943
+ }
1944
+ interface BehavioralMetrics {
1945
+ /** The only trace represented by these metrics; null when spans are empty. */
1946
+ traceId: string | null;
1947
+ llmCallCount: number;
1948
+ /** Causally serial LLM timelines. Parallel branches are never joined. */
1949
+ tokenSequences: BehavioralTokenSequence[];
1950
+ /** Token values from the longest serial timeline, retained for convenience. */
1951
+ inputTokenTrajectory: number[];
1952
+ outputTokenTrajectory: number[];
1953
+ toolHistogram: Record<string, number>;
1954
+ totalToolCalls: number;
1955
+ distinctTools: number;
1956
+ /** distinct/total tool calls; 1.0 when there are no tool calls. */
1957
+ toolDiversityRatio: number;
1958
+ hasSelfVerification: boolean;
1959
+ signals: SuboptimalSignal[];
1960
+ }
1961
+ interface BehavioralTokenSequence {
1962
+ scopeId: string;
1963
+ spanIds: string[];
1964
+ inputTokenTrajectory: Array<number | null>;
1965
+ outputTokenTrajectory: Array<number | null>;
1966
+ }
1967
+
81
1968
  /**
82
1969
  * `behavioralAnalyst` — a DETERMINISTIC analyst (cost.kind = 'deterministic',
83
1970
  * never calls the LLM). It produces the efficiency/behavioral findings a
@@ -104,6 +1991,764 @@ declare function deriveEfficiencyFindings(metrics: BehavioralMetrics, opts?: {
104
1991
  /** The deterministic behavioral/efficiency analyst (no LLM, any-model). */
105
1992
  declare function behavioralAnalyst(): Analyst<TraceAnalysisStore>;
106
1993
 
1994
+ /**
1995
+ * Typed Ax output for analyst findings.
1996
+ *
1997
+ * Replaces the legacy `findings:string[]` pattern (where every bullet
1998
+ * became a flat-severity `AnalystFinding`) with a structured object
1999
+ * array. Ax binds the field as `findings:json[]` so the provider emits
2000
+ * native structured output; at the kind-factory boundary we Zod-validate
2001
+ * each emitted finding so malformed rows fail loud instead of being
2002
+ * silently lifted with default severity.
2003
+ *
2004
+ * Why not `f.object().array()` directly in the signature? The Ax
2005
+ * signature string `question:string -> findings:json[]` already lets
2006
+ * the provider emit JSON arrays. A Zod boundary is required either
2007
+ * way (the provider can return any JSON), and Zod gives us a single
2008
+ * validation surface independent of which Ax version is installed.
2009
+ */
2010
+
2011
+ declare const ANALYST_SEVERITIES: readonly ["critical", "high", "medium", "low", "info"];
2012
+ declare const RawAnalystFindingSchema: z.ZodObject<{
2013
+ severity: z.ZodEnum<{
2014
+ critical: "critical";
2015
+ info: "info";
2016
+ low: "low";
2017
+ high: "high";
2018
+ medium: "medium";
2019
+ }>;
2020
+ claim: z.ZodString;
2021
+ subject: z.ZodOptional<z.ZodString>;
2022
+ evidence_uri: z.ZodString;
2023
+ evidence_excerpt: z.ZodOptional<z.ZodString>;
2024
+ confidence: z.ZodNumber;
2025
+ rationale: z.ZodOptional<z.ZodString>;
2026
+ recommended_action: z.ZodOptional<z.ZodString>;
2027
+ }, z.core.$strict>;
2028
+ type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
2029
+ /**
2030
+ * Description embedded into the actor prompt so the LLM knows what
2031
+ * shape to emit. Kept here so kinds share one source of truth rather
2032
+ * than restating the schema in every prompt.
2033
+ */
2034
+ declare const RAW_FINDING_SCHEMA_PROMPT = "Each finding MUST be a JSON object with these fields:\n - severity: one of \"critical\" | \"high\" | \"medium\" | \"low\" | \"info\"\n - claim: one-sentence statement (max 2000 chars)\n - subject?: the routing locus this finding is about. It MUST be one of the exact subject forms listed in this kind's instructions above (e.g. `system-prompt:<section>`, `agent-knowledge:wiki:<slug>`, `tool-doc:<tool>`). A free phrase, a bare noun, or any form not in that list is REJECTED at parse time and the finding is discarded \u2014 omit subject entirely rather than guess a form.\n - evidence_uri: REQUIRED, never blank. Exactly one of \"span://<trace_id>/<span_id>\" (trace evidence), \"artifact://<relative-path>\" (files), \"metric://<name>\" (named scalars) \u2014 ALWAYS cite a real id surfaced by the tools. If you have no citable id, do not emit the finding.\n - evidence_excerpt?: short quote (<=2000 chars) from the cited span/artifact\n - confidence: number 0..1 \u2014 0.9+ when backed by exact quotes, 0.6-0.8 for inferred patterns, <0.5 for speculative\n - rationale?: one or two sentences explaining the reasoning\n - recommended_action?: concrete change phrased as an imperative (\"Add ...\", \"Replace ...\", \"Stop ...\") \u2014 omit when the finding is purely descriptive\n\nEmit an empty array when the question has no findings to report. Do not fabricate evidence.";
2035
+ /**
2036
+ * Validate one row emitted by the LLM. Returns the typed finding on
2037
+ * success; returns `null` and logs the reason on failure so the kind
2038
+ * factory can skip-and-count rather than abort the whole analyst run.
2039
+ */
2040
+ declare function parseRawFinding(row: unknown, log?: (msg: string, fields?: Record<string, unknown>) => void): RawAnalystFinding | null;
2041
+
2042
+ /**
2043
+ * Analyst-kind factory — the typed way to define trace analysts.
2044
+ *
2045
+ * A "kind" is a specialized analyst whose actor prompt, tool subset,
2046
+ * and Ax recursion config target one failure-mode lens (failure-mode
2047
+ * classification, knowledge gap discovery, knowledge poisoning, recursive
2048
+ * self-improvement, ...). Kinds emit findings in the typed `RawAnalystFinding`
2049
+ * shape via a JSON-array Ax output; the factory validates each row with
2050
+ * Zod and lifts it into `AnalystFinding[]` with no shape guessing.
2051
+ *
2052
+ * Composition rules:
2053
+ * - Each kind owns its actor description. No generic "answer this
2054
+ * question" prompt — the prompt names the failure lens.
2055
+ * - Each kind picks a narrow tool subset from `ANALYST_TOOL_GROUPS`.
2056
+ * A kind that never needs full-trace dumps can drop `viewTrace` /
2057
+ * `viewSpans` and stay cheap.
2058
+ * - Each kind declares its recursion + parallelism budget. Discovery-
2059
+ * heavy kinds (failure-mode) get higher `maxDepth`; lens kinds
2060
+ * (poisoning) usually stay at 0 since they have a tighter brief.
2061
+ *
2062
+ * Optimizer hook: kinds may declare `goldens` — labeled examples used
2063
+ * by `AxMiPRO` / `AxBootstrapFewShot` / `AxGEPA` to fit the actor
2064
+ * description programmatically. Stored on the kind, not the registry,
2065
+ * because the right metric is kind-specific.
2066
+ */
2067
+
2068
+ /**
2069
+ * Per-kind specification. The factory turns this into a regular
2070
+ * `Analyst<TraceAnalysisStore>` ready for `AnalystRegistry.register()`.
2071
+ */
2072
+ interface TraceAnalystKindSpec {
2073
+ /** Stable id. Appears in finding_id, telemetry, and registry exclusions. */
2074
+ id: string;
2075
+ /** One-sentence description shown in `registry.list()`. */
2076
+ description: string;
2077
+ /** Coarse classification stamped on every emitted finding (`failure-mode`, `knowledge-gap`, ...). */
2078
+ area: string;
2079
+ /** Bump on any breaking change to the actor prompt or output schema. */
2080
+ version: string;
2081
+ /** Actor system prompt. Must instruct the LLM to emit `findings` per the schema. */
2082
+ actorDescription: string;
2083
+ /** Responder system prompt; falls back to a minimal "format the findings" instruction. */
2084
+ responderDescription?: string;
2085
+ /** Tool functions the actor may call. Pick narrow subsets via `ANALYST_TOOL_GROUPS`. */
2086
+ buildTools: (store: TraceAnalysisStore) => AxFunction[];
2087
+ /** Recursion budget. `maxDepth: 0` disables subagents. */
2088
+ recursion?: {
2089
+ maxDepth: number;
2090
+ maxParallelSubagents?: number;
2091
+ };
2092
+ /** Actor turn cap. Default 12. */
2093
+ maxTurns?: number;
2094
+ /** Runtime char cap. Default 6000. */
2095
+ maxRuntimeChars?: number;
2096
+ /** Cost classification surfaced in `registry.list()` and budget enforcement. */
2097
+ cost: AnalystCost;
2098
+ /** Per-finding-row hook — kinds may reject / rewrite before lifting. */
2099
+ postProcess?: (row: RawAnalystFinding, ctx: AnalystContext) => RawAnalystFinding | null;
2100
+ /** Optional optimizer hook — populated when a kind wants to fit its prompt against labeled examples. */
2101
+ goldens?: TraceAnalystGolden[];
2102
+ }
2103
+ /**
2104
+ * One labeled example consumed by Ax optimizers (MIPRO / GEPA / Bootstrap).
2105
+ * Each input is the same `{question}` an analyst would receive; `expected`
2106
+ * is the ground-truth finding set a fitted prompt should produce on this
2107
+ * input. Metric: kind-specific (default: F1 on `finding_id` overlap).
2108
+ */
2109
+ interface TraceAnalystGolden {
2110
+ question: string;
2111
+ expected: ReadonlyArray<Omit<RawAnalystFinding, 'confidence'>>;
2112
+ }
2113
+ interface CreateTraceAnalystKindOpts {
2114
+ /** AxAIService bound at registration time. */
2115
+ ai: AxAIService;
2116
+ /** Optional model override; falls back to the AI service's default. */
2117
+ model?: string;
2118
+ /** Override the spec's `version` (e.g. when an optimizer has fitted a new prompt). */
2119
+ versionSuffix?: string;
2120
+ /**
2121
+ * Optional two-phase recovery: when the agentic harvest is empty but the
2122
+ * actor produced a substantive free-form `report`, extract findings from that
2123
+ * prose via a tolerant chat-completions pass (`structureFindings`) — no
2124
+ * strict-emission contract, so it works on weak models. Omit to leave the
2125
+ * actor's harvest as-is (the report is still surfaced fail-loud either way).
2126
+ */
2127
+ recovery?: {
2128
+ baseUrl: string;
2129
+ apiKey?: string;
2130
+ model?: string;
2131
+ fetchImpl?: typeof fetch;
2132
+ };
2133
+ }
2134
+ /**
2135
+ * Build an `Analyst<TraceAnalysisStore>` from a kind spec.
2136
+ *
2137
+ * Lifts the Ax pipeline once at registration time so the registry
2138
+ * gets a stateless analyst. The Ax agent is freshly constructed per
2139
+ * `analyze()` call (the agent carries chat-log + usage state we don't
2140
+ * want shared across analyst runs).
2141
+ */
2142
+ declare function createTraceAnalystKind(spec: TraceAnalystKindSpec, opts: CreateTraceAnalystKindOpts): Analyst<TraceAnalysisStore>;
2143
+ /**
2144
+ * Render a compact prior-findings block the actor reads alongside its
2145
+ * brief. Each row is one line so the actor can scan dozens cheaply.
2146
+ * The kind's prompt instructs the actor to (a) check whether a new
2147
+ * cluster matches a prior `finding_id` (carry the id forward via
2148
+ * `id_basis` to keep diffs stable) and (b) raise severity / confidence
2149
+ * when a prior finding has reappeared without remediation.
2150
+ *
2151
+ * Returns the empty string when there are no prior findings — most
2152
+ * runs are "first-of-its-kind" and the prompt stays unchanged.
2153
+ *
2154
+ * Exported for tests + for consumers that build their own actor
2155
+ * prompts (e.g. specialized analysts living outside the default kinds).
2156
+ */
2157
+ declare function renderPriorFindings(prior: AnalystContext['priorFindings']): string;
2158
+
2159
+ /**
2160
+ * AnalystRegistry — orchestrate N analysts against one run.
2161
+ *
2162
+ * Owns three responsibilities and only three:
2163
+ * 1. Registration — ids must be unique; bad registrations fail loudly
2164
+ * at register-time, not run-time.
2165
+ * 2. Routing — each analyst declares its `inputKind`; the registry
2166
+ * picks the matching field from AnalystRunInputs and skips the
2167
+ * analyst with a logged reason if it's missing.
2168
+ * 3. Isolation — one analyst's exception MUST NOT stop other analysts.
2169
+ * Failed analysts produce zero findings + a 'failed' summary row.
2170
+ *
2171
+ * Cross-cutting concerns (telemetry, error → finding conversion, cost
2172
+ * ingestion, storage rotation) live in `AnalystHooks`. Budget shaping
2173
+ * (equal split vs weighted vs custom) lives in `BudgetPolicy`. Both
2174
+ * have sensible defaults; consumers override only what they need.
2175
+ */
2176
+
2177
+ interface AnalystHooks {
2178
+ /** Before analyze() — last chance to mutate ctx (e.g. inject tags, override budget). */
2179
+ onBeforeAnalyze?(args: {
2180
+ analyst: Analyst;
2181
+ ctx: AnalystContext;
2182
+ runId: string;
2183
+ }): void | Promise<void>;
2184
+ /** After every analyst (ok | failed | skipped). Use for telemetry, ingestion, rotation. */
2185
+ onAfterAnalyze?(args: {
2186
+ analyst: Analyst;
2187
+ summary: AnalystRunSummary;
2188
+ findings: AnalystFinding[];
2189
+ runId: string;
2190
+ }): void | Promise<void>;
2191
+ /**
2192
+ * On analyst exception. Hook MAY return findings to convert the
2193
+ * error into structured findings; the summary still reports 'failed'.
2194
+ * Return void to keep the default empty-findings behavior.
2195
+ */
2196
+ onError?(args: {
2197
+ analyst: Analyst;
2198
+ error: Error;
2199
+ runId: string;
2200
+ }): AnalystFinding[] | undefined | Promise<AnalystFinding[] | undefined>;
2201
+ /** Once after registry.run() completes. Use for final aggregation, persistence. */
2202
+ onComplete?(args: {
2203
+ result: AnalystRunResult;
2204
+ }): void | Promise<void>;
2205
+ }
2206
+ interface BudgetPolicy {
2207
+ /** Overall USD cap across the registry.run(). */
2208
+ totalUsd?: number;
2209
+ /** Per-analyst weight for the default allocator. Missing ids get weight 1. */
2210
+ weights?: Record<string, number>;
2211
+ /**
2212
+ * Custom allocator — receives the analyst, remaining/total budget, and
2213
+ * the count of analysts that will run. Returns the per-analyst budget
2214
+ * (or undefined to leave it uncapped). Overrides weights when set.
2215
+ */
2216
+ allocate?: (args: {
2217
+ analyst: Analyst;
2218
+ totalUsd: number | undefined;
2219
+ remainingUsd: number | undefined;
2220
+ runningCount: number;
2221
+ }) => number | undefined;
2222
+ }
2223
+ interface AnalystRegistryOptions {
2224
+ /** Shared chat client passed to every LLM analyst via AnalystContext. */
2225
+ chat?: ChatClient;
2226
+ /** Logger callback. Defaults to a no-op. */
2227
+ log?: (msg: string, fields?: Record<string, unknown>) => void;
2228
+ /** Hooks invoked around analyze() — observability + customization seam. */
2229
+ hooks?: AnalystHooks;
2230
+ /** Default budget when run() doesn't override. */
2231
+ defaultBudget?: BudgetPolicy;
2232
+ }
2233
+ interface RegistryRunOpts {
2234
+ /** Restrict to a subset of registered analysts by id. */
2235
+ only?: string[];
2236
+ /** Skip these analysts even if registered. Useful for cheap iteration. */
2237
+ skip?: string[];
2238
+ /** Budget policy — totalUsd + optional weights/allocator. Falls back to options.defaultBudget. */
2239
+ budget?: BudgetPolicy;
2240
+ /** Wall-clock cap. Analysts SHOULD honor `ctx.deadlineMs`. */
2241
+ timeoutMs?: number;
2242
+ /** Abort signal — forwarded into every analyst's context. */
2243
+ signal?: AbortSignal;
2244
+ /** Tags echoed into AnalystContext.tags — useful for tracking environment/version in findings. */
2245
+ tags?: Record<string, string>;
2246
+ /**
2247
+ * Prior-run findings made available as retrieval context to every
2248
+ * analyst via `ctx.priorFindings`. The registry forwards the slice
2249
+ * whose `analyst_id` matches each registered analyst so a kind sees
2250
+ * only its own history. Pass `{ '*': findings }` to broadcast to
2251
+ * every analyst (useful for cross-kind chaining where the improvement
2252
+ * analyst consumes upstream failure findings).
2253
+ */
2254
+ priorFindings?: ReadonlyArray<AnalystFinding> | Record<string, ReadonlyArray<AnalystFinding>>;
2255
+ }
2256
+ declare class AnalystRegistry {
2257
+ private readonly analysts;
2258
+ private readonly options;
2259
+ constructor(options?: AnalystRegistryOptions);
2260
+ register(analyst: Analyst): void;
2261
+ list(): ReadonlyArray<{
2262
+ id: string;
2263
+ description: string;
2264
+ version: string;
2265
+ cost: Analyst['cost'];
2266
+ }>;
2267
+ run(runId: string, inputs: AnalystRunInputs, runOpts?: RegistryRunOpts): Promise<AnalystRunResult>;
2268
+ /**
2269
+ * Streaming counterpart to `run()`. Emits `AnalystRunEvent` values
2270
+ * in real time — `run-started`, then per-analyst `skipped` /
2271
+ * `started` / `completed`, then a terminal `run-completed` whose
2272
+ * payload is the full `AnalystRunResult`. UIs use this to render
2273
+ * progress; persistence consumers use `run()` and read the result.
2274
+ *
2275
+ * Hooks (`onBeforeAnalyze` / `onAfterAnalyze` / `onError` /
2276
+ * `onComplete`) fire as before — streaming is additive, not a hook
2277
+ * replacement.
2278
+ */
2279
+ runStream(runId: string, inputs: AnalystRunInputs, runOpts?: RegistryRunOpts): AsyncGenerator<AnalystRunEvent, void, void>;
2280
+ private selectAnalysts;
2281
+ private routeInput;
2282
+ }
2283
+
2284
+ /**
2285
+ * `buildDefaultAnalystRegistry` — the canonical analyst suite, so consumers
2286
+ * stop hand-wiring `new AnalystRegistry()` + per-kind `createTraceAnalystKind`.
2287
+ *
2288
+ * The deterministic `behavioralAnalyst` is ALWAYS registered (it needs no
2289
+ * model and is model-agnostic by construction). The agentic RLM kinds are
2290
+ * registered only when an `ai` service is supplied — so a caller with no LLM
2291
+ * still gets the full behavioral/efficiency diagnosis, and the substrate's
2292
+ * "any model (including no model)" guarantee holds at the suite level.
2293
+ */
2294
+
2295
+ interface DefaultAnalystRegistryOptions {
2296
+ /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */
2297
+ ai?: AxAIService;
2298
+ /** Model for the agentic kinds (falls back to the ai service default). */
2299
+ model?: string;
2300
+ /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */
2301
+ kinds?: readonly TraceAnalystKindSpec[];
2302
+ /** Set false to omit the deterministic behavioral analyst (default: include). */
2303
+ includeBehavioral?: boolean;
2304
+ /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */
2305
+ registry?: AnalystRegistryOptions;
2306
+ }
2307
+ declare function buildDefaultAnalystRegistry(opts?: DefaultAnalystRegistryOptions): AnalystRegistry;
2308
+
2309
+ /**
2310
+ * Typed `FindingSubject` — the canonical grammar every analyst kind emits.
2311
+ *
2312
+ * Background: kind actor prompts have always documented a subject grammar
2313
+ * (e.g. `system-prompt:<section>`, `agent-knowledge:wiki:<slug>`) but the
2314
+ * LLM was unconstrained — it could emit `subject: "fix the prompt"`
2315
+ * (prose) and downstream adapters routed on `startsWith(...)` would
2316
+ * silently skip it. Every per-vertical `ImprovementAdapter` had a
2317
+ * routing table that mostly caught nothing.
2318
+ *
2319
+ * This module fixes that:
2320
+ * - `parseFindingSubject(raw)` — returns the typed `FindingSubject`
2321
+ * when `raw` matches the grammar, else `null`. Used at the
2322
+ * `RawAnalystFindingSchema` boundary so malformed subjects are
2323
+ * rejected loudly instead of silently lifted into the registry.
2324
+ * - `FindingSubjectKind` — the union of valid locus categories. Each
2325
+ * variant carries the typed components downstream adapters resolve
2326
+ * against the agent's surface manifest (no string parsing in the
2327
+ * adapter).
2328
+ * - `FINDING_SUBJECT_GRAMMAR_PROMPT` — single source of truth for the
2329
+ * grammar string embedded in kind actor prompts. Drift between
2330
+ * prompt and parser is impossible if every kind imports this.
2331
+ *
2332
+ * The grammar is intentionally NARROW — only loci the substrate's
2333
+ * default `ImprovementAdapter` / `KnowledgeAdapter` can act on. A
2334
+ * finding with a subject outside this set fails the parser; the kind
2335
+ * author either extends the grammar here (and adds adapter routing)
2336
+ * or rephrases the prompt to map onto an existing variant.
2337
+ *
2338
+ * `failure-mode` is the one exception — its subjects are free-form
2339
+ * cluster labels, not loci. The schema preserves them as
2340
+ * `{ kind: 'cluster', label }` and the adapters skip them (cluster
2341
+ * findings are evidence, not actionable mutations).
2342
+ */
2343
+
2344
+ /**
2345
+ * Discriminated union of every locus the substrate can route findings to.
2346
+ *
2347
+ * Adapters narrow on `kind` and use the typed components (no string
2348
+ * parsing). Adding a variant here REQUIRES updating the parser, the
2349
+ * grammar prompt, and at least one adapter — by design.
2350
+ */
2351
+ type FindingSubject = {
2352
+ kind: 'knowledge.wiki';
2353
+ slug: string;
2354
+ heading?: string;
2355
+ } | {
2356
+ kind: 'knowledge.claim';
2357
+ topic: string;
2358
+ } | {
2359
+ kind: 'knowledge.raw';
2360
+ sourceId: string;
2361
+ } | {
2362
+ kind: 'knowledge.stale';
2363
+ slug: string;
2364
+ } | {
2365
+ kind: 'system-prompt';
2366
+ section: string;
2367
+ } | {
2368
+ kind: 'skill';
2369
+ name: string;
2370
+ } | {
2371
+ kind: 'tool-doc';
2372
+ tool: string;
2373
+ aspect?: string;
2374
+ } | {
2375
+ kind: 'new-tool';
2376
+ name: string;
2377
+ } | {
2378
+ kind: 'mcp';
2379
+ server: string;
2380
+ tool?: string;
2381
+ } | {
2382
+ kind: 'hook';
2383
+ name: string;
2384
+ } | {
2385
+ kind: 'subagent';
2386
+ name: string;
2387
+ } | {
2388
+ kind: 'workflow';
2389
+ name: string;
2390
+ } | {
2391
+ kind: 'rollout-policy';
2392
+ field: string;
2393
+ } | {
2394
+ kind: 'agent-profile';
2395
+ field: string;
2396
+ } | {
2397
+ kind: 'code';
2398
+ path: string;
2399
+ } | {
2400
+ kind: 'rag';
2401
+ corpus: string;
2402
+ docId: string;
2403
+ } | {
2404
+ kind: 'memory';
2405
+ key: string;
2406
+ } | {
2407
+ kind: 'scaffolding';
2408
+ concern: string;
2409
+ } | {
2410
+ kind: 'output-schema';
2411
+ field: string;
2412
+ } | {
2413
+ kind: 'websearch.outdated';
2414
+ topic: string;
2415
+ } | {
2416
+ kind: 'prior-run-summary';
2417
+ topic: string;
2418
+ } | {
2419
+ kind: 'cluster';
2420
+ label: string;
2421
+ };
2422
+ type FindingSubjectKind = FindingSubject['kind'];
2423
+ declare const FINDING_SUBJECT_KINDS: ReadonlyArray<FindingSubjectKind>;
2424
+ /**
2425
+ * Parse a raw subject string emitted by an analyst kind's actor.
2426
+ *
2427
+ * Returns the typed `FindingSubject` when `raw` matches the grammar,
2428
+ * else `null`. Callers use the `null` return as a signal to either
2429
+ * (a) reject the finding at parse time (kinds that emit typed loci —
2430
+ * knowledge-gap, improvement, knowledge-poisoning) or (b) lift it as
2431
+ * a cluster label (failure-mode).
2432
+ *
2433
+ * Slugs are constrained to `[a-z0-9-]+` (lowercase kebab) to keep file
2434
+ * paths sane downstream. Topics / keys / sections allow any non-empty
2435
+ * string (free-form for the LLM's voice) but get trimmed.
2436
+ *
2437
+ * Empty / whitespace-only inputs return `null`. `undefined` returns
2438
+ * `null`. Both are surfaced by the caller as a rejected subject.
2439
+ */
2440
+ declare function parseFindingSubject(raw: string | null | undefined): FindingSubject | null;
2441
+ /**
2442
+ * Render the parsed subject back to its canonical string form. Inverse
2443
+ * of `parseFindingSubject`; useful when the substrate constructs new
2444
+ * findings programmatically (e.g. for tests, replays, or
2445
+ * `id_basis` carry-forward).
2446
+ */
2447
+ declare function renderFindingSubject(s: FindingSubject): string;
2448
+ /**
2449
+ * The grammar text embedded into kind actor prompts. Kinds opt into
2450
+ * the subset of variants they emit (e.g. `improvement` excludes the
2451
+ * cluster variant; `failure-mode` includes ONLY the cluster variant).
2452
+ *
2453
+ * Drift between prompt and parser is impossible: every kind imports
2454
+ * this constant + the matching `expects` set, and the unit tests below
2455
+ * lock the table to the parser.
2456
+ */
2457
+ declare const FINDING_SUBJECT_SYNTAX: Readonly<Record<FindingSubjectKind, string>>;
2458
+ declare const FINDING_SUBJECT_GRAMMAR_PROMPT: string;
2459
+ /**
2460
+ * The variants each kind is allowed to emit. Used at the kind factory
2461
+ * boundary so a knowledge-gap finding can't sneak in a `system-prompt:*`
2462
+ * subject (the improvement-analyst's job) and vice versa.
2463
+ *
2464
+ * `failure-mode` is restricted to `cluster` — the only kind that emits
2465
+ * a non-locus subject.
2466
+ */
2467
+ declare const KIND_EXPECTED_SUBJECTS: Record<string, ReadonlyArray<FindingSubjectKind>>;
2468
+ /** Render only the subject forms one analyst kind is permitted to emit. */
2469
+ declare function findingSubjectGrammarPromptFor(kindId: string): string;
2470
+ /**
2471
+ * Zod schema that validates a raw subject string and returns the parsed
2472
+ * `FindingSubject`. Embedded in `RawAnalystFindingSchema` via
2473
+ * `transform`, so `subject` arrives at the kind factory either as a
2474
+ * typed locus or as a parse error attached to a single Zod issue.
2475
+ *
2476
+ * Optionality is preserved: subjects ARE optional on the wire (some
2477
+ * findings are descriptive, not actionable). When present, they MUST
2478
+ * parse — emitting a malformed subject is a contract violation, not a
2479
+ * soft signal.
2480
+ */
2481
+ declare const FindingSubjectStringSchema: z.ZodString;
2482
+
2483
+ /**
2484
+ * FindingsStore — durable persistence for AnalystFinding rows + a diff
2485
+ * helper so we can answer "what changed since the last run?" without
2486
+ * recomputing analysts.
2487
+ *
2488
+ * On-disk shape is JSONL: one finding per line, append-only, locked via
2489
+ * LockedJsonlAppender. Operators get crash-safety (no partial JSON),
2490
+ * cheap reads (sequential parse), and trivial backup (rsync the file).
2491
+ *
2492
+ * Reads are non-locking: a reader sees a consistent snapshot of all
2493
+ * fully-written lines and skips an incomplete trailing line if the
2494
+ * writer is mid-append. Cross-process locking is intentionally out of
2495
+ * scope (see locked-jsonl-appender.ts).
2496
+ *
2497
+ * The store is run-scoped: callers pass `runId` on append and on load,
2498
+ * which keeps multi-run files cleanly partitioned. The `diffFindings`
2499
+ * helper compares two run-id sets using stable `finding_id` semantics —
2500
+ * the diff is the cross-run signal the regression dashboard renders.
2501
+ */
2502
+
2503
+ /**
2504
+ * One persisted row. We attach `run_id` on disk so a single file can
2505
+ * hold multiple runs and the diff helper can query without re-walking
2506
+ * separate files.
2507
+ */
2508
+ interface PersistedFinding extends AnalystFinding {
2509
+ run_id: string;
2510
+ }
2511
+ declare class FindingsStore {
2512
+ readonly path: string;
2513
+ private readonly appender;
2514
+ constructor(path: string);
2515
+ append(runId: string, findings: AnalystFinding[]): Promise<void>;
2516
+ /** Load every persisted finding. Discards malformed trailing lines silently. */
2517
+ loadAll(): PersistedFinding[];
2518
+ /** Filter to a single run. */
2519
+ loadRun(runId: string): PersistedFinding[];
2520
+ }
2521
+ interface FindingsDiff {
2522
+ /** New finding ids in `current` that weren't in `previous`. */
2523
+ appeared: PersistedFinding[];
2524
+ /** Finding ids in `previous` that aren't in `current`. */
2525
+ disappeared: PersistedFinding[];
2526
+ /** Same finding id present in both runs and unchanged per the materiality test. */
2527
+ persisted: PersistedFinding[];
2528
+ /**
2529
+ * Same finding id in both runs but at least one non-identity field
2530
+ * shifted per `DiffPolicy.isMaterial`. Reported as [previous, current].
2531
+ */
2532
+ changed: Array<{
2533
+ previous: PersistedFinding;
2534
+ current: PersistedFinding;
2535
+ }>;
2536
+ }
2537
+ interface DiffPolicy {
2538
+ /**
2539
+ * Predicate that decides whether two findings (same finding_id) count
2540
+ * as a material change. Defaults to {@link defaultIsMaterial}: severity
2541
+ * shift, confidence Δ > 0.05, or evidence count change. Compliance /
2542
+ * perf consumers MAY supply a stricter predicate (e.g. rationale text
2543
+ * diff, metric Δ thresholds).
2544
+ */
2545
+ isMaterial?: (previous: AnalystFinding, current: AnalystFinding) => boolean;
2546
+ }
2547
+ /**
2548
+ * Default materiality test. Deliberately narrow so LLM-reword churn
2549
+ * doesn't flood the diff. Stricter tests are opt-in via DiffPolicy.
2550
+ */
2551
+ declare function defaultIsMaterial(a: AnalystFinding, b: AnalystFinding): boolean;
2552
+ /**
2553
+ * Diff two findings sets by stable finding_id. Callers typically load
2554
+ * the two run-id slices from the same store and pass them in.
2555
+ */
2556
+ declare function diffFindings(previous: PersistedFinding[], current: PersistedFinding[], policy?: DiffPolicy): FindingsDiff;
2557
+
2558
+ /**
2559
+ * Failure-mode analyst — classifies what went wrong and why.
2560
+ *
2561
+ * Brief: read the trace dataset, identify the top failure modes across
2562
+ * runs, classify each with severity + evidence, and surface them as
2563
+ * findings. The actor's job is *taxonomy + evidence*, not fix-design —
2564
+ * that's the improvement-analyst's job.
2565
+ *
2566
+ * Recursion is deep (`maxDepth: 3`) because real failure-mode
2567
+ * discovery is genuinely tree-shaped: the actor splits the dataset
2568
+ * into candidate clusters, each cluster spawns a focused investigator
2569
+ * that drills into representative traces, and a deeply-recursed
2570
+ * investigator may itself split a confounded mode into two sub-modes.
2571
+ * Each level fans out 4-way, so the analyst can investigate up to
2572
+ * ~16 leaf clusters before hitting the depth ceiling.
2573
+ */
2574
+
2575
+ declare const FAILURE_MODE_KIND_SPEC: TraceAnalystKindSpec;
2576
+
2577
+ /**
2578
+ * Improvement analyst — actionable, recursive self-improvement findings.
2579
+ *
2580
+ * Brief: read findings from upstream analysts (failure-mode,
2581
+ * knowledge-gap, knowledge-poisoning) AND the trace dataset itself,
2582
+ * then propose **concrete edits** to the agent's runtime: prompt
2583
+ * additions, RAG documents to ingest, tool descriptions to rewrite,
2584
+ * scaffolding changes to make, memory entries to invalidate. Each
2585
+ * finding is one proposed edit with the locus, the diff, and the
2586
+ * expected effect.
2587
+ *
2588
+ * This is the recursive-self-improvement loop's last mile: the prior
2589
+ * kinds describe *what's wrong*; this kind describes *what to change*.
2590
+ *
2591
+ * Recursion is deep (`maxDepth: 3`) because real improvement proposals
2592
+ * are competitive: for each failure-mode there are usually 2-3 viable
2593
+ * fix directions (tighten prompt vs add tool vs adjust scaffolding),
2594
+ * and the actor should explore each with a focused subagent before
2595
+ * picking the highest-leverage one to recommend.
2596
+ */
2597
+
2598
+ declare const IMPROVEMENT_KIND_SPEC: TraceAnalystKindSpec;
2599
+
2600
+ /**
2601
+ * Knowledge-gap analyst — what did the agent NOT know that it needed?
2602
+ *
2603
+ * Brief: find moments in the trace where the agent had to guess, ask
2604
+ * the user to fill in context, recover from a wrong assumption, or
2605
+ * loop on a retrieval. Each finding names a *missing or outdated piece
2606
+ * of knowledge* the agent's curated knowledge base should have held —
2607
+ * or a downstream lookup (web, docs, tool description) that surfaced
2608
+ * stale or outdated information.
2609
+ *
2610
+ * The primary expected store is `@tangle-network/agent-knowledge`: a
2611
+ * Karpathy-style wiki the agent maintains with raw ↔ curated pages,
2612
+ * source anchors, and claim/relation triples. A gap is anything the
2613
+ * agent had to discover at run-time that should already have lived
2614
+ * there. Secondary loci: web-search results that returned outdated
2615
+ * pages, tool descriptions that omitted critical behavior, system-
2616
+ * prompt sections that didn't cover the case.
2617
+ *
2618
+ * Distinct from failure-mode: failure-mode classifies *how* it broke;
2619
+ * knowledge-gap names the *information* whose absence (or staleness)
2620
+ * caused the break. One failure-mode often maps to several gaps.
2621
+ *
2622
+ * Recursion (`maxDepth: 2`) is enough to fan out one subagent per
2623
+ * candidate gap-source layer; each subagent runs a focused detection.
2624
+ */
2625
+
2626
+ declare const KNOWLEDGE_GAP_KIND_SPEC: TraceAnalystKindSpec;
2627
+
2628
+ /**
2629
+ * Knowledge-poisoning analyst — what FALSE information misled the agent?
2630
+ *
2631
+ * Brief: find moments where the agent acted on information that was
2632
+ * *wrong* — stale memory, RAG documents that contradicted ground truth,
2633
+ * tool descriptions that lied about return shapes, system-prompt
2634
+ * instructions that no longer matched reality, prior-run summaries that
2635
+ * cached a wrong decision.
2636
+ *
2637
+ * Distinct from knowledge-gap: a gap is "the agent didn't know X"; a
2638
+ * poisoning is "the agent confidently used X, but X was wrong." Gaps
2639
+ * surface as questions / self-correction; poisonings surface as
2640
+ * confident-but-wrong actions that downstream evidence contradicts.
2641
+ *
2642
+ * Recursion is moderate (`maxDepth: 2`) because each candidate
2643
+ * poisoning typically needs two sub-investigations: one to confirm
2644
+ * the agent acted on the false belief, one to confirm the belief
2645
+ * itself is actually false in ground truth.
2646
+ */
2647
+
2648
+ declare const KNOWLEDGE_POISONING_KIND_SPEC: TraceAnalystKindSpec;
2649
+
2650
+ /**
2651
+ * Default analyst kinds focused on agent failure + recursive
2652
+ * self-improvement.
2653
+ *
2654
+ * The four kinds chain: failure-mode classifies; knowledge-gap and
2655
+ * knowledge-poisoning explain *why* in two orthogonal ways; improvement
2656
+ * proposes concrete edits. Register all four against the same trace
2657
+ * store and the registry runs them in dependency order if the operator
2658
+ * pipes findings between them.
2659
+ */
2660
+
2661
+ /**
2662
+ * The default kind suite. Order is the run order operators should
2663
+ * use: failure-mode first (no upstream deps), gap + poisoning next
2664
+ * (both depend on failures), improvement last (chains all three).
2665
+ */
2666
+ declare const DEFAULT_TRACE_ANALYST_KINDS: readonly TraceAnalystKindSpec[];
2667
+
2668
+ /**
2669
+ * Skill-usage analyst — a DETERMINISTIC `Analyst` over a Claude/Codex skill
2670
+ * library + its trace corpus. Unlike the trace-store kinds (failure-mode,
2671
+ * improvement, ...) this kind calls no LLM: it mines real usage and skill
2672
+ * structure and emits findings by rule.
2673
+ *
2674
+ * It exists because the naive "Skill-tool invocation count" lies low — it
2675
+ * misses orchestrated sub-dispatch (a leaf skill run BY /pursue or /governor
2676
+ * logs under the parent), slash-command entry, local-script bypass, and
2677
+ * on-disk artifacts. The 2026-05-30 skill audit found 39/53 skills at zero
2678
+ * direct invocations, yet only one was a genuine cut: the rest were
2679
+ * measurement-invisible or discovery-limited. This analyst encodes that
2680
+ * lesson as a multi-signal usage model so a cheap repeatable pass can keep
2681
+ * the library honest, and so the expensive audit workflow's verdicts can
2682
+ * GEPA-distill it toward agreement (see `gold/skill-verdicts.gold.jsonl`).
2683
+ *
2684
+ * Report-building (`buildSkillUsageReport`, an fs scan) is separated from
2685
+ * finding emission (`SkillUsageAnalyst.analyze`, pure) so the slow scan runs
2686
+ * once at the registry boundary and the rule logic stays unit-testable.
2687
+ */
2688
+
2689
+ type SkillKind = 'public' | 'private';
2690
+ /** One skill's multi-signal usage + structure. All counts are deterministic. */
2691
+ interface SkillUsageRecord {
2692
+ name: string;
2693
+ kind: SkillKind;
2694
+ /** Absolute path to the skill's SKILL.md. */
2695
+ path: string;
2696
+ lines: number;
2697
+ /** `"skill":"<name>"` Skill-tool invocations across the trace corpus. */
2698
+ directInvocations: number;
2699
+ /** `<command-name>/<name>` slash invocations across the trace corpus. */
2700
+ slashInvocations: number;
2701
+ /** Sibling skills whose SKILL.md dispatches to this one (`/<name>`). Proxy
2702
+ * for orchestrated sub-dispatch the per-skill counter cannot see. */
2703
+ inboundRefs: number;
2704
+ /** On-disk artifacts attributable to the skill (e.g. `.evolve/<name>/**`). */
2705
+ artifactCount: number;
2706
+ /** Tangle-private reference count in the body (leak signal for public skills). */
2707
+ tanglePrivateRefs: number;
2708
+ hasReferencesDir: boolean;
2709
+ hasEvalsDir: boolean;
2710
+ /** Body mentions `skill-runs.jsonl` (visible to /reflect + /governor). */
2711
+ logsRuns: boolean;
2712
+ /** Description carries an explicit `Triggers:` clause / trigger phrases. */
2713
+ hasTriggerPhrases: boolean;
2714
+ }
2715
+ interface SkillUsageReport {
2716
+ generatedFromTraces: number;
2717
+ records: SkillUsageRecord[];
2718
+ }
2719
+ interface SkillUsageScanConfig {
2720
+ /** Dirs holding `*.jsonl` transcripts (Claude `~/.claude/projects`, Codex sessions). */
2721
+ transcriptDirs: string[];
2722
+ /** Skill roots to scan; each dir directly under `root` with a `SKILL.md` is a skill. */
2723
+ skillRoots: {
2724
+ root: string;
2725
+ kind: SkillKind;
2726
+ }[];
2727
+ /** Roots scanned for `<root>/.evolve/<skill>` artifact dirs. */
2728
+ artifactRoots?: string[];
2729
+ /** Token-prefixed mappings: skill name → extra artifact subpaths under an artifactRoot
2730
+ * (e.g. reflect → `.evolve/reflections`). Catches non-eponymous artifact dirs. */
2731
+ artifactAliases?: Record<string, string[]>;
2732
+ /** Cap files read per transcript dir (bounds a huge corpus); 0 = unbounded. */
2733
+ maxTranscriptsPerDir?: number;
2734
+ }
2735
+ /** Scan the corpus + skill roots into a {@link SkillUsageReport}. Deterministic. */
2736
+ declare function buildSkillUsageReport(config: SkillUsageScanConfig): SkillUsageReport;
2737
+ /** Pure rule pass over a report → findings. Exported for direct/unit use. */
2738
+ declare function emitSkillUsageFindings(report: SkillUsageReport, producedAt: string): AnalystFinding[];
2739
+ declare class SkillUsageAnalyst implements Analyst<SkillUsageReport> {
2740
+ readonly id = "skill-usage";
2741
+ readonly description = "Deterministic multi-signal skill-usage analysis: flags dead skills, measurement-invisible (orchestrated) usage, discovery gaps, public-repo leaks, bloat, missing evals, and missing run-logging.";
2742
+ readonly inputKind: "custom";
2743
+ readonly cost: {
2744
+ kind: "deterministic";
2745
+ est_usd_per_run: number;
2746
+ };
2747
+ readonly version = "1.0.0";
2748
+ analyze(input: SkillUsageReport, ctx: AnalystContext): Promise<AnalystFinding[]>;
2749
+ }
2750
+ declare const SKILL_USAGE_ANALYST: SkillUsageAnalyst;
2751
+
107
2752
  /**
108
2753
  * Forgiving pre-parse for analyst findings. Weak models routinely emit
109
2754
  * schema-correct content in an unusable wrapper — fenced ```json blocks, a
@@ -130,6 +2775,112 @@ declare function coerceJson(text: string): unknown;
130
2775
  */
131
2776
  declare function coerceToFindingRows(raw: unknown): unknown[];
132
2777
 
2778
+ type PolicyEditSchemaVersion = 'policy-edit/v1';
2779
+ declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
2780
+ type PolicyEditAxis = (typeof POLICY_EDIT_AXES)[number];
2781
+ declare const POLICY_EDIT_TARGET_SURFACES: readonly ["prompt", "tool-contract", "runtime-config", "memory", "agent-profile", "code", "deployment"];
2782
+ type PolicyEditTargetSurface = (typeof POLICY_EDIT_TARGET_SURFACES)[number];
2783
+ type PolicyEditRisk = 'low' | 'medium' | 'high' | 'unknown';
2784
+ type PolicyEditGainDirection = 'increase' | 'decrease';
2785
+ type PolicyEditGainUnit = 'absolute' | 'relative' | 'percent' | 'score';
2786
+ interface PolicyEditTarget {
2787
+ surface: PolicyEditTargetSurface;
2788
+ /** Stable path inside the target surface, for example `system-prompt:tools`
2789
+ * or `budget.maxTurns`. */
2790
+ path?: string;
2791
+ /** Optional canonical deployment identity. Store the existing cell, not a
2792
+ * local profile shape. */
2793
+ agentProfileCell?: AgentProfileCell;
2794
+ /** Human label when the path is not enough for a readable audit trail. */
2795
+ label?: string;
2796
+ }
2797
+ type PolicyEditChange = {
2798
+ kind: 'text';
2799
+ mode: 'append' | 'prepend' | 'replace';
2800
+ value: string;
2801
+ /** Required when `mode === 'replace'`; exact match only. */
2802
+ find?: string;
2803
+ } | {
2804
+ kind: 'json';
2805
+ mode: 'set' | 'merge' | 'remove';
2806
+ path: string;
2807
+ value?: AgentProfileJson;
2808
+ };
2809
+ interface PolicyEditExpectedGain {
2810
+ /** Metric this edit is expected to move, e.g. `holdout.composite`. */
2811
+ metric: string;
2812
+ direction: PolicyEditGainDirection;
2813
+ /** Positive magnitude in the metric's native units. */
2814
+ amount: number;
2815
+ unit?: PolicyEditGainUnit;
2816
+ rationale?: string;
2817
+ }
2818
+ interface PolicyEditSource {
2819
+ findingIds: string[];
2820
+ analystIds: string[];
2821
+ evidenceRefs: EvidenceRef[];
2822
+ /** Mirrors `AnalystFinding.derived_from_judge`; admission rejects it. */
2823
+ derivedFromJudge?: boolean;
2824
+ }
2825
+ interface PolicyEdit {
2826
+ schemaVersion: PolicyEditSchemaVersion;
2827
+ editId: string;
2828
+ axis: PolicyEditAxis;
2829
+ target: PolicyEditTarget;
2830
+ change: PolicyEditChange;
2831
+ claim: string;
2832
+ expectedGain: PolicyEditExpectedGain;
2833
+ confidence: number;
2834
+ risk: PolicyEditRisk;
2835
+ source: PolicyEditSource;
2836
+ rationale?: string;
2837
+ validationPlan?: string;
2838
+ metadata?: Record<string, unknown>;
2839
+ }
2840
+ declare const POLICY_EDIT_CANDIDATE_RECORD_SCHEMA: "tangle.policy-edit-candidate.v1";
2841
+ /** JSON-safe attribution carried with a measured candidate and its scores. */
2842
+ interface PolicyEditCandidateRecord {
2843
+ schema: typeof POLICY_EDIT_CANDIDATE_RECORD_SCHEMA;
2844
+ policyEdit: PolicyEdit;
2845
+ }
2846
+ type PolicyEditInit = Omit<PolicyEdit, 'schemaVersion' | 'editId'> & {
2847
+ schemaVersion?: PolicyEditSchemaVersion;
2848
+ editId?: string;
2849
+ };
2850
+ declare class PolicyEditValidationError extends ValidationError {
2851
+ readonly path: string;
2852
+ constructor(message: string, path?: string);
2853
+ }
2854
+ interface FindingToPolicyEditOptions {
2855
+ expectedGain?: PolicyEditExpectedGain | ((finding: AnalystFinding) => PolicyEditExpectedGain | null | undefined);
2856
+ risk?: PolicyEditRisk | ((finding: AnalystFinding) => PolicyEditRisk);
2857
+ defaultAxis?: PolicyEditAxis;
2858
+ defaultTargetSurface?: PolicyEditTargetSurface;
2859
+ }
2860
+ interface PolicyEditAdmissionOptions {
2861
+ minScore?: number;
2862
+ minExpectedGain?: number;
2863
+ allowHighRisk?: boolean;
2864
+ requireEvidence?: boolean;
2865
+ }
2866
+ interface PolicyEditAdmission {
2867
+ edit: PolicyEdit;
2868
+ decision: 'admit' | 'reject';
2869
+ score: number;
2870
+ reasons: string[];
2871
+ }
2872
+ declare function makePolicyEdit(init: PolicyEditInit): PolicyEdit;
2873
+ declare function computePolicyEditId(edit: Omit<PolicyEdit, 'editId'> | PolicyEdit): string;
2874
+ declare function validatePolicyEdit(input: unknown): PolicyEdit;
2875
+ declare function makePolicyEditCandidateRecord(edit: PolicyEdit): PolicyEditCandidateRecord;
2876
+ declare function validatePolicyEditCandidateRecord(input: unknown): PolicyEditCandidateRecord;
2877
+ declare function isPolicyEdit(input: unknown): input is PolicyEdit;
2878
+ declare function policyEditsFromFindings(findings: ReadonlyArray<AnalystFinding>, opts?: FindingToPolicyEditOptions): PolicyEdit[];
2879
+ declare function policyEditFromFinding(finding: AnalystFinding, opts?: FindingToPolicyEditOptions): PolicyEdit | null;
2880
+ declare function scorePolicyEditReadiness(edit: PolicyEdit, opts?: PolicyEditAdmissionOptions): number;
2881
+ declare function admitPolicyEdit(edit: PolicyEdit, opts?: PolicyEditAdmissionOptions): PolicyEditAdmission;
2882
+ declare function applyPolicyEditToSurface(surface: unknown, edit: PolicyEdit): unknown;
2883
+
133
2884
  /** DESCRIPTIVE predicate: does the finding cite at least one observable
134
2885
  * (span/event/artifact) evidence ref. Useful for ranking evidence quality or
135
2886
  * rendering — it is NOT the steer gate. Evidence presence is the WRONG
@@ -237,4 +2988,4 @@ type TraceToolGroupName =
237
2988
  */
238
2989
  declare function buildTraceToolsForGroup(group: TraceToolGroupName, store: TraceAnalysisStore): AxFunction[];
239
2990
 
240
- export { Analyst, AnalystFinding, AnalystSeverity, type JudgeAdapterOpts, type RunCriticAdapterOpts, type SemanticConceptJudgeAdapterOpts, type StructureFindingsOptions, type StructureFindingsResult, type TraceToolGroupName, type VerifierAdapterOpts, assertNoJudgeVerdict, behavioralAnalyst, buildTraceToolsForGroup, coerceJson, coerceToFindingRows, createJudgeAdapter, createRunCriticAdapter, createSemanticConceptJudgeAdapter, createVerifierAdapter, deriveEfficiencyFindings, isJudgeVerdict, isTraceObservable, liftSeverity, stripCodeFences, structureFindings };
2991
+ export { ANALYST_SEVERITIES, type Analyst, type AnalystContext, type AnalystCost, type AnalystFinding, type AnalystHooks, type AnalystInputKind, AnalystRegistry, type AnalystRegistryOptions, type AnalystRequirements, type AnalystRunEvent, type AnalystRunInputs, type AnalystRunResult, type AnalystRunSummary, type AnalystSeverity, type BudgetPolicy, type ChatCallOpts, type ChatClient, type ChatRequest, type ChatResponse, type ChatTransport, type CliBridgeTransportOpts, type CreateAnalystAiConfig, type CreateChatClientOpts, type CreateTraceAnalystKindOpts, DEFAULT_TRACE_ANALYST_KINDS, type DefaultAnalystRegistryOptions, type DiffPolicy, type DirectProviderTransportOpts, type EvidenceRef, FAILURE_MODE_KIND_SPEC, FINDING_SUBJECT_GRAMMAR_PROMPT, FINDING_SUBJECT_KINDS, FINDING_SUBJECT_SYNTAX, type FindingSubject, type FindingSubjectKind, FindingSubjectStringSchema, type FindingToPolicyEditOptions, type FindingsDiff, FindingsStore, IMPROVEMENT_KIND_SPEC, type JudgeAdapterOpts, KIND_EXPECTED_SUBJECTS, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, type MockTransportOpts, POLICY_EDIT_AXES, POLICY_EDIT_CANDIDATE_RECORD_SCHEMA, POLICY_EDIT_TARGET_SURFACES, type PersistedFinding, type PolicyEdit, type PolicyEditAdmission, type PolicyEditAdmissionOptions, type PolicyEditAxis, type PolicyEditCandidateRecord, type PolicyEditChange, type PolicyEditExpectedGain, type PolicyEditGainDirection, type PolicyEditGainUnit, type PolicyEditInit, type PolicyEditRisk, type PolicyEditSchemaVersion, type PolicyEditSource, type PolicyEditTarget, type PolicyEditTargetSurface, PolicyEditValidationError, RAW_FINDING_SCHEMA_PROMPT, type RawAnalystFinding, RawAnalystFindingSchema, type RegistryRunOpts, type RouterTransportOpts, type RunCriticAdapterOpts, SKILL_USAGE_ANALYST, type SandboxSdkTransportOpts, type SemanticConceptJudgeAdapterOpts, SkillUsageAnalyst, type SkillUsageRecord, type SkillUsageReport, type SkillUsageScanConfig, type StructureFindingsOptions, type StructureFindingsResult, type TraceAnalystGolden, type TraceAnalystKindSpec, type TraceToolGroupName, type VerifierAdapterOpts, admitPolicyEdit, applyPolicyEditToSurface, assertNoJudgeVerdict, behavioralAnalyst, buildDefaultAnalystRegistry, buildSkillUsageReport, buildTraceToolsForGroup, coerceJson, coerceToFindingRows, computeFindingId, computePolicyEditId, createAnalystAi, createChatClient, createJudgeAdapter, createRunCriticAdapter, createSemanticConceptJudgeAdapter, createTraceAnalystKind, createVerifierAdapter, defaultIsMaterial, deriveEfficiencyFindings, diffFindings, emitSkillUsageFindings, findingSubjectGrammarPromptFor, isJudgeVerdict, isPolicyEdit, isTraceObservable, liftSeverity, makeFinding, makePolicyEdit, makePolicyEditCandidateRecord, parseFindingSubject, parseRawFinding, policyEditFromFinding, policyEditsFromFindings, renderFindingSubject, renderPriorFindings, scorePolicyEditReadiness, stripCodeFences, structureFindings, validatePolicyEdit, validatePolicyEditCandidateRecord };