@tangle-network/agent-eval 0.118.3 → 0.119.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/README.md +26 -0
  3. package/dist/analyst/index.d.ts +172 -59
  4. package/dist/analyst/index.js +50 -12
  5. package/dist/analyst/index.js.map +1 -1
  6. package/dist/benchmarks/index.d.ts +50 -40
  7. package/dist/benchmarks/index.js +8 -8
  8. package/dist/campaign/index.d.ts +258 -184
  9. package/dist/campaign/index.js +7 -7
  10. package/dist/{chunk-JTMFT5QZ.js → chunk-4I2E3LLO.js} +2 -2
  11. package/dist/{chunk-KK2O4VWA.js → chunk-D7AEXSM5.js} +2 -2
  12. package/dist/{chunk-MLHRFWQK.js → chunk-JHOJHHU7.js} +85 -16
  13. package/dist/chunk-JHOJHHU7.js.map +1 -0
  14. package/dist/{chunk-F5A4GDQW.js → chunk-JM2SKQMS.js} +3 -15
  15. package/dist/chunk-JM2SKQMS.js.map +1 -0
  16. package/dist/{chunk-EGNRE7VA.js → chunk-PAHNGS65.js} +2 -2
  17. package/dist/{chunk-QWRLW4CT.js → chunk-PSXJ32SR.js} +160 -6
  18. package/dist/chunk-PSXJ32SR.js.map +1 -0
  19. package/dist/{chunk-6QIM2EAP.js → chunk-PXD6ZFNY.js} +99 -1
  20. package/dist/chunk-PXD6ZFNY.js.map +1 -0
  21. package/dist/{chunk-267UCQWI.js → chunk-QWMPPZ3X.js} +3 -3
  22. package/dist/{chunk-267UCQWI.js.map → chunk-QWMPPZ3X.js.map} +1 -1
  23. package/dist/{chunk-VNLWDTDA.js → chunk-S5TT5R3L.js} +1010 -285
  24. package/dist/chunk-S5TT5R3L.js.map +1 -0
  25. package/dist/{chunk-ZYHJNKI3.js → chunk-ULOKLHIQ.js} +42 -5
  26. package/dist/chunk-ULOKLHIQ.js.map +1 -0
  27. package/dist/{chunk-FIUFRXP3.js → chunk-WW2A73HW.js} +49 -91
  28. package/dist/chunk-WW2A73HW.js.map +1 -0
  29. package/dist/{chunk-7V3QDWNL.js → chunk-XJ7JVCHB.js} +2 -2
  30. package/dist/{chunk-EKHHBKS6.js → chunk-XMBOU5W7.js} +48 -190
  31. package/dist/chunk-XMBOU5W7.js.map +1 -0
  32. package/dist/cli.js +2 -2
  33. package/dist/contract/index.d.ts +281 -175
  34. package/dist/contract/index.js +284 -8
  35. package/dist/contract/index.js.map +1 -1
  36. package/dist/fuzz.d.ts +19 -1
  37. package/dist/fuzz.js +1 -1
  38. package/dist/index.d.ts +223 -114
  39. package/dist/index.js +38 -37
  40. package/dist/index.js.map +1 -1
  41. package/dist/openapi.json +1 -1
  42. package/dist/primeintellect/index.d.ts +311 -0
  43. package/dist/primeintellect/index.js +348 -0
  44. package/dist/primeintellect/index.js.map +1 -0
  45. package/dist/rl.d.ts +4 -0
  46. package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js} +3 -3
  47. package/dist/traces.d.ts +39 -25
  48. package/dist/traces.js +3 -5
  49. package/dist/wire/index.d.ts +19 -1
  50. package/dist/wire/index.js +2 -2
  51. package/docs/trace-analysis.md +6 -2
  52. package/package.json +7 -2
  53. package/dist/chunk-6QIM2EAP.js.map +0 -1
  54. package/dist/chunk-EKHHBKS6.js.map +0 -1
  55. package/dist/chunk-F5A4GDQW.js.map +0 -1
  56. package/dist/chunk-FIUFRXP3.js.map +0 -1
  57. package/dist/chunk-MLHRFWQK.js.map +0 -1
  58. package/dist/chunk-QWRLW4CT.js.map +0 -1
  59. package/dist/chunk-VNLWDTDA.js.map +0 -1
  60. package/dist/chunk-ZYHJNKI3.js.map +0 -1
  61. /package/dist/{chunk-JTMFT5QZ.js.map → chunk-4I2E3LLO.js.map} +0 -0
  62. /package/dist/{chunk-KK2O4VWA.js.map → chunk-D7AEXSM5.js.map} +0 -0
  63. /package/dist/{chunk-EGNRE7VA.js.map → chunk-PAHNGS65.js.map} +0 -0
  64. /package/dist/{chunk-7V3QDWNL.js.map → chunk-XJ7JVCHB.js.map} +0 -0
  65. /package/dist/{run-campaign-DWC67KJP.js.map → run-campaign-OBXSN5WK.js.map} +0 -0
package/dist/cli.js CHANGED
@@ -5,9 +5,9 @@ import {
5
5
  runRpcBatch,
6
6
  runRpcOnce,
7
7
  startServer
8
- } from "./chunk-JTMFT5QZ.js";
8
+ } from "./chunk-4I2E3LLO.js";
9
9
  import "./chunk-NJC7U437.js";
10
- import "./chunk-MLHRFWQK.js";
10
+ import "./chunk-JHOJHHU7.js";
11
11
  import "./chunk-VI2UW6B6.js";
12
12
  import "./chunk-PC4UYEBM.js";
13
13
  import "./chunk-ONWEPEDO.js";
@@ -1,3 +1,4 @@
1
+ import { AgentImprovementMeasuredComparison, Sha256Digest } from '@tangle-network/agent-interface';
1
2
  import { AxFunction, AxAIService } from '@ax-llm/ax';
2
3
  import { z } from 'zod';
3
4
 
@@ -28,6 +29,174 @@ interface AgentProfileCell {
28
29
  dimensions?: Record<string, AgentProfileDimensionValue>;
29
30
  }
30
31
 
32
+ type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
33
+ interface CostUsage {
34
+ inputTokens: number;
35
+ /** Includes reasoning tokens when the provider bills them as output. */
36
+ outputTokens: number;
37
+ /** Reasoning-token subset of outputTokens, when reported. */
38
+ reasoningTokens?: number;
39
+ /** Prompt tokens served from a provider cache. */
40
+ cachedTokens?: number;
41
+ /** Prompt tokens written into a provider cache. */
42
+ cacheWriteTokens?: number;
43
+ }
44
+ interface CostCallBase {
45
+ callId: string;
46
+ channel: CostChannel;
47
+ phase: string;
48
+ actor: string;
49
+ model: string;
50
+ maximumCostUsd?: number;
51
+ tags?: Record<string, string>;
52
+ timestamp: number;
53
+ }
54
+ interface CostReceipt extends CostCallBase, CostUsage {
55
+ status: 'settled';
56
+ costUsd: number;
57
+ costUnknown: boolean;
58
+ usageUnknown?: boolean;
59
+ pricing?: {
60
+ inputUsdPerThousand: number;
61
+ outputUsdPerThousand: number;
62
+ };
63
+ actualCostUsd?: number;
64
+ error?: string;
65
+ }
66
+ interface CostReceiptInput extends CostUsage {
67
+ model: string;
68
+ actualCostUsd?: number;
69
+ costUnknown?: boolean;
70
+ usageUnknown?: boolean;
71
+ }
72
+ type MaximumCharge = {
73
+ externallyEnforcedMaximumUsd: number;
74
+ } | ({
75
+ model: string;
76
+ } & CostUsage);
77
+ interface RunPaidCallInput<T> {
78
+ callId?: string;
79
+ channel: CostChannel;
80
+ phase: string;
81
+ actor: string;
82
+ /** Used before a provider receipt exists and on failures without one. */
83
+ model?: string;
84
+ tags?: Record<string, string>;
85
+ signal?: AbortSignal;
86
+ /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
87
+ maximumCharge?: MaximumCharge;
88
+ /** `callId` can be forwarded as the provider's idempotency key. */
89
+ execute(signal: AbortSignal, callId: string): Promise<T>;
90
+ receipt(value: T): CostReceiptInput;
91
+ receiptFromError?(error: Error): CostReceiptInput | undefined;
92
+ }
93
+ type PaidCallResult<T> = {
94
+ succeeded: true;
95
+ callId: string;
96
+ value: T;
97
+ receipt: CostReceipt;
98
+ } | {
99
+ succeeded: false;
100
+ callId?: string;
101
+ error: Error;
102
+ receipt?: CostReceipt;
103
+ };
104
+ interface ChannelRollup {
105
+ channel: CostChannel;
106
+ calls: number;
107
+ inputTokens: number;
108
+ outputTokens: number;
109
+ reasoningTokens?: number;
110
+ cachedTokens: number;
111
+ cacheWriteTokens?: number;
112
+ costUsd: number;
113
+ unpricedCalls: number;
114
+ unknownUsageCalls: number;
115
+ }
116
+ interface CostLedgerSummary {
117
+ totalCalls: number;
118
+ pendingCalls: number;
119
+ unresolvedCalls: number;
120
+ reservedCostUsd: number;
121
+ inputTokens: number;
122
+ outputTokens: number;
123
+ reasoningTokens?: number;
124
+ cachedTokens: number;
125
+ cacheWriteTokens?: number;
126
+ totalCostUsd: number;
127
+ byChannel: ChannelRollup[];
128
+ unpricedModels: string[];
129
+ fullyPriced: boolean;
130
+ usageComplete: boolean;
131
+ accountingComplete: boolean;
132
+ incompleteReasons: string[];
133
+ }
134
+ interface CostLedgerFilter {
135
+ channel?: CostChannel;
136
+ phase?: string;
137
+ tags?: Record<string, string>;
138
+ }
139
+ interface CostLedgerWaitOptions {
140
+ /** Maximum time to wait for active provider calls. Default 5 seconds. */
141
+ timeoutMs?: number;
142
+ }
143
+ /** Append-only storage. `append` must atomically reject stale revisions. */
144
+ interface CostLedgerPersistence {
145
+ read(): {
146
+ revision: string;
147
+ events: string;
148
+ };
149
+ append(expectedRevision: string, event: string): string | undefined;
150
+ }
151
+ interface CostLedgerOptions {
152
+ costCeilingUsd?: number;
153
+ persistence?: CostLedgerPersistence;
154
+ /** Import already-settled receipts without admitting new paid work. */
155
+ receipts?: readonly CostReceipt[];
156
+ }
157
+ /** Run-wide paid-call admission, durable call state, receipts, and summaries. */
158
+ declare class CostLedger {
159
+ private readonly records;
160
+ private readonly activeCallIds;
161
+ private readonly lateCallIds;
162
+ private readonly idleWaiters;
163
+ private completedTasks;
164
+ private revision;
165
+ private costLimitPersisted;
166
+ readonly costCeilingUsd?: number;
167
+ private readonly persistence?;
168
+ constructor(input?: number | CostLedgerOptions);
169
+ runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
170
+ /** Wait until every call started by this ledger has produced a durable outcome. */
171
+ waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
172
+ /** Settle a call left pending by a crashed process after reconciling with the provider. */
173
+ reconcile(callId: string, observed: CostReceiptInput, options?: {
174
+ error?: string;
175
+ }): CostReceipt;
176
+ list(filter?: CostLedgerFilter): CostReceipt[];
177
+ summary(filter?: CostLedgerFilter): CostLedgerSummary;
178
+ markCompleted(count?: number): void;
179
+ costPerCompletedTask(): number | null;
180
+ private execute;
181
+ private captureLateOutcome;
182
+ private releaseActiveCall;
183
+ private commitOutcome;
184
+ private captureFailure;
185
+ private commitReceipt;
186
+ private resolveMaximum;
187
+ private hasIncompleteSettledCall;
188
+ private appendRecord;
189
+ private ensureCostLimitPersisted;
190
+ private appendEvent;
191
+ }
192
+ /** Public callback surface for a shared cost ledger.
193
+ *
194
+ * Declaration bundles may expose this type through multiple package subpaths.
195
+ * Keeping callback contracts structural lets those subpaths compose while the
196
+ * concrete {@link CostLedger} retains its private durable state.
197
+ */
198
+ type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
199
+
31
200
  type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
32
201
 
33
202
  /**
@@ -475,156 +644,6 @@ interface TraceAnalysisStore {
475
644
  }): Promise<SearchSpanResult>;
476
645
  }
477
646
 
478
- type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
479
- interface CostUsage {
480
- inputTokens: number;
481
- outputTokens: number;
482
- cachedTokens?: number;
483
- }
484
- interface CostCallBase {
485
- callId: string;
486
- channel: CostChannel;
487
- phase: string;
488
- actor: string;
489
- model: string;
490
- maximumCostUsd?: number;
491
- tags?: Record<string, string>;
492
- timestamp: number;
493
- }
494
- interface CostReceipt extends CostCallBase, CostUsage {
495
- status: 'settled';
496
- costUsd: number;
497
- costUnknown: boolean;
498
- usageUnknown?: boolean;
499
- pricing?: {
500
- inputUsdPerThousand: number;
501
- outputUsdPerThousand: number;
502
- };
503
- actualCostUsd?: number;
504
- error?: string;
505
- }
506
- interface CostReceiptInput extends CostUsage {
507
- model: string;
508
- actualCostUsd?: number;
509
- costUnknown?: boolean;
510
- usageUnknown?: boolean;
511
- }
512
- type MaximumCharge = {
513
- externallyEnforcedMaximumUsd: number;
514
- } | ({
515
- model: string;
516
- } & CostUsage);
517
- interface RunPaidCallInput<T> {
518
- callId?: string;
519
- channel: CostChannel;
520
- phase: string;
521
- actor: string;
522
- /** Used before a provider receipt exists and on failures without one. */
523
- model?: string;
524
- tags?: Record<string, string>;
525
- signal?: AbortSignal;
526
- /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
527
- maximumCharge?: MaximumCharge;
528
- /** `callId` can be forwarded as the provider's idempotency key. */
529
- execute(signal: AbortSignal, callId: string): Promise<T>;
530
- receipt(value: T): CostReceiptInput;
531
- receiptFromError?(error: Error): CostReceiptInput | undefined;
532
- }
533
- type PaidCallResult<T> = {
534
- succeeded: true;
535
- callId: string;
536
- value: T;
537
- receipt: CostReceipt;
538
- } | {
539
- succeeded: false;
540
- callId?: string;
541
- error: Error;
542
- receipt?: CostReceipt;
543
- };
544
- interface ChannelRollup {
545
- channel: CostChannel;
546
- calls: number;
547
- inputTokens: number;
548
- outputTokens: number;
549
- cachedTokens: number;
550
- costUsd: number;
551
- unpricedCalls: number;
552
- unknownUsageCalls: number;
553
- }
554
- interface CostLedgerSummary {
555
- totalCalls: number;
556
- pendingCalls: number;
557
- unresolvedCalls: number;
558
- reservedCostUsd: number;
559
- inputTokens: number;
560
- outputTokens: number;
561
- cachedTokens: number;
562
- totalCostUsd: number;
563
- byChannel: ChannelRollup[];
564
- unpricedModels: string[];
565
- fullyPriced: boolean;
566
- usageComplete: boolean;
567
- accountingComplete: boolean;
568
- incompleteReasons: string[];
569
- }
570
- interface CostLedgerFilter {
571
- channel?: CostChannel;
572
- phase?: string;
573
- tags?: Record<string, string>;
574
- }
575
- /** Append-only storage. `append` must atomically reject stale revisions. */
576
- interface CostLedgerPersistence {
577
- read(): {
578
- revision: string;
579
- events: string;
580
- };
581
- append(expectedRevision: string, event: string): string | undefined;
582
- }
583
- interface CostLedgerOptions {
584
- costCeilingUsd?: number;
585
- persistence?: CostLedgerPersistence;
586
- /** Import already-settled receipts without admitting new paid work. */
587
- receipts?: readonly CostReceipt[];
588
- }
589
- /** Run-wide paid-call admission, durable call state, receipts, and summaries. */
590
- declare class CostLedger {
591
- private readonly records;
592
- private readonly activeCallIds;
593
- private readonly lateCallIds;
594
- private completedTasks;
595
- private revision;
596
- private costLimitPersisted;
597
- readonly costCeilingUsd?: number;
598
- private readonly persistence?;
599
- constructor(input?: number | CostLedgerOptions);
600
- runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
601
- /** Settle a call left pending by a crashed process after reconciling with the provider. */
602
- reconcile(callId: string, observed: CostReceiptInput, options?: {
603
- error?: string;
604
- }): CostReceipt;
605
- list(filter?: CostLedgerFilter): CostReceipt[];
606
- summary(filter?: CostLedgerFilter): CostLedgerSummary;
607
- markCompleted(count?: number): void;
608
- costPerCompletedTask(): number | null;
609
- private execute;
610
- private captureLateOutcome;
611
- private commitOutcome;
612
- private captureFailure;
613
- private commitReceipt;
614
- private resolveMaximum;
615
- private hasIncompleteSettledCall;
616
- private appendRecord;
617
- private ensureCostLimitPersisted;
618
- private appendEvent;
619
- }
620
- /** Public callback surface for a shared cost ledger.
621
- *
622
- * Declaration bundles may expose this type through multiple package subpaths.
623
- * Keeping callback contracts structural lets those subpaths compose while the
624
- * concrete {@link CostLedger} retains its private durable state.
625
- */
626
- type CostLedgerHandle = Pick<CostLedger, keyof CostLedger>;
627
-
628
647
  interface Scenario$1 {
629
648
  id: string;
630
649
  persona: string;
@@ -1104,6 +1123,8 @@ interface AnalystCost {
1104
1123
  est_usd_per_run?: number;
1105
1124
  /** Models the analyst expects to use (informational). */
1106
1125
  models?: string[];
1126
+ /** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */
1127
+ settlement_timeout_ms?: number;
1107
1128
  }
1108
1129
  interface AnalystRequirements {
1109
1130
  /** Min number of shots / samples the analyst needs to produce signal. */
@@ -1127,10 +1148,14 @@ interface AnalystContext {
1127
1148
  runId: string;
1128
1149
  /** Stable correlation id so logs from a single registry.run() share a tag. */
1129
1150
  correlationId: string;
1130
- /** Wall-clock deadline (epoch ms). Analysts SHOULD honor for graceful cancel. */
1151
+ /** Enforced wall-clock deadline (epoch ms). */
1131
1152
  deadlineMs?: number;
1132
1153
  /** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
1133
1154
  budgetUsd?: number;
1155
+ /** Shared paid-call account when the analyst runs inside a larger campaign. */
1156
+ costLedger?: CostLedgerHandle;
1157
+ /** Attribution phase used when writing to the shared paid-call account. */
1158
+ costPhase?: string;
1134
1159
  /**
1135
1160
  * Shared chat client. Analysts that call an LLM go through this so
1136
1161
  * the operator picks transport (sandbox-sdk | router | cli-bridge |
@@ -1148,6 +1173,18 @@ interface AnalystContext {
1148
1173
  * filter. Empty / absent means no cross-run context.
1149
1174
  */
1150
1175
  priorFindings?: ReadonlyArray<AnalystFinding>;
1176
+ /**
1177
+ * Findings emitted by analysts that completed earlier in this registry run.
1178
+ * This is separate from `priorFindings`: upstream findings are dependency
1179
+ * context for the current pass, while prior findings are cross-run memory.
1180
+ * The registry populates this only when `RegistryRunOpts.chainFindings` is on.
1181
+ */
1182
+ upstreamFindings?: ReadonlyArray<AnalystFinding>;
1183
+ /**
1184
+ * Report metered work independently of findings. This keeps an empty finding
1185
+ * set from erasing token/cost telemetry. Multiple receipts are accumulated.
1186
+ */
1187
+ recordUsage?: (receipt: AnalystUsageReceipt) => void;
1151
1188
  /** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */
1152
1189
  tags?: Record<string, string>;
1153
1190
  /** Logger callback — analysts SHOULD prefer this over console.* for testability. */
@@ -1173,6 +1210,17 @@ interface Analyst<TInput = unknown> {
1173
1210
  readonly version: string;
1174
1211
  analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>;
1175
1212
  }
1213
+ /** Metered work performed by one analyst call. */
1214
+ interface AnalystUsageReceipt {
1215
+ /** Number of model-usage records observed at the provider boundary. */
1216
+ calls: number | null;
1217
+ /** Null when the provider did not return token accounting. */
1218
+ tokens: RunTokenUsage | null;
1219
+ /** Observed, estimated, or explicitly uncaptured dollar cost. */
1220
+ cost: RunCostProvenance;
1221
+ /** Known lower bound when one or more calls have uncaptured cost. */
1222
+ knownCostUsd?: number;
1223
+ }
1176
1224
  interface AnalystRunSummary {
1177
1225
  analyst_id: string;
1178
1226
  status: 'ok' | 'skipped' | 'failed';
@@ -1181,6 +1229,12 @@ interface AnalystRunSummary {
1181
1229
  findings_count: number;
1182
1230
  latency_ms: number;
1183
1231
  cost_usd: number;
1232
+ /**
1233
+ * Additive receipt for model usage. Registry-produced summaries populate it
1234
+ * even when the analyst emits no findings. `cost_usd` remains the legacy
1235
+ * numeric field; inspect `usage.cost` before treating zero as observed.
1236
+ */
1237
+ usage?: AnalystUsageReceipt;
1184
1238
  /** When `status='failed'`: the error class + message, never the full stack. */
1185
1239
  error?: {
1186
1240
  class: string;
@@ -1196,6 +1250,11 @@ interface AnalystRunResult {
1196
1250
  per_analyst: AnalystRunSummary[];
1197
1251
  /** Total LLM cost in USD across all analysts in this registry.run(). */
1198
1252
  total_cost_usd: number;
1253
+ /**
1254
+ * Provenance for `total_cost_usd`. When uncaptured, the numeric field is only
1255
+ * the known subtotal and must not be treated as the run's total spend.
1256
+ */
1257
+ total_cost_provenance?: RunCostProvenance;
1199
1258
  }
1200
1259
  /**
1201
1260
  * Events emitted by `AnalystRegistry.runStream(...)` in real time as
@@ -3944,6 +4003,15 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
3944
4003
  */
3945
4004
  declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
3946
4005
 
4006
+ interface MeasuredComparisonFromSelfImproveResultOptions<TScenario extends Scenario, TArtifact> {
4007
+ result: SelfImproveResult<TScenario, TArtifact>;
4008
+ benchmark: AgentImprovementMeasuredComparison['benchmark'];
4009
+ baselineProfileDigest: Sha256Digest;
4010
+ candidateBundleDigest: Sha256Digest;
4011
+ }
4012
+ /** Convert one paired self-improvement result into the portable Interface evidence record. */
4013
+ declare function measuredComparisonFromSelfImproveResult<TScenario extends Scenario, TArtifact>(options: MeasuredComparisonFromSelfImproveResultOptions<TScenario, TArtifact>): AgentImprovementMeasuredComparison;
4014
+
3947
4015
  /**
3948
4016
  * Typed Ax output for analyst findings.
3949
4017
  *
@@ -3961,7 +4029,10 @@ declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults
3961
4029
  * validation surface independent of which Ax version is installed.
3962
4030
  */
3963
4031
 
4032
+ /** Original public schema retained for stored rows and callback contracts. */
3964
4033
  declare const RawAnalystFindingSchema: z.ZodObject<{
4034
+ evidence_uri: z.ZodString;
4035
+ evidence_excerpt: z.ZodOptional<z.ZodString>;
3965
4036
  severity: z.ZodEnum<{
3966
4037
  low: "low";
3967
4038
  high: "high";
@@ -3971,23 +4042,46 @@ declare const RawAnalystFindingSchema: z.ZodObject<{
3971
4042
  }>;
3972
4043
  claim: z.ZodString;
3973
4044
  subject: z.ZodOptional<z.ZodString>;
3974
- evidence_uri: z.ZodString;
3975
- evidence_excerpt: z.ZodOptional<z.ZodString>;
3976
4045
  confidence: z.ZodNumber;
3977
4046
  rationale: z.ZodOptional<z.ZodString>;
3978
4047
  recommended_action: z.ZodOptional<z.ZodString>;
3979
4048
  }, z.core.$strict>;
3980
4049
  type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
4050
+ /**
4051
+ * Canonical plural-evidence contract. The preprocessor accepts the original
4052
+ * `evidence_uri` / `evidence_excerpt` pair and normalizes it into one evidence
4053
+ * item so persisted rows and older model fixtures remain readable. New output
4054
+ * always receives the plural shape.
4055
+ */
4056
+ declare const CanonicalRawAnalystFindingSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodObject<{
4057
+ evidence: z.ZodArray<z.ZodObject<{
4058
+ uri: z.ZodString;
4059
+ excerpt: z.ZodOptional<z.ZodString>;
4060
+ }, z.core.$strict>>;
4061
+ severity: z.ZodEnum<{
4062
+ low: "low";
4063
+ high: "high";
4064
+ critical: "critical";
4065
+ medium: "medium";
4066
+ info: "info";
4067
+ }>;
4068
+ claim: z.ZodString;
4069
+ subject: z.ZodOptional<z.ZodString>;
4070
+ confidence: z.ZodNumber;
4071
+ rationale: z.ZodOptional<z.ZodString>;
4072
+ recommended_action: z.ZodOptional<z.ZodString>;
4073
+ }, z.core.$strict>>;
4074
+ type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchema>;
3981
4075
 
3982
4076
  /**
3983
4077
  * Analyst-kind factory — the typed way to define trace analysts.
3984
4078
  *
3985
4079
  * A "kind" is a specialized analyst whose actor prompt, tool subset,
3986
- * and Ax recursion config target one failure-mode lens (failure-mode
3987
- * classification, knowledge gap discovery, knowledge poisoning, recursive
3988
- * self-improvement, ...). Kinds emit findings in the typed `RawAnalystFinding`
3989
- * shape via a JSON-array Ax output; the factory validates each row with
3990
- * Zod and lifts it into `AnalystFinding[]` with no shape guessing.
4080
+ * and bounded Ax subqueries target one failure-mode lens (failure-mode
4081
+ * classification, knowledge gap discovery, knowledge poisoning,
4082
+ * self-improvement, ...). Kinds emit findings in the typed
4083
+ * `CanonicalRawAnalystFinding` shape via a JSON-array Ax output; the factory
4084
+ * validates each row with Zod and lifts it into `AnalystFinding[]`.
3991
4085
  *
3992
4086
  * Composition rules:
3993
4087
  * - Each kind owns its actor description. No generic "answer this
@@ -3995,12 +4089,11 @@ type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
3995
4089
  * - Each kind picks a narrow tool subset from `ANALYST_TOOL_GROUPS`.
3996
4090
  * A kind that never needs full-trace dumps can drop `viewTrace` /
3997
4091
  * `viewSpans` and stay cheap.
3998
- * - Each kind declares its recursion + parallelism budget. Discovery-
3999
- * heavy kinds (failure-mode) get higher `maxDepth`; lens kinds
4000
- * (poisoning) usually stay at 0 since they have a tighter brief.
4092
+ * - Each kind declares its subquery + parallelism budget. Discovery-heavy
4093
+ * kinds can fan out more bounded semantic questions than narrow lenses.
4001
4094
  *
4002
4095
  * Optimizer hook: kinds may declare `goldens` — labeled examples used
4003
- * by `AxMiPRO` / `AxBootstrapFewShot` / `AxGEPA` to fit the actor
4096
+ * by `AxBootstrapFewShot` / `AxGEPA` to fit the actor
4004
4097
  * description programmatically. Stored on the kind, not the registry,
4005
4098
  * because the right metric is kind-specific.
4006
4099
  */
@@ -4020,23 +4113,25 @@ interface TraceAnalystKindSpec {
4020
4113
  version: string;
4021
4114
  /** Actor system prompt. Must instruct the LLM to emit `findings` per the schema. */
4022
4115
  actorDescription: string;
4023
- /** Responder system prompt; falls back to a minimal "format the findings" instruction. */
4024
- responderDescription?: string;
4025
4116
  /** Tool functions the actor may call. Pick narrow subsets via `ANALYST_TOOL_GROUPS`. */
4026
4117
  buildTools: (store: TraceAnalysisStore) => AxFunction[];
4027
- /** Recursion budget. `maxDepth: 0` disables subagents. */
4028
- recursion?: {
4029
- maxDepth: number;
4030
- maxParallelSubagents?: number;
4118
+ /** Bounded semantic subqueries. `maxCalls: 0` disables model fan-out. */
4119
+ subqueries?: {
4120
+ maxCalls: number;
4121
+ maxParallel?: number;
4031
4122
  };
4032
4123
  /** Actor turn cap. Default 12. */
4033
4124
  maxTurns?: number;
4034
4125
  /** Runtime char cap. Default 6000. */
4035
4126
  maxRuntimeChars?: number;
4127
+ /** Maximum output tokens for every actor and subquery model call. Default 4096. */
4128
+ maxOutputTokens?: number;
4036
4129
  /** Cost classification surfaced in `registry.list()` and budget enforcement. */
4037
4130
  cost: AnalystCost;
4038
4131
  /** Per-finding-row hook — kinds may reject / rewrite before lifting. */
4039
4132
  postProcess?: (row: RawAnalystFinding, ctx: AnalystContext) => RawAnalystFinding | null;
4133
+ /** Minimum citations per finding. Default 1; rows below it are rejected. */
4134
+ minimumEvidenceCitations?: number;
4040
4135
  /** Optional optimizer hook — populated when a kind wants to fit its prompt against labeled examples. */
4041
4136
  goldens?: TraceAnalystGolden[];
4042
4137
  }
@@ -4048,7 +4143,7 @@ interface TraceAnalystKindSpec {
4048
4143
  */
4049
4144
  interface TraceAnalystGolden {
4050
4145
  question: string;
4051
- expected: ReadonlyArray<Omit<RawAnalystFinding, 'confidence'>>;
4146
+ expected: ReadonlyArray<Omit<CanonicalRawAnalystFinding, 'confidence'>>;
4052
4147
  }
4053
4148
 
4054
4149
  /**
@@ -4106,7 +4201,8 @@ interface BudgetPolicy {
4106
4201
  /**
4107
4202
  * Custom allocator — receives the analyst, remaining/total budget, and
4108
4203
  * the count of analysts that will run. Returns the per-analyst budget
4109
- * (or undefined to leave it uncapped). Overrides weights when set.
4204
+ * (or undefined only when the run has no overall cap). Overrides weights
4205
+ * when set.
4110
4206
  */
4111
4207
  allocate?: (args: {
4112
4208
  analyst: Analyst;
@@ -4132,10 +4228,14 @@ interface RegistryRunOpts {
4132
4228
  skip?: string[];
4133
4229
  /** Budget policy — totalUsd + optional weights/allocator. Falls back to options.defaultBudget. */
4134
4230
  budget?: BudgetPolicy;
4135
- /** Wall-clock cap. Analysts SHOULD honor `ctx.deadlineMs`. */
4231
+ /** Active-work cap for the complete registry run. Model receipt settlement may follow. */
4136
4232
  timeoutMs?: number;
4137
4233
  /** Abort signal — forwarded into every analyst's context. */
4138
4234
  signal?: AbortSignal;
4235
+ /** Shared paid-call account forwarded to every analyst. */
4236
+ costLedger?: CostLedgerHandle;
4237
+ /** Attribution phase for calls written to `costLedger`. */
4238
+ costPhase?: string;
4139
4239
  /** Tags echoed into AnalystContext.tags — useful for tracking environment/version in findings. */
4140
4240
  tags?: Record<string, string>;
4141
4241
  /**
@@ -4143,10 +4243,16 @@ interface RegistryRunOpts {
4143
4243
  * analyst via `ctx.priorFindings`. The registry forwards the slice
4144
4244
  * whose `analyst_id` matches each registered analyst so a kind sees
4145
4245
  * only its own history. Pass `{ '*': findings }` to broadcast to
4146
- * every analyst (useful for cross-kind chaining where the improvement
4147
- * analyst consumes upstream failure findings).
4246
+ * every analyst (useful when several kinds share the same historical
4247
+ * context). For findings from this run, use `chainFindings` instead.
4148
4248
  */
4149
4249
  priorFindings?: ReadonlyArray<AnalystFinding> | Record<string, ReadonlyArray<AnalystFinding>>;
4250
+ /**
4251
+ * Pass findings produced earlier in this registry run to each later analyst
4252
+ * via `ctx.upstreamFindings`. Registration order is dependency order.
4253
+ * Disabled by default because independent analyst suites must opt in.
4254
+ */
4255
+ chainFindings?: boolean;
4150
4256
  }
4151
4257
  declare class AnalystRegistry {
4152
4258
  private readonly analysts;
@@ -4190,7 +4296,7 @@ declare class AnalystRegistry {
4190
4296
  interface DefaultAnalystRegistryOptions {
4191
4297
  /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */
4192
4298
  ai?: AxAIService;
4193
- /** Model for the agentic kinds (falls back to the ai service default). */
4299
+ /** Required unless `ai` was created by `createAnalystAi`. */
4194
4300
  model?: string;
4195
4301
  /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */
4196
4302
  kinds?: readonly TraceAnalystKindSpec[];
@@ -4836,4 +4942,4 @@ interface FromOtelSpansOptions {
4836
4942
  }
4837
4943
  declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
4838
4944
 
4839
- export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type ChatClient, type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, type CodeSurface, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runEval, runImprovementLoop, runReferenceEquivalenceJudge, selfImprove, summarizeExecution };
4945
+ export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type ChatClient, type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, type CodeSurface, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MeasuredComparisonFromSelfImproveResultOptions, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, measuredComparisonFromSelfImproveResult, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runEval, runImprovementLoop, runReferenceEquivalenceJudge, selfImprove, summarizeExecution };