@tangle-network/agent-eval 0.118.3 → 0.119.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/README.md +26 -0
  3. package/dist/analyst/index.d.ts +143 -21
  4. package/dist/analyst/index.js +44 -10
  5. package/dist/analyst/index.js.map +1 -1
  6. package/dist/benchmarks/index.d.ts +50 -40
  7. package/dist/benchmarks/index.js +6 -6
  8. package/dist/campaign/index.d.ts +231 -156
  9. package/dist/campaign/index.js +5 -5
  10. package/dist/{chunk-JTMFT5QZ.js → chunk-4I2E3LLO.js} +2 -2
  11. package/dist/{chunk-VNLWDTDA.js → chunk-7A4LIMMY.js} +845 -252
  12. package/dist/chunk-7A4LIMMY.js.map +1 -0
  13. package/dist/{chunk-KK2O4VWA.js → chunk-D7AEXSM5.js} +2 -2
  14. package/dist/{chunk-EKHHBKS6.js → chunk-F6YUH3L4.js} +39 -28
  15. package/dist/{chunk-EKHHBKS6.js.map → chunk-F6YUH3L4.js.map} +1 -1
  16. package/dist/{chunk-267UCQWI.js → chunk-GERDAIAL.js} +2 -2
  17. package/dist/{chunk-267UCQWI.js.map → chunk-GERDAIAL.js.map} +1 -1
  18. package/dist/{chunk-MLHRFWQK.js → chunk-JHOJHHU7.js} +85 -16
  19. package/dist/chunk-JHOJHHU7.js.map +1 -0
  20. package/dist/{chunk-F5A4GDQW.js → chunk-LMJ2TGWJ.js} +3 -15
  21. package/dist/chunk-LMJ2TGWJ.js.map +1 -0
  22. package/dist/{chunk-EGNRE7VA.js → chunk-PAHNGS65.js} +2 -2
  23. package/dist/{chunk-QWRLW4CT.js → chunk-RLCJQ6VZ.js} +5 -5
  24. package/dist/{chunk-7V3QDWNL.js → chunk-XJ7JVCHB.js} +2 -2
  25. package/dist/cli.js +2 -2
  26. package/dist/contract/index.d.ts +257 -160
  27. package/dist/contract/index.js +6 -6
  28. package/dist/fuzz.d.ts +19 -1
  29. package/dist/fuzz.js +1 -1
  30. package/dist/index.d.ts +154 -48
  31. package/dist/index.js +12 -10
  32. package/dist/index.js.map +1 -1
  33. package/dist/openapi.json +1 -1
  34. package/dist/primeintellect/index.d.ts +311 -0
  35. package/dist/primeintellect/index.js +348 -0
  36. package/dist/primeintellect/index.js.map +1 -0
  37. package/dist/rl.d.ts +4 -0
  38. package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js} +3 -3
  39. package/dist/wire/index.d.ts +19 -1
  40. package/dist/wire/index.js +2 -2
  41. package/package.json +6 -1
  42. package/dist/chunk-F5A4GDQW.js.map +0 -1
  43. package/dist/chunk-MLHRFWQK.js.map +0 -1
  44. package/dist/chunk-VNLWDTDA.js.map +0 -1
  45. /package/dist/{chunk-JTMFT5QZ.js.map → chunk-4I2E3LLO.js.map} +0 -0
  46. /package/dist/{chunk-KK2O4VWA.js.map → chunk-D7AEXSM5.js.map} +0 -0
  47. /package/dist/{chunk-EGNRE7VA.js.map → chunk-PAHNGS65.js.map} +0 -0
  48. /package/dist/{chunk-QWRLW4CT.js.map → chunk-RLCJQ6VZ.js.map} +0 -0
  49. /package/dist/{chunk-7V3QDWNL.js.map → chunk-XJ7JVCHB.js.map} +0 -0
  50. /package/dist/{run-campaign-DWC67KJP.js.map → run-campaign-OBXSN5WK.js.map} +0 -0
@@ -28,6 +28,174 @@ interface AgentProfileCell {
28
28
  dimensions?: Record<string, AgentProfileDimensionValue>;
29
29
  }
30
30
 
31
+ type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
32
+ interface CostUsage {
33
+ inputTokens: number;
34
+ /** Includes reasoning tokens when the provider bills them as output. */
35
+ outputTokens: number;
36
+ /** Reasoning-token subset of outputTokens, when reported. */
37
+ reasoningTokens?: number;
38
+ /** Prompt tokens served from a provider cache. */
39
+ cachedTokens?: number;
40
+ /** Prompt tokens written into a provider cache. */
41
+ cacheWriteTokens?: number;
42
+ }
43
+ interface CostCallBase {
44
+ callId: string;
45
+ channel: CostChannel;
46
+ phase: string;
47
+ actor: string;
48
+ model: string;
49
+ maximumCostUsd?: number;
50
+ tags?: Record<string, string>;
51
+ timestamp: number;
52
+ }
53
+ interface CostReceipt extends CostCallBase, CostUsage {
54
+ status: 'settled';
55
+ costUsd: number;
56
+ costUnknown: boolean;
57
+ usageUnknown?: boolean;
58
+ pricing?: {
59
+ inputUsdPerThousand: number;
60
+ outputUsdPerThousand: number;
61
+ };
62
+ actualCostUsd?: number;
63
+ error?: string;
64
+ }
65
+ interface CostReceiptInput extends CostUsage {
66
+ model: string;
67
+ actualCostUsd?: number;
68
+ costUnknown?: boolean;
69
+ usageUnknown?: boolean;
70
+ }
71
+ type MaximumCharge = {
72
+ externallyEnforcedMaximumUsd: number;
73
+ } | ({
74
+ model: string;
75
+ } & CostUsage);
76
+ interface RunPaidCallInput<T> {
77
+ callId?: string;
78
+ channel: CostChannel;
79
+ phase: string;
80
+ actor: string;
81
+ /** Used before a provider receipt exists and on failures without one. */
82
+ model?: string;
83
+ tags?: Record<string, string>;
84
+ signal?: AbortSignal;
85
+ /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
86
+ maximumCharge?: MaximumCharge;
87
+ /** `callId` can be forwarded as the provider's idempotency key. */
88
+ execute(signal: AbortSignal, callId: string): Promise<T>;
89
+ receipt(value: T): CostReceiptInput;
90
+ receiptFromError?(error: Error): CostReceiptInput | undefined;
91
+ }
92
+ type PaidCallResult<T> = {
93
+ succeeded: true;
94
+ callId: string;
95
+ value: T;
96
+ receipt: CostReceipt;
97
+ } | {
98
+ succeeded: false;
99
+ callId?: string;
100
+ error: Error;
101
+ receipt?: CostReceipt;
102
+ };
103
+ interface ChannelRollup {
104
+ channel: CostChannel;
105
+ calls: number;
106
+ inputTokens: number;
107
+ outputTokens: number;
108
+ reasoningTokens?: number;
109
+ cachedTokens: number;
110
+ cacheWriteTokens?: number;
111
+ costUsd: number;
112
+ unpricedCalls: number;
113
+ unknownUsageCalls: number;
114
+ }
115
+ interface CostLedgerSummary {
116
+ totalCalls: number;
117
+ pendingCalls: number;
118
+ unresolvedCalls: number;
119
+ reservedCostUsd: number;
120
+ inputTokens: number;
121
+ outputTokens: number;
122
+ reasoningTokens?: number;
123
+ cachedTokens: number;
124
+ cacheWriteTokens?: number;
125
+ totalCostUsd: number;
126
+ byChannel: ChannelRollup[];
127
+ unpricedModels: string[];
128
+ fullyPriced: boolean;
129
+ usageComplete: boolean;
130
+ accountingComplete: boolean;
131
+ incompleteReasons: string[];
132
+ }
133
+ interface CostLedgerFilter {
134
+ channel?: CostChannel;
135
+ phase?: string;
136
+ tags?: Record<string, string>;
137
+ }
138
+ interface CostLedgerWaitOptions {
139
+ /** Maximum time to wait for active provider calls. Default 5 seconds. */
140
+ timeoutMs?: number;
141
+ }
142
+ /** Append-only storage. `append` must atomically reject stale revisions. */
143
+ interface CostLedgerPersistence {
144
+ read(): {
145
+ revision: string;
146
+ events: string;
147
+ };
148
+ append(expectedRevision: string, event: string): string | undefined;
149
+ }
150
+ interface CostLedgerOptions {
151
+ costCeilingUsd?: number;
152
+ persistence?: CostLedgerPersistence;
153
+ /** Import already-settled receipts without admitting new paid work. */
154
+ receipts?: readonly CostReceipt[];
155
+ }
156
+ /** Run-wide paid-call admission, durable call state, receipts, and summaries. */
157
+ declare class CostLedger {
158
+ private readonly records;
159
+ private readonly activeCallIds;
160
+ private readonly lateCallIds;
161
+ private readonly idleWaiters;
162
+ private completedTasks;
163
+ private revision;
164
+ private costLimitPersisted;
165
+ readonly costCeilingUsd?: number;
166
+ private readonly persistence?;
167
+ constructor(input?: number | CostLedgerOptions);
168
+ runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
169
+ /** Wait until every call started by this ledger has produced a durable outcome. */
170
+ waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
171
+ /** Settle a call left pending by a crashed process after reconciling with the provider. */
172
+ reconcile(callId: string, observed: CostReceiptInput, options?: {
173
+ error?: string;
174
+ }): CostReceipt;
175
+ list(filter?: CostLedgerFilter): CostReceipt[];
176
+ summary(filter?: CostLedgerFilter): CostLedgerSummary;
177
+ markCompleted(count?: number): void;
178
+ costPerCompletedTask(): number | null;
179
+ private execute;
180
+ private captureLateOutcome;
181
+ private releaseActiveCall;
182
+ private commitOutcome;
183
+ private captureFailure;
184
+ private commitReceipt;
185
+ private resolveMaximum;
186
+ private hasIncompleteSettledCall;
187
+ private appendRecord;
188
+ private ensureCostLimitPersisted;
189
+ private appendEvent;
190
+ }
191
+ /** Public callback surface for a shared cost ledger.
192
+ *
193
+ * Declaration bundles may expose this type through multiple package subpaths.
194
+ * Keeping callback contracts structural lets those subpaths compose while the
195
+ * concrete {@link CostLedger} retains its private durable state.
196
+ */
197
+ type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
198
+
31
199
  type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
32
200
 
33
201
  /**
@@ -475,156 +643,6 @@ interface TraceAnalysisStore {
475
643
  }): Promise<SearchSpanResult>;
476
644
  }
477
645
 
478
- type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
479
- interface CostUsage {
480
- inputTokens: number;
481
- outputTokens: number;
482
- cachedTokens?: number;
483
- }
484
- interface CostCallBase {
485
- callId: string;
486
- channel: CostChannel;
487
- phase: string;
488
- actor: string;
489
- model: string;
490
- maximumCostUsd?: number;
491
- tags?: Record<string, string>;
492
- timestamp: number;
493
- }
494
- interface CostReceipt extends CostCallBase, CostUsage {
495
- status: 'settled';
496
- costUsd: number;
497
- costUnknown: boolean;
498
- usageUnknown?: boolean;
499
- pricing?: {
500
- inputUsdPerThousand: number;
501
- outputUsdPerThousand: number;
502
- };
503
- actualCostUsd?: number;
504
- error?: string;
505
- }
506
- interface CostReceiptInput extends CostUsage {
507
- model: string;
508
- actualCostUsd?: number;
509
- costUnknown?: boolean;
510
- usageUnknown?: boolean;
511
- }
512
- type MaximumCharge = {
513
- externallyEnforcedMaximumUsd: number;
514
- } | ({
515
- model: string;
516
- } & CostUsage);
517
- interface RunPaidCallInput<T> {
518
- callId?: string;
519
- channel: CostChannel;
520
- phase: string;
521
- actor: string;
522
- /** Used before a provider receipt exists and on failures without one. */
523
- model?: string;
524
- tags?: Record<string, string>;
525
- signal?: AbortSignal;
526
- /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
527
- maximumCharge?: MaximumCharge;
528
- /** `callId` can be forwarded as the provider's idempotency key. */
529
- execute(signal: AbortSignal, callId: string): Promise<T>;
530
- receipt(value: T): CostReceiptInput;
531
- receiptFromError?(error: Error): CostReceiptInput | undefined;
532
- }
533
- type PaidCallResult<T> = {
534
- succeeded: true;
535
- callId: string;
536
- value: T;
537
- receipt: CostReceipt;
538
- } | {
539
- succeeded: false;
540
- callId?: string;
541
- error: Error;
542
- receipt?: CostReceipt;
543
- };
544
- interface ChannelRollup {
545
- channel: CostChannel;
546
- calls: number;
547
- inputTokens: number;
548
- outputTokens: number;
549
- cachedTokens: number;
550
- costUsd: number;
551
- unpricedCalls: number;
552
- unknownUsageCalls: number;
553
- }
554
- interface CostLedgerSummary {
555
- totalCalls: number;
556
- pendingCalls: number;
557
- unresolvedCalls: number;
558
- reservedCostUsd: number;
559
- inputTokens: number;
560
- outputTokens: number;
561
- cachedTokens: number;
562
- totalCostUsd: number;
563
- byChannel: ChannelRollup[];
564
- unpricedModels: string[];
565
- fullyPriced: boolean;
566
- usageComplete: boolean;
567
- accountingComplete: boolean;
568
- incompleteReasons: string[];
569
- }
570
- interface CostLedgerFilter {
571
- channel?: CostChannel;
572
- phase?: string;
573
- tags?: Record<string, string>;
574
- }
575
- /** Append-only storage. `append` must atomically reject stale revisions. */
576
- interface CostLedgerPersistence {
577
- read(): {
578
- revision: string;
579
- events: string;
580
- };
581
- append(expectedRevision: string, event: string): string | undefined;
582
- }
583
- interface CostLedgerOptions {
584
- costCeilingUsd?: number;
585
- persistence?: CostLedgerPersistence;
586
- /** Import already-settled receipts without admitting new paid work. */
587
- receipts?: readonly CostReceipt[];
588
- }
589
- /** Run-wide paid-call admission, durable call state, receipts, and summaries. */
590
- declare class CostLedger {
591
- private readonly records;
592
- private readonly activeCallIds;
593
- private readonly lateCallIds;
594
- private completedTasks;
595
- private revision;
596
- private costLimitPersisted;
597
- readonly costCeilingUsd?: number;
598
- private readonly persistence?;
599
- constructor(input?: number | CostLedgerOptions);
600
- runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
601
- /** Settle a call left pending by a crashed process after reconciling with the provider. */
602
- reconcile(callId: string, observed: CostReceiptInput, options?: {
603
- error?: string;
604
- }): CostReceipt;
605
- list(filter?: CostLedgerFilter): CostReceipt[];
606
- summary(filter?: CostLedgerFilter): CostLedgerSummary;
607
- markCompleted(count?: number): void;
608
- costPerCompletedTask(): number | null;
609
- private execute;
610
- private captureLateOutcome;
611
- private commitOutcome;
612
- private captureFailure;
613
- private commitReceipt;
614
- private resolveMaximum;
615
- private hasIncompleteSettledCall;
616
- private appendRecord;
617
- private ensureCostLimitPersisted;
618
- private appendEvent;
619
- }
620
- /** Public callback surface for a shared cost ledger.
621
- *
622
- * Declaration bundles may expose this type through multiple package subpaths.
623
- * Keeping callback contracts structural lets those subpaths compose while the
624
- * concrete {@link CostLedger} retains its private durable state.
625
- */
626
- type CostLedgerHandle = Pick<CostLedger, keyof CostLedger>;
627
-
628
646
  interface Scenario$1 {
629
647
  id: string;
630
648
  persona: string;
@@ -1131,6 +1149,10 @@ interface AnalystContext {
1131
1149
  deadlineMs?: number;
1132
1150
  /** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
1133
1151
  budgetUsd?: number;
1152
+ /** Shared paid-call account when the analyst runs inside a larger campaign. */
1153
+ costLedger?: CostLedgerHandle;
1154
+ /** Attribution phase used when writing to the shared paid-call account. */
1155
+ costPhase?: string;
1134
1156
  /**
1135
1157
  * Shared chat client. Analysts that call an LLM go through this so
1136
1158
  * the operator picks transport (sandbox-sdk | router | cli-bridge |
@@ -1148,6 +1170,18 @@ interface AnalystContext {
1148
1170
  * filter. Empty / absent means no cross-run context.
1149
1171
  */
1150
1172
  priorFindings?: ReadonlyArray<AnalystFinding>;
1173
+ /**
1174
+ * Findings emitted by analysts that completed earlier in this registry run.
1175
+ * This is separate from `priorFindings`: upstream findings are dependency
1176
+ * context for the current pass, while prior findings are cross-run memory.
1177
+ * The registry populates this only when `RegistryRunOpts.chainFindings` is on.
1178
+ */
1179
+ upstreamFindings?: ReadonlyArray<AnalystFinding>;
1180
+ /**
1181
+ * Report metered work independently of findings. This keeps an empty finding
1182
+ * set from erasing token/cost telemetry. Multiple receipts are accumulated.
1183
+ */
1184
+ recordUsage?: (receipt: AnalystUsageReceipt) => void;
1151
1185
  /** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */
1152
1186
  tags?: Record<string, string>;
1153
1187
  /** Logger callback — analysts SHOULD prefer this over console.* for testability. */
@@ -1173,6 +1207,17 @@ interface Analyst<TInput = unknown> {
1173
1207
  readonly version: string;
1174
1208
  analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>;
1175
1209
  }
1210
+ /** Metered work performed by one analyst call. */
1211
+ interface AnalystUsageReceipt {
1212
+ /** Number of model-usage records observed at the provider boundary. */
1213
+ calls: number | null;
1214
+ /** Null when the provider did not return token accounting. */
1215
+ tokens: RunTokenUsage | null;
1216
+ /** Observed, estimated, or explicitly uncaptured dollar cost. */
1217
+ cost: RunCostProvenance;
1218
+ /** Known lower bound when one or more calls have uncaptured cost. */
1219
+ knownCostUsd?: number;
1220
+ }
1176
1221
  interface AnalystRunSummary {
1177
1222
  analyst_id: string;
1178
1223
  status: 'ok' | 'skipped' | 'failed';
@@ -1181,6 +1226,12 @@ interface AnalystRunSummary {
1181
1226
  findings_count: number;
1182
1227
  latency_ms: number;
1183
1228
  cost_usd: number;
1229
+ /**
1230
+ * Additive receipt for model usage. Registry-produced summaries populate it
1231
+ * even when the analyst emits no findings. `cost_usd` remains the legacy
1232
+ * numeric field; inspect `usage.cost` before treating zero as observed.
1233
+ */
1234
+ usage?: AnalystUsageReceipt;
1184
1235
  /** When `status='failed'`: the error class + message, never the full stack. */
1185
1236
  error?: {
1186
1237
  class: string;
@@ -1196,6 +1247,11 @@ interface AnalystRunResult {
1196
1247
  per_analyst: AnalystRunSummary[];
1197
1248
  /** Total LLM cost in USD across all analysts in this registry.run(). */
1198
1249
  total_cost_usd: number;
1250
+ /**
1251
+ * Provenance for `total_cost_usd`. When uncaptured, the numeric field is only
1252
+ * the known subtotal and must not be treated as the run's total spend.
1253
+ */
1254
+ total_cost_provenance?: RunCostProvenance;
1199
1255
  }
1200
1256
  /**
1201
1257
  * Events emitted by `AnalystRegistry.runStream(...)` in real time as
@@ -3961,7 +4017,10 @@ declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults
3961
4017
  * validation surface independent of which Ax version is installed.
3962
4018
  */
3963
4019
 
4020
+ /** Original public schema retained for stored rows and callback contracts. */
3964
4021
  declare const RawAnalystFindingSchema: z.ZodObject<{
4022
+ evidence_uri: z.ZodString;
4023
+ evidence_excerpt: z.ZodOptional<z.ZodString>;
3965
4024
  severity: z.ZodEnum<{
3966
4025
  low: "low";
3967
4026
  high: "high";
@@ -3971,13 +4030,36 @@ declare const RawAnalystFindingSchema: z.ZodObject<{
3971
4030
  }>;
3972
4031
  claim: z.ZodString;
3973
4032
  subject: z.ZodOptional<z.ZodString>;
3974
- evidence_uri: z.ZodString;
3975
- evidence_excerpt: z.ZodOptional<z.ZodString>;
3976
4033
  confidence: z.ZodNumber;
3977
4034
  rationale: z.ZodOptional<z.ZodString>;
3978
4035
  recommended_action: z.ZodOptional<z.ZodString>;
3979
4036
  }, z.core.$strict>;
3980
4037
  type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
4038
+ /**
4039
+ * Canonical plural-evidence contract. The preprocessor accepts the original
4040
+ * `evidence_uri` / `evidence_excerpt` pair and normalizes it into one evidence
4041
+ * item so persisted rows and older model fixtures remain readable. New output
4042
+ * always receives the plural shape.
4043
+ */
4044
+ declare const CanonicalRawAnalystFindingSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodObject<{
4045
+ evidence: z.ZodArray<z.ZodObject<{
4046
+ uri: z.ZodString;
4047
+ excerpt: z.ZodOptional<z.ZodString>;
4048
+ }, z.core.$strict>>;
4049
+ severity: z.ZodEnum<{
4050
+ low: "low";
4051
+ high: "high";
4052
+ critical: "critical";
4053
+ medium: "medium";
4054
+ info: "info";
4055
+ }>;
4056
+ claim: z.ZodString;
4057
+ subject: z.ZodOptional<z.ZodString>;
4058
+ confidence: z.ZodNumber;
4059
+ rationale: z.ZodOptional<z.ZodString>;
4060
+ recommended_action: z.ZodOptional<z.ZodString>;
4061
+ }, z.core.$strict>>;
4062
+ type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchema>;
3981
4063
 
3982
4064
  /**
3983
4065
  * Analyst-kind factory — the typed way to define trace analysts.
@@ -3985,9 +4067,9 @@ type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
3985
4067
  * A "kind" is a specialized analyst whose actor prompt, tool subset,
3986
4068
  * and Ax recursion config target one failure-mode lens (failure-mode
3987
4069
  * classification, knowledge gap discovery, knowledge poisoning, recursive
3988
- * self-improvement, ...). Kinds emit findings in the typed `RawAnalystFinding`
3989
- * shape via a JSON-array Ax output; the factory validates each row with
3990
- * Zod and lifts it into `AnalystFinding[]` with no shape guessing.
4070
+ * self-improvement, ...). Kinds emit findings in the typed
4071
+ * `CanonicalRawAnalystFinding` shape via a JSON-array Ax output; the factory
4072
+ * validates each row with Zod and lifts it into `AnalystFinding[]`.
3991
4073
  *
3992
4074
  * Composition rules:
3993
4075
  * - Each kind owns its actor description. No generic "answer this
@@ -4033,10 +4115,14 @@ interface TraceAnalystKindSpec {
4033
4115
  maxTurns?: number;
4034
4116
  /** Runtime char cap. Default 6000. */
4035
4117
  maxRuntimeChars?: number;
4118
+ /** Maximum output tokens for every actor, responder, and recursive model call. Default 4096. */
4119
+ maxOutputTokens?: number;
4036
4120
  /** Cost classification surfaced in `registry.list()` and budget enforcement. */
4037
4121
  cost: AnalystCost;
4038
4122
  /** Per-finding-row hook — kinds may reject / rewrite before lifting. */
4039
4123
  postProcess?: (row: RawAnalystFinding, ctx: AnalystContext) => RawAnalystFinding | null;
4124
+ /** Minimum citations per finding. Default 1; rows below it are rejected. */
4125
+ minimumEvidenceCitations?: number;
4040
4126
  /** Optional optimizer hook — populated when a kind wants to fit its prompt against labeled examples. */
4041
4127
  goldens?: TraceAnalystGolden[];
4042
4128
  }
@@ -4048,7 +4134,7 @@ interface TraceAnalystKindSpec {
4048
4134
  */
4049
4135
  interface TraceAnalystGolden {
4050
4136
  question: string;
4051
- expected: ReadonlyArray<Omit<RawAnalystFinding, 'confidence'>>;
4137
+ expected: ReadonlyArray<Omit<CanonicalRawAnalystFinding, 'confidence'>>;
4052
4138
  }
4053
4139
 
4054
4140
  /**
@@ -4106,7 +4192,8 @@ interface BudgetPolicy {
4106
4192
  /**
4107
4193
  * Custom allocator — receives the analyst, remaining/total budget, and
4108
4194
  * the count of analysts that will run. Returns the per-analyst budget
4109
- * (or undefined to leave it uncapped). Overrides weights when set.
4195
+ * (or undefined only when the run has no overall cap). Overrides weights
4196
+ * when set.
4110
4197
  */
4111
4198
  allocate?: (args: {
4112
4199
  analyst: Analyst;
@@ -4136,6 +4223,10 @@ interface RegistryRunOpts {
4136
4223
  timeoutMs?: number;
4137
4224
  /** Abort signal — forwarded into every analyst's context. */
4138
4225
  signal?: AbortSignal;
4226
+ /** Shared paid-call account forwarded to every analyst. */
4227
+ costLedger?: CostLedgerHandle;
4228
+ /** Attribution phase for calls written to `costLedger`. */
4229
+ costPhase?: string;
4139
4230
  /** Tags echoed into AnalystContext.tags — useful for tracking environment/version in findings. */
4140
4231
  tags?: Record<string, string>;
4141
4232
  /**
@@ -4143,10 +4234,16 @@ interface RegistryRunOpts {
4143
4234
  * analyst via `ctx.priorFindings`. The registry forwards the slice
4144
4235
  * whose `analyst_id` matches each registered analyst so a kind sees
4145
4236
  * only its own history. Pass `{ '*': findings }` to broadcast to
4146
- * every analyst (useful for cross-kind chaining where the improvement
4147
- * analyst consumes upstream failure findings).
4237
+ * every analyst (useful when several kinds share the same historical
4238
+ * context). For findings from this run, use `chainFindings` instead.
4148
4239
  */
4149
4240
  priorFindings?: ReadonlyArray<AnalystFinding> | Record<string, ReadonlyArray<AnalystFinding>>;
4241
+ /**
4242
+ * Pass findings produced earlier in this registry run to each later analyst
4243
+ * via `ctx.upstreamFindings`. Registration order is dependency order.
4244
+ * Disabled by default because independent analyst suites must opt in.
4245
+ */
4246
+ chainFindings?: boolean;
4150
4247
  }
4151
4248
  declare class AnalystRegistry {
4152
4249
  private readonly analysts;
@@ -4190,7 +4287,7 @@ declare class AnalystRegistry {
4190
4287
  interface DefaultAnalystRegistryOptions {
4191
4288
  /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */
4192
4289
  ai?: AxAIService;
4193
- /** Model for the agentic kinds (falls back to the ai service default). */
4290
+ /** Required unless `ai` was created by `createAnalystAi`. */
4194
4291
  model?: string;
4195
4292
  /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */
4196
4293
  kinds?: readonly TraceAnalystKindSpec[];
@@ -13,7 +13,7 @@ import {
13
13
  import {
14
14
  analyzeRuns,
15
15
  summarizeExecution
16
- } from "../chunk-7V3QDWNL.js";
16
+ } from "../chunk-XJ7JVCHB.js";
17
17
  import {
18
18
  REFERENCE_EQUIVALENCE_INPUT_LIMITS,
19
19
  REFERENCE_EQUIVALENCE_JUDGE_VERSION,
@@ -34,20 +34,20 @@ import {
34
34
  runReferenceEquivalenceJudge,
35
35
  surfaceContentHash,
36
36
  surfaceHash
37
- } from "../chunk-QWRLW4CT.js";
37
+ } from "../chunk-RLCJQ6VZ.js";
38
38
  import {
39
39
  createRunCostLedger,
40
40
  fsCampaignStorage,
41
41
  inMemoryCampaignStorage,
42
42
  resolveRunDir,
43
43
  runCampaign
44
- } from "../chunk-EGNRE7VA.js";
44
+ } from "../chunk-PAHNGS65.js";
45
45
  import {
46
46
  buildDefaultAnalystRegistry,
47
47
  createChatClient
48
- } from "../chunk-267UCQWI.js";
48
+ } from "../chunk-GERDAIAL.js";
49
49
  import "../chunk-HHWE3POT.js";
50
- import "../chunk-VNLWDTDA.js";
50
+ import "../chunk-7A4LIMMY.js";
51
51
  import {
52
52
  FileSystemOutcomeStore,
53
53
  InMemoryOutcomeStore
@@ -56,7 +56,7 @@ import "../chunk-ARU2PZFM.js";
56
56
  import "../chunk-NJC7U437.js";
57
57
  import "../chunk-DPZAEKA6.js";
58
58
  import "../chunk-PJQFMIOX.js";
59
- import "../chunk-MLHRFWQK.js";
59
+ import "../chunk-JHOJHHU7.js";
60
60
  import "../chunk-VI2UW6B6.js";
61
61
  import {
62
62
  recordAggregateMeasurements,
package/dist/fuzz.d.ts CHANGED
@@ -1,8 +1,14 @@
1
1
  type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
2
2
  interface CostUsage {
3
3
  inputTokens: number;
4
+ /** Includes reasoning tokens when the provider bills them as output. */
4
5
  outputTokens: number;
6
+ /** Reasoning-token subset of outputTokens, when reported. */
7
+ reasoningTokens?: number;
8
+ /** Prompt tokens served from a provider cache. */
5
9
  cachedTokens?: number;
10
+ /** Prompt tokens written into a provider cache. */
11
+ cacheWriteTokens?: number;
6
12
  }
7
13
  interface CostCallBase {
8
14
  callId: string;
@@ -69,7 +75,9 @@ interface ChannelRollup {
69
75
  calls: number;
70
76
  inputTokens: number;
71
77
  outputTokens: number;
78
+ reasoningTokens?: number;
72
79
  cachedTokens: number;
80
+ cacheWriteTokens?: number;
73
81
  costUsd: number;
74
82
  unpricedCalls: number;
75
83
  unknownUsageCalls: number;
@@ -81,7 +89,9 @@ interface CostLedgerSummary {
81
89
  reservedCostUsd: number;
82
90
  inputTokens: number;
83
91
  outputTokens: number;
92
+ reasoningTokens?: number;
84
93
  cachedTokens: number;
94
+ cacheWriteTokens?: number;
85
95
  totalCostUsd: number;
86
96
  byChannel: ChannelRollup[];
87
97
  unpricedModels: string[];
@@ -95,6 +105,10 @@ interface CostLedgerFilter {
95
105
  phase?: string;
96
106
  tags?: Record<string, string>;
97
107
  }
108
+ interface CostLedgerWaitOptions {
109
+ /** Maximum time to wait for active provider calls. Default 5 seconds. */
110
+ timeoutMs?: number;
111
+ }
98
112
  /** Append-only storage. `append` must atomically reject stale revisions. */
99
113
  interface CostLedgerPersistence {
100
114
  read(): {
@@ -114,6 +128,7 @@ declare class CostLedger {
114
128
  private readonly records;
115
129
  private readonly activeCallIds;
116
130
  private readonly lateCallIds;
131
+ private readonly idleWaiters;
117
132
  private completedTasks;
118
133
  private revision;
119
134
  private costLimitPersisted;
@@ -121,6 +136,8 @@ declare class CostLedger {
121
136
  private readonly persistence?;
122
137
  constructor(input?: number | CostLedgerOptions);
123
138
  runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
139
+ /** Wait until every call started by this ledger has produced a durable outcome. */
140
+ waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
124
141
  /** Settle a call left pending by a crashed process after reconciling with the provider. */
125
142
  reconcile(callId: string, observed: CostReceiptInput, options?: {
126
143
  error?: string;
@@ -131,6 +148,7 @@ declare class CostLedger {
131
148
  costPerCompletedTask(): number | null;
132
149
  private execute;
133
150
  private captureLateOutcome;
151
+ private releaseActiveCall;
134
152
  private commitOutcome;
135
153
  private captureFailure;
136
154
  private commitReceipt;
@@ -146,7 +164,7 @@ declare class CostLedger {
146
164
  * Keeping callback contracts structural lets those subpaths compose while the
147
165
  * concrete {@link CostLedger} retains its private durable state.
148
166
  */
149
- type CostLedgerHandle = Pick<CostLedger, keyof CostLedger>;
167
+ type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
150
168
 
151
169
  /**
152
170
  * Adversarial mutation contract.
package/dist/fuzz.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import {
2
2
  CostLedger,
3
3
  CostReceiptCaptureError
4
- } from "./chunk-MLHRFWQK.js";
4
+ } from "./chunk-JHOJHHU7.js";
5
5
  import "./chunk-VI2UW6B6.js";
6
6
  import {
7
7
  varianceBasedCurriculum