@tangle-network/agent-eval 0.118.3 → 0.119.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +26 -0
- package/dist/analyst/index.d.ts +143 -21
- package/dist/analyst/index.js +44 -10
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +50 -40
- package/dist/benchmarks/index.js +6 -6
- package/dist/campaign/index.d.ts +231 -156
- package/dist/campaign/index.js +5 -5
- package/dist/{chunk-JTMFT5QZ.js → chunk-4I2E3LLO.js} +2 -2
- package/dist/{chunk-VNLWDTDA.js → chunk-7A4LIMMY.js} +845 -252
- package/dist/chunk-7A4LIMMY.js.map +1 -0
- package/dist/{chunk-KK2O4VWA.js → chunk-D7AEXSM5.js} +2 -2
- package/dist/{chunk-EKHHBKS6.js → chunk-F6YUH3L4.js} +39 -28
- package/dist/{chunk-EKHHBKS6.js.map → chunk-F6YUH3L4.js.map} +1 -1
- package/dist/{chunk-267UCQWI.js → chunk-GERDAIAL.js} +2 -2
- package/dist/{chunk-267UCQWI.js.map → chunk-GERDAIAL.js.map} +1 -1
- package/dist/{chunk-MLHRFWQK.js → chunk-JHOJHHU7.js} +85 -16
- package/dist/chunk-JHOJHHU7.js.map +1 -0
- package/dist/{chunk-F5A4GDQW.js → chunk-LMJ2TGWJ.js} +3 -15
- package/dist/chunk-LMJ2TGWJ.js.map +1 -0
- package/dist/{chunk-EGNRE7VA.js → chunk-PAHNGS65.js} +2 -2
- package/dist/{chunk-QWRLW4CT.js → chunk-RLCJQ6VZ.js} +5 -5
- package/dist/{chunk-7V3QDWNL.js → chunk-XJ7JVCHB.js} +2 -2
- package/dist/cli.js +2 -2
- package/dist/contract/index.d.ts +257 -160
- package/dist/contract/index.js +6 -6
- package/dist/fuzz.d.ts +19 -1
- package/dist/fuzz.js +1 -1
- package/dist/index.d.ts +154 -48
- package/dist/index.js +12 -10
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/primeintellect/index.d.ts +311 -0
- package/dist/primeintellect/index.js +348 -0
- package/dist/primeintellect/index.js.map +1 -0
- package/dist/rl.d.ts +4 -0
- package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js} +3 -3
- package/dist/wire/index.d.ts +19 -1
- package/dist/wire/index.js +2 -2
- package/package.json +6 -1
- package/dist/chunk-F5A4GDQW.js.map +0 -1
- package/dist/chunk-MLHRFWQK.js.map +0 -1
- package/dist/chunk-VNLWDTDA.js.map +0 -1
- /package/dist/{chunk-JTMFT5QZ.js.map → chunk-4I2E3LLO.js.map} +0 -0
- /package/dist/{chunk-KK2O4VWA.js.map → chunk-D7AEXSM5.js.map} +0 -0
- /package/dist/{chunk-EGNRE7VA.js.map → chunk-PAHNGS65.js.map} +0 -0
- /package/dist/{chunk-QWRLW4CT.js.map → chunk-RLCJQ6VZ.js.map} +0 -0
- /package/dist/{chunk-7V3QDWNL.js.map → chunk-XJ7JVCHB.js.map} +0 -0
- /package/dist/{run-campaign-DWC67KJP.js.map → run-campaign-OBXSN5WK.js.map} +0 -0
package/dist/contract/index.d.ts
CHANGED
|
@@ -28,6 +28,174 @@ interface AgentProfileCell {
|
|
|
28
28
|
dimensions?: Record<string, AgentProfileDimensionValue>;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
32
|
+
interface CostUsage {
|
|
33
|
+
inputTokens: number;
|
|
34
|
+
/** Includes reasoning tokens when the provider bills them as output. */
|
|
35
|
+
outputTokens: number;
|
|
36
|
+
/** Reasoning-token subset of outputTokens, when reported. */
|
|
37
|
+
reasoningTokens?: number;
|
|
38
|
+
/** Prompt tokens served from a provider cache. */
|
|
39
|
+
cachedTokens?: number;
|
|
40
|
+
/** Prompt tokens written into a provider cache. */
|
|
41
|
+
cacheWriteTokens?: number;
|
|
42
|
+
}
|
|
43
|
+
interface CostCallBase {
|
|
44
|
+
callId: string;
|
|
45
|
+
channel: CostChannel;
|
|
46
|
+
phase: string;
|
|
47
|
+
actor: string;
|
|
48
|
+
model: string;
|
|
49
|
+
maximumCostUsd?: number;
|
|
50
|
+
tags?: Record<string, string>;
|
|
51
|
+
timestamp: number;
|
|
52
|
+
}
|
|
53
|
+
interface CostReceipt extends CostCallBase, CostUsage {
|
|
54
|
+
status: 'settled';
|
|
55
|
+
costUsd: number;
|
|
56
|
+
costUnknown: boolean;
|
|
57
|
+
usageUnknown?: boolean;
|
|
58
|
+
pricing?: {
|
|
59
|
+
inputUsdPerThousand: number;
|
|
60
|
+
outputUsdPerThousand: number;
|
|
61
|
+
};
|
|
62
|
+
actualCostUsd?: number;
|
|
63
|
+
error?: string;
|
|
64
|
+
}
|
|
65
|
+
interface CostReceiptInput extends CostUsage {
|
|
66
|
+
model: string;
|
|
67
|
+
actualCostUsd?: number;
|
|
68
|
+
costUnknown?: boolean;
|
|
69
|
+
usageUnknown?: boolean;
|
|
70
|
+
}
|
|
71
|
+
type MaximumCharge = {
|
|
72
|
+
externallyEnforcedMaximumUsd: number;
|
|
73
|
+
} | ({
|
|
74
|
+
model: string;
|
|
75
|
+
} & CostUsage);
|
|
76
|
+
interface RunPaidCallInput<T> {
|
|
77
|
+
callId?: string;
|
|
78
|
+
channel: CostChannel;
|
|
79
|
+
phase: string;
|
|
80
|
+
actor: string;
|
|
81
|
+
/** Used before a provider receipt exists and on failures without one. */
|
|
82
|
+
model?: string;
|
|
83
|
+
tags?: Record<string, string>;
|
|
84
|
+
signal?: AbortSignal;
|
|
85
|
+
/** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
|
|
86
|
+
maximumCharge?: MaximumCharge;
|
|
87
|
+
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
88
|
+
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
89
|
+
receipt(value: T): CostReceiptInput;
|
|
90
|
+
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
91
|
+
}
|
|
92
|
+
type PaidCallResult<T> = {
|
|
93
|
+
succeeded: true;
|
|
94
|
+
callId: string;
|
|
95
|
+
value: T;
|
|
96
|
+
receipt: CostReceipt;
|
|
97
|
+
} | {
|
|
98
|
+
succeeded: false;
|
|
99
|
+
callId?: string;
|
|
100
|
+
error: Error;
|
|
101
|
+
receipt?: CostReceipt;
|
|
102
|
+
};
|
|
103
|
+
interface ChannelRollup {
|
|
104
|
+
channel: CostChannel;
|
|
105
|
+
calls: number;
|
|
106
|
+
inputTokens: number;
|
|
107
|
+
outputTokens: number;
|
|
108
|
+
reasoningTokens?: number;
|
|
109
|
+
cachedTokens: number;
|
|
110
|
+
cacheWriteTokens?: number;
|
|
111
|
+
costUsd: number;
|
|
112
|
+
unpricedCalls: number;
|
|
113
|
+
unknownUsageCalls: number;
|
|
114
|
+
}
|
|
115
|
+
interface CostLedgerSummary {
|
|
116
|
+
totalCalls: number;
|
|
117
|
+
pendingCalls: number;
|
|
118
|
+
unresolvedCalls: number;
|
|
119
|
+
reservedCostUsd: number;
|
|
120
|
+
inputTokens: number;
|
|
121
|
+
outputTokens: number;
|
|
122
|
+
reasoningTokens?: number;
|
|
123
|
+
cachedTokens: number;
|
|
124
|
+
cacheWriteTokens?: number;
|
|
125
|
+
totalCostUsd: number;
|
|
126
|
+
byChannel: ChannelRollup[];
|
|
127
|
+
unpricedModels: string[];
|
|
128
|
+
fullyPriced: boolean;
|
|
129
|
+
usageComplete: boolean;
|
|
130
|
+
accountingComplete: boolean;
|
|
131
|
+
incompleteReasons: string[];
|
|
132
|
+
}
|
|
133
|
+
interface CostLedgerFilter {
|
|
134
|
+
channel?: CostChannel;
|
|
135
|
+
phase?: string;
|
|
136
|
+
tags?: Record<string, string>;
|
|
137
|
+
}
|
|
138
|
+
interface CostLedgerWaitOptions {
|
|
139
|
+
/** Maximum time to wait for active provider calls. Default 5 seconds. */
|
|
140
|
+
timeoutMs?: number;
|
|
141
|
+
}
|
|
142
|
+
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
143
|
+
interface CostLedgerPersistence {
|
|
144
|
+
read(): {
|
|
145
|
+
revision: string;
|
|
146
|
+
events: string;
|
|
147
|
+
};
|
|
148
|
+
append(expectedRevision: string, event: string): string | undefined;
|
|
149
|
+
}
|
|
150
|
+
interface CostLedgerOptions {
|
|
151
|
+
costCeilingUsd?: number;
|
|
152
|
+
persistence?: CostLedgerPersistence;
|
|
153
|
+
/** Import already-settled receipts without admitting new paid work. */
|
|
154
|
+
receipts?: readonly CostReceipt[];
|
|
155
|
+
}
|
|
156
|
+
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
157
|
+
declare class CostLedger {
|
|
158
|
+
private readonly records;
|
|
159
|
+
private readonly activeCallIds;
|
|
160
|
+
private readonly lateCallIds;
|
|
161
|
+
private readonly idleWaiters;
|
|
162
|
+
private completedTasks;
|
|
163
|
+
private revision;
|
|
164
|
+
private costLimitPersisted;
|
|
165
|
+
readonly costCeilingUsd?: number;
|
|
166
|
+
private readonly persistence?;
|
|
167
|
+
constructor(input?: number | CostLedgerOptions);
|
|
168
|
+
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
169
|
+
/** Wait until every call started by this ledger has produced a durable outcome. */
|
|
170
|
+
waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
|
|
171
|
+
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
172
|
+
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
173
|
+
error?: string;
|
|
174
|
+
}): CostReceipt;
|
|
175
|
+
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
176
|
+
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
177
|
+
markCompleted(count?: number): void;
|
|
178
|
+
costPerCompletedTask(): number | null;
|
|
179
|
+
private execute;
|
|
180
|
+
private captureLateOutcome;
|
|
181
|
+
private releaseActiveCall;
|
|
182
|
+
private commitOutcome;
|
|
183
|
+
private captureFailure;
|
|
184
|
+
private commitReceipt;
|
|
185
|
+
private resolveMaximum;
|
|
186
|
+
private hasIncompleteSettledCall;
|
|
187
|
+
private appendRecord;
|
|
188
|
+
private ensureCostLimitPersisted;
|
|
189
|
+
private appendEvent;
|
|
190
|
+
}
|
|
191
|
+
/** Public callback surface for a shared cost ledger.
|
|
192
|
+
*
|
|
193
|
+
* Declaration bundles may expose this type through multiple package subpaths.
|
|
194
|
+
* Keeping callback contracts structural lets those subpaths compose while the
|
|
195
|
+
* concrete {@link CostLedger} retains its private durable state.
|
|
196
|
+
*/
|
|
197
|
+
type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
|
|
198
|
+
|
|
31
199
|
type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
|
|
32
200
|
|
|
33
201
|
/**
|
|
@@ -475,156 +643,6 @@ interface TraceAnalysisStore {
|
|
|
475
643
|
}): Promise<SearchSpanResult>;
|
|
476
644
|
}
|
|
477
645
|
|
|
478
|
-
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
479
|
-
interface CostUsage {
|
|
480
|
-
inputTokens: number;
|
|
481
|
-
outputTokens: number;
|
|
482
|
-
cachedTokens?: number;
|
|
483
|
-
}
|
|
484
|
-
interface CostCallBase {
|
|
485
|
-
callId: string;
|
|
486
|
-
channel: CostChannel;
|
|
487
|
-
phase: string;
|
|
488
|
-
actor: string;
|
|
489
|
-
model: string;
|
|
490
|
-
maximumCostUsd?: number;
|
|
491
|
-
tags?: Record<string, string>;
|
|
492
|
-
timestamp: number;
|
|
493
|
-
}
|
|
494
|
-
interface CostReceipt extends CostCallBase, CostUsage {
|
|
495
|
-
status: 'settled';
|
|
496
|
-
costUsd: number;
|
|
497
|
-
costUnknown: boolean;
|
|
498
|
-
usageUnknown?: boolean;
|
|
499
|
-
pricing?: {
|
|
500
|
-
inputUsdPerThousand: number;
|
|
501
|
-
outputUsdPerThousand: number;
|
|
502
|
-
};
|
|
503
|
-
actualCostUsd?: number;
|
|
504
|
-
error?: string;
|
|
505
|
-
}
|
|
506
|
-
interface CostReceiptInput extends CostUsage {
|
|
507
|
-
model: string;
|
|
508
|
-
actualCostUsd?: number;
|
|
509
|
-
costUnknown?: boolean;
|
|
510
|
-
usageUnknown?: boolean;
|
|
511
|
-
}
|
|
512
|
-
type MaximumCharge = {
|
|
513
|
-
externallyEnforcedMaximumUsd: number;
|
|
514
|
-
} | ({
|
|
515
|
-
model: string;
|
|
516
|
-
} & CostUsage);
|
|
517
|
-
interface RunPaidCallInput<T> {
|
|
518
|
-
callId?: string;
|
|
519
|
-
channel: CostChannel;
|
|
520
|
-
phase: string;
|
|
521
|
-
actor: string;
|
|
522
|
-
/** Used before a provider receipt exists and on failures without one. */
|
|
523
|
-
model?: string;
|
|
524
|
-
tags?: Record<string, string>;
|
|
525
|
-
signal?: AbortSignal;
|
|
526
|
-
/** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
|
|
527
|
-
maximumCharge?: MaximumCharge;
|
|
528
|
-
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
529
|
-
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
530
|
-
receipt(value: T): CostReceiptInput;
|
|
531
|
-
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
532
|
-
}
|
|
533
|
-
type PaidCallResult<T> = {
|
|
534
|
-
succeeded: true;
|
|
535
|
-
callId: string;
|
|
536
|
-
value: T;
|
|
537
|
-
receipt: CostReceipt;
|
|
538
|
-
} | {
|
|
539
|
-
succeeded: false;
|
|
540
|
-
callId?: string;
|
|
541
|
-
error: Error;
|
|
542
|
-
receipt?: CostReceipt;
|
|
543
|
-
};
|
|
544
|
-
interface ChannelRollup {
|
|
545
|
-
channel: CostChannel;
|
|
546
|
-
calls: number;
|
|
547
|
-
inputTokens: number;
|
|
548
|
-
outputTokens: number;
|
|
549
|
-
cachedTokens: number;
|
|
550
|
-
costUsd: number;
|
|
551
|
-
unpricedCalls: number;
|
|
552
|
-
unknownUsageCalls: number;
|
|
553
|
-
}
|
|
554
|
-
interface CostLedgerSummary {
|
|
555
|
-
totalCalls: number;
|
|
556
|
-
pendingCalls: number;
|
|
557
|
-
unresolvedCalls: number;
|
|
558
|
-
reservedCostUsd: number;
|
|
559
|
-
inputTokens: number;
|
|
560
|
-
outputTokens: number;
|
|
561
|
-
cachedTokens: number;
|
|
562
|
-
totalCostUsd: number;
|
|
563
|
-
byChannel: ChannelRollup[];
|
|
564
|
-
unpricedModels: string[];
|
|
565
|
-
fullyPriced: boolean;
|
|
566
|
-
usageComplete: boolean;
|
|
567
|
-
accountingComplete: boolean;
|
|
568
|
-
incompleteReasons: string[];
|
|
569
|
-
}
|
|
570
|
-
interface CostLedgerFilter {
|
|
571
|
-
channel?: CostChannel;
|
|
572
|
-
phase?: string;
|
|
573
|
-
tags?: Record<string, string>;
|
|
574
|
-
}
|
|
575
|
-
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
576
|
-
interface CostLedgerPersistence {
|
|
577
|
-
read(): {
|
|
578
|
-
revision: string;
|
|
579
|
-
events: string;
|
|
580
|
-
};
|
|
581
|
-
append(expectedRevision: string, event: string): string | undefined;
|
|
582
|
-
}
|
|
583
|
-
interface CostLedgerOptions {
|
|
584
|
-
costCeilingUsd?: number;
|
|
585
|
-
persistence?: CostLedgerPersistence;
|
|
586
|
-
/** Import already-settled receipts without admitting new paid work. */
|
|
587
|
-
receipts?: readonly CostReceipt[];
|
|
588
|
-
}
|
|
589
|
-
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
590
|
-
declare class CostLedger {
|
|
591
|
-
private readonly records;
|
|
592
|
-
private readonly activeCallIds;
|
|
593
|
-
private readonly lateCallIds;
|
|
594
|
-
private completedTasks;
|
|
595
|
-
private revision;
|
|
596
|
-
private costLimitPersisted;
|
|
597
|
-
readonly costCeilingUsd?: number;
|
|
598
|
-
private readonly persistence?;
|
|
599
|
-
constructor(input?: number | CostLedgerOptions);
|
|
600
|
-
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
601
|
-
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
602
|
-
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
603
|
-
error?: string;
|
|
604
|
-
}): CostReceipt;
|
|
605
|
-
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
606
|
-
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
607
|
-
markCompleted(count?: number): void;
|
|
608
|
-
costPerCompletedTask(): number | null;
|
|
609
|
-
private execute;
|
|
610
|
-
private captureLateOutcome;
|
|
611
|
-
private commitOutcome;
|
|
612
|
-
private captureFailure;
|
|
613
|
-
private commitReceipt;
|
|
614
|
-
private resolveMaximum;
|
|
615
|
-
private hasIncompleteSettledCall;
|
|
616
|
-
private appendRecord;
|
|
617
|
-
private ensureCostLimitPersisted;
|
|
618
|
-
private appendEvent;
|
|
619
|
-
}
|
|
620
|
-
/** Public callback surface for a shared cost ledger.
|
|
621
|
-
*
|
|
622
|
-
* Declaration bundles may expose this type through multiple package subpaths.
|
|
623
|
-
* Keeping callback contracts structural lets those subpaths compose while the
|
|
624
|
-
* concrete {@link CostLedger} retains its private durable state.
|
|
625
|
-
*/
|
|
626
|
-
type CostLedgerHandle = Pick<CostLedger, keyof CostLedger>;
|
|
627
|
-
|
|
628
646
|
interface Scenario$1 {
|
|
629
647
|
id: string;
|
|
630
648
|
persona: string;
|
|
@@ -1131,6 +1149,10 @@ interface AnalystContext {
|
|
|
1131
1149
|
deadlineMs?: number;
|
|
1132
1150
|
/** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
|
|
1133
1151
|
budgetUsd?: number;
|
|
1152
|
+
/** Shared paid-call account when the analyst runs inside a larger campaign. */
|
|
1153
|
+
costLedger?: CostLedgerHandle;
|
|
1154
|
+
/** Attribution phase used when writing to the shared paid-call account. */
|
|
1155
|
+
costPhase?: string;
|
|
1134
1156
|
/**
|
|
1135
1157
|
* Shared chat client. Analysts that call an LLM go through this so
|
|
1136
1158
|
* the operator picks transport (sandbox-sdk | router | cli-bridge |
|
|
@@ -1148,6 +1170,18 @@ interface AnalystContext {
|
|
|
1148
1170
|
* filter. Empty / absent means no cross-run context.
|
|
1149
1171
|
*/
|
|
1150
1172
|
priorFindings?: ReadonlyArray<AnalystFinding>;
|
|
1173
|
+
/**
|
|
1174
|
+
* Findings emitted by analysts that completed earlier in this registry run.
|
|
1175
|
+
* This is separate from `priorFindings`: upstream findings are dependency
|
|
1176
|
+
* context for the current pass, while prior findings are cross-run memory.
|
|
1177
|
+
* The registry populates this only when `RegistryRunOpts.chainFindings` is on.
|
|
1178
|
+
*/
|
|
1179
|
+
upstreamFindings?: ReadonlyArray<AnalystFinding>;
|
|
1180
|
+
/**
|
|
1181
|
+
* Report metered work independently of findings. This keeps an empty finding
|
|
1182
|
+
* set from erasing token/cost telemetry. Multiple receipts are accumulated.
|
|
1183
|
+
*/
|
|
1184
|
+
recordUsage?: (receipt: AnalystUsageReceipt) => void;
|
|
1151
1185
|
/** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */
|
|
1152
1186
|
tags?: Record<string, string>;
|
|
1153
1187
|
/** Logger callback — analysts SHOULD prefer this over console.* for testability. */
|
|
@@ -1173,6 +1207,17 @@ interface Analyst<TInput = unknown> {
|
|
|
1173
1207
|
readonly version: string;
|
|
1174
1208
|
analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>;
|
|
1175
1209
|
}
|
|
1210
|
+
/** Metered work performed by one analyst call. */
|
|
1211
|
+
interface AnalystUsageReceipt {
|
|
1212
|
+
/** Number of model-usage records observed at the provider boundary. */
|
|
1213
|
+
calls: number | null;
|
|
1214
|
+
/** Null when the provider did not return token accounting. */
|
|
1215
|
+
tokens: RunTokenUsage | null;
|
|
1216
|
+
/** Observed, estimated, or explicitly uncaptured dollar cost. */
|
|
1217
|
+
cost: RunCostProvenance;
|
|
1218
|
+
/** Known lower bound when one or more calls have uncaptured cost. */
|
|
1219
|
+
knownCostUsd?: number;
|
|
1220
|
+
}
|
|
1176
1221
|
interface AnalystRunSummary {
|
|
1177
1222
|
analyst_id: string;
|
|
1178
1223
|
status: 'ok' | 'skipped' | 'failed';
|
|
@@ -1181,6 +1226,12 @@ interface AnalystRunSummary {
|
|
|
1181
1226
|
findings_count: number;
|
|
1182
1227
|
latency_ms: number;
|
|
1183
1228
|
cost_usd: number;
|
|
1229
|
+
/**
|
|
1230
|
+
* Additive receipt for model usage. Registry-produced summaries populate it
|
|
1231
|
+
* even when the analyst emits no findings. `cost_usd` remains the legacy
|
|
1232
|
+
* numeric field; inspect `usage.cost` before treating zero as observed.
|
|
1233
|
+
*/
|
|
1234
|
+
usage?: AnalystUsageReceipt;
|
|
1184
1235
|
/** When `status='failed'`: the error class + message, never the full stack. */
|
|
1185
1236
|
error?: {
|
|
1186
1237
|
class: string;
|
|
@@ -1196,6 +1247,11 @@ interface AnalystRunResult {
|
|
|
1196
1247
|
per_analyst: AnalystRunSummary[];
|
|
1197
1248
|
/** Total LLM cost in USD across all analysts in this registry.run(). */
|
|
1198
1249
|
total_cost_usd: number;
|
|
1250
|
+
/**
|
|
1251
|
+
* Provenance for `total_cost_usd`. When uncaptured, the numeric field is only
|
|
1252
|
+
* the known subtotal and must not be treated as the run's total spend.
|
|
1253
|
+
*/
|
|
1254
|
+
total_cost_provenance?: RunCostProvenance;
|
|
1199
1255
|
}
|
|
1200
1256
|
/**
|
|
1201
1257
|
* Events emitted by `AnalystRegistry.runStream(...)` in real time as
|
|
@@ -3961,7 +4017,10 @@ declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults
|
|
|
3961
4017
|
* validation surface independent of which Ax version is installed.
|
|
3962
4018
|
*/
|
|
3963
4019
|
|
|
4020
|
+
/** Original public schema retained for stored rows and callback contracts. */
|
|
3964
4021
|
declare const RawAnalystFindingSchema: z.ZodObject<{
|
|
4022
|
+
evidence_uri: z.ZodString;
|
|
4023
|
+
evidence_excerpt: z.ZodOptional<z.ZodString>;
|
|
3965
4024
|
severity: z.ZodEnum<{
|
|
3966
4025
|
low: "low";
|
|
3967
4026
|
high: "high";
|
|
@@ -3971,13 +4030,36 @@ declare const RawAnalystFindingSchema: z.ZodObject<{
|
|
|
3971
4030
|
}>;
|
|
3972
4031
|
claim: z.ZodString;
|
|
3973
4032
|
subject: z.ZodOptional<z.ZodString>;
|
|
3974
|
-
evidence_uri: z.ZodString;
|
|
3975
|
-
evidence_excerpt: z.ZodOptional<z.ZodString>;
|
|
3976
4033
|
confidence: z.ZodNumber;
|
|
3977
4034
|
rationale: z.ZodOptional<z.ZodString>;
|
|
3978
4035
|
recommended_action: z.ZodOptional<z.ZodString>;
|
|
3979
4036
|
}, z.core.$strict>;
|
|
3980
4037
|
type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
|
|
4038
|
+
/**
|
|
4039
|
+
* Canonical plural-evidence contract. The preprocessor accepts the original
|
|
4040
|
+
* `evidence_uri` / `evidence_excerpt` pair and normalizes it into one evidence
|
|
4041
|
+
* item so persisted rows and older model fixtures remain readable. New output
|
|
4042
|
+
* always receives the plural shape.
|
|
4043
|
+
*/
|
|
4044
|
+
declare const CanonicalRawAnalystFindingSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodObject<{
|
|
4045
|
+
evidence: z.ZodArray<z.ZodObject<{
|
|
4046
|
+
uri: z.ZodString;
|
|
4047
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
4048
|
+
}, z.core.$strict>>;
|
|
4049
|
+
severity: z.ZodEnum<{
|
|
4050
|
+
low: "low";
|
|
4051
|
+
high: "high";
|
|
4052
|
+
critical: "critical";
|
|
4053
|
+
medium: "medium";
|
|
4054
|
+
info: "info";
|
|
4055
|
+
}>;
|
|
4056
|
+
claim: z.ZodString;
|
|
4057
|
+
subject: z.ZodOptional<z.ZodString>;
|
|
4058
|
+
confidence: z.ZodNumber;
|
|
4059
|
+
rationale: z.ZodOptional<z.ZodString>;
|
|
4060
|
+
recommended_action: z.ZodOptional<z.ZodString>;
|
|
4061
|
+
}, z.core.$strict>>;
|
|
4062
|
+
type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchema>;
|
|
3981
4063
|
|
|
3982
4064
|
/**
|
|
3983
4065
|
* Analyst-kind factory — the typed way to define trace analysts.
|
|
@@ -3985,9 +4067,9 @@ type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
|
|
|
3985
4067
|
* A "kind" is a specialized analyst whose actor prompt, tool subset,
|
|
3986
4068
|
* and Ax recursion config target one failure-mode lens (failure-mode
|
|
3987
4069
|
* classification, knowledge gap discovery, knowledge poisoning, recursive
|
|
3988
|
-
* self-improvement, ...). Kinds emit findings in the typed
|
|
3989
|
-
* shape via a JSON-array Ax output; the factory
|
|
3990
|
-
* Zod and lifts it into `AnalystFinding[]
|
|
4070
|
+
* self-improvement, ...). Kinds emit findings in the typed
|
|
4071
|
+
* `CanonicalRawAnalystFinding` shape via a JSON-array Ax output; the factory
|
|
4072
|
+
* validates each row with Zod and lifts it into `AnalystFinding[]`.
|
|
3991
4073
|
*
|
|
3992
4074
|
* Composition rules:
|
|
3993
4075
|
* - Each kind owns its actor description. No generic "answer this
|
|
@@ -4033,10 +4115,14 @@ interface TraceAnalystKindSpec {
|
|
|
4033
4115
|
maxTurns?: number;
|
|
4034
4116
|
/** Runtime char cap. Default 6000. */
|
|
4035
4117
|
maxRuntimeChars?: number;
|
|
4118
|
+
/** Maximum output tokens for every actor, responder, and recursive model call. Default 4096. */
|
|
4119
|
+
maxOutputTokens?: number;
|
|
4036
4120
|
/** Cost classification surfaced in `registry.list()` and budget enforcement. */
|
|
4037
4121
|
cost: AnalystCost;
|
|
4038
4122
|
/** Per-finding-row hook — kinds may reject / rewrite before lifting. */
|
|
4039
4123
|
postProcess?: (row: RawAnalystFinding, ctx: AnalystContext) => RawAnalystFinding | null;
|
|
4124
|
+
/** Minimum citations per finding. Default 1; rows below it are rejected. */
|
|
4125
|
+
minimumEvidenceCitations?: number;
|
|
4040
4126
|
/** Optional optimizer hook — populated when a kind wants to fit its prompt against labeled examples. */
|
|
4041
4127
|
goldens?: TraceAnalystGolden[];
|
|
4042
4128
|
}
|
|
@@ -4048,7 +4134,7 @@ interface TraceAnalystKindSpec {
|
|
|
4048
4134
|
*/
|
|
4049
4135
|
interface TraceAnalystGolden {
|
|
4050
4136
|
question: string;
|
|
4051
|
-
expected: ReadonlyArray<Omit<
|
|
4137
|
+
expected: ReadonlyArray<Omit<CanonicalRawAnalystFinding, 'confidence'>>;
|
|
4052
4138
|
}
|
|
4053
4139
|
|
|
4054
4140
|
/**
|
|
@@ -4106,7 +4192,8 @@ interface BudgetPolicy {
|
|
|
4106
4192
|
/**
|
|
4107
4193
|
* Custom allocator — receives the analyst, remaining/total budget, and
|
|
4108
4194
|
* the count of analysts that will run. Returns the per-analyst budget
|
|
4109
|
-
* (or undefined
|
|
4195
|
+
* (or undefined only when the run has no overall cap). Overrides weights
|
|
4196
|
+
* when set.
|
|
4110
4197
|
*/
|
|
4111
4198
|
allocate?: (args: {
|
|
4112
4199
|
analyst: Analyst;
|
|
@@ -4136,6 +4223,10 @@ interface RegistryRunOpts {
|
|
|
4136
4223
|
timeoutMs?: number;
|
|
4137
4224
|
/** Abort signal — forwarded into every analyst's context. */
|
|
4138
4225
|
signal?: AbortSignal;
|
|
4226
|
+
/** Shared paid-call account forwarded to every analyst. */
|
|
4227
|
+
costLedger?: CostLedgerHandle;
|
|
4228
|
+
/** Attribution phase for calls written to `costLedger`. */
|
|
4229
|
+
costPhase?: string;
|
|
4139
4230
|
/** Tags echoed into AnalystContext.tags — useful for tracking environment/version in findings. */
|
|
4140
4231
|
tags?: Record<string, string>;
|
|
4141
4232
|
/**
|
|
@@ -4143,10 +4234,16 @@ interface RegistryRunOpts {
|
|
|
4143
4234
|
* analyst via `ctx.priorFindings`. The registry forwards the slice
|
|
4144
4235
|
* whose `analyst_id` matches each registered analyst so a kind sees
|
|
4145
4236
|
* only its own history. Pass `{ '*': findings }` to broadcast to
|
|
4146
|
-
* every analyst (useful
|
|
4147
|
-
*
|
|
4237
|
+
* every analyst (useful when several kinds share the same historical
|
|
4238
|
+
* context). For findings from this run, use `chainFindings` instead.
|
|
4148
4239
|
*/
|
|
4149
4240
|
priorFindings?: ReadonlyArray<AnalystFinding> | Record<string, ReadonlyArray<AnalystFinding>>;
|
|
4241
|
+
/**
|
|
4242
|
+
* Pass findings produced earlier in this registry run to each later analyst
|
|
4243
|
+
* via `ctx.upstreamFindings`. Registration order is dependency order.
|
|
4244
|
+
* Disabled by default because independent analyst suites must opt in.
|
|
4245
|
+
*/
|
|
4246
|
+
chainFindings?: boolean;
|
|
4150
4247
|
}
|
|
4151
4248
|
declare class AnalystRegistry {
|
|
4152
4249
|
private readonly analysts;
|
|
@@ -4190,7 +4287,7 @@ declare class AnalystRegistry {
|
|
|
4190
4287
|
interface DefaultAnalystRegistryOptions {
|
|
4191
4288
|
/** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */
|
|
4192
4289
|
ai?: AxAIService;
|
|
4193
|
-
/**
|
|
4290
|
+
/** Required unless `ai` was created by `createAnalystAi`. */
|
|
4194
4291
|
model?: string;
|
|
4195
4292
|
/** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */
|
|
4196
4293
|
kinds?: readonly TraceAnalystKindSpec[];
|
package/dist/contract/index.js
CHANGED
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
import {
|
|
14
14
|
analyzeRuns,
|
|
15
15
|
summarizeExecution
|
|
16
|
-
} from "../chunk-
|
|
16
|
+
} from "../chunk-XJ7JVCHB.js";
|
|
17
17
|
import {
|
|
18
18
|
REFERENCE_EQUIVALENCE_INPUT_LIMITS,
|
|
19
19
|
REFERENCE_EQUIVALENCE_JUDGE_VERSION,
|
|
@@ -34,20 +34,20 @@ import {
|
|
|
34
34
|
runReferenceEquivalenceJudge,
|
|
35
35
|
surfaceContentHash,
|
|
36
36
|
surfaceHash
|
|
37
|
-
} from "../chunk-
|
|
37
|
+
} from "../chunk-RLCJQ6VZ.js";
|
|
38
38
|
import {
|
|
39
39
|
createRunCostLedger,
|
|
40
40
|
fsCampaignStorage,
|
|
41
41
|
inMemoryCampaignStorage,
|
|
42
42
|
resolveRunDir,
|
|
43
43
|
runCampaign
|
|
44
|
-
} from "../chunk-
|
|
44
|
+
} from "../chunk-PAHNGS65.js";
|
|
45
45
|
import {
|
|
46
46
|
buildDefaultAnalystRegistry,
|
|
47
47
|
createChatClient
|
|
48
|
-
} from "../chunk-
|
|
48
|
+
} from "../chunk-GERDAIAL.js";
|
|
49
49
|
import "../chunk-HHWE3POT.js";
|
|
50
|
-
import "../chunk-
|
|
50
|
+
import "../chunk-7A4LIMMY.js";
|
|
51
51
|
import {
|
|
52
52
|
FileSystemOutcomeStore,
|
|
53
53
|
InMemoryOutcomeStore
|
|
@@ -56,7 +56,7 @@ import "../chunk-ARU2PZFM.js";
|
|
|
56
56
|
import "../chunk-NJC7U437.js";
|
|
57
57
|
import "../chunk-DPZAEKA6.js";
|
|
58
58
|
import "../chunk-PJQFMIOX.js";
|
|
59
|
-
import "../chunk-
|
|
59
|
+
import "../chunk-JHOJHHU7.js";
|
|
60
60
|
import "../chunk-VI2UW6B6.js";
|
|
61
61
|
import {
|
|
62
62
|
recordAggregateMeasurements,
|
package/dist/fuzz.d.ts
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
2
2
|
interface CostUsage {
|
|
3
3
|
inputTokens: number;
|
|
4
|
+
/** Includes reasoning tokens when the provider bills them as output. */
|
|
4
5
|
outputTokens: number;
|
|
6
|
+
/** Reasoning-token subset of outputTokens, when reported. */
|
|
7
|
+
reasoningTokens?: number;
|
|
8
|
+
/** Prompt tokens served from a provider cache. */
|
|
5
9
|
cachedTokens?: number;
|
|
10
|
+
/** Prompt tokens written into a provider cache. */
|
|
11
|
+
cacheWriteTokens?: number;
|
|
6
12
|
}
|
|
7
13
|
interface CostCallBase {
|
|
8
14
|
callId: string;
|
|
@@ -69,7 +75,9 @@ interface ChannelRollup {
|
|
|
69
75
|
calls: number;
|
|
70
76
|
inputTokens: number;
|
|
71
77
|
outputTokens: number;
|
|
78
|
+
reasoningTokens?: number;
|
|
72
79
|
cachedTokens: number;
|
|
80
|
+
cacheWriteTokens?: number;
|
|
73
81
|
costUsd: number;
|
|
74
82
|
unpricedCalls: number;
|
|
75
83
|
unknownUsageCalls: number;
|
|
@@ -81,7 +89,9 @@ interface CostLedgerSummary {
|
|
|
81
89
|
reservedCostUsd: number;
|
|
82
90
|
inputTokens: number;
|
|
83
91
|
outputTokens: number;
|
|
92
|
+
reasoningTokens?: number;
|
|
84
93
|
cachedTokens: number;
|
|
94
|
+
cacheWriteTokens?: number;
|
|
85
95
|
totalCostUsd: number;
|
|
86
96
|
byChannel: ChannelRollup[];
|
|
87
97
|
unpricedModels: string[];
|
|
@@ -95,6 +105,10 @@ interface CostLedgerFilter {
|
|
|
95
105
|
phase?: string;
|
|
96
106
|
tags?: Record<string, string>;
|
|
97
107
|
}
|
|
108
|
+
interface CostLedgerWaitOptions {
|
|
109
|
+
/** Maximum time to wait for active provider calls. Default 5 seconds. */
|
|
110
|
+
timeoutMs?: number;
|
|
111
|
+
}
|
|
98
112
|
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
99
113
|
interface CostLedgerPersistence {
|
|
100
114
|
read(): {
|
|
@@ -114,6 +128,7 @@ declare class CostLedger {
|
|
|
114
128
|
private readonly records;
|
|
115
129
|
private readonly activeCallIds;
|
|
116
130
|
private readonly lateCallIds;
|
|
131
|
+
private readonly idleWaiters;
|
|
117
132
|
private completedTasks;
|
|
118
133
|
private revision;
|
|
119
134
|
private costLimitPersisted;
|
|
@@ -121,6 +136,8 @@ declare class CostLedger {
|
|
|
121
136
|
private readonly persistence?;
|
|
122
137
|
constructor(input?: number | CostLedgerOptions);
|
|
123
138
|
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
139
|
+
/** Wait until every call started by this ledger has produced a durable outcome. */
|
|
140
|
+
waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
|
|
124
141
|
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
125
142
|
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
126
143
|
error?: string;
|
|
@@ -131,6 +148,7 @@ declare class CostLedger {
|
|
|
131
148
|
costPerCompletedTask(): number | null;
|
|
132
149
|
private execute;
|
|
133
150
|
private captureLateOutcome;
|
|
151
|
+
private releaseActiveCall;
|
|
134
152
|
private commitOutcome;
|
|
135
153
|
private captureFailure;
|
|
136
154
|
private commitReceipt;
|
|
@@ -146,7 +164,7 @@ declare class CostLedger {
|
|
|
146
164
|
* Keeping callback contracts structural lets those subpaths compose while the
|
|
147
165
|
* concrete {@link CostLedger} retains its private durable state.
|
|
148
166
|
*/
|
|
149
|
-
type CostLedgerHandle = Pick<CostLedger, keyof CostLedger
|
|
167
|
+
type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
|
|
150
168
|
|
|
151
169
|
/**
|
|
152
170
|
* Adversarial mutation contract.
|