@tangle-network/agent-eval 0.118.3 → 0.119.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +26 -0
- package/dist/analyst/index.d.ts +172 -59
- package/dist/analyst/index.js +50 -12
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +50 -40
- package/dist/benchmarks/index.js +8 -8
- package/dist/campaign/index.d.ts +258 -184
- package/dist/campaign/index.js +7 -7
- package/dist/{chunk-JTMFT5QZ.js → chunk-4I2E3LLO.js} +2 -2
- package/dist/{chunk-KK2O4VWA.js → chunk-D7AEXSM5.js} +2 -2
- package/dist/{chunk-MLHRFWQK.js → chunk-JHOJHHU7.js} +85 -16
- package/dist/chunk-JHOJHHU7.js.map +1 -0
- package/dist/{chunk-F5A4GDQW.js → chunk-JM2SKQMS.js} +3 -15
- package/dist/chunk-JM2SKQMS.js.map +1 -0
- package/dist/{chunk-EGNRE7VA.js → chunk-PAHNGS65.js} +2 -2
- package/dist/{chunk-QWRLW4CT.js → chunk-PSXJ32SR.js} +160 -6
- package/dist/chunk-PSXJ32SR.js.map +1 -0
- package/dist/{chunk-6QIM2EAP.js → chunk-PXD6ZFNY.js} +99 -1
- package/dist/chunk-PXD6ZFNY.js.map +1 -0
- package/dist/{chunk-267UCQWI.js → chunk-QWMPPZ3X.js} +3 -3
- package/dist/{chunk-267UCQWI.js.map → chunk-QWMPPZ3X.js.map} +1 -1
- package/dist/{chunk-VNLWDTDA.js → chunk-S5TT5R3L.js} +1010 -285
- package/dist/chunk-S5TT5R3L.js.map +1 -0
- package/dist/{chunk-ZYHJNKI3.js → chunk-ULOKLHIQ.js} +42 -5
- package/dist/chunk-ULOKLHIQ.js.map +1 -0
- package/dist/{chunk-FIUFRXP3.js → chunk-WW2A73HW.js} +49 -91
- package/dist/chunk-WW2A73HW.js.map +1 -0
- package/dist/{chunk-7V3QDWNL.js → chunk-XJ7JVCHB.js} +2 -2
- package/dist/{chunk-EKHHBKS6.js → chunk-XMBOU5W7.js} +48 -190
- package/dist/chunk-XMBOU5W7.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/contract/index.d.ts +281 -175
- package/dist/contract/index.js +284 -8
- package/dist/contract/index.js.map +1 -1
- package/dist/fuzz.d.ts +19 -1
- package/dist/fuzz.js +1 -1
- package/dist/index.d.ts +223 -114
- package/dist/index.js +38 -37
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/primeintellect/index.d.ts +311 -0
- package/dist/primeintellect/index.js +348 -0
- package/dist/primeintellect/index.js.map +1 -0
- package/dist/rl.d.ts +4 -0
- package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js} +3 -3
- package/dist/traces.d.ts +39 -25
- package/dist/traces.js +3 -5
- package/dist/wire/index.d.ts +19 -1
- package/dist/wire/index.js +2 -2
- package/docs/trace-analysis.md +6 -2
- package/package.json +7 -2
- package/dist/chunk-6QIM2EAP.js.map +0 -1
- package/dist/chunk-EKHHBKS6.js.map +0 -1
- package/dist/chunk-F5A4GDQW.js.map +0 -1
- package/dist/chunk-FIUFRXP3.js.map +0 -1
- package/dist/chunk-MLHRFWQK.js.map +0 -1
- package/dist/chunk-QWRLW4CT.js.map +0 -1
- package/dist/chunk-VNLWDTDA.js.map +0 -1
- package/dist/chunk-ZYHJNKI3.js.map +0 -1
- /package/dist/{chunk-JTMFT5QZ.js.map → chunk-4I2E3LLO.js.map} +0 -0
- /package/dist/{chunk-KK2O4VWA.js.map → chunk-D7AEXSM5.js.map} +0 -0
- /package/dist/{chunk-EGNRE7VA.js.map → chunk-PAHNGS65.js.map} +0 -0
- /package/dist/{chunk-7V3QDWNL.js.map → chunk-XJ7JVCHB.js.map} +0 -0
- /package/dist/{run-campaign-DWC67KJP.js.map → run-campaign-OBXSN5WK.js.map} +0 -0
package/dist/cli.js
CHANGED
|
@@ -5,9 +5,9 @@ import {
|
|
|
5
5
|
runRpcBatch,
|
|
6
6
|
runRpcOnce,
|
|
7
7
|
startServer
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-4I2E3LLO.js";
|
|
9
9
|
import "./chunk-NJC7U437.js";
|
|
10
|
-
import "./chunk-
|
|
10
|
+
import "./chunk-JHOJHHU7.js";
|
|
11
11
|
import "./chunk-VI2UW6B6.js";
|
|
12
12
|
import "./chunk-PC4UYEBM.js";
|
|
13
13
|
import "./chunk-ONWEPEDO.js";
|
package/dist/contract/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AgentImprovementMeasuredComparison, Sha256Digest } from '@tangle-network/agent-interface';
|
|
1
2
|
import { AxFunction, AxAIService } from '@ax-llm/ax';
|
|
2
3
|
import { z } from 'zod';
|
|
3
4
|
|
|
@@ -28,6 +29,174 @@ interface AgentProfileCell {
|
|
|
28
29
|
dimensions?: Record<string, AgentProfileDimensionValue>;
|
|
29
30
|
}
|
|
30
31
|
|
|
32
|
+
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
33
|
+
interface CostUsage {
|
|
34
|
+
inputTokens: number;
|
|
35
|
+
/** Includes reasoning tokens when the provider bills them as output. */
|
|
36
|
+
outputTokens: number;
|
|
37
|
+
/** Reasoning-token subset of outputTokens, when reported. */
|
|
38
|
+
reasoningTokens?: number;
|
|
39
|
+
/** Prompt tokens served from a provider cache. */
|
|
40
|
+
cachedTokens?: number;
|
|
41
|
+
/** Prompt tokens written into a provider cache. */
|
|
42
|
+
cacheWriteTokens?: number;
|
|
43
|
+
}
|
|
44
|
+
interface CostCallBase {
|
|
45
|
+
callId: string;
|
|
46
|
+
channel: CostChannel;
|
|
47
|
+
phase: string;
|
|
48
|
+
actor: string;
|
|
49
|
+
model: string;
|
|
50
|
+
maximumCostUsd?: number;
|
|
51
|
+
tags?: Record<string, string>;
|
|
52
|
+
timestamp: number;
|
|
53
|
+
}
|
|
54
|
+
interface CostReceipt extends CostCallBase, CostUsage {
|
|
55
|
+
status: 'settled';
|
|
56
|
+
costUsd: number;
|
|
57
|
+
costUnknown: boolean;
|
|
58
|
+
usageUnknown?: boolean;
|
|
59
|
+
pricing?: {
|
|
60
|
+
inputUsdPerThousand: number;
|
|
61
|
+
outputUsdPerThousand: number;
|
|
62
|
+
};
|
|
63
|
+
actualCostUsd?: number;
|
|
64
|
+
error?: string;
|
|
65
|
+
}
|
|
66
|
+
interface CostReceiptInput extends CostUsage {
|
|
67
|
+
model: string;
|
|
68
|
+
actualCostUsd?: number;
|
|
69
|
+
costUnknown?: boolean;
|
|
70
|
+
usageUnknown?: boolean;
|
|
71
|
+
}
|
|
72
|
+
type MaximumCharge = {
|
|
73
|
+
externallyEnforcedMaximumUsd: number;
|
|
74
|
+
} | ({
|
|
75
|
+
model: string;
|
|
76
|
+
} & CostUsage);
|
|
77
|
+
interface RunPaidCallInput<T> {
|
|
78
|
+
callId?: string;
|
|
79
|
+
channel: CostChannel;
|
|
80
|
+
phase: string;
|
|
81
|
+
actor: string;
|
|
82
|
+
/** Used before a provider receipt exists and on failures without one. */
|
|
83
|
+
model?: string;
|
|
84
|
+
tags?: Record<string, string>;
|
|
85
|
+
signal?: AbortSignal;
|
|
86
|
+
/** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
|
|
87
|
+
maximumCharge?: MaximumCharge;
|
|
88
|
+
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
89
|
+
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
90
|
+
receipt(value: T): CostReceiptInput;
|
|
91
|
+
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
92
|
+
}
|
|
93
|
+
type PaidCallResult<T> = {
|
|
94
|
+
succeeded: true;
|
|
95
|
+
callId: string;
|
|
96
|
+
value: T;
|
|
97
|
+
receipt: CostReceipt;
|
|
98
|
+
} | {
|
|
99
|
+
succeeded: false;
|
|
100
|
+
callId?: string;
|
|
101
|
+
error: Error;
|
|
102
|
+
receipt?: CostReceipt;
|
|
103
|
+
};
|
|
104
|
+
interface ChannelRollup {
|
|
105
|
+
channel: CostChannel;
|
|
106
|
+
calls: number;
|
|
107
|
+
inputTokens: number;
|
|
108
|
+
outputTokens: number;
|
|
109
|
+
reasoningTokens?: number;
|
|
110
|
+
cachedTokens: number;
|
|
111
|
+
cacheWriteTokens?: number;
|
|
112
|
+
costUsd: number;
|
|
113
|
+
unpricedCalls: number;
|
|
114
|
+
unknownUsageCalls: number;
|
|
115
|
+
}
|
|
116
|
+
interface CostLedgerSummary {
|
|
117
|
+
totalCalls: number;
|
|
118
|
+
pendingCalls: number;
|
|
119
|
+
unresolvedCalls: number;
|
|
120
|
+
reservedCostUsd: number;
|
|
121
|
+
inputTokens: number;
|
|
122
|
+
outputTokens: number;
|
|
123
|
+
reasoningTokens?: number;
|
|
124
|
+
cachedTokens: number;
|
|
125
|
+
cacheWriteTokens?: number;
|
|
126
|
+
totalCostUsd: number;
|
|
127
|
+
byChannel: ChannelRollup[];
|
|
128
|
+
unpricedModels: string[];
|
|
129
|
+
fullyPriced: boolean;
|
|
130
|
+
usageComplete: boolean;
|
|
131
|
+
accountingComplete: boolean;
|
|
132
|
+
incompleteReasons: string[];
|
|
133
|
+
}
|
|
134
|
+
interface CostLedgerFilter {
|
|
135
|
+
channel?: CostChannel;
|
|
136
|
+
phase?: string;
|
|
137
|
+
tags?: Record<string, string>;
|
|
138
|
+
}
|
|
139
|
+
interface CostLedgerWaitOptions {
|
|
140
|
+
/** Maximum time to wait for active provider calls. Default 5 seconds. */
|
|
141
|
+
timeoutMs?: number;
|
|
142
|
+
}
|
|
143
|
+
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
144
|
+
interface CostLedgerPersistence {
|
|
145
|
+
read(): {
|
|
146
|
+
revision: string;
|
|
147
|
+
events: string;
|
|
148
|
+
};
|
|
149
|
+
append(expectedRevision: string, event: string): string | undefined;
|
|
150
|
+
}
|
|
151
|
+
interface CostLedgerOptions {
|
|
152
|
+
costCeilingUsd?: number;
|
|
153
|
+
persistence?: CostLedgerPersistence;
|
|
154
|
+
/** Import already-settled receipts without admitting new paid work. */
|
|
155
|
+
receipts?: readonly CostReceipt[];
|
|
156
|
+
}
|
|
157
|
+
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
158
|
+
declare class CostLedger {
|
|
159
|
+
private readonly records;
|
|
160
|
+
private readonly activeCallIds;
|
|
161
|
+
private readonly lateCallIds;
|
|
162
|
+
private readonly idleWaiters;
|
|
163
|
+
private completedTasks;
|
|
164
|
+
private revision;
|
|
165
|
+
private costLimitPersisted;
|
|
166
|
+
readonly costCeilingUsd?: number;
|
|
167
|
+
private readonly persistence?;
|
|
168
|
+
constructor(input?: number | CostLedgerOptions);
|
|
169
|
+
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
170
|
+
/** Wait until every call started by this ledger has produced a durable outcome. */
|
|
171
|
+
waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
|
|
172
|
+
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
173
|
+
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
174
|
+
error?: string;
|
|
175
|
+
}): CostReceipt;
|
|
176
|
+
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
177
|
+
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
178
|
+
markCompleted(count?: number): void;
|
|
179
|
+
costPerCompletedTask(): number | null;
|
|
180
|
+
private execute;
|
|
181
|
+
private captureLateOutcome;
|
|
182
|
+
private releaseActiveCall;
|
|
183
|
+
private commitOutcome;
|
|
184
|
+
private captureFailure;
|
|
185
|
+
private commitReceipt;
|
|
186
|
+
private resolveMaximum;
|
|
187
|
+
private hasIncompleteSettledCall;
|
|
188
|
+
private appendRecord;
|
|
189
|
+
private ensureCostLimitPersisted;
|
|
190
|
+
private appendEvent;
|
|
191
|
+
}
|
|
192
|
+
/** Public callback surface for a shared cost ledger.
|
|
193
|
+
*
|
|
194
|
+
* Declaration bundles may expose this type through multiple package subpaths.
|
|
195
|
+
* Keeping callback contracts structural lets those subpaths compose while the
|
|
196
|
+
* concrete {@link CostLedger} retains its private durable state.
|
|
197
|
+
*/
|
|
198
|
+
type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
|
|
199
|
+
|
|
31
200
|
type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
|
|
32
201
|
|
|
33
202
|
/**
|
|
@@ -475,156 +644,6 @@ interface TraceAnalysisStore {
|
|
|
475
644
|
}): Promise<SearchSpanResult>;
|
|
476
645
|
}
|
|
477
646
|
|
|
478
|
-
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
479
|
-
interface CostUsage {
|
|
480
|
-
inputTokens: number;
|
|
481
|
-
outputTokens: number;
|
|
482
|
-
cachedTokens?: number;
|
|
483
|
-
}
|
|
484
|
-
interface CostCallBase {
|
|
485
|
-
callId: string;
|
|
486
|
-
channel: CostChannel;
|
|
487
|
-
phase: string;
|
|
488
|
-
actor: string;
|
|
489
|
-
model: string;
|
|
490
|
-
maximumCostUsd?: number;
|
|
491
|
-
tags?: Record<string, string>;
|
|
492
|
-
timestamp: number;
|
|
493
|
-
}
|
|
494
|
-
interface CostReceipt extends CostCallBase, CostUsage {
|
|
495
|
-
status: 'settled';
|
|
496
|
-
costUsd: number;
|
|
497
|
-
costUnknown: boolean;
|
|
498
|
-
usageUnknown?: boolean;
|
|
499
|
-
pricing?: {
|
|
500
|
-
inputUsdPerThousand: number;
|
|
501
|
-
outputUsdPerThousand: number;
|
|
502
|
-
};
|
|
503
|
-
actualCostUsd?: number;
|
|
504
|
-
error?: string;
|
|
505
|
-
}
|
|
506
|
-
interface CostReceiptInput extends CostUsage {
|
|
507
|
-
model: string;
|
|
508
|
-
actualCostUsd?: number;
|
|
509
|
-
costUnknown?: boolean;
|
|
510
|
-
usageUnknown?: boolean;
|
|
511
|
-
}
|
|
512
|
-
type MaximumCharge = {
|
|
513
|
-
externallyEnforcedMaximumUsd: number;
|
|
514
|
-
} | ({
|
|
515
|
-
model: string;
|
|
516
|
-
} & CostUsage);
|
|
517
|
-
interface RunPaidCallInput<T> {
|
|
518
|
-
callId?: string;
|
|
519
|
-
channel: CostChannel;
|
|
520
|
-
phase: string;
|
|
521
|
-
actor: string;
|
|
522
|
-
/** Used before a provider receipt exists and on failures without one. */
|
|
523
|
-
model?: string;
|
|
524
|
-
tags?: Record<string, string>;
|
|
525
|
-
signal?: AbortSignal;
|
|
526
|
-
/** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
|
|
527
|
-
maximumCharge?: MaximumCharge;
|
|
528
|
-
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
529
|
-
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
530
|
-
receipt(value: T): CostReceiptInput;
|
|
531
|
-
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
532
|
-
}
|
|
533
|
-
type PaidCallResult<T> = {
|
|
534
|
-
succeeded: true;
|
|
535
|
-
callId: string;
|
|
536
|
-
value: T;
|
|
537
|
-
receipt: CostReceipt;
|
|
538
|
-
} | {
|
|
539
|
-
succeeded: false;
|
|
540
|
-
callId?: string;
|
|
541
|
-
error: Error;
|
|
542
|
-
receipt?: CostReceipt;
|
|
543
|
-
};
|
|
544
|
-
interface ChannelRollup {
|
|
545
|
-
channel: CostChannel;
|
|
546
|
-
calls: number;
|
|
547
|
-
inputTokens: number;
|
|
548
|
-
outputTokens: number;
|
|
549
|
-
cachedTokens: number;
|
|
550
|
-
costUsd: number;
|
|
551
|
-
unpricedCalls: number;
|
|
552
|
-
unknownUsageCalls: number;
|
|
553
|
-
}
|
|
554
|
-
interface CostLedgerSummary {
|
|
555
|
-
totalCalls: number;
|
|
556
|
-
pendingCalls: number;
|
|
557
|
-
unresolvedCalls: number;
|
|
558
|
-
reservedCostUsd: number;
|
|
559
|
-
inputTokens: number;
|
|
560
|
-
outputTokens: number;
|
|
561
|
-
cachedTokens: number;
|
|
562
|
-
totalCostUsd: number;
|
|
563
|
-
byChannel: ChannelRollup[];
|
|
564
|
-
unpricedModels: string[];
|
|
565
|
-
fullyPriced: boolean;
|
|
566
|
-
usageComplete: boolean;
|
|
567
|
-
accountingComplete: boolean;
|
|
568
|
-
incompleteReasons: string[];
|
|
569
|
-
}
|
|
570
|
-
interface CostLedgerFilter {
|
|
571
|
-
channel?: CostChannel;
|
|
572
|
-
phase?: string;
|
|
573
|
-
tags?: Record<string, string>;
|
|
574
|
-
}
|
|
575
|
-
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
576
|
-
interface CostLedgerPersistence {
|
|
577
|
-
read(): {
|
|
578
|
-
revision: string;
|
|
579
|
-
events: string;
|
|
580
|
-
};
|
|
581
|
-
append(expectedRevision: string, event: string): string | undefined;
|
|
582
|
-
}
|
|
583
|
-
interface CostLedgerOptions {
|
|
584
|
-
costCeilingUsd?: number;
|
|
585
|
-
persistence?: CostLedgerPersistence;
|
|
586
|
-
/** Import already-settled receipts without admitting new paid work. */
|
|
587
|
-
receipts?: readonly CostReceipt[];
|
|
588
|
-
}
|
|
589
|
-
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
590
|
-
declare class CostLedger {
|
|
591
|
-
private readonly records;
|
|
592
|
-
private readonly activeCallIds;
|
|
593
|
-
private readonly lateCallIds;
|
|
594
|
-
private completedTasks;
|
|
595
|
-
private revision;
|
|
596
|
-
private costLimitPersisted;
|
|
597
|
-
readonly costCeilingUsd?: number;
|
|
598
|
-
private readonly persistence?;
|
|
599
|
-
constructor(input?: number | CostLedgerOptions);
|
|
600
|
-
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
601
|
-
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
602
|
-
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
603
|
-
error?: string;
|
|
604
|
-
}): CostReceipt;
|
|
605
|
-
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
606
|
-
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
607
|
-
markCompleted(count?: number): void;
|
|
608
|
-
costPerCompletedTask(): number | null;
|
|
609
|
-
private execute;
|
|
610
|
-
private captureLateOutcome;
|
|
611
|
-
private commitOutcome;
|
|
612
|
-
private captureFailure;
|
|
613
|
-
private commitReceipt;
|
|
614
|
-
private resolveMaximum;
|
|
615
|
-
private hasIncompleteSettledCall;
|
|
616
|
-
private appendRecord;
|
|
617
|
-
private ensureCostLimitPersisted;
|
|
618
|
-
private appendEvent;
|
|
619
|
-
}
|
|
620
|
-
/** Public callback surface for a shared cost ledger.
|
|
621
|
-
*
|
|
622
|
-
* Declaration bundles may expose this type through multiple package subpaths.
|
|
623
|
-
* Keeping callback contracts structural lets those subpaths compose while the
|
|
624
|
-
* concrete {@link CostLedger} retains its private durable state.
|
|
625
|
-
*/
|
|
626
|
-
type CostLedgerHandle = Pick<CostLedger, keyof CostLedger>;
|
|
627
|
-
|
|
628
647
|
interface Scenario$1 {
|
|
629
648
|
id: string;
|
|
630
649
|
persona: string;
|
|
@@ -1104,6 +1123,8 @@ interface AnalystCost {
|
|
|
1104
1123
|
est_usd_per_run?: number;
|
|
1105
1124
|
/** Models the analyst expects to use (informational). */
|
|
1106
1125
|
models?: string[];
|
|
1126
|
+
/** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */
|
|
1127
|
+
settlement_timeout_ms?: number;
|
|
1107
1128
|
}
|
|
1108
1129
|
interface AnalystRequirements {
|
|
1109
1130
|
/** Min number of shots / samples the analyst needs to produce signal. */
|
|
@@ -1127,10 +1148,14 @@ interface AnalystContext {
|
|
|
1127
1148
|
runId: string;
|
|
1128
1149
|
/** Stable correlation id so logs from a single registry.run() share a tag. */
|
|
1129
1150
|
correlationId: string;
|
|
1130
|
-
/**
|
|
1151
|
+
/** Enforced wall-clock deadline (epoch ms). */
|
|
1131
1152
|
deadlineMs?: number;
|
|
1132
1153
|
/** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
|
|
1133
1154
|
budgetUsd?: number;
|
|
1155
|
+
/** Shared paid-call account when the analyst runs inside a larger campaign. */
|
|
1156
|
+
costLedger?: CostLedgerHandle;
|
|
1157
|
+
/** Attribution phase used when writing to the shared paid-call account. */
|
|
1158
|
+
costPhase?: string;
|
|
1134
1159
|
/**
|
|
1135
1160
|
* Shared chat client. Analysts that call an LLM go through this so
|
|
1136
1161
|
* the operator picks transport (sandbox-sdk | router | cli-bridge |
|
|
@@ -1148,6 +1173,18 @@ interface AnalystContext {
|
|
|
1148
1173
|
* filter. Empty / absent means no cross-run context.
|
|
1149
1174
|
*/
|
|
1150
1175
|
priorFindings?: ReadonlyArray<AnalystFinding>;
|
|
1176
|
+
/**
|
|
1177
|
+
* Findings emitted by analysts that completed earlier in this registry run.
|
|
1178
|
+
* This is separate from `priorFindings`: upstream findings are dependency
|
|
1179
|
+
* context for the current pass, while prior findings are cross-run memory.
|
|
1180
|
+
* The registry populates this only when `RegistryRunOpts.chainFindings` is on.
|
|
1181
|
+
*/
|
|
1182
|
+
upstreamFindings?: ReadonlyArray<AnalystFinding>;
|
|
1183
|
+
/**
|
|
1184
|
+
* Report metered work independently of findings. This keeps an empty finding
|
|
1185
|
+
* set from erasing token/cost telemetry. Multiple receipts are accumulated.
|
|
1186
|
+
*/
|
|
1187
|
+
recordUsage?: (receipt: AnalystUsageReceipt) => void;
|
|
1151
1188
|
/** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */
|
|
1152
1189
|
tags?: Record<string, string>;
|
|
1153
1190
|
/** Logger callback — analysts SHOULD prefer this over console.* for testability. */
|
|
@@ -1173,6 +1210,17 @@ interface Analyst<TInput = unknown> {
|
|
|
1173
1210
|
readonly version: string;
|
|
1174
1211
|
analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>;
|
|
1175
1212
|
}
|
|
1213
|
+
/** Metered work performed by one analyst call. */
|
|
1214
|
+
interface AnalystUsageReceipt {
|
|
1215
|
+
/** Number of model-usage records observed at the provider boundary. */
|
|
1216
|
+
calls: number | null;
|
|
1217
|
+
/** Null when the provider did not return token accounting. */
|
|
1218
|
+
tokens: RunTokenUsage | null;
|
|
1219
|
+
/** Observed, estimated, or explicitly uncaptured dollar cost. */
|
|
1220
|
+
cost: RunCostProvenance;
|
|
1221
|
+
/** Known lower bound when one or more calls have uncaptured cost. */
|
|
1222
|
+
knownCostUsd?: number;
|
|
1223
|
+
}
|
|
1176
1224
|
interface AnalystRunSummary {
|
|
1177
1225
|
analyst_id: string;
|
|
1178
1226
|
status: 'ok' | 'skipped' | 'failed';
|
|
@@ -1181,6 +1229,12 @@ interface AnalystRunSummary {
|
|
|
1181
1229
|
findings_count: number;
|
|
1182
1230
|
latency_ms: number;
|
|
1183
1231
|
cost_usd: number;
|
|
1232
|
+
/**
|
|
1233
|
+
* Additive receipt for model usage. Registry-produced summaries populate it
|
|
1234
|
+
* even when the analyst emits no findings. `cost_usd` remains the legacy
|
|
1235
|
+
* numeric field; inspect `usage.cost` before treating zero as observed.
|
|
1236
|
+
*/
|
|
1237
|
+
usage?: AnalystUsageReceipt;
|
|
1184
1238
|
/** When `status='failed'`: the error class + message, never the full stack. */
|
|
1185
1239
|
error?: {
|
|
1186
1240
|
class: string;
|
|
@@ -1196,6 +1250,11 @@ interface AnalystRunResult {
|
|
|
1196
1250
|
per_analyst: AnalystRunSummary[];
|
|
1197
1251
|
/** Total LLM cost in USD across all analysts in this registry.run(). */
|
|
1198
1252
|
total_cost_usd: number;
|
|
1253
|
+
/**
|
|
1254
|
+
* Provenance for `total_cost_usd`. When uncaptured, the numeric field is only
|
|
1255
|
+
* the known subtotal and must not be treated as the run's total spend.
|
|
1256
|
+
*/
|
|
1257
|
+
total_cost_provenance?: RunCostProvenance;
|
|
1199
1258
|
}
|
|
1200
1259
|
/**
|
|
1201
1260
|
* Events emitted by `AnalystRegistry.runStream(...)` in real time as
|
|
@@ -3944,6 +4003,15 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
|
|
|
3944
4003
|
*/
|
|
3945
4004
|
declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
|
|
3946
4005
|
|
|
4006
|
+
interface MeasuredComparisonFromSelfImproveResultOptions<TScenario extends Scenario, TArtifact> {
|
|
4007
|
+
result: SelfImproveResult<TScenario, TArtifact>;
|
|
4008
|
+
benchmark: AgentImprovementMeasuredComparison['benchmark'];
|
|
4009
|
+
baselineProfileDigest: Sha256Digest;
|
|
4010
|
+
candidateBundleDigest: Sha256Digest;
|
|
4011
|
+
}
|
|
4012
|
+
/** Convert one paired self-improvement result into the portable Interface evidence record. */
|
|
4013
|
+
declare function measuredComparisonFromSelfImproveResult<TScenario extends Scenario, TArtifact>(options: MeasuredComparisonFromSelfImproveResultOptions<TScenario, TArtifact>): AgentImprovementMeasuredComparison;
|
|
4014
|
+
|
|
3947
4015
|
/**
|
|
3948
4016
|
* Typed Ax output for analyst findings.
|
|
3949
4017
|
*
|
|
@@ -3961,7 +4029,10 @@ declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults
|
|
|
3961
4029
|
* validation surface independent of which Ax version is installed.
|
|
3962
4030
|
*/
|
|
3963
4031
|
|
|
4032
|
+
/** Original public schema retained for stored rows and callback contracts. */
|
|
3964
4033
|
declare const RawAnalystFindingSchema: z.ZodObject<{
|
|
4034
|
+
evidence_uri: z.ZodString;
|
|
4035
|
+
evidence_excerpt: z.ZodOptional<z.ZodString>;
|
|
3965
4036
|
severity: z.ZodEnum<{
|
|
3966
4037
|
low: "low";
|
|
3967
4038
|
high: "high";
|
|
@@ -3971,23 +4042,46 @@ declare const RawAnalystFindingSchema: z.ZodObject<{
|
|
|
3971
4042
|
}>;
|
|
3972
4043
|
claim: z.ZodString;
|
|
3973
4044
|
subject: z.ZodOptional<z.ZodString>;
|
|
3974
|
-
evidence_uri: z.ZodString;
|
|
3975
|
-
evidence_excerpt: z.ZodOptional<z.ZodString>;
|
|
3976
4045
|
confidence: z.ZodNumber;
|
|
3977
4046
|
rationale: z.ZodOptional<z.ZodString>;
|
|
3978
4047
|
recommended_action: z.ZodOptional<z.ZodString>;
|
|
3979
4048
|
}, z.core.$strict>;
|
|
3980
4049
|
type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
|
|
4050
|
+
/**
|
|
4051
|
+
* Canonical plural-evidence contract. The preprocessor accepts the original
|
|
4052
|
+
* `evidence_uri` / `evidence_excerpt` pair and normalizes it into one evidence
|
|
4053
|
+
* item so persisted rows and older model fixtures remain readable. New output
|
|
4054
|
+
* always receives the plural shape.
|
|
4055
|
+
*/
|
|
4056
|
+
declare const CanonicalRawAnalystFindingSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodObject<{
|
|
4057
|
+
evidence: z.ZodArray<z.ZodObject<{
|
|
4058
|
+
uri: z.ZodString;
|
|
4059
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
4060
|
+
}, z.core.$strict>>;
|
|
4061
|
+
severity: z.ZodEnum<{
|
|
4062
|
+
low: "low";
|
|
4063
|
+
high: "high";
|
|
4064
|
+
critical: "critical";
|
|
4065
|
+
medium: "medium";
|
|
4066
|
+
info: "info";
|
|
4067
|
+
}>;
|
|
4068
|
+
claim: z.ZodString;
|
|
4069
|
+
subject: z.ZodOptional<z.ZodString>;
|
|
4070
|
+
confidence: z.ZodNumber;
|
|
4071
|
+
rationale: z.ZodOptional<z.ZodString>;
|
|
4072
|
+
recommended_action: z.ZodOptional<z.ZodString>;
|
|
4073
|
+
}, z.core.$strict>>;
|
|
4074
|
+
type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchema>;
|
|
3981
4075
|
|
|
3982
4076
|
/**
|
|
3983
4077
|
* Analyst-kind factory — the typed way to define trace analysts.
|
|
3984
4078
|
*
|
|
3985
4079
|
* A "kind" is a specialized analyst whose actor prompt, tool subset,
|
|
3986
|
-
* and Ax
|
|
3987
|
-
* classification, knowledge gap discovery, knowledge poisoning,
|
|
3988
|
-
* self-improvement, ...). Kinds emit findings in the typed
|
|
3989
|
-
* shape via a JSON-array Ax output; the factory
|
|
3990
|
-
* Zod and lifts it into `AnalystFinding[]
|
|
4080
|
+
* and bounded Ax subqueries target one failure-mode lens (failure-mode
|
|
4081
|
+
* classification, knowledge gap discovery, knowledge poisoning,
|
|
4082
|
+
* self-improvement, ...). Kinds emit findings in the typed
|
|
4083
|
+
* `CanonicalRawAnalystFinding` shape via a JSON-array Ax output; the factory
|
|
4084
|
+
* validates each row with Zod and lifts it into `AnalystFinding[]`.
|
|
3991
4085
|
*
|
|
3992
4086
|
* Composition rules:
|
|
3993
4087
|
* - Each kind owns its actor description. No generic "answer this
|
|
@@ -3995,12 +4089,11 @@ type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
|
|
|
3995
4089
|
* - Each kind picks a narrow tool subset from `ANALYST_TOOL_GROUPS`.
|
|
3996
4090
|
* A kind that never needs full-trace dumps can drop `viewTrace` /
|
|
3997
4091
|
* `viewSpans` and stay cheap.
|
|
3998
|
-
* - Each kind declares its
|
|
3999
|
-
*
|
|
4000
|
-
* (poisoning) usually stay at 0 since they have a tighter brief.
|
|
4092
|
+
* - Each kind declares its subquery + parallelism budget. Discovery-heavy
|
|
4093
|
+
* kinds can fan out more bounded semantic questions than narrow lenses.
|
|
4001
4094
|
*
|
|
4002
4095
|
* Optimizer hook: kinds may declare `goldens` — labeled examples used
|
|
4003
|
-
* by `
|
|
4096
|
+
* by `AxBootstrapFewShot` / `AxGEPA` to fit the actor
|
|
4004
4097
|
* description programmatically. Stored on the kind, not the registry,
|
|
4005
4098
|
* because the right metric is kind-specific.
|
|
4006
4099
|
*/
|
|
@@ -4020,23 +4113,25 @@ interface TraceAnalystKindSpec {
|
|
|
4020
4113
|
version: string;
|
|
4021
4114
|
/** Actor system prompt. Must instruct the LLM to emit `findings` per the schema. */
|
|
4022
4115
|
actorDescription: string;
|
|
4023
|
-
/** Responder system prompt; falls back to a minimal "format the findings" instruction. */
|
|
4024
|
-
responderDescription?: string;
|
|
4025
4116
|
/** Tool functions the actor may call. Pick narrow subsets via `ANALYST_TOOL_GROUPS`. */
|
|
4026
4117
|
buildTools: (store: TraceAnalysisStore) => AxFunction[];
|
|
4027
|
-
/**
|
|
4028
|
-
|
|
4029
|
-
|
|
4030
|
-
|
|
4118
|
+
/** Bounded semantic subqueries. `maxCalls: 0` disables model fan-out. */
|
|
4119
|
+
subqueries?: {
|
|
4120
|
+
maxCalls: number;
|
|
4121
|
+
maxParallel?: number;
|
|
4031
4122
|
};
|
|
4032
4123
|
/** Actor turn cap. Default 12. */
|
|
4033
4124
|
maxTurns?: number;
|
|
4034
4125
|
/** Runtime char cap. Default 6000. */
|
|
4035
4126
|
maxRuntimeChars?: number;
|
|
4127
|
+
/** Maximum output tokens for every actor and subquery model call. Default 4096. */
|
|
4128
|
+
maxOutputTokens?: number;
|
|
4036
4129
|
/** Cost classification surfaced in `registry.list()` and budget enforcement. */
|
|
4037
4130
|
cost: AnalystCost;
|
|
4038
4131
|
/** Per-finding-row hook — kinds may reject / rewrite before lifting. */
|
|
4039
4132
|
postProcess?: (row: RawAnalystFinding, ctx: AnalystContext) => RawAnalystFinding | null;
|
|
4133
|
+
/** Minimum citations per finding. Default 1; rows below it are rejected. */
|
|
4134
|
+
minimumEvidenceCitations?: number;
|
|
4040
4135
|
/** Optional optimizer hook — populated when a kind wants to fit its prompt against labeled examples. */
|
|
4041
4136
|
goldens?: TraceAnalystGolden[];
|
|
4042
4137
|
}
|
|
@@ -4048,7 +4143,7 @@ interface TraceAnalystKindSpec {
|
|
|
4048
4143
|
*/
|
|
4049
4144
|
interface TraceAnalystGolden {
|
|
4050
4145
|
question: string;
|
|
4051
|
-
expected: ReadonlyArray<Omit<
|
|
4146
|
+
expected: ReadonlyArray<Omit<CanonicalRawAnalystFinding, 'confidence'>>;
|
|
4052
4147
|
}
|
|
4053
4148
|
|
|
4054
4149
|
/**
|
|
@@ -4106,7 +4201,8 @@ interface BudgetPolicy {
|
|
|
4106
4201
|
/**
|
|
4107
4202
|
* Custom allocator — receives the analyst, remaining/total budget, and
|
|
4108
4203
|
* the count of analysts that will run. Returns the per-analyst budget
|
|
4109
|
-
* (or undefined
|
|
4204
|
+
* (or undefined only when the run has no overall cap). Overrides weights
|
|
4205
|
+
* when set.
|
|
4110
4206
|
*/
|
|
4111
4207
|
allocate?: (args: {
|
|
4112
4208
|
analyst: Analyst;
|
|
@@ -4132,10 +4228,14 @@ interface RegistryRunOpts {
|
|
|
4132
4228
|
skip?: string[];
|
|
4133
4229
|
/** Budget policy — totalUsd + optional weights/allocator. Falls back to options.defaultBudget. */
|
|
4134
4230
|
budget?: BudgetPolicy;
|
|
4135
|
-
/**
|
|
4231
|
+
/** Active-work cap for the complete registry run. Model receipt settlement may follow. */
|
|
4136
4232
|
timeoutMs?: number;
|
|
4137
4233
|
/** Abort signal — forwarded into every analyst's context. */
|
|
4138
4234
|
signal?: AbortSignal;
|
|
4235
|
+
/** Shared paid-call account forwarded to every analyst. */
|
|
4236
|
+
costLedger?: CostLedgerHandle;
|
|
4237
|
+
/** Attribution phase for calls written to `costLedger`. */
|
|
4238
|
+
costPhase?: string;
|
|
4139
4239
|
/** Tags echoed into AnalystContext.tags — useful for tracking environment/version in findings. */
|
|
4140
4240
|
tags?: Record<string, string>;
|
|
4141
4241
|
/**
|
|
@@ -4143,10 +4243,16 @@ interface RegistryRunOpts {
|
|
|
4143
4243
|
* analyst via `ctx.priorFindings`. The registry forwards the slice
|
|
4144
4244
|
* whose `analyst_id` matches each registered analyst so a kind sees
|
|
4145
4245
|
* only its own history. Pass `{ '*': findings }` to broadcast to
|
|
4146
|
-
* every analyst (useful
|
|
4147
|
-
*
|
|
4246
|
+
* every analyst (useful when several kinds share the same historical
|
|
4247
|
+
* context). For findings from this run, use `chainFindings` instead.
|
|
4148
4248
|
*/
|
|
4149
4249
|
priorFindings?: ReadonlyArray<AnalystFinding> | Record<string, ReadonlyArray<AnalystFinding>>;
|
|
4250
|
+
/**
|
|
4251
|
+
* Pass findings produced earlier in this registry run to each later analyst
|
|
4252
|
+
* via `ctx.upstreamFindings`. Registration order is dependency order.
|
|
4253
|
+
* Disabled by default because independent analyst suites must opt in.
|
|
4254
|
+
*/
|
|
4255
|
+
chainFindings?: boolean;
|
|
4150
4256
|
}
|
|
4151
4257
|
declare class AnalystRegistry {
|
|
4152
4258
|
private readonly analysts;
|
|
@@ -4190,7 +4296,7 @@ declare class AnalystRegistry {
|
|
|
4190
4296
|
interface DefaultAnalystRegistryOptions {
|
|
4191
4297
|
/** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */
|
|
4192
4298
|
ai?: AxAIService;
|
|
4193
|
-
/**
|
|
4299
|
+
/** Required unless `ai` was created by `createAnalystAi`. */
|
|
4194
4300
|
model?: string;
|
|
4195
4301
|
/** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */
|
|
4196
4302
|
kinds?: readonly TraceAnalystKindSpec[];
|
|
@@ -4836,4 +4942,4 @@ interface FromOtelSpansOptions {
|
|
|
4836
4942
|
}
|
|
4837
4943
|
declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
|
|
4838
4944
|
|
|
4839
|
-
export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type ChatClient, type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, type CodeSurface, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runEval, runImprovementLoop, runReferenceEquivalenceJudge, selfImprove, summarizeExecution };
|
|
4945
|
+
export { type AgentEvalAgent, type AgentEvalEvaluateOptions, type AgentEvalImproveOptions, type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AnalystFinding, type AnalyzeRunsOptions, type AuthoringProvenance, type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, type CampaignAggregates, type CampaignArtifactWriter, type CampaignCellResult, type CampaignCostMeter, type CampaignResult, type CampaignStorage, type CampaignTraceWriter, type ChatClient, type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, type CodeSurface, type CostLedgerHandle, type CostProvenanceSummary, type CreateChatClientOpts, type DefaultAnalystRegistryOptions, type DefaultProductionGateOptions, type DefineAgentEvalOptions, type DefinedAgentEval, type DeploymentOutcome, type DispatchFn as Dispatch, type DispatchContext, type EvalCellScoreDelta, type EvalDimensionDelta, type EvalGenerationDiff, type EvalReportingSuiteInput, type EvalReportingSuiteOptions, type EvalReportingSuiteResult, type EvalRunDiff, type EvidenceVector, type EvolutionaryProposerOptions, type ExecutionInsight, type ExecutionReport, type FailureClusterInsight, type FeedbackTableMeta, type FeedbackTableRow, FileSystemOutcomeStore, type FileSystemOutcomeStoreOptions, type FromFeedbackTableOptions, type FromFeedbackTableResult, type FromOtelSpansOptions, type FromRunRecordDirOptions, type FromRunRecordDirResult, type Gate, type GateContext, type GateDecision, type GateResult, type GenerationCandidate, type GenerationRecord, type GepaProposerOptions, type HeldOutGateOptions, type HostedTenant, InMemoryOutcomeStore, type InsightReport, type InterRaterInsight, type JudgeConfig, type JudgeDimension, type JudgeInsight, type JudgeScore, type LiftInsight, type MeasuredComparisonFromSelfImproveResultOptions, type MutableSurface, type Mutator, type ObjectiveSource, type OptimizationProposer, type OptimizerConfig, type OutcomeCorrelationInsight, type OutcomeStore, type ParetoSignificanceGateOptions, type ParsedCodeAgentJsonl, type PartitionByAuthoringModelResult, type PromotionObjective, type PromotionPolicy, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, type Recommendation, type ReferenceEquivalenceJudgeInput, type ReferenceEquivalenceJudgeOptions, type ReferenceEquivalenceJudgeResult, type ReferenceEquivalenceScenario, type ReleaseSummary, type RunCampaignOptions, type RunEvalOptions, type RunImprovementLoopOptions, type RunImprovementLoopResult, type RunRecordRejection, type ScalarDistribution, type Scenario, type SelfImproveBudget, type SelfImproveLlm, type SelfImproveOptions, type SelfImproveProgressEvent, type SelfImproveResult, SelfImproveRunError, type SessionScript, type SummarizeExecutionOptions, type SurfaceProposer, type TokenUsageInsight, analyzeRuns, buildDefaultAnalystRegistry, buildEvidenceVector, composeGate, createChatClient, createReferenceEquivalenceJudge, defaultProductionGate, defineAgentEval, diffGenerations, diffRunBaselineToWinner, diffRuns, evalReportingSuite, evolutionaryProposer, fromClaudeCodeSession, fromCodexSession, fromFeedbackTable, fromKimiCodeSession, fromOpenCodeSession, fromOtelSpans, fromPiSession, fromPigraphSession, fromRunRecordDir, fsCampaignStorage, gepaProposer, heldOutGate, inMemoryCampaignStorage, measuredComparisonFromSelfImproveResult, paretoPolicy, paretoSignificanceGate, parseAgentTrace, parseCodeAgentJsonl, partitionRunsByAuthoringModel, runCampaign, runEval, runImprovementLoop, runReferenceEquivalenceJudge, selfImprove, summarizeExecution };
|