@tangle-network/agent-eval 0.118.3 → 0.119.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +26 -0
- package/dist/analyst/index.d.ts +172 -59
- package/dist/analyst/index.js +50 -12
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +50 -40
- package/dist/benchmarks/index.js +8 -8
- package/dist/campaign/index.d.ts +258 -184
- package/dist/campaign/index.js +7 -7
- package/dist/{chunk-JTMFT5QZ.js → chunk-4I2E3LLO.js} +2 -2
- package/dist/{chunk-KK2O4VWA.js → chunk-D7AEXSM5.js} +2 -2
- package/dist/{chunk-MLHRFWQK.js → chunk-JHOJHHU7.js} +85 -16
- package/dist/chunk-JHOJHHU7.js.map +1 -0
- package/dist/{chunk-F5A4GDQW.js → chunk-JM2SKQMS.js} +3 -15
- package/dist/chunk-JM2SKQMS.js.map +1 -0
- package/dist/{chunk-EGNRE7VA.js → chunk-PAHNGS65.js} +2 -2
- package/dist/{chunk-QWRLW4CT.js → chunk-PSXJ32SR.js} +160 -6
- package/dist/chunk-PSXJ32SR.js.map +1 -0
- package/dist/{chunk-6QIM2EAP.js → chunk-PXD6ZFNY.js} +99 -1
- package/dist/chunk-PXD6ZFNY.js.map +1 -0
- package/dist/{chunk-267UCQWI.js → chunk-QWMPPZ3X.js} +3 -3
- package/dist/{chunk-267UCQWI.js.map → chunk-QWMPPZ3X.js.map} +1 -1
- package/dist/{chunk-VNLWDTDA.js → chunk-S5TT5R3L.js} +1010 -285
- package/dist/chunk-S5TT5R3L.js.map +1 -0
- package/dist/{chunk-ZYHJNKI3.js → chunk-ULOKLHIQ.js} +42 -5
- package/dist/chunk-ULOKLHIQ.js.map +1 -0
- package/dist/{chunk-FIUFRXP3.js → chunk-WW2A73HW.js} +49 -91
- package/dist/chunk-WW2A73HW.js.map +1 -0
- package/dist/{chunk-7V3QDWNL.js → chunk-XJ7JVCHB.js} +2 -2
- package/dist/{chunk-EKHHBKS6.js → chunk-XMBOU5W7.js} +48 -190
- package/dist/chunk-XMBOU5W7.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/contract/index.d.ts +281 -175
- package/dist/contract/index.js +284 -8
- package/dist/contract/index.js.map +1 -1
- package/dist/fuzz.d.ts +19 -1
- package/dist/fuzz.js +1 -1
- package/dist/index.d.ts +223 -114
- package/dist/index.js +38 -37
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/primeintellect/index.d.ts +311 -0
- package/dist/primeintellect/index.js +348 -0
- package/dist/primeintellect/index.js.map +1 -0
- package/dist/rl.d.ts +4 -0
- package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js} +3 -3
- package/dist/traces.d.ts +39 -25
- package/dist/traces.js +3 -5
- package/dist/wire/index.d.ts +19 -1
- package/dist/wire/index.js +2 -2
- package/docs/trace-analysis.md +6 -2
- package/package.json +7 -2
- package/dist/chunk-6QIM2EAP.js.map +0 -1
- package/dist/chunk-EKHHBKS6.js.map +0 -1
- package/dist/chunk-F5A4GDQW.js.map +0 -1
- package/dist/chunk-FIUFRXP3.js.map +0 -1
- package/dist/chunk-MLHRFWQK.js.map +0 -1
- package/dist/chunk-QWRLW4CT.js.map +0 -1
- package/dist/chunk-VNLWDTDA.js.map +0 -1
- package/dist/chunk-ZYHJNKI3.js.map +0 -1
- /package/dist/{chunk-JTMFT5QZ.js.map → chunk-4I2E3LLO.js.map} +0 -0
- /package/dist/{chunk-KK2O4VWA.js.map → chunk-D7AEXSM5.js.map} +0 -0
- /package/dist/{chunk-EGNRE7VA.js.map → chunk-PAHNGS65.js.map} +0 -0
- /package/dist/{chunk-7V3QDWNL.js.map → chunk-XJ7JVCHB.js.map} +0 -0
- /package/dist/{run-campaign-DWC67KJP.js.map → run-campaign-OBXSN5WK.js.map} +0 -0
package/dist/campaign/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { AxAIService, AxFunction } from '@ax-llm/ax';
|
|
2
|
+
import { AxAIService, AxAgentActorTurnCallbackArgs, AxFunction, AxAIArgs } from '@ax-llm/ax';
|
|
3
3
|
import { AgentProfile } from '@tangle-network/agent-interface';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -299,6 +299,174 @@ interface AgentProfileCell {
|
|
|
299
299
|
dimensions?: Record<string, AgentProfileDimensionValue>;
|
|
300
300
|
}
|
|
301
301
|
|
|
302
|
+
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
303
|
+
interface CostUsage {
|
|
304
|
+
inputTokens: number;
|
|
305
|
+
/** Includes reasoning tokens when the provider bills them as output. */
|
|
306
|
+
outputTokens: number;
|
|
307
|
+
/** Reasoning-token subset of outputTokens, when reported. */
|
|
308
|
+
reasoningTokens?: number;
|
|
309
|
+
/** Prompt tokens served from a provider cache. */
|
|
310
|
+
cachedTokens?: number;
|
|
311
|
+
/** Prompt tokens written into a provider cache. */
|
|
312
|
+
cacheWriteTokens?: number;
|
|
313
|
+
}
|
|
314
|
+
interface CostCallBase {
|
|
315
|
+
callId: string;
|
|
316
|
+
channel: CostChannel;
|
|
317
|
+
phase: string;
|
|
318
|
+
actor: string;
|
|
319
|
+
model: string;
|
|
320
|
+
maximumCostUsd?: number;
|
|
321
|
+
tags?: Record<string, string>;
|
|
322
|
+
timestamp: number;
|
|
323
|
+
}
|
|
324
|
+
interface CostReceipt extends CostCallBase, CostUsage {
|
|
325
|
+
status: 'settled';
|
|
326
|
+
costUsd: number;
|
|
327
|
+
costUnknown: boolean;
|
|
328
|
+
usageUnknown?: boolean;
|
|
329
|
+
pricing?: {
|
|
330
|
+
inputUsdPerThousand: number;
|
|
331
|
+
outputUsdPerThousand: number;
|
|
332
|
+
};
|
|
333
|
+
actualCostUsd?: number;
|
|
334
|
+
error?: string;
|
|
335
|
+
}
|
|
336
|
+
interface CostReceiptInput extends CostUsage {
|
|
337
|
+
model: string;
|
|
338
|
+
actualCostUsd?: number;
|
|
339
|
+
costUnknown?: boolean;
|
|
340
|
+
usageUnknown?: boolean;
|
|
341
|
+
}
|
|
342
|
+
type MaximumCharge = {
|
|
343
|
+
externallyEnforcedMaximumUsd: number;
|
|
344
|
+
} | ({
|
|
345
|
+
model: string;
|
|
346
|
+
} & CostUsage);
|
|
347
|
+
interface RunPaidCallInput<T> {
|
|
348
|
+
callId?: string;
|
|
349
|
+
channel: CostChannel;
|
|
350
|
+
phase: string;
|
|
351
|
+
actor: string;
|
|
352
|
+
/** Used before a provider receipt exists and on failures without one. */
|
|
353
|
+
model?: string;
|
|
354
|
+
tags?: Record<string, string>;
|
|
355
|
+
signal?: AbortSignal;
|
|
356
|
+
/** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
|
|
357
|
+
maximumCharge?: MaximumCharge;
|
|
358
|
+
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
359
|
+
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
360
|
+
receipt(value: T): CostReceiptInput;
|
|
361
|
+
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
362
|
+
}
|
|
363
|
+
type PaidCallResult<T> = {
|
|
364
|
+
succeeded: true;
|
|
365
|
+
callId: string;
|
|
366
|
+
value: T;
|
|
367
|
+
receipt: CostReceipt;
|
|
368
|
+
} | {
|
|
369
|
+
succeeded: false;
|
|
370
|
+
callId?: string;
|
|
371
|
+
error: Error;
|
|
372
|
+
receipt?: CostReceipt;
|
|
373
|
+
};
|
|
374
|
+
interface ChannelRollup {
|
|
375
|
+
channel: CostChannel;
|
|
376
|
+
calls: number;
|
|
377
|
+
inputTokens: number;
|
|
378
|
+
outputTokens: number;
|
|
379
|
+
reasoningTokens?: number;
|
|
380
|
+
cachedTokens: number;
|
|
381
|
+
cacheWriteTokens?: number;
|
|
382
|
+
costUsd: number;
|
|
383
|
+
unpricedCalls: number;
|
|
384
|
+
unknownUsageCalls: number;
|
|
385
|
+
}
|
|
386
|
+
interface CostLedgerSummary {
|
|
387
|
+
totalCalls: number;
|
|
388
|
+
pendingCalls: number;
|
|
389
|
+
unresolvedCalls: number;
|
|
390
|
+
reservedCostUsd: number;
|
|
391
|
+
inputTokens: number;
|
|
392
|
+
outputTokens: number;
|
|
393
|
+
reasoningTokens?: number;
|
|
394
|
+
cachedTokens: number;
|
|
395
|
+
cacheWriteTokens?: number;
|
|
396
|
+
totalCostUsd: number;
|
|
397
|
+
byChannel: ChannelRollup[];
|
|
398
|
+
unpricedModels: string[];
|
|
399
|
+
fullyPriced: boolean;
|
|
400
|
+
usageComplete: boolean;
|
|
401
|
+
accountingComplete: boolean;
|
|
402
|
+
incompleteReasons: string[];
|
|
403
|
+
}
|
|
404
|
+
interface CostLedgerFilter {
|
|
405
|
+
channel?: CostChannel;
|
|
406
|
+
phase?: string;
|
|
407
|
+
tags?: Record<string, string>;
|
|
408
|
+
}
|
|
409
|
+
interface CostLedgerWaitOptions {
|
|
410
|
+
/** Maximum time to wait for active provider calls. Default 5 seconds. */
|
|
411
|
+
timeoutMs?: number;
|
|
412
|
+
}
|
|
413
|
+
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
414
|
+
interface CostLedgerPersistence {
|
|
415
|
+
read(): {
|
|
416
|
+
revision: string;
|
|
417
|
+
events: string;
|
|
418
|
+
};
|
|
419
|
+
append(expectedRevision: string, event: string): string | undefined;
|
|
420
|
+
}
|
|
421
|
+
interface CostLedgerOptions {
|
|
422
|
+
costCeilingUsd?: number;
|
|
423
|
+
persistence?: CostLedgerPersistence;
|
|
424
|
+
/** Import already-settled receipts without admitting new paid work. */
|
|
425
|
+
receipts?: readonly CostReceipt[];
|
|
426
|
+
}
|
|
427
|
+
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
428
|
+
declare class CostLedger {
|
|
429
|
+
private readonly records;
|
|
430
|
+
private readonly activeCallIds;
|
|
431
|
+
private readonly lateCallIds;
|
|
432
|
+
private readonly idleWaiters;
|
|
433
|
+
private completedTasks;
|
|
434
|
+
private revision;
|
|
435
|
+
private costLimitPersisted;
|
|
436
|
+
readonly costCeilingUsd?: number;
|
|
437
|
+
private readonly persistence?;
|
|
438
|
+
constructor(input?: number | CostLedgerOptions);
|
|
439
|
+
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
440
|
+
/** Wait until every call started by this ledger has produced a durable outcome. */
|
|
441
|
+
waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
|
|
442
|
+
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
443
|
+
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
444
|
+
error?: string;
|
|
445
|
+
}): CostReceipt;
|
|
446
|
+
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
447
|
+
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
448
|
+
markCompleted(count?: number): void;
|
|
449
|
+
costPerCompletedTask(): number | null;
|
|
450
|
+
private execute;
|
|
451
|
+
private captureLateOutcome;
|
|
452
|
+
private releaseActiveCall;
|
|
453
|
+
private commitOutcome;
|
|
454
|
+
private captureFailure;
|
|
455
|
+
private commitReceipt;
|
|
456
|
+
private resolveMaximum;
|
|
457
|
+
private hasIncompleteSettledCall;
|
|
458
|
+
private appendRecord;
|
|
459
|
+
private ensureCostLimitPersisted;
|
|
460
|
+
private appendEvent;
|
|
461
|
+
}
|
|
462
|
+
/** Public callback surface for a shared cost ledger.
|
|
463
|
+
*
|
|
464
|
+
* Declaration bundles may expose this type through multiple package subpaths.
|
|
465
|
+
* Keeping callback contracts structural lets those subpaths compose while the
|
|
466
|
+
* concrete {@link CostLedger} retains its private durable state.
|
|
467
|
+
*/
|
|
468
|
+
type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
|
|
469
|
+
|
|
302
470
|
type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
|
|
303
471
|
|
|
304
472
|
/**
|
|
@@ -746,156 +914,6 @@ interface TraceAnalysisStore {
|
|
|
746
914
|
}): Promise<SearchSpanResult>;
|
|
747
915
|
}
|
|
748
916
|
|
|
749
|
-
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
750
|
-
interface CostUsage {
|
|
751
|
-
inputTokens: number;
|
|
752
|
-
outputTokens: number;
|
|
753
|
-
cachedTokens?: number;
|
|
754
|
-
}
|
|
755
|
-
interface CostCallBase {
|
|
756
|
-
callId: string;
|
|
757
|
-
channel: CostChannel;
|
|
758
|
-
phase: string;
|
|
759
|
-
actor: string;
|
|
760
|
-
model: string;
|
|
761
|
-
maximumCostUsd?: number;
|
|
762
|
-
tags?: Record<string, string>;
|
|
763
|
-
timestamp: number;
|
|
764
|
-
}
|
|
765
|
-
interface CostReceipt extends CostCallBase, CostUsage {
|
|
766
|
-
status: 'settled';
|
|
767
|
-
costUsd: number;
|
|
768
|
-
costUnknown: boolean;
|
|
769
|
-
usageUnknown?: boolean;
|
|
770
|
-
pricing?: {
|
|
771
|
-
inputUsdPerThousand: number;
|
|
772
|
-
outputUsdPerThousand: number;
|
|
773
|
-
};
|
|
774
|
-
actualCostUsd?: number;
|
|
775
|
-
error?: string;
|
|
776
|
-
}
|
|
777
|
-
interface CostReceiptInput extends CostUsage {
|
|
778
|
-
model: string;
|
|
779
|
-
actualCostUsd?: number;
|
|
780
|
-
costUnknown?: boolean;
|
|
781
|
-
usageUnknown?: boolean;
|
|
782
|
-
}
|
|
783
|
-
type MaximumCharge = {
|
|
784
|
-
externallyEnforcedMaximumUsd: number;
|
|
785
|
-
} | ({
|
|
786
|
-
model: string;
|
|
787
|
-
} & CostUsage);
|
|
788
|
-
interface RunPaidCallInput<T> {
|
|
789
|
-
callId?: string;
|
|
790
|
-
channel: CostChannel;
|
|
791
|
-
phase: string;
|
|
792
|
-
actor: string;
|
|
793
|
-
/** Used before a provider receipt exists and on failures without one. */
|
|
794
|
-
model?: string;
|
|
795
|
-
tags?: Record<string, string>;
|
|
796
|
-
signal?: AbortSignal;
|
|
797
|
-
/** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
|
|
798
|
-
maximumCharge?: MaximumCharge;
|
|
799
|
-
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
800
|
-
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
801
|
-
receipt(value: T): CostReceiptInput;
|
|
802
|
-
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
803
|
-
}
|
|
804
|
-
type PaidCallResult<T> = {
|
|
805
|
-
succeeded: true;
|
|
806
|
-
callId: string;
|
|
807
|
-
value: T;
|
|
808
|
-
receipt: CostReceipt;
|
|
809
|
-
} | {
|
|
810
|
-
succeeded: false;
|
|
811
|
-
callId?: string;
|
|
812
|
-
error: Error;
|
|
813
|
-
receipt?: CostReceipt;
|
|
814
|
-
};
|
|
815
|
-
interface ChannelRollup {
|
|
816
|
-
channel: CostChannel;
|
|
817
|
-
calls: number;
|
|
818
|
-
inputTokens: number;
|
|
819
|
-
outputTokens: number;
|
|
820
|
-
cachedTokens: number;
|
|
821
|
-
costUsd: number;
|
|
822
|
-
unpricedCalls: number;
|
|
823
|
-
unknownUsageCalls: number;
|
|
824
|
-
}
|
|
825
|
-
interface CostLedgerSummary {
|
|
826
|
-
totalCalls: number;
|
|
827
|
-
pendingCalls: number;
|
|
828
|
-
unresolvedCalls: number;
|
|
829
|
-
reservedCostUsd: number;
|
|
830
|
-
inputTokens: number;
|
|
831
|
-
outputTokens: number;
|
|
832
|
-
cachedTokens: number;
|
|
833
|
-
totalCostUsd: number;
|
|
834
|
-
byChannel: ChannelRollup[];
|
|
835
|
-
unpricedModels: string[];
|
|
836
|
-
fullyPriced: boolean;
|
|
837
|
-
usageComplete: boolean;
|
|
838
|
-
accountingComplete: boolean;
|
|
839
|
-
incompleteReasons: string[];
|
|
840
|
-
}
|
|
841
|
-
interface CostLedgerFilter {
|
|
842
|
-
channel?: CostChannel;
|
|
843
|
-
phase?: string;
|
|
844
|
-
tags?: Record<string, string>;
|
|
845
|
-
}
|
|
846
|
-
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
847
|
-
interface CostLedgerPersistence {
|
|
848
|
-
read(): {
|
|
849
|
-
revision: string;
|
|
850
|
-
events: string;
|
|
851
|
-
};
|
|
852
|
-
append(expectedRevision: string, event: string): string | undefined;
|
|
853
|
-
}
|
|
854
|
-
interface CostLedgerOptions {
|
|
855
|
-
costCeilingUsd?: number;
|
|
856
|
-
persistence?: CostLedgerPersistence;
|
|
857
|
-
/** Import already-settled receipts without admitting new paid work. */
|
|
858
|
-
receipts?: readonly CostReceipt[];
|
|
859
|
-
}
|
|
860
|
-
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
861
|
-
declare class CostLedger {
|
|
862
|
-
private readonly records;
|
|
863
|
-
private readonly activeCallIds;
|
|
864
|
-
private readonly lateCallIds;
|
|
865
|
-
private completedTasks;
|
|
866
|
-
private revision;
|
|
867
|
-
private costLimitPersisted;
|
|
868
|
-
readonly costCeilingUsd?: number;
|
|
869
|
-
private readonly persistence?;
|
|
870
|
-
constructor(input?: number | CostLedgerOptions);
|
|
871
|
-
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
872
|
-
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
873
|
-
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
874
|
-
error?: string;
|
|
875
|
-
}): CostReceipt;
|
|
876
|
-
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
877
|
-
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
878
|
-
markCompleted(count?: number): void;
|
|
879
|
-
costPerCompletedTask(): number | null;
|
|
880
|
-
private execute;
|
|
881
|
-
private captureLateOutcome;
|
|
882
|
-
private commitOutcome;
|
|
883
|
-
private captureFailure;
|
|
884
|
-
private commitReceipt;
|
|
885
|
-
private resolveMaximum;
|
|
886
|
-
private hasIncompleteSettledCall;
|
|
887
|
-
private appendRecord;
|
|
888
|
-
private ensureCostLimitPersisted;
|
|
889
|
-
private appendEvent;
|
|
890
|
-
}
|
|
891
|
-
/** Public callback surface for a shared cost ledger.
|
|
892
|
-
*
|
|
893
|
-
* Declaration bundles may expose this type through multiple package subpaths.
|
|
894
|
-
* Keeping callback contracts structural lets those subpaths compose while the
|
|
895
|
-
* concrete {@link CostLedger} retains its private durable state.
|
|
896
|
-
*/
|
|
897
|
-
type CostLedgerHandle = Pick<CostLedger, keyof CostLedger>;
|
|
898
|
-
|
|
899
917
|
/**
|
|
900
918
|
* RawProviderSink — first-class persistence for the actual HTTP-level
|
|
901
919
|
* request/response bodies of every LLM provider call.
|
|
@@ -1261,15 +1279,21 @@ interface AnalystCost {
|
|
|
1261
1279
|
est_usd_per_run?: number;
|
|
1262
1280
|
/** Models the analyst expects to use (informational). */
|
|
1263
1281
|
models?: string[];
|
|
1282
|
+
/** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */
|
|
1283
|
+
settlement_timeout_ms?: number;
|
|
1264
1284
|
}
|
|
1265
1285
|
interface AnalystContext {
|
|
1266
1286
|
runId: string;
|
|
1267
1287
|
/** Stable correlation id so logs from a single registry.run() share a tag. */
|
|
1268
1288
|
correlationId: string;
|
|
1269
|
-
/**
|
|
1289
|
+
/** Enforced wall-clock deadline (epoch ms). */
|
|
1270
1290
|
deadlineMs?: number;
|
|
1271
1291
|
/** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
|
|
1272
1292
|
budgetUsd?: number;
|
|
1293
|
+
/** Shared paid-call account when the analyst runs inside a larger campaign. */
|
|
1294
|
+
costLedger?: CostLedgerHandle;
|
|
1295
|
+
/** Attribution phase used when writing to the shared paid-call account. */
|
|
1296
|
+
costPhase?: string;
|
|
1273
1297
|
/**
|
|
1274
1298
|
* Shared chat client. Analysts that call an LLM go through this so
|
|
1275
1299
|
* the operator picks transport (sandbox-sdk | router | cli-bridge |
|
|
@@ -1287,6 +1311,18 @@ interface AnalystContext {
|
|
|
1287
1311
|
* filter. Empty / absent means no cross-run context.
|
|
1288
1312
|
*/
|
|
1289
1313
|
priorFindings?: ReadonlyArray<AnalystFinding>;
|
|
1314
|
+
/**
|
|
1315
|
+
* Findings emitted by analysts that completed earlier in this registry run.
|
|
1316
|
+
* This is separate from `priorFindings`: upstream findings are dependency
|
|
1317
|
+
* context for the current pass, while prior findings are cross-run memory.
|
|
1318
|
+
* The registry populates this only when `RegistryRunOpts.chainFindings` is on.
|
|
1319
|
+
*/
|
|
1320
|
+
upstreamFindings?: ReadonlyArray<AnalystFinding>;
|
|
1321
|
+
/**
|
|
1322
|
+
* Report metered work independently of findings. This keeps an empty finding
|
|
1323
|
+
* set from erasing token/cost telemetry. Multiple receipts are accumulated.
|
|
1324
|
+
*/
|
|
1325
|
+
recordUsage?: (receipt: AnalystUsageReceipt) => void;
|
|
1290
1326
|
/** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */
|
|
1291
1327
|
tags?: Record<string, string>;
|
|
1292
1328
|
/** Logger callback — analysts SHOULD prefer this over console.* for testability. */
|
|
@@ -1294,6 +1330,17 @@ interface AnalystContext {
|
|
|
1294
1330
|
/** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */
|
|
1295
1331
|
signal?: AbortSignal;
|
|
1296
1332
|
}
|
|
1333
|
+
/** Metered work performed by one analyst call. */
|
|
1334
|
+
interface AnalystUsageReceipt {
|
|
1335
|
+
/** Number of model-usage records observed at the provider boundary. */
|
|
1336
|
+
calls: number | null;
|
|
1337
|
+
/** Null when the provider did not return token accounting. */
|
|
1338
|
+
tokens: RunTokenUsage | null;
|
|
1339
|
+
/** Observed, estimated, or explicitly uncaptured dollar cost. */
|
|
1340
|
+
cost: RunCostProvenance;
|
|
1341
|
+
/** Known lower bound when one or more calls have uncaptured cost. */
|
|
1342
|
+
knownCostUsd?: number;
|
|
1343
|
+
}
|
|
1297
1344
|
|
|
1298
1345
|
type PolicyEditSchemaVersion = 'policy-edit/v1';
|
|
1299
1346
|
declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
|
|
@@ -2112,11 +2159,11 @@ interface AnalyzeTracesInput {
|
|
|
2112
2159
|
question: string;
|
|
2113
2160
|
}
|
|
2114
2161
|
interface AnalyzeTracesResult {
|
|
2115
|
-
/** The
|
|
2162
|
+
/** The actor's submitted prose answer. */
|
|
2116
2163
|
answer: string;
|
|
2117
|
-
/** Bulleted findings
|
|
2164
|
+
/** Bulleted findings from the actor's structured completion. */
|
|
2118
2165
|
findings: string[];
|
|
2119
|
-
/** Per-
|
|
2166
|
+
/** Per-turn snapshots captured via `actorTurnCallback`. */
|
|
2120
2167
|
turns: AnalyzeTracesTurnSnapshot[];
|
|
2121
2168
|
/** Total turns the actor took. */
|
|
2122
2169
|
turnCount: number;
|
|
@@ -2142,13 +2189,14 @@ interface TraceAnalystChatMessage {
|
|
|
2142
2189
|
[key: string]: unknown;
|
|
2143
2190
|
}
|
|
2144
2191
|
interface AnalyzeTracesTurnSnapshot {
|
|
2192
|
+
stage: AxAgentActorTurnCallbackArgs['stage'];
|
|
2145
2193
|
turn: number;
|
|
2146
2194
|
isError: boolean;
|
|
2147
2195
|
/** The JS code the actor produced for this turn. */
|
|
2148
2196
|
code: string;
|
|
2149
2197
|
/** The formatted action-log entry the actor sees on the next turn. */
|
|
2150
2198
|
output: string;
|
|
2151
|
-
/** Provider thought (when `
|
|
2199
|
+
/** Provider thought (when `executorOptions.showThoughts` is true and the
|
|
2152
2200
|
* provider returns it). */
|
|
2153
2201
|
thought?: string;
|
|
2154
2202
|
}
|
|
@@ -2157,18 +2205,18 @@ interface AnalyzeTracesOptions {
|
|
|
2157
2205
|
source: string | TraceAnalysisStore;
|
|
2158
2206
|
/** Caller-provided AxAIService. */
|
|
2159
2207
|
ai: AxAIService;
|
|
2160
|
-
/** Model id forwarded to actor
|
|
2208
|
+
/** Model id forwarded to the actor. */
|
|
2161
2209
|
model?: string;
|
|
2162
|
-
/**
|
|
2163
|
-
|
|
2210
|
+
/** Maximum model subqueries. 0 disables model fan-out. Default 4. */
|
|
2211
|
+
maxSubqueries?: number;
|
|
2164
2212
|
/** Maximum actor turns. Default 12. */
|
|
2165
2213
|
maxTurns?: number;
|
|
2166
|
-
/** Maximum parallel
|
|
2167
|
-
|
|
2214
|
+
/** Maximum parallel model subqueries. Default 2. */
|
|
2215
|
+
maxParallelSubqueries?: number;
|
|
2216
|
+
/** Cancels in-flight model and tool work. */
|
|
2217
|
+
signal?: AbortSignal;
|
|
2168
2218
|
/** Override the actor description. */
|
|
2169
2219
|
actorDescription?: string;
|
|
2170
|
-
/** Override the subagent description. */
|
|
2171
|
-
subagentDescription?: string;
|
|
2172
2220
|
/** Per-turn observability hook. */
|
|
2173
2221
|
onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>;
|
|
2174
2222
|
/** Override max runtime characters per turn. Default 6000. */
|
|
@@ -2176,8 +2224,7 @@ interface AnalyzeTracesOptions {
|
|
|
2176
2224
|
/** When set, every turn's snapshot is appended to this JSONL file
|
|
2177
2225
|
* immediately. If the analyst crashes mid-loop (provider 503,
|
|
2178
2226
|
* network error, validator reject) the partial reasoning is still
|
|
2179
|
-
* on disk
|
|
2180
|
-
* evidence. */
|
|
2227
|
+
* on disk for diagnosis and recovery. */
|
|
2181
2228
|
progressLogPath?: string;
|
|
2182
2229
|
}
|
|
2183
2230
|
|
|
@@ -5819,7 +5866,10 @@ declare function assertPolicyEditAuthorContextBudget(value: unknown, maxChars: n
|
|
|
5819
5866
|
* validation surface independent of which Ax version is installed.
|
|
5820
5867
|
*/
|
|
5821
5868
|
|
|
5869
|
+
/** Original public schema retained for stored rows and callback contracts. */
|
|
5822
5870
|
declare const RawAnalystFindingSchema: z.ZodObject<{
|
|
5871
|
+
evidence_uri: z.ZodString;
|
|
5872
|
+
evidence_excerpt: z.ZodOptional<z.ZodString>;
|
|
5823
5873
|
severity: z.ZodEnum<{
|
|
5824
5874
|
low: "low";
|
|
5825
5875
|
high: "high";
|
|
@@ -5829,23 +5879,46 @@ declare const RawAnalystFindingSchema: z.ZodObject<{
|
|
|
5829
5879
|
}>;
|
|
5830
5880
|
claim: z.ZodString;
|
|
5831
5881
|
subject: z.ZodOptional<z.ZodString>;
|
|
5832
|
-
evidence_uri: z.ZodString;
|
|
5833
|
-
evidence_excerpt: z.ZodOptional<z.ZodString>;
|
|
5834
5882
|
confidence: z.ZodNumber;
|
|
5835
5883
|
rationale: z.ZodOptional<z.ZodString>;
|
|
5836
5884
|
recommended_action: z.ZodOptional<z.ZodString>;
|
|
5837
5885
|
}, z.core.$strict>;
|
|
5838
5886
|
type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
|
|
5887
|
+
/**
|
|
5888
|
+
* Canonical plural-evidence contract. The preprocessor accepts the original
|
|
5889
|
+
* `evidence_uri` / `evidence_excerpt` pair and normalizes it into one evidence
|
|
5890
|
+
* item so persisted rows and older model fixtures remain readable. New output
|
|
5891
|
+
* always receives the plural shape.
|
|
5892
|
+
*/
|
|
5893
|
+
declare const CanonicalRawAnalystFindingSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodObject<{
|
|
5894
|
+
evidence: z.ZodArray<z.ZodObject<{
|
|
5895
|
+
uri: z.ZodString;
|
|
5896
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
5897
|
+
}, z.core.$strict>>;
|
|
5898
|
+
severity: z.ZodEnum<{
|
|
5899
|
+
low: "low";
|
|
5900
|
+
high: "high";
|
|
5901
|
+
critical: "critical";
|
|
5902
|
+
medium: "medium";
|
|
5903
|
+
info: "info";
|
|
5904
|
+
}>;
|
|
5905
|
+
claim: z.ZodString;
|
|
5906
|
+
subject: z.ZodOptional<z.ZodString>;
|
|
5907
|
+
confidence: z.ZodNumber;
|
|
5908
|
+
rationale: z.ZodOptional<z.ZodString>;
|
|
5909
|
+
recommended_action: z.ZodOptional<z.ZodString>;
|
|
5910
|
+
}, z.core.$strict>>;
|
|
5911
|
+
type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchema>;
|
|
5839
5912
|
|
|
5840
5913
|
/**
|
|
5841
5914
|
* Analyst-kind factory — the typed way to define trace analysts.
|
|
5842
5915
|
*
|
|
5843
5916
|
* A "kind" is a specialized analyst whose actor prompt, tool subset,
|
|
5844
|
-
* and Ax
|
|
5845
|
-
* classification, knowledge gap discovery, knowledge poisoning,
|
|
5846
|
-
* self-improvement, ...). Kinds emit findings in the typed
|
|
5847
|
-
* shape via a JSON-array Ax output; the factory
|
|
5848
|
-
* Zod and lifts it into `AnalystFinding[]
|
|
5917
|
+
* and bounded Ax subqueries target one failure-mode lens (failure-mode
|
|
5918
|
+
* classification, knowledge gap discovery, knowledge poisoning,
|
|
5919
|
+
* self-improvement, ...). Kinds emit findings in the typed
|
|
5920
|
+
* `CanonicalRawAnalystFinding` shape via a JSON-array Ax output; the factory
|
|
5921
|
+
* validates each row with Zod and lifts it into `AnalystFinding[]`.
|
|
5849
5922
|
*
|
|
5850
5923
|
* Composition rules:
|
|
5851
5924
|
* - Each kind owns its actor description. No generic "answer this
|
|
@@ -5853,12 +5926,11 @@ type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
|
|
|
5853
5926
|
* - Each kind picks a narrow tool subset from `ANALYST_TOOL_GROUPS`.
|
|
5854
5927
|
* A kind that never needs full-trace dumps can drop `viewTrace` /
|
|
5855
5928
|
* `viewSpans` and stay cheap.
|
|
5856
|
-
* - Each kind declares its
|
|
5857
|
-
*
|
|
5858
|
-
* (poisoning) usually stay at 0 since they have a tighter brief.
|
|
5929
|
+
* - Each kind declares its subquery + parallelism budget. Discovery-heavy
|
|
5930
|
+
* kinds can fan out more bounded semantic questions than narrow lenses.
|
|
5859
5931
|
*
|
|
5860
5932
|
* Optimizer hook: kinds may declare `goldens` — labeled examples used
|
|
5861
|
-
* by `
|
|
5933
|
+
* by `AxBootstrapFewShot` / `AxGEPA` to fit the actor
|
|
5862
5934
|
* description programmatically. Stored on the kind, not the registry,
|
|
5863
5935
|
* because the right metric is kind-specific.
|
|
5864
5936
|
*/
|
|
@@ -5878,23 +5950,25 @@ interface TraceAnalystKindSpec {
|
|
|
5878
5950
|
version: string;
|
|
5879
5951
|
/** Actor system prompt. Must instruct the LLM to emit `findings` per the schema. */
|
|
5880
5952
|
actorDescription: string;
|
|
5881
|
-
/** Responder system prompt; falls back to a minimal "format the findings" instruction. */
|
|
5882
|
-
responderDescription?: string;
|
|
5883
5953
|
/** Tool functions the actor may call. Pick narrow subsets via `ANALYST_TOOL_GROUPS`. */
|
|
5884
5954
|
buildTools: (store: TraceAnalysisStore) => AxFunction[];
|
|
5885
|
-
/**
|
|
5886
|
-
|
|
5887
|
-
|
|
5888
|
-
|
|
5955
|
+
/** Bounded semantic subqueries. `maxCalls: 0` disables model fan-out. */
|
|
5956
|
+
subqueries?: {
|
|
5957
|
+
maxCalls: number;
|
|
5958
|
+
maxParallel?: number;
|
|
5889
5959
|
};
|
|
5890
5960
|
/** Actor turn cap. Default 12. */
|
|
5891
5961
|
maxTurns?: number;
|
|
5892
5962
|
/** Runtime char cap. Default 6000. */
|
|
5893
5963
|
maxRuntimeChars?: number;
|
|
5964
|
+
/** Maximum output tokens for every actor and subquery model call. Default 4096. */
|
|
5965
|
+
maxOutputTokens?: number;
|
|
5894
5966
|
/** Cost classification surfaced in `registry.list()` and budget enforcement. */
|
|
5895
5967
|
cost: AnalystCost;
|
|
5896
5968
|
/** Per-finding-row hook — kinds may reject / rewrite before lifting. */
|
|
5897
5969
|
postProcess?: (row: RawAnalystFinding, ctx: AnalystContext) => RawAnalystFinding | null;
|
|
5970
|
+
/** Minimum citations per finding. Default 1; rows below it are rejected. */
|
|
5971
|
+
minimumEvidenceCitations?: number;
|
|
5898
5972
|
/** Optional optimizer hook — populated when a kind wants to fit its prompt against labeled examples. */
|
|
5899
5973
|
goldens?: TraceAnalystGolden[];
|
|
5900
5974
|
}
|
|
@@ -5906,7 +5980,7 @@ interface TraceAnalystKindSpec {
|
|
|
5906
5980
|
*/
|
|
5907
5981
|
interface TraceAnalystGolden {
|
|
5908
5982
|
question: string;
|
|
5909
|
-
expected: ReadonlyArray<Omit<
|
|
5983
|
+
expected: ReadonlyArray<Omit<CanonicalRawAnalystFinding, 'confidence'>>;
|
|
5910
5984
|
}
|
|
5911
5985
|
|
|
5912
5986
|
/**
|
|
@@ -5944,7 +6018,7 @@ interface TraceAnalystProposerOptions {
|
|
|
5944
6018
|
applyMaxTokens?: number;
|
|
5945
6019
|
/** Ax provider name. Default 'openai' — works for any OpenAI-compatible base
|
|
5946
6020
|
* via `apiURL`. Use 'deepseek' to hit DeepSeek's native provider. */
|
|
5947
|
-
provider?:
|
|
6021
|
+
provider?: AxAIArgs<unknown>['name'];
|
|
5948
6022
|
/** Which analyst kinds to run. Default = the full shipped suite. */
|
|
5949
6023
|
kinds?: readonly TraceAnalystKindSpec[];
|
|
5950
6024
|
/** Resolve the OTLP traces (JSONL string) the analyst should read for THIS
|
package/dist/campaign/index.js
CHANGED
|
@@ -66,7 +66,7 @@ import {
|
|
|
66
66
|
userStoryScoreboard,
|
|
67
67
|
validateSearchLedgerEvent,
|
|
68
68
|
verifyCodeSurface
|
|
69
|
-
} from "../chunk-
|
|
69
|
+
} from "../chunk-XMBOU5W7.js";
|
|
70
70
|
import {
|
|
71
71
|
assertCodeSurfaceIdentity,
|
|
72
72
|
buildEvidenceVector,
|
|
@@ -103,7 +103,7 @@ import {
|
|
|
103
103
|
runOptimization,
|
|
104
104
|
surfaceContentHash,
|
|
105
105
|
surfaceHash
|
|
106
|
-
} from "../chunk-
|
|
106
|
+
} from "../chunk-PSXJ32SR.js";
|
|
107
107
|
import {
|
|
108
108
|
SearchLedgerConflictError,
|
|
109
109
|
SearchLedgerError,
|
|
@@ -115,20 +115,20 @@ import {
|
|
|
115
115
|
resolveRunDir,
|
|
116
116
|
runCampaign,
|
|
117
117
|
tangleTracesRoot
|
|
118
|
-
} from "../chunk-
|
|
118
|
+
} from "../chunk-PAHNGS65.js";
|
|
119
119
|
import "../chunk-3YYRZDON.js";
|
|
120
120
|
import {
|
|
121
121
|
POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
|
|
122
122
|
validatePolicyEditCandidateRecord
|
|
123
|
-
} from "../chunk-
|
|
123
|
+
} from "../chunk-S5TT5R3L.js";
|
|
124
124
|
import "../chunk-ARU2PZFM.js";
|
|
125
125
|
import "../chunk-NJC7U437.js";
|
|
126
126
|
import "../chunk-PJQFMIOX.js";
|
|
127
|
-
import "../chunk-
|
|
127
|
+
import "../chunk-JHOJHHU7.js";
|
|
128
128
|
import "../chunk-VI2UW6B6.js";
|
|
129
|
-
import "../chunk-
|
|
129
|
+
import "../chunk-WW2A73HW.js";
|
|
130
130
|
import "../chunk-GGE4NNQT.js";
|
|
131
|
-
import "../chunk-
|
|
131
|
+
import "../chunk-PXD6ZFNY.js";
|
|
132
132
|
import "../chunk-PC4UYEBM.js";
|
|
133
133
|
import "../chunk-S3UZOQ5Y.js";
|
|
134
134
|
import "../chunk-MA6HLL3S.js";
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
} from "./chunk-NJC7U437.js";
|
|
7
7
|
import {
|
|
8
8
|
CostLedger
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-JHOJHHU7.js";
|
|
10
10
|
|
|
11
11
|
// src/wire/schemas.ts
|
|
12
12
|
import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi";
|
|
@@ -1027,4 +1027,4 @@ export {
|
|
|
1027
1027
|
startServer,
|
|
1028
1028
|
startServerAsync
|
|
1029
1029
|
};
|
|
1030
|
-
//# sourceMappingURL=chunk-
|
|
1030
|
+
//# sourceMappingURL=chunk-4I2E3LLO.js.map
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
fsCampaignStorage,
|
|
3
3
|
runCampaign
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-PAHNGS65.js";
|
|
5
5
|
import {
|
|
6
6
|
__export
|
|
7
7
|
} from "./chunk-PZ5AY32C.js";
|
|
@@ -763,4 +763,4 @@ export {
|
|
|
763
763
|
retrievalMetricsAtCutoff,
|
|
764
764
|
benchmarks_exports
|
|
765
765
|
};
|
|
766
|
-
//# sourceMappingURL=chunk-
|
|
766
|
+
//# sourceMappingURL=chunk-D7AEXSM5.js.map
|