@tangle-network/agent-eval 0.118.2 → 0.119.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +26 -0
- package/dist/analyst/index.d.ts +143 -21
- package/dist/analyst/index.js +46 -12
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +50 -40
- package/dist/benchmarks/index.js +9 -9
- package/dist/campaign/index.d.ts +231 -156
- package/dist/campaign/index.js +8 -8
- package/dist/{chunk-JTMFT5QZ.js → chunk-4I2E3LLO.js} +2 -2
- package/dist/{chunk-BUBU3YHT.js → chunk-6QIM2EAP.js} +6 -116
- package/dist/chunk-6QIM2EAP.js.map +1 -0
- package/dist/{chunk-DZBCDDIE.js → chunk-7A4LIMMY.js} +846 -253
- package/dist/chunk-7A4LIMMY.js.map +1 -0
- package/dist/{chunk-KK2O4VWA.js → chunk-D7AEXSM5.js} +2 -2
- package/dist/{chunk-GCT3DQWU.js → chunk-F6YUH3L4.js} +41 -30
- package/dist/{chunk-GCT3DQWU.js.map → chunk-F6YUH3L4.js.map} +1 -1
- package/dist/{chunk-63KG3ASG.js → chunk-FIUFRXP3.js} +2 -2
- package/dist/{chunk-J23QK2U5.js → chunk-GERDAIAL.js} +8 -8
- package/dist/{chunk-J23QK2U5.js.map → chunk-GERDAIAL.js.map} +1 -1
- package/dist/{chunk-TVXPJJ7H.js → chunk-H5UD2323.js} +2 -2
- package/dist/{chunk-MLHRFWQK.js → chunk-JHOJHHU7.js} +85 -16
- package/dist/chunk-JHOJHHU7.js.map +1 -0
- package/dist/chunk-K4DBDHLK.js +158 -0
- package/dist/chunk-K4DBDHLK.js.map +1 -0
- package/dist/{chunk-DRSFTBKU.js → chunk-LMJ2TGWJ.js} +3 -15
- package/dist/chunk-LMJ2TGWJ.js.map +1 -0
- package/dist/{chunk-EGNRE7VA.js → chunk-PAHNGS65.js} +2 -2
- package/dist/{chunk-I42AAGHL.js → chunk-RLCJQ6VZ.js} +5 -5
- package/dist/{chunk-7V3QDWNL.js → chunk-XJ7JVCHB.js} +2 -2
- package/dist/{chunk-6JQE3YVE.js → chunk-ZYHJNKI3.js} +6 -6
- package/dist/cli.js +2 -2
- package/dist/contract/index.d.ts +257 -160
- package/dist/contract/index.js +11 -11
- package/dist/fuzz.d.ts +19 -1
- package/dist/fuzz.js +1 -1
- package/dist/index.d.ts +170 -64
- package/dist/index.js +30 -28
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/primeintellect/index.d.ts +311 -0
- package/dist/primeintellect/index.js +348 -0
- package/dist/primeintellect/index.js.map +1 -0
- package/dist/rl.d.ts +4 -0
- package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js} +3 -3
- package/dist/trace-attributes.d.ts +17 -1
- package/dist/trace-attributes.js +27 -3
- package/dist/traces.d.ts +16 -16
- package/dist/traces.js +18 -18
- package/dist/wire/index.d.ts +19 -1
- package/dist/wire/index.js +2 -2
- package/package.json +6 -1
- package/dist/chunk-BUBU3YHT.js.map +0 -1
- package/dist/chunk-DRSFTBKU.js.map +0 -1
- package/dist/chunk-DZBCDDIE.js.map +0 -1
- package/dist/chunk-MLHRFWQK.js.map +0 -1
- package/dist/chunk-NW4AQBXD.js +0 -53
- package/dist/chunk-NW4AQBXD.js.map +0 -1
- /package/dist/{chunk-JTMFT5QZ.js.map → chunk-4I2E3LLO.js.map} +0 -0
- /package/dist/{chunk-KK2O4VWA.js.map → chunk-D7AEXSM5.js.map} +0 -0
- /package/dist/{chunk-63KG3ASG.js.map → chunk-FIUFRXP3.js.map} +0 -0
- /package/dist/{chunk-TVXPJJ7H.js.map → chunk-H5UD2323.js.map} +0 -0
- /package/dist/{chunk-EGNRE7VA.js.map → chunk-PAHNGS65.js.map} +0 -0
- /package/dist/{chunk-I42AAGHL.js.map → chunk-RLCJQ6VZ.js.map} +0 -0
- /package/dist/{chunk-7V3QDWNL.js.map → chunk-XJ7JVCHB.js.map} +0 -0
- /package/dist/{chunk-6JQE3YVE.js.map → chunk-ZYHJNKI3.js.map} +0 -0
- /package/dist/{run-campaign-DWC67KJP.js.map → run-campaign-OBXSN5WK.js.map} +0 -0
package/dist/campaign/index.d.ts
CHANGED
|
@@ -299,6 +299,174 @@ interface AgentProfileCell {
|
|
|
299
299
|
dimensions?: Record<string, AgentProfileDimensionValue>;
|
|
300
300
|
}
|
|
301
301
|
|
|
302
|
+
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
303
|
+
interface CostUsage {
|
|
304
|
+
inputTokens: number;
|
|
305
|
+
/** Includes reasoning tokens when the provider bills them as output. */
|
|
306
|
+
outputTokens: number;
|
|
307
|
+
/** Reasoning-token subset of outputTokens, when reported. */
|
|
308
|
+
reasoningTokens?: number;
|
|
309
|
+
/** Prompt tokens served from a provider cache. */
|
|
310
|
+
cachedTokens?: number;
|
|
311
|
+
/** Prompt tokens written into a provider cache. */
|
|
312
|
+
cacheWriteTokens?: number;
|
|
313
|
+
}
|
|
314
|
+
interface CostCallBase {
|
|
315
|
+
callId: string;
|
|
316
|
+
channel: CostChannel;
|
|
317
|
+
phase: string;
|
|
318
|
+
actor: string;
|
|
319
|
+
model: string;
|
|
320
|
+
maximumCostUsd?: number;
|
|
321
|
+
tags?: Record<string, string>;
|
|
322
|
+
timestamp: number;
|
|
323
|
+
}
|
|
324
|
+
interface CostReceipt extends CostCallBase, CostUsage {
|
|
325
|
+
status: 'settled';
|
|
326
|
+
costUsd: number;
|
|
327
|
+
costUnknown: boolean;
|
|
328
|
+
usageUnknown?: boolean;
|
|
329
|
+
pricing?: {
|
|
330
|
+
inputUsdPerThousand: number;
|
|
331
|
+
outputUsdPerThousand: number;
|
|
332
|
+
};
|
|
333
|
+
actualCostUsd?: number;
|
|
334
|
+
error?: string;
|
|
335
|
+
}
|
|
336
|
+
interface CostReceiptInput extends CostUsage {
|
|
337
|
+
model: string;
|
|
338
|
+
actualCostUsd?: number;
|
|
339
|
+
costUnknown?: boolean;
|
|
340
|
+
usageUnknown?: boolean;
|
|
341
|
+
}
|
|
342
|
+
type MaximumCharge = {
|
|
343
|
+
externallyEnforcedMaximumUsd: number;
|
|
344
|
+
} | ({
|
|
345
|
+
model: string;
|
|
346
|
+
} & CostUsage);
|
|
347
|
+
interface RunPaidCallInput<T> {
|
|
348
|
+
callId?: string;
|
|
349
|
+
channel: CostChannel;
|
|
350
|
+
phase: string;
|
|
351
|
+
actor: string;
|
|
352
|
+
/** Used before a provider receipt exists and on failures without one. */
|
|
353
|
+
model?: string;
|
|
354
|
+
tags?: Record<string, string>;
|
|
355
|
+
signal?: AbortSignal;
|
|
356
|
+
/** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
|
|
357
|
+
maximumCharge?: MaximumCharge;
|
|
358
|
+
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
359
|
+
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
360
|
+
receipt(value: T): CostReceiptInput;
|
|
361
|
+
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
362
|
+
}
|
|
363
|
+
type PaidCallResult<T> = {
|
|
364
|
+
succeeded: true;
|
|
365
|
+
callId: string;
|
|
366
|
+
value: T;
|
|
367
|
+
receipt: CostReceipt;
|
|
368
|
+
} | {
|
|
369
|
+
succeeded: false;
|
|
370
|
+
callId?: string;
|
|
371
|
+
error: Error;
|
|
372
|
+
receipt?: CostReceipt;
|
|
373
|
+
};
|
|
374
|
+
interface ChannelRollup {
|
|
375
|
+
channel: CostChannel;
|
|
376
|
+
calls: number;
|
|
377
|
+
inputTokens: number;
|
|
378
|
+
outputTokens: number;
|
|
379
|
+
reasoningTokens?: number;
|
|
380
|
+
cachedTokens: number;
|
|
381
|
+
cacheWriteTokens?: number;
|
|
382
|
+
costUsd: number;
|
|
383
|
+
unpricedCalls: number;
|
|
384
|
+
unknownUsageCalls: number;
|
|
385
|
+
}
|
|
386
|
+
interface CostLedgerSummary {
|
|
387
|
+
totalCalls: number;
|
|
388
|
+
pendingCalls: number;
|
|
389
|
+
unresolvedCalls: number;
|
|
390
|
+
reservedCostUsd: number;
|
|
391
|
+
inputTokens: number;
|
|
392
|
+
outputTokens: number;
|
|
393
|
+
reasoningTokens?: number;
|
|
394
|
+
cachedTokens: number;
|
|
395
|
+
cacheWriteTokens?: number;
|
|
396
|
+
totalCostUsd: number;
|
|
397
|
+
byChannel: ChannelRollup[];
|
|
398
|
+
unpricedModels: string[];
|
|
399
|
+
fullyPriced: boolean;
|
|
400
|
+
usageComplete: boolean;
|
|
401
|
+
accountingComplete: boolean;
|
|
402
|
+
incompleteReasons: string[];
|
|
403
|
+
}
|
|
404
|
+
interface CostLedgerFilter {
|
|
405
|
+
channel?: CostChannel;
|
|
406
|
+
phase?: string;
|
|
407
|
+
tags?: Record<string, string>;
|
|
408
|
+
}
|
|
409
|
+
interface CostLedgerWaitOptions {
|
|
410
|
+
/** Maximum time to wait for active provider calls. Default 5 seconds. */
|
|
411
|
+
timeoutMs?: number;
|
|
412
|
+
}
|
|
413
|
+
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
414
|
+
interface CostLedgerPersistence {
|
|
415
|
+
read(): {
|
|
416
|
+
revision: string;
|
|
417
|
+
events: string;
|
|
418
|
+
};
|
|
419
|
+
append(expectedRevision: string, event: string): string | undefined;
|
|
420
|
+
}
|
|
421
|
+
interface CostLedgerOptions {
|
|
422
|
+
costCeilingUsd?: number;
|
|
423
|
+
persistence?: CostLedgerPersistence;
|
|
424
|
+
/** Import already-settled receipts without admitting new paid work. */
|
|
425
|
+
receipts?: readonly CostReceipt[];
|
|
426
|
+
}
|
|
427
|
+
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
428
|
+
declare class CostLedger {
|
|
429
|
+
private readonly records;
|
|
430
|
+
private readonly activeCallIds;
|
|
431
|
+
private readonly lateCallIds;
|
|
432
|
+
private readonly idleWaiters;
|
|
433
|
+
private completedTasks;
|
|
434
|
+
private revision;
|
|
435
|
+
private costLimitPersisted;
|
|
436
|
+
readonly costCeilingUsd?: number;
|
|
437
|
+
private readonly persistence?;
|
|
438
|
+
constructor(input?: number | CostLedgerOptions);
|
|
439
|
+
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
440
|
+
/** Wait until every call started by this ledger has produced a durable outcome. */
|
|
441
|
+
waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
|
|
442
|
+
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
443
|
+
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
444
|
+
error?: string;
|
|
445
|
+
}): CostReceipt;
|
|
446
|
+
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
447
|
+
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
448
|
+
markCompleted(count?: number): void;
|
|
449
|
+
costPerCompletedTask(): number | null;
|
|
450
|
+
private execute;
|
|
451
|
+
private captureLateOutcome;
|
|
452
|
+
private releaseActiveCall;
|
|
453
|
+
private commitOutcome;
|
|
454
|
+
private captureFailure;
|
|
455
|
+
private commitReceipt;
|
|
456
|
+
private resolveMaximum;
|
|
457
|
+
private hasIncompleteSettledCall;
|
|
458
|
+
private appendRecord;
|
|
459
|
+
private ensureCostLimitPersisted;
|
|
460
|
+
private appendEvent;
|
|
461
|
+
}
|
|
462
|
+
/** Public callback surface for a shared cost ledger.
|
|
463
|
+
*
|
|
464
|
+
* Declaration bundles may expose this type through multiple package subpaths.
|
|
465
|
+
* Keeping callback contracts structural lets those subpaths compose while the
|
|
466
|
+
* concrete {@link CostLedger} retains its private durable state.
|
|
467
|
+
*/
|
|
468
|
+
type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
|
|
469
|
+
|
|
302
470
|
type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
|
|
303
471
|
|
|
304
472
|
/**
|
|
@@ -746,156 +914,6 @@ interface TraceAnalysisStore {
|
|
|
746
914
|
}): Promise<SearchSpanResult>;
|
|
747
915
|
}
|
|
748
916
|
|
|
749
|
-
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
750
|
-
interface CostUsage {
|
|
751
|
-
inputTokens: number;
|
|
752
|
-
outputTokens: number;
|
|
753
|
-
cachedTokens?: number;
|
|
754
|
-
}
|
|
755
|
-
interface CostCallBase {
|
|
756
|
-
callId: string;
|
|
757
|
-
channel: CostChannel;
|
|
758
|
-
phase: string;
|
|
759
|
-
actor: string;
|
|
760
|
-
model: string;
|
|
761
|
-
maximumCostUsd?: number;
|
|
762
|
-
tags?: Record<string, string>;
|
|
763
|
-
timestamp: number;
|
|
764
|
-
}
|
|
765
|
-
interface CostReceipt extends CostCallBase, CostUsage {
|
|
766
|
-
status: 'settled';
|
|
767
|
-
costUsd: number;
|
|
768
|
-
costUnknown: boolean;
|
|
769
|
-
usageUnknown?: boolean;
|
|
770
|
-
pricing?: {
|
|
771
|
-
inputUsdPerThousand: number;
|
|
772
|
-
outputUsdPerThousand: number;
|
|
773
|
-
};
|
|
774
|
-
actualCostUsd?: number;
|
|
775
|
-
error?: string;
|
|
776
|
-
}
|
|
777
|
-
interface CostReceiptInput extends CostUsage {
|
|
778
|
-
model: string;
|
|
779
|
-
actualCostUsd?: number;
|
|
780
|
-
costUnknown?: boolean;
|
|
781
|
-
usageUnknown?: boolean;
|
|
782
|
-
}
|
|
783
|
-
type MaximumCharge = {
|
|
784
|
-
externallyEnforcedMaximumUsd: number;
|
|
785
|
-
} | ({
|
|
786
|
-
model: string;
|
|
787
|
-
} & CostUsage);
|
|
788
|
-
interface RunPaidCallInput<T> {
|
|
789
|
-
callId?: string;
|
|
790
|
-
channel: CostChannel;
|
|
791
|
-
phase: string;
|
|
792
|
-
actor: string;
|
|
793
|
-
/** Used before a provider receipt exists and on failures without one. */
|
|
794
|
-
model?: string;
|
|
795
|
-
tags?: Record<string, string>;
|
|
796
|
-
signal?: AbortSignal;
|
|
797
|
-
/** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
|
|
798
|
-
maximumCharge?: MaximumCharge;
|
|
799
|
-
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
800
|
-
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
801
|
-
receipt(value: T): CostReceiptInput;
|
|
802
|
-
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
803
|
-
}
|
|
804
|
-
type PaidCallResult<T> = {
|
|
805
|
-
succeeded: true;
|
|
806
|
-
callId: string;
|
|
807
|
-
value: T;
|
|
808
|
-
receipt: CostReceipt;
|
|
809
|
-
} | {
|
|
810
|
-
succeeded: false;
|
|
811
|
-
callId?: string;
|
|
812
|
-
error: Error;
|
|
813
|
-
receipt?: CostReceipt;
|
|
814
|
-
};
|
|
815
|
-
interface ChannelRollup {
|
|
816
|
-
channel: CostChannel;
|
|
817
|
-
calls: number;
|
|
818
|
-
inputTokens: number;
|
|
819
|
-
outputTokens: number;
|
|
820
|
-
cachedTokens: number;
|
|
821
|
-
costUsd: number;
|
|
822
|
-
unpricedCalls: number;
|
|
823
|
-
unknownUsageCalls: number;
|
|
824
|
-
}
|
|
825
|
-
interface CostLedgerSummary {
|
|
826
|
-
totalCalls: number;
|
|
827
|
-
pendingCalls: number;
|
|
828
|
-
unresolvedCalls: number;
|
|
829
|
-
reservedCostUsd: number;
|
|
830
|
-
inputTokens: number;
|
|
831
|
-
outputTokens: number;
|
|
832
|
-
cachedTokens: number;
|
|
833
|
-
totalCostUsd: number;
|
|
834
|
-
byChannel: ChannelRollup[];
|
|
835
|
-
unpricedModels: string[];
|
|
836
|
-
fullyPriced: boolean;
|
|
837
|
-
usageComplete: boolean;
|
|
838
|
-
accountingComplete: boolean;
|
|
839
|
-
incompleteReasons: string[];
|
|
840
|
-
}
|
|
841
|
-
interface CostLedgerFilter {
|
|
842
|
-
channel?: CostChannel;
|
|
843
|
-
phase?: string;
|
|
844
|
-
tags?: Record<string, string>;
|
|
845
|
-
}
|
|
846
|
-
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
847
|
-
interface CostLedgerPersistence {
|
|
848
|
-
read(): {
|
|
849
|
-
revision: string;
|
|
850
|
-
events: string;
|
|
851
|
-
};
|
|
852
|
-
append(expectedRevision: string, event: string): string | undefined;
|
|
853
|
-
}
|
|
854
|
-
interface CostLedgerOptions {
|
|
855
|
-
costCeilingUsd?: number;
|
|
856
|
-
persistence?: CostLedgerPersistence;
|
|
857
|
-
/** Import already-settled receipts without admitting new paid work. */
|
|
858
|
-
receipts?: readonly CostReceipt[];
|
|
859
|
-
}
|
|
860
|
-
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
861
|
-
declare class CostLedger {
|
|
862
|
-
private readonly records;
|
|
863
|
-
private readonly activeCallIds;
|
|
864
|
-
private readonly lateCallIds;
|
|
865
|
-
private completedTasks;
|
|
866
|
-
private revision;
|
|
867
|
-
private costLimitPersisted;
|
|
868
|
-
readonly costCeilingUsd?: number;
|
|
869
|
-
private readonly persistence?;
|
|
870
|
-
constructor(input?: number | CostLedgerOptions);
|
|
871
|
-
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
872
|
-
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
873
|
-
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
874
|
-
error?: string;
|
|
875
|
-
}): CostReceipt;
|
|
876
|
-
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
877
|
-
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
878
|
-
markCompleted(count?: number): void;
|
|
879
|
-
costPerCompletedTask(): number | null;
|
|
880
|
-
private execute;
|
|
881
|
-
private captureLateOutcome;
|
|
882
|
-
private commitOutcome;
|
|
883
|
-
private captureFailure;
|
|
884
|
-
private commitReceipt;
|
|
885
|
-
private resolveMaximum;
|
|
886
|
-
private hasIncompleteSettledCall;
|
|
887
|
-
private appendRecord;
|
|
888
|
-
private ensureCostLimitPersisted;
|
|
889
|
-
private appendEvent;
|
|
890
|
-
}
|
|
891
|
-
/** Public callback surface for a shared cost ledger.
|
|
892
|
-
*
|
|
893
|
-
* Declaration bundles may expose this type through multiple package subpaths.
|
|
894
|
-
* Keeping callback contracts structural lets those subpaths compose while the
|
|
895
|
-
* concrete {@link CostLedger} retains its private durable state.
|
|
896
|
-
*/
|
|
897
|
-
type CostLedgerHandle = Pick<CostLedger, keyof CostLedger>;
|
|
898
|
-
|
|
899
917
|
/**
|
|
900
918
|
* RawProviderSink — first-class persistence for the actual HTTP-level
|
|
901
919
|
* request/response bodies of every LLM provider call.
|
|
@@ -1270,6 +1288,10 @@ interface AnalystContext {
|
|
|
1270
1288
|
deadlineMs?: number;
|
|
1271
1289
|
/** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */
|
|
1272
1290
|
budgetUsd?: number;
|
|
1291
|
+
/** Shared paid-call account when the analyst runs inside a larger campaign. */
|
|
1292
|
+
costLedger?: CostLedgerHandle;
|
|
1293
|
+
/** Attribution phase used when writing to the shared paid-call account. */
|
|
1294
|
+
costPhase?: string;
|
|
1273
1295
|
/**
|
|
1274
1296
|
* Shared chat client. Analysts that call an LLM go through this so
|
|
1275
1297
|
* the operator picks transport (sandbox-sdk | router | cli-bridge |
|
|
@@ -1287,6 +1309,18 @@ interface AnalystContext {
|
|
|
1287
1309
|
* filter. Empty / absent means no cross-run context.
|
|
1288
1310
|
*/
|
|
1289
1311
|
priorFindings?: ReadonlyArray<AnalystFinding>;
|
|
1312
|
+
/**
|
|
1313
|
+
* Findings emitted by analysts that completed earlier in this registry run.
|
|
1314
|
+
* This is separate from `priorFindings`: upstream findings are dependency
|
|
1315
|
+
* context for the current pass, while prior findings are cross-run memory.
|
|
1316
|
+
* The registry populates this only when `RegistryRunOpts.chainFindings` is on.
|
|
1317
|
+
*/
|
|
1318
|
+
upstreamFindings?: ReadonlyArray<AnalystFinding>;
|
|
1319
|
+
/**
|
|
1320
|
+
* Report metered work independently of findings. This keeps an empty finding
|
|
1321
|
+
* set from erasing token/cost telemetry. Multiple receipts are accumulated.
|
|
1322
|
+
*/
|
|
1323
|
+
recordUsage?: (receipt: AnalystUsageReceipt) => void;
|
|
1290
1324
|
/** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */
|
|
1291
1325
|
tags?: Record<string, string>;
|
|
1292
1326
|
/** Logger callback — analysts SHOULD prefer this over console.* for testability. */
|
|
@@ -1294,6 +1328,17 @@ interface AnalystContext {
|
|
|
1294
1328
|
/** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */
|
|
1295
1329
|
signal?: AbortSignal;
|
|
1296
1330
|
}
|
|
1331
|
+
/** Metered work performed by one analyst call. */
|
|
1332
|
+
interface AnalystUsageReceipt {
|
|
1333
|
+
/** Number of model-usage records observed at the provider boundary. */
|
|
1334
|
+
calls: number | null;
|
|
1335
|
+
/** Null when the provider did not return token accounting. */
|
|
1336
|
+
tokens: RunTokenUsage | null;
|
|
1337
|
+
/** Observed, estimated, or explicitly uncaptured dollar cost. */
|
|
1338
|
+
cost: RunCostProvenance;
|
|
1339
|
+
/** Known lower bound when one or more calls have uncaptured cost. */
|
|
1340
|
+
knownCostUsd?: number;
|
|
1341
|
+
}
|
|
1297
1342
|
|
|
1298
1343
|
type PolicyEditSchemaVersion = 'policy-edit/v1';
|
|
1299
1344
|
declare const POLICY_EDIT_AXES: readonly ["carrier", "representation", "budget", "sampling", "output_contract", "tool_contract", "routing", "memory", "agent_profile", "deployment_target"];
|
|
@@ -5819,7 +5864,10 @@ declare function assertPolicyEditAuthorContextBudget(value: unknown, maxChars: n
|
|
|
5819
5864
|
* validation surface independent of which Ax version is installed.
|
|
5820
5865
|
*/
|
|
5821
5866
|
|
|
5867
|
+
/** Original public schema retained for stored rows and callback contracts. */
|
|
5822
5868
|
declare const RawAnalystFindingSchema: z.ZodObject<{
|
|
5869
|
+
evidence_uri: z.ZodString;
|
|
5870
|
+
evidence_excerpt: z.ZodOptional<z.ZodString>;
|
|
5823
5871
|
severity: z.ZodEnum<{
|
|
5824
5872
|
low: "low";
|
|
5825
5873
|
high: "high";
|
|
@@ -5829,13 +5877,36 @@ declare const RawAnalystFindingSchema: z.ZodObject<{
|
|
|
5829
5877
|
}>;
|
|
5830
5878
|
claim: z.ZodString;
|
|
5831
5879
|
subject: z.ZodOptional<z.ZodString>;
|
|
5832
|
-
evidence_uri: z.ZodString;
|
|
5833
|
-
evidence_excerpt: z.ZodOptional<z.ZodString>;
|
|
5834
5880
|
confidence: z.ZodNumber;
|
|
5835
5881
|
rationale: z.ZodOptional<z.ZodString>;
|
|
5836
5882
|
recommended_action: z.ZodOptional<z.ZodString>;
|
|
5837
5883
|
}, z.core.$strict>;
|
|
5838
5884
|
type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
|
|
5885
|
+
/**
|
|
5886
|
+
* Canonical plural-evidence contract. The preprocessor accepts the original
|
|
5887
|
+
* `evidence_uri` / `evidence_excerpt` pair and normalizes it into one evidence
|
|
5888
|
+
* item so persisted rows and older model fixtures remain readable. New output
|
|
5889
|
+
* always receives the plural shape.
|
|
5890
|
+
*/
|
|
5891
|
+
declare const CanonicalRawAnalystFindingSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodObject<{
|
|
5892
|
+
evidence: z.ZodArray<z.ZodObject<{
|
|
5893
|
+
uri: z.ZodString;
|
|
5894
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
5895
|
+
}, z.core.$strict>>;
|
|
5896
|
+
severity: z.ZodEnum<{
|
|
5897
|
+
low: "low";
|
|
5898
|
+
high: "high";
|
|
5899
|
+
critical: "critical";
|
|
5900
|
+
medium: "medium";
|
|
5901
|
+
info: "info";
|
|
5902
|
+
}>;
|
|
5903
|
+
claim: z.ZodString;
|
|
5904
|
+
subject: z.ZodOptional<z.ZodString>;
|
|
5905
|
+
confidence: z.ZodNumber;
|
|
5906
|
+
rationale: z.ZodOptional<z.ZodString>;
|
|
5907
|
+
recommended_action: z.ZodOptional<z.ZodString>;
|
|
5908
|
+
}, z.core.$strict>>;
|
|
5909
|
+
type CanonicalRawAnalystFinding = z.infer<typeof CanonicalRawAnalystFindingSchema>;
|
|
5839
5910
|
|
|
5840
5911
|
/**
|
|
5841
5912
|
* Analyst-kind factory — the typed way to define trace analysts.
|
|
@@ -5843,9 +5914,9 @@ type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
|
|
|
5843
5914
|
* A "kind" is a specialized analyst whose actor prompt, tool subset,
|
|
5844
5915
|
* and Ax recursion config target one failure-mode lens (failure-mode
|
|
5845
5916
|
* classification, knowledge gap discovery, knowledge poisoning, recursive
|
|
5846
|
-
* self-improvement, ...). Kinds emit findings in the typed
|
|
5847
|
-
* shape via a JSON-array Ax output; the factory
|
|
5848
|
-
* Zod and lifts it into `AnalystFinding[]
|
|
5917
|
+
* self-improvement, ...). Kinds emit findings in the typed
|
|
5918
|
+
* `CanonicalRawAnalystFinding` shape via a JSON-array Ax output; the factory
|
|
5919
|
+
* validates each row with Zod and lifts it into `AnalystFinding[]`.
|
|
5849
5920
|
*
|
|
5850
5921
|
* Composition rules:
|
|
5851
5922
|
* - Each kind owns its actor description. No generic "answer this
|
|
@@ -5891,10 +5962,14 @@ interface TraceAnalystKindSpec {
|
|
|
5891
5962
|
maxTurns?: number;
|
|
5892
5963
|
/** Runtime char cap. Default 6000. */
|
|
5893
5964
|
maxRuntimeChars?: number;
|
|
5965
|
+
/** Maximum output tokens for every actor, responder, and recursive model call. Default 4096. */
|
|
5966
|
+
maxOutputTokens?: number;
|
|
5894
5967
|
/** Cost classification surfaced in `registry.list()` and budget enforcement. */
|
|
5895
5968
|
cost: AnalystCost;
|
|
5896
5969
|
/** Per-finding-row hook — kinds may reject / rewrite before lifting. */
|
|
5897
5970
|
postProcess?: (row: RawAnalystFinding, ctx: AnalystContext) => RawAnalystFinding | null;
|
|
5971
|
+
/** Minimum citations per finding. Default 1; rows below it are rejected. */
|
|
5972
|
+
minimumEvidenceCitations?: number;
|
|
5898
5973
|
/** Optional optimizer hook — populated when a kind wants to fit its prompt against labeled examples. */
|
|
5899
5974
|
goldens?: TraceAnalystGolden[];
|
|
5900
5975
|
}
|
|
@@ -5906,7 +5981,7 @@ interface TraceAnalystKindSpec {
|
|
|
5906
5981
|
*/
|
|
5907
5982
|
interface TraceAnalystGolden {
|
|
5908
5983
|
question: string;
|
|
5909
|
-
expected: ReadonlyArray<Omit<
|
|
5984
|
+
expected: ReadonlyArray<Omit<CanonicalRawAnalystFinding, 'confidence'>>;
|
|
5910
5985
|
}
|
|
5911
5986
|
|
|
5912
5987
|
/**
|
package/dist/campaign/index.js
CHANGED
|
@@ -66,7 +66,7 @@ import {
|
|
|
66
66
|
userStoryScoreboard,
|
|
67
67
|
validateSearchLedgerEvent,
|
|
68
68
|
verifyCodeSurface
|
|
69
|
-
} from "../chunk-
|
|
69
|
+
} from "../chunk-F6YUH3L4.js";
|
|
70
70
|
import {
|
|
71
71
|
assertCodeSurfaceIdentity,
|
|
72
72
|
buildEvidenceVector,
|
|
@@ -103,7 +103,7 @@ import {
|
|
|
103
103
|
runOptimization,
|
|
104
104
|
surfaceContentHash,
|
|
105
105
|
surfaceHash
|
|
106
|
-
} from "../chunk-
|
|
106
|
+
} from "../chunk-RLCJQ6VZ.js";
|
|
107
107
|
import {
|
|
108
108
|
SearchLedgerConflictError,
|
|
109
109
|
SearchLedgerError,
|
|
@@ -115,27 +115,27 @@ import {
|
|
|
115
115
|
resolveRunDir,
|
|
116
116
|
runCampaign,
|
|
117
117
|
tangleTracesRoot
|
|
118
|
-
} from "../chunk-
|
|
118
|
+
} from "../chunk-PAHNGS65.js";
|
|
119
119
|
import "../chunk-3YYRZDON.js";
|
|
120
120
|
import {
|
|
121
121
|
POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
|
|
122
122
|
validatePolicyEditCandidateRecord
|
|
123
|
-
} from "../chunk-
|
|
123
|
+
} from "../chunk-7A4LIMMY.js";
|
|
124
124
|
import "../chunk-ARU2PZFM.js";
|
|
125
125
|
import "../chunk-NJC7U437.js";
|
|
126
126
|
import "../chunk-PJQFMIOX.js";
|
|
127
|
-
import "../chunk-
|
|
127
|
+
import "../chunk-JHOJHHU7.js";
|
|
128
128
|
import "../chunk-VI2UW6B6.js";
|
|
129
|
-
import "../chunk-
|
|
129
|
+
import "../chunk-FIUFRXP3.js";
|
|
130
130
|
import "../chunk-GGE4NNQT.js";
|
|
131
|
-
import "../chunk-
|
|
131
|
+
import "../chunk-6QIM2EAP.js";
|
|
132
132
|
import "../chunk-PC4UYEBM.js";
|
|
133
133
|
import "../chunk-S3UZOQ5Y.js";
|
|
134
134
|
import "../chunk-MA6HLL3S.js";
|
|
135
135
|
import "../chunk-XJYR7XFV.js";
|
|
136
136
|
import "../chunk-VSMTAMNK.js";
|
|
137
137
|
import "../chunk-ONWEPEDO.js";
|
|
138
|
-
import "../chunk-
|
|
138
|
+
import "../chunk-K4DBDHLK.js";
|
|
139
139
|
import "../chunk-PZ5AY32C.js";
|
|
140
140
|
export {
|
|
141
141
|
DEFAULT_POLICY_EDIT_HISTORY_LIMITS,
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
} from "./chunk-NJC7U437.js";
|
|
7
7
|
import {
|
|
8
8
|
CostLedger
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-JHOJHHU7.js";
|
|
10
10
|
|
|
11
11
|
// src/wire/schemas.ts
|
|
12
12
|
import { extendZodWithOpenApi } from "@asteasolutions/zod-to-openapi";
|
|
@@ -1027,4 +1027,4 @@ export {
|
|
|
1027
1027
|
startServer,
|
|
1028
1028
|
startServerAsync
|
|
1029
1029
|
};
|
|
1030
|
-
//# sourceMappingURL=chunk-
|
|
1030
|
+
//# sourceMappingURL=chunk-4I2E3LLO.js.map
|
|
@@ -3,26 +3,16 @@ import {
|
|
|
3
3
|
} from "./chunk-ONWEPEDO.js";
|
|
4
4
|
import {
|
|
5
5
|
INPUT_VALUE,
|
|
6
|
-
|
|
7
|
-
LLM_CACHE_WRITE_TOKENS,
|
|
8
|
-
LLM_COST_USD,
|
|
9
|
-
LLM_INPUT_TOKENS,
|
|
10
|
-
LLM_MODEL_NAME,
|
|
11
|
-
LLM_OUTPUT_TOKENS,
|
|
12
|
-
LLM_REASONING_TOKENS,
|
|
13
|
-
OPENINFERENCE_SPAN_KIND,
|
|
6
|
+
LLM_MODEL_ATTR_KEYS,
|
|
14
7
|
OUTPUT_VALUE,
|
|
8
|
+
SPAN_KIND_ATTR_KEYS,
|
|
15
9
|
TOOL_ARGS_CAPTURED,
|
|
16
10
|
TOOL_LATENCY_MS,
|
|
17
|
-
TOOL_NAME
|
|
18
|
-
|
|
11
|
+
TOOL_NAME,
|
|
12
|
+
TOOL_NAME_ATTR_KEYS
|
|
13
|
+
} from "./chunk-K4DBDHLK.js";
|
|
19
14
|
|
|
20
15
|
// src/trace/otlp-attributes.ts
|
|
21
|
-
import {
|
|
22
|
-
GEN_AI_INPUT_TOKEN_KEYS,
|
|
23
|
-
GEN_AI_MODEL_KEYS,
|
|
24
|
-
GEN_AI_OUTPUT_TOKEN_KEYS
|
|
25
|
-
} from "@tangle-network/agent-core/telemetry";
|
|
26
16
|
var TOOL_SPAN_ATTRIBUTE_KEYS = [
|
|
27
17
|
TOOL_NAME,
|
|
28
18
|
TOOL_ARGS_CAPTURED,
|
|
@@ -30,79 +20,6 @@ var TOOL_SPAN_ATTRIBUTE_KEYS = [
|
|
|
30
20
|
INPUT_VALUE,
|
|
31
21
|
OUTPUT_VALUE
|
|
32
22
|
];
|
|
33
|
-
var SPAN_KIND_ATTR_KEYS = [OPENINFERENCE_SPAN_KIND, "inference.observation_kind"];
|
|
34
|
-
var LLM_MODEL_ATTR_KEYS = [
|
|
35
|
-
LLM_MODEL_NAME,
|
|
36
|
-
"inference.llm.model_name",
|
|
37
|
-
"llm.model",
|
|
38
|
-
...GEN_AI_MODEL_KEYS,
|
|
39
|
-
"tangle.model"
|
|
40
|
-
];
|
|
41
|
-
var LLM_INPUT_TOKEN_ATTR_KEYS = [
|
|
42
|
-
LLM_INPUT_TOKENS,
|
|
43
|
-
"inference.llm.input_tokens",
|
|
44
|
-
"llm.input_tokens",
|
|
45
|
-
...GEN_AI_INPUT_TOKEN_KEYS,
|
|
46
|
-
"tangle.tokens.in",
|
|
47
|
-
"tokens.in"
|
|
48
|
-
];
|
|
49
|
-
var LLM_OUTPUT_TOKEN_ATTR_KEYS = [
|
|
50
|
-
LLM_OUTPUT_TOKENS,
|
|
51
|
-
"inference.llm.output_tokens",
|
|
52
|
-
"llm.output_tokens",
|
|
53
|
-
...GEN_AI_OUTPUT_TOKEN_KEYS,
|
|
54
|
-
"tangle.tokens.out",
|
|
55
|
-
"tokens.out"
|
|
56
|
-
];
|
|
57
|
-
var LLM_REASONING_TOKEN_ATTR_KEYS = [
|
|
58
|
-
LLM_REASONING_TOKENS,
|
|
59
|
-
"inference.llm.reasoning_tokens",
|
|
60
|
-
"llm.reasoning_tokens",
|
|
61
|
-
"gen_ai.usage.reasoning_tokens",
|
|
62
|
-
"gen_ai.usage.reasoning_output_tokens",
|
|
63
|
-
"reasoning_tokens",
|
|
64
|
-
"reasoning_output_tokens",
|
|
65
|
-
"tangle.tokens.reasoning",
|
|
66
|
-
"gen_ai.usage.output_tokens_details.reasoning_tokens",
|
|
67
|
-
"gen_ai.usage.completion_tokens_details.reasoning_tokens"
|
|
68
|
-
];
|
|
69
|
-
var LLM_CACHED_TOKEN_ATTR_KEYS = [
|
|
70
|
-
LLM_CACHED_TOKENS,
|
|
71
|
-
"inference.llm.cached_tokens",
|
|
72
|
-
"llm.cached_tokens",
|
|
73
|
-
"gen_ai.usage.cached_tokens",
|
|
74
|
-
"gen_ai.usage.prompt_tokens_details.cached_tokens",
|
|
75
|
-
"gen_ai.usage.input_tokens_details.cached_tokens",
|
|
76
|
-
"gen_ai.usage.cache_read_tokens",
|
|
77
|
-
"gen_ai.usage.cache_read_input_tokens",
|
|
78
|
-
"cache_read_tokens",
|
|
79
|
-
"cache_read_input_tokens",
|
|
80
|
-
"input_cache_read",
|
|
81
|
-
"tangle.tokens.cached"
|
|
82
|
-
];
|
|
83
|
-
var LLM_CACHE_WRITE_TOKEN_ATTR_KEYS = [
|
|
84
|
-
LLM_CACHE_WRITE_TOKENS,
|
|
85
|
-
"inference.llm.cache_write_tokens",
|
|
86
|
-
"llm.cache_write_tokens",
|
|
87
|
-
"gen_ai.usage.cache_creation_tokens",
|
|
88
|
-
"gen_ai.usage.cache_creation_input_tokens",
|
|
89
|
-
"cache_creation_tokens",
|
|
90
|
-
"cache_creation_input_tokens",
|
|
91
|
-
"input_cache_creation",
|
|
92
|
-
"tangle.tokens.cache_write"
|
|
93
|
-
];
|
|
94
|
-
var LLM_COST_ATTR_KEYS = [
|
|
95
|
-
LLM_COST_USD,
|
|
96
|
-
"inference.llm.cost.total",
|
|
97
|
-
"llm.cost.total",
|
|
98
|
-
"gen_ai.usage.cost",
|
|
99
|
-
"gen_ai.usage.cost_usd",
|
|
100
|
-
"tangle.cost.usd",
|
|
101
|
-
"cost.usd",
|
|
102
|
-
"cost"
|
|
103
|
-
];
|
|
104
|
-
var RUN_COST_ATTR_KEYS = ["tangle.cost.usd", "cost.usd"];
|
|
105
|
-
var TOOL_NAME_ATTR_KEYS = [TOOL_NAME, "inference.tool.name"];
|
|
106
23
|
function toolSpanOtlpAttributes(span) {
|
|
107
24
|
const argsCaptured = span.argsCaptured !== false;
|
|
108
25
|
const attributes = {
|
|
@@ -239,21 +156,6 @@ function stringField(raw, key) {
|
|
|
239
156
|
function asString(v) {
|
|
240
157
|
return typeof v === "string" && v.length > 0 ? v : null;
|
|
241
158
|
}
|
|
242
|
-
function asNumber(v) {
|
|
243
|
-
if (typeof v === "number") return Number.isFinite(v) ? v : null;
|
|
244
|
-
if (typeof v === "string" && v.length > 0) {
|
|
245
|
-
const n = Number(v);
|
|
246
|
-
return Number.isFinite(n) ? n : null;
|
|
247
|
-
}
|
|
248
|
-
return null;
|
|
249
|
-
}
|
|
250
|
-
function firstNumberAttr(attrs, keys) {
|
|
251
|
-
for (const k of keys) {
|
|
252
|
-
const n = asNumber(attrs[k]);
|
|
253
|
-
if (n !== null) return n;
|
|
254
|
-
}
|
|
255
|
-
return null;
|
|
256
|
-
}
|
|
257
159
|
function firstStringAttr(attrs, keys) {
|
|
258
160
|
for (const k of keys) {
|
|
259
161
|
const s = asString(attrs[k]);
|
|
@@ -1084,16 +986,6 @@ function assertStringArray(v, label) {
|
|
|
1084
986
|
}
|
|
1085
987
|
|
|
1086
988
|
export {
|
|
1087
|
-
SPAN_KIND_ATTR_KEYS,
|
|
1088
|
-
LLM_MODEL_ATTR_KEYS,
|
|
1089
|
-
LLM_INPUT_TOKEN_ATTR_KEYS,
|
|
1090
|
-
LLM_OUTPUT_TOKEN_ATTR_KEYS,
|
|
1091
|
-
LLM_REASONING_TOKEN_ATTR_KEYS,
|
|
1092
|
-
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
1093
|
-
LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
|
|
1094
|
-
LLM_COST_ATTR_KEYS,
|
|
1095
|
-
RUN_COST_ATTR_KEYS,
|
|
1096
|
-
TOOL_NAME_ATTR_KEYS,
|
|
1097
989
|
applyToolSpanOtlpAttributes,
|
|
1098
990
|
traceSpanKindToOpenInferenceKind,
|
|
1099
991
|
projectOtlpFlatLine,
|
|
@@ -1102,8 +994,6 @@ export {
|
|
|
1102
994
|
extractOtlpAttributes,
|
|
1103
995
|
stringField,
|
|
1104
996
|
asString,
|
|
1105
|
-
asNumber,
|
|
1106
|
-
firstNumberAttr,
|
|
1107
997
|
firstStringAttr,
|
|
1108
998
|
spanEpochMillis,
|
|
1109
999
|
compareSpanTime,
|
|
@@ -1116,4 +1006,4 @@ export {
|
|
|
1116
1006
|
buildTraceAnalystTools,
|
|
1117
1007
|
traceAnalystFunctionGroup
|
|
1118
1008
|
};
|
|
1119
|
-
//# sourceMappingURL=chunk-
|
|
1009
|
+
//# sourceMappingURL=chunk-6QIM2EAP.js.map
|