@tangle-network/agent-eval 0.118.2 → 0.119.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/README.md +26 -0
  3. package/dist/analyst/index.d.ts +143 -21
  4. package/dist/analyst/index.js +46 -12
  5. package/dist/analyst/index.js.map +1 -1
  6. package/dist/benchmarks/index.d.ts +50 -40
  7. package/dist/benchmarks/index.js +9 -9
  8. package/dist/campaign/index.d.ts +231 -156
  9. package/dist/campaign/index.js +8 -8
  10. package/dist/{chunk-JTMFT5QZ.js → chunk-4I2E3LLO.js} +2 -2
  11. package/dist/{chunk-BUBU3YHT.js → chunk-6QIM2EAP.js} +6 -116
  12. package/dist/chunk-6QIM2EAP.js.map +1 -0
  13. package/dist/{chunk-DZBCDDIE.js → chunk-7A4LIMMY.js} +846 -253
  14. package/dist/chunk-7A4LIMMY.js.map +1 -0
  15. package/dist/{chunk-KK2O4VWA.js → chunk-D7AEXSM5.js} +2 -2
  16. package/dist/{chunk-GCT3DQWU.js → chunk-F6YUH3L4.js} +41 -30
  17. package/dist/{chunk-GCT3DQWU.js.map → chunk-F6YUH3L4.js.map} +1 -1
  18. package/dist/{chunk-63KG3ASG.js → chunk-FIUFRXP3.js} +2 -2
  19. package/dist/{chunk-J23QK2U5.js → chunk-GERDAIAL.js} +8 -8
  20. package/dist/{chunk-J23QK2U5.js.map → chunk-GERDAIAL.js.map} +1 -1
  21. package/dist/{chunk-TVXPJJ7H.js → chunk-H5UD2323.js} +2 -2
  22. package/dist/{chunk-MLHRFWQK.js → chunk-JHOJHHU7.js} +85 -16
  23. package/dist/chunk-JHOJHHU7.js.map +1 -0
  24. package/dist/chunk-K4DBDHLK.js +158 -0
  25. package/dist/chunk-K4DBDHLK.js.map +1 -0
  26. package/dist/{chunk-DRSFTBKU.js → chunk-LMJ2TGWJ.js} +3 -15
  27. package/dist/chunk-LMJ2TGWJ.js.map +1 -0
  28. package/dist/{chunk-EGNRE7VA.js → chunk-PAHNGS65.js} +2 -2
  29. package/dist/{chunk-I42AAGHL.js → chunk-RLCJQ6VZ.js} +5 -5
  30. package/dist/{chunk-7V3QDWNL.js → chunk-XJ7JVCHB.js} +2 -2
  31. package/dist/{chunk-6JQE3YVE.js → chunk-ZYHJNKI3.js} +6 -6
  32. package/dist/cli.js +2 -2
  33. package/dist/contract/index.d.ts +257 -160
  34. package/dist/contract/index.js +11 -11
  35. package/dist/fuzz.d.ts +19 -1
  36. package/dist/fuzz.js +1 -1
  37. package/dist/index.d.ts +170 -64
  38. package/dist/index.js +30 -28
  39. package/dist/index.js.map +1 -1
  40. package/dist/openapi.json +1 -1
  41. package/dist/primeintellect/index.d.ts +311 -0
  42. package/dist/primeintellect/index.js +348 -0
  43. package/dist/primeintellect/index.js.map +1 -0
  44. package/dist/rl.d.ts +4 -0
  45. package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js} +3 -3
  46. package/dist/trace-attributes.d.ts +17 -1
  47. package/dist/trace-attributes.js +27 -3
  48. package/dist/traces.d.ts +16 -16
  49. package/dist/traces.js +18 -18
  50. package/dist/wire/index.d.ts +19 -1
  51. package/dist/wire/index.js +2 -2
  52. package/package.json +6 -1
  53. package/dist/chunk-BUBU3YHT.js.map +0 -1
  54. package/dist/chunk-DRSFTBKU.js.map +0 -1
  55. package/dist/chunk-DZBCDDIE.js.map +0 -1
  56. package/dist/chunk-MLHRFWQK.js.map +0 -1
  57. package/dist/chunk-NW4AQBXD.js +0 -53
  58. package/dist/chunk-NW4AQBXD.js.map +0 -1
  59. /package/dist/{chunk-JTMFT5QZ.js.map → chunk-4I2E3LLO.js.map} +0 -0
  60. /package/dist/{chunk-KK2O4VWA.js.map → chunk-D7AEXSM5.js.map} +0 -0
  61. /package/dist/{chunk-63KG3ASG.js.map → chunk-FIUFRXP3.js.map} +0 -0
  62. /package/dist/{chunk-TVXPJJ7H.js.map → chunk-H5UD2323.js.map} +0 -0
  63. /package/dist/{chunk-EGNRE7VA.js.map → chunk-PAHNGS65.js.map} +0 -0
  64. /package/dist/{chunk-I42AAGHL.js.map → chunk-RLCJQ6VZ.js.map} +0 -0
  65. /package/dist/{chunk-7V3QDWNL.js.map → chunk-XJ7JVCHB.js.map} +0 -0
  66. /package/dist/{chunk-6JQE3YVE.js.map → chunk-ZYHJNKI3.js.map} +0 -0
  67. /package/dist/{run-campaign-DWC67KJP.js.map → run-campaign-OBXSN5WK.js.map} +0 -0
@@ -6,11 +6,11 @@ import {
6
6
  } from "./chunk-NJC7U437.js";
7
7
  import {
8
8
  CostLedger
9
- } from "./chunk-MLHRFWQK.js";
9
+ } from "./chunk-JHOJHHU7.js";
10
10
  import {
11
11
  TraceFileMissingError,
12
12
  buildTraceAnalystTools
13
- } from "./chunk-BUBU3YHT.js";
13
+ } from "./chunk-6QIM2EAP.js";
14
14
  import {
15
15
  validateAgentProfileCell
16
16
  } from "./chunk-XJYR7XFV.js";
@@ -21,6 +21,39 @@ import {
21
21
  ValidationError
22
22
  } from "./chunk-ONWEPEDO.js";
23
23
 
24
+ // src/analyst/ax-service.ts
25
+ import { ai } from "@ax-llm/ax";
26
+ var configuredModels = /* @__PURE__ */ new WeakMap();
27
+ function createAnalystAi(config) {
28
+ const model = config.model.trim();
29
+ if (!model) throw new TypeError("createAnalystAi: model must be a non-empty string");
30
+ const service = ai({
31
+ name: config.provider ?? "openai",
32
+ apiKey: config.apiKey,
33
+ apiURL: config.baseUrl,
34
+ config: { model }
35
+ });
36
+ configuredModels.set(service, model);
37
+ return service;
38
+ }
39
+ function getConfiguredAnalystModel(service) {
40
+ return configuredModels.get(service);
41
+ }
42
+ function resolveAnalystModel(service, override) {
43
+ if (override !== void 0) {
44
+ const model2 = override.trim();
45
+ if (!model2) throw new TypeError("createTraceAnalystKind: model must be a non-empty string");
46
+ return model2;
47
+ }
48
+ const model = getConfiguredAnalystModel(service)?.trim();
49
+ if (!model) {
50
+ throw new TypeError(
51
+ "createTraceAnalystKind: model is required for Ax services not created by createAnalystAi()"
52
+ );
53
+ }
54
+ return model;
55
+ }
56
+
24
57
  // src/analyst/types.ts
25
58
  import { createHash } from "crypto";
26
59
  function computeFindingId(input) {
@@ -345,47 +378,63 @@ function coerceToFindingRows(raw) {
345
378
  // src/analyst/finding-signature.ts
346
379
  import { z as z2 } from "zod";
347
380
  var ANALYST_SEVERITIES = ["critical", "high", "medium", "low", "info"];
348
- var RawAnalystFindingSchema = z2.object({
381
+ var RawAnalystEvidenceSchema = z2.object({
382
+ uri: z2.string().trim().min(1).max(2e3),
383
+ excerpt: z2.string().max(2e3).optional()
384
+ }).strict();
385
+ var RawAnalystFindingBaseShape = {
349
386
  severity: z2.enum(ANALYST_SEVERITIES),
350
387
  claim: z2.string().min(1).max(2e3),
351
- /**
352
- * Subject locus the finding is about. Validated at parse time
353
- * against the documented grammar (`finding-subject.ts`). Findings
354
- * with a malformed subject are rejected — they would have been
355
- * silently skipped by every downstream adapter, so failing loud at
356
- * parse time turns a hidden no-op into a kind-prompt audit signal.
357
- *
358
- * Optional because purely descriptive findings (no actionable
359
- * locus) are legitimate; they just don't route through the
360
- * KnowledgeAdapter / ImprovementAdapter.
361
- */
362
- subject: z2.string().max(400).refine((s) => parseFindingSubject(s) !== null, {
388
+ subject: z2.string().max(400).refine((subject) => parseFindingSubject(subject) !== null, {
363
389
  message: "subject does not match the finding-subject grammar"
364
390
  }).optional(),
365
- evidence_uri: z2.string().min(1).max(2e3),
366
- evidence_excerpt: z2.string().max(2e3).optional(),
367
391
  confidence: z2.number().min(0).max(1),
368
392
  rationale: z2.string().max(4e3).optional(),
369
393
  recommended_action: z2.string().max(2e3).optional()
394
+ };
395
+ var RawAnalystFindingSchema = z2.object({
396
+ ...RawAnalystFindingBaseShape,
397
+ evidence_uri: z2.string().min(1).max(2e3).refine((uri) => uri.trim().length > 0, { message: "evidence_uri must not be blank" }),
398
+ evidence_excerpt: z2.string().max(2e3).optional()
399
+ }).strict();
400
+ var CanonicalRawAnalystFindingObjectSchema = z2.object({
401
+ ...RawAnalystFindingBaseShape,
402
+ evidence: z2.array(RawAnalystEvidenceSchema).min(1)
370
403
  }).strict();
371
- var RAW_FINDING_SCHEMA_PROMPT = `Each finding MUST be a JSON object with these fields:
372
- - severity: one of "critical" | "high" | "medium" | "low" | "info"
404
+ var CanonicalRawAnalystFindingSchema = z2.preprocess(
405
+ normalizeLegacySingleCitation,
406
+ CanonicalRawAnalystFindingObjectSchema
407
+ );
408
+ var RAW_FINDING_SCHEMA_PROMPT = `Each finding MUST be a strict JSON object with:
409
+ - severity: "critical" | "high" | "medium" | "low" | "info"
373
410
  - claim: one-sentence statement (max 2000 chars)
374
- - subject?: the routing locus this finding is about. It MUST be one of the exact subject forms listed in this kind's instructions above (e.g. \`system-prompt:<section>\`, \`agent-knowledge:wiki:<slug>\`, \`tool-doc:<tool>\`). A free phrase, a bare noun, or any form not in that list is REJECTED at parse time and the finding is discarded \u2014 omit subject entirely rather than guess a form.
375
- - evidence_uri: REQUIRED, never blank. Exactly one of "span://<trace_id>/<span_id>" (trace evidence), "artifact://<relative-path>" (files), "metric://<name>" (named scalars) \u2014 ALWAYS cite a real id surfaced by the tools. If you have no citable id, do not emit the finding.
376
- - evidence_excerpt?: short quote (<=2000 chars) from the cited span/artifact
377
- - confidence: number 0..1 \u2014 0.9+ when backed by exact quotes, 0.6-0.8 for inferred patterns, <0.5 for speculative
378
- - rationale?: one or two sentences explaining the reasoning
379
- - recommended_action?: concrete change phrased as an imperative ("Add ...", "Replace ...", "Stop ...") \u2014 omit when the finding is purely descriptive
411
+ - subject?: one exact subject form listed by this kind; omit rather than guess
412
+ - evidence: REQUIRED non-empty array of {"uri": string, "excerpt"?: string}. Use real identifiers with span://, event://, artifact://, metric://, or finding://. Include a short exact quote in excerpt when available. If nothing is citable, do not emit the finding.
413
+ - confidence: number 0..1 (0.9+ exact evidence; 0.6-0.8 inferred pattern; <0.5 speculative)
414
+ - rationale?: one or two reasoning sentences
415
+ - recommended_action?: concrete imperative change; omit for descriptive findings
380
416
 
381
- Emit an empty array when the question has no findings to report. Do not fabricate evidence.`;
417
+ Unknown fields are rejected. Do not emit area; the factory assigns it. Emit [] when there are no findings. Never fabricate evidence.`;
418
+ function evidenceRefsFromRawFinding(finding) {
419
+ return finding.evidence.map(({ uri, excerpt }) => ({
420
+ kind: evidenceKindFromUri(uri),
421
+ uri,
422
+ excerpt
423
+ }));
424
+ }
382
425
  function parseRawFinding(row, log) {
383
- const result = RawAnalystFindingSchema.safeParse(row);
426
+ return parseFindingWithSchema(RawAnalystFindingSchema, row, log);
427
+ }
428
+ function parseCanonicalRawFinding(row, log) {
429
+ return parseFindingWithSchema(CanonicalRawAnalystFindingSchema, row, log);
430
+ }
431
+ function parseFindingWithSchema(schema, row, log) {
432
+ const result = schema.safeParse(row);
384
433
  if (result.success) return result.data;
385
434
  if (typeof row === "string") {
386
435
  const coerced = coerceJson(row);
387
436
  if (coerced !== void 0) {
388
- const retry = RawAnalystFindingSchema.safeParse(coerced);
437
+ const retry = schema.safeParse(coerced);
389
438
  if (retry.success) return retry.data;
390
439
  }
391
440
  }
@@ -398,40 +447,98 @@ function parseRawFinding(row, log) {
398
447
  });
399
448
  return null;
400
449
  }
450
+ function normalizeLegacySingleCitation(value) {
451
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return value;
452
+ const row = value;
453
+ if ("evidence" in row) {
454
+ if (!("evidence_uri" in row) || !Array.isArray(row.evidence)) return value;
455
+ const { evidence, evidence_uri: uri2, evidence_excerpt: excerpt2, ...rest2 } = row;
456
+ let alreadyPresent = false;
457
+ const mergedEvidence = evidence.map((citation) => {
458
+ if (citation === null || typeof citation !== "object" || Array.isArray(citation) || citation.uri !== uri2) {
459
+ return citation;
460
+ }
461
+ alreadyPresent = true;
462
+ const current = citation;
463
+ return excerpt2 === void 0 || current.excerpt !== void 0 ? citation : { ...current, excerpt: excerpt2 };
464
+ });
465
+ return {
466
+ ...rest2,
467
+ evidence: alreadyPresent ? mergedEvidence : [...evidence, { uri: uri2, ...excerpt2 === void 0 ? {} : { excerpt: excerpt2 } }]
468
+ };
469
+ }
470
+ if (!("evidence_uri" in row)) return value;
471
+ const { evidence_uri: uri, evidence_excerpt: excerpt, ...rest } = row;
472
+ return {
473
+ ...rest,
474
+ evidence: [{ uri, ...excerpt === void 0 ? {} : { excerpt } }]
475
+ };
476
+ }
477
+ function toLegacyRawAnalystFinding(finding) {
478
+ const primaryEvidence = finding.evidence[0];
479
+ if (!primaryEvidence) {
480
+ throw new TypeError("Canonical raw analyst findings require at least one evidence citation");
481
+ }
482
+ const { evidence: _evidence, ...rest } = finding;
483
+ return {
484
+ ...rest,
485
+ evidence_uri: primaryEvidence.uri,
486
+ ...primaryEvidence.excerpt === void 0 ? {} : { evidence_excerpt: primaryEvidence.excerpt }
487
+ };
488
+ }
489
+ function applyLegacyRawFindingCallback(finding, callback, log) {
490
+ const callbackResult = callback(toLegacyRawAnalystFinding(finding));
491
+ if (callbackResult === null) return null;
492
+ const parsed = parseCanonicalRawFinding(callbackResult, log);
493
+ if (!parsed) return null;
494
+ const primaryEvidence = parsed.evidence[0];
495
+ if (!primaryEvidence) {
496
+ throw new TypeError("Canonical raw analyst findings require at least one evidence citation");
497
+ }
498
+ const remainingEvidence = finding.evidence.slice(1).filter((citation) => citation.uri !== primaryEvidence.uri);
499
+ return {
500
+ ...parsed,
501
+ evidence: [primaryEvidence, ...remainingEvidence]
502
+ };
503
+ }
504
+ function evidenceKindFromUri(uri) {
505
+ if (uri.startsWith("span://")) return "span";
506
+ if (uri.startsWith("event://")) return "event";
507
+ if (uri.startsWith("finding://")) return "finding";
508
+ if (uri.startsWith("metric://")) return "metric";
509
+ return "artifact";
510
+ }
401
511
 
402
512
  // src/analyst/structure-findings.ts
403
513
  var SYSTEM = [
404
514
  "You convert a free-form trace-analysis report into a STRICT JSON array of findings.",
405
515
  "Output ONLY the JSON array \u2014 no prose, no code fences.",
406
- 'Each element: {"severity":"critical|high|medium|low|info","claim":string,"evidence_uri":string,',
407
- '"subject"?:string,"rationale"?:string,"recommended_action"?:string,"confidence":number(0..1)}.',
408
- 'evidence_uri cites the trace element the report referenced (e.g. "span://<trace>/<span>") or "report://summary".',
516
+ RAW_FINDING_SCHEMA_PROMPT,
517
+ "Omit subject when the report does not contain an exact valid locus.",
409
518
  "If the report asserts NO problems, output exactly []."
410
519
  ].join(" ");
411
- function buildRows(raw, analystId, area) {
520
+ function buildRows(raw, opts) {
412
521
  const rows = coerceToFindingRows(raw);
413
522
  const out = [];
414
523
  for (const row of rows) {
415
- const normalized = row && typeof row === "object" && !Array.isArray(row) && !row.evidence_uri ? { ...row, evidence_uri: "report://summary" } : row;
416
- const parsed = parseRawFinding(normalized);
524
+ const parsed = parseCanonicalRawFinding(row);
417
525
  if (!parsed) continue;
526
+ const callbackProcessed = opts.processRow ? applyLegacyRawFindingCallback(parsed, opts.processRow) : parsed;
527
+ if (!callbackProcessed) continue;
528
+ const processed = opts.processCanonicalRow ? opts.processCanonicalRow(callbackProcessed) : callbackProcessed;
529
+ if (!processed) continue;
418
530
  out.push(
419
531
  makeFinding({
420
- analyst_id: analystId,
421
- area,
422
- subject: parsed.subject,
423
- claim: parsed.claim,
424
- rationale: parsed.rationale,
425
- severity: parsed.severity,
426
- confidence: parsed.confidence,
427
- evidence_refs: [
428
- {
429
- kind: parsed.evidence_uri.startsWith("span://") ? "span" : "artifact",
430
- uri: parsed.evidence_uri,
431
- excerpt: parsed.evidence_excerpt
432
- }
433
- ],
434
- recommended_action: parsed.recommended_action
532
+ analyst_id: opts.analystId,
533
+ area: opts.area,
534
+ subject: processed.subject,
535
+ claim: processed.claim,
536
+ rationale: processed.rationale,
537
+ severity: processed.severity,
538
+ confidence: processed.confidence,
539
+ evidence_refs: evidenceRefsFromRawFinding(processed),
540
+ recommended_action: processed.recommended_action,
541
+ ...opts.findingMetadata ? { metadata: { ...opts.findingMetadata } } : {}
435
542
  })
436
543
  );
437
544
  }
@@ -439,6 +546,9 @@ function buildRows(raw, analystId, area) {
439
546
  }
440
547
  async function structureFindings(opts) {
441
548
  const maxReasks = opts.maxReasks ?? 1;
549
+ if (!Number.isSafeInteger(maxReasks) || maxReasks < 0) {
550
+ throw new RangeError("structureFindings: maxReasks must be a non-negative safe integer");
551
+ }
442
552
  const llm = { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl };
443
553
  const costLedger = opts.costLedger ?? new CostLedger();
444
554
  let user = `TRACE-ANALYSIS REPORT:
@@ -459,8 +569,9 @@ Return the findings JSON array.`;
459
569
  phase: opts.costPhase ?? "analyst.structure-findings",
460
570
  actor: "structure-findings",
461
571
  model: opts.model,
572
+ signal: opts.signal,
462
573
  maximumCharge: maximumChargeForLlmRequest(request, llm),
463
- tags: { analystId: opts.analystId, attempt: String(attempt) },
574
+ tags: { ...opts.costTags, analystId: opts.analystId, attempt: String(attempt) },
464
575
  execute: (signal, callId) => callLlm(request, { ...llm, signal, idempotencyKey: callId }),
465
576
  receipt: costReceiptFromLlm,
466
577
  receiptFromError: costReceiptFromLlmError
@@ -468,7 +579,7 @@ Return the findings JSON array.`;
468
579
  if (!paid.succeeded) throw paid.error;
469
580
  const res = paid.value;
470
581
  const text = res.content.trim();
471
- const findings = buildRows(text, opts.analystId, opts.area);
582
+ const findings = buildRows(text, opts);
472
583
  if (findings.length > 0) return { findings, outcome: "ok" };
473
584
  if (opts.report.trim().length < 200) return { findings: [], outcome: "ok" };
474
585
  user = `${user}
@@ -480,8 +591,249 @@ That produced no valid findings. The report DOES describe issues \u2014 re-extra
480
591
 
481
592
  // src/analyst/kind-factory.ts
482
593
  import { AxJSRuntime, agent } from "@ax-llm/ax";
594
+
595
+ // src/analyst/ax-cost-service.ts
596
+ function meterAxChatService(ai2, options) {
597
+ assertPositiveInteger(options.maxOutputTokens, "maxOutputTokens");
598
+ const source = ai2;
599
+ if (typeof source.chat !== "function") {
600
+ throw new TypeError("meterAxChatService: Ax service must implement chat()");
601
+ }
602
+ const providerChat = source.chat.bind(ai2);
603
+ const chat = async (request, callOptions = {}) => {
604
+ const boundedRequest = boundOutputTokens(request, options.maxOutputTokens);
605
+ const model = modelName(boundedRequest.model) || options.defaultModel || "";
606
+ const canTurnOffThinking = canDisableThinking(ai2, model);
607
+ const combined = combineSignals(options.signal, callOptions.abortSignal);
608
+ try {
609
+ const paid = await options.ledger.runPaidCall({
610
+ channel: "analyst",
611
+ phase: options.phase ?? "analyst.ax.chat",
612
+ actor: options.actor,
613
+ model,
614
+ tags: options.tags,
615
+ signal: combined.signal,
616
+ maximumCharge: maximumChargeForAxChatRequest(boundedRequest, model),
617
+ execute: async (executionSignal) => {
618
+ const providerOptions = {
619
+ ...callOptions,
620
+ abortSignal: executionSignal,
621
+ retry: { ...callOptions.retry, maxRetries: 0 },
622
+ stream: false,
623
+ showThoughts: false
624
+ };
625
+ if (canTurnOffThinking) providerOptions.thinkingTokenBudget = "none";
626
+ else Reflect.deleteProperty(providerOptions, "thinkingTokenBudget");
627
+ const response = await providerChat(boundedRequest, providerOptions);
628
+ if (response instanceof ReadableStream) {
629
+ throw new Error("meterAxChatService: provider returned a stream after stream:false");
630
+ }
631
+ return response;
632
+ },
633
+ receipt: (response) => costReceiptFromAxResponse(response, model)
634
+ });
635
+ if (!paid.succeeded) throw paid.error;
636
+ return paid.value;
637
+ } finally {
638
+ combined.dispose();
639
+ }
640
+ };
641
+ return new Proxy(ai2, {
642
+ get(target, property) {
643
+ if (property === "chat") return chat;
644
+ const value = Reflect.get(target, property, target);
645
+ return typeof value === "function" ? value.bind(target) : value;
646
+ }
647
+ });
648
+ }
649
+ function maximumChargeForAxChatRequest(request, defaultModel) {
650
+ const model = modelName(request.model) || defaultModel || "";
651
+ const maxTokens = request.modelConfig?.maxTokens;
652
+ const completions = request.modelConfig?.n;
653
+ if (!model || maxTokens === void 0 || completions === void 0) return void 0;
654
+ assertPositiveInteger(maxTokens, "request.modelConfig.maxTokens");
655
+ assertPositiveInteger(completions, "request.modelConfig.n");
656
+ const maximumOutputTokens = maxTokens * completions;
657
+ assertPositiveInteger(maximumOutputTokens, "maximum output tokens");
658
+ if (containsUnboundedOrCacheableContent(request)) return void 0;
659
+ let inputTokens;
660
+ try {
661
+ const pricedRequest = request.model === void 0 ? { ...request, model } : request;
662
+ inputTokens = new TextEncoder().encode(JSON.stringify(pricedRequest)).byteLength * completions;
663
+ } catch {
664
+ return void 0;
665
+ }
666
+ assertPositiveInteger(inputTokens, "maximum input tokens");
667
+ return { model, inputTokens, outputTokens: maximumOutputTokens };
668
+ }
669
+ function boundOutputTokens(request, limit) {
670
+ const requested = request.modelConfig?.maxTokens;
671
+ if (requested !== void 0) {
672
+ assertPositiveInteger(requested, "request.modelConfig.maxTokens");
673
+ }
674
+ const completions = request.modelConfig?.n ?? 1;
675
+ assertPositiveInteger(completions, "request.modelConfig.n");
676
+ const maxTokens = requested === void 0 ? limit : Math.min(requested, limit);
677
+ return {
678
+ ...request,
679
+ modelConfig: { ...request.modelConfig, maxTokens, n: completions }
680
+ };
681
+ }
682
+ function costReceiptFromAxResponse(response, fallbackModel) {
683
+ const usage = response.modelUsage;
684
+ const tokens = usage?.tokens;
685
+ const model = usage?.model || fallbackModel;
686
+ if (!tokens || !validUsage(tokens.promptTokens) || !validUsage(tokens.completionTokens) || !validUsage(tokens.totalTokens) || tokens.totalTokens < tokens.promptTokens + tokens.completionTokens || !validOptionalUsage(tokens.cacheReadTokens) || !validOptionalUsage(tokens.cacheCreationTokens) || !validOptionalUsage(tokens.reasoningTokens) || !validOptionalUsage(tokens.thoughtsTokens)) {
687
+ return {
688
+ model,
689
+ inputTokens: 0,
690
+ outputTokens: 0,
691
+ usageUnknown: true
692
+ };
693
+ }
694
+ const cacheReadTokens = validUsage(tokens.cacheReadTokens) ? tokens.cacheReadTokens : 0;
695
+ const cacheCreationTokens = validUsage(tokens.cacheCreationTokens) ? tokens.cacheCreationTokens : 0;
696
+ const totalCacheTokens = cacheReadTokens + cacheCreationTokens;
697
+ const thoughtsTokens = validUsage(tokens.thoughtsTokens) ? tokens.thoughtsTokens : 0;
698
+ const reasoningTokens = Math.max(
699
+ validUsage(tokens.reasoningTokens) ? tokens.reasoningTokens : 0,
700
+ thoughtsTokens
701
+ );
702
+ const outputTokens = tokens.completionTokens + thoughtsTokens;
703
+ const directTotal = tokens.promptTokens + tokens.completionTokens;
704
+ const classifiedTotal = directTotal + thoughtsTokens + totalCacheTokens;
705
+ if (reasoningTokens > outputTokens || tokens.totalTokens !== directTotal && tokens.totalTokens !== classifiedTotal) {
706
+ return {
707
+ model,
708
+ inputTokens: 0,
709
+ outputTokens: 0,
710
+ usageUnknown: true
711
+ };
712
+ }
713
+ return {
714
+ model,
715
+ inputTokens: tokens.promptTokens,
716
+ outputTokens,
717
+ ...reasoningTokens > 0 ? { reasoningTokens } : {},
718
+ ...cacheReadTokens > 0 ? { cachedTokens: cacheReadTokens } : {},
719
+ ...cacheCreationTokens > 0 ? { cacheWriteTokens: cacheCreationTokens } : {}
720
+ };
721
+ }
722
+ function combineSignals(first, second) {
723
+ if (!first) return { signal: second, dispose: () => {
724
+ } };
725
+ if (!second || first === second) return { signal: first, dispose: () => {
726
+ } };
727
+ if (typeof AbortSignal.any === "function") {
728
+ return { signal: AbortSignal.any([first, second]), dispose: () => {
729
+ } };
730
+ }
731
+ const controller = new AbortController();
732
+ const dispose = () => {
733
+ first.removeEventListener("abort", abortFromFirst);
734
+ second.removeEventListener("abort", abortFromSecond);
735
+ };
736
+ const abortFrom = (source) => {
737
+ if (!controller.signal.aborted) controller.abort(source.reason);
738
+ dispose();
739
+ };
740
+ const abortFromFirst = () => abortFrom(first);
741
+ const abortFromSecond = () => abortFrom(second);
742
+ if (first.aborted) abortFrom(first);
743
+ else if (second.aborted) abortFrom(second);
744
+ else {
745
+ first.addEventListener("abort", abortFromFirst, { once: true });
746
+ second.addEventListener("abort", abortFromSecond, { once: true });
747
+ }
748
+ return { signal: controller.signal, dispose };
749
+ }
750
+ function containsUnboundedOrCacheableContent(request) {
751
+ if (request.functions?.some((fn) => fn.cache === true)) return true;
752
+ return request.chatPrompt.some((message) => {
753
+ if (message.cache === true) return true;
754
+ return Array.isArray(message.content) && message.content.some((part) => part.cache === true || part.type !== "text");
755
+ });
756
+ }
757
+ function modelName(value) {
758
+ return typeof value === "string" ? value : "";
759
+ }
760
+ function canDisableThinking(ai2, model) {
761
+ const namedAi = ai2;
762
+ const serviceName = typeof namedAi.getName === "function" ? namedAi.getName() : "";
763
+ const modelId = model.slice(model.lastIndexOf("/") + 1);
764
+ return serviceName !== "GoogleGeminiAI" || !/^gemini-3(?:[.-]|$)/i.test(modelId);
765
+ }
766
+ function validUsage(value) {
767
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
768
+ }
769
+ function validOptionalUsage(value) {
770
+ return value === void 0 || validUsage(value);
771
+ }
772
+ function assertPositiveInteger(value, field) {
773
+ if (!Number.isSafeInteger(value) || value <= 0) {
774
+ throw new RangeError(`meterAxChatService: ${field} must be a positive integer`);
775
+ }
776
+ }
777
+
778
+ // src/analyst/usage-receipt.ts
779
+ var DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS = 5e3;
780
+ function usageReceiptFromCostLedger(ledger, filter = "analyst") {
781
+ const resolvedFilter = typeof filter === "string" ? { channel: filter } : filter;
782
+ const summary = ledger.summary(resolvedFilter);
783
+ const receipts = ledger.list(resolvedFilter);
784
+ const hasReasoningUsage = receipts.some((receipt) => receipt.reasoningTokens !== void 0);
785
+ const hasCacheWriteUsage = receipts.some((receipt) => receipt.cacheWriteTokens !== void 0);
786
+ const costUncaptured = summary.pendingCalls > 0 || receipts.some((receipt) => receipt.costUnknown);
787
+ const cost = costUncaptured ? { kind: "uncaptured", usd: null } : receipts.every((receipt) => receipt.actualCostUsd !== void 0) ? { kind: "observed", usd: summary.totalCostUsd } : { kind: "estimated", usd: summary.totalCostUsd };
788
+ return {
789
+ calls: summary.totalCalls + summary.pendingCalls,
790
+ tokens: summary.usageComplete ? {
791
+ input: summary.inputTokens,
792
+ output: summary.outputTokens,
793
+ ...hasReasoningUsage ? { reasoning: summary.reasoningTokens ?? 0 } : {},
794
+ ...summary.cachedTokens > 0 ? { cached: summary.cachedTokens } : {},
795
+ ...hasCacheWriteUsage ? { cacheWrite: summary.cacheWriteTokens ?? 0 } : {}
796
+ } : null,
797
+ cost,
798
+ ...cost.kind === "uncaptured" ? { knownCostUsd: summary.totalCostUsd } : {}
799
+ };
800
+ }
801
+ async function settleUsageReceiptFromCostLedger(ledger, options = {}) {
802
+ const { timeoutMs: requestedTimeoutMs, ...requestedFilter } = options;
803
+ const filter = {
804
+ channel: requestedFilter.channel ?? "analyst",
805
+ ...requestedFilter.phase === void 0 ? {} : { phase: requestedFilter.phase },
806
+ ...requestedFilter.tags === void 0 ? {} : { tags: requestedFilter.tags }
807
+ };
808
+ const timeoutMs = validateUsageSettlementTimeout(requestedTimeoutMs);
809
+ const initial = ledger.summary(filter);
810
+ const waitResult = initial.pendingCalls === 0 ? true : ledger.waitForIdle ? await ledger.waitForIdle({ timeoutMs }) : false;
811
+ const pendingCalls = ledger.summary(filter).pendingCalls;
812
+ return {
813
+ settled: waitResult && pendingCalls === 0,
814
+ pendingCalls,
815
+ receipt: usageReceiptFromCostLedger(ledger, filter)
816
+ };
817
+ }
818
+ function validateUsageSettlementTimeout(timeoutMs) {
819
+ const resolved = timeoutMs ?? DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS;
820
+ if (!Number.isSafeInteger(resolved) || resolved < 0 || resolved > 2147483647) {
821
+ throw new TypeError(
822
+ "settlementTimeoutMs must be a non-negative safe integer no greater than 2147483647"
823
+ );
824
+ }
825
+ return resolved;
826
+ }
827
+
828
+ // src/analyst/kind-factory.ts
483
829
  function createTraceAnalystKind(spec, opts) {
484
830
  const version = opts.versionSuffix ? `${spec.version}+${opts.versionSuffix}` : spec.version;
831
+ const model = resolveAnalystModel(opts.ai, opts.model);
832
+ const minimumEvidenceCitations = spec.minimumEvidenceCitations ?? 1;
833
+ if (!Number.isInteger(minimumEvidenceCitations) || minimumEvidenceCitations < 1) {
834
+ throw new TypeError("minimumEvidenceCitations must be a positive integer");
835
+ }
836
+ const settlementTimeoutMs = validateUsageSettlementTimeout(opts.settlementTimeoutMs);
485
837
  return {
486
838
  id: spec.id,
487
839
  description: spec.description,
@@ -489,130 +841,201 @@ function createTraceAnalystKind(spec, opts) {
489
841
  cost: spec.cost,
490
842
  version,
491
843
  async analyze(store, ctx) {
492
- const tools = spec.buildTools(store);
493
- const maxDepth = spec.recursion?.maxDepth ?? 0;
494
- const maxParallel = spec.recursion?.maxParallelSubagents ?? 2;
495
- const priorContext = renderPriorFindings(ctx.priorFindings);
496
- const functions = tools;
497
- const actorDescription = spec.actorDescription.trim() + priorContext + "\n\n" + RAW_FINDING_SCHEMA_PROMPT + "\n\nFirst write `report`: a concise free-form prose diagnosis of what the traces show \u2014 what succeeded, what was suboptimal or failed \u2014 with concrete trace ids and numbers. THEN return the structured `findings` array (it MAY be empty when there is nothing to report). Use `final(...)` with the `{ report, findings }` payload when you are done.";
498
- const ax = agent(
499
- "question:string -> report:string, findings:json[]",
500
- {
501
- agentIdentity: {
502
- name: spec.id,
503
- description: spec.description
504
- },
505
- contextFields: ["question"],
506
- runtime: new AxJSRuntime({
507
- permissions: [],
508
- blockDynamicImport: true,
509
- allowedModules: [],
510
- freezeIntrinsics: true,
511
- blockShadowRealm: true,
512
- preventGlobalThisExtensions: false
513
- }),
514
- mode: maxDepth > 0 ? "advanced" : "simple",
515
- recursionOptions: maxDepth > 0 ? { maxDepth } : void 0,
516
- maxTurns: spec.maxTurns ?? 12,
517
- maxRuntimeChars: spec.maxRuntimeChars ?? 6e3,
518
- maxBatchedLlmQueryConcurrency: maxParallel,
519
- promptLevel: "detailed",
520
- // Trace analysis depends on exact prior tool results and runtime variables.
521
- contextPolicy: { preset: "full", budget: "balanced" },
522
- functions,
523
- actorOptions: {
524
- description: actorDescription,
525
- ...opts.model ? { model: opts.model } : {},
526
- showThoughts: false,
527
- thinkingTokenBudget: "none"
528
- },
529
- responderOptions: {
530
- description: spec.responderDescription ?? "Pass through the actor's `report` prose verbatim, and format the `findings` array exactly as the actor produced it. Do not add, drop, or summarize entries.",
531
- ...opts.model ? { model: opts.model } : {},
532
- showThoughts: false
533
- },
534
- bubbleErrors: [TraceFileMissingError]
535
- }
536
- );
537
- ctx.log?.(`analyst.kind ${spec.id} forward`, {
538
- max_depth: maxDepth,
539
- tool_count: tools.length,
540
- tags: ctx.tags
844
+ const maxOutputTokens = spec.maxOutputTokens ?? 4096;
845
+ const costLedger = ctx.costLedger ?? new CostLedger(ctx.budgetUsd);
846
+ const costTags = {
847
+ ...ctx.tags ?? {},
848
+ analystId: spec.id,
849
+ ...ctx.correlationId ? { analystRunId: ctx.correlationId } : {}
850
+ };
851
+ const meteredAi = meterAxChatService(opts.ai, {
852
+ ledger: costLedger,
853
+ actor: spec.id,
854
+ maxOutputTokens,
855
+ defaultModel: model,
856
+ phase: ctx.costPhase,
857
+ signal: ctx.signal,
858
+ tags: costTags
541
859
  });
542
- const result = await ax.forward(opts.ai, { question: deriveQuestion(ctx, spec) });
543
- const expectedSubjects = KIND_EXPECTED_SUBJECTS[spec.id];
544
- const out = [];
545
- const rawRows = Array.isArray(result.findings) ? result.findings : [];
546
- let rejectedWrongKind = 0;
547
- for (const row of rawRows) {
548
- const parsed = parseRawFinding(row, ctx.log);
549
- if (!parsed) continue;
550
- if (expectedSubjects && parsed.subject !== void 0) {
551
- const parsedSubject = parseFindingSubject(parsed.subject);
552
- if (parsedSubject === null) {
553
- ctx.log?.("finding rejected: subject failed to parse", {
554
- kind: spec.id,
555
- subject: parsed.subject
556
- });
557
- rejectedWrongKind += 1;
558
- continue;
860
+ try {
861
+ const tools = spec.buildTools(store);
862
+ const maxDepth = spec.recursion?.maxDepth ?? 0;
863
+ const maxParallel = spec.recursion?.maxParallelSubagents ?? 2;
864
+ const priorContext = renderPriorFindings(ctx.priorFindings);
865
+ const upstreamContext = renderUpstreamFindings(ctx.upstreamFindings);
866
+ const functions = tools;
867
+ const actorDescription = spec.actorDescription.trim() + priorContext + upstreamContext + "\n\n" + RAW_FINDING_SCHEMA_PROMPT + (minimumEvidenceCitations > 1 ? `
868
+
869
+ This kind requires at least ${minimumEvidenceCitations} evidence citations per finding; rows with fewer are rejected.` : "") + "\n\nFirst write `report`: a concise free-form prose diagnosis of what the traces show \u2014 what succeeded, what was suboptimal or failed \u2014 with concrete trace ids and numbers. THEN return the structured `findings` array (it MAY be empty when there is nothing to report). Call `final({ report, findings })` exactly once when done.";
870
+ const ax = agent(
871
+ "question:string -> report:string, findings:json[]",
872
+ {
873
+ agentIdentity: {
874
+ name: spec.id,
875
+ description: spec.description
876
+ },
877
+ contextFields: ["question"],
878
+ runtime: new AxJSRuntime({
879
+ permissions: [],
880
+ blockDynamicImport: true,
881
+ allowedModules: [],
882
+ freezeIntrinsics: true,
883
+ blockShadowRealm: true,
884
+ preventGlobalThisExtensions: false
885
+ }),
886
+ mode: maxDepth > 0 ? "advanced" : "simple",
887
+ recursionOptions: maxDepth > 0 ? { maxDepth } : void 0,
888
+ maxTurns: spec.maxTurns ?? 12,
889
+ maxRuntimeChars: spec.maxRuntimeChars ?? 6e3,
890
+ maxBatchedLlmQueryConcurrency: maxParallel,
891
+ promptLevel: "detailed",
892
+ // Trace analysis depends on exact prior tool results and runtime variables.
893
+ contextPolicy: { preset: "full", budget: "balanced" },
894
+ functions,
895
+ actorOptions: {
896
+ description: actorDescription,
897
+ model,
898
+ showThoughts: false,
899
+ thinkingTokenBudget: "none"
900
+ },
901
+ responderOptions: {
902
+ description: spec.responderDescription ?? "Pass through the actor's `report` prose verbatim, and format the `findings` array exactly as the actor produced it. Do not add, drop, or summarize entries.",
903
+ model,
904
+ showThoughts: false
905
+ },
906
+ bubbleErrors: [TraceFileMissingError]
907
+ }
908
+ );
909
+ ctx.log?.(`analyst.kind ${spec.id} forward`, {
910
+ max_depth: maxDepth,
911
+ tool_count: tools.length,
912
+ tags: ctx.tags
913
+ });
914
+ const result = await ax.forward(meteredAi, { question: deriveQuestion(ctx, spec) });
915
+ const expectedSubjects = KIND_EXPECTED_SUBJECTS[spec.id];
916
+ const out = [];
917
+ const rawRows = Array.isArray(result.findings) ? result.findings : [];
918
+ let rejectedWrongKind = 0;
919
+ let rejectedInsufficientEvidence = 0;
920
+ const processRow = (parsed) => {
921
+ const postProcessed = spec.postProcess ? applyLegacyRawFindingCallback(
922
+ parsed,
923
+ (row) => spec.postProcess?.(row, ctx) ?? null,
924
+ ctx.log
925
+ ) : parsed;
926
+ if (!postProcessed) return null;
927
+ if (expectedSubjects && postProcessed.subject !== void 0) {
928
+ const parsedSubject = parseFindingSubject(postProcessed.subject);
929
+ if (parsedSubject === null) {
930
+ ctx.log?.("finding rejected: subject failed to parse", {
931
+ kind: spec.id,
932
+ subject: postProcessed.subject
933
+ });
934
+ rejectedWrongKind += 1;
935
+ return null;
936
+ }
937
+ if (!expectedSubjects.includes(parsedSubject.kind)) {
938
+ ctx.log?.("finding rejected: subject variant not allowed for this kind", {
939
+ kind: spec.id,
940
+ subject_kind: parsedSubject.kind,
941
+ subject: postProcessed.subject,
942
+ allowed: expectedSubjects
943
+ });
944
+ rejectedWrongKind += 1;
945
+ return null;
946
+ }
559
947
  }
560
- if (!expectedSubjects.includes(parsedSubject.kind)) {
561
- ctx.log?.("finding rejected: subject variant not allowed for this kind", {
948
+ const distinctEvidenceCitations = new Set(
949
+ postProcessed.evidence.map((citation) => citation.uri.trim())
950
+ ).size;
951
+ if (distinctEvidenceCitations < minimumEvidenceCitations) {
952
+ ctx.log?.("finding rejected: insufficient evidence citations", {
562
953
  kind: spec.id,
563
- subject_kind: parsedSubject.kind,
564
- subject: parsed.subject,
565
- allowed: expectedSubjects
954
+ required: minimumEvidenceCitations,
955
+ received: postProcessed.evidence.length,
956
+ distinct: distinctEvidenceCitations
566
957
  });
567
- rejectedWrongKind += 1;
568
- continue;
958
+ rejectedInsufficientEvidence += 1;
959
+ return null;
569
960
  }
961
+ return postProcessed;
962
+ };
963
+ for (const row of rawRows) {
964
+ const parsed = parseCanonicalRawFinding(row, ctx.log);
965
+ if (!parsed) continue;
966
+ const postProcessed = processRow(parsed);
967
+ if (!postProcessed) continue;
968
+ out.push(toAnalystFinding(spec, version, postProcessed));
570
969
  }
571
- const postProcessed = spec.postProcess?.(parsed, ctx) ?? parsed;
572
- if (!postProcessed) continue;
573
- out.push(toAnalystFinding(spec, postProcessed));
574
- }
575
- ctx.log?.(`analyst.kind ${spec.id} done`, {
576
- emitted: rawRows.length,
577
- accepted: out.length,
578
- rejected_wrong_subject: rejectedWrongKind
579
- });
580
- const report = typeof result.report === "string" ? result.report : "";
581
- if (out.length === 0 && report.trim().length >= 200) {
582
- if (opts.recovery) {
583
- const recovered = await structureFindings({
584
- report,
585
- analystId: spec.id,
586
- area: spec.area,
587
- model: opts.recovery.model ?? opts.model ?? "",
588
- baseUrl: opts.recovery.baseUrl,
589
- apiKey: opts.recovery.apiKey,
590
- fetchImpl: opts.recovery.fetchImpl
591
- });
592
- out.push(...recovered.findings);
593
- ctx.log?.(`analyst.kind ${spec.id} recovery`, {
594
- outcome: recovered.outcome,
595
- recovered: recovered.findings.length
596
- });
597
- }
598
- if (out.length === 0) {
599
- out.push(
600
- makeFinding({
601
- analyst_id: spec.id,
970
+ ctx.log?.(`analyst.kind ${spec.id} done`, {
971
+ emitted: rawRows.length,
972
+ accepted: out.length,
973
+ rejected_wrong_subject: rejectedWrongKind,
974
+ rejected_insufficient_evidence: rejectedInsufficientEvidence
975
+ });
976
+ const report = typeof result.report === "string" ? result.report : "";
977
+ if (out.length === 0 && report.trim().length >= 200) {
978
+ if (opts.recovery) {
979
+ const wrongKindBefore = rejectedWrongKind;
980
+ const insufficientEvidenceBefore = rejectedInsufficientEvidence;
981
+ const recovered = await structureFindings({
982
+ report,
983
+ analystId: spec.id,
602
984
  area: spec.area,
985
+ model: opts.recovery.model ?? model,
986
+ baseUrl: opts.recovery.baseUrl,
987
+ apiKey: opts.recovery.apiKey,
988
+ fetchImpl: opts.recovery.fetchImpl,
989
+ costLedger,
990
+ costPhase: ctx.costPhase,
991
+ costTags,
992
+ signal: ctx.signal,
993
+ maxTokens: Math.min(maxOutputTokens, 2e3),
994
+ processCanonicalRow: processRow,
995
+ findingMetadata: { kind_version: version }
996
+ });
997
+ out.push(...recovered.findings);
998
+ ctx.log?.(`analyst.kind ${spec.id} recovery`, {
999
+ outcome: recovered.outcome,
1000
+ recovered: recovered.findings.length,
1001
+ rejected_wrong_subject: rejectedWrongKind - wrongKindBefore,
1002
+ rejected_insufficient_evidence: rejectedInsufficientEvidence - insufficientEvidenceBefore
1003
+ });
1004
+ }
1005
+ if (out.length === 0) {
1006
+ const fallback = processRow({
603
1007
  claim: "Analyst produced a diagnosis but no structured findings \u2014 see report.",
604
1008
  rationale: report.slice(0, 1500),
605
1009
  severity: "info",
606
1010
  confidence: 0.3,
607
- evidence_refs: [
608
- { kind: "artifact", uri: "report://summary", excerpt: report.slice(0, 2e3) }
609
- ],
610
- metadata: { outcome: "extraction_failed" }
611
- })
612
- );
1011
+ evidence: [{ uri: "report://summary", excerpt: report.slice(0, 2e3) }]
1012
+ });
1013
+ if (fallback) {
1014
+ out.push(toAnalystFinding(spec, version, fallback, { outcome: "extraction_failed" }));
1015
+ } else {
1016
+ throw new Error(
1017
+ `Trace analyst '${spec.id}' produced a substantive report, but no finding satisfied its acceptance rules`
1018
+ );
1019
+ }
1020
+ }
613
1021
  }
1022
+ return out;
1023
+ } finally {
1024
+ const usage = await settleUsageReceiptFromCostLedger(costLedger, {
1025
+ tags: {
1026
+ analystId: spec.id,
1027
+ ...ctx.correlationId ? { analystRunId: ctx.correlationId } : {}
1028
+ },
1029
+ timeoutMs: settlementTimeoutMs
1030
+ });
1031
+ if (!usage.settled) {
1032
+ ctx.log?.(`analyst.kind ${spec.id} provider settlement timed out`, {
1033
+ pending_calls: usage.pendingCalls,
1034
+ timeout_ms: settlementTimeoutMs
1035
+ });
1036
+ }
1037
+ ctx.recordUsage?.(usage.receipt);
614
1038
  }
615
- return out;
616
1039
  }
617
1040
  };
618
1041
  }
@@ -621,7 +1044,7 @@ function deriveQuestion(ctx, spec) {
621
1044
  const task = `Analyze this trace dataset with the available tools and report ${spec.area} findings. ${spec.description}`;
622
1045
  return focus ? `${task} Focus: ${focus}.` : task;
623
1046
  }
624
- function toAnalystFinding(spec, raw) {
1047
+ function toAnalystFinding(spec, version, raw, metadata = {}) {
625
1048
  return makeFinding({
626
1049
  analyst_id: spec.id,
627
1050
  area: spec.area,
@@ -630,25 +1053,11 @@ function toAnalystFinding(spec, raw) {
630
1053
  rationale: raw.rationale,
631
1054
  severity: raw.severity,
632
1055
  confidence: raw.confidence,
633
- evidence_refs: [
634
- {
635
- kind: evidenceKindFromUri(raw.evidence_uri),
636
- uri: raw.evidence_uri,
637
- excerpt: raw.evidence_excerpt
638
- }
639
- ],
1056
+ evidence_refs: evidenceRefsFromRawFinding(raw),
640
1057
  recommended_action: raw.recommended_action,
641
- metadata: { kind_version: spec.version }
1058
+ metadata: { kind_version: version, ...metadata }
642
1059
  });
643
1060
  }
644
- function evidenceKindFromUri(uri) {
645
- if (uri.startsWith("span://")) return "span";
646
- if (uri.startsWith("artifact://")) return "artifact";
647
- if (uri.startsWith("metric://")) return "metric";
648
- if (uri.startsWith("event://")) return "event";
649
- if (uri.startsWith("finding://")) return "finding";
650
- return "artifact";
651
- }
652
1061
  function renderPriorFindings(prior) {
653
1062
  if (!prior || prior.length === 0) return "";
654
1063
  const MAX_ROWS = 40;
@@ -668,6 +1077,26 @@ function renderPriorFindings(prior) {
668
1077
  overflow
669
1078
  ].filter(Boolean).join("\n");
670
1079
  }
1080
+ function renderUpstreamFindings(upstream) {
1081
+ if (!upstream || upstream.length === 0) return "";
1082
+ const MAX_ROWS = 40;
1083
+ const rows = upstream.slice(0, MAX_ROWS).map((finding) => {
1084
+ const subject = finding.subject ? ` [${finding.subject}]` : "";
1085
+ const action = finding.recommended_action ? ` action=${truncateForContext(finding.recommended_action, 120)}` : "";
1086
+ const evidence = finding.evidence_refs[0] ? ` evidence=${truncateForContext(finding.evidence_refs[0].uri, 120)}` : "";
1087
+ return ` - id=${finding.finding_id} source=${finding.analyst_id} ${finding.severity}${subject} claim=${truncateForContext(finding.claim, 160)}${action}${evidence}`;
1088
+ });
1089
+ const overflow = upstream.length > MAX_ROWS ? `
1090
+ ... +${upstream.length - MAX_ROWS} more upstream findings (truncated)` : "";
1091
+ return [
1092
+ "",
1093
+ "",
1094
+ "UPSTREAM FINDINGS (produced earlier in this same registry run):",
1095
+ "Use these as intermediate evidence. Build on them instead of repeating the same diagnosis, and cite a dependency with `finding://<id>`.",
1096
+ ...rows,
1097
+ overflow
1098
+ ].filter(Boolean).join("\n");
1099
+ }
671
1100
  function truncateForContext(s, max) {
672
1101
  if (s.length <= max) return s;
673
1102
  return `${s.slice(0, max - 1).trimEnd()}\u2026`;
@@ -713,15 +1142,7 @@ DISCOVERY \u2192 CLUSTER \u2192 CITE protocol:
713
1142
  2. Use \`traces.queryTraces({ filters: { has_errors: true }, limit })\` to pull error-bearing traces. Combine with \`traces.countTraces\` to see what fraction of the dataset failed.
714
1143
  3. For each candidate failure cluster, use \`traces.searchTrace\` with regex like \`STATUS_CODE_ERROR\`, \`MaxTurnsExceeded\`, \`assertion\`, \`unauthorized\`, \`timeout\`, \`429\`, \`5\\d\\d\`, the agent's specific error strings, or the names of its tools. Pull one or two representative traces per cluster, **not all** of them.
715
1144
  4. **Cluster, do not enumerate.** Two failures with the same root cause should be ONE finding citing both traces, not two findings. The point of this analyst is to compress N runs into K modes.
716
- 5. For each cluster you can defend with evidence, emit ONE finding with:
717
- - \`area\` = "failure-mode"
718
- - \`subject\` = a lowercase cluster label matching the subject grammar above ("tool-call-loop", "auth-revoked-mid-run", ...)
719
- - \`claim\` = one sentence stating the mode
720
- - \`severity\` = "critical" when it blocks the run, "high" when the run finished degraded, "medium" when it slowed convergence
721
- - \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the most representative span
722
- - \`evidence_excerpt\` = the exact quote (e.g. error message, stuck tool call payload, contradictory turn output)
723
- - \`confidence\` = 0.85+ when multiple traces show the same shape; 0.6-0.8 for a single-trace inference; <0.5 for speculative.
724
- - \`recommended_action\` = imperative-phrased fix idea (kept short \u2014 the improvement-analyst will expand on these)
1145
+ 5. For each defensible cluster, emit ONE finding. Use a lowercase cluster label matching the subject grammar ("tool-call-loop", "auth-revoked-mid-run", ...). Rate it critical when it blocks the run, high when the run finishes degraded, and medium when it slows convergence. Cite representative spans and include exact error, payload, or contradictory-output quotes. Use confidence 0.85+ when multiple traces show the same shape, 0.6-0.8 for a single-trace inference, and <0.5 for speculation. Keep the imperative fix idea short; the improvement analyst expands it.
725
1146
 
726
1147
  If the dataset has no failures, return an empty findings array \u2014 do NOT pad with low-confidence speculation.
727
1148
 
@@ -729,17 +1150,16 @@ If the dataset has no failures, return an empty findings array \u2014 do NOT pad
729
1150
  - After your first \`getDatasetOverview\` + \`queryTraces\` calls, you should have 3-6 candidate failure clusters in mind. Spawn one \`llmQuery\` per cluster in a single batch \u2014 they investigate in parallel.
730
1151
  - A sub-investigator that finds its cluster is actually two distinct modes should split again at its own level. Recursion is meant to discover sub-modes, not to do trivial drilling that the parent could do in-line.
731
1152
  - Pass narrow context to each subagent: { question: 'investigate the auth-revoked-mid-run cluster', context: { trace_ids: ['abc', 'def'], suspected_root_cause: 'token refresh skipped on idle sessions' } }. Subagents need enough context to skip re-discovery but not the whole conversation.
732
- - Each subagent returns its findings as JSON; the parent merges them. Do NOT have subagents call \`final()\` \u2014 they return their findings list to you, and you call \`final()\` once at the top.
1153
+ - Each subagent returns candidate cluster evidence; the parent merges it into the final finding set.
733
1154
 
734
1155
  OBSERVABILITY rules:
735
1156
  - Each non-final turn must emit at least one \`console.log\` for evidence.
736
- - Reuse runtime variables across turns; don't recompute.
737
- - Call \`final({ findings: [...] })\` exactly once, after you've gathered evidence for every cluster you intend to report.`;
1157
+ - Reuse runtime variables across turns; don't recompute.`;
738
1158
  var FAILURE_MODE_KIND_SPEC = {
739
1159
  id: "failure-mode",
740
1160
  description: "Clusters trace-dataset failures into distinct failure modes with cited evidence and a short recommended action.",
741
1161
  area: "failure-mode",
742
- version: "1.0.0",
1162
+ version: "1.1.0",
743
1163
  actorDescription: ACTOR_PROMPT,
744
1164
  buildTools: (store) => buildTraceToolsForGroup("all", store),
745
1165
  recursion: { maxDepth: 3, maxParallelSubagents: 4 },
@@ -771,31 +1191,21 @@ DISCOVERY \u2192 CANDIDATE-FIXES \u2192 COMPETE \u2192 CITE protocol:
771
1191
  - **Code** \u2014 change an implementation path when profile edits cannot repair the behavior
772
1192
  3. **Compete candidate fixes via subagents.** For each failure cluster, spawn one \`llmQuery\` per candidate-fix axis you want to evaluate. Each subagent's job: simulate the fix on the cited traces and report (i) likely effect, (ii) side effects, (iii) implementation cost as small/medium/large. Pass the cluster's failing trace_ids and the candidate axis as context.
773
1193
  4. After subagents return, **pick the winning candidate per cluster** based on (effect / cost) and emit ONE finding. Discard the losing candidates \u2014 the output is the recommendation, not the candidate set.
774
- 5. **Cross-reference upstream findings.** If a finding cites a prior failure-mode or knowledge-gap finding, use \`evidence_uri = "finding://<prior-finding-id>"\` (the registry supports this kind). This builds the dependency graph that lets the dashboard show "fix #X resolves failure modes A, B, C."
1194
+ 5. **Cross-reference upstream findings.** Cite prior failure-mode or knowledge-gap findings as \`finding://<prior-finding-id>\`. This builds the dependency graph that lets the dashboard show "fix #X resolves failure modes A, B, C."
775
1195
 
776
- For each winning recommendation, emit ONE finding with:
777
- - \`area\` = "improvement"
778
- - \`subject\` = one exact locus form from the subject grammar above
779
- - \`claim\` = one sentence stating the edit ("Add a precondition check to refuse tool X calls without arg Y")
780
- - \`severity\` = leverage rating: "critical" when fix resolves a critical failure mode; "high" when it resolves a high; "medium" when it's a quality-of-life win; "info" when it's a cleanup with no behavioral effect
781
- - \`evidence_uri\` = the failure-mode finding id this fix targets (\`finding://<id>\`) when it exists; else the most representative span
782
- - \`evidence_excerpt\` = a fragment showing the problem the fix targets
783
- - \`confidence\` = 0.85+ when the fix is mechanical and the failure mode is well-evidenced; 0.6-0.8 when the fix requires judgment; <0.5 for speculative
784
- - \`rationale\` = why this candidate beat its alternatives (2 sentences max)
785
- - \`recommended_action\` = the **literal edit**, phrased as a diff or a quoted replacement: "Replace section X with: '...'" or "Add tool with description: '...'" or "Set retry policy to max_attempts=3 with exponential backoff"
1196
+ For each winning recommendation, emit ONE finding. Use one exact locus from the subject grammar and state the edit in one sentence. Match leverage to the source failure's severity; use medium for quality-of-life changes and info for cleanup with no behavioral effect. Cite the targeted \`finding://<id>\` when available and the most representative span when useful. Quote the problem being fixed. Use confidence 0.85+ for a mechanical fix to a well-evidenced failure, 0.6-0.8 when judgment is required, and <0.5 for speculation. Explain in at most two sentences why this candidate beat its alternatives. The recommended action must be the literal diff, quoted replacement, tool description, or setting change.
786
1197
 
787
1198
  If no upstream failure findings exist in this run, derive your own from the trace dataset using the failure-mode protocol inline (\`searchTrace\` for STATUS_CODE_ERROR / MaxTurnsExceeded / etc.). But prefer to consume upstream findings when present \u2014 the kinds are designed to chain.
788
1199
 
789
1200
  Do NOT propose a fix you cannot defend with evidence. "Tighten the prompt" is not a finding; "Add 'When the user asks for X, always Y' to the system prompt section "request-classification"" is.
790
1201
 
791
1202
  OBSERVABILITY rules:
792
- - Each non-final turn must emit at least one \`console.log\` for evidence.
793
- - Call \`final({ findings: [...] })\` exactly once at the top level.`;
1203
+ - Each non-final turn must emit at least one \`console.log\` for evidence.`;
794
1204
  var IMPROVEMENT_KIND_SPEC = {
795
1205
  id: "improvement",
796
1206
  description: "Converts upstream failure / gap / poisoning findings into concrete locus-named edits (prompt, tool-doc, RAG, scaffolding) with leverage grades.",
797
1207
  area: "improvement",
798
- version: "1.0.0",
1208
+ version: "1.1.0",
799
1209
  actorDescription: ACTOR_PROMPT2,
800
1210
  buildTools: (store) => buildTraceToolsForGroup("all", store),
801
1211
  recursion: { maxDepth: 3, maxParallelSubagents: 4 },
@@ -825,28 +1235,19 @@ DISCOVERY \u2192 ATTRIBUTE-TO-LAYER \u2192 CITE protocol:
825
1235
  - Fabricated identifiers that don't appear in dataset \`sample_trace_ids\`
826
1236
  Use \`traces.searchTrace\` with patterns like \`I (don.?t|do not) know\`, \`assumed\`, \`unclear\`, \`could you (clarify|tell me|provide)\`, \`not found\`, \`undefined\`, \`unknown\`, \`null\`, dates older than the analysis window, or the agent's specific clarification phrases.
827
1237
  3. For each gap, identify the **layer of the runtime that should have prevented it** and use its exact locus from the subject grammar above.
828
- 4. For each gap you can defend with evidence, emit ONE finding with:
829
- - \`area\` = "knowledge-gap"
830
- - \`subject\` = one exact locus form from the subject grammar above
831
- - \`claim\` = a sentence naming the missing or stale knowledge ("wiki has no page on invoice line-item shape, agent had to re-derive it from raw spans")
832
- - \`severity\` = "high" when the gap caused a failure or a clarifying question; "medium" when it caused unnecessary turns; "low" when it caused minor inefficiency
833
- - \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the moment the gap surfaced (the question, the self-correction, the retrieval miss, the stale web result)
834
- - \`evidence_excerpt\` = exact quote where the agent showed the gap
835
- - \`confidence\` = 0.85+ when the agent itself articulated the gap; 0.6-0.8 when inferred from behavior
836
- - \`recommended_action\` = phrased as a wiki edit when the locus is \`agent-knowledge:*\` ("Create wiki page \`invoice-line-items\` with claims: ..."), or as a prompt/tool-doc edit otherwise
1238
+ 4. For each defensible gap, emit ONE finding. Use an exact locus from the subject grammar and name the missing or stale knowledge (for example, "wiki has no page on invoice line-item shape; agent re-derived it from raw spans"). Rate it high when it caused failure or a clarifying question, medium for unnecessary turns, and low for minor inefficiency. Cite the span where the question, correction, retrieval miss, or stale result surfaced and quote it exactly. Use confidence 0.85+ when the agent articulated the gap and 0.6-0.8 when inferred. Recommend a concrete wiki edit for an agent-knowledge locus or a prompt/tool-description edit otherwise.
837
1239
 
838
- **Delegate per layer.** After your first scan, you should have candidates spread across \`agent-knowledge:*\`, \`websearch:outdated\`, \`tool-doc:*\`, \`system-prompt:*\`, and \`memory:*\`. Spawn one \`llmQuery\` per layer in parallel \u2014 each subagent runs a focused detection (e.g. the \`agent-knowledge\` subagent looks for both missing-pages AND stale-pages; the \`websearch\` subagent looks specifically for date staleness signals; the \`tool-doc\` subagent looks for tool-call argument errors a fuller description would have prevented). Subagents return findings; you merge and emit one \`final({ findings })\` at the top.
1240
+ **Delegate per layer.** After your first scan, you should have candidates spread across \`agent-knowledge:*\`, \`websearch:outdated\`, \`tool-doc:*\`, \`system-prompt:*\`, and \`memory:*\`. Spawn one \`llmQuery\` per layer in parallel \u2014 each subagent runs a focused detection (e.g. the \`agent-knowledge\` subagent looks for both missing-pages AND stale-pages; the \`websearch\` subagent looks specifically for date staleness signals; the \`tool-doc\` subagent looks for tool-call argument errors a fuller description would have prevented). Merge their evidence into the final finding set.
839
1241
 
840
1242
  Do NOT report a gap that the agent later recovered from cleanly within the same turn \u2014 that's resilience, not a gap. Cite the *non-recovery* version when both exist.
841
1243
 
842
1244
  OBSERVABILITY rules:
843
- - Each non-final turn must emit at least one \`console.log\` for evidence.
844
- - Call \`final({ findings: [...] })\` exactly once at the top level.`;
1245
+ - Each non-final turn must emit at least one \`console.log\` for evidence.`;
845
1246
  var KNOWLEDGE_GAP_KIND_SPEC = {
846
1247
  id: "knowledge-gap",
847
1248
  description: "Identifies missing or stale pieces of knowledge \u2014 primarily against the agent-knowledge wiki \u2014 and attributes each to the runtime layer (wiki page, claim, raw source, websearch, tool-doc, system-prompt, memory) that should have held it.",
848
1249
  area: "knowledge-gap",
849
- version: "1.0.0",
1250
+ version: "1.1.0",
850
1251
  actorDescription: ACTOR_PROMPT3,
851
1252
  buildTools: (store) => buildTraceToolsForGroup("discoveryAndSearch", store),
852
1253
  recursion: { maxDepth: 2, maxParallelSubagents: 4 },
@@ -878,30 +1279,22 @@ DISCOVERY \u2192 DUAL-VERIFY \u2192 CITE protocol:
878
1279
 
879
1280
  **Delegate the dual-verify.** Use the recursion budget so each candidate poisoning gets one subagent investigating "did the agent act?" and one investigating "is the belief false?". After your first scan, fire off N parallel \`llmQuery\` pairs (one cluster per pair). Subagents return their findings; you accept only the ones where BOTH halves of the pair were confirmed.
880
1281
 
881
- For each confirmed poisoning, emit ONE finding with:
882
- - \`area\` = "knowledge-poisoning"
883
- - \`subject\` = the source of the false belief using one exact form from the subject grammar above
884
- - \`claim\` = one sentence: "agent believed X (from source S); evidence in trace shows X is false"
885
- - \`severity\` = "critical" when poisoning caused a wrong user-visible action; "high" when caught internally but wasted significant work; "medium" for inefficiency only
886
- - \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the action span (the moment the agent acted on the false belief)
887
- - \`evidence_excerpt\` = exact quote of the confident-but-wrong claim or action
888
- - \`confidence\` = 0.85+ when both halves are exact-quote backed; 0.6-0.8 when one half is inferred
889
- - \`recommended_action\` = where the source should be updated and how ("Update wiki page \`X\` claim \`Y\` to '...'", "Invalidate raw source \`Z\` and re-curate", "Replace system-prompt section X with 'tool foo now returns Y'")
1282
+ For each confirmed poisoning, emit ONE finding. Use the source of the false belief as the exact subject. State "agent believed X (from source S); trace evidence shows X is false." Rate it critical for a wrong user-visible action, high when caught internally after significant waste, and medium for inefficiency. Cite BOTH the action span and the contradicting span with exact quotes. Use confidence 0.85+ when both halves have exact quotes and 0.6-0.8 when one half is inferred. Recommend the literal source correction: update the wiki claim, invalidate and re-curate the raw source, or replace the stale prompt/tool instruction.
890
1283
 
891
1284
  Do NOT report a finding if the agent caught and corrected the false belief in the same turn \u2014 that's the system working. Reserve poisoning for cases where the false belief shaped downstream action.
892
1285
 
893
1286
  OBSERVABILITY rules:
894
- - Each non-final turn must emit at least one \`console.log\` for evidence.
895
- - Call \`final({ findings: [...] })\` exactly once at the top level.`;
1287
+ - Each non-final turn must emit at least one \`console.log\` for evidence.`;
896
1288
  var KNOWLEDGE_POISONING_KIND_SPEC = {
897
1289
  id: "knowledge-poisoning",
898
1290
  description: "Identifies confident-but-wrong actions caused by stale memory, contradicting RAG, deprecated tool docs, or outdated system-prompt instructions.",
899
1291
  area: "knowledge-poisoning",
900
- version: "1.0.0",
1292
+ version: "1.1.0",
901
1293
  actorDescription: ACTOR_PROMPT4,
902
1294
  buildTools: (store) => buildTraceToolsForGroup("all", store),
903
1295
  recursion: { maxDepth: 2, maxParallelSubagents: 4 },
904
1296
  maxTurns: 20,
1297
+ minimumEvidenceCitations: 2,
905
1298
  cost: { kind: "llm" }
906
1299
  };
907
1300
 
@@ -966,6 +1359,7 @@ var AnalystRegistry = class {
966
1359
  const deadlineMs = runOpts.timeoutMs ? started + runOpts.timeoutMs : void 0;
967
1360
  const selected = this.selectAnalysts(runOpts);
968
1361
  const budget = runOpts.budget ?? this.options.defaultBudget;
1362
+ validateBudgetPolicy(budget);
969
1363
  yield {
970
1364
  type: "run-started",
971
1365
  run_id: runId,
@@ -977,9 +1371,13 @@ var AnalystRegistry = class {
977
1371
  const allFindings = [];
978
1372
  let totalCost = 0;
979
1373
  let remainingUsd = budget?.totalUsd;
980
- const runnableCount = selected.filter(
981
- (a) => this.routeInput(a, inputs).kind !== "missing"
982
- ).length;
1374
+ const runnableAnalysts = selected.filter((a) => this.routeInput(a, inputs).kind !== "missing");
1375
+ const runnableCount = runnableAnalysts.length;
1376
+ const weights = budget?.weights;
1377
+ const totalWeight = weights && budget?.totalUsd != null && !budget.allocate && runnableCount > 0 ? runnableAnalysts.reduce((sum, analyst) => sum + analystWeight(weights, analyst.id), 0) : void 0;
1378
+ if (totalWeight === 0) {
1379
+ throw new Error("BudgetPolicy.weights must allocate positive weight to a runnable analyst");
1380
+ }
983
1381
  for (const analyst of selected) {
984
1382
  const t0 = Date.now();
985
1383
  const input = this.routeInput(analyst, inputs);
@@ -990,7 +1388,8 @@ var AnalystRegistry = class {
990
1388
  reason: `missing input of kind '${analyst.inputKind}'`,
991
1389
  findings_count: 0,
992
1390
  latency_ms: 0,
993
- cost_usd: 0
1391
+ cost_usd: 0,
1392
+ usage: zeroUsage()
994
1393
  };
995
1394
  summaries.push(summary);
996
1395
  log(`[analyst] skip ${analyst.id} \u2014 missing input`, { runId, kind: analyst.inputKind });
@@ -1001,20 +1400,30 @@ var AnalystRegistry = class {
1001
1400
  const perBudget = allocateBudget(budget, {
1002
1401
  analyst,
1003
1402
  remainingUsd,
1004
- runningCount: runnableCount
1403
+ runningCount: runnableCount,
1404
+ totalWeight
1005
1405
  });
1406
+ const usageReceipts = [];
1006
1407
  const ctx = {
1007
1408
  runId,
1008
1409
  correlationId,
1009
1410
  deadlineMs,
1010
1411
  budgetUsd: perBudget,
1412
+ costLedger: runOpts.costLedger,
1413
+ costPhase: runOpts.costPhase,
1011
1414
  chat: this.options.chat,
1012
1415
  tags: runOpts.tags,
1013
1416
  log: (msg, fields) => log(`[${analyst.id}] ${msg}`, { runId, correlationId, ...fields }),
1014
1417
  signal: runOpts.signal,
1015
- priorFindings: selectPriorFindings(runOpts.priorFindings, analyst.id)
1418
+ priorFindings: selectPriorFindings(runOpts.priorFindings, analyst.id),
1419
+ upstreamFindings: runOpts.chainFindings && allFindings.length > 0 ? [...allFindings] : void 0,
1420
+ recordUsage: (receipt) => {
1421
+ assertValidUsageReceipt(receipt);
1422
+ usageReceipts.push(receipt);
1423
+ }
1016
1424
  };
1017
1425
  await hooks.onBeforeAnalyze?.({ analyst, ctx, runId });
1426
+ const effectiveBudget = validateEffectiveBudget(ctx.budgetUsd, remainingUsd, analyst.id);
1018
1427
  yield {
1019
1428
  type: "analyst-started",
1020
1429
  analyst_id: analyst.id,
@@ -1023,24 +1432,38 @@ var AnalystRegistry = class {
1023
1432
  try {
1024
1433
  const findings = await analyst.analyze(input.value, ctx);
1025
1434
  const latency = Date.now() - t0;
1026
- const cost = sumFindingCost(findings);
1435
+ const usage = resolveUsage(analyst, findings, usageReceipts);
1436
+ const cost = knownCostUsd(usage);
1027
1437
  totalCost += cost;
1028
- if (typeof remainingUsd === "number") remainingUsd = Math.max(0, remainingUsd - cost);
1438
+ if (typeof remainingUsd === "number") {
1439
+ remainingUsd = Math.max(0, remainingUsd - budgetDebit(usage, effectiveBudget));
1440
+ }
1029
1441
  allFindings.push(...findings);
1030
1442
  const summary = {
1031
1443
  analyst_id: analyst.id,
1032
1444
  status: "ok",
1033
1445
  findings_count: findings.length,
1034
1446
  latency_ms: latency,
1035
- cost_usd: cost
1447
+ cost_usd: cost,
1448
+ usage
1036
1449
  };
1037
1450
  summaries.push(summary);
1038
1451
  log(`[analyst] ok ${analyst.id}`, {
1039
1452
  runId,
1040
1453
  findings: findings.length,
1041
1454
  latency_ms: latency,
1042
- cost_usd: cost
1455
+ cost_usd: cost,
1456
+ cost_kind: usage.cost.kind,
1457
+ input_tokens: usage.tokens?.input ?? null,
1458
+ output_tokens: usage.tokens?.output ?? null
1043
1459
  });
1460
+ if (effectiveBudget !== void 0 && usage.cost.kind === "uncaptured") {
1461
+ log(`[analyst] WARN ${analyst.id} \u2014 USD cost uncaptured; budget not reconciled`, {
1462
+ runId,
1463
+ budget_usd: effectiveBudget,
1464
+ cost_captured: false
1465
+ });
1466
+ }
1044
1467
  await hooks.onAfterAnalyze?.({ analyst, summary, findings, runId });
1045
1468
  yield { type: "analyst-completed", summary, findings };
1046
1469
  } catch (err) {
@@ -1048,20 +1471,36 @@ var AnalystRegistry = class {
1048
1471
  const e = err instanceof Error ? err : new Error(String(err));
1049
1472
  const hookFindings = await hooks.onError?.({ analyst, error: e, runId }) ?? [];
1050
1473
  if (hookFindings.length) allFindings.push(...hookFindings);
1474
+ const usage = resolveUsage(analyst, hookFindings, usageReceipts);
1475
+ const cost = knownCostUsd(usage);
1476
+ totalCost += cost;
1477
+ if (typeof remainingUsd === "number") {
1478
+ remainingUsd = Math.max(0, remainingUsd - budgetDebit(usage, effectiveBudget));
1479
+ }
1051
1480
  const summary = {
1052
1481
  analyst_id: analyst.id,
1053
1482
  status: "failed",
1054
1483
  findings_count: hookFindings.length,
1055
1484
  latency_ms: latency,
1056
- cost_usd: 0,
1485
+ cost_usd: cost,
1486
+ usage,
1057
1487
  error: { class: e.constructor.name, message: e.message }
1058
1488
  };
1059
1489
  summaries.push(summary);
1060
1490
  log(`[analyst] FAIL ${analyst.id}`, {
1061
1491
  runId,
1062
1492
  error_class: e.constructor.name,
1063
- error: e.message
1493
+ error: e.message,
1494
+ cost_usd: cost,
1495
+ cost_kind: usage.cost.kind
1064
1496
  });
1497
+ if (effectiveBudget !== void 0 && usage.cost.kind === "uncaptured") {
1498
+ log(`[analyst] WARN ${analyst.id} \u2014 USD cost uncaptured; budget not reconciled`, {
1499
+ runId,
1500
+ budget_usd: effectiveBudget,
1501
+ cost_captured: false
1502
+ });
1503
+ }
1065
1504
  await hooks.onAfterAnalyze?.({ analyst, summary, findings: hookFindings, runId });
1066
1505
  yield { type: "analyst-completed", summary, findings: hookFindings };
1067
1506
  }
@@ -1073,7 +1512,10 @@ var AnalystRegistry = class {
1073
1512
  ended_at: (/* @__PURE__ */ new Date()).toISOString(),
1074
1513
  findings: allFindings,
1075
1514
  per_analyst: summaries,
1076
- total_cost_usd: totalCost
1515
+ total_cost_usd: totalCost,
1516
+ total_cost_provenance: aggregateCostProvenance(
1517
+ summaries.map((summary) => summary.usage?.cost ?? { kind: "uncaptured", usd: null })
1518
+ )
1077
1519
  };
1078
1520
  await hooks.onComplete?.({ result });
1079
1521
  yield { type: "run-completed", result };
@@ -1110,28 +1552,171 @@ var AnalystRegistry = class {
1110
1552
  function allocateBudget(policy, args) {
1111
1553
  if (!policy) return void 0;
1112
1554
  if (policy.allocate) {
1113
- return policy.allocate({
1555
+ const allocated2 = policy.allocate({
1114
1556
  analyst: args.analyst,
1115
1557
  totalUsd: policy.totalUsd,
1116
1558
  remainingUsd: args.remainingUsd,
1117
1559
  runningCount: args.runningCount
1118
1560
  });
1561
+ if (allocated2 === void 0) {
1562
+ if (policy.totalUsd !== void 0) {
1563
+ throw new Error(
1564
+ `BudgetPolicy.allocate('${args.analyst.id}') cannot return undefined when totalUsd is set`
1565
+ );
1566
+ }
1567
+ return void 0;
1568
+ }
1569
+ assertBudgetAmount(allocated2, `BudgetPolicy.allocate('${args.analyst.id}')`);
1570
+ return args.remainingUsd === void 0 ? allocated2 : Math.min(allocated2, args.remainingUsd);
1119
1571
  }
1120
1572
  if (policy.totalUsd == null) return void 0;
1121
- if (policy.weights) {
1122
- const w = policy.weights[args.analyst.id] ?? 1;
1123
- const totalWeight = Math.max(1, args.runningCount);
1124
- return policy.totalUsd * w / totalWeight;
1573
+ const allocated = policy.weights ? policy.totalUsd * analystWeight(policy.weights, args.analyst.id) / args.totalWeight : policy.totalUsd / Math.max(1, args.runningCount);
1574
+ return args.remainingUsd === void 0 ? allocated : Math.min(allocated, args.remainingUsd);
1575
+ }
1576
+ function validateBudgetPolicy(policy) {
1577
+ if (!policy) return;
1578
+ if (policy.totalUsd !== void 0) assertBudgetAmount(policy.totalUsd, "BudgetPolicy.totalUsd");
1579
+ for (const [analystId, weight] of Object.entries(policy.weights ?? {})) {
1580
+ assertBudgetAmount(weight, `BudgetPolicy.weights['${analystId}']`);
1581
+ }
1582
+ }
1583
+ function assertBudgetAmount(value, field) {
1584
+ if (!Number.isFinite(value) || value < 0) {
1585
+ throw new Error(`${field} must be a non-negative finite number`);
1586
+ }
1587
+ }
1588
+ function validateEffectiveBudget(budgetUsd, remainingUsd, analystId) {
1589
+ if (budgetUsd !== void 0) {
1590
+ assertBudgetAmount(budgetUsd, `AnalystContext.budgetUsd for '${analystId}'`);
1591
+ }
1592
+ if (remainingUsd === void 0) return budgetUsd;
1593
+ if (budgetUsd === void 0) {
1594
+ throw new Error(
1595
+ `AnalystContext.budgetUsd for '${analystId}' cannot be removed while an overall budget remains`
1596
+ );
1597
+ }
1598
+ if (budgetUsd > remainingUsd) {
1599
+ throw new Error(
1600
+ `AnalystContext.budgetUsd for '${analystId}' (${budgetUsd}) exceeds the remaining overall budget (${remainingUsd})`
1601
+ );
1602
+ }
1603
+ return budgetUsd;
1604
+ }
1605
+ function analystWeight(weights, analystId) {
1606
+ const weight = weights[analystId] ?? 1;
1607
+ assertBudgetAmount(weight, `BudgetPolicy.weights['${analystId}']`);
1608
+ return weight;
1609
+ }
1610
+ function zeroUsage() {
1611
+ return {
1612
+ calls: 0,
1613
+ tokens: { input: 0, output: 0 },
1614
+ cost: { kind: "observed", usd: 0 }
1615
+ };
1616
+ }
1617
+ function resolveUsage(analyst, findings, receipts) {
1618
+ const legacyCost = sumFindingCost(findings);
1619
+ if (receipts.length > 0) {
1620
+ const merged = mergeUsageReceipts(receipts);
1621
+ return merged.cost.kind === "uncaptured" && legacyCost.captured ? { ...merged, knownCostUsd: Math.max(merged.knownCostUsd ?? 0, legacyCost.usd) } : merged;
1622
+ }
1623
+ if (legacyCost.captured) {
1624
+ return {
1625
+ calls: null,
1626
+ tokens: null,
1627
+ cost: { kind: "observed", usd: legacyCost.usd }
1628
+ };
1629
+ }
1630
+ if (analyst.cost.kind === "deterministic") return zeroUsage();
1631
+ return { calls: null, tokens: null, cost: { kind: "uncaptured", usd: null } };
1632
+ }
1633
+ function mergeUsageReceipts(receipts) {
1634
+ const calls = receipts.every((receipt) => receipt.calls !== null) ? receipts.reduce((sum, receipt) => sum + (receipt.calls ?? 0), 0) : null;
1635
+ const tokens = receipts.every((receipt) => receipt.tokens !== null) ? receipts.reduce(
1636
+ (sum, receipt) => ({
1637
+ input: sum.input + (receipt.tokens?.input ?? 0),
1638
+ output: sum.output + (receipt.tokens?.output ?? 0),
1639
+ ...sum.reasoning !== void 0 || receipt.tokens?.reasoning !== void 0 ? { reasoning: (sum.reasoning ?? 0) + (receipt.tokens?.reasoning ?? 0) } : {},
1640
+ ...sum.cached !== void 0 || receipt.tokens?.cached !== void 0 ? { cached: (sum.cached ?? 0) + (receipt.tokens?.cached ?? 0) } : {},
1641
+ ...sum.cacheWrite !== void 0 || receipt.tokens?.cacheWrite !== void 0 ? { cacheWrite: (sum.cacheWrite ?? 0) + (receipt.tokens?.cacheWrite ?? 0) } : {}
1642
+ }),
1643
+ { input: 0, output: 0 }
1644
+ ) : null;
1645
+ const cost = aggregateCostProvenance(receipts.map((receipt) => receipt.cost));
1646
+ return {
1647
+ calls,
1648
+ tokens,
1649
+ cost,
1650
+ ...cost.kind === "uncaptured" ? {
1651
+ knownCostUsd: receipts.reduce((sum, receipt) => sum + knownCostUsd(receipt), 0)
1652
+ } : {}
1653
+ };
1654
+ }
1655
+ function knownCostUsd(receipt) {
1656
+ return receipt.cost.kind === "uncaptured" ? receipt.knownCostUsd ?? 0 : receipt.cost.usd;
1657
+ }
1658
+ function budgetDebit(receipt, allocatedUsd) {
1659
+ const known = knownCostUsd(receipt);
1660
+ return receipt.cost.kind === "uncaptured" && allocatedUsd !== void 0 ? Math.max(known, allocatedUsd) : known;
1661
+ }
1662
+ function aggregateCostProvenance(costs) {
1663
+ if (costs.some((cost) => cost.kind === "uncaptured")) {
1664
+ return { kind: "uncaptured", usd: null };
1665
+ }
1666
+ const usd = costs.reduce((sum, cost) => sum + (cost.usd ?? 0), 0);
1667
+ return costs.some((cost) => cost.kind === "estimated") ? { kind: "estimated", usd } : { kind: "observed", usd };
1668
+ }
1669
+ function assertValidUsageReceipt(receipt) {
1670
+ if (receipt.calls !== null && (!Number.isInteger(receipt.calls) || receipt.calls < 0)) {
1671
+ throw new Error("AnalystContext.recordUsage: calls must be a non-negative integer or null");
1672
+ }
1673
+ if (receipt.tokens) {
1674
+ assertNonNegativeFinite(receipt.tokens.input, "tokens.input");
1675
+ assertNonNegativeFinite(receipt.tokens.output, "tokens.output");
1676
+ if (receipt.tokens.reasoning !== void 0) {
1677
+ assertNonNegativeFinite(receipt.tokens.reasoning, "tokens.reasoning");
1678
+ if (receipt.tokens.reasoning > receipt.tokens.output) {
1679
+ throw new Error(
1680
+ "AnalystContext.recordUsage: tokens.reasoning must not exceed tokens.output"
1681
+ );
1682
+ }
1683
+ }
1684
+ if (receipt.tokens.cached !== void 0) {
1685
+ assertNonNegativeFinite(receipt.tokens.cached, "tokens.cached");
1686
+ }
1687
+ if (receipt.tokens.cacheWrite !== void 0) {
1688
+ assertNonNegativeFinite(receipt.tokens.cacheWrite, "tokens.cacheWrite");
1689
+ }
1690
+ }
1691
+ if (receipt.cost.kind !== "uncaptured") {
1692
+ assertNonNegativeFinite(receipt.cost.usd, "cost.usd");
1693
+ } else if (receipt.cost.usd !== null) {
1694
+ throw new Error("AnalystContext.recordUsage: uncaptured cost.usd must be null");
1695
+ }
1696
+ if (receipt.knownCostUsd !== void 0) {
1697
+ assertNonNegativeFinite(receipt.knownCostUsd, "knownCostUsd");
1698
+ }
1699
+ }
1700
+ function assertNonNegativeFinite(value, field) {
1701
+ if (!Number.isFinite(value) || value < 0) {
1702
+ throw new Error(`AnalystContext.recordUsage: ${field} must be a non-negative finite number`);
1125
1703
  }
1126
- return policy.totalUsd / Math.max(1, args.runningCount);
1127
1704
  }
1128
1705
  function sumFindingCost(findings) {
1129
1706
  let sum = 0;
1707
+ let captured = false;
1130
1708
  for (const f of findings) {
1131
1709
  const c = f.metadata?.cost_usd;
1132
- if (typeof c === "number" && Number.isFinite(c)) sum += c;
1710
+ if (c === void 0) continue;
1711
+ if (typeof c !== "number" || !Number.isFinite(c) || c < 0) {
1712
+ throw new Error(
1713
+ `Analyst finding '${f.finding_id}' metadata.cost_usd must be a non-negative finite number`
1714
+ );
1715
+ }
1716
+ sum += c;
1717
+ captured = true;
1133
1718
  }
1134
- return sum;
1719
+ return { usd: sum, captured };
1135
1720
  }
1136
1721
  function selectPriorFindings(source, analystId) {
1137
1722
  if (!source) return void 0;
@@ -1891,6 +2476,7 @@ function finiteOrZero(value) {
1891
2476
  }
1892
2477
 
1893
2478
  export {
2479
+ createAnalystAi,
1894
2480
  computeFindingId,
1895
2481
  makeFinding,
1896
2482
  FINDING_SUBJECT_KINDS,
@@ -1905,12 +2491,19 @@ export {
1905
2491
  coerceJson,
1906
2492
  coerceToFindingRows,
1907
2493
  ANALYST_SEVERITIES,
2494
+ RawAnalystEvidenceSchema,
1908
2495
  RawAnalystFindingSchema,
2496
+ CanonicalRawAnalystFindingSchema,
1909
2497
  RAW_FINDING_SCHEMA_PROMPT,
2498
+ evidenceRefsFromRawFinding,
1910
2499
  parseRawFinding,
2500
+ parseCanonicalRawFinding,
1911
2501
  structureFindings,
2502
+ settleUsageReceiptFromCostLedger,
2503
+ validateUsageSettlementTimeout,
1912
2504
  createTraceAnalystKind,
1913
2505
  renderPriorFindings,
2506
+ renderUpstreamFindings,
1914
2507
  buildTraceToolsForGroup,
1915
2508
  FAILURE_MODE_KIND_SPEC,
1916
2509
  IMPROVEMENT_KIND_SPEC,
@@ -1940,4 +2533,4 @@ export {
1940
2533
  aggregateRunScore,
1941
2534
  clamp012 as clamp01
1942
2535
  };
1943
- //# sourceMappingURL=chunk-DZBCDDIE.js.map
2536
+ //# sourceMappingURL=chunk-7A4LIMMY.js.map