@tangle-network/agent-eval 0.118.3 → 0.119.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +26 -0
- package/dist/analyst/index.d.ts +143 -21
- package/dist/analyst/index.js +44 -10
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +50 -40
- package/dist/benchmarks/index.js +6 -6
- package/dist/campaign/index.d.ts +231 -156
- package/dist/campaign/index.js +5 -5
- package/dist/{chunk-JTMFT5QZ.js → chunk-4I2E3LLO.js} +2 -2
- package/dist/{chunk-VNLWDTDA.js → chunk-7A4LIMMY.js} +845 -252
- package/dist/chunk-7A4LIMMY.js.map +1 -0
- package/dist/{chunk-KK2O4VWA.js → chunk-D7AEXSM5.js} +2 -2
- package/dist/{chunk-EKHHBKS6.js → chunk-F6YUH3L4.js} +39 -28
- package/dist/{chunk-EKHHBKS6.js.map → chunk-F6YUH3L4.js.map} +1 -1
- package/dist/{chunk-267UCQWI.js → chunk-GERDAIAL.js} +2 -2
- package/dist/{chunk-267UCQWI.js.map → chunk-GERDAIAL.js.map} +1 -1
- package/dist/{chunk-MLHRFWQK.js → chunk-JHOJHHU7.js} +85 -16
- package/dist/chunk-JHOJHHU7.js.map +1 -0
- package/dist/{chunk-F5A4GDQW.js → chunk-LMJ2TGWJ.js} +3 -15
- package/dist/chunk-LMJ2TGWJ.js.map +1 -0
- package/dist/{chunk-EGNRE7VA.js → chunk-PAHNGS65.js} +2 -2
- package/dist/{chunk-QWRLW4CT.js → chunk-RLCJQ6VZ.js} +5 -5
- package/dist/{chunk-7V3QDWNL.js → chunk-XJ7JVCHB.js} +2 -2
- package/dist/cli.js +2 -2
- package/dist/contract/index.d.ts +257 -160
- package/dist/contract/index.js +6 -6
- package/dist/fuzz.d.ts +19 -1
- package/dist/fuzz.js +1 -1
- package/dist/index.d.ts +154 -48
- package/dist/index.js +12 -10
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/primeintellect/index.d.ts +311 -0
- package/dist/primeintellect/index.js +348 -0
- package/dist/primeintellect/index.js.map +1 -0
- package/dist/rl.d.ts +4 -0
- package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js} +3 -3
- package/dist/wire/index.d.ts +19 -1
- package/dist/wire/index.js +2 -2
- package/package.json +6 -1
- package/dist/chunk-F5A4GDQW.js.map +0 -1
- package/dist/chunk-MLHRFWQK.js.map +0 -1
- package/dist/chunk-VNLWDTDA.js.map +0 -1
- /package/dist/{chunk-JTMFT5QZ.js.map → chunk-4I2E3LLO.js.map} +0 -0
- /package/dist/{chunk-KK2O4VWA.js.map → chunk-D7AEXSM5.js.map} +0 -0
- /package/dist/{chunk-EGNRE7VA.js.map → chunk-PAHNGS65.js.map} +0 -0
- /package/dist/{chunk-QWRLW4CT.js.map → chunk-RLCJQ6VZ.js.map} +0 -0
- /package/dist/{chunk-7V3QDWNL.js.map → chunk-XJ7JVCHB.js.map} +0 -0
- /package/dist/{run-campaign-DWC67KJP.js.map → run-campaign-OBXSN5WK.js.map} +0 -0
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
} from "./chunk-NJC7U437.js";
|
|
7
7
|
import {
|
|
8
8
|
CostLedger
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-JHOJHHU7.js";
|
|
10
10
|
import {
|
|
11
11
|
TraceFileMissingError,
|
|
12
12
|
buildTraceAnalystTools
|
|
@@ -21,6 +21,39 @@ import {
|
|
|
21
21
|
ValidationError
|
|
22
22
|
} from "./chunk-ONWEPEDO.js";
|
|
23
23
|
|
|
24
|
+
// src/analyst/ax-service.ts
|
|
25
|
+
import { ai } from "@ax-llm/ax";
|
|
26
|
+
var configuredModels = /* @__PURE__ */ new WeakMap();
|
|
27
|
+
function createAnalystAi(config) {
|
|
28
|
+
const model = config.model.trim();
|
|
29
|
+
if (!model) throw new TypeError("createAnalystAi: model must be a non-empty string");
|
|
30
|
+
const service = ai({
|
|
31
|
+
name: config.provider ?? "openai",
|
|
32
|
+
apiKey: config.apiKey,
|
|
33
|
+
apiURL: config.baseUrl,
|
|
34
|
+
config: { model }
|
|
35
|
+
});
|
|
36
|
+
configuredModels.set(service, model);
|
|
37
|
+
return service;
|
|
38
|
+
}
|
|
39
|
+
function getConfiguredAnalystModel(service) {
|
|
40
|
+
return configuredModels.get(service);
|
|
41
|
+
}
|
|
42
|
+
function resolveAnalystModel(service, override) {
|
|
43
|
+
if (override !== void 0) {
|
|
44
|
+
const model2 = override.trim();
|
|
45
|
+
if (!model2) throw new TypeError("createTraceAnalystKind: model must be a non-empty string");
|
|
46
|
+
return model2;
|
|
47
|
+
}
|
|
48
|
+
const model = getConfiguredAnalystModel(service)?.trim();
|
|
49
|
+
if (!model) {
|
|
50
|
+
throw new TypeError(
|
|
51
|
+
"createTraceAnalystKind: model is required for Ax services not created by createAnalystAi()"
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
return model;
|
|
55
|
+
}
|
|
56
|
+
|
|
24
57
|
// src/analyst/types.ts
|
|
25
58
|
import { createHash } from "crypto";
|
|
26
59
|
function computeFindingId(input) {
|
|
@@ -345,47 +378,63 @@ function coerceToFindingRows(raw) {
|
|
|
345
378
|
// src/analyst/finding-signature.ts
|
|
346
379
|
import { z as z2 } from "zod";
|
|
347
380
|
var ANALYST_SEVERITIES = ["critical", "high", "medium", "low", "info"];
|
|
348
|
-
var
|
|
381
|
+
var RawAnalystEvidenceSchema = z2.object({
|
|
382
|
+
uri: z2.string().trim().min(1).max(2e3),
|
|
383
|
+
excerpt: z2.string().max(2e3).optional()
|
|
384
|
+
}).strict();
|
|
385
|
+
var RawAnalystFindingBaseShape = {
|
|
349
386
|
severity: z2.enum(ANALYST_SEVERITIES),
|
|
350
387
|
claim: z2.string().min(1).max(2e3),
|
|
351
|
-
|
|
352
|
-
* Subject locus the finding is about. Validated at parse time
|
|
353
|
-
* against the documented grammar (`finding-subject.ts`). Findings
|
|
354
|
-
* with a malformed subject are rejected — they would have been
|
|
355
|
-
* silently skipped by every downstream adapter, so failing loud at
|
|
356
|
-
* parse time turns a hidden no-op into a kind-prompt audit signal.
|
|
357
|
-
*
|
|
358
|
-
* Optional because purely descriptive findings (no actionable
|
|
359
|
-
* locus) are legitimate; they just don't route through the
|
|
360
|
-
* KnowledgeAdapter / ImprovementAdapter.
|
|
361
|
-
*/
|
|
362
|
-
subject: z2.string().max(400).refine((s) => parseFindingSubject(s) !== null, {
|
|
388
|
+
subject: z2.string().max(400).refine((subject) => parseFindingSubject(subject) !== null, {
|
|
363
389
|
message: "subject does not match the finding-subject grammar"
|
|
364
390
|
}).optional(),
|
|
365
|
-
evidence_uri: z2.string().min(1).max(2e3),
|
|
366
|
-
evidence_excerpt: z2.string().max(2e3).optional(),
|
|
367
391
|
confidence: z2.number().min(0).max(1),
|
|
368
392
|
rationale: z2.string().max(4e3).optional(),
|
|
369
393
|
recommended_action: z2.string().max(2e3).optional()
|
|
394
|
+
};
|
|
395
|
+
var RawAnalystFindingSchema = z2.object({
|
|
396
|
+
...RawAnalystFindingBaseShape,
|
|
397
|
+
evidence_uri: z2.string().min(1).max(2e3).refine((uri) => uri.trim().length > 0, { message: "evidence_uri must not be blank" }),
|
|
398
|
+
evidence_excerpt: z2.string().max(2e3).optional()
|
|
399
|
+
}).strict();
|
|
400
|
+
var CanonicalRawAnalystFindingObjectSchema = z2.object({
|
|
401
|
+
...RawAnalystFindingBaseShape,
|
|
402
|
+
evidence: z2.array(RawAnalystEvidenceSchema).min(1)
|
|
370
403
|
}).strict();
|
|
371
|
-
var
|
|
372
|
-
|
|
404
|
+
var CanonicalRawAnalystFindingSchema = z2.preprocess(
|
|
405
|
+
normalizeLegacySingleCitation,
|
|
406
|
+
CanonicalRawAnalystFindingObjectSchema
|
|
407
|
+
);
|
|
408
|
+
var RAW_FINDING_SCHEMA_PROMPT = `Each finding MUST be a strict JSON object with:
|
|
409
|
+
- severity: "critical" | "high" | "medium" | "low" | "info"
|
|
373
410
|
- claim: one-sentence statement (max 2000 chars)
|
|
374
|
-
- subject?:
|
|
375
|
-
-
|
|
376
|
-
-
|
|
377
|
-
-
|
|
378
|
-
-
|
|
379
|
-
- recommended_action?: concrete change phrased as an imperative ("Add ...", "Replace ...", "Stop ...") \u2014 omit when the finding is purely descriptive
|
|
411
|
+
- subject?: one exact subject form listed by this kind; omit rather than guess
|
|
412
|
+
- evidence: REQUIRED non-empty array of {"uri": string, "excerpt"?: string}. Use real identifiers with span://, event://, artifact://, metric://, or finding://. Include a short exact quote in excerpt when available. If nothing is citable, do not emit the finding.
|
|
413
|
+
- confidence: number 0..1 (0.9+ exact evidence; 0.6-0.8 inferred pattern; <0.5 speculative)
|
|
414
|
+
- rationale?: one or two reasoning sentences
|
|
415
|
+
- recommended_action?: concrete imperative change; omit for descriptive findings
|
|
380
416
|
|
|
381
|
-
|
|
417
|
+
Unknown fields are rejected. Do not emit area; the factory assigns it. Emit [] when there are no findings. Never fabricate evidence.`;
|
|
418
|
+
function evidenceRefsFromRawFinding(finding) {
|
|
419
|
+
return finding.evidence.map(({ uri, excerpt }) => ({
|
|
420
|
+
kind: evidenceKindFromUri(uri),
|
|
421
|
+
uri,
|
|
422
|
+
excerpt
|
|
423
|
+
}));
|
|
424
|
+
}
|
|
382
425
|
function parseRawFinding(row, log) {
|
|
383
|
-
|
|
426
|
+
return parseFindingWithSchema(RawAnalystFindingSchema, row, log);
|
|
427
|
+
}
|
|
428
|
+
function parseCanonicalRawFinding(row, log) {
|
|
429
|
+
return parseFindingWithSchema(CanonicalRawAnalystFindingSchema, row, log);
|
|
430
|
+
}
|
|
431
|
+
function parseFindingWithSchema(schema, row, log) {
|
|
432
|
+
const result = schema.safeParse(row);
|
|
384
433
|
if (result.success) return result.data;
|
|
385
434
|
if (typeof row === "string") {
|
|
386
435
|
const coerced = coerceJson(row);
|
|
387
436
|
if (coerced !== void 0) {
|
|
388
|
-
const retry =
|
|
437
|
+
const retry = schema.safeParse(coerced);
|
|
389
438
|
if (retry.success) return retry.data;
|
|
390
439
|
}
|
|
391
440
|
}
|
|
@@ -398,40 +447,98 @@ function parseRawFinding(row, log) {
|
|
|
398
447
|
});
|
|
399
448
|
return null;
|
|
400
449
|
}
|
|
450
|
+
function normalizeLegacySingleCitation(value) {
|
|
451
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return value;
|
|
452
|
+
const row = value;
|
|
453
|
+
if ("evidence" in row) {
|
|
454
|
+
if (!("evidence_uri" in row) || !Array.isArray(row.evidence)) return value;
|
|
455
|
+
const { evidence, evidence_uri: uri2, evidence_excerpt: excerpt2, ...rest2 } = row;
|
|
456
|
+
let alreadyPresent = false;
|
|
457
|
+
const mergedEvidence = evidence.map((citation) => {
|
|
458
|
+
if (citation === null || typeof citation !== "object" || Array.isArray(citation) || citation.uri !== uri2) {
|
|
459
|
+
return citation;
|
|
460
|
+
}
|
|
461
|
+
alreadyPresent = true;
|
|
462
|
+
const current = citation;
|
|
463
|
+
return excerpt2 === void 0 || current.excerpt !== void 0 ? citation : { ...current, excerpt: excerpt2 };
|
|
464
|
+
});
|
|
465
|
+
return {
|
|
466
|
+
...rest2,
|
|
467
|
+
evidence: alreadyPresent ? mergedEvidence : [...evidence, { uri: uri2, ...excerpt2 === void 0 ? {} : { excerpt: excerpt2 } }]
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
if (!("evidence_uri" in row)) return value;
|
|
471
|
+
const { evidence_uri: uri, evidence_excerpt: excerpt, ...rest } = row;
|
|
472
|
+
return {
|
|
473
|
+
...rest,
|
|
474
|
+
evidence: [{ uri, ...excerpt === void 0 ? {} : { excerpt } }]
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
function toLegacyRawAnalystFinding(finding) {
|
|
478
|
+
const primaryEvidence = finding.evidence[0];
|
|
479
|
+
if (!primaryEvidence) {
|
|
480
|
+
throw new TypeError("Canonical raw analyst findings require at least one evidence citation");
|
|
481
|
+
}
|
|
482
|
+
const { evidence: _evidence, ...rest } = finding;
|
|
483
|
+
return {
|
|
484
|
+
...rest,
|
|
485
|
+
evidence_uri: primaryEvidence.uri,
|
|
486
|
+
...primaryEvidence.excerpt === void 0 ? {} : { evidence_excerpt: primaryEvidence.excerpt }
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
function applyLegacyRawFindingCallback(finding, callback, log) {
|
|
490
|
+
const callbackResult = callback(toLegacyRawAnalystFinding(finding));
|
|
491
|
+
if (callbackResult === null) return null;
|
|
492
|
+
const parsed = parseCanonicalRawFinding(callbackResult, log);
|
|
493
|
+
if (!parsed) return null;
|
|
494
|
+
const primaryEvidence = parsed.evidence[0];
|
|
495
|
+
if (!primaryEvidence) {
|
|
496
|
+
throw new TypeError("Canonical raw analyst findings require at least one evidence citation");
|
|
497
|
+
}
|
|
498
|
+
const remainingEvidence = finding.evidence.slice(1).filter((citation) => citation.uri !== primaryEvidence.uri);
|
|
499
|
+
return {
|
|
500
|
+
...parsed,
|
|
501
|
+
evidence: [primaryEvidence, ...remainingEvidence]
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
function evidenceKindFromUri(uri) {
|
|
505
|
+
if (uri.startsWith("span://")) return "span";
|
|
506
|
+
if (uri.startsWith("event://")) return "event";
|
|
507
|
+
if (uri.startsWith("finding://")) return "finding";
|
|
508
|
+
if (uri.startsWith("metric://")) return "metric";
|
|
509
|
+
return "artifact";
|
|
510
|
+
}
|
|
401
511
|
|
|
402
512
|
// src/analyst/structure-findings.ts
|
|
403
513
|
var SYSTEM = [
|
|
404
514
|
"You convert a free-form trace-analysis report into a STRICT JSON array of findings.",
|
|
405
515
|
"Output ONLY the JSON array \u2014 no prose, no code fences.",
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
'evidence_uri cites the trace element the report referenced (e.g. "span://<trace>/<span>") or "report://summary".',
|
|
516
|
+
RAW_FINDING_SCHEMA_PROMPT,
|
|
517
|
+
"Omit subject when the report does not contain an exact valid locus.",
|
|
409
518
|
"If the report asserts NO problems, output exactly []."
|
|
410
519
|
].join(" ");
|
|
411
|
-
function buildRows(raw,
|
|
520
|
+
function buildRows(raw, opts) {
|
|
412
521
|
const rows = coerceToFindingRows(raw);
|
|
413
522
|
const out = [];
|
|
414
523
|
for (const row of rows) {
|
|
415
|
-
const
|
|
416
|
-
const parsed = parseRawFinding(normalized);
|
|
524
|
+
const parsed = parseCanonicalRawFinding(row);
|
|
417
525
|
if (!parsed) continue;
|
|
526
|
+
const callbackProcessed = opts.processRow ? applyLegacyRawFindingCallback(parsed, opts.processRow) : parsed;
|
|
527
|
+
if (!callbackProcessed) continue;
|
|
528
|
+
const processed = opts.processCanonicalRow ? opts.processCanonicalRow(callbackProcessed) : callbackProcessed;
|
|
529
|
+
if (!processed) continue;
|
|
418
530
|
out.push(
|
|
419
531
|
makeFinding({
|
|
420
|
-
analyst_id: analystId,
|
|
421
|
-
area,
|
|
422
|
-
subject:
|
|
423
|
-
claim:
|
|
424
|
-
rationale:
|
|
425
|
-
severity:
|
|
426
|
-
confidence:
|
|
427
|
-
evidence_refs:
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
uri: parsed.evidence_uri,
|
|
431
|
-
excerpt: parsed.evidence_excerpt
|
|
432
|
-
}
|
|
433
|
-
],
|
|
434
|
-
recommended_action: parsed.recommended_action
|
|
532
|
+
analyst_id: opts.analystId,
|
|
533
|
+
area: opts.area,
|
|
534
|
+
subject: processed.subject,
|
|
535
|
+
claim: processed.claim,
|
|
536
|
+
rationale: processed.rationale,
|
|
537
|
+
severity: processed.severity,
|
|
538
|
+
confidence: processed.confidence,
|
|
539
|
+
evidence_refs: evidenceRefsFromRawFinding(processed),
|
|
540
|
+
recommended_action: processed.recommended_action,
|
|
541
|
+
...opts.findingMetadata ? { metadata: { ...opts.findingMetadata } } : {}
|
|
435
542
|
})
|
|
436
543
|
);
|
|
437
544
|
}
|
|
@@ -439,6 +546,9 @@ function buildRows(raw, analystId, area) {
|
|
|
439
546
|
}
|
|
440
547
|
async function structureFindings(opts) {
|
|
441
548
|
const maxReasks = opts.maxReasks ?? 1;
|
|
549
|
+
if (!Number.isSafeInteger(maxReasks) || maxReasks < 0) {
|
|
550
|
+
throw new RangeError("structureFindings: maxReasks must be a non-negative safe integer");
|
|
551
|
+
}
|
|
442
552
|
const llm = { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl };
|
|
443
553
|
const costLedger = opts.costLedger ?? new CostLedger();
|
|
444
554
|
let user = `TRACE-ANALYSIS REPORT:
|
|
@@ -459,8 +569,9 @@ Return the findings JSON array.`;
|
|
|
459
569
|
phase: opts.costPhase ?? "analyst.structure-findings",
|
|
460
570
|
actor: "structure-findings",
|
|
461
571
|
model: opts.model,
|
|
572
|
+
signal: opts.signal,
|
|
462
573
|
maximumCharge: maximumChargeForLlmRequest(request, llm),
|
|
463
|
-
tags: { analystId: opts.analystId, attempt: String(attempt) },
|
|
574
|
+
tags: { ...opts.costTags, analystId: opts.analystId, attempt: String(attempt) },
|
|
464
575
|
execute: (signal, callId) => callLlm(request, { ...llm, signal, idempotencyKey: callId }),
|
|
465
576
|
receipt: costReceiptFromLlm,
|
|
466
577
|
receiptFromError: costReceiptFromLlmError
|
|
@@ -468,7 +579,7 @@ Return the findings JSON array.`;
|
|
|
468
579
|
if (!paid.succeeded) throw paid.error;
|
|
469
580
|
const res = paid.value;
|
|
470
581
|
const text = res.content.trim();
|
|
471
|
-
const findings = buildRows(text, opts
|
|
582
|
+
const findings = buildRows(text, opts);
|
|
472
583
|
if (findings.length > 0) return { findings, outcome: "ok" };
|
|
473
584
|
if (opts.report.trim().length < 200) return { findings: [], outcome: "ok" };
|
|
474
585
|
user = `${user}
|
|
@@ -480,8 +591,249 @@ That produced no valid findings. The report DOES describe issues \u2014 re-extra
|
|
|
480
591
|
|
|
481
592
|
// src/analyst/kind-factory.ts
|
|
482
593
|
import { AxJSRuntime, agent } from "@ax-llm/ax";
|
|
594
|
+
|
|
595
|
+
// src/analyst/ax-cost-service.ts
|
|
596
|
+
function meterAxChatService(ai2, options) {
|
|
597
|
+
assertPositiveInteger(options.maxOutputTokens, "maxOutputTokens");
|
|
598
|
+
const source = ai2;
|
|
599
|
+
if (typeof source.chat !== "function") {
|
|
600
|
+
throw new TypeError("meterAxChatService: Ax service must implement chat()");
|
|
601
|
+
}
|
|
602
|
+
const providerChat = source.chat.bind(ai2);
|
|
603
|
+
const chat = async (request, callOptions = {}) => {
|
|
604
|
+
const boundedRequest = boundOutputTokens(request, options.maxOutputTokens);
|
|
605
|
+
const model = modelName(boundedRequest.model) || options.defaultModel || "";
|
|
606
|
+
const canTurnOffThinking = canDisableThinking(ai2, model);
|
|
607
|
+
const combined = combineSignals(options.signal, callOptions.abortSignal);
|
|
608
|
+
try {
|
|
609
|
+
const paid = await options.ledger.runPaidCall({
|
|
610
|
+
channel: "analyst",
|
|
611
|
+
phase: options.phase ?? "analyst.ax.chat",
|
|
612
|
+
actor: options.actor,
|
|
613
|
+
model,
|
|
614
|
+
tags: options.tags,
|
|
615
|
+
signal: combined.signal,
|
|
616
|
+
maximumCharge: maximumChargeForAxChatRequest(boundedRequest, model),
|
|
617
|
+
execute: async (executionSignal) => {
|
|
618
|
+
const providerOptions = {
|
|
619
|
+
...callOptions,
|
|
620
|
+
abortSignal: executionSignal,
|
|
621
|
+
retry: { ...callOptions.retry, maxRetries: 0 },
|
|
622
|
+
stream: false,
|
|
623
|
+
showThoughts: false
|
|
624
|
+
};
|
|
625
|
+
if (canTurnOffThinking) providerOptions.thinkingTokenBudget = "none";
|
|
626
|
+
else Reflect.deleteProperty(providerOptions, "thinkingTokenBudget");
|
|
627
|
+
const response = await providerChat(boundedRequest, providerOptions);
|
|
628
|
+
if (response instanceof ReadableStream) {
|
|
629
|
+
throw new Error("meterAxChatService: provider returned a stream after stream:false");
|
|
630
|
+
}
|
|
631
|
+
return response;
|
|
632
|
+
},
|
|
633
|
+
receipt: (response) => costReceiptFromAxResponse(response, model)
|
|
634
|
+
});
|
|
635
|
+
if (!paid.succeeded) throw paid.error;
|
|
636
|
+
return paid.value;
|
|
637
|
+
} finally {
|
|
638
|
+
combined.dispose();
|
|
639
|
+
}
|
|
640
|
+
};
|
|
641
|
+
return new Proxy(ai2, {
|
|
642
|
+
get(target, property) {
|
|
643
|
+
if (property === "chat") return chat;
|
|
644
|
+
const value = Reflect.get(target, property, target);
|
|
645
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
646
|
+
}
|
|
647
|
+
});
|
|
648
|
+
}
|
|
649
|
+
function maximumChargeForAxChatRequest(request, defaultModel) {
|
|
650
|
+
const model = modelName(request.model) || defaultModel || "";
|
|
651
|
+
const maxTokens = request.modelConfig?.maxTokens;
|
|
652
|
+
const completions = request.modelConfig?.n;
|
|
653
|
+
if (!model || maxTokens === void 0 || completions === void 0) return void 0;
|
|
654
|
+
assertPositiveInteger(maxTokens, "request.modelConfig.maxTokens");
|
|
655
|
+
assertPositiveInteger(completions, "request.modelConfig.n");
|
|
656
|
+
const maximumOutputTokens = maxTokens * completions;
|
|
657
|
+
assertPositiveInteger(maximumOutputTokens, "maximum output tokens");
|
|
658
|
+
if (containsUnboundedOrCacheableContent(request)) return void 0;
|
|
659
|
+
let inputTokens;
|
|
660
|
+
try {
|
|
661
|
+
const pricedRequest = request.model === void 0 ? { ...request, model } : request;
|
|
662
|
+
inputTokens = new TextEncoder().encode(JSON.stringify(pricedRequest)).byteLength * completions;
|
|
663
|
+
} catch {
|
|
664
|
+
return void 0;
|
|
665
|
+
}
|
|
666
|
+
assertPositiveInteger(inputTokens, "maximum input tokens");
|
|
667
|
+
return { model, inputTokens, outputTokens: maximumOutputTokens };
|
|
668
|
+
}
|
|
669
|
+
function boundOutputTokens(request, limit) {
|
|
670
|
+
const requested = request.modelConfig?.maxTokens;
|
|
671
|
+
if (requested !== void 0) {
|
|
672
|
+
assertPositiveInteger(requested, "request.modelConfig.maxTokens");
|
|
673
|
+
}
|
|
674
|
+
const completions = request.modelConfig?.n ?? 1;
|
|
675
|
+
assertPositiveInteger(completions, "request.modelConfig.n");
|
|
676
|
+
const maxTokens = requested === void 0 ? limit : Math.min(requested, limit);
|
|
677
|
+
return {
|
|
678
|
+
...request,
|
|
679
|
+
modelConfig: { ...request.modelConfig, maxTokens, n: completions }
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
function costReceiptFromAxResponse(response, fallbackModel) {
|
|
683
|
+
const usage = response.modelUsage;
|
|
684
|
+
const tokens = usage?.tokens;
|
|
685
|
+
const model = usage?.model || fallbackModel;
|
|
686
|
+
if (!tokens || !validUsage(tokens.promptTokens) || !validUsage(tokens.completionTokens) || !validUsage(tokens.totalTokens) || tokens.totalTokens < tokens.promptTokens + tokens.completionTokens || !validOptionalUsage(tokens.cacheReadTokens) || !validOptionalUsage(tokens.cacheCreationTokens) || !validOptionalUsage(tokens.reasoningTokens) || !validOptionalUsage(tokens.thoughtsTokens)) {
|
|
687
|
+
return {
|
|
688
|
+
model,
|
|
689
|
+
inputTokens: 0,
|
|
690
|
+
outputTokens: 0,
|
|
691
|
+
usageUnknown: true
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
const cacheReadTokens = validUsage(tokens.cacheReadTokens) ? tokens.cacheReadTokens : 0;
|
|
695
|
+
const cacheCreationTokens = validUsage(tokens.cacheCreationTokens) ? tokens.cacheCreationTokens : 0;
|
|
696
|
+
const totalCacheTokens = cacheReadTokens + cacheCreationTokens;
|
|
697
|
+
const thoughtsTokens = validUsage(tokens.thoughtsTokens) ? tokens.thoughtsTokens : 0;
|
|
698
|
+
const reasoningTokens = Math.max(
|
|
699
|
+
validUsage(tokens.reasoningTokens) ? tokens.reasoningTokens : 0,
|
|
700
|
+
thoughtsTokens
|
|
701
|
+
);
|
|
702
|
+
const outputTokens = tokens.completionTokens + thoughtsTokens;
|
|
703
|
+
const directTotal = tokens.promptTokens + tokens.completionTokens;
|
|
704
|
+
const classifiedTotal = directTotal + thoughtsTokens + totalCacheTokens;
|
|
705
|
+
if (reasoningTokens > outputTokens || tokens.totalTokens !== directTotal && tokens.totalTokens !== classifiedTotal) {
|
|
706
|
+
return {
|
|
707
|
+
model,
|
|
708
|
+
inputTokens: 0,
|
|
709
|
+
outputTokens: 0,
|
|
710
|
+
usageUnknown: true
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
return {
|
|
714
|
+
model,
|
|
715
|
+
inputTokens: tokens.promptTokens,
|
|
716
|
+
outputTokens,
|
|
717
|
+
...reasoningTokens > 0 ? { reasoningTokens } : {},
|
|
718
|
+
...cacheReadTokens > 0 ? { cachedTokens: cacheReadTokens } : {},
|
|
719
|
+
...cacheCreationTokens > 0 ? { cacheWriteTokens: cacheCreationTokens } : {}
|
|
720
|
+
};
|
|
721
|
+
}
|
|
722
|
+
function combineSignals(first, second) {
|
|
723
|
+
if (!first) return { signal: second, dispose: () => {
|
|
724
|
+
} };
|
|
725
|
+
if (!second || first === second) return { signal: first, dispose: () => {
|
|
726
|
+
} };
|
|
727
|
+
if (typeof AbortSignal.any === "function") {
|
|
728
|
+
return { signal: AbortSignal.any([first, second]), dispose: () => {
|
|
729
|
+
} };
|
|
730
|
+
}
|
|
731
|
+
const controller = new AbortController();
|
|
732
|
+
const dispose = () => {
|
|
733
|
+
first.removeEventListener("abort", abortFromFirst);
|
|
734
|
+
second.removeEventListener("abort", abortFromSecond);
|
|
735
|
+
};
|
|
736
|
+
const abortFrom = (source) => {
|
|
737
|
+
if (!controller.signal.aborted) controller.abort(source.reason);
|
|
738
|
+
dispose();
|
|
739
|
+
};
|
|
740
|
+
const abortFromFirst = () => abortFrom(first);
|
|
741
|
+
const abortFromSecond = () => abortFrom(second);
|
|
742
|
+
if (first.aborted) abortFrom(first);
|
|
743
|
+
else if (second.aborted) abortFrom(second);
|
|
744
|
+
else {
|
|
745
|
+
first.addEventListener("abort", abortFromFirst, { once: true });
|
|
746
|
+
second.addEventListener("abort", abortFromSecond, { once: true });
|
|
747
|
+
}
|
|
748
|
+
return { signal: controller.signal, dispose };
|
|
749
|
+
}
|
|
750
|
+
function containsUnboundedOrCacheableContent(request) {
|
|
751
|
+
if (request.functions?.some((fn) => fn.cache === true)) return true;
|
|
752
|
+
return request.chatPrompt.some((message) => {
|
|
753
|
+
if (message.cache === true) return true;
|
|
754
|
+
return Array.isArray(message.content) && message.content.some((part) => part.cache === true || part.type !== "text");
|
|
755
|
+
});
|
|
756
|
+
}
|
|
757
|
+
function modelName(value) {
|
|
758
|
+
return typeof value === "string" ? value : "";
|
|
759
|
+
}
|
|
760
|
+
function canDisableThinking(ai2, model) {
|
|
761
|
+
const namedAi = ai2;
|
|
762
|
+
const serviceName = typeof namedAi.getName === "function" ? namedAi.getName() : "";
|
|
763
|
+
const modelId = model.slice(model.lastIndexOf("/") + 1);
|
|
764
|
+
return serviceName !== "GoogleGeminiAI" || !/^gemini-3(?:[.-]|$)/i.test(modelId);
|
|
765
|
+
}
|
|
766
|
+
function validUsage(value) {
|
|
767
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
768
|
+
}
|
|
769
|
+
function validOptionalUsage(value) {
|
|
770
|
+
return value === void 0 || validUsage(value);
|
|
771
|
+
}
|
|
772
|
+
function assertPositiveInteger(value, field) {
|
|
773
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
774
|
+
throw new RangeError(`meterAxChatService: ${field} must be a positive integer`);
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
// src/analyst/usage-receipt.ts
|
|
779
|
+
var DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS = 5e3;
|
|
780
|
+
function usageReceiptFromCostLedger(ledger, filter = "analyst") {
|
|
781
|
+
const resolvedFilter = typeof filter === "string" ? { channel: filter } : filter;
|
|
782
|
+
const summary = ledger.summary(resolvedFilter);
|
|
783
|
+
const receipts = ledger.list(resolvedFilter);
|
|
784
|
+
const hasReasoningUsage = receipts.some((receipt) => receipt.reasoningTokens !== void 0);
|
|
785
|
+
const hasCacheWriteUsage = receipts.some((receipt) => receipt.cacheWriteTokens !== void 0);
|
|
786
|
+
const costUncaptured = summary.pendingCalls > 0 || receipts.some((receipt) => receipt.costUnknown);
|
|
787
|
+
const cost = costUncaptured ? { kind: "uncaptured", usd: null } : receipts.every((receipt) => receipt.actualCostUsd !== void 0) ? { kind: "observed", usd: summary.totalCostUsd } : { kind: "estimated", usd: summary.totalCostUsd };
|
|
788
|
+
return {
|
|
789
|
+
calls: summary.totalCalls + summary.pendingCalls,
|
|
790
|
+
tokens: summary.usageComplete ? {
|
|
791
|
+
input: summary.inputTokens,
|
|
792
|
+
output: summary.outputTokens,
|
|
793
|
+
...hasReasoningUsage ? { reasoning: summary.reasoningTokens ?? 0 } : {},
|
|
794
|
+
...summary.cachedTokens > 0 ? { cached: summary.cachedTokens } : {},
|
|
795
|
+
...hasCacheWriteUsage ? { cacheWrite: summary.cacheWriteTokens ?? 0 } : {}
|
|
796
|
+
} : null,
|
|
797
|
+
cost,
|
|
798
|
+
...cost.kind === "uncaptured" ? { knownCostUsd: summary.totalCostUsd } : {}
|
|
799
|
+
};
|
|
800
|
+
}
|
|
801
|
+
async function settleUsageReceiptFromCostLedger(ledger, options = {}) {
|
|
802
|
+
const { timeoutMs: requestedTimeoutMs, ...requestedFilter } = options;
|
|
803
|
+
const filter = {
|
|
804
|
+
channel: requestedFilter.channel ?? "analyst",
|
|
805
|
+
...requestedFilter.phase === void 0 ? {} : { phase: requestedFilter.phase },
|
|
806
|
+
...requestedFilter.tags === void 0 ? {} : { tags: requestedFilter.tags }
|
|
807
|
+
};
|
|
808
|
+
const timeoutMs = validateUsageSettlementTimeout(requestedTimeoutMs);
|
|
809
|
+
const initial = ledger.summary(filter);
|
|
810
|
+
const waitResult = initial.pendingCalls === 0 ? true : ledger.waitForIdle ? await ledger.waitForIdle({ timeoutMs }) : false;
|
|
811
|
+
const pendingCalls = ledger.summary(filter).pendingCalls;
|
|
812
|
+
return {
|
|
813
|
+
settled: waitResult && pendingCalls === 0,
|
|
814
|
+
pendingCalls,
|
|
815
|
+
receipt: usageReceiptFromCostLedger(ledger, filter)
|
|
816
|
+
};
|
|
817
|
+
}
|
|
818
|
+
function validateUsageSettlementTimeout(timeoutMs) {
|
|
819
|
+
const resolved = timeoutMs ?? DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS;
|
|
820
|
+
if (!Number.isSafeInteger(resolved) || resolved < 0 || resolved > 2147483647) {
|
|
821
|
+
throw new TypeError(
|
|
822
|
+
"settlementTimeoutMs must be a non-negative safe integer no greater than 2147483647"
|
|
823
|
+
);
|
|
824
|
+
}
|
|
825
|
+
return resolved;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
// src/analyst/kind-factory.ts
|
|
483
829
|
function createTraceAnalystKind(spec, opts) {
|
|
484
830
|
const version = opts.versionSuffix ? `${spec.version}+${opts.versionSuffix}` : spec.version;
|
|
831
|
+
const model = resolveAnalystModel(opts.ai, opts.model);
|
|
832
|
+
const minimumEvidenceCitations = spec.minimumEvidenceCitations ?? 1;
|
|
833
|
+
if (!Number.isInteger(minimumEvidenceCitations) || minimumEvidenceCitations < 1) {
|
|
834
|
+
throw new TypeError("minimumEvidenceCitations must be a positive integer");
|
|
835
|
+
}
|
|
836
|
+
const settlementTimeoutMs = validateUsageSettlementTimeout(opts.settlementTimeoutMs);
|
|
485
837
|
return {
|
|
486
838
|
id: spec.id,
|
|
487
839
|
description: spec.description,
|
|
@@ -489,130 +841,201 @@ function createTraceAnalystKind(spec, opts) {
|
|
|
489
841
|
cost: spec.cost,
|
|
490
842
|
version,
|
|
491
843
|
async analyze(store, ctx) {
|
|
492
|
-
const
|
|
493
|
-
const
|
|
494
|
-
const
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
permissions: [],
|
|
508
|
-
blockDynamicImport: true,
|
|
509
|
-
allowedModules: [],
|
|
510
|
-
freezeIntrinsics: true,
|
|
511
|
-
blockShadowRealm: true,
|
|
512
|
-
preventGlobalThisExtensions: false
|
|
513
|
-
}),
|
|
514
|
-
mode: maxDepth > 0 ? "advanced" : "simple",
|
|
515
|
-
recursionOptions: maxDepth > 0 ? { maxDepth } : void 0,
|
|
516
|
-
maxTurns: spec.maxTurns ?? 12,
|
|
517
|
-
maxRuntimeChars: spec.maxRuntimeChars ?? 6e3,
|
|
518
|
-
maxBatchedLlmQueryConcurrency: maxParallel,
|
|
519
|
-
promptLevel: "detailed",
|
|
520
|
-
// Trace analysis depends on exact prior tool results and runtime variables.
|
|
521
|
-
contextPolicy: { preset: "full", budget: "balanced" },
|
|
522
|
-
functions,
|
|
523
|
-
actorOptions: {
|
|
524
|
-
description: actorDescription,
|
|
525
|
-
...opts.model ? { model: opts.model } : {},
|
|
526
|
-
showThoughts: false,
|
|
527
|
-
thinkingTokenBudget: "none"
|
|
528
|
-
},
|
|
529
|
-
responderOptions: {
|
|
530
|
-
description: spec.responderDescription ?? "Pass through the actor's `report` prose verbatim, and format the `findings` array exactly as the actor produced it. Do not add, drop, or summarize entries.",
|
|
531
|
-
...opts.model ? { model: opts.model } : {},
|
|
532
|
-
showThoughts: false
|
|
533
|
-
},
|
|
534
|
-
bubbleErrors: [TraceFileMissingError]
|
|
535
|
-
}
|
|
536
|
-
);
|
|
537
|
-
ctx.log?.(`analyst.kind ${spec.id} forward`, {
|
|
538
|
-
max_depth: maxDepth,
|
|
539
|
-
tool_count: tools.length,
|
|
540
|
-
tags: ctx.tags
|
|
844
|
+
const maxOutputTokens = spec.maxOutputTokens ?? 4096;
|
|
845
|
+
const costLedger = ctx.costLedger ?? new CostLedger(ctx.budgetUsd);
|
|
846
|
+
const costTags = {
|
|
847
|
+
...ctx.tags ?? {},
|
|
848
|
+
analystId: spec.id,
|
|
849
|
+
...ctx.correlationId ? { analystRunId: ctx.correlationId } : {}
|
|
850
|
+
};
|
|
851
|
+
const meteredAi = meterAxChatService(opts.ai, {
|
|
852
|
+
ledger: costLedger,
|
|
853
|
+
actor: spec.id,
|
|
854
|
+
maxOutputTokens,
|
|
855
|
+
defaultModel: model,
|
|
856
|
+
phase: ctx.costPhase,
|
|
857
|
+
signal: ctx.signal,
|
|
858
|
+
tags: costTags
|
|
541
859
|
});
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
const
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
860
|
+
try {
|
|
861
|
+
const tools = spec.buildTools(store);
|
|
862
|
+
const maxDepth = spec.recursion?.maxDepth ?? 0;
|
|
863
|
+
const maxParallel = spec.recursion?.maxParallelSubagents ?? 2;
|
|
864
|
+
const priorContext = renderPriorFindings(ctx.priorFindings);
|
|
865
|
+
const upstreamContext = renderUpstreamFindings(ctx.upstreamFindings);
|
|
866
|
+
const functions = tools;
|
|
867
|
+
const actorDescription = spec.actorDescription.trim() + priorContext + upstreamContext + "\n\n" + RAW_FINDING_SCHEMA_PROMPT + (minimumEvidenceCitations > 1 ? `
|
|
868
|
+
|
|
869
|
+
This kind requires at least ${minimumEvidenceCitations} evidence citations per finding; rows with fewer are rejected.` : "") + "\n\nFirst write `report`: a concise free-form prose diagnosis of what the traces show \u2014 what succeeded, what was suboptimal or failed \u2014 with concrete trace ids and numbers. THEN return the structured `findings` array (it MAY be empty when there is nothing to report). Call `final({ report, findings })` exactly once when done.";
|
|
870
|
+
const ax = agent(
|
|
871
|
+
"question:string -> report:string, findings:json[]",
|
|
872
|
+
{
|
|
873
|
+
agentIdentity: {
|
|
874
|
+
name: spec.id,
|
|
875
|
+
description: spec.description
|
|
876
|
+
},
|
|
877
|
+
contextFields: ["question"],
|
|
878
|
+
runtime: new AxJSRuntime({
|
|
879
|
+
permissions: [],
|
|
880
|
+
blockDynamicImport: true,
|
|
881
|
+
allowedModules: [],
|
|
882
|
+
freezeIntrinsics: true,
|
|
883
|
+
blockShadowRealm: true,
|
|
884
|
+
preventGlobalThisExtensions: false
|
|
885
|
+
}),
|
|
886
|
+
mode: maxDepth > 0 ? "advanced" : "simple",
|
|
887
|
+
recursionOptions: maxDepth > 0 ? { maxDepth } : void 0,
|
|
888
|
+
maxTurns: spec.maxTurns ?? 12,
|
|
889
|
+
maxRuntimeChars: spec.maxRuntimeChars ?? 6e3,
|
|
890
|
+
maxBatchedLlmQueryConcurrency: maxParallel,
|
|
891
|
+
promptLevel: "detailed",
|
|
892
|
+
// Trace analysis depends on exact prior tool results and runtime variables.
|
|
893
|
+
contextPolicy: { preset: "full", budget: "balanced" },
|
|
894
|
+
functions,
|
|
895
|
+
actorOptions: {
|
|
896
|
+
description: actorDescription,
|
|
897
|
+
model,
|
|
898
|
+
showThoughts: false,
|
|
899
|
+
thinkingTokenBudget: "none"
|
|
900
|
+
},
|
|
901
|
+
responderOptions: {
|
|
902
|
+
description: spec.responderDescription ?? "Pass through the actor's `report` prose verbatim, and format the `findings` array exactly as the actor produced it. Do not add, drop, or summarize entries.",
|
|
903
|
+
model,
|
|
904
|
+
showThoughts: false
|
|
905
|
+
},
|
|
906
|
+
bubbleErrors: [TraceFileMissingError]
|
|
907
|
+
}
|
|
908
|
+
);
|
|
909
|
+
ctx.log?.(`analyst.kind ${spec.id} forward`, {
|
|
910
|
+
max_depth: maxDepth,
|
|
911
|
+
tool_count: tools.length,
|
|
912
|
+
tags: ctx.tags
|
|
913
|
+
});
|
|
914
|
+
const result = await ax.forward(meteredAi, { question: deriveQuestion(ctx, spec) });
|
|
915
|
+
const expectedSubjects = KIND_EXPECTED_SUBJECTS[spec.id];
|
|
916
|
+
const out = [];
|
|
917
|
+
const rawRows = Array.isArray(result.findings) ? result.findings : [];
|
|
918
|
+
let rejectedWrongKind = 0;
|
|
919
|
+
let rejectedInsufficientEvidence = 0;
|
|
920
|
+
const processRow = (parsed) => {
|
|
921
|
+
const postProcessed = spec.postProcess ? applyLegacyRawFindingCallback(
|
|
922
|
+
parsed,
|
|
923
|
+
(row) => spec.postProcess?.(row, ctx) ?? null,
|
|
924
|
+
ctx.log
|
|
925
|
+
) : parsed;
|
|
926
|
+
if (!postProcessed) return null;
|
|
927
|
+
if (expectedSubjects && postProcessed.subject !== void 0) {
|
|
928
|
+
const parsedSubject = parseFindingSubject(postProcessed.subject);
|
|
929
|
+
if (parsedSubject === null) {
|
|
930
|
+
ctx.log?.("finding rejected: subject failed to parse", {
|
|
931
|
+
kind: spec.id,
|
|
932
|
+
subject: postProcessed.subject
|
|
933
|
+
});
|
|
934
|
+
rejectedWrongKind += 1;
|
|
935
|
+
return null;
|
|
936
|
+
}
|
|
937
|
+
if (!expectedSubjects.includes(parsedSubject.kind)) {
|
|
938
|
+
ctx.log?.("finding rejected: subject variant not allowed for this kind", {
|
|
939
|
+
kind: spec.id,
|
|
940
|
+
subject_kind: parsedSubject.kind,
|
|
941
|
+
subject: postProcessed.subject,
|
|
942
|
+
allowed: expectedSubjects
|
|
943
|
+
});
|
|
944
|
+
rejectedWrongKind += 1;
|
|
945
|
+
return null;
|
|
946
|
+
}
|
|
559
947
|
}
|
|
560
|
-
|
|
561
|
-
|
|
948
|
+
const distinctEvidenceCitations = new Set(
|
|
949
|
+
postProcessed.evidence.map((citation) => citation.uri.trim())
|
|
950
|
+
).size;
|
|
951
|
+
if (distinctEvidenceCitations < minimumEvidenceCitations) {
|
|
952
|
+
ctx.log?.("finding rejected: insufficient evidence citations", {
|
|
562
953
|
kind: spec.id,
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
954
|
+
required: minimumEvidenceCitations,
|
|
955
|
+
received: postProcessed.evidence.length,
|
|
956
|
+
distinct: distinctEvidenceCitations
|
|
566
957
|
});
|
|
567
|
-
|
|
568
|
-
|
|
958
|
+
rejectedInsufficientEvidence += 1;
|
|
959
|
+
return null;
|
|
569
960
|
}
|
|
961
|
+
return postProcessed;
|
|
962
|
+
};
|
|
963
|
+
for (const row of rawRows) {
|
|
964
|
+
const parsed = parseCanonicalRawFinding(row, ctx.log);
|
|
965
|
+
if (!parsed) continue;
|
|
966
|
+
const postProcessed = processRow(parsed);
|
|
967
|
+
if (!postProcessed) continue;
|
|
968
|
+
out.push(toAnalystFinding(spec, version, postProcessed));
|
|
570
969
|
}
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
analystId: spec.id,
|
|
586
|
-
area: spec.area,
|
|
587
|
-
model: opts.recovery.model ?? opts.model ?? "",
|
|
588
|
-
baseUrl: opts.recovery.baseUrl,
|
|
589
|
-
apiKey: opts.recovery.apiKey,
|
|
590
|
-
fetchImpl: opts.recovery.fetchImpl
|
|
591
|
-
});
|
|
592
|
-
out.push(...recovered.findings);
|
|
593
|
-
ctx.log?.(`analyst.kind ${spec.id} recovery`, {
|
|
594
|
-
outcome: recovered.outcome,
|
|
595
|
-
recovered: recovered.findings.length
|
|
596
|
-
});
|
|
597
|
-
}
|
|
598
|
-
if (out.length === 0) {
|
|
599
|
-
out.push(
|
|
600
|
-
makeFinding({
|
|
601
|
-
analyst_id: spec.id,
|
|
970
|
+
ctx.log?.(`analyst.kind ${spec.id} done`, {
|
|
971
|
+
emitted: rawRows.length,
|
|
972
|
+
accepted: out.length,
|
|
973
|
+
rejected_wrong_subject: rejectedWrongKind,
|
|
974
|
+
rejected_insufficient_evidence: rejectedInsufficientEvidence
|
|
975
|
+
});
|
|
976
|
+
const report = typeof result.report === "string" ? result.report : "";
|
|
977
|
+
if (out.length === 0 && report.trim().length >= 200) {
|
|
978
|
+
if (opts.recovery) {
|
|
979
|
+
const wrongKindBefore = rejectedWrongKind;
|
|
980
|
+
const insufficientEvidenceBefore = rejectedInsufficientEvidence;
|
|
981
|
+
const recovered = await structureFindings({
|
|
982
|
+
report,
|
|
983
|
+
analystId: spec.id,
|
|
602
984
|
area: spec.area,
|
|
985
|
+
model: opts.recovery.model ?? model,
|
|
986
|
+
baseUrl: opts.recovery.baseUrl,
|
|
987
|
+
apiKey: opts.recovery.apiKey,
|
|
988
|
+
fetchImpl: opts.recovery.fetchImpl,
|
|
989
|
+
costLedger,
|
|
990
|
+
costPhase: ctx.costPhase,
|
|
991
|
+
costTags,
|
|
992
|
+
signal: ctx.signal,
|
|
993
|
+
maxTokens: Math.min(maxOutputTokens, 2e3),
|
|
994
|
+
processCanonicalRow: processRow,
|
|
995
|
+
findingMetadata: { kind_version: version }
|
|
996
|
+
});
|
|
997
|
+
out.push(...recovered.findings);
|
|
998
|
+
ctx.log?.(`analyst.kind ${spec.id} recovery`, {
|
|
999
|
+
outcome: recovered.outcome,
|
|
1000
|
+
recovered: recovered.findings.length,
|
|
1001
|
+
rejected_wrong_subject: rejectedWrongKind - wrongKindBefore,
|
|
1002
|
+
rejected_insufficient_evidence: rejectedInsufficientEvidence - insufficientEvidenceBefore
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
if (out.length === 0) {
|
|
1006
|
+
const fallback = processRow({
|
|
603
1007
|
claim: "Analyst produced a diagnosis but no structured findings \u2014 see report.",
|
|
604
1008
|
rationale: report.slice(0, 1500),
|
|
605
1009
|
severity: "info",
|
|
606
1010
|
confidence: 0.3,
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
}
|
|
612
|
-
|
|
1011
|
+
evidence: [{ uri: "report://summary", excerpt: report.slice(0, 2e3) }]
|
|
1012
|
+
});
|
|
1013
|
+
if (fallback) {
|
|
1014
|
+
out.push(toAnalystFinding(spec, version, fallback, { outcome: "extraction_failed" }));
|
|
1015
|
+
} else {
|
|
1016
|
+
throw new Error(
|
|
1017
|
+
`Trace analyst '${spec.id}' produced a substantive report, but no finding satisfied its acceptance rules`
|
|
1018
|
+
);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
613
1021
|
}
|
|
1022
|
+
return out;
|
|
1023
|
+
} finally {
|
|
1024
|
+
const usage = await settleUsageReceiptFromCostLedger(costLedger, {
|
|
1025
|
+
tags: {
|
|
1026
|
+
analystId: spec.id,
|
|
1027
|
+
...ctx.correlationId ? { analystRunId: ctx.correlationId } : {}
|
|
1028
|
+
},
|
|
1029
|
+
timeoutMs: settlementTimeoutMs
|
|
1030
|
+
});
|
|
1031
|
+
if (!usage.settled) {
|
|
1032
|
+
ctx.log?.(`analyst.kind ${spec.id} provider settlement timed out`, {
|
|
1033
|
+
pending_calls: usage.pendingCalls,
|
|
1034
|
+
timeout_ms: settlementTimeoutMs
|
|
1035
|
+
});
|
|
1036
|
+
}
|
|
1037
|
+
ctx.recordUsage?.(usage.receipt);
|
|
614
1038
|
}
|
|
615
|
-
return out;
|
|
616
1039
|
}
|
|
617
1040
|
};
|
|
618
1041
|
}
|
|
@@ -621,7 +1044,7 @@ function deriveQuestion(ctx, spec) {
|
|
|
621
1044
|
const task = `Analyze this trace dataset with the available tools and report ${spec.area} findings. ${spec.description}`;
|
|
622
1045
|
return focus ? `${task} Focus: ${focus}.` : task;
|
|
623
1046
|
}
|
|
624
|
-
function toAnalystFinding(spec, raw) {
|
|
1047
|
+
function toAnalystFinding(spec, version, raw, metadata = {}) {
|
|
625
1048
|
return makeFinding({
|
|
626
1049
|
analyst_id: spec.id,
|
|
627
1050
|
area: spec.area,
|
|
@@ -630,25 +1053,11 @@ function toAnalystFinding(spec, raw) {
|
|
|
630
1053
|
rationale: raw.rationale,
|
|
631
1054
|
severity: raw.severity,
|
|
632
1055
|
confidence: raw.confidence,
|
|
633
|
-
evidence_refs:
|
|
634
|
-
{
|
|
635
|
-
kind: evidenceKindFromUri(raw.evidence_uri),
|
|
636
|
-
uri: raw.evidence_uri,
|
|
637
|
-
excerpt: raw.evidence_excerpt
|
|
638
|
-
}
|
|
639
|
-
],
|
|
1056
|
+
evidence_refs: evidenceRefsFromRawFinding(raw),
|
|
640
1057
|
recommended_action: raw.recommended_action,
|
|
641
|
-
metadata: { kind_version:
|
|
1058
|
+
metadata: { kind_version: version, ...metadata }
|
|
642
1059
|
});
|
|
643
1060
|
}
|
|
644
|
-
function evidenceKindFromUri(uri) {
|
|
645
|
-
if (uri.startsWith("span://")) return "span";
|
|
646
|
-
if (uri.startsWith("artifact://")) return "artifact";
|
|
647
|
-
if (uri.startsWith("metric://")) return "metric";
|
|
648
|
-
if (uri.startsWith("event://")) return "event";
|
|
649
|
-
if (uri.startsWith("finding://")) return "finding";
|
|
650
|
-
return "artifact";
|
|
651
|
-
}
|
|
652
1061
|
function renderPriorFindings(prior) {
|
|
653
1062
|
if (!prior || prior.length === 0) return "";
|
|
654
1063
|
const MAX_ROWS = 40;
|
|
@@ -668,6 +1077,26 @@ function renderPriorFindings(prior) {
|
|
|
668
1077
|
overflow
|
|
669
1078
|
].filter(Boolean).join("\n");
|
|
670
1079
|
}
|
|
1080
|
+
function renderUpstreamFindings(upstream) {
|
|
1081
|
+
if (!upstream || upstream.length === 0) return "";
|
|
1082
|
+
const MAX_ROWS = 40;
|
|
1083
|
+
const rows = upstream.slice(0, MAX_ROWS).map((finding) => {
|
|
1084
|
+
const subject = finding.subject ? ` [${finding.subject}]` : "";
|
|
1085
|
+
const action = finding.recommended_action ? ` action=${truncateForContext(finding.recommended_action, 120)}` : "";
|
|
1086
|
+
const evidence = finding.evidence_refs[0] ? ` evidence=${truncateForContext(finding.evidence_refs[0].uri, 120)}` : "";
|
|
1087
|
+
return ` - id=${finding.finding_id} source=${finding.analyst_id} ${finding.severity}${subject} claim=${truncateForContext(finding.claim, 160)}${action}${evidence}`;
|
|
1088
|
+
});
|
|
1089
|
+
const overflow = upstream.length > MAX_ROWS ? `
|
|
1090
|
+
... +${upstream.length - MAX_ROWS} more upstream findings (truncated)` : "";
|
|
1091
|
+
return [
|
|
1092
|
+
"",
|
|
1093
|
+
"",
|
|
1094
|
+
"UPSTREAM FINDINGS (produced earlier in this same registry run):",
|
|
1095
|
+
"Use these as intermediate evidence. Build on them instead of repeating the same diagnosis, and cite a dependency with `finding://<id>`.",
|
|
1096
|
+
...rows,
|
|
1097
|
+
overflow
|
|
1098
|
+
].filter(Boolean).join("\n");
|
|
1099
|
+
}
|
|
671
1100
|
function truncateForContext(s, max) {
|
|
672
1101
|
if (s.length <= max) return s;
|
|
673
1102
|
return `${s.slice(0, max - 1).trimEnd()}\u2026`;
|
|
@@ -713,15 +1142,7 @@ DISCOVERY \u2192 CLUSTER \u2192 CITE protocol:
|
|
|
713
1142
|
2. Use \`traces.queryTraces({ filters: { has_errors: true }, limit })\` to pull error-bearing traces. Combine with \`traces.countTraces\` to see what fraction of the dataset failed.
|
|
714
1143
|
3. For each candidate failure cluster, use \`traces.searchTrace\` with regex like \`STATUS_CODE_ERROR\`, \`MaxTurnsExceeded\`, \`assertion\`, \`unauthorized\`, \`timeout\`, \`429\`, \`5\\d\\d\`, the agent's specific error strings, or the names of its tools. Pull one or two representative traces per cluster, **not all** of them.
|
|
715
1144
|
4. **Cluster, do not enumerate.** Two failures with the same root cause should be ONE finding citing both traces, not two findings. The point of this analyst is to compress N runs into K modes.
|
|
716
|
-
5. For each cluster
|
|
717
|
-
- \`area\` = "failure-mode"
|
|
718
|
-
- \`subject\` = a lowercase cluster label matching the subject grammar above ("tool-call-loop", "auth-revoked-mid-run", ...)
|
|
719
|
-
- \`claim\` = one sentence stating the mode
|
|
720
|
-
- \`severity\` = "critical" when it blocks the run, "high" when the run finished degraded, "medium" when it slowed convergence
|
|
721
|
-
- \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the most representative span
|
|
722
|
-
- \`evidence_excerpt\` = the exact quote (e.g. error message, stuck tool call payload, contradictory turn output)
|
|
723
|
-
- \`confidence\` = 0.85+ when multiple traces show the same shape; 0.6-0.8 for a single-trace inference; <0.5 for speculative.
|
|
724
|
-
- \`recommended_action\` = imperative-phrased fix idea (kept short \u2014 the improvement-analyst will expand on these)
|
|
1145
|
+
5. For each defensible cluster, emit ONE finding. Use a lowercase cluster label matching the subject grammar ("tool-call-loop", "auth-revoked-mid-run", ...). Rate it critical when it blocks the run, high when the run finishes degraded, and medium when it slows convergence. Cite representative spans and include exact error, payload, or contradictory-output quotes. Use confidence 0.85+ when multiple traces show the same shape, 0.6-0.8 for a single-trace inference, and <0.5 for speculation. Keep the imperative fix idea short; the improvement analyst expands it.
|
|
725
1146
|
|
|
726
1147
|
If the dataset has no failures, return an empty findings array \u2014 do NOT pad with low-confidence speculation.
|
|
727
1148
|
|
|
@@ -729,17 +1150,16 @@ If the dataset has no failures, return an empty findings array \u2014 do NOT pad
|
|
|
729
1150
|
- After your first \`getDatasetOverview\` + \`queryTraces\` calls, you should have 3-6 candidate failure clusters in mind. Spawn one \`llmQuery\` per cluster in a single batch \u2014 they investigate in parallel.
|
|
730
1151
|
- A sub-investigator that finds its cluster is actually two distinct modes should split again at its own level. Recursion is meant to discover sub-modes, not to do trivial drilling that the parent could do in-line.
|
|
731
1152
|
- Pass narrow context to each subagent: { question: 'investigate the auth-revoked-mid-run cluster', context: { trace_ids: ['abc', 'def'], suspected_root_cause: 'token refresh skipped on idle sessions' } }. Subagents need enough context to skip re-discovery but not the whole conversation.
|
|
732
|
-
- Each subagent returns
|
|
1153
|
+
- Each subagent returns candidate cluster evidence; the parent merges it into the final finding set.
|
|
733
1154
|
|
|
734
1155
|
OBSERVABILITY rules:
|
|
735
1156
|
- Each non-final turn must emit at least one \`console.log\` for evidence.
|
|
736
|
-
- Reuse runtime variables across turns; don't recompute
|
|
737
|
-
- Call \`final({ findings: [...] })\` exactly once, after you've gathered evidence for every cluster you intend to report.`;
|
|
1157
|
+
- Reuse runtime variables across turns; don't recompute.`;
|
|
738
1158
|
var FAILURE_MODE_KIND_SPEC = {
|
|
739
1159
|
id: "failure-mode",
|
|
740
1160
|
description: "Clusters trace-dataset failures into distinct failure modes with cited evidence and a short recommended action.",
|
|
741
1161
|
area: "failure-mode",
|
|
742
|
-
version: "1.
|
|
1162
|
+
version: "1.1.0",
|
|
743
1163
|
actorDescription: ACTOR_PROMPT,
|
|
744
1164
|
buildTools: (store) => buildTraceToolsForGroup("all", store),
|
|
745
1165
|
recursion: { maxDepth: 3, maxParallelSubagents: 4 },
|
|
@@ -771,31 +1191,21 @@ DISCOVERY \u2192 CANDIDATE-FIXES \u2192 COMPETE \u2192 CITE protocol:
|
|
|
771
1191
|
- **Code** \u2014 change an implementation path when profile edits cannot repair the behavior
|
|
772
1192
|
3. **Compete candidate fixes via subagents.** For each failure cluster, spawn one \`llmQuery\` per candidate-fix axis you want to evaluate. Each subagent's job: simulate the fix on the cited traces and report (i) likely effect, (ii) side effects, (iii) implementation cost as small/medium/large. Pass the cluster's failing trace_ids and the candidate axis as context.
|
|
773
1193
|
4. After subagents return, **pick the winning candidate per cluster** based on (effect / cost) and emit ONE finding. Discard the losing candidates \u2014 the output is the recommendation, not the candidate set.
|
|
774
|
-
5. **Cross-reference upstream findings.**
|
|
1194
|
+
5. **Cross-reference upstream findings.** Cite prior failure-mode or knowledge-gap findings as \`finding://<prior-finding-id>\`. This builds the dependency graph that lets the dashboard show "fix #X resolves failure modes A, B, C."
|
|
775
1195
|
|
|
776
|
-
For each winning recommendation, emit ONE finding with
|
|
777
|
-
- \`area\` = "improvement"
|
|
778
|
-
- \`subject\` = one exact locus form from the subject grammar above
|
|
779
|
-
- \`claim\` = one sentence stating the edit ("Add a precondition check to refuse tool X calls without arg Y")
|
|
780
|
-
- \`severity\` = leverage rating: "critical" when fix resolves a critical failure mode; "high" when it resolves a high; "medium" when it's a quality-of-life win; "info" when it's a cleanup with no behavioral effect
|
|
781
|
-
- \`evidence_uri\` = the failure-mode finding id this fix targets (\`finding://<id>\`) when it exists; else the most representative span
|
|
782
|
-
- \`evidence_excerpt\` = a fragment showing the problem the fix targets
|
|
783
|
-
- \`confidence\` = 0.85+ when the fix is mechanical and the failure mode is well-evidenced; 0.6-0.8 when the fix requires judgment; <0.5 for speculative
|
|
784
|
-
- \`rationale\` = why this candidate beat its alternatives (2 sentences max)
|
|
785
|
-
- \`recommended_action\` = the **literal edit**, phrased as a diff or a quoted replacement: "Replace section X with: '...'" or "Add tool with description: '...'" or "Set retry policy to max_attempts=3 with exponential backoff"
|
|
1196
|
+
For each winning recommendation, emit ONE finding. Use one exact locus from the subject grammar and state the edit in one sentence. Match leverage to the source failure's severity; use medium for quality-of-life changes and info for cleanup with no behavioral effect. Cite the targeted \`finding://<id>\` when available and the most representative span when useful. Quote the problem being fixed. Use confidence 0.85+ for a mechanical fix to a well-evidenced failure, 0.6-0.8 when judgment is required, and <0.5 for speculation. Explain in at most two sentences why this candidate beat its alternatives. The recommended action must be the literal diff, quoted replacement, tool description, or setting change.
|
|
786
1197
|
|
|
787
1198
|
If no upstream failure findings exist in this run, derive your own from the trace dataset using the failure-mode protocol inline (\`searchTrace\` for STATUS_CODE_ERROR / MaxTurnsExceeded / etc.). But prefer to consume upstream findings when present \u2014 the kinds are designed to chain.
|
|
788
1199
|
|
|
789
1200
|
Do NOT propose a fix you cannot defend with evidence. "Tighten the prompt" is not a finding; "Add 'When the user asks for X, always Y' to the system prompt section "request-classification"" is.
|
|
790
1201
|
|
|
791
1202
|
OBSERVABILITY rules:
|
|
792
|
-
- Each non-final turn must emit at least one \`console.log\` for evidence
|
|
793
|
-
- Call \`final({ findings: [...] })\` exactly once at the top level.`;
|
|
1203
|
+
- Each non-final turn must emit at least one \`console.log\` for evidence.`;
|
|
794
1204
|
var IMPROVEMENT_KIND_SPEC = {
|
|
795
1205
|
id: "improvement",
|
|
796
1206
|
description: "Converts upstream failure / gap / poisoning findings into concrete locus-named edits (prompt, tool-doc, RAG, scaffolding) with leverage grades.",
|
|
797
1207
|
area: "improvement",
|
|
798
|
-
version: "1.
|
|
1208
|
+
version: "1.1.0",
|
|
799
1209
|
actorDescription: ACTOR_PROMPT2,
|
|
800
1210
|
buildTools: (store) => buildTraceToolsForGroup("all", store),
|
|
801
1211
|
recursion: { maxDepth: 3, maxParallelSubagents: 4 },
|
|
@@ -825,28 +1235,19 @@ DISCOVERY \u2192 ATTRIBUTE-TO-LAYER \u2192 CITE protocol:
|
|
|
825
1235
|
- Fabricated identifiers that don't appear in dataset \`sample_trace_ids\`
|
|
826
1236
|
Use \`traces.searchTrace\` with patterns like \`I (don.?t|do not) know\`, \`assumed\`, \`unclear\`, \`could you (clarify|tell me|provide)\`, \`not found\`, \`undefined\`, \`unknown\`, \`null\`, dates older than the analysis window, or the agent's specific clarification phrases.
|
|
827
1237
|
3. For each gap, identify the **layer of the runtime that should have prevented it** and use its exact locus from the subject grammar above.
|
|
828
|
-
4. For each gap
|
|
829
|
-
- \`area\` = "knowledge-gap"
|
|
830
|
-
- \`subject\` = one exact locus form from the subject grammar above
|
|
831
|
-
- \`claim\` = a sentence naming the missing or stale knowledge ("wiki has no page on invoice line-item shape, agent had to re-derive it from raw spans")
|
|
832
|
-
- \`severity\` = "high" when the gap caused a failure or a clarifying question; "medium" when it caused unnecessary turns; "low" when it caused minor inefficiency
|
|
833
|
-
- \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the moment the gap surfaced (the question, the self-correction, the retrieval miss, the stale web result)
|
|
834
|
-
- \`evidence_excerpt\` = exact quote where the agent showed the gap
|
|
835
|
-
- \`confidence\` = 0.85+ when the agent itself articulated the gap; 0.6-0.8 when inferred from behavior
|
|
836
|
-
- \`recommended_action\` = phrased as a wiki edit when the locus is \`agent-knowledge:*\` ("Create wiki page \`invoice-line-items\` with claims: ..."), or as a prompt/tool-doc edit otherwise
|
|
1238
|
+
4. For each defensible gap, emit ONE finding. Use an exact locus from the subject grammar and name the missing or stale knowledge (for example, "wiki has no page on invoice line-item shape; agent re-derived it from raw spans"). Rate it high when it caused failure or a clarifying question, medium for unnecessary turns, and low for minor inefficiency. Cite the span where the question, correction, retrieval miss, or stale result surfaced and quote it exactly. Use confidence 0.85+ when the agent articulated the gap and 0.6-0.8 when inferred. Recommend a concrete wiki edit for an agent-knowledge locus or a prompt/tool-description edit otherwise.
|
|
837
1239
|
|
|
838
|
-
**Delegate per layer.** After your first scan, you should have candidates spread across \`agent-knowledge:*\`, \`websearch:outdated\`, \`tool-doc:*\`, \`system-prompt:*\`, and \`memory:*\`. Spawn one \`llmQuery\` per layer in parallel \u2014 each subagent runs a focused detection (e.g. the \`agent-knowledge\` subagent looks for both missing-pages AND stale-pages; the \`websearch\` subagent looks specifically for date staleness signals; the \`tool-doc\` subagent looks for tool-call argument errors a fuller description would have prevented).
|
|
1240
|
+
**Delegate per layer.** After your first scan, you should have candidates spread across \`agent-knowledge:*\`, \`websearch:outdated\`, \`tool-doc:*\`, \`system-prompt:*\`, and \`memory:*\`. Spawn one \`llmQuery\` per layer in parallel \u2014 each subagent runs a focused detection (e.g. the \`agent-knowledge\` subagent looks for both missing-pages AND stale-pages; the \`websearch\` subagent looks specifically for date staleness signals; the \`tool-doc\` subagent looks for tool-call argument errors a fuller description would have prevented). Merge their evidence into the final finding set.
|
|
839
1241
|
|
|
840
1242
|
Do NOT report a gap that the agent later recovered from cleanly within the same turn \u2014 that's resilience, not a gap. Cite the *non-recovery* version when both exist.
|
|
841
1243
|
|
|
842
1244
|
OBSERVABILITY rules:
|
|
843
|
-
- Each non-final turn must emit at least one \`console.log\` for evidence
|
|
844
|
-
- Call \`final({ findings: [...] })\` exactly once at the top level.`;
|
|
1245
|
+
- Each non-final turn must emit at least one \`console.log\` for evidence.`;
|
|
845
1246
|
var KNOWLEDGE_GAP_KIND_SPEC = {
|
|
846
1247
|
id: "knowledge-gap",
|
|
847
1248
|
description: "Identifies missing or stale pieces of knowledge \u2014 primarily against the agent-knowledge wiki \u2014 and attributes each to the runtime layer (wiki page, claim, raw source, websearch, tool-doc, system-prompt, memory) that should have held it.",
|
|
848
1249
|
area: "knowledge-gap",
|
|
849
|
-
version: "1.
|
|
1250
|
+
version: "1.1.0",
|
|
850
1251
|
actorDescription: ACTOR_PROMPT3,
|
|
851
1252
|
buildTools: (store) => buildTraceToolsForGroup("discoveryAndSearch", store),
|
|
852
1253
|
recursion: { maxDepth: 2, maxParallelSubagents: 4 },
|
|
@@ -878,30 +1279,22 @@ DISCOVERY \u2192 DUAL-VERIFY \u2192 CITE protocol:
|
|
|
878
1279
|
|
|
879
1280
|
**Delegate the dual-verify.** Use the recursion budget so each candidate poisoning gets one subagent investigating "did the agent act?" and one investigating "is the belief false?". After your first scan, fire off N parallel \`llmQuery\` pairs (one cluster per pair). Subagents return their findings; you accept only the ones where BOTH halves of the pair were confirmed.
|
|
880
1281
|
|
|
881
|
-
For each confirmed poisoning, emit ONE finding with:
|
|
882
|
-
- \`area\` = "knowledge-poisoning"
|
|
883
|
-
- \`subject\` = the source of the false belief using one exact form from the subject grammar above
|
|
884
|
-
- \`claim\` = one sentence: "agent believed X (from source S); evidence in trace shows X is false"
|
|
885
|
-
- \`severity\` = "critical" when poisoning caused a wrong user-visible action; "high" when caught internally but wasted significant work; "medium" for inefficiency only
|
|
886
|
-
- \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the action span (the moment the agent acted on the false belief)
|
|
887
|
-
- \`evidence_excerpt\` = exact quote of the confident-but-wrong claim or action
|
|
888
|
-
- \`confidence\` = 0.85+ when both halves are exact-quote backed; 0.6-0.8 when one half is inferred
|
|
889
|
-
- \`recommended_action\` = where the source should be updated and how ("Update wiki page \`X\` claim \`Y\` to '...'", "Invalidate raw source \`Z\` and re-curate", "Replace system-prompt section X with 'tool foo now returns Y'")
|
|
1282
|
+
For each confirmed poisoning, emit ONE finding. Use the source of the false belief as the exact subject. State "agent believed X (from source S); trace evidence shows X is false." Rate it critical for a wrong user-visible action, high when caught internally after significant waste, and medium for inefficiency. Cite BOTH the action span and the contradicting span with exact quotes. Use confidence 0.85+ when both halves have exact quotes and 0.6-0.8 when one half is inferred. Recommend the literal source correction: update the wiki claim, invalidate and re-curate the raw source, or replace the stale prompt/tool instruction.
|
|
890
1283
|
|
|
891
1284
|
Do NOT report a finding if the agent caught and corrected the false belief in the same turn \u2014 that's the system working. Reserve poisoning for cases where the false belief shaped downstream action.
|
|
892
1285
|
|
|
893
1286
|
OBSERVABILITY rules:
|
|
894
|
-
- Each non-final turn must emit at least one \`console.log\` for evidence
|
|
895
|
-
- Call \`final({ findings: [...] })\` exactly once at the top level.`;
|
|
1287
|
+
- Each non-final turn must emit at least one \`console.log\` for evidence.`;
|
|
896
1288
|
var KNOWLEDGE_POISONING_KIND_SPEC = {
|
|
897
1289
|
id: "knowledge-poisoning",
|
|
898
1290
|
description: "Identifies confident-but-wrong actions caused by stale memory, contradicting RAG, deprecated tool docs, or outdated system-prompt instructions.",
|
|
899
1291
|
area: "knowledge-poisoning",
|
|
900
|
-
version: "1.
|
|
1292
|
+
version: "1.1.0",
|
|
901
1293
|
actorDescription: ACTOR_PROMPT4,
|
|
902
1294
|
buildTools: (store) => buildTraceToolsForGroup("all", store),
|
|
903
1295
|
recursion: { maxDepth: 2, maxParallelSubagents: 4 },
|
|
904
1296
|
maxTurns: 20,
|
|
1297
|
+
minimumEvidenceCitations: 2,
|
|
905
1298
|
cost: { kind: "llm" }
|
|
906
1299
|
};
|
|
907
1300
|
|
|
@@ -966,6 +1359,7 @@ var AnalystRegistry = class {
|
|
|
966
1359
|
const deadlineMs = runOpts.timeoutMs ? started + runOpts.timeoutMs : void 0;
|
|
967
1360
|
const selected = this.selectAnalysts(runOpts);
|
|
968
1361
|
const budget = runOpts.budget ?? this.options.defaultBudget;
|
|
1362
|
+
validateBudgetPolicy(budget);
|
|
969
1363
|
yield {
|
|
970
1364
|
type: "run-started",
|
|
971
1365
|
run_id: runId,
|
|
@@ -977,9 +1371,13 @@ var AnalystRegistry = class {
|
|
|
977
1371
|
const allFindings = [];
|
|
978
1372
|
let totalCost = 0;
|
|
979
1373
|
let remainingUsd = budget?.totalUsd;
|
|
980
|
-
const
|
|
981
|
-
|
|
982
|
-
|
|
1374
|
+
const runnableAnalysts = selected.filter((a) => this.routeInput(a, inputs).kind !== "missing");
|
|
1375
|
+
const runnableCount = runnableAnalysts.length;
|
|
1376
|
+
const weights = budget?.weights;
|
|
1377
|
+
const totalWeight = weights && budget?.totalUsd != null && !budget.allocate && runnableCount > 0 ? runnableAnalysts.reduce((sum, analyst) => sum + analystWeight(weights, analyst.id), 0) : void 0;
|
|
1378
|
+
if (totalWeight === 0) {
|
|
1379
|
+
throw new Error("BudgetPolicy.weights must allocate positive weight to a runnable analyst");
|
|
1380
|
+
}
|
|
983
1381
|
for (const analyst of selected) {
|
|
984
1382
|
const t0 = Date.now();
|
|
985
1383
|
const input = this.routeInput(analyst, inputs);
|
|
@@ -990,7 +1388,8 @@ var AnalystRegistry = class {
|
|
|
990
1388
|
reason: `missing input of kind '${analyst.inputKind}'`,
|
|
991
1389
|
findings_count: 0,
|
|
992
1390
|
latency_ms: 0,
|
|
993
|
-
cost_usd: 0
|
|
1391
|
+
cost_usd: 0,
|
|
1392
|
+
usage: zeroUsage()
|
|
994
1393
|
};
|
|
995
1394
|
summaries.push(summary);
|
|
996
1395
|
log(`[analyst] skip ${analyst.id} \u2014 missing input`, { runId, kind: analyst.inputKind });
|
|
@@ -1001,20 +1400,30 @@ var AnalystRegistry = class {
|
|
|
1001
1400
|
const perBudget = allocateBudget(budget, {
|
|
1002
1401
|
analyst,
|
|
1003
1402
|
remainingUsd,
|
|
1004
|
-
runningCount: runnableCount
|
|
1403
|
+
runningCount: runnableCount,
|
|
1404
|
+
totalWeight
|
|
1005
1405
|
});
|
|
1406
|
+
const usageReceipts = [];
|
|
1006
1407
|
const ctx = {
|
|
1007
1408
|
runId,
|
|
1008
1409
|
correlationId,
|
|
1009
1410
|
deadlineMs,
|
|
1010
1411
|
budgetUsd: perBudget,
|
|
1412
|
+
costLedger: runOpts.costLedger,
|
|
1413
|
+
costPhase: runOpts.costPhase,
|
|
1011
1414
|
chat: this.options.chat,
|
|
1012
1415
|
tags: runOpts.tags,
|
|
1013
1416
|
log: (msg, fields) => log(`[${analyst.id}] ${msg}`, { runId, correlationId, ...fields }),
|
|
1014
1417
|
signal: runOpts.signal,
|
|
1015
|
-
priorFindings: selectPriorFindings(runOpts.priorFindings, analyst.id)
|
|
1418
|
+
priorFindings: selectPriorFindings(runOpts.priorFindings, analyst.id),
|
|
1419
|
+
upstreamFindings: runOpts.chainFindings && allFindings.length > 0 ? [...allFindings] : void 0,
|
|
1420
|
+
recordUsage: (receipt) => {
|
|
1421
|
+
assertValidUsageReceipt(receipt);
|
|
1422
|
+
usageReceipts.push(receipt);
|
|
1423
|
+
}
|
|
1016
1424
|
};
|
|
1017
1425
|
await hooks.onBeforeAnalyze?.({ analyst, ctx, runId });
|
|
1426
|
+
const effectiveBudget = validateEffectiveBudget(ctx.budgetUsd, remainingUsd, analyst.id);
|
|
1018
1427
|
yield {
|
|
1019
1428
|
type: "analyst-started",
|
|
1020
1429
|
analyst_id: analyst.id,
|
|
@@ -1023,24 +1432,38 @@ var AnalystRegistry = class {
|
|
|
1023
1432
|
try {
|
|
1024
1433
|
const findings = await analyst.analyze(input.value, ctx);
|
|
1025
1434
|
const latency = Date.now() - t0;
|
|
1026
|
-
const
|
|
1435
|
+
const usage = resolveUsage(analyst, findings, usageReceipts);
|
|
1436
|
+
const cost = knownCostUsd(usage);
|
|
1027
1437
|
totalCost += cost;
|
|
1028
|
-
if (typeof remainingUsd === "number")
|
|
1438
|
+
if (typeof remainingUsd === "number") {
|
|
1439
|
+
remainingUsd = Math.max(0, remainingUsd - budgetDebit(usage, effectiveBudget));
|
|
1440
|
+
}
|
|
1029
1441
|
allFindings.push(...findings);
|
|
1030
1442
|
const summary = {
|
|
1031
1443
|
analyst_id: analyst.id,
|
|
1032
1444
|
status: "ok",
|
|
1033
1445
|
findings_count: findings.length,
|
|
1034
1446
|
latency_ms: latency,
|
|
1035
|
-
cost_usd: cost
|
|
1447
|
+
cost_usd: cost,
|
|
1448
|
+
usage
|
|
1036
1449
|
};
|
|
1037
1450
|
summaries.push(summary);
|
|
1038
1451
|
log(`[analyst] ok ${analyst.id}`, {
|
|
1039
1452
|
runId,
|
|
1040
1453
|
findings: findings.length,
|
|
1041
1454
|
latency_ms: latency,
|
|
1042
|
-
cost_usd: cost
|
|
1455
|
+
cost_usd: cost,
|
|
1456
|
+
cost_kind: usage.cost.kind,
|
|
1457
|
+
input_tokens: usage.tokens?.input ?? null,
|
|
1458
|
+
output_tokens: usage.tokens?.output ?? null
|
|
1043
1459
|
});
|
|
1460
|
+
if (effectiveBudget !== void 0 && usage.cost.kind === "uncaptured") {
|
|
1461
|
+
log(`[analyst] WARN ${analyst.id} \u2014 USD cost uncaptured; budget not reconciled`, {
|
|
1462
|
+
runId,
|
|
1463
|
+
budget_usd: effectiveBudget,
|
|
1464
|
+
cost_captured: false
|
|
1465
|
+
});
|
|
1466
|
+
}
|
|
1044
1467
|
await hooks.onAfterAnalyze?.({ analyst, summary, findings, runId });
|
|
1045
1468
|
yield { type: "analyst-completed", summary, findings };
|
|
1046
1469
|
} catch (err) {
|
|
@@ -1048,20 +1471,36 @@ var AnalystRegistry = class {
|
|
|
1048
1471
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
1049
1472
|
const hookFindings = await hooks.onError?.({ analyst, error: e, runId }) ?? [];
|
|
1050
1473
|
if (hookFindings.length) allFindings.push(...hookFindings);
|
|
1474
|
+
const usage = resolveUsage(analyst, hookFindings, usageReceipts);
|
|
1475
|
+
const cost = knownCostUsd(usage);
|
|
1476
|
+
totalCost += cost;
|
|
1477
|
+
if (typeof remainingUsd === "number") {
|
|
1478
|
+
remainingUsd = Math.max(0, remainingUsd - budgetDebit(usage, effectiveBudget));
|
|
1479
|
+
}
|
|
1051
1480
|
const summary = {
|
|
1052
1481
|
analyst_id: analyst.id,
|
|
1053
1482
|
status: "failed",
|
|
1054
1483
|
findings_count: hookFindings.length,
|
|
1055
1484
|
latency_ms: latency,
|
|
1056
|
-
cost_usd:
|
|
1485
|
+
cost_usd: cost,
|
|
1486
|
+
usage,
|
|
1057
1487
|
error: { class: e.constructor.name, message: e.message }
|
|
1058
1488
|
};
|
|
1059
1489
|
summaries.push(summary);
|
|
1060
1490
|
log(`[analyst] FAIL ${analyst.id}`, {
|
|
1061
1491
|
runId,
|
|
1062
1492
|
error_class: e.constructor.name,
|
|
1063
|
-
error: e.message
|
|
1493
|
+
error: e.message,
|
|
1494
|
+
cost_usd: cost,
|
|
1495
|
+
cost_kind: usage.cost.kind
|
|
1064
1496
|
});
|
|
1497
|
+
if (effectiveBudget !== void 0 && usage.cost.kind === "uncaptured") {
|
|
1498
|
+
log(`[analyst] WARN ${analyst.id} \u2014 USD cost uncaptured; budget not reconciled`, {
|
|
1499
|
+
runId,
|
|
1500
|
+
budget_usd: effectiveBudget,
|
|
1501
|
+
cost_captured: false
|
|
1502
|
+
});
|
|
1503
|
+
}
|
|
1065
1504
|
await hooks.onAfterAnalyze?.({ analyst, summary, findings: hookFindings, runId });
|
|
1066
1505
|
yield { type: "analyst-completed", summary, findings: hookFindings };
|
|
1067
1506
|
}
|
|
@@ -1073,7 +1512,10 @@ var AnalystRegistry = class {
|
|
|
1073
1512
|
ended_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1074
1513
|
findings: allFindings,
|
|
1075
1514
|
per_analyst: summaries,
|
|
1076
|
-
total_cost_usd: totalCost
|
|
1515
|
+
total_cost_usd: totalCost,
|
|
1516
|
+
total_cost_provenance: aggregateCostProvenance(
|
|
1517
|
+
summaries.map((summary) => summary.usage?.cost ?? { kind: "uncaptured", usd: null })
|
|
1518
|
+
)
|
|
1077
1519
|
};
|
|
1078
1520
|
await hooks.onComplete?.({ result });
|
|
1079
1521
|
yield { type: "run-completed", result };
|
|
@@ -1110,28 +1552,171 @@ var AnalystRegistry = class {
|
|
|
1110
1552
|
function allocateBudget(policy, args) {
|
|
1111
1553
|
if (!policy) return void 0;
|
|
1112
1554
|
if (policy.allocate) {
|
|
1113
|
-
|
|
1555
|
+
const allocated2 = policy.allocate({
|
|
1114
1556
|
analyst: args.analyst,
|
|
1115
1557
|
totalUsd: policy.totalUsd,
|
|
1116
1558
|
remainingUsd: args.remainingUsd,
|
|
1117
1559
|
runningCount: args.runningCount
|
|
1118
1560
|
});
|
|
1561
|
+
if (allocated2 === void 0) {
|
|
1562
|
+
if (policy.totalUsd !== void 0) {
|
|
1563
|
+
throw new Error(
|
|
1564
|
+
`BudgetPolicy.allocate('${args.analyst.id}') cannot return undefined when totalUsd is set`
|
|
1565
|
+
);
|
|
1566
|
+
}
|
|
1567
|
+
return void 0;
|
|
1568
|
+
}
|
|
1569
|
+
assertBudgetAmount(allocated2, `BudgetPolicy.allocate('${args.analyst.id}')`);
|
|
1570
|
+
return args.remainingUsd === void 0 ? allocated2 : Math.min(allocated2, args.remainingUsd);
|
|
1119
1571
|
}
|
|
1120
1572
|
if (policy.totalUsd == null) return void 0;
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1573
|
+
const allocated = policy.weights ? policy.totalUsd * analystWeight(policy.weights, args.analyst.id) / args.totalWeight : policy.totalUsd / Math.max(1, args.runningCount);
|
|
1574
|
+
return args.remainingUsd === void 0 ? allocated : Math.min(allocated, args.remainingUsd);
|
|
1575
|
+
}
|
|
1576
|
+
function validateBudgetPolicy(policy) {
|
|
1577
|
+
if (!policy) return;
|
|
1578
|
+
if (policy.totalUsd !== void 0) assertBudgetAmount(policy.totalUsd, "BudgetPolicy.totalUsd");
|
|
1579
|
+
for (const [analystId, weight] of Object.entries(policy.weights ?? {})) {
|
|
1580
|
+
assertBudgetAmount(weight, `BudgetPolicy.weights['${analystId}']`);
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1583
|
+
function assertBudgetAmount(value, field) {
|
|
1584
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
1585
|
+
throw new Error(`${field} must be a non-negative finite number`);
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
function validateEffectiveBudget(budgetUsd, remainingUsd, analystId) {
|
|
1589
|
+
if (budgetUsd !== void 0) {
|
|
1590
|
+
assertBudgetAmount(budgetUsd, `AnalystContext.budgetUsd for '${analystId}'`);
|
|
1591
|
+
}
|
|
1592
|
+
if (remainingUsd === void 0) return budgetUsd;
|
|
1593
|
+
if (budgetUsd === void 0) {
|
|
1594
|
+
throw new Error(
|
|
1595
|
+
`AnalystContext.budgetUsd for '${analystId}' cannot be removed while an overall budget remains`
|
|
1596
|
+
);
|
|
1597
|
+
}
|
|
1598
|
+
if (budgetUsd > remainingUsd) {
|
|
1599
|
+
throw new Error(
|
|
1600
|
+
`AnalystContext.budgetUsd for '${analystId}' (${budgetUsd}) exceeds the remaining overall budget (${remainingUsd})`
|
|
1601
|
+
);
|
|
1602
|
+
}
|
|
1603
|
+
return budgetUsd;
|
|
1604
|
+
}
|
|
1605
|
+
function analystWeight(weights, analystId) {
|
|
1606
|
+
const weight = weights[analystId] ?? 1;
|
|
1607
|
+
assertBudgetAmount(weight, `BudgetPolicy.weights['${analystId}']`);
|
|
1608
|
+
return weight;
|
|
1609
|
+
}
|
|
1610
|
+
function zeroUsage() {
|
|
1611
|
+
return {
|
|
1612
|
+
calls: 0,
|
|
1613
|
+
tokens: { input: 0, output: 0 },
|
|
1614
|
+
cost: { kind: "observed", usd: 0 }
|
|
1615
|
+
};
|
|
1616
|
+
}
|
|
1617
|
+
function resolveUsage(analyst, findings, receipts) {
|
|
1618
|
+
const legacyCost = sumFindingCost(findings);
|
|
1619
|
+
if (receipts.length > 0) {
|
|
1620
|
+
const merged = mergeUsageReceipts(receipts);
|
|
1621
|
+
return merged.cost.kind === "uncaptured" && legacyCost.captured ? { ...merged, knownCostUsd: Math.max(merged.knownCostUsd ?? 0, legacyCost.usd) } : merged;
|
|
1622
|
+
}
|
|
1623
|
+
if (legacyCost.captured) {
|
|
1624
|
+
return {
|
|
1625
|
+
calls: null,
|
|
1626
|
+
tokens: null,
|
|
1627
|
+
cost: { kind: "observed", usd: legacyCost.usd }
|
|
1628
|
+
};
|
|
1629
|
+
}
|
|
1630
|
+
if (analyst.cost.kind === "deterministic") return zeroUsage();
|
|
1631
|
+
return { calls: null, tokens: null, cost: { kind: "uncaptured", usd: null } };
|
|
1632
|
+
}
|
|
1633
|
+
function mergeUsageReceipts(receipts) {
|
|
1634
|
+
const calls = receipts.every((receipt) => receipt.calls !== null) ? receipts.reduce((sum, receipt) => sum + (receipt.calls ?? 0), 0) : null;
|
|
1635
|
+
const tokens = receipts.every((receipt) => receipt.tokens !== null) ? receipts.reduce(
|
|
1636
|
+
(sum, receipt) => ({
|
|
1637
|
+
input: sum.input + (receipt.tokens?.input ?? 0),
|
|
1638
|
+
output: sum.output + (receipt.tokens?.output ?? 0),
|
|
1639
|
+
...sum.reasoning !== void 0 || receipt.tokens?.reasoning !== void 0 ? { reasoning: (sum.reasoning ?? 0) + (receipt.tokens?.reasoning ?? 0) } : {},
|
|
1640
|
+
...sum.cached !== void 0 || receipt.tokens?.cached !== void 0 ? { cached: (sum.cached ?? 0) + (receipt.tokens?.cached ?? 0) } : {},
|
|
1641
|
+
...sum.cacheWrite !== void 0 || receipt.tokens?.cacheWrite !== void 0 ? { cacheWrite: (sum.cacheWrite ?? 0) + (receipt.tokens?.cacheWrite ?? 0) } : {}
|
|
1642
|
+
}),
|
|
1643
|
+
{ input: 0, output: 0 }
|
|
1644
|
+
) : null;
|
|
1645
|
+
const cost = aggregateCostProvenance(receipts.map((receipt) => receipt.cost));
|
|
1646
|
+
return {
|
|
1647
|
+
calls,
|
|
1648
|
+
tokens,
|
|
1649
|
+
cost,
|
|
1650
|
+
...cost.kind === "uncaptured" ? {
|
|
1651
|
+
knownCostUsd: receipts.reduce((sum, receipt) => sum + knownCostUsd(receipt), 0)
|
|
1652
|
+
} : {}
|
|
1653
|
+
};
|
|
1654
|
+
}
|
|
1655
|
+
function knownCostUsd(receipt) {
|
|
1656
|
+
return receipt.cost.kind === "uncaptured" ? receipt.knownCostUsd ?? 0 : receipt.cost.usd;
|
|
1657
|
+
}
|
|
1658
|
+
function budgetDebit(receipt, allocatedUsd) {
|
|
1659
|
+
const known = knownCostUsd(receipt);
|
|
1660
|
+
return receipt.cost.kind === "uncaptured" && allocatedUsd !== void 0 ? Math.max(known, allocatedUsd) : known;
|
|
1661
|
+
}
|
|
1662
|
+
function aggregateCostProvenance(costs) {
|
|
1663
|
+
if (costs.some((cost) => cost.kind === "uncaptured")) {
|
|
1664
|
+
return { kind: "uncaptured", usd: null };
|
|
1665
|
+
}
|
|
1666
|
+
const usd = costs.reduce((sum, cost) => sum + (cost.usd ?? 0), 0);
|
|
1667
|
+
return costs.some((cost) => cost.kind === "estimated") ? { kind: "estimated", usd } : { kind: "observed", usd };
|
|
1668
|
+
}
|
|
1669
|
+
function assertValidUsageReceipt(receipt) {
|
|
1670
|
+
if (receipt.calls !== null && (!Number.isInteger(receipt.calls) || receipt.calls < 0)) {
|
|
1671
|
+
throw new Error("AnalystContext.recordUsage: calls must be a non-negative integer or null");
|
|
1672
|
+
}
|
|
1673
|
+
if (receipt.tokens) {
|
|
1674
|
+
assertNonNegativeFinite(receipt.tokens.input, "tokens.input");
|
|
1675
|
+
assertNonNegativeFinite(receipt.tokens.output, "tokens.output");
|
|
1676
|
+
if (receipt.tokens.reasoning !== void 0) {
|
|
1677
|
+
assertNonNegativeFinite(receipt.tokens.reasoning, "tokens.reasoning");
|
|
1678
|
+
if (receipt.tokens.reasoning > receipt.tokens.output) {
|
|
1679
|
+
throw new Error(
|
|
1680
|
+
"AnalystContext.recordUsage: tokens.reasoning must not exceed tokens.output"
|
|
1681
|
+
);
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
if (receipt.tokens.cached !== void 0) {
|
|
1685
|
+
assertNonNegativeFinite(receipt.tokens.cached, "tokens.cached");
|
|
1686
|
+
}
|
|
1687
|
+
if (receipt.tokens.cacheWrite !== void 0) {
|
|
1688
|
+
assertNonNegativeFinite(receipt.tokens.cacheWrite, "tokens.cacheWrite");
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
if (receipt.cost.kind !== "uncaptured") {
|
|
1692
|
+
assertNonNegativeFinite(receipt.cost.usd, "cost.usd");
|
|
1693
|
+
} else if (receipt.cost.usd !== null) {
|
|
1694
|
+
throw new Error("AnalystContext.recordUsage: uncaptured cost.usd must be null");
|
|
1695
|
+
}
|
|
1696
|
+
if (receipt.knownCostUsd !== void 0) {
|
|
1697
|
+
assertNonNegativeFinite(receipt.knownCostUsd, "knownCostUsd");
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
function assertNonNegativeFinite(value, field) {
|
|
1701
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
1702
|
+
throw new Error(`AnalystContext.recordUsage: ${field} must be a non-negative finite number`);
|
|
1125
1703
|
}
|
|
1126
|
-
return policy.totalUsd / Math.max(1, args.runningCount);
|
|
1127
1704
|
}
|
|
1128
1705
|
function sumFindingCost(findings) {
|
|
1129
1706
|
let sum = 0;
|
|
1707
|
+
let captured = false;
|
|
1130
1708
|
for (const f of findings) {
|
|
1131
1709
|
const c = f.metadata?.cost_usd;
|
|
1132
|
-
if (
|
|
1710
|
+
if (c === void 0) continue;
|
|
1711
|
+
if (typeof c !== "number" || !Number.isFinite(c) || c < 0) {
|
|
1712
|
+
throw new Error(
|
|
1713
|
+
`Analyst finding '${f.finding_id}' metadata.cost_usd must be a non-negative finite number`
|
|
1714
|
+
);
|
|
1715
|
+
}
|
|
1716
|
+
sum += c;
|
|
1717
|
+
captured = true;
|
|
1133
1718
|
}
|
|
1134
|
-
return sum;
|
|
1719
|
+
return { usd: sum, captured };
|
|
1135
1720
|
}
|
|
1136
1721
|
function selectPriorFindings(source, analystId) {
|
|
1137
1722
|
if (!source) return void 0;
|
|
@@ -1891,6 +2476,7 @@ function finiteOrZero(value) {
|
|
|
1891
2476
|
}
|
|
1892
2477
|
|
|
1893
2478
|
export {
|
|
2479
|
+
createAnalystAi,
|
|
1894
2480
|
computeFindingId,
|
|
1895
2481
|
makeFinding,
|
|
1896
2482
|
FINDING_SUBJECT_KINDS,
|
|
@@ -1905,12 +2491,19 @@ export {
|
|
|
1905
2491
|
coerceJson,
|
|
1906
2492
|
coerceToFindingRows,
|
|
1907
2493
|
ANALYST_SEVERITIES,
|
|
2494
|
+
RawAnalystEvidenceSchema,
|
|
1908
2495
|
RawAnalystFindingSchema,
|
|
2496
|
+
CanonicalRawAnalystFindingSchema,
|
|
1909
2497
|
RAW_FINDING_SCHEMA_PROMPT,
|
|
2498
|
+
evidenceRefsFromRawFinding,
|
|
1910
2499
|
parseRawFinding,
|
|
2500
|
+
parseCanonicalRawFinding,
|
|
1911
2501
|
structureFindings,
|
|
2502
|
+
settleUsageReceiptFromCostLedger,
|
|
2503
|
+
validateUsageSettlementTimeout,
|
|
1912
2504
|
createTraceAnalystKind,
|
|
1913
2505
|
renderPriorFindings,
|
|
2506
|
+
renderUpstreamFindings,
|
|
1914
2507
|
buildTraceToolsForGroup,
|
|
1915
2508
|
FAILURE_MODE_KIND_SPEC,
|
|
1916
2509
|
IMPROVEMENT_KIND_SPEC,
|
|
@@ -1940,4 +2533,4 @@ export {
|
|
|
1940
2533
|
aggregateRunScore,
|
|
1941
2534
|
clamp012 as clamp01
|
|
1942
2535
|
};
|
|
1943
|
-
//# sourceMappingURL=chunk-
|
|
2536
|
+
//# sourceMappingURL=chunk-7A4LIMMY.js.map
|