@tangle-network/agent-eval 0.118.3 → 0.119.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +26 -0
- package/dist/analyst/index.d.ts +172 -59
- package/dist/analyst/index.js +50 -12
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +50 -40
- package/dist/benchmarks/index.js +8 -8
- package/dist/campaign/index.d.ts +258 -184
- package/dist/campaign/index.js +7 -7
- package/dist/{chunk-JTMFT5QZ.js → chunk-4I2E3LLO.js} +2 -2
- package/dist/{chunk-KK2O4VWA.js → chunk-D7AEXSM5.js} +2 -2
- package/dist/{chunk-MLHRFWQK.js → chunk-JHOJHHU7.js} +85 -16
- package/dist/chunk-JHOJHHU7.js.map +1 -0
- package/dist/{chunk-F5A4GDQW.js → chunk-JM2SKQMS.js} +3 -15
- package/dist/chunk-JM2SKQMS.js.map +1 -0
- package/dist/{chunk-EGNRE7VA.js → chunk-PAHNGS65.js} +2 -2
- package/dist/{chunk-QWRLW4CT.js → chunk-PSXJ32SR.js} +160 -6
- package/dist/chunk-PSXJ32SR.js.map +1 -0
- package/dist/{chunk-6QIM2EAP.js → chunk-PXD6ZFNY.js} +99 -1
- package/dist/chunk-PXD6ZFNY.js.map +1 -0
- package/dist/{chunk-267UCQWI.js → chunk-QWMPPZ3X.js} +3 -3
- package/dist/{chunk-267UCQWI.js.map → chunk-QWMPPZ3X.js.map} +1 -1
- package/dist/{chunk-VNLWDTDA.js → chunk-S5TT5R3L.js} +1010 -285
- package/dist/chunk-S5TT5R3L.js.map +1 -0
- package/dist/{chunk-ZYHJNKI3.js → chunk-ULOKLHIQ.js} +42 -5
- package/dist/chunk-ULOKLHIQ.js.map +1 -0
- package/dist/{chunk-FIUFRXP3.js → chunk-WW2A73HW.js} +49 -91
- package/dist/chunk-WW2A73HW.js.map +1 -0
- package/dist/{chunk-7V3QDWNL.js → chunk-XJ7JVCHB.js} +2 -2
- package/dist/{chunk-EKHHBKS6.js → chunk-XMBOU5W7.js} +48 -190
- package/dist/chunk-XMBOU5W7.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/contract/index.d.ts +281 -175
- package/dist/contract/index.js +284 -8
- package/dist/contract/index.js.map +1 -1
- package/dist/fuzz.d.ts +19 -1
- package/dist/fuzz.js +1 -1
- package/dist/index.d.ts +223 -114
- package/dist/index.js +38 -37
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/primeintellect/index.d.ts +311 -0
- package/dist/primeintellect/index.js +348 -0
- package/dist/primeintellect/index.js.map +1 -0
- package/dist/rl.d.ts +4 -0
- package/dist/{run-campaign-DWC67KJP.js → run-campaign-OBXSN5WK.js} +3 -3
- package/dist/traces.d.ts +39 -25
- package/dist/traces.js +3 -5
- package/dist/wire/index.d.ts +19 -1
- package/dist/wire/index.js +2 -2
- package/docs/trace-analysis.md +6 -2
- package/package.json +7 -2
- package/dist/chunk-6QIM2EAP.js.map +0 -1
- package/dist/chunk-EKHHBKS6.js.map +0 -1
- package/dist/chunk-F5A4GDQW.js.map +0 -1
- package/dist/chunk-FIUFRXP3.js.map +0 -1
- package/dist/chunk-MLHRFWQK.js.map +0 -1
- package/dist/chunk-QWRLW4CT.js.map +0 -1
- package/dist/chunk-VNLWDTDA.js.map +0 -1
- package/dist/chunk-ZYHJNKI3.js.map +0 -1
- /package/dist/{chunk-JTMFT5QZ.js.map → chunk-4I2E3LLO.js.map} +0 -0
- /package/dist/{chunk-KK2O4VWA.js.map → chunk-D7AEXSM5.js.map} +0 -0
- /package/dist/{chunk-EGNRE7VA.js.map → chunk-PAHNGS65.js.map} +0 -0
- /package/dist/{chunk-7V3QDWNL.js.map → chunk-XJ7JVCHB.js.map} +0 -0
- /package/dist/{run-campaign-DWC67KJP.js.map → run-campaign-OBXSN5WK.js.map} +0 -0
|
@@ -6,11 +6,11 @@ import {
|
|
|
6
6
|
} from "./chunk-NJC7U437.js";
|
|
7
7
|
import {
|
|
8
8
|
CostLedger
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-JHOJHHU7.js";
|
|
10
10
|
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
} from "./chunk-
|
|
11
|
+
buildTraceAnalystTools,
|
|
12
|
+
runTraceAnalysisLoop
|
|
13
|
+
} from "./chunk-PXD6ZFNY.js";
|
|
14
14
|
import {
|
|
15
15
|
validateAgentProfileCell
|
|
16
16
|
} from "./chunk-XJYR7XFV.js";
|
|
@@ -21,6 +21,41 @@ import {
|
|
|
21
21
|
ValidationError
|
|
22
22
|
} from "./chunk-ONWEPEDO.js";
|
|
23
23
|
|
|
24
|
+
// src/analyst/ax-service.ts
|
|
25
|
+
import { ai } from "@ax-llm/ax";
|
|
26
|
+
var configuredModels = /* @__PURE__ */ new WeakMap();
|
|
27
|
+
function createAnalystAi(config) {
|
|
28
|
+
const model = config.model.trim();
|
|
29
|
+
if (!model) throw new TypeError("createAnalystAi: model must be a non-empty string");
|
|
30
|
+
const args = {
|
|
31
|
+
name: config.provider ?? "openai",
|
|
32
|
+
apiKey: config.apiKey,
|
|
33
|
+
...config.baseUrl ? { apiURL: config.baseUrl } : {},
|
|
34
|
+
...config.headers ? { headers: config.headers } : {},
|
|
35
|
+
config: { model }
|
|
36
|
+
};
|
|
37
|
+
const service = ai(args);
|
|
38
|
+
configuredModels.set(service, model);
|
|
39
|
+
return service;
|
|
40
|
+
}
|
|
41
|
+
function getConfiguredAnalystModel(service) {
|
|
42
|
+
return configuredModels.get(service);
|
|
43
|
+
}
|
|
44
|
+
function resolveAnalystModel(service, override) {
|
|
45
|
+
if (override !== void 0) {
|
|
46
|
+
const model2 = override.trim();
|
|
47
|
+
if (!model2) throw new TypeError("createTraceAnalystKind: model must be a non-empty string");
|
|
48
|
+
return model2;
|
|
49
|
+
}
|
|
50
|
+
const model = getConfiguredAnalystModel(service)?.trim();
|
|
51
|
+
if (!model) {
|
|
52
|
+
throw new TypeError(
|
|
53
|
+
"createTraceAnalystKind: model is required for Ax services not created by createAnalystAi()"
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
return model;
|
|
57
|
+
}
|
|
58
|
+
|
|
24
59
|
// src/analyst/types.ts
|
|
25
60
|
import { createHash } from "crypto";
|
|
26
61
|
function computeFindingId(input) {
|
|
@@ -345,47 +380,63 @@ function coerceToFindingRows(raw) {
|
|
|
345
380
|
// src/analyst/finding-signature.ts
|
|
346
381
|
import { z as z2 } from "zod";
|
|
347
382
|
var ANALYST_SEVERITIES = ["critical", "high", "medium", "low", "info"];
|
|
348
|
-
var
|
|
383
|
+
var RawAnalystEvidenceSchema = z2.object({
|
|
384
|
+
uri: z2.string().trim().min(1).max(2e3),
|
|
385
|
+
excerpt: z2.string().max(2e3).optional()
|
|
386
|
+
}).strict();
|
|
387
|
+
var RawAnalystFindingBaseShape = {
|
|
349
388
|
severity: z2.enum(ANALYST_SEVERITIES),
|
|
350
389
|
claim: z2.string().min(1).max(2e3),
|
|
351
|
-
|
|
352
|
-
* Subject locus the finding is about. Validated at parse time
|
|
353
|
-
* against the documented grammar (`finding-subject.ts`). Findings
|
|
354
|
-
* with a malformed subject are rejected — they would have been
|
|
355
|
-
* silently skipped by every downstream adapter, so failing loud at
|
|
356
|
-
* parse time turns a hidden no-op into a kind-prompt audit signal.
|
|
357
|
-
*
|
|
358
|
-
* Optional because purely descriptive findings (no actionable
|
|
359
|
-
* locus) are legitimate; they just don't route through the
|
|
360
|
-
* KnowledgeAdapter / ImprovementAdapter.
|
|
361
|
-
*/
|
|
362
|
-
subject: z2.string().max(400).refine((s) => parseFindingSubject(s) !== null, {
|
|
390
|
+
subject: z2.string().max(400).refine((subject) => parseFindingSubject(subject) !== null, {
|
|
363
391
|
message: "subject does not match the finding-subject grammar"
|
|
364
392
|
}).optional(),
|
|
365
|
-
evidence_uri: z2.string().min(1).max(2e3),
|
|
366
|
-
evidence_excerpt: z2.string().max(2e3).optional(),
|
|
367
393
|
confidence: z2.number().min(0).max(1),
|
|
368
394
|
rationale: z2.string().max(4e3).optional(),
|
|
369
395
|
recommended_action: z2.string().max(2e3).optional()
|
|
396
|
+
};
|
|
397
|
+
var RawAnalystFindingSchema = z2.object({
|
|
398
|
+
...RawAnalystFindingBaseShape,
|
|
399
|
+
evidence_uri: z2.string().min(1).max(2e3).refine((uri) => uri.trim().length > 0, { message: "evidence_uri must not be blank" }),
|
|
400
|
+
evidence_excerpt: z2.string().max(2e3).optional()
|
|
370
401
|
}).strict();
|
|
371
|
-
var
|
|
372
|
-
|
|
402
|
+
var CanonicalRawAnalystFindingObjectSchema = z2.object({
|
|
403
|
+
...RawAnalystFindingBaseShape,
|
|
404
|
+
evidence: z2.array(RawAnalystEvidenceSchema).min(1)
|
|
405
|
+
}).strict();
|
|
406
|
+
var CanonicalRawAnalystFindingSchema = z2.preprocess(
|
|
407
|
+
normalizeLegacySingleCitation,
|
|
408
|
+
CanonicalRawAnalystFindingObjectSchema
|
|
409
|
+
);
|
|
410
|
+
var RAW_FINDING_SCHEMA_PROMPT = `Each finding MUST be a strict JSON object with:
|
|
411
|
+
- severity: "critical" | "high" | "medium" | "low" | "info"
|
|
373
412
|
- claim: one-sentence statement (max 2000 chars)
|
|
374
|
-
- subject?:
|
|
375
|
-
-
|
|
376
|
-
-
|
|
377
|
-
-
|
|
378
|
-
-
|
|
379
|
-
- recommended_action?: concrete change phrased as an imperative ("Add ...", "Replace ...", "Stop ...") \u2014 omit when the finding is purely descriptive
|
|
413
|
+
- subject?: one exact subject form listed by this kind; omit rather than guess
|
|
414
|
+
- evidence: REQUIRED non-empty array of {"uri": string, "excerpt"?: string}. Use real identifiers with span://, event://, artifact://, metric://, or finding://. Include a short exact quote in excerpt when available. If nothing is citable, do not emit the finding.
|
|
415
|
+
- confidence: number 0..1 (0.9+ exact evidence; 0.6-0.8 inferred pattern; <0.5 speculative)
|
|
416
|
+
- rationale?: one or two reasoning sentences
|
|
417
|
+
- recommended_action?: concrete imperative change; omit for descriptive findings
|
|
380
418
|
|
|
381
|
-
|
|
419
|
+
Unknown fields are rejected. Do not emit area; the factory assigns it. Emit [] when there are no findings. Never fabricate evidence.`;
|
|
420
|
+
function evidenceRefsFromRawFinding(finding) {
|
|
421
|
+
return finding.evidence.map(({ uri, excerpt }) => ({
|
|
422
|
+
kind: evidenceKindFromUri(uri),
|
|
423
|
+
uri,
|
|
424
|
+
excerpt
|
|
425
|
+
}));
|
|
426
|
+
}
|
|
382
427
|
function parseRawFinding(row, log) {
|
|
383
|
-
|
|
428
|
+
return parseFindingWithSchema(RawAnalystFindingSchema, row, log);
|
|
429
|
+
}
|
|
430
|
+
function parseCanonicalRawFinding(row, log) {
|
|
431
|
+
return parseFindingWithSchema(CanonicalRawAnalystFindingSchema, row, log);
|
|
432
|
+
}
|
|
433
|
+
function parseFindingWithSchema(schema, row, log) {
|
|
434
|
+
const result = schema.safeParse(row);
|
|
384
435
|
if (result.success) return result.data;
|
|
385
436
|
if (typeof row === "string") {
|
|
386
437
|
const coerced = coerceJson(row);
|
|
387
438
|
if (coerced !== void 0) {
|
|
388
|
-
const retry =
|
|
439
|
+
const retry = schema.safeParse(coerced);
|
|
389
440
|
if (retry.success) return retry.data;
|
|
390
441
|
}
|
|
391
442
|
}
|
|
@@ -398,40 +449,98 @@ function parseRawFinding(row, log) {
|
|
|
398
449
|
});
|
|
399
450
|
return null;
|
|
400
451
|
}
|
|
452
|
+
function normalizeLegacySingleCitation(value) {
|
|
453
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return value;
|
|
454
|
+
const row = value;
|
|
455
|
+
if ("evidence" in row) {
|
|
456
|
+
if (!("evidence_uri" in row) || !Array.isArray(row.evidence)) return value;
|
|
457
|
+
const { evidence, evidence_uri: uri2, evidence_excerpt: excerpt2, ...rest2 } = row;
|
|
458
|
+
let alreadyPresent = false;
|
|
459
|
+
const mergedEvidence = evidence.map((citation) => {
|
|
460
|
+
if (citation === null || typeof citation !== "object" || Array.isArray(citation) || citation.uri !== uri2) {
|
|
461
|
+
return citation;
|
|
462
|
+
}
|
|
463
|
+
alreadyPresent = true;
|
|
464
|
+
const current = citation;
|
|
465
|
+
return excerpt2 === void 0 || current.excerpt !== void 0 ? citation : { ...current, excerpt: excerpt2 };
|
|
466
|
+
});
|
|
467
|
+
return {
|
|
468
|
+
...rest2,
|
|
469
|
+
evidence: alreadyPresent ? mergedEvidence : [...evidence, { uri: uri2, ...excerpt2 === void 0 ? {} : { excerpt: excerpt2 } }]
|
|
470
|
+
};
|
|
471
|
+
}
|
|
472
|
+
if (!("evidence_uri" in row)) return value;
|
|
473
|
+
const { evidence_uri: uri, evidence_excerpt: excerpt, ...rest } = row;
|
|
474
|
+
return {
|
|
475
|
+
...rest,
|
|
476
|
+
evidence: [{ uri, ...excerpt === void 0 ? {} : { excerpt } }]
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
function toLegacyRawAnalystFinding(finding) {
|
|
480
|
+
const primaryEvidence = finding.evidence[0];
|
|
481
|
+
if (!primaryEvidence) {
|
|
482
|
+
throw new TypeError("Canonical raw analyst findings require at least one evidence citation");
|
|
483
|
+
}
|
|
484
|
+
const { evidence: _evidence, ...rest } = finding;
|
|
485
|
+
return {
|
|
486
|
+
...rest,
|
|
487
|
+
evidence_uri: primaryEvidence.uri,
|
|
488
|
+
...primaryEvidence.excerpt === void 0 ? {} : { evidence_excerpt: primaryEvidence.excerpt }
|
|
489
|
+
};
|
|
490
|
+
}
|
|
491
|
+
function applyLegacyRawFindingCallback(finding, callback, log) {
|
|
492
|
+
const callbackResult = callback(toLegacyRawAnalystFinding(finding));
|
|
493
|
+
if (callbackResult === null) return null;
|
|
494
|
+
const parsed = parseCanonicalRawFinding(callbackResult, log);
|
|
495
|
+
if (!parsed) return null;
|
|
496
|
+
const primaryEvidence = parsed.evidence[0];
|
|
497
|
+
if (!primaryEvidence) {
|
|
498
|
+
throw new TypeError("Canonical raw analyst findings require at least one evidence citation");
|
|
499
|
+
}
|
|
500
|
+
const remainingEvidence = finding.evidence.slice(1).filter((citation) => citation.uri !== primaryEvidence.uri);
|
|
501
|
+
return {
|
|
502
|
+
...parsed,
|
|
503
|
+
evidence: [primaryEvidence, ...remainingEvidence]
|
|
504
|
+
};
|
|
505
|
+
}
|
|
506
|
+
function evidenceKindFromUri(uri) {
|
|
507
|
+
if (uri.startsWith("span://")) return "span";
|
|
508
|
+
if (uri.startsWith("event://")) return "event";
|
|
509
|
+
if (uri.startsWith("finding://")) return "finding";
|
|
510
|
+
if (uri.startsWith("metric://")) return "metric";
|
|
511
|
+
return "artifact";
|
|
512
|
+
}
|
|
401
513
|
|
|
402
514
|
// src/analyst/structure-findings.ts
|
|
403
515
|
var SYSTEM = [
|
|
404
516
|
"You convert a free-form trace-analysis report into a STRICT JSON array of findings.",
|
|
405
517
|
"Output ONLY the JSON array \u2014 no prose, no code fences.",
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
'evidence_uri cites the trace element the report referenced (e.g. "span://<trace>/<span>") or "report://summary".',
|
|
518
|
+
RAW_FINDING_SCHEMA_PROMPT,
|
|
519
|
+
"Omit subject when the report does not contain an exact valid locus.",
|
|
409
520
|
"If the report asserts NO problems, output exactly []."
|
|
410
521
|
].join(" ");
|
|
411
|
-
function buildRows(raw,
|
|
522
|
+
function buildRows(raw, opts) {
|
|
412
523
|
const rows = coerceToFindingRows(raw);
|
|
413
524
|
const out = [];
|
|
414
525
|
for (const row of rows) {
|
|
415
|
-
const
|
|
416
|
-
const parsed = parseRawFinding(normalized);
|
|
526
|
+
const parsed = parseCanonicalRawFinding(row);
|
|
417
527
|
if (!parsed) continue;
|
|
528
|
+
const callbackProcessed = opts.processRow ? applyLegacyRawFindingCallback(parsed, opts.processRow) : parsed;
|
|
529
|
+
if (!callbackProcessed) continue;
|
|
530
|
+
const processed = opts.processCanonicalRow ? opts.processCanonicalRow(callbackProcessed) : callbackProcessed;
|
|
531
|
+
if (!processed) continue;
|
|
418
532
|
out.push(
|
|
419
533
|
makeFinding({
|
|
420
|
-
analyst_id: analystId,
|
|
421
|
-
area,
|
|
422
|
-
subject:
|
|
423
|
-
claim:
|
|
424
|
-
rationale:
|
|
425
|
-
severity:
|
|
426
|
-
confidence:
|
|
427
|
-
evidence_refs:
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
uri: parsed.evidence_uri,
|
|
431
|
-
excerpt: parsed.evidence_excerpt
|
|
432
|
-
}
|
|
433
|
-
],
|
|
434
|
-
recommended_action: parsed.recommended_action
|
|
534
|
+
analyst_id: opts.analystId,
|
|
535
|
+
area: opts.area,
|
|
536
|
+
subject: processed.subject,
|
|
537
|
+
claim: processed.claim,
|
|
538
|
+
rationale: processed.rationale,
|
|
539
|
+
severity: processed.severity,
|
|
540
|
+
confidence: processed.confidence,
|
|
541
|
+
evidence_refs: evidenceRefsFromRawFinding(processed),
|
|
542
|
+
recommended_action: processed.recommended_action,
|
|
543
|
+
...opts.findingMetadata ? { metadata: { ...opts.findingMetadata } } : {}
|
|
435
544
|
})
|
|
436
545
|
);
|
|
437
546
|
}
|
|
@@ -439,6 +548,9 @@ function buildRows(raw, analystId, area) {
|
|
|
439
548
|
}
|
|
440
549
|
async function structureFindings(opts) {
|
|
441
550
|
const maxReasks = opts.maxReasks ?? 1;
|
|
551
|
+
if (!Number.isSafeInteger(maxReasks) || maxReasks < 0) {
|
|
552
|
+
throw new RangeError("structureFindings: maxReasks must be a non-negative safe integer");
|
|
553
|
+
}
|
|
442
554
|
const llm = { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl };
|
|
443
555
|
const costLedger = opts.costLedger ?? new CostLedger();
|
|
444
556
|
let user = `TRACE-ANALYSIS REPORT:
|
|
@@ -459,8 +571,9 @@ Return the findings JSON array.`;
|
|
|
459
571
|
phase: opts.costPhase ?? "analyst.structure-findings",
|
|
460
572
|
actor: "structure-findings",
|
|
461
573
|
model: opts.model,
|
|
574
|
+
signal: opts.signal,
|
|
462
575
|
maximumCharge: maximumChargeForLlmRequest(request, llm),
|
|
463
|
-
tags: { analystId: opts.analystId, attempt: String(attempt) },
|
|
576
|
+
tags: { ...opts.costTags, analystId: opts.analystId, attempt: String(attempt) },
|
|
464
577
|
execute: (signal, callId) => callLlm(request, { ...llm, signal, idempotencyKey: callId }),
|
|
465
578
|
receipt: costReceiptFromLlm,
|
|
466
579
|
receiptFromError: costReceiptFromLlmError
|
|
@@ -468,7 +581,7 @@ Return the findings JSON array.`;
|
|
|
468
581
|
if (!paid.succeeded) throw paid.error;
|
|
469
582
|
const res = paid.value;
|
|
470
583
|
const text = res.content.trim();
|
|
471
|
-
const findings = buildRows(text, opts
|
|
584
|
+
const findings = buildRows(text, opts);
|
|
472
585
|
if (findings.length > 0) return { findings, outcome: "ok" };
|
|
473
586
|
if (opts.report.trim().length < 200) return { findings: [], outcome: "ok" };
|
|
474
587
|
user = `${user}
|
|
@@ -478,150 +591,459 @@ That produced no valid findings. The report DOES describe issues \u2014 re-extra
|
|
|
478
591
|
return { findings: [], outcome: "extraction_failed" };
|
|
479
592
|
}
|
|
480
593
|
|
|
594
|
+
// src/analyst/ax-cost-service.ts
|
|
595
|
+
function meterAxChatService(ai2, options) {
|
|
596
|
+
assertPositiveInteger(options.maxOutputTokens, "maxOutputTokens");
|
|
597
|
+
const source = ai2;
|
|
598
|
+
if (typeof source.chat !== "function") {
|
|
599
|
+
throw new TypeError("meterAxChatService: Ax service must implement chat()");
|
|
600
|
+
}
|
|
601
|
+
const providerChat = source.chat.bind(ai2);
|
|
602
|
+
const chat = async (request, callOptions = {}) => {
|
|
603
|
+
const boundedRequest = boundOutputTokens(request, options.maxOutputTokens);
|
|
604
|
+
const model = modelName(boundedRequest.model) || options.defaultModel || "";
|
|
605
|
+
const canTurnOffThinking = canDisableThinking(ai2, model);
|
|
606
|
+
const combined = combineSignals(options.signal, callOptions.abortSignal);
|
|
607
|
+
try {
|
|
608
|
+
const paid = await options.ledger.runPaidCall({
|
|
609
|
+
channel: "analyst",
|
|
610
|
+
phase: options.phase ?? "analyst.ax.chat",
|
|
611
|
+
actor: options.actor,
|
|
612
|
+
model,
|
|
613
|
+
tags: options.tags,
|
|
614
|
+
signal: combined.signal,
|
|
615
|
+
maximumCharge: maximumChargeForAxChatRequest(boundedRequest, model),
|
|
616
|
+
execute: async (executionSignal) => {
|
|
617
|
+
const providerOptions = {
|
|
618
|
+
...callOptions,
|
|
619
|
+
abortSignal: executionSignal,
|
|
620
|
+
retry: { ...callOptions.retry, maxRetries: 0 },
|
|
621
|
+
stream: false,
|
|
622
|
+
showThoughts: false
|
|
623
|
+
};
|
|
624
|
+
if (canTurnOffThinking) providerOptions.thinkingTokenBudget = "none";
|
|
625
|
+
else Reflect.deleteProperty(providerOptions, "thinkingTokenBudget");
|
|
626
|
+
const response = await providerChat(boundedRequest, providerOptions);
|
|
627
|
+
if (response instanceof ReadableStream) {
|
|
628
|
+
throw new Error("meterAxChatService: provider returned a stream after stream:false");
|
|
629
|
+
}
|
|
630
|
+
return response;
|
|
631
|
+
},
|
|
632
|
+
receipt: (response) => costReceiptFromAxResponse(response, model)
|
|
633
|
+
});
|
|
634
|
+
if (!paid.succeeded) throw paid.error;
|
|
635
|
+
return paid.value;
|
|
636
|
+
} finally {
|
|
637
|
+
combined.dispose();
|
|
638
|
+
}
|
|
639
|
+
};
|
|
640
|
+
return new Proxy(ai2, {
|
|
641
|
+
get(target, property) {
|
|
642
|
+
if (property === "chat") return chat;
|
|
643
|
+
const value = Reflect.get(target, property, target);
|
|
644
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
645
|
+
}
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
function maximumChargeForAxChatRequest(request, defaultModel) {
|
|
649
|
+
const model = modelName(request.model) || defaultModel || "";
|
|
650
|
+
const maxTokens = request.modelConfig?.maxTokens;
|
|
651
|
+
const completions = request.modelConfig?.n;
|
|
652
|
+
if (!model || maxTokens === void 0 || completions === void 0) return void 0;
|
|
653
|
+
assertPositiveInteger(maxTokens, "request.modelConfig.maxTokens");
|
|
654
|
+
assertPositiveInteger(completions, "request.modelConfig.n");
|
|
655
|
+
const maximumOutputTokens = maxTokens * completions;
|
|
656
|
+
assertPositiveInteger(maximumOutputTokens, "maximum output tokens");
|
|
657
|
+
if (containsUnboundedOrCacheableContent(request)) return void 0;
|
|
658
|
+
let inputTokens;
|
|
659
|
+
try {
|
|
660
|
+
const pricedRequest = request.model === void 0 ? { ...request, model } : request;
|
|
661
|
+
inputTokens = new TextEncoder().encode(JSON.stringify(pricedRequest)).byteLength * completions;
|
|
662
|
+
} catch {
|
|
663
|
+
return void 0;
|
|
664
|
+
}
|
|
665
|
+
assertPositiveInteger(inputTokens, "maximum input tokens");
|
|
666
|
+
return { model, inputTokens, outputTokens: maximumOutputTokens };
|
|
667
|
+
}
|
|
668
|
+
function boundOutputTokens(request, limit) {
|
|
669
|
+
const requested = request.modelConfig?.maxTokens;
|
|
670
|
+
if (requested !== void 0) {
|
|
671
|
+
assertPositiveInteger(requested, "request.modelConfig.maxTokens");
|
|
672
|
+
}
|
|
673
|
+
const completions = request.modelConfig?.n ?? 1;
|
|
674
|
+
assertPositiveInteger(completions, "request.modelConfig.n");
|
|
675
|
+
const maxTokens = requested === void 0 ? limit : Math.min(requested, limit);
|
|
676
|
+
return {
|
|
677
|
+
...request,
|
|
678
|
+
modelConfig: { ...request.modelConfig, maxTokens, n: completions }
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
function costReceiptFromAxResponse(response, fallbackModel) {
|
|
682
|
+
const usage = response.modelUsage;
|
|
683
|
+
const tokens = usage?.tokens;
|
|
684
|
+
const model = usage?.model || fallbackModel;
|
|
685
|
+
if (!tokens || !validUsage(tokens.promptTokens) || !validUsage(tokens.completionTokens) || !validUsage(tokens.totalTokens) || tokens.totalTokens < tokens.promptTokens + tokens.completionTokens || !validOptionalUsage(tokens.cacheReadTokens) || !validOptionalUsage(tokens.cacheCreationTokens) || !validOptionalUsage(tokens.reasoningTokens) || !validOptionalUsage(tokens.thoughtsTokens)) {
|
|
686
|
+
return {
|
|
687
|
+
model,
|
|
688
|
+
inputTokens: 0,
|
|
689
|
+
outputTokens: 0,
|
|
690
|
+
usageUnknown: true
|
|
691
|
+
};
|
|
692
|
+
}
|
|
693
|
+
const cacheReadTokens = validUsage(tokens.cacheReadTokens) ? tokens.cacheReadTokens : 0;
|
|
694
|
+
const cacheCreationTokens = validUsage(tokens.cacheCreationTokens) ? tokens.cacheCreationTokens : 0;
|
|
695
|
+
const totalCacheTokens = cacheReadTokens + cacheCreationTokens;
|
|
696
|
+
const thoughtsTokens = validUsage(tokens.thoughtsTokens) ? tokens.thoughtsTokens : 0;
|
|
697
|
+
const reasoningTokens = Math.max(
|
|
698
|
+
validUsage(tokens.reasoningTokens) ? tokens.reasoningTokens : 0,
|
|
699
|
+
thoughtsTokens
|
|
700
|
+
);
|
|
701
|
+
const separateReasoningTokens = reasoningTokens > tokens.completionTokens ? reasoningTokens : 0;
|
|
702
|
+
const additionalOutputTokens = Math.max(thoughtsTokens, separateReasoningTokens);
|
|
703
|
+
const outputTokens = tokens.completionTokens + additionalOutputTokens;
|
|
704
|
+
const directTotal = tokens.promptTokens + tokens.completionTokens;
|
|
705
|
+
const validTotals = /* @__PURE__ */ new Set([
|
|
706
|
+
directTotal,
|
|
707
|
+
directTotal + totalCacheTokens,
|
|
708
|
+
directTotal + additionalOutputTokens,
|
|
709
|
+
directTotal + totalCacheTokens + additionalOutputTokens
|
|
710
|
+
]);
|
|
711
|
+
if (!validTotals.has(tokens.totalTokens)) {
|
|
712
|
+
return {
|
|
713
|
+
model,
|
|
714
|
+
inputTokens: 0,
|
|
715
|
+
outputTokens: 0,
|
|
716
|
+
usageUnknown: true
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
return {
|
|
720
|
+
model,
|
|
721
|
+
inputTokens: tokens.promptTokens,
|
|
722
|
+
outputTokens,
|
|
723
|
+
...reasoningTokens > 0 ? { reasoningTokens } : {},
|
|
724
|
+
...cacheReadTokens > 0 ? { cachedTokens: cacheReadTokens } : {},
|
|
725
|
+
...cacheCreationTokens > 0 ? { cacheWriteTokens: cacheCreationTokens } : {}
|
|
726
|
+
};
|
|
727
|
+
}
|
|
728
|
+
function combineSignals(first, second) {
|
|
729
|
+
if (!first) return { signal: second, dispose: () => {
|
|
730
|
+
} };
|
|
731
|
+
if (!second || first === second) return { signal: first, dispose: () => {
|
|
732
|
+
} };
|
|
733
|
+
if (typeof AbortSignal.any === "function") {
|
|
734
|
+
return { signal: AbortSignal.any([first, second]), dispose: () => {
|
|
735
|
+
} };
|
|
736
|
+
}
|
|
737
|
+
const controller = new AbortController();
|
|
738
|
+
const dispose = () => {
|
|
739
|
+
first.removeEventListener("abort", abortFromFirst);
|
|
740
|
+
second.removeEventListener("abort", abortFromSecond);
|
|
741
|
+
};
|
|
742
|
+
const abortFrom = (source) => {
|
|
743
|
+
if (!controller.signal.aborted) controller.abort(source.reason);
|
|
744
|
+
dispose();
|
|
745
|
+
};
|
|
746
|
+
const abortFromFirst = () => abortFrom(first);
|
|
747
|
+
const abortFromSecond = () => abortFrom(second);
|
|
748
|
+
if (first.aborted) abortFrom(first);
|
|
749
|
+
else if (second.aborted) abortFrom(second);
|
|
750
|
+
else {
|
|
751
|
+
first.addEventListener("abort", abortFromFirst, { once: true });
|
|
752
|
+
second.addEventListener("abort", abortFromSecond, { once: true });
|
|
753
|
+
}
|
|
754
|
+
return { signal: controller.signal, dispose };
|
|
755
|
+
}
|
|
756
|
+
function containsUnboundedOrCacheableContent(request) {
|
|
757
|
+
if (request.functions?.some((fn) => fn.cache === true)) return true;
|
|
758
|
+
return request.chatPrompt.some((message) => {
|
|
759
|
+
if (message.cache === true) return true;
|
|
760
|
+
if (!("content" in message) || !Array.isArray(message.content)) return false;
|
|
761
|
+
return message.content.some((part) => part.cache === true || part.type !== "text");
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
function modelName(value) {
|
|
765
|
+
return typeof value === "string" ? value : "";
|
|
766
|
+
}
|
|
767
|
+
function canDisableThinking(ai2, model) {
|
|
768
|
+
const namedAi = ai2;
|
|
769
|
+
const serviceName = typeof namedAi.getName === "function" ? namedAi.getName() : "";
|
|
770
|
+
const modelId = model.slice(model.lastIndexOf("/") + 1);
|
|
771
|
+
return serviceName !== "GoogleGeminiAI" || !/^gemini-3(?:[.-]|$)/i.test(modelId);
|
|
772
|
+
}
|
|
773
|
+
function validUsage(value) {
|
|
774
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
775
|
+
}
|
|
776
|
+
function validOptionalUsage(value) {
|
|
777
|
+
return value === void 0 || validUsage(value);
|
|
778
|
+
}
|
|
779
|
+
function assertPositiveInteger(value, field) {
|
|
780
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
781
|
+
throw new RangeError(`meterAxChatService: ${field} must be a positive integer`);
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
// src/analyst/usage-receipt.ts
|
|
786
|
+
var DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS = 5e3;
|
|
787
|
+
function usageReceiptFromCostLedger(ledger, filter = "analyst") {
|
|
788
|
+
const resolvedFilter = typeof filter === "string" ? { channel: filter } : filter;
|
|
789
|
+
const summary = ledger.summary(resolvedFilter);
|
|
790
|
+
const receipts = ledger.list(resolvedFilter);
|
|
791
|
+
const hasReasoningUsage = receipts.some((receipt) => receipt.reasoningTokens !== void 0);
|
|
792
|
+
const hasCacheWriteUsage = receipts.some((receipt) => receipt.cacheWriteTokens !== void 0);
|
|
793
|
+
const costUncaptured = summary.pendingCalls > 0 || receipts.some((receipt) => receipt.costUnknown);
|
|
794
|
+
const cost = costUncaptured ? { kind: "uncaptured", usd: null } : receipts.every((receipt) => receipt.actualCostUsd !== void 0) ? { kind: "observed", usd: summary.totalCostUsd } : { kind: "estimated", usd: summary.totalCostUsd };
|
|
795
|
+
return {
|
|
796
|
+
calls: summary.totalCalls + summary.pendingCalls,
|
|
797
|
+
tokens: summary.usageComplete ? {
|
|
798
|
+
input: summary.inputTokens,
|
|
799
|
+
output: summary.outputTokens,
|
|
800
|
+
...hasReasoningUsage ? { reasoning: summary.reasoningTokens ?? 0 } : {},
|
|
801
|
+
...summary.cachedTokens > 0 ? { cached: summary.cachedTokens } : {},
|
|
802
|
+
...hasCacheWriteUsage ? { cacheWrite: summary.cacheWriteTokens ?? 0 } : {}
|
|
803
|
+
} : null,
|
|
804
|
+
cost,
|
|
805
|
+
...cost.kind === "uncaptured" ? { knownCostUsd: summary.totalCostUsd } : {}
|
|
806
|
+
};
|
|
807
|
+
}
|
|
808
|
+
async function settleUsageReceiptFromCostLedger(ledger, options = {}) {
|
|
809
|
+
const { timeoutMs: requestedTimeoutMs, ...requestedFilter } = options;
|
|
810
|
+
const filter = {
|
|
811
|
+
channel: requestedFilter.channel ?? "analyst",
|
|
812
|
+
...requestedFilter.phase === void 0 ? {} : { phase: requestedFilter.phase },
|
|
813
|
+
...requestedFilter.tags === void 0 ? {} : { tags: requestedFilter.tags }
|
|
814
|
+
};
|
|
815
|
+
const timeoutMs = validateUsageSettlementTimeout(requestedTimeoutMs);
|
|
816
|
+
const initial = ledger.summary(filter);
|
|
817
|
+
const waitResult = initial.pendingCalls === 0 ? true : ledger.waitForIdle ? await ledger.waitForIdle({ timeoutMs }) : false;
|
|
818
|
+
const pendingCalls = ledger.summary(filter).pendingCalls;
|
|
819
|
+
return {
|
|
820
|
+
settled: waitResult && pendingCalls === 0,
|
|
821
|
+
pendingCalls,
|
|
822
|
+
receipt: usageReceiptFromCostLedger(ledger, filter)
|
|
823
|
+
};
|
|
824
|
+
}
|
|
825
|
+
function validateUsageSettlementTimeout(timeoutMs) {
|
|
826
|
+
const resolved = timeoutMs ?? DEFAULT_USAGE_SETTLEMENT_TIMEOUT_MS;
|
|
827
|
+
if (!Number.isSafeInteger(resolved) || resolved < 0 || resolved > 2147483647) {
|
|
828
|
+
throw new TypeError(
|
|
829
|
+
"settlementTimeoutMs must be a non-negative safe integer no greater than 2147483647"
|
|
830
|
+
);
|
|
831
|
+
}
|
|
832
|
+
return resolved;
|
|
833
|
+
}
|
|
834
|
+
|
|
481
835
|
// src/analyst/kind-factory.ts
|
|
482
|
-
import { AxJSRuntime, agent } from "@ax-llm/ax";
|
|
483
836
|
function createTraceAnalystKind(spec, opts) {
|
|
837
|
+
rejectRemovedKindOptions(spec);
|
|
484
838
|
const version = opts.versionSuffix ? `${spec.version}+${opts.versionSuffix}` : spec.version;
|
|
839
|
+
const model = resolveAnalystModel(opts.ai, opts.model);
|
|
840
|
+
const minimumEvidenceCitations = spec.minimumEvidenceCitations ?? 1;
|
|
841
|
+
if (!Number.isInteger(minimumEvidenceCitations) || minimumEvidenceCitations < 1) {
|
|
842
|
+
throw new TypeError("minimumEvidenceCitations must be a positive integer");
|
|
843
|
+
}
|
|
844
|
+
const settlementTimeoutMs = validateUsageSettlementTimeout(opts.settlementTimeoutMs);
|
|
485
845
|
return {
|
|
486
846
|
id: spec.id,
|
|
487
847
|
description: spec.description,
|
|
488
848
|
inputKind: "trace-store",
|
|
489
|
-
cost: spec.cost,
|
|
849
|
+
cost: { ...spec.cost, settlement_timeout_ms: settlementTimeoutMs },
|
|
490
850
|
version,
|
|
491
851
|
async analyze(store, ctx) {
|
|
492
|
-
const
|
|
493
|
-
const
|
|
494
|
-
const
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
852
|
+
const maxOutputTokens = spec.maxOutputTokens ?? 4096;
|
|
853
|
+
const costLedger = ctx.costLedger ?? new CostLedger(ctx.budgetUsd);
|
|
854
|
+
const costTags = {
|
|
855
|
+
...ctx.tags ?? {},
|
|
856
|
+
analystId: spec.id,
|
|
857
|
+
...ctx.correlationId ? { analystRunId: ctx.correlationId } : {}
|
|
858
|
+
};
|
|
859
|
+
const meteredAi = meterAxChatService(opts.ai, {
|
|
860
|
+
ledger: costLedger,
|
|
861
|
+
actor: spec.id,
|
|
862
|
+
maxOutputTokens,
|
|
863
|
+
defaultModel: model,
|
|
864
|
+
phase: ctx.costPhase,
|
|
865
|
+
signal: ctx.signal,
|
|
866
|
+
tags: costTags
|
|
867
|
+
});
|
|
868
|
+
try {
|
|
869
|
+
const tools = spec.buildTools(store);
|
|
870
|
+
const maxSubqueries = spec.subqueries?.maxCalls ?? 0;
|
|
871
|
+
const maxParallel = spec.subqueries?.maxParallel ?? 2;
|
|
872
|
+
const priorContext = renderPriorFindings(ctx.priorFindings);
|
|
873
|
+
const upstreamContext = renderUpstreamFindings(ctx.upstreamFindings);
|
|
874
|
+
const actorDescription = spec.actorDescription.trim() + priorContext + upstreamContext + "\n\n" + RAW_FINDING_SCHEMA_PROMPT + (minimumEvidenceCitations > 1 ? `
|
|
875
|
+
|
|
876
|
+
This kind requires at least ${minimumEvidenceCitations} evidence citations per finding; rows with fewer are rejected.` : "") + "\n\nFirst write `report`: a concise free-form prose diagnosis of what the traces show \u2014 what succeeded, what was suboptimal or failed \u2014 with concrete trace ids and numbers. THEN return the structured `findings` array (it MAY be empty when there is nothing to report).";
|
|
877
|
+
ctx.log?.(`analyst.kind ${spec.id} forward`, {
|
|
878
|
+
max_subqueries: maxSubqueries,
|
|
879
|
+
tool_count: tools.length,
|
|
880
|
+
tags: ctx.tags
|
|
881
|
+
});
|
|
882
|
+
const completed = await runTraceAnalysisLoop({
|
|
883
|
+
id: spec.id,
|
|
884
|
+
description: spec.description,
|
|
885
|
+
prompt: actorDescription,
|
|
886
|
+
question: deriveQuestion(ctx, spec),
|
|
887
|
+
ai: meteredAi,
|
|
888
|
+
model,
|
|
889
|
+
tools,
|
|
890
|
+
findingType: "object",
|
|
891
|
+
maxSubqueries,
|
|
892
|
+
maxParallelSubqueries: maxParallel,
|
|
516
893
|
maxTurns: spec.maxTurns ?? 12,
|
|
517
894
|
maxRuntimeChars: spec.maxRuntimeChars ?? 6e3,
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
ctx.log?.("finding rejected: subject failed to parse", {
|
|
554
|
-
kind: spec.id,
|
|
555
|
-
subject: parsed.subject
|
|
556
|
-
});
|
|
557
|
-
rejectedWrongKind += 1;
|
|
558
|
-
continue;
|
|
895
|
+
...ctx.signal ? { signal: ctx.signal } : {}
|
|
896
|
+
});
|
|
897
|
+
const { report, findings: submittedFindings } = completed;
|
|
898
|
+
const expectedSubjects = KIND_EXPECTED_SUBJECTS[spec.id];
|
|
899
|
+
const out = [];
|
|
900
|
+
const rawRows = submittedFindings;
|
|
901
|
+
let rejectedWrongKind = 0;
|
|
902
|
+
let rejectedInsufficientEvidence = 0;
|
|
903
|
+
const processRow = (parsed) => {
|
|
904
|
+
const postProcessed = spec.postProcess ? applyLegacyRawFindingCallback(
|
|
905
|
+
parsed,
|
|
906
|
+
(row) => spec.postProcess?.(row, ctx) ?? null,
|
|
907
|
+
ctx.log
|
|
908
|
+
) : parsed;
|
|
909
|
+
if (!postProcessed) return null;
|
|
910
|
+
if (expectedSubjects && postProcessed.subject !== void 0) {
|
|
911
|
+
const parsedSubject = parseFindingSubject(postProcessed.subject);
|
|
912
|
+
if (parsedSubject === null) {
|
|
913
|
+
ctx.log?.("finding rejected: subject failed to parse", {
|
|
914
|
+
kind: spec.id,
|
|
915
|
+
subject: postProcessed.subject
|
|
916
|
+
});
|
|
917
|
+
rejectedWrongKind += 1;
|
|
918
|
+
return null;
|
|
919
|
+
}
|
|
920
|
+
if (!expectedSubjects.includes(parsedSubject.kind)) {
|
|
921
|
+
ctx.log?.("finding rejected: subject variant not allowed for this kind", {
|
|
922
|
+
kind: spec.id,
|
|
923
|
+
subject_kind: parsedSubject.kind,
|
|
924
|
+
subject: postProcessed.subject,
|
|
925
|
+
allowed: expectedSubjects
|
|
926
|
+
});
|
|
927
|
+
rejectedWrongKind += 1;
|
|
928
|
+
return null;
|
|
929
|
+
}
|
|
559
930
|
}
|
|
560
|
-
|
|
561
|
-
|
|
931
|
+
const distinctEvidenceCitations = new Set(
|
|
932
|
+
postProcessed.evidence.map((citation) => citation.uri.trim())
|
|
933
|
+
).size;
|
|
934
|
+
if (distinctEvidenceCitations < minimumEvidenceCitations) {
|
|
935
|
+
ctx.log?.("finding rejected: insufficient evidence citations", {
|
|
562
936
|
kind: spec.id,
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
937
|
+
required: minimumEvidenceCitations,
|
|
938
|
+
received: postProcessed.evidence.length,
|
|
939
|
+
distinct: distinctEvidenceCitations
|
|
566
940
|
});
|
|
567
|
-
|
|
568
|
-
|
|
941
|
+
rejectedInsufficientEvidence += 1;
|
|
942
|
+
return null;
|
|
569
943
|
}
|
|
944
|
+
return postProcessed;
|
|
945
|
+
};
|
|
946
|
+
for (const row of rawRows) {
|
|
947
|
+
const parsed = parseCanonicalRawFinding(row, ctx.log);
|
|
948
|
+
if (!parsed) continue;
|
|
949
|
+
const postProcessed = processRow(parsed);
|
|
950
|
+
if (!postProcessed) continue;
|
|
951
|
+
out.push(toAnalystFinding(spec, version, postProcessed));
|
|
570
952
|
}
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
report,
|
|
585
|
-
analystId: spec.id,
|
|
586
|
-
area: spec.area,
|
|
587
|
-
model: opts.recovery.model ?? opts.model ?? "",
|
|
588
|
-
baseUrl: opts.recovery.baseUrl,
|
|
589
|
-
apiKey: opts.recovery.apiKey,
|
|
590
|
-
fetchImpl: opts.recovery.fetchImpl
|
|
591
|
-
});
|
|
592
|
-
out.push(...recovered.findings);
|
|
593
|
-
ctx.log?.(`analyst.kind ${spec.id} recovery`, {
|
|
594
|
-
outcome: recovered.outcome,
|
|
595
|
-
recovered: recovered.findings.length
|
|
596
|
-
});
|
|
597
|
-
}
|
|
598
|
-
if (out.length === 0) {
|
|
599
|
-
out.push(
|
|
600
|
-
makeFinding({
|
|
601
|
-
analyst_id: spec.id,
|
|
953
|
+
ctx.log?.(`analyst.kind ${spec.id} done`, {
|
|
954
|
+
emitted: rawRows.length,
|
|
955
|
+
accepted: out.length,
|
|
956
|
+
rejected_wrong_subject: rejectedWrongKind,
|
|
957
|
+
rejected_insufficient_evidence: rejectedInsufficientEvidence
|
|
958
|
+
});
|
|
959
|
+
if (out.length === 0 && report.trim().length >= 200) {
|
|
960
|
+
if (opts.recovery) {
|
|
961
|
+
const wrongKindBefore = rejectedWrongKind;
|
|
962
|
+
const insufficientEvidenceBefore = rejectedInsufficientEvidence;
|
|
963
|
+
const recovered = await structureFindings({
|
|
964
|
+
report,
|
|
965
|
+
analystId: spec.id,
|
|
602
966
|
area: spec.area,
|
|
967
|
+
model: opts.recovery.model ?? model,
|
|
968
|
+
baseUrl: opts.recovery.baseUrl,
|
|
969
|
+
apiKey: opts.recovery.apiKey,
|
|
970
|
+
fetchImpl: opts.recovery.fetchImpl,
|
|
971
|
+
costLedger,
|
|
972
|
+
costPhase: ctx.costPhase,
|
|
973
|
+
costTags,
|
|
974
|
+
signal: ctx.signal,
|
|
975
|
+
maxTokens: Math.min(maxOutputTokens, 2e3),
|
|
976
|
+
processCanonicalRow: processRow,
|
|
977
|
+
findingMetadata: { kind_version: version }
|
|
978
|
+
});
|
|
979
|
+
out.push(...recovered.findings);
|
|
980
|
+
ctx.log?.(`analyst.kind ${spec.id} recovery`, {
|
|
981
|
+
outcome: recovered.outcome,
|
|
982
|
+
recovered: recovered.findings.length,
|
|
983
|
+
rejected_wrong_subject: rejectedWrongKind - wrongKindBefore,
|
|
984
|
+
rejected_insufficient_evidence: rejectedInsufficientEvidence - insufficientEvidenceBefore
|
|
985
|
+
});
|
|
986
|
+
}
|
|
987
|
+
if (out.length === 0) {
|
|
988
|
+
const fallback = processRow({
|
|
603
989
|
claim: "Analyst produced a diagnosis but no structured findings \u2014 see report.",
|
|
604
990
|
rationale: report.slice(0, 1500),
|
|
605
991
|
severity: "info",
|
|
606
992
|
confidence: 0.3,
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
}
|
|
612
|
-
|
|
993
|
+
evidence: [{ uri: "report://summary", excerpt: report.slice(0, 2e3) }]
|
|
994
|
+
});
|
|
995
|
+
if (fallback) {
|
|
996
|
+
out.push(toAnalystFinding(spec, version, fallback, { outcome: "extraction_failed" }));
|
|
997
|
+
} else {
|
|
998
|
+
throw new Error(
|
|
999
|
+
`Trace analyst '${spec.id}' produced a substantive report, but no finding satisfied its acceptance rules`
|
|
1000
|
+
);
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
613
1003
|
}
|
|
1004
|
+
return out;
|
|
1005
|
+
} finally {
|
|
1006
|
+
const usage = await settleUsageReceiptFromCostLedger(costLedger, {
|
|
1007
|
+
tags: {
|
|
1008
|
+
analystId: spec.id,
|
|
1009
|
+
...ctx.correlationId ? { analystRunId: ctx.correlationId } : {}
|
|
1010
|
+
},
|
|
1011
|
+
timeoutMs: settlementTimeoutMs
|
|
1012
|
+
});
|
|
1013
|
+
if (!usage.settled) {
|
|
1014
|
+
ctx.log?.(`analyst.kind ${spec.id} provider settlement timed out`, {
|
|
1015
|
+
pending_calls: usage.pendingCalls,
|
|
1016
|
+
timeout_ms: settlementTimeoutMs
|
|
1017
|
+
});
|
|
1018
|
+
}
|
|
1019
|
+
ctx.recordUsage?.(usage.receipt);
|
|
614
1020
|
}
|
|
615
|
-
return out;
|
|
616
1021
|
}
|
|
617
1022
|
};
|
|
618
1023
|
}
|
|
1024
|
+
function rejectRemovedKindOptions(spec) {
|
|
1025
|
+
const supplied = spec;
|
|
1026
|
+
const migrations = [
|
|
1027
|
+
["recursion", "subqueries"],
|
|
1028
|
+
["responderDescription", "actorDescription"],
|
|
1029
|
+
["maxDepth", "subqueries"],
|
|
1030
|
+
["maxParallelSubagents", "subqueries.maxParallel"],
|
|
1031
|
+
["subagentDescription", "actorDescription"]
|
|
1032
|
+
];
|
|
1033
|
+
for (const [removed, replacement] of migrations) {
|
|
1034
|
+
if (removed in supplied) {
|
|
1035
|
+
throw new TypeError(
|
|
1036
|
+
`createTraceAnalystKind: '${removed}' is unsupported; use '${replacement}'`
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
619
1041
|
function deriveQuestion(ctx, spec) {
|
|
620
1042
|
const focus = ctx.tags?.focus?.trim();
|
|
621
1043
|
const task = `Analyze this trace dataset with the available tools and report ${spec.area} findings. ${spec.description}`;
|
|
622
1044
|
return focus ? `${task} Focus: ${focus}.` : task;
|
|
623
1045
|
}
|
|
624
|
-
function toAnalystFinding(spec, raw) {
|
|
1046
|
+
function toAnalystFinding(spec, version, raw, metadata = {}) {
|
|
625
1047
|
return makeFinding({
|
|
626
1048
|
analyst_id: spec.id,
|
|
627
1049
|
area: spec.area,
|
|
@@ -630,25 +1052,11 @@ function toAnalystFinding(spec, raw) {
|
|
|
630
1052
|
rationale: raw.rationale,
|
|
631
1053
|
severity: raw.severity,
|
|
632
1054
|
confidence: raw.confidence,
|
|
633
|
-
evidence_refs:
|
|
634
|
-
{
|
|
635
|
-
kind: evidenceKindFromUri(raw.evidence_uri),
|
|
636
|
-
uri: raw.evidence_uri,
|
|
637
|
-
excerpt: raw.evidence_excerpt
|
|
638
|
-
}
|
|
639
|
-
],
|
|
1055
|
+
evidence_refs: evidenceRefsFromRawFinding(raw),
|
|
640
1056
|
recommended_action: raw.recommended_action,
|
|
641
|
-
metadata: { kind_version:
|
|
1057
|
+
metadata: { kind_version: version, ...metadata }
|
|
642
1058
|
});
|
|
643
1059
|
}
|
|
644
|
-
function evidenceKindFromUri(uri) {
|
|
645
|
-
if (uri.startsWith("span://")) return "span";
|
|
646
|
-
if (uri.startsWith("artifact://")) return "artifact";
|
|
647
|
-
if (uri.startsWith("metric://")) return "metric";
|
|
648
|
-
if (uri.startsWith("event://")) return "event";
|
|
649
|
-
if (uri.startsWith("finding://")) return "finding";
|
|
650
|
-
return "artifact";
|
|
651
|
-
}
|
|
652
1060
|
function renderPriorFindings(prior) {
|
|
653
1061
|
if (!prior || prior.length === 0) return "";
|
|
654
1062
|
const MAX_ROWS = 40;
|
|
@@ -668,6 +1076,26 @@ function renderPriorFindings(prior) {
|
|
|
668
1076
|
overflow
|
|
669
1077
|
].filter(Boolean).join("\n");
|
|
670
1078
|
}
|
|
1079
|
+
function renderUpstreamFindings(upstream) {
|
|
1080
|
+
if (!upstream || upstream.length === 0) return "";
|
|
1081
|
+
const MAX_ROWS = 40;
|
|
1082
|
+
const rows = upstream.slice(0, MAX_ROWS).map((finding) => {
|
|
1083
|
+
const subject = finding.subject ? ` [${finding.subject}]` : "";
|
|
1084
|
+
const action = finding.recommended_action ? ` action=${truncateForContext(finding.recommended_action, 120)}` : "";
|
|
1085
|
+
const evidence = finding.evidence_refs[0] ? ` evidence=${truncateForContext(finding.evidence_refs[0].uri, 120)}` : "";
|
|
1086
|
+
return ` - id=${finding.finding_id} source=${finding.analyst_id} ${finding.severity}${subject} claim=${truncateForContext(finding.claim, 160)}${action}${evidence}`;
|
|
1087
|
+
});
|
|
1088
|
+
const overflow = upstream.length > MAX_ROWS ? `
|
|
1089
|
+
... +${upstream.length - MAX_ROWS} more upstream findings (truncated)` : "";
|
|
1090
|
+
return [
|
|
1091
|
+
"",
|
|
1092
|
+
"",
|
|
1093
|
+
"UPSTREAM FINDINGS (produced earlier in this same registry run):",
|
|
1094
|
+
"Use these as intermediate evidence. Build on them instead of repeating the same diagnosis, and cite a dependency with `finding://<id>`.",
|
|
1095
|
+
...rows,
|
|
1096
|
+
overflow
|
|
1097
|
+
].filter(Boolean).join("\n");
|
|
1098
|
+
}
|
|
671
1099
|
function truncateForContext(s, max) {
|
|
672
1100
|
if (s.length <= max) return s;
|
|
673
1101
|
return `${s.slice(0, max - 1).trimEnd()}\u2026`;
|
|
@@ -713,43 +1141,30 @@ DISCOVERY \u2192 CLUSTER \u2192 CITE protocol:
|
|
|
713
1141
|
2. Use \`traces.queryTraces({ filters: { has_errors: true }, limit })\` to pull error-bearing traces. Combine with \`traces.countTraces\` to see what fraction of the dataset failed.
|
|
714
1142
|
3. For each candidate failure cluster, use \`traces.searchTrace\` with regex like \`STATUS_CODE_ERROR\`, \`MaxTurnsExceeded\`, \`assertion\`, \`unauthorized\`, \`timeout\`, \`429\`, \`5\\d\\d\`, the agent's specific error strings, or the names of its tools. Pull one or two representative traces per cluster, **not all** of them.
|
|
715
1143
|
4. **Cluster, do not enumerate.** Two failures with the same root cause should be ONE finding citing both traces, not two findings. The point of this analyst is to compress N runs into K modes.
|
|
716
|
-
5. For each cluster
|
|
717
|
-
- \`area\` = "failure-mode"
|
|
718
|
-
- \`subject\` = a lowercase cluster label matching the subject grammar above ("tool-call-loop", "auth-revoked-mid-run", ...)
|
|
719
|
-
- \`claim\` = one sentence stating the mode
|
|
720
|
-
- \`severity\` = "critical" when it blocks the run, "high" when the run finished degraded, "medium" when it slowed convergence
|
|
721
|
-
- \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the most representative span
|
|
722
|
-
- \`evidence_excerpt\` = the exact quote (e.g. error message, stuck tool call payload, contradictory turn output)
|
|
723
|
-
- \`confidence\` = 0.85+ when multiple traces show the same shape; 0.6-0.8 for a single-trace inference; <0.5 for speculative.
|
|
724
|
-
- \`recommended_action\` = imperative-phrased fix idea (kept short \u2014 the improvement-analyst will expand on these)
|
|
1144
|
+
5. For each defensible cluster, emit ONE finding. Use a lowercase cluster label matching the subject grammar ("tool-call-loop", "auth-revoked-mid-run", ...). Rate it critical when it blocks the run, high when the run finishes degraded, and medium when it slows convergence. Cite representative spans and include exact error, payload, or contradictory-output quotes. Use confidence 0.85+ when multiple traces show the same shape, 0.6-0.8 for a single-trace inference, and <0.5 for speculation. Keep the imperative fix idea short; the improvement analyst expands it.
|
|
725
1145
|
|
|
726
1146
|
If the dataset has no failures, return an empty findings array \u2014 do NOT pad with low-confidence speculation.
|
|
727
1147
|
|
|
728
|
-
**
|
|
729
|
-
- After your first \`getDatasetOverview\` + \`queryTraces\` calls, you should have 3-6 candidate failure clusters in mind. Spawn one \`llmQuery\` per cluster in a single batch \u2014 they investigate in parallel.
|
|
730
|
-
- A sub-investigator that finds its cluster is actually two distinct modes should split again at its own level. Recursion is meant to discover sub-modes, not to do trivial drilling that the parent could do in-line.
|
|
731
|
-
- Pass narrow context to each subagent: { question: 'investigate the auth-revoked-mid-run cluster', context: { trace_ids: ['abc', 'def'], suspected_root_cause: 'token refresh skipped on idle sessions' } }. Subagents need enough context to skip re-discovery but not the whole conversation.
|
|
732
|
-
- Each subagent returns its findings as JSON; the parent merges them. Do NOT have subagents call \`final()\` \u2014 they return their findings list to you, and you call \`final()\` once at the top.
|
|
1148
|
+
**Use subqueries over loaded evidence.** After the first scan, load representative span excerpts for each candidate cluster. Then send one bounded \`llmQuery\` per cluster in one batch, including the exact excerpts and asking it to classify the root cause. Subqueries cannot call trace tools. Merge or split clusters yourself from their classifications and the cited source evidence.
|
|
733
1149
|
|
|
734
1150
|
OBSERVABILITY rules:
|
|
735
1151
|
- Each non-final turn must emit at least one \`console.log\` for evidence.
|
|
736
|
-
- Reuse runtime variables across turns; don't recompute
|
|
737
|
-
- Call \`final({ findings: [...] })\` exactly once, after you've gathered evidence for every cluster you intend to report.`;
|
|
1152
|
+
- Reuse runtime variables across turns; don't recompute.`;
|
|
738
1153
|
var FAILURE_MODE_KIND_SPEC = {
|
|
739
1154
|
id: "failure-mode",
|
|
740
1155
|
description: "Clusters trace-dataset failures into distinct failure modes with cited evidence and a short recommended action.",
|
|
741
1156
|
area: "failure-mode",
|
|
742
|
-
version: "1.
|
|
1157
|
+
version: "1.2.0",
|
|
743
1158
|
actorDescription: ACTOR_PROMPT,
|
|
744
1159
|
buildTools: (store) => buildTraceToolsForGroup("all", store),
|
|
745
|
-
|
|
1160
|
+
subqueries: { maxCalls: 8, maxParallel: 4 },
|
|
746
1161
|
maxTurns: 24,
|
|
747
1162
|
cost: { kind: "llm" }
|
|
748
1163
|
};
|
|
749
1164
|
|
|
750
1165
|
// src/analyst/kinds/improvement.ts
|
|
751
1166
|
var subjectGrammar2 = findingSubjectGrammarPromptFor("improvement");
|
|
752
|
-
var ACTOR_PROMPT2 = `You are a
|
|
1167
|
+
var ACTOR_PROMPT2 = `You are a self-improvement analyst. Your job is to propose **concrete, locus-named edits** the agent's runtime should adopt to fix the failure modes, knowledge gaps, and poisonings present in this dataset.
|
|
753
1168
|
|
|
754
1169
|
Upstream analysts have already classified the problems. Your job is to convert each problem into a *change to make* and grade its expected leverage. Each finding is one proposed edit.
|
|
755
1170
|
|
|
@@ -769,36 +1184,26 @@ DISCOVERY \u2192 CANDIDATE-FIXES \u2192 COMPETE \u2192 CITE protocol:
|
|
|
769
1184
|
- **Skill / MCP / hook / subagent** \u2014 change the reusable profile component responsible for the behavior
|
|
770
1185
|
- **Workflow / rollout policy** \u2014 change orchestration, budget, sampling, or stopping behavior
|
|
771
1186
|
- **Code** \u2014 change an implementation path when profile edits cannot repair the behavior
|
|
772
|
-
3. **
|
|
773
|
-
4. After
|
|
774
|
-
5. **Cross-reference upstream findings.**
|
|
1187
|
+
3. **Compare candidate fixes with bounded subqueries.** Load the representative failure excerpts, then send one \`llmQuery\` per candidate-fix axis the same evidence. Ask for likely effect, side effects, and implementation scope. Subqueries cannot call trace tools; trace ids alone are insufficient context.
|
|
1188
|
+
4. After the comparisons return, **pick the winning candidate per cluster** based on expected effect and risk, then emit ONE finding. Keep the alternatives and rejection reasons in the rationale so the recommendation is auditable.
|
|
1189
|
+
5. **Cross-reference upstream findings.** Cite prior failure-mode or knowledge-gap findings as \`finding://<prior-finding-id>\`. This builds the dependency graph that lets the dashboard show "fix #X resolves failure modes A, B, C."
|
|
775
1190
|
|
|
776
|
-
For each winning recommendation, emit ONE finding with
|
|
777
|
-
- \`area\` = "improvement"
|
|
778
|
-
- \`subject\` = one exact locus form from the subject grammar above
|
|
779
|
-
- \`claim\` = one sentence stating the edit ("Add a precondition check to refuse tool X calls without arg Y")
|
|
780
|
-
- \`severity\` = leverage rating: "critical" when fix resolves a critical failure mode; "high" when it resolves a high; "medium" when it's a quality-of-life win; "info" when it's a cleanup with no behavioral effect
|
|
781
|
-
- \`evidence_uri\` = the failure-mode finding id this fix targets (\`finding://<id>\`) when it exists; else the most representative span
|
|
782
|
-
- \`evidence_excerpt\` = a fragment showing the problem the fix targets
|
|
783
|
-
- \`confidence\` = 0.85+ when the fix is mechanical and the failure mode is well-evidenced; 0.6-0.8 when the fix requires judgment; <0.5 for speculative
|
|
784
|
-
- \`rationale\` = why this candidate beat its alternatives (2 sentences max)
|
|
785
|
-
- \`recommended_action\` = the **literal edit**, phrased as a diff or a quoted replacement: "Replace section X with: '...'" or "Add tool with description: '...'" or "Set retry policy to max_attempts=3 with exponential backoff"
|
|
1191
|
+
For each winning recommendation, emit ONE finding. Use one exact locus from the subject grammar and state the edit in one sentence. Match leverage to the source failure's severity; use medium for quality-of-life changes and info for cleanup with no behavioral effect. Cite the targeted \`finding://<id>\` when available and the most representative span when useful. Quote the problem being fixed. Use confidence 0.85+ for a mechanical fix to a well-evidenced failure, 0.6-0.8 when judgment is required, and <0.5 for speculation. Explain in at most two sentences why this candidate beat its alternatives. The recommended action must be the literal diff, quoted replacement, tool description, or setting change.
|
|
786
1192
|
|
|
787
1193
|
If no upstream failure findings exist in this run, derive your own from the trace dataset using the failure-mode protocol inline (\`searchTrace\` for STATUS_CODE_ERROR / MaxTurnsExceeded / etc.). But prefer to consume upstream findings when present \u2014 the kinds are designed to chain.
|
|
788
1194
|
|
|
789
1195
|
Do NOT propose a fix you cannot defend with evidence. "Tighten the prompt" is not a finding; "Add 'When the user asks for X, always Y' to the system prompt section "request-classification"" is.
|
|
790
1196
|
|
|
791
1197
|
OBSERVABILITY rules:
|
|
792
|
-
- Each non-final turn must emit at least one \`console.log\` for evidence
|
|
793
|
-
- Call \`final({ findings: [...] })\` exactly once at the top level.`;
|
|
1198
|
+
- Each non-final turn must emit at least one \`console.log\` for evidence.`;
|
|
794
1199
|
var IMPROVEMENT_KIND_SPEC = {
|
|
795
1200
|
id: "improvement",
|
|
796
1201
|
description: "Converts upstream failure / gap / poisoning findings into concrete locus-named edits (prompt, tool-doc, RAG, scaffolding) with leverage grades.",
|
|
797
1202
|
area: "improvement",
|
|
798
|
-
version: "1.
|
|
1203
|
+
version: "1.2.0",
|
|
799
1204
|
actorDescription: ACTOR_PROMPT2,
|
|
800
1205
|
buildTools: (store) => buildTraceToolsForGroup("all", store),
|
|
801
|
-
|
|
1206
|
+
subqueries: { maxCalls: 8, maxParallel: 4 },
|
|
802
1207
|
maxTurns: 30,
|
|
803
1208
|
maxRuntimeChars: 12e3,
|
|
804
1209
|
cost: { kind: "llm" }
|
|
@@ -825,31 +1230,22 @@ DISCOVERY \u2192 ATTRIBUTE-TO-LAYER \u2192 CITE protocol:
|
|
|
825
1230
|
- Fabricated identifiers that don't appear in dataset \`sample_trace_ids\`
|
|
826
1231
|
Use \`traces.searchTrace\` with patterns like \`I (don.?t|do not) know\`, \`assumed\`, \`unclear\`, \`could you (clarify|tell me|provide)\`, \`not found\`, \`undefined\`, \`unknown\`, \`null\`, dates older than the analysis window, or the agent's specific clarification phrases.
|
|
827
1232
|
3. For each gap, identify the **layer of the runtime that should have prevented it** and use its exact locus from the subject grammar above.
|
|
828
|
-
4. For each gap
|
|
829
|
-
- \`area\` = "knowledge-gap"
|
|
830
|
-
- \`subject\` = one exact locus form from the subject grammar above
|
|
831
|
-
- \`claim\` = a sentence naming the missing or stale knowledge ("wiki has no page on invoice line-item shape, agent had to re-derive it from raw spans")
|
|
832
|
-
- \`severity\` = "high" when the gap caused a failure or a clarifying question; "medium" when it caused unnecessary turns; "low" when it caused minor inefficiency
|
|
833
|
-
- \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the moment the gap surfaced (the question, the self-correction, the retrieval miss, the stale web result)
|
|
834
|
-
- \`evidence_excerpt\` = exact quote where the agent showed the gap
|
|
835
|
-
- \`confidence\` = 0.85+ when the agent itself articulated the gap; 0.6-0.8 when inferred from behavior
|
|
836
|
-
- \`recommended_action\` = phrased as a wiki edit when the locus is \`agent-knowledge:*\` ("Create wiki page \`invoice-line-items\` with claims: ..."), or as a prompt/tool-doc edit otherwise
|
|
1233
|
+
4. For each defensible gap, emit ONE finding. Use an exact locus from the subject grammar and name the missing or stale knowledge (for example, "wiki has no page on invoice line-item shape; agent re-derived it from raw spans"). Rate it high when it caused failure or a clarifying question, medium for unnecessary turns, and low for minor inefficiency. Cite the span where the question, correction, retrieval miss, or stale result surfaced and quote it exactly. Use confidence 0.85+ when the agent articulated the gap and 0.6-0.8 when inferred. Recommend a concrete wiki edit for an agent-knowledge locus or a prompt/tool-description edit otherwise.
|
|
837
1234
|
|
|
838
|
-
**
|
|
1235
|
+
**Compare layers over loaded evidence.** After the first scan, load the exact excerpts behind candidates across \`agent-knowledge:*\`, \`websearch:outdated\`, \`tool-doc:*\`, \`system-prompt:*\`, and \`memory:*\`. Use one bounded \`llmQuery\` per layer to classify those excerpts. Subqueries cannot call trace tools. Merge their classifications into the final finding set only when the source excerpts support them.
|
|
839
1236
|
|
|
840
1237
|
Do NOT report a gap that the agent later recovered from cleanly within the same turn \u2014 that's resilience, not a gap. Cite the *non-recovery* version when both exist.
|
|
841
1238
|
|
|
842
1239
|
OBSERVABILITY rules:
|
|
843
|
-
- Each non-final turn must emit at least one \`console.log\` for evidence
|
|
844
|
-
- Call \`final({ findings: [...] })\` exactly once at the top level.`;
|
|
1240
|
+
- Each non-final turn must emit at least one \`console.log\` for evidence.`;
|
|
845
1241
|
var KNOWLEDGE_GAP_KIND_SPEC = {
|
|
846
1242
|
id: "knowledge-gap",
|
|
847
1243
|
description: "Identifies missing or stale pieces of knowledge \u2014 primarily against the agent-knowledge wiki \u2014 and attributes each to the runtime layer (wiki page, claim, raw source, websearch, tool-doc, system-prompt, memory) that should have held it.",
|
|
848
1244
|
area: "knowledge-gap",
|
|
849
|
-
version: "1.
|
|
1245
|
+
version: "1.2.0",
|
|
850
1246
|
actorDescription: ACTOR_PROMPT3,
|
|
851
1247
|
buildTools: (store) => buildTraceToolsForGroup("discoveryAndSearch", store),
|
|
852
|
-
|
|
1248
|
+
subqueries: { maxCalls: 5, maxParallel: 4 },
|
|
853
1249
|
maxTurns: 18,
|
|
854
1250
|
cost: { kind: "llm" }
|
|
855
1251
|
};
|
|
@@ -876,32 +1272,24 @@ DISCOVERY \u2192 DUAL-VERIFY \u2192 CITE protocol:
|
|
|
876
1272
|
- Confirm the belief is actually false in this trace's own evidence (cite the span that contradicts it)
|
|
877
1273
|
Only emit a finding when both halves are nailed down. If you can only nail one, drop it \u2014 single-evidence poisoning findings are too speculative to be useful.
|
|
878
1274
|
|
|
879
|
-
**
|
|
1275
|
+
**Independently assess both halves.** Load the action excerpt and contradicting excerpt yourself, then send bounded \`llmQuery\` calls the exact evidence for "did the agent act?" and "does the trace contradict the belief?" Subqueries cannot call trace tools. Accept a poisoning only when both assessments and the source excerpts support it.
|
|
880
1276
|
|
|
881
|
-
For each confirmed poisoning, emit ONE finding with:
|
|
882
|
-
- \`area\` = "knowledge-poisoning"
|
|
883
|
-
- \`subject\` = the source of the false belief using one exact form from the subject grammar above
|
|
884
|
-
- \`claim\` = one sentence: "agent believed X (from source S); evidence in trace shows X is false"
|
|
885
|
-
- \`severity\` = "critical" when poisoning caused a wrong user-visible action; "high" when caught internally but wasted significant work; "medium" for inefficiency only
|
|
886
|
-
- \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the action span (the moment the agent acted on the false belief)
|
|
887
|
-
- \`evidence_excerpt\` = exact quote of the confident-but-wrong claim or action
|
|
888
|
-
- \`confidence\` = 0.85+ when both halves are exact-quote backed; 0.6-0.8 when one half is inferred
|
|
889
|
-
- \`recommended_action\` = where the source should be updated and how ("Update wiki page \`X\` claim \`Y\` to '...'", "Invalidate raw source \`Z\` and re-curate", "Replace system-prompt section X with 'tool foo now returns Y'")
|
|
1277
|
+
For each confirmed poisoning, emit ONE finding. Use the source of the false belief as the exact subject. State "agent believed X (from source S); trace evidence shows X is false." Rate it critical for a wrong user-visible action, high when caught internally after significant waste, and medium for inefficiency. Cite BOTH the action span and the contradicting span with exact quotes. Use confidence 0.85+ when both halves have exact quotes and 0.6-0.8 when one half is inferred. Recommend the literal source correction: update the wiki claim, invalidate and re-curate the raw source, or replace the stale prompt/tool instruction.
|
|
890
1278
|
|
|
891
1279
|
Do NOT report a finding if the agent caught and corrected the false belief in the same turn \u2014 that's the system working. Reserve poisoning for cases where the false belief shaped downstream action.
|
|
892
1280
|
|
|
893
1281
|
OBSERVABILITY rules:
|
|
894
|
-
- Each non-final turn must emit at least one \`console.log\` for evidence
|
|
895
|
-
- Call \`final({ findings: [...] })\` exactly once at the top level.`;
|
|
1282
|
+
- Each non-final turn must emit at least one \`console.log\` for evidence.`;
|
|
896
1283
|
var KNOWLEDGE_POISONING_KIND_SPEC = {
|
|
897
1284
|
id: "knowledge-poisoning",
|
|
898
1285
|
description: "Identifies confident-but-wrong actions caused by stale memory, contradicting RAG, deprecated tool docs, or outdated system-prompt instructions.",
|
|
899
1286
|
area: "knowledge-poisoning",
|
|
900
|
-
version: "1.
|
|
1287
|
+
version: "1.2.0",
|
|
901
1288
|
actorDescription: ACTOR_PROMPT4,
|
|
902
1289
|
buildTools: (store) => buildTraceToolsForGroup("all", store),
|
|
903
|
-
|
|
1290
|
+
subqueries: { maxCalls: 8, maxParallel: 4 },
|
|
904
1291
|
maxTurns: 20,
|
|
1292
|
+
minimumEvidenceCitations: 2,
|
|
905
1293
|
cost: { kind: "llm" }
|
|
906
1294
|
};
|
|
907
1295
|
|
|
@@ -929,6 +1317,14 @@ var AnalystRegistry = class {
|
|
|
929
1317
|
if (!analyst.version) {
|
|
930
1318
|
throw new Error(`AnalystRegistry.register: analyst "${analyst.id}" must declare a version`);
|
|
931
1319
|
}
|
|
1320
|
+
if (analyst.cost.kind === "deterministic" && analyst.cost.settlement_timeout_ms !== void 0) {
|
|
1321
|
+
throw new TypeError(
|
|
1322
|
+
`AnalystRegistry.register: deterministic analyst "${analyst.id}" cannot declare settlement_timeout_ms`
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
if (analyst.cost.settlement_timeout_ms !== void 0) {
|
|
1326
|
+
validateUsageSettlementTimeout(analyst.cost.settlement_timeout_ms);
|
|
1327
|
+
}
|
|
932
1328
|
this.analysts.set(analyst.id, analyst);
|
|
933
1329
|
}
|
|
934
1330
|
list() {
|
|
@@ -963,9 +1359,13 @@ var AnalystRegistry = class {
|
|
|
963
1359
|
const hooks = this.options.hooks ?? {};
|
|
964
1360
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
965
1361
|
const started = Date.now();
|
|
966
|
-
const
|
|
1362
|
+
const timeoutMs = validateTimeout(runOpts.timeoutMs);
|
|
1363
|
+
const deadlineMs = timeoutMs === void 0 ? void 0 : started + timeoutMs;
|
|
1364
|
+
const timeoutSignal = timeoutMs === void 0 ? void 0 : AbortSignal.timeout(timeoutMs);
|
|
1365
|
+
const runSignal = combineAbortSignals(runOpts.signal, timeoutSignal);
|
|
967
1366
|
const selected = this.selectAnalysts(runOpts);
|
|
968
1367
|
const budget = runOpts.budget ?? this.options.defaultBudget;
|
|
1368
|
+
validateBudgetPolicy(budget);
|
|
969
1369
|
yield {
|
|
970
1370
|
type: "run-started",
|
|
971
1371
|
run_id: runId,
|
|
@@ -977,94 +1377,173 @@ var AnalystRegistry = class {
|
|
|
977
1377
|
const allFindings = [];
|
|
978
1378
|
let totalCost = 0;
|
|
979
1379
|
let remainingUsd = budget?.totalUsd;
|
|
980
|
-
const
|
|
981
|
-
|
|
982
|
-
|
|
1380
|
+
const runnableAnalysts = selected.filter((a) => this.routeInput(a, inputs).kind !== "missing");
|
|
1381
|
+
const runnableCount = runnableAnalysts.length;
|
|
1382
|
+
const weights = budget?.weights;
|
|
1383
|
+
const totalWeight = weights && budget?.totalUsd != null && !budget.allocate && runnableCount > 0 ? runnableAnalysts.reduce((sum, analyst) => sum + analystWeight(weights, analyst.id), 0) : void 0;
|
|
1384
|
+
if (totalWeight === 0) {
|
|
1385
|
+
throw new Error("BudgetPolicy.weights must allocate positive weight to a runnable analyst");
|
|
1386
|
+
}
|
|
983
1387
|
for (const analyst of selected) {
|
|
984
1388
|
const t0 = Date.now();
|
|
1389
|
+
if (runSignal?.aborted) {
|
|
1390
|
+
const summary2 = abortedBeforeStartSummary(analyst, runSignal);
|
|
1391
|
+
summaries.push(summary2);
|
|
1392
|
+
log(`[analyst] skip ${analyst.id} \u2014 run aborted`, { runId, reason: summary2.reason });
|
|
1393
|
+
yield { type: "analyst-skipped", summary: summary2 };
|
|
1394
|
+
continue;
|
|
1395
|
+
}
|
|
985
1396
|
const input = this.routeInput(analyst, inputs);
|
|
986
1397
|
if (input.kind === "missing") {
|
|
987
|
-
const
|
|
1398
|
+
const summary2 = {
|
|
988
1399
|
analyst_id: analyst.id,
|
|
989
1400
|
status: "skipped",
|
|
990
1401
|
reason: `missing input of kind '${analyst.inputKind}'`,
|
|
991
1402
|
findings_count: 0,
|
|
992
1403
|
latency_ms: 0,
|
|
993
|
-
cost_usd: 0
|
|
1404
|
+
cost_usd: 0,
|
|
1405
|
+
usage: zeroUsage()
|
|
994
1406
|
};
|
|
995
|
-
summaries.push(
|
|
1407
|
+
summaries.push(summary2);
|
|
996
1408
|
log(`[analyst] skip ${analyst.id} \u2014 missing input`, { runId, kind: analyst.inputKind });
|
|
997
|
-
await
|
|
998
|
-
|
|
1409
|
+
await waitForHook(
|
|
1410
|
+
hooks.onAfterAnalyze ? () => hooks.onAfterAnalyze?.({ analyst, summary: summary2, findings: [], runId }) : void 0,
|
|
1411
|
+
runSignal
|
|
1412
|
+
);
|
|
1413
|
+
yield { type: "analyst-skipped", summary: summary2 };
|
|
999
1414
|
continue;
|
|
1000
1415
|
}
|
|
1001
1416
|
const perBudget = allocateBudget(budget, {
|
|
1002
1417
|
analyst,
|
|
1003
1418
|
remainingUsd,
|
|
1004
|
-
runningCount: runnableCount
|
|
1419
|
+
runningCount: runnableCount,
|
|
1420
|
+
totalWeight
|
|
1005
1421
|
});
|
|
1422
|
+
const usageReceipts = [];
|
|
1006
1423
|
const ctx = {
|
|
1007
1424
|
runId,
|
|
1008
1425
|
correlationId,
|
|
1009
1426
|
deadlineMs,
|
|
1010
1427
|
budgetUsd: perBudget,
|
|
1428
|
+
costLedger: runOpts.costLedger,
|
|
1429
|
+
costPhase: runOpts.costPhase,
|
|
1011
1430
|
chat: this.options.chat,
|
|
1012
1431
|
tags: runOpts.tags,
|
|
1013
1432
|
log: (msg, fields) => log(`[${analyst.id}] ${msg}`, { runId, correlationId, ...fields }),
|
|
1014
|
-
signal:
|
|
1015
|
-
priorFindings: selectPriorFindings(runOpts.priorFindings, analyst.id)
|
|
1433
|
+
signal: runSignal,
|
|
1434
|
+
priorFindings: selectPriorFindings(runOpts.priorFindings, analyst.id),
|
|
1435
|
+
upstreamFindings: runOpts.chainFindings && allFindings.length > 0 ? [...allFindings] : void 0,
|
|
1436
|
+
recordUsage: (receipt) => {
|
|
1437
|
+
assertValidUsageReceipt(receipt);
|
|
1438
|
+
usageReceipts.push(receipt);
|
|
1439
|
+
}
|
|
1016
1440
|
};
|
|
1017
|
-
await
|
|
1441
|
+
await waitForHook(
|
|
1442
|
+
hooks.onBeforeAnalyze ? () => hooks.onBeforeAnalyze?.({ analyst, ctx, runId }) : void 0,
|
|
1443
|
+
runSignal
|
|
1444
|
+
);
|
|
1445
|
+
if (runSignal?.aborted) {
|
|
1446
|
+
const summary2 = abortedBeforeStartSummary(analyst, runSignal, Date.now() - t0);
|
|
1447
|
+
summaries.push(summary2);
|
|
1448
|
+
log(`[analyst] skip ${analyst.id} \u2014 run aborted`, { runId, reason: summary2.reason });
|
|
1449
|
+
yield { type: "analyst-skipped", summary: summary2 };
|
|
1450
|
+
continue;
|
|
1451
|
+
}
|
|
1452
|
+
const effectiveBudget = validateEffectiveBudget(ctx.budgetUsd, remainingUsd, analyst.id);
|
|
1018
1453
|
yield {
|
|
1019
1454
|
type: "analyst-started",
|
|
1020
1455
|
analyst_id: analyst.id,
|
|
1021
1456
|
started_at: new Date(t0).toISOString()
|
|
1022
1457
|
};
|
|
1458
|
+
let findings;
|
|
1459
|
+
let summary;
|
|
1023
1460
|
try {
|
|
1024
|
-
|
|
1461
|
+
if (runSignal?.aborted) throw abortReason(runSignal);
|
|
1462
|
+
findings = await waitForOperation(
|
|
1463
|
+
analyst.analyze(input.value, ctx),
|
|
1464
|
+
runSignal,
|
|
1465
|
+
analystAbortGraceMs(analyst)
|
|
1466
|
+
);
|
|
1025
1467
|
const latency = Date.now() - t0;
|
|
1026
|
-
const
|
|
1468
|
+
const usage = resolveUsage(analyst, findings, usageReceipts);
|
|
1469
|
+
const cost = knownCostUsd(usage);
|
|
1027
1470
|
totalCost += cost;
|
|
1028
|
-
if (typeof remainingUsd === "number")
|
|
1471
|
+
if (typeof remainingUsd === "number") {
|
|
1472
|
+
remainingUsd = Math.max(0, remainingUsd - budgetDebit(usage, effectiveBudget));
|
|
1473
|
+
}
|
|
1029
1474
|
allFindings.push(...findings);
|
|
1030
|
-
|
|
1475
|
+
summary = {
|
|
1031
1476
|
analyst_id: analyst.id,
|
|
1032
1477
|
status: "ok",
|
|
1033
1478
|
findings_count: findings.length,
|
|
1034
1479
|
latency_ms: latency,
|
|
1035
|
-
cost_usd: cost
|
|
1480
|
+
cost_usd: cost,
|
|
1481
|
+
usage
|
|
1036
1482
|
};
|
|
1037
1483
|
summaries.push(summary);
|
|
1038
1484
|
log(`[analyst] ok ${analyst.id}`, {
|
|
1039
1485
|
runId,
|
|
1040
1486
|
findings: findings.length,
|
|
1041
1487
|
latency_ms: latency,
|
|
1042
|
-
cost_usd: cost
|
|
1488
|
+
cost_usd: cost,
|
|
1489
|
+
cost_kind: usage.cost.kind,
|
|
1490
|
+
input_tokens: usage.tokens?.input ?? null,
|
|
1491
|
+
output_tokens: usage.tokens?.output ?? null
|
|
1043
1492
|
});
|
|
1044
|
-
|
|
1045
|
-
|
|
1493
|
+
if (effectiveBudget !== void 0 && usage.cost.kind === "uncaptured") {
|
|
1494
|
+
log(`[analyst] WARN ${analyst.id} \u2014 USD cost uncaptured; budget not reconciled`, {
|
|
1495
|
+
runId,
|
|
1496
|
+
budget_usd: effectiveBudget,
|
|
1497
|
+
cost_captured: false
|
|
1498
|
+
});
|
|
1499
|
+
}
|
|
1046
1500
|
} catch (err) {
|
|
1047
1501
|
const latency = Date.now() - t0;
|
|
1048
1502
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
1049
|
-
const hookFindings = await hooks.onError?.({ analyst, error: e, runId }) ?? [];
|
|
1503
|
+
const hookFindings = runSignal?.aborted ? [] : await hooks.onError?.({ analyst, error: e, runId }) ?? [];
|
|
1050
1504
|
if (hookFindings.length) allFindings.push(...hookFindings);
|
|
1051
|
-
const
|
|
1505
|
+
const usage = resolveUsage(analyst, hookFindings, usageReceipts);
|
|
1506
|
+
const cost = knownCostUsd(usage);
|
|
1507
|
+
totalCost += cost;
|
|
1508
|
+
if (typeof remainingUsd === "number") {
|
|
1509
|
+
remainingUsd = Math.max(0, remainingUsd - budgetDebit(usage, effectiveBudget));
|
|
1510
|
+
}
|
|
1511
|
+
const summary2 = {
|
|
1052
1512
|
analyst_id: analyst.id,
|
|
1053
1513
|
status: "failed",
|
|
1054
1514
|
findings_count: hookFindings.length,
|
|
1055
1515
|
latency_ms: latency,
|
|
1056
|
-
cost_usd:
|
|
1516
|
+
cost_usd: cost,
|
|
1517
|
+
usage,
|
|
1057
1518
|
error: { class: e.constructor.name, message: e.message }
|
|
1058
1519
|
};
|
|
1059
|
-
summaries.push(
|
|
1520
|
+
summaries.push(summary2);
|
|
1060
1521
|
log(`[analyst] FAIL ${analyst.id}`, {
|
|
1061
1522
|
runId,
|
|
1062
1523
|
error_class: e.constructor.name,
|
|
1063
|
-
error: e.message
|
|
1524
|
+
error: e.message,
|
|
1525
|
+
cost_usd: cost,
|
|
1526
|
+
cost_kind: usage.cost.kind
|
|
1064
1527
|
});
|
|
1065
|
-
|
|
1066
|
-
|
|
1528
|
+
if (effectiveBudget !== void 0 && usage.cost.kind === "uncaptured") {
|
|
1529
|
+
log(`[analyst] WARN ${analyst.id} \u2014 USD cost uncaptured; budget not reconciled`, {
|
|
1530
|
+
runId,
|
|
1531
|
+
budget_usd: effectiveBudget,
|
|
1532
|
+
cost_captured: false
|
|
1533
|
+
});
|
|
1534
|
+
}
|
|
1535
|
+
await waitForHook(
|
|
1536
|
+
hooks.onAfterAnalyze ? () => hooks.onAfterAnalyze?.({ analyst, summary: summary2, findings: hookFindings, runId }) : void 0,
|
|
1537
|
+
runSignal
|
|
1538
|
+
);
|
|
1539
|
+
yield { type: "analyst-completed", summary: summary2, findings: hookFindings };
|
|
1540
|
+
continue;
|
|
1067
1541
|
}
|
|
1542
|
+
await waitForHook(
|
|
1543
|
+
hooks.onAfterAnalyze ? () => hooks.onAfterAnalyze?.({ analyst, summary, findings, runId }) : void 0,
|
|
1544
|
+
runSignal
|
|
1545
|
+
);
|
|
1546
|
+
yield { type: "analyst-completed", summary, findings };
|
|
1068
1547
|
}
|
|
1069
1548
|
const result = {
|
|
1070
1549
|
run_id: runId,
|
|
@@ -1073,9 +1552,15 @@ var AnalystRegistry = class {
|
|
|
1073
1552
|
ended_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1074
1553
|
findings: allFindings,
|
|
1075
1554
|
per_analyst: summaries,
|
|
1076
|
-
total_cost_usd: totalCost
|
|
1555
|
+
total_cost_usd: totalCost,
|
|
1556
|
+
total_cost_provenance: aggregateCostProvenance(
|
|
1557
|
+
summaries.map((summary) => summary.usage?.cost ?? { kind: "uncaptured", usd: null })
|
|
1558
|
+
)
|
|
1077
1559
|
};
|
|
1078
|
-
await
|
|
1560
|
+
await waitForHook(
|
|
1561
|
+
hooks.onComplete ? () => hooks.onComplete?.({ result }) : void 0,
|
|
1562
|
+
runSignal
|
|
1563
|
+
);
|
|
1079
1564
|
yield { type: "run-completed", result };
|
|
1080
1565
|
}
|
|
1081
1566
|
selectAnalysts(opts) {
|
|
@@ -1107,31 +1592,263 @@ var AnalystRegistry = class {
|
|
|
1107
1592
|
}
|
|
1108
1593
|
}
|
|
1109
1594
|
};
|
|
1595
|
+
function validateTimeout(timeoutMs) {
|
|
1596
|
+
if (timeoutMs === void 0) return void 0;
|
|
1597
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2147483647) {
|
|
1598
|
+
throw new TypeError(
|
|
1599
|
+
"RegistryRunOpts.timeoutMs must be a positive safe integer no greater than 2147483647"
|
|
1600
|
+
);
|
|
1601
|
+
}
|
|
1602
|
+
return timeoutMs;
|
|
1603
|
+
}
|
|
1604
|
+
function combineAbortSignals(caller, timeout) {
|
|
1605
|
+
if (!caller) return timeout;
|
|
1606
|
+
if (!timeout) return caller;
|
|
1607
|
+
return AbortSignal.any([caller, timeout]);
|
|
1608
|
+
}
|
|
1609
|
+
async function waitForOperation(operation, signal, abortGraceMs) {
|
|
1610
|
+
if (!signal) return operation;
|
|
1611
|
+
if (signal.aborted) {
|
|
1612
|
+
void operation.catch(() => {
|
|
1613
|
+
});
|
|
1614
|
+
throw abortReason(signal);
|
|
1615
|
+
}
|
|
1616
|
+
return new Promise((resolve, reject) => {
|
|
1617
|
+
let settlementTimer;
|
|
1618
|
+
const cleanup = () => {
|
|
1619
|
+
signal.removeEventListener("abort", onAbort);
|
|
1620
|
+
if (settlementTimer) clearTimeout(settlementTimer);
|
|
1621
|
+
};
|
|
1622
|
+
const onAbort = () => {
|
|
1623
|
+
if (abortGraceMs === 0) {
|
|
1624
|
+
cleanup();
|
|
1625
|
+
reject(abortReason(signal));
|
|
1626
|
+
return;
|
|
1627
|
+
}
|
|
1628
|
+
settlementTimer = setTimeout(() => {
|
|
1629
|
+
cleanup();
|
|
1630
|
+
reject(abortReason(signal));
|
|
1631
|
+
}, abortGraceMs);
|
|
1632
|
+
};
|
|
1633
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1634
|
+
operation.then(
|
|
1635
|
+
(value) => {
|
|
1636
|
+
cleanup();
|
|
1637
|
+
if (signal.aborted) reject(abortReason(signal));
|
|
1638
|
+
else resolve(value);
|
|
1639
|
+
},
|
|
1640
|
+
(error) => {
|
|
1641
|
+
cleanup();
|
|
1642
|
+
reject(signal.aborted ? abortReason(signal) : error);
|
|
1643
|
+
}
|
|
1644
|
+
);
|
|
1645
|
+
});
|
|
1646
|
+
}
|
|
1647
|
+
async function waitForHook(operation, signal) {
|
|
1648
|
+
if (operation === void 0 || signal?.aborted) return void 0;
|
|
1649
|
+
try {
|
|
1650
|
+
return await waitForOperation(
|
|
1651
|
+
Promise.resolve().then(() => {
|
|
1652
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
1653
|
+
return operation();
|
|
1654
|
+
}),
|
|
1655
|
+
signal,
|
|
1656
|
+
0
|
|
1657
|
+
);
|
|
1658
|
+
} catch (error) {
|
|
1659
|
+
if (signal?.aborted) return void 0;
|
|
1660
|
+
throw error;
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
function analystAbortGraceMs(analyst) {
|
|
1664
|
+
if (analyst.cost.kind === "deterministic") return 0;
|
|
1665
|
+
const settlementMs = validateUsageSettlementTimeout(analyst.cost.settlement_timeout_ms);
|
|
1666
|
+
if (settlementMs === 0) return 0;
|
|
1667
|
+
return Math.min(settlementMs + 100, 2147483647);
|
|
1668
|
+
}
|
|
1669
|
+
function abortedBeforeStartSummary(analyst, signal, latencyMs = 0) {
|
|
1670
|
+
const reason = abortReason(signal);
|
|
1671
|
+
return {
|
|
1672
|
+
analyst_id: analyst.id,
|
|
1673
|
+
status: "skipped",
|
|
1674
|
+
reason: `${reason.name}: ${reason.message}`,
|
|
1675
|
+
findings_count: 0,
|
|
1676
|
+
latency_ms: latencyMs,
|
|
1677
|
+
cost_usd: 0,
|
|
1678
|
+
usage: zeroUsage()
|
|
1679
|
+
};
|
|
1680
|
+
}
|
|
1681
|
+
function abortReason(signal) {
|
|
1682
|
+
return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
|
|
1683
|
+
}
|
|
1110
1684
|
function allocateBudget(policy, args) {
|
|
1111
1685
|
if (!policy) return void 0;
|
|
1112
1686
|
if (policy.allocate) {
|
|
1113
|
-
|
|
1687
|
+
const allocated2 = policy.allocate({
|
|
1114
1688
|
analyst: args.analyst,
|
|
1115
1689
|
totalUsd: policy.totalUsd,
|
|
1116
1690
|
remainingUsd: args.remainingUsd,
|
|
1117
1691
|
runningCount: args.runningCount
|
|
1118
1692
|
});
|
|
1693
|
+
if (allocated2 === void 0) {
|
|
1694
|
+
if (policy.totalUsd !== void 0) {
|
|
1695
|
+
throw new Error(
|
|
1696
|
+
`BudgetPolicy.allocate('${args.analyst.id}') cannot return undefined when totalUsd is set`
|
|
1697
|
+
);
|
|
1698
|
+
}
|
|
1699
|
+
return void 0;
|
|
1700
|
+
}
|
|
1701
|
+
assertBudgetAmount(allocated2, `BudgetPolicy.allocate('${args.analyst.id}')`);
|
|
1702
|
+
return args.remainingUsd === void 0 ? allocated2 : Math.min(allocated2, args.remainingUsd);
|
|
1119
1703
|
}
|
|
1120
1704
|
if (policy.totalUsd == null) return void 0;
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1705
|
+
const allocated = policy.weights ? policy.totalUsd * analystWeight(policy.weights, args.analyst.id) / args.totalWeight : policy.totalUsd / Math.max(1, args.runningCount);
|
|
1706
|
+
return args.remainingUsd === void 0 ? allocated : Math.min(allocated, args.remainingUsd);
|
|
1707
|
+
}
|
|
1708
|
+
function validateBudgetPolicy(policy) {
|
|
1709
|
+
if (!policy) return;
|
|
1710
|
+
if (policy.totalUsd !== void 0) assertBudgetAmount(policy.totalUsd, "BudgetPolicy.totalUsd");
|
|
1711
|
+
for (const [analystId, weight] of Object.entries(policy.weights ?? {})) {
|
|
1712
|
+
assertBudgetAmount(weight, `BudgetPolicy.weights['${analystId}']`);
|
|
1713
|
+
}
|
|
1714
|
+
}
|
|
1715
|
+
function assertBudgetAmount(value, field) {
|
|
1716
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
1717
|
+
throw new Error(`${field} must be a non-negative finite number`);
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
function validateEffectiveBudget(budgetUsd, remainingUsd, analystId) {
|
|
1721
|
+
if (budgetUsd !== void 0) {
|
|
1722
|
+
assertBudgetAmount(budgetUsd, `AnalystContext.budgetUsd for '${analystId}'`);
|
|
1723
|
+
}
|
|
1724
|
+
if (remainingUsd === void 0) return budgetUsd;
|
|
1725
|
+
if (budgetUsd === void 0) {
|
|
1726
|
+
throw new Error(
|
|
1727
|
+
`AnalystContext.budgetUsd for '${analystId}' cannot be removed while an overall budget remains`
|
|
1728
|
+
);
|
|
1729
|
+
}
|
|
1730
|
+
if (budgetUsd > remainingUsd) {
|
|
1731
|
+
throw new Error(
|
|
1732
|
+
`AnalystContext.budgetUsd for '${analystId}' (${budgetUsd}) exceeds the remaining overall budget (${remainingUsd})`
|
|
1733
|
+
);
|
|
1734
|
+
}
|
|
1735
|
+
return budgetUsd;
|
|
1736
|
+
}
|
|
1737
|
+
function analystWeight(weights, analystId) {
|
|
1738
|
+
const weight = weights[analystId] ?? 1;
|
|
1739
|
+
assertBudgetAmount(weight, `BudgetPolicy.weights['${analystId}']`);
|
|
1740
|
+
return weight;
|
|
1741
|
+
}
|
|
1742
|
+
function zeroUsage() {
|
|
1743
|
+
return {
|
|
1744
|
+
calls: 0,
|
|
1745
|
+
tokens: { input: 0, output: 0 },
|
|
1746
|
+
cost: { kind: "observed", usd: 0 }
|
|
1747
|
+
};
|
|
1748
|
+
}
|
|
1749
|
+
function resolveUsage(analyst, findings, receipts) {
|
|
1750
|
+
const legacyCost = sumFindingCost(findings);
|
|
1751
|
+
if (receipts.length > 0) {
|
|
1752
|
+
const merged = mergeUsageReceipts(receipts);
|
|
1753
|
+
return merged.cost.kind === "uncaptured" && legacyCost.captured ? { ...merged, knownCostUsd: Math.max(merged.knownCostUsd ?? 0, legacyCost.usd) } : merged;
|
|
1754
|
+
}
|
|
1755
|
+
if (legacyCost.captured) {
|
|
1756
|
+
return {
|
|
1757
|
+
calls: null,
|
|
1758
|
+
tokens: null,
|
|
1759
|
+
cost: { kind: "observed", usd: legacyCost.usd }
|
|
1760
|
+
};
|
|
1761
|
+
}
|
|
1762
|
+
if (analyst.cost.kind === "deterministic") return zeroUsage();
|
|
1763
|
+
return { calls: null, tokens: null, cost: { kind: "uncaptured", usd: null } };
|
|
1764
|
+
}
|
|
1765
|
+
function mergeUsageReceipts(receipts) {
|
|
1766
|
+
const calls = receipts.every((receipt) => receipt.calls !== null) ? receipts.reduce((sum, receipt) => sum + (receipt.calls ?? 0), 0) : null;
|
|
1767
|
+
const tokens = receipts.every((receipt) => receipt.tokens !== null) ? receipts.reduce(
|
|
1768
|
+
(sum, receipt) => ({
|
|
1769
|
+
input: sum.input + (receipt.tokens?.input ?? 0),
|
|
1770
|
+
output: sum.output + (receipt.tokens?.output ?? 0),
|
|
1771
|
+
...sum.reasoning !== void 0 || receipt.tokens?.reasoning !== void 0 ? { reasoning: (sum.reasoning ?? 0) + (receipt.tokens?.reasoning ?? 0) } : {},
|
|
1772
|
+
...sum.cached !== void 0 || receipt.tokens?.cached !== void 0 ? { cached: (sum.cached ?? 0) + (receipt.tokens?.cached ?? 0) } : {},
|
|
1773
|
+
...sum.cacheWrite !== void 0 || receipt.tokens?.cacheWrite !== void 0 ? { cacheWrite: (sum.cacheWrite ?? 0) + (receipt.tokens?.cacheWrite ?? 0) } : {}
|
|
1774
|
+
}),
|
|
1775
|
+
{ input: 0, output: 0 }
|
|
1776
|
+
) : null;
|
|
1777
|
+
const cost = aggregateCostProvenance(receipts.map((receipt) => receipt.cost));
|
|
1778
|
+
return {
|
|
1779
|
+
calls,
|
|
1780
|
+
tokens,
|
|
1781
|
+
cost,
|
|
1782
|
+
...cost.kind === "uncaptured" ? {
|
|
1783
|
+
knownCostUsd: receipts.reduce((sum, receipt) => sum + knownCostUsd(receipt), 0)
|
|
1784
|
+
} : {}
|
|
1785
|
+
};
|
|
1786
|
+
}
|
|
1787
|
+
function knownCostUsd(receipt) {
|
|
1788
|
+
return receipt.cost.kind === "uncaptured" ? receipt.knownCostUsd ?? 0 : receipt.cost.usd;
|
|
1789
|
+
}
|
|
1790
|
+
function budgetDebit(receipt, allocatedUsd) {
|
|
1791
|
+
const known = knownCostUsd(receipt);
|
|
1792
|
+
return receipt.cost.kind === "uncaptured" && allocatedUsd !== void 0 ? Math.max(known, allocatedUsd) : known;
|
|
1793
|
+
}
|
|
1794
|
+
function aggregateCostProvenance(costs) {
|
|
1795
|
+
if (costs.some((cost) => cost.kind === "uncaptured")) {
|
|
1796
|
+
return { kind: "uncaptured", usd: null };
|
|
1797
|
+
}
|
|
1798
|
+
const usd = costs.reduce((sum, cost) => sum + (cost.usd ?? 0), 0);
|
|
1799
|
+
return costs.some((cost) => cost.kind === "estimated") ? { kind: "estimated", usd } : { kind: "observed", usd };
|
|
1800
|
+
}
|
|
1801
|
+
function assertValidUsageReceipt(receipt) {
|
|
1802
|
+
if (receipt.calls !== null && (!Number.isInteger(receipt.calls) || receipt.calls < 0)) {
|
|
1803
|
+
throw new Error("AnalystContext.recordUsage: calls must be a non-negative integer or null");
|
|
1804
|
+
}
|
|
1805
|
+
if (receipt.tokens) {
|
|
1806
|
+
assertNonNegativeFinite(receipt.tokens.input, "tokens.input");
|
|
1807
|
+
assertNonNegativeFinite(receipt.tokens.output, "tokens.output");
|
|
1808
|
+
if (receipt.tokens.reasoning !== void 0) {
|
|
1809
|
+
assertNonNegativeFinite(receipt.tokens.reasoning, "tokens.reasoning");
|
|
1810
|
+
if (receipt.tokens.reasoning > receipt.tokens.output) {
|
|
1811
|
+
throw new Error(
|
|
1812
|
+
"AnalystContext.recordUsage: tokens.reasoning must not exceed tokens.output"
|
|
1813
|
+
);
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
if (receipt.tokens.cached !== void 0) {
|
|
1817
|
+
assertNonNegativeFinite(receipt.tokens.cached, "tokens.cached");
|
|
1818
|
+
}
|
|
1819
|
+
if (receipt.tokens.cacheWrite !== void 0) {
|
|
1820
|
+
assertNonNegativeFinite(receipt.tokens.cacheWrite, "tokens.cacheWrite");
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
if (receipt.cost.kind !== "uncaptured") {
|
|
1824
|
+
assertNonNegativeFinite(receipt.cost.usd, "cost.usd");
|
|
1825
|
+
} else if (receipt.cost.usd !== null) {
|
|
1826
|
+
throw new Error("AnalystContext.recordUsage: uncaptured cost.usd must be null");
|
|
1827
|
+
}
|
|
1828
|
+
if (receipt.knownCostUsd !== void 0) {
|
|
1829
|
+
assertNonNegativeFinite(receipt.knownCostUsd, "knownCostUsd");
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
function assertNonNegativeFinite(value, field) {
|
|
1833
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
1834
|
+
throw new Error(`AnalystContext.recordUsage: ${field} must be a non-negative finite number`);
|
|
1125
1835
|
}
|
|
1126
|
-
return policy.totalUsd / Math.max(1, args.runningCount);
|
|
1127
1836
|
}
|
|
1128
1837
|
function sumFindingCost(findings) {
|
|
1129
1838
|
let sum = 0;
|
|
1839
|
+
let captured = false;
|
|
1130
1840
|
for (const f of findings) {
|
|
1131
1841
|
const c = f.metadata?.cost_usd;
|
|
1132
|
-
if (
|
|
1842
|
+
if (c === void 0) continue;
|
|
1843
|
+
if (typeof c !== "number" || !Number.isFinite(c) || c < 0) {
|
|
1844
|
+
throw new Error(
|
|
1845
|
+
`Analyst finding '${f.finding_id}' metadata.cost_usd must be a non-negative finite number`
|
|
1846
|
+
);
|
|
1847
|
+
}
|
|
1848
|
+
sum += c;
|
|
1849
|
+
captured = true;
|
|
1133
1850
|
}
|
|
1134
|
-
return sum;
|
|
1851
|
+
return { usd: sum, captured };
|
|
1135
1852
|
}
|
|
1136
1853
|
function selectPriorFindings(source, analystId) {
|
|
1137
1854
|
if (!source) return void 0;
|
|
@@ -1891,6 +2608,7 @@ function finiteOrZero(value) {
|
|
|
1891
2608
|
}
|
|
1892
2609
|
|
|
1893
2610
|
export {
|
|
2611
|
+
createAnalystAi,
|
|
1894
2612
|
computeFindingId,
|
|
1895
2613
|
makeFinding,
|
|
1896
2614
|
FINDING_SUBJECT_KINDS,
|
|
@@ -1905,12 +2623,19 @@ export {
|
|
|
1905
2623
|
coerceJson,
|
|
1906
2624
|
coerceToFindingRows,
|
|
1907
2625
|
ANALYST_SEVERITIES,
|
|
2626
|
+
RawAnalystEvidenceSchema,
|
|
1908
2627
|
RawAnalystFindingSchema,
|
|
2628
|
+
CanonicalRawAnalystFindingSchema,
|
|
1909
2629
|
RAW_FINDING_SCHEMA_PROMPT,
|
|
2630
|
+
evidenceRefsFromRawFinding,
|
|
1910
2631
|
parseRawFinding,
|
|
2632
|
+
parseCanonicalRawFinding,
|
|
1911
2633
|
structureFindings,
|
|
2634
|
+
settleUsageReceiptFromCostLedger,
|
|
2635
|
+
validateUsageSettlementTimeout,
|
|
1912
2636
|
createTraceAnalystKind,
|
|
1913
2637
|
renderPriorFindings,
|
|
2638
|
+
renderUpstreamFindings,
|
|
1914
2639
|
buildTraceToolsForGroup,
|
|
1915
2640
|
FAILURE_MODE_KIND_SPEC,
|
|
1916
2641
|
IMPROVEMENT_KIND_SPEC,
|
|
@@ -1940,4 +2665,4 @@ export {
|
|
|
1940
2665
|
aggregateRunScore,
|
|
1941
2666
|
clamp012 as clamp01
|
|
1942
2667
|
};
|
|
1943
|
-
//# sourceMappingURL=chunk-
|
|
2668
|
+
//# sourceMappingURL=chunk-S5TT5R3L.js.map
|