@tangle-network/agent-eval 0.137.0 → 0.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +78 -0
- package/README.md +34 -0
- package/dist/analyst/index.d.ts +485 -104
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +10 -607
- package/dist/analyst/index.js.map +1 -1
- package/dist/{benchmark-YDrpumqB.js → benchmark-CYtcIF2V.js} +299 -159
- package/dist/benchmark-CYtcIF2V.js.map +1 -0
- package/dist/{benchmark-CHX4orG7.d.ts → benchmark-DDVdWcwA.d.ts} +67 -15
- package/dist/benchmark-DDVdWcwA.d.ts.map +1 -0
- package/dist/benchmark-command-BKfjOBJ5.js +4537 -0
- package/dist/benchmark-command-BKfjOBJ5.js.map +1 -0
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-DCLkQOmc.js → benchmarks-zxhy1QV3.js} +4 -3
- package/dist/{benchmarks-DCLkQOmc.js.map → benchmarks-zxhy1QV3.js.map} +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +4 -3
- package/dist/{campaign-lgObcHFC.js → campaign-DrS6_hLd.js} +19 -11
- package/dist/campaign-DrS6_hLd.js.map +1 -0
- package/dist/canonical-D011XM8r.js +86 -0
- package/dist/canonical-D011XM8r.js.map +1 -0
- package/dist/cli.js +10 -3
- package/dist/cli.js.map +1 -1
- package/dist/{client-C8L6h6Wf.d.ts → client-BohnDFBq.d.ts} +4 -4
- package/dist/{client-C8L6h6Wf.d.ts.map → client-BohnDFBq.d.ts.map} +1 -1
- package/dist/{completion-verifier-DSyRNVzU.d.ts → completion-verifier-IPoP4fQO.d.ts} +178 -4
- package/dist/completion-verifier-IPoP4fQO.d.ts.map +1 -0
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +9 -8
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/{cost-ledger-D-5_-dhi.js → cost-ledger-CZ9diLxY.js} +90 -45
- package/dist/cost-ledger-CZ9diLxY.js.map +1 -0
- package/dist/{cost-ledger-D2o6JOrL.d.ts → cost-ledger-DKgyIWRj.d.ts} +6 -2
- package/dist/cost-ledger-DKgyIWRj.d.ts.map +1 -0
- package/dist/default-registry-B8vf7Rmf.d.ts +118 -0
- package/dist/default-registry-B8vf7Rmf.d.ts.map +1 -0
- package/dist/default-registry-BgJJItGr.js +2364 -0
- package/dist/default-registry-BgJJItGr.js.map +1 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js +344 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js.map +1 -0
- package/dist/{eval-campaign-CHqfLnff.js → eval-campaign-BmptJj50.js} +2 -2
- package/dist/{eval-campaign-CHqfLnff.js.map → eval-campaign-BmptJj50.js.map} +1 -1
- package/dist/exact-types-MaaFcllV.d.ts +234 -0
- package/dist/exact-types-MaaFcllV.d.ts.map +1 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts +32 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts.map +1 -0
- package/dist/{extract-usage-p-56bh8q.js → extract-usage-DZs601Va.js} +2 -2
- package/dist/{extract-usage-p-56bh8q.js.map → extract-usage-DZs601Va.js.map} +1 -1
- package/dist/{feedback-trajectory-N_F0PwHz.d.ts → feedback-trajectory-BJUWOkJM.d.ts} +3 -2
- package/dist/feedback-trajectory-BJUWOkJM.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/{hf-dataset-DBJXXoY1.js → hf-dataset-XggBupCr.js} +2 -2
- package/dist/{hf-dataset-DBJXXoY1.js.map → hf-dataset-XggBupCr.js.map} +1 -1
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-C-Pr4OWg.d.ts → index-BTm_P9aC.d.ts} +12 -11
- package/dist/index-BTm_P9aC.d.ts.map +1 -0
- package/dist/{index-U3RHOShi.d.ts → index-CWOPCJiw.d.ts} +2 -2
- package/dist/{index-U3RHOShi.d.ts.map → index-CWOPCJiw.d.ts.map} +1 -1
- package/dist/{index-DRNl6g_N.d.ts → index-CtR1xh4V.d.ts} +3 -3
- package/dist/{index-DRNl6g_N.d.ts.map → index-CtR1xh4V.d.ts.map} +1 -1
- package/dist/index-DEb46kc6.d.ts.map +1 -1
- package/dist/{index-BnP1QJUv.d.ts → index-_66rVpwN.d.ts} +5 -5
- package/dist/{index-BnP1QJUv.d.ts.map → index-_66rVpwN.d.ts.map} +1 -1
- package/dist/index.d.ts +35 -55
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +55 -514
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-B9ooYH_g.d.ts → insight-report-Bu5Wi9tG.d.ts} +4 -4
- package/dist/{insight-report-B9ooYH_g.d.ts.map → insight-report-Bu5Wi9tG.d.ts.map} +1 -1
- package/dist/{integrity-CKxosZ5Z.d.ts → integrity-COTh3DTH.d.ts} +2 -2
- package/dist/{integrity-CKxosZ5Z.d.ts.map → integrity-COTh3DTH.d.ts.map} +1 -1
- package/dist/kind-factory-CFxA0JQX.js +2133 -0
- package/dist/kind-factory-CFxA0JQX.js.map +1 -0
- package/dist/ledger-core/index.js +2 -1
- package/dist/{ledger-core-t6sItivm.js → ledger-core-Dxz0Rkwa.js} +210 -99
- package/dist/ledger-core-Dxz0Rkwa.js.map +1 -0
- package/dist/{llm-client-DKB25jV8.js → llm-client-bkztEfIx.js} +5 -5
- package/dist/llm-client-bkztEfIx.js.map +1 -0
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{proposal-findings-DCawte-y.js → proposal-findings-2GIUo1et.js} +2 -68
- package/dist/proposal-findings-2GIUo1et.js.map +1 -0
- package/dist/{release-report-CofgVNZt.d.ts → release-report-fZarvIm-.d.ts} +3 -3
- package/dist/{release-report-CofgVNZt.d.ts.map → release-report-fZarvIm-.d.ts.map} +1 -1
- package/dist/{replay-K8FaC0CB.d.ts → replay-DjG4IG60.d.ts} +34 -143
- package/dist/replay-DjG4IG60.d.ts.map +1 -0
- package/dist/{replay-Bju0T8Ls.js → replay-SA4OB7O7.js} +48 -136
- package/dist/replay-SA4OB7O7.js.map +1 -0
- package/dist/reporting.d.ts +4 -4
- package/dist/{researcher-Da0Wj-bt.d.ts → researcher-BxhtGfKa.d.ts} +5 -5
- package/dist/{researcher-Da0Wj-bt.d.ts.map → researcher-BxhtGfKa.d.ts.map} +1 -1
- package/dist/{reward-hacking-CQ3hTCO3.d.ts → reward-hacking-CqSLiV51.d.ts} +2 -2
- package/dist/{reward-hacking-CQ3hTCO3.d.ts.map → reward-hacking-CqSLiV51.d.ts.map} +1 -1
- package/dist/rl.d.ts +5 -5
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-DQFl0UXA.js → rollout-8nj3mYvx.js} +2 -2
- package/dist/{rollout-DQFl0UXA.js.map → rollout-8nj3mYvx.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C4sztLR3.d.ts → rubric-predictive-validity-DQBQj6uV.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C4sztLR3.d.ts.map → rubric-predictive-validity-DQBQj6uV.d.ts.map} +1 -1
- package/dist/{run-evidence-BDIircdA.d.ts → run-evidence-C4RcRQT5.d.ts} +3 -3
- package/dist/{run-evidence-BDIircdA.d.ts.map → run-evidence-C4RcRQT5.d.ts.map} +1 -1
- package/dist/{run-record-BPCa2rQ8.d.ts → run-record-CztDMXVF.d.ts} +2 -2
- package/dist/{run-record-BPCa2rQ8.d.ts.map → run-record-CztDMXVF.d.ts.map} +1 -1
- package/dist/{semantic-concept-judge-Bz64IckK.js → semantic-concept-judge-BuIJ9IfB.js} +49 -6
- package/dist/semantic-concept-judge-BuIJ9IfB.js.map +1 -0
- package/dist/{server-KjXZZUDX.js → server-DaCpLfi0.js} +3 -3
- package/dist/{server-KjXZZUDX.js.map → server-DaCpLfi0.js.map} +1 -1
- package/dist/single-run-lock-BTTtPZ9N.js +989 -0
- package/dist/single-run-lock-BTTtPZ9N.js.map +1 -0
- package/dist/{skill-usage-CFDLLlhF.d.ts → skill-usage-B-BFS8M2.d.ts} +65 -40
- package/dist/skill-usage-B-BFS8M2.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-f4o9sUT4.js → skillopt-optimization-method-BbGnCC53.js} +20 -979
- package/dist/skillopt-optimization-method-BbGnCC53.js.map +1 -0
- package/dist/{skillopt-optimization-method-BpbnlvAZ.d.ts → skillopt-optimization-method-_s0Tub7Y.d.ts} +11 -39
- package/dist/skillopt-optimization-method-_s0Tub7Y.d.ts.map +1 -0
- package/dist/{statistics-_7P642CN.d.ts → statistics-B5d0Zd-z.d.ts} +2 -2
- package/dist/{statistics-_7P642CN.d.ts.map → statistics-B5d0Zd-z.d.ts.map} +1 -1
- package/dist/store-otlp-DX4fGIcf.js +757 -0
- package/dist/store-otlp-DX4fGIcf.js.map +1 -0
- package/dist/{summary-report-DHipz9Kx.d.ts → summary-report-Cg7BifAM.d.ts} +3 -3
- package/dist/{summary-report-DHipz9Kx.d.ts.map → summary-report-Cg7BifAM.d.ts.map} +1 -1
- package/dist/tool-groups-CdYq22lX.d.ts +258 -0
- package/dist/tool-groups-CdYq22lX.d.ts.map +1 -0
- package/dist/traces.d.ts +7 -6
- package/dist/traces.js +5 -4
- package/dist/{types-CTGbIm57.d.ts → types-BBFNHxSK.d.ts} +5 -5
- package/dist/{types-CTGbIm57.d.ts.map → types-BBFNHxSK.d.ts.map} +1 -1
- package/dist/{types-CKswbJGO.d.ts → types-DoEYskCd.d.ts} +5 -5
- package/dist/{types-CKswbJGO.d.ts.map → types-DoEYskCd.d.ts.map} +1 -1
- package/dist/{types-CTvKfr5F.d.ts → types-uPrS6mD-.d.ts} +2 -2
- package/dist/{types-CTvKfr5F.d.ts.map → types-uPrS6mD-.d.ts.map} +1 -1
- package/dist/usage-receipt-CgxMEBZq.js +134 -0
- package/dist/usage-receipt-CgxMEBZq.js.map +1 -0
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/trace-analysis.md +191 -385
- package/package.json +5 -4
- package/dist/analyze-runs-PVtnfjvA.d.ts +0 -72
- package/dist/analyze-runs-PVtnfjvA.d.ts.map +0 -1
- package/dist/benchmark-CHX4orG7.d.ts.map +0 -1
- package/dist/benchmark-YDrpumqB.js.map +0 -1
- package/dist/campaign-lgObcHFC.js.map +0 -1
- package/dist/completion-verifier-DSyRNVzU.d.ts.map +0 -1
- package/dist/concurrency-MUjT7VjM.js +0 -109
- package/dist/concurrency-MUjT7VjM.js.map +0 -1
- package/dist/cost-ledger-D-5_-dhi.js.map +0 -1
- package/dist/cost-ledger-D2o6JOrL.d.ts.map +0 -1
- package/dist/default-registry-CLXbRt0f.js +0 -2594
- package/dist/default-registry-CLXbRt0f.js.map +0 -1
- package/dist/default-registry-Dc5D_Loc.d.ts +0 -202
- package/dist/default-registry-Dc5D_Loc.d.ts.map +0 -1
- package/dist/feedback-trajectory-N_F0PwHz.d.ts.map +0 -1
- package/dist/index-C-Pr4OWg.d.ts.map +0 -1
- package/dist/ledger-core-t6sItivm.js.map +0 -1
- package/dist/llm-client-DKB25jV8.js.map +0 -1
- package/dist/proposal-findings-DCawte-y.js.map +0 -1
- package/dist/registry-BdM7SuTr.d.ts +0 -124
- package/dist/registry-BdM7SuTr.d.ts.map +0 -1
- package/dist/replay-Bju0T8Ls.js.map +0 -1
- package/dist/replay-K8FaC0CB.d.ts.map +0 -1
- package/dist/semantic-concept-judge-Bz64IckK.js.map +0 -1
- package/dist/skill-usage-CFDLLlhF.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-BpbnlvAZ.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-f4o9sUT4.js.map +0 -1
- package/dist/tools-DZk2Jn64.js +0 -1876
- package/dist/tools-DZk2Jn64.js.map +0 -1
|
@@ -0,0 +1,2364 @@
|
|
|
1
|
+
import { n as LlmClient } from "./llm-client-bkztEfIx.js";
|
|
2
|
+
import { LLM_CONTEXT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_OUTPUT_TOKEN_ATTR_KEYS, TOOL_NAME_ATTR_KEYS } from "./trace-attributes.js";
|
|
3
|
+
import { t as executionTrackByLane } from "./execution-tracks-CpgFPpS5.js";
|
|
4
|
+
import { $ as spanEpochMillis, H as snapshotExactExecutionPlan, R as findingSubjectGrammarPromptFor, U as deepFreezeCanonicalJson, V as snapshotExactExecutionComponentIdentity, t as createTraceAnalyst } from "./kind-factory-CFxA0JQX.js";
|
|
5
|
+
import { i as validateUsageSettlementTimeout, o as makeFinding, t as assertValidAnalystUsageReceipt } from "./usage-receipt-CgxMEBZq.js";
|
|
6
|
+
import { i as hashCanonical, r as canonicalString } from "./canonical-D011XM8r.js";
|
|
7
|
+
import { t as analyzeSupervisorRunIntegrity } from "./integrity-CCXTftiL.js";
|
|
8
|
+
import { o as combineAbortSignals } from "./proposal-findings-2GIUo1et.js";
|
|
9
|
+
import { z } from "zod";
|
|
10
|
+
import { randomUUID } from "node:crypto";
|
|
11
|
+
//#region src/analyst/chat-client.ts
|
|
12
|
+
/**
|
|
13
|
+
* Provider-neutral chat contract for every model call made by agent-eval.
|
|
14
|
+
*
|
|
15
|
+
* Callers choose the transport at the package boundary with `createChatClient`.
|
|
16
|
+
* Evaluation code receives canonical requests and results without importing a
|
|
17
|
+
* provider SDK.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Build a ChatClient bound to a specific transport. The returned client
|
|
21
|
+
* is safe to share across analysts in a single registry run.
|
|
22
|
+
*/
|
|
23
|
+
function createChatClient(opts) {
|
|
24
|
+
switch (opts.transport) {
|
|
25
|
+
case "router": return wrapLlmClient(opts.transport, opts.defaultModel, new LlmClient({
|
|
26
|
+
baseUrl: opts.baseUrl ?? "https://router.tangle.tools/v1",
|
|
27
|
+
apiKey: opts.apiKey,
|
|
28
|
+
maximumAttempts: opts.maximumAttempts
|
|
29
|
+
}));
|
|
30
|
+
case "cli-bridge": return wrapLlmClient(opts.transport, opts.defaultModel, new LlmClient({
|
|
31
|
+
baseUrl: opts.baseUrl ?? "http://127.0.0.1:3344/v1",
|
|
32
|
+
apiKey: opts.bearer ?? "",
|
|
33
|
+
maximumAttempts: opts.maximumAttempts
|
|
34
|
+
}));
|
|
35
|
+
case "direct-provider": return wrapLlmClient(opts.transport, opts.defaultModel, new LlmClient({
|
|
36
|
+
baseUrl: opts.baseUrl,
|
|
37
|
+
apiKey: opts.apiKey,
|
|
38
|
+
maximumAttempts: opts.maximumAttempts
|
|
39
|
+
}));
|
|
40
|
+
case "sandbox-sdk": return {
|
|
41
|
+
transport: "sandbox-sdk",
|
|
42
|
+
defaultModel: opts.defaultModel,
|
|
43
|
+
maximumAttempts: opts.maximumAttempts,
|
|
44
|
+
chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts)
|
|
45
|
+
};
|
|
46
|
+
case "custom": return {
|
|
47
|
+
transport: "custom",
|
|
48
|
+
defaultModel: opts.defaultModel,
|
|
49
|
+
maximumAttempts: opts.maximumAttempts,
|
|
50
|
+
chat: async (req, callOpts) => opts.chat(resolveModel(req, opts.defaultModel), callOpts)
|
|
51
|
+
};
|
|
52
|
+
case "mock": return {
|
|
53
|
+
transport: "mock",
|
|
54
|
+
defaultModel: opts.defaultModel,
|
|
55
|
+
maximumAttempts: 1,
|
|
56
|
+
chat: async (req, callOpts) => opts.handler(resolveModel(req, opts.defaultModel), callOpts)
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function wrapLlmClient(transport, defaultModel, inner) {
|
|
61
|
+
return {
|
|
62
|
+
transport,
|
|
63
|
+
defaultModel,
|
|
64
|
+
maximumAttempts: inner.maximumAttempts,
|
|
65
|
+
chat: (req, callOpts) => {
|
|
66
|
+
const request = {
|
|
67
|
+
model: resolveModel(req, defaultModel).model,
|
|
68
|
+
messages: req.messages,
|
|
69
|
+
jsonMode: req.jsonMode,
|
|
70
|
+
jsonSchema: req.jsonSchema,
|
|
71
|
+
temperature: req.temperature,
|
|
72
|
+
maxTokens: req.maxTokens,
|
|
73
|
+
thinking: req.thinking,
|
|
74
|
+
timeoutMs: req.timeoutMs
|
|
75
|
+
};
|
|
76
|
+
return inner.call(request, {
|
|
77
|
+
signal: callOpts?.signal,
|
|
78
|
+
idempotencyKey: callOpts?.idempotencyKey
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
function resolveModel(req, defaultModel) {
|
|
84
|
+
if (req.model) return req;
|
|
85
|
+
if (!defaultModel) throw new Error("ChatClient.chat: no model on request and no defaultModel on the client. Either pass req.model or bind defaultModel at createChatClient().");
|
|
86
|
+
return {
|
|
87
|
+
...req,
|
|
88
|
+
model: defaultModel
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
//#endregion
|
|
92
|
+
//#region src/trace-analyst/behavioral-metrics.ts
|
|
93
|
+
/**
|
|
94
|
+
* Deterministic behavioral metrics over OTLP spans — pure arithmetic, no LLM.
|
|
95
|
+
*
|
|
96
|
+
* It computes token growth, output decay, tool monoculture, and missing
|
|
97
|
+
* self-verification once in TypeScript with no model judgment.
|
|
98
|
+
*
|
|
99
|
+
* General, not trace-specific: the detectors key off token trajectories and
|
|
100
|
+
* tool usage present in any agentic OTLP trace, not any one benchmark.
|
|
101
|
+
*/
|
|
102
|
+
/** ≥ this input-token growth ratio across a run, with no compression, fires. */
|
|
103
|
+
const INPUT_GROWTH_FACTOR = 3;
|
|
104
|
+
/** Tool-usage signals need at least this many calls to be meaningful. */
|
|
105
|
+
const MIN_TOOL_CALLS = 3;
|
|
106
|
+
/** Tool names that read or check state count as self-verification, not mutation.
|
|
107
|
+
* Covers the inspect verbs plus the read/search tools real harnesses use to
|
|
108
|
+
* verify (Claude Code Read/Grep/Glob, codex read_file/ls/cat, git status/diff,
|
|
109
|
+
* test/lint). A pure shell tool (Bash/exec_command) is intentionally NOT matched
|
|
110
|
+
* — its name can't tell a `pytest` from an `rm`. */
|
|
111
|
+
const VERIFY_RE = /verif|eval|inspect|check|assert|validat|review|confirm|read|grep|glob|search|view|\blist\b|\bls\b|\bcat\b|\bfind\b|diff|status|\btest|lint|typecheck/i;
|
|
112
|
+
function num(v) {
|
|
113
|
+
return typeof v === "number" && Number.isFinite(v) ? v : null;
|
|
114
|
+
}
|
|
115
|
+
function numAttr(attrs, keys) {
|
|
116
|
+
for (const key of keys) {
|
|
117
|
+
const value = num(attrs[key]);
|
|
118
|
+
if (value !== null) return value;
|
|
119
|
+
}
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
function inputTokensOf(s) {
|
|
123
|
+
const exactContext = num(s.attributes[LLM_CONTEXT_TOKENS]);
|
|
124
|
+
if (exactContext !== null) return exactContext;
|
|
125
|
+
return numAttr(s.attributes, LLM_INPUT_TOKEN_ATTR_KEYS) ?? num(s.attributes["llm.usage.input_tokens"]);
|
|
126
|
+
}
|
|
127
|
+
function outputTokensOf(s) {
|
|
128
|
+
return numAttr(s.attributes, LLM_OUTPUT_TOKEN_ATTR_KEYS) ?? num(s.attributes["llm.usage.output_tokens"]);
|
|
129
|
+
}
|
|
130
|
+
function stepOf(s) {
|
|
131
|
+
return num(s.attributes.step);
|
|
132
|
+
}
|
|
133
|
+
function toolNameOf(s) {
|
|
134
|
+
if (s.tool_name) return s.tool_name;
|
|
135
|
+
for (const key of TOOL_NAME_ATTR_KEYS) {
|
|
136
|
+
const t = s.attributes[key];
|
|
137
|
+
if (typeof t === "string" && t.length > 0) return t;
|
|
138
|
+
}
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Reduce a span list to behavioral metrics + fired suboptimality signals.
|
|
143
|
+
* Pure + deterministic: same spans → same output, on any machine, no model.
|
|
144
|
+
*/
|
|
145
|
+
function computeTraceMetrics(spans) {
|
|
146
|
+
const traceIds = new Set(spans.map((span) => span.trace_id));
|
|
147
|
+
if (traceIds.size > 1) throw new Error(`computeTraceMetrics: expected spans from one trace, received ${traceIds.size} traces`);
|
|
148
|
+
const traceId = traceIds.values().next().value ?? null;
|
|
149
|
+
const samples = spans.map((span) => ({
|
|
150
|
+
span,
|
|
151
|
+
input: inputTokensOf(span),
|
|
152
|
+
output: outputTokensOf(span),
|
|
153
|
+
step: stepOf(span)
|
|
154
|
+
}));
|
|
155
|
+
const llmSamples = samples.filter((sample) => sample.span.kind === "LLM");
|
|
156
|
+
const tokenSamples = llmSamples.length > 0 ? llmSamples : samples.filter((sample) => sample.input !== null || sample.output !== null);
|
|
157
|
+
const tokenSequences = buildTokenSequences(tokenSamples, spans);
|
|
158
|
+
const primarySequence = tokenSequences[0];
|
|
159
|
+
const inputTokenTrajectory = primarySequence?.inputTokenTrajectory.filter((value) => value !== null) ?? [];
|
|
160
|
+
const outputTokenTrajectory = primarySequence?.outputTokenTrajectory.filter((value) => value !== null) ?? [];
|
|
161
|
+
const toolHistogram = {};
|
|
162
|
+
let hasSelfVerification = false;
|
|
163
|
+
for (const s of spans) {
|
|
164
|
+
const tool = toolNameOf(s);
|
|
165
|
+
if (tool) {
|
|
166
|
+
toolHistogram[tool] = (toolHistogram[tool] ?? 0) + 1;
|
|
167
|
+
if (VERIFY_RE.test(tool)) hasSelfVerification = true;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
const totalToolCalls = Object.values(toolHistogram).reduce((a, b) => a + b, 0);
|
|
171
|
+
const distinctTools = Object.keys(toolHistogram).length;
|
|
172
|
+
const toolDiversityRatio = totalToolCalls === 0 ? 1 : distinctTools / totalToolCalls;
|
|
173
|
+
const signals = [];
|
|
174
|
+
const seenTokenSignals = /* @__PURE__ */ new Set();
|
|
175
|
+
for (const sequence of tokenSequences) for (const signal of tokenSignals(sequence)) {
|
|
176
|
+
if (seenTokenSignals.has(signal.code)) continue;
|
|
177
|
+
seenTokenSignals.add(signal.code);
|
|
178
|
+
signals.push(signal);
|
|
179
|
+
}
|
|
180
|
+
if (totalToolCalls >= MIN_TOOL_CALLS && distinctTools === 1) {
|
|
181
|
+
const only = Object.keys(toolHistogram)[0];
|
|
182
|
+
signals.push({
|
|
183
|
+
code: "single-tool-dependency",
|
|
184
|
+
severity: "medium",
|
|
185
|
+
detail: `All ${totalToolCalls} observed tool calls are \`${only}\`; no alternate tool call was observed.`,
|
|
186
|
+
evidence: {
|
|
187
|
+
tool: only,
|
|
188
|
+
calls: totalToolCalls,
|
|
189
|
+
distinct_tools: 1
|
|
190
|
+
}
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
if (totalToolCalls >= MIN_TOOL_CALLS && !hasSelfVerification) signals.push({
|
|
194
|
+
code: "no-self-verification",
|
|
195
|
+
severity: "medium",
|
|
196
|
+
detail: `${totalToolCalls} tool calls were observed without a verification-named tool call.`,
|
|
197
|
+
evidence: {
|
|
198
|
+
tool_calls: totalToolCalls,
|
|
199
|
+
verification_calls: 0
|
|
200
|
+
}
|
|
201
|
+
});
|
|
202
|
+
return {
|
|
203
|
+
traceId,
|
|
204
|
+
llmCallCount: tokenSamples.length,
|
|
205
|
+
tokenSequences,
|
|
206
|
+
inputTokenTrajectory,
|
|
207
|
+
outputTokenTrajectory,
|
|
208
|
+
toolHistogram,
|
|
209
|
+
totalToolCalls,
|
|
210
|
+
distinctTools,
|
|
211
|
+
toolDiversityRatio,
|
|
212
|
+
hasSelfVerification,
|
|
213
|
+
signals
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
function buildTokenSequences(samples, spans) {
|
|
217
|
+
const executionScopeFor = createTokenExecutionScopeResolver(new Map(spans.map((span) => [span.span_id, span])));
|
|
218
|
+
const scopedSamples = samples.map((sample) => ({
|
|
219
|
+
sample,
|
|
220
|
+
...executionScopeFor(sample.span)
|
|
221
|
+
}));
|
|
222
|
+
const trackByLane = executionTrackByLane(scopedSamples);
|
|
223
|
+
const byTrack = /* @__PURE__ */ new Map();
|
|
224
|
+
for (const scoped of scopedSamples) {
|
|
225
|
+
const trackId = trackByLane.get(scoped.key);
|
|
226
|
+
const track = byTrack.get(trackId) ?? {
|
|
227
|
+
scopeId: scoped.scopeId,
|
|
228
|
+
samples: []
|
|
229
|
+
};
|
|
230
|
+
track.samples.push(scoped.sample);
|
|
231
|
+
byTrack.set(trackId, track);
|
|
232
|
+
}
|
|
233
|
+
const sequences = [];
|
|
234
|
+
for (const { scopeId, samples: tracked } of byTrack.values()) {
|
|
235
|
+
const runs = serialTokenRuns([...tracked].sort(compareTokenSamples));
|
|
236
|
+
runs.forEach((run, index) => {
|
|
237
|
+
sequences.push({
|
|
238
|
+
scopeId: runs.length === 1 ? scopeId : `${scopeId}#${index + 1}`,
|
|
239
|
+
spanIds: run.map((sample) => sample.span.span_id),
|
|
240
|
+
inputTokenTrajectory: run.map((sample) => sample.input),
|
|
241
|
+
outputTokenTrajectory: run.map((sample) => sample.output)
|
|
242
|
+
});
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
return sequences.sort((a, b) => b.spanIds.length - a.spanIds.length || a.scopeId.localeCompare(b.scopeId) || a.spanIds[0].localeCompare(b.spanIds[0]));
|
|
246
|
+
}
|
|
247
|
+
function createTokenExecutionScopeResolver(spansById) {
|
|
248
|
+
const cache = /* @__PURE__ */ new Map();
|
|
249
|
+
return (span) => {
|
|
250
|
+
const ancestry = resolveAncestorScope(span.parent_span_id, spansById, cache);
|
|
251
|
+
const rootId = ancestry.rootId;
|
|
252
|
+
const scopeId = ancestry.agentId ? `span:${ancestry.agentId}` : ancestry.missingParentId ? `parent:${ancestry.missingParentId}` : rootId ? `root:${rootId}` : span.agent_name ? `agent:${span.agent_name}` : `trace:${span.trace_id}`;
|
|
253
|
+
const scopeSpanId = ancestry.agentId ?? ancestry.missingParentId ?? rootId;
|
|
254
|
+
const laneSpan = ancestry.laneSpanId ? spansById.get(ancestry.laneSpanId) : void 0;
|
|
255
|
+
const direct = scopeSpanId === null || ancestry.laneSpanId === scopeSpanId;
|
|
256
|
+
const timedSpan = direct ? span : laneSpan;
|
|
257
|
+
return {
|
|
258
|
+
key: JSON.stringify([scopeId, direct ? span.span_id : ancestry.laneSpanId]),
|
|
259
|
+
scopeKey: scopeId,
|
|
260
|
+
scopeId,
|
|
261
|
+
start: timedSpan ? spanEpochMillis(timedSpan.start_time) : null,
|
|
262
|
+
end: timedSpan ? spanEpochMillis(timedSpan.end_time) : null
|
|
263
|
+
};
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
function resolveAncestorScope(startId, spansById, cache) {
|
|
267
|
+
const empty = {
|
|
268
|
+
agentId: null,
|
|
269
|
+
rootId: null,
|
|
270
|
+
missingParentId: null,
|
|
271
|
+
laneSpanId: null
|
|
272
|
+
};
|
|
273
|
+
if (!startId) return empty;
|
|
274
|
+
const path = [];
|
|
275
|
+
const pathIndex = /* @__PURE__ */ new Map();
|
|
276
|
+
let currentId = startId;
|
|
277
|
+
let resolved = empty;
|
|
278
|
+
while (currentId) {
|
|
279
|
+
const cached = cache.get(currentId);
|
|
280
|
+
if (cached) {
|
|
281
|
+
resolved = cached;
|
|
282
|
+
break;
|
|
283
|
+
}
|
|
284
|
+
const cycleStart = pathIndex.get(currentId);
|
|
285
|
+
if (cycleStart !== void 0) {
|
|
286
|
+
resolved = {
|
|
287
|
+
agentId: null,
|
|
288
|
+
rootId: [...path.slice(cycleStart)].sort()[0],
|
|
289
|
+
missingParentId: null,
|
|
290
|
+
laneSpanId: [...path.slice(cycleStart)].sort()[0]
|
|
291
|
+
};
|
|
292
|
+
break;
|
|
293
|
+
}
|
|
294
|
+
const current = spansById.get(currentId);
|
|
295
|
+
if (!current) {
|
|
296
|
+
resolved = {
|
|
297
|
+
agentId: null,
|
|
298
|
+
rootId: null,
|
|
299
|
+
missingParentId: currentId,
|
|
300
|
+
laneSpanId: currentId
|
|
301
|
+
};
|
|
302
|
+
break;
|
|
303
|
+
}
|
|
304
|
+
if (current.kind === "AGENT") {
|
|
305
|
+
resolved = {
|
|
306
|
+
agentId: current.span_id,
|
|
307
|
+
rootId: null,
|
|
308
|
+
missingParentId: null,
|
|
309
|
+
laneSpanId: current.span_id
|
|
310
|
+
};
|
|
311
|
+
break;
|
|
312
|
+
}
|
|
313
|
+
pathIndex.set(currentId, path.length);
|
|
314
|
+
path.push(currentId);
|
|
315
|
+
currentId = current.parent_span_id;
|
|
316
|
+
}
|
|
317
|
+
for (let index = path.length - 1; index >= 0; index -= 1) {
|
|
318
|
+
if (resolved.agentId === null && resolved.rootId === null) resolved = {
|
|
319
|
+
...resolved,
|
|
320
|
+
rootId: path[index],
|
|
321
|
+
laneSpanId: path[index]
|
|
322
|
+
};
|
|
323
|
+
else if (resolved.laneSpanId === (resolved.agentId ?? resolved.missingParentId ?? resolved.rootId)) resolved = {
|
|
324
|
+
...resolved,
|
|
325
|
+
laneSpanId: path[index]
|
|
326
|
+
};
|
|
327
|
+
cache.set(path[index], resolved);
|
|
328
|
+
}
|
|
329
|
+
return resolved;
|
|
330
|
+
}
|
|
331
|
+
function compareTokenSamples(a, b) {
|
|
332
|
+
const aStart = spanEpochMillis(a.span.start_time);
|
|
333
|
+
const bStart = spanEpochMillis(b.span.start_time);
|
|
334
|
+
if (aStart === null && bStart !== null) return 1;
|
|
335
|
+
if (aStart !== null && bStart === null) return -1;
|
|
336
|
+
if (aStart !== null && bStart !== null && aStart !== bStart) return aStart - bStart;
|
|
337
|
+
if (a.step !== null && b.step !== null && a.step !== b.step) return a.step - b.step;
|
|
338
|
+
return a.span.span_id.localeCompare(b.span.span_id);
|
|
339
|
+
}
|
|
340
|
+
function serialTokenRuns(ordered) {
|
|
341
|
+
const runs = [];
|
|
342
|
+
let serial = [];
|
|
343
|
+
let overlap = [];
|
|
344
|
+
let overlapEnd = Number.NEGATIVE_INFINITY;
|
|
345
|
+
const flushSerial = () => {
|
|
346
|
+
if (serial.length > 0) runs.push(serial);
|
|
347
|
+
serial = [];
|
|
348
|
+
};
|
|
349
|
+
const flushOverlap = () => {
|
|
350
|
+
if (overlap.length === 1) serial.push(overlap[0]);
|
|
351
|
+
else if (overlap.length > 1) {
|
|
352
|
+
flushSerial();
|
|
353
|
+
for (const sample of overlap) runs.push([sample]);
|
|
354
|
+
}
|
|
355
|
+
overlap = [];
|
|
356
|
+
overlapEnd = Number.NEGATIVE_INFINITY;
|
|
357
|
+
};
|
|
358
|
+
for (const sample of ordered) {
|
|
359
|
+
const start = spanEpochMillis(sample.span.start_time);
|
|
360
|
+
const end = spanEpochMillis(sample.span.end_time);
|
|
361
|
+
if (start === null || end === null || sample.span.duration_ms <= 0 || end < start) {
|
|
362
|
+
flushOverlap();
|
|
363
|
+
flushSerial();
|
|
364
|
+
runs.push([sample]);
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
if (overlap.length > 0 && start >= overlapEnd) flushOverlap();
|
|
368
|
+
overlap.push(sample);
|
|
369
|
+
overlapEnd = Math.max(overlapEnd, end);
|
|
370
|
+
}
|
|
371
|
+
flushOverlap();
|
|
372
|
+
flushSerial();
|
|
373
|
+
return runs;
|
|
374
|
+
}
|
|
375
|
+
function tokenSignals(sequence) {
|
|
376
|
+
const signals = [];
|
|
377
|
+
const inputs = sequence.inputTokenTrajectory;
|
|
378
|
+
const outputs = sequence.outputTokenTrajectory;
|
|
379
|
+
if (inputs.length >= 3 && inputs.every((value) => value !== null)) {
|
|
380
|
+
const first = inputs[0];
|
|
381
|
+
const last = inputs[inputs.length - 1];
|
|
382
|
+
const isMonotonic = everyAdjacent(inputs, (previous, current) => current >= previous);
|
|
383
|
+
const growthFromZero = first === 0 && last > 0;
|
|
384
|
+
const growth = growthFromZero ? Infinity : first > 0 ? last / first : 0;
|
|
385
|
+
if (isMonotonic && last > first && growth >= INPUT_GROWTH_FACTOR) {
|
|
386
|
+
const growthLabel = growthFromZero ? "0→nonzero (unbounded)" : `${growth.toFixed(1)}x`;
|
|
387
|
+
signals.push({
|
|
388
|
+
code: "monotonic-input-growth",
|
|
389
|
+
severity: "high",
|
|
390
|
+
detail: `LLM input tokens grew ${growthLabel} (${first}→${last}) across ${inputs.length} serial calls without an intervening decrease.`,
|
|
391
|
+
evidence: {
|
|
392
|
+
first,
|
|
393
|
+
last,
|
|
394
|
+
growth_x: growthFromZero ? "unbounded" : Number(growth.toFixed(2)),
|
|
395
|
+
calls: inputs.length,
|
|
396
|
+
scope: sequence.scopeId,
|
|
397
|
+
first_span_id: sequence.spanIds[0],
|
|
398
|
+
last_span_id: sequence.spanIds[sequence.spanIds.length - 1]
|
|
399
|
+
}
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
if (inputs.length >= 3 && inputs.length === outputs.length && inputs.every((value) => value !== null) && outputs.every((value) => value !== null)) {
|
|
404
|
+
const first = outputs[0];
|
|
405
|
+
const last = outputs[outputs.length - 1];
|
|
406
|
+
const inputIsMonotonic = everyAdjacent(inputs, (previous, current) => current >= previous);
|
|
407
|
+
const outputIsMonotonic = everyAdjacent(outputs, (previous, current) => current <= previous);
|
|
408
|
+
const inputGrew = inputs[inputs.length - 1] > inputs[0];
|
|
409
|
+
if (inputIsMonotonic && inputGrew && outputIsMonotonic && last < first) signals.push({
|
|
410
|
+
code: "output-length-decay",
|
|
411
|
+
severity: "medium",
|
|
412
|
+
detail: `LLM output tokens shrank ${first}→${last} over ${outputs.length} serial calls while input tokens increased monotonically.`,
|
|
413
|
+
evidence: {
|
|
414
|
+
first,
|
|
415
|
+
last,
|
|
416
|
+
calls: outputs.length,
|
|
417
|
+
scope: sequence.scopeId,
|
|
418
|
+
first_span_id: sequence.spanIds[0],
|
|
419
|
+
last_span_id: sequence.spanIds[sequence.spanIds.length - 1]
|
|
420
|
+
}
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
return signals;
|
|
424
|
+
}
|
|
425
|
+
function everyAdjacent(values, predicate) {
|
|
426
|
+
return values.slice(1).every((current, index) => predicate(values[index], current));
|
|
427
|
+
}
|
|
428
|
+
//#endregion
|
|
429
|
+
//#region src/analyst/behavioral-analyst.ts
|
|
430
|
+
/**
|
|
431
|
+
* Deterministic behavioral analysis over arithmetic in trace spans.
|
|
432
|
+
* This pass is cheap and repeatable; semantic analysis remains the job of
|
|
433
|
+
* model-backed analysts. Relative quality requires a labeled comparison.
|
|
434
|
+
*/
|
|
435
|
+
const RECOMMENDED_ACTION = {
|
|
436
|
+
"monotonic-input-growth": "Inspect context assembly; if prior history is repeatedly included, summarize completed work before the next model call.",
|
|
437
|
+
"output-length-decay": "Check late-step completeness; if shorter responses omit required work, add explicit completion criteria to the agent instructions.",
|
|
438
|
+
"single-tool-dependency": "Test whether an inspect or verification tool improves outcomes after the repeated call fails or returns no progress.",
|
|
439
|
+
"no-self-verification": "After state-changing actions, require an observable check before the agent proceeds."
|
|
440
|
+
};
|
|
441
|
+
const ANALYST_ID$1 = "efficiency-behavioral";
|
|
442
|
+
const DEFAULT_MAX_TRACES = 1e3;
|
|
443
|
+
const DEFAULT_MAX_EVIDENCE_REFS = 20;
|
|
444
|
+
const TRACE_PAGE_SIZE = 200;
|
|
445
|
+
const AGGREGATE_CLAIM = {
|
|
446
|
+
"monotonic-input-growth": (observed, analyzed) => `${observed}/${analyzed} analyzed traces showed input tokens grow from zero to nonzero or to at least 3x their initial value across at least 3 serial model calls without a decrease.`,
|
|
447
|
+
"output-length-decay": (observed, analyzed) => `${observed}/${analyzed} analyzed traces showed output tokens decrease while input tokens increased monotonically across at least 3 serial model calls.`,
|
|
448
|
+
"single-tool-dependency": (observed, analyzed) => `${observed}/${analyzed} analyzed traces used only one named tool across at least 3 tool calls.`,
|
|
449
|
+
"no-self-verification": (observed, analyzed) => `${observed}/${analyzed} analyzed traces had at least 3 tool calls without a verification-named tool call.`
|
|
450
|
+
};
|
|
451
|
+
async function listTraceIds(store, maxTraces, signal) {
|
|
452
|
+
const traceIds = /* @__PURE__ */ new Set();
|
|
453
|
+
let offset = 0;
|
|
454
|
+
let expectedTotal;
|
|
455
|
+
while (true) {
|
|
456
|
+
signal?.throwIfAborted();
|
|
457
|
+
const page = await store.queryTraces({
|
|
458
|
+
limit: TRACE_PAGE_SIZE,
|
|
459
|
+
offset
|
|
460
|
+
});
|
|
461
|
+
if (expectedTotal === void 0) expectedTotal = page.total;
|
|
462
|
+
if (page.total !== expectedTotal) throw new Error(`behavioralAnalyst: trace count changed during pagination (${expectedTotal} to ${page.total})`);
|
|
463
|
+
if (page.total > maxTraces) throw new RangeError(`behavioralAnalyst: ${page.total} traces exceed maxTraces=${maxTraces}; filter the store or raise the explicit limit`);
|
|
464
|
+
for (const trace of page.traces) traceIds.add(trace.trace_id);
|
|
465
|
+
if (traceIds.size > maxTraces) throw new RangeError(`behavioralAnalyst: more than maxTraces=${maxTraces} unique traces were returned`);
|
|
466
|
+
if (!page.has_more) break;
|
|
467
|
+
if (page.traces.length === 0) throw new Error("behavioralAnalyst: trace store returned an empty page with has_more=true");
|
|
468
|
+
offset += page.traces.length;
|
|
469
|
+
}
|
|
470
|
+
if (traceIds.size !== expectedTotal) throw new Error(`behavioralAnalyst: pagination returned ${traceIds.size}/${expectedTotal ?? 0} unique traces`);
|
|
471
|
+
return [...traceIds].sort();
|
|
472
|
+
}
|
|
473
|
+
/**
|
|
474
|
+
* Map computed signals → structured AnalystFindings. Pure: no LLM, no clock
|
|
475
|
+
* dependence beyond `produced_at` (overridable for deterministic tests).
|
|
476
|
+
*/
|
|
477
|
+
function deriveEfficiencyFindings(metrics, opts = {}) {
|
|
478
|
+
const analystId = opts.analystId ?? ANALYST_ID$1;
|
|
479
|
+
const traceId = metrics.traceId;
|
|
480
|
+
return metrics.signals.map((sig) => makeFinding({
|
|
481
|
+
analyst_id: analystId,
|
|
482
|
+
area: "efficiency",
|
|
483
|
+
subject: sig.code,
|
|
484
|
+
claim: sig.detail,
|
|
485
|
+
severity: sig.severity,
|
|
486
|
+
confidence: 1,
|
|
487
|
+
evidence_refs: [{
|
|
488
|
+
kind: "metric",
|
|
489
|
+
uri: traceId ? `metric://trace/${encodeURIComponent(traceId)}/efficiency/${sig.code}` : `metric://efficiency/${sig.code}`,
|
|
490
|
+
excerpt: JSON.stringify(sig.evidence)
|
|
491
|
+
}],
|
|
492
|
+
recommended_action: RECOMMENDED_ACTION[sig.code],
|
|
493
|
+
metadata: {
|
|
494
|
+
deterministic: true,
|
|
495
|
+
evidence: sig.evidence,
|
|
496
|
+
...traceId ? { trace_id: traceId } : {}
|
|
497
|
+
},
|
|
498
|
+
id_basis: sig.code,
|
|
499
|
+
...opts.producedAt ? { produced_at: opts.producedAt } : {}
|
|
500
|
+
}));
|
|
501
|
+
}
|
|
502
|
+
/** The deterministic behavioral/efficiency analyst (no LLM, any-model). */
|
|
503
|
+
function behavioralAnalyst(options = {}) {
|
|
504
|
+
const maxTraces = positiveInteger(options.maxTraces ?? DEFAULT_MAX_TRACES, "maxTraces");
|
|
505
|
+
const maxEvidenceRefsPerFinding = positiveInteger(options.maxEvidenceRefsPerFinding ?? DEFAULT_MAX_EVIDENCE_REFS, "maxEvidenceRefsPerFinding");
|
|
506
|
+
return {
|
|
507
|
+
id: ANALYST_ID$1,
|
|
508
|
+
description: "Deterministic behavioral/efficiency findings over OTLP spans — token-growth, output-decay, tool-monoculture, missing self-verification. Zero LLM; model-agnostic by construction.",
|
|
509
|
+
inputKind: "trace-store",
|
|
510
|
+
cost: { kind: "deterministic" },
|
|
511
|
+
version: "2.0.0",
|
|
512
|
+
executionConfig: {
|
|
513
|
+
kind: "behavioral-efficiency",
|
|
514
|
+
max_traces: maxTraces,
|
|
515
|
+
max_evidence_refs_per_finding: maxEvidenceRefsPerFinding
|
|
516
|
+
},
|
|
517
|
+
async analyze(store, context) {
|
|
518
|
+
const analyzedTraceIds = await listTraceIds(store, maxTraces, context.signal);
|
|
519
|
+
const findingsById = /* @__PURE__ */ new Map();
|
|
520
|
+
for (const traceId of analyzedTraceIds) {
|
|
521
|
+
context.signal?.throwIfAborted();
|
|
522
|
+
const viewed = await store.viewTrace({ trace_id: traceId });
|
|
523
|
+
if (viewed.trace_id !== traceId) throw new Error(`behavioralAnalyst: requested trace '${traceId}', received '${viewed.trace_id}'`);
|
|
524
|
+
if (!viewed.spans) throw new Error(`behavioralAnalyst: trace '${traceId}' is oversized; complete spans are required`);
|
|
525
|
+
const metrics = computeTraceMetrics(viewed.spans);
|
|
526
|
+
if (metrics.traceId !== null && metrics.traceId !== traceId) throw new Error(`behavioralAnalyst: requested trace '${traceId}', received '${metrics.traceId}'`);
|
|
527
|
+
for (const finding of deriveEfficiencyFindings(metrics)) {
|
|
528
|
+
const current = findingsById.get(finding.finding_id);
|
|
529
|
+
if (!current) {
|
|
530
|
+
findingsById.set(finding.finding_id, {
|
|
531
|
+
finding,
|
|
532
|
+
observedTraceCount: 1,
|
|
533
|
+
evidenceTraceIds: [traceId],
|
|
534
|
+
evidence: [...finding.evidence_refs]
|
|
535
|
+
});
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
current.observedTraceCount += 1;
|
|
539
|
+
if (current.evidence.length < maxEvidenceRefsPerFinding) {
|
|
540
|
+
current.evidenceTraceIds.push(traceId);
|
|
541
|
+
current.evidence.push(...finding.evidence_refs);
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
return [...findingsById.values()].map(({ finding, observedTraceCount, evidenceTraceIds, evidence }) => ({
|
|
546
|
+
...finding,
|
|
547
|
+
claim: AGGREGATE_CLAIM[finding.subject](observedTraceCount, analyzedTraceIds.length),
|
|
548
|
+
rationale: `${observedTraceCount}/${analyzedTraceIds.length} analyzed traces exhibited this pattern.`,
|
|
549
|
+
evidence_refs: evidence,
|
|
550
|
+
metadata: {
|
|
551
|
+
deterministic: true,
|
|
552
|
+
evidence_trace_ids: evidenceTraceIds,
|
|
553
|
+
omitted_evidence_trace_count: observedTraceCount - evidenceTraceIds.length,
|
|
554
|
+
observed_trace_count: observedTraceCount,
|
|
555
|
+
analyzed_trace_count: analyzedTraceIds.length
|
|
556
|
+
}
|
|
557
|
+
}));
|
|
558
|
+
}
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
function positiveInteger(value, name) {
|
|
562
|
+
if (!Number.isSafeInteger(value) || value < 1) throw new RangeError(`behavioralAnalyst: ${name} must be a positive safe integer`);
|
|
563
|
+
return value;
|
|
564
|
+
}
|
|
565
|
+
//#endregion
|
|
566
|
+
//#region src/analyst/kinds/control-integrity.ts
|
|
567
|
+
const ANALYST_ID = "control-integrity";
|
|
568
|
+
function shown(value) {
|
|
569
|
+
if (value === void 0) return "<absent>";
|
|
570
|
+
const encoded = JSON.stringify(value);
|
|
571
|
+
return encoded === void 0 ? String(value) : encoded;
|
|
572
|
+
}
|
|
573
|
+
function evidenceRef(namespace, value) {
|
|
574
|
+
return {
|
|
575
|
+
kind: "metric",
|
|
576
|
+
uri: `supervisor-run://${encodeURIComponent(namespace)}/${value.path}`,
|
|
577
|
+
excerpt: shown(value.value)
|
|
578
|
+
};
|
|
579
|
+
}
|
|
580
|
+
/** Translate typed supervisor-run integrity issues into the shared analyst envelope. */
|
|
581
|
+
function emitControlIntegrityFindings(input, producedAt) {
|
|
582
|
+
const report = analyzeSupervisorRunIntegrity(input, { capturedAt: producedAt });
|
|
583
|
+
return report.issues.map((issue) => makeFinding({
|
|
584
|
+
analyst_id: ANALYST_ID,
|
|
585
|
+
produced_at: producedAt,
|
|
586
|
+
area: issue.area,
|
|
587
|
+
severity: issue.severity,
|
|
588
|
+
subject: `${report.runRef}/${issue.subject}`,
|
|
589
|
+
claim: issue.claim,
|
|
590
|
+
rationale: issue.detail,
|
|
591
|
+
evidence_refs: issue.evidence.map((value) => evidenceRef(report.runRef, value)),
|
|
592
|
+
recommended_action: issue.recommendedAction,
|
|
593
|
+
validation_plan: "Re-run this deterministic analyst on the retained SupervisorRunSources or SupervisorRunTree after correcting the producer.",
|
|
594
|
+
confidence: 1,
|
|
595
|
+
metadata: {
|
|
596
|
+
integrity_code: issue.code,
|
|
597
|
+
integrity_input: report.input,
|
|
598
|
+
integrity_run_ref: report.runRef,
|
|
599
|
+
integrity_subject: issue.subject,
|
|
600
|
+
...issue.metadata
|
|
601
|
+
}
|
|
602
|
+
}));
|
|
603
|
+
}
|
|
604
|
+
/** Deterministic Analyst adapter for `SupervisorRunSources | SupervisorRunTree`. */
|
|
605
|
+
var ControlIntegrityAnalyst = class {
|
|
606
|
+
id = ANALYST_ID;
|
|
607
|
+
description = "Deterministic supervisor-run integrity checks with explicit unavailable evidence.";
|
|
608
|
+
inputKind = "custom";
|
|
609
|
+
cost = {
|
|
610
|
+
kind: "deterministic",
|
|
611
|
+
est_usd_per_run: 0
|
|
612
|
+
};
|
|
613
|
+
version = "2.0.0";
|
|
614
|
+
executionConfig = {
|
|
615
|
+
kind: "control-integrity",
|
|
616
|
+
produced_at_source: "tags.producedAt-or-system-clock"
|
|
617
|
+
};
|
|
618
|
+
async analyze(input, ctx) {
|
|
619
|
+
const findings = emitControlIntegrityFindings(input, ctx.tags?.producedAt ?? (/* @__PURE__ */ new Date()).toISOString());
|
|
620
|
+
ctx.log?.(`control-integrity: ${findings.length} finding(s)`, { input: "nodes" in input ? "SupervisorRunTree" : "SupervisorRunSources" });
|
|
621
|
+
return findings;
|
|
622
|
+
}
|
|
623
|
+
};
|
|
624
|
+
const CONTROL_INTEGRITY_ANALYST = new ControlIntegrityAnalyst();
|
|
625
|
+
const FAILURE_MODE_KIND_SPEC = {
|
|
626
|
+
id: "failure-mode",
|
|
627
|
+
description: "Clusters trace-dataset failures into distinct failure modes with cited evidence and a short recommended action.",
|
|
628
|
+
area: "failure-mode",
|
|
629
|
+
version: "1.2.0",
|
|
630
|
+
instructions: `You are a failure-mode classifier for an OTLP trace dataset. Your job is to identify the **distinct ways agents failed** in this dataset, not to grade individual runs.
|
|
631
|
+
|
|
632
|
+
${findingSubjectGrammarPromptFor("failure-mode")}
|
|
633
|
+
|
|
634
|
+
DISCOVERY → CLUSTER → CITE protocol:
|
|
635
|
+
|
|
636
|
+
1. Call \`getDatasetOverview({})\` first. Use \`has_errors\`, \`models\`, \`agent_names\`, \`tools\`, and \`sample_trace_ids\` to size the failure surface.
|
|
637
|
+
2. Use \`queryTraces(filters={"has_errors": true}, limit=...)\` to pull error-bearing traces. Combine with \`countTraces\` to see what fraction of the dataset failed.
|
|
638
|
+
3. For each candidate failure cluster, use \`searchTrace\` with regex like \`STATUS_CODE_ERROR\`, \`MaxTurnsExceeded\`, \`assertion\`, \`unauthorized\`, \`timeout\`, \`429\`, \`5\\d\\d\`, the agent's specific error strings, or the names of its tools. Pull one or two representative traces per cluster, **not all** of them.
|
|
639
|
+
4. **Cluster, do not enumerate.** Two failures with the same root cause should be ONE finding citing both traces, not two findings. The point of this analyst is to compress N runs into K modes.
|
|
640
|
+
5. For each defensible cluster, emit ONE finding. Use a lowercase cluster label matching the subject grammar ("tool-call-loop", "auth-revoked-mid-run", ...). Rate it critical when it blocks the run, high when the run finishes degraded, and medium when it slows convergence. Cite representative spans and include exact error, payload, or contradictory-output quotes. Use confidence 0.85+ when multiple traces show the same shape, 0.6-0.8 for a single-trace inference, and <0.5 for speculation. Keep the imperative fix idea short; the improvement analyst expands it.
|
|
641
|
+
|
|
642
|
+
If the dataset has no failures, return an empty findings array — do NOT pad with low-confidence speculation.
|
|
643
|
+
|
|
644
|
+
**Use subqueries over loaded evidence.** After the first scan, load representative span excerpts for each candidate cluster. Then send one bounded \`llm_query\` per cluster in one batch, including the exact excerpts and asking it to classify the root cause. Subqueries cannot call trace tools. Merge or split clusters yourself from their classifications and the cited source evidence.`,
|
|
645
|
+
toolGroup: "all",
|
|
646
|
+
limits: {
|
|
647
|
+
maxLlmCalls: 8,
|
|
648
|
+
maxIterations: 24,
|
|
649
|
+
maxToolCalls: 64
|
|
650
|
+
}
|
|
651
|
+
};
|
|
652
|
+
const IMPROVEMENT_KIND_SPEC = {
|
|
653
|
+
id: "improvement",
|
|
654
|
+
description: "Converts upstream failure / gap / poisoning findings into concrete locus-named edits (prompt, tool-doc, RAG, scaffolding) with leverage grades.",
|
|
655
|
+
area: "improvement",
|
|
656
|
+
version: "1.2.0",
|
|
657
|
+
instructions: `You are a self-improvement analyst. Your job is to propose **concrete, locus-named edits** the agent's runtime should adopt to fix the failure modes, knowledge gaps, and poisonings present in this dataset.
|
|
658
|
+
|
|
659
|
+
Upstream analysts have already classified the problems. Your job is to convert each problem into a *change to make* and grade its expected leverage. Each finding is one proposed edit.
|
|
660
|
+
|
|
661
|
+
${findingSubjectGrammarPromptFor("improvement")}
|
|
662
|
+
|
|
663
|
+
DISCOVERY → CANDIDATE-FIXES → COMPETE → CITE protocol:
|
|
664
|
+
|
|
665
|
+
1. \`getDatasetOverview({})\` first. Note the agents, tools, and any system-prompt fingerprints (look for the prompt text echoed in early spans).
|
|
666
|
+
2. For each high-severity failure pattern, generate 2-3 candidate fixes. Real candidate axes:
|
|
667
|
+
- **System-prompt edit** — add an instruction, remove a misleading one, restructure precedence
|
|
668
|
+
- **Tool description edit** — rewrite a tool's description so the agent picks it correctly / passes valid args
|
|
669
|
+
- **New tool** — add a tool the agent kept emulating in code
|
|
670
|
+
- **RAG ingestion** — add a document or correct a stale one
|
|
671
|
+
- **Memory invalidation** — clear cached prior-run decisions that no longer apply
|
|
672
|
+
- **Scaffolding** — add a precondition check, a retry policy, a turn budget, a verification step
|
|
673
|
+
- **Output schema** — narrow the agent's output to forbid the failure shape
|
|
674
|
+
- **Skill / MCP / hook / subagent** — change the reusable profile component responsible for the behavior
|
|
675
|
+
- **Workflow / rollout policy** — change orchestration, budget, sampling, or stopping behavior
|
|
676
|
+
- **Code** — change an implementation path when profile edits cannot repair the behavior
|
|
677
|
+
3. **Compare candidate fixes with bounded subqueries.** Load the representative failure excerpts, then send one \`llm_query\` per candidate-fix axis the same evidence. Ask for likely effect, side effects, and implementation scope. Subqueries cannot call trace tools; trace ids alone are insufficient context.
|
|
678
|
+
4. After the comparisons return, **pick the winning candidate per cluster** based on expected effect and risk, then emit ONE finding. Keep the alternatives and rejection reasons in the rationale so the recommendation is auditable.
|
|
679
|
+
5. **Cross-reference upstream findings.** Cite prior failure-mode or knowledge-gap findings as \`finding://<prior-finding-id>\`. This builds the dependency graph that lets the dashboard show "fix #X resolves failure modes A, B, C."
|
|
680
|
+
|
|
681
|
+
For each winning recommendation, emit ONE finding. Use one exact locus from the subject grammar and state the edit in one sentence. Match leverage to the source failure's severity; use medium for quality-of-life changes and info for cleanup with no behavioral effect. Cite the targeted \`finding://<id>\` when available and the most representative span when useful. Quote the problem being fixed. Use confidence 0.85+ for a mechanical fix to a well-evidenced failure, 0.6-0.8 when judgment is required, and <0.5 for speculation. Explain in at most two sentences why this candidate beat its alternatives. The recommended action must be the literal diff, quoted replacement, tool description, or setting change.
|
|
682
|
+
|
|
683
|
+
If no upstream failure findings exist in this run, derive your own from the trace dataset using the failure-mode protocol inline (\`searchTrace\` for STATUS_CODE_ERROR / MaxTurnsExceeded / etc.). Prefer upstream findings when present because the analysts are designed to chain.
|
|
684
|
+
|
|
685
|
+
Do NOT propose a fix you cannot defend with evidence. "Tighten the prompt" is not a finding; "Add 'When the user asks for X, always Y' to the system prompt section "request-classification"" is.`,
|
|
686
|
+
toolGroup: "all",
|
|
687
|
+
limits: {
|
|
688
|
+
maxLlmCalls: 8,
|
|
689
|
+
maxIterations: 30,
|
|
690
|
+
maxToolCalls: 80,
|
|
691
|
+
maxOutputChars: 12e3
|
|
692
|
+
}
|
|
693
|
+
};
|
|
694
|
+
const KNOWLEDGE_GAP_KIND_SPEC = {
|
|
695
|
+
id: "knowledge-gap",
|
|
696
|
+
description: "Identifies missing or stale pieces of knowledge — primarily against the agent-knowledge wiki — and attributes each to the runtime layer (wiki page, claim, raw source, websearch, tool-doc, system-prompt, memory) that should have held it.",
|
|
697
|
+
area: "knowledge-gap",
|
|
698
|
+
version: "1.2.0",
|
|
699
|
+
instructions: `You are a knowledge-gap analyst for an OTLP trace dataset. Your job is to identify the **specific pieces of information the agent lacked, or that were stale**, that caused poor decisions.
|
|
700
|
+
|
|
701
|
+
The agent under analysis maintains a curated knowledge base via \`@tangle-network/agent-knowledge\` — a wiki of \`KnowledgePage\`s with raw source anchors, claims, and relations. The primary expected store of agent-knowable facts IS that wiki. A "knowledge gap" is anything the agent had to discover or guess at run-time that the wiki should have held — or an outdated/contradictory fact the agent picked up from a non-wiki source.
|
|
702
|
+
|
|
703
|
+
${findingSubjectGrammarPromptFor("knowledge-gap")}
|
|
704
|
+
|
|
705
|
+
DISCOVERY → ATTRIBUTE-TO-LAYER → CITE protocol:
|
|
706
|
+
|
|
707
|
+
1. \`getDatasetOverview({})\` first. Note which agents, tools, and models appear.
|
|
708
|
+
2. Pull traces where the agent shows gap signals. The strongest signals are:
|
|
709
|
+
- Self-correction turns ("I assumed X but…", "let me re-check", "actually,")
|
|
710
|
+
- Clarifying-question turns where the agent asked the user something the runtime should have surfaced
|
|
711
|
+
- Repeated retrieval / lookup calls for the same artifact with slightly varied queries
|
|
712
|
+
- Tool errors that name a missing argument or unknown resource
|
|
713
|
+
- Web-search calls returning pages dated before a known cutoff for content that changes (versioned APIs, schemas, policies)
|
|
714
|
+
- Agent quoting a tool's docs / system prompt incorrectly because the actual text was insufficient
|
|
715
|
+
- Fabricated identifiers that don't appear in dataset \`sample_trace_ids\`
|
|
716
|
+
Use \`searchTrace\` with patterns like \`I (don.?t|do not) know\`, \`assumed\`, \`unclear\`, \`could you (clarify|tell me|provide)\`, \`not found\`, \`undefined\`, \`unknown\`, \`null\`, dates older than the analysis window, or the agent's specific clarification phrases.
|
|
717
|
+
3. For each gap, identify the **layer of the runtime that should have prevented it** and use its exact locus from the subject grammar above.
|
|
718
|
+
4. For each defensible gap, emit ONE finding. Use an exact locus from the subject grammar and name the missing or stale knowledge (for example, "wiki has no page on invoice line-item shape; agent re-derived it from raw spans"). Rate it high when it caused failure or a clarifying question, medium for unnecessary turns, and low for minor inefficiency. Cite the span where the question, correction, retrieval miss, or stale result surfaced and quote it exactly. Use confidence 0.85+ when the agent articulated the gap and 0.6-0.8 when inferred. Recommend a concrete wiki edit for an agent-knowledge locus or a prompt/tool-description edit otherwise.
|
|
719
|
+
|
|
720
|
+
**Compare layers over loaded evidence.** After the first scan, load the exact excerpts behind candidates across \`agent-knowledge:*\`, \`websearch:outdated\`, \`tool-doc:*\`, \`system-prompt:*\`, and \`memory:*\`. Use one bounded \`llm_query\` per layer to classify those excerpts. Subqueries cannot call trace tools. Merge their classifications into the final finding set only when the source excerpts support them.
|
|
721
|
+
|
|
722
|
+
Do NOT report a gap that the agent later recovered from cleanly within the same turn. That is resilience, not a gap. Cite the non-recovery version when both exist.`,
|
|
723
|
+
toolGroup: "discoveryAndSearch",
|
|
724
|
+
limits: {
|
|
725
|
+
maxLlmCalls: 5,
|
|
726
|
+
maxIterations: 18,
|
|
727
|
+
maxToolCalls: 48
|
|
728
|
+
}
|
|
729
|
+
};
|
|
730
|
+
const KNOWLEDGE_POISONING_KIND_SPEC = {
|
|
731
|
+
id: "knowledge-poisoning",
|
|
732
|
+
description: "Identifies confident-but-wrong actions caused by stale memory, contradicting RAG, deprecated tool docs, or outdated system-prompt instructions.",
|
|
733
|
+
area: "knowledge-poisoning",
|
|
734
|
+
version: "1.2.0",
|
|
735
|
+
instructions: `You are a knowledge-poisoning analyst for an OTLP trace dataset. Your job is to identify cases where the agent **confidently used wrong information** — not where it lacked information (that's the knowledge-gap analyst).
|
|
736
|
+
|
|
737
|
+
${findingSubjectGrammarPromptFor("knowledge-poisoning")}
|
|
738
|
+
|
|
739
|
+
DISCOVERY → DUAL-VERIFY → CITE protocol:
|
|
740
|
+
|
|
741
|
+
1. \`getDatasetOverview({})\` first. Identify the agents, models, and tools.
|
|
742
|
+
2. Pull traces where the agent's confident action was later contradicted. Strongest signals:
|
|
743
|
+
- Agent stated a fact in one span; a later span surfaced contradictory evidence; the agent then proceeded anyway or fabricated reconciliation.
|
|
744
|
+
- Tool call with stale arguments (an id that no longer exists, an API shape that changed).
|
|
745
|
+
- Agent cited an \`agent-knowledge\` wiki page or claim whose content contradicts the trace's own evidence — the wiki itself drifted.
|
|
746
|
+
- Web-search result the agent cited that returned an outdated page; agent treated it as canonical.
|
|
747
|
+
- System-prompt instruction the agent followed that ground-truth evidence in the trace contradicts (e.g. prompt says "use endpoint A"; tool reply says "endpoint A deprecated, use B").
|
|
748
|
+
- Repeated wrong-shape parsing despite the tool's actual output proving the shape.
|
|
749
|
+
3. Use \`searchTrace\` with regex on phrases like \`actually\`, \`turns out\`, \`previously assumed\`, \`old version\`, \`deprecated\`, \`updated to\`, \`now uses\`, or specific entity names you suspect have changed.
|
|
750
|
+
4. For each candidate poisoning, **DUAL-VERIFY**:
|
|
751
|
+
- Confirm the agent actually acted on the false belief (cite the span where it did)
|
|
752
|
+
- Confirm the belief is actually false in this trace's own evidence (cite the span that contradicts it)
|
|
753
|
+
Only emit a finding when both halves are supported. If you can only support one, drop it because single-evidence poisoning findings are too speculative to be useful.
|
|
754
|
+
|
|
755
|
+
**Independently assess both halves.** Load the action excerpt and contradicting excerpt yourself, then send bounded \`llm_query\` calls the exact evidence for "did the agent act?" and "does the trace contradict the belief?" Subqueries cannot call trace tools. Accept a poisoning only when both assessments and the source excerpts support it.
|
|
756
|
+
|
|
757
|
+
For each confirmed poisoning, emit ONE finding. Use the source of the false belief as the exact subject. State "agent believed X (from source S); trace evidence shows X is false." Rate it critical for a wrong user-visible action, high when caught internally after significant waste, and medium for inefficiency. Cite BOTH the action span and the contradicting span with exact quotes. Use confidence 0.85+ when both halves have exact quotes and 0.6-0.8 when one half is inferred. Recommend the literal source correction: update the wiki claim, invalidate and re-curate the raw source, or replace the stale prompt/tool instruction.
|
|
758
|
+
|
|
759
|
+
Do NOT report a finding if the agent caught and corrected the false belief in the same turn. Reserve poisoning for cases where the false belief shaped downstream action.`,
|
|
760
|
+
toolGroup: "all",
|
|
761
|
+
limits: {
|
|
762
|
+
maxLlmCalls: 8,
|
|
763
|
+
maxIterations: 20,
|
|
764
|
+
maxToolCalls: 64
|
|
765
|
+
},
|
|
766
|
+
minimumEvidenceCitations: 2
|
|
767
|
+
};
|
|
768
|
+
//#endregion
|
|
769
|
+
//#region src/analyst/kinds/index.ts
|
|
770
|
+
/**
|
|
771
|
+
* The default kind suite. Order is the run order operators should
|
|
772
|
+
* use: failure-mode first (no upstream deps), gap + poisoning next
|
|
773
|
+
* (both depend on failures), improvement last (chains all three).
|
|
774
|
+
*/
|
|
775
|
+
const DEFAULT_TRACE_ANALYST_KINDS = [
|
|
776
|
+
FAILURE_MODE_KIND_SPEC,
|
|
777
|
+
KNOWLEDGE_GAP_KIND_SPEC,
|
|
778
|
+
KNOWLEDGE_POISONING_KIND_SPEC,
|
|
779
|
+
IMPROVEMENT_KIND_SPEC
|
|
780
|
+
];
|
|
781
|
+
//#endregion
|
|
782
|
+
//#region src/feedback-trajectory-review.ts
|
|
783
|
+
/** Bind an analyst finding's complete canonical JSON content to a stable digest. */
|
|
784
|
+
function analystFindingDigest(finding) {
|
|
785
|
+
return hashCanonical(snapshotAnalystFinding(finding, "analyst finding"));
|
|
786
|
+
}
|
|
787
|
+
/** Bind the complete analyst result to one immutable review target. */
|
|
788
|
+
function analystRunDigest(run) {
|
|
789
|
+
return hashCanonical(snapshotAnalystRun(run, "analyst run"));
|
|
790
|
+
}
|
|
791
|
+
function snapshotAnalystRun(value, context = "analyst run") {
|
|
792
|
+
const snapshot = snapshotAnalystRunRecord(value, context);
|
|
793
|
+
if (snapshot.execution_plan !== void 0) return sealExactAnalystRunReceipt(snapshot, context);
|
|
794
|
+
if (snapshot.completion !== void 0) throw new TypeError(`${context} completion is valid only for an exact run`);
|
|
795
|
+
return snapshot;
|
|
796
|
+
}
|
|
797
|
+
/** Canonicalize, validate, and deeply freeze one complete or failed exact-run receipt. */
|
|
798
|
+
function snapshotExactAnalystRunReceipt(value, context = "exact analyst run receipt") {
|
|
799
|
+
return sealExactAnalystRunReceipt(snapshotAnalystRunRecord(value, context), context);
|
|
800
|
+
}
|
|
801
|
+
function snapshotAnalystRunRecord(value, context) {
|
|
802
|
+
let snapshot;
|
|
803
|
+
try {
|
|
804
|
+
snapshot = JSON.parse(canonicalString(value));
|
|
805
|
+
} catch (cause) {
|
|
806
|
+
throw new TypeError(`${context} must have a canonical JSON representation`, { cause });
|
|
807
|
+
}
|
|
808
|
+
if (!isRecord(snapshot)) throw new TypeError(`${context} must be an object`);
|
|
809
|
+
assertOnlyKeys(snapshot, [
|
|
810
|
+
"run_id",
|
|
811
|
+
"correlation_id",
|
|
812
|
+
"started_at",
|
|
813
|
+
"ended_at",
|
|
814
|
+
"findings",
|
|
815
|
+
"per_analyst",
|
|
816
|
+
"total_cost_usd",
|
|
817
|
+
"total_cost_provenance",
|
|
818
|
+
"execution_plan",
|
|
819
|
+
"completion"
|
|
820
|
+
], context);
|
|
821
|
+
requiredString(snapshot.run_id, `${context} run_id`);
|
|
822
|
+
requiredString(snapshot.correlation_id, `${context} correlation_id`);
|
|
823
|
+
canonicalTimestamp(snapshot.started_at, `${context} started_at`);
|
|
824
|
+
canonicalTimestamp(snapshot.ended_at, `${context} ended_at`);
|
|
825
|
+
snapshot.findings = snapshotAnalystFindings(snapshot.findings, `${context} findings`);
|
|
826
|
+
if (!Array.isArray(snapshot.per_analyst)) throw new TypeError(`${context} per_analyst must be an array`);
|
|
827
|
+
for (const [index, summary] of snapshot.per_analyst.entries()) assertAnalystRunSummary(summary, `${context} per_analyst ${index}`);
|
|
828
|
+
if (typeof snapshot.total_cost_usd !== "number" || !Number.isFinite(snapshot.total_cost_usd) || snapshot.total_cost_usd < 0) throw new TypeError(`${context} total_cost_usd must be a finite non-negative number`);
|
|
829
|
+
if (snapshot.total_cost_provenance !== void 0) assertCostProvenance(snapshot.total_cost_provenance, `${context} total_cost_provenance`);
|
|
830
|
+
return snapshot;
|
|
831
|
+
}
|
|
832
|
+
function assertAnalystRunSummary(value, context) {
|
|
833
|
+
if (!isRecord(value)) throw new TypeError(`${context} must be an object`);
|
|
834
|
+
assertOnlyKeys(value, [
|
|
835
|
+
"analyst_id",
|
|
836
|
+
"status",
|
|
837
|
+
"reason",
|
|
838
|
+
"findings_count",
|
|
839
|
+
"latency_ms",
|
|
840
|
+
"usage",
|
|
841
|
+
"allocated_budget_usd",
|
|
842
|
+
"error"
|
|
843
|
+
], context);
|
|
844
|
+
requiredString(value.analyst_id, `${context} analyst_id`);
|
|
845
|
+
if (value.status !== "ok" && value.status !== "skipped" && value.status !== "failed") throw new TypeError(`${context} status is invalid`);
|
|
846
|
+
if (value.reason !== void 0) requiredString(value.reason, `${context} reason`);
|
|
847
|
+
if (value.status === "skipped" && value.reason === void 0) throw new TypeError(`${context} skipped summary requires reason`);
|
|
848
|
+
nonnegativeSafeInteger(value.findings_count, `${context} findings_count`);
|
|
849
|
+
finiteNonnegative(value.latency_ms, `${context} latency_ms`);
|
|
850
|
+
assertAnalystUsageReceipt(value.usage, `${context} usage`);
|
|
851
|
+
if (value.allocated_budget_usd !== void 0 && value.allocated_budget_usd !== null) finiteNonnegative(value.allocated_budget_usd, `${context} allocated_budget_usd`);
|
|
852
|
+
if (value.error !== void 0) {
|
|
853
|
+
if (value.status !== "failed" || !isRecord(value.error)) throw new TypeError(`${context} error is valid only for failed summaries`);
|
|
854
|
+
assertOnlyKeys(value.error, ["class", "message"], `${context} error`);
|
|
855
|
+
requiredString(value.error.class, `${context} error class`);
|
|
856
|
+
requiredString(value.error.message, `${context} error message`);
|
|
857
|
+
} else if (value.status === "failed") throw new TypeError(`${context} failed summary requires error`);
|
|
858
|
+
}
|
|
859
|
+
function assertAnalystUsageReceipt(value, context) {
|
|
860
|
+
if (!isRecord(value)) throw new TypeError(`${context} must be an object`);
|
|
861
|
+
assertOnlyKeys(value, [
|
|
862
|
+
"calls",
|
|
863
|
+
"tokens",
|
|
864
|
+
"cost",
|
|
865
|
+
"knownCostUsd"
|
|
866
|
+
], context);
|
|
867
|
+
for (const field of [
|
|
868
|
+
"calls",
|
|
869
|
+
"tokens",
|
|
870
|
+
"cost"
|
|
871
|
+
]) if (!Object.hasOwn(value, field)) throw new TypeError(`${context} ${field} is required`);
|
|
872
|
+
if (value.tokens !== null) {
|
|
873
|
+
if (!isRecord(value.tokens)) throw new TypeError(`${context} tokens must be an object or null`);
|
|
874
|
+
assertOnlyKeys(value.tokens, [
|
|
875
|
+
"input",
|
|
876
|
+
"output",
|
|
877
|
+
"reasoning",
|
|
878
|
+
"cached",
|
|
879
|
+
"cacheWrite"
|
|
880
|
+
], `${context} tokens`);
|
|
881
|
+
}
|
|
882
|
+
assertCostProvenance(value.cost, `${context} cost`);
|
|
883
|
+
assertValidAnalystUsageReceipt(value, context);
|
|
884
|
+
}
|
|
885
|
+
function assertCostProvenance(value, context) {
|
|
886
|
+
if (!isRecord(value)) throw new TypeError(`${context} must be an object`);
|
|
887
|
+
assertOnlyKeys(value, ["kind", "usd"], context);
|
|
888
|
+
if (value.kind === "uncaptured") {
|
|
889
|
+
if (value.usd !== null) throw new TypeError(`${context} uncaptured usd must be null`);
|
|
890
|
+
return;
|
|
891
|
+
}
|
|
892
|
+
if (value.kind !== "observed" && value.kind !== "estimated") throw new TypeError(`${context} kind is invalid`);
|
|
893
|
+
finiteNonnegative(value.usd, `${context} usd`);
|
|
894
|
+
}
|
|
895
|
+
function sealExactAnalystRunReceipt(run, context) {
|
|
896
|
+
if (run.execution_plan === void 0) throw new TypeError(`${context} exact run requires execution_plan`);
|
|
897
|
+
const plan = snapshotExactExecutionPlan(run.execution_plan, `${context} execution_plan`);
|
|
898
|
+
const completion = snapshotExactRunCompletion(run.completion, `${context} completion`);
|
|
899
|
+
run.execution_plan = plan;
|
|
900
|
+
run.completion = completion;
|
|
901
|
+
const summaries = run.per_analyst;
|
|
902
|
+
const findings = run.findings;
|
|
903
|
+
const planned = plan.analysts.map((analyst) => analyst.id);
|
|
904
|
+
const completed = summaries.map((summary) => summary.analyst_id);
|
|
905
|
+
if (!completed.every((analystId, index) => analystId === planned[index]) || completion.status === "complete" && completed.length !== planned.length) throw new TypeError(completion.status === "complete" ? `${context} complete receipt must contain every execution_plan analyst in exact order` : `${context} failed receipt per_analyst must be an execution_plan prefix`);
|
|
906
|
+
const completedIds = new Set(completed);
|
|
907
|
+
for (const finding of findings) if (!completedIds.has(finding.analyst_id)) throw new TypeError(`${context} finding names an analyst absent from per_analyst`);
|
|
908
|
+
for (const summary of summaries) {
|
|
909
|
+
const actual = findings.filter((finding) => finding.analyst_id === summary.analyst_id).length;
|
|
910
|
+
if (summary.findings_count !== actual) throw new TypeError(`${context} findings_count does not match findings for "${summary.analyst_id}"`);
|
|
911
|
+
const hasAllocation = Object.hasOwn(summary, "allocated_budget_usd");
|
|
912
|
+
if (summary.status === "skipped") {
|
|
913
|
+
if (hasAllocation) throw new TypeError(`${context} skipped summary "${summary.analyst_id}" cannot report an allocated budget`);
|
|
914
|
+
continue;
|
|
915
|
+
}
|
|
916
|
+
const allocation = summary.allocated_budget_usd;
|
|
917
|
+
if (!(hasAllocation && (plan.policy.budget.kind === "none" ? allocation === null : typeof allocation === "number" && plan.policy.budget.allocations_usd[summary.analyst_id] !== null && plan.policy.budget.allocations_usd[summary.analyst_id] !== void 0 && allocation <= plan.policy.budget.allocations_usd[summary.analyst_id]))) throw new TypeError(`${context} summary "${summary.analyst_id}" allocation does not match its execution plan`);
|
|
918
|
+
}
|
|
919
|
+
let knownCost = 0;
|
|
920
|
+
for (const summary of summaries) {
|
|
921
|
+
const amount = summary.usage.cost.kind === "uncaptured" ? summary.usage.knownCostUsd ?? 0 : summary.usage.cost.usd ?? 0;
|
|
922
|
+
knownCost = finiteNonnegative(knownCost + amount, `${context} aggregate known cost`);
|
|
923
|
+
}
|
|
924
|
+
if (run.total_cost_usd !== knownCost) throw new TypeError(`${context} total_cost_usd does not match per_analyst usage`);
|
|
925
|
+
if (run.total_cost_provenance === void 0) throw new TypeError(`${context} exact run requires total_cost_provenance`);
|
|
926
|
+
const costs = summaries.map((summary) => summary.usage.cost);
|
|
927
|
+
const expectedProvenance = costs.some((cost) => cost.kind === "uncaptured") ? {
|
|
928
|
+
kind: "uncaptured",
|
|
929
|
+
usd: null
|
|
930
|
+
} : {
|
|
931
|
+
kind: costs.some((cost) => cost.kind === "estimated") ? "estimated" : "observed",
|
|
932
|
+
usd: costs.reduce((sum, cost) => finiteNonnegative(sum + (cost.usd ?? 0), `${context} aggregate captured cost`), 0)
|
|
933
|
+
};
|
|
934
|
+
if (hashCanonical(run.total_cost_provenance) !== hashCanonical(expectedProvenance)) throw new TypeError(`${context} total_cost_provenance does not match per_analyst usage`);
|
|
935
|
+
return deepFreezeCanonicalJson(run);
|
|
936
|
+
}
|
|
937
|
+
function snapshotExactRunCompletion(value, context) {
|
|
938
|
+
if (!isRecord(value)) throw new TypeError(`${context} must be an object`);
|
|
939
|
+
if (value.status === "complete") {
|
|
940
|
+
assertOnlyKeys(value, ["status"], context);
|
|
941
|
+
return value;
|
|
942
|
+
}
|
|
943
|
+
if (value.status !== "failed") throw new TypeError(`${context} status must be complete or failed`);
|
|
944
|
+
assertOnlyKeys(value, ["status", "error"], context);
|
|
945
|
+
if (!isRecord(value.error)) throw new TypeError(`${context} failed receipt requires error`);
|
|
946
|
+
assertOnlyKeys(value.error, ["class", "message"], `${context} error`);
|
|
947
|
+
requiredString(value.error.class, `${context} error class`);
|
|
948
|
+
requiredString(value.error.message, `${context} error message`);
|
|
949
|
+
return value;
|
|
950
|
+
}
|
|
951
|
+
function finiteNonnegative(value, context) {
|
|
952
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new TypeError(`${context} must be a non-negative finite number`);
|
|
953
|
+
return value;
|
|
954
|
+
}
|
|
955
|
+
function nonnegativeSafeInteger(value, context) {
|
|
956
|
+
if (!Number.isSafeInteger(value) || value < 0) throw new TypeError(`${context} must be a non-negative safe integer`);
|
|
957
|
+
return value;
|
|
958
|
+
}
|
|
959
|
+
function snapshotAnalystFindings(value, context = "analyst run findings") {
|
|
960
|
+
if (!Array.isArray(value)) throw new TypeError(`${context} must be an array`);
|
|
961
|
+
const findings = value.map((finding, index) => snapshotAnalystFinding(finding, `${context} finding ${index}`));
|
|
962
|
+
assertUniqueFindingIds(findings.map((finding) => finding.finding_id));
|
|
963
|
+
return findings;
|
|
964
|
+
}
|
|
965
|
+
function readAnalystReview(trajectory) {
|
|
966
|
+
const analystAttempts = trajectory.attempts.filter((attempt) => isRecord(attempt.artifact) && attempt.artifact.type === "analyst-run");
|
|
967
|
+
const analysis = isRecord(trajectory.metadata?.analysis) ? trajectory.metadata.analysis : void 0;
|
|
968
|
+
if (analystAttempts.length === 0) {
|
|
969
|
+
if (analysis?.kind === "analyst-run") throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" is missing its archived run`);
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
972
|
+
if (analystAttempts.length !== 1) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" must contain exactly one archived run`);
|
|
973
|
+
if (analysis?.kind !== "analyst-run") throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" is missing review state`);
|
|
974
|
+
const artifact = analystAttempts[0].artifact;
|
|
975
|
+
if (!isRecord(artifact)) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" has an invalid archived run`);
|
|
976
|
+
const runId = requiredString(artifact.analystRunId, `analyst trajectory "${trajectory.id}" run id`);
|
|
977
|
+
if (analysis.runId !== runId) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" run identity does not match its review state`);
|
|
978
|
+
const artifactRunDigest = requiredDigest(artifact.runDigest, `analyst trajectory "${trajectory.id}" archived run digest`);
|
|
979
|
+
const storedRunDigest = requiredDigest(analysis.runDigest, `analyst trajectory "${trajectory.id}" review run digest`);
|
|
980
|
+
if (artifactRunDigest !== storedRunDigest) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" run digest does not match its review state`);
|
|
981
|
+
const findings = snapshotAnalystFindings(artifact.findings, `analyst trajectory "${trajectory.id}"`);
|
|
982
|
+
const findingIds = findings.map((finding) => finding.finding_id);
|
|
983
|
+
const analystIds = stringArray(artifact.analystIds, `analyst trajectory "${trajectory.id}" analyst ids`);
|
|
984
|
+
const attemptMetadata = analystAttempts[0].metadata;
|
|
985
|
+
if (!isRecord(attemptMetadata)) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" is missing archived run metadata`);
|
|
986
|
+
const archivedRun = snapshotAnalystRun({
|
|
987
|
+
run_id: runId,
|
|
988
|
+
correlation_id: artifact.correlationId,
|
|
989
|
+
started_at: analysis.startedAt,
|
|
990
|
+
ended_at: analysis.endedAt,
|
|
991
|
+
findings,
|
|
992
|
+
per_analyst: attemptMetadata.perAnalyst,
|
|
993
|
+
total_cost_usd: analysis.knownCostUsd,
|
|
994
|
+
...analysis.costProvenance === void 0 ? {} : { total_cost_provenance: analysis.costProvenance },
|
|
995
|
+
...artifact.executionPlan === void 0 ? {} : {
|
|
996
|
+
execution_plan: artifact.executionPlan,
|
|
997
|
+
completion: artifact.completion
|
|
998
|
+
}
|
|
999
|
+
}, `analyst trajectory "${trajectory.id}" archived run`);
|
|
1000
|
+
const knownAnalystIds = new Set(analystIds);
|
|
1001
|
+
for (const [index, finding] of findings.entries()) if (!knownAnalystIds.has(finding.analyst_id)) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" omits generating analyst "${finding.analyst_id}" at finding ${index}`);
|
|
1002
|
+
const reviewDecisions = validateAnalystReviewDecisions({
|
|
1003
|
+
runId,
|
|
1004
|
+
runDigest: storedRunDigest,
|
|
1005
|
+
findings,
|
|
1006
|
+
analystIds,
|
|
1007
|
+
decisions: analysis.reviewDecisions,
|
|
1008
|
+
requireComplete: true
|
|
1009
|
+
});
|
|
1010
|
+
const expectedRunDigest = analystRunDigest(archivedRun);
|
|
1011
|
+
if (storedRunDigest !== expectedRunDigest) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" archived run digest mismatch`);
|
|
1012
|
+
return {
|
|
1013
|
+
runId,
|
|
1014
|
+
runDigest: expectedRunDigest,
|
|
1015
|
+
findings,
|
|
1016
|
+
findingIds,
|
|
1017
|
+
analystIds,
|
|
1018
|
+
reviewDecisions
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
1021
|
+
function completedAnalystReviewQuality(review) {
|
|
1022
|
+
const findingDecisions = review.reviewDecisions.filter((decision) => decision.verdict !== "completeness_assessed");
|
|
1023
|
+
const completeness = review.reviewDecisions.filter((decision) => decision.verdict === "completeness_assessed");
|
|
1024
|
+
if (completeness.length !== 1) throw new TypeError("feedbackTrajectoryToOptimizerRow: analyst run requires exactly one independent completeness_assessed decision");
|
|
1025
|
+
const confirmed = findingDecisions.filter((decision) => decision.verdict === "confirmed").length;
|
|
1026
|
+
const rejected = findingDecisions.length - confirmed;
|
|
1027
|
+
const emitted = review.findingIds.length;
|
|
1028
|
+
const missed = completeness[0].missedIssues.length;
|
|
1029
|
+
const precision = emitted === 0 ? 1 : confirmed / emitted;
|
|
1030
|
+
const recallDenominator = confirmed + missed;
|
|
1031
|
+
const recall = recallDenominator === 0 ? 1 : confirmed / recallDenominator;
|
|
1032
|
+
return {
|
|
1033
|
+
precision,
|
|
1034
|
+
recall,
|
|
1035
|
+
f1: precision + recall === 0 ? 0 : 2 * precision * recall / (precision + recall),
|
|
1036
|
+
counts: {
|
|
1037
|
+
emitted,
|
|
1038
|
+
confirmed,
|
|
1039
|
+
rejected,
|
|
1040
|
+
missed
|
|
1041
|
+
}
|
|
1042
|
+
};
|
|
1043
|
+
}
|
|
1044
|
+
function validateAnalystReviewDecisions(input) {
|
|
1045
|
+
if (!Array.isArray(input.decisions)) throw new TypeError("analyst review decisions must be an array");
|
|
1046
|
+
const findings = snapshotAnalystFindings(input.findings);
|
|
1047
|
+
const expectedRunDigest = requiredDigest(input.runDigest, "analyst review run digest");
|
|
1048
|
+
const findingsById = new Map(findings.map((finding) => [finding.finding_id, finding]));
|
|
1049
|
+
const generatingAnalystIds = new Set(input.analystIds);
|
|
1050
|
+
const seenFindingIds = /* @__PURE__ */ new Set();
|
|
1051
|
+
let completenessCount = 0;
|
|
1052
|
+
const decisions = input.decisions.map((value, index) => {
|
|
1053
|
+
if (!isRecord(value)) throw new TypeError(`analyst review decision ${index} must be an object`);
|
|
1054
|
+
const source = requiredString(value.source, `analyst review decision ${index} source`);
|
|
1055
|
+
if (!isAnalystReviewSource(source)) throw new TypeError(`analyst review decision ${index} source must be user, judge, environment, metric, or policy`);
|
|
1056
|
+
const reviewerId = requiredString(value.reviewerId, `analyst review decision ${index} reviewerId`);
|
|
1057
|
+
if (generatingAnalystIds.has(reviewerId)) throw new TypeError(`analyst review decision ${index} reviewerId must differ from the generating analyst`);
|
|
1058
|
+
const reviewId = requiredString(value.reviewId, `analyst review decision ${index} reviewId`);
|
|
1059
|
+
if (reviewId === input.runId) throw new TypeError(`analyst review decision ${index} reviewId must identify an independent review`);
|
|
1060
|
+
const reason = requiredString(value.reason, `analyst review decision ${index} reason`);
|
|
1061
|
+
const decidedAt = canonicalTimestamp(value.decidedAt, `analyst review decision ${index} decidedAt`);
|
|
1062
|
+
const runDigest = requiredDigest(value.runDigest, `analyst review decision ${index} runDigest`);
|
|
1063
|
+
if (runDigest !== expectedRunDigest) throw new TypeError(`analyst review decision ${index} run digest mismatch`);
|
|
1064
|
+
if (value.verdict === "completeness_assessed") {
|
|
1065
|
+
assertOnlyKeys(value, [
|
|
1066
|
+
"runDigest",
|
|
1067
|
+
"verdict",
|
|
1068
|
+
"missedIssues",
|
|
1069
|
+
"source",
|
|
1070
|
+
"reviewerId",
|
|
1071
|
+
"reviewId",
|
|
1072
|
+
"reason",
|
|
1073
|
+
"decidedAt"
|
|
1074
|
+
], `analyst review decision ${index}`);
|
|
1075
|
+
completenessCount += 1;
|
|
1076
|
+
if (completenessCount > 1) throw new TypeError("duplicate completeness_assessed analyst review decision");
|
|
1077
|
+
return {
|
|
1078
|
+
runDigest,
|
|
1079
|
+
verdict: "completeness_assessed",
|
|
1080
|
+
missedIssues: validateMissedIssues(value.missedIssues, findingsById, `analyst review decision ${index}`),
|
|
1081
|
+
source,
|
|
1082
|
+
reviewerId,
|
|
1083
|
+
reviewId,
|
|
1084
|
+
reason,
|
|
1085
|
+
decidedAt
|
|
1086
|
+
};
|
|
1087
|
+
}
|
|
1088
|
+
if (value.verdict !== "confirmed" && value.verdict !== "rejected") throw new TypeError(`analyst review decision ${index} verdict must be confirmed, rejected, or completeness_assessed`);
|
|
1089
|
+
assertOnlyKeys(value, [
|
|
1090
|
+
"runDigest",
|
|
1091
|
+
"findingId",
|
|
1092
|
+
"findingDigest",
|
|
1093
|
+
"verdict",
|
|
1094
|
+
"source",
|
|
1095
|
+
"reviewerId",
|
|
1096
|
+
"reviewId",
|
|
1097
|
+
"reason",
|
|
1098
|
+
"decidedAt"
|
|
1099
|
+
], `analyst review decision ${index}`);
|
|
1100
|
+
const findingId = requiredString(value.findingId, `analyst review decision ${index} findingId`);
|
|
1101
|
+
const finding = findingsById.get(findingId);
|
|
1102
|
+
if (!finding) throw new TypeError(`analyst review decision references unknown finding id "${findingId}"`);
|
|
1103
|
+
if (seenFindingIds.has(findingId)) throw new TypeError(`duplicate analyst review decision for finding id "${findingId}"`);
|
|
1104
|
+
seenFindingIds.add(findingId);
|
|
1105
|
+
const findingDigest = requiredString(value.findingDigest, `analyst review decision ${index} findingDigest`);
|
|
1106
|
+
const expectedDigest = analystFindingDigest(finding);
|
|
1107
|
+
if (findingDigest !== expectedDigest) throw new TypeError(`analyst review decision ${index} digest mismatch for finding id "${findingId}"`);
|
|
1108
|
+
return {
|
|
1109
|
+
runDigest,
|
|
1110
|
+
findingId,
|
|
1111
|
+
findingDigest: expectedDigest,
|
|
1112
|
+
verdict: value.verdict,
|
|
1113
|
+
source,
|
|
1114
|
+
reviewerId,
|
|
1115
|
+
reviewId,
|
|
1116
|
+
reason,
|
|
1117
|
+
decidedAt
|
|
1118
|
+
};
|
|
1119
|
+
});
|
|
1120
|
+
if (input.requireComplete) {
|
|
1121
|
+
const missing = findings.map((finding) => finding.finding_id).filter((findingId) => !seenFindingIds.has(findingId));
|
|
1122
|
+
if (missing.length > 0) throw new TypeError(`feedbackTrajectoryToOptimizerRow: missing independent decisions for finding ids: ${missing.join(", ")}`);
|
|
1123
|
+
if (completenessCount !== 1) throw new TypeError("feedbackTrajectoryToOptimizerRow: analyst run requires exactly one independent completeness_assessed decision");
|
|
1124
|
+
}
|
|
1125
|
+
return decisions;
|
|
1126
|
+
}
|
|
1127
|
+
function assertUniqueFindingIds(findingIds) {
|
|
1128
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1129
|
+
for (const findingId of findingIds) {
|
|
1130
|
+
if (findingId.trim().length === 0) throw new TypeError("analyst finding id must not be empty");
|
|
1131
|
+
if (seen.has(findingId)) throw new TypeError(`analyst run contains duplicate finding id "${findingId}"`);
|
|
1132
|
+
seen.add(findingId);
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
function snapshotAnalystFinding(value, context) {
|
|
1136
|
+
let snapshot;
|
|
1137
|
+
try {
|
|
1138
|
+
snapshot = JSON.parse(canonicalString(value));
|
|
1139
|
+
} catch (cause) {
|
|
1140
|
+
throw new TypeError(`${context} must have a canonical JSON representation`, { cause });
|
|
1141
|
+
}
|
|
1142
|
+
assertAnalystFinding(snapshot, context);
|
|
1143
|
+
return snapshot;
|
|
1144
|
+
}
|
|
1145
|
+
function assertAnalystFinding(value, context) {
|
|
1146
|
+
if (!isRecord(value)) throw new TypeError(`${context} must be an object`);
|
|
1147
|
+
assertOnlyKeys(value, [
|
|
1148
|
+
"schema_version",
|
|
1149
|
+
"finding_id",
|
|
1150
|
+
"analyst_id",
|
|
1151
|
+
"produced_at",
|
|
1152
|
+
"severity",
|
|
1153
|
+
"area",
|
|
1154
|
+
"claim",
|
|
1155
|
+
"rationale",
|
|
1156
|
+
"evidence_refs",
|
|
1157
|
+
"recommended_action",
|
|
1158
|
+
"validation_plan",
|
|
1159
|
+
"confidence",
|
|
1160
|
+
"subject",
|
|
1161
|
+
"derived_from_judge",
|
|
1162
|
+
"metadata"
|
|
1163
|
+
], context);
|
|
1164
|
+
if (value.schema_version !== "1.0.0") throw new TypeError(`${context} schema_version must be "1.0.0"`);
|
|
1165
|
+
requiredString(value.finding_id, `${context} finding_id`);
|
|
1166
|
+
requiredString(value.analyst_id, `${context} analyst_id`);
|
|
1167
|
+
canonicalTimestamp(value.produced_at, `${context} produced_at`);
|
|
1168
|
+
if (value.severity !== "critical" && value.severity !== "high" && value.severity !== "medium" && value.severity !== "low" && value.severity !== "info") throw new TypeError(`${context} severity is invalid`);
|
|
1169
|
+
requiredString(value.area, `${context} area`);
|
|
1170
|
+
requiredString(value.claim, `${context} claim`);
|
|
1171
|
+
optionalString(value.rationale, `${context} rationale`);
|
|
1172
|
+
value.evidence_refs = validateEvidenceRefs(value.evidence_refs, `${context} evidence_refs`);
|
|
1173
|
+
optionalString(value.recommended_action, `${context} recommended_action`);
|
|
1174
|
+
optionalString(value.validation_plan, `${context} validation_plan`);
|
|
1175
|
+
if (typeof value.confidence !== "number" || !Number.isFinite(value.confidence) || value.confidence < 0 || value.confidence > 1) throw new TypeError(`${context} confidence must be a finite number from 0 through 1`);
|
|
1176
|
+
optionalString(value.subject, `${context} subject`);
|
|
1177
|
+
if (value.derived_from_judge !== void 0 && typeof value.derived_from_judge !== "boolean") throw new TypeError(`${context} derived_from_judge must be a boolean`);
|
|
1178
|
+
if (value.metadata !== void 0 && !isRecord(value.metadata)) throw new TypeError(`${context} metadata must be an object`);
|
|
1179
|
+
}
|
|
1180
|
+
function validateMissedIssues(value, findingsById, context) {
|
|
1181
|
+
if (!Array.isArray(value)) throw new TypeError(`${context} missedIssues must be an array`);
|
|
1182
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1183
|
+
return value.map((issue, index) => {
|
|
1184
|
+
const issueContext = `${context} missedIssues ${index}`;
|
|
1185
|
+
if (!isRecord(issue)) throw new TypeError(`${issueContext} must be an object`);
|
|
1186
|
+
assertOnlyKeys(issue, [
|
|
1187
|
+
"id",
|
|
1188
|
+
"reason",
|
|
1189
|
+
"evidence"
|
|
1190
|
+
], issueContext);
|
|
1191
|
+
const id = requiredString(issue.id, `${issueContext} id`);
|
|
1192
|
+
if (findingsById.has(id)) throw new TypeError(`${issueContext} id "${id}" is already an emitted finding id`);
|
|
1193
|
+
if (seen.has(id)) throw new TypeError(`duplicate missed issue id "${id}"`);
|
|
1194
|
+
seen.add(id);
|
|
1195
|
+
return {
|
|
1196
|
+
id,
|
|
1197
|
+
reason: requiredString(issue.reason, `${issueContext} reason`),
|
|
1198
|
+
...issue.evidence === void 0 ? {} : { evidence: validateEvidenceRefs(issue.evidence, `${issueContext} evidence`) }
|
|
1199
|
+
};
|
|
1200
|
+
});
|
|
1201
|
+
}
|
|
1202
|
+
function validateEvidenceRefs(value, context) {
|
|
1203
|
+
if (!Array.isArray(value)) throw new TypeError(`${context} must be an array`);
|
|
1204
|
+
return value.map((evidence, index) => {
|
|
1205
|
+
const evidenceContext = `${context} ${index}`;
|
|
1206
|
+
if (!isRecord(evidence)) throw new TypeError(`${evidenceContext} must be an object`);
|
|
1207
|
+
assertOnlyKeys(evidence, [
|
|
1208
|
+
"kind",
|
|
1209
|
+
"uri",
|
|
1210
|
+
"excerpt"
|
|
1211
|
+
], evidenceContext);
|
|
1212
|
+
if (evidence.kind !== "span" && evidence.kind !== "event" && evidence.kind !== "artifact" && evidence.kind !== "finding" && evidence.kind !== "metric") throw new TypeError(`${evidenceContext} kind is invalid`);
|
|
1213
|
+
const uri = requiredString(evidence.uri, `${evidenceContext} uri`);
|
|
1214
|
+
const excerpt = evidence.excerpt;
|
|
1215
|
+
optionalString(excerpt, `${evidenceContext} excerpt`);
|
|
1216
|
+
return {
|
|
1217
|
+
kind: evidence.kind,
|
|
1218
|
+
uri,
|
|
1219
|
+
...excerpt === void 0 ? {} : { excerpt }
|
|
1220
|
+
};
|
|
1221
|
+
});
|
|
1222
|
+
}
|
|
1223
|
+
function assertOnlyKeys(value, allowed, name) {
|
|
1224
|
+
const allowedKeys = new Set(allowed);
|
|
1225
|
+
const unexpected = Object.keys(value).filter((key) => !allowedKeys.has(key));
|
|
1226
|
+
if (unexpected.length > 0) throw new TypeError(`${name} contains unknown fields: ${unexpected.sort().join(", ")}`);
|
|
1227
|
+
}
|
|
1228
|
+
function stringArray(value, name) {
|
|
1229
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) throw new TypeError(`${name} must be an array of strings`);
|
|
1230
|
+
const strings = value.map((item) => requiredString(item, name));
|
|
1231
|
+
if (new Set(strings).size !== strings.length) throw new TypeError(`${name} must contain unique values`);
|
|
1232
|
+
return strings;
|
|
1233
|
+
}
|
|
1234
|
+
function requiredString(value, name) {
|
|
1235
|
+
if (typeof value !== "string" || value.trim().length === 0) throw new TypeError(`${name} must be a non-empty string`);
|
|
1236
|
+
return value;
|
|
1237
|
+
}
|
|
1238
|
+
function requiredDigest(value, name) {
|
|
1239
|
+
const digest = requiredString(value, name);
|
|
1240
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(digest)) throw new TypeError(`${name} must be a sha256 digest`);
|
|
1241
|
+
return digest;
|
|
1242
|
+
}
|
|
1243
|
+
function optionalString(value, name) {
|
|
1244
|
+
if (value !== void 0 && typeof value !== "string") throw new TypeError(`${name} must be a string`);
|
|
1245
|
+
}
|
|
1246
|
+
function canonicalTimestamp(value, name) {
|
|
1247
|
+
const timestamp = requiredString(value, name);
|
|
1248
|
+
const parsed = new Date(timestamp);
|
|
1249
|
+
if (Number.isNaN(parsed.valueOf()) || parsed.toISOString() !== timestamp) throw new TypeError(`${name} must be a canonical ISO 8601 UTC timestamp`);
|
|
1250
|
+
return timestamp;
|
|
1251
|
+
}
|
|
1252
|
+
function isAnalystReviewSource(value) {
|
|
1253
|
+
return value === "user" || value === "judge" || value === "environment" || value === "metric" || value === "policy";
|
|
1254
|
+
}
|
|
1255
|
+
function isRecord(value) {
|
|
1256
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1257
|
+
}
|
|
1258
|
+
//#endregion
|
|
1259
|
+
//#region src/analyst/registry.ts
|
|
1260
|
+
/**
|
|
1261
|
+
* AnalystRegistry — orchestrate N analysts against one run.
|
|
1262
|
+
*
|
|
1263
|
+
* Owns three responsibilities and only three:
|
|
1264
|
+
* 1. Registration — ids must be unique; bad registrations fail loudly
|
|
1265
|
+
* at register-time, not run-time.
|
|
1266
|
+
* 2. Routing — each analyst declares its `inputKind`; the registry
|
|
1267
|
+
* picks the matching field from AnalystRunInputs and skips the
|
|
1268
|
+
* analyst with a logged reason if it's missing.
|
|
1269
|
+
* 3. Isolation — one analyst's exception MUST NOT stop other analysts.
|
|
1270
|
+
* Failed analysts produce zero findings + a 'failed' summary row.
|
|
1271
|
+
*
|
|
1272
|
+
* Cross-cutting concerns (telemetry, error → finding conversion, cost
|
|
1273
|
+
* ingestion, storage rotation) live in `AnalystHooks`. Budget shaping
|
|
1274
|
+
* (equal split vs weighted vs custom) lives in `BudgetPolicy`. Both
|
|
1275
|
+
* have sensible defaults; consumers override only what they need.
|
|
1276
|
+
*/
|
|
1277
|
+
/** A post-start exact-run failure; completed work remains attached for accounting and review. */
|
|
1278
|
+
var ExactAnalystRunExecutionError = class extends Error {
|
|
1279
|
+
name = "ExactAnalystRunExecutionError";
|
|
1280
|
+
result;
|
|
1281
|
+
constructor(message, result, options) {
|
|
1282
|
+
super(message, options);
|
|
1283
|
+
const snapshot = snapshotExactAnalystRunReceipt(result, "ExactAnalystRunExecutionError result");
|
|
1284
|
+
if (snapshot.completion.status !== "failed") throw new TypeError("ExactAnalystRunExecutionError result must be a failed receipt");
|
|
1285
|
+
this.result = snapshot;
|
|
1286
|
+
}
|
|
1287
|
+
};
|
|
1288
|
+
var AnalystRegistry = class {
|
|
1289
|
+
analysts = /* @__PURE__ */ new Map();
|
|
1290
|
+
options;
|
|
1291
|
+
constructor(options = {}) {
|
|
1292
|
+
this.options = options;
|
|
1293
|
+
}
|
|
1294
|
+
register(analyst) {
|
|
1295
|
+
const id = analyst.id;
|
|
1296
|
+
const version = analyst.version;
|
|
1297
|
+
const cost = analyst.cost;
|
|
1298
|
+
if (!id) throw new Error("AnalystRegistry.register: analyst.id is required");
|
|
1299
|
+
if (this.analysts.has(id)) throw new Error(`AnalystRegistry.register: duplicate analyst id "${id}"`);
|
|
1300
|
+
if (!version) throw new Error(`AnalystRegistry.register: analyst "${id}" must declare a version`);
|
|
1301
|
+
if (cost.kind === "deterministic" && cost.settlement_timeout_ms !== void 0) throw new TypeError(`AnalystRegistry.register: deterministic analyst "${id}" cannot declare settlement_timeout_ms`);
|
|
1302
|
+
if (cost.settlement_timeout_ms !== void 0) validateUsageSettlementTimeout(cost.settlement_timeout_ms);
|
|
1303
|
+
this.analysts.set(id, analyst);
|
|
1304
|
+
}
|
|
1305
|
+
list() {
|
|
1306
|
+
return Array.from(this.analysts.values()).map((a) => ({
|
|
1307
|
+
id: a.id,
|
|
1308
|
+
description: a.description,
|
|
1309
|
+
version: a.version,
|
|
1310
|
+
cost: a.cost
|
|
1311
|
+
}));
|
|
1312
|
+
}
|
|
1313
|
+
async run(runId, inputs, runOpts = {}) {
|
|
1314
|
+
for await (const ev of this.runStream(runId, inputs, runOpts)) if (ev.type === "run-completed") return ev.result;
|
|
1315
|
+
throw new Error("AnalystRegistry.run: stream completed without run-completed event");
|
|
1316
|
+
}
|
|
1317
|
+
/** Run exactly the ordered analysts and complete policy supplied by the caller. */
|
|
1318
|
+
async runExact(runId, inputs, runOpts) {
|
|
1319
|
+
for await (const ev of this.runExactStream(runId, inputs, runOpts)) if (ev.type === "run-completed") return ev.result;
|
|
1320
|
+
throw new Error("AnalystRegistry.runExact: stream completed without run-completed event");
|
|
1321
|
+
}
|
|
1322
|
+
/** Streaming counterpart to {@link runExact}. */
|
|
1323
|
+
async *runExactStream(runId, inputs, runOpts) {
|
|
1324
|
+
for await (const event of this.executePlanStream(this.normalizeExactPlan(runId, inputs, runOpts))) yield event;
|
|
1325
|
+
}
|
|
1326
|
+
/**
|
|
1327
|
+
* Streaming counterpart to `run()`. Emits `AnalystRunEvent` values
|
|
1328
|
+
* in real time — `run-started`, then per-analyst `skipped` /
|
|
1329
|
+
* `started` / `completed`, then a terminal `run-completed` whose
|
|
1330
|
+
* payload is the full `AnalystRunResult`. UIs use this to render
|
|
1331
|
+
* progress; persistence consumers use `run()` and read the result.
|
|
1332
|
+
*
|
|
1333
|
+
* Hooks (`onBeforeAnalyze` / `onAfterAnalyze` / `onError` /
|
|
1334
|
+
* `onComplete`) fire as before — streaming is additive, not a hook
|
|
1335
|
+
* replacement.
|
|
1336
|
+
*/
|
|
1337
|
+
async *runStream(runId, inputs, runOpts = {}) {
|
|
1338
|
+
yield* this.executePlanStream(this.normalizeLegacyPlan(runId, inputs, runOpts));
|
|
1339
|
+
}
|
|
1340
|
+
normalizeLegacyPlan(runId, inputs, runOpts) {
|
|
1341
|
+
const timeoutMs = validateTimeout(runOpts.timeoutMs) ?? null;
|
|
1342
|
+
const budget = runOpts.budget ?? this.options.defaultBudget;
|
|
1343
|
+
validateBudgetPolicy(budget);
|
|
1344
|
+
return {
|
|
1345
|
+
runId,
|
|
1346
|
+
prepared: this.selectAnalysts(runOpts).map((analyst) => ({
|
|
1347
|
+
analyst,
|
|
1348
|
+
input: this.routeInput(analyst, inputs)
|
|
1349
|
+
})),
|
|
1350
|
+
budget: budget ? {
|
|
1351
|
+
kind: "dynamic",
|
|
1352
|
+
policy: budget
|
|
1353
|
+
} : { kind: "none" },
|
|
1354
|
+
totalTimeoutMs: timeoutMs,
|
|
1355
|
+
signal: runOpts.signal ?? null,
|
|
1356
|
+
costLedger: runOpts.costLedger ?? null,
|
|
1357
|
+
costPhase: runOpts.costPhase ?? null,
|
|
1358
|
+
tags: runOpts.tags ?? null,
|
|
1359
|
+
priorFindings: runOpts.priorFindings ?? null,
|
|
1360
|
+
chainFindings: runOpts.chainFindings ?? false,
|
|
1361
|
+
hooks: this.options.hooks ?? {},
|
|
1362
|
+
chat: this.options.chat,
|
|
1363
|
+
log: this.options.log ?? (() => {}),
|
|
1364
|
+
executionSnapshot: void 0
|
|
1365
|
+
};
|
|
1366
|
+
}
|
|
1367
|
+
normalizeExactPlan(runId, inputs, runOpts) {
|
|
1368
|
+
const exactRunId = snapshotExactRunId(runId);
|
|
1369
|
+
const exact = snapshotExactRegistryRunOpts(runOpts);
|
|
1370
|
+
const selected = normalizeExactAnalysts(this.selectExactAnalysts(exact.analystIds));
|
|
1371
|
+
const registryChat = this.options.chat;
|
|
1372
|
+
const registryChatIdentity = this.options.chatIdentity;
|
|
1373
|
+
const registryHooks = this.options.hooks;
|
|
1374
|
+
const registryHooksIdentity = this.options.hooksIdentity;
|
|
1375
|
+
if (exact.useRegistryChat && registryChat === void 0) throw new TypeError("ExactRegistryRunOpts.useRegistryChat is true but the registry has no chat client");
|
|
1376
|
+
if (exact.applyRegistryHooks && !hasRegistryHooks(registryHooks)) throw new TypeError("ExactRegistryRunOpts.applyRegistryHooks is true but the registry has no lifecycle hooks");
|
|
1377
|
+
const inputSnapshot = snapshotAnalystRunInputChannels(inputs);
|
|
1378
|
+
const prepared = selected.map((analyst) => ({
|
|
1379
|
+
analyst,
|
|
1380
|
+
input: this.routeInput(analyst, inputSnapshot)
|
|
1381
|
+
}));
|
|
1382
|
+
if (exact.missingInputMode === "abort") {
|
|
1383
|
+
const missing = prepared.find((candidate) => candidate.input.kind === "missing")?.analyst;
|
|
1384
|
+
if (missing) throw new TypeError(`ExactRegistryRunOpts.missingInputMode abort preflight found no "${missing.inputKind}" input for "${missing.id}"`);
|
|
1385
|
+
}
|
|
1386
|
+
const hooksIdentity = exact.applyRegistryHooks && registryHooks ? requireExactComponentIdentity(registryHooksIdentity, "registry hooks") : null;
|
|
1387
|
+
const chatIdentity = exact.useRegistryChat ? requireExactComponentIdentity(registryChatIdentity, "registry chat") : null;
|
|
1388
|
+
const costLedgerIdentity = exact.costLedger === null ? null : requireExactComponentIdentity(exact.costLedgerIdentity ?? void 0, "cost ledger");
|
|
1389
|
+
const executionSnapshot = exactExecutionSnapshot(selected, exact, exactFixedBudgets(exact.budget, prepared.filter((candidate) => candidate.input.kind === "present").map((candidate) => candidate.analyst), selected), costLedgerIdentity, hooksIdentity, chatIdentity);
|
|
1390
|
+
return {
|
|
1391
|
+
runId: exactRunId,
|
|
1392
|
+
prepared,
|
|
1393
|
+
budget: { kind: "none" },
|
|
1394
|
+
totalTimeoutMs: exact.totalTimeoutMs,
|
|
1395
|
+
signal: exact.signal,
|
|
1396
|
+
costLedger: exact.costLedger,
|
|
1397
|
+
costPhase: exact.costPhase,
|
|
1398
|
+
tags: exact.tags,
|
|
1399
|
+
priorFindings: exact.priorFindings,
|
|
1400
|
+
chainFindings: exact.chainFindings,
|
|
1401
|
+
hooks: exact.applyRegistryHooks && registryHooks ? snapshotHooks(registryHooks) : {},
|
|
1402
|
+
chat: exact.useRegistryChat && registryChat ? snapshotChat(registryChat) : void 0,
|
|
1403
|
+
log: () => {},
|
|
1404
|
+
executionSnapshot
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
async *executePlanStream(plan) {
|
|
1408
|
+
const exact = plan.executionSnapshot !== void 0;
|
|
1409
|
+
if (exact && plan.signal?.aborted) throw abortReason(plan.signal);
|
|
1410
|
+
const correlationId = `ar_${randomUUID().slice(0, 12)}`;
|
|
1411
|
+
const log = plan.log;
|
|
1412
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1413
|
+
const started = Date.now();
|
|
1414
|
+
const timeoutSignal = plan.totalTimeoutMs === null ? void 0 : AbortSignal.timeout(plan.totalTimeoutMs);
|
|
1415
|
+
const runSignal = combineAbortSignals(plan.signal ?? void 0, timeoutSignal);
|
|
1416
|
+
const deadlineMs = plan.totalTimeoutMs === null ? void 0 : started + plan.totalTimeoutMs;
|
|
1417
|
+
const runnable = plan.prepared.filter((candidate) => candidate.input.kind === "present").map((candidate) => candidate.analyst);
|
|
1418
|
+
let remainingUsd = plan.budget.kind === "dynamic" ? plan.budget.policy.totalUsd : void 0;
|
|
1419
|
+
const weights = plan.budget.kind === "dynamic" ? plan.budget.policy.weights : void 0;
|
|
1420
|
+
const totalWeight = weights && plan.budget.kind === "dynamic" && plan.budget.policy.totalUsd != null && !plan.budget.policy.allocate && runnable.length > 0 ? runnable.reduce((sum, analyst) => sum + analystWeight(weights, analyst.id), 0) : void 0;
|
|
1421
|
+
if (totalWeight === 0) throw new Error("BudgetPolicy.weights must allocate positive weight to a runnable analyst");
|
|
1422
|
+
const upstreamFindings = [];
|
|
1423
|
+
yield snapshotExecutionEvent({
|
|
1424
|
+
type: "run-started",
|
|
1425
|
+
run_id: plan.runId,
|
|
1426
|
+
correlation_id: correlationId,
|
|
1427
|
+
started_at: startedAt,
|
|
1428
|
+
analyst_ids: plan.prepared.map(({ analyst }) => analyst.id),
|
|
1429
|
+
...plan.executionSnapshot === void 0 ? {} : { execution_plan: plan.executionSnapshot }
|
|
1430
|
+
}, exact);
|
|
1431
|
+
const executions = [];
|
|
1432
|
+
let executionFailure;
|
|
1433
|
+
for (const { analyst, input } of plan.prepared) {
|
|
1434
|
+
const t0 = Date.now();
|
|
1435
|
+
if (runSignal?.aborted) {
|
|
1436
|
+
const summary = abortedBeforeStartSummary(analyst, runSignal);
|
|
1437
|
+
executions.push({
|
|
1438
|
+
summary,
|
|
1439
|
+
findings: [],
|
|
1440
|
+
budgetDebitUsd: 0
|
|
1441
|
+
});
|
|
1442
|
+
log(`[analyst] skip ${analyst.id} — run aborted`, {
|
|
1443
|
+
runId: plan.runId,
|
|
1444
|
+
reason: summary.reason
|
|
1445
|
+
});
|
|
1446
|
+
yield snapshotExecutionEvent({
|
|
1447
|
+
type: "analyst-skipped",
|
|
1448
|
+
summary
|
|
1449
|
+
}, exact);
|
|
1450
|
+
if (exact) {
|
|
1451
|
+
executionFailure = abortReason(runSignal);
|
|
1452
|
+
break;
|
|
1453
|
+
}
|
|
1454
|
+
continue;
|
|
1455
|
+
}
|
|
1456
|
+
if (input.kind === "missing") {
|
|
1457
|
+
const summary = {
|
|
1458
|
+
analyst_id: analyst.id,
|
|
1459
|
+
status: "skipped",
|
|
1460
|
+
reason: `missing input of kind '${analyst.inputKind}'`,
|
|
1461
|
+
findings_count: 0,
|
|
1462
|
+
latency_ms: 0,
|
|
1463
|
+
usage: zeroUsage()
|
|
1464
|
+
};
|
|
1465
|
+
const execution = {
|
|
1466
|
+
summary,
|
|
1467
|
+
findings: [],
|
|
1468
|
+
budgetDebitUsd: 0
|
|
1469
|
+
};
|
|
1470
|
+
executions.push(execution);
|
|
1471
|
+
log(`[analyst] skip ${analyst.id} — missing input`, {
|
|
1472
|
+
runId: plan.runId,
|
|
1473
|
+
kind: analyst.inputKind
|
|
1474
|
+
});
|
|
1475
|
+
const hookValues = snapshotAfterHookValues(summary, [], exact);
|
|
1476
|
+
try {
|
|
1477
|
+
await waitForHook(plan.hooks.onAfterAnalyze ? () => plan.hooks.onAfterAnalyze?.({
|
|
1478
|
+
analyst,
|
|
1479
|
+
summary: hookValues.summary,
|
|
1480
|
+
findings: hookValues.findings,
|
|
1481
|
+
runId: plan.runId
|
|
1482
|
+
}) : void 0, runSignal);
|
|
1483
|
+
} catch (error) {
|
|
1484
|
+
if (!exact) throw error;
|
|
1485
|
+
executionFailure = error;
|
|
1486
|
+
}
|
|
1487
|
+
yield snapshotExecutionEvent({
|
|
1488
|
+
type: "analyst-skipped",
|
|
1489
|
+
summary
|
|
1490
|
+
}, exact);
|
|
1491
|
+
if (executionFailure !== void 0) break;
|
|
1492
|
+
continue;
|
|
1493
|
+
}
|
|
1494
|
+
const allocatedUsd = plan.executionSnapshot === void 0 ? allocateBudget(plan.budget.kind === "dynamic" ? plan.budget.policy : void 0, {
|
|
1495
|
+
analyst,
|
|
1496
|
+
remainingUsd,
|
|
1497
|
+
runningCount: runnable.length,
|
|
1498
|
+
totalWeight
|
|
1499
|
+
}) : exactPlannedAllocation(plan.executionSnapshot, analyst.id);
|
|
1500
|
+
const budgetCeilingUsd = plan.executionSnapshot === void 0 ? remainingUsd : allocatedUsd;
|
|
1501
|
+
const usageReceipts = [];
|
|
1502
|
+
const contextTags = plan.tags === null ? void 0 : { ...plan.tags };
|
|
1503
|
+
const priorFindings = selectPriorFindings(plan.priorFindings ?? void 0, analyst.id);
|
|
1504
|
+
const chainedFindings = plan.chainFindings && upstreamFindings.length > 0 ? [...upstreamFindings] : void 0;
|
|
1505
|
+
const ctx = {
|
|
1506
|
+
runId: plan.runId,
|
|
1507
|
+
correlationId,
|
|
1508
|
+
deadlineMs,
|
|
1509
|
+
budgetUsd: allocatedUsd,
|
|
1510
|
+
costLedger: plan.costLedger ?? void 0,
|
|
1511
|
+
costPhase: plan.costPhase ?? void 0,
|
|
1512
|
+
chat: plan.chat,
|
|
1513
|
+
tags: contextTags,
|
|
1514
|
+
log: (message, fields) => log(`[${analyst.id}] ${message}`, {
|
|
1515
|
+
runId: plan.runId,
|
|
1516
|
+
correlationId,
|
|
1517
|
+
...fields
|
|
1518
|
+
}),
|
|
1519
|
+
signal: runSignal,
|
|
1520
|
+
priorFindings,
|
|
1521
|
+
upstreamFindings: chainedFindings,
|
|
1522
|
+
recordUsage: (receipt) => {
|
|
1523
|
+
if (!exact) {
|
|
1524
|
+
assertValidAnalystUsageReceipt(receipt);
|
|
1525
|
+
usageReceipts.push(receipt);
|
|
1526
|
+
return;
|
|
1527
|
+
}
|
|
1528
|
+
usageReceipts.push(snapshotUsageReceiptOnce(receipt, `AnalystRegistry.runExact analyst "${analyst.id}" usage`));
|
|
1529
|
+
}
|
|
1530
|
+
};
|
|
1531
|
+
if (exact) {
|
|
1532
|
+
if (contextTags) deepFreezeCanonicalJson(contextTags);
|
|
1533
|
+
if (priorFindings) deepFreezeCanonicalJson(priorFindings);
|
|
1534
|
+
if (chainedFindings) deepFreezeCanonicalJson(chainedFindings);
|
|
1535
|
+
Object.freeze(ctx);
|
|
1536
|
+
}
|
|
1537
|
+
try {
|
|
1538
|
+
await waitForHook(plan.hooks.onBeforeAnalyze ? () => plan.hooks.onBeforeAnalyze?.({
|
|
1539
|
+
analyst,
|
|
1540
|
+
ctx,
|
|
1541
|
+
runId: plan.runId
|
|
1542
|
+
}) : void 0, runSignal);
|
|
1543
|
+
} catch (error) {
|
|
1544
|
+
if (!exact) throw error;
|
|
1545
|
+
executionFailure = error;
|
|
1546
|
+
break;
|
|
1547
|
+
}
|
|
1548
|
+
if (runSignal?.aborted) {
|
|
1549
|
+
const summary = abortedBeforeStartSummary(analyst, runSignal, Date.now() - t0);
|
|
1550
|
+
executions.push({
|
|
1551
|
+
summary,
|
|
1552
|
+
findings: [],
|
|
1553
|
+
budgetDebitUsd: 0
|
|
1554
|
+
});
|
|
1555
|
+
yield snapshotExecutionEvent({
|
|
1556
|
+
type: "analyst-skipped",
|
|
1557
|
+
summary
|
|
1558
|
+
}, exact);
|
|
1559
|
+
if (exact) {
|
|
1560
|
+
executionFailure = abortReason(runSignal);
|
|
1561
|
+
break;
|
|
1562
|
+
}
|
|
1563
|
+
continue;
|
|
1564
|
+
}
|
|
1565
|
+
let effectiveBudget;
|
|
1566
|
+
try {
|
|
1567
|
+
effectiveBudget = validateEffectiveBudget(ctx.budgetUsd, budgetCeilingUsd, analyst.id);
|
|
1568
|
+
} catch (error) {
|
|
1569
|
+
if (!exact) throw error;
|
|
1570
|
+
executionFailure = error;
|
|
1571
|
+
break;
|
|
1572
|
+
}
|
|
1573
|
+
const analystContext = exact ? ctx : { ...ctx };
|
|
1574
|
+
const executionSignal = exact ? ctx.signal : runSignal;
|
|
1575
|
+
yield snapshotExecutionEvent({
|
|
1576
|
+
type: "analyst-started",
|
|
1577
|
+
analyst_id: analyst.id,
|
|
1578
|
+
started_at: new Date(t0).toISOString()
|
|
1579
|
+
}, exact);
|
|
1580
|
+
let findings;
|
|
1581
|
+
let summary;
|
|
1582
|
+
let lifecycleFailure;
|
|
1583
|
+
let analysisFailure;
|
|
1584
|
+
try {
|
|
1585
|
+
if (runSignal?.aborted) throw abortReason(runSignal);
|
|
1586
|
+
findings = snapshotExecutionFindings(await waitForOperation(analyst.analyze(input.value, analystContext), executionSignal, analystAbortGraceMs(analyst)), exact, `AnalystRegistry.runExact analyst "${analyst.id}" findings`);
|
|
1587
|
+
} catch (error) {
|
|
1588
|
+
const cause = error instanceof Error ? error : new Error(String(error));
|
|
1589
|
+
analysisFailure = cause;
|
|
1590
|
+
let hookFindings = [];
|
|
1591
|
+
if (!executionSignal?.aborted) try {
|
|
1592
|
+
hookFindings = snapshotExecutionFindings(await waitForHook(plan.hooks.onError ? () => plan.hooks.onError?.({
|
|
1593
|
+
analyst,
|
|
1594
|
+
error: cause,
|
|
1595
|
+
runId: plan.runId
|
|
1596
|
+
}) : void 0, executionSignal) ?? [], exact, `AnalystRegistry.runExact analyst "${analyst.id}" onError findings`);
|
|
1597
|
+
} catch (error) {
|
|
1598
|
+
lifecycleFailure = error;
|
|
1599
|
+
}
|
|
1600
|
+
findings = hookFindings;
|
|
1601
|
+
}
|
|
1602
|
+
let usage;
|
|
1603
|
+
try {
|
|
1604
|
+
usage = resolveUsage(analyst, usageReceipts, exact);
|
|
1605
|
+
} catch (error) {
|
|
1606
|
+
if (!exact) throw error;
|
|
1607
|
+
executionFailure = error;
|
|
1608
|
+
break;
|
|
1609
|
+
}
|
|
1610
|
+
if (analysisFailure === void 0) {
|
|
1611
|
+
summary = {
|
|
1612
|
+
analyst_id: analyst.id,
|
|
1613
|
+
status: "ok",
|
|
1614
|
+
findings_count: findings.length,
|
|
1615
|
+
latency_ms: Date.now() - t0,
|
|
1616
|
+
usage,
|
|
1617
|
+
...exact ? { allocated_budget_usd: effectiveBudget ?? null } : {}
|
|
1618
|
+
};
|
|
1619
|
+
log(`[analyst] ok ${analyst.id}`, {
|
|
1620
|
+
runId: plan.runId,
|
|
1621
|
+
findings: findings.length,
|
|
1622
|
+
latency_ms: summary.latency_ms,
|
|
1623
|
+
cost_usd: knownCostUsd(usage),
|
|
1624
|
+
cost_kind: usage.cost.kind,
|
|
1625
|
+
input_tokens: usage.tokens?.input ?? null,
|
|
1626
|
+
output_tokens: usage.tokens?.output ?? null
|
|
1627
|
+
});
|
|
1628
|
+
} else {
|
|
1629
|
+
const errorClass = analysisFailure.constructor.name || "Error";
|
|
1630
|
+
const errorMessage = exact && analysisFailure.message.length === 0 ? "Analyst failed without an error message" : analysisFailure.message;
|
|
1631
|
+
summary = {
|
|
1632
|
+
analyst_id: analyst.id,
|
|
1633
|
+
status: "failed",
|
|
1634
|
+
findings_count: findings.length,
|
|
1635
|
+
latency_ms: Date.now() - t0,
|
|
1636
|
+
usage,
|
|
1637
|
+
...exact ? { allocated_budget_usd: effectiveBudget ?? null } : {},
|
|
1638
|
+
error: {
|
|
1639
|
+
class: errorClass,
|
|
1640
|
+
message: errorMessage
|
|
1641
|
+
}
|
|
1642
|
+
};
|
|
1643
|
+
log(`[analyst] FAIL ${analyst.id}`, {
|
|
1644
|
+
runId: plan.runId,
|
|
1645
|
+
error_class: errorClass,
|
|
1646
|
+
error: errorMessage,
|
|
1647
|
+
cost_usd: knownCostUsd(usage),
|
|
1648
|
+
cost_kind: usage.cost.kind
|
|
1649
|
+
});
|
|
1650
|
+
}
|
|
1651
|
+
logUncapturedBudgetWarning({
|
|
1652
|
+
analyst,
|
|
1653
|
+
runId: plan.runId,
|
|
1654
|
+
budgetUsd: effectiveBudget,
|
|
1655
|
+
usage,
|
|
1656
|
+
log
|
|
1657
|
+
});
|
|
1658
|
+
const execution = {
|
|
1659
|
+
summary,
|
|
1660
|
+
findings,
|
|
1661
|
+
budgetDebitUsd: budgetDebit(summary.usage, effectiveBudget)
|
|
1662
|
+
};
|
|
1663
|
+
if (exact) try {
|
|
1664
|
+
executionCost([...executions, execution], true);
|
|
1665
|
+
} catch (error) {
|
|
1666
|
+
executionFailure = error;
|
|
1667
|
+
break;
|
|
1668
|
+
}
|
|
1669
|
+
executions.push(execution);
|
|
1670
|
+
if (plan.budget.kind === "dynamic" && remainingUsd !== void 0) remainingUsd = Math.max(0, remainingUsd - execution.budgetDebitUsd);
|
|
1671
|
+
if (plan.chainFindings) upstreamFindings.push(...findings);
|
|
1672
|
+
if (lifecycleFailure !== void 0) {
|
|
1673
|
+
if (!exact) throw lifecycleFailure;
|
|
1674
|
+
executionFailure = lifecycleFailure;
|
|
1675
|
+
break;
|
|
1676
|
+
}
|
|
1677
|
+
const hookValues = snapshotAfterHookValues(summary, findings, exact);
|
|
1678
|
+
try {
|
|
1679
|
+
await waitForHook(plan.hooks.onAfterAnalyze ? () => plan.hooks.onAfterAnalyze?.({
|
|
1680
|
+
analyst,
|
|
1681
|
+
summary: hookValues.summary,
|
|
1682
|
+
findings: hookValues.findings,
|
|
1683
|
+
runId: plan.runId
|
|
1684
|
+
}) : void 0, executionSignal);
|
|
1685
|
+
} catch (error) {
|
|
1686
|
+
if (!exact) throw error;
|
|
1687
|
+
executionFailure = error;
|
|
1688
|
+
break;
|
|
1689
|
+
}
|
|
1690
|
+
yield snapshotExecutionEvent({
|
|
1691
|
+
type: "analyst-completed",
|
|
1692
|
+
summary,
|
|
1693
|
+
findings
|
|
1694
|
+
}, exact);
|
|
1695
|
+
if (exact && runSignal?.aborted) {
|
|
1696
|
+
executionFailure = abortReason(runSignal);
|
|
1697
|
+
break;
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
const summaries = executions.map(({ summary }) => summary);
|
|
1701
|
+
const findings = executions.flatMap((execution) => execution.findings);
|
|
1702
|
+
const cost = executionCost(executions, exact);
|
|
1703
|
+
const baseResult = {
|
|
1704
|
+
run_id: plan.runId,
|
|
1705
|
+
correlation_id: correlationId,
|
|
1706
|
+
started_at: startedAt,
|
|
1707
|
+
ended_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1708
|
+
findings,
|
|
1709
|
+
per_analyst: summaries,
|
|
1710
|
+
total_cost_usd: cost.known,
|
|
1711
|
+
total_cost_provenance: cost.provenance
|
|
1712
|
+
};
|
|
1713
|
+
if (plan.executionSnapshot === void 0) {
|
|
1714
|
+
await waitForHook(plan.hooks.onComplete ? () => plan.hooks.onComplete?.({ result: baseResult }) : void 0, runSignal);
|
|
1715
|
+
yield {
|
|
1716
|
+
type: "run-completed",
|
|
1717
|
+
result: baseResult
|
|
1718
|
+
};
|
|
1719
|
+
return;
|
|
1720
|
+
}
|
|
1721
|
+
let completeResult;
|
|
1722
|
+
if (executionFailure === void 0) try {
|
|
1723
|
+
completeResult = snapshotExactAnalystRunReceipt({
|
|
1724
|
+
...baseResult,
|
|
1725
|
+
execution_plan: plan.executionSnapshot,
|
|
1726
|
+
completion: { status: "complete" }
|
|
1727
|
+
}, "AnalystRegistry.runExact result");
|
|
1728
|
+
await waitForHook(plan.hooks.onComplete ? () => plan.hooks.onComplete?.({ result: completeResult }) : void 0, runSignal);
|
|
1729
|
+
} catch (error) {
|
|
1730
|
+
executionFailure = error;
|
|
1731
|
+
}
|
|
1732
|
+
if (runSignal?.aborted) executionFailure ??= abortReason(runSignal);
|
|
1733
|
+
if (executionFailure === void 0 && completeResult) {
|
|
1734
|
+
yield snapshotExecutionEvent({
|
|
1735
|
+
type: "run-completed",
|
|
1736
|
+
result: completeResult
|
|
1737
|
+
}, true);
|
|
1738
|
+
return;
|
|
1739
|
+
}
|
|
1740
|
+
const cause = executionFailure instanceof Error ? executionFailure : new Error(String(executionFailure));
|
|
1741
|
+
const errorClass = cause.constructor.name || "Error";
|
|
1742
|
+
const errorMessage = cause.message.trim().length === 0 ? "Exact analyst run failed without a message" : cause.message;
|
|
1743
|
+
throw new ExactAnalystRunExecutionError(`exact analyst run failed after starting: ${errorMessage}; partial result is attached`, {
|
|
1744
|
+
...baseResult,
|
|
1745
|
+
execution_plan: plan.executionSnapshot,
|
|
1746
|
+
completion: {
|
|
1747
|
+
status: "failed",
|
|
1748
|
+
error: {
|
|
1749
|
+
class: errorClass,
|
|
1750
|
+
message: errorMessage
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
}, { cause });
|
|
1754
|
+
}
|
|
1755
|
+
selectAnalysts(opts) {
|
|
1756
|
+
let candidates = Array.from(this.analysts.values());
|
|
1757
|
+
if (opts.only?.length) {
|
|
1758
|
+
const only = new Set(opts.only);
|
|
1759
|
+
candidates = candidates.filter((a) => only.has(a.id));
|
|
1760
|
+
}
|
|
1761
|
+
if (opts.skip?.length) {
|
|
1762
|
+
const skip = new Set(opts.skip);
|
|
1763
|
+
candidates = candidates.filter((a) => !skip.has(a.id));
|
|
1764
|
+
}
|
|
1765
|
+
return candidates;
|
|
1766
|
+
}
|
|
1767
|
+
selectExactAnalysts(ids) {
|
|
1768
|
+
return ids.map((id) => {
|
|
1769
|
+
const analyst = this.analysts.get(id);
|
|
1770
|
+
if (!analyst) throw new Error(`ExactRegistryRunOpts.analystIds names unknown analyst "${id}"`);
|
|
1771
|
+
return {
|
|
1772
|
+
registeredId: id,
|
|
1773
|
+
analyst
|
|
1774
|
+
};
|
|
1775
|
+
});
|
|
1776
|
+
}
|
|
1777
|
+
routeInput(analyst, inputs) {
|
|
1778
|
+
switch (analyst.inputKind) {
|
|
1779
|
+
case "trace-store": {
|
|
1780
|
+
const value = inputs.traceStore;
|
|
1781
|
+
return value ? {
|
|
1782
|
+
kind: "present",
|
|
1783
|
+
value
|
|
1784
|
+
} : { kind: "missing" };
|
|
1785
|
+
}
|
|
1786
|
+
case "artifact-dir": {
|
|
1787
|
+
const value = inputs.artifactDir;
|
|
1788
|
+
return value ? {
|
|
1789
|
+
kind: "present",
|
|
1790
|
+
value
|
|
1791
|
+
} : { kind: "missing" };
|
|
1792
|
+
}
|
|
1793
|
+
case "run-record": {
|
|
1794
|
+
const value = inputs.runRecord;
|
|
1795
|
+
return value ? {
|
|
1796
|
+
kind: "present",
|
|
1797
|
+
value
|
|
1798
|
+
} : { kind: "missing" };
|
|
1799
|
+
}
|
|
1800
|
+
case "judge-input": {
|
|
1801
|
+
const value = inputs.judgeInput;
|
|
1802
|
+
return value ? {
|
|
1803
|
+
kind: "present",
|
|
1804
|
+
value
|
|
1805
|
+
} : { kind: "missing" };
|
|
1806
|
+
}
|
|
1807
|
+
case "custom": {
|
|
1808
|
+
const value = inputs.custom?.[analyst.id];
|
|
1809
|
+
return value !== void 0 ? {
|
|
1810
|
+
kind: "present",
|
|
1811
|
+
value
|
|
1812
|
+
} : { kind: "missing" };
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
};
|
|
1817
|
+
const exactRunFields = [
|
|
1818
|
+
"analystIds",
|
|
1819
|
+
"budget",
|
|
1820
|
+
"totalTimeoutMs",
|
|
1821
|
+
"signal",
|
|
1822
|
+
"costLedger",
|
|
1823
|
+
"costLedgerIdentity",
|
|
1824
|
+
"costPhase",
|
|
1825
|
+
"tags",
|
|
1826
|
+
"priorFindings",
|
|
1827
|
+
"chainFindings",
|
|
1828
|
+
"missingInputMode",
|
|
1829
|
+
"applyRegistryHooks",
|
|
1830
|
+
"useRegistryChat"
|
|
1831
|
+
];
|
|
1832
|
+
const exactNonEmptyString = z.string().min(1);
|
|
1833
|
+
const exactFiniteNonnegative = z.number().finite().nonnegative();
|
|
1834
|
+
const exactBudgetSchema = z.discriminatedUnion("kind", [z.strictObject({
|
|
1835
|
+
kind: z.literal("equal"),
|
|
1836
|
+
totalUsd: exactFiniteNonnegative
|
|
1837
|
+
}), z.strictObject({
|
|
1838
|
+
kind: z.literal("weighted"),
|
|
1839
|
+
totalUsd: exactFiniteNonnegative,
|
|
1840
|
+
weights: z.record(exactNonEmptyString, exactFiniteNonnegative)
|
|
1841
|
+
})]);
|
|
1842
|
+
const exactRunDataSchema = z.strictObject({
|
|
1843
|
+
analystIds: z.array(exactNonEmptyString).min(1),
|
|
1844
|
+
budget: exactBudgetSchema.nullable(),
|
|
1845
|
+
totalTimeoutMs: z.number().int().positive().max(2147483647).nullable(),
|
|
1846
|
+
costLedgerIdentity: z.unknown().nullable(),
|
|
1847
|
+
costPhase: exactNonEmptyString.nullable(),
|
|
1848
|
+
tags: z.record(z.string(), z.string()).nullable(),
|
|
1849
|
+
chainFindings: z.boolean(),
|
|
1850
|
+
missingInputMode: z.enum(["skip", "abort"]),
|
|
1851
|
+
applyRegistryHooks: z.boolean(),
|
|
1852
|
+
useRegistryChat: z.boolean()
|
|
1853
|
+
}).superRefine((policy, context) => {
|
|
1854
|
+
const issue = (path, message) => context.addIssue({
|
|
1855
|
+
code: "custom",
|
|
1856
|
+
path,
|
|
1857
|
+
message
|
|
1858
|
+
});
|
|
1859
|
+
if (new Set(policy.analystIds).size !== policy.analystIds.length) issue(["analystIds"], "must not contain duplicates");
|
|
1860
|
+
if (policy.budget?.kind === "weighted" && Object.values(policy.budget.weights).every((weight) => weight === 0)) issue(["budget", "weights"], "must allocate positive weight to at least one analyst");
|
|
1861
|
+
if (policy.budget?.kind === "weighted") {
|
|
1862
|
+
const selected = [...policy.analystIds].sort();
|
|
1863
|
+
const weighted = Object.keys(policy.budget.weights).sort();
|
|
1864
|
+
if (selected.length !== weighted.length || selected.some((id, index) => id !== weighted[index])) issue(["budget", "weights"], "must name every selected analyst and no others");
|
|
1865
|
+
}
|
|
1866
|
+
});
|
|
1867
|
+
/** Validate the canonical exact-run policy before any analyst can start. */
|
|
1868
|
+
function assertExactRegistryRunOpts(value) {
|
|
1869
|
+
snapshotExactRegistryRunOpts(value);
|
|
1870
|
+
}
|
|
1871
|
+
function snapshotExactRunId(value) {
|
|
1872
|
+
if (typeof value !== "string" || value.length === 0) throw new TypeError("AnalystRegistry.runExact: runId must be a non-empty string");
|
|
1873
|
+
return canonicalJsonSnapshot(value, "AnalystRegistry.runExact runId");
|
|
1874
|
+
}
|
|
1875
|
+
function snapshotAnalystRunInputChannels(inputs) {
|
|
1876
|
+
if (!inputs || typeof inputs !== "object" || Array.isArray(inputs)) throw new TypeError("AnalystRegistry.runExact: inputs must be an object");
|
|
1877
|
+
const traceStore = inputs.traceStore;
|
|
1878
|
+
const artifactDir = inputs.artifactDir;
|
|
1879
|
+
const runRecord = inputs.runRecord;
|
|
1880
|
+
const judgeInput = inputs.judgeInput;
|
|
1881
|
+
const custom = inputs.custom;
|
|
1882
|
+
return Object.freeze({
|
|
1883
|
+
traceStore,
|
|
1884
|
+
artifactDir,
|
|
1885
|
+
runRecord,
|
|
1886
|
+
judgeInput,
|
|
1887
|
+
custom
|
|
1888
|
+
});
|
|
1889
|
+
}
|
|
1890
|
+
/**
|
|
1891
|
+
* Read the untrusted caller object once, then validate and execute only this frozen snapshot.
|
|
1892
|
+
* Functions and resource handles retain identity; all data fields are copied canonically.
|
|
1893
|
+
*/
|
|
1894
|
+
function snapshotExactRegistryRunOpts(value) {
|
|
1895
|
+
const captured = readOwnFields(value, exactRunFields, "ExactRegistryRunOpts");
|
|
1896
|
+
const missing = exactRunFields.find((field) => !Object.hasOwn(captured, field));
|
|
1897
|
+
if (missing) throw new TypeError(`ExactRegistryRunOpts.${missing} must be supplied explicitly`);
|
|
1898
|
+
const { signal, costLedger, priorFindings, ...rawData } = captured;
|
|
1899
|
+
const data = canonicalJsonSnapshot(rawData, "ExactRegistryRunOpts");
|
|
1900
|
+
const parsed = exactRunDataSchema.safeParse(data);
|
|
1901
|
+
if (!parsed.success) {
|
|
1902
|
+
const issue = parsed.error.issues[0];
|
|
1903
|
+
if (issue?.code === "unrecognized_keys" && issue.path.join(".") === "budget") {
|
|
1904
|
+
const required = isPlainRecord(data.budget) && data.budget.kind === "weighted" ? "kind, totalUsd, weights" : "kind, totalUsd";
|
|
1905
|
+
throw new TypeError(`ExactRegistryRunOpts.budget must contain exactly ${required}`);
|
|
1906
|
+
}
|
|
1907
|
+
const path = issue?.path.length ? `.${issue.path.join(".")}` : "";
|
|
1908
|
+
throw new TypeError(`ExactRegistryRunOpts${path}: ${issue?.message ?? "is invalid"}`);
|
|
1909
|
+
}
|
|
1910
|
+
if (signal !== null && (!signal || typeof signal !== "object" || typeof signal.addEventListener !== "function")) throw new TypeError("ExactRegistryRunOpts.signal must be an AbortSignal or null");
|
|
1911
|
+
if (costLedger !== null && (!costLedger || typeof costLedger !== "object")) throw new TypeError("ExactRegistryRunOpts.costLedger must be a CostLedgerHandle or null");
|
|
1912
|
+
if (costLedger === null && parsed.data.costLedgerIdentity !== null) throw new TypeError("ExactRegistryRunOpts.costLedgerIdentity must be null without costLedger");
|
|
1913
|
+
if (costLedger !== null && parsed.data.costLedgerIdentity === null) throw new TypeError("ExactRegistryRunOpts.costLedgerIdentity is required with costLedger");
|
|
1914
|
+
if (costLedger === null && parsed.data.costPhase !== null) throw new TypeError("ExactRegistryRunOpts.costPhase requires a non-null costLedger");
|
|
1915
|
+
return Object.freeze({
|
|
1916
|
+
...deepFreezeCanonicalJson(parsed.data),
|
|
1917
|
+
signal,
|
|
1918
|
+
costLedger,
|
|
1919
|
+
priorFindings: snapshotExactPriorFindings(priorFindings)
|
|
1920
|
+
});
|
|
1921
|
+
}
|
|
1922
|
+
function snapshotExactPriorFindings(value) {
|
|
1923
|
+
if (value === null) return null;
|
|
1924
|
+
if (Array.isArray(value)) return snapshotAnalystFindings(value, "ExactRegistryRunOpts.priorFindings");
|
|
1925
|
+
if (!isPlainRecord(value)) throw new TypeError("ExactRegistryRunOpts.priorFindings must be an array, a findings record, or null");
|
|
1926
|
+
const result = {};
|
|
1927
|
+
for (const [key, findings] of Object.entries(value)) {
|
|
1928
|
+
if (!Array.isArray(findings)) throw new TypeError(`ExactRegistryRunOpts.priorFindings.${key} must be an array`);
|
|
1929
|
+
result[key] = snapshotAnalystFindings(findings, `ExactRegistryRunOpts.priorFindings.${key}`);
|
|
1930
|
+
}
|
|
1931
|
+
return deepFreezeCanonicalJson(result);
|
|
1932
|
+
}
|
|
1933
|
+
function normalizeExactAnalysts(selections) {
|
|
1934
|
+
return selections.map(({ registeredId, analyst }) => {
|
|
1935
|
+
const exactAnalyst = analyst;
|
|
1936
|
+
const id = analyst.id;
|
|
1937
|
+
const description = analyst.description;
|
|
1938
|
+
const inputKind = analyst.inputKind;
|
|
1939
|
+
const rawCostValue = analyst.cost;
|
|
1940
|
+
const requiresValue = analyst.requires;
|
|
1941
|
+
const version = analyst.version;
|
|
1942
|
+
const executionConfigValue = exactAnalyst.executionConfig;
|
|
1943
|
+
const analyzeValue = analyst.analyze;
|
|
1944
|
+
if (id !== registeredId) throw new TypeError(`AnalystRegistry.runExact: registered analyst "${registeredId}" changed id to "${id}"`);
|
|
1945
|
+
if (executionConfigValue === void 0) throw new TypeError(`AnalystRegistry.runExact: analyst "${id}" must declare executionConfig`);
|
|
1946
|
+
const executionConfig = canonicalJsonSnapshot(executionConfigValue, `AnalystRegistry.runExact analyst "${id}" executionConfig`);
|
|
1947
|
+
if (!isPlainRecord(executionConfig)) throw new TypeError(`AnalystRegistry.runExact analyst "${id}" executionConfig must be an object`);
|
|
1948
|
+
const rawCost = canonicalJsonSnapshot(rawCostValue, `AnalystRegistry.runExact analyst "${id}" cost`);
|
|
1949
|
+
const cost = rawCost.kind === "llm" ? Object.freeze({
|
|
1950
|
+
...rawCost,
|
|
1951
|
+
settlement_timeout_ms: validateUsageSettlementTimeout(rawCost.settlement_timeout_ms)
|
|
1952
|
+
}) : rawCost;
|
|
1953
|
+
const requires = requiresValue === void 0 ? void 0 : canonicalJsonSnapshot(requiresValue, `AnalystRegistry.runExact analyst "${id}" requirements`);
|
|
1954
|
+
const analyze = analyzeValue.bind(analyst);
|
|
1955
|
+
return Object.freeze({
|
|
1956
|
+
id,
|
|
1957
|
+
description,
|
|
1958
|
+
inputKind,
|
|
1959
|
+
cost,
|
|
1960
|
+
...requires === void 0 ? {} : { requires },
|
|
1961
|
+
version,
|
|
1962
|
+
executionConfig,
|
|
1963
|
+
analyze
|
|
1964
|
+
});
|
|
1965
|
+
});
|
|
1966
|
+
}
|
|
1967
|
+
function hasRegistryHooks(hooks) {
|
|
1968
|
+
return Boolean(hooks && (hooks.onBeforeAnalyze || hooks.onAfterAnalyze || hooks.onError || hooks.onComplete));
|
|
1969
|
+
}
|
|
1970
|
+
function snapshotHooks(hooks) {
|
|
1971
|
+
const onBeforeAnalyze = hooks.onBeforeAnalyze;
|
|
1972
|
+
const onAfterAnalyze = hooks.onAfterAnalyze;
|
|
1973
|
+
const onError = hooks.onError;
|
|
1974
|
+
const onComplete = hooks.onComplete;
|
|
1975
|
+
return Object.freeze({
|
|
1976
|
+
...onBeforeAnalyze === void 0 ? {} : { onBeforeAnalyze: onBeforeAnalyze.bind(hooks) },
|
|
1977
|
+
...onAfterAnalyze === void 0 ? {} : { onAfterAnalyze: onAfterAnalyze.bind(hooks) },
|
|
1978
|
+
...onError === void 0 ? {} : { onError: onError.bind(hooks) },
|
|
1979
|
+
...onComplete === void 0 ? {} : { onComplete: onComplete.bind(hooks) }
|
|
1980
|
+
});
|
|
1981
|
+
}
|
|
1982
|
+
function snapshotChat(chat) {
|
|
1983
|
+
const transport = chat.transport;
|
|
1984
|
+
const defaultModel = chat.defaultModel;
|
|
1985
|
+
const maximumAttempts = chat.maximumAttempts;
|
|
1986
|
+
const call = chat.chat;
|
|
1987
|
+
return Object.freeze({
|
|
1988
|
+
transport,
|
|
1989
|
+
...defaultModel === void 0 ? {} : { defaultModel },
|
|
1990
|
+
...maximumAttempts === void 0 ? {} : { maximumAttempts },
|
|
1991
|
+
chat: call.bind(chat)
|
|
1992
|
+
});
|
|
1993
|
+
}
|
|
1994
|
+
function requireExactComponentIdentity(value, label) {
|
|
1995
|
+
if (value === void 0) throw new TypeError(`AnalystRegistry.runExact: ${label} requires a versioned identity`);
|
|
1996
|
+
return snapshotExactExecutionComponentIdentity(value, `AnalystRegistry.runExact ${label} identity`);
|
|
1997
|
+
}
|
|
1998
|
+
function exactExecutionSnapshot(analysts, opts, allocations, costLedger, hooks, chat) {
|
|
1999
|
+
const priorFindings = exactPriorFindingsSnapshot(opts.priorFindings);
|
|
2000
|
+
const budget = opts.budget === null ? { kind: "none" } : opts.budget.kind === "equal" ? {
|
|
2001
|
+
kind: "equal",
|
|
2002
|
+
total_usd: opts.budget.totalUsd,
|
|
2003
|
+
allocations_usd: { ...allocations }
|
|
2004
|
+
} : {
|
|
2005
|
+
kind: "weighted",
|
|
2006
|
+
total_usd: opts.budget.totalUsd,
|
|
2007
|
+
weights: { ...opts.budget.weights },
|
|
2008
|
+
allocations_usd: { ...allocations }
|
|
2009
|
+
};
|
|
2010
|
+
const material = {
|
|
2011
|
+
schema_version: "1.0.0",
|
|
2012
|
+
analysts: analysts.map((analyst) => ({
|
|
2013
|
+
id: analyst.id,
|
|
2014
|
+
version: analyst.version,
|
|
2015
|
+
input_kind: analyst.inputKind,
|
|
2016
|
+
cost: analyst.cost,
|
|
2017
|
+
requirements: analyst.requires ?? null,
|
|
2018
|
+
execution_config_digest: hashCanonical(analyst.executionConfig)
|
|
2019
|
+
})),
|
|
2020
|
+
policy: {
|
|
2021
|
+
budget,
|
|
2022
|
+
total_timeout_ms: opts.totalTimeoutMs,
|
|
2023
|
+
signal_provided: opts.signal !== null,
|
|
2024
|
+
cost_ledger: costLedger,
|
|
2025
|
+
cost_phase: opts.costPhase,
|
|
2026
|
+
tags: opts.tags === null ? null : { ...opts.tags },
|
|
2027
|
+
prior_findings: priorFindings,
|
|
2028
|
+
chain_findings: opts.chainFindings,
|
|
2029
|
+
missing_input_mode: opts.missingInputMode,
|
|
2030
|
+
registry_hooks: hooks,
|
|
2031
|
+
registry_chat: chat
|
|
2032
|
+
}
|
|
2033
|
+
};
|
|
2034
|
+
return snapshotExactExecutionPlan({
|
|
2035
|
+
...material,
|
|
2036
|
+
digest: hashCanonical(material)
|
|
2037
|
+
}, "AnalystRegistry.runExact execution plan");
|
|
2038
|
+
}
|
|
2039
|
+
function exactPriorFindingsSnapshot(findings) {
|
|
2040
|
+
if (findings === null) return { kind: "none" };
|
|
2041
|
+
if (Array.isArray(findings)) return {
|
|
2042
|
+
kind: "ordered",
|
|
2043
|
+
count: findings.length,
|
|
2044
|
+
digest: hashCanonical(findings)
|
|
2045
|
+
};
|
|
2046
|
+
const record = findings;
|
|
2047
|
+
const keys = Object.keys(record).sort();
|
|
2048
|
+
return {
|
|
2049
|
+
kind: "by_analyst",
|
|
2050
|
+
keys,
|
|
2051
|
+
count: keys.reduce((sum, key) => sum + (record[key]?.length ?? 0), 0),
|
|
2052
|
+
digest: hashCanonical(record)
|
|
2053
|
+
};
|
|
2054
|
+
}
|
|
2055
|
+
function canonicalJsonSnapshot(value, label) {
|
|
2056
|
+
let snapshot;
|
|
2057
|
+
try {
|
|
2058
|
+
snapshot = JSON.parse(canonicalString(value));
|
|
2059
|
+
} catch (cause) {
|
|
2060
|
+
throw new TypeError(`${label} must be canonical JSON`, { cause });
|
|
2061
|
+
}
|
|
2062
|
+
return deepFreezeCanonicalJson(snapshot);
|
|
2063
|
+
}
|
|
2064
|
+
function snapshotUsageReceiptOnce(receipt, context) {
|
|
2065
|
+
const data = readOwnFields(receipt, [
|
|
2066
|
+
"calls",
|
|
2067
|
+
"tokens",
|
|
2068
|
+
"cost",
|
|
2069
|
+
"knownCostUsd"
|
|
2070
|
+
], context);
|
|
2071
|
+
data.tokens = data.tokens === null ? null : readOwnFields(data.tokens, [
|
|
2072
|
+
"input",
|
|
2073
|
+
"output",
|
|
2074
|
+
"reasoning",
|
|
2075
|
+
"cached",
|
|
2076
|
+
"cacheWrite"
|
|
2077
|
+
], `${context} tokens`);
|
|
2078
|
+
data.cost = readOwnFields(data.cost, ["kind", "usd"], `${context} cost`);
|
|
2079
|
+
const snapshot = canonicalJsonSnapshot(data, context);
|
|
2080
|
+
assertValidAnalystUsageReceipt(snapshot, context);
|
|
2081
|
+
return snapshot;
|
|
2082
|
+
}
|
|
2083
|
+
function readOwnFields(value, fields, context) {
|
|
2084
|
+
if (!isPlainRecord(value)) throw new TypeError(`${context} must be a plain object`);
|
|
2085
|
+
const unexpected = Object.keys(value).filter((key) => !fields.includes(key));
|
|
2086
|
+
if (unexpected.length > 0) throw new TypeError(`${context} contains unknown fields: ${unexpected.sort().join(", ")}`);
|
|
2087
|
+
return Object.fromEntries(fields.flatMap((field) => Object.hasOwn(value, field) ? [[field, value[field]]] : []));
|
|
2088
|
+
}
|
|
2089
|
+
function isPlainRecord(value) {
|
|
2090
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
2091
|
+
const prototype = Object.getPrototypeOf(value);
|
|
2092
|
+
return prototype === Object.prototype || prototype === null;
|
|
2093
|
+
}
|
|
2094
|
+
function exactFixedBudgets(exact, runnable, selected) {
|
|
2095
|
+
if (exact === null) return {};
|
|
2096
|
+
const allocations = Object.fromEntries(selected.map((analyst) => [analyst.id, null]));
|
|
2097
|
+
if (runnable.length === 0) return deepFreezeCanonicalJson(allocations);
|
|
2098
|
+
if (exact.kind === "equal") {
|
|
2099
|
+
const each = exact.totalUsd / runnable.length;
|
|
2100
|
+
for (const analyst of runnable) allocations[analyst.id] = each;
|
|
2101
|
+
return deepFreezeCanonicalJson(allocations);
|
|
2102
|
+
}
|
|
2103
|
+
const totalWeight = runnable.reduce((sum, analyst) => sum + exact.weights[analyst.id], 0);
|
|
2104
|
+
if (totalWeight === 0) throw new Error("ExactRegistryRunOpts weighted budget must allocate positive weight to a runnable analyst");
|
|
2105
|
+
for (const analyst of runnable) allocations[analyst.id] = exact.totalUsd * exact.weights[analyst.id] / totalWeight;
|
|
2106
|
+
return deepFreezeCanonicalJson(allocations);
|
|
2107
|
+
}
|
|
2108
|
+
function snapshotExecutionFindings(findings, exact, context) {
|
|
2109
|
+
return exact ? deepFreezeCanonicalJson(snapshotAnalystFindings(findings, context)) : findings;
|
|
2110
|
+
}
|
|
2111
|
+
function snapshotAfterHookValues(summary, findings, exact) {
|
|
2112
|
+
if (!exact) return {
|
|
2113
|
+
summary,
|
|
2114
|
+
findings
|
|
2115
|
+
};
|
|
2116
|
+
return {
|
|
2117
|
+
summary: canonicalJsonSnapshot(summary, "AnalystRegistry.runExact onAfterAnalyze summary"),
|
|
2118
|
+
findings: deepFreezeCanonicalJson(snapshotAnalystFindings(findings, "AnalystRegistry.runExact onAfterAnalyze findings"))
|
|
2119
|
+
};
|
|
2120
|
+
}
|
|
2121
|
+
function exactPlannedAllocation(plan, analystId) {
|
|
2122
|
+
const budget = plan.policy.budget;
|
|
2123
|
+
if (budget.kind === "none") return void 0;
|
|
2124
|
+
const allocated = budget.allocations_usd[analystId];
|
|
2125
|
+
return allocated === null ? void 0 : allocated;
|
|
2126
|
+
}
|
|
2127
|
+
function snapshotExecutionEvent(event, exact) {
|
|
2128
|
+
return exact ? canonicalJsonSnapshot(event, "AnalystRegistry.runExact event") : event;
|
|
2129
|
+
}
|
|
2130
|
+
function logUncapturedBudgetWarning(args) {
|
|
2131
|
+
if (args.budgetUsd === void 0 || args.usage.cost.kind !== "uncaptured") return;
|
|
2132
|
+
args.log(`[analyst] WARN ${args.analyst.id} — USD cost uncaptured; budget not reconciled`, {
|
|
2133
|
+
runId: args.runId,
|
|
2134
|
+
budget_usd: args.budgetUsd,
|
|
2135
|
+
cost_captured: false
|
|
2136
|
+
});
|
|
2137
|
+
}
|
|
2138
|
+
function validateTimeout(timeoutMs) {
|
|
2139
|
+
if (timeoutMs === void 0) return void 0;
|
|
2140
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2147483647) throw new TypeError("RegistryRunOpts.timeoutMs must be a positive safe integer no greater than 2147483647");
|
|
2141
|
+
return timeoutMs;
|
|
2142
|
+
}
|
|
2143
|
+
async function waitForOperation(operation, signal, abortGraceMs) {
|
|
2144
|
+
if (!signal) return operation;
|
|
2145
|
+
if (signal.aborted) {
|
|
2146
|
+
operation.catch(() => {});
|
|
2147
|
+
throw abortReason(signal);
|
|
2148
|
+
}
|
|
2149
|
+
return new Promise((resolve, reject) => {
|
|
2150
|
+
let settlementTimer;
|
|
2151
|
+
const cleanup = () => {
|
|
2152
|
+
signal.removeEventListener("abort", onAbort);
|
|
2153
|
+
if (settlementTimer) clearTimeout(settlementTimer);
|
|
2154
|
+
};
|
|
2155
|
+
const onAbort = () => {
|
|
2156
|
+
if (abortGraceMs === 0) {
|
|
2157
|
+
cleanup();
|
|
2158
|
+
reject(abortReason(signal));
|
|
2159
|
+
return;
|
|
2160
|
+
}
|
|
2161
|
+
settlementTimer = setTimeout(() => {
|
|
2162
|
+
cleanup();
|
|
2163
|
+
reject(abortReason(signal));
|
|
2164
|
+
}, abortGraceMs);
|
|
2165
|
+
};
|
|
2166
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
2167
|
+
operation.then((value) => {
|
|
2168
|
+
cleanup();
|
|
2169
|
+
if (signal.aborted) reject(abortReason(signal));
|
|
2170
|
+
else resolve(value);
|
|
2171
|
+
}, (error) => {
|
|
2172
|
+
cleanup();
|
|
2173
|
+
reject(signal.aborted ? abortReason(signal) : error);
|
|
2174
|
+
});
|
|
2175
|
+
});
|
|
2176
|
+
}
|
|
2177
|
+
async function waitForHook(operation, signal) {
|
|
2178
|
+
if (operation === void 0 || signal?.aborted) return void 0;
|
|
2179
|
+
try {
|
|
2180
|
+
return await waitForOperation(Promise.resolve().then(() => {
|
|
2181
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
2182
|
+
return operation();
|
|
2183
|
+
}), signal, 0);
|
|
2184
|
+
} catch (error) {
|
|
2185
|
+
if (signal?.aborted) return void 0;
|
|
2186
|
+
throw error;
|
|
2187
|
+
}
|
|
2188
|
+
}
|
|
2189
|
+
function analystAbortGraceMs(analyst) {
|
|
2190
|
+
if (analyst.cost.kind === "deterministic") return 0;
|
|
2191
|
+
const settlementMs = validateUsageSettlementTimeout(analyst.cost.settlement_timeout_ms);
|
|
2192
|
+
if (settlementMs === 0) return 0;
|
|
2193
|
+
return Math.min(settlementMs + 100, 2147483647);
|
|
2194
|
+
}
|
|
2195
|
+
function abortedBeforeStartSummary(analyst, signal, latencyMs = 0) {
|
|
2196
|
+
const reason = abortReason(signal);
|
|
2197
|
+
return {
|
|
2198
|
+
analyst_id: analyst.id,
|
|
2199
|
+
status: "skipped",
|
|
2200
|
+
reason: `${reason.name}: ${reason.message}`,
|
|
2201
|
+
findings_count: 0,
|
|
2202
|
+
latency_ms: latencyMs,
|
|
2203
|
+
usage: zeroUsage()
|
|
2204
|
+
};
|
|
2205
|
+
}
|
|
2206
|
+
function abortReason(signal) {
|
|
2207
|
+
return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError");
|
|
2208
|
+
}
|
|
2209
|
+
/**
|
|
2210
|
+
* Default budget allocator: prefer the custom `allocate` callback if
|
|
2211
|
+
* provided; else weighted split when weights are set; else equal split
|
|
2212
|
+
* across `runningCount`. Returns undefined when no totalUsd is known.
|
|
2213
|
+
*/
|
|
2214
|
+
function allocateBudget(policy, args) {
|
|
2215
|
+
if (!policy) return void 0;
|
|
2216
|
+
if (policy.allocate) {
|
|
2217
|
+
const allocated = policy.allocate({
|
|
2218
|
+
analyst: args.analyst,
|
|
2219
|
+
totalUsd: policy.totalUsd,
|
|
2220
|
+
remainingUsd: args.remainingUsd,
|
|
2221
|
+
runningCount: args.runningCount
|
|
2222
|
+
});
|
|
2223
|
+
if (allocated === void 0) {
|
|
2224
|
+
if (policy.totalUsd !== void 0) throw new Error(`BudgetPolicy.allocate('${args.analyst.id}') cannot return undefined when totalUsd is set`);
|
|
2225
|
+
return;
|
|
2226
|
+
}
|
|
2227
|
+
assertBudgetAmount(allocated, `BudgetPolicy.allocate('${args.analyst.id}')`);
|
|
2228
|
+
return args.remainingUsd === void 0 ? allocated : Math.min(allocated, args.remainingUsd);
|
|
2229
|
+
}
|
|
2230
|
+
if (policy.totalUsd == null) return void 0;
|
|
2231
|
+
const allocated = policy.weights ? policy.totalUsd * analystWeight(policy.weights, args.analyst.id) / args.totalWeight : policy.totalUsd / Math.max(1, args.runningCount);
|
|
2232
|
+
return args.remainingUsd === void 0 ? allocated : Math.min(allocated, args.remainingUsd);
|
|
2233
|
+
}
|
|
2234
|
+
function validateBudgetPolicy(policy) {
|
|
2235
|
+
if (!policy) return;
|
|
2236
|
+
if (policy.totalUsd !== void 0) assertBudgetAmount(policy.totalUsd, "BudgetPolicy.totalUsd");
|
|
2237
|
+
for (const [analystId, weight] of Object.entries(policy.weights ?? {})) assertBudgetAmount(weight, `BudgetPolicy.weights['${analystId}']`);
|
|
2238
|
+
}
|
|
2239
|
+
function assertBudgetAmount(value, field) {
|
|
2240
|
+
if (!Number.isFinite(value) || value < 0) throw new Error(`${field} must be a non-negative finite number`);
|
|
2241
|
+
}
|
|
2242
|
+
function validateEffectiveBudget(budgetUsd, remainingUsd, analystId) {
|
|
2243
|
+
if (budgetUsd !== void 0) assertBudgetAmount(budgetUsd, `AnalystContext.budgetUsd for '${analystId}'`);
|
|
2244
|
+
if (remainingUsd === void 0) return budgetUsd;
|
|
2245
|
+
if (budgetUsd === void 0) throw new Error(`AnalystContext.budgetUsd for '${analystId}' cannot be removed while an overall budget remains`);
|
|
2246
|
+
if (budgetUsd > remainingUsd) throw new Error(`AnalystContext.budgetUsd for '${analystId}' (${budgetUsd}) exceeds the remaining overall budget (${remainingUsd})`);
|
|
2247
|
+
return budgetUsd;
|
|
2248
|
+
}
|
|
2249
|
+
function analystWeight(weights, analystId) {
|
|
2250
|
+
const weight = weights[analystId] ?? 1;
|
|
2251
|
+
assertBudgetAmount(weight, `BudgetPolicy.weights['${analystId}']`);
|
|
2252
|
+
return weight;
|
|
2253
|
+
}
|
|
2254
|
+
function zeroUsage() {
|
|
2255
|
+
return {
|
|
2256
|
+
calls: 0,
|
|
2257
|
+
tokens: {
|
|
2258
|
+
input: 0,
|
|
2259
|
+
output: 0
|
|
2260
|
+
},
|
|
2261
|
+
cost: {
|
|
2262
|
+
kind: "observed",
|
|
2263
|
+
usd: 0
|
|
2264
|
+
}
|
|
2265
|
+
};
|
|
2266
|
+
}
|
|
2267
|
+
function resolveUsage(analyst, receipts, exact = false) {
|
|
2268
|
+
if (receipts.length > 0) return mergeUsageReceipts(receipts, exact);
|
|
2269
|
+
if (analyst.cost.kind === "deterministic") return zeroUsage();
|
|
2270
|
+
return {
|
|
2271
|
+
calls: null,
|
|
2272
|
+
tokens: null,
|
|
2273
|
+
cost: {
|
|
2274
|
+
kind: "uncaptured",
|
|
2275
|
+
usd: null
|
|
2276
|
+
}
|
|
2277
|
+
};
|
|
2278
|
+
}
|
|
2279
|
+
function mergeUsageReceipts(receipts, exact = false) {
|
|
2280
|
+
const calls = receipts.every((receipt) => receipt.calls !== null) ? usageSum(receipts.map((receipt) => receipt.calls ?? 0), exact, "calls", true) : null;
|
|
2281
|
+
const tokens = receipts.every((receipt) => receipt.tokens !== null) ? Object.fromEntries([
|
|
2282
|
+
"input",
|
|
2283
|
+
"output",
|
|
2284
|
+
"reasoning",
|
|
2285
|
+
"cached",
|
|
2286
|
+
"cacheWrite"
|
|
2287
|
+
].flatMap((field) => field === "input" || field === "output" || receipts.some((receipt) => receipt.tokens?.[field] !== void 0) ? [[field, usageSum(receipts.map((receipt) => receipt.tokens?.[field] ?? 0), exact, `tokens.${field}`, true)]] : [])) : null;
|
|
2288
|
+
const cost = aggregateCostProvenance(receipts.map((receipt) => receipt.cost), exact);
|
|
2289
|
+
return {
|
|
2290
|
+
calls,
|
|
2291
|
+
tokens,
|
|
2292
|
+
cost,
|
|
2293
|
+
...cost.kind === "uncaptured" ? { knownCostUsd: usageSum(receipts.map(knownCostUsd), exact, "known cost") } : {}
|
|
2294
|
+
};
|
|
2295
|
+
}
|
|
2296
|
+
function knownCostUsd(receipt) {
|
|
2297
|
+
return receipt.cost.kind === "uncaptured" ? receipt.knownCostUsd ?? 0 : receipt.cost.usd;
|
|
2298
|
+
}
|
|
2299
|
+
function budgetDebit(receipt, allocatedUsd) {
|
|
2300
|
+
const known = knownCostUsd(receipt);
|
|
2301
|
+
return receipt.cost.kind === "uncaptured" && allocatedUsd !== void 0 ? Math.max(known, allocatedUsd) : known;
|
|
2302
|
+
}
|
|
2303
|
+
function aggregateCostProvenance(costs, exact = false) {
|
|
2304
|
+
if (costs.some((cost) => cost.kind === "uncaptured")) return {
|
|
2305
|
+
kind: "uncaptured",
|
|
2306
|
+
usd: null
|
|
2307
|
+
};
|
|
2308
|
+
const usd = usageSum(costs.map((cost) => cost.usd ?? 0), exact, "captured cost");
|
|
2309
|
+
return costs.some((cost) => cost.kind === "estimated") ? {
|
|
2310
|
+
kind: "estimated",
|
|
2311
|
+
usd
|
|
2312
|
+
} : {
|
|
2313
|
+
kind: "observed",
|
|
2314
|
+
usd
|
|
2315
|
+
};
|
|
2316
|
+
}
|
|
2317
|
+
function executionCost(executions, exact) {
|
|
2318
|
+
const usages = executions.map((execution) => execution.summary.usage);
|
|
2319
|
+
return {
|
|
2320
|
+
known: usageSum(usages.map(knownCostUsd), exact, "run known cost"),
|
|
2321
|
+
provenance: aggregateCostProvenance(usages.map((usage) => usage.cost), exact)
|
|
2322
|
+
};
|
|
2323
|
+
}
|
|
2324
|
+
function usageSum(values, exact, field, integer = false) {
|
|
2325
|
+
const sum = values.reduce((total, value) => total + value, 0);
|
|
2326
|
+
if (exact && (integer ? !Number.isSafeInteger(sum) : !Number.isFinite(sum))) throw new RangeError(`exact analyst usage ${field} aggregate ${integer ? "exceeds a safe integer" : "is not finite"}`);
|
|
2327
|
+
return sum;
|
|
2328
|
+
}
|
|
2329
|
+
/**
|
|
2330
|
+
* Resolve the `priorFindings` slice an analyst sees.
|
|
2331
|
+
*
|
|
2332
|
+
* - Array form → the analyst sees only findings whose `analyst_id`
|
|
2333
|
+
* matches its own id, so a kind never reads
|
|
2334
|
+
* another kind's history by accident.
|
|
2335
|
+
* - Record form → the analyst gets the entry keyed by its id, with
|
|
2336
|
+
* the `'*'` wildcard appended (in that order). Use
|
|
2337
|
+
* the wildcard when several kinds should see the same
|
|
2338
|
+
* historical findings.
|
|
2339
|
+
*/
|
|
2340
|
+
function selectPriorFindings(source, analystId) {
|
|
2341
|
+
if (!source) return void 0;
|
|
2342
|
+
if (Array.isArray(source)) {
|
|
2343
|
+
const own = source.filter((f) => f.analyst_id === analystId);
|
|
2344
|
+
return own.length > 0 ? own : void 0;
|
|
2345
|
+
}
|
|
2346
|
+
const record = source;
|
|
2347
|
+
const own = record[analystId] ?? [];
|
|
2348
|
+
const wildcard = record["*"] ?? [];
|
|
2349
|
+
const merged = [...own, ...wildcard];
|
|
2350
|
+
return merged.length > 0 ? merged : void 0;
|
|
2351
|
+
}
|
|
2352
|
+
//#endregion
|
|
2353
|
+
//#region src/analyst/default-registry.ts
|
|
2354
|
+
function buildDefaultAnalystRegistry(options = {}) {
|
|
2355
|
+
if (options.definitions && !options.engine) throw new TypeError("buildDefaultAnalystRegistry: definitions require an engine — a definition cannot run without one");
|
|
2356
|
+
const registry = new AnalystRegistry(options.registry);
|
|
2357
|
+
if (options.includeBehavioral !== false) registry.register(behavioralAnalyst(options.behavioral));
|
|
2358
|
+
if (options.engine) for (const definition of options.definitions ?? DEFAULT_TRACE_ANALYST_KINDS) registry.register(createTraceAnalyst(definition, { engine: options.engine }));
|
|
2359
|
+
return registry;
|
|
2360
|
+
}
|
|
2361
|
+
//#endregion
|
|
2362
|
+
export { createChatClient as C, computeTraceMetrics as S, CONTROL_INTEGRITY_ANALYST as _, analystFindingDigest as a, behavioralAnalyst as b, completedAnalystReviewQuality as c, validateAnalystReviewDecisions as d, DEFAULT_TRACE_ANALYST_KINDS as f, FAILURE_MODE_KIND_SPEC as g, IMPROVEMENT_KIND_SPEC as h, assertExactRegistryRunOpts as i, readAnalystReview as l, KNOWLEDGE_GAP_KIND_SPEC as m, AnalystRegistry as n, analystRunDigest as o, KNOWLEDGE_POISONING_KIND_SPEC as p, ExactAnalystRunExecutionError as r, assertUniqueFindingIds as s, buildDefaultAnalystRegistry as t, snapshotAnalystRun as u, ControlIntegrityAnalyst as v, deriveEfficiencyFindings as x, emitControlIntegrityFindings as y };
|
|
2363
|
+
|
|
2364
|
+
//# sourceMappingURL=default-registry-BgJJItGr.js.map
|