@tangle-network/agent-eval 0.138.0 → 0.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +2 -1
- package/dist/analyst/index.d.ts +41 -94
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +9 -24
- package/dist/analyst/index.js.map +1 -1
- package/dist/{benchmark-D8dkki-J.js → benchmark-CYtcIF2V.js} +2 -2
- package/dist/{benchmark-D8dkki-J.js.map → benchmark-CYtcIF2V.js.map} +1 -1
- package/dist/{benchmark-DlQgU_XI.d.ts → benchmark-DDVdWcwA.d.ts} +3 -3
- package/dist/{benchmark-DlQgU_XI.d.ts.map → benchmark-DDVdWcwA.d.ts.map} +1 -1
- package/dist/{benchmark-command-CMqVqReF.js → benchmark-command-BKfjOBJ5.js} +243 -38
- package/dist/benchmark-command-BKfjOBJ5.js.map +1 -0
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-BJ_xK5rQ.js → benchmarks-zxhy1QV3.js} +4 -4
- package/dist/{benchmarks-BJ_xK5rQ.js.map → benchmarks-zxhy1QV3.js.map} +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +4 -3
- package/dist/{campaign-BIBS-NHV.js → campaign-DrS6_hLd.js} +10 -9
- package/dist/campaign-DrS6_hLd.js.map +1 -0
- package/dist/canonical-D011XM8r.js +86 -0
- package/dist/canonical-D011XM8r.js.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/{client-BwPKohkJ.d.ts → client-BohnDFBq.d.ts} +4 -4
- package/dist/{client-BwPKohkJ.d.ts.map → client-BohnDFBq.d.ts.map} +1 -1
- package/dist/{completion-verifier-B4-IMYcS.d.ts → completion-verifier-IPoP4fQO.d.ts} +178 -4
- package/dist/completion-verifier-IPoP4fQO.d.ts.map +1 -0
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +8 -7
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/{cost-ledger-CHDLA0Ss.js → cost-ledger-CZ9diLxY.js} +7 -7
- package/dist/cost-ledger-CZ9diLxY.js.map +1 -0
- package/dist/{cost-ledger-B1D3COAc.d.ts → cost-ledger-DKgyIWRj.d.ts} +5 -2
- package/dist/cost-ledger-DKgyIWRj.d.ts.map +1 -0
- package/dist/default-registry-B8vf7Rmf.d.ts +118 -0
- package/dist/default-registry-B8vf7Rmf.d.ts.map +1 -0
- package/dist/{default-registry-lp5R0lve.js → default-registry-BgJJItGr.js} +57 -1532
- package/dist/default-registry-BgJJItGr.js.map +1 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js +344 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js.map +1 -0
- package/dist/{eval-campaign-9MozgKL7.js → eval-campaign-BmptJj50.js} +2 -2
- package/dist/{eval-campaign-9MozgKL7.js.map → eval-campaign-BmptJj50.js.map} +1 -1
- package/dist/{exact-types-Dpw2LeHA.d.ts → exact-types-MaaFcllV.d.ts} +2 -2
- package/dist/{exact-types-Dpw2LeHA.d.ts.map → exact-types-MaaFcllV.d.ts.map} +1 -1
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts +32 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts.map +1 -0
- package/dist/{extract-usage-CS391dOE.js → extract-usage-DZs601Va.js} +2 -2
- package/dist/{extract-usage-CS391dOE.js.map → extract-usage-DZs601Va.js.map} +1 -1
- package/dist/{feedback-trajectory-CoNep7rl.d.ts → feedback-trajectory-BJUWOkJM.d.ts} +3 -3
- package/dist/{feedback-trajectory-CoNep7rl.d.ts.map → feedback-trajectory-BJUWOkJM.d.ts.map} +1 -1
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/{hf-dataset-DBJXXoY1.js → hf-dataset-XggBupCr.js} +2 -2
- package/dist/{hf-dataset-DBJXXoY1.js.map → hf-dataset-XggBupCr.js.map} +1 -1
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-D0cxAdaV.d.ts → index-BTm_P9aC.d.ts} +11 -11
- package/dist/{index-D0cxAdaV.d.ts.map → index-BTm_P9aC.d.ts.map} +1 -1
- package/dist/{index-B2-IxCMB.d.ts → index-CWOPCJiw.d.ts} +2 -2
- package/dist/{index-B2-IxCMB.d.ts.map → index-CWOPCJiw.d.ts.map} +1 -1
- package/dist/{index-sMN_hI4E.d.ts → index-CtR1xh4V.d.ts} +3 -3
- package/dist/{index-sMN_hI4E.d.ts.map → index-CtR1xh4V.d.ts.map} +1 -1
- package/dist/{index-CjVYlVBK.d.ts → index-_66rVpwN.d.ts} +5 -5
- package/dist/{index-CjVYlVBK.d.ts.map → index-_66rVpwN.d.ts.map} +1 -1
- package/dist/index.d.ts +35 -56
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +51 -176
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-CXd8VBDR.d.ts → insight-report-Bu5Wi9tG.d.ts} +4 -4
- package/dist/{insight-report-CXd8VBDR.d.ts.map → insight-report-Bu5Wi9tG.d.ts.map} +1 -1
- package/dist/{integrity-B-MLFz0I.d.ts → integrity-COTh3DTH.d.ts} +2 -2
- package/dist/{integrity-B-MLFz0I.d.ts.map → integrity-COTh3DTH.d.ts.map} +1 -1
- package/dist/kind-factory-CFxA0JQX.js +2133 -0
- package/dist/kind-factory-CFxA0JQX.js.map +1 -0
- package/dist/ledger-core/index.js +2 -1
- package/dist/{ledger-core-C0Yx1I14.js → ledger-core-Dxz0Rkwa.js} +3 -85
- package/dist/ledger-core-Dxz0Rkwa.js.map +1 -0
- package/dist/{llm-client-Cj3c7PEm.js → llm-client-bkztEfIx.js} +2 -2
- package/dist/{llm-client-Cj3c7PEm.js.map → llm-client-bkztEfIx.js.map} +1 -1
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{release-report-CoyvyLBs.d.ts → release-report-fZarvIm-.d.ts} +3 -3
- package/dist/{release-report-CoyvyLBs.d.ts.map → release-report-fZarvIm-.d.ts.map} +1 -1
- package/dist/{replay-DbIYwso6.d.ts → replay-DjG4IG60.d.ts} +34 -143
- package/dist/replay-DjG4IG60.d.ts.map +1 -0
- package/dist/{replay-Cb-4Vf0k.js → replay-SA4OB7O7.js} +48 -137
- package/dist/replay-SA4OB7O7.js.map +1 -0
- package/dist/reporting.d.ts +4 -4
- package/dist/{researcher-BCeOEjtR.d.ts → researcher-BxhtGfKa.d.ts} +5 -5
- package/dist/{researcher-BCeOEjtR.d.ts.map → researcher-BxhtGfKa.d.ts.map} +1 -1
- package/dist/{reward-hacking-sE2l_NV6.d.ts → reward-hacking-CqSLiV51.d.ts} +2 -2
- package/dist/{reward-hacking-sE2l_NV6.d.ts.map → reward-hacking-CqSLiV51.d.ts.map} +1 -1
- package/dist/rl.d.ts +5 -5
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-DQFl0UXA.js → rollout-8nj3mYvx.js} +2 -2
- package/dist/{rollout-DQFl0UXA.js.map → rollout-8nj3mYvx.js.map} +1 -1
- package/dist/{rubric-predictive-validity-w2klGv1u.d.ts → rubric-predictive-validity-DQBQj6uV.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-w2klGv1u.d.ts.map → rubric-predictive-validity-DQBQj6uV.d.ts.map} +1 -1
- package/dist/{run-evidence-CbE0A8Xg.d.ts → run-evidence-C4RcRQT5.d.ts} +3 -3
- package/dist/{run-evidence-CbE0A8Xg.d.ts.map → run-evidence-C4RcRQT5.d.ts.map} +1 -1
- package/dist/{run-record-DwHMk1Ai.d.ts → run-record-CztDMXVF.d.ts} +2 -2
- package/dist/{run-record-DwHMk1Ai.d.ts.map → run-record-CztDMXVF.d.ts.map} +1 -1
- package/dist/{semantic-concept-judge-DYXDPZW0.js → semantic-concept-judge-BuIJ9IfB.js} +43 -6
- package/dist/semantic-concept-judge-BuIJ9IfB.js.map +1 -0
- package/dist/{server-DLEvyW2z.js → server-DaCpLfi0.js} +3 -3
- package/dist/{server-DLEvyW2z.js.map → server-DaCpLfi0.js.map} +1 -1
- package/dist/single-run-lock-BTTtPZ9N.js +989 -0
- package/dist/single-run-lock-BTTtPZ9N.js.map +1 -0
- package/dist/{skill-usage-Bv3G4VkA.d.ts → skill-usage-B-BFS8M2.d.ts} +54 -39
- package/dist/skill-usage-B-BFS8M2.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-CjKMZy0d.js → skillopt-optimization-method-BbGnCC53.js} +18 -802
- package/dist/skillopt-optimization-method-BbGnCC53.js.map +1 -0
- package/dist/{skillopt-optimization-method-CzfnA8O-.d.ts → skillopt-optimization-method-_s0Tub7Y.d.ts} +11 -39
- package/dist/skillopt-optimization-method-_s0Tub7Y.d.ts.map +1 -0
- package/dist/{statistics-mf70aXKp.d.ts → statistics-B5d0Zd-z.d.ts} +2 -2
- package/dist/{statistics-mf70aXKp.d.ts.map → statistics-B5d0Zd-z.d.ts.map} +1 -1
- package/dist/store-otlp-DX4fGIcf.js +757 -0
- package/dist/store-otlp-DX4fGIcf.js.map +1 -0
- package/dist/{summary-report-BKinV4yD.d.ts → summary-report-Cg7BifAM.d.ts} +3 -3
- package/dist/{summary-report-BKinV4yD.d.ts.map → summary-report-Cg7BifAM.d.ts.map} +1 -1
- package/dist/tool-groups-CdYq22lX.d.ts +258 -0
- package/dist/tool-groups-CdYq22lX.d.ts.map +1 -0
- package/dist/traces.d.ts +7 -6
- package/dist/traces.js +5 -5
- package/dist/{types-zFYez3PK.d.ts → types-BBFNHxSK.d.ts} +5 -5
- package/dist/{types-zFYez3PK.d.ts.map → types-BBFNHxSK.d.ts.map} +1 -1
- package/dist/{types-BtJhn8v6.d.ts → types-DoEYskCd.d.ts} +5 -5
- package/dist/{types-BtJhn8v6.d.ts.map → types-DoEYskCd.d.ts.map} +1 -1
- package/dist/{types-5q2T25iW.d.ts → types-uPrS6mD-.d.ts} +2 -2
- package/dist/{types-5q2T25iW.d.ts.map → types-uPrS6mD-.d.ts.map} +1 -1
- package/dist/usage-receipt-CgxMEBZq.js +134 -0
- package/dist/usage-receipt-CgxMEBZq.js.map +1 -0
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/trace-analysis.md +170 -484
- package/package.json +1 -2
- package/dist/analyze-runs-CPYxfPWT.d.ts +0 -72
- package/dist/analyze-runs-CPYxfPWT.d.ts.map +0 -1
- package/dist/benchmark-command-CMqVqReF.js.map +0 -1
- package/dist/campaign-BIBS-NHV.js.map +0 -1
- package/dist/completion-verifier-B4-IMYcS.d.ts.map +0 -1
- package/dist/cost-ledger-B1D3COAc.d.ts.map +0 -1
- package/dist/cost-ledger-CHDLA0Ss.js.map +0 -1
- package/dist/default-registry-PUhIVRWz.d.ts +0 -215
- package/dist/default-registry-PUhIVRWz.d.ts.map +0 -1
- package/dist/default-registry-lp5R0lve.js.map +0 -1
- package/dist/ledger-core-C0Yx1I14.js.map +0 -1
- package/dist/registry-C4yJTza7.d.ts +0 -178
- package/dist/registry-C4yJTza7.d.ts.map +0 -1
- package/dist/replay-Cb-4Vf0k.js.map +0 -1
- package/dist/replay-DbIYwso6.d.ts.map +0 -1
- package/dist/semantic-concept-judge-DYXDPZW0.js.map +0 -1
- package/dist/single-run-lock-D_bS5xhj.js +0 -318
- package/dist/single-run-lock-D_bS5xhj.js.map +0 -1
- package/dist/skill-usage-Bv3G4VkA.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-CjKMZy0d.js.map +0 -1
- package/dist/skillopt-optimization-method-CzfnA8O-.d.ts.map +0 -1
- package/dist/store-otlp-BenKynPE.js +0 -1688
- package/dist/store-otlp-BenKynPE.js.map +0 -1
- package/dist/tools-DZGdROtG.js +0 -255
- package/dist/tools-DZGdROtG.js.map +0 -1
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import { c as ReplayError, l as ValidationError, o as LimitExceededError, r as CaptureIntegrityError, s as NotFoundError, t as AgentEvalError } from "./errors-DkfjIDvD.js";
|
|
2
|
+
import { c as CostLedgerHandle } from "./cost-ledger-DKgyIWRj.js";
|
|
2
3
|
import { S as ToolSpan, f as Run } from "./schema-BtVldJ3T.js";
|
|
3
|
-
import { a as RunRecord, l as RunTerminalOutcome, s as RunSplitTag, u as RunTokenUsage } from "./run-record-
|
|
4
|
-
import { _t as RawProviderSink, gt as RawProviderEvent } from "./types-
|
|
4
|
+
import { a as RunRecord, l as RunTerminalOutcome, s as RunSplitTag, u as RunTokenUsage } from "./run-record-CztDMXVF.js";
|
|
5
|
+
import { _t as RawProviderSink, gt as RawProviderEvent } from "./types-uPrS6mD-.js";
|
|
5
6
|
import { s as TraceStore } from "./store-CT9YIIve.js";
|
|
6
7
|
import { n as RunCompleteHookContext, t as RunCompleteHook } from "./emitter-DGQGoLyj.js";
|
|
7
|
-
import { D as SearchSpanResult, E as QueryTracesPage, F as TraceAnalystSpanStatus, L as ViewSpansResult, M as TraceAnalystFilters, O as SearchTraceResult, P as TraceAnalystSpanKind, S as TraceAnalysisStoreContext, w as DatasetOverview, x as TraceAnalysisStore, y as BoundedTraceAnalysisStoreOptions, z as ViewTraceResult } from "./types-
|
|
8
|
-
import {
|
|
9
|
-
import { r as CorrectnessChecker } from "./completion-verifier-B4-IMYcS.js";
|
|
10
|
-
import { t as AnalyzeRunsOptions } from "./analyze-runs-CPYxfPWT.js";
|
|
11
|
-
import { AxAIService, AxAgentActorTurnCallbackArgs, AxFunction } from "@ax-llm/ax";
|
|
8
|
+
import { D as SearchSpanResult, E as QueryTracesPage, F as TraceAnalystSpanStatus, L as ViewSpansResult, M as TraceAnalystFilters, O as SearchTraceResult, P as TraceAnalystSpanKind, S as TraceAnalysisStoreContext, f as AnalystUsageReceipt, i as AnalystFinding, w as DatasetOverview, x as TraceAnalysisStore, y as BoundedTraceAnalysisStoreOptions, z as ViewTraceResult } from "./types-DoEYskCd.js";
|
|
9
|
+
import { i as TraceAnalysisEngine, o as TraceAnalysisEngineResult, s as TraceAnalystLimits, t as TraceToolGroupName } from "./tool-groups-CdYq22lX.js";
|
|
12
10
|
//#region src/trace/otel.d.ts
|
|
13
11
|
declare const OTEL_AGENT_EVAL_SCOPE: {
|
|
14
12
|
name: string;
|
|
@@ -243,86 +241,38 @@ declare function createBoundedTraceAnalysisStore(source: TraceAnalysisStore, opt
|
|
|
243
241
|
//#endregion
|
|
244
242
|
//#region src/trace-analyst/analyst.d.ts
|
|
245
243
|
interface AnalyzeTracesInput {
|
|
246
|
-
/** The user-facing question. Domain framing belongs here, not in the
|
|
247
|
-
* actor description. */
|
|
248
244
|
question: string;
|
|
245
|
+
id?: string;
|
|
246
|
+
description?: string;
|
|
247
|
+
area?: string;
|
|
249
248
|
}
|
|
250
|
-
|
|
251
|
-
/** The actor's submitted prose answer. */
|
|
252
|
-
answer: string;
|
|
253
|
-
/** Bulleted findings from the actor's structured completion. */
|
|
254
|
-
findings: string[];
|
|
255
|
-
/** Per-turn snapshots captured via `actorTurnCallback`. */
|
|
256
|
-
turns: AnalyzeTracesTurnSnapshot[];
|
|
257
|
-
/** Total turns the actor took. */
|
|
258
|
-
turnCount: number;
|
|
259
|
-
/** Token usage by role. */
|
|
260
|
-
usage: TraceAnalystUsage;
|
|
261
|
-
/** Full system + assistant + tool message log by role. */
|
|
262
|
-
chatLog: TraceAnalystChatLog;
|
|
263
|
-
/** Prompt version that produced this run. */
|
|
264
|
-
actorPromptVersion: string;
|
|
265
|
-
}
|
|
266
|
-
interface TraceAnalystUsage {
|
|
267
|
-
actor: TraceAnalystUsageEntry[];
|
|
268
|
-
responder: TraceAnalystUsageEntry[];
|
|
269
|
-
}
|
|
270
|
-
interface TraceAnalystUsageEntry {
|
|
271
|
-
[key: string]: unknown;
|
|
272
|
-
}
|
|
273
|
-
interface TraceAnalystChatLog {
|
|
274
|
-
actor: TraceAnalystChatMessage[];
|
|
275
|
-
responder: TraceAnalystChatMessage[];
|
|
276
|
-
}
|
|
277
|
-
interface TraceAnalystChatMessage {
|
|
278
|
-
[key: string]: unknown;
|
|
279
|
-
}
|
|
280
|
-
interface AnalyzeTracesTurnSnapshot {
|
|
281
|
-
stage: AxAgentActorTurnCallbackArgs['stage'];
|
|
282
|
-
turn: number;
|
|
283
|
-
isError: boolean;
|
|
284
|
-
/** The JS code the actor produced for this turn. */
|
|
285
|
-
code: string;
|
|
286
|
-
/** The formatted action-log entry the actor sees on the next turn. */
|
|
287
|
-
output: string;
|
|
288
|
-
/** Provider thought (when `executorOptions.showThoughts` is true and the
|
|
289
|
-
* provider returns it). */
|
|
290
|
-
thought?: string;
|
|
291
|
-
}
|
|
249
|
+
type AnalyzeTracesResult = TraceAnalysisEngineResult;
|
|
292
250
|
interface AnalyzeTracesOptions {
|
|
293
|
-
/**
|
|
251
|
+
/** OTLP JSONL path or a caller-owned store. */
|
|
294
252
|
source: string | TraceAnalysisStore;
|
|
295
|
-
/**
|
|
296
|
-
|
|
297
|
-
/**
|
|
298
|
-
|
|
299
|
-
/**
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
253
|
+
/** Recursive research engine. */
|
|
254
|
+
engine: TraceAnalysisEngine;
|
|
255
|
+
/** Override the general trace-research policy. */
|
|
256
|
+
instructions?: string;
|
|
257
|
+
/** Smallest tool set that can answer the question. Default: all. */
|
|
258
|
+
toolGroup?: TraceToolGroupName;
|
|
259
|
+
limits?: Partial<TraceAnalystLimits>;
|
|
260
|
+
runId?: string;
|
|
261
|
+
budgetUsd?: number;
|
|
262
|
+
costLedger?: CostLedgerHandle;
|
|
263
|
+
costPhase?: string;
|
|
264
|
+
priorFindings?: readonly AnalystFinding[];
|
|
265
|
+
upstreamFindings?: readonly AnalystFinding[];
|
|
266
|
+
recordUsage?: (receipt: AnalystUsageReceipt) => void;
|
|
267
|
+
tags?: Record<string, string>;
|
|
268
|
+
log?: (message: string, fields?: Record<string, unknown>) => void;
|
|
306
269
|
signal?: AbortSignal;
|
|
307
|
-
/** Override the actor description. */
|
|
308
|
-
actorDescription?: string;
|
|
309
|
-
/** Per-turn observability hook. */
|
|
310
|
-
onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>;
|
|
311
|
-
/** Override max runtime characters per turn. Default 6000. */
|
|
312
|
-
maxRuntimeChars?: number;
|
|
313
|
-
/** When set, every turn's snapshot is appended to this JSONL file
|
|
314
|
-
* immediately. If the analyst crashes mid-loop (provider 503,
|
|
315
|
-
* network error, validator reject) the partial reasoning is still
|
|
316
|
-
* on disk for diagnosis and recovery. */
|
|
317
|
-
progressLogPath?: string;
|
|
318
270
|
}
|
|
319
271
|
/**
|
|
320
|
-
*
|
|
272
|
+
* Answer one question by recursively inspecting a trace store.
|
|
321
273
|
*
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
* - `AxAgentClarificationError` if the analyst asks for clarification.
|
|
325
|
-
* - Provider errors (auth, rate limits) propagate from the AI service.
|
|
274
|
+
* The returned answer, cited findings, engine steps, call counts, and runtime
|
|
275
|
+
* identity are one audit record. A direct one-shot model call is not used.
|
|
326
276
|
*/
|
|
327
277
|
declare function analyzeTraces(input: AnalyzeTracesInput, options: AnalyzeTracesOptions): Promise<AnalyzeTracesResult>;
|
|
328
278
|
//#endregion
|
|
@@ -678,9 +628,9 @@ declare function otlpToTraceRunRecords(otlpJsonl: string, opts: OtlpToRunRecords
|
|
|
678
628
|
declare function otlpRowsToTraceRunRecords(rows: Iterable<object>, opts: OtlpToRunRecordsOptions): OtlpTraceRunRecord[];
|
|
679
629
|
//#endregion
|
|
680
630
|
//#region src/trace-analyst/prompts.d.ts
|
|
681
|
-
/**
|
|
682
|
-
declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "
|
|
683
|
-
declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-
|
|
631
|
+
/** General policy for recursive, evidence-backed trace analysis. */
|
|
632
|
+
declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "Answer the question by inspecting the OTLP trace dataset with the available tools.\n\n1. Call getDatasetOverview first. Use its real trace ids and dataset size to plan the investigation.\n2. Narrow with queryTraces and countTraces before scanning large payloads.\n3. For a small trace, use viewTrace. For a large trace, use searchTrace and then viewSpans or searchSpan.\n4. Never invent a trace id, span id, tool result, error, frequency, or final outcome.\n5. When a search reports has_more, refine the query before drawing a conclusion.\n6. Use llm_query only over evidence already loaded. A recursive query cannot inspect traces itself.\n7. Cite exact evidence URIs returned by the tools. Include a short exact excerpt when it supports the claim.\n8. Return no finding when the available evidence cannot support one.\n\nThe prose answer must directly answer the question and state important uncertainty.\nThe findings array contains only actionable or decision-relevant claims supported by inspected evidence.";
|
|
633
|
+
declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-research-v1-2026-07-30";
|
|
684
634
|
//#endregion
|
|
685
635
|
//#region src/trace-analyst/store-otlp.d.ts
|
|
686
636
|
interface ToolSpansToTraceAnalysisStoreOptions {
|
|
@@ -785,65 +735,6 @@ declare class ToolTraceMissingError extends CaptureIntegrityError {
|
|
|
785
735
|
*/
|
|
786
736
|
declare function toolSpansToTraceAnalysisStore(spans: readonly ToolSpan[] | null | undefined, options?: ToolSpansToTraceAnalysisStoreOptions): TraceAnalysisStore;
|
|
787
737
|
//#endregion
|
|
788
|
-
//#region src/eval-tools.d.ts
|
|
789
|
-
/** One agent-callable tool. `parameters` is a JSON Schema (draft-07+) object. */
|
|
790
|
-
interface EvalToolDef {
|
|
791
|
-
name: string;
|
|
792
|
-
description: string;
|
|
793
|
-
parameters: Record<string, unknown>;
|
|
794
|
-
handler: (args: unknown, ctx?: {
|
|
795
|
-
signal?: AbortSignal;
|
|
796
|
-
}) => Promise<unknown>;
|
|
797
|
-
}
|
|
798
|
-
interface MakeEvalToolsConfig {
|
|
799
|
-
/** Judges available to `run_judges`. Omit to exclude the tool. */
|
|
800
|
-
judges?: Array<JudgeConfig<unknown>>;
|
|
801
|
-
/** Host-side correctness checker for `verify_completion` (the third
|
|
802
|
-
* `verifyCompletion` argument — a function, so it cannot cross the wire).
|
|
803
|
-
* Omit to exclude the tool. */
|
|
804
|
-
completion?: {
|
|
805
|
-
checkCorrectness: CorrectnessChecker;
|
|
806
|
-
};
|
|
807
|
-
/** `analyzeRuns` options minus `runs` (runs arrive as tool args, inline or
|
|
808
|
-
* via `path`). Omit to exclude the tool. */
|
|
809
|
-
analyze?: Omit<AnalyzeRunsOptions, 'runs'>;
|
|
810
|
-
}
|
|
811
|
-
/** OpenAI function-tool wire shape for an `EvalToolDef`. */
|
|
812
|
-
declare function toOpenAiTool(def: EvalToolDef): {
|
|
813
|
-
type: 'function';
|
|
814
|
-
function: {
|
|
815
|
-
name: string;
|
|
816
|
-
description: string;
|
|
817
|
-
parameters: Record<string, unknown>;
|
|
818
|
-
};
|
|
819
|
-
};
|
|
820
|
-
/**
|
|
821
|
-
* Build the eval toolset for the supplied config. Only sections present in
|
|
822
|
-
* `cfg` produce tools, so the agent's tool list mirrors what the host
|
|
823
|
-
* actually wired. Handlers fail loud on malformed args — no silent defaults.
|
|
824
|
-
*/
|
|
825
|
-
declare function makeEvalTools(cfg: MakeEvalToolsConfig): EvalToolDef[];
|
|
826
|
-
//#endregion
|
|
827
|
-
//#region src/trace-analyst/tools.d.ts
|
|
828
|
-
declare const TRACE_ANALYST_TOOL_NAMESPACE: 'traces';
|
|
829
|
-
interface BuildTraceAnalysisToolsOptions extends BoundedTraceAnalysisStoreOptions {
|
|
830
|
-
store: TraceAnalysisStore;
|
|
831
|
-
}
|
|
832
|
-
interface TraceAnalysisToolDescriptor extends EvalToolDef {
|
|
833
|
-
namespace: typeof TRACE_ANALYST_TOOL_NAMESPACE;
|
|
834
|
-
}
|
|
835
|
-
/** Bind all seven trace reads without exposing an agent framework type. */
|
|
836
|
-
declare function buildTraceAnalysisToolDescriptors(options: BuildTraceAnalysisToolsOptions): TraceAnalysisToolDescriptor[];
|
|
837
|
-
/** Adapt the canonical descriptors into Ax functions used by the built-in analyst. */
|
|
838
|
-
declare function buildTraceAnalystTools(options: BuildTraceAnalysisToolsOptions): AxFunction[];
|
|
839
|
-
declare function traceAnalystFunctionGroup(options: BuildTraceAnalysisToolsOptions): {
|
|
840
|
-
namespace: string;
|
|
841
|
-
title: string;
|
|
842
|
-
selectionCriteria: string;
|
|
843
|
-
description: string;
|
|
844
|
-
functions: AxFunction[];
|
|
845
|
-
};
|
|
846
|
-
//#endregion
|
|
847
738
|
//#region src/replay.d.ts
|
|
848
739
|
declare class ReplayCacheMissError extends ReplayError {
|
|
849
740
|
readonly url: string;
|
|
@@ -938,5 +829,5 @@ declare function iterateRawCalls(sink: RawProviderSink, filter?: {
|
|
|
938
829
|
spanId?: string;
|
|
939
830
|
}): AsyncGenerator<ReplayCacheEntry>;
|
|
940
831
|
//#endregion
|
|
941
|
-
export {
|
|
942
|
-
//# sourceMappingURL=replay-
|
|
832
|
+
export { traceAnalystOnRunComplete as $, stringField as A, OtlpSpanRoleInput as At, TraceInsightReadiness as B, exportRunAsOtlp as Bt, ProjectedOtlpSpan as C, createOtelTracingStore as Ct, inferOtlpKind as D, OtelExporter as Dt, firstStringAttr as E, OtelExportConfig as Et, TraceInsightFinding as F, traceSpanKindToOpenInferenceKind as Ft, defaultTraceInsightPanel as G, TraceInsightTask as H, TraceInsightPanelRole as I, OTEL_AGENT_EVAL_SCOPE as It, inferDomainKeywords as J, describeTraceInsightScope as K, TraceInsightPromptInput as L, OtlpExport as Lt, flattenOtlpExportToNdjson as M, applyToolSpanOtlpAttributes as Mt, OtlpFlatLine as N, classifyOtlpSpanRole as Nt, projectOtlpFlatLine as O, createOtelExporter as Ot, TraceInsightContext as P, isOtlpModelCall as Pt, TraceAnalystHookOptions as Q, TraceInsightQualityGate as R, OtlpResourceSpans as Rt, otlpToTraceRunRecords as S, redactValue as St, extractOtlpAttributes as T, ExportableSpan as Tt, buildTraceInsightContext as U, TraceInsightSuite as V, buildTraceInsightPrompt as W, scoreTraceInsightReadiness as X, planTraceInsightQuestions as Y, tokenizeDomainWords as Z, OtlpTraceRunRecord as _, DEFAULT_REDACTION_RULES as _t, ReplayFetchOptions as a, TraceFileMissingError as at, otlpRowsToTraceRunRecords as b, RedactionRule as bt, ToolTraceMissingError as c, AnalyzeTracesInput as ct, OtlpFileTraceStoreOptions as d, analyzeTraces as dt, SpanNotFoundError as et, ToolSpansToTraceAnalysisStoreOptions as f, createBoundedTraceAnalysisStore as ft, OtlpToRunRecordsOptions as g, convertTraceStoresToOtlp as gt, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION as h, TracesToOtlpResult as ht, ReplayCacheStats as i, TraceFileMalformedError as it, FlattenOtlpOptions as j, ToolSpanOtlpInput as jt, readOtlpStatus as k, OtlpSpanRole as kt, toolSpansToTraceAnalysisStore as l, AnalyzeTracesOptions as lt, TRACE_ANALYST_ACTOR_DESCRIPTION as m, TraceStoreToOtlpOptions as mt, ReplayCacheEntry as n, TraceAnalysisStoreContractError as nt, createReplayFetch as o, TraceFileTooLargeError as ot, otlpTextToTraceAnalysisStore as p, TraceStoreSource as pt, domainEvidencePattern as q, ReplayCacheMissError as r, TraceAnalysisValidationError as rt, iterateRawCalls as s, TraceNotFoundError as st, ReplayCache as t, TraceAnalysisLimitError as tt, OtlpFileTraceStore as u, AnalyzeTracesResult as ut, TraceAggregate as v, REDACTION_VERSION as vt, asString as w, otelRunCompleteHook as wt, otlpToRunRecords as x, redactString as xt, otlpRowsToRunRecords as y, RedactionReport as yt, TraceInsightQuestion as z, OtlpSpan as zt };
|
|
833
|
+
//# sourceMappingURL=replay-DjG4IG60.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replay-DjG4IG60.d.ts","names":[],"sources":["../src/trace/otel.ts","../src/trace/otlp-attributes.ts","../src/trace/otel-export.ts","../src/trace/otel-bridge.ts","../src/trace/redact.ts","../src/trace/store-to-otlp.ts","../src/trace-analyst/store-boundary.ts","../src/trace-analyst/analyst.ts","../src/trace-analyst/errors.ts","../src/trace-analyst/hook.ts","../src/trace-analyst/insights.ts","../src/trace/otlp-flat.ts","../src/trace-analyst/otlp-flatten.ts","../src/trace-analyst/otlp-span.ts","../src/trace-analyst/otlp-to-run-records.ts","../src/trace-analyst/prompts.ts","../src/trace-analyst/store-otlp.ts","../src/trace-analyst/store-tool-spans.ts","../src/replay.ts"],"mappings":";;;;;;;;;;cAoBa;EAA0B;EAAoC;;UAE1D;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,YAAY;IACV;IACA;MAAS;MAAsB;MAAmB;MAAsB;;;EAE1E,SAAS;IAAQ;IAAsB;IAAc,aAAa;;EAClE;IAAW;IAAc;;;UAGV;EACf;IAAY,YAAY;;EACxB,YAAY;IAAQ,cAAc;IAAuB,OAAO;;;UAGjD;EACf,eAAe;;;iBAIK,gBACpB,OAAO,YACP,eACA,gBAAe,4CACd,QAAQ;;;KC5BC,oBAAoB,KAC9B;KAIU;UAUK;EACf;EACA,YAAY;EACZ;;;;;;;iBAkBc,qBAAqB,OAAO,oBAAoB;iBA2BhD,gBAAgB,OAAO;iBAkBvB,4BACd,YAAY,yBACZ,MAAM;iBAMQ,iCAAiC;;;UC/FhC;;EAEf;;EAEA,UAAU;;EAEV;;EAEA;;EAEA,qBAAqB;;EAErB;;UAGe;;EAEf,WAAW,MAAM;;EAEjB,SAAS;;EAET,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;EACP,aAAa;;;;;;iBAOC,mBAAmB,SAAS,mBAAmB;;;;;;;iBC9C/C,oBAAoB,UAAU,eAAe;;;;;;;;;iBAoB7C,uBACd,OAAO,YACP,UAAU,cACV,kBACC;;;;;;;;;;;;;;;;UC/Bc;EACf;EACA,SAAS;;EAET;;UAGe;EACf;EACA,QAAQ;;;cAIG,yBAAyB;cAezB;;;;;iBAMG,aACd,eACA,QAAO;EACJ;EAAgB,QAAQ;;;;;;;iBAuBb,YACd,gBACA,QAAO,iBACP,SAAQ;EACL;EAAgB,QAAQ;;;;UC1CZ;;EAEf;;EAEA;;EAEA;;;EAGA;;;;;;;;;;;;UAae;;EAEf;;EAEA;;EAEA;;;;UAKe;;;EAGf;;;EAGA,sBAAsB,KAAK,QAAQ;;EAEnC,iBAAiB,KAAK,QAAQ;;;;;;;;;;;iBAYhB,yBACd,iBAAiB,4BAA4B,oBAC7C,iBACA,OAAM,0BACL;;;;iBChEa,gCACd,QAAQ,oBACR,UAAS,mCACR;;;UClBc;EACf;EACA;EACA;EACA;;KAGU,sBAAsB;UAEjB;;EAEf,iBAAiB;;EAEjB,QAAQ;;EAER;;EAEA,YAAY;EACZ,SAAS,QAAQ;EACjB;EACA;EACA,aAAa;EACb;EACA,yBAAyB;EACzB,4BAA4B;EAC5B,eAAe,SAAS;EACxB,OAAO;EACP,OAAO,iBAAiB,SAAS;EACjC,SAAS;;;;;;;;iBASW,cACpB,OAAO,oBACP,SAAS,uBACR,QAAQ;;;;cC7CE,qCAAqC;WAC/B;;;cAIN,gCAAgC;WAC1B;WACR;WACA;WACA;EAET,YAAY,mBAAmB,gBAAgB,eAAe;;;cASnD,wCAAwC;WAClC;WACR;EAET,YAAY,mBAAmB,iBAAiB;IAAY;;;cAMjD,8BAA8B;WACxB;WACR;EAET,YAAY;;cAMD,+BAA+B;WACjC;WACA;WACA;EAET,YAAY,cAAc,oBAAoB;;cAcnC,gCAAgC;WAClC;WACA;WACA;EAET,YAAY,cAAc,qBAAqB,qBAAqB;;cAQzD,2BAA2B;WACrB;WACR;EAET,YAAY;;cAMD,0BAA0B;WACpB;WACR;WACA;EAET,YAAY,kBAAkB;;;;UC1Ef;;;;;;EAMf,SAAS,KAAK;IAAoC,SAAS;;;;;;;EAM3D;;;;;;EAMA,QAAQ,QAAQ,qBAAqB,KAAK,2BAA2B;;;;;EAKrE,aAAa,KAAK;;;;;;EAMlB,UAAU,QAAQ,qBAAqB,KAAK;;iBAM9B,0BAA0B,MAAM,0BAA0B;;;UCzDzD;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf,OAAO;EACP,WAAW;EACX,QAAQ;EACR,SAAS;EACT;;UAGe;EACf,OAAO;EACP;EACA;EACA,WAAW;EACX,OAAO;EACP,UAAU;EACV,OAAO;EACP,QAAQ;;UAGO;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,OAAO;;iBA0BO,oBAAoB;iBAMpB,oBAAoB,OAAO;iBAuB3B,sBAAsB,qBAAqB;iBAS3C,0BAA0B,OAAO;iBAsBjC,0BAA0B,OAAO,0BAA0B;iBAsD3D,yBAAyB,OAAO,0BAA0B;iBAa1D,2BAA2B,SAAS,sBAAsB;iBA2D1D,4BAA4B;iBA6B5B,wBAAwB,OAAO;;;KChTnC;UAEK;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE,MAAM;IACN;;EAEF;IAAY,YAAY;;EACxB,YAAY;EACZ,SAAS;IAAQ;IAAc;IAAuB,aAAa;;;;;UCEpD;;;;EAIf;;EAEA,UAAU,QAAQ;;iBA+DJ,0BACd,YAAY,YACZ,OAAM,qBACL;;;;;;;;;UC1Dc;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;EAEA,YAAY;;;;;;iBAOE,oBAAoB,KAAK,0BAA0B;iBAkEnD,eAAe,KAAK;EAClC,MAAM;EACN;;iBAkBc,cAAc,OAAO,0BAA0B;;;;;;iBAwB/C,sBAAsB,KAAK,0BAA0B;iBAoBrD,YAAY,KAAK,yBAAyB;iBAK1C,SAAS;;iBAKT,gBACd,OAAO,yBACP;;;UCpIe;;EAEf;;;EAGA;;;EAGA,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;;;;;;EAOA;;;;;;EAMA;;;;;;;;;EASA,wBAAwB;;;;;;;;;EASxB,iBAAiB,eAAe,MAAM;;;;;EAKtC,yBAAyB,kBAAkB;;;;;;UAO5B;EACf,QAAQ;;EAER;;EAEA;;;UAIe;EACf;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,YAAY;;EAEZ;EACA;EACA;EACA;EACA;;EAEA,iBAAiB;;;;;;;iBAeH,iBAAiB,mBAAmB,MAAM,0BAA0B;;;;;;iBASpE,qBACd,MAAM,kBACN,MAAM,0BACL;;iBAKa,sBACd,mBACA,MAAM,0BACL;;iBAQa,0BACd,MAAM,kBACN,MAAM,0BACL;;;;cC9MU;cAcA;;;UC0HI;;EAEf;;EAEA;;EAEA;;EAEA;;KAGG,gCAAgC;UAEpB,kCAAkC;;EAEjD;;;;;;;;EAQA;;uBAMa,kCAAkC;mBAC9B;mBACA;mBACA;mBACA;UACT;UACA;UACA;UACA;EAER,YAAY,MAAM;EAyBZ,SAAS,kBAAkB,UAAU,4BAA4B;EAOjE,SACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT;EAeG,YACJ,UAAU,qBACV,UAAU,4BACT,QAAQ;EAoDL,YACJ;IAAQ,UAAU;IAAqB;IAAe;KACtD,UAAU,4BACT,QAAQ;EAmBL,YACJ,UAAU,qBACV,UAAU,4BACT;EAMG,UACJ;IAAQ;IAAkB;KAC1B,UAAU,4BACT,QAAQ;EAuCL,UACJ;IAAQ;IAAkB;IAA6B;KACvD,UAAU,4BACT,QAAQ;EAsFL,YACJ;IAAQ;IAAkB;IAAuB;KACjD,UAAU,4BACT,QAAQ;EA4CL,WACJ;IAAQ;IAAkB;IAAiB;IAAuB;KAClE,UAAU,4BACT,QAAQ;;;EA6CL,cAAc,UAAU,4BAA4B;UAI5C;qBAmBK,WAAW,SAAS,cAAc,QAAQ;qBAE1C;UAEL;UAmBA;UAsHA;UA+DN;UAkBA;UAwDA;UAmBM;;cA+CH,2BAA2B;mBACrB;mBACA;EAEjB,YAAY,MAAM;;;YAYF,WAAW,SAAS,cAAc,QAAQ;YAkBhD;;;iBAwBI,6BACd,cACA,OAAM,uCACL;;;;cC/7BU,8BAA8B;EACzC;;;;;;iBASc,8BACd,gBAAgB,+BAChB,UAAS,uCACR;;;cCSU,6BAA6B;WAEtB;WACA;EAFlB,YACkB,aACA,oBAChB;;UAMa;EACf,SAAS;EACT,UAAU;;UAGK;EACf;EACA,YAAY;EACZ,SAAS;;EAET;;;;;;;;;;cAWW;UACH;UACA;UACA;UACA;;;;;SAMK,SACX,MAAM,iBACN;IAAU;IAAgB;MACzB,QAAQ;;SASE,WAAW,QAAQ,qBAAqB,QAAQ;;EA4B7D;EAIA,SAAS;;EAUR,WAAW,iBAAiB;;;;;;EASvB,OAAO,uBAAuB,QAAQ;;UAM7B;;;;;;;EAOf;EACA,uBAAuB;;EAEvB,SAAS;IAAQ;IAAa;IAAkB;;;EAEhD,gBAAgB;IAAQ;IAAa;;;;;;;;;;;;;iBAavB,kBAAkB,OAAO,aAAa,OAAM,4BAAiC;;;;;;iBAsDtE,gBACrB,MAAM,iBACN;EAAU;EAAgB;IACzB,eAAe"}
|
|
@@ -2,158 +2,69 @@ import { n as CaptureIntegrityError, s as ReplayError } from "./errors-D-LKuDhb.
|
|
|
2
2
|
import { h as hashJson, p as canonicalize } from "./agent-profile-cell-CbfBm2g6.js";
|
|
3
3
|
import { a as providerFromBaseUrl, i as defaultProviderRedactor } from "./raw-provider-sink-BQd7mzyT.js";
|
|
4
4
|
import { LLM_INPUT_TOKENS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, OPENINFERENCE_SPAN_KIND, TOOL_NAME, applyLlmSpanOtlpAttributes } from "./trace-attributes.js";
|
|
5
|
-
import {
|
|
6
|
-
import { a as runTraceAnalysisLoop, r as buildTraceAnalystTools } from "./tools-DZGdROtG.js";
|
|
5
|
+
import { $ as spanEpochMillis, Y as firstStringAttr, Z as projectOtlpFlatLine, i as runTraceAnalyst, it as traceSpanKindToOpenInferenceKind, nt as classifyOtlpSpanRole, q as compareSpanTime, rt as isOtlpModelCall, tt as applyToolSpanOtlpAttributes } from "./kind-factory-CFxA0JQX.js";
|
|
7
6
|
import { s as validateRunRecord } from "./run-record-vRgqWmJw.js";
|
|
8
|
-
import {
|
|
7
|
+
import { n as OtlpFileTraceStore, r as createOtlpBufferTraceStore } from "./store-otlp-DX4fGIcf.js";
|
|
8
|
+
import { a as recordAggregateMeasurements, i as readTaskFailureLabels, o as summarizeExecutionMeasurements, r as extractUsageFromSse, s as summarizeTraceErrors, t as extractUsage } from "./extract-usage-DZs601Va.js";
|
|
9
9
|
import { readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
10
10
|
import { join } from "node:path";
|
|
11
|
+
import { randomUUID } from "node:crypto";
|
|
11
12
|
//#region src/trace-analyst/prompts.ts
|
|
12
|
-
/**
|
|
13
|
-
const TRACE_ANALYST_ACTOR_DESCRIPTION = `
|
|
13
|
+
/** General policy for recursive, evidence-backed trace analysis. */
|
|
14
|
+
const TRACE_ANALYST_ACTOR_DESCRIPTION = `Answer the question by inspecting the OTLP trace dataset with the available tools.
|
|
14
15
|
|
|
15
|
-
|
|
16
|
+
1. Call getDatasetOverview first. Use its real trace ids and dataset size to plan the investigation.
|
|
17
|
+
2. Narrow with queryTraces and countTraces before scanning large payloads.
|
|
18
|
+
3. For a small trace, use viewTrace. For a large trace, use searchTrace and then viewSpans or searchSpan.
|
|
19
|
+
4. Never invent a trace id, span id, tool result, error, frequency, or final outcome.
|
|
20
|
+
5. When a search reports has_more, refine the query before drawing a conclusion.
|
|
21
|
+
6. Use llm_query only over evidence already loaded. A recursive query cannot inspect traces itself.
|
|
22
|
+
7. Cite exact evidence URIs returned by the tools. Include a short exact excerpt when it supports the claim.
|
|
23
|
+
8. Return no finding when the available evidence cannot support one.
|
|
16
24
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
3. To list more traces than the sample, call \`traces.queryTraces({ filters?, limit, offset? })\`. Each summary carries raw_jsonl_bytes — use it to choose between viewTrace and searchTrace BEFORE calling either.
|
|
22
|
-
|
|
23
|
-
4. Per-trace inspection:
|
|
24
|
-
- SMALL trace (raw_jsonl_bytes well under 150_000): call \`traces.viewTrace({ trace_id })\`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large \`input.value\` / \`output.value\` / \`llm.input_messages\` will show a \`[trace-analyst truncated: N bytes]\` marker.
|
|
25
|
-
- LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an \`oversized\` response): use \`traces.searchTrace({ trace_id, regex_pattern })\` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call \`traces.viewSpans({ trace_id, span_ids: [...] })\` for surgical reads (~16KB cap, 4× higher than discovery), or \`traces.searchSpan({ trace_id, span_id, regex_pattern })\` for one large span. Stays bounded regardless of trace size.
|
|
26
|
-
- Useful regex patterns: \`STATUS_CODE_ERROR\` (failures), tool names like \`grep\` or \`view_trace\`, error strings like \`MaxTurnsExceeded\`, model names, attribute keys.
|
|
27
|
-
|
|
28
|
-
5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.
|
|
29
|
-
|
|
30
|
-
5a. **Result-shape contract** — searchTrace and searchSpan return \`{ trace_id, hits, has_more }\`. Iterate \`result.hits\` (NOT result.matches). Each hit has \`{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }\`. viewTrace returns \`{ trace_id, spans }\` (or \`oversized\`). viewSpans returns \`{ trace_id, spans, missing_span_ids, omitted_span_ids, has_more, truncated_attribute_count }\`; retry only \`omitted_span_ids\`. Never assume a field name — log the result shape first if unsure.
|
|
31
|
-
|
|
32
|
-
6. If viewTrace returns an \`oversized\` summary instead of \`spans\`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.
|
|
33
|
-
|
|
34
|
-
7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.
|
|
35
|
-
|
|
36
|
-
8. If a tool errors (invalid regex, range error), STOP and reconsider — don't retry with a guessed id or argument. Use the discovery tools above to recover.
|
|
37
|
-
|
|
38
|
-
9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.
|
|
39
|
-
|
|
40
|
-
10. If the question splits into independent reasoning branches, use bounded \`llmQuery(...)\` calls over evidence you already loaded. Subqueries cannot inspect the trace store, so pass the exact trace excerpts they need. Example:
|
|
41
|
-
|
|
42
|
-
const reviews = await llmQuery([
|
|
43
|
-
{ query: 'Classify the failure mechanism in this excerpt.', context: traceAbcExcerpt },
|
|
44
|
-
{ query: 'Classify the failure mechanism in this excerpt.', context: traceDefExcerpt },
|
|
45
|
-
]);
|
|
46
|
-
|
|
47
|
-
OBSERVABILITY rules:
|
|
48
|
-
- Each discovery turn must emit at least one concise \`console.log(...)\` showing what evidence was learned.
|
|
49
|
-
- Finish gathering evidence before submitting the analysis.
|
|
50
|
-
- Reuse runtime variables across turns; don't recompute.
|
|
51
|
-
|
|
52
|
-
OUTPUT contract — your final answer must include:
|
|
53
|
-
- A clear prose conclusion answering the user's question.
|
|
54
|
-
- Trace ids and span ids cited as evidence for each claim.
|
|
55
|
-
- Failure modes named in the user's domain language, with frequency and concrete examples.
|
|
56
|
-
- A concise findings array containing only claims supported by inspected evidence.
|
|
57
|
-
|
|
58
|
-
Do NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.`;
|
|
59
|
-
const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v6-2026-07-14";
|
|
25
|
+
The prose answer must directly answer the question and state important uncertainty.
|
|
26
|
+
The findings array contains only actionable or decision-relevant claims supported by inspected evidence.`;
|
|
27
|
+
const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-research-v1-2026-07-30";
|
|
60
28
|
//#endregion
|
|
61
29
|
//#region src/trace-analyst/analyst.ts
|
|
62
30
|
/**
|
|
63
|
-
*
|
|
31
|
+
* Answer one question by recursively inspecting a trace store.
|
|
64
32
|
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
* - `AxAgentClarificationError` if the analyst asks for clarification.
|
|
68
|
-
* - Provider errors (auth, rate limits) propagate from the AI service.
|
|
33
|
+
* The returned answer, cited findings, engine steps, call counts, and runtime
|
|
34
|
+
* identity are one audit record. A direct one-shot model call is not used.
|
|
69
35
|
*/
|
|
70
36
|
async function analyzeTraces(input, options) {
|
|
71
|
-
if (
|
|
72
|
-
|
|
37
|
+
if (typeof input.question !== "string" || !input.question.trim()) throw new TypeError("analyzeTraces: input.question must be a non-empty string");
|
|
38
|
+
const id = input.id?.trim() || "trace-analysis";
|
|
73
39
|
const store = typeof options.source === "string" ? new OtlpFileTraceStore({ path: options.source }) : options.source;
|
|
74
40
|
if (store instanceof OtlpFileTraceStore) await store.ensureIndexed(options.signal ? { signal: options.signal } : void 0);
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
const { dirname } = await import("node:path");
|
|
82
|
-
await mkdir(dirname(options.progressLogPath), { recursive: true });
|
|
83
|
-
progressFs = createWriteStream(options.progressLogPath, { flags: "a" });
|
|
84
|
-
}
|
|
85
|
-
const actorTurnCallback = async (turn) => {
|
|
86
|
-
const snap = {
|
|
87
|
-
stage: turn.stage,
|
|
88
|
-
turn: turn.turn,
|
|
89
|
-
isError: turn.isError,
|
|
90
|
-
code: turn.code,
|
|
91
|
-
output: turn.output,
|
|
92
|
-
thought: turn.thought
|
|
93
|
-
};
|
|
94
|
-
turns.push(snap);
|
|
95
|
-
if (progressFs) try {
|
|
96
|
-
progressFs.write(`${JSON.stringify({
|
|
97
|
-
...snap,
|
|
98
|
-
ts: Date.now()
|
|
99
|
-
})}\n`);
|
|
100
|
-
} catch {}
|
|
101
|
-
if (options.onTurn) await options.onTurn(snap);
|
|
102
|
-
};
|
|
103
|
-
const maxSubqueries = options.maxSubqueries ?? 4;
|
|
104
|
-
const maxTurns = options.maxTurns ?? 12;
|
|
105
|
-
const maxParallelSubqueries = options.maxParallelSubqueries ?? 2;
|
|
106
|
-
const maxRuntimeChars = options.maxRuntimeChars ?? 6e3;
|
|
107
|
-
let completed;
|
|
108
|
-
try {
|
|
109
|
-
completed = await runTraceAnalysisLoop({
|
|
110
|
-
id: "TraceAnalyst",
|
|
111
|
-
description: "Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes.",
|
|
112
|
-
prompt: `${options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION}
|
|
113
|
-
|
|
114
|
-
The report must answer the user's question, and findings must be an array of concise evidence-backed strings.`,
|
|
41
|
+
return runTraceAnalyst({
|
|
42
|
+
definition: {
|
|
43
|
+
id,
|
|
44
|
+
description: input.description?.trim() || "Answers a caller-defined question by recursively inspecting trace evidence.",
|
|
45
|
+
area: input.area?.trim() || "trace-analysis",
|
|
46
|
+
version: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
|
|
115
47
|
question: input.question,
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
findingType: "string",
|
|
120
|
-
maxSubqueries,
|
|
121
|
-
maxParallelSubqueries,
|
|
122
|
-
maxTurns,
|
|
123
|
-
maxRuntimeChars,
|
|
124
|
-
...options.signal ? { signal: options.signal } : {},
|
|
125
|
-
onTurn: actorTurnCallback
|
|
126
|
-
});
|
|
127
|
-
} finally {
|
|
128
|
-
if (progressFs) await new Promise((resolve) => progressFs.end(() => resolve()));
|
|
129
|
-
}
|
|
130
|
-
return {
|
|
131
|
-
answer: completed.report,
|
|
132
|
-
findings: completed.findings,
|
|
133
|
-
turns,
|
|
134
|
-
turnCount: turns.length,
|
|
135
|
-
usage: {
|
|
136
|
-
actor: normalizeRecordArray(completed.usage.actor),
|
|
137
|
-
responder: normalizeRecordArray(completed.usage.responder)
|
|
138
|
-
},
|
|
139
|
-
chatLog: {
|
|
140
|
-
actor: normalizeRecordArray(completed.chatLog.actor),
|
|
141
|
-
responder: normalizeRecordArray(completed.chatLog.responder)
|
|
48
|
+
instructions: options.instructions ?? TRACE_ANALYST_ACTOR_DESCRIPTION,
|
|
49
|
+
toolGroup: options.toolGroup ?? "all",
|
|
50
|
+
limits: options.limits
|
|
142
51
|
},
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
52
|
+
engine: options.engine,
|
|
53
|
+
store,
|
|
54
|
+
context: {
|
|
55
|
+
runId: options.runId ?? id,
|
|
56
|
+
correlationId: randomUUID(),
|
|
57
|
+
budgetUsd: options.budgetUsd,
|
|
58
|
+
costLedger: options.costLedger,
|
|
59
|
+
costPhase: options.costPhase ?? "trace-analysis",
|
|
60
|
+
priorFindings: options.priorFindings,
|
|
61
|
+
upstreamFindings: options.upstreamFindings,
|
|
62
|
+
recordUsage: options.recordUsage,
|
|
63
|
+
tags: options.tags,
|
|
64
|
+
log: options.log,
|
|
65
|
+
signal: options.signal
|
|
66
|
+
}
|
|
67
|
+
});
|
|
157
68
|
}
|
|
158
69
|
//#endregion
|
|
159
70
|
//#region src/trace-analyst/hook.ts
|
|
@@ -1960,4 +1871,4 @@ async function keyFromBody(body) {
|
|
|
1960
1871
|
//#endregion
|
|
1961
1872
|
export { TRACE_ANALYST_ACTOR_DESCRIPTION as A, domainEvidencePattern as C, tokenizeDomainWords as D, scoreTraceInsightReadiness as E, traceAnalystOnRunComplete as O, describeTraceInsightScope as S, planTraceInsightQuestions as T, otlpToTraceRunRecords as _, convertTraceStoresToOtlp as a, buildTraceInsightPrompt as b, otelRunCompleteHook as c, captureFetchToRawSink as d, ToolTraceMissingError as f, otlpToRunRecords as g, otlpRowsToTraceRunRecords as h, iterateRawCalls as i, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION as j, analyzeTraces as k, OTEL_AGENT_EVAL_SCOPE as l, otlpRowsToRunRecords as m, ReplayCacheMissError as n, createOtelExporter as o, toolSpansToTraceAnalysisStore as p, createReplayFetch as r, createOtelTracingStore as s, ReplayCache as t, exportRunAsOtlp as u, flattenOtlpExportToNdjson as v, inferDomainKeywords as w, defaultTraceInsightPanel as x, buildTraceInsightContext as y };
|
|
1962
1873
|
|
|
1963
|
-
//# sourceMappingURL=replay-
|
|
1874
|
+
//# sourceMappingURL=replay-SA4OB7O7.js.map
|