@tangle-network/agent-eval 0.27.2 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/dist/{baseline-4R5deP0N.d.ts → baseline-BwdCXUS8.d.ts} +1 -1
- package/dist/builder-eval/index.d.ts +3 -3
- package/dist/chunk-UW4NOOZI.js +1561 -0
- package/dist/chunk-UW4NOOZI.js.map +1 -0
- package/dist/{control-BT4qnXiS.d.ts → control-rJhEDdpy.d.ts} +4 -4
- package/dist/{control-runtime-BZ_lVLYW.d.ts → control-runtime-BRdQ0wrx.d.ts} +2 -2
- package/dist/control.d.ts +5 -5
- package/dist/{emitter-DP_cSSiw.d.ts → emitter-BqjeOvJh.d.ts} +1 -1
- package/dist/{failure-cluster-Cw65_5FY.d.ts → failure-cluster-D1NZKqYu.d.ts} +1 -1
- package/dist/{feedback-trajectory-D1aGKusy.d.ts → feedback-trajectory-j0nJFgC6.d.ts} +1 -1
- package/dist/governance/index.d.ts +2 -2
- package/dist/{index-BhLlu-qO.d.ts → index-Cgt3DKXr.d.ts} +1 -1
- package/dist/index.d.ts +1190 -335
- package/dist/index.js +1580 -489
- package/dist/index.js.map +1 -1
- package/dist/{integrity-DK2EBVZC.d.ts → integrity-BAxLGJ9I.d.ts} +2 -2
- package/dist/knowledge/index.d.ts +3 -3
- package/dist/meta-eval/index.d.ts +1 -1
- package/dist/{multi-layer-verifier-U-c8ge1k.d.ts → multi-layer-verifier-BNi4-8lR.d.ts} +1 -1
- package/dist/openapi.json +1 -1
- package/dist/optimization.d.ts +8 -8
- package/dist/pipelines/index.d.ts +6 -6
- package/dist/prm/index.d.ts +4 -4
- package/dist/{query-DODUYdPg.d.ts → query-BFDT0kX_.d.ts} +1 -1
- package/dist/{release-report-CCQqnK46.d.ts → release-report-PWhGlpfO.d.ts} +1 -1
- package/dist/replay-BX5Fm8en.d.ts +529 -0
- package/dist/reporting.d.ts +4 -4
- package/dist/{researcher-G81CWc0q.d.ts → researcher-ClDX3KZx.d.ts} +5 -5
- package/dist/rl.d.ts +8 -8
- package/dist/{rubric-D5tjHNJQ.d.ts → rubric-DgSqjqqj.d.ts} +2 -2
- package/dist/{store-Db2Bv8Cf.d.ts → store-BP5be6s7.d.ts} +1 -1
- package/dist/{summary-report-Dl4akLKX.d.ts → summary-report-jrSGb2xZ.d.ts} +1 -1
- package/dist/{test-graded-scenario-B2kWEdh9.d.ts → test-graded-scenario-BJ54PDan.d.ts} +2 -2
- package/dist/traces.d.ts +9 -311
- package/dist/traces.js +15 -986
- package/dist/traces.js.map +1 -1
- package/dist/{trajectory-CnoBo-JY.d.ts → trajectory-BFmveYZt.d.ts} +1 -1
- package/dist/wire/index.d.ts +4 -4
- package/package.json +1 -1
- package/dist/chunk-4U4BKCXK.js +0 -569
- package/dist/chunk-4U4BKCXK.js.map +0 -1
- package/dist/replay-D7z0J43-.d.ts +0 -225
package/dist/traces.js
CHANGED
|
@@ -1,17 +1,29 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DEFAULT_REDACTION_RULES,
|
|
3
|
+
DEFAULT_TRACE_ANALYST_BUDGETS,
|
|
3
4
|
FileSystemTraceStore,
|
|
4
5
|
InMemoryTraceStore,
|
|
5
6
|
OTEL_AGENT_EVAL_SCOPE,
|
|
7
|
+
OtlpFileTraceStore,
|
|
6
8
|
REDACTION_VERSION,
|
|
7
9
|
ReplayCache,
|
|
8
10
|
ReplayCacheMissError,
|
|
11
|
+
SpanNotFoundError,
|
|
12
|
+
TRACE_ANALYST_ACTOR_DESCRIPTION,
|
|
13
|
+
TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
|
|
14
|
+
TRACE_ANALYST_SUBAGENT_DESCRIPTION,
|
|
15
|
+
TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
|
|
16
|
+
TraceFileMissingError,
|
|
17
|
+
TraceNotFoundError,
|
|
18
|
+
analyzeTraces,
|
|
19
|
+
buildTraceAnalystTools,
|
|
9
20
|
createReplayFetch,
|
|
10
21
|
exportRunAsOtlp,
|
|
11
22
|
iterateRawCalls,
|
|
12
23
|
redactString,
|
|
13
|
-
redactValue
|
|
14
|
-
|
|
24
|
+
redactValue,
|
|
25
|
+
traceAnalystFunctionGroup
|
|
26
|
+
} from "./chunk-UW4NOOZI.js";
|
|
15
27
|
import {
|
|
16
28
|
aggregateLlm,
|
|
17
29
|
argHash,
|
|
@@ -48,992 +60,9 @@ import {
|
|
|
48
60
|
llmSpanFromProvider
|
|
49
61
|
} from "./chunk-TVVP3ZZQ.js";
|
|
50
62
|
import "./chunk-VSMTAMNK.js";
|
|
51
|
-
import
|
|
52
|
-
NotFoundError
|
|
53
|
-
} from "./chunk-NG236HPC.js";
|
|
63
|
+
import "./chunk-NG236HPC.js";
|
|
54
64
|
import "./chunk-PZ5AY32C.js";
|
|
55
65
|
|
|
56
|
-
// src/trace-analyst/analyst.ts
|
|
57
|
-
import { AxJSRuntime, agent } from "@ax-llm/ax";
|
|
58
|
-
|
|
59
|
-
// src/trace-analyst/prompts.ts
|
|
60
|
-
var TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \`traces\` namespace.
|
|
61
|
-
|
|
62
|
-
DISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol \u2014 follow exactly:
|
|
63
|
-
|
|
64
|
-
1. ALWAYS call \`traces.getDatasetOverview({})\` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids \u2014 never fabricate one).
|
|
65
|
-
|
|
66
|
-
2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. \`filters.regex_pattern\` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces \u2014 narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.
|
|
67
|
-
|
|
68
|
-
3. To list more traces than the sample, call \`traces.queryTraces({ filters?, limit, offset? })\`. Each summary carries raw_jsonl_bytes \u2014 use it to choose between viewTrace and searchTrace BEFORE calling either.
|
|
69
|
-
|
|
70
|
-
4. Per-trace inspection:
|
|
71
|
-
- SMALL trace (raw_jsonl_bytes well under 150_000): call \`traces.viewTrace({ trace_id })\`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large \`input.value\` / \`output.value\` / \`llm.input_messages\` will show a \`[trace-analyst truncated: N bytes]\` marker.
|
|
72
|
-
- LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an \`oversized\` response): use \`traces.searchTrace({ trace_id, regex_pattern })\` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call \`traces.viewSpans({ trace_id, span_ids: [...] })\` for surgical reads (~16KB cap, 4\xD7 higher than discovery), or \`traces.searchSpan({ trace_id, span_id, regex_pattern })\` for one large span. Stays bounded regardless of trace size.
|
|
73
|
-
- Useful regex patterns: \`STATUS_CODE_ERROR\` (failures), tool names like \`grep\` or \`view_trace\`, error strings like \`MaxTurnsExceeded\`, model names, attribute keys.
|
|
74
|
-
|
|
75
|
-
5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.
|
|
76
|
-
|
|
77
|
-
5a. **Result-shape contract** \u2014 searchTrace and searchSpan return \`{ trace_id, hits, total_matches, has_more }\`. Iterate \`result.hits\` (NOT result.matches). Each hit has \`{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }\`. viewTrace returns \`{ trace_id, spans }\` (or \`oversized\`). viewSpans returns \`{ trace_id, spans, missing_span_ids, truncated_attribute_count }\`. Never assume a field name \u2014 log the result shape first if unsure.
|
|
78
|
-
|
|
79
|
-
6. If viewTrace returns an \`oversized\` summary instead of \`spans\`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.
|
|
80
|
-
|
|
81
|
-
7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.
|
|
82
|
-
|
|
83
|
-
8. If a tool errors (invalid regex, range error), STOP and reconsider \u2014 don't retry with a guessed id or argument. Use the discovery tools above to recover.
|
|
84
|
-
|
|
85
|
-
9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.
|
|
86
|
-
|
|
87
|
-
10. If maxDepth > 0 and the question splits into independent semantic branches, delegate well-defined subtasks to subagents using \`await llmQuery(...)\`. Pass narrow context and a focused query. Examples:
|
|
88
|
-
|
|
89
|
-
const reviews = await llmQuery([
|
|
90
|
-
{ query: 'Drill into trace abc123 \u2014 what tool calls preceded the failure?', context: { trace_id: 'abc123' } },
|
|
91
|
-
{ query: 'Drill into trace def456 \u2014 same failure mode?', context: { trace_id: 'def456' } },
|
|
92
|
-
]);
|
|
93
|
-
|
|
94
|
-
OBSERVABILITY rules:
|
|
95
|
-
- Each non-final actor turn must emit at least one \`console.log(...)\` for evidence. Up to 3 logs per turn is fine when correlating multiple data sources (e.g. one log for findings list, one for source-file content, one for derived analysis).
|
|
96
|
-
- Do NOT combine \`console.log\` with \`final(...)\` or \`askClarification(...)\` in the same turn \u2014 finish gathering data first, then call final on its own turn.
|
|
97
|
-
- Reuse runtime variables across turns; don't recompute.
|
|
98
|
-
- When done, call \`await final(answer)\` with the fully-formed report. The responder rewrites the answer into output fields; if you only pass a vague summary string the responder has nothing concrete to format.
|
|
99
|
-
|
|
100
|
-
CRITICAL \u2014 \`final()\` payload contract for evidence-grounded analysis tasks:
|
|
101
|
-
- Pass a STRUCTURED object as the second arg with the actual data the responder needs to format the answer. Do NOT pass abstract instructions; pass evidence.
|
|
102
|
-
- Example for per-item verdict tasks:
|
|
103
|
-
\`\`\`js
|
|
104
|
-
await final("Format the per-item verdict report from the evidence below.", {
|
|
105
|
-
findings: [
|
|
106
|
-
{ id: 'sub-1-finding-1', claim: '...', verdict: 'TRUE-POSITIVE', evidence: 'lines 42-45 of contracts/X.sol show ...' },
|
|
107
|
-
...all items
|
|
108
|
-
],
|
|
109
|
-
systemic_summary: '3 sentences I wrote based on the evidence above'
|
|
110
|
-
});
|
|
111
|
-
\`\`\`
|
|
112
|
-
- Calling \`final("answer", {})\` with no evidence is a failure mode \u2014 the responder will hallucinate or echo back the field names. Always include the gathered data.
|
|
113
|
-
- Premature final after a single viewSpans call is INSUFFICIENT for per-finding analysis tasks. Read the requested attributes (e.g. \`spans[i].attributes['redteam.finding.title']\`), and for each one perform the requested cross-reference (e.g. read the source SPAN's \`attributes['source.content']\`).
|
|
114
|
-
|
|
115
|
-
OUTPUT contract \u2014 your final answer must include:
|
|
116
|
-
- A clear prose conclusion answering the user's question.
|
|
117
|
-
- Trace ids and span ids cited as evidence for each claim.
|
|
118
|
-
- Failure modes named in the user's domain language, with frequency and concrete examples.
|
|
119
|
-
|
|
120
|
-
Do NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.`;
|
|
121
|
-
var TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v5-2026-05-06";
|
|
122
|
-
var TRACE_ANALYST_SUBAGENT_DESCRIPTION = `You are a trace-analyst subagent. Your parent has delegated a focused trace-inspection question. Use the same DISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol but stay tightly scoped: do exactly what was asked, return a concise compact answer, do NOT spawn further subagents unless the parent's question is genuinely multi-branch.
|
|
123
|
-
|
|
124
|
-
Cite trace ids and span ids for every claim. Do NOT invent ids.`;
|
|
125
|
-
|
|
126
|
-
// src/trace-analyst/store-otlp.ts
|
|
127
|
-
import { readFile, stat } from "fs/promises";
|
|
128
|
-
|
|
129
|
-
// src/trace-analyst/store.ts
|
|
130
|
-
function compileSearchRegex(pattern) {
|
|
131
|
-
let source = pattern;
|
|
132
|
-
let flags = "m";
|
|
133
|
-
if (source.startsWith("(?i)")) {
|
|
134
|
-
source = source.slice(4);
|
|
135
|
-
flags += "i";
|
|
136
|
-
}
|
|
137
|
-
return new RegExp(source, flags);
|
|
138
|
-
}
|
|
139
|
-
function truncateForBudget(value, byteCap) {
|
|
140
|
-
const original = Buffer.byteLength(value, "utf8");
|
|
141
|
-
if (original <= byteCap) return value;
|
|
142
|
-
const ratio = byteCap / original;
|
|
143
|
-
let cut = Math.max(0, Math.floor(value.length * ratio));
|
|
144
|
-
while (cut > 0 && Buffer.byteLength(value.slice(0, cut), "utf8") > byteCap) {
|
|
145
|
-
cut -= 1;
|
|
146
|
-
}
|
|
147
|
-
return `${value.slice(0, cut)}
|
|
148
|
-
[trace-analyst truncated: original ${original} bytes]`;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
// src/trace-analyst/types.ts
|
|
152
|
-
var DEFAULT_TRACE_ANALYST_BUDGETS = {
|
|
153
|
-
perCallByteCeiling: 15e4,
|
|
154
|
-
perAttributeViewBudget: 4096,
|
|
155
|
-
perAttributeSpanBudget: 16384,
|
|
156
|
-
perMatchTextBudget: 1024
|
|
157
|
-
};
|
|
158
|
-
var TRACE_ANALYST_TRUNCATION_MARKER_PREFIX = "[trace-analyst truncated:";
|
|
159
|
-
|
|
160
|
-
// src/trace-analyst/store-otlp.ts
|
|
161
|
-
var OtlpFileTraceStore = class {
|
|
162
|
-
path;
|
|
163
|
-
perAttributeViewBudget;
|
|
164
|
-
perAttributeSpanBudget;
|
|
165
|
-
perCallByteCeiling;
|
|
166
|
-
perMatchTextBudget;
|
|
167
|
-
indexPromise;
|
|
168
|
-
/** Cached UTF-8 buffer of the file. We pin it once because every
|
|
169
|
-
* read needs slice access and re-reading on each call balloons the
|
|
170
|
-
* syscall count. */
|
|
171
|
-
bufferPromise;
|
|
172
|
-
constructor(opts) {
|
|
173
|
-
this.path = opts.path;
|
|
174
|
-
this.perAttributeViewBudget = opts.perAttributeViewBudget ?? DEFAULT_TRACE_ANALYST_BUDGETS.perAttributeViewBudget;
|
|
175
|
-
this.perAttributeSpanBudget = opts.perAttributeSpanBudget ?? DEFAULT_TRACE_ANALYST_BUDGETS.perAttributeSpanBudget;
|
|
176
|
-
this.perCallByteCeiling = opts.perCallByteCeiling ?? DEFAULT_TRACE_ANALYST_BUDGETS.perCallByteCeiling;
|
|
177
|
-
this.perMatchTextBudget = opts.perMatchTextBudget ?? DEFAULT_TRACE_ANALYST_BUDGETS.perMatchTextBudget;
|
|
178
|
-
}
|
|
179
|
-
// ─── Public API ────────────────────────────────────────────────────
|
|
180
|
-
async getOverview(filters) {
|
|
181
|
-
const idx = await this.index();
|
|
182
|
-
const matched = await this.matchedTraces(idx, filters);
|
|
183
|
-
const services = /* @__PURE__ */ new Set();
|
|
184
|
-
const agents = /* @__PURE__ */ new Set();
|
|
185
|
-
const models = /* @__PURE__ */ new Set();
|
|
186
|
-
const tools = /* @__PURE__ */ new Set();
|
|
187
|
-
let rawBytes = 0;
|
|
188
|
-
let earliest = null;
|
|
189
|
-
let latest = null;
|
|
190
|
-
let errorTraceCount = 0;
|
|
191
|
-
let errorSpanCount = 0;
|
|
192
|
-
for (const t of matched) {
|
|
193
|
-
if (t.service_name) services.add(t.service_name);
|
|
194
|
-
if (t.agent_name) agents.add(t.agent_name);
|
|
195
|
-
for (const m of t.models) models.add(m);
|
|
196
|
-
for (const tn of t.tools) tools.add(tn);
|
|
197
|
-
rawBytes += t.raw_jsonl_bytes;
|
|
198
|
-
if (!earliest || t.start_time < earliest) earliest = t.start_time;
|
|
199
|
-
if (!latest || t.end_time > latest) latest = t.end_time;
|
|
200
|
-
if (t.has_errors) {
|
|
201
|
-
errorTraceCount += 1;
|
|
202
|
-
for (const s of t.spans) if (s.status === "ERROR") errorSpanCount += 1;
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
const sample_trace_ids = matched.slice(0, 20).map((t) => t.trace_id);
|
|
206
|
-
return {
|
|
207
|
-
total_traces: matched.length,
|
|
208
|
-
raw_jsonl_bytes: rawBytes,
|
|
209
|
-
services: [...services].sort(),
|
|
210
|
-
agents: [...agents].sort(),
|
|
211
|
-
models: [...models].sort(),
|
|
212
|
-
tool_names: [...tools].sort(),
|
|
213
|
-
sample_trace_ids,
|
|
214
|
-
errors: { trace_count: errorTraceCount, span_count: errorSpanCount },
|
|
215
|
-
time_range: earliest && latest ? { earliest, latest } : null
|
|
216
|
-
};
|
|
217
|
-
}
|
|
218
|
-
async queryTraces(opts) {
|
|
219
|
-
if (!Number.isInteger(opts.limit) || opts.limit < 1 || opts.limit > 200) {
|
|
220
|
-
throw new RangeError(`queryTraces.limit must be 1..200, got ${opts.limit}`);
|
|
221
|
-
}
|
|
222
|
-
const offset = opts.offset ?? 0;
|
|
223
|
-
if (!Number.isInteger(offset) || offset < 0) {
|
|
224
|
-
throw new RangeError(`queryTraces.offset must be >=0, got ${offset}`);
|
|
225
|
-
}
|
|
226
|
-
const idx = await this.index();
|
|
227
|
-
const matched = await this.matchedTraces(idx, opts.filters);
|
|
228
|
-
const slice = matched.slice(offset, offset + opts.limit);
|
|
229
|
-
return {
|
|
230
|
-
traces: slice.map((t) => this.toSummary(t)),
|
|
231
|
-
total: matched.length,
|
|
232
|
-
has_more: offset + slice.length < matched.length
|
|
233
|
-
};
|
|
234
|
-
}
|
|
235
|
-
async countTraces(filters) {
|
|
236
|
-
const idx = await this.index();
|
|
237
|
-
const matched = await this.matchedTraces(idx, filters);
|
|
238
|
-
return matched.length;
|
|
239
|
-
}
|
|
240
|
-
async viewTrace(opts) {
|
|
241
|
-
const idx = await this.index();
|
|
242
|
-
const trace = idx.byTrace.get(opts.trace_id);
|
|
243
|
-
if (!trace) {
|
|
244
|
-
throw new TraceNotFoundError(opts.trace_id);
|
|
245
|
-
}
|
|
246
|
-
const cap = opts.per_attribute_byte_cap ?? this.perAttributeViewBudget;
|
|
247
|
-
const buf = await this.buffer();
|
|
248
|
-
const spans = [];
|
|
249
|
-
let runningBytes = 0;
|
|
250
|
-
let span_response_bytes_max = 0;
|
|
251
|
-
for (const s of trace.spans) {
|
|
252
|
-
const projected = await this.projectSpan(buf, trace.trace_id, s, cap);
|
|
253
|
-
const bytes = Buffer.byteLength(JSON.stringify(projected), "utf8");
|
|
254
|
-
span_response_bytes_max = Math.max(span_response_bytes_max, bytes);
|
|
255
|
-
runningBytes += bytes;
|
|
256
|
-
if (runningBytes > this.perCallByteCeiling) {
|
|
257
|
-
return {
|
|
258
|
-
trace_id: trace.trace_id,
|
|
259
|
-
oversized: this.buildOversizedSummary(trace, span_response_bytes_max)
|
|
260
|
-
};
|
|
261
|
-
}
|
|
262
|
-
spans.push(projected);
|
|
263
|
-
}
|
|
264
|
-
return { trace_id: trace.trace_id, spans };
|
|
265
|
-
}
|
|
266
|
-
async viewSpans(opts) {
|
|
267
|
-
const idx = await this.index();
|
|
268
|
-
const trace = idx.byTrace.get(opts.trace_id);
|
|
269
|
-
if (!trace) throw new TraceNotFoundError(opts.trace_id);
|
|
270
|
-
if (opts.span_ids.length === 0) {
|
|
271
|
-
return {
|
|
272
|
-
trace_id: trace.trace_id,
|
|
273
|
-
spans: [],
|
|
274
|
-
missing_span_ids: [],
|
|
275
|
-
truncated_attribute_count: 0
|
|
276
|
-
};
|
|
277
|
-
}
|
|
278
|
-
if (opts.span_ids.length > 100) {
|
|
279
|
-
throw new RangeError(`viewSpans.span_ids cap is 100, got ${opts.span_ids.length}`);
|
|
280
|
-
}
|
|
281
|
-
const cap = opts.per_attribute_byte_cap ?? this.perAttributeSpanBudget;
|
|
282
|
-
const wantSet = new Set(opts.span_ids);
|
|
283
|
-
const found = trace.spans.filter((s) => wantSet.has(s.span_id));
|
|
284
|
-
const missing = opts.span_ids.filter((id) => !found.some((f2) => f2.span_id === id));
|
|
285
|
-
const buf = await this.buffer();
|
|
286
|
-
const spans = [];
|
|
287
|
-
let truncated = 0;
|
|
288
|
-
let runningBytes = 0;
|
|
289
|
-
for (const s of found) {
|
|
290
|
-
const before = truncationCounter(this);
|
|
291
|
-
const projected = await this.projectSpan(buf, trace.trace_id, s, cap);
|
|
292
|
-
truncated += before.delta();
|
|
293
|
-
const bytes = Buffer.byteLength(JSON.stringify(projected), "utf8");
|
|
294
|
-
runningBytes += bytes;
|
|
295
|
-
if (runningBytes > this.perCallByteCeiling) {
|
|
296
|
-
break;
|
|
297
|
-
}
|
|
298
|
-
spans.push(projected);
|
|
299
|
-
}
|
|
300
|
-
return {
|
|
301
|
-
trace_id: trace.trace_id,
|
|
302
|
-
spans,
|
|
303
|
-
missing_span_ids: missing,
|
|
304
|
-
truncated_attribute_count: truncated
|
|
305
|
-
};
|
|
306
|
-
}
|
|
307
|
-
async searchTrace(opts) {
|
|
308
|
-
const max_matches = opts.max_matches ?? 50;
|
|
309
|
-
if (!Number.isInteger(max_matches) || max_matches < 1 || max_matches > 500) {
|
|
310
|
-
throw new RangeError(`searchTrace.max_matches must be 1..500, got ${max_matches}`);
|
|
311
|
-
}
|
|
312
|
-
const idx = await this.index();
|
|
313
|
-
const trace = idx.byTrace.get(opts.trace_id);
|
|
314
|
-
if (!trace) throw new TraceNotFoundError(opts.trace_id);
|
|
315
|
-
const re = compileSearchRegex(opts.regex_pattern);
|
|
316
|
-
const buf = await this.buffer();
|
|
317
|
-
const hits = [];
|
|
318
|
-
let total = 0;
|
|
319
|
-
let capped = false;
|
|
320
|
-
for (const s of trace.spans) {
|
|
321
|
-
const remaining = max_matches - hits.length;
|
|
322
|
-
const localHits = await this.scanSpanForMatches(
|
|
323
|
-
buf,
|
|
324
|
-
trace.trace_id,
|
|
325
|
-
s,
|
|
326
|
-
re,
|
|
327
|
-
this.perMatchTextBudget,
|
|
328
|
-
remaining
|
|
329
|
-
);
|
|
330
|
-
total += localHits.total;
|
|
331
|
-
for (const h of localHits.records) {
|
|
332
|
-
if (hits.length >= max_matches) break;
|
|
333
|
-
hits.push(h);
|
|
334
|
-
}
|
|
335
|
-
if (hits.length >= max_matches) {
|
|
336
|
-
capped = true;
|
|
337
|
-
total = Math.max(total, hits.length + 1);
|
|
338
|
-
break;
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
return {
|
|
342
|
-
trace_id: trace.trace_id,
|
|
343
|
-
hits,
|
|
344
|
-
total_matches: total,
|
|
345
|
-
has_more: capped || total > hits.length
|
|
346
|
-
};
|
|
347
|
-
}
|
|
348
|
-
async searchSpan(opts) {
|
|
349
|
-
const max_matches = opts.max_matches ?? 50;
|
|
350
|
-
if (!Number.isInteger(max_matches) || max_matches < 1 || max_matches > 500) {
|
|
351
|
-
throw new RangeError(`searchSpan.max_matches must be 1..500, got ${max_matches}`);
|
|
352
|
-
}
|
|
353
|
-
const idx = await this.index();
|
|
354
|
-
const trace = idx.byTrace.get(opts.trace_id);
|
|
355
|
-
if (!trace) throw new TraceNotFoundError(opts.trace_id);
|
|
356
|
-
const span = trace.spans.find((s) => s.span_id === opts.span_id);
|
|
357
|
-
if (!span) {
|
|
358
|
-
throw new SpanNotFoundError(opts.trace_id, opts.span_id);
|
|
359
|
-
}
|
|
360
|
-
const re = compileSearchRegex(opts.regex_pattern);
|
|
361
|
-
const buf = await this.buffer();
|
|
362
|
-
const localHits = await this.scanSpanForMatches(
|
|
363
|
-
buf,
|
|
364
|
-
trace.trace_id,
|
|
365
|
-
span,
|
|
366
|
-
re,
|
|
367
|
-
this.perMatchTextBudget,
|
|
368
|
-
max_matches
|
|
369
|
-
);
|
|
370
|
-
return {
|
|
371
|
-
trace_id: trace.trace_id,
|
|
372
|
-
span_id: span.span_id,
|
|
373
|
-
hits: localHits.records,
|
|
374
|
-
total_matches: localHits.total,
|
|
375
|
-
has_more: localHits.total > localHits.records.length
|
|
376
|
-
};
|
|
377
|
-
}
|
|
378
|
-
// ─── Index building ────────────────────────────────────────────────
|
|
379
|
-
/** Force the index to materialise. Useful to amortise startup cost
|
|
380
|
-
* before the first agent call. */
|
|
381
|
-
async ensureIndexed() {
|
|
382
|
-
await this.index();
|
|
383
|
-
}
|
|
384
|
-
async buffer() {
|
|
385
|
-
if (!this.bufferPromise) {
|
|
386
|
-
this.bufferPromise = readFile(this.path);
|
|
387
|
-
}
|
|
388
|
-
return this.bufferPromise;
|
|
389
|
-
}
|
|
390
|
-
async index() {
|
|
391
|
-
if (!this.indexPromise) {
|
|
392
|
-
this.indexPromise = this.buildIndex();
|
|
393
|
-
}
|
|
394
|
-
return this.indexPromise;
|
|
395
|
-
}
|
|
396
|
-
async buildIndex() {
|
|
397
|
-
let buf;
|
|
398
|
-
try {
|
|
399
|
-
buf = await this.buffer();
|
|
400
|
-
} catch (err) {
|
|
401
|
-
const stats = await stat(this.path).catch(() => null);
|
|
402
|
-
if (!stats) {
|
|
403
|
-
throw new TraceFileMissingError(this.path);
|
|
404
|
-
}
|
|
405
|
-
throw err;
|
|
406
|
-
}
|
|
407
|
-
const byTrace = /* @__PURE__ */ new Map();
|
|
408
|
-
let cursor = 0;
|
|
409
|
-
while (cursor < buf.length) {
|
|
410
|
-
const newlineIndex = buf.indexOf(10, cursor);
|
|
411
|
-
const lineEnd = newlineIndex === -1 ? buf.length : newlineIndex;
|
|
412
|
-
const lineLength = lineEnd - cursor;
|
|
413
|
-
if (lineLength === 0) {
|
|
414
|
-
cursor = lineEnd + 1;
|
|
415
|
-
continue;
|
|
416
|
-
}
|
|
417
|
-
const lineSlice = buf.subarray(cursor, lineEnd).toString("utf8");
|
|
418
|
-
const lineOffset = cursor;
|
|
419
|
-
cursor = lineEnd + 1;
|
|
420
|
-
let parsed;
|
|
421
|
-
try {
|
|
422
|
-
parsed = JSON.parse(lineSlice);
|
|
423
|
-
} catch {
|
|
424
|
-
continue;
|
|
425
|
-
}
|
|
426
|
-
if (!parsed || typeof parsed !== "object") continue;
|
|
427
|
-
const span = readOtlpSpan(parsed);
|
|
428
|
-
if (!span) continue;
|
|
429
|
-
let entry = byTrace.get(span.trace_id);
|
|
430
|
-
if (!entry) {
|
|
431
|
-
entry = {
|
|
432
|
-
trace_id: span.trace_id,
|
|
433
|
-
service_name: span.service_name,
|
|
434
|
-
agent_name: span.agent_name,
|
|
435
|
-
span_count: 0,
|
|
436
|
-
has_errors: false,
|
|
437
|
-
start_time: span.start_time,
|
|
438
|
-
end_time: span.end_time,
|
|
439
|
-
duration_ms: 0,
|
|
440
|
-
raw_jsonl_bytes: 0,
|
|
441
|
-
models: /* @__PURE__ */ new Set(),
|
|
442
|
-
tools: /* @__PURE__ */ new Set(),
|
|
443
|
-
spans: []
|
|
444
|
-
};
|
|
445
|
-
byTrace.set(span.trace_id, entry);
|
|
446
|
-
} else {
|
|
447
|
-
if (!entry.service_name && span.service_name) entry.service_name = span.service_name;
|
|
448
|
-
if (!entry.agent_name && span.agent_name) entry.agent_name = span.agent_name;
|
|
449
|
-
}
|
|
450
|
-
const indexEntry = {
|
|
451
|
-
span_id: span.span_id,
|
|
452
|
-
parent_span_id: span.parent_span_id,
|
|
453
|
-
name: span.name,
|
|
454
|
-
kind: span.kind,
|
|
455
|
-
start_time: span.start_time,
|
|
456
|
-
end_time: span.end_time,
|
|
457
|
-
duration_ms: span.duration_ms,
|
|
458
|
-
status: span.status,
|
|
459
|
-
status_message: span.status_message,
|
|
460
|
-
service_name: span.service_name,
|
|
461
|
-
agent_name: span.agent_name,
|
|
462
|
-
model_name: span.model_name,
|
|
463
|
-
tool_name: span.tool_name,
|
|
464
|
-
line_byte_offset: lineOffset,
|
|
465
|
-
line_byte_length: lineLength
|
|
466
|
-
};
|
|
467
|
-
entry.spans.push(indexEntry);
|
|
468
|
-
entry.span_count += 1;
|
|
469
|
-
entry.raw_jsonl_bytes += lineLength + 1;
|
|
470
|
-
if (span.status === "ERROR") entry.has_errors = true;
|
|
471
|
-
if (span.start_time < entry.start_time) entry.start_time = span.start_time;
|
|
472
|
-
if (span.end_time > entry.end_time) entry.end_time = span.end_time;
|
|
473
|
-
if (span.model_name) entry.models.add(span.model_name);
|
|
474
|
-
if (span.tool_name) entry.tools.add(span.tool_name);
|
|
475
|
-
}
|
|
476
|
-
let totalRawBytes = 0;
|
|
477
|
-
for (const t of byTrace.values()) {
|
|
478
|
-
totalRawBytes += t.raw_jsonl_bytes;
|
|
479
|
-
t.spans.sort(
|
|
480
|
-
(a, b) => a.start_time.localeCompare(b.start_time) || a.line_byte_offset - b.line_byte_offset
|
|
481
|
-
);
|
|
482
|
-
t.duration_ms = Math.max(0, new Date(t.end_time).getTime() - new Date(t.start_time).getTime());
|
|
483
|
-
}
|
|
484
|
-
const sortedTraceIds = [...byTrace.keys()].sort();
|
|
485
|
-
return { byTrace, totalRawBytes, sortedTraceIds };
|
|
486
|
-
}
|
|
487
|
-
// ─── Filter pipeline ───────────────────────────────────────────────
|
|
488
|
-
async matchedTraces(idx, filters) {
|
|
489
|
-
const traces = idx.sortedTraceIds.map((id) => idx.byTrace.get(id)).filter(isPresent);
|
|
490
|
-
if (!filters) return traces;
|
|
491
|
-
const indexedFiltered = traces.filter((t) => {
|
|
492
|
-
if (filters.has_errors !== void 0 && t.has_errors !== filters.has_errors) return false;
|
|
493
|
-
if (filters.service_names && filters.service_names.length > 0) {
|
|
494
|
-
if (!t.service_name || !filters.service_names.includes(t.service_name)) return false;
|
|
495
|
-
}
|
|
496
|
-
if (filters.agent_names && filters.agent_names.length > 0) {
|
|
497
|
-
if (!t.agent_name || !filters.agent_names.includes(t.agent_name)) return false;
|
|
498
|
-
}
|
|
499
|
-
if (filters.model_names && filters.model_names.length > 0) {
|
|
500
|
-
if (![...t.models].some((m) => filters.model_names.includes(m))) return false;
|
|
501
|
-
}
|
|
502
|
-
if (filters.tool_names && filters.tool_names.length > 0) {
|
|
503
|
-
if (![...t.tools].some((tn) => filters.tool_names.includes(tn))) return false;
|
|
504
|
-
}
|
|
505
|
-
if (filters.start_time_after && t.start_time < filters.start_time_after) return false;
|
|
506
|
-
if (filters.start_time_before && t.start_time > filters.start_time_before) return false;
|
|
507
|
-
return true;
|
|
508
|
-
});
|
|
509
|
-
if (!filters.regex_pattern) return indexedFiltered;
|
|
510
|
-
const re = compileSearchRegex(filters.regex_pattern);
|
|
511
|
-
const buf = await this.buffer();
|
|
512
|
-
const out = [];
|
|
513
|
-
for (const t of indexedFiltered) {
|
|
514
|
-
let matched = false;
|
|
515
|
-
for (const s of t.spans) {
|
|
516
|
-
const slice = buf.subarray(s.line_byte_offset, s.line_byte_offset + s.line_byte_length);
|
|
517
|
-
if (re.test(slice.toString("utf8"))) {
|
|
518
|
-
matched = true;
|
|
519
|
-
break;
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
if (matched) out.push(t);
|
|
523
|
-
}
|
|
524
|
-
return out;
|
|
525
|
-
}
|
|
526
|
-
toSummary(t) {
|
|
527
|
-
return {
|
|
528
|
-
trace_id: t.trace_id,
|
|
529
|
-
service_name: t.service_name,
|
|
530
|
-
agent_name: t.agent_name,
|
|
531
|
-
span_count: t.span_count,
|
|
532
|
-
has_errors: t.has_errors,
|
|
533
|
-
start_time: t.start_time,
|
|
534
|
-
end_time: t.end_time,
|
|
535
|
-
duration_ms: t.duration_ms,
|
|
536
|
-
raw_jsonl_bytes: t.raw_jsonl_bytes,
|
|
537
|
-
models: [...t.models].sort(),
|
|
538
|
-
tools: [...t.tools].sort()
|
|
539
|
-
};
|
|
540
|
-
}
|
|
541
|
-
// ─── Span projection (lazy attribute reads) ────────────────────────
|
|
542
|
-
async projectSpan(buf, trace_id, s, perAttrCap) {
|
|
543
|
-
const slice = buf.subarray(s.line_byte_offset, s.line_byte_offset + s.line_byte_length).toString("utf8");
|
|
544
|
-
let raw = {};
|
|
545
|
-
try {
|
|
546
|
-
const parsed = JSON.parse(slice);
|
|
547
|
-
if (parsed && typeof parsed === "object") raw = parsed;
|
|
548
|
-
} catch {
|
|
549
|
-
}
|
|
550
|
-
const attrs = extractAttributes(raw);
|
|
551
|
-
const projected = {};
|
|
552
|
-
for (const [k, v] of Object.entries(attrs)) {
|
|
553
|
-
if (typeof v === "string") {
|
|
554
|
-
const trunc = truncateForBudget(v, perAttrCap);
|
|
555
|
-
if (trunc !== v) trackTruncation(this);
|
|
556
|
-
projected[k] = trunc;
|
|
557
|
-
} else if (Array.isArray(v) || v && typeof v === "object") {
|
|
558
|
-
const json = JSON.stringify(v);
|
|
559
|
-
const trunc = truncateForBudget(json, perAttrCap);
|
|
560
|
-
if (trunc !== json) {
|
|
561
|
-
trackTruncation(this);
|
|
562
|
-
projected[k] = trunc;
|
|
563
|
-
} else {
|
|
564
|
-
projected[k] = v;
|
|
565
|
-
}
|
|
566
|
-
} else {
|
|
567
|
-
projected[k] = v;
|
|
568
|
-
}
|
|
569
|
-
}
|
|
570
|
-
return {
|
|
571
|
-
trace_id,
|
|
572
|
-
span_id: s.span_id,
|
|
573
|
-
parent_span_id: s.parent_span_id,
|
|
574
|
-
name: s.name,
|
|
575
|
-
kind: s.kind,
|
|
576
|
-
start_time: s.start_time,
|
|
577
|
-
end_time: s.end_time,
|
|
578
|
-
duration_ms: s.duration_ms,
|
|
579
|
-
status: s.status,
|
|
580
|
-
status_message: s.status_message,
|
|
581
|
-
service_name: s.service_name,
|
|
582
|
-
agent_name: s.agent_name,
|
|
583
|
-
model_name: s.model_name,
|
|
584
|
-
tool_name: s.tool_name,
|
|
585
|
-
attributes: projected
|
|
586
|
-
};
|
|
587
|
-
}
|
|
588
|
-
buildOversizedSummary(t, span_response_bytes_max) {
|
|
589
|
-
const counts = /* @__PURE__ */ new Map();
|
|
590
|
-
let errorCount = 0;
|
|
591
|
-
for (const s of t.spans) {
|
|
592
|
-
counts.set(s.name, (counts.get(s.name) ?? 0) + 1);
|
|
593
|
-
if (s.status === "ERROR") errorCount += 1;
|
|
594
|
-
}
|
|
595
|
-
const top = [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20);
|
|
596
|
-
return {
|
|
597
|
-
span_count: t.span_count,
|
|
598
|
-
top_span_names: top,
|
|
599
|
-
span_response_bytes_max,
|
|
600
|
-
error_span_count: errorCount
|
|
601
|
-
};
|
|
602
|
-
}
|
|
603
|
-
async scanSpanForMatches(buf, trace_id, s, re, textBudget, recordCap) {
|
|
604
|
-
const slice = buf.subarray(s.line_byte_offset, s.line_byte_offset + s.line_byte_length).toString("utf8");
|
|
605
|
-
const records = [];
|
|
606
|
-
const globalRe = new RegExp(re.source, re.flags.includes("g") ? re.flags : `${re.flags}g`);
|
|
607
|
-
let total = 0;
|
|
608
|
-
let hasMore = false;
|
|
609
|
-
let m;
|
|
610
|
-
while ((m = globalRe.exec(slice)) !== null) {
|
|
611
|
-
total += 1;
|
|
612
|
-
if (m.index === globalRe.lastIndex) globalRe.lastIndex += 1;
|
|
613
|
-
if (records.length >= recordCap) {
|
|
614
|
-
hasMore = true;
|
|
615
|
-
break;
|
|
616
|
-
}
|
|
617
|
-
const before = slice.slice(Math.max(0, m.index - textBudget / 2), m.index);
|
|
618
|
-
const after = slice.slice(
|
|
619
|
-
m.index + m[0].length,
|
|
620
|
-
m.index + m[0].length + Math.floor(textBudget / 2)
|
|
621
|
-
);
|
|
622
|
-
records.push({
|
|
623
|
-
trace_id,
|
|
624
|
-
span_id: s.span_id,
|
|
625
|
-
span_name: s.name,
|
|
626
|
-
span_kind: s.kind,
|
|
627
|
-
attribute_path: bestAttributePathForOffset(slice, m.index) ?? "span.raw",
|
|
628
|
-
matched_text: truncateForBudget(m[0], textBudget),
|
|
629
|
-
context_before: truncateForBudget(before, textBudget),
|
|
630
|
-
context_after: truncateForBudget(after, textBudget),
|
|
631
|
-
match_offset: m.index
|
|
632
|
-
});
|
|
633
|
-
}
|
|
634
|
-
return { records, total, hasMore };
|
|
635
|
-
}
|
|
636
|
-
};
|
|
637
|
-
var TraceFileMissingError = class extends NotFoundError {
|
|
638
|
-
constructor(path) {
|
|
639
|
-
super(`trace file not found: ${path}`);
|
|
640
|
-
}
|
|
641
|
-
};
|
|
642
|
-
var TraceNotFoundError = class extends NotFoundError {
|
|
643
|
-
trace_id;
|
|
644
|
-
constructor(trace_id) {
|
|
645
|
-
super(`trace not found: ${trace_id}`);
|
|
646
|
-
this.trace_id = trace_id;
|
|
647
|
-
}
|
|
648
|
-
};
|
|
649
|
-
var SpanNotFoundError = class extends NotFoundError {
|
|
650
|
-
trace_id;
|
|
651
|
-
span_id;
|
|
652
|
-
constructor(trace_id, span_id) {
|
|
653
|
-
super(`span ${span_id} not found in trace ${trace_id}`);
|
|
654
|
-
this.trace_id = trace_id;
|
|
655
|
-
this.span_id = span_id;
|
|
656
|
-
}
|
|
657
|
-
};
|
|
658
|
-
function readOtlpSpan(raw) {
|
|
659
|
-
const trace_id = stringField(raw, "trace_id") ?? stringField(raw, "traceId");
|
|
660
|
-
const span_id = stringField(raw, "span_id") ?? stringField(raw, "spanId");
|
|
661
|
-
if (!trace_id || !span_id) return null;
|
|
662
|
-
const parent_id = stringField(raw, "parent_span_id") ?? stringField(raw, "parentSpanId") ?? null;
|
|
663
|
-
const name = stringField(raw, "name") ?? "unknown";
|
|
664
|
-
const start_time = stringField(raw, "start_time") ?? stringField(raw, "startTime") ?? "";
|
|
665
|
-
const end_time = stringField(raw, "end_time") ?? stringField(raw, "endTime") ?? start_time;
|
|
666
|
-
const status = readStatus(raw);
|
|
667
|
-
const attrs = extractAttributes(raw);
|
|
668
|
-
const service_name = asString(attrs["service.name"]) ?? asString(attrs["resource.attributes.service.name"]) ?? null;
|
|
669
|
-
const agent_name = asString(attrs["agent.name"]) ?? asString(attrs["inference.agent.name"]) ?? null;
|
|
670
|
-
const model_name = asString(attrs["llm.model_name"]) ?? asString(attrs["inference.llm.model_name"]) ?? null;
|
|
671
|
-
const tool_name = asString(attrs["tool.name"]) ?? asString(attrs["inference.tool.name"]) ?? null;
|
|
672
|
-
const kind = inferKind(attrs);
|
|
673
|
-
let duration_ms = 0;
|
|
674
|
-
if (start_time && end_time) {
|
|
675
|
-
const a = Date.parse(start_time);
|
|
676
|
-
const b = Date.parse(end_time);
|
|
677
|
-
if (!Number.isNaN(a) && !Number.isNaN(b)) duration_ms = Math.max(0, b - a);
|
|
678
|
-
}
|
|
679
|
-
return {
|
|
680
|
-
trace_id,
|
|
681
|
-
span_id,
|
|
682
|
-
parent_span_id: parent_id && parent_id.length > 0 ? parent_id : null,
|
|
683
|
-
name,
|
|
684
|
-
kind,
|
|
685
|
-
start_time,
|
|
686
|
-
end_time,
|
|
687
|
-
duration_ms,
|
|
688
|
-
status: status.code,
|
|
689
|
-
status_message: status.message,
|
|
690
|
-
service_name,
|
|
691
|
-
agent_name,
|
|
692
|
-
model_name,
|
|
693
|
-
tool_name
|
|
694
|
-
};
|
|
695
|
-
}
|
|
696
|
-
function readStatus(raw) {
|
|
697
|
-
const status = raw.status;
|
|
698
|
-
if (status && typeof status === "object" && !Array.isArray(status)) {
|
|
699
|
-
const codeRaw = status.code;
|
|
700
|
-
const code = codeRaw === "STATUS_CODE_OK" || codeRaw === "OK" ? "OK" : codeRaw === "STATUS_CODE_ERROR" || codeRaw === "ERROR" ? "ERROR" : "UNSET";
|
|
701
|
-
const messageRaw = status.message;
|
|
702
|
-
const message = typeof messageRaw === "string" && messageRaw.length > 0 ? messageRaw : void 0;
|
|
703
|
-
return { code, message };
|
|
704
|
-
}
|
|
705
|
-
return { code: "UNSET", message: void 0 };
|
|
706
|
-
}
|
|
707
|
-
function inferKind(attrs) {
|
|
708
|
-
const opik = asString(attrs["openinference.span.kind"]) ?? asString(attrs["inference.observation_kind"]);
|
|
709
|
-
if (opik) {
|
|
710
|
-
const upper = opik.toUpperCase();
|
|
711
|
-
if (upper === "AGENT" || upper === "LLM" || upper === "TOOL" || upper === "CHAIN" || upper === "GUARDRAIL" || upper === "SPAN") {
|
|
712
|
-
return upper;
|
|
713
|
-
}
|
|
714
|
-
}
|
|
715
|
-
return "UNKNOWN";
|
|
716
|
-
}
|
|
717
|
-
function extractAttributes(raw) {
|
|
718
|
-
const out = {};
|
|
719
|
-
const resource = raw.resource;
|
|
720
|
-
if (resource && typeof resource === "object" && !Array.isArray(resource)) {
|
|
721
|
-
const ra = resource.attributes;
|
|
722
|
-
if (ra && typeof ra === "object" && !Array.isArray(ra)) {
|
|
723
|
-
for (const [k, v] of Object.entries(ra)) {
|
|
724
|
-
out[k] = v;
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
const spanAttrs = raw.attributes;
|
|
729
|
-
if (spanAttrs && typeof spanAttrs === "object" && !Array.isArray(spanAttrs)) {
|
|
730
|
-
for (const [k, v] of Object.entries(spanAttrs)) {
|
|
731
|
-
out[k] = v;
|
|
732
|
-
}
|
|
733
|
-
}
|
|
734
|
-
return out;
|
|
735
|
-
}
|
|
736
|
-
function stringField(raw, key) {
|
|
737
|
-
const v = raw[key];
|
|
738
|
-
return typeof v === "string" ? v : void 0;
|
|
739
|
-
}
|
|
740
|
-
function asString(v) {
|
|
741
|
-
return typeof v === "string" && v.length > 0 ? v : null;
|
|
742
|
-
}
|
|
743
|
-
function isPresent(v) {
|
|
744
|
-
return v !== void 0;
|
|
745
|
-
}
|
|
746
|
-
var truncationCounters = /* @__PURE__ */ new WeakMap();
|
|
747
|
-
function trackTruncation(store) {
|
|
748
|
-
let c = truncationCounters.get(store);
|
|
749
|
-
if (!c) {
|
|
750
|
-
c = { value: 0 };
|
|
751
|
-
truncationCounters.set(store, c);
|
|
752
|
-
}
|
|
753
|
-
c.value += 1;
|
|
754
|
-
}
|
|
755
|
-
function truncationCounter(store) {
|
|
756
|
-
const before = truncationCounters.get(store)?.value ?? 0;
|
|
757
|
-
return {
|
|
758
|
-
delta() {
|
|
759
|
-
const after = truncationCounters.get(store)?.value ?? 0;
|
|
760
|
-
return after - before;
|
|
761
|
-
}
|
|
762
|
-
};
|
|
763
|
-
}
|
|
764
|
-
function bestAttributePathForOffset(slice, offset) {
|
|
765
|
-
let i = offset;
|
|
766
|
-
while (i > 0 && slice[i] !== '"') i -= 1;
|
|
767
|
-
if (i <= 0) return null;
|
|
768
|
-
let j = i - 1;
|
|
769
|
-
while (j > 0 && slice[j] !== ":") j -= 1;
|
|
770
|
-
if (j <= 0) return null;
|
|
771
|
-
let k = j - 1;
|
|
772
|
-
while (k > 0 && slice[k] !== '"') k -= 1;
|
|
773
|
-
let l = k - 1;
|
|
774
|
-
while (l > 0 && slice[l] !== '"') l -= 1;
|
|
775
|
-
if (l <= 0) return null;
|
|
776
|
-
return slice.slice(l + 1, k);
|
|
777
|
-
}
|
|
778
|
-
|
|
779
|
-
// src/trace-analyst/tools.ts
|
|
780
|
-
import { f, fn } from "@ax-llm/ax";
|
|
781
|
-
var NAMESPACE = "traces";
|
|
782
|
-
var filtersField = f.json("Filter set. ALL fields are AND-composed. Leave empty to scan everything.").optional();
|
|
783
|
-
function buildTraceAnalystTools(opts) {
|
|
784
|
-
const { store } = opts;
|
|
785
|
-
const getDatasetOverview = fn("getDatasetOverview").description(
|
|
786
|
-
"Dataset rollup: total traces, raw_jsonl_bytes, services, agents, models, tools, and sample_trace_ids (real ids passable to view/search). Always call this FIRST without a regex_pattern."
|
|
787
|
-
).namespace(NAMESPACE).arg("filters", filtersField).returns(f.json("DatasetOverview")).handler(async ({ filters }) => store.getOverview(parseFilters(filters))).build();
|
|
788
|
-
const queryTraces = fn("queryTraces").description(
|
|
789
|
-
"Paginated trace summaries. Each summary carries raw_jsonl_bytes \u2014 use it to size traces BEFORE calling viewTrace. Narrow with indexed filters before adding regex_pattern."
|
|
790
|
-
).namespace(NAMESPACE).arg("filters", filtersField).arg("limit", f.number("Page size, 1..200")).arg("offset", f.number("Page offset; default 0").optional()).returns(f.json("QueryTracesPage")).handler(
|
|
791
|
-
async ({ filters, limit, offset }) => store.queryTraces({
|
|
792
|
-
filters: parseFilters(filters),
|
|
793
|
-
limit: assertPageLimit(limit),
|
|
794
|
-
offset: assertOffset(offset)
|
|
795
|
-
})
|
|
796
|
-
).build();
|
|
797
|
-
const countTraces = fn("countTraces").description(
|
|
798
|
-
"Count traces matching `filters`. Use as a cheap pre-flight before opting into a regex_pattern scan."
|
|
799
|
-
).namespace(NAMESPACE).arg("filters", filtersField).returns(f.number("count")).handler(async ({ filters }) => store.countTraces(parseFilters(filters))).build();
|
|
800
|
-
const viewTrace = fn("viewTrace").description(
|
|
801
|
-
"Return ALL spans for a single trace, with each attribute capped at ~4KB. If the response would exceed the per-call ceiling the result carries `oversized` instead of `spans` \u2014 DO NOT retry with the same trace_id; switch to searchTrace / viewSpans."
|
|
802
|
-
).namespace(NAMESPACE).arg("trace_id", f.string("Real trace id from a prior overview/query")).returns(f.json("ViewTraceResult")).handler(
|
|
803
|
-
async ({ trace_id }) => store.viewTrace({ trace_id: assertString(trace_id, "trace_id") })
|
|
804
|
-
).build();
|
|
805
|
-
const viewSpans = fn("viewSpans").description(
|
|
806
|
-
"Surgical read of specific spans within a trace, with each attribute capped at ~16KB (4\xD7 the discovery cap). Use after searchTrace narrows to specific span_ids."
|
|
807
|
-
).namespace(NAMESPACE).arg("trace_id", f.string("Real trace id")).arg("span_ids", f.string("Span ids to fetch").array()).returns(f.json("ViewSpansResult")).handler(
|
|
808
|
-
async ({ trace_id, span_ids }) => store.viewSpans({
|
|
809
|
-
trace_id: assertString(trace_id, "trace_id"),
|
|
810
|
-
span_ids: assertStringArray(span_ids, "span_ids")
|
|
811
|
-
})
|
|
812
|
-
).build();
|
|
813
|
-
const searchTrace = fn("searchTrace").description(
|
|
814
|
-
"Regex search across all spans of one trace. Returns `{trace_id, hits: SpanMatchRecord[], total_matches, has_more}`. **Iterate `result.hits`, NOT `result.matches`** \u2014 the field is `hits`. Each hit has `{span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset}`. Bounded regardless of trace size by max_matches (1..500, default 50). If has_more=true, REFINE the regex rather than blindly raising max_matches."
|
|
815
|
-
).namespace(NAMESPACE).arg("trace_id", f.string("Real trace id")).arg("regex_pattern", f.string("JS-compatible regex, multiline")).arg("max_matches", f.number("Max records returned, 1..500; default 50").optional()).returns(f.json("SearchTraceResult")).handler(
|
|
816
|
-
async ({ trace_id, regex_pattern, max_matches }) => store.searchTrace({
|
|
817
|
-
trace_id: assertString(trace_id, "trace_id"),
|
|
818
|
-
regex_pattern: assertRegex(regex_pattern),
|
|
819
|
-
max_matches: assertMaxMatches(max_matches)
|
|
820
|
-
})
|
|
821
|
-
).build();
|
|
822
|
-
const searchSpan = fn("searchSpan").description(
|
|
823
|
-
"Regex search inside a single span. Use when viewSpans returned a 16KB-truncated payload and you need to narrow further. Returns `{trace_id, span_id, hits: SpanMatchRecord[], total_matches, has_more}` \u2014 iterate `result.hits`, NOT `result.matches`."
|
|
824
|
-
).namespace(NAMESPACE).arg("trace_id", f.string("Real trace id")).arg("span_id", f.string("Real span id within trace")).arg("regex_pattern", f.string("JS-compatible regex, multiline")).arg("max_matches", f.number("Max records, 1..500; default 50").optional()).returns(f.json("SearchSpanResult")).handler(
|
|
825
|
-
async ({ trace_id, span_id, regex_pattern, max_matches }) => store.searchSpan({
|
|
826
|
-
trace_id: assertString(trace_id, "trace_id"),
|
|
827
|
-
span_id: assertString(span_id, "span_id"),
|
|
828
|
-
regex_pattern: assertRegex(regex_pattern),
|
|
829
|
-
max_matches: assertMaxMatches(max_matches)
|
|
830
|
-
})
|
|
831
|
-
).build();
|
|
832
|
-
return [
|
|
833
|
-
getDatasetOverview,
|
|
834
|
-
queryTraces,
|
|
835
|
-
countTraces,
|
|
836
|
-
viewTrace,
|
|
837
|
-
viewSpans,
|
|
838
|
-
searchTrace,
|
|
839
|
-
searchSpan
|
|
840
|
-
];
|
|
841
|
-
}
|
|
842
|
-
function traceAnalystFunctionGroup(opts) {
|
|
843
|
-
return {
|
|
844
|
-
namespace: NAMESPACE,
|
|
845
|
-
title: "Trace Analysis",
|
|
846
|
-
selectionCriteria: "Use for any inspection of OTLP-shaped trace data.",
|
|
847
|
-
description: "Discovery \u2192 narrow \u2192 deep-read tools over a JSONL trace dataset. Always call getDatasetOverview first.",
|
|
848
|
-
functions: buildTraceAnalystTools(opts)
|
|
849
|
-
};
|
|
850
|
-
}
|
|
851
|
-
function parseFilters(input) {
|
|
852
|
-
if (input == null) return void 0;
|
|
853
|
-
if (typeof input !== "object" || Array.isArray(input)) {
|
|
854
|
-
throw new TypeError(`filters must be an object, got ${typeof input}`);
|
|
855
|
-
}
|
|
856
|
-
const f2 = input;
|
|
857
|
-
const out = {};
|
|
858
|
-
if (typeof f2.has_errors === "boolean") out.has_errors = f2.has_errors;
|
|
859
|
-
out.service_names = stringArrayOrUndefined(f2.service_names, "service_names");
|
|
860
|
-
out.agent_names = stringArrayOrUndefined(f2.agent_names, "agent_names");
|
|
861
|
-
out.model_names = stringArrayOrUndefined(f2.model_names, "model_names");
|
|
862
|
-
out.tool_names = stringArrayOrUndefined(f2.tool_names, "tool_names");
|
|
863
|
-
if (typeof f2.start_time_after === "string") out.start_time_after = f2.start_time_after;
|
|
864
|
-
if (typeof f2.start_time_before === "string") out.start_time_before = f2.start_time_before;
|
|
865
|
-
if (typeof f2.regex_pattern === "string") {
|
|
866
|
-
if (f2.regex_pattern.length === 0) {
|
|
867
|
-
throw new TypeError("filters.regex_pattern cannot be empty");
|
|
868
|
-
}
|
|
869
|
-
out.regex_pattern = f2.regex_pattern;
|
|
870
|
-
}
|
|
871
|
-
return out;
|
|
872
|
-
}
|
|
873
|
-
function stringArrayOrUndefined(v, label) {
|
|
874
|
-
if (v === void 0 || v === null) return void 0;
|
|
875
|
-
if (!Array.isArray(v)) throw new TypeError(`${label} must be an array of strings`);
|
|
876
|
-
if (v.some((x) => typeof x !== "string")) {
|
|
877
|
-
throw new TypeError(`${label} entries must be strings`);
|
|
878
|
-
}
|
|
879
|
-
return v;
|
|
880
|
-
}
|
|
881
|
-
function assertPageLimit(limit) {
|
|
882
|
-
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < 1 || limit > 200) {
|
|
883
|
-
throw new RangeError(`limit must be an integer 1..200`);
|
|
884
|
-
}
|
|
885
|
-
return limit;
|
|
886
|
-
}
|
|
887
|
-
function assertOffset(offset) {
|
|
888
|
-
if (offset === void 0) return void 0;
|
|
889
|
-
if (typeof offset !== "number" || !Number.isInteger(offset) || offset < 0) {
|
|
890
|
-
throw new RangeError(`offset must be a non-negative integer`);
|
|
891
|
-
}
|
|
892
|
-
return offset;
|
|
893
|
-
}
|
|
894
|
-
function assertRegex(pattern) {
|
|
895
|
-
if (typeof pattern !== "string" || pattern.length === 0) {
|
|
896
|
-
throw new TypeError(`regex_pattern must be a non-empty string`);
|
|
897
|
-
}
|
|
898
|
-
new RegExp(pattern, "m");
|
|
899
|
-
return pattern;
|
|
900
|
-
}
|
|
901
|
-
function assertMaxMatches(n) {
|
|
902
|
-
if (n === void 0) return void 0;
|
|
903
|
-
if (typeof n !== "number" || !Number.isInteger(n) || n < 1 || n > 500) {
|
|
904
|
-
throw new RangeError(`max_matches must be an integer 1..500`);
|
|
905
|
-
}
|
|
906
|
-
return n;
|
|
907
|
-
}
|
|
908
|
-
function assertString(v, label) {
|
|
909
|
-
if (typeof v !== "string" || v.length === 0) {
|
|
910
|
-
throw new TypeError(`${label} must be a non-empty string`);
|
|
911
|
-
}
|
|
912
|
-
return v;
|
|
913
|
-
}
|
|
914
|
-
function assertStringArray(v, label) {
|
|
915
|
-
if (!Array.isArray(v)) throw new TypeError(`${label} must be an array of strings`);
|
|
916
|
-
if (v.some((x) => typeof x !== "string")) {
|
|
917
|
-
throw new TypeError(`${label} entries must be strings`);
|
|
918
|
-
}
|
|
919
|
-
return v;
|
|
920
|
-
}
|
|
921
|
-
|
|
922
|
-
// src/trace-analyst/analyst.ts
|
|
923
|
-
async function analyzeTraces(input, options) {
|
|
924
|
-
if (!input.question || typeof input.question !== "string") {
|
|
925
|
-
throw new TypeError("analyzeTraces: input.question must be a non-empty string");
|
|
926
|
-
}
|
|
927
|
-
const store = typeof options.source === "string" ? new OtlpFileTraceStore({ path: options.source }) : options.source;
|
|
928
|
-
if (store instanceof OtlpFileTraceStore) {
|
|
929
|
-
await store.ensureIndexed();
|
|
930
|
-
}
|
|
931
|
-
const tools = buildTraceAnalystTools({ store });
|
|
932
|
-
const turns = [];
|
|
933
|
-
let progressFs;
|
|
934
|
-
if (options.progressLogPath) {
|
|
935
|
-
const { createWriteStream } = await import("fs");
|
|
936
|
-
const { mkdir } = await import("fs/promises");
|
|
937
|
-
const { dirname } = await import("path");
|
|
938
|
-
await mkdir(dirname(options.progressLogPath), { recursive: true });
|
|
939
|
-
progressFs = createWriteStream(options.progressLogPath, { flags: "a" });
|
|
940
|
-
}
|
|
941
|
-
const actorTurnCallback = async (turn) => {
|
|
942
|
-
const snap = {
|
|
943
|
-
turn: turn.turn,
|
|
944
|
-
isError: turn.isError,
|
|
945
|
-
code: turn.code,
|
|
946
|
-
output: turn.output,
|
|
947
|
-
thought: turn.thought
|
|
948
|
-
};
|
|
949
|
-
turns.push(snap);
|
|
950
|
-
if (progressFs) {
|
|
951
|
-
try {
|
|
952
|
-
progressFs.write(`${JSON.stringify({ ...snap, ts: Date.now() })}
|
|
953
|
-
`);
|
|
954
|
-
} catch {
|
|
955
|
-
}
|
|
956
|
-
}
|
|
957
|
-
if (options.onTurn) await options.onTurn(snap);
|
|
958
|
-
};
|
|
959
|
-
const maxDepth = options.maxDepth ?? 1;
|
|
960
|
-
const maxTurns = options.maxTurns ?? 12;
|
|
961
|
-
const maxParallelSubagents = options.maxParallelSubagents ?? 2;
|
|
962
|
-
const maxRuntimeChars = options.maxRuntimeChars ?? 6e3;
|
|
963
|
-
const analyst = agent(
|
|
964
|
-
"question:string -> answer:string, findings:string[]",
|
|
965
|
-
{
|
|
966
|
-
agentIdentity: {
|
|
967
|
-
name: "TraceAnalyst",
|
|
968
|
-
description: "Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes."
|
|
969
|
-
},
|
|
970
|
-
contextFields: ["question"],
|
|
971
|
-
runtime: new AxJSRuntime({
|
|
972
|
-
permissions: [],
|
|
973
|
-
blockDynamicImport: true,
|
|
974
|
-
allowedModules: [],
|
|
975
|
-
freezeIntrinsics: true,
|
|
976
|
-
blockShadowRealm: true,
|
|
977
|
-
// RLM stdout mode relies on runtime bindings persisting across turns.
|
|
978
|
-
preventGlobalThisExtensions: false
|
|
979
|
-
}),
|
|
980
|
-
mode: maxDepth > 0 ? "advanced" : "simple",
|
|
981
|
-
recursionOptions: maxDepth > 0 ? { maxDepth } : void 0,
|
|
982
|
-
maxTurns,
|
|
983
|
-
maxRuntimeChars,
|
|
984
|
-
maxBatchedLlmQueryConcurrency: maxParallelSubagents,
|
|
985
|
-
promptLevel: "detailed",
|
|
986
|
-
// Trace analysis depends on exact prior tool results and runtime variables.
|
|
987
|
-
contextPolicy: { preset: "full", budget: "balanced" },
|
|
988
|
-
functions: { local: tools },
|
|
989
|
-
actorOptions: {
|
|
990
|
-
description: options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION,
|
|
991
|
-
...options.model ? { model: options.model } : {},
|
|
992
|
-
// Keep actor messages tool-call/content shaped across reasoning models.
|
|
993
|
-
showThoughts: false,
|
|
994
|
-
thinkingTokenBudget: "none"
|
|
995
|
-
},
|
|
996
|
-
responderOptions: {
|
|
997
|
-
...options.model ? { model: options.model } : {},
|
|
998
|
-
description: options.subagentDescription ?? TRACE_ANALYST_SUBAGENT_DESCRIPTION,
|
|
999
|
-
showThoughts: false
|
|
1000
|
-
},
|
|
1001
|
-
actorTurnCallback,
|
|
1002
|
-
bubbleErrors: [TraceFileMissingError]
|
|
1003
|
-
}
|
|
1004
|
-
);
|
|
1005
|
-
let result;
|
|
1006
|
-
try {
|
|
1007
|
-
result = await analyst.forward(options.ai, { question: input.question });
|
|
1008
|
-
} finally {
|
|
1009
|
-
if (progressFs) {
|
|
1010
|
-
await new Promise((resolve) => progressFs.end(() => resolve()));
|
|
1011
|
-
}
|
|
1012
|
-
}
|
|
1013
|
-
return {
|
|
1014
|
-
answer: typeof result.answer === "string" ? result.answer : String(result.answer ?? ""),
|
|
1015
|
-
findings: Array.isArray(result.findings) ? result.findings.filter((s) => typeof s === "string") : [],
|
|
1016
|
-
turns,
|
|
1017
|
-
turnCount: turns.length,
|
|
1018
|
-
usage: normalizeRoleArrays(analyst.getUsage()),
|
|
1019
|
-
chatLog: normalizeRoleArrays(analyst.getChatLog()),
|
|
1020
|
-
actorPromptVersion: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION
|
|
1021
|
-
};
|
|
1022
|
-
}
|
|
1023
|
-
function normalizeRoleArrays(value) {
|
|
1024
|
-
const record = value && typeof value === "object" ? value : {};
|
|
1025
|
-
return {
|
|
1026
|
-
actor: normalizeRecordArray(record.actor),
|
|
1027
|
-
responder: normalizeRecordArray(record.responder)
|
|
1028
|
-
};
|
|
1029
|
-
}
|
|
1030
|
-
function normalizeRecordArray(value) {
|
|
1031
|
-
if (!Array.isArray(value)) return [];
|
|
1032
|
-
return value.map(
|
|
1033
|
-
(item) => item && typeof item === "object" ? { ...item } : { value: item }
|
|
1034
|
-
);
|
|
1035
|
-
}
|
|
1036
|
-
|
|
1037
66
|
// src/trace-analyst/hook.ts
|
|
1038
67
|
var DEFAULT_QUESTION = "Summarise what happened in this run. Surface any failure modes, surprising findings, or evidence that the run's verdict is wrong.";
|
|
1039
68
|
function traceAnalystOnRunComplete(opts) {
|