agent-inspect 6.7.2 → 6.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +1 -1
- package/docs/EXPORTS.md +9 -0
- package/docs/PERFORMANCE.md +18 -0
- package/docs/STANDARDS.md +4 -0
- package/package.json +2 -2
- package/packages/cli/dist/{chunk-M6PC5KUT.mjs → chunk-FVEQ7JFF.mjs} +251 -209
- package/packages/cli/dist/chunk-FVEQ7JFF.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +292 -229
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +35 -15
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-27LC2BB3.mjs → src-I5NA7BLK.mjs} +3 -3
- package/packages/cli/dist/{src-27LC2BB3.mjs.map → src-I5NA7BLK.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +249 -207
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.mjs +51 -250
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +56 -4
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-X2FLDF7M.mjs → chunk-CTA6XNDP.mjs} +3 -3
- package/packages/core/dist/{chunk-X2FLDF7M.mjs.map → chunk-CTA6XNDP.mjs.map} +1 -1
- package/packages/core/dist/{chunk-EWHN6NA4.mjs → chunk-EQVXDGGU.mjs} +58 -6
- package/packages/core/dist/chunk-EQVXDGGU.mjs.map +1 -0
- package/packages/core/dist/{chunk-3USJVBLA.mjs → chunk-KDDU2R5P.mjs} +236 -5
- package/packages/core/dist/chunk-KDDU2R5P.mjs.map +1 -0
- package/packages/core/dist/{chunk-E5F2LQCX.mjs → chunk-T5MASTIW.mjs} +15 -4
- package/packages/core/dist/chunk-T5MASTIW.mjs.map +1 -0
- package/packages/core/dist/{chunk-KNYL56KZ.mjs → chunk-UOPVGCOB.mjs} +3 -3
- package/packages/core/dist/{chunk-KNYL56KZ.mjs.map → chunk-UOPVGCOB.mjs.map} +1 -1
- package/packages/core/dist/diff.cjs +4 -0
- package/packages/core/dist/diff.cjs.map +1 -1
- package/packages/core/dist/diff.mjs +4 -0
- package/packages/core/dist/diff.mjs.map +1 -1
- package/packages/core/dist/exporters.cjs +145 -133
- package/packages/core/dist/exporters.cjs.map +1 -1
- package/packages/core/dist/exporters.d.cts +3 -2
- package/packages/core/dist/exporters.d.ts +3 -2
- package/packages/core/dist/exporters.mjs +10 -5
- package/packages/core/dist/exporters.mjs.map +1 -1
- package/packages/core/dist/logs.cjs +16 -4
- package/packages/core/dist/logs.cjs.map +1 -1
- package/packages/core/dist/logs.mjs +5 -4
- package/packages/core/dist/logs.mjs.map +1 -1
- package/packages/core/dist/persisted.cjs +13 -2
- package/packages/core/dist/persisted.cjs.map +1 -1
- package/packages/core/dist/persisted.mjs +3 -3
- package/packages/core/dist/readers.cjs +13 -2
- package/packages/core/dist/readers.cjs.map +1 -1
- package/packages/core/dist/readers.mjs +3 -3
- package/packages/cli/dist/chunk-M6PC5KUT.mjs.map +0 -1
- package/packages/core/dist/chunk-3USJVBLA.mjs.map +0 -1
- package/packages/core/dist/chunk-E5F2LQCX.mjs.map +0 -1
- package/packages/core/dist/chunk-EWHN6NA4.mjs.map +0 -1
|
@@ -1904,6 +1904,202 @@ function filterTraces(traces, options) {
|
|
|
1904
1904
|
return out;
|
|
1905
1905
|
}
|
|
1906
1906
|
|
|
1907
|
+
// packages/core/src/stats.ts
|
|
1908
|
+
function percentile(sorted, p) {
|
|
1909
|
+
if (sorted.length === 0) return void 0;
|
|
1910
|
+
const idx = Math.min(
|
|
1911
|
+
sorted.length - 1,
|
|
1912
|
+
Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
|
|
1913
|
+
);
|
|
1914
|
+
return sorted[idx];
|
|
1915
|
+
}
|
|
1916
|
+
async function readRunStartedMetadata(filePath) {
|
|
1917
|
+
try {
|
|
1918
|
+
const events = await readTraceEventsFromFile(filePath);
|
|
1919
|
+
for (const event of events) {
|
|
1920
|
+
if (event.event !== "run_started") continue;
|
|
1921
|
+
const rs = event;
|
|
1922
|
+
if (rs.metadata && typeof rs.metadata === "object") {
|
|
1923
|
+
return rs.metadata;
|
|
1924
|
+
}
|
|
1925
|
+
return void 0;
|
|
1926
|
+
}
|
|
1927
|
+
} catch {
|
|
1928
|
+
}
|
|
1929
|
+
return void 0;
|
|
1930
|
+
}
|
|
1931
|
+
function metaMatchesCorrelation(metadata, correlationId, groupId) {
|
|
1932
|
+
if (correlationId) {
|
|
1933
|
+
const v = metadata?.correlationId;
|
|
1934
|
+
if (typeof v !== "string" || v !== correlationId) return false;
|
|
1935
|
+
}
|
|
1936
|
+
if (groupId) {
|
|
1937
|
+
const v = metadata?.groupId;
|
|
1938
|
+
if (typeof v !== "string" || v !== groupId) return false;
|
|
1939
|
+
}
|
|
1940
|
+
return true;
|
|
1941
|
+
}
|
|
1942
|
+
async function buildTraceStats(metas, options) {
|
|
1943
|
+
let filtered = filterTraces(metas, { since: options.since });
|
|
1944
|
+
if (options.correlationId || options.groupId) {
|
|
1945
|
+
const next = [];
|
|
1946
|
+
for (const m of filtered) {
|
|
1947
|
+
const md = await readRunStartedMetadata(m.filePath);
|
|
1948
|
+
if (metaMatchesCorrelation(md, options.correlationId, options.groupId)) {
|
|
1949
|
+
next.push(m);
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
filtered = next;
|
|
1953
|
+
}
|
|
1954
|
+
let successCount = 0;
|
|
1955
|
+
let errorCount = 0;
|
|
1956
|
+
let runningCount = 0;
|
|
1957
|
+
let unknownCount = 0;
|
|
1958
|
+
const durations = [];
|
|
1959
|
+
let totalSteps = 0;
|
|
1960
|
+
let totalLlmSteps = 0;
|
|
1961
|
+
let totalToolSteps = 0;
|
|
1962
|
+
let totalErrorSteps = 0;
|
|
1963
|
+
const slowestRuns = [];
|
|
1964
|
+
const slowestSteps = [];
|
|
1965
|
+
for (const m of filtered) {
|
|
1966
|
+
if (m.status === "success") successCount += 1;
|
|
1967
|
+
else if (m.status === "error") errorCount += 1;
|
|
1968
|
+
else if (m.status === "running") runningCount += 1;
|
|
1969
|
+
else unknownCount += 1;
|
|
1970
|
+
if (typeof m.durationMs === "number" && Number.isFinite(m.durationMs) && m.durationMs >= 0) {
|
|
1971
|
+
durations.push(m.durationMs);
|
|
1972
|
+
slowestRuns.push({
|
|
1973
|
+
runId: m.runId,
|
|
1974
|
+
name: m.name,
|
|
1975
|
+
durationMs: m.durationMs,
|
|
1976
|
+
status: m.status
|
|
1977
|
+
});
|
|
1978
|
+
}
|
|
1979
|
+
try {
|
|
1980
|
+
const events = await readTraceEventsFromFile(m.filePath);
|
|
1981
|
+
if (events.length === 0) continue;
|
|
1982
|
+
const summary = buildRunSummary(events);
|
|
1983
|
+
totalSteps += summary.totalSteps;
|
|
1984
|
+
totalLlmSteps += summary.llmSteps;
|
|
1985
|
+
totalToolSteps += summary.toolSteps;
|
|
1986
|
+
totalErrorSteps += summary.errorSteps;
|
|
1987
|
+
const steps = collectCompletedSteps(events, m.runId);
|
|
1988
|
+
for (const s of steps) {
|
|
1989
|
+
slowestSteps.push(s);
|
|
1990
|
+
}
|
|
1991
|
+
} catch {
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1994
|
+
slowestRuns.sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0));
|
|
1995
|
+
slowestSteps.sort((a, b) => b.durationMs - a.durationMs);
|
|
1996
|
+
const runLimit = options.slowRunLimit ?? 5;
|
|
1997
|
+
const stepLimit = options.slowStepLimit ?? 5;
|
|
1998
|
+
const sortedDur = [...durations].sort((a, b) => a - b);
|
|
1999
|
+
const totalRuns = filtered.length;
|
|
2000
|
+
const errorRate = totalRuns > 0 ? errorCount / totalRuns : 0;
|
|
2001
|
+
const sumDur = durations.reduce((a, b) => a + b, 0);
|
|
2002
|
+
return {
|
|
2003
|
+
traceDir: options.traceDir,
|
|
2004
|
+
...options.since ? { since: options.since } : {},
|
|
2005
|
+
...options.correlationId ? { correlationId: options.correlationId } : {},
|
|
2006
|
+
...options.groupId ? { groupId: options.groupId } : {},
|
|
2007
|
+
totalRuns,
|
|
2008
|
+
successCount,
|
|
2009
|
+
errorCount,
|
|
2010
|
+
runningCount,
|
|
2011
|
+
unknownCount,
|
|
2012
|
+
errorRate,
|
|
2013
|
+
duration: {
|
|
2014
|
+
...sortedDur.length > 0 ? {
|
|
2015
|
+
minMs: sortedDur[0],
|
|
2016
|
+
maxMs: sortedDur[sortedDur.length - 1],
|
|
2017
|
+
avgMs: sumDur / sortedDur.length,
|
|
2018
|
+
p50Ms: percentile(sortedDur, 50),
|
|
2019
|
+
p95Ms: percentile(sortedDur, 95)
|
|
2020
|
+
} : {}
|
|
2021
|
+
},
|
|
2022
|
+
totalSteps,
|
|
2023
|
+
avgStepsPerRun: totalRuns > 0 ? totalSteps / totalRuns : 0,
|
|
2024
|
+
totalLlmSteps,
|
|
2025
|
+
totalToolSteps,
|
|
2026
|
+
totalErrorSteps,
|
|
2027
|
+
slowestRuns: slowestRuns.slice(0, runLimit),
|
|
2028
|
+
slowestSteps: slowestSteps.slice(0, stepLimit)
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
function collectCompletedSteps(events, runId) {
|
|
2032
|
+
const started = /* @__PURE__ */ new Map();
|
|
2033
|
+
const out = [];
|
|
2034
|
+
for (const e of events) {
|
|
2035
|
+
if (e.event === "step_started") {
|
|
2036
|
+
const s = e;
|
|
2037
|
+
started.set(s.stepId, { name: s.name, type: s.type });
|
|
2038
|
+
}
|
|
2039
|
+
if (e.event === "step_completed") {
|
|
2040
|
+
const c = e;
|
|
2041
|
+
if (c.status !== "success" && c.status !== "error") continue;
|
|
2042
|
+
if (typeof c.durationMs !== "number" || !Number.isFinite(c.durationMs)) {
|
|
2043
|
+
continue;
|
|
2044
|
+
}
|
|
2045
|
+
const meta = started.get(c.stepId);
|
|
2046
|
+
out.push({
|
|
2047
|
+
runId,
|
|
2048
|
+
stepName: meta?.name ?? c.stepId,
|
|
2049
|
+
stepType: meta?.type ?? "logic",
|
|
2050
|
+
durationMs: c.durationMs
|
|
2051
|
+
});
|
|
2052
|
+
}
|
|
2053
|
+
}
|
|
2054
|
+
return out;
|
|
2055
|
+
}
|
|
2056
|
+
function formatStepLabel(stepType, stepName) {
|
|
2057
|
+
return stepName.startsWith(`${stepType}:`) ? stepName : `${stepType}:${stepName}`;
|
|
2058
|
+
}
|
|
2059
|
+
function renderTraceStats(stats) {
|
|
2060
|
+
const lines = [];
|
|
2061
|
+
lines.push("Trace stats (local)");
|
|
2062
|
+
lines.push(`Directory: ${stats.traceDir}`);
|
|
2063
|
+
if (stats.since) lines.push(`Since: ${stats.since}`);
|
|
2064
|
+
if (stats.correlationId) lines.push(`Correlation ID: ${stats.correlationId}`);
|
|
2065
|
+
if (stats.groupId) lines.push(`Group ID: ${stats.groupId}`);
|
|
2066
|
+
lines.push("");
|
|
2067
|
+
lines.push(`Runs: ${stats.totalRuns}`);
|
|
2068
|
+
lines.push(
|
|
2069
|
+
` success: ${stats.successCount} error: ${stats.errorCount} running: ${stats.runningCount} unknown: ${stats.unknownCount}`
|
|
2070
|
+
);
|
|
2071
|
+
lines.push(`Error rate: ${(stats.errorRate * 100).toFixed(1)}%`);
|
|
2072
|
+
if (stats.duration.avgMs !== void 0) {
|
|
2073
|
+
lines.push(
|
|
2074
|
+
`Duration: min ${formatDuration2(stats.duration.minMs ?? 0)} | avg ${formatDuration2(stats.duration.avgMs)} | p50 ${formatDuration2(stats.duration.p50Ms ?? 0)} | p95 ${formatDuration2(stats.duration.p95Ms ?? 0)} | max ${formatDuration2(stats.duration.maxMs ?? 0)}`
|
|
2075
|
+
);
|
|
2076
|
+
}
|
|
2077
|
+
lines.push("");
|
|
2078
|
+
lines.push(`Steps: ${stats.totalSteps} (avg ${stats.avgStepsPerRun.toFixed(1)} per run)`);
|
|
2079
|
+
lines.push(
|
|
2080
|
+
` LLM: ${stats.totalLlmSteps} tool: ${stats.totalToolSteps} errors: ${stats.totalErrorSteps}`
|
|
2081
|
+
);
|
|
2082
|
+
if (stats.slowestRuns.length > 0) {
|
|
2083
|
+
lines.push("");
|
|
2084
|
+
lines.push("Slowest runs:");
|
|
2085
|
+
for (const r of stats.slowestRuns) {
|
|
2086
|
+
lines.push(
|
|
2087
|
+
` ${r.runId} | ${r.name ?? "-"} | ${formatDuration2(r.durationMs ?? 0)} | ${r.status}`
|
|
2088
|
+
);
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
if (stats.slowestSteps.length > 0) {
|
|
2092
|
+
lines.push("");
|
|
2093
|
+
lines.push("Slowest steps:");
|
|
2094
|
+
for (const s of stats.slowestSteps) {
|
|
2095
|
+
lines.push(
|
|
2096
|
+
` ${s.runId} | ${formatStepLabel(s.stepType, s.stepName)} | ${formatDuration2(s.durationMs)}`
|
|
2097
|
+
);
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
return lines.join("\n");
|
|
2101
|
+
}
|
|
2102
|
+
|
|
1907
2103
|
// packages/core/src/timeline.ts
|
|
1908
2104
|
function finite(n) {
|
|
1909
2105
|
return typeof n === "number" && Number.isFinite(n);
|
|
@@ -2061,7 +2257,7 @@ function renderTimeline(timeline, options = {}) {
|
|
|
2061
2257
|
const dur = e.durationMs !== void 0 ? formatDuration2(e.durationMs) : "-";
|
|
2062
2258
|
const err = e.isError ? " error" : "";
|
|
2063
2259
|
const off = formatDuration2(e.offsetMs);
|
|
2064
|
-
let line = `${prefix}+${off} ${typeTag
|
|
2260
|
+
let line = `${prefix}+${off} ${formatStepLabel(typeTag, e.name)} (${dur})${err}`;
|
|
2065
2261
|
if (e.streaming?.chunkCount !== void 0) {
|
|
2066
2262
|
line += ` chunks=${e.streaming.chunkCount}`;
|
|
2067
2263
|
}
|
|
@@ -2213,6 +2409,12 @@ function renderRunWhat(summary, options = {}) {
|
|
|
2213
2409
|
}
|
|
2214
2410
|
|
|
2215
2411
|
// packages/core/src/explain.ts
|
|
2412
|
+
function toDisplayStatus(status) {
|
|
2413
|
+
if (status === "ok") return "success";
|
|
2414
|
+
if (status === "error") return "error";
|
|
2415
|
+
if (status === "running") return "running";
|
|
2416
|
+
return "unknown";
|
|
2417
|
+
}
|
|
2216
2418
|
function flatten(nodes, out = []) {
|
|
2217
2419
|
for (const node of nodes) {
|
|
2218
2420
|
out.push({ node, index: out.length + 1 });
|
|
@@ -2270,7 +2472,7 @@ function buildFacts(run, redactor) {
|
|
|
2270
2472
|
const facts = [
|
|
2271
2473
|
fact("run.id", "Run id", run.runId, redactor),
|
|
2272
2474
|
fact("run.name", "Run name", run.name ?? run.runId, redactor),
|
|
2273
|
-
fact("run.status", "Run status", run.status
|
|
2475
|
+
fact("run.status", "Run status", toDisplayStatus(run.status), redactor),
|
|
2274
2476
|
fact("run.totalEvents", "Total events", run.metadata.totalEvents, redactor),
|
|
2275
2477
|
fact("run.stepCount", "Top-level step count", run.children.length, redactor),
|
|
2276
2478
|
fact("run.nodeCount", "Total node count", nodes.length, redactor),
|
|
@@ -2346,7 +2548,7 @@ function buildLocalExplanation(run, options = {}) {
|
|
|
2346
2548
|
mode,
|
|
2347
2549
|
runId: String(redactValue(redactor, "runId", run.runId)),
|
|
2348
2550
|
...run.name !== void 0 ? { name: String(redactValue(redactor, "name", run.name)) } : {},
|
|
2349
|
-
...run.status !== void 0 ? { status: run.status } : {},
|
|
2551
|
+
...run.status !== void 0 ? { status: toDisplayStatus(run.status) } : {},
|
|
2350
2552
|
redactionProfile,
|
|
2351
2553
|
facts,
|
|
2352
2554
|
inferences: mode === "dry-run" ? [] : buildInferences(run, facts),
|
|
@@ -2357,199 +2559,6 @@ function buildLocalExplanation(run, options = {}) {
|
|
|
2357
2559
|
};
|
|
2358
2560
|
}
|
|
2359
2561
|
|
|
2360
|
-
// packages/core/src/stats.ts
|
|
2361
|
-
function percentile(sorted, p) {
|
|
2362
|
-
if (sorted.length === 0) return void 0;
|
|
2363
|
-
const idx = Math.min(
|
|
2364
|
-
sorted.length - 1,
|
|
2365
|
-
Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
|
|
2366
|
-
);
|
|
2367
|
-
return sorted[idx];
|
|
2368
|
-
}
|
|
2369
|
-
async function readRunStartedMetadata(filePath) {
|
|
2370
|
-
try {
|
|
2371
|
-
const events = await readTraceEventsFromFile(filePath);
|
|
2372
|
-
for (const event of events) {
|
|
2373
|
-
if (event.event !== "run_started") continue;
|
|
2374
|
-
const rs = event;
|
|
2375
|
-
if (rs.metadata && typeof rs.metadata === "object") {
|
|
2376
|
-
return rs.metadata;
|
|
2377
|
-
}
|
|
2378
|
-
return void 0;
|
|
2379
|
-
}
|
|
2380
|
-
} catch {
|
|
2381
|
-
}
|
|
2382
|
-
return void 0;
|
|
2383
|
-
}
|
|
2384
|
-
function metaMatchesCorrelation(metadata, correlationId, groupId) {
|
|
2385
|
-
if (correlationId) {
|
|
2386
|
-
const v = metadata?.correlationId;
|
|
2387
|
-
if (typeof v !== "string" || v !== correlationId) return false;
|
|
2388
|
-
}
|
|
2389
|
-
if (groupId) {
|
|
2390
|
-
const v = metadata?.groupId;
|
|
2391
|
-
if (typeof v !== "string" || v !== groupId) return false;
|
|
2392
|
-
}
|
|
2393
|
-
return true;
|
|
2394
|
-
}
|
|
2395
|
-
async function buildTraceStats(metas, options) {
|
|
2396
|
-
let filtered = filterTraces(metas, { since: options.since });
|
|
2397
|
-
if (options.correlationId || options.groupId) {
|
|
2398
|
-
const next = [];
|
|
2399
|
-
for (const m of filtered) {
|
|
2400
|
-
const md = await readRunStartedMetadata(m.filePath);
|
|
2401
|
-
if (metaMatchesCorrelation(md, options.correlationId, options.groupId)) {
|
|
2402
|
-
next.push(m);
|
|
2403
|
-
}
|
|
2404
|
-
}
|
|
2405
|
-
filtered = next;
|
|
2406
|
-
}
|
|
2407
|
-
let successCount = 0;
|
|
2408
|
-
let errorCount = 0;
|
|
2409
|
-
let runningCount = 0;
|
|
2410
|
-
let unknownCount = 0;
|
|
2411
|
-
const durations = [];
|
|
2412
|
-
let totalSteps = 0;
|
|
2413
|
-
let totalLlmSteps = 0;
|
|
2414
|
-
let totalToolSteps = 0;
|
|
2415
|
-
let totalErrorSteps = 0;
|
|
2416
|
-
const slowestRuns = [];
|
|
2417
|
-
const slowestSteps = [];
|
|
2418
|
-
for (const m of filtered) {
|
|
2419
|
-
if (m.status === "success") successCount += 1;
|
|
2420
|
-
else if (m.status === "error") errorCount += 1;
|
|
2421
|
-
else if (m.status === "running") runningCount += 1;
|
|
2422
|
-
else unknownCount += 1;
|
|
2423
|
-
if (typeof m.durationMs === "number" && Number.isFinite(m.durationMs) && m.durationMs >= 0) {
|
|
2424
|
-
durations.push(m.durationMs);
|
|
2425
|
-
slowestRuns.push({
|
|
2426
|
-
runId: m.runId,
|
|
2427
|
-
name: m.name,
|
|
2428
|
-
durationMs: m.durationMs,
|
|
2429
|
-
status: m.status
|
|
2430
|
-
});
|
|
2431
|
-
}
|
|
2432
|
-
try {
|
|
2433
|
-
const events = await readTraceEventsFromFile(m.filePath);
|
|
2434
|
-
if (events.length === 0) continue;
|
|
2435
|
-
const summary = buildRunSummary(events);
|
|
2436
|
-
totalSteps += summary.totalSteps;
|
|
2437
|
-
totalLlmSteps += summary.llmSteps;
|
|
2438
|
-
totalToolSteps += summary.toolSteps;
|
|
2439
|
-
totalErrorSteps += summary.errorSteps;
|
|
2440
|
-
const steps = collectCompletedSteps(events, m.runId);
|
|
2441
|
-
for (const s of steps) {
|
|
2442
|
-
slowestSteps.push(s);
|
|
2443
|
-
}
|
|
2444
|
-
} catch {
|
|
2445
|
-
}
|
|
2446
|
-
}
|
|
2447
|
-
slowestRuns.sort((a, b) => (b.durationMs ?? 0) - (a.durationMs ?? 0));
|
|
2448
|
-
slowestSteps.sort((a, b) => b.durationMs - a.durationMs);
|
|
2449
|
-
const runLimit = options.slowRunLimit ?? 5;
|
|
2450
|
-
const stepLimit = options.slowStepLimit ?? 5;
|
|
2451
|
-
const sortedDur = [...durations].sort((a, b) => a - b);
|
|
2452
|
-
const totalRuns = filtered.length;
|
|
2453
|
-
const errorRate = totalRuns > 0 ? errorCount / totalRuns : 0;
|
|
2454
|
-
const sumDur = durations.reduce((a, b) => a + b, 0);
|
|
2455
|
-
return {
|
|
2456
|
-
traceDir: options.traceDir,
|
|
2457
|
-
...options.since ? { since: options.since } : {},
|
|
2458
|
-
...options.correlationId ? { correlationId: options.correlationId } : {},
|
|
2459
|
-
...options.groupId ? { groupId: options.groupId } : {},
|
|
2460
|
-
totalRuns,
|
|
2461
|
-
successCount,
|
|
2462
|
-
errorCount,
|
|
2463
|
-
runningCount,
|
|
2464
|
-
unknownCount,
|
|
2465
|
-
errorRate,
|
|
2466
|
-
duration: {
|
|
2467
|
-
...sortedDur.length > 0 ? {
|
|
2468
|
-
minMs: sortedDur[0],
|
|
2469
|
-
maxMs: sortedDur[sortedDur.length - 1],
|
|
2470
|
-
avgMs: sumDur / sortedDur.length,
|
|
2471
|
-
p50Ms: percentile(sortedDur, 50),
|
|
2472
|
-
p95Ms: percentile(sortedDur, 95)
|
|
2473
|
-
} : {}
|
|
2474
|
-
},
|
|
2475
|
-
totalSteps,
|
|
2476
|
-
avgStepsPerRun: totalRuns > 0 ? totalSteps / totalRuns : 0,
|
|
2477
|
-
totalLlmSteps,
|
|
2478
|
-
totalToolSteps,
|
|
2479
|
-
totalErrorSteps,
|
|
2480
|
-
slowestRuns: slowestRuns.slice(0, runLimit),
|
|
2481
|
-
slowestSteps: slowestSteps.slice(0, stepLimit)
|
|
2482
|
-
};
|
|
2483
|
-
}
|
|
2484
|
-
function collectCompletedSteps(events, runId) {
|
|
2485
|
-
const started = /* @__PURE__ */ new Map();
|
|
2486
|
-
const out = [];
|
|
2487
|
-
for (const e of events) {
|
|
2488
|
-
if (e.event === "step_started") {
|
|
2489
|
-
const s = e;
|
|
2490
|
-
started.set(s.stepId, { name: s.name, type: s.type });
|
|
2491
|
-
}
|
|
2492
|
-
if (e.event === "step_completed") {
|
|
2493
|
-
const c = e;
|
|
2494
|
-
if (c.status !== "success" && c.status !== "error") continue;
|
|
2495
|
-
if (typeof c.durationMs !== "number" || !Number.isFinite(c.durationMs)) {
|
|
2496
|
-
continue;
|
|
2497
|
-
}
|
|
2498
|
-
const meta = started.get(c.stepId);
|
|
2499
|
-
out.push({
|
|
2500
|
-
runId,
|
|
2501
|
-
stepName: meta?.name ?? c.stepId,
|
|
2502
|
-
stepType: meta?.type ?? "logic",
|
|
2503
|
-
durationMs: c.durationMs
|
|
2504
|
-
});
|
|
2505
|
-
}
|
|
2506
|
-
}
|
|
2507
|
-
return out;
|
|
2508
|
-
}
|
|
2509
|
-
function renderTraceStats(stats) {
|
|
2510
|
-
const lines = [];
|
|
2511
|
-
lines.push("Trace stats (local)");
|
|
2512
|
-
lines.push(`Directory: ${stats.traceDir}`);
|
|
2513
|
-
if (stats.since) lines.push(`Since: ${stats.since}`);
|
|
2514
|
-
if (stats.correlationId) lines.push(`Correlation ID: ${stats.correlationId}`);
|
|
2515
|
-
if (stats.groupId) lines.push(`Group ID: ${stats.groupId}`);
|
|
2516
|
-
lines.push("");
|
|
2517
|
-
lines.push(`Runs: ${stats.totalRuns}`);
|
|
2518
|
-
lines.push(
|
|
2519
|
-
` success: ${stats.successCount} error: ${stats.errorCount} running: ${stats.runningCount} unknown: ${stats.unknownCount}`
|
|
2520
|
-
);
|
|
2521
|
-
lines.push(`Error rate: ${(stats.errorRate * 100).toFixed(1)}%`);
|
|
2522
|
-
if (stats.duration.avgMs !== void 0) {
|
|
2523
|
-
lines.push(
|
|
2524
|
-
`Duration: min ${formatDuration2(stats.duration.minMs ?? 0)} | avg ${formatDuration2(stats.duration.avgMs)} | p50 ${formatDuration2(stats.duration.p50Ms ?? 0)} | p95 ${formatDuration2(stats.duration.p95Ms ?? 0)} | max ${formatDuration2(stats.duration.maxMs ?? 0)}`
|
|
2525
|
-
);
|
|
2526
|
-
}
|
|
2527
|
-
lines.push("");
|
|
2528
|
-
lines.push(`Steps: ${stats.totalSteps} (avg ${stats.avgStepsPerRun.toFixed(1)} per run)`);
|
|
2529
|
-
lines.push(
|
|
2530
|
-
` LLM: ${stats.totalLlmSteps} tool: ${stats.totalToolSteps} errors: ${stats.totalErrorSteps}`
|
|
2531
|
-
);
|
|
2532
|
-
if (stats.slowestRuns.length > 0) {
|
|
2533
|
-
lines.push("");
|
|
2534
|
-
lines.push("Slowest runs:");
|
|
2535
|
-
for (const r of stats.slowestRuns) {
|
|
2536
|
-
lines.push(
|
|
2537
|
-
` ${r.runId} | ${r.name ?? "-"} | ${formatDuration2(r.durationMs ?? 0)} | ${r.status}`
|
|
2538
|
-
);
|
|
2539
|
-
}
|
|
2540
|
-
}
|
|
2541
|
-
if (stats.slowestSteps.length > 0) {
|
|
2542
|
-
lines.push("");
|
|
2543
|
-
lines.push("Slowest steps:");
|
|
2544
|
-
for (const s of stats.slowestSteps) {
|
|
2545
|
-
lines.push(
|
|
2546
|
-
` ${s.runId} | ${s.stepType}:${s.stepName} | ${formatDuration2(s.durationMs)}`
|
|
2547
|
-
);
|
|
2548
|
-
}
|
|
2549
|
-
}
|
|
2550
|
-
return lines.join("\n");
|
|
2551
|
-
}
|
|
2552
|
-
|
|
2553
2562
|
// packages/core/src/outcomes/types.ts
|
|
2554
2563
|
var OBSERVED_OUTCOME_STATUSES = [
|
|
2555
2564
|
"passed",
|
|
@@ -2752,6 +2761,12 @@ async function searchTraces(metas, options) {
|
|
|
2752
2761
|
...sessionLabel ? { sessionId: sessionLabel } : {}
|
|
2753
2762
|
});
|
|
2754
2763
|
}
|
|
2764
|
+
results.sort((a, b) => {
|
|
2765
|
+
const ta = a.timestamp ?? 0;
|
|
2766
|
+
const tb = b.timestamp ?? 0;
|
|
2767
|
+
if (ta !== tb) return tb - ta;
|
|
2768
|
+
return a.runId.localeCompare(b.runId);
|
|
2769
|
+
});
|
|
2755
2770
|
return results.slice(0, limit);
|
|
2756
2771
|
}
|
|
2757
2772
|
for (const m of filtered) {
|
|
@@ -2799,7 +2814,7 @@ async function searchTraces(metas, options) {
|
|
|
2799
2814
|
results.sort((a, b) => {
|
|
2800
2815
|
const ta = a.timestamp ?? 0;
|
|
2801
2816
|
const tb = b.timestamp ?? 0;
|
|
2802
|
-
if (ta !== tb) return
|
|
2817
|
+
if (ta !== tb) return tb - ta;
|
|
2803
2818
|
const runCmp = a.runId.localeCompare(b.runId);
|
|
2804
2819
|
if (runCmp !== 0) return runCmp;
|
|
2805
2820
|
return (a.stepName ?? "").localeCompare(b.stepName ?? "");
|
|
@@ -5814,13 +5829,24 @@ function inc(map, key) {
|
|
|
5814
5829
|
map[key] = (map[key] ?? 0) + 1;
|
|
5815
5830
|
}
|
|
5816
5831
|
function computeRunStatus(events) {
|
|
5832
|
+
let runTerminal;
|
|
5833
|
+
let sawRunEvent = false;
|
|
5834
|
+
for (const e of events) {
|
|
5835
|
+
if (e.kind !== "RUN") continue;
|
|
5836
|
+
sawRunEvent = true;
|
|
5837
|
+
if (e.status === "ok" || e.status === "error") {
|
|
5838
|
+
runTerminal = e.status;
|
|
5839
|
+
}
|
|
5840
|
+
}
|
|
5841
|
+
if (sawRunEvent) {
|
|
5842
|
+
return runTerminal ?? "running";
|
|
5843
|
+
}
|
|
5817
5844
|
let hasRunning = false;
|
|
5818
5845
|
for (const e of events) {
|
|
5819
5846
|
if (e.status === "error") return "error";
|
|
5820
5847
|
if (e.status === "running") hasRunning = true;
|
|
5821
5848
|
}
|
|
5822
|
-
|
|
5823
|
-
return "ok";
|
|
5849
|
+
return hasRunning ? "running" : "ok";
|
|
5824
5850
|
}
|
|
5825
5851
|
var TreeBuilder = class {
|
|
5826
5852
|
constructor(options) {
|
|
@@ -8137,7 +8163,9 @@ function compareCohortAggregates(groups, options) {
|
|
|
8137
8163
|
metric,
|
|
8138
8164
|
"Average duration (ms)",
|
|
8139
8165
|
baselineAgg?.avgDurationMs,
|
|
8140
|
-
candidateAgg?.avgDurationMs
|
|
8166
|
+
candidateAgg?.avgDurationMs,
|
|
8167
|
+
true,
|
|
8168
|
+
tolerance
|
|
8141
8169
|
);
|
|
8142
8170
|
if (item) comparisons.push(item);
|
|
8143
8171
|
break;
|
|
@@ -8147,7 +8175,9 @@ function compareCohortAggregates(groups, options) {
|
|
|
8147
8175
|
metric,
|
|
8148
8176
|
"Average LLM calls",
|
|
8149
8177
|
baselineAgg?.avgLlmCallCount,
|
|
8150
|
-
candidateAgg?.avgLlmCallCount
|
|
8178
|
+
candidateAgg?.avgLlmCallCount,
|
|
8179
|
+
true,
|
|
8180
|
+
tolerance
|
|
8151
8181
|
);
|
|
8152
8182
|
if (item) comparisons.push(item);
|
|
8153
8183
|
break;
|
|
@@ -8157,7 +8187,9 @@ function compareCohortAggregates(groups, options) {
|
|
|
8157
8187
|
metric,
|
|
8158
8188
|
"Average token usage",
|
|
8159
8189
|
baselineAgg?.avgTokenUsage,
|
|
8160
|
-
candidateAgg?.avgTokenUsage
|
|
8190
|
+
candidateAgg?.avgTokenUsage,
|
|
8191
|
+
true,
|
|
8192
|
+
tolerance
|
|
8161
8193
|
);
|
|
8162
8194
|
if (item) comparisons.push(item);
|
|
8163
8195
|
break;
|
|
@@ -8167,7 +8199,9 @@ function compareCohortAggregates(groups, options) {
|
|
|
8167
8199
|
metric,
|
|
8168
8200
|
"Average retries",
|
|
8169
8201
|
baselineAgg?.avgRetryCount,
|
|
8170
|
-
candidateAgg?.avgRetryCount
|
|
8202
|
+
candidateAgg?.avgRetryCount,
|
|
8203
|
+
true,
|
|
8204
|
+
tolerance
|
|
8171
8205
|
);
|
|
8172
8206
|
if (item) comparisons.push(item);
|
|
8173
8207
|
break;
|
|
@@ -8177,7 +8211,9 @@ function compareCohortAggregates(groups, options) {
|
|
|
8177
8211
|
metric,
|
|
8178
8212
|
"Observation failure rate",
|
|
8179
8213
|
baselineAgg?.observationFailureRate,
|
|
8180
|
-
candidateAgg?.observationFailureRate
|
|
8214
|
+
candidateAgg?.observationFailureRate,
|
|
8215
|
+
true,
|
|
8216
|
+
tolerance
|
|
8181
8217
|
);
|
|
8182
8218
|
if (item) comparisons.push(item);
|
|
8183
8219
|
break;
|
|
@@ -8213,7 +8249,9 @@ function compareCohortAggregates(groups, options) {
|
|
|
8213
8249
|
metric,
|
|
8214
8250
|
"Guardrail failures",
|
|
8215
8251
|
baselineAgg?.avgGuardrailFailures,
|
|
8216
|
-
candidateAgg?.avgGuardrailFailures
|
|
8252
|
+
candidateAgg?.avgGuardrailFailures,
|
|
8253
|
+
true,
|
|
8254
|
+
tolerance
|
|
8217
8255
|
);
|
|
8218
8256
|
if (item) comparisons.push(item);
|
|
8219
8257
|
break;
|
|
@@ -8223,7 +8261,9 @@ function compareCohortAggregates(groups, options) {
|
|
|
8223
8261
|
metric,
|
|
8224
8262
|
"Circuit violations",
|
|
8225
8263
|
baselineAgg?.avgCircuitViolations,
|
|
8226
|
-
candidateAgg?.avgCircuitViolations
|
|
8264
|
+
candidateAgg?.avgCircuitViolations,
|
|
8265
|
+
true,
|
|
8266
|
+
tolerance
|
|
8227
8267
|
);
|
|
8228
8268
|
if (item) comparisons.push(item);
|
|
8229
8269
|
break;
|
|
@@ -8233,7 +8273,9 @@ function compareCohortAggregates(groups, options) {
|
|
|
8233
8273
|
metric,
|
|
8234
8274
|
"Redaction warnings",
|
|
8235
8275
|
baselineAgg?.avgRedactionWarnings,
|
|
8236
|
-
candidateAgg?.avgRedactionWarnings
|
|
8276
|
+
candidateAgg?.avgRedactionWarnings,
|
|
8277
|
+
true,
|
|
8278
|
+
tolerance
|
|
8237
8279
|
);
|
|
8238
8280
|
if (item) comparisons.push(item);
|
|
8239
8281
|
break;
|
|
@@ -9045,5 +9087,5 @@ function renderGateReport(result, options = {}) {
|
|
|
9045
9087
|
}
|
|
9046
9088
|
|
|
9047
9089
|
export { COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, Redactor, TraceDirectory, TraceReadError, TreeBuilder, __commonJS, __require, __toESM, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, assertBundlePathContained, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, bundleRunAssetRelativePath, compactAttributes3 as compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, enrichSessionRunRecord, escapeHtml, escapeMarkdown, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, isAgentInspectTrace, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveSuiteTemplate, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, sanitizeBundleRunId, searchTraces, source_default, stableJson, summarizeObservedOutcomes, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, zeroKinds };
|
|
9048
|
-
//# sourceMappingURL=chunk-
|
|
9049
|
-
//# sourceMappingURL=chunk-
|
|
9090
|
+
//# sourceMappingURL=chunk-FVEQ7JFF.mjs.map
|
|
9091
|
+
//# sourceMappingURL=chunk-FVEQ7JFF.mjs.map
|