agent-inspect 5.0.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/docs/CLI.md +65 -0
- package/package.json +1 -1
- package/packages/cli/dist/{chunk-TB45H3QT.mjs → chunk-SJ5R2XBE.mjs} +1043 -31
- package/packages/cli/dist/chunk-SJ5R2XBE.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +1543 -312
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +276 -164
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-ZRFSYML6.mjs → src-PK22BBMH.mjs} +3 -3
- package/packages/cli/dist/{src-ZRFSYML6.mjs.map → src-PK22BBMH.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +944 -12
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +134 -1
- package/packages/core/dist/advanced.d.ts +134 -1
- package/packages/core/dist/advanced.mjs +921 -8
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/cli/dist/chunk-TB45H3QT.mjs.map +0 -1
|
@@ -1789,7 +1789,7 @@ function buildRunSummary(events) {
|
|
|
1789
1789
|
}
|
|
1790
1790
|
let totalSteps = 0;
|
|
1791
1791
|
let llmSteps = 0;
|
|
1792
|
-
let
|
|
1792
|
+
let toolSteps2 = 0;
|
|
1793
1793
|
let logicSteps = 0;
|
|
1794
1794
|
let errorSteps = 0;
|
|
1795
1795
|
let maxDepth = 0;
|
|
@@ -1819,7 +1819,7 @@ function buildRunSummary(events) {
|
|
|
1819
1819
|
for (const [id, s] of steps.entries()) {
|
|
1820
1820
|
totalSteps += 1;
|
|
1821
1821
|
if (s.type === "llm") llmSteps += 1;
|
|
1822
|
-
else if (s.type === "tool")
|
|
1822
|
+
else if (s.type === "tool") toolSteps2 += 1;
|
|
1823
1823
|
else logicSteps += 1;
|
|
1824
1824
|
if (s.status === "error") errorSteps += 1;
|
|
1825
1825
|
const depth = computeDepth(id);
|
|
@@ -1853,7 +1853,7 @@ function buildRunSummary(events) {
|
|
|
1853
1853
|
durationMs,
|
|
1854
1854
|
totalSteps,
|
|
1855
1855
|
llmSteps,
|
|
1856
|
-
toolSteps,
|
|
1856
|
+
toolSteps: toolSteps2,
|
|
1857
1857
|
logicSteps,
|
|
1858
1858
|
errorSteps,
|
|
1859
1859
|
maxDepth,
|
|
@@ -2732,9 +2732,9 @@ async function searchTraces(metas, options) {
|
|
|
2732
2732
|
}
|
|
2733
2733
|
const limit = options.limit ?? 50;
|
|
2734
2734
|
const sessionId = options.session?.trim();
|
|
2735
|
-
const
|
|
2735
|
+
const observationStatus2 = parseObservationFilter(options.observation);
|
|
2736
2736
|
const hasContentFilter = Boolean(
|
|
2737
|
-
options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter ||
|
|
2737
|
+
options.status || stepTypeFilter || nameQuery || toolQuery || durationFilter || observationStatus2
|
|
2738
2738
|
);
|
|
2739
2739
|
const results = [];
|
|
2740
2740
|
const sessionLabel = sessionId && sessionId !== "" ? sessionId : void 0;
|
|
@@ -2779,9 +2779,9 @@ async function searchTraces(metas, options) {
|
|
|
2779
2779
|
statusFilter: options.status
|
|
2780
2780
|
});
|
|
2781
2781
|
results.push(...stepMatches);
|
|
2782
|
-
if (
|
|
2782
|
+
if (observationStatus2) {
|
|
2783
2783
|
const outcomes = extractOutcomesFromTraceEvents(events);
|
|
2784
|
-
const matched = outcomes.filter((outcome) => outcome.status ===
|
|
2784
|
+
const matched = outcomes.filter((outcome) => outcome.status === observationStatus2);
|
|
2785
2785
|
for (const outcome of matched) {
|
|
2786
2786
|
results.push({
|
|
2787
2787
|
runId: m.runId,
|
|
@@ -3460,12 +3460,12 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
3460
3460
|
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
3461
3461
|
);
|
|
3462
3462
|
const ordered = [...runs].sort(compareRuns);
|
|
3463
|
-
const
|
|
3463
|
+
const path12 = [];
|
|
3464
3464
|
const visited = /* @__PURE__ */ new Set();
|
|
3465
3465
|
const pushRun = (run, confidence, source) => {
|
|
3466
3466
|
if (visited.has(run.runId)) return;
|
|
3467
3467
|
visited.add(run.runId);
|
|
3468
|
-
|
|
3468
|
+
path12.push({
|
|
3469
3469
|
runId: run.runId,
|
|
3470
3470
|
name: run.name,
|
|
3471
3471
|
startedAt: run.startedAt,
|
|
@@ -3490,7 +3490,7 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
3490
3490
|
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
3491
3491
|
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
3492
3492
|
}
|
|
3493
|
-
return
|
|
3493
|
+
return path12;
|
|
3494
3494
|
}
|
|
3495
3495
|
function metaRunIdMatches(run, token, runById) {
|
|
3496
3496
|
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
@@ -4396,7 +4396,7 @@ function stripPrefix(name, prefixes) {
|
|
|
4396
4396
|
}
|
|
4397
4397
|
return name;
|
|
4398
4398
|
}
|
|
4399
|
-
function eventEvidence(event,
|
|
4399
|
+
function eventEvidence(event, path12) {
|
|
4400
4400
|
return {
|
|
4401
4401
|
runId: event.runId,
|
|
4402
4402
|
eventId: event.eventId,
|
|
@@ -4406,7 +4406,7 @@ function eventEvidence(event, path11) {
|
|
|
4406
4406
|
kind: event.kind,
|
|
4407
4407
|
name: event.name,
|
|
4408
4408
|
status: event.status,
|
|
4409
|
-
...
|
|
4409
|
+
...path12 ? { path: path12 } : {}
|
|
4410
4410
|
};
|
|
4411
4411
|
}
|
|
4412
4412
|
function runEvidence(run) {
|
|
@@ -4469,9 +4469,9 @@ function eventEndMs(event) {
|
|
|
4469
4469
|
function normalizedKey(value) {
|
|
4470
4470
|
return value.toLowerCase().replace(/[^a-z0-9_]/g, "");
|
|
4471
4471
|
}
|
|
4472
|
-
function lastPathSegment(
|
|
4473
|
-
const parts =
|
|
4474
|
-
return parts[parts.length - 1] ??
|
|
4472
|
+
function lastPathSegment(path12) {
|
|
4473
|
+
const parts = path12.split(".");
|
|
4474
|
+
return parts[parts.length - 1] ?? path12;
|
|
4475
4475
|
}
|
|
4476
4476
|
function valueType(value) {
|
|
4477
4477
|
if (Array.isArray(value)) return "array";
|
|
@@ -4485,12 +4485,12 @@ function serializedByteLength(value) {
|
|
|
4485
4485
|
return void 0;
|
|
4486
4486
|
}
|
|
4487
4487
|
}
|
|
4488
|
-
function pushValueEntries(entries, event, value,
|
|
4489
|
-
entries.push({ event, path:
|
|
4488
|
+
function pushValueEntries(entries, event, value, path12, key, depth = 0) {
|
|
4489
|
+
entries.push({ event, path: path12, key, value });
|
|
4490
4490
|
if (depth >= 8) return;
|
|
4491
4491
|
if (Array.isArray(value)) {
|
|
4492
4492
|
for (const [index, item] of value.entries()) {
|
|
4493
|
-
pushValueEntries(entries, event, item, `${
|
|
4493
|
+
pushValueEntries(entries, event, item, `${path12}.${index}`, String(index), depth + 1);
|
|
4494
4494
|
}
|
|
4495
4495
|
return;
|
|
4496
4496
|
}
|
|
@@ -4500,7 +4500,7 @@ function pushValueEntries(entries, event, value, path11, key, depth = 0) {
|
|
|
4500
4500
|
entries,
|
|
4501
4501
|
event,
|
|
4502
4502
|
value[nestedKey],
|
|
4503
|
-
`${
|
|
4503
|
+
`${path12}.${nestedKey}`,
|
|
4504
4504
|
nestedKey,
|
|
4505
4505
|
depth + 1
|
|
4506
4506
|
);
|
|
@@ -4581,9 +4581,9 @@ function eventDurationMs(event) {
|
|
|
4581
4581
|
}
|
|
4582
4582
|
function treeShape(nodes) {
|
|
4583
4583
|
const lines = [];
|
|
4584
|
-
const visit = (node,
|
|
4585
|
-
lines.push(`${
|
|
4586
|
-
node.children.forEach((child, index) => visit(child, `${
|
|
4584
|
+
const visit = (node, path12) => {
|
|
4585
|
+
lines.push(`${path12}:${node.event.kind}:${node.event.name}:${node.event.status ?? "unknown"}`);
|
|
4586
|
+
node.children.forEach((child, index) => visit(child, `${path12}.${index}`));
|
|
4587
4587
|
};
|
|
4588
4588
|
nodes.forEach((node, index) => visit(node, String(index)));
|
|
4589
4589
|
return lines;
|
|
@@ -4632,9 +4632,9 @@ function retrievalShape(context) {
|
|
|
4632
4632
|
function guardrailShape(context) {
|
|
4633
4633
|
return guardrailEvents(context).map((event) => signalName(event, ["guardrailName", "guardrail", "guardrailId"], ["guardrail:"])).sort((a, b) => a.localeCompare(b));
|
|
4634
4634
|
}
|
|
4635
|
-
function firstEvidenceForKind(context, kind,
|
|
4635
|
+
function firstEvidenceForKind(context, kind, path12) {
|
|
4636
4636
|
const event = context.events.find((candidate) => candidate.kind === kind);
|
|
4637
|
-
return event ? [eventEvidence(event,
|
|
4637
|
+
return event ? [eventEvidence(event, path12)] : runEvidence(context.selectedRun);
|
|
4638
4638
|
}
|
|
4639
4639
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
4640
4640
|
return failFinding("baseline.regression", message, evidence, expected, actual);
|
|
@@ -4984,13 +4984,13 @@ function createStructureCycleRule() {
|
|
|
4984
4984
|
const seenCycles = /* @__PURE__ */ new Set();
|
|
4985
4985
|
const findings = [];
|
|
4986
4986
|
for (const event of [...context.events].sort((a, b) => a.eventId.localeCompare(b.eventId))) {
|
|
4987
|
-
const
|
|
4987
|
+
const path12 = [];
|
|
4988
4988
|
const seenAt = /* @__PURE__ */ new Map();
|
|
4989
4989
|
let current = event;
|
|
4990
4990
|
while (current) {
|
|
4991
4991
|
const existing = seenAt.get(current.eventId);
|
|
4992
4992
|
if (existing !== void 0) {
|
|
4993
|
-
const cycle =
|
|
4993
|
+
const cycle = path12.slice(existing);
|
|
4994
4994
|
const key = cycle.map((item) => item.eventId).sort().join("\0");
|
|
4995
4995
|
if (!seenCycles.has(key)) {
|
|
4996
4996
|
seenCycles.add(key);
|
|
@@ -5006,8 +5006,8 @@ function createStructureCycleRule() {
|
|
|
5006
5006
|
}
|
|
5007
5007
|
break;
|
|
5008
5008
|
}
|
|
5009
|
-
seenAt.set(current.eventId,
|
|
5010
|
-
|
|
5009
|
+
seenAt.set(current.eventId, path12.length);
|
|
5010
|
+
path12.push(current);
|
|
5011
5011
|
current = current.parentId ? byId.get(current.parentId) : void 0;
|
|
5012
5012
|
}
|
|
5013
5013
|
}
|
|
@@ -7876,6 +7876,1018 @@ function renderSuiteReport(result, options = {}) {
|
|
|
7876
7876
|
return renderSuiteReportMarkdown(result);
|
|
7877
7877
|
}
|
|
7878
7878
|
|
|
7879
|
-
|
|
7880
|
-
|
|
7881
|
-
|
|
7879
|
+
// packages/core/src/cohort/types.ts
|
|
7880
|
+
var COHORT_METRIC_IDS = [
|
|
7881
|
+
"errorRate",
|
|
7882
|
+
"duration",
|
|
7883
|
+
"toolChoice",
|
|
7884
|
+
"toolOrdering",
|
|
7885
|
+
"llmCallCount",
|
|
7886
|
+
"tokenUsage",
|
|
7887
|
+
"retryCount",
|
|
7888
|
+
"observationFailure",
|
|
7889
|
+
"guardrailFailure",
|
|
7890
|
+
"circuitViolation",
|
|
7891
|
+
"redactionWarning"
|
|
7892
|
+
];
|
|
7893
|
+
|
|
7894
|
+
// packages/core/src/cohort/grouping.ts
|
|
7895
|
+
function parseCohortMetricList(value) {
|
|
7896
|
+
if (value === void 0 || value.trim() === "") return [];
|
|
7897
|
+
return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
|
|
7898
|
+
}
|
|
7899
|
+
function parseGroupBySpec(groupBy) {
|
|
7900
|
+
const raw = (groupBy ?? "model").trim();
|
|
7901
|
+
if (raw === "model") return { kind: "model" };
|
|
7902
|
+
if (raw === "session") return { kind: "session" };
|
|
7903
|
+
if (raw === "group") return { kind: "group" };
|
|
7904
|
+
if (raw.startsWith("metadata.")) {
|
|
7905
|
+
const metadataKey = raw.slice("metadata.".length).trim();
|
|
7906
|
+
if (metadataKey === "") throw new Error("metadata group-by requires a key.");
|
|
7907
|
+
return { kind: "metadata", metadataKey };
|
|
7908
|
+
}
|
|
7909
|
+
throw new Error(`Unsupported --group-by value: ${raw}`);
|
|
7910
|
+
}
|
|
7911
|
+
function metadataString(metadata, key) {
|
|
7912
|
+
const value = metadata?.[key];
|
|
7913
|
+
return typeof value === "string" && value.trim() !== "" ? value.trim() : void 0;
|
|
7914
|
+
}
|
|
7915
|
+
function resolveRunGroupKey(run, groupBy) {
|
|
7916
|
+
const metadata = run.metadata ?? {};
|
|
7917
|
+
switch (groupBy.kind) {
|
|
7918
|
+
case "model":
|
|
7919
|
+
return metadataString(metadata, "model") ?? "unknown";
|
|
7920
|
+
case "session":
|
|
7921
|
+
return extractSessionWorkflowMetadata(metadata)?.sessionId ?? metadataString(metadata, "sessionId") ?? "__unscoped__";
|
|
7922
|
+
case "group":
|
|
7923
|
+
return extractSessionWorkflowMetadata(metadata)?.groupId ?? metadataString(metadata, "groupId") ?? "__unscoped__";
|
|
7924
|
+
case "metadata":
|
|
7925
|
+
return metadataString(metadata, groupBy.metadataKey) ?? "__missing__";
|
|
7926
|
+
default:
|
|
7927
|
+
return "unknown";
|
|
7928
|
+
}
|
|
7929
|
+
}
|
|
7930
|
+
function resolveCohortLabel(run, cohortKey, baseline, candidate) {
|
|
7931
|
+
const label = metadataString(run.metadata, cohortKey);
|
|
7932
|
+
if (label === void 0) return void 0;
|
|
7933
|
+
if (baseline !== void 0 && label === baseline) return baseline;
|
|
7934
|
+
if (candidate !== void 0 && label === candidate) return candidate;
|
|
7935
|
+
if (baseline === void 0 && candidate === void 0) return label;
|
|
7936
|
+
return void 0;
|
|
7937
|
+
}
|
|
7938
|
+
function filterRunsForCohort(runs, options) {
|
|
7939
|
+
const warnings = [];
|
|
7940
|
+
if (options.baseline === void 0 && options.candidate === void 0) {
|
|
7941
|
+
return { runs: [...runs], warnings };
|
|
7942
|
+
}
|
|
7943
|
+
const selected = [];
|
|
7944
|
+
for (const run of runs) {
|
|
7945
|
+
const label = resolveCohortLabel(
|
|
7946
|
+
run,
|
|
7947
|
+
options.cohortKey,
|
|
7948
|
+
options.baseline,
|
|
7949
|
+
options.candidate
|
|
7950
|
+
);
|
|
7951
|
+
if (label !== void 0) selected.push(run);
|
|
7952
|
+
}
|
|
7953
|
+
if (selected.length === 0) {
|
|
7954
|
+
warnings.push(
|
|
7955
|
+
`No runs matched baseline/candidate labels on metadata.${options.cohortKey}.`
|
|
7956
|
+
);
|
|
7957
|
+
}
|
|
7958
|
+
return { runs: selected, warnings };
|
|
7959
|
+
}
|
|
7960
|
+
|
|
7961
|
+
// packages/core/src/cohort/compare.ts
|
|
7962
|
+
function compareNumber(metric, label, baseline, candidate, higherIsWorse = true) {
|
|
7963
|
+
if (baseline === void 0 && candidate === void 0) return void 0;
|
|
7964
|
+
const delta = baseline !== void 0 && candidate !== void 0 ? candidate - baseline : void 0;
|
|
7965
|
+
const regression = delta !== void 0 && (higherIsWorse && delta > 0 || !higherIsWorse && delta < 0);
|
|
7966
|
+
return {
|
|
7967
|
+
metric,
|
|
7968
|
+
baseline,
|
|
7969
|
+
candidate,
|
|
7970
|
+
delta,
|
|
7971
|
+
regression,
|
|
7972
|
+
message: `${label}: baseline=${baseline ?? "n/a"} candidate=${candidate ?? "n/a"}${delta !== void 0 ? ` (delta ${delta})` : ""}`
|
|
7973
|
+
};
|
|
7974
|
+
}
|
|
7975
|
+
function pickAggregate(groups, cohortLabel, groupKey) {
|
|
7976
|
+
return groups.find(
|
|
7977
|
+
(group) => group.cohortLabel === cohortLabel && (groupKey === void 0 || group.groupKey === groupKey)
|
|
7978
|
+
);
|
|
7979
|
+
}
|
|
7980
|
+
function compareCohortAggregates(groups, options) {
|
|
7981
|
+
const baselineAgg = pickAggregate(groups, options.baseline, options.groupKey);
|
|
7982
|
+
const candidateAgg = pickAggregate(groups, options.candidate, options.groupKey);
|
|
7983
|
+
const comparisons = [];
|
|
7984
|
+
for (const metric of options.metrics) {
|
|
7985
|
+
switch (metric) {
|
|
7986
|
+
case "errorRate": {
|
|
7987
|
+
const item = compareNumber(
|
|
7988
|
+
metric,
|
|
7989
|
+
"Error rate",
|
|
7990
|
+
baselineAgg?.errorRate,
|
|
7991
|
+
candidateAgg?.errorRate
|
|
7992
|
+
);
|
|
7993
|
+
if (item) comparisons.push(item);
|
|
7994
|
+
break;
|
|
7995
|
+
}
|
|
7996
|
+
case "duration": {
|
|
7997
|
+
const item = compareNumber(
|
|
7998
|
+
metric,
|
|
7999
|
+
"Average duration (ms)",
|
|
8000
|
+
baselineAgg?.avgDurationMs,
|
|
8001
|
+
candidateAgg?.avgDurationMs
|
|
8002
|
+
);
|
|
8003
|
+
if (item) comparisons.push(item);
|
|
8004
|
+
break;
|
|
8005
|
+
}
|
|
8006
|
+
case "llmCallCount": {
|
|
8007
|
+
const item = compareNumber(
|
|
8008
|
+
metric,
|
|
8009
|
+
"Average LLM calls",
|
|
8010
|
+
baselineAgg?.avgLlmCallCount,
|
|
8011
|
+
candidateAgg?.avgLlmCallCount
|
|
8012
|
+
);
|
|
8013
|
+
if (item) comparisons.push(item);
|
|
8014
|
+
break;
|
|
8015
|
+
}
|
|
8016
|
+
case "tokenUsage": {
|
|
8017
|
+
const item = compareNumber(
|
|
8018
|
+
metric,
|
|
8019
|
+
"Average token usage",
|
|
8020
|
+
baselineAgg?.avgTokenUsage,
|
|
8021
|
+
candidateAgg?.avgTokenUsage
|
|
8022
|
+
);
|
|
8023
|
+
if (item) comparisons.push(item);
|
|
8024
|
+
break;
|
|
8025
|
+
}
|
|
8026
|
+
case "retryCount": {
|
|
8027
|
+
const item = compareNumber(
|
|
8028
|
+
metric,
|
|
8029
|
+
"Average retries",
|
|
8030
|
+
baselineAgg?.avgRetryCount,
|
|
8031
|
+
candidateAgg?.avgRetryCount
|
|
8032
|
+
);
|
|
8033
|
+
if (item) comparisons.push(item);
|
|
8034
|
+
break;
|
|
8035
|
+
}
|
|
8036
|
+
case "observationFailure": {
|
|
8037
|
+
const item = compareNumber(
|
|
8038
|
+
metric,
|
|
8039
|
+
"Observation failure rate",
|
|
8040
|
+
baselineAgg?.observationFailureRate,
|
|
8041
|
+
candidateAgg?.observationFailureRate
|
|
8042
|
+
);
|
|
8043
|
+
if (item) comparisons.push(item);
|
|
8044
|
+
break;
|
|
8045
|
+
}
|
|
8046
|
+
case "toolChoice": {
|
|
8047
|
+
const baselineValue = baselineAgg?.dominantToolChoice;
|
|
8048
|
+
const candidateValue = candidateAgg?.dominantToolChoice;
|
|
8049
|
+
comparisons.push({
|
|
8050
|
+
metric,
|
|
8051
|
+
baseline: baselineValue,
|
|
8052
|
+
candidate: candidateValue,
|
|
8053
|
+
delta: baselineValue === candidateValue ? "same" : "changed",
|
|
8054
|
+
regression: baselineValue !== candidateValue,
|
|
8055
|
+
message: `Tool choice: baseline=${baselineValue ?? "n/a"} candidate=${candidateValue ?? "n/a"}`
|
|
8056
|
+
});
|
|
8057
|
+
break;
|
|
8058
|
+
}
|
|
8059
|
+
case "toolOrdering": {
|
|
8060
|
+
const baselineValue = baselineAgg?.toolOrderingSignature;
|
|
8061
|
+
const candidateValue = candidateAgg?.toolOrderingSignature;
|
|
8062
|
+
comparisons.push({
|
|
8063
|
+
metric,
|
|
8064
|
+
baseline: baselineValue,
|
|
8065
|
+
candidate: candidateValue,
|
|
8066
|
+
delta: baselineValue === candidateValue ? "same" : "changed",
|
|
8067
|
+
regression: baselineValue !== candidateValue,
|
|
8068
|
+
message: `Tool ordering: baseline=${baselineValue ?? "n/a"} candidate=${candidateValue ?? "n/a"}`
|
|
8069
|
+
});
|
|
8070
|
+
break;
|
|
8071
|
+
}
|
|
8072
|
+
case "guardrailFailure": {
|
|
8073
|
+
const item = compareNumber(
|
|
8074
|
+
metric,
|
|
8075
|
+
"Guardrail failures",
|
|
8076
|
+
baselineAgg?.avgGuardrailFailures,
|
|
8077
|
+
candidateAgg?.avgGuardrailFailures
|
|
8078
|
+
);
|
|
8079
|
+
if (item) comparisons.push(item);
|
|
8080
|
+
break;
|
|
8081
|
+
}
|
|
8082
|
+
case "circuitViolation": {
|
|
8083
|
+
const item = compareNumber(
|
|
8084
|
+
metric,
|
|
8085
|
+
"Circuit violations",
|
|
8086
|
+
baselineAgg?.avgCircuitViolations,
|
|
8087
|
+
candidateAgg?.avgCircuitViolations
|
|
8088
|
+
);
|
|
8089
|
+
if (item) comparisons.push(item);
|
|
8090
|
+
break;
|
|
8091
|
+
}
|
|
8092
|
+
case "redactionWarning": {
|
|
8093
|
+
const item = compareNumber(
|
|
8094
|
+
metric,
|
|
8095
|
+
"Redaction warnings",
|
|
8096
|
+
baselineAgg?.avgRedactionWarnings,
|
|
8097
|
+
candidateAgg?.avgRedactionWarnings
|
|
8098
|
+
);
|
|
8099
|
+
if (item) comparisons.push(item);
|
|
8100
|
+
break;
|
|
8101
|
+
}
|
|
8102
|
+
}
|
|
8103
|
+
}
|
|
8104
|
+
return comparisons;
|
|
8105
|
+
}
|
|
8106
|
+
|
|
8107
|
+
// packages/core/src/cohort/metrics.ts
|
|
8108
|
+
function asNumber(value) {
|
|
8109
|
+
return typeof value === "number" && Number.isFinite(value) ? value : void 0;
|
|
8110
|
+
}
|
|
8111
|
+
function toolSteps(events) {
|
|
8112
|
+
const ordering = [];
|
|
8113
|
+
const choices = /* @__PURE__ */ new Set();
|
|
8114
|
+
const sorted = [...events].filter((event) => event.event === "step_started").sort((a, b) => a.timestamp - b.timestamp);
|
|
8115
|
+
for (const event of sorted) {
|
|
8116
|
+
const step = event;
|
|
8117
|
+
if (step.type !== "tool") continue;
|
|
8118
|
+
const name = typeof step.metadata?.toolName === "string" ? step.metadata.toolName : step.name;
|
|
8119
|
+
ordering.push(name);
|
|
8120
|
+
choices.add(name);
|
|
8121
|
+
}
|
|
8122
|
+
return { choices: [...choices].sort(), ordering };
|
|
8123
|
+
}
|
|
8124
|
+
async function computeCohortRunMetrics(input) {
|
|
8125
|
+
const events = await readTraceEventsFromFile(input.filePath);
|
|
8126
|
+
const summary = buildRunSummary(events);
|
|
8127
|
+
const tools = toolSteps(events);
|
|
8128
|
+
const outcomes = extractOutcomesFromTraceEvents(events);
|
|
8129
|
+
const observationFailures = outcomes.filter((item) => item.status === "failed").length;
|
|
8130
|
+
const metadata = input.metadata ?? {};
|
|
8131
|
+
const retryCount2 = asNumber(metadata.attempt) !== void 0 && asNumber(metadata.attempt) > 1 ? asNumber(metadata.attempt) - 1 : typeof metadata.retryOf === "string" ? 1 : 0;
|
|
8132
|
+
return {
|
|
8133
|
+
runId: input.runId,
|
|
8134
|
+
...input.cohortLabel !== void 0 ? { cohortLabel: input.cohortLabel } : {},
|
|
8135
|
+
groupKey: input.groupKey,
|
|
8136
|
+
status: summary.status,
|
|
8137
|
+
error: summary.status === "error",
|
|
8138
|
+
durationMs: summary.durationMs ?? input.durationMs,
|
|
8139
|
+
llmCallCount: summary.llmSteps,
|
|
8140
|
+
tokenUsageTotal: summary.totalTokens?.total,
|
|
8141
|
+
retryCount: retryCount2,
|
|
8142
|
+
observationFailures,
|
|
8143
|
+
guardrailFailures: asNumber(metadata.guardrailFailures) ?? 0,
|
|
8144
|
+
circuitViolations: asNumber(metadata.circuitViolations) ?? 0,
|
|
8145
|
+
redactionWarnings: asNumber(metadata.redactionWarnings) ?? 0,
|
|
8146
|
+
toolChoices: tools.choices,
|
|
8147
|
+
toolOrdering: tools.ordering
|
|
8148
|
+
};
|
|
8149
|
+
}
|
|
8150
|
+
function percentile2(values, p) {
|
|
8151
|
+
if (values.length === 0) return void 0;
|
|
8152
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
8153
|
+
const idx = Math.min(
|
|
8154
|
+
sorted.length - 1,
|
|
8155
|
+
Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
|
|
8156
|
+
);
|
|
8157
|
+
return sorted[idx];
|
|
8158
|
+
}
|
|
8159
|
+
function dominantToolChoice(runs) {
|
|
8160
|
+
const counts = /* @__PURE__ */ new Map();
|
|
8161
|
+
for (const run of runs) {
|
|
8162
|
+
const signature = run.toolChoices.join(",");
|
|
8163
|
+
if (signature === "") continue;
|
|
8164
|
+
counts.set(signature, (counts.get(signature) ?? 0) + 1);
|
|
8165
|
+
}
|
|
8166
|
+
let best;
|
|
8167
|
+
let bestCount = 0;
|
|
8168
|
+
for (const [key, count] of counts) {
|
|
8169
|
+
if (count > bestCount) {
|
|
8170
|
+
best = key;
|
|
8171
|
+
bestCount = count;
|
|
8172
|
+
}
|
|
8173
|
+
}
|
|
8174
|
+
return best;
|
|
8175
|
+
}
|
|
8176
|
+
function orderingSignature(runs) {
|
|
8177
|
+
const counts = /* @__PURE__ */ new Map();
|
|
8178
|
+
for (const run of runs) {
|
|
8179
|
+
const signature = run.toolOrdering.join(">");
|
|
8180
|
+
if (signature === "") continue;
|
|
8181
|
+
counts.set(signature, (counts.get(signature) ?? 0) + 1);
|
|
8182
|
+
}
|
|
8183
|
+
let best;
|
|
8184
|
+
let bestCount = 0;
|
|
8185
|
+
for (const [key, count] of counts) {
|
|
8186
|
+
if (count > bestCount) {
|
|
8187
|
+
best = key;
|
|
8188
|
+
bestCount = count;
|
|
8189
|
+
}
|
|
8190
|
+
}
|
|
8191
|
+
return best;
|
|
8192
|
+
}
|
|
8193
|
+
function aggregateCohortMetrics(runs, groupKey, cohortLabel) {
|
|
8194
|
+
const durations = runs.map((run) => run.durationMs).filter((value) => typeof value === "number");
|
|
8195
|
+
const tokenValues = runs.map((run) => run.tokenUsageTotal).filter((value) => typeof value === "number");
|
|
8196
|
+
const errors = runs.filter((run) => run.error).length;
|
|
8197
|
+
const observationFailures = runs.reduce((sum, run) => sum + run.observationFailures, 0);
|
|
8198
|
+
return {
|
|
8199
|
+
groupKey,
|
|
8200
|
+
...cohortLabel !== void 0 ? { cohortLabel } : {},
|
|
8201
|
+
runCount: runs.length,
|
|
8202
|
+
errorRate: runs.length > 0 ? errors / runs.length : 0,
|
|
8203
|
+
avgDurationMs: durations.length > 0 ? durations.reduce((sum, value) => sum + value, 0) / durations.length : void 0,
|
|
8204
|
+
p95DurationMs: percentile2(durations, 95),
|
|
8205
|
+
avgLlmCallCount: runs.length > 0 ? runs.reduce((sum, run) => sum + run.llmCallCount, 0) / runs.length : 0,
|
|
8206
|
+
avgTokenUsage: tokenValues.length > 0 ? tokenValues.reduce((sum, value) => sum + value, 0) / tokenValues.length : void 0,
|
|
8207
|
+
avgRetryCount: runs.length > 0 ? runs.reduce((sum, run) => sum + run.retryCount, 0) / runs.length : 0,
|
|
8208
|
+
observationFailureRate: runs.length > 0 ? observationFailures / runs.length : 0,
|
|
8209
|
+
avgGuardrailFailures: runs.length > 0 ? runs.reduce((sum, run) => sum + run.guardrailFailures, 0) / runs.length : 0,
|
|
8210
|
+
avgCircuitViolations: runs.length > 0 ? runs.reduce((sum, run) => sum + run.circuitViolations, 0) / runs.length : 0,
|
|
8211
|
+
avgRedactionWarnings: runs.length > 0 ? runs.reduce((sum, run) => sum + run.redactionWarnings, 0) / runs.length : 0,
|
|
8212
|
+
dominantToolChoice: dominantToolChoice(runs),
|
|
8213
|
+
toolOrderingSignature: orderingSignature(runs)
|
|
8214
|
+
};
|
|
8215
|
+
}
|
|
8216
|
+
|
|
8217
|
+
// packages/core/src/cohort/analyze.ts
|
|
8218
|
+
var DEFAULT_METRICS = [
|
|
8219
|
+
"errorRate",
|
|
8220
|
+
"duration",
|
|
8221
|
+
"toolChoice",
|
|
8222
|
+
"observationFailure"
|
|
8223
|
+
];
|
|
8224
|
+
function normalizeMetrics(metrics) {
|
|
8225
|
+
if (metrics === void 0 || metrics.length === 0) return [...DEFAULT_METRICS];
|
|
8226
|
+
const allowed = new Set(COHORT_METRIC_IDS);
|
|
8227
|
+
return metrics.filter((metric) => allowed.has(metric));
|
|
8228
|
+
}
|
|
8229
|
+
async function analyzeCohort(runsInput, options) {
|
|
8230
|
+
const cohortKey = options.cohortKey ?? "cohort";
|
|
8231
|
+
const groupBySpec = parseGroupBySpec(options.groupBy);
|
|
8232
|
+
const metrics = normalizeMetrics(options.metrics);
|
|
8233
|
+
const { runs: filteredRuns, warnings } = filterRunsForCohort(runsInput, {
|
|
8234
|
+
cohortKey,
|
|
8235
|
+
baseline: options.baseline,
|
|
8236
|
+
candidate: options.candidate
|
|
8237
|
+
});
|
|
8238
|
+
const runMetrics = [];
|
|
8239
|
+
for (const run of filteredRuns) {
|
|
8240
|
+
if (run.filePath === void 0) continue;
|
|
8241
|
+
const cohortLabel = resolveCohortLabel(
|
|
8242
|
+
run,
|
|
8243
|
+
cohortKey,
|
|
8244
|
+
options.baseline,
|
|
8245
|
+
options.candidate
|
|
8246
|
+
);
|
|
8247
|
+
runMetrics.push(
|
|
8248
|
+
await computeCohortRunMetrics({
|
|
8249
|
+
runId: run.runId,
|
|
8250
|
+
filePath: run.filePath,
|
|
8251
|
+
metadata: run.metadata,
|
|
8252
|
+
status: run.status,
|
|
8253
|
+
durationMs: run.durationMs,
|
|
8254
|
+
groupKey: resolveRunGroupKey(run, groupBySpec),
|
|
8255
|
+
cohortLabel
|
|
8256
|
+
})
|
|
8257
|
+
);
|
|
8258
|
+
}
|
|
8259
|
+
const groupMap = /* @__PURE__ */ new Map();
|
|
8260
|
+
for (const run of runMetrics) {
|
|
8261
|
+
const key = `${run.cohortLabel ?? "*"}::${run.groupKey}`;
|
|
8262
|
+
const bucket = groupMap.get(key) ?? [];
|
|
8263
|
+
bucket.push(run);
|
|
8264
|
+
groupMap.set(key, bucket);
|
|
8265
|
+
}
|
|
8266
|
+
const groups = [...groupMap.entries()].sort(([a], [b]) => a.localeCompare(b)).map(
|
|
8267
|
+
([, bucket]) => aggregateCohortMetrics(
|
|
8268
|
+
bucket,
|
|
8269
|
+
bucket[0].groupKey,
|
|
8270
|
+
bucket[0]?.cohortLabel
|
|
8271
|
+
)
|
|
8272
|
+
);
|
|
8273
|
+
const comparisons = options.baseline !== void 0 && options.candidate !== void 0 ? (() => {
|
|
8274
|
+
const groupKeys = [
|
|
8275
|
+
...new Set(groups.map((group) => group.groupKey))
|
|
8276
|
+
].sort((a, b) => a.localeCompare(b));
|
|
8277
|
+
const items = [];
|
|
8278
|
+
for (const groupKey of groupKeys) {
|
|
8279
|
+
items.push(
|
|
8280
|
+
...compareCohortAggregates(groups, {
|
|
8281
|
+
baseline: options.baseline,
|
|
8282
|
+
candidate: options.candidate,
|
|
8283
|
+
metrics,
|
|
8284
|
+
groupKey
|
|
8285
|
+
})
|
|
8286
|
+
);
|
|
8287
|
+
}
|
|
8288
|
+
return items;
|
|
8289
|
+
})() : [];
|
|
8290
|
+
const regression = comparisons.some((item) => item.regression);
|
|
8291
|
+
return {
|
|
8292
|
+
ok: !regression,
|
|
8293
|
+
traceDir: options.traceDir,
|
|
8294
|
+
...options.baseline !== void 0 ? { baseline: options.baseline } : {},
|
|
8295
|
+
...options.candidate !== void 0 ? { candidate: options.candidate } : {},
|
|
8296
|
+
cohortKey,
|
|
8297
|
+
groupBy: options.groupBy ?? "model",
|
|
8298
|
+
metrics,
|
|
8299
|
+
groups,
|
|
8300
|
+
comparisons,
|
|
8301
|
+
runs: runMetrics,
|
|
8302
|
+
warnings
|
|
8303
|
+
};
|
|
8304
|
+
}
|
|
8305
|
+
|
|
8306
|
+
// packages/core/src/exporters/helpers.ts
|
|
8307
|
+
var REDACT_SUBSTRINGS = [
|
|
8308
|
+
"authorization",
|
|
8309
|
+
"cookie",
|
|
8310
|
+
"token",
|
|
8311
|
+
"apikey",
|
|
8312
|
+
"password",
|
|
8313
|
+
"secret",
|
|
8314
|
+
"email"
|
|
8315
|
+
];
|
|
8316
|
+
function shouldRedactKey(key) {
|
|
8317
|
+
const k = key.toLowerCase();
|
|
8318
|
+
for (const s of REDACT_SUBSTRINGS) {
|
|
8319
|
+
if (k.includes(s)) return true;
|
|
8320
|
+
}
|
|
8321
|
+
return false;
|
|
8322
|
+
}
|
|
8323
|
+
function safeString(value, maxLength) {
|
|
8324
|
+
if (value === null || value === void 0) return "";
|
|
8325
|
+
let s;
|
|
8326
|
+
if (typeof value === "string") s = value;
|
|
8327
|
+
else if (typeof value === "number" || typeof value === "boolean") s = String(value);
|
|
8328
|
+
else s = stableJson(value, false);
|
|
8329
|
+
if (maxLength !== void 0 && maxLength >= 0 && s.length > maxLength) {
|
|
8330
|
+
return `${s.slice(0, maxLength)}\u2026`;
|
|
8331
|
+
}
|
|
8332
|
+
return s;
|
|
8333
|
+
}
|
|
8334
|
+
function escapeMarkdown(value) {
|
|
8335
|
+
return value.replace(/\|/g, "\\|").replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n/g, " ");
|
|
8336
|
+
}
|
|
8337
|
+
function escapeHtml(value) {
|
|
8338
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
8339
|
+
}
|
|
8340
|
+
function sortKeysDeep(input) {
|
|
8341
|
+
if (input === null || typeof input !== "object") return input;
|
|
8342
|
+
if (Array.isArray(input)) return input.map(sortKeysDeep);
|
|
8343
|
+
const o = input;
|
|
8344
|
+
const out = {};
|
|
8345
|
+
for (const k of Object.keys(o).sort()) {
|
|
8346
|
+
out[k] = sortKeysDeep(o[k]);
|
|
8347
|
+
}
|
|
8348
|
+
return out;
|
|
8349
|
+
}
|
|
8350
|
+
function stableJson(value, pretty) {
|
|
8351
|
+
const sorted = sortKeysDeep(value);
|
|
8352
|
+
return pretty === true ? JSON.stringify(sorted, null, 2) : JSON.stringify(sorted);
|
|
8353
|
+
}
|
|
8354
|
+
function compactAttributes3(attrs, options) {
|
|
8355
|
+
if (attrs === void 0) return {};
|
|
8356
|
+
const maxLen = options?.maxLength ?? 500;
|
|
8357
|
+
const redacted = options?.redacted ?? true;
|
|
8358
|
+
const out = {};
|
|
8359
|
+
for (const key of Object.keys(attrs).sort()) {
|
|
8360
|
+
if (redacted && shouldRedactKey(key)) {
|
|
8361
|
+
out[key] = "[REDACTED]";
|
|
8362
|
+
continue;
|
|
8363
|
+
}
|
|
8364
|
+
const v = attrs[key];
|
|
8365
|
+
out[key] = compactValue(v, maxLen, redacted);
|
|
8366
|
+
}
|
|
8367
|
+
return out;
|
|
8368
|
+
}
|
|
8369
|
+
function compactValue(value, maxLen, redacted) {
|
|
8370
|
+
if (value === null || typeof value !== "object") {
|
|
8371
|
+
return typeof value === "string" ? safeString(value, maxLen) : value;
|
|
8372
|
+
}
|
|
8373
|
+
if (Array.isArray(value)) {
|
|
8374
|
+
const arr = value.slice(0, 20).map((x) => compactValue(x, maxLen, redacted));
|
|
8375
|
+
if (value.length > 20) arr.push(`\u2026(+${value.length - 20} more)`);
|
|
8376
|
+
return arr;
|
|
8377
|
+
}
|
|
8378
|
+
const o = value;
|
|
8379
|
+
const inner = {};
|
|
8380
|
+
for (const k of Object.keys(o)) {
|
|
8381
|
+
if (redacted && shouldRedactKey(k)) inner[k] = "[REDACTED]";
|
|
8382
|
+
else inner[k] = compactValue(o[k], maxLen, redacted);
|
|
8383
|
+
}
|
|
8384
|
+
return inner;
|
|
8385
|
+
}
|
|
8386
|
+
function flattenTree(tree) {
|
|
8387
|
+
const out = [];
|
|
8388
|
+
function walk(nodes) {
|
|
8389
|
+
for (const n of nodes) {
|
|
8390
|
+
out.push(n);
|
|
8391
|
+
if (n.children.length > 0) walk(n.children);
|
|
8392
|
+
}
|
|
8393
|
+
}
|
|
8394
|
+
walk(tree.children);
|
|
8395
|
+
return out;
|
|
8396
|
+
}
|
|
8397
|
+
function zeroKinds() {
|
|
8398
|
+
return {
|
|
8399
|
+
RUN: 0,
|
|
8400
|
+
AGENT: 0,
|
|
8401
|
+
LLM: 0,
|
|
8402
|
+
TOOL: 0,
|
|
8403
|
+
CHAIN: 0,
|
|
8404
|
+
RETRIEVER: 0,
|
|
8405
|
+
DECISION: 0,
|
|
8406
|
+
RESULT: 0,
|
|
8407
|
+
ERROR: 0,
|
|
8408
|
+
LOGIC: 0,
|
|
8409
|
+
LOG: 0,
|
|
8410
|
+
OUTCOME: 0
|
|
8411
|
+
};
|
|
8412
|
+
}
|
|
8413
|
+
|
|
8414
|
+
// packages/core/src/cohort/render.ts
|
|
8415
|
+
function formatRate(value) {
|
|
8416
|
+
if (value === void 0) return "n/a";
|
|
8417
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
8418
|
+
}
|
|
8419
|
+
function renderCohortSummaryMarkdown(result) {
|
|
8420
|
+
const lines = [];
|
|
8421
|
+
lines.push("# Cohort analysis");
|
|
8422
|
+
lines.push("");
|
|
8423
|
+
lines.push(`Trace directory: \`${result.traceDir}\``);
|
|
8424
|
+
lines.push(`Group by: \`${result.groupBy}\``);
|
|
8425
|
+
if (result.baseline !== void 0 && result.candidate !== void 0) {
|
|
8426
|
+
lines.push(
|
|
8427
|
+
`Baseline/Candidate key: \`${result.cohortKey}\` (${result.baseline} vs ${result.candidate})`
|
|
8428
|
+
);
|
|
8429
|
+
}
|
|
8430
|
+
lines.push(`Status: **${result.ok ? "PASS" : "REGRESSION"}**`);
|
|
8431
|
+
lines.push("");
|
|
8432
|
+
if (result.warnings.length > 0) {
|
|
8433
|
+
lines.push("## Warnings");
|
|
8434
|
+
for (const warning of result.warnings) lines.push(`- ${warning}`);
|
|
8435
|
+
lines.push("");
|
|
8436
|
+
}
|
|
8437
|
+
lines.push("## Groups");
|
|
8438
|
+
for (const group of result.groups) {
|
|
8439
|
+
lines.push(
|
|
8440
|
+
`### ${group.cohortLabel ?? "all"} / ${group.groupKey} (${group.runCount} runs)`
|
|
8441
|
+
);
|
|
8442
|
+
lines.push(`- Error rate: ${formatRate(group.errorRate)}`);
|
|
8443
|
+
if (group.avgDurationMs !== void 0) {
|
|
8444
|
+
lines.push(`- Avg duration: ${Math.round(group.avgDurationMs)} ms`);
|
|
8445
|
+
}
|
|
8446
|
+
if (group.dominantToolChoice !== void 0) {
|
|
8447
|
+
lines.push(`- Dominant tools: ${group.dominantToolChoice}`);
|
|
8448
|
+
}
|
|
8449
|
+
lines.push(
|
|
8450
|
+
`- Observation failure rate: ${formatRate(group.observationFailureRate)}`
|
|
8451
|
+
);
|
|
8452
|
+
lines.push("");
|
|
8453
|
+
}
|
|
8454
|
+
if (result.comparisons.length > 0) {
|
|
8455
|
+
lines.push("## Comparisons");
|
|
8456
|
+
for (const comparison of result.comparisons) {
|
|
8457
|
+
const flag = comparison.regression ? " **REGRESSION**" : "";
|
|
8458
|
+
lines.push(`- ${comparison.message}${flag}`);
|
|
8459
|
+
}
|
|
8460
|
+
lines.push("");
|
|
8461
|
+
}
|
|
8462
|
+
return lines.join("\n").trimEnd();
|
|
8463
|
+
}
|
|
8464
|
+
function renderCohortReportHtml(result) {
|
|
8465
|
+
const rows = result.groups.map(
|
|
8466
|
+
(group) => `<tr><td>${escapeHtml(group.cohortLabel ?? "all")}</td><td>${escapeHtml(group.groupKey)}</td><td>${group.runCount}</td><td>${escapeHtml(formatRate(group.errorRate))}</td><td>${group.avgDurationMs !== void 0 ? Math.round(group.avgDurationMs) : "n/a"}</td></tr>`
|
|
8467
|
+
).join("");
|
|
8468
|
+
const comparisons = result.comparisons.map(
|
|
8469
|
+
(item) => `<li>${escapeHtml(item.message)}${item.regression ? " <strong>REGRESSION</strong>" : ""}</li>`
|
|
8470
|
+
).join("");
|
|
8471
|
+
return `<!DOCTYPE html>
|
|
8472
|
+
<html lang="en">
|
|
8473
|
+
<head>
|
|
8474
|
+
<meta charset="utf-8" />
|
|
8475
|
+
<title>Cohort report</title>
|
|
8476
|
+
<style>
|
|
8477
|
+
body { font-family: system-ui, sans-serif; margin: 2rem; }
|
|
8478
|
+
table { border-collapse: collapse; width: 100%; }
|
|
8479
|
+
th, td { border: 1px solid #ddd; padding: 0.5rem; text-align: left; }
|
|
8480
|
+
th { background: #f6f6f6; }
|
|
8481
|
+
</style>
|
|
8482
|
+
</head>
|
|
8483
|
+
<body>
|
|
8484
|
+
<h1>Cohort analysis</h1>
|
|
8485
|
+
<p>Status: <strong>${result.ok ? "PASS" : "REGRESSION"}</strong></p>
|
|
8486
|
+
<p>Trace directory: <code>${escapeHtml(result.traceDir)}</code></p>
|
|
8487
|
+
<h2>Groups</h2>
|
|
8488
|
+
<table>
|
|
8489
|
+
<thead><tr><th>Cohort</th><th>Group</th><th>Runs</th><th>Error rate</th><th>Avg duration (ms)</th></tr></thead>
|
|
8490
|
+
<tbody>${rows}</tbody>
|
|
8491
|
+
</table>
|
|
8492
|
+
${result.comparisons.length > 0 ? `<h2>Comparisons</h2><ul>${comparisons}</ul>` : ""}
|
|
8493
|
+
</body>
|
|
8494
|
+
</html>`;
|
|
8495
|
+
}
|
|
8496
|
+
function renderCohortReport(result, options = {}) {
|
|
8497
|
+
const format = options.format ?? "markdown";
|
|
8498
|
+
if (format === "json") return JSON.stringify(result, null, 2);
|
|
8499
|
+
if (format === "html") return renderCohortReportHtml(result);
|
|
8500
|
+
return renderCohortSummaryMarkdown(result);
|
|
8501
|
+
}
|
|
8502
|
+
|
|
8503
|
+
// packages/core/src/gate/parse.ts
|
|
8504
|
+
function parseGateList(value) {
|
|
8505
|
+
if (value === void 0 || value.trim() === "") return [];
|
|
8506
|
+
return value.split(",").map((item) => item.trim()).filter((item) => item.length > 0);
|
|
8507
|
+
}
|
|
8508
|
+
|
|
8509
|
+
// packages/core/src/gate/evaluate.ts
|
|
8510
|
+
function percentile3(values, p) {
|
|
8511
|
+
if (values.length === 0) return void 0;
|
|
8512
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
8513
|
+
const idx = Math.min(
|
|
8514
|
+
sorted.length - 1,
|
|
8515
|
+
Math.max(0, Math.ceil(p / 100 * sorted.length) - 1)
|
|
8516
|
+
);
|
|
8517
|
+
return sorted[idx];
|
|
8518
|
+
}
|
|
8519
|
+
function hasThresholds(options) {
|
|
8520
|
+
return options.maxErrorRate !== void 0 || options.maxP95DurationMs !== void 0 || (options.forbidTools?.length ?? 0) > 0 || (options.requireObservations?.length ?? 0) > 0;
|
|
8521
|
+
}
|
|
8522
|
+
function gateHasThresholds(options) {
|
|
8523
|
+
return hasThresholds(options);
|
|
8524
|
+
}
|
|
8525
|
+
async function loadRunMetrics(runs) {
|
|
8526
|
+
const metrics = [];
|
|
8527
|
+
for (const run of runs) {
|
|
8528
|
+
if (run.filePath === void 0) continue;
|
|
8529
|
+
metrics.push(
|
|
8530
|
+
await computeCohortRunMetrics({
|
|
8531
|
+
runId: run.runId,
|
|
8532
|
+
filePath: run.filePath,
|
|
8533
|
+
metadata: run.metadata,
|
|
8534
|
+
status: run.status,
|
|
8535
|
+
durationMs: run.durationMs,
|
|
8536
|
+
groupKey: "all"
|
|
8537
|
+
})
|
|
8538
|
+
);
|
|
8539
|
+
}
|
|
8540
|
+
return metrics;
|
|
8541
|
+
}
|
|
8542
|
+
async function observationStatus(filePath, name) {
|
|
8543
|
+
const events = await readTraceEventsFromFile(filePath);
|
|
8544
|
+
const outcomes = extractOutcomesFromTraceEvents(events);
|
|
8545
|
+
const match = outcomes.find((item) => item.name === name);
|
|
8546
|
+
if (!match) return "missing";
|
|
8547
|
+
return match.status === "passed" ? "passed" : "failed";
|
|
8548
|
+
}
|
|
8549
|
+
async function evaluateGateThresholds(runs, options) {
|
|
8550
|
+
const checks = [];
|
|
8551
|
+
const readErrors = [];
|
|
8552
|
+
if (!hasThresholds(options)) {
|
|
8553
|
+
return { checks, readErrors };
|
|
8554
|
+
}
|
|
8555
|
+
if (runs.length === 0) {
|
|
8556
|
+
readErrors.push("No trace runs found in the gate directory.");
|
|
8557
|
+
return { checks, readErrors };
|
|
8558
|
+
}
|
|
8559
|
+
let runMetrics;
|
|
8560
|
+
try {
|
|
8561
|
+
runMetrics = await loadRunMetrics(runs);
|
|
8562
|
+
} catch (error) {
|
|
8563
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
8564
|
+
readErrors.push(message);
|
|
8565
|
+
return { checks, readErrors };
|
|
8566
|
+
}
|
|
8567
|
+
if (options.maxErrorRate !== void 0) {
|
|
8568
|
+
const errors = runMetrics.filter((run) => run.error).length;
|
|
8569
|
+
const actual = runMetrics.length > 0 ? errors / runMetrics.length * 100 : 0;
|
|
8570
|
+
const ok = actual <= options.maxErrorRate;
|
|
8571
|
+
checks.push({
|
|
8572
|
+
id: "maxErrorRate",
|
|
8573
|
+
name: "Max error rate",
|
|
8574
|
+
ok,
|
|
8575
|
+
expected: options.maxErrorRate,
|
|
8576
|
+
actual: Math.round(actual * 10) / 10,
|
|
8577
|
+
message: ok ? `Error rate ${actual.toFixed(1)}% within limit ${options.maxErrorRate}%` : `Error rate ${actual.toFixed(1)}% exceeds limit ${options.maxErrorRate}%`
|
|
8578
|
+
});
|
|
8579
|
+
}
|
|
8580
|
+
if (options.maxP95DurationMs !== void 0) {
|
|
8581
|
+
const durations = runMetrics.map((run) => run.durationMs).filter((value) => typeof value === "number");
|
|
8582
|
+
const actual = percentile3(durations, 95);
|
|
8583
|
+
const ok = actual !== void 0 && actual <= options.maxP95DurationMs;
|
|
8584
|
+
checks.push({
|
|
8585
|
+
id: "maxP95Duration",
|
|
8586
|
+
name: "Max p95 duration (ms)",
|
|
8587
|
+
ok,
|
|
8588
|
+
expected: options.maxP95DurationMs,
|
|
8589
|
+
actual: actual ?? "n/a",
|
|
8590
|
+
message: actual === void 0 ? "No duration samples available for p95 check." : ok ? `P95 duration ${Math.round(actual)} ms within limit ${options.maxP95DurationMs} ms` : `P95 duration ${Math.round(actual)} ms exceeds limit ${options.maxP95DurationMs} ms`
|
|
8591
|
+
});
|
|
8592
|
+
}
|
|
8593
|
+
for (const tool of options.forbidTools ?? []) {
|
|
8594
|
+
let violated = false;
|
|
8595
|
+
for (const run of runMetrics) {
|
|
8596
|
+
const used = run.toolChoices.includes(tool) || run.toolOrdering.includes(tool);
|
|
8597
|
+
if (used) {
|
|
8598
|
+
violated = true;
|
|
8599
|
+
checks.push({
|
|
8600
|
+
id: "forbidTool",
|
|
8601
|
+
name: `Forbid tool: ${tool}`,
|
|
8602
|
+
ok: false,
|
|
8603
|
+
expected: `not used`,
|
|
8604
|
+
actual: "used",
|
|
8605
|
+
runId: run.runId,
|
|
8606
|
+
message: `Forbidden tool "${tool}" used in run ${run.runId}`
|
|
8607
|
+
});
|
|
8608
|
+
}
|
|
8609
|
+
}
|
|
8610
|
+
if (!violated) {
|
|
8611
|
+
checks.push({
|
|
8612
|
+
id: "forbidTool",
|
|
8613
|
+
name: `Forbid tool: ${tool}`,
|
|
8614
|
+
ok: true,
|
|
8615
|
+
message: `Forbidden tool "${tool}" not used`
|
|
8616
|
+
});
|
|
8617
|
+
}
|
|
8618
|
+
}
|
|
8619
|
+
for (const observation of options.requireObservations ?? []) {
|
|
8620
|
+
for (const run of runs) {
|
|
8621
|
+
if (run.filePath === void 0) continue;
|
|
8622
|
+
try {
|
|
8623
|
+
const status = await observationStatus(run.filePath, observation);
|
|
8624
|
+
const ok = status === "passed";
|
|
8625
|
+
checks.push({
|
|
8626
|
+
id: "requireObservation",
|
|
8627
|
+
name: `Require observation: ${observation}`,
|
|
8628
|
+
ok,
|
|
8629
|
+
expected: "passed",
|
|
8630
|
+
actual: status,
|
|
8631
|
+
runId: run.runId,
|
|
8632
|
+
message: ok ? `Observation "${observation}" passed in run ${run.runId}` : status === "missing" ? `Observation "${observation}" missing in run ${run.runId}` : `Observation "${observation}" failed in run ${run.runId}`
|
|
8633
|
+
});
|
|
8634
|
+
} catch (error) {
|
|
8635
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
8636
|
+
readErrors.push(`Run ${run.runId}: ${message}`);
|
|
8637
|
+
}
|
|
8638
|
+
}
|
|
8639
|
+
}
|
|
8640
|
+
return { checks, readErrors };
|
|
8641
|
+
}
|
|
8642
|
+
function checksFromSuiteResult(suiteResult) {
|
|
8643
|
+
const checks = [
|
|
8644
|
+
{
|
|
8645
|
+
id: "suite",
|
|
8646
|
+
name: `Suite: ${suiteResult.suiteName}`,
|
|
8647
|
+
ok: suiteResult.ok,
|
|
8648
|
+
message: suiteResult.ok ? `Suite passed (${suiteResult.summary.passed} cases)` : `Suite failed (${suiteResult.summary.failed} failed, ${suiteResult.summary.errors} errors)`
|
|
8649
|
+
}
|
|
8650
|
+
];
|
|
8651
|
+
for (const suiteCase of suiteResult.cases) {
|
|
8652
|
+
if (suiteCase.status === "pass") continue;
|
|
8653
|
+
checks.push({
|
|
8654
|
+
id: "suite",
|
|
8655
|
+
name: `Case: ${suiteCase.id}`,
|
|
8656
|
+
ok: false,
|
|
8657
|
+
message: suiteCase.message ?? `Case status: ${suiteCase.status}`
|
|
8658
|
+
});
|
|
8659
|
+
}
|
|
8660
|
+
return checks;
|
|
8661
|
+
}
|
|
8662
|
+
function resolveExitCode(input) {
|
|
8663
|
+
if (input.configError) return 2;
|
|
8664
|
+
if (input.readError) return 3;
|
|
8665
|
+
if (!input.ok) return 1;
|
|
8666
|
+
return 0;
|
|
8667
|
+
}
|
|
8668
|
+
function validateOptions(options) {
|
|
8669
|
+
const errors = [];
|
|
8670
|
+
const hasSuite = options.suitePath !== void 0 && options.suitePath.trim() !== "";
|
|
8671
|
+
const hasThresholds2 = gateHasThresholds(options);
|
|
8672
|
+
if (!hasSuite && !hasThresholds2) {
|
|
8673
|
+
errors.push(
|
|
8674
|
+
"No gate rules specified. Pass --suite or at least one threshold flag."
|
|
8675
|
+
);
|
|
8676
|
+
}
|
|
8677
|
+
if (hasThresholds2 && (options.traceDir === void 0 || options.traceDir.trim() === "")) {
|
|
8678
|
+
if (!hasSuite) {
|
|
8679
|
+
errors.push("Threshold flags require --dir <trace-directory>.");
|
|
8680
|
+
}
|
|
8681
|
+
}
|
|
8682
|
+
if (options.maxErrorRate !== void 0 && options.maxErrorRate < 0) {
|
|
8683
|
+
errors.push("--max-error-rate must be a non-negative percentage.");
|
|
8684
|
+
}
|
|
8685
|
+
if (options.maxP95DurationMs !== void 0 && options.maxP95DurationMs < 0) {
|
|
8686
|
+
errors.push("--max-p95-duration must be a non-negative millisecond value.");
|
|
8687
|
+
}
|
|
8688
|
+
return errors;
|
|
8689
|
+
}
|
|
8690
|
+
function isConfigLoadError(error) {
|
|
8691
|
+
if (!(error instanceof Error)) return false;
|
|
8692
|
+
const ext = path7.extname(error.message);
|
|
8693
|
+
if (error.message.includes("Unsupported suite config extension")) return true;
|
|
8694
|
+
if (error.message.includes("TypeScript suite configs require")) return true;
|
|
8695
|
+
if (error.message.includes("No suite config found")) return true;
|
|
8696
|
+
if (error.message.includes("AI_SUITE_CONFIG")) return true;
|
|
8697
|
+
if (ext === ".ts" || ext === ".mts" || ext === ".cts") return true;
|
|
8698
|
+
return "diagnostics" in error;
|
|
8699
|
+
}
|
|
8700
|
+
async function runGate(runs, options) {
|
|
8701
|
+
const diagnostics = [];
|
|
8702
|
+
const checks = [];
|
|
8703
|
+
const validationErrors = validateOptions(options);
|
|
8704
|
+
if (validationErrors.length > 0) {
|
|
8705
|
+
return {
|
|
8706
|
+
ok: false,
|
|
8707
|
+
exitCode: 2,
|
|
8708
|
+
runCount: 0,
|
|
8709
|
+
checks,
|
|
8710
|
+
diagnostics: validationErrors
|
|
8711
|
+
};
|
|
8712
|
+
}
|
|
8713
|
+
let traceDir = options.traceDir?.trim();
|
|
8714
|
+
let suiteResult;
|
|
8715
|
+
if (options.suitePath !== void 0 && options.suitePath.trim() !== "") {
|
|
8716
|
+
try {
|
|
8717
|
+
suiteResult = await runSuite({
|
|
8718
|
+
configPath: options.suitePath,
|
|
8719
|
+
cwd: options.cwd
|
|
8720
|
+
});
|
|
8721
|
+
traceDir = traceDir ?? suiteResult.tracesDir;
|
|
8722
|
+
checks.push(...checksFromSuiteResult(suiteResult));
|
|
8723
|
+
} catch (error) {
|
|
8724
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
8725
|
+
diagnostics.push(message);
|
|
8726
|
+
return {
|
|
8727
|
+
ok: false,
|
|
8728
|
+
exitCode: isConfigLoadError(error) ? 2 : 3,
|
|
8729
|
+
traceDir,
|
|
8730
|
+
suitePath: options.suitePath,
|
|
8731
|
+
runCount: 0,
|
|
8732
|
+
checks,
|
|
8733
|
+
diagnostics
|
|
8734
|
+
};
|
|
8735
|
+
}
|
|
8736
|
+
}
|
|
8737
|
+
if (gateHasThresholds(options)) {
|
|
8738
|
+
const thresholdDir = traceDir;
|
|
8739
|
+
if (thresholdDir === void 0 || thresholdDir.trim() === "") {
|
|
8740
|
+
return {
|
|
8741
|
+
ok: false,
|
|
8742
|
+
exitCode: 2,
|
|
8743
|
+
traceDir,
|
|
8744
|
+
suitePath: options.suitePath,
|
|
8745
|
+
runCount: runs.length,
|
|
8746
|
+
checks,
|
|
8747
|
+
diagnostics: ["Threshold evaluation requires a trace directory."],
|
|
8748
|
+
...suiteResult !== void 0 ? { suiteResult } : {}
|
|
8749
|
+
};
|
|
8750
|
+
}
|
|
8751
|
+
const thresholdRuns = runs.length > 0 ? runs : [];
|
|
8752
|
+
const { checks: thresholdChecks, readErrors } = await evaluateGateThresholds(
|
|
8753
|
+
thresholdRuns,
|
|
8754
|
+
options
|
|
8755
|
+
);
|
|
8756
|
+
checks.push(...thresholdChecks);
|
|
8757
|
+
diagnostics.push(...readErrors);
|
|
8758
|
+
if (readErrors.length > 0) {
|
|
8759
|
+
const ok2 = checks.length > 0 && checks.every((item) => item.ok);
|
|
8760
|
+
return {
|
|
8761
|
+
ok: ok2,
|
|
8762
|
+
exitCode: resolveExitCode({
|
|
8763
|
+
ok: ok2,
|
|
8764
|
+
configError: false,
|
|
8765
|
+
readError: true
|
|
8766
|
+
}),
|
|
8767
|
+
traceDir: thresholdDir,
|
|
8768
|
+
suitePath: options.suitePath,
|
|
8769
|
+
runCount: thresholdRuns.length,
|
|
8770
|
+
checks,
|
|
8771
|
+
diagnostics,
|
|
8772
|
+
...suiteResult !== void 0 ? { suiteResult } : {}
|
|
8773
|
+
};
|
|
8774
|
+
}
|
|
8775
|
+
}
|
|
8776
|
+
const ok = checks.length > 0 && checks.every((item) => item.ok);
|
|
8777
|
+
return {
|
|
8778
|
+
ok,
|
|
8779
|
+
exitCode: resolveExitCode({ ok, configError: false, readError: false }),
|
|
8780
|
+
traceDir,
|
|
8781
|
+
suitePath: options.suitePath,
|
|
8782
|
+
runCount: runs.length,
|
|
8783
|
+
checks,
|
|
8784
|
+
diagnostics,
|
|
8785
|
+
...suiteResult !== void 0 ? { suiteResult } : {}
|
|
8786
|
+
};
|
|
8787
|
+
}
|
|
8788
|
+
|
|
8789
|
+
// packages/core/src/gate/render.ts
|
|
8790
|
+
function escapeXml(value) {
|
|
8791
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
8792
|
+
}
|
|
8793
|
+
function renderGateSummaryMarkdown(result) {
|
|
8794
|
+
const lines = [];
|
|
8795
|
+
lines.push("# AgentInspect gate");
|
|
8796
|
+
lines.push("");
|
|
8797
|
+
lines.push(`Status: **${result.ok ? "PASS" : "FAIL"}** (exit ${result.exitCode})`);
|
|
8798
|
+
if (result.traceDir !== void 0) {
|
|
8799
|
+
lines.push(`Trace directory: \`${result.traceDir}\``);
|
|
8800
|
+
}
|
|
8801
|
+
if (result.suitePath !== void 0) {
|
|
8802
|
+
lines.push(`Suite config: \`${result.suitePath}\``);
|
|
8803
|
+
}
|
|
8804
|
+
lines.push(`Runs evaluated: ${result.runCount}`);
|
|
8805
|
+
lines.push("");
|
|
8806
|
+
if (result.diagnostics.length > 0) {
|
|
8807
|
+
lines.push("## Diagnostics");
|
|
8808
|
+
for (const item of result.diagnostics) lines.push(`- ${item}`);
|
|
8809
|
+
lines.push("");
|
|
8810
|
+
}
|
|
8811
|
+
lines.push("## Checks");
|
|
8812
|
+
for (const check of result.checks) {
|
|
8813
|
+
const flag = check.ok ? "PASS" : "FAIL";
|
|
8814
|
+
lines.push(`- [${flag}] ${check.name}: ${check.message}`);
|
|
8815
|
+
}
|
|
8816
|
+
lines.push("");
|
|
8817
|
+
return lines.join("\n").trimEnd();
|
|
8818
|
+
}
|
|
8819
|
+
function renderGateGithubStepSummary(result) {
|
|
8820
|
+
const lines = [];
|
|
8821
|
+
lines.push(`## AgentInspect gate: ${result.ok ? "PASS" : "FAIL"}`);
|
|
8822
|
+
lines.push("");
|
|
8823
|
+
lines.push("| Check | Status | Details |");
|
|
8824
|
+
lines.push("| --- | --- | --- |");
|
|
8825
|
+
for (const check of result.checks) {
|
|
8826
|
+
lines.push(
|
|
8827
|
+
`| ${check.name} | ${check.ok ? "pass" : "fail"} | ${check.message.replace(/\|/g, "/")} |`
|
|
8828
|
+
);
|
|
8829
|
+
}
|
|
8830
|
+
if (result.diagnostics.length > 0) {
|
|
8831
|
+
lines.push("");
|
|
8832
|
+
lines.push("**Diagnostics**");
|
|
8833
|
+
for (const item of result.diagnostics) lines.push(`- ${item}`);
|
|
8834
|
+
}
|
|
8835
|
+
return lines.join("\n").trimEnd();
|
|
8836
|
+
}
|
|
8837
|
+
function renderGateReportHtml(result) {
|
|
8838
|
+
const rows = result.checks.map(
|
|
8839
|
+
(check) => `<tr><td>${escapeHtml(check.name)}</td><td>${check.ok ? "PASS" : "FAIL"}</td><td>${escapeHtml(check.message)}</td></tr>`
|
|
8840
|
+
).join("");
|
|
8841
|
+
return `<!DOCTYPE html>
|
|
8842
|
+
<html lang="en">
|
|
8843
|
+
<head>
|
|
8844
|
+
<meta charset="utf-8" />
|
|
8845
|
+
<title>Gate report</title>
|
|
8846
|
+
<style>
|
|
8847
|
+
body { font-family: system-ui, sans-serif; margin: 2rem; }
|
|
8848
|
+
table { border-collapse: collapse; width: 100%; }
|
|
8849
|
+
th, td { border: 1px solid #ddd; padding: 0.5rem; text-align: left; }
|
|
8850
|
+
th { background: #f6f6f6; }
|
|
8851
|
+
</style>
|
|
8852
|
+
</head>
|
|
8853
|
+
<body>
|
|
8854
|
+
<h1>AgentInspect gate</h1>
|
|
8855
|
+
<p>Status: <strong>${result.ok ? "PASS" : "FAIL"}</strong> (exit ${result.exitCode})</p>
|
|
8856
|
+
<h2>Checks</h2>
|
|
8857
|
+
<table>
|
|
8858
|
+
<thead><tr><th>Check</th><th>Status</th><th>Details</th></tr></thead>
|
|
8859
|
+
<tbody>${rows}</tbody>
|
|
8860
|
+
</table>
|
|
8861
|
+
</body>
|
|
8862
|
+
</html>`;
|
|
8863
|
+
}
|
|
8864
|
+
function renderGateJUnit(result) {
|
|
8865
|
+
const failures = result.checks.filter((check) => !check.ok).length;
|
|
8866
|
+
const tests = result.checks.length;
|
|
8867
|
+
const cases = result.checks.map((check) => {
|
|
8868
|
+
if (check.ok) {
|
|
8869
|
+
return ` <testcase name="${escapeXml(check.name)}" classname="gate" />`;
|
|
8870
|
+
}
|
|
8871
|
+
return ` <testcase name="${escapeXml(check.name)}" classname="gate">
|
|
8872
|
+
<failure message="${escapeXml(check.message)}">${escapeXml(check.message)}</failure>
|
|
8873
|
+
</testcase>`;
|
|
8874
|
+
}).join("\n");
|
|
8875
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
8876
|
+
<testsuites tests="${tests}" failures="${failures}" errors="0" time="0">
|
|
8877
|
+
<testsuite name="agent-inspect-gate" tests="${tests}" failures="${failures}" errors="0" time="0">
|
|
8878
|
+
${cases}
|
|
8879
|
+
</testsuite>
|
|
8880
|
+
</testsuites>`;
|
|
8881
|
+
}
|
|
8882
|
+
function renderGateReport(result, options = {}) {
|
|
8883
|
+
const format = options.format ?? "markdown";
|
|
8884
|
+
if (format === "json") return JSON.stringify(result, null, 2);
|
|
8885
|
+
if (format === "html") return renderGateReportHtml(result);
|
|
8886
|
+
if (format === "junit") return renderGateJUnit(result);
|
|
8887
|
+
if (format === "github") return renderGateGithubStepSummary(result);
|
|
8888
|
+
return renderGateSummaryMarkdown(result);
|
|
8889
|
+
}
|
|
8890
|
+
|
|
8891
|
+
export { COHORT_METRIC_IDS, DEFAULT_SUITE_ARTIFACTS_DIR, Redactor, TraceDirectory, TraceReadError, TreeBuilder, __commonJS, __require, __toESM, aggregateBundleSafeStatus, aggregateSessionCheckResults, analyzeCohort, applyProfileMetadataCaps, buildActivitySummary, buildBundleMetadata, buildBundleSummaryMarkdown, buildLocalExplanation, buildPlaceholderArtifact, buildRunSummary, buildRunTimeline, buildRunWhatSummary, buildSessionIndex, buildTraceStats, bundleFailsOnSafety, compactAttributes3 as compactAttributes, createBaselineRegressionRule, createLlmUsageRule, createMaxStepDurationRule, createObservedOutcomeRule, createRequireCompletedRule, createRunDepthRule, createRunDurationRule, createRunStatusRule, createSafetyOversizedAttributeRule, createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createStallDetectionRule, createStructureCycleRule, createStructureOrphanRule, createStructureParallelWidthRule, createStructureRelationshipRule, createToolUsageRule, defaultBundleOutputPath, defaultSuiteConfigTemplate, enrichSessionRunRecord, escapeHtml, escapeMarkdown, extractMetadata, extractOutcomesFromTraceEvents, filterMetasBySessionScope, filterTraces, flattenTree, formatDuration2 as formatDuration, formatTimestamp, gateHasThresholds, getIndent, getTraceFilePath, isAgentInspectTrace, isPersistedInspectEvent, loadSessionRunRecords, loadSuiteConfig, loadTraceMetadataList, nanoid, normalizeBundleOutputPath, openTrace, parseCohortMetricList, parseDuration, parseDurationFilter, parseGateList, parseTraceJsonl, persistedInspectEventsToTraceEvents, renderActivitySummaryHuman, renderCohortReport, renderErrorLine, renderGateReport, renderObservedOutcomesHtml, renderObservedOutcomesMarkdown, renderRunWhat, renderStepLine, renderSuiteReport, renderTimeline, renderTraceStats, resolveBundleRunIds, resolveRedactionProfile, resolveTraceDir, runGate, runSuite, runTraceChecks, safeString, searchTraces, source_default, stableJson, summarizeObservedOutcomes, traceEventToPersistedInspectEvent, truncateName, truncateStringForProfile, validateEvent, validateSuiteConfig, zeroKinds };
|
|
8892
|
+
//# sourceMappingURL=chunk-SJ5R2XBE.mjs.map
|
|
8893
|
+
//# sourceMappingURL=chunk-SJ5R2XBE.mjs.map
|