agent-inspect 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +33 -2
- package/SECURITY.md +18 -1
- package/docs/ADAPTERS.md +41 -0
- package/docs/CLI.md +22 -2
- package/docs/LIMITATIONS.md +6 -0
- package/package.json +2 -2
- package/packages/cli/dist/index.cjs +1074 -177
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +1074 -177
- package/packages/cli/dist/index.mjs.map +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
4
|
var fs = require('fs');
|
|
5
|
-
var
|
|
5
|
+
var path14 = require('path');
|
|
6
6
|
var url = require('url');
|
|
7
7
|
var commander = require('commander');
|
|
8
8
|
var async_hooks = require('async_hooks');
|
|
@@ -12,18 +12,19 @@ var os = require('os');
|
|
|
12
12
|
var process2 = require('process');
|
|
13
13
|
var tty = require('tty');
|
|
14
14
|
var readline = require('readline');
|
|
15
|
+
var http = require('http');
|
|
15
16
|
|
|
16
17
|
var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
|
|
17
18
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
18
19
|
|
|
19
|
-
var
|
|
20
|
+
var path14__default = /*#__PURE__*/_interopDefault(path14);
|
|
20
21
|
var crypto__default = /*#__PURE__*/_interopDefault(crypto);
|
|
21
22
|
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
22
23
|
var process2__default = /*#__PURE__*/_interopDefault(process2);
|
|
23
24
|
var tty__default = /*#__PURE__*/_interopDefault(tty);
|
|
24
25
|
|
|
25
26
|
// package.json
|
|
26
|
-
var version = "2.
|
|
27
|
+
var version = "2.6.0";
|
|
27
28
|
|
|
28
29
|
// packages/core/src/correlation-metadata.ts
|
|
29
30
|
var TRACE_CORRELATION_KEYS = [
|
|
@@ -755,7 +756,7 @@ function formatDuration(ms) {
|
|
|
755
756
|
// packages/core/src/utils.ts
|
|
756
757
|
var DEFAULT_TRACE_DIR_NAME = ".agent-inspect";
|
|
757
758
|
var RUNS_DIR_NAME = "runs";
|
|
758
|
-
var FALLBACK_TRACE_DIR =
|
|
759
|
+
var FALLBACK_TRACE_DIR = path14__default.default.join(
|
|
759
760
|
os__default.default.tmpdir(),
|
|
760
761
|
"agent-inspect",
|
|
761
762
|
RUNS_DIR_NAME
|
|
@@ -790,7 +791,7 @@ function getDefaultTraceDir() {
|
|
|
790
791
|
if (typeof home !== "string" || home.trim() === "") {
|
|
791
792
|
return FALLBACK_TRACE_DIR;
|
|
792
793
|
}
|
|
793
|
-
return
|
|
794
|
+
return path14__default.default.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
|
|
794
795
|
} catch {
|
|
795
796
|
return FALLBACK_TRACE_DIR;
|
|
796
797
|
}
|
|
@@ -798,11 +799,11 @@ function getDefaultTraceDir() {
|
|
|
798
799
|
function getTraceFilePath(runId, traceDir) {
|
|
799
800
|
const baseDir = traceDir ?? getDefaultTraceDir();
|
|
800
801
|
let safeId = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
801
|
-
safeId =
|
|
802
|
+
safeId = path14__default.default.basename(safeId);
|
|
802
803
|
if (safeId === "" || safeId === "." || safeId === "..") {
|
|
803
804
|
safeId = "run_unknown";
|
|
804
805
|
}
|
|
805
|
-
return
|
|
806
|
+
return path14__default.default.join(baseDir, `${safeId}.jsonl`);
|
|
806
807
|
}
|
|
807
808
|
function formatError(error) {
|
|
808
809
|
if (error instanceof Error) {
|
|
@@ -1440,18 +1441,18 @@ var proto = Object.defineProperties(() => {
|
|
|
1440
1441
|
}
|
|
1441
1442
|
}
|
|
1442
1443
|
});
|
|
1443
|
-
var createStyler = (
|
|
1444
|
+
var createStyler = (open3, close, parent) => {
|
|
1444
1445
|
let openAll;
|
|
1445
1446
|
let closeAll;
|
|
1446
1447
|
if (parent === void 0) {
|
|
1447
|
-
openAll =
|
|
1448
|
+
openAll = open3;
|
|
1448
1449
|
closeAll = close;
|
|
1449
1450
|
} else {
|
|
1450
|
-
openAll = parent.openAll +
|
|
1451
|
+
openAll = parent.openAll + open3;
|
|
1451
1452
|
closeAll = close + parent.closeAll;
|
|
1452
1453
|
}
|
|
1453
1454
|
return {
|
|
1454
|
-
open:
|
|
1455
|
+
open: open3,
|
|
1455
1456
|
close,
|
|
1456
1457
|
openAll,
|
|
1457
1458
|
closeAll,
|
|
@@ -1516,15 +1517,15 @@ function getStatusIcon(status) {
|
|
|
1516
1517
|
if (status === "error") return source_default.red("\u2716");
|
|
1517
1518
|
return source_default.yellow("\u23F3");
|
|
1518
1519
|
}
|
|
1519
|
-
function renderStepLine(name,
|
|
1520
|
+
function renderStepLine(name, durationMs2, status, depth) {
|
|
1520
1521
|
try {
|
|
1521
1522
|
const nm = formatTerminalName(name);
|
|
1522
1523
|
const ind = getIndent(depth ?? 0);
|
|
1523
|
-
if (status === "running" &&
|
|
1524
|
+
if (status === "running" && durationMs2 === void 0) {
|
|
1524
1525
|
return `${ind}${source_default.yellow("\u23F3")} ${nm}`;
|
|
1525
1526
|
}
|
|
1526
|
-
const hasDur =
|
|
1527
|
-
const dur = hasDur ? formatDuration2(
|
|
1527
|
+
const hasDur = durationMs2 !== void 0 && Number.isFinite(durationMs2);
|
|
1528
|
+
const dur = hasDur ? formatDuration2(durationMs2) : void 0;
|
|
1528
1529
|
if (status === "running") {
|
|
1529
1530
|
return dur !== void 0 ? `${ind}${source_default.yellow("\u23F3")} ${nm} (${dur})` : `${ind}${source_default.yellow("\u23F3")} ${nm}`;
|
|
1530
1531
|
}
|
|
@@ -1564,7 +1565,7 @@ var TraceDirectory = class {
|
|
|
1564
1565
|
this.#dir = resolveTraceDir(options);
|
|
1565
1566
|
}
|
|
1566
1567
|
getPath(filename) {
|
|
1567
|
-
return filename ?
|
|
1568
|
+
return filename ? path14__default.default.join(this.#dir, filename) : this.#dir;
|
|
1568
1569
|
}
|
|
1569
1570
|
async list() {
|
|
1570
1571
|
try {
|
|
@@ -1591,7 +1592,7 @@ function parseIsoToMs2(value) {
|
|
|
1591
1592
|
}
|
|
1592
1593
|
async function extractMetadata(filePath, _quickScan) {
|
|
1593
1594
|
const stats = await promises.stat(filePath);
|
|
1594
|
-
let runIdFromFile =
|
|
1595
|
+
let runIdFromFile = path14__default.default.basename(filePath);
|
|
1595
1596
|
if (runIdFromFile.endsWith(".jsonl")) {
|
|
1596
1597
|
runIdFromFile = runIdFromFile.slice(0, -".jsonl".length);
|
|
1597
1598
|
}
|
|
@@ -1677,14 +1678,14 @@ async function extractMetadata(filePath, _quickScan) {
|
|
|
1677
1678
|
} else {
|
|
1678
1679
|
status = "unknown";
|
|
1679
1680
|
}
|
|
1680
|
-
const
|
|
1681
|
+
const durationMs2 = explicitDurationMs ?? (startedAt !== void 0 && endedAt !== void 0 && Number.isFinite(startedAt) && Number.isFinite(endedAt) && endedAt >= startedAt ? endedAt - startedAt : void 0);
|
|
1681
1682
|
return {
|
|
1682
1683
|
runId: resolvedRunId,
|
|
1683
1684
|
name,
|
|
1684
1685
|
status,
|
|
1685
1686
|
startedAt,
|
|
1686
1687
|
endedAt,
|
|
1687
|
-
durationMs,
|
|
1688
|
+
durationMs: durationMs2,
|
|
1688
1689
|
eventCount: parsedTrace.sourceEventCount,
|
|
1689
1690
|
filePath,
|
|
1690
1691
|
fileSize: stats.size,
|
|
@@ -1705,7 +1706,7 @@ function buildRunSummary(events) {
|
|
|
1705
1706
|
const runId = started?.runId ?? events.find((e) => typeof e.runId === "string")?.runId ?? "unknown-run";
|
|
1706
1707
|
const name = typeof started?.name === "string" && started.name.trim() !== "" ? started.name : void 0;
|
|
1707
1708
|
const status = lastCompleted ? lastCompleted.status : started ? "running" : "unknown";
|
|
1708
|
-
const
|
|
1709
|
+
const durationMs2 = lastCompleted && isFiniteNumber(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
1709
1710
|
started && isFiniteNumber(started.startTime) ? started.startTime : void 0;
|
|
1710
1711
|
const steps = /* @__PURE__ */ new Map();
|
|
1711
1712
|
for (const e of events) {
|
|
@@ -1795,7 +1796,7 @@ function buildRunSummary(events) {
|
|
|
1795
1796
|
runId,
|
|
1796
1797
|
name,
|
|
1797
1798
|
status,
|
|
1798
|
-
durationMs,
|
|
1799
|
+
durationMs: durationMs2,
|
|
1799
1800
|
totalSteps,
|
|
1800
1801
|
llmSteps,
|
|
1801
1802
|
toolSteps,
|
|
@@ -3069,12 +3070,12 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
3069
3070
|
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
3070
3071
|
);
|
|
3071
3072
|
const ordered = [...runs].sort(compareRuns);
|
|
3072
|
-
const
|
|
3073
|
+
const path16 = [];
|
|
3073
3074
|
const visited = /* @__PURE__ */ new Set();
|
|
3074
3075
|
const pushRun = (run, confidence, source) => {
|
|
3075
3076
|
if (visited.has(run.runId)) return;
|
|
3076
3077
|
visited.add(run.runId);
|
|
3077
|
-
|
|
3078
|
+
path16.push({
|
|
3078
3079
|
runId: run.runId,
|
|
3079
3080
|
name: run.name,
|
|
3080
3081
|
startedAt: run.startedAt,
|
|
@@ -3099,7 +3100,7 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
3099
3100
|
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
3100
3101
|
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
3101
3102
|
}
|
|
3102
|
-
return
|
|
3103
|
+
return path16;
|
|
3103
3104
|
}
|
|
3104
3105
|
function metaRunIdMatches(run, token, runById) {
|
|
3105
3106
|
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
@@ -3226,10 +3227,10 @@ function statusIcon(status) {
|
|
|
3226
3227
|
if (status === "running") return "\u23F3";
|
|
3227
3228
|
return "?";
|
|
3228
3229
|
}
|
|
3229
|
-
function durationCell(status,
|
|
3230
|
+
function durationCell(status, durationMs2) {
|
|
3230
3231
|
if (status === "running" || status === "unknown") return "-";
|
|
3231
|
-
if (
|
|
3232
|
-
return formatDuration2(
|
|
3232
|
+
if (durationMs2 !== void 0 && Number.isFinite(durationMs2)) {
|
|
3233
|
+
return formatDuration2(durationMs2);
|
|
3233
3234
|
}
|
|
3234
3235
|
return "-";
|
|
3235
3236
|
}
|
|
@@ -3886,7 +3887,7 @@ var TreeBuilder = class {
|
|
|
3886
3887
|
const startedAt = sorted.length > 0 ? sorted[0].timestamp : void 0;
|
|
3887
3888
|
const endedAt = sorted.length > 0 ? sorted[sorted.length - 1].timestamp : void 0;
|
|
3888
3889
|
const status = computeRunStatus(sorted);
|
|
3889
|
-
const
|
|
3890
|
+
const durationMs2 = startedAt !== void 0 && endedAt !== void 0 && Number.isFinite(startedAt) && Number.isFinite(endedAt) && endedAt >= startedAt && status !== "running" ? endedAt - startedAt : void 0;
|
|
3890
3891
|
const name = sorted.find((e) => e.kind === "RUN")?.name;
|
|
3891
3892
|
out.push({
|
|
3892
3893
|
runId,
|
|
@@ -3894,7 +3895,7 @@ var TreeBuilder = class {
|
|
|
3894
3895
|
status,
|
|
3895
3896
|
startedAt,
|
|
3896
3897
|
endedAt: status === "running" ? void 0 : endedAt,
|
|
3897
|
-
durationMs,
|
|
3898
|
+
durationMs: durationMs2,
|
|
3898
3899
|
children: roots,
|
|
3899
3900
|
metadata: {
|
|
3900
3901
|
totalEvents: sorted.length,
|
|
@@ -4037,7 +4038,7 @@ function findReaderByFormat(format, readers) {
|
|
|
4037
4038
|
}
|
|
4038
4039
|
async function jsonlFilesInDirectory(dirPath) {
|
|
4039
4040
|
const entries = await promises.readdir(dirPath, { withFileTypes: true });
|
|
4040
|
-
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) =>
|
|
4041
|
+
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path14__default.default.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
|
|
4041
4042
|
}
|
|
4042
4043
|
async function resolveInput(input3) {
|
|
4043
4044
|
const cached = resolvedInputCache.get(input3);
|
|
@@ -4589,9 +4590,9 @@ function mapOpenInferenceSpan(span, index, version2) {
|
|
|
4589
4590
|
if (endedAt !== void 0) {
|
|
4590
4591
|
event.endedAt = endedAt;
|
|
4591
4592
|
}
|
|
4592
|
-
const
|
|
4593
|
-
if (
|
|
4594
|
-
event.durationMs =
|
|
4593
|
+
const durationMs2 = durationBetweenIso(startedAt, endedAt);
|
|
4594
|
+
if (durationMs2 !== void 0) {
|
|
4595
|
+
event.durationMs = durationMs2;
|
|
4595
4596
|
}
|
|
4596
4597
|
if (tokenUsage !== void 0) {
|
|
4597
4598
|
event.tokenUsage = tokenUsage;
|
|
@@ -5176,9 +5177,9 @@ function mapOtlpSpan(context) {
|
|
|
5176
5177
|
if (endedAt !== void 0) {
|
|
5177
5178
|
event.endedAt = endedAt;
|
|
5178
5179
|
}
|
|
5179
|
-
const
|
|
5180
|
-
if (
|
|
5181
|
-
event.durationMs =
|
|
5180
|
+
const durationMs2 = durationBetweenIso(startedAt, endedAt);
|
|
5181
|
+
if (durationMs2 !== void 0) {
|
|
5182
|
+
event.durationMs = durationMs2;
|
|
5182
5183
|
}
|
|
5183
5184
|
if (tokenUsage !== void 0) {
|
|
5184
5185
|
event.tokenUsage = tokenUsage;
|
|
@@ -6183,13 +6184,13 @@ var EventNormalizer = class {
|
|
|
6183
6184
|
const timestampMissing = parsedTs === void 0;
|
|
6184
6185
|
const parentIdKey = cfg.parentIdKey;
|
|
6185
6186
|
const parentId = parentIdKey ? safeString(raw[parentIdKey]) : void 0;
|
|
6186
|
-
let
|
|
6187
|
+
let durationMs2;
|
|
6187
6188
|
const durationKey = cfg.durationKey;
|
|
6188
6189
|
if (durationKey) {
|
|
6189
6190
|
const v = raw[durationKey];
|
|
6190
|
-
if (isFiniteNumber2(v))
|
|
6191
|
+
if (isFiniteNumber2(v)) durationMs2 = v;
|
|
6191
6192
|
} else if (isFiniteNumber2(raw.durationMs)) {
|
|
6192
|
-
|
|
6193
|
+
durationMs2 = raw.durationMs;
|
|
6193
6194
|
}
|
|
6194
6195
|
let status;
|
|
6195
6196
|
const statusKey = cfg.statusKey;
|
|
@@ -6236,7 +6237,7 @@ var EventNormalizer = class {
|
|
|
6236
6237
|
kind,
|
|
6237
6238
|
timestamp,
|
|
6238
6239
|
...status ? { status } : {},
|
|
6239
|
-
...
|
|
6240
|
+
...durationMs2 !== void 0 ? { durationMs: durationMs2 } : {},
|
|
6240
6241
|
...Object.keys(attributes).length > 0 ? { attributes } : {},
|
|
6241
6242
|
confidence,
|
|
6242
6243
|
source: {
|
|
@@ -7824,7 +7825,7 @@ function manualTraceEventsToRunTree(events) {
|
|
|
7824
7825
|
}
|
|
7825
7826
|
const startedAt = started.startTime;
|
|
7826
7827
|
const endedAt = lastCompleted !== void 0 && runStatus !== "running" ? lastCompleted.endTime : void 0;
|
|
7827
|
-
const
|
|
7828
|
+
const durationMs2 = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
7828
7829
|
const steps = /* @__PURE__ */ new Map();
|
|
7829
7830
|
for (const e of events) {
|
|
7830
7831
|
if (e.event !== "step_started") continue;
|
|
@@ -7913,7 +7914,7 @@ function manualTraceEventsToRunTree(events) {
|
|
|
7913
7914
|
status: runStatus,
|
|
7914
7915
|
startedAt,
|
|
7915
7916
|
endedAt,
|
|
7916
|
-
durationMs,
|
|
7917
|
+
durationMs: durationMs2,
|
|
7917
7918
|
children: roots,
|
|
7918
7919
|
metadata: {
|
|
7919
7920
|
totalEvents: inspectNodes.size,
|
|
@@ -8163,9 +8164,9 @@ Trace directory: ${traceDir}`);
|
|
|
8163
8164
|
if (validation !== void 0 && !validation.ok) {
|
|
8164
8165
|
process.exitCode = 1;
|
|
8165
8166
|
}
|
|
8166
|
-
const outPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
8167
|
+
const outPath = options.output !== void 0 && options.output.trim() !== "" ? path14__default.default.resolve(options.output.trim()) : void 0;
|
|
8167
8168
|
if (outPath !== void 0) {
|
|
8168
|
-
await promises.mkdir(
|
|
8169
|
+
await promises.mkdir(path14__default.default.dirname(outPath), { recursive: true });
|
|
8169
8170
|
await promises.writeFile(outPath, result.content, "utf-8");
|
|
8170
8171
|
const vlabel = validation !== void 0 ? validation.ok ? "ok" : "failed" : "skipped";
|
|
8171
8172
|
console.log(`Wrote ${result.fileExtension} export to ${outPath} (validation: ${vlabel})`);
|
|
@@ -8226,7 +8227,7 @@ function manualTraceEventsToComparableRun(events) {
|
|
|
8226
8227
|
let runStatus;
|
|
8227
8228
|
if (lastCompleted === void 0) runStatus = "running";
|
|
8228
8229
|
else runStatus = lastCompleted.status;
|
|
8229
|
-
const
|
|
8230
|
+
const durationMs2 = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
8230
8231
|
const steps = /* @__PURE__ */ new Map();
|
|
8231
8232
|
let order = 0;
|
|
8232
8233
|
for (const e of events) {
|
|
@@ -8297,7 +8298,7 @@ function manualTraceEventsToComparableRun(events) {
|
|
|
8297
8298
|
runId,
|
|
8298
8299
|
name: rs.name,
|
|
8299
8300
|
status: runStatus,
|
|
8300
|
-
durationMs,
|
|
8301
|
+
durationMs: durationMs2,
|
|
8301
8302
|
steps: roots
|
|
8302
8303
|
};
|
|
8303
8304
|
}
|
|
@@ -8342,13 +8343,13 @@ function pairSteps(left, right) {
|
|
|
8342
8343
|
return pairs;
|
|
8343
8344
|
}
|
|
8344
8345
|
function compareLeafSteps(L, R, segments, opts, out) {
|
|
8345
|
-
const
|
|
8346
|
+
const path16 = buildPath(segments);
|
|
8346
8347
|
if (L.name !== R.name) {
|
|
8347
8348
|
out.push({
|
|
8348
8349
|
kind: "structure",
|
|
8349
8350
|
severity: "warning",
|
|
8350
8351
|
message: "Step name differs",
|
|
8351
|
-
path:
|
|
8352
|
+
path: path16,
|
|
8352
8353
|
left: L.name,
|
|
8353
8354
|
right: R.name
|
|
8354
8355
|
});
|
|
@@ -8358,7 +8359,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8358
8359
|
kind: "step-type",
|
|
8359
8360
|
severity: "warning",
|
|
8360
8361
|
message: "Step type differs",
|
|
8361
|
-
path:
|
|
8362
|
+
path: path16,
|
|
8362
8363
|
left: L.type,
|
|
8363
8364
|
right: R.type
|
|
8364
8365
|
});
|
|
@@ -8368,7 +8369,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8368
8369
|
kind: "step-status",
|
|
8369
8370
|
severity: "warning",
|
|
8370
8371
|
message: "Step status differs",
|
|
8371
|
-
path:
|
|
8372
|
+
path: path16,
|
|
8372
8373
|
left: L.status,
|
|
8373
8374
|
right: R.status
|
|
8374
8375
|
});
|
|
@@ -8380,7 +8381,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8380
8381
|
kind: "error",
|
|
8381
8382
|
severity: "error",
|
|
8382
8383
|
message: "Step error message differs",
|
|
8383
|
-
path:
|
|
8384
|
+
path: path16,
|
|
8384
8385
|
left: le || void 0,
|
|
8385
8386
|
right: re || void 0
|
|
8386
8387
|
});
|
|
@@ -8398,7 +8399,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8398
8399
|
kind: "duration",
|
|
8399
8400
|
severity: "info",
|
|
8400
8401
|
message: "Step duration differs",
|
|
8401
|
-
path:
|
|
8402
|
+
path: path16,
|
|
8402
8403
|
left: ld,
|
|
8403
8404
|
right: rd
|
|
8404
8405
|
});
|
|
@@ -8411,7 +8412,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8411
8412
|
kind: "metadata",
|
|
8412
8413
|
severity: "info",
|
|
8413
8414
|
message: "Step metadata differs",
|
|
8414
|
-
path:
|
|
8415
|
+
path: path16,
|
|
8415
8416
|
left: L.metadata,
|
|
8416
8417
|
right: R.metadata
|
|
8417
8418
|
});
|
|
@@ -8423,7 +8424,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8423
8424
|
kind: "output",
|
|
8424
8425
|
severity: "info",
|
|
8425
8426
|
message: "Output preview differs",
|
|
8426
|
-
path:
|
|
8427
|
+
path: path16,
|
|
8427
8428
|
left: L.outputPreview,
|
|
8428
8429
|
right: R.outputPreview
|
|
8429
8430
|
});
|
|
@@ -8583,11 +8584,11 @@ function diffRuns(left, right, options) {
|
|
|
8583
8584
|
}
|
|
8584
8585
|
|
|
8585
8586
|
// packages/core/src/diff/renderer.ts
|
|
8586
|
-
function formatPath(
|
|
8587
|
-
if (
|
|
8587
|
+
function formatPath(path16) {
|
|
8588
|
+
if (path16 === void 0 || path16.path.length === 0) {
|
|
8588
8589
|
return "(run)";
|
|
8589
8590
|
}
|
|
8590
|
-
return
|
|
8591
|
+
return path16.path.map((s) => s.name).join(" > ");
|
|
8591
8592
|
}
|
|
8592
8593
|
function formatValue(v, verbose) {
|
|
8593
8594
|
if (v === void 0) return "(undefined)";
|
|
@@ -9216,9 +9217,9 @@ async function reportCommand(runId, options = {}) {
|
|
|
9216
9217
|
redactionProfile,
|
|
9217
9218
|
correlation: !options.noCorrelation
|
|
9218
9219
|
});
|
|
9219
|
-
const outPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
9220
|
+
const outPath = options.output !== void 0 && options.output.trim() !== "" ? path14__default.default.resolve(options.output.trim()) : void 0;
|
|
9220
9221
|
if (outPath !== void 0) {
|
|
9221
|
-
await promises.mkdir(
|
|
9222
|
+
await promises.mkdir(path14__default.default.dirname(outPath), { recursive: true });
|
|
9222
9223
|
await promises.writeFile(outPath, result.content, "utf-8");
|
|
9223
9224
|
console.log(`Wrote ${result.fileExtension} report to ${outPath}`);
|
|
9224
9225
|
}
|
|
@@ -9465,17 +9466,17 @@ function applyRule(rule, value, replacement) {
|
|
|
9465
9466
|
}
|
|
9466
9467
|
return value;
|
|
9467
9468
|
}
|
|
9468
|
-
function childPath(
|
|
9469
|
+
function childPath(path16, key) {
|
|
9469
9470
|
if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)) {
|
|
9470
|
-
return
|
|
9471
|
+
return path16 ? `${path16}.${key}` : key;
|
|
9471
9472
|
}
|
|
9472
|
-
return `${
|
|
9473
|
+
return `${path16 || "$"}[${JSON.stringify(key)}]`;
|
|
9473
9474
|
}
|
|
9474
|
-
function indexPath(
|
|
9475
|
-
return `${
|
|
9475
|
+
function indexPath(path16, index) {
|
|
9476
|
+
return `${path16 || "$"}[${index}]`;
|
|
9476
9477
|
}
|
|
9477
|
-
function makeFinding(
|
|
9478
|
-
return preview === void 0 ? { path:
|
|
9478
|
+
function makeFinding(path16, detector, action, matchKind, severity = "warning", preview) {
|
|
9479
|
+
return preview === void 0 ? { path: path16, detector, action, severity, matchKind } : { path: path16, detector, action, severity, matchKind, preview };
|
|
9479
9480
|
}
|
|
9480
9481
|
function createRedactionProfile(profile = "local") {
|
|
9481
9482
|
switch (profile) {
|
|
@@ -9544,11 +9545,11 @@ var Redactor2 = class {
|
|
|
9544
9545
|
#recordFinding(state, finding) {
|
|
9545
9546
|
if (this.#collectFindings) state.findings.push(finding);
|
|
9546
9547
|
}
|
|
9547
|
-
#redactValue(value, key,
|
|
9548
|
+
#redactValue(value, key, path16, depth, state) {
|
|
9548
9549
|
if (depth > this.#maxDepth) {
|
|
9549
9550
|
this.#recordFinding(
|
|
9550
9551
|
state,
|
|
9551
|
-
makeFinding(
|
|
9552
|
+
makeFinding(path16, "structure.maxDepth", "truncate", "value", "warning")
|
|
9552
9553
|
);
|
|
9553
9554
|
return "[Truncated]";
|
|
9554
9555
|
}
|
|
@@ -9557,19 +9558,19 @@ var Redactor2 = class {
|
|
|
9557
9558
|
if (rule) {
|
|
9558
9559
|
this.#recordFinding(
|
|
9559
9560
|
state,
|
|
9560
|
-
makeFinding(
|
|
9561
|
+
makeFinding(path16, `key.${rule.key}`, actionForRule(rule), "key", "warning")
|
|
9561
9562
|
);
|
|
9562
9563
|
return applyRule(rule, value, this.#replacement);
|
|
9563
9564
|
}
|
|
9564
9565
|
}
|
|
9565
9566
|
for (const detector of this.#detectors) {
|
|
9566
|
-
const detections = detector.detect({ path:
|
|
9567
|
+
const detections = detector.detect({ path: path16, key, value });
|
|
9567
9568
|
for (const detection of detections) {
|
|
9568
9569
|
const action = detection.action ?? "replace";
|
|
9569
9570
|
this.#recordFinding(
|
|
9570
9571
|
state,
|
|
9571
9572
|
makeFinding(
|
|
9572
|
-
|
|
9573
|
+
path16,
|
|
9573
9574
|
detector.id,
|
|
9574
9575
|
action,
|
|
9575
9576
|
detection.matchKind ?? detector.matchKind ?? "custom",
|
|
@@ -9587,7 +9588,7 @@ var Redactor2 = class {
|
|
|
9587
9588
|
const out = [];
|
|
9588
9589
|
state.seen.set(value, out);
|
|
9589
9590
|
value.forEach((item, index) => {
|
|
9590
|
-
out[index] = this.#redactValue(item, void 0, indexPath(
|
|
9591
|
+
out[index] = this.#redactValue(item, void 0, indexPath(path16, index), depth + 1, state);
|
|
9591
9592
|
});
|
|
9592
9593
|
return out;
|
|
9593
9594
|
}
|
|
@@ -9599,7 +9600,7 @@ var Redactor2 = class {
|
|
|
9599
9600
|
out[entryKey] = this.#redactValue(
|
|
9600
9601
|
entryValue,
|
|
9601
9602
|
entryKey,
|
|
9602
|
-
childPath(
|
|
9603
|
+
childPath(path16 === "$" ? "" : path16, entryKey),
|
|
9603
9604
|
depth + 1,
|
|
9604
9605
|
state
|
|
9605
9606
|
);
|
|
@@ -10046,14 +10047,14 @@ function uniqueSorted(values) {
|
|
|
10046
10047
|
return [...new Set(values)].sort();
|
|
10047
10048
|
}
|
|
10048
10049
|
function isWithinDirectory(child, parent) {
|
|
10049
|
-
const relative =
|
|
10050
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
10050
|
+
const relative = path14__default.default.relative(parent, child);
|
|
10051
|
+
return relative === "" || !relative.startsWith("..") && !path14__default.default.isAbsolute(relative);
|
|
10051
10052
|
}
|
|
10052
10053
|
async function resolveOutputPath(inputPath, output2, force) {
|
|
10053
10054
|
if (output2 === void 0 || output2.trim() === "") return void 0;
|
|
10054
|
-
const inputAbs =
|
|
10055
|
-
const outputAbs =
|
|
10056
|
-
const inputDir =
|
|
10055
|
+
const inputAbs = path14__default.default.resolve(inputPath);
|
|
10056
|
+
const outputAbs = path14__default.default.resolve(output2.trim());
|
|
10057
|
+
const inputDir = path14__default.default.dirname(inputAbs);
|
|
10057
10058
|
if (!isWithinDirectory(outputAbs, inputDir)) {
|
|
10058
10059
|
throw new Error("Refusing to write migrated output outside the input directory.");
|
|
10059
10060
|
}
|
|
@@ -10174,7 +10175,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
10174
10175
|
process.exitCode = 1;
|
|
10175
10176
|
return;
|
|
10176
10177
|
}
|
|
10177
|
-
const inputPath =
|
|
10178
|
+
const inputPath = path14__default.default.resolve(input3.trim());
|
|
10178
10179
|
const dryRun = options.dryRun === true;
|
|
10179
10180
|
if (!dryRun && (options.output === void 0 || options.output.trim() === "")) {
|
|
10180
10181
|
console.error("migrate requires --dry-run or --output <path>.");
|
|
@@ -10193,7 +10194,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
10193
10194
|
);
|
|
10194
10195
|
const result = await buildMigration(inputPath, outputPath);
|
|
10195
10196
|
if (!dryRun && outputPath !== void 0) {
|
|
10196
|
-
await promises.mkdir(
|
|
10197
|
+
await promises.mkdir(path14__default.default.dirname(outputPath), { recursive: true });
|
|
10197
10198
|
await promises.writeFile(outputPath, result.content, "utf-8");
|
|
10198
10199
|
}
|
|
10199
10200
|
printSummary2(result, dryRun);
|
|
@@ -10471,7 +10472,7 @@ function stripPrefix(name, prefixes) {
|
|
|
10471
10472
|
}
|
|
10472
10473
|
return name;
|
|
10473
10474
|
}
|
|
10474
|
-
function eventEvidence(event,
|
|
10475
|
+
function eventEvidence(event, path16) {
|
|
10475
10476
|
return {
|
|
10476
10477
|
runId: event.runId,
|
|
10477
10478
|
eventId: event.eventId,
|
|
@@ -10481,7 +10482,7 @@ function eventEvidence(event, path15) {
|
|
|
10481
10482
|
kind: event.kind,
|
|
10482
10483
|
name: event.name,
|
|
10483
10484
|
status: event.status,
|
|
10484
|
-
...
|
|
10485
|
+
...path16 ? { path: path16 } : {}
|
|
10485
10486
|
};
|
|
10486
10487
|
}
|
|
10487
10488
|
function runEvidence(run) {
|
|
@@ -10544,9 +10545,9 @@ function eventEndMs(event) {
|
|
|
10544
10545
|
function normalizedKey(value) {
|
|
10545
10546
|
return value.toLowerCase().replace(/[^a-z0-9_]/g, "");
|
|
10546
10547
|
}
|
|
10547
|
-
function lastPathSegment(
|
|
10548
|
-
const parts =
|
|
10549
|
-
return parts[parts.length - 1] ??
|
|
10548
|
+
function lastPathSegment(path16) {
|
|
10549
|
+
const parts = path16.split(".");
|
|
10550
|
+
return parts[parts.length - 1] ?? path16;
|
|
10550
10551
|
}
|
|
10551
10552
|
function valueType(value) {
|
|
10552
10553
|
if (Array.isArray(value)) return "array";
|
|
@@ -10560,12 +10561,12 @@ function serializedByteLength(value) {
|
|
|
10560
10561
|
return void 0;
|
|
10561
10562
|
}
|
|
10562
10563
|
}
|
|
10563
|
-
function pushValueEntries(entries, event, value,
|
|
10564
|
-
entries.push({ event, path:
|
|
10564
|
+
function pushValueEntries(entries, event, value, path16, key, depth = 0) {
|
|
10565
|
+
entries.push({ event, path: path16, key, value });
|
|
10565
10566
|
if (depth >= 8) return;
|
|
10566
10567
|
if (Array.isArray(value)) {
|
|
10567
10568
|
for (const [index, item] of value.entries()) {
|
|
10568
|
-
pushValueEntries(entries, event, item, `${
|
|
10569
|
+
pushValueEntries(entries, event, item, `${path16}.${index}`, String(index), depth + 1);
|
|
10569
10570
|
}
|
|
10570
10571
|
return;
|
|
10571
10572
|
}
|
|
@@ -10575,7 +10576,7 @@ function pushValueEntries(entries, event, value, path15, key, depth = 0) {
|
|
|
10575
10576
|
entries,
|
|
10576
10577
|
event,
|
|
10577
10578
|
value[nestedKey],
|
|
10578
|
-
`${
|
|
10579
|
+
`${path16}.${nestedKey}`,
|
|
10579
10580
|
nestedKey,
|
|
10580
10581
|
depth + 1
|
|
10581
10582
|
);
|
|
@@ -10656,9 +10657,9 @@ function eventDurationMs(event) {
|
|
|
10656
10657
|
}
|
|
10657
10658
|
function treeShape(nodes) {
|
|
10658
10659
|
const lines = [];
|
|
10659
|
-
const visit = (node,
|
|
10660
|
-
lines.push(`${
|
|
10661
|
-
node.children.forEach((child, index) => visit(child, `${
|
|
10660
|
+
const visit = (node, path16) => {
|
|
10661
|
+
lines.push(`${path16}:${node.event.kind}:${node.event.name}:${node.event.status ?? "unknown"}`);
|
|
10662
|
+
node.children.forEach((child, index) => visit(child, `${path16}.${index}`));
|
|
10662
10663
|
};
|
|
10663
10664
|
nodes.forEach((node, index) => visit(node, String(index)));
|
|
10664
10665
|
return lines;
|
|
@@ -10707,9 +10708,9 @@ function retrievalShape(context) {
|
|
|
10707
10708
|
function guardrailShape(context) {
|
|
10708
10709
|
return guardrailEvents(context).map((event) => signalName(event, ["guardrailName", "guardrail", "guardrailId"], ["guardrail:"])).sort((a, b) => a.localeCompare(b));
|
|
10709
10710
|
}
|
|
10710
|
-
function firstEvidenceForKind(context, kind,
|
|
10711
|
+
function firstEvidenceForKind(context, kind, path16) {
|
|
10711
10712
|
const event = context.events.find((candidate) => candidate.kind === kind);
|
|
10712
|
-
return event ? [eventEvidence(event,
|
|
10713
|
+
return event ? [eventEvidence(event, path16)] : runEvidence(context.selectedRun);
|
|
10713
10714
|
}
|
|
10714
10715
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
10715
10716
|
return failFinding("baseline.regression", message, evidence, expected, actual);
|
|
@@ -10957,13 +10958,13 @@ function createStructureCycleRule() {
|
|
|
10957
10958
|
const seenCycles = /* @__PURE__ */ new Set();
|
|
10958
10959
|
const findings = [];
|
|
10959
10960
|
for (const event of [...context.events].sort((a, b) => a.eventId.localeCompare(b.eventId))) {
|
|
10960
|
-
const
|
|
10961
|
+
const path16 = [];
|
|
10961
10962
|
const seenAt = /* @__PURE__ */ new Map();
|
|
10962
10963
|
let current = event;
|
|
10963
10964
|
while (current) {
|
|
10964
10965
|
const existing = seenAt.get(current.eventId);
|
|
10965
10966
|
if (existing !== void 0) {
|
|
10966
|
-
const cycle =
|
|
10967
|
+
const cycle = path16.slice(existing);
|
|
10967
10968
|
const key = cycle.map((item) => item.eventId).sort().join("\0");
|
|
10968
10969
|
if (!seenCycles.has(key)) {
|
|
10969
10970
|
seenCycles.add(key);
|
|
@@ -10979,8 +10980,8 @@ function createStructureCycleRule() {
|
|
|
10979
10980
|
}
|
|
10980
10981
|
break;
|
|
10981
10982
|
}
|
|
10982
|
-
seenAt.set(current.eventId,
|
|
10983
|
-
|
|
10983
|
+
seenAt.set(current.eventId, path16.length);
|
|
10984
|
+
path16.push(current);
|
|
10984
10985
|
current = current.parentId ? byId.get(current.parentId) : void 0;
|
|
10985
10986
|
}
|
|
10986
10987
|
}
|
|
@@ -11191,7 +11192,7 @@ function createSafetyRawContentRule(options = {}) {
|
|
|
11191
11192
|
}
|
|
11192
11193
|
function createSafetySecretPatternRule(options = {}) {
|
|
11193
11194
|
const patterns = options.patterns ?? DEFAULT_SECRET_PATTERNS;
|
|
11194
|
-
const
|
|
11195
|
+
const maxStringLength2 = options.maxStringLength ?? 4096;
|
|
11195
11196
|
return {
|
|
11196
11197
|
id: "safety.secretPattern",
|
|
11197
11198
|
category: "safety",
|
|
@@ -11201,7 +11202,7 @@ function createSafetySecretPatternRule(options = {}) {
|
|
|
11201
11202
|
for (const event of context.events) {
|
|
11202
11203
|
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
11203
11204
|
if (typeof entry.value !== "string") continue;
|
|
11204
|
-
const sample = entry.value.slice(0,
|
|
11205
|
+
const sample = entry.value.slice(0, maxStringLength2);
|
|
11205
11206
|
for (const pattern of patterns) {
|
|
11206
11207
|
pattern.pattern.lastIndex = 0;
|
|
11207
11208
|
if (!pattern.pattern.test(sample)) continue;
|
|
@@ -11457,6 +11458,621 @@ function runTraceChecks(input3, options = {}) {
|
|
|
11457
11458
|
diagnostics
|
|
11458
11459
|
};
|
|
11459
11460
|
}
|
|
11461
|
+
var ALL_RULES = [
|
|
11462
|
+
"circuit.same-tool-repetition",
|
|
11463
|
+
"circuit.same-args-repetition",
|
|
11464
|
+
"circuit.max-loop-iterations",
|
|
11465
|
+
"circuit.max-retries",
|
|
11466
|
+
"circuit.tool-timeout",
|
|
11467
|
+
"circuit.runaway-llm-loop",
|
|
11468
|
+
"circuit.excessive-branch-width"
|
|
11469
|
+
];
|
|
11470
|
+
function closed(ruleId, message) {
|
|
11471
|
+
return { ruleId, status: "closed", severity: "info", message, evidence: [] };
|
|
11472
|
+
}
|
|
11473
|
+
function open2(ruleId, message, evidence, severity = "error") {
|
|
11474
|
+
return {
|
|
11475
|
+
ruleId,
|
|
11476
|
+
status: severity === "warning" ? "warn" : "open",
|
|
11477
|
+
severity,
|
|
11478
|
+
message,
|
|
11479
|
+
evidence
|
|
11480
|
+
};
|
|
11481
|
+
}
|
|
11482
|
+
function isToolEvent(event) {
|
|
11483
|
+
const name = event.name.toLowerCase();
|
|
11484
|
+
return event.kind === "tool" || name.startsWith("tool:") || name.startsWith("function:") || name.includes(".tool.") || name.startsWith("mcp:");
|
|
11485
|
+
}
|
|
11486
|
+
function isLlmEvent(event) {
|
|
11487
|
+
const name = event.name.toLowerCase();
|
|
11488
|
+
return event.kind === "llm" || name.startsWith("llm:") || name.includes(".llm.") || name.includes("generation");
|
|
11489
|
+
}
|
|
11490
|
+
function toolLabel(event) {
|
|
11491
|
+
const attrs = event.attributes ?? {};
|
|
11492
|
+
const fromAttr = attrs.toolName ?? attrs.tool ?? attrs.function;
|
|
11493
|
+
if (typeof fromAttr === "string" && fromAttr.length > 0) return fromAttr;
|
|
11494
|
+
return event.name.replace(/^(tool:|function:|mcp:)/i, "");
|
|
11495
|
+
}
|
|
11496
|
+
function argsHash(toolName2, args) {
|
|
11497
|
+
return crypto.createHash("sha256").update(`${toolName2}:${JSON.stringify(args ?? null)}`).digest("hex").slice(0, 16);
|
|
11498
|
+
}
|
|
11499
|
+
function toolArgs(event) {
|
|
11500
|
+
const attrs = event.attributes ?? {};
|
|
11501
|
+
return attrs.arguments ?? attrs.args ?? attrs.input ?? attrs.parameters;
|
|
11502
|
+
}
|
|
11503
|
+
function durationMs(event) {
|
|
11504
|
+
if (typeof event.durationMs === "number") return event.durationMs;
|
|
11505
|
+
const attrs = event.attributes ?? {};
|
|
11506
|
+
const fromAttr = attrs.durationMs ?? attrs.duration;
|
|
11507
|
+
return typeof fromAttr === "number" ? fromAttr : void 0;
|
|
11508
|
+
}
|
|
11509
|
+
function attemptNumber(event) {
|
|
11510
|
+
const attrs = event.attributes ?? {};
|
|
11511
|
+
const value = attrs.attempt ?? attrs.retryAttempt ?? attrs.retryCount;
|
|
11512
|
+
return typeof value === "number" ? value : void 0;
|
|
11513
|
+
}
|
|
11514
|
+
function evaluateSameToolRepetition(events, maxRepeats) {
|
|
11515
|
+
const ruleId = "circuit.same-tool-repetition";
|
|
11516
|
+
const counts = /* @__PURE__ */ new Map();
|
|
11517
|
+
for (const event of events.filter(isToolEvent)) {
|
|
11518
|
+
const label = toolLabel(event);
|
|
11519
|
+
counts.set(label, (counts.get(label) ?? 0) + 1);
|
|
11520
|
+
}
|
|
11521
|
+
const evidence = [];
|
|
11522
|
+
for (const [toolName2, count] of counts) {
|
|
11523
|
+
if (count > maxRepeats) {
|
|
11524
|
+
evidence.push({ ruleId, toolName: toolName2, count, threshold: maxRepeats });
|
|
11525
|
+
}
|
|
11526
|
+
}
|
|
11527
|
+
if (evidence.length === 0) {
|
|
11528
|
+
return closed(ruleId, "Tool repetition within threshold.");
|
|
11529
|
+
}
|
|
11530
|
+
return open2(ruleId, "Same tool repeated beyond threshold.", evidence);
|
|
11531
|
+
}
|
|
11532
|
+
function evaluateSameArgsRepetition(events, maxRepeats) {
|
|
11533
|
+
const ruleId = "circuit.same-args-repetition";
|
|
11534
|
+
const counts = /* @__PURE__ */ new Map();
|
|
11535
|
+
for (const event of events.filter(isToolEvent)) {
|
|
11536
|
+
const label = toolLabel(event);
|
|
11537
|
+
const hash = argsHash(label, toolArgs(event));
|
|
11538
|
+
const key = `${label}:${hash}`;
|
|
11539
|
+
const current = counts.get(key) ?? { toolName: label, count: 0 };
|
|
11540
|
+
current.count += 1;
|
|
11541
|
+
counts.set(key, current);
|
|
11542
|
+
}
|
|
11543
|
+
const evidence = [];
|
|
11544
|
+
for (const entry of counts.values()) {
|
|
11545
|
+
if (entry.count > maxRepeats) {
|
|
11546
|
+
evidence.push({ ruleId, toolName: entry.toolName, count: entry.count, threshold: maxRepeats });
|
|
11547
|
+
}
|
|
11548
|
+
}
|
|
11549
|
+
if (evidence.length === 0) {
|
|
11550
|
+
return closed(ruleId, "Tool argument repetition within threshold.");
|
|
11551
|
+
}
|
|
11552
|
+
return open2(ruleId, "Same tool arguments repeated beyond threshold.", evidence);
|
|
11553
|
+
}
|
|
11554
|
+
function evaluateMaxLoopIterations(events, maxIterations) {
|
|
11555
|
+
const ruleId = "circuit.max-loop-iterations";
|
|
11556
|
+
const iterationEvents = events.filter((event) => {
|
|
11557
|
+
const attrs = event.attributes ?? {};
|
|
11558
|
+
return typeof attrs.iteration === "number" || event.name.toLowerCase().includes("loop");
|
|
11559
|
+
});
|
|
11560
|
+
const maxSeen = iterationEvents.reduce((max, event) => {
|
|
11561
|
+
const attrs = event.attributes ?? {};
|
|
11562
|
+
const iteration = typeof attrs.iteration === "number" ? attrs.iteration : max;
|
|
11563
|
+
return Math.max(max, iteration);
|
|
11564
|
+
}, iterationEvents.length);
|
|
11565
|
+
if (maxSeen <= maxIterations) {
|
|
11566
|
+
return closed(ruleId, "Loop iterations within threshold.");
|
|
11567
|
+
}
|
|
11568
|
+
return open2(ruleId, "Loop iterations exceeded threshold.", [
|
|
11569
|
+
{ ruleId, count: maxSeen, threshold: maxIterations }
|
|
11570
|
+
]);
|
|
11571
|
+
}
|
|
11572
|
+
function evaluateMaxRetries(events, maxRetries) {
|
|
11573
|
+
const ruleId = "circuit.max-retries";
|
|
11574
|
+
const attempts = events.map(attemptNumber).filter((value) => value !== void 0);
|
|
11575
|
+
const maxAttempt = attempts.length > 0 ? Math.max(...attempts) : 0;
|
|
11576
|
+
if (maxAttempt <= maxRetries) {
|
|
11577
|
+
return closed(ruleId, "Retry count within threshold.");
|
|
11578
|
+
}
|
|
11579
|
+
return open2(ruleId, "Retry count exceeded threshold.", [
|
|
11580
|
+
{ ruleId, count: maxAttempt, threshold: maxRetries }
|
|
11581
|
+
]);
|
|
11582
|
+
}
|
|
11583
|
+
function evaluateToolTimeout(events, maxDurationMs) {
|
|
11584
|
+
const ruleId = "circuit.tool-timeout";
|
|
11585
|
+
const evidence = [];
|
|
11586
|
+
for (const event of events.filter(isToolEvent)) {
|
|
11587
|
+
const duration = durationMs(event);
|
|
11588
|
+
if (duration !== void 0 && duration > maxDurationMs) {
|
|
11589
|
+
evidence.push({
|
|
11590
|
+
ruleId,
|
|
11591
|
+
toolName: toolLabel(event),
|
|
11592
|
+
count: duration,
|
|
11593
|
+
threshold: maxDurationMs,
|
|
11594
|
+
eventId: event.eventId
|
|
11595
|
+
});
|
|
11596
|
+
}
|
|
11597
|
+
}
|
|
11598
|
+
if (evidence.length === 0) {
|
|
11599
|
+
return closed(ruleId, "Tool durations within timeout.");
|
|
11600
|
+
}
|
|
11601
|
+
return open2(ruleId, "Tool call exceeded configured timeout.", evidence, "warning");
|
|
11602
|
+
}
|
|
11603
|
+
function evaluateRunawayLlmLoop(events, maxLlmCalls) {
|
|
11604
|
+
const ruleId = "circuit.runaway-llm-loop";
|
|
11605
|
+
const llmCount = events.filter(isLlmEvent).length;
|
|
11606
|
+
const hasTerminal = events.some((event) => {
|
|
11607
|
+
const status = (event.status ?? event.attributes?.status ?? "").toString().toLowerCase();
|
|
11608
|
+
return status === "ok" || status === "success" || status === "completed";
|
|
11609
|
+
});
|
|
11610
|
+
if (llmCount <= maxLlmCalls || hasTerminal) {
|
|
11611
|
+
return closed(ruleId, "LLM call count within threshold or run completed.");
|
|
11612
|
+
}
|
|
11613
|
+
return open2(ruleId, "Runaway LLM loop detected.", [
|
|
11614
|
+
{ ruleId, count: llmCount, threshold: maxLlmCalls }
|
|
11615
|
+
]);
|
|
11616
|
+
}
|
|
11617
|
+
function evaluateExcessiveBranchWidth(events, maxWidth) {
|
|
11618
|
+
const ruleId = "circuit.excessive-branch-width";
|
|
11619
|
+
const children = /* @__PURE__ */ new Map();
|
|
11620
|
+
for (const event of events) {
|
|
11621
|
+
const parentId = event.parentId;
|
|
11622
|
+
if (!parentId) continue;
|
|
11623
|
+
children.set(parentId, (children.get(parentId) ?? 0) + 1);
|
|
11624
|
+
}
|
|
11625
|
+
const evidence = [];
|
|
11626
|
+
for (const [parentId, count] of children) {
|
|
11627
|
+
if (count > maxWidth) {
|
|
11628
|
+
evidence.push({ ruleId, path: parentId, count, threshold: maxWidth });
|
|
11629
|
+
}
|
|
11630
|
+
}
|
|
11631
|
+
if (evidence.length === 0) {
|
|
11632
|
+
return closed(ruleId, "Branch width within threshold.");
|
|
11633
|
+
}
|
|
11634
|
+
return open2(ruleId, "Excessive parallel branch width detected.", evidence, "warning");
|
|
11635
|
+
}
|
|
11636
|
+
function runRule(ruleId, events, options) {
|
|
11637
|
+
switch (ruleId) {
|
|
11638
|
+
case "circuit.same-tool-repetition":
|
|
11639
|
+
if (options.sameToolRepetition === void 0) return void 0;
|
|
11640
|
+
return evaluateSameToolRepetition(events, options.sameToolRepetition.maxRepeats);
|
|
11641
|
+
case "circuit.same-args-repetition":
|
|
11642
|
+
if (options.sameArgsRepetition === void 0) return void 0;
|
|
11643
|
+
return evaluateSameArgsRepetition(events, options.sameArgsRepetition.maxRepeats);
|
|
11644
|
+
case "circuit.max-loop-iterations":
|
|
11645
|
+
if (options.maxLoopIterations === void 0) return void 0;
|
|
11646
|
+
return evaluateMaxLoopIterations(events, options.maxLoopIterations.maxIterations);
|
|
11647
|
+
case "circuit.max-retries":
|
|
11648
|
+
if (options.maxRetries === void 0) return void 0;
|
|
11649
|
+
return evaluateMaxRetries(events, options.maxRetries.maxRetries);
|
|
11650
|
+
case "circuit.tool-timeout":
|
|
11651
|
+
if (options.toolTimeout === void 0) return void 0;
|
|
11652
|
+
return evaluateToolTimeout(events, options.toolTimeout.maxDurationMs);
|
|
11653
|
+
case "circuit.runaway-llm-loop":
|
|
11654
|
+
if (options.runawayLlmLoop === void 0) return void 0;
|
|
11655
|
+
return evaluateRunawayLlmLoop(events, options.runawayLlmLoop.maxLlmCalls);
|
|
11656
|
+
case "circuit.excessive-branch-width":
|
|
11657
|
+
if (options.excessiveBranchWidth === void 0) return void 0;
|
|
11658
|
+
return evaluateExcessiveBranchWidth(events, options.excessiveBranchWidth.maxWidth);
|
|
11659
|
+
default:
|
|
11660
|
+
return void 0;
|
|
11661
|
+
}
|
|
11662
|
+
}
|
|
11663
|
+
function runCircuits(events, options = {}) {
|
|
11664
|
+
const selected = options.rules ?? ALL_RULES;
|
|
11665
|
+
const results = [];
|
|
11666
|
+
for (const ruleId of selected) {
|
|
11667
|
+
const result = runRule(ruleId, events, options);
|
|
11668
|
+
if (result) results.push(result);
|
|
11669
|
+
}
|
|
11670
|
+
const ok = !results.some((result) => result.status === "open" && result.severity === "error");
|
|
11671
|
+
return { ok, results };
|
|
11672
|
+
}
|
|
11673
|
+
|
|
11674
|
+
// packages/guardrails/src/rules.ts
|
|
11675
|
+
var DEFAULT_INJECTION_PATTERNS = [
|
|
11676
|
+
"ignore previous instructions",
|
|
11677
|
+
"ignore all prior",
|
|
11678
|
+
"disregard your instructions",
|
|
11679
|
+
"system prompt",
|
|
11680
|
+
"you are now",
|
|
11681
|
+
"jailbreak"
|
|
11682
|
+
];
|
|
11683
|
+
function pass(ruleId, message) {
|
|
11684
|
+
return { ruleId, status: "pass", severity: "info", message, evidence: [] };
|
|
11685
|
+
}
|
|
11686
|
+
function fail(ruleId, message, evidence, severity = "error") {
|
|
11687
|
+
return { ruleId, status: severity === "warning" ? "warn" : "fail", severity, message, evidence };
|
|
11688
|
+
}
|
|
11689
|
+
function boundedPreview(value, max = 80) {
|
|
11690
|
+
if (value.length <= max) return value;
|
|
11691
|
+
return `${value.slice(0, max - 3)}...`;
|
|
11692
|
+
}
|
|
11693
|
+
function evaluateBannedPhrase(text, options) {
|
|
11694
|
+
const ruleId = "guardrail.banned-phrase";
|
|
11695
|
+
const haystack = options.caseInsensitive !== false ? text.toLowerCase() : text;
|
|
11696
|
+
const evidence = [];
|
|
11697
|
+
for (const phrase of options.phrases) {
|
|
11698
|
+
const needle = options.caseInsensitive !== false ? phrase.toLowerCase() : phrase;
|
|
11699
|
+
if (needle.length > 0 && haystack.includes(needle)) {
|
|
11700
|
+
evidence.push({ ruleId, match: phrase, preview: boundedPreview(text) });
|
|
11701
|
+
}
|
|
11702
|
+
}
|
|
11703
|
+
if (evidence.length === 0) {
|
|
11704
|
+
return pass(ruleId, "No banned phrases matched.");
|
|
11705
|
+
}
|
|
11706
|
+
return fail(ruleId, `Matched ${evidence.length} banned phrase(s).`, evidence);
|
|
11707
|
+
}
|
|
11708
|
+
var SEVERITY_RANK2 = { info: 0, warning: 1, error: 2 };
|
|
11709
|
+
function evaluatePiiLeak(value, options = {}) {
|
|
11710
|
+
const ruleId = "guardrail.pii-leak";
|
|
11711
|
+
const minSeverity = options.minSeverity ?? "warning";
|
|
11712
|
+
const result = redact(value, { profile: options.profile ?? "share", collectFindings: true });
|
|
11713
|
+
const findings = result.findings.filter(
|
|
11714
|
+
(finding) => SEVERITY_RANK2[finding.severity] >= SEVERITY_RANK2[minSeverity]
|
|
11715
|
+
);
|
|
11716
|
+
if (findings.length === 0) {
|
|
11717
|
+
return pass(ruleId, "No PII-style redaction findings.");
|
|
11718
|
+
}
|
|
11719
|
+
const evidence = findings.map((finding) => ({
|
|
11720
|
+
ruleId,
|
|
11721
|
+
path: finding.path,
|
|
11722
|
+
detector: finding.detector,
|
|
11723
|
+
preview: finding.preview
|
|
11724
|
+
}));
|
|
11725
|
+
return fail(ruleId, `Detected ${findings.length} PII-style finding(s).`, evidence);
|
|
11726
|
+
}
|
|
11727
|
+
function measureDepth(value) {
|
|
11728
|
+
if (value === null || typeof value !== "object") return 0;
|
|
11729
|
+
if (Array.isArray(value)) {
|
|
11730
|
+
return 1 + Math.max(0, ...value.map((item) => measureDepth(item)));
|
|
11731
|
+
}
|
|
11732
|
+
const depths = Object.values(value).map((item) => measureDepth(item));
|
|
11733
|
+
return 1 + (depths.length === 0 ? 0 : Math.max(...depths));
|
|
11734
|
+
}
|
|
11735
|
+
function maxStringLength(value) {
|
|
11736
|
+
if (typeof value === "string") return value.length;
|
|
11737
|
+
if (value === null || typeof value !== "object") return 0;
|
|
11738
|
+
if (Array.isArray(value)) {
|
|
11739
|
+
return Math.max(0, ...value.map((item) => maxStringLength(item)));
|
|
11740
|
+
}
|
|
11741
|
+
return Math.max(0, ...Object.values(value).map((item) => maxStringLength(item)));
|
|
11742
|
+
}
|
|
11743
|
+
function evaluateUnsafeToolArgs(toolName2, toolArgs2, options = {}) {
|
|
11744
|
+
const ruleId = "guardrail.unsafe-tool-args";
|
|
11745
|
+
const blocked = new Set((options.blockedTools ?? []).map((name) => name.toLowerCase()));
|
|
11746
|
+
const evidence = [];
|
|
11747
|
+
if (blocked.has(toolName2.toLowerCase())) {
|
|
11748
|
+
evidence.push({ ruleId, preview: toolName2, match: toolName2 });
|
|
11749
|
+
}
|
|
11750
|
+
const maxDepth = options.maxDepth ?? 12;
|
|
11751
|
+
const depth = measureDepth(toolArgs2);
|
|
11752
|
+
if (depth > maxDepth) {
|
|
11753
|
+
evidence.push({ ruleId, path: "args", preview: `depth=${depth}` });
|
|
11754
|
+
}
|
|
11755
|
+
const maxLen = options.maxStringLength ?? 16384;
|
|
11756
|
+
const longest = maxStringLength(toolArgs2);
|
|
11757
|
+
if (longest > maxLen) {
|
|
11758
|
+
evidence.push({ ruleId, path: "args", preview: `maxStringLength=${longest}` });
|
|
11759
|
+
}
|
|
11760
|
+
if (evidence.length === 0) {
|
|
11761
|
+
return pass(ruleId, "Tool arguments within configured bounds.");
|
|
11762
|
+
}
|
|
11763
|
+
return fail(ruleId, "Unsafe or oversized tool arguments detected.", evidence);
|
|
11764
|
+
}
|
|
11765
|
+
function evaluatePromptInjection(text, options = {}) {
|
|
11766
|
+
const ruleId = "guardrail.prompt-injection";
|
|
11767
|
+
const patterns = options.patterns ?? DEFAULT_INJECTION_PATTERNS;
|
|
11768
|
+
const haystack = text.toLowerCase();
|
|
11769
|
+
const evidence = [];
|
|
11770
|
+
for (const pattern of patterns) {
|
|
11771
|
+
const needle = pattern.toLowerCase();
|
|
11772
|
+
if (needle.length > 0 && haystack.includes(needle)) {
|
|
11773
|
+
evidence.push({ ruleId, match: pattern, preview: boundedPreview(text) });
|
|
11774
|
+
}
|
|
11775
|
+
}
|
|
11776
|
+
if (evidence.length === 0) {
|
|
11777
|
+
return pass(ruleId, "No prompt-injection patterns matched.");
|
|
11778
|
+
}
|
|
11779
|
+
return fail(ruleId, `Matched ${evidence.length} injection pattern(s).`, evidence, "warning");
|
|
11780
|
+
}
|
|
11781
|
+
function validateSchemaField(value, field, path16, evidence) {
|
|
11782
|
+
const ruleId = "guardrail.structured-output";
|
|
11783
|
+
if (field.type) {
|
|
11784
|
+
const actual = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
|
|
11785
|
+
if (actual !== field.type) {
|
|
11786
|
+
evidence.push({ ruleId, path: path16, preview: `expected ${field.type}, got ${actual}` });
|
|
11787
|
+
return;
|
|
11788
|
+
}
|
|
11789
|
+
}
|
|
11790
|
+
if (field.enum && !field.enum.some((item) => Object.is(item, value))) {
|
|
11791
|
+
evidence.push({ ruleId, path: path16, preview: "value not in enum" });
|
|
11792
|
+
}
|
|
11793
|
+
if (field.type === "object" && field.required && value && typeof value === "object" && !Array.isArray(value)) {
|
|
11794
|
+
const record = value;
|
|
11795
|
+
for (const key of field.required) {
|
|
11796
|
+
if (!(key in record)) {
|
|
11797
|
+
evidence.push({ ruleId, path: `${path16}.${key}`, preview: "missing required key" });
|
|
11798
|
+
}
|
|
11799
|
+
}
|
|
11800
|
+
}
|
|
11801
|
+
}
|
|
11802
|
+
function evaluateStructuredOutput(value, options) {
|
|
11803
|
+
const ruleId = "guardrail.structured-output";
|
|
11804
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
11805
|
+
return fail(ruleId, "Structured output must be an object.", [
|
|
11806
|
+
{ ruleId, preview: typeof value }
|
|
11807
|
+
]);
|
|
11808
|
+
}
|
|
11809
|
+
const record = value;
|
|
11810
|
+
const evidence = [];
|
|
11811
|
+
for (const [key, field] of Object.entries(options.schema)) {
|
|
11812
|
+
validateSchemaField(record[key], field, key, evidence);
|
|
11813
|
+
}
|
|
11814
|
+
if (evidence.length === 0) {
|
|
11815
|
+
return pass(ruleId, "Structured output matches schema subset.");
|
|
11816
|
+
}
|
|
11817
|
+
return fail(ruleId, "Structured output schema violation.", evidence);
|
|
11818
|
+
}
|
|
11819
|
+
function evaluateOversizeOutput(value, options = {}) {
|
|
11820
|
+
const ruleId = "guardrail.oversize-output";
|
|
11821
|
+
const text = typeof value === "string" ? value : JSON.stringify(value);
|
|
11822
|
+
const maxLength = options.maxLength ?? options.maxSerializedLength ?? 32768;
|
|
11823
|
+
if (text.length <= maxLength) {
|
|
11824
|
+
return pass(ruleId, "Output within size limits.");
|
|
11825
|
+
}
|
|
11826
|
+
return fail(ruleId, `Output exceeds max length (${text.length} > ${maxLength}).`, [
|
|
11827
|
+
{ ruleId, preview: `length=${text.length}` }
|
|
11828
|
+
]);
|
|
11829
|
+
}
|
|
11830
|
+
function evaluateRequiredJsonShape(value, options) {
|
|
11831
|
+
const ruleId = "guardrail.required-json-shape";
|
|
11832
|
+
let parsed = value;
|
|
11833
|
+
if (typeof value === "string") {
|
|
11834
|
+
try {
|
|
11835
|
+
parsed = JSON.parse(value);
|
|
11836
|
+
} catch {
|
|
11837
|
+
return fail(ruleId, "Value is not valid JSON.", [{ ruleId, preview: boundedPreview(value) }]);
|
|
11838
|
+
}
|
|
11839
|
+
}
|
|
11840
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
11841
|
+
return fail(ruleId, "JSON value must be an object.", [{ ruleId, preview: typeof parsed }]);
|
|
11842
|
+
}
|
|
11843
|
+
const record = parsed;
|
|
11844
|
+
const evidence = [];
|
|
11845
|
+
for (const key of options.requiredKeys) {
|
|
11846
|
+
if (!(key in record)) {
|
|
11847
|
+
evidence.push({ ruleId, path: key, preview: "missing required key" });
|
|
11848
|
+
}
|
|
11849
|
+
}
|
|
11850
|
+
if (evidence.length === 0) {
|
|
11851
|
+
return pass(ruleId, "Required JSON keys present.");
|
|
11852
|
+
}
|
|
11853
|
+
return fail(ruleId, "Missing required JSON keys.", evidence);
|
|
11854
|
+
}
|
|
11855
|
+
|
|
11856
|
+
// packages/guardrails/src/run.ts
|
|
11857
|
+
var ALL_RULES2 = [
|
|
11858
|
+
"guardrail.banned-phrase",
|
|
11859
|
+
"guardrail.pii-leak",
|
|
11860
|
+
"guardrail.unsafe-tool-args",
|
|
11861
|
+
"guardrail.prompt-injection",
|
|
11862
|
+
"guardrail.structured-output",
|
|
11863
|
+
"guardrail.oversize-output",
|
|
11864
|
+
"guardrail.required-json-shape"
|
|
11865
|
+
];
|
|
11866
|
+
function isErrorFailure(result) {
|
|
11867
|
+
return result.status === "fail" && result.severity === "error";
|
|
11868
|
+
}
|
|
11869
|
+
function runRule2(ruleId, input3, options) {
|
|
11870
|
+
switch (ruleId) {
|
|
11871
|
+
case "guardrail.banned-phrase": {
|
|
11872
|
+
if (!options.bannedPhrase || input3.text === void 0) return void 0;
|
|
11873
|
+
return evaluateBannedPhrase(input3.text, options.bannedPhrase);
|
|
11874
|
+
}
|
|
11875
|
+
case "guardrail.pii-leak": {
|
|
11876
|
+
if (input3.value === void 0 && input3.text === void 0) return void 0;
|
|
11877
|
+
return evaluatePiiLeak(input3.value ?? input3.text, options.piiLeak);
|
|
11878
|
+
}
|
|
11879
|
+
case "guardrail.unsafe-tool-args": {
|
|
11880
|
+
if (!input3.toolName) return void 0;
|
|
11881
|
+
return evaluateUnsafeToolArgs(input3.toolName, input3.toolArgs ?? {}, options.unsafeToolArgs);
|
|
11882
|
+
}
|
|
11883
|
+
case "guardrail.prompt-injection": {
|
|
11884
|
+
if (input3.text === void 0) return void 0;
|
|
11885
|
+
return evaluatePromptInjection(input3.text, options.promptInjection);
|
|
11886
|
+
}
|
|
11887
|
+
case "guardrail.structured-output": {
|
|
11888
|
+
if (!options.structuredOutput || input3.value === void 0) return void 0;
|
|
11889
|
+
return evaluateStructuredOutput(input3.value, options.structuredOutput);
|
|
11890
|
+
}
|
|
11891
|
+
case "guardrail.oversize-output": {
|
|
11892
|
+
if (input3.value === void 0 && input3.text === void 0) return void 0;
|
|
11893
|
+
return evaluateOversizeOutput(input3.value ?? input3.text, options.oversizeOutput);
|
|
11894
|
+
}
|
|
11895
|
+
case "guardrail.required-json-shape": {
|
|
11896
|
+
if (!options.requiredJsonShape || input3.value === void 0 && input3.text === void 0) return void 0;
|
|
11897
|
+
return evaluateRequiredJsonShape(input3.value ?? input3.text, options.requiredJsonShape);
|
|
11898
|
+
}
|
|
11899
|
+
default:
|
|
11900
|
+
return void 0;
|
|
11901
|
+
}
|
|
11902
|
+
}
|
|
11903
|
+
function runGuardrails(input3, options = {}) {
|
|
11904
|
+
const selected = options.rules ?? ALL_RULES2;
|
|
11905
|
+
const results = [];
|
|
11906
|
+
for (const ruleId of selected) {
|
|
11907
|
+
const result = runRule2(ruleId, input3, options);
|
|
11908
|
+
if (result) results.push(result);
|
|
11909
|
+
}
|
|
11910
|
+
const ok = !results.some(isErrorFailure);
|
|
11911
|
+
return { ok, results };
|
|
11912
|
+
}
|
|
11913
|
+
|
|
11914
|
+
// packages/cli/src/safety-extensions.ts
|
|
11915
|
+
var GUARDRAIL_ALIASES = {
|
|
11916
|
+
"banned-phrase": "guardrail.banned-phrase",
|
|
11917
|
+
"pii-leak": "guardrail.pii-leak",
|
|
11918
|
+
"unsafe-tool-args": "guardrail.unsafe-tool-args",
|
|
11919
|
+
"prompt-injection": "guardrail.prompt-injection",
|
|
11920
|
+
"structured-output": "guardrail.structured-output",
|
|
11921
|
+
"oversize-output": "guardrail.oversize-output",
|
|
11922
|
+
"required-json-shape": "guardrail.required-json-shape"
|
|
11923
|
+
};
|
|
11924
|
+
var CIRCUIT_ALIASES = {
|
|
11925
|
+
"same-tool-repetition": "circuit.same-tool-repetition",
|
|
11926
|
+
"same-args-repetition": "circuit.same-args-repetition",
|
|
11927
|
+
"max-loop-iterations": "circuit.max-loop-iterations",
|
|
11928
|
+
"max-retries": "circuit.max-retries",
|
|
11929
|
+
"tool-timeout": "circuit.tool-timeout",
|
|
11930
|
+
"runaway-llm-loop": "circuit.runaway-llm-loop",
|
|
11931
|
+
"excessive-branch-width": "circuit.excessive-branch-width"
|
|
11932
|
+
};
|
|
11933
|
+
function parseGuardrailRules(values) {
|
|
11934
|
+
if (!values?.length) return void 0;
|
|
11935
|
+
return values.map((value) => {
|
|
11936
|
+
const rule = GUARDRAIL_ALIASES[value] ?? value;
|
|
11937
|
+
return rule;
|
|
11938
|
+
});
|
|
11939
|
+
}
|
|
11940
|
+
function parseCircuitRules(values) {
|
|
11941
|
+
if (!values?.length) return void 0;
|
|
11942
|
+
return values.map((value) => {
|
|
11943
|
+
const rule = CIRCUIT_ALIASES[value] ?? value;
|
|
11944
|
+
return rule;
|
|
11945
|
+
});
|
|
11946
|
+
}
|
|
11947
|
+
function toSeverity(severity) {
|
|
11948
|
+
return severity;
|
|
11949
|
+
}
|
|
11950
|
+
function guardrailFinding(result) {
|
|
11951
|
+
return {
|
|
11952
|
+
ruleId: result.ruleId,
|
|
11953
|
+
severity: toSeverity(result.severity),
|
|
11954
|
+
status: result.status === "pass" ? "pass" : result.status === "warn" ? "warning" : "fail",
|
|
11955
|
+
message: result.message,
|
|
11956
|
+
evidence: result.evidence.map(
|
|
11957
|
+
(item) => ({
|
|
11958
|
+
path: item.preview ? `${item.path ?? "value"} (${item.preview})` : item.path
|
|
11959
|
+
})
|
|
11960
|
+
)
|
|
11961
|
+
};
|
|
11962
|
+
}
|
|
11963
|
+
function circuitFinding(result) {
|
|
11964
|
+
return {
|
|
11965
|
+
ruleId: result.ruleId,
|
|
11966
|
+
severity: toSeverity(result.severity),
|
|
11967
|
+
status: result.status === "closed" ? "pass" : result.status === "warn" ? "warning" : "fail",
|
|
11968
|
+
message: result.message,
|
|
11969
|
+
evidence: result.evidence.map(
|
|
11970
|
+
(item) => ({
|
|
11971
|
+
runId: item.runId,
|
|
11972
|
+
eventId: item.eventId,
|
|
11973
|
+
path: item.path,
|
|
11974
|
+
name: item.toolName
|
|
11975
|
+
})
|
|
11976
|
+
),
|
|
11977
|
+
actual: result.evidence[0]?.count,
|
|
11978
|
+
expected: result.evidence[0]?.threshold
|
|
11979
|
+
};
|
|
11980
|
+
}
|
|
11981
|
+
function eventToCircuit(event) {
|
|
11982
|
+
return {
|
|
11983
|
+
eventId: event.eventId,
|
|
11984
|
+
runId: event.runId,
|
|
11985
|
+
name: event.name,
|
|
11986
|
+
kind: event.kind,
|
|
11987
|
+
parentId: event.parentId,
|
|
11988
|
+
startedAt: event.startedAt,
|
|
11989
|
+
endedAt: event.endedAt,
|
|
11990
|
+
durationMs: event.durationMs,
|
|
11991
|
+
attributes: event.attributes,
|
|
11992
|
+
status: event.status
|
|
11993
|
+
};
|
|
11994
|
+
}
|
|
11995
|
+
function collectGuardrailInputs(read) {
|
|
11996
|
+
const inputs = [];
|
|
11997
|
+
for (const event of read.events) {
|
|
11998
|
+
const attrs = event.attributes ?? {};
|
|
11999
|
+
for (const key of ["output", "answer", "text", "content", "message"]) {
|
|
12000
|
+
const value = attrs[key];
|
|
12001
|
+
if (typeof value === "string") inputs.push({ text: value });
|
|
12002
|
+
else if (value !== void 0) inputs.push({ value });
|
|
12003
|
+
}
|
|
12004
|
+
if (event.kind === "TOOL" || event.name.startsWith("tool:")) {
|
|
12005
|
+
inputs.push({
|
|
12006
|
+
toolName: String(attrs.toolName ?? attrs.tool ?? event.name),
|
|
12007
|
+
toolArgs: attrs.arguments ?? attrs.args ?? attrs.input
|
|
12008
|
+
});
|
|
12009
|
+
}
|
|
12010
|
+
}
|
|
12011
|
+
return inputs;
|
|
12012
|
+
}
|
|
12013
|
+
var DEFAULT_GUARDRAIL_OPTIONS = {
|
|
12014
|
+
bannedPhrase: { phrases: ["delete all data", "ignore all instructions"] },
|
|
12015
|
+
promptInjection: {},
|
|
12016
|
+
piiLeak: { profile: "share" }
|
|
12017
|
+
};
|
|
12018
|
+
var DEFAULT_CIRCUIT_OPTIONS = {
|
|
12019
|
+
sameToolRepetition: { maxRepeats: 3 },
|
|
12020
|
+
sameArgsRepetition: { maxRepeats: 2 },
|
|
12021
|
+
maxLoopIterations: { maxIterations: 20 },
|
|
12022
|
+
maxRetries: { maxRetries: 3 },
|
|
12023
|
+
toolTimeout: { maxDurationMs: 6e4 },
|
|
12024
|
+
runawayLlmLoop: { maxLlmCalls: 12 },
|
|
12025
|
+
excessiveBranchWidth: { maxWidth: 8 }
|
|
12026
|
+
};
|
|
12027
|
+
function mergeSafetyExtensions(result, read, options) {
|
|
12028
|
+
const findings = [...result.findings];
|
|
12029
|
+
let failed = result.summary.failed;
|
|
12030
|
+
let warnings = result.summary.warnings;
|
|
12031
|
+
let passed = result.summary.passed;
|
|
12032
|
+
const guardrailRules = parseGuardrailRules(options.guardrails);
|
|
12033
|
+
if (guardrailRules) {
|
|
12034
|
+
for (const input3 of collectGuardrailInputs(read)) {
|
|
12035
|
+
const run = runGuardrails(input3, {
|
|
12036
|
+
...DEFAULT_GUARDRAIL_OPTIONS,
|
|
12037
|
+
rules: guardrailRules
|
|
12038
|
+
});
|
|
12039
|
+
for (const item of run.results) {
|
|
12040
|
+
const finding = guardrailFinding(item);
|
|
12041
|
+
findings.push(finding);
|
|
12042
|
+
if (finding.status === "fail") failed += 1;
|
|
12043
|
+
else if (finding.status === "warning") warnings += 1;
|
|
12044
|
+
else passed += 1;
|
|
12045
|
+
}
|
|
12046
|
+
}
|
|
12047
|
+
}
|
|
12048
|
+
const circuitRules = parseCircuitRules(options.circuits);
|
|
12049
|
+
if (circuitRules) {
|
|
12050
|
+
const circuitRun = runCircuits(
|
|
12051
|
+
read.events.map(eventToCircuit),
|
|
12052
|
+
{ ...DEFAULT_CIRCUIT_OPTIONS, rules: circuitRules }
|
|
12053
|
+
);
|
|
12054
|
+
for (const item of circuitRun.results) {
|
|
12055
|
+
const finding = circuitFinding(item);
|
|
12056
|
+
findings.push(finding);
|
|
12057
|
+
if (finding.status === "fail") failed += 1;
|
|
12058
|
+
else if (finding.status === "warning") warnings += 1;
|
|
12059
|
+
else passed += 1;
|
|
12060
|
+
}
|
|
12061
|
+
}
|
|
12062
|
+
const ok = failed === 0 && result.summary.errors === 0;
|
|
12063
|
+
return {
|
|
12064
|
+
...result,
|
|
12065
|
+
ok,
|
|
12066
|
+
status: failed > 0 ? "fail" : result.status,
|
|
12067
|
+
summary: {
|
|
12068
|
+
passed,
|
|
12069
|
+
failed,
|
|
12070
|
+
warnings,
|
|
12071
|
+
errors: result.summary.errors
|
|
12072
|
+
},
|
|
12073
|
+
findings
|
|
12074
|
+
};
|
|
12075
|
+
}
|
|
11460
12076
|
|
|
11461
12077
|
// packages/cli/src/check.ts
|
|
11462
12078
|
var DEFAULT_SELECT = ["run.status"];
|
|
@@ -11505,7 +12121,7 @@ function asConfig(value) {
|
|
|
11505
12121
|
}
|
|
11506
12122
|
async function loadConfig(configPath) {
|
|
11507
12123
|
if (configPath === void 0) return {};
|
|
11508
|
-
const extension =
|
|
12124
|
+
const extension = path14__default.default.extname(configPath);
|
|
11509
12125
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
11510
12126
|
throw new Error(
|
|
11511
12127
|
"TypeScript check configs require an explicit precompiled JavaScript config or future --config-loader support."
|
|
@@ -11514,7 +12130,7 @@ async function loadConfig(configPath) {
|
|
|
11514
12130
|
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
11515
12131
|
throw new Error("Unsupported check config extension. Use .json, .js, .mjs, or .cjs.");
|
|
11516
12132
|
}
|
|
11517
|
-
const absolute =
|
|
12133
|
+
const absolute = path14__default.default.resolve(configPath);
|
|
11518
12134
|
if (extension === ".json") {
|
|
11519
12135
|
const raw = await promises.readFile(absolute, "utf-8");
|
|
11520
12136
|
return asConfig(JSON.parse(raw));
|
|
@@ -11643,10 +12259,10 @@ function printHuman(result) {
|
|
|
11643
12259
|
console.log(`- ${diagnostic4.code}: ${diagnostic4.message}`);
|
|
11644
12260
|
}
|
|
11645
12261
|
for (const finding of result.findings) {
|
|
11646
|
-
const
|
|
12262
|
+
const path16 = finding.evidence[0]?.path;
|
|
11647
12263
|
const run = finding.evidence[0]?.runId;
|
|
11648
12264
|
const runPrefix = run ? `[${run}] ` : "";
|
|
11649
|
-
console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${
|
|
12265
|
+
console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${path16 ? ` (${path16})` : ""}`);
|
|
11650
12266
|
}
|
|
11651
12267
|
}
|
|
11652
12268
|
function readErrorResult(error) {
|
|
@@ -11696,12 +12312,19 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
11696
12312
|
}
|
|
11697
12313
|
);
|
|
11698
12314
|
perRun.push(
|
|
11699
|
-
|
|
11700
|
-
|
|
12315
|
+
mergeSafetyExtensions(
|
|
12316
|
+
runTraceChecks(
|
|
12317
|
+
{ read },
|
|
12318
|
+
{
|
|
12319
|
+
rules: built.rules,
|
|
12320
|
+
select: built.select,
|
|
12321
|
+
runId: meta.runId
|
|
12322
|
+
}
|
|
12323
|
+
),
|
|
12324
|
+
read,
|
|
11701
12325
|
{
|
|
11702
|
-
|
|
11703
|
-
|
|
11704
|
-
runId: meta.runId
|
|
12326
|
+
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
12327
|
+
...options.circuit ? { circuits: options.circuit } : {}
|
|
11705
12328
|
}
|
|
11706
12329
|
)
|
|
11707
12330
|
);
|
|
@@ -11720,12 +12343,19 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
11720
12343
|
const read = await openTrace(input3, {
|
|
11721
12344
|
...options.format !== void 0 ? { format: options.format } : {}
|
|
11722
12345
|
});
|
|
11723
|
-
result =
|
|
11724
|
-
|
|
12346
|
+
result = mergeSafetyExtensions(
|
|
12347
|
+
runTraceChecks(
|
|
12348
|
+
{ read },
|
|
12349
|
+
{
|
|
12350
|
+
rules: built.rules,
|
|
12351
|
+
select: built.select,
|
|
12352
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
12353
|
+
}
|
|
12354
|
+
),
|
|
12355
|
+
read,
|
|
11725
12356
|
{
|
|
11726
|
-
|
|
11727
|
-
|
|
11728
|
-
...options.run !== void 0 ? { runId: options.run } : {}
|
|
12357
|
+
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
12358
|
+
...options.circuit ? { circuits: options.circuit } : {}
|
|
11729
12359
|
}
|
|
11730
12360
|
);
|
|
11731
12361
|
}
|
|
@@ -11746,6 +12376,262 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
11746
12376
|
else printHuman(result);
|
|
11747
12377
|
}
|
|
11748
12378
|
|
|
12379
|
+
// packages/viewer/src/html.ts
|
|
12380
|
+
var viewerIndexHtml = `<!DOCTYPE html>
|
|
12381
|
+
<html lang="en">
|
|
12382
|
+
<head>
|
|
12383
|
+
<meta charset="utf-8" />
|
|
12384
|
+
<title>AgentInspect Viewer</title>
|
|
12385
|
+
<style>
|
|
12386
|
+
body { font-family: system-ui, sans-serif; margin: 1.5rem; line-height: 1.4; }
|
|
12387
|
+
h1 { font-size: 1.25rem; }
|
|
12388
|
+
pre { background: #f4f4f5; padding: 1rem; overflow: auto; max-height: 70vh; }
|
|
12389
|
+
a { color: #0b57d0; }
|
|
12390
|
+
.muted { color: #666; }
|
|
12391
|
+
</style>
|
|
12392
|
+
</head>
|
|
12393
|
+
<body>
|
|
12394
|
+
<h1>AgentInspect local viewer</h1>
|
|
12395
|
+
<p class="muted">Read-only. JSONL on disk remains canonical.</p>
|
|
12396
|
+
<p><a href="/api/health">/api/health</a> \xB7 <a href="/api/traces">/api/traces</a> \xB7 <a href="/api/sessions">/api/sessions</a></p>
|
|
12397
|
+
<pre id="out">Loading traces\u2026</pre>
|
|
12398
|
+
<script>
|
|
12399
|
+
fetch("/api/traces").then((r) => r.json()).then((data) => {
|
|
12400
|
+
document.getElementById("out").textContent = JSON.stringify(data, null, 2);
|
|
12401
|
+
}).catch((err) => {
|
|
12402
|
+
document.getElementById("out").textContent = String(err);
|
|
12403
|
+
});
|
|
12404
|
+
</script>
|
|
12405
|
+
</body>
|
|
12406
|
+
</html>
|
|
12407
|
+
`;
|
|
12408
|
+
|
|
12409
|
+
// packages/viewer/src/server.ts
|
|
12410
|
+
var DEFAULT_HOST = "127.0.0.1";
|
|
12411
|
+
var DEFAULT_PORT = 7337;
|
|
12412
|
+
var DEFAULT_MAX_EVENTS = 500;
|
|
12413
|
+
function sendJson(res, status, body) {
|
|
12414
|
+
const payload = JSON.stringify(body);
|
|
12415
|
+
res.writeHead(status, {
|
|
12416
|
+
"content-type": "application/json; charset=utf-8",
|
|
12417
|
+
"cache-control": "no-store"
|
|
12418
|
+
});
|
|
12419
|
+
res.end(payload);
|
|
12420
|
+
}
|
|
12421
|
+
function notFound(res, message) {
|
|
12422
|
+
sendJson(res, 404, { error: message });
|
|
12423
|
+
}
|
|
12424
|
+
function badRequest(res, message) {
|
|
12425
|
+
sendJson(res, 400, { error: message });
|
|
12426
|
+
}
|
|
12427
|
+
function decodeId(segment) {
|
|
12428
|
+
if (!segment) return "";
|
|
12429
|
+
try {
|
|
12430
|
+
return decodeURIComponent(segment);
|
|
12431
|
+
} catch {
|
|
12432
|
+
return segment;
|
|
12433
|
+
}
|
|
12434
|
+
}
|
|
12435
|
+
function boundedEvents(events, maxEvents) {
|
|
12436
|
+
if (events.length <= maxEvents) return [...events];
|
|
12437
|
+
return events.slice(0, maxEvents);
|
|
12438
|
+
}
|
|
12439
|
+
function createViewerServer(options = {}) {
|
|
12440
|
+
const traceDir = resolveTraceDir({ dir: options.traceDir });
|
|
12441
|
+
const host = options.host ?? DEFAULT_HOST;
|
|
12442
|
+
const port = options.port ?? DEFAULT_PORT;
|
|
12443
|
+
const maxEvents = options.maxEvents ?? DEFAULT_MAX_EVENTS;
|
|
12444
|
+
if (host === "0.0.0.0") {
|
|
12445
|
+
console.warn(
|
|
12446
|
+
"[AgentInspect viewer] Binding to 0.0.0.0 exposes traces on the network. Use 127.0.0.1 unless you accept that risk."
|
|
12447
|
+
);
|
|
12448
|
+
}
|
|
12449
|
+
const server = http.createServer(async (req, res) => {
|
|
12450
|
+
try {
|
|
12451
|
+
if (req.method !== "GET" && req.method !== "HEAD") {
|
|
12452
|
+
return badRequest(res, "Only GET is supported.");
|
|
12453
|
+
}
|
|
12454
|
+
const url = new URL(req.url ?? "/", `http://${host}:${port}`);
|
|
12455
|
+
const pathname = url.pathname;
|
|
12456
|
+
if (pathname === "/" || pathname === "/index.html") {
|
|
12457
|
+
res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
|
|
12458
|
+
res.end(viewerIndexHtml);
|
|
12459
|
+
return;
|
|
12460
|
+
}
|
|
12461
|
+
if (pathname === "/api/health") {
|
|
12462
|
+
return sendJson(res, 200, {
|
|
12463
|
+
ok: true,
|
|
12464
|
+
readOnly: true,
|
|
12465
|
+
traceDir: path14__default.default.resolve(traceDir)
|
|
12466
|
+
});
|
|
12467
|
+
}
|
|
12468
|
+
const td = new TraceDirectory({ dir: traceDir });
|
|
12469
|
+
if (pathname === "/api/traces") {
|
|
12470
|
+
const files = await td.list();
|
|
12471
|
+
const metas = await loadTraceMetadataList(
|
|
12472
|
+
traceDir,
|
|
12473
|
+
files,
|
|
12474
|
+
(fileName) => td.getPath(fileName)
|
|
12475
|
+
);
|
|
12476
|
+
return sendJson(
|
|
12477
|
+
res,
|
|
12478
|
+
200,
|
|
12479
|
+
metas.map((meta) => ({
|
|
12480
|
+
runId: meta.runId,
|
|
12481
|
+
name: meta.name,
|
|
12482
|
+
status: meta.status,
|
|
12483
|
+
file: path14__default.default.basename(meta.filePath),
|
|
12484
|
+
startedAt: meta.startedAt,
|
|
12485
|
+
durationMs: meta.durationMs
|
|
12486
|
+
}))
|
|
12487
|
+
);
|
|
12488
|
+
}
|
|
12489
|
+
if (pathname === "/api/sessions") {
|
|
12490
|
+
const files = await td.list();
|
|
12491
|
+
const metas = await loadTraceMetadataList(
|
|
12492
|
+
traceDir,
|
|
12493
|
+
files,
|
|
12494
|
+
(fileName) => td.getPath(fileName)
|
|
12495
|
+
);
|
|
12496
|
+
const runs = await loadSessionRunRecords(metas);
|
|
12497
|
+
const index = buildSessionIndex(runs, {
|
|
12498
|
+
correlateByGroupId: url.searchParams.get("correlateGroup") === "true"
|
|
12499
|
+
});
|
|
12500
|
+
return sendJson(res, 200, index);
|
|
12501
|
+
}
|
|
12502
|
+
const sessionMatch = pathname.match(/^\/api\/session\/([^/]+)$/);
|
|
12503
|
+
if (sessionMatch) {
|
|
12504
|
+
const sessionId = decodeId(sessionMatch[1]);
|
|
12505
|
+
const files = await td.list();
|
|
12506
|
+
const metas = await loadTraceMetadataList(
|
|
12507
|
+
traceDir,
|
|
12508
|
+
files,
|
|
12509
|
+
(fileName) => td.getPath(fileName)
|
|
12510
|
+
);
|
|
12511
|
+
const runs = await loadSessionRunRecords(metas);
|
|
12512
|
+
const index = buildSessionIndex(runs, {
|
|
12513
|
+
correlateByGroupId: url.searchParams.get("correlateGroup") === "true"
|
|
12514
|
+
});
|
|
12515
|
+
const session = index.sessions.find((item) => item.sessionId === sessionId);
|
|
12516
|
+
if (!session) return notFound(res, `Session not found: ${sessionId}`);
|
|
12517
|
+
return sendJson(res, 200, session);
|
|
12518
|
+
}
|
|
12519
|
+
const traceMatch = pathname.match(/^\/api\/trace\/([^/]+)$/);
|
|
12520
|
+
if (traceMatch) {
|
|
12521
|
+
const runId = decodeId(traceMatch[1]);
|
|
12522
|
+
const files = await td.list();
|
|
12523
|
+
const metas = await loadTraceMetadataList(
|
|
12524
|
+
traceDir,
|
|
12525
|
+
files,
|
|
12526
|
+
(fileName) => td.getPath(fileName)
|
|
12527
|
+
);
|
|
12528
|
+
const meta = metas.find((item) => item.runId === runId);
|
|
12529
|
+
if (!meta) return notFound(res, `Run not found: ${runId}`);
|
|
12530
|
+
const read = await openTrace({ type: "file", path: meta.filePath });
|
|
12531
|
+
const run = read.runs.find((item) => item.runId === runId) ?? read.runs[0];
|
|
12532
|
+
return sendJson(res, 200, {
|
|
12533
|
+
runId,
|
|
12534
|
+
format: read.format,
|
|
12535
|
+
run,
|
|
12536
|
+
events: boundedEvents(read.events, maxEvents),
|
|
12537
|
+
warnings: read.warnings,
|
|
12538
|
+
truncated: read.events.length > maxEvents
|
|
12539
|
+
});
|
|
12540
|
+
}
|
|
12541
|
+
const timelineMatch = pathname.match(/^\/api\/trace\/([^/]+)\/timeline$/);
|
|
12542
|
+
if (timelineMatch) {
|
|
12543
|
+
const runId = decodeId(timelineMatch[1]);
|
|
12544
|
+
const files = await td.list();
|
|
12545
|
+
const metas = await loadTraceMetadataList(
|
|
12546
|
+
traceDir,
|
|
12547
|
+
files,
|
|
12548
|
+
(fileName) => td.getPath(fileName)
|
|
12549
|
+
);
|
|
12550
|
+
const meta = metas.find((item) => item.runId === runId);
|
|
12551
|
+
if (!meta) return notFound(res, `Run not found: ${runId}`);
|
|
12552
|
+
const read = await openTrace({ type: "file", path: meta.filePath });
|
|
12553
|
+
const legacyEvents = persistedInspectEventsToTraceEvents(
|
|
12554
|
+
boundedEvents(read.events, maxEvents)
|
|
12555
|
+
);
|
|
12556
|
+
const timeline = buildRunTimeline(legacyEvents, { focus: "all" });
|
|
12557
|
+
return sendJson(res, 200, { runId, timeline });
|
|
12558
|
+
}
|
|
12559
|
+
const checkMatch = pathname.match(/^\/api\/trace\/([^/]+)\/check$/);
|
|
12560
|
+
if (checkMatch) {
|
|
12561
|
+
const runId = decodeId(checkMatch[1]);
|
|
12562
|
+
const files = await td.list();
|
|
12563
|
+
const metas = await loadTraceMetadataList(
|
|
12564
|
+
traceDir,
|
|
12565
|
+
files,
|
|
12566
|
+
(fileName) => td.getPath(fileName)
|
|
12567
|
+
);
|
|
12568
|
+
const meta = metas.find((item) => item.runId === runId);
|
|
12569
|
+
if (!meta) return notFound(res, `Run not found: ${runId}`);
|
|
12570
|
+
const read = await openTrace({ type: "file", path: meta.filePath });
|
|
12571
|
+
const result = runTraceChecks(
|
|
12572
|
+
{ read },
|
|
12573
|
+
{ rules: [createRunStatusRule()], select: ["run.status"], runId }
|
|
12574
|
+
);
|
|
12575
|
+
return sendJson(res, 200, result);
|
|
12576
|
+
}
|
|
12577
|
+
return notFound(res, `Unknown route: ${pathname}`);
|
|
12578
|
+
} catch (error) {
|
|
12579
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
12580
|
+
sendJson(res, 500, { error: message });
|
|
12581
|
+
}
|
|
12582
|
+
});
|
|
12583
|
+
return server;
|
|
12584
|
+
}
|
|
12585
|
+
function startViewerServer(options = {}) {
|
|
12586
|
+
const host = options.host ?? DEFAULT_HOST;
|
|
12587
|
+
const port = options.port ?? DEFAULT_PORT;
|
|
12588
|
+
const traceDir = resolveTraceDir({ dir: options.traceDir });
|
|
12589
|
+
const server = createViewerServer(options);
|
|
12590
|
+
return new Promise((resolve, reject) => {
|
|
12591
|
+
server.once("error", reject);
|
|
12592
|
+
server.listen(port, host, () => {
|
|
12593
|
+
const address = server.address();
|
|
12594
|
+
const resolvedPort = typeof address === "object" && address ? address.port : port;
|
|
12595
|
+
resolve({
|
|
12596
|
+
host,
|
|
12597
|
+
port: resolvedPort,
|
|
12598
|
+
traceDir: path14__default.default.resolve(traceDir),
|
|
12599
|
+
url: `http://${host}:${resolvedPort}`
|
|
12600
|
+
});
|
|
12601
|
+
});
|
|
12602
|
+
});
|
|
12603
|
+
}
|
|
12604
|
+
|
|
12605
|
+
// packages/cli/src/serve.ts
|
|
12606
|
+
function parsePort(value) {
|
|
12607
|
+
if (value === void 0) return void 0;
|
|
12608
|
+
const parsed = Number(value);
|
|
12609
|
+
if (!Number.isInteger(parsed) || parsed <= 0 || parsed > 65535) {
|
|
12610
|
+
throw new Error("--port must be an integer between 1 and 65535.");
|
|
12611
|
+
}
|
|
12612
|
+
return parsed;
|
|
12613
|
+
}
|
|
12614
|
+
async function serveCommand(options = {}) {
|
|
12615
|
+
const port = parsePort(options.port);
|
|
12616
|
+
const host = options.host?.trim() || "127.0.0.1";
|
|
12617
|
+
const info = await startViewerServer({
|
|
12618
|
+
...options.dir !== void 0 ? { traceDir: options.dir } : {},
|
|
12619
|
+
host,
|
|
12620
|
+
...port !== void 0 ? { port } : {}
|
|
12621
|
+
});
|
|
12622
|
+
console.log(`AgentInspect viewer (read-only): ${info.url}`);
|
|
12623
|
+
console.log(`Trace directory: ${info.traceDir}`);
|
|
12624
|
+
if (options.open === true && host !== "127.0.0.1" && host !== "localhost") {
|
|
12625
|
+
console.warn("Skipping browser open for non-local host binding.");
|
|
12626
|
+
} else if (options.open === true) {
|
|
12627
|
+
const openMod = await import('child_process');
|
|
12628
|
+
openMod.exec(`open ${info.url}`, () => {
|
|
12629
|
+
});
|
|
12630
|
+
}
|
|
12631
|
+
await new Promise(() => {
|
|
12632
|
+
});
|
|
12633
|
+
}
|
|
12634
|
+
|
|
11749
12635
|
// packages/eval/src/index.ts
|
|
11750
12636
|
function isTraceReadResult(value) {
|
|
11751
12637
|
return value !== null && typeof value === "object" && "runs" in value && "events" in value && "format" in value;
|
|
@@ -11914,10 +12800,10 @@ async function evalRun(input3, options = {}) {
|
|
|
11914
12800
|
diagnostics: []
|
|
11915
12801
|
};
|
|
11916
12802
|
}
|
|
11917
|
-
function evidenceForRun(run,
|
|
11918
|
-
return [{ runId: run.runId, ...
|
|
12803
|
+
function evidenceForRun(run, path16) {
|
|
12804
|
+
return [{ runId: run.runId, ...path16 !== void 0 ? { path: path16 } : {} }];
|
|
11919
12805
|
}
|
|
11920
|
-
function evidenceForEvent(event,
|
|
12806
|
+
function evidenceForEvent(event, path16) {
|
|
11921
12807
|
return [
|
|
11922
12808
|
{
|
|
11923
12809
|
runId: event.runId,
|
|
@@ -11925,11 +12811,11 @@ function evidenceForEvent(event, path15) {
|
|
|
11925
12811
|
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
11926
12812
|
kind: event.kind,
|
|
11927
12813
|
name: event.name,
|
|
11928
|
-
...
|
|
12814
|
+
...path16 !== void 0 ? { path: path16 } : {}
|
|
11929
12815
|
}
|
|
11930
12816
|
];
|
|
11931
12817
|
}
|
|
11932
|
-
function
|
|
12818
|
+
function fail2(ruleId, message, evidence, expected, actual) {
|
|
11933
12819
|
return {
|
|
11934
12820
|
ruleId,
|
|
11935
12821
|
status: "fail",
|
|
@@ -12083,9 +12969,9 @@ function collectTextFields(nodes, keys, preferredKinds = []) {
|
|
|
12083
12969
|
function tokenize(text) {
|
|
12084
12970
|
return [...text.toLowerCase().matchAll(/[a-z0-9][a-z0-9'-]{2,}/g)].map((match) => match[0].replace(/^['-]+|['-]+$/g, "")).filter((token) => token.length > 2 && !STOP_WORDS.has(token));
|
|
12085
12971
|
}
|
|
12086
|
-
function firstEvidence(fields, run,
|
|
12972
|
+
function firstEvidence(fields, run, path16) {
|
|
12087
12973
|
const first = fields[0];
|
|
12088
|
-
return first === void 0 ? evidenceForRun(run,
|
|
12974
|
+
return first === void 0 ? evidenceForRun(run, path16) : evidenceForEvent(first.node.event, first.path);
|
|
12089
12975
|
}
|
|
12090
12976
|
function collectSourceIds(nodes, keys) {
|
|
12091
12977
|
const wanted = keySet(keys);
|
|
@@ -12125,7 +13011,7 @@ var checks = {
|
|
|
12125
13011
|
"eval.requireSuccess",
|
|
12126
13012
|
"run",
|
|
12127
13013
|
(context) => context.run.status === "ok" ? [] : [
|
|
12128
|
-
|
|
13014
|
+
fail2(
|
|
12129
13015
|
"eval.requireSuccess",
|
|
12130
13016
|
"Run did not complete successfully.",
|
|
12131
13017
|
evidenceForRun(context.run, "status"),
|
|
@@ -12140,7 +13026,7 @@ var checks = {
|
|
|
12140
13026
|
return createRule("eval.requiredTools", "tool", (context) => {
|
|
12141
13027
|
const tools = new Set(nodeNames(context.nodes, "TOOL"));
|
|
12142
13028
|
return expected.filter((name) => !tools.has(name)).map(
|
|
12143
|
-
(name) =>
|
|
13029
|
+
(name) => fail2(
|
|
12144
13030
|
"eval.requiredTools",
|
|
12145
13031
|
`Required tool ${name} did not appear.`,
|
|
12146
13032
|
evidenceForRun(context.run, "children"),
|
|
@@ -12156,7 +13042,7 @@ var checks = {
|
|
|
12156
13042
|
"eval.forbiddenTools",
|
|
12157
13043
|
"tool",
|
|
12158
13044
|
(context) => context.nodes.filter((node) => node.event.kind === "TOOL" && blocked.includes(node.event.name)).map(
|
|
12159
|
-
(node) =>
|
|
13045
|
+
(node) => fail2(
|
|
12160
13046
|
"eval.forbiddenTools",
|
|
12161
13047
|
`Forbidden tool ${node.event.name} appeared.`,
|
|
12162
13048
|
evidenceForEvent(node.event, "name"),
|
|
@@ -12171,7 +13057,7 @@ var checks = {
|
|
|
12171
13057
|
"eval.maxDurationMs",
|
|
12172
13058
|
"run",
|
|
12173
13059
|
(context) => context.run.durationMs !== void 0 && context.run.durationMs > maxDurationMs ? [
|
|
12174
|
-
|
|
13060
|
+
fail2(
|
|
12175
13061
|
"eval.maxDurationMs",
|
|
12176
13062
|
`Run duration exceeded ${maxDurationMs}ms.`,
|
|
12177
13063
|
evidenceForRun(context.run, "durationMs"),
|
|
@@ -12188,7 +13074,7 @@ var checks = {
|
|
|
12188
13074
|
void 0
|
|
12189
13075
|
);
|
|
12190
13076
|
return deepest !== void 0 && deepest.depth > maxDepth ? [
|
|
12191
|
-
|
|
13077
|
+
fail2(
|
|
12192
13078
|
"eval.maxDepth",
|
|
12193
13079
|
`Run tree depth exceeded ${maxDepth}.`,
|
|
12194
13080
|
evidenceForEvent(deepest.event, "depth"),
|
|
@@ -12205,7 +13091,7 @@ var checks = {
|
|
|
12205
13091
|
(context) => context.nodes.flatMap((node) => {
|
|
12206
13092
|
const retries = numericAttribute(node, ["retryCount", "retries", "attempt"]);
|
|
12207
13093
|
return retries !== void 0 && retries > maxRetries ? [
|
|
12208
|
-
|
|
13094
|
+
fail2(
|
|
12209
13095
|
"eval.maxRetries",
|
|
12210
13096
|
`Retry count exceeded ${maxRetries}.`,
|
|
12211
13097
|
evidenceForEvent(node.event, "attributes.retryCount"),
|
|
@@ -12220,7 +13106,7 @@ var checks = {
|
|
|
12220
13106
|
return createRule("eval.maxTotalTokens", "llm", (context) => {
|
|
12221
13107
|
const total = totalTokenCount(context.events);
|
|
12222
13108
|
return total > maxTotalTokens ? [
|
|
12223
|
-
|
|
13109
|
+
fail2(
|
|
12224
13110
|
"eval.maxTotalTokens",
|
|
12225
13111
|
`Total token usage exceeded ${maxTotalTokens}.`,
|
|
12226
13112
|
evidenceForRun(context.run, "tokenUsage.total"),
|
|
@@ -12235,7 +13121,7 @@ var checks = {
|
|
|
12235
13121
|
"eval.noFailedSteps",
|
|
12236
13122
|
"run",
|
|
12237
13123
|
(context) => context.nodes.filter((node) => node.event.status === "error" || node.event.kind === "ERROR").map(
|
|
12238
|
-
(node) =>
|
|
13124
|
+
(node) => fail2(
|
|
12239
13125
|
"eval.noFailedSteps",
|
|
12240
13126
|
"Run contains a failed step or error node.",
|
|
12241
13127
|
evidenceForEvent(node.event, "status"),
|
|
@@ -12253,7 +13139,7 @@ var checks = {
|
|
|
12253
13139
|
(node, index) => index < firstLlmIndex && node.event.kind === "RETRIEVER"
|
|
12254
13140
|
);
|
|
12255
13141
|
return retrievalIndex === -1 ? [
|
|
12256
|
-
|
|
13142
|
+
fail2(
|
|
12257
13143
|
"eval.requiredRetrievalBeforeGeneration",
|
|
12258
13144
|
"No retrieval step appeared before the first LLM generation.",
|
|
12259
13145
|
evidenceForEvent(context.nodes[firstLlmIndex].event, "kind"),
|
|
@@ -12269,7 +13155,7 @@ var checks = {
|
|
|
12269
13155
|
const decisions = context.nodes.filter((node) => node.event.kind === "DECISION");
|
|
12270
13156
|
if (decisions.length === 0) {
|
|
12271
13157
|
return [
|
|
12272
|
-
|
|
13158
|
+
fail2(
|
|
12273
13159
|
"eval.requiredDecisionMetadata",
|
|
12274
13160
|
"No decision node is available for required metadata.",
|
|
12275
13161
|
evidenceForRun(context.run, "children"),
|
|
@@ -12280,7 +13166,7 @@ var checks = {
|
|
|
12280
13166
|
}
|
|
12281
13167
|
return decisions.flatMap(
|
|
12282
13168
|
(node) => required.filter((key) => !hasAttribute(node, key)).map(
|
|
12283
|
-
(key) =>
|
|
13169
|
+
(key) => fail2(
|
|
12284
13170
|
"eval.requiredDecisionMetadata",
|
|
12285
13171
|
`Decision metadata ${key} is missing.`,
|
|
12286
13172
|
evidenceForEvent(node.event, `attributes.${key}`),
|
|
@@ -12301,7 +13187,7 @@ var checks = {
|
|
|
12301
13187
|
const contexts = collectTextFields(context.nodes, contextKeys, ["RETRIEVER", "TOOL"]);
|
|
12302
13188
|
if (answers.length === 0 || contexts.length === 0) {
|
|
12303
13189
|
return [
|
|
12304
|
-
|
|
13190
|
+
fail2(
|
|
12305
13191
|
"eval.contextOverlap",
|
|
12306
13192
|
"Answer and context text are required for overlap evaluation.",
|
|
12307
13193
|
firstEvidence(answers.length > 0 ? answers : contexts, context.run, "children"),
|
|
@@ -12315,7 +13201,7 @@ var checks = {
|
|
|
12315
13201
|
const sharedTerms = [...answerTerms].filter((term) => contextTerms.has(term)).length;
|
|
12316
13202
|
const overlap = answerTerms.size === 0 ? 0 : sharedTerms / answerTerms.size;
|
|
12317
13203
|
return sharedTerms < minSharedTerms || overlap < minOverlap ? [
|
|
12318
|
-
|
|
13204
|
+
fail2(
|
|
12319
13205
|
"eval.contextOverlap",
|
|
12320
13206
|
"Answer text did not sufficiently overlap retrieved context.",
|
|
12321
13207
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12343,7 +13229,7 @@ var checks = {
|
|
|
12343
13229
|
const quotes = quotedSnippets(answerText, minQuoteLength);
|
|
12344
13230
|
if (quotes.length === 0) {
|
|
12345
13231
|
return requireQuote ? [
|
|
12346
|
-
|
|
13232
|
+
fail2(
|
|
12347
13233
|
"eval.quoteOverlap",
|
|
12348
13234
|
"Answer did not contain a quote for overlap evaluation.",
|
|
12349
13235
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12354,7 +13240,7 @@ var checks = {
|
|
|
12354
13240
|
}
|
|
12355
13241
|
const missing = quotes.filter((quote) => !contextText.includes(quote.toLowerCase()));
|
|
12356
13242
|
return missing.length > 0 ? [
|
|
12357
|
-
|
|
13243
|
+
fail2(
|
|
12358
13244
|
"eval.quoteOverlap",
|
|
12359
13245
|
"Quoted answer text did not appear in retrieved context.",
|
|
12360
13246
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12372,7 +13258,7 @@ var checks = {
|
|
|
12372
13258
|
const citations = collectTextFields(context.nodes, citationKeys);
|
|
12373
13259
|
const count = citationCount(answers.map((field) => field.text).join(" "), citations);
|
|
12374
13260
|
return count === 0 ? [
|
|
12375
|
-
|
|
13261
|
+
fail2(
|
|
12376
13262
|
"eval.citationPresence",
|
|
12377
13263
|
"Answer did not include citations or source references.",
|
|
12378
13264
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12390,7 +13276,7 @@ var checks = {
|
|
|
12390
13276
|
const availableSet = new Set(available);
|
|
12391
13277
|
const missing = expected.filter((id) => !availableSet.has(id));
|
|
12392
13278
|
return missing.length > 0 ? [
|
|
12393
|
-
|
|
13279
|
+
fail2(
|
|
12394
13280
|
"eval.requiredSourceIds",
|
|
12395
13281
|
"Required source IDs were not present in trace context or citations.",
|
|
12396
13282
|
evidenceForRun(context.run, "children"),
|
|
@@ -12410,7 +13296,7 @@ var checks = {
|
|
|
12410
13296
|
const tooShort = options.minCharacters !== void 0 && characters < options.minCharacters || options.minWords !== void 0 && words < options.minWords;
|
|
12411
13297
|
const tooLong = options.maxCharacters !== void 0 && characters > options.maxCharacters || options.maxWords !== void 0 && words > options.maxWords;
|
|
12412
13298
|
return answer.length === 0 || tooShort || tooLong ? [
|
|
12413
|
-
|
|
13299
|
+
fail2(
|
|
12414
13300
|
"eval.answerLengthBounds",
|
|
12415
13301
|
"Answer length fell outside required bounds.",
|
|
12416
13302
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12433,7 +13319,7 @@ var checks = {
|
|
|
12433
13319
|
const answer = answers.map((field) => field.text).join(" ").toLowerCase();
|
|
12434
13320
|
const matches = banned.filter((phrase) => answer.includes(phrase));
|
|
12435
13321
|
return matches.length > 0 ? [
|
|
12436
|
-
|
|
13322
|
+
fail2(
|
|
12437
13323
|
"eval.bannedUnsupportedPhrases",
|
|
12438
13324
|
"Answer contained banned unsupported-answer phrasing.",
|
|
12439
13325
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12462,8 +13348,8 @@ function renderEvalMarkdown(result) {
|
|
|
12462
13348
|
if (result.findings.length > 0) {
|
|
12463
13349
|
lines.push("", "## Findings");
|
|
12464
13350
|
for (const finding of result.findings) {
|
|
12465
|
-
const
|
|
12466
|
-
lines.push(`- ${finding.ruleId}: ${finding.message}${
|
|
13351
|
+
const path16 = finding.evidence[0]?.path;
|
|
13352
|
+
lines.push(`- ${finding.ruleId}: ${finding.message}${path16 ? ` (${path16})` : ""}`);
|
|
12467
13353
|
}
|
|
12468
13354
|
}
|
|
12469
13355
|
return `${lines.join("\n")}
|
|
@@ -12510,7 +13396,7 @@ function asConfig2(value) {
|
|
|
12510
13396
|
}
|
|
12511
13397
|
async function loadConfig2(configPath) {
|
|
12512
13398
|
if (configPath === void 0) return {};
|
|
12513
|
-
const extension =
|
|
13399
|
+
const extension = path14__default.default.extname(configPath);
|
|
12514
13400
|
if (TS_CONFIG_EXTENSIONS2.has(extension)) {
|
|
12515
13401
|
throw new Error(
|
|
12516
13402
|
"TypeScript eval configs require an explicit precompiled JavaScript config or future --config-loader support."
|
|
@@ -12519,7 +13405,7 @@ async function loadConfig2(configPath) {
|
|
|
12519
13405
|
if (!CONFIG_EXTENSIONS2.has(extension)) {
|
|
12520
13406
|
throw new Error("Unsupported eval config extension. Use .json, .js, .mjs, or .cjs.");
|
|
12521
13407
|
}
|
|
12522
|
-
const absolute =
|
|
13408
|
+
const absolute = path14__default.default.resolve(configPath);
|
|
12523
13409
|
if (extension === ".json") {
|
|
12524
13410
|
const raw = await promises.readFile(absolute, "utf-8");
|
|
12525
13411
|
return asConfig2(JSON.parse(raw));
|
|
@@ -12656,8 +13542,8 @@ function printHuman2(result) {
|
|
|
12656
13542
|
console.log(`- ${diagnostic4.code}: ${diagnostic4.message}`);
|
|
12657
13543
|
}
|
|
12658
13544
|
for (const finding of result.findings) {
|
|
12659
|
-
const
|
|
12660
|
-
console.log(`- ${finding.ruleId}: ${finding.message}${
|
|
13545
|
+
const path16 = finding.evidence[0]?.path;
|
|
13546
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path16 ? ` (${path16})` : ""}`);
|
|
12661
13547
|
}
|
|
12662
13548
|
}
|
|
12663
13549
|
function readErrorResult2(error) {
|
|
@@ -12811,7 +13697,7 @@ function invalidArgumentResult(command, error) {
|
|
|
12811
13697
|
});
|
|
12812
13698
|
}
|
|
12813
13699
|
function buildSafetyRules(options) {
|
|
12814
|
-
const
|
|
13700
|
+
const maxStringLength2 = parseLimit3(options.maxStringLength, "--max-string-length") ?? DEFAULT_MAX_STRING_LENGTH;
|
|
12815
13701
|
const maxArrayLength = parseLimit3(options.maxArrayLength, "--max-array-length") ?? DEFAULT_MAX_ARRAY_LENGTH;
|
|
12816
13702
|
const maxObjectKeys = parseLimit3(options.maxObjectKeys, "--max-object-keys") ?? DEFAULT_MAX_OBJECT_KEYS;
|
|
12817
13703
|
const maxSerializedBytes = parseLimit3(options.maxSerializedBytes, "--max-serialized-bytes") ?? DEFAULT_MAX_SERIALIZED_BYTES;
|
|
@@ -12820,7 +13706,7 @@ function buildSafetyRules(options) {
|
|
|
12820
13706
|
createSafetyRedactionRule(),
|
|
12821
13707
|
createSafetySecretPatternRule(),
|
|
12822
13708
|
createSafetyOversizedAttributeRule({
|
|
12823
|
-
maxStringLength,
|
|
13709
|
+
maxStringLength: maxStringLength2,
|
|
12824
13710
|
maxArrayLength,
|
|
12825
13711
|
maxObjectKeys,
|
|
12826
13712
|
maxSerializedBytes
|
|
@@ -12895,8 +13781,8 @@ function printHuman3(result) {
|
|
|
12895
13781
|
console.log(`- ${diagnostic4.code}: ${diagnostic4.message}`);
|
|
12896
13782
|
}
|
|
12897
13783
|
for (const finding of result.findings) {
|
|
12898
|
-
const
|
|
12899
|
-
console.log(`- ${finding.ruleId}: ${finding.message}${
|
|
13784
|
+
const path16 = finding.evidence[0]?.path;
|
|
13785
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path16 ? ` (${path16})` : ""}`);
|
|
12900
13786
|
}
|
|
12901
13787
|
console.log(`Note: ${result.note}`);
|
|
12902
13788
|
}
|
|
@@ -13015,8 +13901,8 @@ function renderCheckSection(result) {
|
|
|
13015
13901
|
`Diagnostics: ${result.diagnostics.length}`
|
|
13016
13902
|
];
|
|
13017
13903
|
for (const finding of result.findings.slice(0, 10)) {
|
|
13018
|
-
const
|
|
13019
|
-
lines.push(`- ${finding.ruleId}: ${finding.message} (${
|
|
13904
|
+
const path16 = finding.evidence[0]?.path ?? "(run)";
|
|
13905
|
+
lines.push(`- ${finding.ruleId}: ${finding.message} (${path16})`);
|
|
13020
13906
|
}
|
|
13021
13907
|
for (const diagnostic4 of result.diagnostics.slice(0, 10)) {
|
|
13022
13908
|
lines.push(`- ${diagnostic4.code}: ${diagnostic4.message}`);
|
|
@@ -13094,8 +13980,8 @@ function renderHtml(trace, check, diff) {
|
|
|
13094
13980
|
`;
|
|
13095
13981
|
}
|
|
13096
13982
|
async function writeArtifact(outputDir, relativePath, content, files) {
|
|
13097
|
-
const outPath =
|
|
13098
|
-
await promises.mkdir(
|
|
13983
|
+
const outPath = path14__default.default.join(outputDir, relativePath);
|
|
13984
|
+
await promises.mkdir(path14__default.default.dirname(outPath), { recursive: true });
|
|
13099
13985
|
await promises.writeFile(outPath, content, "utf-8");
|
|
13100
13986
|
files.push(relativePath);
|
|
13101
13987
|
}
|
|
@@ -13111,7 +13997,7 @@ function manifestStatus(check, diff) {
|
|
|
13111
13997
|
return "ok";
|
|
13112
13998
|
}
|
|
13113
13999
|
async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
13114
|
-
const outputDir = options.outputDir !== void 0 && options.outputDir.trim() !== "" ?
|
|
14000
|
+
const outputDir = options.outputDir !== void 0 && options.outputDir.trim() !== "" ? path14__default.default.resolve(options.outputDir.trim()) : "";
|
|
13115
14001
|
if (outputDir === "") {
|
|
13116
14002
|
console.error("--output-dir is required.");
|
|
13117
14003
|
process.exitCode = 1;
|
|
@@ -13177,8 +14063,8 @@ async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
|
13177
14063
|
await writeArtifact(outputDir, "report.html", renderHtml(trace, check, diff), files);
|
|
13178
14064
|
const summaryTarget = options.githubSummary ?? process.env.GITHUB_STEP_SUMMARY;
|
|
13179
14065
|
if (summaryTarget !== void 0 && summaryTarget.trim() !== "") {
|
|
13180
|
-
await promises.mkdir(
|
|
13181
|
-
await promises.appendFile(
|
|
14066
|
+
await promises.mkdir(path14__default.default.dirname(path14__default.default.resolve(summaryTarget)), { recursive: true });
|
|
14067
|
+
await promises.appendFile(path14__default.default.resolve(summaryTarget), `
|
|
13182
14068
|
${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
13183
14069
|
}
|
|
13184
14070
|
const manifestFiles = [...files, "manifest.json"].sort((a, b) => a.localeCompare(b));
|
|
@@ -13197,10 +14083,10 @@ ${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
|
13197
14083
|
findings: diff?.findings.length ?? 0,
|
|
13198
14084
|
diagnostics: diff?.diagnostics.length ?? 0
|
|
13199
14085
|
},
|
|
13200
|
-
...summaryTarget !== void 0 && summaryTarget.trim() !== "" ? { githubSummary:
|
|
14086
|
+
...summaryTarget !== void 0 && summaryTarget.trim() !== "" ? { githubSummary: path14__default.default.resolve(summaryTarget) } : {},
|
|
13201
14087
|
note: NOTE
|
|
13202
14088
|
};
|
|
13203
|
-
await promises.writeFile(
|
|
14089
|
+
await promises.writeFile(path14__default.default.join(outputDir, "manifest.json"), writeJson3(manifest), "utf-8");
|
|
13204
14090
|
if (options.json === true) {
|
|
13205
14091
|
console.log(writeJson3(manifest).trimEnd());
|
|
13206
14092
|
} else {
|
|
@@ -13212,7 +14098,7 @@ ${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
|
13212
14098
|
}
|
|
13213
14099
|
}
|
|
13214
14100
|
function validateReporterArtifactPath(options) {
|
|
13215
|
-
const outputDir =
|
|
14101
|
+
const outputDir = path14__default.default.resolve(options.outputDir);
|
|
13216
14102
|
const diagnostics = [];
|
|
13217
14103
|
const rawPath = options.relativePath;
|
|
13218
14104
|
if (rawPath.length === 0) {
|
|
@@ -13232,7 +14118,7 @@ function validateReporterArtifactPath(options) {
|
|
|
13232
14118
|
});
|
|
13233
14119
|
return { ok: false, outputDir, diagnostics };
|
|
13234
14120
|
}
|
|
13235
|
-
if (
|
|
14121
|
+
if (path14__default.default.isAbsolute(rawPath) || path14__default.default.win32.isAbsolute(rawPath)) {
|
|
13236
14122
|
diagnostics.push({
|
|
13237
14123
|
code: "artifact_path_absolute",
|
|
13238
14124
|
severity: "error",
|
|
@@ -13241,7 +14127,7 @@ function validateReporterArtifactPath(options) {
|
|
|
13241
14127
|
});
|
|
13242
14128
|
return { ok: false, outputDir, diagnostics };
|
|
13243
14129
|
}
|
|
13244
|
-
const normalized =
|
|
14130
|
+
const normalized = path14__default.default.posix.normalize(rawPath.replace(/\\/g, "/"));
|
|
13245
14131
|
const segments = normalized.split("/");
|
|
13246
14132
|
if (normalized === "." || normalized.startsWith("../") || segments.some((segment) => segment === "..")) {
|
|
13247
14133
|
diagnostics.push({
|
|
@@ -13252,9 +14138,9 @@ function validateReporterArtifactPath(options) {
|
|
|
13252
14138
|
});
|
|
13253
14139
|
return { ok: false, outputDir, diagnostics };
|
|
13254
14140
|
}
|
|
13255
|
-
const absolutePath =
|
|
13256
|
-
const relFromOutput =
|
|
13257
|
-
if (relFromOutput.length === 0 || relFromOutput.startsWith("..") ||
|
|
14141
|
+
const absolutePath = path14__default.default.resolve(outputDir, normalized);
|
|
14142
|
+
const relFromOutput = path14__default.default.relative(outputDir, absolutePath);
|
|
14143
|
+
if (relFromOutput.length === 0 || relFromOutput.startsWith("..") || path14__default.default.isAbsolute(relFromOutput)) {
|
|
13258
14144
|
diagnostics.push({
|
|
13259
14145
|
code: "artifact_path_escape",
|
|
13260
14146
|
severity: "error",
|
|
@@ -13400,23 +14286,23 @@ function readManifestDocument(value) {
|
|
|
13400
14286
|
};
|
|
13401
14287
|
}
|
|
13402
14288
|
function cwdRelative(filePath) {
|
|
13403
|
-
const relative =
|
|
13404
|
-
if (relative === "" || relative.startsWith("../") ||
|
|
13405
|
-
return
|
|
14289
|
+
const relative = path14__default.default.relative(process.cwd(), path14__default.default.resolve(filePath)).replace(/\\/g, "/");
|
|
14290
|
+
if (relative === "" || relative.startsWith("../") || path14__default.default.isAbsolute(relative)) {
|
|
14291
|
+
return path14__default.default.basename(filePath);
|
|
13406
14292
|
}
|
|
13407
14293
|
return relative;
|
|
13408
14294
|
}
|
|
13409
14295
|
async function readReporterManifest(filePath) {
|
|
13410
|
-
const absolute =
|
|
14296
|
+
const absolute = path14__default.default.resolve(filePath);
|
|
13411
14297
|
const raw = await promises.readFile(absolute, "utf-8");
|
|
13412
14298
|
const document = readManifestDocument(JSON.parse(raw));
|
|
13413
14299
|
const manifest = document.manifest;
|
|
13414
14300
|
const results = manifest.results.map((result) => ({
|
|
13415
14301
|
testId: safeText(result.testId),
|
|
13416
14302
|
name: safeText(result.name),
|
|
13417
|
-
...result.file === void 0 ? {} : { file: safeText(
|
|
14303
|
+
...result.file === void 0 ? {} : { file: safeText(path14__default.default.basename(result.file)) },
|
|
13418
14304
|
status: result.status,
|
|
13419
|
-
...result.tracePath === void 0 ? {} : { tracePath: safeText(
|
|
14305
|
+
...result.tracePath === void 0 ? {} : { tracePath: safeText(path14__default.default.basename(result.tracePath)) },
|
|
13420
14306
|
artifacts: result.artifacts,
|
|
13421
14307
|
diagnostics: result.diagnostics
|
|
13422
14308
|
}));
|
|
@@ -13555,15 +14441,15 @@ async function ciSummaryCommand(manifestPaths, options = {}) {
|
|
|
13555
14441
|
return;
|
|
13556
14442
|
}
|
|
13557
14443
|
const markdown = renderMarkdown2(result);
|
|
13558
|
-
const outputPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
14444
|
+
const outputPath = options.output !== void 0 && options.output.trim() !== "" ? path14__default.default.resolve(options.output.trim()) : void 0;
|
|
13559
14445
|
if (outputPath !== void 0) {
|
|
13560
|
-
await promises.mkdir(
|
|
14446
|
+
await promises.mkdir(path14__default.default.dirname(outputPath), { recursive: true });
|
|
13561
14447
|
await promises.writeFile(outputPath, markdown, "utf-8");
|
|
13562
14448
|
}
|
|
13563
14449
|
const summaryTarget = options.githubSummary ?? process.env.GITHUB_STEP_SUMMARY;
|
|
13564
14450
|
if (summaryTarget !== void 0 && summaryTarget.trim() !== "") {
|
|
13565
|
-
const summaryPath =
|
|
13566
|
-
await promises.mkdir(
|
|
14451
|
+
const summaryPath = path14__default.default.resolve(summaryTarget);
|
|
14452
|
+
await promises.mkdir(path14__default.default.dirname(summaryPath), { recursive: true });
|
|
13567
14453
|
await promises.appendFile(summaryPath, `
|
|
13568
14454
|
${markdown}`, "utf-8");
|
|
13569
14455
|
}
|
|
@@ -13704,9 +14590,20 @@ function createCliProgram() {
|
|
|
13704
14590
|
]).option("--max-total-tokens <number>", "add llm.usage with a max total-token budget").option("--session <id>", "check all runs in a workflow session (requires --dir)").option("--group <id>", "check all runs sharing a groupId (requires --dir)").option(
|
|
13705
14591
|
"--correlate-group",
|
|
13706
14592
|
"when using --session, also match synthetic group: session keys"
|
|
14593
|
+
).option(
|
|
14594
|
+
"--guardrails <rule>",
|
|
14595
|
+
"run optional guardrail rules (repeatable): banned-phrase, pii-leak, prompt-injection, ...",
|
|
14596
|
+
(value, previous = []) => [...previous, value]
|
|
14597
|
+
).option(
|
|
14598
|
+
"--circuit <rule>",
|
|
14599
|
+
"run optional circuit rules (repeatable): same-tool-repetition, max-retries, ...",
|
|
14600
|
+
(value, previous = []) => [...previous, value]
|
|
13707
14601
|
).action((target, opts) => {
|
|
13708
14602
|
runCommand(() => checkCommand(target, opts));
|
|
13709
14603
|
});
|
|
14604
|
+
program.command("serve").description("Start optional localhost read-only trace viewer").option("--dir <path>", "trace directory to serve").option("--host <host>", "bind host (default 127.0.0.1)", "127.0.0.1").option("--port <number>", "bind port (default 7337)", "7337").option("--open", "open browser locally when host is localhost").action((opts) => {
|
|
14605
|
+
runCommand(() => serveCommand(opts));
|
|
14606
|
+
});
|
|
13710
14607
|
program.command("eval").description("Run deterministic local evals against a trace").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
13711
14608
|
new commander.Option("--format <format>", "trace input format").choices([
|
|
13712
14609
|
"agent-inspect-jsonl",
|
|
@@ -13871,9 +14768,9 @@ function isPrimaryModule() {
|
|
|
13871
14768
|
if (!entry) return false;
|
|
13872
14769
|
const selfPath = url.fileURLToPath((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href)));
|
|
13873
14770
|
try {
|
|
13874
|
-
return fs.realpathSync(
|
|
14771
|
+
return fs.realpathSync(path14__default.default.resolve(entry)) === fs.realpathSync(path14__default.default.resolve(selfPath));
|
|
13875
14772
|
} catch {
|
|
13876
|
-
return
|
|
14773
|
+
return path14__default.default.resolve(entry) === path14__default.default.resolve(selfPath);
|
|
13877
14774
|
}
|
|
13878
14775
|
}
|
|
13879
14776
|
if (isPrimaryModule()) {
|