agent-inspect 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +33 -2
- package/SECURITY.md +18 -1
- package/docs/ADAPTERS.md +41 -0
- package/docs/CLI.md +22 -2
- package/docs/LIMITATIONS.md +6 -0
- package/package.json +2 -2
- package/packages/cli/dist/index.cjs +1074 -177
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +1074 -177
- package/packages/cli/dist/index.mjs.map +1 -1
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { realpathSync, createReadStream } from 'fs';
|
|
3
|
-
import
|
|
3
|
+
import path14 from 'path';
|
|
4
4
|
import { fileURLToPath, pathToFileURL } from 'url';
|
|
5
5
|
import { Command, Option } from 'commander';
|
|
6
6
|
import { AsyncLocalStorage } from 'async_hooks';
|
|
7
|
-
import crypto, { webcrypto } from 'crypto';
|
|
7
|
+
import crypto, { webcrypto, createHash } from 'crypto';
|
|
8
8
|
import { unlink, stat, mkdir, writeFile, appendFile, readdir, readFile, access, open } from 'fs/promises';
|
|
9
9
|
import os from 'os';
|
|
10
10
|
import process2, { stdin, stdout } from 'process';
|
|
11
11
|
import tty from 'tty';
|
|
12
12
|
import { createInterface } from 'readline';
|
|
13
|
+
import { createServer } from 'http';
|
|
13
14
|
|
|
14
15
|
// package.json
|
|
15
|
-
var version = "2.
|
|
16
|
+
var version = "2.6.0";
|
|
16
17
|
|
|
17
18
|
// packages/core/src/correlation-metadata.ts
|
|
18
19
|
var TRACE_CORRELATION_KEYS = [
|
|
@@ -744,7 +745,7 @@ function formatDuration(ms) {
|
|
|
744
745
|
// packages/core/src/utils.ts
|
|
745
746
|
var DEFAULT_TRACE_DIR_NAME = ".agent-inspect";
|
|
746
747
|
var RUNS_DIR_NAME = "runs";
|
|
747
|
-
var FALLBACK_TRACE_DIR =
|
|
748
|
+
var FALLBACK_TRACE_DIR = path14.join(
|
|
748
749
|
os.tmpdir(),
|
|
749
750
|
"agent-inspect",
|
|
750
751
|
RUNS_DIR_NAME
|
|
@@ -779,7 +780,7 @@ function getDefaultTraceDir() {
|
|
|
779
780
|
if (typeof home !== "string" || home.trim() === "") {
|
|
780
781
|
return FALLBACK_TRACE_DIR;
|
|
781
782
|
}
|
|
782
|
-
return
|
|
783
|
+
return path14.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
|
|
783
784
|
} catch {
|
|
784
785
|
return FALLBACK_TRACE_DIR;
|
|
785
786
|
}
|
|
@@ -787,11 +788,11 @@ function getDefaultTraceDir() {
|
|
|
787
788
|
function getTraceFilePath(runId, traceDir) {
|
|
788
789
|
const baseDir = traceDir ?? getDefaultTraceDir();
|
|
789
790
|
let safeId = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
790
|
-
safeId =
|
|
791
|
+
safeId = path14.basename(safeId);
|
|
791
792
|
if (safeId === "" || safeId === "." || safeId === "..") {
|
|
792
793
|
safeId = "run_unknown";
|
|
793
794
|
}
|
|
794
|
-
return
|
|
795
|
+
return path14.join(baseDir, `${safeId}.jsonl`);
|
|
795
796
|
}
|
|
796
797
|
function formatError(error) {
|
|
797
798
|
if (error instanceof Error) {
|
|
@@ -1429,18 +1430,18 @@ var proto = Object.defineProperties(() => {
|
|
|
1429
1430
|
}
|
|
1430
1431
|
}
|
|
1431
1432
|
});
|
|
1432
|
-
var createStyler = (
|
|
1433
|
+
var createStyler = (open3, close, parent) => {
|
|
1433
1434
|
let openAll;
|
|
1434
1435
|
let closeAll;
|
|
1435
1436
|
if (parent === void 0) {
|
|
1436
|
-
openAll =
|
|
1437
|
+
openAll = open3;
|
|
1437
1438
|
closeAll = close;
|
|
1438
1439
|
} else {
|
|
1439
|
-
openAll = parent.openAll +
|
|
1440
|
+
openAll = parent.openAll + open3;
|
|
1440
1441
|
closeAll = close + parent.closeAll;
|
|
1441
1442
|
}
|
|
1442
1443
|
return {
|
|
1443
|
-
open:
|
|
1444
|
+
open: open3,
|
|
1444
1445
|
close,
|
|
1445
1446
|
openAll,
|
|
1446
1447
|
closeAll,
|
|
@@ -1505,15 +1506,15 @@ function getStatusIcon(status) {
|
|
|
1505
1506
|
if (status === "error") return source_default.red("\u2716");
|
|
1506
1507
|
return source_default.yellow("\u23F3");
|
|
1507
1508
|
}
|
|
1508
|
-
function renderStepLine(name,
|
|
1509
|
+
function renderStepLine(name, durationMs2, status, depth) {
|
|
1509
1510
|
try {
|
|
1510
1511
|
const nm = formatTerminalName(name);
|
|
1511
1512
|
const ind = getIndent(depth ?? 0);
|
|
1512
|
-
if (status === "running" &&
|
|
1513
|
+
if (status === "running" && durationMs2 === void 0) {
|
|
1513
1514
|
return `${ind}${source_default.yellow("\u23F3")} ${nm}`;
|
|
1514
1515
|
}
|
|
1515
|
-
const hasDur =
|
|
1516
|
-
const dur = hasDur ? formatDuration2(
|
|
1516
|
+
const hasDur = durationMs2 !== void 0 && Number.isFinite(durationMs2);
|
|
1517
|
+
const dur = hasDur ? formatDuration2(durationMs2) : void 0;
|
|
1517
1518
|
if (status === "running") {
|
|
1518
1519
|
return dur !== void 0 ? `${ind}${source_default.yellow("\u23F3")} ${nm} (${dur})` : `${ind}${source_default.yellow("\u23F3")} ${nm}`;
|
|
1519
1520
|
}
|
|
@@ -1553,7 +1554,7 @@ var TraceDirectory = class {
|
|
|
1553
1554
|
this.#dir = resolveTraceDir(options);
|
|
1554
1555
|
}
|
|
1555
1556
|
getPath(filename) {
|
|
1556
|
-
return filename ?
|
|
1557
|
+
return filename ? path14.join(this.#dir, filename) : this.#dir;
|
|
1557
1558
|
}
|
|
1558
1559
|
async list() {
|
|
1559
1560
|
try {
|
|
@@ -1580,7 +1581,7 @@ function parseIsoToMs2(value) {
|
|
|
1580
1581
|
}
|
|
1581
1582
|
async function extractMetadata(filePath, _quickScan) {
|
|
1582
1583
|
const stats = await stat(filePath);
|
|
1583
|
-
let runIdFromFile =
|
|
1584
|
+
let runIdFromFile = path14.basename(filePath);
|
|
1584
1585
|
if (runIdFromFile.endsWith(".jsonl")) {
|
|
1585
1586
|
runIdFromFile = runIdFromFile.slice(0, -".jsonl".length);
|
|
1586
1587
|
}
|
|
@@ -1666,14 +1667,14 @@ async function extractMetadata(filePath, _quickScan) {
|
|
|
1666
1667
|
} else {
|
|
1667
1668
|
status = "unknown";
|
|
1668
1669
|
}
|
|
1669
|
-
const
|
|
1670
|
+
const durationMs2 = explicitDurationMs ?? (startedAt !== void 0 && endedAt !== void 0 && Number.isFinite(startedAt) && Number.isFinite(endedAt) && endedAt >= startedAt ? endedAt - startedAt : void 0);
|
|
1670
1671
|
return {
|
|
1671
1672
|
runId: resolvedRunId,
|
|
1672
1673
|
name,
|
|
1673
1674
|
status,
|
|
1674
1675
|
startedAt,
|
|
1675
1676
|
endedAt,
|
|
1676
|
-
durationMs,
|
|
1677
|
+
durationMs: durationMs2,
|
|
1677
1678
|
eventCount: parsedTrace.sourceEventCount,
|
|
1678
1679
|
filePath,
|
|
1679
1680
|
fileSize: stats.size,
|
|
@@ -1694,7 +1695,7 @@ function buildRunSummary(events) {
|
|
|
1694
1695
|
const runId = started?.runId ?? events.find((e) => typeof e.runId === "string")?.runId ?? "unknown-run";
|
|
1695
1696
|
const name = typeof started?.name === "string" && started.name.trim() !== "" ? started.name : void 0;
|
|
1696
1697
|
const status = lastCompleted ? lastCompleted.status : started ? "running" : "unknown";
|
|
1697
|
-
const
|
|
1698
|
+
const durationMs2 = lastCompleted && isFiniteNumber(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
1698
1699
|
started && isFiniteNumber(started.startTime) ? started.startTime : void 0;
|
|
1699
1700
|
const steps = /* @__PURE__ */ new Map();
|
|
1700
1701
|
for (const e of events) {
|
|
@@ -1784,7 +1785,7 @@ function buildRunSummary(events) {
|
|
|
1784
1785
|
runId,
|
|
1785
1786
|
name,
|
|
1786
1787
|
status,
|
|
1787
|
-
durationMs,
|
|
1788
|
+
durationMs: durationMs2,
|
|
1788
1789
|
totalSteps,
|
|
1789
1790
|
llmSteps,
|
|
1790
1791
|
toolSteps,
|
|
@@ -3058,12 +3059,12 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
3058
3059
|
handoffs.filter((edge) => edge.confidence === "explicit").map((edge) => edge.from)
|
|
3059
3060
|
);
|
|
3060
3061
|
const ordered = [...runs].sort(compareRuns);
|
|
3061
|
-
const
|
|
3062
|
+
const path16 = [];
|
|
3062
3063
|
const visited = /* @__PURE__ */ new Set();
|
|
3063
3064
|
const pushRun = (run, confidence, source) => {
|
|
3064
3065
|
if (visited.has(run.runId)) return;
|
|
3065
3066
|
visited.add(run.runId);
|
|
3066
|
-
|
|
3067
|
+
path16.push({
|
|
3067
3068
|
runId: run.runId,
|
|
3068
3069
|
name: run.name,
|
|
3069
3070
|
startedAt: run.startedAt,
|
|
@@ -3088,7 +3089,7 @@ function buildCriticalPath(runs, handoffs) {
|
|
|
3088
3089
|
const confidence = explicitTargets.has(run.runId) || explicitSources.has(run.runId) ? "explicit" : "correlated";
|
|
3089
3090
|
pushRun(run, confidence, confidence === "explicit" ? "manual" : "inferred");
|
|
3090
3091
|
}
|
|
3091
|
-
return
|
|
3092
|
+
return path16;
|
|
3092
3093
|
}
|
|
3093
3094
|
function metaRunIdMatches(run, token, runById) {
|
|
3094
3095
|
const meta = extractSessionWorkflowMetadata(run.metadata);
|
|
@@ -3215,10 +3216,10 @@ function statusIcon(status) {
|
|
|
3215
3216
|
if (status === "running") return "\u23F3";
|
|
3216
3217
|
return "?";
|
|
3217
3218
|
}
|
|
3218
|
-
function durationCell(status,
|
|
3219
|
+
function durationCell(status, durationMs2) {
|
|
3219
3220
|
if (status === "running" || status === "unknown") return "-";
|
|
3220
|
-
if (
|
|
3221
|
-
return formatDuration2(
|
|
3221
|
+
if (durationMs2 !== void 0 && Number.isFinite(durationMs2)) {
|
|
3222
|
+
return formatDuration2(durationMs2);
|
|
3222
3223
|
}
|
|
3223
3224
|
return "-";
|
|
3224
3225
|
}
|
|
@@ -3875,7 +3876,7 @@ var TreeBuilder = class {
|
|
|
3875
3876
|
const startedAt = sorted.length > 0 ? sorted[0].timestamp : void 0;
|
|
3876
3877
|
const endedAt = sorted.length > 0 ? sorted[sorted.length - 1].timestamp : void 0;
|
|
3877
3878
|
const status = computeRunStatus(sorted);
|
|
3878
|
-
const
|
|
3879
|
+
const durationMs2 = startedAt !== void 0 && endedAt !== void 0 && Number.isFinite(startedAt) && Number.isFinite(endedAt) && endedAt >= startedAt && status !== "running" ? endedAt - startedAt : void 0;
|
|
3879
3880
|
const name = sorted.find((e) => e.kind === "RUN")?.name;
|
|
3880
3881
|
out.push({
|
|
3881
3882
|
runId,
|
|
@@ -3883,7 +3884,7 @@ var TreeBuilder = class {
|
|
|
3883
3884
|
status,
|
|
3884
3885
|
startedAt,
|
|
3885
3886
|
endedAt: status === "running" ? void 0 : endedAt,
|
|
3886
|
-
durationMs,
|
|
3887
|
+
durationMs: durationMs2,
|
|
3887
3888
|
children: roots,
|
|
3888
3889
|
metadata: {
|
|
3889
3890
|
totalEvents: sorted.length,
|
|
@@ -4026,7 +4027,7 @@ function findReaderByFormat(format, readers) {
|
|
|
4026
4027
|
}
|
|
4027
4028
|
async function jsonlFilesInDirectory(dirPath) {
|
|
4028
4029
|
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
4029
|
-
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) =>
|
|
4030
|
+
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path14.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
|
|
4030
4031
|
}
|
|
4031
4032
|
async function resolveInput(input3) {
|
|
4032
4033
|
const cached = resolvedInputCache.get(input3);
|
|
@@ -4578,9 +4579,9 @@ function mapOpenInferenceSpan(span, index, version2) {
|
|
|
4578
4579
|
if (endedAt !== void 0) {
|
|
4579
4580
|
event.endedAt = endedAt;
|
|
4580
4581
|
}
|
|
4581
|
-
const
|
|
4582
|
-
if (
|
|
4583
|
-
event.durationMs =
|
|
4582
|
+
const durationMs2 = durationBetweenIso(startedAt, endedAt);
|
|
4583
|
+
if (durationMs2 !== void 0) {
|
|
4584
|
+
event.durationMs = durationMs2;
|
|
4584
4585
|
}
|
|
4585
4586
|
if (tokenUsage !== void 0) {
|
|
4586
4587
|
event.tokenUsage = tokenUsage;
|
|
@@ -5165,9 +5166,9 @@ function mapOtlpSpan(context) {
|
|
|
5165
5166
|
if (endedAt !== void 0) {
|
|
5166
5167
|
event.endedAt = endedAt;
|
|
5167
5168
|
}
|
|
5168
|
-
const
|
|
5169
|
-
if (
|
|
5170
|
-
event.durationMs =
|
|
5169
|
+
const durationMs2 = durationBetweenIso(startedAt, endedAt);
|
|
5170
|
+
if (durationMs2 !== void 0) {
|
|
5171
|
+
event.durationMs = durationMs2;
|
|
5171
5172
|
}
|
|
5172
5173
|
if (tokenUsage !== void 0) {
|
|
5173
5174
|
event.tokenUsage = tokenUsage;
|
|
@@ -6172,13 +6173,13 @@ var EventNormalizer = class {
|
|
|
6172
6173
|
const timestampMissing = parsedTs === void 0;
|
|
6173
6174
|
const parentIdKey = cfg.parentIdKey;
|
|
6174
6175
|
const parentId = parentIdKey ? safeString(raw[parentIdKey]) : void 0;
|
|
6175
|
-
let
|
|
6176
|
+
let durationMs2;
|
|
6176
6177
|
const durationKey = cfg.durationKey;
|
|
6177
6178
|
if (durationKey) {
|
|
6178
6179
|
const v = raw[durationKey];
|
|
6179
|
-
if (isFiniteNumber2(v))
|
|
6180
|
+
if (isFiniteNumber2(v)) durationMs2 = v;
|
|
6180
6181
|
} else if (isFiniteNumber2(raw.durationMs)) {
|
|
6181
|
-
|
|
6182
|
+
durationMs2 = raw.durationMs;
|
|
6182
6183
|
}
|
|
6183
6184
|
let status;
|
|
6184
6185
|
const statusKey = cfg.statusKey;
|
|
@@ -6225,7 +6226,7 @@ var EventNormalizer = class {
|
|
|
6225
6226
|
kind,
|
|
6226
6227
|
timestamp,
|
|
6227
6228
|
...status ? { status } : {},
|
|
6228
|
-
...
|
|
6229
|
+
...durationMs2 !== void 0 ? { durationMs: durationMs2 } : {},
|
|
6229
6230
|
...Object.keys(attributes).length > 0 ? { attributes } : {},
|
|
6230
6231
|
confidence,
|
|
6231
6232
|
source: {
|
|
@@ -7813,7 +7814,7 @@ function manualTraceEventsToRunTree(events) {
|
|
|
7813
7814
|
}
|
|
7814
7815
|
const startedAt = started.startTime;
|
|
7815
7816
|
const endedAt = lastCompleted !== void 0 && runStatus !== "running" ? lastCompleted.endTime : void 0;
|
|
7816
|
-
const
|
|
7817
|
+
const durationMs2 = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
7817
7818
|
const steps = /* @__PURE__ */ new Map();
|
|
7818
7819
|
for (const e of events) {
|
|
7819
7820
|
if (e.event !== "step_started") continue;
|
|
@@ -7902,7 +7903,7 @@ function manualTraceEventsToRunTree(events) {
|
|
|
7902
7903
|
status: runStatus,
|
|
7903
7904
|
startedAt,
|
|
7904
7905
|
endedAt,
|
|
7905
|
-
durationMs,
|
|
7906
|
+
durationMs: durationMs2,
|
|
7906
7907
|
children: roots,
|
|
7907
7908
|
metadata: {
|
|
7908
7909
|
totalEvents: inspectNodes.size,
|
|
@@ -8152,9 +8153,9 @@ Trace directory: ${traceDir}`);
|
|
|
8152
8153
|
if (validation !== void 0 && !validation.ok) {
|
|
8153
8154
|
process.exitCode = 1;
|
|
8154
8155
|
}
|
|
8155
|
-
const outPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
8156
|
+
const outPath = options.output !== void 0 && options.output.trim() !== "" ? path14.resolve(options.output.trim()) : void 0;
|
|
8156
8157
|
if (outPath !== void 0) {
|
|
8157
|
-
await mkdir(
|
|
8158
|
+
await mkdir(path14.dirname(outPath), { recursive: true });
|
|
8158
8159
|
await writeFile(outPath, result.content, "utf-8");
|
|
8159
8160
|
const vlabel = validation !== void 0 ? validation.ok ? "ok" : "failed" : "skipped";
|
|
8160
8161
|
console.log(`Wrote ${result.fileExtension} export to ${outPath} (validation: ${vlabel})`);
|
|
@@ -8215,7 +8216,7 @@ function manualTraceEventsToComparableRun(events) {
|
|
|
8215
8216
|
let runStatus;
|
|
8216
8217
|
if (lastCompleted === void 0) runStatus = "running";
|
|
8217
8218
|
else runStatus = lastCompleted.status;
|
|
8218
|
-
const
|
|
8219
|
+
const durationMs2 = lastCompleted !== void 0 && Number.isFinite(lastCompleted.durationMs) ? lastCompleted.durationMs : void 0;
|
|
8219
8220
|
const steps = /* @__PURE__ */ new Map();
|
|
8220
8221
|
let order = 0;
|
|
8221
8222
|
for (const e of events) {
|
|
@@ -8286,7 +8287,7 @@ function manualTraceEventsToComparableRun(events) {
|
|
|
8286
8287
|
runId,
|
|
8287
8288
|
name: rs.name,
|
|
8288
8289
|
status: runStatus,
|
|
8289
|
-
durationMs,
|
|
8290
|
+
durationMs: durationMs2,
|
|
8290
8291
|
steps: roots
|
|
8291
8292
|
};
|
|
8292
8293
|
}
|
|
@@ -8331,13 +8332,13 @@ function pairSteps(left, right) {
|
|
|
8331
8332
|
return pairs;
|
|
8332
8333
|
}
|
|
8333
8334
|
function compareLeafSteps(L, R, segments, opts, out) {
|
|
8334
|
-
const
|
|
8335
|
+
const path16 = buildPath(segments);
|
|
8335
8336
|
if (L.name !== R.name) {
|
|
8336
8337
|
out.push({
|
|
8337
8338
|
kind: "structure",
|
|
8338
8339
|
severity: "warning",
|
|
8339
8340
|
message: "Step name differs",
|
|
8340
|
-
path:
|
|
8341
|
+
path: path16,
|
|
8341
8342
|
left: L.name,
|
|
8342
8343
|
right: R.name
|
|
8343
8344
|
});
|
|
@@ -8347,7 +8348,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8347
8348
|
kind: "step-type",
|
|
8348
8349
|
severity: "warning",
|
|
8349
8350
|
message: "Step type differs",
|
|
8350
|
-
path:
|
|
8351
|
+
path: path16,
|
|
8351
8352
|
left: L.type,
|
|
8352
8353
|
right: R.type
|
|
8353
8354
|
});
|
|
@@ -8357,7 +8358,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8357
8358
|
kind: "step-status",
|
|
8358
8359
|
severity: "warning",
|
|
8359
8360
|
message: "Step status differs",
|
|
8360
|
-
path:
|
|
8361
|
+
path: path16,
|
|
8361
8362
|
left: L.status,
|
|
8362
8363
|
right: R.status
|
|
8363
8364
|
});
|
|
@@ -8369,7 +8370,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8369
8370
|
kind: "error",
|
|
8370
8371
|
severity: "error",
|
|
8371
8372
|
message: "Step error message differs",
|
|
8372
|
-
path:
|
|
8373
|
+
path: path16,
|
|
8373
8374
|
left: le || void 0,
|
|
8374
8375
|
right: re || void 0
|
|
8375
8376
|
});
|
|
@@ -8387,7 +8388,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8387
8388
|
kind: "duration",
|
|
8388
8389
|
severity: "info",
|
|
8389
8390
|
message: "Step duration differs",
|
|
8390
|
-
path:
|
|
8391
|
+
path: path16,
|
|
8391
8392
|
left: ld,
|
|
8392
8393
|
right: rd
|
|
8393
8394
|
});
|
|
@@ -8400,7 +8401,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8400
8401
|
kind: "metadata",
|
|
8401
8402
|
severity: "info",
|
|
8402
8403
|
message: "Step metadata differs",
|
|
8403
|
-
path:
|
|
8404
|
+
path: path16,
|
|
8404
8405
|
left: L.metadata,
|
|
8405
8406
|
right: R.metadata
|
|
8406
8407
|
});
|
|
@@ -8412,7 +8413,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
8412
8413
|
kind: "output",
|
|
8413
8414
|
severity: "info",
|
|
8414
8415
|
message: "Output preview differs",
|
|
8415
|
-
path:
|
|
8416
|
+
path: path16,
|
|
8416
8417
|
left: L.outputPreview,
|
|
8417
8418
|
right: R.outputPreview
|
|
8418
8419
|
});
|
|
@@ -8572,11 +8573,11 @@ function diffRuns(left, right, options) {
|
|
|
8572
8573
|
}
|
|
8573
8574
|
|
|
8574
8575
|
// packages/core/src/diff/renderer.ts
|
|
8575
|
-
function formatPath(
|
|
8576
|
-
if (
|
|
8576
|
+
function formatPath(path16) {
|
|
8577
|
+
if (path16 === void 0 || path16.path.length === 0) {
|
|
8577
8578
|
return "(run)";
|
|
8578
8579
|
}
|
|
8579
|
-
return
|
|
8580
|
+
return path16.path.map((s) => s.name).join(" > ");
|
|
8580
8581
|
}
|
|
8581
8582
|
function formatValue(v, verbose) {
|
|
8582
8583
|
if (v === void 0) return "(undefined)";
|
|
@@ -9205,9 +9206,9 @@ async function reportCommand(runId, options = {}) {
|
|
|
9205
9206
|
redactionProfile,
|
|
9206
9207
|
correlation: !options.noCorrelation
|
|
9207
9208
|
});
|
|
9208
|
-
const outPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
9209
|
+
const outPath = options.output !== void 0 && options.output.trim() !== "" ? path14.resolve(options.output.trim()) : void 0;
|
|
9209
9210
|
if (outPath !== void 0) {
|
|
9210
|
-
await mkdir(
|
|
9211
|
+
await mkdir(path14.dirname(outPath), { recursive: true });
|
|
9211
9212
|
await writeFile(outPath, result.content, "utf-8");
|
|
9212
9213
|
console.log(`Wrote ${result.fileExtension} report to ${outPath}`);
|
|
9213
9214
|
}
|
|
@@ -9454,17 +9455,17 @@ function applyRule(rule, value, replacement) {
|
|
|
9454
9455
|
}
|
|
9455
9456
|
return value;
|
|
9456
9457
|
}
|
|
9457
|
-
function childPath(
|
|
9458
|
+
function childPath(path16, key) {
|
|
9458
9459
|
if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)) {
|
|
9459
|
-
return
|
|
9460
|
+
return path16 ? `${path16}.${key}` : key;
|
|
9460
9461
|
}
|
|
9461
|
-
return `${
|
|
9462
|
+
return `${path16 || "$"}[${JSON.stringify(key)}]`;
|
|
9462
9463
|
}
|
|
9463
|
-
function indexPath(
|
|
9464
|
-
return `${
|
|
9464
|
+
function indexPath(path16, index) {
|
|
9465
|
+
return `${path16 || "$"}[${index}]`;
|
|
9465
9466
|
}
|
|
9466
|
-
function makeFinding(
|
|
9467
|
-
return preview === void 0 ? { path:
|
|
9467
|
+
function makeFinding(path16, detector, action, matchKind, severity = "warning", preview) {
|
|
9468
|
+
return preview === void 0 ? { path: path16, detector, action, severity, matchKind } : { path: path16, detector, action, severity, matchKind, preview };
|
|
9468
9469
|
}
|
|
9469
9470
|
function createRedactionProfile(profile = "local") {
|
|
9470
9471
|
switch (profile) {
|
|
@@ -9533,11 +9534,11 @@ var Redactor2 = class {
|
|
|
9533
9534
|
#recordFinding(state, finding) {
|
|
9534
9535
|
if (this.#collectFindings) state.findings.push(finding);
|
|
9535
9536
|
}
|
|
9536
|
-
#redactValue(value, key,
|
|
9537
|
+
#redactValue(value, key, path16, depth, state) {
|
|
9537
9538
|
if (depth > this.#maxDepth) {
|
|
9538
9539
|
this.#recordFinding(
|
|
9539
9540
|
state,
|
|
9540
|
-
makeFinding(
|
|
9541
|
+
makeFinding(path16, "structure.maxDepth", "truncate", "value", "warning")
|
|
9541
9542
|
);
|
|
9542
9543
|
return "[Truncated]";
|
|
9543
9544
|
}
|
|
@@ -9546,19 +9547,19 @@ var Redactor2 = class {
|
|
|
9546
9547
|
if (rule) {
|
|
9547
9548
|
this.#recordFinding(
|
|
9548
9549
|
state,
|
|
9549
|
-
makeFinding(
|
|
9550
|
+
makeFinding(path16, `key.${rule.key}`, actionForRule(rule), "key", "warning")
|
|
9550
9551
|
);
|
|
9551
9552
|
return applyRule(rule, value, this.#replacement);
|
|
9552
9553
|
}
|
|
9553
9554
|
}
|
|
9554
9555
|
for (const detector of this.#detectors) {
|
|
9555
|
-
const detections = detector.detect({ path:
|
|
9556
|
+
const detections = detector.detect({ path: path16, key, value });
|
|
9556
9557
|
for (const detection of detections) {
|
|
9557
9558
|
const action = detection.action ?? "replace";
|
|
9558
9559
|
this.#recordFinding(
|
|
9559
9560
|
state,
|
|
9560
9561
|
makeFinding(
|
|
9561
|
-
|
|
9562
|
+
path16,
|
|
9562
9563
|
detector.id,
|
|
9563
9564
|
action,
|
|
9564
9565
|
detection.matchKind ?? detector.matchKind ?? "custom",
|
|
@@ -9576,7 +9577,7 @@ var Redactor2 = class {
|
|
|
9576
9577
|
const out = [];
|
|
9577
9578
|
state.seen.set(value, out);
|
|
9578
9579
|
value.forEach((item, index) => {
|
|
9579
|
-
out[index] = this.#redactValue(item, void 0, indexPath(
|
|
9580
|
+
out[index] = this.#redactValue(item, void 0, indexPath(path16, index), depth + 1, state);
|
|
9580
9581
|
});
|
|
9581
9582
|
return out;
|
|
9582
9583
|
}
|
|
@@ -9588,7 +9589,7 @@ var Redactor2 = class {
|
|
|
9588
9589
|
out[entryKey] = this.#redactValue(
|
|
9589
9590
|
entryValue,
|
|
9590
9591
|
entryKey,
|
|
9591
|
-
childPath(
|
|
9592
|
+
childPath(path16 === "$" ? "" : path16, entryKey),
|
|
9592
9593
|
depth + 1,
|
|
9593
9594
|
state
|
|
9594
9595
|
);
|
|
@@ -10035,14 +10036,14 @@ function uniqueSorted(values) {
|
|
|
10035
10036
|
return [...new Set(values)].sort();
|
|
10036
10037
|
}
|
|
10037
10038
|
function isWithinDirectory(child, parent) {
|
|
10038
|
-
const relative =
|
|
10039
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
10039
|
+
const relative = path14.relative(parent, child);
|
|
10040
|
+
return relative === "" || !relative.startsWith("..") && !path14.isAbsolute(relative);
|
|
10040
10041
|
}
|
|
10041
10042
|
async function resolveOutputPath(inputPath, output2, force) {
|
|
10042
10043
|
if (output2 === void 0 || output2.trim() === "") return void 0;
|
|
10043
|
-
const inputAbs =
|
|
10044
|
-
const outputAbs =
|
|
10045
|
-
const inputDir =
|
|
10044
|
+
const inputAbs = path14.resolve(inputPath);
|
|
10045
|
+
const outputAbs = path14.resolve(output2.trim());
|
|
10046
|
+
const inputDir = path14.dirname(inputAbs);
|
|
10046
10047
|
if (!isWithinDirectory(outputAbs, inputDir)) {
|
|
10047
10048
|
throw new Error("Refusing to write migrated output outside the input directory.");
|
|
10048
10049
|
}
|
|
@@ -10163,7 +10164,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
10163
10164
|
process.exitCode = 1;
|
|
10164
10165
|
return;
|
|
10165
10166
|
}
|
|
10166
|
-
const inputPath =
|
|
10167
|
+
const inputPath = path14.resolve(input3.trim());
|
|
10167
10168
|
const dryRun = options.dryRun === true;
|
|
10168
10169
|
if (!dryRun && (options.output === void 0 || options.output.trim() === "")) {
|
|
10169
10170
|
console.error("migrate requires --dry-run or --output <path>.");
|
|
@@ -10182,7 +10183,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
10182
10183
|
);
|
|
10183
10184
|
const result = await buildMigration(inputPath, outputPath);
|
|
10184
10185
|
if (!dryRun && outputPath !== void 0) {
|
|
10185
|
-
await mkdir(
|
|
10186
|
+
await mkdir(path14.dirname(outputPath), { recursive: true });
|
|
10186
10187
|
await writeFile(outputPath, result.content, "utf-8");
|
|
10187
10188
|
}
|
|
10188
10189
|
printSummary2(result, dryRun);
|
|
@@ -10460,7 +10461,7 @@ function stripPrefix(name, prefixes) {
|
|
|
10460
10461
|
}
|
|
10461
10462
|
return name;
|
|
10462
10463
|
}
|
|
10463
|
-
function eventEvidence(event,
|
|
10464
|
+
function eventEvidence(event, path16) {
|
|
10464
10465
|
return {
|
|
10465
10466
|
runId: event.runId,
|
|
10466
10467
|
eventId: event.eventId,
|
|
@@ -10470,7 +10471,7 @@ function eventEvidence(event, path15) {
|
|
|
10470
10471
|
kind: event.kind,
|
|
10471
10472
|
name: event.name,
|
|
10472
10473
|
status: event.status,
|
|
10473
|
-
...
|
|
10474
|
+
...path16 ? { path: path16 } : {}
|
|
10474
10475
|
};
|
|
10475
10476
|
}
|
|
10476
10477
|
function runEvidence(run) {
|
|
@@ -10533,9 +10534,9 @@ function eventEndMs(event) {
|
|
|
10533
10534
|
function normalizedKey(value) {
|
|
10534
10535
|
return value.toLowerCase().replace(/[^a-z0-9_]/g, "");
|
|
10535
10536
|
}
|
|
10536
|
-
function lastPathSegment(
|
|
10537
|
-
const parts =
|
|
10538
|
-
return parts[parts.length - 1] ??
|
|
10537
|
+
function lastPathSegment(path16) {
|
|
10538
|
+
const parts = path16.split(".");
|
|
10539
|
+
return parts[parts.length - 1] ?? path16;
|
|
10539
10540
|
}
|
|
10540
10541
|
function valueType(value) {
|
|
10541
10542
|
if (Array.isArray(value)) return "array";
|
|
@@ -10549,12 +10550,12 @@ function serializedByteLength(value) {
|
|
|
10549
10550
|
return void 0;
|
|
10550
10551
|
}
|
|
10551
10552
|
}
|
|
10552
|
-
function pushValueEntries(entries, event, value,
|
|
10553
|
-
entries.push({ event, path:
|
|
10553
|
+
function pushValueEntries(entries, event, value, path16, key, depth = 0) {
|
|
10554
|
+
entries.push({ event, path: path16, key, value });
|
|
10554
10555
|
if (depth >= 8) return;
|
|
10555
10556
|
if (Array.isArray(value)) {
|
|
10556
10557
|
for (const [index, item] of value.entries()) {
|
|
10557
|
-
pushValueEntries(entries, event, item, `${
|
|
10558
|
+
pushValueEntries(entries, event, item, `${path16}.${index}`, String(index), depth + 1);
|
|
10558
10559
|
}
|
|
10559
10560
|
return;
|
|
10560
10561
|
}
|
|
@@ -10564,7 +10565,7 @@ function pushValueEntries(entries, event, value, path15, key, depth = 0) {
|
|
|
10564
10565
|
entries,
|
|
10565
10566
|
event,
|
|
10566
10567
|
value[nestedKey],
|
|
10567
|
-
`${
|
|
10568
|
+
`${path16}.${nestedKey}`,
|
|
10568
10569
|
nestedKey,
|
|
10569
10570
|
depth + 1
|
|
10570
10571
|
);
|
|
@@ -10645,9 +10646,9 @@ function eventDurationMs(event) {
|
|
|
10645
10646
|
}
|
|
10646
10647
|
function treeShape(nodes) {
|
|
10647
10648
|
const lines = [];
|
|
10648
|
-
const visit = (node,
|
|
10649
|
-
lines.push(`${
|
|
10650
|
-
node.children.forEach((child, index) => visit(child, `${
|
|
10649
|
+
const visit = (node, path16) => {
|
|
10650
|
+
lines.push(`${path16}:${node.event.kind}:${node.event.name}:${node.event.status ?? "unknown"}`);
|
|
10651
|
+
node.children.forEach((child, index) => visit(child, `${path16}.${index}`));
|
|
10651
10652
|
};
|
|
10652
10653
|
nodes.forEach((node, index) => visit(node, String(index)));
|
|
10653
10654
|
return lines;
|
|
@@ -10696,9 +10697,9 @@ function retrievalShape(context) {
|
|
|
10696
10697
|
function guardrailShape(context) {
|
|
10697
10698
|
return guardrailEvents(context).map((event) => signalName(event, ["guardrailName", "guardrail", "guardrailId"], ["guardrail:"])).sort((a, b) => a.localeCompare(b));
|
|
10698
10699
|
}
|
|
10699
|
-
function firstEvidenceForKind(context, kind,
|
|
10700
|
+
function firstEvidenceForKind(context, kind, path16) {
|
|
10700
10701
|
const event = context.events.find((candidate) => candidate.kind === kind);
|
|
10701
|
-
return event ? [eventEvidence(event,
|
|
10702
|
+
return event ? [eventEvidence(event, path16)] : runEvidence(context.selectedRun);
|
|
10702
10703
|
}
|
|
10703
10704
|
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
10704
10705
|
return failFinding("baseline.regression", message, evidence, expected, actual);
|
|
@@ -10946,13 +10947,13 @@ function createStructureCycleRule() {
|
|
|
10946
10947
|
const seenCycles = /* @__PURE__ */ new Set();
|
|
10947
10948
|
const findings = [];
|
|
10948
10949
|
for (const event of [...context.events].sort((a, b) => a.eventId.localeCompare(b.eventId))) {
|
|
10949
|
-
const
|
|
10950
|
+
const path16 = [];
|
|
10950
10951
|
const seenAt = /* @__PURE__ */ new Map();
|
|
10951
10952
|
let current = event;
|
|
10952
10953
|
while (current) {
|
|
10953
10954
|
const existing = seenAt.get(current.eventId);
|
|
10954
10955
|
if (existing !== void 0) {
|
|
10955
|
-
const cycle =
|
|
10956
|
+
const cycle = path16.slice(existing);
|
|
10956
10957
|
const key = cycle.map((item) => item.eventId).sort().join("\0");
|
|
10957
10958
|
if (!seenCycles.has(key)) {
|
|
10958
10959
|
seenCycles.add(key);
|
|
@@ -10968,8 +10969,8 @@ function createStructureCycleRule() {
|
|
|
10968
10969
|
}
|
|
10969
10970
|
break;
|
|
10970
10971
|
}
|
|
10971
|
-
seenAt.set(current.eventId,
|
|
10972
|
-
|
|
10972
|
+
seenAt.set(current.eventId, path16.length);
|
|
10973
|
+
path16.push(current);
|
|
10973
10974
|
current = current.parentId ? byId.get(current.parentId) : void 0;
|
|
10974
10975
|
}
|
|
10975
10976
|
}
|
|
@@ -11180,7 +11181,7 @@ function createSafetyRawContentRule(options = {}) {
|
|
|
11180
11181
|
}
|
|
11181
11182
|
function createSafetySecretPatternRule(options = {}) {
|
|
11182
11183
|
const patterns = options.patterns ?? DEFAULT_SECRET_PATTERNS;
|
|
11183
|
-
const
|
|
11184
|
+
const maxStringLength2 = options.maxStringLength ?? 4096;
|
|
11184
11185
|
return {
|
|
11185
11186
|
id: "safety.secretPattern",
|
|
11186
11187
|
category: "safety",
|
|
@@ -11190,7 +11191,7 @@ function createSafetySecretPatternRule(options = {}) {
|
|
|
11190
11191
|
for (const event of context.events) {
|
|
11191
11192
|
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
11192
11193
|
if (typeof entry.value !== "string") continue;
|
|
11193
|
-
const sample = entry.value.slice(0,
|
|
11194
|
+
const sample = entry.value.slice(0, maxStringLength2);
|
|
11194
11195
|
for (const pattern of patterns) {
|
|
11195
11196
|
pattern.pattern.lastIndex = 0;
|
|
11196
11197
|
if (!pattern.pattern.test(sample)) continue;
|
|
@@ -11446,6 +11447,621 @@ function runTraceChecks(input3, options = {}) {
|
|
|
11446
11447
|
diagnostics
|
|
11447
11448
|
};
|
|
11448
11449
|
}
|
|
11450
|
+
var ALL_RULES = [
|
|
11451
|
+
"circuit.same-tool-repetition",
|
|
11452
|
+
"circuit.same-args-repetition",
|
|
11453
|
+
"circuit.max-loop-iterations",
|
|
11454
|
+
"circuit.max-retries",
|
|
11455
|
+
"circuit.tool-timeout",
|
|
11456
|
+
"circuit.runaway-llm-loop",
|
|
11457
|
+
"circuit.excessive-branch-width"
|
|
11458
|
+
];
|
|
11459
|
+
function closed(ruleId, message) {
|
|
11460
|
+
return { ruleId, status: "closed", severity: "info", message, evidence: [] };
|
|
11461
|
+
}
|
|
11462
|
+
function open2(ruleId, message, evidence, severity = "error") {
|
|
11463
|
+
return {
|
|
11464
|
+
ruleId,
|
|
11465
|
+
status: severity === "warning" ? "warn" : "open",
|
|
11466
|
+
severity,
|
|
11467
|
+
message,
|
|
11468
|
+
evidence
|
|
11469
|
+
};
|
|
11470
|
+
}
|
|
11471
|
+
function isToolEvent(event) {
|
|
11472
|
+
const name = event.name.toLowerCase();
|
|
11473
|
+
return event.kind === "tool" || name.startsWith("tool:") || name.startsWith("function:") || name.includes(".tool.") || name.startsWith("mcp:");
|
|
11474
|
+
}
|
|
11475
|
+
function isLlmEvent(event) {
|
|
11476
|
+
const name = event.name.toLowerCase();
|
|
11477
|
+
return event.kind === "llm" || name.startsWith("llm:") || name.includes(".llm.") || name.includes("generation");
|
|
11478
|
+
}
|
|
11479
|
+
function toolLabel(event) {
|
|
11480
|
+
const attrs = event.attributes ?? {};
|
|
11481
|
+
const fromAttr = attrs.toolName ?? attrs.tool ?? attrs.function;
|
|
11482
|
+
if (typeof fromAttr === "string" && fromAttr.length > 0) return fromAttr;
|
|
11483
|
+
return event.name.replace(/^(tool:|function:|mcp:)/i, "");
|
|
11484
|
+
}
|
|
11485
|
+
function argsHash(toolName2, args) {
|
|
11486
|
+
return createHash("sha256").update(`${toolName2}:${JSON.stringify(args ?? null)}`).digest("hex").slice(0, 16);
|
|
11487
|
+
}
|
|
11488
|
+
function toolArgs(event) {
|
|
11489
|
+
const attrs = event.attributes ?? {};
|
|
11490
|
+
return attrs.arguments ?? attrs.args ?? attrs.input ?? attrs.parameters;
|
|
11491
|
+
}
|
|
11492
|
+
function durationMs(event) {
|
|
11493
|
+
if (typeof event.durationMs === "number") return event.durationMs;
|
|
11494
|
+
const attrs = event.attributes ?? {};
|
|
11495
|
+
const fromAttr = attrs.durationMs ?? attrs.duration;
|
|
11496
|
+
return typeof fromAttr === "number" ? fromAttr : void 0;
|
|
11497
|
+
}
|
|
11498
|
+
function attemptNumber(event) {
|
|
11499
|
+
const attrs = event.attributes ?? {};
|
|
11500
|
+
const value = attrs.attempt ?? attrs.retryAttempt ?? attrs.retryCount;
|
|
11501
|
+
return typeof value === "number" ? value : void 0;
|
|
11502
|
+
}
|
|
11503
|
+
function evaluateSameToolRepetition(events, maxRepeats) {
|
|
11504
|
+
const ruleId = "circuit.same-tool-repetition";
|
|
11505
|
+
const counts = /* @__PURE__ */ new Map();
|
|
11506
|
+
for (const event of events.filter(isToolEvent)) {
|
|
11507
|
+
const label = toolLabel(event);
|
|
11508
|
+
counts.set(label, (counts.get(label) ?? 0) + 1);
|
|
11509
|
+
}
|
|
11510
|
+
const evidence = [];
|
|
11511
|
+
for (const [toolName2, count] of counts) {
|
|
11512
|
+
if (count > maxRepeats) {
|
|
11513
|
+
evidence.push({ ruleId, toolName: toolName2, count, threshold: maxRepeats });
|
|
11514
|
+
}
|
|
11515
|
+
}
|
|
11516
|
+
if (evidence.length === 0) {
|
|
11517
|
+
return closed(ruleId, "Tool repetition within threshold.");
|
|
11518
|
+
}
|
|
11519
|
+
return open2(ruleId, "Same tool repeated beyond threshold.", evidence);
|
|
11520
|
+
}
|
|
11521
|
+
function evaluateSameArgsRepetition(events, maxRepeats) {
|
|
11522
|
+
const ruleId = "circuit.same-args-repetition";
|
|
11523
|
+
const counts = /* @__PURE__ */ new Map();
|
|
11524
|
+
for (const event of events.filter(isToolEvent)) {
|
|
11525
|
+
const label = toolLabel(event);
|
|
11526
|
+
const hash = argsHash(label, toolArgs(event));
|
|
11527
|
+
const key = `${label}:${hash}`;
|
|
11528
|
+
const current = counts.get(key) ?? { toolName: label, count: 0 };
|
|
11529
|
+
current.count += 1;
|
|
11530
|
+
counts.set(key, current);
|
|
11531
|
+
}
|
|
11532
|
+
const evidence = [];
|
|
11533
|
+
for (const entry of counts.values()) {
|
|
11534
|
+
if (entry.count > maxRepeats) {
|
|
11535
|
+
evidence.push({ ruleId, toolName: entry.toolName, count: entry.count, threshold: maxRepeats });
|
|
11536
|
+
}
|
|
11537
|
+
}
|
|
11538
|
+
if (evidence.length === 0) {
|
|
11539
|
+
return closed(ruleId, "Tool argument repetition within threshold.");
|
|
11540
|
+
}
|
|
11541
|
+
return open2(ruleId, "Same tool arguments repeated beyond threshold.", evidence);
|
|
11542
|
+
}
|
|
11543
|
+
function evaluateMaxLoopIterations(events, maxIterations) {
|
|
11544
|
+
const ruleId = "circuit.max-loop-iterations";
|
|
11545
|
+
const iterationEvents = events.filter((event) => {
|
|
11546
|
+
const attrs = event.attributes ?? {};
|
|
11547
|
+
return typeof attrs.iteration === "number" || event.name.toLowerCase().includes("loop");
|
|
11548
|
+
});
|
|
11549
|
+
const maxSeen = iterationEvents.reduce((max, event) => {
|
|
11550
|
+
const attrs = event.attributes ?? {};
|
|
11551
|
+
const iteration = typeof attrs.iteration === "number" ? attrs.iteration : max;
|
|
11552
|
+
return Math.max(max, iteration);
|
|
11553
|
+
}, iterationEvents.length);
|
|
11554
|
+
if (maxSeen <= maxIterations) {
|
|
11555
|
+
return closed(ruleId, "Loop iterations within threshold.");
|
|
11556
|
+
}
|
|
11557
|
+
return open2(ruleId, "Loop iterations exceeded threshold.", [
|
|
11558
|
+
{ ruleId, count: maxSeen, threshold: maxIterations }
|
|
11559
|
+
]);
|
|
11560
|
+
}
|
|
11561
|
+
function evaluateMaxRetries(events, maxRetries) {
|
|
11562
|
+
const ruleId = "circuit.max-retries";
|
|
11563
|
+
const attempts = events.map(attemptNumber).filter((value) => value !== void 0);
|
|
11564
|
+
const maxAttempt = attempts.length > 0 ? Math.max(...attempts) : 0;
|
|
11565
|
+
if (maxAttempt <= maxRetries) {
|
|
11566
|
+
return closed(ruleId, "Retry count within threshold.");
|
|
11567
|
+
}
|
|
11568
|
+
return open2(ruleId, "Retry count exceeded threshold.", [
|
|
11569
|
+
{ ruleId, count: maxAttempt, threshold: maxRetries }
|
|
11570
|
+
]);
|
|
11571
|
+
}
|
|
11572
|
+
function evaluateToolTimeout(events, maxDurationMs) {
|
|
11573
|
+
const ruleId = "circuit.tool-timeout";
|
|
11574
|
+
const evidence = [];
|
|
11575
|
+
for (const event of events.filter(isToolEvent)) {
|
|
11576
|
+
const duration = durationMs(event);
|
|
11577
|
+
if (duration !== void 0 && duration > maxDurationMs) {
|
|
11578
|
+
evidence.push({
|
|
11579
|
+
ruleId,
|
|
11580
|
+
toolName: toolLabel(event),
|
|
11581
|
+
count: duration,
|
|
11582
|
+
threshold: maxDurationMs,
|
|
11583
|
+
eventId: event.eventId
|
|
11584
|
+
});
|
|
11585
|
+
}
|
|
11586
|
+
}
|
|
11587
|
+
if (evidence.length === 0) {
|
|
11588
|
+
return closed(ruleId, "Tool durations within timeout.");
|
|
11589
|
+
}
|
|
11590
|
+
return open2(ruleId, "Tool call exceeded configured timeout.", evidence, "warning");
|
|
11591
|
+
}
|
|
11592
|
+
function evaluateRunawayLlmLoop(events, maxLlmCalls) {
|
|
11593
|
+
const ruleId = "circuit.runaway-llm-loop";
|
|
11594
|
+
const llmCount = events.filter(isLlmEvent).length;
|
|
11595
|
+
const hasTerminal = events.some((event) => {
|
|
11596
|
+
const status = (event.status ?? event.attributes?.status ?? "").toString().toLowerCase();
|
|
11597
|
+
return status === "ok" || status === "success" || status === "completed";
|
|
11598
|
+
});
|
|
11599
|
+
if (llmCount <= maxLlmCalls || hasTerminal) {
|
|
11600
|
+
return closed(ruleId, "LLM call count within threshold or run completed.");
|
|
11601
|
+
}
|
|
11602
|
+
return open2(ruleId, "Runaway LLM loop detected.", [
|
|
11603
|
+
{ ruleId, count: llmCount, threshold: maxLlmCalls }
|
|
11604
|
+
]);
|
|
11605
|
+
}
|
|
11606
|
+
function evaluateExcessiveBranchWidth(events, maxWidth) {
|
|
11607
|
+
const ruleId = "circuit.excessive-branch-width";
|
|
11608
|
+
const children = /* @__PURE__ */ new Map();
|
|
11609
|
+
for (const event of events) {
|
|
11610
|
+
const parentId = event.parentId;
|
|
11611
|
+
if (!parentId) continue;
|
|
11612
|
+
children.set(parentId, (children.get(parentId) ?? 0) + 1);
|
|
11613
|
+
}
|
|
11614
|
+
const evidence = [];
|
|
11615
|
+
for (const [parentId, count] of children) {
|
|
11616
|
+
if (count > maxWidth) {
|
|
11617
|
+
evidence.push({ ruleId, path: parentId, count, threshold: maxWidth });
|
|
11618
|
+
}
|
|
11619
|
+
}
|
|
11620
|
+
if (evidence.length === 0) {
|
|
11621
|
+
return closed(ruleId, "Branch width within threshold.");
|
|
11622
|
+
}
|
|
11623
|
+
return open2(ruleId, "Excessive parallel branch width detected.", evidence, "warning");
|
|
11624
|
+
}
|
|
11625
|
+
function runRule(ruleId, events, options) {
|
|
11626
|
+
switch (ruleId) {
|
|
11627
|
+
case "circuit.same-tool-repetition":
|
|
11628
|
+
if (options.sameToolRepetition === void 0) return void 0;
|
|
11629
|
+
return evaluateSameToolRepetition(events, options.sameToolRepetition.maxRepeats);
|
|
11630
|
+
case "circuit.same-args-repetition":
|
|
11631
|
+
if (options.sameArgsRepetition === void 0) return void 0;
|
|
11632
|
+
return evaluateSameArgsRepetition(events, options.sameArgsRepetition.maxRepeats);
|
|
11633
|
+
case "circuit.max-loop-iterations":
|
|
11634
|
+
if (options.maxLoopIterations === void 0) return void 0;
|
|
11635
|
+
return evaluateMaxLoopIterations(events, options.maxLoopIterations.maxIterations);
|
|
11636
|
+
case "circuit.max-retries":
|
|
11637
|
+
if (options.maxRetries === void 0) return void 0;
|
|
11638
|
+
return evaluateMaxRetries(events, options.maxRetries.maxRetries);
|
|
11639
|
+
case "circuit.tool-timeout":
|
|
11640
|
+
if (options.toolTimeout === void 0) return void 0;
|
|
11641
|
+
return evaluateToolTimeout(events, options.toolTimeout.maxDurationMs);
|
|
11642
|
+
case "circuit.runaway-llm-loop":
|
|
11643
|
+
if (options.runawayLlmLoop === void 0) return void 0;
|
|
11644
|
+
return evaluateRunawayLlmLoop(events, options.runawayLlmLoop.maxLlmCalls);
|
|
11645
|
+
case "circuit.excessive-branch-width":
|
|
11646
|
+
if (options.excessiveBranchWidth === void 0) return void 0;
|
|
11647
|
+
return evaluateExcessiveBranchWidth(events, options.excessiveBranchWidth.maxWidth);
|
|
11648
|
+
default:
|
|
11649
|
+
return void 0;
|
|
11650
|
+
}
|
|
11651
|
+
}
|
|
11652
|
+
function runCircuits(events, options = {}) {
|
|
11653
|
+
const selected = options.rules ?? ALL_RULES;
|
|
11654
|
+
const results = [];
|
|
11655
|
+
for (const ruleId of selected) {
|
|
11656
|
+
const result = runRule(ruleId, events, options);
|
|
11657
|
+
if (result) results.push(result);
|
|
11658
|
+
}
|
|
11659
|
+
const ok = !results.some((result) => result.status === "open" && result.severity === "error");
|
|
11660
|
+
return { ok, results };
|
|
11661
|
+
}
|
|
11662
|
+
|
|
11663
|
+
// packages/guardrails/src/rules.ts
|
|
11664
|
+
var DEFAULT_INJECTION_PATTERNS = [
|
|
11665
|
+
"ignore previous instructions",
|
|
11666
|
+
"ignore all prior",
|
|
11667
|
+
"disregard your instructions",
|
|
11668
|
+
"system prompt",
|
|
11669
|
+
"you are now",
|
|
11670
|
+
"jailbreak"
|
|
11671
|
+
];
|
|
11672
|
+
function pass(ruleId, message) {
|
|
11673
|
+
return { ruleId, status: "pass", severity: "info", message, evidence: [] };
|
|
11674
|
+
}
|
|
11675
|
+
function fail(ruleId, message, evidence, severity = "error") {
|
|
11676
|
+
return { ruleId, status: severity === "warning" ? "warn" : "fail", severity, message, evidence };
|
|
11677
|
+
}
|
|
11678
|
+
function boundedPreview(value, max = 80) {
|
|
11679
|
+
if (value.length <= max) return value;
|
|
11680
|
+
return `${value.slice(0, max - 3)}...`;
|
|
11681
|
+
}
|
|
11682
|
+
function evaluateBannedPhrase(text, options) {
|
|
11683
|
+
const ruleId = "guardrail.banned-phrase";
|
|
11684
|
+
const haystack = options.caseInsensitive !== false ? text.toLowerCase() : text;
|
|
11685
|
+
const evidence = [];
|
|
11686
|
+
for (const phrase of options.phrases) {
|
|
11687
|
+
const needle = options.caseInsensitive !== false ? phrase.toLowerCase() : phrase;
|
|
11688
|
+
if (needle.length > 0 && haystack.includes(needle)) {
|
|
11689
|
+
evidence.push({ ruleId, match: phrase, preview: boundedPreview(text) });
|
|
11690
|
+
}
|
|
11691
|
+
}
|
|
11692
|
+
if (evidence.length === 0) {
|
|
11693
|
+
return pass(ruleId, "No banned phrases matched.");
|
|
11694
|
+
}
|
|
11695
|
+
return fail(ruleId, `Matched ${evidence.length} banned phrase(s).`, evidence);
|
|
11696
|
+
}
|
|
11697
|
+
var SEVERITY_RANK2 = { info: 0, warning: 1, error: 2 };
|
|
11698
|
+
function evaluatePiiLeak(value, options = {}) {
|
|
11699
|
+
const ruleId = "guardrail.pii-leak";
|
|
11700
|
+
const minSeverity = options.minSeverity ?? "warning";
|
|
11701
|
+
const result = redact(value, { profile: options.profile ?? "share", collectFindings: true });
|
|
11702
|
+
const findings = result.findings.filter(
|
|
11703
|
+
(finding) => SEVERITY_RANK2[finding.severity] >= SEVERITY_RANK2[minSeverity]
|
|
11704
|
+
);
|
|
11705
|
+
if (findings.length === 0) {
|
|
11706
|
+
return pass(ruleId, "No PII-style redaction findings.");
|
|
11707
|
+
}
|
|
11708
|
+
const evidence = findings.map((finding) => ({
|
|
11709
|
+
ruleId,
|
|
11710
|
+
path: finding.path,
|
|
11711
|
+
detector: finding.detector,
|
|
11712
|
+
preview: finding.preview
|
|
11713
|
+
}));
|
|
11714
|
+
return fail(ruleId, `Detected ${findings.length} PII-style finding(s).`, evidence);
|
|
11715
|
+
}
|
|
11716
|
+
function measureDepth(value) {
|
|
11717
|
+
if (value === null || typeof value !== "object") return 0;
|
|
11718
|
+
if (Array.isArray(value)) {
|
|
11719
|
+
return 1 + Math.max(0, ...value.map((item) => measureDepth(item)));
|
|
11720
|
+
}
|
|
11721
|
+
const depths = Object.values(value).map((item) => measureDepth(item));
|
|
11722
|
+
return 1 + (depths.length === 0 ? 0 : Math.max(...depths));
|
|
11723
|
+
}
|
|
11724
|
+
function maxStringLength(value) {
|
|
11725
|
+
if (typeof value === "string") return value.length;
|
|
11726
|
+
if (value === null || typeof value !== "object") return 0;
|
|
11727
|
+
if (Array.isArray(value)) {
|
|
11728
|
+
return Math.max(0, ...value.map((item) => maxStringLength(item)));
|
|
11729
|
+
}
|
|
11730
|
+
return Math.max(0, ...Object.values(value).map((item) => maxStringLength(item)));
|
|
11731
|
+
}
|
|
11732
|
+
function evaluateUnsafeToolArgs(toolName2, toolArgs2, options = {}) {
|
|
11733
|
+
const ruleId = "guardrail.unsafe-tool-args";
|
|
11734
|
+
const blocked = new Set((options.blockedTools ?? []).map((name) => name.toLowerCase()));
|
|
11735
|
+
const evidence = [];
|
|
11736
|
+
if (blocked.has(toolName2.toLowerCase())) {
|
|
11737
|
+
evidence.push({ ruleId, preview: toolName2, match: toolName2 });
|
|
11738
|
+
}
|
|
11739
|
+
const maxDepth = options.maxDepth ?? 12;
|
|
11740
|
+
const depth = measureDepth(toolArgs2);
|
|
11741
|
+
if (depth > maxDepth) {
|
|
11742
|
+
evidence.push({ ruleId, path: "args", preview: `depth=${depth}` });
|
|
11743
|
+
}
|
|
11744
|
+
const maxLen = options.maxStringLength ?? 16384;
|
|
11745
|
+
const longest = maxStringLength(toolArgs2);
|
|
11746
|
+
if (longest > maxLen) {
|
|
11747
|
+
evidence.push({ ruleId, path: "args", preview: `maxStringLength=${longest}` });
|
|
11748
|
+
}
|
|
11749
|
+
if (evidence.length === 0) {
|
|
11750
|
+
return pass(ruleId, "Tool arguments within configured bounds.");
|
|
11751
|
+
}
|
|
11752
|
+
return fail(ruleId, "Unsafe or oversized tool arguments detected.", evidence);
|
|
11753
|
+
}
|
|
11754
|
+
function evaluatePromptInjection(text, options = {}) {
|
|
11755
|
+
const ruleId = "guardrail.prompt-injection";
|
|
11756
|
+
const patterns = options.patterns ?? DEFAULT_INJECTION_PATTERNS;
|
|
11757
|
+
const haystack = text.toLowerCase();
|
|
11758
|
+
const evidence = [];
|
|
11759
|
+
for (const pattern of patterns) {
|
|
11760
|
+
const needle = pattern.toLowerCase();
|
|
11761
|
+
if (needle.length > 0 && haystack.includes(needle)) {
|
|
11762
|
+
evidence.push({ ruleId, match: pattern, preview: boundedPreview(text) });
|
|
11763
|
+
}
|
|
11764
|
+
}
|
|
11765
|
+
if (evidence.length === 0) {
|
|
11766
|
+
return pass(ruleId, "No prompt-injection patterns matched.");
|
|
11767
|
+
}
|
|
11768
|
+
return fail(ruleId, `Matched ${evidence.length} injection pattern(s).`, evidence, "warning");
|
|
11769
|
+
}
|
|
11770
|
+
function validateSchemaField(value, field, path16, evidence) {
|
|
11771
|
+
const ruleId = "guardrail.structured-output";
|
|
11772
|
+
if (field.type) {
|
|
11773
|
+
const actual = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
|
|
11774
|
+
if (actual !== field.type) {
|
|
11775
|
+
evidence.push({ ruleId, path: path16, preview: `expected ${field.type}, got ${actual}` });
|
|
11776
|
+
return;
|
|
11777
|
+
}
|
|
11778
|
+
}
|
|
11779
|
+
if (field.enum && !field.enum.some((item) => Object.is(item, value))) {
|
|
11780
|
+
evidence.push({ ruleId, path: path16, preview: "value not in enum" });
|
|
11781
|
+
}
|
|
11782
|
+
if (field.type === "object" && field.required && value && typeof value === "object" && !Array.isArray(value)) {
|
|
11783
|
+
const record = value;
|
|
11784
|
+
for (const key of field.required) {
|
|
11785
|
+
if (!(key in record)) {
|
|
11786
|
+
evidence.push({ ruleId, path: `${path16}.${key}`, preview: "missing required key" });
|
|
11787
|
+
}
|
|
11788
|
+
}
|
|
11789
|
+
}
|
|
11790
|
+
}
|
|
11791
|
+
function evaluateStructuredOutput(value, options) {
|
|
11792
|
+
const ruleId = "guardrail.structured-output";
|
|
11793
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
11794
|
+
return fail(ruleId, "Structured output must be an object.", [
|
|
11795
|
+
{ ruleId, preview: typeof value }
|
|
11796
|
+
]);
|
|
11797
|
+
}
|
|
11798
|
+
const record = value;
|
|
11799
|
+
const evidence = [];
|
|
11800
|
+
for (const [key, field] of Object.entries(options.schema)) {
|
|
11801
|
+
validateSchemaField(record[key], field, key, evidence);
|
|
11802
|
+
}
|
|
11803
|
+
if (evidence.length === 0) {
|
|
11804
|
+
return pass(ruleId, "Structured output matches schema subset.");
|
|
11805
|
+
}
|
|
11806
|
+
return fail(ruleId, "Structured output schema violation.", evidence);
|
|
11807
|
+
}
|
|
11808
|
+
function evaluateOversizeOutput(value, options = {}) {
|
|
11809
|
+
const ruleId = "guardrail.oversize-output";
|
|
11810
|
+
const text = typeof value === "string" ? value : JSON.stringify(value);
|
|
11811
|
+
const maxLength = options.maxLength ?? options.maxSerializedLength ?? 32768;
|
|
11812
|
+
if (text.length <= maxLength) {
|
|
11813
|
+
return pass(ruleId, "Output within size limits.");
|
|
11814
|
+
}
|
|
11815
|
+
return fail(ruleId, `Output exceeds max length (${text.length} > ${maxLength}).`, [
|
|
11816
|
+
{ ruleId, preview: `length=${text.length}` }
|
|
11817
|
+
]);
|
|
11818
|
+
}
|
|
11819
|
+
function evaluateRequiredJsonShape(value, options) {
|
|
11820
|
+
const ruleId = "guardrail.required-json-shape";
|
|
11821
|
+
let parsed = value;
|
|
11822
|
+
if (typeof value === "string") {
|
|
11823
|
+
try {
|
|
11824
|
+
parsed = JSON.parse(value);
|
|
11825
|
+
} catch {
|
|
11826
|
+
return fail(ruleId, "Value is not valid JSON.", [{ ruleId, preview: boundedPreview(value) }]);
|
|
11827
|
+
}
|
|
11828
|
+
}
|
|
11829
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
11830
|
+
return fail(ruleId, "JSON value must be an object.", [{ ruleId, preview: typeof parsed }]);
|
|
11831
|
+
}
|
|
11832
|
+
const record = parsed;
|
|
11833
|
+
const evidence = [];
|
|
11834
|
+
for (const key of options.requiredKeys) {
|
|
11835
|
+
if (!(key in record)) {
|
|
11836
|
+
evidence.push({ ruleId, path: key, preview: "missing required key" });
|
|
11837
|
+
}
|
|
11838
|
+
}
|
|
11839
|
+
if (evidence.length === 0) {
|
|
11840
|
+
return pass(ruleId, "Required JSON keys present.");
|
|
11841
|
+
}
|
|
11842
|
+
return fail(ruleId, "Missing required JSON keys.", evidence);
|
|
11843
|
+
}
|
|
11844
|
+
|
|
11845
|
+
// packages/guardrails/src/run.ts
|
|
11846
|
+
var ALL_RULES2 = [
|
|
11847
|
+
"guardrail.banned-phrase",
|
|
11848
|
+
"guardrail.pii-leak",
|
|
11849
|
+
"guardrail.unsafe-tool-args",
|
|
11850
|
+
"guardrail.prompt-injection",
|
|
11851
|
+
"guardrail.structured-output",
|
|
11852
|
+
"guardrail.oversize-output",
|
|
11853
|
+
"guardrail.required-json-shape"
|
|
11854
|
+
];
|
|
11855
|
+
function isErrorFailure(result) {
|
|
11856
|
+
return result.status === "fail" && result.severity === "error";
|
|
11857
|
+
}
|
|
11858
|
+
function runRule2(ruleId, input3, options) {
|
|
11859
|
+
switch (ruleId) {
|
|
11860
|
+
case "guardrail.banned-phrase": {
|
|
11861
|
+
if (!options.bannedPhrase || input3.text === void 0) return void 0;
|
|
11862
|
+
return evaluateBannedPhrase(input3.text, options.bannedPhrase);
|
|
11863
|
+
}
|
|
11864
|
+
case "guardrail.pii-leak": {
|
|
11865
|
+
if (input3.value === void 0 && input3.text === void 0) return void 0;
|
|
11866
|
+
return evaluatePiiLeak(input3.value ?? input3.text, options.piiLeak);
|
|
11867
|
+
}
|
|
11868
|
+
case "guardrail.unsafe-tool-args": {
|
|
11869
|
+
if (!input3.toolName) return void 0;
|
|
11870
|
+
return evaluateUnsafeToolArgs(input3.toolName, input3.toolArgs ?? {}, options.unsafeToolArgs);
|
|
11871
|
+
}
|
|
11872
|
+
case "guardrail.prompt-injection": {
|
|
11873
|
+
if (input3.text === void 0) return void 0;
|
|
11874
|
+
return evaluatePromptInjection(input3.text, options.promptInjection);
|
|
11875
|
+
}
|
|
11876
|
+
case "guardrail.structured-output": {
|
|
11877
|
+
if (!options.structuredOutput || input3.value === void 0) return void 0;
|
|
11878
|
+
return evaluateStructuredOutput(input3.value, options.structuredOutput);
|
|
11879
|
+
}
|
|
11880
|
+
case "guardrail.oversize-output": {
|
|
11881
|
+
if (input3.value === void 0 && input3.text === void 0) return void 0;
|
|
11882
|
+
return evaluateOversizeOutput(input3.value ?? input3.text, options.oversizeOutput);
|
|
11883
|
+
}
|
|
11884
|
+
case "guardrail.required-json-shape": {
|
|
11885
|
+
if (!options.requiredJsonShape || input3.value === void 0 && input3.text === void 0) return void 0;
|
|
11886
|
+
return evaluateRequiredJsonShape(input3.value ?? input3.text, options.requiredJsonShape);
|
|
11887
|
+
}
|
|
11888
|
+
default:
|
|
11889
|
+
return void 0;
|
|
11890
|
+
}
|
|
11891
|
+
}
|
|
11892
|
+
function runGuardrails(input3, options = {}) {
|
|
11893
|
+
const selected = options.rules ?? ALL_RULES2;
|
|
11894
|
+
const results = [];
|
|
11895
|
+
for (const ruleId of selected) {
|
|
11896
|
+
const result = runRule2(ruleId, input3, options);
|
|
11897
|
+
if (result) results.push(result);
|
|
11898
|
+
}
|
|
11899
|
+
const ok = !results.some(isErrorFailure);
|
|
11900
|
+
return { ok, results };
|
|
11901
|
+
}
|
|
11902
|
+
|
|
11903
|
+
// packages/cli/src/safety-extensions.ts
|
|
11904
|
+
var GUARDRAIL_ALIASES = {
|
|
11905
|
+
"banned-phrase": "guardrail.banned-phrase",
|
|
11906
|
+
"pii-leak": "guardrail.pii-leak",
|
|
11907
|
+
"unsafe-tool-args": "guardrail.unsafe-tool-args",
|
|
11908
|
+
"prompt-injection": "guardrail.prompt-injection",
|
|
11909
|
+
"structured-output": "guardrail.structured-output",
|
|
11910
|
+
"oversize-output": "guardrail.oversize-output",
|
|
11911
|
+
"required-json-shape": "guardrail.required-json-shape"
|
|
11912
|
+
};
|
|
11913
|
+
var CIRCUIT_ALIASES = {
|
|
11914
|
+
"same-tool-repetition": "circuit.same-tool-repetition",
|
|
11915
|
+
"same-args-repetition": "circuit.same-args-repetition",
|
|
11916
|
+
"max-loop-iterations": "circuit.max-loop-iterations",
|
|
11917
|
+
"max-retries": "circuit.max-retries",
|
|
11918
|
+
"tool-timeout": "circuit.tool-timeout",
|
|
11919
|
+
"runaway-llm-loop": "circuit.runaway-llm-loop",
|
|
11920
|
+
"excessive-branch-width": "circuit.excessive-branch-width"
|
|
11921
|
+
};
|
|
11922
|
+
function parseGuardrailRules(values) {
|
|
11923
|
+
if (!values?.length) return void 0;
|
|
11924
|
+
return values.map((value) => {
|
|
11925
|
+
const rule = GUARDRAIL_ALIASES[value] ?? value;
|
|
11926
|
+
return rule;
|
|
11927
|
+
});
|
|
11928
|
+
}
|
|
11929
|
+
function parseCircuitRules(values) {
|
|
11930
|
+
if (!values?.length) return void 0;
|
|
11931
|
+
return values.map((value) => {
|
|
11932
|
+
const rule = CIRCUIT_ALIASES[value] ?? value;
|
|
11933
|
+
return rule;
|
|
11934
|
+
});
|
|
11935
|
+
}
|
|
11936
|
+
function toSeverity(severity) {
|
|
11937
|
+
return severity;
|
|
11938
|
+
}
|
|
11939
|
+
function guardrailFinding(result) {
|
|
11940
|
+
return {
|
|
11941
|
+
ruleId: result.ruleId,
|
|
11942
|
+
severity: toSeverity(result.severity),
|
|
11943
|
+
status: result.status === "pass" ? "pass" : result.status === "warn" ? "warning" : "fail",
|
|
11944
|
+
message: result.message,
|
|
11945
|
+
evidence: result.evidence.map(
|
|
11946
|
+
(item) => ({
|
|
11947
|
+
path: item.preview ? `${item.path ?? "value"} (${item.preview})` : item.path
|
|
11948
|
+
})
|
|
11949
|
+
)
|
|
11950
|
+
};
|
|
11951
|
+
}
|
|
11952
|
+
function circuitFinding(result) {
|
|
11953
|
+
return {
|
|
11954
|
+
ruleId: result.ruleId,
|
|
11955
|
+
severity: toSeverity(result.severity),
|
|
11956
|
+
status: result.status === "closed" ? "pass" : result.status === "warn" ? "warning" : "fail",
|
|
11957
|
+
message: result.message,
|
|
11958
|
+
evidence: result.evidence.map(
|
|
11959
|
+
(item) => ({
|
|
11960
|
+
runId: item.runId,
|
|
11961
|
+
eventId: item.eventId,
|
|
11962
|
+
path: item.path,
|
|
11963
|
+
name: item.toolName
|
|
11964
|
+
})
|
|
11965
|
+
),
|
|
11966
|
+
actual: result.evidence[0]?.count,
|
|
11967
|
+
expected: result.evidence[0]?.threshold
|
|
11968
|
+
};
|
|
11969
|
+
}
|
|
11970
|
+
function eventToCircuit(event) {
|
|
11971
|
+
return {
|
|
11972
|
+
eventId: event.eventId,
|
|
11973
|
+
runId: event.runId,
|
|
11974
|
+
name: event.name,
|
|
11975
|
+
kind: event.kind,
|
|
11976
|
+
parentId: event.parentId,
|
|
11977
|
+
startedAt: event.startedAt,
|
|
11978
|
+
endedAt: event.endedAt,
|
|
11979
|
+
durationMs: event.durationMs,
|
|
11980
|
+
attributes: event.attributes,
|
|
11981
|
+
status: event.status
|
|
11982
|
+
};
|
|
11983
|
+
}
|
|
11984
|
+
function collectGuardrailInputs(read) {
|
|
11985
|
+
const inputs = [];
|
|
11986
|
+
for (const event of read.events) {
|
|
11987
|
+
const attrs = event.attributes ?? {};
|
|
11988
|
+
for (const key of ["output", "answer", "text", "content", "message"]) {
|
|
11989
|
+
const value = attrs[key];
|
|
11990
|
+
if (typeof value === "string") inputs.push({ text: value });
|
|
11991
|
+
else if (value !== void 0) inputs.push({ value });
|
|
11992
|
+
}
|
|
11993
|
+
if (event.kind === "TOOL" || event.name.startsWith("tool:")) {
|
|
11994
|
+
inputs.push({
|
|
11995
|
+
toolName: String(attrs.toolName ?? attrs.tool ?? event.name),
|
|
11996
|
+
toolArgs: attrs.arguments ?? attrs.args ?? attrs.input
|
|
11997
|
+
});
|
|
11998
|
+
}
|
|
11999
|
+
}
|
|
12000
|
+
return inputs;
|
|
12001
|
+
}
|
|
12002
|
+
var DEFAULT_GUARDRAIL_OPTIONS = {
|
|
12003
|
+
bannedPhrase: { phrases: ["delete all data", "ignore all instructions"] },
|
|
12004
|
+
promptInjection: {},
|
|
12005
|
+
piiLeak: { profile: "share" }
|
|
12006
|
+
};
|
|
12007
|
+
var DEFAULT_CIRCUIT_OPTIONS = {
|
|
12008
|
+
sameToolRepetition: { maxRepeats: 3 },
|
|
12009
|
+
sameArgsRepetition: { maxRepeats: 2 },
|
|
12010
|
+
maxLoopIterations: { maxIterations: 20 },
|
|
12011
|
+
maxRetries: { maxRetries: 3 },
|
|
12012
|
+
toolTimeout: { maxDurationMs: 6e4 },
|
|
12013
|
+
runawayLlmLoop: { maxLlmCalls: 12 },
|
|
12014
|
+
excessiveBranchWidth: { maxWidth: 8 }
|
|
12015
|
+
};
|
|
12016
|
+
function mergeSafetyExtensions(result, read, options) {
|
|
12017
|
+
const findings = [...result.findings];
|
|
12018
|
+
let failed = result.summary.failed;
|
|
12019
|
+
let warnings = result.summary.warnings;
|
|
12020
|
+
let passed = result.summary.passed;
|
|
12021
|
+
const guardrailRules = parseGuardrailRules(options.guardrails);
|
|
12022
|
+
if (guardrailRules) {
|
|
12023
|
+
for (const input3 of collectGuardrailInputs(read)) {
|
|
12024
|
+
const run = runGuardrails(input3, {
|
|
12025
|
+
...DEFAULT_GUARDRAIL_OPTIONS,
|
|
12026
|
+
rules: guardrailRules
|
|
12027
|
+
});
|
|
12028
|
+
for (const item of run.results) {
|
|
12029
|
+
const finding = guardrailFinding(item);
|
|
12030
|
+
findings.push(finding);
|
|
12031
|
+
if (finding.status === "fail") failed += 1;
|
|
12032
|
+
else if (finding.status === "warning") warnings += 1;
|
|
12033
|
+
else passed += 1;
|
|
12034
|
+
}
|
|
12035
|
+
}
|
|
12036
|
+
}
|
|
12037
|
+
const circuitRules = parseCircuitRules(options.circuits);
|
|
12038
|
+
if (circuitRules) {
|
|
12039
|
+
const circuitRun = runCircuits(
|
|
12040
|
+
read.events.map(eventToCircuit),
|
|
12041
|
+
{ ...DEFAULT_CIRCUIT_OPTIONS, rules: circuitRules }
|
|
12042
|
+
);
|
|
12043
|
+
for (const item of circuitRun.results) {
|
|
12044
|
+
const finding = circuitFinding(item);
|
|
12045
|
+
findings.push(finding);
|
|
12046
|
+
if (finding.status === "fail") failed += 1;
|
|
12047
|
+
else if (finding.status === "warning") warnings += 1;
|
|
12048
|
+
else passed += 1;
|
|
12049
|
+
}
|
|
12050
|
+
}
|
|
12051
|
+
const ok = failed === 0 && result.summary.errors === 0;
|
|
12052
|
+
return {
|
|
12053
|
+
...result,
|
|
12054
|
+
ok,
|
|
12055
|
+
status: failed > 0 ? "fail" : result.status,
|
|
12056
|
+
summary: {
|
|
12057
|
+
passed,
|
|
12058
|
+
failed,
|
|
12059
|
+
warnings,
|
|
12060
|
+
errors: result.summary.errors
|
|
12061
|
+
},
|
|
12062
|
+
findings
|
|
12063
|
+
};
|
|
12064
|
+
}
|
|
11449
12065
|
|
|
11450
12066
|
// packages/cli/src/check.ts
|
|
11451
12067
|
var DEFAULT_SELECT = ["run.status"];
|
|
@@ -11494,7 +12110,7 @@ function asConfig(value) {
|
|
|
11494
12110
|
}
|
|
11495
12111
|
async function loadConfig(configPath) {
|
|
11496
12112
|
if (configPath === void 0) return {};
|
|
11497
|
-
const extension =
|
|
12113
|
+
const extension = path14.extname(configPath);
|
|
11498
12114
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
11499
12115
|
throw new Error(
|
|
11500
12116
|
"TypeScript check configs require an explicit precompiled JavaScript config or future --config-loader support."
|
|
@@ -11503,7 +12119,7 @@ async function loadConfig(configPath) {
|
|
|
11503
12119
|
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
11504
12120
|
throw new Error("Unsupported check config extension. Use .json, .js, .mjs, or .cjs.");
|
|
11505
12121
|
}
|
|
11506
|
-
const absolute =
|
|
12122
|
+
const absolute = path14.resolve(configPath);
|
|
11507
12123
|
if (extension === ".json") {
|
|
11508
12124
|
const raw = await readFile(absolute, "utf-8");
|
|
11509
12125
|
return asConfig(JSON.parse(raw));
|
|
@@ -11632,10 +12248,10 @@ function printHuman(result) {
|
|
|
11632
12248
|
console.log(`- ${diagnostic4.code}: ${diagnostic4.message}`);
|
|
11633
12249
|
}
|
|
11634
12250
|
for (const finding of result.findings) {
|
|
11635
|
-
const
|
|
12251
|
+
const path16 = finding.evidence[0]?.path;
|
|
11636
12252
|
const run = finding.evidence[0]?.runId;
|
|
11637
12253
|
const runPrefix = run ? `[${run}] ` : "";
|
|
11638
|
-
console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${
|
|
12254
|
+
console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${path16 ? ` (${path16})` : ""}`);
|
|
11639
12255
|
}
|
|
11640
12256
|
}
|
|
11641
12257
|
function readErrorResult(error) {
|
|
@@ -11685,12 +12301,19 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
11685
12301
|
}
|
|
11686
12302
|
);
|
|
11687
12303
|
perRun.push(
|
|
11688
|
-
|
|
11689
|
-
|
|
12304
|
+
mergeSafetyExtensions(
|
|
12305
|
+
runTraceChecks(
|
|
12306
|
+
{ read },
|
|
12307
|
+
{
|
|
12308
|
+
rules: built.rules,
|
|
12309
|
+
select: built.select,
|
|
12310
|
+
runId: meta.runId
|
|
12311
|
+
}
|
|
12312
|
+
),
|
|
12313
|
+
read,
|
|
11690
12314
|
{
|
|
11691
|
-
|
|
11692
|
-
|
|
11693
|
-
runId: meta.runId
|
|
12315
|
+
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
12316
|
+
...options.circuit ? { circuits: options.circuit } : {}
|
|
11694
12317
|
}
|
|
11695
12318
|
)
|
|
11696
12319
|
);
|
|
@@ -11709,12 +12332,19 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
11709
12332
|
const read = await openTrace(input3, {
|
|
11710
12333
|
...options.format !== void 0 ? { format: options.format } : {}
|
|
11711
12334
|
});
|
|
11712
|
-
result =
|
|
11713
|
-
|
|
12335
|
+
result = mergeSafetyExtensions(
|
|
12336
|
+
runTraceChecks(
|
|
12337
|
+
{ read },
|
|
12338
|
+
{
|
|
12339
|
+
rules: built.rules,
|
|
12340
|
+
select: built.select,
|
|
12341
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
12342
|
+
}
|
|
12343
|
+
),
|
|
12344
|
+
read,
|
|
11714
12345
|
{
|
|
11715
|
-
|
|
11716
|
-
|
|
11717
|
-
...options.run !== void 0 ? { runId: options.run } : {}
|
|
12346
|
+
...options.guardrails ? { guardrails: options.guardrails } : {},
|
|
12347
|
+
...options.circuit ? { circuits: options.circuit } : {}
|
|
11718
12348
|
}
|
|
11719
12349
|
);
|
|
11720
12350
|
}
|
|
@@ -11735,6 +12365,262 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
11735
12365
|
else printHuman(result);
|
|
11736
12366
|
}
|
|
11737
12367
|
|
|
12368
|
+
// packages/viewer/src/html.ts
|
|
12369
|
+
var viewerIndexHtml = `<!DOCTYPE html>
|
|
12370
|
+
<html lang="en">
|
|
12371
|
+
<head>
|
|
12372
|
+
<meta charset="utf-8" />
|
|
12373
|
+
<title>AgentInspect Viewer</title>
|
|
12374
|
+
<style>
|
|
12375
|
+
body { font-family: system-ui, sans-serif; margin: 1.5rem; line-height: 1.4; }
|
|
12376
|
+
h1 { font-size: 1.25rem; }
|
|
12377
|
+
pre { background: #f4f4f5; padding: 1rem; overflow: auto; max-height: 70vh; }
|
|
12378
|
+
a { color: #0b57d0; }
|
|
12379
|
+
.muted { color: #666; }
|
|
12380
|
+
</style>
|
|
12381
|
+
</head>
|
|
12382
|
+
<body>
|
|
12383
|
+
<h1>AgentInspect local viewer</h1>
|
|
12384
|
+
<p class="muted">Read-only. JSONL on disk remains canonical.</p>
|
|
12385
|
+
<p><a href="/api/health">/api/health</a> \xB7 <a href="/api/traces">/api/traces</a> \xB7 <a href="/api/sessions">/api/sessions</a></p>
|
|
12386
|
+
<pre id="out">Loading traces\u2026</pre>
|
|
12387
|
+
<script>
|
|
12388
|
+
fetch("/api/traces").then((r) => r.json()).then((data) => {
|
|
12389
|
+
document.getElementById("out").textContent = JSON.stringify(data, null, 2);
|
|
12390
|
+
}).catch((err) => {
|
|
12391
|
+
document.getElementById("out").textContent = String(err);
|
|
12392
|
+
});
|
|
12393
|
+
</script>
|
|
12394
|
+
</body>
|
|
12395
|
+
</html>
|
|
12396
|
+
`;
|
|
12397
|
+
|
|
12398
|
+
// packages/viewer/src/server.ts
|
|
12399
|
+
var DEFAULT_HOST = "127.0.0.1";
|
|
12400
|
+
var DEFAULT_PORT = 7337;
|
|
12401
|
+
var DEFAULT_MAX_EVENTS = 500;
|
|
12402
|
+
function sendJson(res, status, body) {
|
|
12403
|
+
const payload = JSON.stringify(body);
|
|
12404
|
+
res.writeHead(status, {
|
|
12405
|
+
"content-type": "application/json; charset=utf-8",
|
|
12406
|
+
"cache-control": "no-store"
|
|
12407
|
+
});
|
|
12408
|
+
res.end(payload);
|
|
12409
|
+
}
|
|
12410
|
+
function notFound(res, message) {
|
|
12411
|
+
sendJson(res, 404, { error: message });
|
|
12412
|
+
}
|
|
12413
|
+
function badRequest(res, message) {
|
|
12414
|
+
sendJson(res, 400, { error: message });
|
|
12415
|
+
}
|
|
12416
|
+
function decodeId(segment) {
|
|
12417
|
+
if (!segment) return "";
|
|
12418
|
+
try {
|
|
12419
|
+
return decodeURIComponent(segment);
|
|
12420
|
+
} catch {
|
|
12421
|
+
return segment;
|
|
12422
|
+
}
|
|
12423
|
+
}
|
|
12424
|
+
function boundedEvents(events, maxEvents) {
|
|
12425
|
+
if (events.length <= maxEvents) return [...events];
|
|
12426
|
+
return events.slice(0, maxEvents);
|
|
12427
|
+
}
|
|
12428
|
+
function createViewerServer(options = {}) {
|
|
12429
|
+
const traceDir = resolveTraceDir({ dir: options.traceDir });
|
|
12430
|
+
const host = options.host ?? DEFAULT_HOST;
|
|
12431
|
+
const port = options.port ?? DEFAULT_PORT;
|
|
12432
|
+
const maxEvents = options.maxEvents ?? DEFAULT_MAX_EVENTS;
|
|
12433
|
+
if (host === "0.0.0.0") {
|
|
12434
|
+
console.warn(
|
|
12435
|
+
"[AgentInspect viewer] Binding to 0.0.0.0 exposes traces on the network. Use 127.0.0.1 unless you accept that risk."
|
|
12436
|
+
);
|
|
12437
|
+
}
|
|
12438
|
+
const server = createServer(async (req, res) => {
|
|
12439
|
+
try {
|
|
12440
|
+
if (req.method !== "GET" && req.method !== "HEAD") {
|
|
12441
|
+
return badRequest(res, "Only GET is supported.");
|
|
12442
|
+
}
|
|
12443
|
+
const url = new URL(req.url ?? "/", `http://${host}:${port}`);
|
|
12444
|
+
const pathname = url.pathname;
|
|
12445
|
+
if (pathname === "/" || pathname === "/index.html") {
|
|
12446
|
+
res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
|
|
12447
|
+
res.end(viewerIndexHtml);
|
|
12448
|
+
return;
|
|
12449
|
+
}
|
|
12450
|
+
if (pathname === "/api/health") {
|
|
12451
|
+
return sendJson(res, 200, {
|
|
12452
|
+
ok: true,
|
|
12453
|
+
readOnly: true,
|
|
12454
|
+
traceDir: path14.resolve(traceDir)
|
|
12455
|
+
});
|
|
12456
|
+
}
|
|
12457
|
+
const td = new TraceDirectory({ dir: traceDir });
|
|
12458
|
+
if (pathname === "/api/traces") {
|
|
12459
|
+
const files = await td.list();
|
|
12460
|
+
const metas = await loadTraceMetadataList(
|
|
12461
|
+
traceDir,
|
|
12462
|
+
files,
|
|
12463
|
+
(fileName) => td.getPath(fileName)
|
|
12464
|
+
);
|
|
12465
|
+
return sendJson(
|
|
12466
|
+
res,
|
|
12467
|
+
200,
|
|
12468
|
+
metas.map((meta) => ({
|
|
12469
|
+
runId: meta.runId,
|
|
12470
|
+
name: meta.name,
|
|
12471
|
+
status: meta.status,
|
|
12472
|
+
file: path14.basename(meta.filePath),
|
|
12473
|
+
startedAt: meta.startedAt,
|
|
12474
|
+
durationMs: meta.durationMs
|
|
12475
|
+
}))
|
|
12476
|
+
);
|
|
12477
|
+
}
|
|
12478
|
+
if (pathname === "/api/sessions") {
|
|
12479
|
+
const files = await td.list();
|
|
12480
|
+
const metas = await loadTraceMetadataList(
|
|
12481
|
+
traceDir,
|
|
12482
|
+
files,
|
|
12483
|
+
(fileName) => td.getPath(fileName)
|
|
12484
|
+
);
|
|
12485
|
+
const runs = await loadSessionRunRecords(metas);
|
|
12486
|
+
const index = buildSessionIndex(runs, {
|
|
12487
|
+
correlateByGroupId: url.searchParams.get("correlateGroup") === "true"
|
|
12488
|
+
});
|
|
12489
|
+
return sendJson(res, 200, index);
|
|
12490
|
+
}
|
|
12491
|
+
const sessionMatch = pathname.match(/^\/api\/session\/([^/]+)$/);
|
|
12492
|
+
if (sessionMatch) {
|
|
12493
|
+
const sessionId = decodeId(sessionMatch[1]);
|
|
12494
|
+
const files = await td.list();
|
|
12495
|
+
const metas = await loadTraceMetadataList(
|
|
12496
|
+
traceDir,
|
|
12497
|
+
files,
|
|
12498
|
+
(fileName) => td.getPath(fileName)
|
|
12499
|
+
);
|
|
12500
|
+
const runs = await loadSessionRunRecords(metas);
|
|
12501
|
+
const index = buildSessionIndex(runs, {
|
|
12502
|
+
correlateByGroupId: url.searchParams.get("correlateGroup") === "true"
|
|
12503
|
+
});
|
|
12504
|
+
const session = index.sessions.find((item) => item.sessionId === sessionId);
|
|
12505
|
+
if (!session) return notFound(res, `Session not found: ${sessionId}`);
|
|
12506
|
+
return sendJson(res, 200, session);
|
|
12507
|
+
}
|
|
12508
|
+
const traceMatch = pathname.match(/^\/api\/trace\/([^/]+)$/);
|
|
12509
|
+
if (traceMatch) {
|
|
12510
|
+
const runId = decodeId(traceMatch[1]);
|
|
12511
|
+
const files = await td.list();
|
|
12512
|
+
const metas = await loadTraceMetadataList(
|
|
12513
|
+
traceDir,
|
|
12514
|
+
files,
|
|
12515
|
+
(fileName) => td.getPath(fileName)
|
|
12516
|
+
);
|
|
12517
|
+
const meta = metas.find((item) => item.runId === runId);
|
|
12518
|
+
if (!meta) return notFound(res, `Run not found: ${runId}`);
|
|
12519
|
+
const read = await openTrace({ type: "file", path: meta.filePath });
|
|
12520
|
+
const run = read.runs.find((item) => item.runId === runId) ?? read.runs[0];
|
|
12521
|
+
return sendJson(res, 200, {
|
|
12522
|
+
runId,
|
|
12523
|
+
format: read.format,
|
|
12524
|
+
run,
|
|
12525
|
+
events: boundedEvents(read.events, maxEvents),
|
|
12526
|
+
warnings: read.warnings,
|
|
12527
|
+
truncated: read.events.length > maxEvents
|
|
12528
|
+
});
|
|
12529
|
+
}
|
|
12530
|
+
const timelineMatch = pathname.match(/^\/api\/trace\/([^/]+)\/timeline$/);
|
|
12531
|
+
if (timelineMatch) {
|
|
12532
|
+
const runId = decodeId(timelineMatch[1]);
|
|
12533
|
+
const files = await td.list();
|
|
12534
|
+
const metas = await loadTraceMetadataList(
|
|
12535
|
+
traceDir,
|
|
12536
|
+
files,
|
|
12537
|
+
(fileName) => td.getPath(fileName)
|
|
12538
|
+
);
|
|
12539
|
+
const meta = metas.find((item) => item.runId === runId);
|
|
12540
|
+
if (!meta) return notFound(res, `Run not found: ${runId}`);
|
|
12541
|
+
const read = await openTrace({ type: "file", path: meta.filePath });
|
|
12542
|
+
const legacyEvents = persistedInspectEventsToTraceEvents(
|
|
12543
|
+
boundedEvents(read.events, maxEvents)
|
|
12544
|
+
);
|
|
12545
|
+
const timeline = buildRunTimeline(legacyEvents, { focus: "all" });
|
|
12546
|
+
return sendJson(res, 200, { runId, timeline });
|
|
12547
|
+
}
|
|
12548
|
+
const checkMatch = pathname.match(/^\/api\/trace\/([^/]+)\/check$/);
|
|
12549
|
+
if (checkMatch) {
|
|
12550
|
+
const runId = decodeId(checkMatch[1]);
|
|
12551
|
+
const files = await td.list();
|
|
12552
|
+
const metas = await loadTraceMetadataList(
|
|
12553
|
+
traceDir,
|
|
12554
|
+
files,
|
|
12555
|
+
(fileName) => td.getPath(fileName)
|
|
12556
|
+
);
|
|
12557
|
+
const meta = metas.find((item) => item.runId === runId);
|
|
12558
|
+
if (!meta) return notFound(res, `Run not found: ${runId}`);
|
|
12559
|
+
const read = await openTrace({ type: "file", path: meta.filePath });
|
|
12560
|
+
const result = runTraceChecks(
|
|
12561
|
+
{ read },
|
|
12562
|
+
{ rules: [createRunStatusRule()], select: ["run.status"], runId }
|
|
12563
|
+
);
|
|
12564
|
+
return sendJson(res, 200, result);
|
|
12565
|
+
}
|
|
12566
|
+
return notFound(res, `Unknown route: ${pathname}`);
|
|
12567
|
+
} catch (error) {
|
|
12568
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
12569
|
+
sendJson(res, 500, { error: message });
|
|
12570
|
+
}
|
|
12571
|
+
});
|
|
12572
|
+
return server;
|
|
12573
|
+
}
|
|
12574
|
+
function startViewerServer(options = {}) {
|
|
12575
|
+
const host = options.host ?? DEFAULT_HOST;
|
|
12576
|
+
const port = options.port ?? DEFAULT_PORT;
|
|
12577
|
+
const traceDir = resolveTraceDir({ dir: options.traceDir });
|
|
12578
|
+
const server = createViewerServer(options);
|
|
12579
|
+
return new Promise((resolve, reject) => {
|
|
12580
|
+
server.once("error", reject);
|
|
12581
|
+
server.listen(port, host, () => {
|
|
12582
|
+
const address = server.address();
|
|
12583
|
+
const resolvedPort = typeof address === "object" && address ? address.port : port;
|
|
12584
|
+
resolve({
|
|
12585
|
+
host,
|
|
12586
|
+
port: resolvedPort,
|
|
12587
|
+
traceDir: path14.resolve(traceDir),
|
|
12588
|
+
url: `http://${host}:${resolvedPort}`
|
|
12589
|
+
});
|
|
12590
|
+
});
|
|
12591
|
+
});
|
|
12592
|
+
}
|
|
12593
|
+
|
|
12594
|
+
// packages/cli/src/serve.ts
|
|
12595
|
+
function parsePort(value) {
|
|
12596
|
+
if (value === void 0) return void 0;
|
|
12597
|
+
const parsed = Number(value);
|
|
12598
|
+
if (!Number.isInteger(parsed) || parsed <= 0 || parsed > 65535) {
|
|
12599
|
+
throw new Error("--port must be an integer between 1 and 65535.");
|
|
12600
|
+
}
|
|
12601
|
+
return parsed;
|
|
12602
|
+
}
|
|
12603
|
+
async function serveCommand(options = {}) {
|
|
12604
|
+
const port = parsePort(options.port);
|
|
12605
|
+
const host = options.host?.trim() || "127.0.0.1";
|
|
12606
|
+
const info = await startViewerServer({
|
|
12607
|
+
...options.dir !== void 0 ? { traceDir: options.dir } : {},
|
|
12608
|
+
host,
|
|
12609
|
+
...port !== void 0 ? { port } : {}
|
|
12610
|
+
});
|
|
12611
|
+
console.log(`AgentInspect viewer (read-only): ${info.url}`);
|
|
12612
|
+
console.log(`Trace directory: ${info.traceDir}`);
|
|
12613
|
+
if (options.open === true && host !== "127.0.0.1" && host !== "localhost") {
|
|
12614
|
+
console.warn("Skipping browser open for non-local host binding.");
|
|
12615
|
+
} else if (options.open === true) {
|
|
12616
|
+
const openMod = await import('child_process');
|
|
12617
|
+
openMod.exec(`open ${info.url}`, () => {
|
|
12618
|
+
});
|
|
12619
|
+
}
|
|
12620
|
+
await new Promise(() => {
|
|
12621
|
+
});
|
|
12622
|
+
}
|
|
12623
|
+
|
|
11738
12624
|
// packages/eval/src/index.ts
|
|
11739
12625
|
function isTraceReadResult(value) {
|
|
11740
12626
|
return value !== null && typeof value === "object" && "runs" in value && "events" in value && "format" in value;
|
|
@@ -11903,10 +12789,10 @@ async function evalRun(input3, options = {}) {
|
|
|
11903
12789
|
diagnostics: []
|
|
11904
12790
|
};
|
|
11905
12791
|
}
|
|
11906
|
-
function evidenceForRun(run,
|
|
11907
|
-
return [{ runId: run.runId, ...
|
|
12792
|
+
function evidenceForRun(run, path16) {
|
|
12793
|
+
return [{ runId: run.runId, ...path16 !== void 0 ? { path: path16 } : {} }];
|
|
11908
12794
|
}
|
|
11909
|
-
function evidenceForEvent(event,
|
|
12795
|
+
function evidenceForEvent(event, path16) {
|
|
11910
12796
|
return [
|
|
11911
12797
|
{
|
|
11912
12798
|
runId: event.runId,
|
|
@@ -11914,11 +12800,11 @@ function evidenceForEvent(event, path15) {
|
|
|
11914
12800
|
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
11915
12801
|
kind: event.kind,
|
|
11916
12802
|
name: event.name,
|
|
11917
|
-
...
|
|
12803
|
+
...path16 !== void 0 ? { path: path16 } : {}
|
|
11918
12804
|
}
|
|
11919
12805
|
];
|
|
11920
12806
|
}
|
|
11921
|
-
function
|
|
12807
|
+
function fail2(ruleId, message, evidence, expected, actual) {
|
|
11922
12808
|
return {
|
|
11923
12809
|
ruleId,
|
|
11924
12810
|
status: "fail",
|
|
@@ -12072,9 +12958,9 @@ function collectTextFields(nodes, keys, preferredKinds = []) {
|
|
|
12072
12958
|
function tokenize(text) {
|
|
12073
12959
|
return [...text.toLowerCase().matchAll(/[a-z0-9][a-z0-9'-]{2,}/g)].map((match) => match[0].replace(/^['-]+|['-]+$/g, "")).filter((token) => token.length > 2 && !STOP_WORDS.has(token));
|
|
12074
12960
|
}
|
|
12075
|
-
function firstEvidence(fields, run,
|
|
12961
|
+
function firstEvidence(fields, run, path16) {
|
|
12076
12962
|
const first = fields[0];
|
|
12077
|
-
return first === void 0 ? evidenceForRun(run,
|
|
12963
|
+
return first === void 0 ? evidenceForRun(run, path16) : evidenceForEvent(first.node.event, first.path);
|
|
12078
12964
|
}
|
|
12079
12965
|
function collectSourceIds(nodes, keys) {
|
|
12080
12966
|
const wanted = keySet(keys);
|
|
@@ -12114,7 +13000,7 @@ var checks = {
|
|
|
12114
13000
|
"eval.requireSuccess",
|
|
12115
13001
|
"run",
|
|
12116
13002
|
(context) => context.run.status === "ok" ? [] : [
|
|
12117
|
-
|
|
13003
|
+
fail2(
|
|
12118
13004
|
"eval.requireSuccess",
|
|
12119
13005
|
"Run did not complete successfully.",
|
|
12120
13006
|
evidenceForRun(context.run, "status"),
|
|
@@ -12129,7 +13015,7 @@ var checks = {
|
|
|
12129
13015
|
return createRule("eval.requiredTools", "tool", (context) => {
|
|
12130
13016
|
const tools = new Set(nodeNames(context.nodes, "TOOL"));
|
|
12131
13017
|
return expected.filter((name) => !tools.has(name)).map(
|
|
12132
|
-
(name) =>
|
|
13018
|
+
(name) => fail2(
|
|
12133
13019
|
"eval.requiredTools",
|
|
12134
13020
|
`Required tool ${name} did not appear.`,
|
|
12135
13021
|
evidenceForRun(context.run, "children"),
|
|
@@ -12145,7 +13031,7 @@ var checks = {
|
|
|
12145
13031
|
"eval.forbiddenTools",
|
|
12146
13032
|
"tool",
|
|
12147
13033
|
(context) => context.nodes.filter((node) => node.event.kind === "TOOL" && blocked.includes(node.event.name)).map(
|
|
12148
|
-
(node) =>
|
|
13034
|
+
(node) => fail2(
|
|
12149
13035
|
"eval.forbiddenTools",
|
|
12150
13036
|
`Forbidden tool ${node.event.name} appeared.`,
|
|
12151
13037
|
evidenceForEvent(node.event, "name"),
|
|
@@ -12160,7 +13046,7 @@ var checks = {
|
|
|
12160
13046
|
"eval.maxDurationMs",
|
|
12161
13047
|
"run",
|
|
12162
13048
|
(context) => context.run.durationMs !== void 0 && context.run.durationMs > maxDurationMs ? [
|
|
12163
|
-
|
|
13049
|
+
fail2(
|
|
12164
13050
|
"eval.maxDurationMs",
|
|
12165
13051
|
`Run duration exceeded ${maxDurationMs}ms.`,
|
|
12166
13052
|
evidenceForRun(context.run, "durationMs"),
|
|
@@ -12177,7 +13063,7 @@ var checks = {
|
|
|
12177
13063
|
void 0
|
|
12178
13064
|
);
|
|
12179
13065
|
return deepest !== void 0 && deepest.depth > maxDepth ? [
|
|
12180
|
-
|
|
13066
|
+
fail2(
|
|
12181
13067
|
"eval.maxDepth",
|
|
12182
13068
|
`Run tree depth exceeded ${maxDepth}.`,
|
|
12183
13069
|
evidenceForEvent(deepest.event, "depth"),
|
|
@@ -12194,7 +13080,7 @@ var checks = {
|
|
|
12194
13080
|
(context) => context.nodes.flatMap((node) => {
|
|
12195
13081
|
const retries = numericAttribute(node, ["retryCount", "retries", "attempt"]);
|
|
12196
13082
|
return retries !== void 0 && retries > maxRetries ? [
|
|
12197
|
-
|
|
13083
|
+
fail2(
|
|
12198
13084
|
"eval.maxRetries",
|
|
12199
13085
|
`Retry count exceeded ${maxRetries}.`,
|
|
12200
13086
|
evidenceForEvent(node.event, "attributes.retryCount"),
|
|
@@ -12209,7 +13095,7 @@ var checks = {
|
|
|
12209
13095
|
return createRule("eval.maxTotalTokens", "llm", (context) => {
|
|
12210
13096
|
const total = totalTokenCount(context.events);
|
|
12211
13097
|
return total > maxTotalTokens ? [
|
|
12212
|
-
|
|
13098
|
+
fail2(
|
|
12213
13099
|
"eval.maxTotalTokens",
|
|
12214
13100
|
`Total token usage exceeded ${maxTotalTokens}.`,
|
|
12215
13101
|
evidenceForRun(context.run, "tokenUsage.total"),
|
|
@@ -12224,7 +13110,7 @@ var checks = {
|
|
|
12224
13110
|
"eval.noFailedSteps",
|
|
12225
13111
|
"run",
|
|
12226
13112
|
(context) => context.nodes.filter((node) => node.event.status === "error" || node.event.kind === "ERROR").map(
|
|
12227
|
-
(node) =>
|
|
13113
|
+
(node) => fail2(
|
|
12228
13114
|
"eval.noFailedSteps",
|
|
12229
13115
|
"Run contains a failed step or error node.",
|
|
12230
13116
|
evidenceForEvent(node.event, "status"),
|
|
@@ -12242,7 +13128,7 @@ var checks = {
|
|
|
12242
13128
|
(node, index) => index < firstLlmIndex && node.event.kind === "RETRIEVER"
|
|
12243
13129
|
);
|
|
12244
13130
|
return retrievalIndex === -1 ? [
|
|
12245
|
-
|
|
13131
|
+
fail2(
|
|
12246
13132
|
"eval.requiredRetrievalBeforeGeneration",
|
|
12247
13133
|
"No retrieval step appeared before the first LLM generation.",
|
|
12248
13134
|
evidenceForEvent(context.nodes[firstLlmIndex].event, "kind"),
|
|
@@ -12258,7 +13144,7 @@ var checks = {
|
|
|
12258
13144
|
const decisions = context.nodes.filter((node) => node.event.kind === "DECISION");
|
|
12259
13145
|
if (decisions.length === 0) {
|
|
12260
13146
|
return [
|
|
12261
|
-
|
|
13147
|
+
fail2(
|
|
12262
13148
|
"eval.requiredDecisionMetadata",
|
|
12263
13149
|
"No decision node is available for required metadata.",
|
|
12264
13150
|
evidenceForRun(context.run, "children"),
|
|
@@ -12269,7 +13155,7 @@ var checks = {
|
|
|
12269
13155
|
}
|
|
12270
13156
|
return decisions.flatMap(
|
|
12271
13157
|
(node) => required.filter((key) => !hasAttribute(node, key)).map(
|
|
12272
|
-
(key) =>
|
|
13158
|
+
(key) => fail2(
|
|
12273
13159
|
"eval.requiredDecisionMetadata",
|
|
12274
13160
|
`Decision metadata ${key} is missing.`,
|
|
12275
13161
|
evidenceForEvent(node.event, `attributes.${key}`),
|
|
@@ -12290,7 +13176,7 @@ var checks = {
|
|
|
12290
13176
|
const contexts = collectTextFields(context.nodes, contextKeys, ["RETRIEVER", "TOOL"]);
|
|
12291
13177
|
if (answers.length === 0 || contexts.length === 0) {
|
|
12292
13178
|
return [
|
|
12293
|
-
|
|
13179
|
+
fail2(
|
|
12294
13180
|
"eval.contextOverlap",
|
|
12295
13181
|
"Answer and context text are required for overlap evaluation.",
|
|
12296
13182
|
firstEvidence(answers.length > 0 ? answers : contexts, context.run, "children"),
|
|
@@ -12304,7 +13190,7 @@ var checks = {
|
|
|
12304
13190
|
const sharedTerms = [...answerTerms].filter((term) => contextTerms.has(term)).length;
|
|
12305
13191
|
const overlap = answerTerms.size === 0 ? 0 : sharedTerms / answerTerms.size;
|
|
12306
13192
|
return sharedTerms < minSharedTerms || overlap < minOverlap ? [
|
|
12307
|
-
|
|
13193
|
+
fail2(
|
|
12308
13194
|
"eval.contextOverlap",
|
|
12309
13195
|
"Answer text did not sufficiently overlap retrieved context.",
|
|
12310
13196
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12332,7 +13218,7 @@ var checks = {
|
|
|
12332
13218
|
const quotes = quotedSnippets(answerText, minQuoteLength);
|
|
12333
13219
|
if (quotes.length === 0) {
|
|
12334
13220
|
return requireQuote ? [
|
|
12335
|
-
|
|
13221
|
+
fail2(
|
|
12336
13222
|
"eval.quoteOverlap",
|
|
12337
13223
|
"Answer did not contain a quote for overlap evaluation.",
|
|
12338
13224
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12343,7 +13229,7 @@ var checks = {
|
|
|
12343
13229
|
}
|
|
12344
13230
|
const missing = quotes.filter((quote) => !contextText.includes(quote.toLowerCase()));
|
|
12345
13231
|
return missing.length > 0 ? [
|
|
12346
|
-
|
|
13232
|
+
fail2(
|
|
12347
13233
|
"eval.quoteOverlap",
|
|
12348
13234
|
"Quoted answer text did not appear in retrieved context.",
|
|
12349
13235
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12361,7 +13247,7 @@ var checks = {
|
|
|
12361
13247
|
const citations = collectTextFields(context.nodes, citationKeys);
|
|
12362
13248
|
const count = citationCount(answers.map((field) => field.text).join(" "), citations);
|
|
12363
13249
|
return count === 0 ? [
|
|
12364
|
-
|
|
13250
|
+
fail2(
|
|
12365
13251
|
"eval.citationPresence",
|
|
12366
13252
|
"Answer did not include citations or source references.",
|
|
12367
13253
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12379,7 +13265,7 @@ var checks = {
|
|
|
12379
13265
|
const availableSet = new Set(available);
|
|
12380
13266
|
const missing = expected.filter((id) => !availableSet.has(id));
|
|
12381
13267
|
return missing.length > 0 ? [
|
|
12382
|
-
|
|
13268
|
+
fail2(
|
|
12383
13269
|
"eval.requiredSourceIds",
|
|
12384
13270
|
"Required source IDs were not present in trace context or citations.",
|
|
12385
13271
|
evidenceForRun(context.run, "children"),
|
|
@@ -12399,7 +13285,7 @@ var checks = {
|
|
|
12399
13285
|
const tooShort = options.minCharacters !== void 0 && characters < options.minCharacters || options.minWords !== void 0 && words < options.minWords;
|
|
12400
13286
|
const tooLong = options.maxCharacters !== void 0 && characters > options.maxCharacters || options.maxWords !== void 0 && words > options.maxWords;
|
|
12401
13287
|
return answer.length === 0 || tooShort || tooLong ? [
|
|
12402
|
-
|
|
13288
|
+
fail2(
|
|
12403
13289
|
"eval.answerLengthBounds",
|
|
12404
13290
|
"Answer length fell outside required bounds.",
|
|
12405
13291
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12422,7 +13308,7 @@ var checks = {
|
|
|
12422
13308
|
const answer = answers.map((field) => field.text).join(" ").toLowerCase();
|
|
12423
13309
|
const matches = banned.filter((phrase) => answer.includes(phrase));
|
|
12424
13310
|
return matches.length > 0 ? [
|
|
12425
|
-
|
|
13311
|
+
fail2(
|
|
12426
13312
|
"eval.bannedUnsupportedPhrases",
|
|
12427
13313
|
"Answer contained banned unsupported-answer phrasing.",
|
|
12428
13314
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
@@ -12451,8 +13337,8 @@ function renderEvalMarkdown(result) {
|
|
|
12451
13337
|
if (result.findings.length > 0) {
|
|
12452
13338
|
lines.push("", "## Findings");
|
|
12453
13339
|
for (const finding of result.findings) {
|
|
12454
|
-
const
|
|
12455
|
-
lines.push(`- ${finding.ruleId}: ${finding.message}${
|
|
13340
|
+
const path16 = finding.evidence[0]?.path;
|
|
13341
|
+
lines.push(`- ${finding.ruleId}: ${finding.message}${path16 ? ` (${path16})` : ""}`);
|
|
12456
13342
|
}
|
|
12457
13343
|
}
|
|
12458
13344
|
return `${lines.join("\n")}
|
|
@@ -12499,7 +13385,7 @@ function asConfig2(value) {
|
|
|
12499
13385
|
}
|
|
12500
13386
|
async function loadConfig2(configPath) {
|
|
12501
13387
|
if (configPath === void 0) return {};
|
|
12502
|
-
const extension =
|
|
13388
|
+
const extension = path14.extname(configPath);
|
|
12503
13389
|
if (TS_CONFIG_EXTENSIONS2.has(extension)) {
|
|
12504
13390
|
throw new Error(
|
|
12505
13391
|
"TypeScript eval configs require an explicit precompiled JavaScript config or future --config-loader support."
|
|
@@ -12508,7 +13394,7 @@ async function loadConfig2(configPath) {
|
|
|
12508
13394
|
if (!CONFIG_EXTENSIONS2.has(extension)) {
|
|
12509
13395
|
throw new Error("Unsupported eval config extension. Use .json, .js, .mjs, or .cjs.");
|
|
12510
13396
|
}
|
|
12511
|
-
const absolute =
|
|
13397
|
+
const absolute = path14.resolve(configPath);
|
|
12512
13398
|
if (extension === ".json") {
|
|
12513
13399
|
const raw = await readFile(absolute, "utf-8");
|
|
12514
13400
|
return asConfig2(JSON.parse(raw));
|
|
@@ -12645,8 +13531,8 @@ function printHuman2(result) {
|
|
|
12645
13531
|
console.log(`- ${diagnostic4.code}: ${diagnostic4.message}`);
|
|
12646
13532
|
}
|
|
12647
13533
|
for (const finding of result.findings) {
|
|
12648
|
-
const
|
|
12649
|
-
console.log(`- ${finding.ruleId}: ${finding.message}${
|
|
13534
|
+
const path16 = finding.evidence[0]?.path;
|
|
13535
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path16 ? ` (${path16})` : ""}`);
|
|
12650
13536
|
}
|
|
12651
13537
|
}
|
|
12652
13538
|
function readErrorResult2(error) {
|
|
@@ -12800,7 +13686,7 @@ function invalidArgumentResult(command, error) {
|
|
|
12800
13686
|
});
|
|
12801
13687
|
}
|
|
12802
13688
|
function buildSafetyRules(options) {
|
|
12803
|
-
const
|
|
13689
|
+
const maxStringLength2 = parseLimit3(options.maxStringLength, "--max-string-length") ?? DEFAULT_MAX_STRING_LENGTH;
|
|
12804
13690
|
const maxArrayLength = parseLimit3(options.maxArrayLength, "--max-array-length") ?? DEFAULT_MAX_ARRAY_LENGTH;
|
|
12805
13691
|
const maxObjectKeys = parseLimit3(options.maxObjectKeys, "--max-object-keys") ?? DEFAULT_MAX_OBJECT_KEYS;
|
|
12806
13692
|
const maxSerializedBytes = parseLimit3(options.maxSerializedBytes, "--max-serialized-bytes") ?? DEFAULT_MAX_SERIALIZED_BYTES;
|
|
@@ -12809,7 +13695,7 @@ function buildSafetyRules(options) {
|
|
|
12809
13695
|
createSafetyRedactionRule(),
|
|
12810
13696
|
createSafetySecretPatternRule(),
|
|
12811
13697
|
createSafetyOversizedAttributeRule({
|
|
12812
|
-
maxStringLength,
|
|
13698
|
+
maxStringLength: maxStringLength2,
|
|
12813
13699
|
maxArrayLength,
|
|
12814
13700
|
maxObjectKeys,
|
|
12815
13701
|
maxSerializedBytes
|
|
@@ -12884,8 +13770,8 @@ function printHuman3(result) {
|
|
|
12884
13770
|
console.log(`- ${diagnostic4.code}: ${diagnostic4.message}`);
|
|
12885
13771
|
}
|
|
12886
13772
|
for (const finding of result.findings) {
|
|
12887
|
-
const
|
|
12888
|
-
console.log(`- ${finding.ruleId}: ${finding.message}${
|
|
13773
|
+
const path16 = finding.evidence[0]?.path;
|
|
13774
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path16 ? ` (${path16})` : ""}`);
|
|
12889
13775
|
}
|
|
12890
13776
|
console.log(`Note: ${result.note}`);
|
|
12891
13777
|
}
|
|
@@ -13004,8 +13890,8 @@ function renderCheckSection(result) {
|
|
|
13004
13890
|
`Diagnostics: ${result.diagnostics.length}`
|
|
13005
13891
|
];
|
|
13006
13892
|
for (const finding of result.findings.slice(0, 10)) {
|
|
13007
|
-
const
|
|
13008
|
-
lines.push(`- ${finding.ruleId}: ${finding.message} (${
|
|
13893
|
+
const path16 = finding.evidence[0]?.path ?? "(run)";
|
|
13894
|
+
lines.push(`- ${finding.ruleId}: ${finding.message} (${path16})`);
|
|
13009
13895
|
}
|
|
13010
13896
|
for (const diagnostic4 of result.diagnostics.slice(0, 10)) {
|
|
13011
13897
|
lines.push(`- ${diagnostic4.code}: ${diagnostic4.message}`);
|
|
@@ -13083,8 +13969,8 @@ function renderHtml(trace, check, diff) {
|
|
|
13083
13969
|
`;
|
|
13084
13970
|
}
|
|
13085
13971
|
async function writeArtifact(outputDir, relativePath, content, files) {
|
|
13086
|
-
const outPath =
|
|
13087
|
-
await mkdir(
|
|
13972
|
+
const outPath = path14.join(outputDir, relativePath);
|
|
13973
|
+
await mkdir(path14.dirname(outPath), { recursive: true });
|
|
13088
13974
|
await writeFile(outPath, content, "utf-8");
|
|
13089
13975
|
files.push(relativePath);
|
|
13090
13976
|
}
|
|
@@ -13100,7 +13986,7 @@ function manifestStatus(check, diff) {
|
|
|
13100
13986
|
return "ok";
|
|
13101
13987
|
}
|
|
13102
13988
|
async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
13103
|
-
const outputDir = options.outputDir !== void 0 && options.outputDir.trim() !== "" ?
|
|
13989
|
+
const outputDir = options.outputDir !== void 0 && options.outputDir.trim() !== "" ? path14.resolve(options.outputDir.trim()) : "";
|
|
13104
13990
|
if (outputDir === "") {
|
|
13105
13991
|
console.error("--output-dir is required.");
|
|
13106
13992
|
process.exitCode = 1;
|
|
@@ -13166,8 +14052,8 @@ async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
|
13166
14052
|
await writeArtifact(outputDir, "report.html", renderHtml(trace, check, diff), files);
|
|
13167
14053
|
const summaryTarget = options.githubSummary ?? process.env.GITHUB_STEP_SUMMARY;
|
|
13168
14054
|
if (summaryTarget !== void 0 && summaryTarget.trim() !== "") {
|
|
13169
|
-
await mkdir(
|
|
13170
|
-
await appendFile(
|
|
14055
|
+
await mkdir(path14.dirname(path14.resolve(summaryTarget)), { recursive: true });
|
|
14056
|
+
await appendFile(path14.resolve(summaryTarget), `
|
|
13171
14057
|
${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
13172
14058
|
}
|
|
13173
14059
|
const manifestFiles = [...files, "manifest.json"].sort((a, b) => a.localeCompare(b));
|
|
@@ -13186,10 +14072,10 @@ ${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
|
13186
14072
|
findings: diff?.findings.length ?? 0,
|
|
13187
14073
|
diagnostics: diff?.diagnostics.length ?? 0
|
|
13188
14074
|
},
|
|
13189
|
-
...summaryTarget !== void 0 && summaryTarget.trim() !== "" ? { githubSummary:
|
|
14075
|
+
...summaryTarget !== void 0 && summaryTarget.trim() !== "" ? { githubSummary: path14.resolve(summaryTarget) } : {},
|
|
13190
14076
|
note: NOTE
|
|
13191
14077
|
};
|
|
13192
|
-
await writeFile(
|
|
14078
|
+
await writeFile(path14.join(outputDir, "manifest.json"), writeJson3(manifest), "utf-8");
|
|
13193
14079
|
if (options.json === true) {
|
|
13194
14080
|
console.log(writeJson3(manifest).trimEnd());
|
|
13195
14081
|
} else {
|
|
@@ -13201,7 +14087,7 @@ ${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
|
13201
14087
|
}
|
|
13202
14088
|
}
|
|
13203
14089
|
function validateReporterArtifactPath(options) {
|
|
13204
|
-
const outputDir =
|
|
14090
|
+
const outputDir = path14.resolve(options.outputDir);
|
|
13205
14091
|
const diagnostics = [];
|
|
13206
14092
|
const rawPath = options.relativePath;
|
|
13207
14093
|
if (rawPath.length === 0) {
|
|
@@ -13221,7 +14107,7 @@ function validateReporterArtifactPath(options) {
|
|
|
13221
14107
|
});
|
|
13222
14108
|
return { ok: false, outputDir, diagnostics };
|
|
13223
14109
|
}
|
|
13224
|
-
if (
|
|
14110
|
+
if (path14.isAbsolute(rawPath) || path14.win32.isAbsolute(rawPath)) {
|
|
13225
14111
|
diagnostics.push({
|
|
13226
14112
|
code: "artifact_path_absolute",
|
|
13227
14113
|
severity: "error",
|
|
@@ -13230,7 +14116,7 @@ function validateReporterArtifactPath(options) {
|
|
|
13230
14116
|
});
|
|
13231
14117
|
return { ok: false, outputDir, diagnostics };
|
|
13232
14118
|
}
|
|
13233
|
-
const normalized =
|
|
14119
|
+
const normalized = path14.posix.normalize(rawPath.replace(/\\/g, "/"));
|
|
13234
14120
|
const segments = normalized.split("/");
|
|
13235
14121
|
if (normalized === "." || normalized.startsWith("../") || segments.some((segment) => segment === "..")) {
|
|
13236
14122
|
diagnostics.push({
|
|
@@ -13241,9 +14127,9 @@ function validateReporterArtifactPath(options) {
|
|
|
13241
14127
|
});
|
|
13242
14128
|
return { ok: false, outputDir, diagnostics };
|
|
13243
14129
|
}
|
|
13244
|
-
const absolutePath =
|
|
13245
|
-
const relFromOutput =
|
|
13246
|
-
if (relFromOutput.length === 0 || relFromOutput.startsWith("..") ||
|
|
14130
|
+
const absolutePath = path14.resolve(outputDir, normalized);
|
|
14131
|
+
const relFromOutput = path14.relative(outputDir, absolutePath);
|
|
14132
|
+
if (relFromOutput.length === 0 || relFromOutput.startsWith("..") || path14.isAbsolute(relFromOutput)) {
|
|
13247
14133
|
diagnostics.push({
|
|
13248
14134
|
code: "artifact_path_escape",
|
|
13249
14135
|
severity: "error",
|
|
@@ -13389,23 +14275,23 @@ function readManifestDocument(value) {
|
|
|
13389
14275
|
};
|
|
13390
14276
|
}
|
|
13391
14277
|
function cwdRelative(filePath) {
|
|
13392
|
-
const relative =
|
|
13393
|
-
if (relative === "" || relative.startsWith("../") ||
|
|
13394
|
-
return
|
|
14278
|
+
const relative = path14.relative(process.cwd(), path14.resolve(filePath)).replace(/\\/g, "/");
|
|
14279
|
+
if (relative === "" || relative.startsWith("../") || path14.isAbsolute(relative)) {
|
|
14280
|
+
return path14.basename(filePath);
|
|
13395
14281
|
}
|
|
13396
14282
|
return relative;
|
|
13397
14283
|
}
|
|
13398
14284
|
async function readReporterManifest(filePath) {
|
|
13399
|
-
const absolute =
|
|
14285
|
+
const absolute = path14.resolve(filePath);
|
|
13400
14286
|
const raw = await readFile(absolute, "utf-8");
|
|
13401
14287
|
const document = readManifestDocument(JSON.parse(raw));
|
|
13402
14288
|
const manifest = document.manifest;
|
|
13403
14289
|
const results = manifest.results.map((result) => ({
|
|
13404
14290
|
testId: safeText(result.testId),
|
|
13405
14291
|
name: safeText(result.name),
|
|
13406
|
-
...result.file === void 0 ? {} : { file: safeText(
|
|
14292
|
+
...result.file === void 0 ? {} : { file: safeText(path14.basename(result.file)) },
|
|
13407
14293
|
status: result.status,
|
|
13408
|
-
...result.tracePath === void 0 ? {} : { tracePath: safeText(
|
|
14294
|
+
...result.tracePath === void 0 ? {} : { tracePath: safeText(path14.basename(result.tracePath)) },
|
|
13409
14295
|
artifacts: result.artifacts,
|
|
13410
14296
|
diagnostics: result.diagnostics
|
|
13411
14297
|
}));
|
|
@@ -13544,15 +14430,15 @@ async function ciSummaryCommand(manifestPaths, options = {}) {
|
|
|
13544
14430
|
return;
|
|
13545
14431
|
}
|
|
13546
14432
|
const markdown = renderMarkdown2(result);
|
|
13547
|
-
const outputPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
14433
|
+
const outputPath = options.output !== void 0 && options.output.trim() !== "" ? path14.resolve(options.output.trim()) : void 0;
|
|
13548
14434
|
if (outputPath !== void 0) {
|
|
13549
|
-
await mkdir(
|
|
14435
|
+
await mkdir(path14.dirname(outputPath), { recursive: true });
|
|
13550
14436
|
await writeFile(outputPath, markdown, "utf-8");
|
|
13551
14437
|
}
|
|
13552
14438
|
const summaryTarget = options.githubSummary ?? process.env.GITHUB_STEP_SUMMARY;
|
|
13553
14439
|
if (summaryTarget !== void 0 && summaryTarget.trim() !== "") {
|
|
13554
|
-
const summaryPath =
|
|
13555
|
-
await mkdir(
|
|
14440
|
+
const summaryPath = path14.resolve(summaryTarget);
|
|
14441
|
+
await mkdir(path14.dirname(summaryPath), { recursive: true });
|
|
13556
14442
|
await appendFile(summaryPath, `
|
|
13557
14443
|
${markdown}`, "utf-8");
|
|
13558
14444
|
}
|
|
@@ -13693,9 +14579,20 @@ function createCliProgram() {
|
|
|
13693
14579
|
]).option("--max-total-tokens <number>", "add llm.usage with a max total-token budget").option("--session <id>", "check all runs in a workflow session (requires --dir)").option("--group <id>", "check all runs sharing a groupId (requires --dir)").option(
|
|
13694
14580
|
"--correlate-group",
|
|
13695
14581
|
"when using --session, also match synthetic group: session keys"
|
|
14582
|
+
).option(
|
|
14583
|
+
"--guardrails <rule>",
|
|
14584
|
+
"run optional guardrail rules (repeatable): banned-phrase, pii-leak, prompt-injection, ...",
|
|
14585
|
+
(value, previous = []) => [...previous, value]
|
|
14586
|
+
).option(
|
|
14587
|
+
"--circuit <rule>",
|
|
14588
|
+
"run optional circuit rules (repeatable): same-tool-repetition, max-retries, ...",
|
|
14589
|
+
(value, previous = []) => [...previous, value]
|
|
13696
14590
|
).action((target, opts) => {
|
|
13697
14591
|
runCommand(() => checkCommand(target, opts));
|
|
13698
14592
|
});
|
|
14593
|
+
program.command("serve").description("Start optional localhost read-only trace viewer").option("--dir <path>", "trace directory to serve").option("--host <host>", "bind host (default 127.0.0.1)", "127.0.0.1").option("--port <number>", "bind port (default 7337)", "7337").option("--open", "open browser locally when host is localhost").action((opts) => {
|
|
14594
|
+
runCommand(() => serveCommand(opts));
|
|
14595
|
+
});
|
|
13699
14596
|
program.command("eval").description("Run deterministic local evals against a trace").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
13700
14597
|
new Option("--format <format>", "trace input format").choices([
|
|
13701
14598
|
"agent-inspect-jsonl",
|
|
@@ -13860,9 +14757,9 @@ function isPrimaryModule() {
|
|
|
13860
14757
|
if (!entry) return false;
|
|
13861
14758
|
const selfPath = fileURLToPath(import.meta.url);
|
|
13862
14759
|
try {
|
|
13863
|
-
return realpathSync(
|
|
14760
|
+
return realpathSync(path14.resolve(entry)) === realpathSync(path14.resolve(selfPath));
|
|
13864
14761
|
} catch {
|
|
13865
|
-
return
|
|
14762
|
+
return path14.resolve(entry) === path14.resolve(selfPath);
|
|
13866
14763
|
}
|
|
13867
14764
|
}
|
|
13868
14765
|
if (isPrimaryModule()) {
|