agent-inspect 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +145 -21
- package/docs/ADAPTER-CONFORMANCE.md +7 -3
- package/docs/ADAPTERS.md +155 -5
- package/docs/API.md +214 -26
- package/docs/CLI.md +189 -7
- package/docs/GETTING-STARTED.md +71 -16
- package/docs/KNOWN-ISSUES.md +7 -1
- package/docs/LIMITATIONS.md +7 -1
- package/docs/LOG-TO-TREE-QUICKSTART.md +1 -2
- package/docs/MIGRATION.md +67 -0
- package/docs/SCHEMA.md +1 -0
- package/package.json +12 -2
- package/packages/cli/dist/index.cjs +2454 -140
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +2454 -140
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +4 -4
- package/packages/core/dist/advanced.d.ts +4 -4
- package/packages/core/dist/checks.cjs +1535 -0
- package/packages/core/dist/checks.cjs.map +1 -0
- package/packages/core/dist/checks.d.cts +585 -0
- package/packages/core/dist/checks.d.ts +585 -0
- package/packages/core/dist/checks.mjs +1512 -0
- package/packages/core/dist/checks.mjs.map +1 -0
- package/packages/core/dist/diff.d.cts +3 -3
- package/packages/core/dist/diff.d.ts +3 -3
- package/packages/core/dist/exporters.d.cts +3 -3
- package/packages/core/dist/exporters.d.ts +3 -3
- package/packages/core/dist/index.cjs +146 -0
- package/packages/core/dist/index.cjs.map +1 -1
- package/packages/core/dist/index.d.cts +44 -7
- package/packages/core/dist/index.d.ts +44 -7
- package/packages/core/dist/index.mjs +148 -1
- package/packages/core/dist/index.mjs.map +1 -1
- package/packages/core/dist/{inspect-event-Des4JDHo.d.cts → inspect-event-CevRYp58.d.cts} +1 -1
- package/packages/core/dist/{inspect-event-Des4JDHo.d.ts → inspect-event-CevRYp58.d.ts} +1 -1
- package/packages/core/dist/{log-config-C1GcJPIM.d.ts → log-config-BPHS4Sds.d.ts} +1 -1
- package/packages/core/dist/{log-config-BnH8Ykcb.d.cts → log-config-DanPV3P9.d.cts} +1 -1
- package/packages/core/dist/logs.d.cts +3 -3
- package/packages/core/dist/logs.d.ts +3 -3
- package/packages/core/dist/{persisted-inspect-event-DiFto0K2.d.ts → persisted-inspect-event-Cw7TeYGr.d.ts} +1 -1
- package/packages/core/dist/{persisted-inspect-event-0kaRADsp.d.cts → persisted-inspect-event-DHPfzUd8.d.cts} +1 -1
- package/packages/core/dist/persisted.d.cts +5 -5
- package/packages/core/dist/persisted.d.ts +5 -5
- package/packages/core/dist/readers.d.cts +2 -2
- package/packages/core/dist/readers.d.ts +2 -2
- package/packages/core/dist/{types-tSix7tfv.d.ts → types-Ap9uMdx_.d.ts} +1 -1
- package/packages/core/dist/{types-DB8jB6Jg.d.cts → types-B2-BU5CS.d.cts} +1 -1
- package/packages/core/dist/writers.d.cts +2 -2
- package/packages/core/dist/writers.d.ts +2 -2
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
4
|
var fs = require('fs');
|
|
5
|
-
var
|
|
5
|
+
var path10 = require('path');
|
|
6
6
|
var url = require('url');
|
|
7
7
|
var commander = require('commander');
|
|
8
8
|
var promises = require('fs/promises');
|
|
@@ -16,14 +16,14 @@ var tty = require('tty');
|
|
|
16
16
|
var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
|
|
17
17
|
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
18
18
|
|
|
19
|
-
var
|
|
19
|
+
var path10__default = /*#__PURE__*/_interopDefault(path10);
|
|
20
20
|
var crypto__default = /*#__PURE__*/_interopDefault(crypto);
|
|
21
21
|
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
22
22
|
var process2__default = /*#__PURE__*/_interopDefault(process2);
|
|
23
23
|
var tty__default = /*#__PURE__*/_interopDefault(tty);
|
|
24
24
|
|
|
25
25
|
// package.json
|
|
26
|
-
var version = "1.
|
|
26
|
+
var version = "1.9.0";
|
|
27
27
|
|
|
28
28
|
// packages/core/src/types.ts
|
|
29
29
|
var STEP_TYPES = [
|
|
@@ -1905,7 +1905,7 @@ function formatDuration(ms) {
|
|
|
1905
1905
|
// packages/core/src/utils.ts
|
|
1906
1906
|
var DEFAULT_TRACE_DIR_NAME = ".agent-inspect";
|
|
1907
1907
|
var RUNS_DIR_NAME = "runs";
|
|
1908
|
-
var FALLBACK_TRACE_DIR =
|
|
1908
|
+
var FALLBACK_TRACE_DIR = path10__default.default.join(
|
|
1909
1909
|
os__default.default.tmpdir(),
|
|
1910
1910
|
"agent-inspect",
|
|
1911
1911
|
RUNS_DIR_NAME
|
|
@@ -1943,7 +1943,7 @@ function getDefaultTraceDir() {
|
|
|
1943
1943
|
if (typeof home !== "string" || home.trim() === "") {
|
|
1944
1944
|
return FALLBACK_TRACE_DIR;
|
|
1945
1945
|
}
|
|
1946
|
-
return
|
|
1946
|
+
return path10__default.default.join(home, DEFAULT_TRACE_DIR_NAME, RUNS_DIR_NAME);
|
|
1947
1947
|
} catch {
|
|
1948
1948
|
return FALLBACK_TRACE_DIR;
|
|
1949
1949
|
}
|
|
@@ -1951,20 +1951,20 @@ function getDefaultTraceDir() {
|
|
|
1951
1951
|
function getTraceFilePath(runId, traceDir) {
|
|
1952
1952
|
const baseDir = traceDir ?? getDefaultTraceDir();
|
|
1953
1953
|
let safeId = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "run_unknown";
|
|
1954
|
-
safeId =
|
|
1954
|
+
safeId = path10__default.default.basename(safeId);
|
|
1955
1955
|
if (safeId === "" || safeId === "." || safeId === "..") {
|
|
1956
1956
|
safeId = "run_unknown";
|
|
1957
1957
|
}
|
|
1958
|
-
return
|
|
1958
|
+
return path10__default.default.join(baseDir, `${safeId}.jsonl`);
|
|
1959
1959
|
}
|
|
1960
1960
|
async function ensureTraceDir(traceDir) {
|
|
1961
|
-
const primary =
|
|
1961
|
+
const primary = path10__default.default.resolve(traceDir);
|
|
1962
1962
|
try {
|
|
1963
1963
|
await promises.mkdir(primary, { recursive: true });
|
|
1964
1964
|
return primary;
|
|
1965
1965
|
} catch {
|
|
1966
1966
|
warn(`Failed to create trace directory: ${primary}`);
|
|
1967
|
-
const fallback =
|
|
1967
|
+
const fallback = path10__default.default.resolve(FALLBACK_TRACE_DIR);
|
|
1968
1968
|
try {
|
|
1969
1969
|
await promises.mkdir(fallback, { recursive: true });
|
|
1970
1970
|
return fallback;
|
|
@@ -2638,7 +2638,7 @@ var TraceDirectory = class {
|
|
|
2638
2638
|
this.#dir = resolveTraceDir(options);
|
|
2639
2639
|
}
|
|
2640
2640
|
getPath(filename) {
|
|
2641
|
-
return filename ?
|
|
2641
|
+
return filename ? path10__default.default.join(this.#dir, filename) : this.#dir;
|
|
2642
2642
|
}
|
|
2643
2643
|
async list() {
|
|
2644
2644
|
try {
|
|
@@ -2665,7 +2665,7 @@ function parseIsoToMs3(value) {
|
|
|
2665
2665
|
}
|
|
2666
2666
|
async function extractMetadata(filePath, _quickScan) {
|
|
2667
2667
|
const stats = await promises.stat(filePath);
|
|
2668
|
-
let runIdFromFile =
|
|
2668
|
+
let runIdFromFile = path10__default.default.basename(filePath);
|
|
2669
2669
|
if (runIdFromFile.endsWith(".jsonl")) {
|
|
2670
2670
|
runIdFromFile = runIdFromFile.slice(0, -".jsonl".length);
|
|
2671
2671
|
}
|
|
@@ -4069,6 +4069,151 @@ ${attrsSection}
|
|
|
4069
4069
|
};
|
|
4070
4070
|
}
|
|
4071
4071
|
|
|
4072
|
+
// packages/core/src/explain.ts
|
|
4073
|
+
function flatten(nodes, out = []) {
|
|
4074
|
+
for (const node of nodes) {
|
|
4075
|
+
out.push({ node, index: out.length + 1 });
|
|
4076
|
+
flatten(node.children, out);
|
|
4077
|
+
}
|
|
4078
|
+
return out;
|
|
4079
|
+
}
|
|
4080
|
+
function redactValue(redactor, key, value) {
|
|
4081
|
+
return redactor.redactValue(key, value);
|
|
4082
|
+
}
|
|
4083
|
+
function fact(id, label, value, redactor) {
|
|
4084
|
+
return {
|
|
4085
|
+
id,
|
|
4086
|
+
label,
|
|
4087
|
+
value: redactValue(redactor, id.split(".").at(-1) ?? id, value),
|
|
4088
|
+
source: "trace",
|
|
4089
|
+
confidence: "observed"
|
|
4090
|
+
};
|
|
4091
|
+
}
|
|
4092
|
+
function topKinds(run) {
|
|
4093
|
+
return Object.entries(run.metadata.kinds).filter(([, count]) => count > 0).sort((a, b) => {
|
|
4094
|
+
if (b[1] !== a[1]) return b[1] - a[1];
|
|
4095
|
+
return a[0].localeCompare(b[0]);
|
|
4096
|
+
}).slice(0, 5).map(([kind, count]) => `${kind}:${count}`);
|
|
4097
|
+
}
|
|
4098
|
+
function countErrorNodes(nodes) {
|
|
4099
|
+
return nodes.filter((entry) => entry.node.event.status === "error").length;
|
|
4100
|
+
}
|
|
4101
|
+
function slowestNode(nodes) {
|
|
4102
|
+
return nodes.filter((entry) => entry.node.event.durationMs !== void 0).sort((a, b) => {
|
|
4103
|
+
const delta = (b.node.event.durationMs ?? 0) - (a.node.event.durationMs ?? 0);
|
|
4104
|
+
return delta !== 0 ? delta : a.index - b.index;
|
|
4105
|
+
})[0];
|
|
4106
|
+
}
|
|
4107
|
+
function attributeFacts(nodes, redactor) {
|
|
4108
|
+
const facts = [];
|
|
4109
|
+
for (const entry of nodes) {
|
|
4110
|
+
const attrs = entry.node.event.attributes;
|
|
4111
|
+
if (attrs === void 0) continue;
|
|
4112
|
+
for (const key of Object.keys(attrs).sort()) {
|
|
4113
|
+
facts.push({
|
|
4114
|
+
id: `node.${entry.index}.attributes.${key}`,
|
|
4115
|
+
label: `${entry.node.event.name} attribute ${key}`,
|
|
4116
|
+
value: redactValue(redactor, key, attrs[key]),
|
|
4117
|
+
source: "trace",
|
|
4118
|
+
confidence: "observed"
|
|
4119
|
+
});
|
|
4120
|
+
if (facts.length >= 8) return facts;
|
|
4121
|
+
}
|
|
4122
|
+
}
|
|
4123
|
+
return facts;
|
|
4124
|
+
}
|
|
4125
|
+
function buildFacts(run, redactor) {
|
|
4126
|
+
const nodes = flatten(run.children);
|
|
4127
|
+
const facts = [
|
|
4128
|
+
fact("run.id", "Run id", run.runId, redactor),
|
|
4129
|
+
fact("run.name", "Run name", run.name ?? run.runId, redactor),
|
|
4130
|
+
fact("run.status", "Run status", run.status ?? "unknown", redactor),
|
|
4131
|
+
fact("run.totalEvents", "Total events", run.metadata.totalEvents, redactor),
|
|
4132
|
+
fact("run.stepCount", "Top-level step count", run.children.length, redactor),
|
|
4133
|
+
fact("run.nodeCount", "Total node count", nodes.length, redactor),
|
|
4134
|
+
fact("run.errorNodeCount", "Error node count", countErrorNodes(nodes), redactor),
|
|
4135
|
+
fact("run.kinds", "Observed kind mix", topKinds(run), redactor)
|
|
4136
|
+
];
|
|
4137
|
+
if (run.durationMs !== void 0) {
|
|
4138
|
+
facts.push(fact("run.durationMs", "Run duration milliseconds", run.durationMs, redactor));
|
|
4139
|
+
}
|
|
4140
|
+
const slowest = slowestNode(nodes);
|
|
4141
|
+
if (slowest !== void 0) {
|
|
4142
|
+
facts.push(
|
|
4143
|
+
fact("run.slowestNode", "Slowest observed node", {
|
|
4144
|
+
name: slowest.node.event.name,
|
|
4145
|
+
kind: slowest.node.event.kind,
|
|
4146
|
+
durationMs: slowest.node.event.durationMs
|
|
4147
|
+
}, redactor)
|
|
4148
|
+
);
|
|
4149
|
+
}
|
|
4150
|
+
facts.push(...attributeFacts(nodes, redactor));
|
|
4151
|
+
return facts;
|
|
4152
|
+
}
|
|
4153
|
+
function buildInferences(run, facts) {
|
|
4154
|
+
const inferences = [];
|
|
4155
|
+
const errorFact = facts.find((item) => item.id === "run.errorNodeCount");
|
|
4156
|
+
const kindFact = facts.find((item) => item.id === "run.kinds");
|
|
4157
|
+
const durationFact = facts.find((item) => item.id === "run.durationMs");
|
|
4158
|
+
const errorNodeCount = typeof errorFact?.value === "number" ? errorFact.value : 0;
|
|
4159
|
+
if (run.status === "error" || errorNodeCount > 0) {
|
|
4160
|
+
inferences.push({
|
|
4161
|
+
id: "outcome.error",
|
|
4162
|
+
label: "Outcome",
|
|
4163
|
+
text: "The run recorded an error status or at least one error node.",
|
|
4164
|
+
evidence: ["run.status", "run.errorNodeCount"],
|
|
4165
|
+
confidence: "deterministic"
|
|
4166
|
+
});
|
|
4167
|
+
} else if (run.status === "ok") {
|
|
4168
|
+
inferences.push({
|
|
4169
|
+
id: "outcome.success",
|
|
4170
|
+
label: "Outcome",
|
|
4171
|
+
text: "The run completed without observed error nodes.",
|
|
4172
|
+
evidence: ["run.status", "run.errorNodeCount"],
|
|
4173
|
+
confidence: "deterministic"
|
|
4174
|
+
});
|
|
4175
|
+
}
|
|
4176
|
+
if (kindFact !== void 0) {
|
|
4177
|
+
inferences.push({
|
|
4178
|
+
id: "shape.kind-mix",
|
|
4179
|
+
label: "Trace shape",
|
|
4180
|
+
text: "The explanation is based on the observed event kind mix, not generated content.",
|
|
4181
|
+
evidence: [kindFact.id],
|
|
4182
|
+
confidence: "deterministic"
|
|
4183
|
+
});
|
|
4184
|
+
}
|
|
4185
|
+
if (durationFact !== void 0) {
|
|
4186
|
+
inferences.push({
|
|
4187
|
+
id: "timing.duration",
|
|
4188
|
+
label: "Timing",
|
|
4189
|
+
text: "Timing claims are limited to persisted duration fields in the trace.",
|
|
4190
|
+
evidence: [durationFact.id],
|
|
4191
|
+
confidence: "deterministic"
|
|
4192
|
+
});
|
|
4193
|
+
}
|
|
4194
|
+
return inferences;
|
|
4195
|
+
}
|
|
4196
|
+
function buildLocalExplanation(run, options = {}) {
|
|
4197
|
+
const redactionProfile = options.redactionProfile ?? "local";
|
|
4198
|
+
const resolved = resolveRedactionProfile(redactionProfile);
|
|
4199
|
+
const redactor = new Redactor({ extraKeys: resolved.extraKeys });
|
|
4200
|
+
const mode = options.mode ?? "local";
|
|
4201
|
+
const facts = buildFacts(run, redactor);
|
|
4202
|
+
return {
|
|
4203
|
+
mode,
|
|
4204
|
+
runId: String(redactValue(redactor, "runId", run.runId)),
|
|
4205
|
+
...run.name !== void 0 ? { name: String(redactValue(redactor, "name", run.name)) } : {},
|
|
4206
|
+
...run.status !== void 0 ? { status: run.status } : {},
|
|
4207
|
+
redactionProfile,
|
|
4208
|
+
facts,
|
|
4209
|
+
inferences: mode === "dry-run" ? [] : buildInferences(run, facts),
|
|
4210
|
+
notes: [
|
|
4211
|
+
"Generated locally without provider or network calls.",
|
|
4212
|
+
"Facts are observed from normalized trace data; inferences are deterministic labels."
|
|
4213
|
+
]
|
|
4214
|
+
};
|
|
4215
|
+
}
|
|
4216
|
+
|
|
4072
4217
|
// packages/core/src/stats.ts
|
|
4073
4218
|
function percentile(sorted, p) {
|
|
4074
4219
|
if (sorted.length === 0) return void 0;
|
|
@@ -4420,8 +4565,8 @@ function matchStepLevel(m, events, opts) {
|
|
|
4420
4565
|
fields.push("step.name");
|
|
4421
4566
|
}
|
|
4422
4567
|
if (opts.toolQuery) {
|
|
4423
|
-
const
|
|
4424
|
-
if (!nameMatches(
|
|
4568
|
+
const toolName2 = typeof s.metadata?.toolName === "string" ? s.metadata.toolName : s.name;
|
|
4569
|
+
if (!nameMatches(toolName2, opts.toolQuery)) continue;
|
|
4425
4570
|
fields.push("step.tool");
|
|
4426
4571
|
}
|
|
4427
4572
|
if (opts.durationFilter) {
|
|
@@ -4626,7 +4771,7 @@ function findReaderByFormat(format, readers) {
|
|
|
4626
4771
|
}
|
|
4627
4772
|
async function jsonlFilesInDirectory(dirPath) {
|
|
4628
4773
|
const entries = await promises.readdir(dirPath, { withFileTypes: true });
|
|
4629
|
-
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) =>
|
|
4774
|
+
return entries.filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")).map((entry) => path10__default.default.join(dirPath, entry.name)).sort((a, b) => a.localeCompare(b));
|
|
4630
4775
|
}
|
|
4631
4776
|
async function resolveInput(input3) {
|
|
4632
4777
|
const cached = resolvedInputCache.get(input3);
|
|
@@ -6206,13 +6351,13 @@ function pairSteps(left, right) {
|
|
|
6206
6351
|
return pairs;
|
|
6207
6352
|
}
|
|
6208
6353
|
function compareLeafSteps(L, R, segments, opts, out) {
|
|
6209
|
-
const
|
|
6354
|
+
const path12 = buildPath(segments);
|
|
6210
6355
|
if (L.name !== R.name) {
|
|
6211
6356
|
out.push({
|
|
6212
6357
|
kind: "structure",
|
|
6213
6358
|
severity: "warning",
|
|
6214
6359
|
message: "Step name differs",
|
|
6215
|
-
path:
|
|
6360
|
+
path: path12,
|
|
6216
6361
|
left: L.name,
|
|
6217
6362
|
right: R.name
|
|
6218
6363
|
});
|
|
@@ -6222,7 +6367,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
6222
6367
|
kind: "step-type",
|
|
6223
6368
|
severity: "warning",
|
|
6224
6369
|
message: "Step type differs",
|
|
6225
|
-
path:
|
|
6370
|
+
path: path12,
|
|
6226
6371
|
left: L.type,
|
|
6227
6372
|
right: R.type
|
|
6228
6373
|
});
|
|
@@ -6232,7 +6377,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
6232
6377
|
kind: "step-status",
|
|
6233
6378
|
severity: "warning",
|
|
6234
6379
|
message: "Step status differs",
|
|
6235
|
-
path:
|
|
6380
|
+
path: path12,
|
|
6236
6381
|
left: L.status,
|
|
6237
6382
|
right: R.status
|
|
6238
6383
|
});
|
|
@@ -6244,7 +6389,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
6244
6389
|
kind: "error",
|
|
6245
6390
|
severity: "error",
|
|
6246
6391
|
message: "Step error message differs",
|
|
6247
|
-
path:
|
|
6392
|
+
path: path12,
|
|
6248
6393
|
left: le || void 0,
|
|
6249
6394
|
right: re || void 0
|
|
6250
6395
|
});
|
|
@@ -6262,7 +6407,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
6262
6407
|
kind: "duration",
|
|
6263
6408
|
severity: "info",
|
|
6264
6409
|
message: "Step duration differs",
|
|
6265
|
-
path:
|
|
6410
|
+
path: path12,
|
|
6266
6411
|
left: ld,
|
|
6267
6412
|
right: rd
|
|
6268
6413
|
});
|
|
@@ -6275,7 +6420,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
6275
6420
|
kind: "metadata",
|
|
6276
6421
|
severity: "info",
|
|
6277
6422
|
message: "Step metadata differs",
|
|
6278
|
-
path:
|
|
6423
|
+
path: path12,
|
|
6279
6424
|
left: L.metadata,
|
|
6280
6425
|
right: R.metadata
|
|
6281
6426
|
});
|
|
@@ -6287,7 +6432,7 @@ function compareLeafSteps(L, R, segments, opts, out) {
|
|
|
6287
6432
|
kind: "output",
|
|
6288
6433
|
severity: "info",
|
|
6289
6434
|
message: "Output preview differs",
|
|
6290
|
-
path:
|
|
6435
|
+
path: path12,
|
|
6291
6436
|
left: L.outputPreview,
|
|
6292
6437
|
right: R.outputPreview
|
|
6293
6438
|
});
|
|
@@ -6937,11 +7082,11 @@ createChalk({ level: stderrColor ? stderrColor.level : 0 });
|
|
|
6937
7082
|
var source_default = chalk;
|
|
6938
7083
|
|
|
6939
7084
|
// packages/core/src/diff/renderer.ts
|
|
6940
|
-
function formatPath(
|
|
6941
|
-
if (
|
|
7085
|
+
function formatPath(path12) {
|
|
7086
|
+
if (path12 === void 0 || path12.path.length === 0) {
|
|
6942
7087
|
return "(run)";
|
|
6943
7088
|
}
|
|
6944
|
-
return
|
|
7089
|
+
return path12.path.map((s) => s.name).join(" > ");
|
|
6945
7090
|
}
|
|
6946
7091
|
function formatValue(v, verbose) {
|
|
6947
7092
|
if (v === void 0) return "(undefined)";
|
|
@@ -7222,8 +7367,8 @@ async function stepLlm(model, fn) {
|
|
|
7222
7367
|
metadata: { model: modelName }
|
|
7223
7368
|
});
|
|
7224
7369
|
}
|
|
7225
|
-
async function stepTool(
|
|
7226
|
-
const normalized = typeof
|
|
7370
|
+
async function stepTool(toolName2, fn) {
|
|
7371
|
+
const normalized = typeof toolName2 === "string" && toolName2.trim() !== "" ? toolName2.trim() : "unknown-tool";
|
|
7227
7372
|
return stepImpl(`tool:${normalized}`, fn, {
|
|
7228
7373
|
type: "tool",
|
|
7229
7374
|
metadata: { toolName: normalized }
|
|
@@ -8534,9 +8679,9 @@ Trace directory: ${traceDir}`);
|
|
|
8534
8679
|
if (validation !== void 0 && !validation.ok) {
|
|
8535
8680
|
process.exitCode = 1;
|
|
8536
8681
|
}
|
|
8537
|
-
const outPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
8682
|
+
const outPath = options.output !== void 0 && options.output.trim() !== "" ? path10__default.default.resolve(options.output.trim()) : void 0;
|
|
8538
8683
|
if (outPath !== void 0) {
|
|
8539
|
-
await promises.mkdir(
|
|
8684
|
+
await promises.mkdir(path10__default.default.dirname(outPath), { recursive: true });
|
|
8540
8685
|
await promises.writeFile(outPath, result.content, "utf-8");
|
|
8541
8686
|
const vlabel = validation !== void 0 ? validation.ok ? "ok" : "failed" : "skipped";
|
|
8542
8687
|
console.log(`Wrote ${result.fileExtension} export to ${outPath} (validation: ${vlabel})`);
|
|
@@ -8928,9 +9073,9 @@ async function reportCommand(runId, options = {}) {
|
|
|
8928
9073
|
redactionProfile,
|
|
8929
9074
|
correlation: !options.noCorrelation
|
|
8930
9075
|
});
|
|
8931
|
-
const outPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
9076
|
+
const outPath = options.output !== void 0 && options.output.trim() !== "" ? path10__default.default.resolve(options.output.trim()) : void 0;
|
|
8932
9077
|
if (outPath !== void 0) {
|
|
8933
|
-
await promises.mkdir(
|
|
9078
|
+
await promises.mkdir(path10__default.default.dirname(outPath), { recursive: true });
|
|
8934
9079
|
await promises.writeFile(outPath, result.content, "utf-8");
|
|
8935
9080
|
console.log(`Wrote ${result.fileExtension} report to ${outPath}`);
|
|
8936
9081
|
}
|
|
@@ -8958,9 +9103,164 @@ async function readStdin(stdin) {
|
|
|
8958
9103
|
}
|
|
8959
9104
|
return content;
|
|
8960
9105
|
}
|
|
9106
|
+
function isMissingFileError2(error) {
|
|
9107
|
+
return error !== null && typeof error === "object" && "code" in error && error.code === "ENOENT";
|
|
9108
|
+
}
|
|
9109
|
+
async function inputFromTarget(target, options, stdin) {
|
|
9110
|
+
if (target === "-") {
|
|
9111
|
+
return { type: "string", content: await readStdin(stdin) };
|
|
9112
|
+
}
|
|
9113
|
+
try {
|
|
9114
|
+
const stats2 = await promises.stat(target);
|
|
9115
|
+
if (stats2.isDirectory()) return { type: "directory", path: target };
|
|
9116
|
+
return { type: "file", path: target };
|
|
9117
|
+
} catch (error) {
|
|
9118
|
+
if (!isMissingFileError2(error)) throw error;
|
|
9119
|
+
}
|
|
9120
|
+
const runPath = getTraceFilePath(target, resolveTraceDir({ dir: options.dir }));
|
|
9121
|
+
const stats = await promises.stat(runPath);
|
|
9122
|
+
if (stats.isDirectory()) return { type: "directory", path: runPath };
|
|
9123
|
+
return { type: "file", path: runPath };
|
|
9124
|
+
}
|
|
9125
|
+
|
|
9126
|
+
// packages/cli/src/explain.ts
|
|
9127
|
+
function parseRedactionProfile3(value) {
|
|
9128
|
+
const profile = (value ?? "local").trim().toLowerCase();
|
|
9129
|
+
if (profile === "local" || profile === "share" || profile === "strict") {
|
|
9130
|
+
return profile;
|
|
9131
|
+
}
|
|
9132
|
+
throw new Error(
|
|
9133
|
+
`Unsupported --redaction-profile "${value ?? ""}". Use local, share, or strict.`
|
|
9134
|
+
);
|
|
9135
|
+
}
|
|
9136
|
+
function selectRun(result, runId) {
|
|
9137
|
+
if (runId !== void 0) {
|
|
9138
|
+
return result.runs.find((run) => run.runId === runId);
|
|
9139
|
+
}
|
|
9140
|
+
return result.runs.length === 1 ? result.runs[0] : void 0;
|
|
9141
|
+
}
|
|
9142
|
+
function printMultipleRuns(result) {
|
|
9143
|
+
console.error(
|
|
9144
|
+
`Trace contains ${result.runs.length} runs. Re-run with --run <run-id>.`
|
|
9145
|
+
);
|
|
9146
|
+
for (const run of result.runs) {
|
|
9147
|
+
console.error(`- ${run.runId}${run.name !== void 0 ? ` name=${run.name}` : ""}`);
|
|
9148
|
+
}
|
|
9149
|
+
}
|
|
9150
|
+
function renderHuman(result) {
|
|
9151
|
+
const lines = [
|
|
9152
|
+
`Explain: ${result.name ?? result.runId}`,
|
|
9153
|
+
`Mode: ${result.mode}`,
|
|
9154
|
+
`Status: ${result.status ?? "unknown"}`,
|
|
9155
|
+
`Redaction: ${result.redactionProfile}`,
|
|
9156
|
+
"",
|
|
9157
|
+
"Facts:"
|
|
9158
|
+
];
|
|
9159
|
+
for (const fact2 of result.facts) {
|
|
9160
|
+
lines.push(`- ${fact2.id}: ${JSON.stringify(fact2.value)}`);
|
|
9161
|
+
}
|
|
9162
|
+
lines.push("", "Inferences:");
|
|
9163
|
+
if (result.inferences.length === 0) {
|
|
9164
|
+
lines.push("- none");
|
|
9165
|
+
} else {
|
|
9166
|
+
for (const inference of result.inferences) {
|
|
9167
|
+
lines.push(`- ${inference.label}: ${inference.text}`);
|
|
9168
|
+
}
|
|
9169
|
+
}
|
|
9170
|
+
lines.push("", "Notes:");
|
|
9171
|
+
for (const note of result.notes) {
|
|
9172
|
+
lines.push(`- ${note}`);
|
|
9173
|
+
}
|
|
9174
|
+
return lines.join("\n");
|
|
9175
|
+
}
|
|
9176
|
+
function writeJson(result) {
|
|
9177
|
+
console.log(JSON.stringify(result, null, 2));
|
|
9178
|
+
}
|
|
9179
|
+
function rejectProvider(provider, json) {
|
|
9180
|
+
process.exitCode = 1;
|
|
9181
|
+
const message = `Provider explain is not implemented in this build: ${provider}. Use --dry-run to inspect the redacted local payload.`;
|
|
9182
|
+
if (json) {
|
|
9183
|
+
writeJson({
|
|
9184
|
+
ok: false,
|
|
9185
|
+
error: { code: "PROVIDER_NOT_IMPLEMENTED", message }
|
|
9186
|
+
});
|
|
9187
|
+
return;
|
|
9188
|
+
}
|
|
9189
|
+
console.error(message);
|
|
9190
|
+
}
|
|
9191
|
+
async function explainCommand(target, options = {}, stdin = process.stdin) {
|
|
9192
|
+
if (options.provider !== void 0) {
|
|
9193
|
+
rejectProvider(options.provider, options.json);
|
|
9194
|
+
return;
|
|
9195
|
+
}
|
|
9196
|
+
let redactionProfile;
|
|
9197
|
+
try {
|
|
9198
|
+
redactionProfile = parseRedactionProfile3(options.redactionProfile);
|
|
9199
|
+
} catch (error) {
|
|
9200
|
+
process.exitCode = 1;
|
|
9201
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
9202
|
+
return;
|
|
9203
|
+
}
|
|
9204
|
+
try {
|
|
9205
|
+
const input3 = await inputFromTarget(target, options, stdin);
|
|
9206
|
+
const read = await openTrace(input3, {
|
|
9207
|
+
...options.format !== void 0 ? { format: options.format } : {}
|
|
9208
|
+
});
|
|
9209
|
+
const selected = selectRun(read, options.run);
|
|
9210
|
+
if (selected === void 0) {
|
|
9211
|
+
process.exitCode = 1;
|
|
9212
|
+
const message = options.run !== void 0 ? `Run not found: ${options.run}` : `Trace contains ${read.runs.length} runs. Specify --run <run-id>.`;
|
|
9213
|
+
if (options.json) {
|
|
9214
|
+
writeJson({ ok: false, error: { message }, runs: read.runs });
|
|
9215
|
+
} else if (options.run !== void 0) {
|
|
9216
|
+
console.error(message);
|
|
9217
|
+
} else {
|
|
9218
|
+
printMultipleRuns(read);
|
|
9219
|
+
}
|
|
9220
|
+
return;
|
|
9221
|
+
}
|
|
9222
|
+
const mode = options.dryRun ? "dry-run" : "local";
|
|
9223
|
+
const explanation = buildLocalExplanation(selected, {
|
|
9224
|
+
mode,
|
|
9225
|
+
redactionProfile
|
|
9226
|
+
});
|
|
9227
|
+
if (options.json) {
|
|
9228
|
+
writeJson({
|
|
9229
|
+
ok: true,
|
|
9230
|
+
format: read.format,
|
|
9231
|
+
sourceFiles: read.sourceFiles,
|
|
9232
|
+
warnings: read.warnings,
|
|
9233
|
+
unsupportedFields: read.unsupportedFields,
|
|
9234
|
+
explanation
|
|
9235
|
+
});
|
|
9236
|
+
return;
|
|
9237
|
+
}
|
|
9238
|
+
console.log(renderHuman(explanation));
|
|
9239
|
+
} catch (error) {
|
|
9240
|
+
process.exitCode = 1;
|
|
9241
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
9242
|
+
const code = error instanceof TraceReadError ? error.code : void 0;
|
|
9243
|
+
if (options.json) {
|
|
9244
|
+
writeJson({
|
|
9245
|
+
ok: false,
|
|
9246
|
+
error: { ...code !== void 0 ? { code } : {}, message }
|
|
9247
|
+
});
|
|
9248
|
+
return;
|
|
9249
|
+
}
|
|
9250
|
+
console.error(message);
|
|
9251
|
+
}
|
|
9252
|
+
}
|
|
9253
|
+
async function readStdin2(stdin) {
|
|
9254
|
+
stdin.setEncoding("utf8");
|
|
9255
|
+
let content = "";
|
|
9256
|
+
for await (const chunk of stdin) {
|
|
9257
|
+
content += typeof chunk === "string" ? chunk : String(chunk);
|
|
9258
|
+
}
|
|
9259
|
+
return content;
|
|
9260
|
+
}
|
|
8961
9261
|
async function inputFromPathOrStdin(input3, stdin) {
|
|
8962
9262
|
if (input3 === void 0 || input3 === "-") {
|
|
8963
|
-
return { type: "string", content: await
|
|
9263
|
+
return { type: "string", content: await readStdin2(stdin) };
|
|
8964
9264
|
}
|
|
8965
9265
|
const stats = await promises.stat(input3);
|
|
8966
9266
|
if (stats.isDirectory()) return { type: "directory", path: input3 };
|
|
@@ -9011,13 +9311,13 @@ function printRun(result, run) {
|
|
|
9011
9311
|
printNode(node, 0);
|
|
9012
9312
|
}
|
|
9013
9313
|
}
|
|
9014
|
-
function
|
|
9314
|
+
function selectRun2(result, runId) {
|
|
9015
9315
|
if (runId !== void 0) {
|
|
9016
9316
|
return result.runs.find((run) => run.runId === runId);
|
|
9017
9317
|
}
|
|
9018
9318
|
return result.runs.length === 1 ? result.runs[0] : void 0;
|
|
9019
9319
|
}
|
|
9020
|
-
function
|
|
9320
|
+
function printMultipleRuns2(result) {
|
|
9021
9321
|
console.error(
|
|
9022
9322
|
`Trace contains ${result.runs.length} runs. Re-run with --run <run-id>.`
|
|
9023
9323
|
);
|
|
@@ -9031,7 +9331,7 @@ function printMultipleRuns(result) {
|
|
|
9031
9331
|
console.error(`- ${bits.join(" ")}`);
|
|
9032
9332
|
}
|
|
9033
9333
|
}
|
|
9034
|
-
function
|
|
9334
|
+
function writeJson2(output2) {
|
|
9035
9335
|
console.log(JSON.stringify(output2, null, 2));
|
|
9036
9336
|
}
|
|
9037
9337
|
async function openCommand(input3, options = {}, stdin = process.stdin) {
|
|
@@ -9040,12 +9340,12 @@ async function openCommand(input3, options = {}, stdin = process.stdin) {
|
|
|
9040
9340
|
const result = await openTrace(traceInput, {
|
|
9041
9341
|
...options.format !== void 0 ? { format: options.format } : {}
|
|
9042
9342
|
});
|
|
9043
|
-
const selected =
|
|
9343
|
+
const selected = selectRun2(result, options.run);
|
|
9044
9344
|
if (selected === void 0) {
|
|
9045
9345
|
process.exitCode = 1;
|
|
9046
9346
|
const message = options.run !== void 0 ? `Run not found: ${options.run}` : `Trace contains ${result.runs.length} runs. Specify --run <run-id>.`;
|
|
9047
9347
|
if (options.json) {
|
|
9048
|
-
|
|
9348
|
+
writeJson2({
|
|
9049
9349
|
format: result.format,
|
|
9050
9350
|
sourceFiles: result.sourceFiles,
|
|
9051
9351
|
warnings: result.warnings,
|
|
@@ -9056,12 +9356,12 @@ async function openCommand(input3, options = {}, stdin = process.stdin) {
|
|
|
9056
9356
|
} else if (options.run !== void 0) {
|
|
9057
9357
|
console.error(message);
|
|
9058
9358
|
} else {
|
|
9059
|
-
|
|
9359
|
+
printMultipleRuns2(result);
|
|
9060
9360
|
}
|
|
9061
9361
|
return;
|
|
9062
9362
|
}
|
|
9063
9363
|
if (options.json) {
|
|
9064
|
-
|
|
9364
|
+
writeJson2({
|
|
9065
9365
|
format: result.format,
|
|
9066
9366
|
sourceFiles: result.sourceFiles,
|
|
9067
9367
|
warnings: result.warnings,
|
|
@@ -9081,7 +9381,7 @@ async function openCommand(input3, options = {}, stdin = process.stdin) {
|
|
|
9081
9381
|
const code = error instanceof TraceReadError ? error.code : void 0;
|
|
9082
9382
|
const warnings = error instanceof TraceReadError ? error.warnings : [];
|
|
9083
9383
|
if (options.json) {
|
|
9084
|
-
|
|
9384
|
+
writeJson2({
|
|
9085
9385
|
warnings,
|
|
9086
9386
|
error: { ...code !== void 0 ? { code } : {}, message }
|
|
9087
9387
|
});
|
|
@@ -9094,108 +9394,2105 @@ async function openCommand(input3, options = {}, stdin = process.stdin) {
|
|
|
9094
9394
|
}
|
|
9095
9395
|
}
|
|
9096
9396
|
|
|
9097
|
-
// packages/
|
|
9098
|
-
|
|
9099
|
-
|
|
9100
|
-
|
|
9101
|
-
|
|
9102
|
-
|
|
9103
|
-
|
|
9397
|
+
// packages/core/src/checks/index.ts
|
|
9398
|
+
var SEVERITY_RANK = {
|
|
9399
|
+
error: 0,
|
|
9400
|
+
warning: 1,
|
|
9401
|
+
info: 2
|
|
9402
|
+
};
|
|
9403
|
+
var STATUS_RANK = {
|
|
9404
|
+
fail: 0,
|
|
9405
|
+
warning: 1,
|
|
9406
|
+
pass: 2
|
|
9407
|
+
};
|
|
9408
|
+
var CONFIDENCE_RANK = {
|
|
9409
|
+
unknown: 0,
|
|
9410
|
+
heuristic: 1,
|
|
9411
|
+
correlated: 2,
|
|
9412
|
+
explicit: 3
|
|
9413
|
+
};
|
|
9414
|
+
var DEFAULT_SENSITIVE_KEYS = [
|
|
9415
|
+
"authorization",
|
|
9416
|
+
"cookie",
|
|
9417
|
+
"token",
|
|
9418
|
+
"apikey",
|
|
9419
|
+
"api_key",
|
|
9420
|
+
"password",
|
|
9421
|
+
"secret",
|
|
9422
|
+
"email"
|
|
9423
|
+
];
|
|
9424
|
+
var DEFAULT_RAW_CONTENT_KEYS = [
|
|
9425
|
+
"body",
|
|
9426
|
+
"headers",
|
|
9427
|
+
"input",
|
|
9428
|
+
"messages",
|
|
9429
|
+
"output",
|
|
9430
|
+
"payload",
|
|
9431
|
+
"prompt",
|
|
9432
|
+
"requestbody",
|
|
9433
|
+
"request_body",
|
|
9434
|
+
"responsebody",
|
|
9435
|
+
"response_body",
|
|
9436
|
+
"rawprompt",
|
|
9437
|
+
"raw_prompt",
|
|
9438
|
+
"rawoutput",
|
|
9439
|
+
"raw_output",
|
|
9440
|
+
"toolinput",
|
|
9441
|
+
"tool_input",
|
|
9442
|
+
"tooloutput",
|
|
9443
|
+
"tool_output"
|
|
9444
|
+
];
|
|
9445
|
+
var DEFAULT_SECRET_PATTERNS = [
|
|
9446
|
+
{ id: "bearer-token", pattern: /Bearer\s+[A-Za-z0-9._~+/-]{12,}=*/ },
|
|
9447
|
+
{ id: "openai-key", pattern: /sk-[A-Za-z0-9_-]{16,}/ },
|
|
9448
|
+
{ id: "aws-access-key", pattern: /AKIA[0-9A-Z]{16}/ },
|
|
9449
|
+
{ id: "github-token", pattern: /gh[opsu]_[A-Za-z0-9_]{20,}/ },
|
|
9450
|
+
{ id: "key-value-secret", pattern: /(api[_-]?key|token|password|secret)=\S{8,}/i }
|
|
9451
|
+
];
|
|
9452
|
+
function compareStrings(a, b) {
|
|
9453
|
+
return (a ?? "").localeCompare(b ?? "");
|
|
9104
9454
|
}
|
|
9105
|
-
function
|
|
9106
|
-
|
|
9107
|
-
|
|
9108
|
-
|
|
9109
|
-
|
|
9110
|
-
|
|
9111
|
-
|
|
9112
|
-
|
|
9113
|
-
|
|
9114
|
-
|
|
9115
|
-
|
|
9116
|
-
|
|
9117
|
-
|
|
9118
|
-
|
|
9119
|
-
|
|
9455
|
+
function diagnostic(code, message, ruleId) {
|
|
9456
|
+
return {
|
|
9457
|
+
code,
|
|
9458
|
+
message,
|
|
9459
|
+
severity: "error",
|
|
9460
|
+
...ruleId ? { ruleId } : {}
|
|
9461
|
+
};
|
|
9462
|
+
}
|
|
9463
|
+
function emptySummary() {
|
|
9464
|
+
return {
|
|
9465
|
+
passed: 0,
|
|
9466
|
+
failed: 0,
|
|
9467
|
+
warnings: 0,
|
|
9468
|
+
errors: 0
|
|
9469
|
+
};
|
|
9470
|
+
}
|
|
9471
|
+
function errorResult(input3, diagnostics, selectedRun) {
|
|
9472
|
+
return {
|
|
9473
|
+
ok: false,
|
|
9474
|
+
status: "error",
|
|
9475
|
+
format: input3.read.format,
|
|
9476
|
+
...selectedRun ? { runId: selectedRun.runId } : {},
|
|
9477
|
+
summary: {
|
|
9478
|
+
...emptySummary(),
|
|
9479
|
+
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
9480
|
+
},
|
|
9481
|
+
findings: [],
|
|
9482
|
+
diagnostics: [...diagnostics]
|
|
9483
|
+
};
|
|
9484
|
+
}
|
|
9485
|
+
function flattenNodes(nodes) {
|
|
9486
|
+
return nodes.flatMap((node) => [node, ...flattenNodes(node.children)]);
|
|
9487
|
+
}
|
|
9488
|
+
function buildFacts2(input3, selectedRun) {
|
|
9489
|
+
const scopedRuns = selectedRun ? [selectedRun] : input3.read.runs;
|
|
9490
|
+
const scopedRunIds = new Set(scopedRuns.map((run) => run.runId));
|
|
9491
|
+
const scopedEvents = selectedRun === void 0 ? input3.read.events : input3.read.events.filter((event) => scopedRunIds.has(event.runId));
|
|
9492
|
+
const nodes = flattenNodes(scopedRuns.flatMap((run) => run.children));
|
|
9493
|
+
const nodesByEventId = /* @__PURE__ */ new Map();
|
|
9494
|
+
const childrenByParentId = /* @__PURE__ */ new Map();
|
|
9495
|
+
for (const node of nodes) {
|
|
9496
|
+
nodesByEventId.set(node.event.eventId, node);
|
|
9497
|
+
const parentId = node.event.parentId;
|
|
9498
|
+
if (parentId) {
|
|
9499
|
+
const children = childrenByParentId.get(parentId) ?? [];
|
|
9500
|
+
children.push(node);
|
|
9501
|
+
childrenByParentId.set(parentId, children);
|
|
9120
9502
|
}
|
|
9121
|
-
|
|
9122
|
-
|
|
9123
|
-
|
|
9124
|
-
|
|
9125
|
-
|
|
9126
|
-
(
|
|
9127
|
-
|
|
9503
|
+
}
|
|
9504
|
+
return {
|
|
9505
|
+
format: input3.read.format,
|
|
9506
|
+
runs: Object.freeze([...input3.read.runs]),
|
|
9507
|
+
events: Object.freeze([...scopedEvents]),
|
|
9508
|
+
readerWarnings: Object.freeze([...input3.read.warnings]),
|
|
9509
|
+
unsupportedFields: Object.freeze([...input3.read.unsupportedFields]),
|
|
9510
|
+
sourceFiles: Object.freeze([...input3.read.sourceFiles]),
|
|
9511
|
+
nodesByEventId,
|
|
9512
|
+
childrenByParentId,
|
|
9513
|
+
rootNodes: Object.freeze(scopedRuns.flatMap((run) => run.children))
|
|
9514
|
+
};
|
|
9515
|
+
}
|
|
9516
|
+
function resolveSelectedRun(input3, runId) {
|
|
9517
|
+
if (input3.selectedRun) {
|
|
9518
|
+
if (runId && input3.selectedRun.runId !== runId) {
|
|
9519
|
+
return {
|
|
9520
|
+
diagnostics: [
|
|
9521
|
+
diagnostic(
|
|
9522
|
+
"AI_CHECK_INVALID_ARGUMENTS",
|
|
9523
|
+
`Selected run ${input3.selectedRun.runId} does not match requested run ${runId}.`
|
|
9524
|
+
)
|
|
9525
|
+
]
|
|
9526
|
+
};
|
|
9128
9527
|
}
|
|
9129
|
-
|
|
9130
|
-
|
|
9131
|
-
|
|
9132
|
-
|
|
9133
|
-
|
|
9134
|
-
|
|
9135
|
-
|
|
9528
|
+
return { run: input3.selectedRun, diagnostics: [] };
|
|
9529
|
+
}
|
|
9530
|
+
if (runId) {
|
|
9531
|
+
const run = input3.read.runs.find((candidate) => candidate.runId === runId);
|
|
9532
|
+
if (!run) {
|
|
9533
|
+
return {
|
|
9534
|
+
diagnostics: [
|
|
9535
|
+
diagnostic("AI_CHECK_RUN_SELECTION_REQUIRED", `Run not found: ${runId}.`)
|
|
9536
|
+
]
|
|
9537
|
+
};
|
|
9136
9538
|
}
|
|
9137
|
-
|
|
9138
|
-
|
|
9139
|
-
|
|
9140
|
-
|
|
9141
|
-
|
|
9142
|
-
|
|
9143
|
-
|
|
9144
|
-
|
|
9145
|
-
|
|
9146
|
-
|
|
9147
|
-
|
|
9148
|
-
|
|
9149
|
-
|
|
9150
|
-
|
|
9151
|
-
|
|
9152
|
-
|
|
9153
|
-
|
|
9154
|
-
|
|
9155
|
-
|
|
9156
|
-
|
|
9157
|
-
|
|
9158
|
-
|
|
9159
|
-
|
|
9160
|
-
|
|
9161
|
-
|
|
9162
|
-
|
|
9163
|
-
|
|
9164
|
-
|
|
9165
|
-
|
|
9166
|
-
|
|
9167
|
-
|
|
9168
|
-
|
|
9169
|
-
|
|
9170
|
-
|
|
9171
|
-
|
|
9172
|
-
|
|
9173
|
-
|
|
9174
|
-
|
|
9175
|
-
|
|
9176
|
-
|
|
9177
|
-
|
|
9178
|
-
|
|
9179
|
-
|
|
9180
|
-
|
|
9181
|
-
|
|
9182
|
-
|
|
9183
|
-
|
|
9184
|
-
|
|
9185
|
-
|
|
9186
|
-
|
|
9187
|
-
|
|
9188
|
-
|
|
9189
|
-
|
|
9190
|
-
|
|
9191
|
-
|
|
9192
|
-
|
|
9193
|
-
|
|
9194
|
-
|
|
9195
|
-
|
|
9196
|
-
|
|
9197
|
-
|
|
9198
|
-
|
|
9539
|
+
return { run, diagnostics: [] };
|
|
9540
|
+
}
|
|
9541
|
+
if (input3.read.runs.length === 1) {
|
|
9542
|
+
return { run: input3.read.runs[0], diagnostics: [] };
|
|
9543
|
+
}
|
|
9544
|
+
if (input3.read.runs.length === 0) {
|
|
9545
|
+
return {
|
|
9546
|
+
diagnostics: [
|
|
9547
|
+
diagnostic("AI_CHECK_RUN_SELECTION_REQUIRED", "No runs are available for checks.")
|
|
9548
|
+
]
|
|
9549
|
+
};
|
|
9550
|
+
}
|
|
9551
|
+
return {
|
|
9552
|
+
diagnostics: [
|
|
9553
|
+
diagnostic(
|
|
9554
|
+
"AI_CHECK_RUN_SELECTION_REQUIRED",
|
|
9555
|
+
"Multiple runs are available; select a run before executing checks."
|
|
9556
|
+
)
|
|
9557
|
+
]
|
|
9558
|
+
};
|
|
9559
|
+
}
|
|
9560
|
+
function selectRules(rules, selectedIds) {
|
|
9561
|
+
const diagnostics = [];
|
|
9562
|
+
const byId = /* @__PURE__ */ new Map();
|
|
9563
|
+
for (const rule of rules) {
|
|
9564
|
+
if (byId.has(rule.id)) {
|
|
9565
|
+
diagnostics.push(
|
|
9566
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Duplicate trace check rule id: ${rule.id}.`, rule.id)
|
|
9567
|
+
);
|
|
9568
|
+
continue;
|
|
9569
|
+
}
|
|
9570
|
+
byId.set(rule.id, rule);
|
|
9571
|
+
}
|
|
9572
|
+
if (selectedIds && selectedIds.length > 0) {
|
|
9573
|
+
const selected = new Set(selectedIds);
|
|
9574
|
+
for (const id of selected) {
|
|
9575
|
+
if (!byId.has(id)) {
|
|
9576
|
+
diagnostics.push(
|
|
9577
|
+
diagnostic("AI_CHECK_INVALID_CONFIG", `Unknown trace check rule id: ${id}.`, id)
|
|
9578
|
+
);
|
|
9579
|
+
}
|
|
9580
|
+
}
|
|
9581
|
+
return {
|
|
9582
|
+
rules: [...byId.values()].filter((rule) => selected.has(rule.id)).sort(compareRules),
|
|
9583
|
+
diagnostics
|
|
9584
|
+
};
|
|
9585
|
+
}
|
|
9586
|
+
return { rules: [...byId.values()].sort(compareRules), diagnostics };
|
|
9587
|
+
}
|
|
9588
|
+
function compareRules(a, b) {
|
|
9589
|
+
return a.id.localeCompare(b.id);
|
|
9590
|
+
}
|
|
9591
|
+
function eventTimestamp(finding, eventById) {
|
|
9592
|
+
const eventId = finding.evidence[0]?.eventId;
|
|
9593
|
+
return eventId ? eventById.get(eventId)?.timestamp ?? "" : "";
|
|
9594
|
+
}
|
|
9595
|
+
function compareFindings(eventById) {
|
|
9596
|
+
return (a, b) => {
|
|
9597
|
+
if (SEVERITY_RANK[a.severity] !== SEVERITY_RANK[b.severity]) {
|
|
9598
|
+
return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity];
|
|
9599
|
+
}
|
|
9600
|
+
const byRule = a.ruleId.localeCompare(b.ruleId);
|
|
9601
|
+
if (byRule !== 0) return byRule;
|
|
9602
|
+
if (STATUS_RANK[a.status] !== STATUS_RANK[b.status]) {
|
|
9603
|
+
return STATUS_RANK[a.status] - STATUS_RANK[b.status];
|
|
9604
|
+
}
|
|
9605
|
+
const byRun = compareStrings(a.evidence[0]?.runId, b.evidence[0]?.runId);
|
|
9606
|
+
if (byRun !== 0) return byRun;
|
|
9607
|
+
const byTime = eventTimestamp(a, eventById).localeCompare(eventTimestamp(b, eventById));
|
|
9608
|
+
if (byTime !== 0) return byTime;
|
|
9609
|
+
const byEvent = compareStrings(a.evidence[0]?.eventId, b.evidence[0]?.eventId);
|
|
9610
|
+
if (byEvent !== 0) return byEvent;
|
|
9611
|
+
return compareStrings(a.evidence[0]?.path, b.evidence[0]?.path);
|
|
9612
|
+
};
|
|
9613
|
+
}
|
|
9614
|
+
function normalizeFinding(rule, finding) {
|
|
9615
|
+
return {
|
|
9616
|
+
ruleId: finding.ruleId || rule.id,
|
|
9617
|
+
severity: finding.severity ?? rule.defaultSeverity,
|
|
9618
|
+
status: finding.status,
|
|
9619
|
+
message: finding.message,
|
|
9620
|
+
...finding.expected !== void 0 ? { expected: finding.expected } : {},
|
|
9621
|
+
...finding.actual !== void 0 ? { actual: finding.actual } : {},
|
|
9622
|
+
evidence: [...finding.evidence ?? []]
|
|
9623
|
+
};
|
|
9624
|
+
}
|
|
9625
|
+
function summarize(findings, diagnostics) {
|
|
9626
|
+
return {
|
|
9627
|
+
passed: findings.filter((finding) => finding.status === "pass").length,
|
|
9628
|
+
failed: findings.filter(
|
|
9629
|
+
(finding) => finding.status === "fail" && finding.severity === "error"
|
|
9630
|
+
).length,
|
|
9631
|
+
warnings: findings.filter(
|
|
9632
|
+
(finding) => finding.status === "warning" || finding.severity === "warning"
|
|
9633
|
+
).length,
|
|
9634
|
+
errors: diagnostics.filter((item) => item.severity === "error").length
|
|
9635
|
+
};
|
|
9636
|
+
}
|
|
9637
|
+
function stringAttr2(event, keys) {
|
|
9638
|
+
for (const key of keys) {
|
|
9639
|
+
const value = event.attributes?.[key];
|
|
9640
|
+
if (typeof value === "string" && value.trim() !== "") return value;
|
|
9641
|
+
}
|
|
9642
|
+
return void 0;
|
|
9643
|
+
}
|
|
9644
|
+
function numericAttr(event, keys) {
|
|
9645
|
+
for (const key of keys) {
|
|
9646
|
+
const value = event.attributes?.[key];
|
|
9647
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
9648
|
+
}
|
|
9649
|
+
return void 0;
|
|
9650
|
+
}
|
|
9651
|
+
function booleanAttr(event, keys) {
|
|
9652
|
+
for (const key of keys) {
|
|
9653
|
+
const value = event.attributes?.[key];
|
|
9654
|
+
if (typeof value === "boolean") return value;
|
|
9655
|
+
}
|
|
9656
|
+
return void 0;
|
|
9657
|
+
}
|
|
9658
|
+
function stripPrefix(name, prefixes) {
|
|
9659
|
+
for (const prefix of prefixes) {
|
|
9660
|
+
if (name.startsWith(prefix)) return name.slice(prefix.length);
|
|
9661
|
+
}
|
|
9662
|
+
return name;
|
|
9663
|
+
}
|
|
9664
|
+
function eventEvidence(event, path12) {
|
|
9665
|
+
return {
|
|
9666
|
+
runId: event.runId,
|
|
9667
|
+
eventId: event.eventId,
|
|
9668
|
+
parentId: event.parentId,
|
|
9669
|
+
traceId: event.trace?.traceId,
|
|
9670
|
+
spanId: event.trace?.spanId,
|
|
9671
|
+
kind: event.kind,
|
|
9672
|
+
name: event.name,
|
|
9673
|
+
status: event.status,
|
|
9674
|
+
...path12 ? { path: path12 } : {}
|
|
9675
|
+
};
|
|
9676
|
+
}
|
|
9677
|
+
function runEvidence(run) {
|
|
9678
|
+
return run ? [{ runId: run.runId, name: run.name, status: run.status }] : [];
|
|
9679
|
+
}
|
|
9680
|
+
function failFinding(ruleId, message, evidence, expected, actual) {
|
|
9681
|
+
return {
|
|
9682
|
+
ruleId,
|
|
9683
|
+
severity: "error",
|
|
9684
|
+
status: "fail",
|
|
9685
|
+
message,
|
|
9686
|
+
...expected !== void 0 ? { expected } : {},
|
|
9687
|
+
...actual !== void 0 ? { actual } : {},
|
|
9688
|
+
evidence: [...evidence]
|
|
9689
|
+
};
|
|
9690
|
+
}
|
|
9691
|
+
function toolName(event) {
|
|
9692
|
+
return stringAttr2(event, ["toolName", "tool"]) ?? stripPrefix(event.name, ["tool:", "function:", "mcp-tools:"]);
|
|
9693
|
+
}
|
|
9694
|
+
function llmModel(event) {
|
|
9695
|
+
return stringAttr2(event, ["model", "modelId", "responseModelId", "modelName", "model_name"]) ?? stripPrefix(event.name, ["llm:", "generation:", "transcription:", "speech:"]);
|
|
9696
|
+
}
|
|
9697
|
+
function llmProvider(event) {
|
|
9698
|
+
return stringAttr2(event, ["provider", "providerName", "provider_name"]);
|
|
9699
|
+
}
|
|
9700
|
+
function llmFinishReason(event) {
|
|
9701
|
+
return stringAttr2(event, ["finishReason", "rawFinishReason", "finish_reason"]);
|
|
9702
|
+
}
|
|
9703
|
+
function retryCount(event) {
|
|
9704
|
+
return numericAttr(event, ["retryCount", "retryAttempt", "retry_attempt", "attempt"]);
|
|
9705
|
+
}
|
|
9706
|
+
function finishedEvents(context, kind) {
|
|
9707
|
+
return context.events.filter(
|
|
9708
|
+
(event) => (kind === void 0 || event.kind === kind) && event.status !== "running"
|
|
9709
|
+
);
|
|
9710
|
+
}
|
|
9711
|
+
function isRecord14(value) {
|
|
9712
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
9713
|
+
}
|
|
9714
|
+
function eventMap(events) {
|
|
9715
|
+
return new Map(events.map((event) => [event.eventId, event]));
|
|
9716
|
+
}
|
|
9717
|
+
function parseEventTime(value) {
|
|
9718
|
+
if (!value) return void 0;
|
|
9719
|
+
const parsed = Date.parse(value);
|
|
9720
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
9721
|
+
}
|
|
9722
|
+
function eventStartMs(event) {
|
|
9723
|
+
return parseEventTime(event.startedAt) ?? parseEventTime(event.timestamp);
|
|
9724
|
+
}
|
|
9725
|
+
function eventEndMs(event) {
|
|
9726
|
+
const endedAt = parseEventTime(event.endedAt);
|
|
9727
|
+
if (endedAt !== void 0) return endedAt;
|
|
9728
|
+
const startedAt = eventStartMs(event);
|
|
9729
|
+
if (startedAt !== void 0 && event.durationMs !== void 0 && Number.isFinite(event.durationMs)) {
|
|
9730
|
+
return startedAt + event.durationMs;
|
|
9731
|
+
}
|
|
9732
|
+
return void 0;
|
|
9733
|
+
}
|
|
9734
|
+
function normalizedKey(value) {
|
|
9735
|
+
return value.toLowerCase().replace(/[^a-z0-9_]/g, "");
|
|
9736
|
+
}
|
|
9737
|
+
function lastPathSegment(path12) {
|
|
9738
|
+
const parts = path12.split(".");
|
|
9739
|
+
return parts[parts.length - 1] ?? path12;
|
|
9740
|
+
}
|
|
9741
|
+
function valueType(value) {
|
|
9742
|
+
if (Array.isArray(value)) return "array";
|
|
9743
|
+
if (value === null) return "null";
|
|
9744
|
+
return typeof value;
|
|
9745
|
+
}
|
|
9746
|
+
function serializedByteLength(value) {
|
|
9747
|
+
try {
|
|
9748
|
+
return Buffer.byteLength(JSON.stringify(value), "utf-8");
|
|
9749
|
+
} catch {
|
|
9750
|
+
return void 0;
|
|
9751
|
+
}
|
|
9752
|
+
}
|
|
9753
|
+
function pushValueEntries(entries, event, value, path12, key, depth = 0) {
|
|
9754
|
+
entries.push({ event, path: path12, key, value });
|
|
9755
|
+
if (depth >= 8) return;
|
|
9756
|
+
if (Array.isArray(value)) {
|
|
9757
|
+
for (const [index, item] of value.entries()) {
|
|
9758
|
+
pushValueEntries(entries, event, item, `${path12}.${index}`, String(index), depth + 1);
|
|
9759
|
+
}
|
|
9760
|
+
return;
|
|
9761
|
+
}
|
|
9762
|
+
if (!isRecord14(value)) return;
|
|
9763
|
+
for (const nestedKey of Object.keys(value).sort((a, b) => a.localeCompare(b))) {
|
|
9764
|
+
pushValueEntries(
|
|
9765
|
+
entries,
|
|
9766
|
+
event,
|
|
9767
|
+
value[nestedKey],
|
|
9768
|
+
`${path12}.${nestedKey}`,
|
|
9769
|
+
nestedKey,
|
|
9770
|
+
depth + 1
|
|
9771
|
+
);
|
|
9772
|
+
}
|
|
9773
|
+
}
|
|
9774
|
+
function eventValueEntries(event, options = {}) {
|
|
9775
|
+
const entries = [];
|
|
9776
|
+
if (event.attributes !== void 0) {
|
|
9777
|
+
pushValueEntries(entries, event, event.attributes, "attributes", "attributes");
|
|
9778
|
+
}
|
|
9779
|
+
if (options.includeSummaries) {
|
|
9780
|
+
if (event.inputSummary !== void 0) {
|
|
9781
|
+
pushValueEntries(entries, event, event.inputSummary, "inputSummary", "inputSummary");
|
|
9782
|
+
}
|
|
9783
|
+
if (event.outputSummary !== void 0) {
|
|
9784
|
+
pushValueEntries(entries, event, event.outputSummary, "outputSummary", "outputSummary");
|
|
9785
|
+
}
|
|
9786
|
+
}
|
|
9787
|
+
if (options.includeError && event.error !== void 0) {
|
|
9788
|
+
pushValueEntries(entries, event, event.error, "error", "error");
|
|
9789
|
+
}
|
|
9790
|
+
return entries;
|
|
9791
|
+
}
|
|
9792
|
+
function limitFindings(findings, maxFindings) {
|
|
9793
|
+
if (maxFindings === void 0 || findings.length <= maxFindings) return findings;
|
|
9794
|
+
return findings.slice(0, Math.max(0, maxFindings));
|
|
9795
|
+
}
|
|
9796
|
+
function hasRedactionMarker(value, markers) {
|
|
9797
|
+
return markers.some((marker) => value.includes(marker)) || /^\[HASH:[A-Za-z0-9_-]+\]$/.test(value);
|
|
9798
|
+
}
|
|
9799
|
+
function isSensitiveKey(key, sensitiveKeys) {
|
|
9800
|
+
if (!key) return false;
|
|
9801
|
+
const normalized = normalizedKey(key);
|
|
9802
|
+
return sensitiveKeys.some((sensitive) => normalized.includes(normalizedKey(sensitive)));
|
|
9803
|
+
}
|
|
9804
|
+
function isRawContentKey(key, forbiddenKeys) {
|
|
9805
|
+
if (!key) return false;
|
|
9806
|
+
const normalized = normalizedKey(key);
|
|
9807
|
+
return forbiddenKeys.some((forbidden) => normalized === normalizedKey(forbidden));
|
|
9808
|
+
}
|
|
9809
|
+
function parentMarkedUnresolved(event) {
|
|
9810
|
+
if (booleanAttr(event, [
|
|
9811
|
+
"parentUnresolved",
|
|
9812
|
+
"unresolvedParent",
|
|
9813
|
+
"relationshipUnresolved",
|
|
9814
|
+
"unresolvedRelationship"
|
|
9815
|
+
]) === true) {
|
|
9816
|
+
return true;
|
|
9817
|
+
}
|
|
9818
|
+
const resolution = stringAttr2(event, [
|
|
9819
|
+
"parentResolution",
|
|
9820
|
+
"relationshipResolution",
|
|
9821
|
+
"relationshipStatus"
|
|
9822
|
+
]);
|
|
9823
|
+
return resolution === "unresolved" || resolution === "missing-parent";
|
|
9824
|
+
}
|
|
9825
|
+
function signalName(event, attributeKeys, prefixes) {
|
|
9826
|
+
return stringAttr2(event, attributeKeys) ?? stripPrefix(event.name, prefixes);
|
|
9827
|
+
}
|
|
9828
|
+
function guardrailEvents(context) {
|
|
9829
|
+
return finishedEvents2().filter((event) => {
|
|
9830
|
+
const name = event.name.toLowerCase();
|
|
9831
|
+
if (name.startsWith("guardrail:") || name.includes(".guardrail.")) return true;
|
|
9832
|
+
return stringAttr2(event, ["guardrailName", "guardrail", "guardrailId"]) !== void 0;
|
|
9833
|
+
});
|
|
9834
|
+
function finishedEvents2() {
|
|
9835
|
+
return context.events.filter((event) => event.status !== "running");
|
|
9836
|
+
}
|
|
9837
|
+
}
|
|
9838
|
+
function retryValue(event) {
|
|
9839
|
+
return retryCount(event) ?? 0;
|
|
9840
|
+
}
|
|
9841
|
+
function eventDurationMs(event) {
|
|
9842
|
+
if (event.durationMs !== void 0) return event.durationMs;
|
|
9843
|
+
const start = eventStartMs(event);
|
|
9844
|
+
const end = eventEndMs(event);
|
|
9845
|
+
return start !== void 0 && end !== void 0 && end >= start ? end - start : void 0;
|
|
9846
|
+
}
|
|
9847
|
+
function treeShape(nodes) {
|
|
9848
|
+
const lines = [];
|
|
9849
|
+
const visit = (node, path12) => {
|
|
9850
|
+
lines.push(`${path12}:${node.event.kind}:${node.event.name}:${node.event.status ?? "unknown"}`);
|
|
9851
|
+
node.children.forEach((child, index) => visit(child, `${path12}.${index}`));
|
|
9852
|
+
};
|
|
9853
|
+
nodes.forEach((node, index) => visit(node, String(index)));
|
|
9854
|
+
return lines;
|
|
9855
|
+
}
|
|
9856
|
+
function statusShape(context) {
|
|
9857
|
+
return context.events.map((event) => `${event.kind}:${event.name}:${event.status ?? "unknown"}`).sort((a, b) => a.localeCompare(b));
|
|
9858
|
+
}
|
|
9859
|
+
function toolShape(context) {
|
|
9860
|
+
return finishedEvents(context, "TOOL").map(
|
|
9861
|
+
(event) => [
|
|
9862
|
+
toolName(event),
|
|
9863
|
+
event.status ?? "unknown",
|
|
9864
|
+
retryValue(event),
|
|
9865
|
+
eventDurationMs(event) ?? "unknown"
|
|
9866
|
+
].join(":")
|
|
9867
|
+
);
|
|
9868
|
+
}
|
|
9869
|
+
function llmShape(context) {
|
|
9870
|
+
return finishedEvents(context, "LLM").map(
|
|
9871
|
+
(event) => [
|
|
9872
|
+
llmProvider(event) ?? "unknown",
|
|
9873
|
+
llmModel(event) ?? "unknown",
|
|
9874
|
+
llmFinishReason(event) ?? "unknown",
|
|
9875
|
+
event.tokenUsage?.input ?? 0,
|
|
9876
|
+
event.tokenUsage?.output ?? 0,
|
|
9877
|
+
event.tokenUsage?.total ?? 0,
|
|
9878
|
+
event.tokenUsage?.cached ?? 0
|
|
9879
|
+
].join(":")
|
|
9880
|
+
);
|
|
9881
|
+
}
|
|
9882
|
+
function errorShape(context) {
|
|
9883
|
+
return context.events.filter((event) => event.status === "error" || event.error !== void 0).map(
|
|
9884
|
+
(event) => [
|
|
9885
|
+
event.kind,
|
|
9886
|
+
event.name,
|
|
9887
|
+
event.error?.name ?? "Error",
|
|
9888
|
+
event.error?.code ?? "unknown"
|
|
9889
|
+
].join(":")
|
|
9890
|
+
).sort((a, b) => a.localeCompare(b));
|
|
9891
|
+
}
|
|
9892
|
+
function retrievalShape(context) {
|
|
9893
|
+
return finishedEvents(context, "RETRIEVER").map(
|
|
9894
|
+
(event) => signalName(event, ["retrievalName", "retrieverName", "retriever"], ["retriever:", "retrieval:"])
|
|
9895
|
+
).sort((a, b) => a.localeCompare(b));
|
|
9896
|
+
}
|
|
9897
|
+
function guardrailShape(context) {
|
|
9898
|
+
return guardrailEvents(context).map((event) => signalName(event, ["guardrailName", "guardrail", "guardrailId"], ["guardrail:"])).sort((a, b) => a.localeCompare(b));
|
|
9899
|
+
}
|
|
9900
|
+
function firstEvidenceForKind(context, kind, path12) {
|
|
9901
|
+
const event = context.events.find((candidate) => candidate.kind === kind);
|
|
9902
|
+
return event ? [eventEvidence(event, path12)] : runEvidence(context.selectedRun);
|
|
9903
|
+
}
|
|
9904
|
+
function baselineDiffFinding(message, evidence, expected, actual) {
|
|
9905
|
+
return failFinding("baseline.regression", message, evidence, expected, actual);
|
|
9906
|
+
}
|
|
9907
|
+
function createRunStatusRule(options = {}) {
|
|
9908
|
+
const expected = options.expected ?? "ok";
|
|
9909
|
+
const allowIncomplete = options.allowIncomplete === true;
|
|
9910
|
+
return {
|
|
9911
|
+
id: "run.status",
|
|
9912
|
+
category: "run",
|
|
9913
|
+
defaultSeverity: "error",
|
|
9914
|
+
evaluate(context) {
|
|
9915
|
+
const findings = [];
|
|
9916
|
+
const actual = context.selectedRun?.status ?? "unknown";
|
|
9917
|
+
if (actual !== expected) {
|
|
9918
|
+
findings.push(
|
|
9919
|
+
failFinding(
|
|
9920
|
+
"run.status",
|
|
9921
|
+
`Run status ${actual} did not match expected ${expected}.`,
|
|
9922
|
+
runEvidence(context.selectedRun),
|
|
9923
|
+
expected,
|
|
9924
|
+
actual
|
|
9925
|
+
)
|
|
9926
|
+
);
|
|
9927
|
+
}
|
|
9928
|
+
if (!allowIncomplete) {
|
|
9929
|
+
const running = context.events.filter((event) => event.status === "running");
|
|
9930
|
+
if (running.length > 0) {
|
|
9931
|
+
findings.push(
|
|
9932
|
+
failFinding(
|
|
9933
|
+
"run.status",
|
|
9934
|
+
"Run contains incomplete running events.",
|
|
9935
|
+
running.map((event) => eventEvidence(event)),
|
|
9936
|
+
"no running events",
|
|
9937
|
+
running.length
|
|
9938
|
+
)
|
|
9939
|
+
);
|
|
9940
|
+
}
|
|
9941
|
+
}
|
|
9942
|
+
return findings;
|
|
9943
|
+
}
|
|
9944
|
+
};
|
|
9945
|
+
}
|
|
9946
|
+
function createRunDurationRule(options) {
|
|
9947
|
+
return {
|
|
9948
|
+
id: "run.duration",
|
|
9949
|
+
category: "run",
|
|
9950
|
+
defaultSeverity: "error",
|
|
9951
|
+
evaluate(context) {
|
|
9952
|
+
const actual = context.selectedRun?.durationMs;
|
|
9953
|
+
if (actual === void 0 || actual <= options.maxDurationMs) return [];
|
|
9954
|
+
return [
|
|
9955
|
+
failFinding(
|
|
9956
|
+
"run.duration",
|
|
9957
|
+
`Run duration ${actual}ms exceeded ${options.maxDurationMs}ms.`,
|
|
9958
|
+
runEvidence(context.selectedRun),
|
|
9959
|
+
{ maxDurationMs: options.maxDurationMs },
|
|
9960
|
+
actual
|
|
9961
|
+
)
|
|
9962
|
+
];
|
|
9963
|
+
}
|
|
9964
|
+
};
|
|
9965
|
+
}
|
|
9966
|
+
function createRunDepthRule(options) {
|
|
9967
|
+
return {
|
|
9968
|
+
id: "run.depth",
|
|
9969
|
+
category: "run",
|
|
9970
|
+
defaultSeverity: "error",
|
|
9971
|
+
evaluate(context) {
|
|
9972
|
+
const nodes = [...context.nodesByEventId.values()];
|
|
9973
|
+
const maxDepth = nodes.reduce((max, node) => Math.max(max, node.depth), 0);
|
|
9974
|
+
if (maxDepth <= options.maxDepth) return [];
|
|
9975
|
+
const deepest = nodes.filter((node) => node.depth === maxDepth);
|
|
9976
|
+
return [
|
|
9977
|
+
failFinding(
|
|
9978
|
+
"run.depth",
|
|
9979
|
+
`Run depth ${maxDepth} exceeded ${options.maxDepth}.`,
|
|
9980
|
+
deepest.map((node) => ({
|
|
9981
|
+
runId: node.event.runId,
|
|
9982
|
+
eventId: node.event.eventId,
|
|
9983
|
+
parentId: node.event.parentId,
|
|
9984
|
+
kind: node.event.kind,
|
|
9985
|
+
name: node.event.name,
|
|
9986
|
+
status: node.event.status
|
|
9987
|
+
})),
|
|
9988
|
+
{ maxDepth: options.maxDepth },
|
|
9989
|
+
maxDepth
|
|
9990
|
+
)
|
|
9991
|
+
];
|
|
9992
|
+
}
|
|
9993
|
+
};
|
|
9994
|
+
}
|
|
9995
|
+
function createToolUsageRule(options) {
|
|
9996
|
+
return {
|
|
9997
|
+
id: "tool.usage",
|
|
9998
|
+
category: "tool",
|
|
9999
|
+
defaultSeverity: "error",
|
|
10000
|
+
evaluate(context) {
|
|
10001
|
+
const tools = finishedEvents(context, "TOOL");
|
|
10002
|
+
const names = tools.map(toolName);
|
|
10003
|
+
const nameSet = new Set(names);
|
|
10004
|
+
const findings = [];
|
|
10005
|
+
for (const required of options.required ?? []) {
|
|
10006
|
+
if (!nameSet.has(required)) {
|
|
10007
|
+
findings.push(
|
|
10008
|
+
failFinding("tool.usage", `Required tool ${required} did not appear.`, runEvidence(context.selectedRun), required, names)
|
|
10009
|
+
);
|
|
10010
|
+
}
|
|
10011
|
+
}
|
|
10012
|
+
const forbidden = new Set(options.forbidden ?? []);
|
|
10013
|
+
const allowed = options.allowed ? new Set(options.allowed) : void 0;
|
|
10014
|
+
for (const event of tools) {
|
|
10015
|
+
const name = toolName(event);
|
|
10016
|
+
if (forbidden.has(name)) {
|
|
10017
|
+
findings.push(
|
|
10018
|
+
failFinding("tool.usage", `Forbidden tool ${name} appeared.`, [eventEvidence(event)], "tool absent", name)
|
|
10019
|
+
);
|
|
10020
|
+
}
|
|
10021
|
+
if (allowed && !allowed.has(name)) {
|
|
10022
|
+
findings.push(
|
|
10023
|
+
failFinding("tool.usage", `Tool ${name} is not in the allowed tool set.`, [eventEvidence(event)], [...allowed].sort(), name)
|
|
10024
|
+
);
|
|
10025
|
+
}
|
|
10026
|
+
}
|
|
10027
|
+
if (options.minCount !== void 0 && tools.length < options.minCount) {
|
|
10028
|
+
findings.push(
|
|
10029
|
+
failFinding("tool.usage", `Tool count ${tools.length} was below minimum ${options.minCount}.`, runEvidence(context.selectedRun), { minCount: options.minCount }, tools.length)
|
|
10030
|
+
);
|
|
10031
|
+
}
|
|
10032
|
+
if (options.maxCount !== void 0 && tools.length > options.maxCount) {
|
|
10033
|
+
findings.push(
|
|
10034
|
+
failFinding("tool.usage", `Tool count ${tools.length} exceeded maximum ${options.maxCount}.`, tools.map((event) => eventEvidence(event)), { maxCount: options.maxCount }, tools.length)
|
|
10035
|
+
);
|
|
10036
|
+
}
|
|
10037
|
+
return findings;
|
|
10038
|
+
}
|
|
10039
|
+
};
|
|
10040
|
+
}
|
|
10041
|
+
function createLlmUsageRule(options) {
|
|
10042
|
+
return {
|
|
10043
|
+
id: "llm.usage",
|
|
10044
|
+
category: "llm",
|
|
10045
|
+
defaultSeverity: "error",
|
|
10046
|
+
evaluate(context) {
|
|
10047
|
+
const llms = finishedEvents(context, "LLM");
|
|
10048
|
+
const findings = [];
|
|
10049
|
+
const allowedModels = options.allowedModels ? new Set(options.allowedModels) : void 0;
|
|
10050
|
+
const allowedProviders = options.allowedProviders ? new Set(options.allowedProviders) : void 0;
|
|
10051
|
+
const finishReasons = options.finishReasons ? new Set(options.finishReasons) : void 0;
|
|
10052
|
+
if (options.maxCalls !== void 0 && llms.length > options.maxCalls) {
|
|
10053
|
+
findings.push(
|
|
10054
|
+
failFinding(
|
|
10055
|
+
"llm.usage",
|
|
10056
|
+
`LLM call count ${llms.length} exceeded ${options.maxCalls}.`,
|
|
10057
|
+
llms.map((event) => eventEvidence(event)),
|
|
10058
|
+
{ maxCalls: options.maxCalls },
|
|
10059
|
+
llms.length
|
|
10060
|
+
)
|
|
10061
|
+
);
|
|
10062
|
+
}
|
|
10063
|
+
for (const event of llms) {
|
|
10064
|
+
const model = llmModel(event);
|
|
10065
|
+
const provider = llmProvider(event);
|
|
10066
|
+
const finishReason = llmFinishReason(event);
|
|
10067
|
+
if (allowedModels && (!model || !allowedModels.has(model))) {
|
|
10068
|
+
findings.push(
|
|
10069
|
+
failFinding("llm.usage", `LLM model ${model ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.model")], [...allowedModels].sort(), model ?? "unknown")
|
|
10070
|
+
);
|
|
10071
|
+
}
|
|
10072
|
+
if (allowedProviders && (!provider || !allowedProviders.has(provider))) {
|
|
10073
|
+
findings.push(
|
|
10074
|
+
failFinding("llm.usage", `LLM provider ${provider ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.provider")], [...allowedProviders].sort(), provider ?? "unknown")
|
|
10075
|
+
);
|
|
10076
|
+
}
|
|
10077
|
+
if (finishReasons && (!finishReason || !finishReasons.has(finishReason))) {
|
|
10078
|
+
findings.push(
|
|
10079
|
+
failFinding("llm.usage", `LLM finish reason ${finishReason ?? "unknown"} is not allowed.`, [eventEvidence(event, "attributes.finishReason")], [...finishReasons].sort(), finishReason ?? "unknown")
|
|
10080
|
+
);
|
|
10081
|
+
}
|
|
10082
|
+
}
|
|
10083
|
+
const tokenTotals = llms.reduce(
|
|
10084
|
+
(totals, event) => ({
|
|
10085
|
+
input: totals.input + (event.tokenUsage?.input ?? 0),
|
|
10086
|
+
output: totals.output + (event.tokenUsage?.output ?? 0),
|
|
10087
|
+
total: totals.total + (event.tokenUsage?.total ?? 0),
|
|
10088
|
+
cached: totals.cached + (event.tokenUsage?.cached ?? 0)
|
|
10089
|
+
}),
|
|
10090
|
+
{ input: 0, output: 0, total: 0, cached: 0 }
|
|
10091
|
+
);
|
|
10092
|
+
const tokenLimits = [
|
|
10093
|
+
["input", options.maxInputTokens],
|
|
10094
|
+
["output", options.maxOutputTokens],
|
|
10095
|
+
["total", options.maxTotalTokens],
|
|
10096
|
+
["cached", options.maxCachedTokens]
|
|
10097
|
+
];
|
|
10098
|
+
for (const [key, limit] of tokenLimits) {
|
|
10099
|
+
if (limit !== void 0 && tokenTotals[key] > limit) {
|
|
10100
|
+
findings.push(
|
|
10101
|
+
failFinding(
|
|
10102
|
+
"llm.usage",
|
|
10103
|
+
`LLM ${key} token count ${tokenTotals[key]} exceeded ${limit}.`,
|
|
10104
|
+
llms.map((event) => eventEvidence(event, `tokenUsage.${key}`)),
|
|
10105
|
+
{ [`max${key[0].toUpperCase()}${key.slice(1)}Tokens`]: limit },
|
|
10106
|
+
tokenTotals[key]
|
|
10107
|
+
)
|
|
10108
|
+
);
|
|
10109
|
+
}
|
|
10110
|
+
}
|
|
10111
|
+
return findings;
|
|
10112
|
+
}
|
|
10113
|
+
};
|
|
10114
|
+
}
|
|
10115
|
+
function createStructureOrphanRule(options = {}) {
|
|
10116
|
+
const allowMarkedUnresolved = options.allowMarkedUnresolved ?? true;
|
|
10117
|
+
return {
|
|
10118
|
+
id: "structure.orphan",
|
|
10119
|
+
category: "structure",
|
|
10120
|
+
defaultSeverity: "error",
|
|
10121
|
+
evaluate(context) {
|
|
10122
|
+
const byId = eventMap(context.events);
|
|
10123
|
+
const orphans = context.events.filter((event) => {
|
|
10124
|
+
if (!event.parentId || byId.has(event.parentId)) return false;
|
|
10125
|
+
return !(allowMarkedUnresolved && parentMarkedUnresolved(event));
|
|
10126
|
+
});
|
|
10127
|
+
if (orphans.length === 0) return [];
|
|
10128
|
+
return [
|
|
10129
|
+
failFinding(
|
|
10130
|
+
"structure.orphan",
|
|
10131
|
+
"Trace contains events whose parentId is not present in the selected run.",
|
|
10132
|
+
orphans.map((event) => eventEvidence(event, "parentId")),
|
|
10133
|
+
"parentId resolves to an event in the selected run",
|
|
10134
|
+
orphans.length
|
|
10135
|
+
)
|
|
10136
|
+
];
|
|
10137
|
+
}
|
|
10138
|
+
};
|
|
10139
|
+
}
|
|
10140
|
+
function createStructureCycleRule() {
|
|
10141
|
+
return {
|
|
10142
|
+
id: "structure.cycle",
|
|
10143
|
+
category: "structure",
|
|
10144
|
+
defaultSeverity: "error",
|
|
10145
|
+
evaluate(context) {
|
|
10146
|
+
const byId = eventMap(context.events);
|
|
10147
|
+
const seenCycles = /* @__PURE__ */ new Set();
|
|
10148
|
+
const findings = [];
|
|
10149
|
+
for (const event of [...context.events].sort((a, b) => a.eventId.localeCompare(b.eventId))) {
|
|
10150
|
+
const path12 = [];
|
|
10151
|
+
const seenAt = /* @__PURE__ */ new Map();
|
|
10152
|
+
let current = event;
|
|
10153
|
+
while (current) {
|
|
10154
|
+
const existing = seenAt.get(current.eventId);
|
|
10155
|
+
if (existing !== void 0) {
|
|
10156
|
+
const cycle = path12.slice(existing);
|
|
10157
|
+
const key = cycle.map((item) => item.eventId).sort().join("\0");
|
|
10158
|
+
if (!seenCycles.has(key)) {
|
|
10159
|
+
seenCycles.add(key);
|
|
10160
|
+
findings.push(
|
|
10161
|
+
failFinding(
|
|
10162
|
+
"structure.cycle",
|
|
10163
|
+
"Trace contains a parentId cycle.",
|
|
10164
|
+
cycle.map((item) => eventEvidence(item, "parentId")),
|
|
10165
|
+
"acyclic parentId graph",
|
|
10166
|
+
cycle.map((item) => item.eventId).sort()
|
|
10167
|
+
)
|
|
10168
|
+
);
|
|
10169
|
+
}
|
|
10170
|
+
break;
|
|
10171
|
+
}
|
|
10172
|
+
seenAt.set(current.eventId, path12.length);
|
|
10173
|
+
path12.push(current);
|
|
10174
|
+
current = current.parentId ? byId.get(current.parentId) : void 0;
|
|
10175
|
+
}
|
|
10176
|
+
}
|
|
10177
|
+
return findings;
|
|
10178
|
+
}
|
|
10179
|
+
};
|
|
10180
|
+
}
|
|
10181
|
+
function createStructureRelationshipRule(options = {}) {
|
|
10182
|
+
return {
|
|
10183
|
+
id: "structure.relationship",
|
|
10184
|
+
category: "structure",
|
|
10185
|
+
defaultSeverity: "error",
|
|
10186
|
+
evaluate(context) {
|
|
10187
|
+
const byId = eventMap(context.events);
|
|
10188
|
+
const findings = [];
|
|
10189
|
+
const minConfidence = options.minConfidence;
|
|
10190
|
+
for (const event of context.events) {
|
|
10191
|
+
if (minConfidence && CONFIDENCE_RANK[event.confidence] < CONFIDENCE_RANK[minConfidence]) {
|
|
10192
|
+
findings.push(
|
|
10193
|
+
failFinding(
|
|
10194
|
+
"structure.relationship",
|
|
10195
|
+
`Event confidence ${event.confidence} is below ${minConfidence}.`,
|
|
10196
|
+
[eventEvidence(event, "confidence")],
|
|
10197
|
+
{ minConfidence },
|
|
10198
|
+
event.confidence
|
|
10199
|
+
)
|
|
10200
|
+
);
|
|
10201
|
+
}
|
|
10202
|
+
if (!event.parentId) continue;
|
|
10203
|
+
if (event.parentId === event.eventId) {
|
|
10204
|
+
findings.push(
|
|
10205
|
+
failFinding(
|
|
10206
|
+
"structure.relationship",
|
|
10207
|
+
"Event parentId points to itself.",
|
|
10208
|
+
[eventEvidence(event, "parentId")],
|
|
10209
|
+
"parentId references a distinct event",
|
|
10210
|
+
"self"
|
|
10211
|
+
)
|
|
10212
|
+
);
|
|
10213
|
+
continue;
|
|
10214
|
+
}
|
|
10215
|
+
const parent = byId.get(event.parentId);
|
|
10216
|
+
if (!parent) continue;
|
|
10217
|
+
if (options.requireParentBeforeChild) {
|
|
10218
|
+
const parentTime = eventStartMs(parent);
|
|
10219
|
+
const childTime = eventStartMs(event);
|
|
10220
|
+
if (parentTime !== void 0 && childTime !== void 0 && parentTime > childTime) {
|
|
10221
|
+
findings.push(
|
|
10222
|
+
failFinding(
|
|
10223
|
+
"structure.relationship",
|
|
10224
|
+
"Parent event starts after child event.",
|
|
10225
|
+
[eventEvidence(parent), eventEvidence(event, "parentId")],
|
|
10226
|
+
"parent start <= child start",
|
|
10227
|
+
{ parentEventId: parent.eventId, childEventId: event.eventId }
|
|
10228
|
+
)
|
|
10229
|
+
);
|
|
10230
|
+
}
|
|
10231
|
+
}
|
|
10232
|
+
if (options.requireTraceParentSpan && parent.trace?.spanId && event.trace) {
|
|
10233
|
+
const actual = event.trace.parentSpanId;
|
|
10234
|
+
if (actual !== parent.trace.spanId) {
|
|
10235
|
+
findings.push(
|
|
10236
|
+
failFinding(
|
|
10237
|
+
"structure.relationship",
|
|
10238
|
+
"Trace parentSpanId does not match parent spanId.",
|
|
10239
|
+
[eventEvidence(event, "trace.parentSpanId")],
|
|
10240
|
+
{ parentSpanId: parent.trace.spanId },
|
|
10241
|
+
actual ?? "missing"
|
|
10242
|
+
)
|
|
10243
|
+
);
|
|
10244
|
+
}
|
|
10245
|
+
}
|
|
10246
|
+
}
|
|
10247
|
+
return findings;
|
|
10248
|
+
}
|
|
10249
|
+
};
|
|
10250
|
+
}
|
|
10251
|
+
function createStructureParallelWidthRule(options) {
|
|
10252
|
+
return {
|
|
10253
|
+
id: "structure.parallelWidth",
|
|
10254
|
+
category: "structure",
|
|
10255
|
+
defaultSeverity: "error",
|
|
10256
|
+
evaluate(context) {
|
|
10257
|
+
const findings = [];
|
|
10258
|
+
const byId = eventMap(context.events);
|
|
10259
|
+
if (options.maxChildren !== void 0) {
|
|
10260
|
+
for (const [parentId, children] of context.childrenByParentId.entries()) {
|
|
10261
|
+
if (children.length <= options.maxChildren) continue;
|
|
10262
|
+
const parent = byId.get(parentId);
|
|
10263
|
+
findings.push(
|
|
10264
|
+
failFinding(
|
|
10265
|
+
"structure.parallelWidth",
|
|
10266
|
+
`Parent ${parentId} has ${children.length} children, exceeding ${options.maxChildren}.`,
|
|
10267
|
+
[
|
|
10268
|
+
...parent ? [eventEvidence(parent)] : [{ runId: context.selectedRun?.runId, eventId: parentId }],
|
|
10269
|
+
...children.map((child) => ({
|
|
10270
|
+
runId: child.event.runId,
|
|
10271
|
+
eventId: child.event.eventId,
|
|
10272
|
+
parentId: child.event.parentId,
|
|
10273
|
+
kind: child.event.kind,
|
|
10274
|
+
name: child.event.name,
|
|
10275
|
+
status: child.event.status
|
|
10276
|
+
}))
|
|
10277
|
+
],
|
|
10278
|
+
{ maxChildren: options.maxChildren },
|
|
10279
|
+
children.length
|
|
10280
|
+
)
|
|
10281
|
+
);
|
|
10282
|
+
}
|
|
10283
|
+
}
|
|
10284
|
+
if (options.maxConcurrent !== void 0) {
|
|
10285
|
+
const intervals = context.events.map((event) => ({ event, start: eventStartMs(event), end: eventEndMs(event) })).filter(
|
|
10286
|
+
(item) => item.start !== void 0 && item.end !== void 0 && item.end > item.start
|
|
10287
|
+
);
|
|
10288
|
+
const points = intervals.flatMap((item) => [
|
|
10289
|
+
{ time: item.start, delta: 1, event: item.event },
|
|
10290
|
+
{ time: item.end, delta: -1, event: item.event }
|
|
10291
|
+
]);
|
|
10292
|
+
points.sort((a, b) => {
|
|
10293
|
+
const byTime = a.time - b.time;
|
|
10294
|
+
if (byTime !== 0) return byTime;
|
|
10295
|
+
const byDelta = a.delta - b.delta;
|
|
10296
|
+
if (byDelta !== 0) return byDelta;
|
|
10297
|
+
return a.event.eventId.localeCompare(b.event.eventId);
|
|
10298
|
+
});
|
|
10299
|
+
const active = /* @__PURE__ */ new Map();
|
|
10300
|
+
let maxActive = [];
|
|
10301
|
+
for (const point of points) {
|
|
10302
|
+
if (point.delta > 0) {
|
|
10303
|
+
active.set(point.event.eventId, point.event);
|
|
10304
|
+
if (active.size > maxActive.length) {
|
|
10305
|
+
maxActive = [...active.values()].sort((a, b) => a.eventId.localeCompare(b.eventId));
|
|
10306
|
+
}
|
|
10307
|
+
} else {
|
|
10308
|
+
active.delete(point.event.eventId);
|
|
10309
|
+
}
|
|
10310
|
+
}
|
|
10311
|
+
if (maxActive.length > options.maxConcurrent) {
|
|
10312
|
+
findings.push(
|
|
10313
|
+
failFinding(
|
|
10314
|
+
"structure.parallelWidth",
|
|
10315
|
+
`Concurrent event width ${maxActive.length} exceeded ${options.maxConcurrent}.`,
|
|
10316
|
+
maxActive.map((event) => eventEvidence(event)),
|
|
10317
|
+
{ maxConcurrent: options.maxConcurrent },
|
|
10318
|
+
maxActive.length
|
|
10319
|
+
)
|
|
10320
|
+
);
|
|
10321
|
+
}
|
|
10322
|
+
}
|
|
10323
|
+
return findings;
|
|
10324
|
+
}
|
|
10325
|
+
};
|
|
10326
|
+
}
|
|
10327
|
+
function createSafetyRedactionRule(options = {}) {
|
|
10328
|
+
const sensitiveKeys = options.sensitiveKeys ?? DEFAULT_SENSITIVE_KEYS;
|
|
10329
|
+
const markers = options.redactedMarkers ?? ["[REDACTED]", "[REDACTED:"];
|
|
10330
|
+
return {
|
|
10331
|
+
id: "safety.redaction",
|
|
10332
|
+
category: "safety",
|
|
10333
|
+
defaultSeverity: "error",
|
|
10334
|
+
evaluate(context) {
|
|
10335
|
+
const findings = [];
|
|
10336
|
+
for (const event of context.events) {
|
|
10337
|
+
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
10338
|
+
if (!isSensitiveKey(entry.key ?? lastPathSegment(entry.path), sensitiveKeys)) continue;
|
|
10339
|
+
if (typeof entry.value === "string" && hasRedactionMarker(entry.value, markers)) continue;
|
|
10340
|
+
findings.push(
|
|
10341
|
+
failFinding(
|
|
10342
|
+
"safety.redaction",
|
|
10343
|
+
`Sensitive-looking field at ${entry.path} is not redacted.`,
|
|
10344
|
+
[eventEvidence(event, entry.path)],
|
|
10345
|
+
"redaction marker",
|
|
10346
|
+
{ path: entry.path, valueType: valueType(entry.value) }
|
|
10347
|
+
)
|
|
10348
|
+
);
|
|
10349
|
+
}
|
|
10350
|
+
}
|
|
10351
|
+
return limitFindings(findings, options.maxFindings);
|
|
10352
|
+
}
|
|
10353
|
+
};
|
|
10354
|
+
}
|
|
10355
|
+
function createSafetyRawContentRule(options = {}) {
|
|
10356
|
+
const forbiddenKeys = options.forbiddenKeys ?? DEFAULT_RAW_CONTENT_KEYS;
|
|
10357
|
+
return {
|
|
10358
|
+
id: "safety.rawPrompt",
|
|
10359
|
+
category: "safety",
|
|
10360
|
+
defaultSeverity: "error",
|
|
10361
|
+
evaluate(context) {
|
|
10362
|
+
const findings = [];
|
|
10363
|
+
for (const event of context.events) {
|
|
10364
|
+
for (const entry of eventValueEntries(event, { includeSummaries: options.includeSummaries })) {
|
|
10365
|
+
const key = entry.key ?? lastPathSegment(entry.path);
|
|
10366
|
+
if (!isRawContentKey(key, forbiddenKeys)) continue;
|
|
10367
|
+
findings.push(
|
|
10368
|
+
failFinding(
|
|
10369
|
+
"safety.rawPrompt",
|
|
10370
|
+
`Raw content-like field ${entry.path} is present.`,
|
|
10371
|
+
[eventEvidence(event, entry.path)],
|
|
10372
|
+
"metadata-only trace fields",
|
|
10373
|
+
{ path: entry.path, valueType: valueType(entry.value) }
|
|
10374
|
+
)
|
|
10375
|
+
);
|
|
10376
|
+
}
|
|
10377
|
+
}
|
|
10378
|
+
return limitFindings(findings, options.maxFindings);
|
|
10379
|
+
}
|
|
10380
|
+
};
|
|
10381
|
+
}
|
|
10382
|
+
function createSafetySecretPatternRule(options = {}) {
|
|
10383
|
+
const patterns = options.patterns ?? DEFAULT_SECRET_PATTERNS;
|
|
10384
|
+
const maxStringLength = options.maxStringLength ?? 4096;
|
|
10385
|
+
return {
|
|
10386
|
+
id: "safety.secretPattern",
|
|
10387
|
+
category: "safety",
|
|
10388
|
+
defaultSeverity: "error",
|
|
10389
|
+
evaluate(context) {
|
|
10390
|
+
const findings = [];
|
|
10391
|
+
for (const event of context.events) {
|
|
10392
|
+
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
10393
|
+
if (typeof entry.value !== "string") continue;
|
|
10394
|
+
const sample = entry.value.slice(0, maxStringLength);
|
|
10395
|
+
for (const pattern of patterns) {
|
|
10396
|
+
pattern.pattern.lastIndex = 0;
|
|
10397
|
+
if (!pattern.pattern.test(sample)) continue;
|
|
10398
|
+
pattern.pattern.lastIndex = 0;
|
|
10399
|
+
findings.push(
|
|
10400
|
+
failFinding(
|
|
10401
|
+
"safety.secretPattern",
|
|
10402
|
+
`Secret-like pattern ${pattern.id} matched at ${entry.path}.`,
|
|
10403
|
+
[eventEvidence(event, entry.path)],
|
|
10404
|
+
"no secret-like strings",
|
|
10405
|
+
{ pattern: pattern.id, path: entry.path }
|
|
10406
|
+
)
|
|
10407
|
+
);
|
|
10408
|
+
break;
|
|
10409
|
+
}
|
|
10410
|
+
}
|
|
10411
|
+
}
|
|
10412
|
+
return limitFindings(findings, options.maxFindings);
|
|
10413
|
+
}
|
|
10414
|
+
};
|
|
10415
|
+
}
|
|
10416
|
+
function createSafetyOversizedAttributeRule(options) {
|
|
10417
|
+
return {
|
|
10418
|
+
id: "safety.oversizedAttribute",
|
|
10419
|
+
category: "safety",
|
|
10420
|
+
defaultSeverity: "error",
|
|
10421
|
+
evaluate(context) {
|
|
10422
|
+
const findings = [];
|
|
10423
|
+
for (const event of context.events) {
|
|
10424
|
+
for (const entry of eventValueEntries(event, { includeSummaries: true, includeError: true })) {
|
|
10425
|
+
if (typeof entry.value === "string" && options.maxStringLength !== void 0 && entry.value.length > options.maxStringLength) {
|
|
10426
|
+
findings.push(
|
|
10427
|
+
failFinding(
|
|
10428
|
+
"safety.oversizedAttribute",
|
|
10429
|
+
`String at ${entry.path} exceeds ${options.maxStringLength} characters.`,
|
|
10430
|
+
[eventEvidence(event, entry.path)],
|
|
10431
|
+
{ maxStringLength: options.maxStringLength },
|
|
10432
|
+
{ path: entry.path, length: entry.value.length }
|
|
10433
|
+
)
|
|
10434
|
+
);
|
|
10435
|
+
}
|
|
10436
|
+
if (Array.isArray(entry.value) && options.maxArrayLength !== void 0 && entry.value.length > options.maxArrayLength) {
|
|
10437
|
+
findings.push(
|
|
10438
|
+
failFinding(
|
|
10439
|
+
"safety.oversizedAttribute",
|
|
10440
|
+
`Array at ${entry.path} exceeds ${options.maxArrayLength} items.`,
|
|
10441
|
+
[eventEvidence(event, entry.path)],
|
|
10442
|
+
{ maxArrayLength: options.maxArrayLength },
|
|
10443
|
+
{ path: entry.path, length: entry.value.length }
|
|
10444
|
+
)
|
|
10445
|
+
);
|
|
10446
|
+
}
|
|
10447
|
+
if (isRecord14(entry.value) && options.maxObjectKeys !== void 0 && Object.keys(entry.value).length > options.maxObjectKeys) {
|
|
10448
|
+
findings.push(
|
|
10449
|
+
failFinding(
|
|
10450
|
+
"safety.oversizedAttribute",
|
|
10451
|
+
`Object at ${entry.path} exceeds ${options.maxObjectKeys} keys.`,
|
|
10452
|
+
[eventEvidence(event, entry.path)],
|
|
10453
|
+
{ maxObjectKeys: options.maxObjectKeys },
|
|
10454
|
+
{ path: entry.path, keys: Object.keys(entry.value).length }
|
|
10455
|
+
)
|
|
10456
|
+
);
|
|
10457
|
+
}
|
|
10458
|
+
if (options.maxSerializedBytes !== void 0) {
|
|
10459
|
+
const bytes = serializedByteLength(entry.value);
|
|
10460
|
+
if (bytes !== void 0 && bytes > options.maxSerializedBytes) {
|
|
10461
|
+
findings.push(
|
|
10462
|
+
failFinding(
|
|
10463
|
+
"safety.oversizedAttribute",
|
|
10464
|
+
`Value at ${entry.path} exceeds ${options.maxSerializedBytes} serialized bytes.`,
|
|
10465
|
+
[eventEvidence(event, entry.path)],
|
|
10466
|
+
{ maxSerializedBytes: options.maxSerializedBytes },
|
|
10467
|
+
{ path: entry.path, bytes }
|
|
10468
|
+
)
|
|
10469
|
+
);
|
|
10470
|
+
}
|
|
10471
|
+
}
|
|
10472
|
+
}
|
|
10473
|
+
}
|
|
10474
|
+
return limitFindings(findings, options.maxFindings);
|
|
10475
|
+
}
|
|
10476
|
+
};
|
|
10477
|
+
}
|
|
10478
|
+
function createBaselineRegressionRule(options) {
|
|
10479
|
+
return {
|
|
10480
|
+
id: "baseline.regression",
|
|
10481
|
+
category: "baseline",
|
|
10482
|
+
defaultSeverity: "error",
|
|
10483
|
+
evaluate(context) {
|
|
10484
|
+
const baselineSelection = resolveSelectedRun(options.baseline, options.baselineRunId);
|
|
10485
|
+
if (baselineSelection.diagnostics.length > 0 || !baselineSelection.run) {
|
|
10486
|
+
return [
|
|
10487
|
+
failFinding(
|
|
10488
|
+
"baseline.regression",
|
|
10489
|
+
"Baseline run could not be selected.",
|
|
10490
|
+
runEvidence(context.selectedRun),
|
|
10491
|
+
"selectable baseline run",
|
|
10492
|
+
baselineSelection.diagnostics.map((item) => item.code)
|
|
10493
|
+
)
|
|
10494
|
+
];
|
|
10495
|
+
}
|
|
10496
|
+
const baselineFacts = buildFacts2(options.baseline, baselineSelection.run);
|
|
10497
|
+
const baselineContext = {
|
|
10498
|
+
...baselineFacts,
|
|
10499
|
+
selectedRun: baselineSelection.run,
|
|
10500
|
+
sourceLabel: options.baseline.sourceLabel
|
|
10501
|
+
};
|
|
10502
|
+
const findings = [];
|
|
10503
|
+
const durationToleranceMs = options.durationToleranceMs ?? 0;
|
|
10504
|
+
if (baselineContext.format !== context.format) {
|
|
10505
|
+
findings.push(
|
|
10506
|
+
baselineDiffFinding(
|
|
10507
|
+
"Trace format differs from baseline.",
|
|
10508
|
+
runEvidence(context.selectedRun),
|
|
10509
|
+
baselineContext.format,
|
|
10510
|
+
context.format
|
|
10511
|
+
)
|
|
10512
|
+
);
|
|
10513
|
+
}
|
|
10514
|
+
const baselineRunStatus = baselineContext.selectedRun?.status ?? "unknown";
|
|
10515
|
+
const candidateRunStatus = context.selectedRun?.status ?? "unknown";
|
|
10516
|
+
if (baselineRunStatus !== candidateRunStatus) {
|
|
10517
|
+
findings.push(
|
|
10518
|
+
baselineDiffFinding(
|
|
10519
|
+
"Run status differs from baseline.",
|
|
10520
|
+
runEvidence(context.selectedRun),
|
|
10521
|
+
baselineRunStatus,
|
|
10522
|
+
candidateRunStatus
|
|
10523
|
+
)
|
|
10524
|
+
);
|
|
10525
|
+
}
|
|
10526
|
+
const baselineDuration = baselineContext.selectedRun?.durationMs;
|
|
10527
|
+
const candidateDuration = context.selectedRun?.durationMs;
|
|
10528
|
+
if (baselineDuration !== void 0 && candidateDuration !== void 0 && Math.abs(candidateDuration - baselineDuration) > durationToleranceMs) {
|
|
10529
|
+
findings.push(
|
|
10530
|
+
baselineDiffFinding(
|
|
10531
|
+
"Run duration differs from baseline beyond tolerance.",
|
|
10532
|
+
runEvidence(context.selectedRun),
|
|
10533
|
+
{ durationMs: baselineDuration, toleranceMs: durationToleranceMs },
|
|
10534
|
+
candidateDuration
|
|
10535
|
+
)
|
|
10536
|
+
);
|
|
10537
|
+
}
|
|
10538
|
+
const comparisons = [
|
|
10539
|
+
{
|
|
10540
|
+
label: "Tree shape",
|
|
10541
|
+
path: "tree",
|
|
10542
|
+
expected: treeShape(baselineContext.rootNodes),
|
|
10543
|
+
actual: treeShape(context.rootNodes),
|
|
10544
|
+
evidence: runEvidence(context.selectedRun)
|
|
10545
|
+
},
|
|
10546
|
+
{
|
|
10547
|
+
label: "Event statuses",
|
|
10548
|
+
path: "status",
|
|
10549
|
+
expected: statusShape(baselineContext),
|
|
10550
|
+
actual: statusShape(context),
|
|
10551
|
+
evidence: runEvidence(context.selectedRun)
|
|
10552
|
+
},
|
|
10553
|
+
{
|
|
10554
|
+
label: "Tool usage",
|
|
10555
|
+
path: "tool",
|
|
10556
|
+
expected: toolShape(baselineContext),
|
|
10557
|
+
actual: toolShape(context),
|
|
10558
|
+
evidence: firstEvidenceForKind(context, "TOOL", "tool")
|
|
10559
|
+
},
|
|
10560
|
+
{
|
|
10561
|
+
label: "LLM usage",
|
|
10562
|
+
path: "llm",
|
|
10563
|
+
expected: llmShape(baselineContext),
|
|
10564
|
+
actual: llmShape(context),
|
|
10565
|
+
evidence: firstEvidenceForKind(context, "LLM", "llm")
|
|
10566
|
+
},
|
|
10567
|
+
{
|
|
10568
|
+
label: "Error profile",
|
|
10569
|
+
path: "error",
|
|
10570
|
+
expected: errorShape(baselineContext),
|
|
10571
|
+
actual: errorShape(context),
|
|
10572
|
+
evidence: firstEvidenceForKind(context, "ERROR", "error")
|
|
10573
|
+
},
|
|
10574
|
+
{
|
|
10575
|
+
label: "Retrieval signals",
|
|
10576
|
+
path: "retrieval",
|
|
10577
|
+
expected: retrievalShape(baselineContext),
|
|
10578
|
+
actual: retrievalShape(context),
|
|
10579
|
+
evidence: firstEvidenceForKind(context, "RETRIEVER", "retrieval")
|
|
10580
|
+
},
|
|
10581
|
+
{
|
|
10582
|
+
label: "Guardrail signals",
|
|
10583
|
+
path: "guardrail",
|
|
10584
|
+
expected: guardrailShape(baselineContext),
|
|
10585
|
+
actual: guardrailShape(context),
|
|
10586
|
+
evidence: guardrailEvents(context)[0] ? [eventEvidence(guardrailEvents(context)[0], "guardrail")] : runEvidence(context.selectedRun)
|
|
10587
|
+
}
|
|
10588
|
+
];
|
|
10589
|
+
for (const comparison of comparisons) {
|
|
10590
|
+
if (JSON.stringify(comparison.expected) === JSON.stringify(comparison.actual)) {
|
|
10591
|
+
continue;
|
|
10592
|
+
}
|
|
10593
|
+
findings.push(
|
|
10594
|
+
baselineDiffFinding(
|
|
10595
|
+
`${comparison.label} differs from baseline.`,
|
|
10596
|
+
comparison.evidence.length > 0 ? comparison.evidence : [{ runId: context.selectedRun?.runId, path: comparison.path }],
|
|
10597
|
+
comparison.expected,
|
|
10598
|
+
comparison.actual
|
|
10599
|
+
)
|
|
10600
|
+
);
|
|
10601
|
+
}
|
|
10602
|
+
return findings;
|
|
10603
|
+
}
|
|
10604
|
+
};
|
|
10605
|
+
}
|
|
10606
|
+
function runTraceChecks(input3, options = {}) {
|
|
10607
|
+
const selected = resolveSelectedRun(input3, options.runId);
|
|
10608
|
+
if (selected.diagnostics.length > 0) {
|
|
10609
|
+
return errorResult(input3, selected.diagnostics, selected.run);
|
|
10610
|
+
}
|
|
10611
|
+
const rules = selectRules(options.rules ?? [], options.select);
|
|
10612
|
+
if (rules.diagnostics.length > 0) {
|
|
10613
|
+
return errorResult(input3, rules.diagnostics, selected.run);
|
|
10614
|
+
}
|
|
10615
|
+
const facts = buildFacts2(input3, selected.run);
|
|
10616
|
+
const context = {
|
|
10617
|
+
...facts,
|
|
10618
|
+
...selected.run ? { selectedRun: selected.run } : {},
|
|
10619
|
+
...input3.sourceLabel ? { sourceLabel: input3.sourceLabel } : {}
|
|
10620
|
+
};
|
|
10621
|
+
const diagnostics = [];
|
|
10622
|
+
const findings = [];
|
|
10623
|
+
for (const rule of rules.rules) {
|
|
10624
|
+
try {
|
|
10625
|
+
findings.push(...rule.evaluate(context).map((finding) => normalizeFinding(rule, finding)));
|
|
10626
|
+
} catch (error) {
|
|
10627
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
10628
|
+
diagnostics.push(
|
|
10629
|
+
diagnostic("AI_CHECK_INTERNAL_ERROR", `Rule ${rule.id} failed: ${message}`, rule.id)
|
|
10630
|
+
);
|
|
10631
|
+
}
|
|
10632
|
+
}
|
|
10633
|
+
if (diagnostics.length > 0) {
|
|
10634
|
+
return errorResult(input3, diagnostics, selected.run);
|
|
10635
|
+
}
|
|
10636
|
+
const eventById = new Map(input3.read.events.map((event) => [event.eventId, event]));
|
|
10637
|
+
const sortedFindings = findings.sort(compareFindings(eventById));
|
|
10638
|
+
const summary = summarize(sortedFindings, diagnostics);
|
|
10639
|
+
const status = summary.failed > 0 ? "fail" : "pass";
|
|
10640
|
+
return {
|
|
10641
|
+
ok: status === "pass",
|
|
10642
|
+
status,
|
|
10643
|
+
format: input3.read.format,
|
|
10644
|
+
...selected.run ? { runId: selected.run.runId } : {},
|
|
10645
|
+
summary,
|
|
10646
|
+
findings: sortedFindings,
|
|
10647
|
+
diagnostics
|
|
10648
|
+
};
|
|
10649
|
+
}
|
|
10650
|
+
|
|
10651
|
+
// packages/cli/src/check.ts
|
|
10652
|
+
var DEFAULT_SELECT = ["run.status"];
|
|
10653
|
+
var CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
10654
|
+
var TS_CONFIG_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
10655
|
+
function diagnostic2(code, message, severity = "error") {
|
|
10656
|
+
return { code, message, severity };
|
|
10657
|
+
}
|
|
10658
|
+
function errorResult2(code, message, format = "unknown") {
|
|
10659
|
+
const diagnostics = [diagnostic2(code, message)];
|
|
10660
|
+
return {
|
|
10661
|
+
ok: false,
|
|
10662
|
+
status: "error",
|
|
10663
|
+
format,
|
|
10664
|
+
summary: {
|
|
10665
|
+
passed: 0,
|
|
10666
|
+
failed: 0,
|
|
10667
|
+
warnings: 0,
|
|
10668
|
+
errors: 1
|
|
10669
|
+
},
|
|
10670
|
+
findings: [],
|
|
10671
|
+
diagnostics
|
|
10672
|
+
};
|
|
10673
|
+
}
|
|
10674
|
+
function parseNumber(value, label) {
|
|
10675
|
+
if (value === void 0) return void 0;
|
|
10676
|
+
const parsed = Number(value);
|
|
10677
|
+
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
10678
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
10679
|
+
}
|
|
10680
|
+
return parsed;
|
|
10681
|
+
}
|
|
10682
|
+
function asStringArray(value) {
|
|
10683
|
+
if (value === void 0) return void 0;
|
|
10684
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
10685
|
+
throw new Error("Expected an array of strings.");
|
|
10686
|
+
}
|
|
10687
|
+
return value;
|
|
10688
|
+
}
|
|
10689
|
+
function asConfig(value) {
|
|
10690
|
+
if (value === void 0 || value === null) return {};
|
|
10691
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
10692
|
+
throw new Error("Config must export an object.");
|
|
10693
|
+
}
|
|
10694
|
+
return value;
|
|
10695
|
+
}
|
|
10696
|
+
async function loadConfig(configPath) {
|
|
10697
|
+
if (configPath === void 0) return {};
|
|
10698
|
+
const extension = path10__default.default.extname(configPath);
|
|
10699
|
+
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
10700
|
+
throw new Error(
|
|
10701
|
+
"TypeScript check configs require an explicit precompiled JavaScript config or future --config-loader support."
|
|
10702
|
+
);
|
|
10703
|
+
}
|
|
10704
|
+
if (!CONFIG_EXTENSIONS.has(extension)) {
|
|
10705
|
+
throw new Error("Unsupported check config extension. Use .json, .js, .mjs, or .cjs.");
|
|
10706
|
+
}
|
|
10707
|
+
const absolute = path10__default.default.resolve(configPath);
|
|
10708
|
+
if (extension === ".json") {
|
|
10709
|
+
const raw = await promises.readFile(absolute, "utf-8");
|
|
10710
|
+
return asConfig(JSON.parse(raw));
|
|
10711
|
+
}
|
|
10712
|
+
const mod = await import(url.pathToFileURL(absolute).href);
|
|
10713
|
+
return asConfig("default" in mod ? mod.default : mod);
|
|
10714
|
+
}
|
|
10715
|
+
function normalizeConfig(config) {
|
|
10716
|
+
if (config.checks === void 0) return {};
|
|
10717
|
+
if (typeof config.checks !== "object" || Array.isArray(config.checks)) {
|
|
10718
|
+
throw new Error("checks config must be an object.");
|
|
10719
|
+
}
|
|
10720
|
+
return config.checks;
|
|
10721
|
+
}
|
|
10722
|
+
function buildRules(config, options) {
|
|
10723
|
+
const diagnostics = [];
|
|
10724
|
+
const checks = normalizeConfig(config);
|
|
10725
|
+
const run = checks.run ?? {};
|
|
10726
|
+
const tool = checks.tool ?? {};
|
|
10727
|
+
const llm = checks.llm ?? {};
|
|
10728
|
+
const structure = checks.structure ?? {};
|
|
10729
|
+
const safety = checks.safety ?? {};
|
|
10730
|
+
const maxDurationMs = parseNumber(options.maxDurationMs, "--max-duration-ms") ?? run.maxDurationMs;
|
|
10731
|
+
const maxTotalTokens = parseNumber(options.maxTotalTokens, "--max-total-tokens") ?? llm.maxTotalTokens;
|
|
10732
|
+
const rules = [
|
|
10733
|
+
createRunStatusRule(run),
|
|
10734
|
+
createStructureOrphanRule(),
|
|
10735
|
+
createStructureCycleRule(),
|
|
10736
|
+
createSafetyRawContentRule(),
|
|
10737
|
+
createSafetySecretPatternRule()
|
|
10738
|
+
];
|
|
10739
|
+
if (maxDurationMs !== void 0) {
|
|
10740
|
+
rules.push(createRunDurationRule({ maxDurationMs }));
|
|
10741
|
+
}
|
|
10742
|
+
if (run.maxDepth !== void 0) {
|
|
10743
|
+
rules.push(createRunDepthRule({ maxDepth: run.maxDepth }));
|
|
10744
|
+
}
|
|
10745
|
+
const toolOptions = {
|
|
10746
|
+
...tool,
|
|
10747
|
+
required: [...tool.required ?? [], ...options.requiredTool ?? []],
|
|
10748
|
+
forbidden: [...tool.forbidden ?? [], ...options.forbiddenTool ?? []]
|
|
10749
|
+
};
|
|
10750
|
+
if (toolOptions.required?.length || toolOptions.forbidden?.length || toolOptions.allowed?.length || toolOptions.minCount !== void 0 || toolOptions.maxCount !== void 0) {
|
|
10751
|
+
rules.push(createToolUsageRule(toolOptions));
|
|
10752
|
+
}
|
|
10753
|
+
const llmOptions = {
|
|
10754
|
+
...llm,
|
|
10755
|
+
allowedModels: [...llm.allowedModels ?? [], ...options.allowedModel ?? []],
|
|
10756
|
+
...maxTotalTokens !== void 0 ? { maxTotalTokens } : {}
|
|
10757
|
+
};
|
|
10758
|
+
if (llmOptions.allowedModels?.length || llmOptions.allowedProviders?.length || llmOptions.finishReasons?.length || llmOptions.maxCalls !== void 0 || llmOptions.maxInputTokens !== void 0 || llmOptions.maxOutputTokens !== void 0 || llmOptions.maxTotalTokens !== void 0 || llmOptions.maxCachedTokens !== void 0) {
|
|
10759
|
+
rules.push(createLlmUsageRule(llmOptions));
|
|
10760
|
+
}
|
|
10761
|
+
if (structure.minConfidence !== void 0 || structure.requireParentBeforeChild !== void 0 || structure.requireTraceParentSpan !== void 0) {
|
|
10762
|
+
rules.push(createStructureRelationshipRule(structure));
|
|
10763
|
+
}
|
|
10764
|
+
if (structure.maxChildren !== void 0 || structure.maxConcurrent !== void 0) {
|
|
10765
|
+
rules.push(
|
|
10766
|
+
createStructureParallelWidthRule({
|
|
10767
|
+
maxChildren: structure.maxChildren,
|
|
10768
|
+
maxConcurrent: structure.maxConcurrent
|
|
10769
|
+
})
|
|
10770
|
+
);
|
|
10771
|
+
}
|
|
10772
|
+
if (safety.redaction) rules.push(createSafetyRedactionRule());
|
|
10773
|
+
if (safety.maxStringLength !== void 0 || safety.maxArrayLength !== void 0 || safety.maxObjectKeys !== void 0 || safety.maxSerializedBytes !== void 0) {
|
|
10774
|
+
rules.push(createSafetyOversizedAttributeRule(safety));
|
|
10775
|
+
}
|
|
10776
|
+
const select = [
|
|
10777
|
+
...asStringArray(checks.select) ?? [],
|
|
10778
|
+
...options.rule ?? []
|
|
10779
|
+
];
|
|
10780
|
+
return {
|
|
10781
|
+
rules,
|
|
10782
|
+
select: select.length > 0 ? select : DEFAULT_SELECT,
|
|
10783
|
+
diagnostics
|
|
10784
|
+
};
|
|
10785
|
+
}
|
|
10786
|
+
function exitCodeFor(result) {
|
|
10787
|
+
if (result.status === "pass") return 0;
|
|
10788
|
+
if (result.status === "fail") return 1;
|
|
10789
|
+
const codes = result.diagnostics.map((item) => item.code);
|
|
10790
|
+
if (codes.some(
|
|
10791
|
+
(code) => code === "AI_CHECK_UNSUPPORTED_FORMAT" || code === "AI_CHECK_AMBIGUOUS_FORMAT"
|
|
10792
|
+
)) {
|
|
10793
|
+
return 4;
|
|
10794
|
+
}
|
|
10795
|
+
if (codes.some(
|
|
10796
|
+
(code) => code === "AI_CHECK_TRACE_UNREADABLE" || code === "AI_CHECK_BASELINE_UNREADABLE"
|
|
10797
|
+
)) {
|
|
10798
|
+
return 3;
|
|
10799
|
+
}
|
|
10800
|
+
if (codes.some(
|
|
10801
|
+
(code) => code === "AI_CHECK_INVALID_ARGUMENTS" || code === "AI_CHECK_INVALID_CONFIG" || code === "AI_CHECK_CONFIG_LOAD_FAILED" || code === "AI_CHECK_RUN_SELECTION_REQUIRED"
|
|
10802
|
+
)) {
|
|
10803
|
+
return 2;
|
|
10804
|
+
}
|
|
10805
|
+
return 1;
|
|
10806
|
+
}
|
|
10807
|
+
function stable(value) {
|
|
10808
|
+
if (Array.isArray(value)) return value.map(stable);
|
|
10809
|
+
if (value === null || typeof value !== "object") return value;
|
|
10810
|
+
const record = value;
|
|
10811
|
+
return Object.fromEntries(
|
|
10812
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable(record[key])])
|
|
10813
|
+
);
|
|
10814
|
+
}
|
|
10815
|
+
function printJson(result) {
|
|
10816
|
+
console.log(JSON.stringify(stable(result), null, 2));
|
|
10817
|
+
}
|
|
10818
|
+
function printHuman(result) {
|
|
10819
|
+
console.log(`Check status: ${result.status}`);
|
|
10820
|
+
console.log(`Format: ${result.format}`);
|
|
10821
|
+
if (result.runId !== void 0) console.log(`Run: ${result.runId}`);
|
|
10822
|
+
console.log(
|
|
10823
|
+
`Summary: ${result.summary.failed} failed, ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
10824
|
+
);
|
|
10825
|
+
for (const diagnostic3 of result.diagnostics) {
|
|
10826
|
+
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
10827
|
+
}
|
|
10828
|
+
for (const finding of result.findings) {
|
|
10829
|
+
const path12 = finding.evidence[0]?.path;
|
|
10830
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path12 ? ` (${path12})` : ""}`);
|
|
10831
|
+
}
|
|
10832
|
+
}
|
|
10833
|
+
function readErrorResult(error) {
|
|
10834
|
+
if (error instanceof TraceReadError) {
|
|
10835
|
+
const code = error.code === "unsupported_format" ? "AI_CHECK_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_CHECK_AMBIGUOUS_FORMAT" : "AI_CHECK_TRACE_UNREADABLE";
|
|
10836
|
+
return errorResult2(code, error.message);
|
|
10837
|
+
}
|
|
10838
|
+
return errorResult2(
|
|
10839
|
+
"AI_CHECK_TRACE_UNREADABLE",
|
|
10840
|
+
error instanceof Error ? error.message : String(error)
|
|
10841
|
+
);
|
|
10842
|
+
}
|
|
10843
|
+
async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
10844
|
+
let result;
|
|
10845
|
+
let phase = "config";
|
|
10846
|
+
try {
|
|
10847
|
+
const config = await loadConfig(options.config);
|
|
10848
|
+
const built = buildRules(config, options);
|
|
10849
|
+
if (built.diagnostics.some((item) => item.severity === "error")) {
|
|
10850
|
+
result = errorResult2("AI_CHECK_INVALID_CONFIG", "Invalid check configuration.");
|
|
10851
|
+
result.diagnostics = [...built.diagnostics];
|
|
10852
|
+
} else {
|
|
10853
|
+
phase = "read";
|
|
10854
|
+
const input3 = await inputFromTarget(target, options, stdin);
|
|
10855
|
+
const read = await openTrace(input3, {
|
|
10856
|
+
...options.format !== void 0 ? { format: options.format } : {}
|
|
10857
|
+
});
|
|
10858
|
+
result = runTraceChecks(
|
|
10859
|
+
{ read },
|
|
10860
|
+
{
|
|
10861
|
+
rules: built.rules,
|
|
10862
|
+
select: built.select,
|
|
10863
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
10864
|
+
}
|
|
10865
|
+
);
|
|
10866
|
+
}
|
|
10867
|
+
} catch (error) {
|
|
10868
|
+
if (phase === "config") {
|
|
10869
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
10870
|
+
const code = message.startsWith("--") ? "AI_CHECK_INVALID_ARGUMENTS" : error instanceof SyntaxError || message.includes("Unsupported check config extension") || message.includes("TypeScript check configs") || message.includes("Config must") || message.includes("checks config") || message.includes("Expected an array") ? "AI_CHECK_INVALID_CONFIG" : "AI_CHECK_CONFIG_LOAD_FAILED";
|
|
10871
|
+
result = errorResult2(
|
|
10872
|
+
code,
|
|
10873
|
+
message
|
|
10874
|
+
);
|
|
10875
|
+
} else {
|
|
10876
|
+
result = readErrorResult(error);
|
|
10877
|
+
}
|
|
10878
|
+
}
|
|
10879
|
+
process.exitCode = exitCodeFor(result);
|
|
10880
|
+
if (options.json) printJson(result);
|
|
10881
|
+
else printHuman(result);
|
|
10882
|
+
}
|
|
10883
|
+
|
|
10884
|
+
// packages/cli/src/safety.ts
|
|
10885
|
+
var BEST_EFFORT_NOTE = "Best-effort local safety verification only; not a compliance, privacy, security, or regulatory certification.";
|
|
10886
|
+
var DEFAULT_MAX_STRING_LENGTH = 16384;
|
|
10887
|
+
var DEFAULT_MAX_ARRAY_LENGTH = 1e3;
|
|
10888
|
+
var DEFAULT_MAX_OBJECT_KEYS = 200;
|
|
10889
|
+
var DEFAULT_MAX_SERIALIZED_BYTES = 128 * 1024;
|
|
10890
|
+
function parseLimit3(value, label) {
|
|
10891
|
+
if (value === void 0) return void 0;
|
|
10892
|
+
const parsed = Number(value);
|
|
10893
|
+
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
10894
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
10895
|
+
}
|
|
10896
|
+
return parsed;
|
|
10897
|
+
}
|
|
10898
|
+
function stable2(value) {
|
|
10899
|
+
if (Array.isArray(value)) return value.map(stable2);
|
|
10900
|
+
if (value === null || typeof value !== "object") return value;
|
|
10901
|
+
const record = value;
|
|
10902
|
+
return Object.fromEntries(
|
|
10903
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable2(record[key])])
|
|
10904
|
+
);
|
|
10905
|
+
}
|
|
10906
|
+
function safetyDiagnostic(code, message, severity = "error") {
|
|
10907
|
+
return { code, message, severity };
|
|
10908
|
+
}
|
|
10909
|
+
function warningDiagnostics(warnings, unsupportedFields) {
|
|
10910
|
+
return [
|
|
10911
|
+
...warnings.map(
|
|
10912
|
+
(warning) => safetyDiagnostic(
|
|
10913
|
+
warning.code,
|
|
10914
|
+
warning.message,
|
|
10915
|
+
warning.severity === "error" ? "error" : "warning"
|
|
10916
|
+
)
|
|
10917
|
+
),
|
|
10918
|
+
...unsupportedFields.map(
|
|
10919
|
+
(field) => safetyDiagnostic(
|
|
10920
|
+
"unsupported_field",
|
|
10921
|
+
`Reader reported unsupported field: ${field}`,
|
|
10922
|
+
"warning"
|
|
10923
|
+
)
|
|
10924
|
+
)
|
|
10925
|
+
];
|
|
10926
|
+
}
|
|
10927
|
+
function diagnosticFromCheck(item) {
|
|
10928
|
+
return safetyDiagnostic(item.code, item.message, item.severity);
|
|
10929
|
+
}
|
|
10930
|
+
function statusFrom(findings, diagnostics) {
|
|
10931
|
+
if (diagnostics.some((item) => item.severity === "error")) return "UNKNOWN";
|
|
10932
|
+
if (findings.some((item) => item.severity === "error")) return "UNSAFE";
|
|
10933
|
+
if (diagnostics.some((item) => item.severity === "warning")) return "SAFE WITH WARNINGS";
|
|
10934
|
+
if (findings.some((item) => item.severity === "warning")) return "SAFE WITH WARNINGS";
|
|
10935
|
+
return "SAFE";
|
|
10936
|
+
}
|
|
10937
|
+
function resultFromParts(parts) {
|
|
10938
|
+
const findings = [...parts.findings ?? []];
|
|
10939
|
+
const diagnostics = [...parts.diagnostics ?? []];
|
|
10940
|
+
const warnings = [...parts.warnings ?? []];
|
|
10941
|
+
const unsupportedFields = [...parts.unsupportedFields ?? []];
|
|
10942
|
+
const status = statusFrom(findings, diagnostics);
|
|
10943
|
+
return {
|
|
10944
|
+
ok: status === "SAFE" || status === "SAFE WITH WARNINGS",
|
|
10945
|
+
command: parts.command,
|
|
10946
|
+
status,
|
|
10947
|
+
format: parts.format,
|
|
10948
|
+
...parts.runId !== void 0 ? { runId: parts.runId } : {},
|
|
10949
|
+
summary: {
|
|
10950
|
+
findings: findings.length,
|
|
10951
|
+
warnings: diagnostics.filter((item) => item.severity === "warning").length + findings.filter((item) => item.severity === "warning").length,
|
|
10952
|
+
errors: diagnostics.filter((item) => item.severity === "error").length + findings.filter((item) => item.severity === "error").length
|
|
10953
|
+
},
|
|
10954
|
+
findings,
|
|
10955
|
+
diagnostics,
|
|
10956
|
+
warnings,
|
|
10957
|
+
unsupportedFields,
|
|
10958
|
+
note: BEST_EFFORT_NOTE
|
|
10959
|
+
};
|
|
10960
|
+
}
|
|
10961
|
+
function readErrorResult2(command, error) {
|
|
10962
|
+
if (error instanceof TraceReadError) {
|
|
10963
|
+
const code = error.code === "unsupported_format" ? "AI_SAFETY_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_SAFETY_AMBIGUOUS_FORMAT" : "AI_SAFETY_TRACE_UNREADABLE";
|
|
10964
|
+
return resultFromParts({
|
|
10965
|
+
command,
|
|
10966
|
+
format: "unknown",
|
|
10967
|
+
diagnostics: [safetyDiagnostic(code, error.message)],
|
|
10968
|
+
warnings: error.warnings
|
|
10969
|
+
});
|
|
10970
|
+
}
|
|
10971
|
+
return resultFromParts({
|
|
10972
|
+
command,
|
|
10973
|
+
format: "unknown",
|
|
10974
|
+
diagnostics: [
|
|
10975
|
+
safetyDiagnostic(
|
|
10976
|
+
"AI_SAFETY_TRACE_UNREADABLE",
|
|
10977
|
+
error instanceof Error ? error.message : String(error)
|
|
10978
|
+
)
|
|
10979
|
+
]
|
|
10980
|
+
});
|
|
10981
|
+
}
|
|
10982
|
+
function invalidArgumentResult(command, error) {
|
|
10983
|
+
return resultFromParts({
|
|
10984
|
+
command,
|
|
10985
|
+
format: "unknown",
|
|
10986
|
+
diagnostics: [
|
|
10987
|
+
safetyDiagnostic(
|
|
10988
|
+
"AI_SAFETY_INVALID_ARGUMENTS",
|
|
10989
|
+
error instanceof Error ? error.message : String(error)
|
|
10990
|
+
)
|
|
10991
|
+
]
|
|
10992
|
+
});
|
|
10993
|
+
}
|
|
10994
|
+
function buildSafetyRules(options) {
|
|
10995
|
+
const maxStringLength = parseLimit3(options.maxStringLength, "--max-string-length") ?? DEFAULT_MAX_STRING_LENGTH;
|
|
10996
|
+
const maxArrayLength = parseLimit3(options.maxArrayLength, "--max-array-length") ?? DEFAULT_MAX_ARRAY_LENGTH;
|
|
10997
|
+
const maxObjectKeys = parseLimit3(options.maxObjectKeys, "--max-object-keys") ?? DEFAULT_MAX_OBJECT_KEYS;
|
|
10998
|
+
const maxSerializedBytes = parseLimit3(options.maxSerializedBytes, "--max-serialized-bytes") ?? DEFAULT_MAX_SERIALIZED_BYTES;
|
|
10999
|
+
return [
|
|
11000
|
+
createSafetyRawContentRule(),
|
|
11001
|
+
createSafetyRedactionRule(),
|
|
11002
|
+
createSafetySecretPatternRule(),
|
|
11003
|
+
createSafetyOversizedAttributeRule({
|
|
11004
|
+
maxStringLength,
|
|
11005
|
+
maxArrayLength,
|
|
11006
|
+
maxObjectKeys,
|
|
11007
|
+
maxSerializedBytes
|
|
11008
|
+
})
|
|
11009
|
+
];
|
|
11010
|
+
}
|
|
11011
|
+
function exitCodeFor2(result) {
|
|
11012
|
+
if (result.status === "SAFE" || result.status === "SAFE WITH WARNINGS") return 0;
|
|
11013
|
+
if (result.status === "UNSAFE") return 1;
|
|
11014
|
+
return 2;
|
|
11015
|
+
}
|
|
11016
|
+
function printJson2(result) {
|
|
11017
|
+
console.log(JSON.stringify(stable2(result), null, 2));
|
|
11018
|
+
}
|
|
11019
|
+
function printHuman2(result) {
|
|
11020
|
+
console.log(`Safety status: ${result.status}`);
|
|
11021
|
+
console.log(`Format: ${result.format}`);
|
|
11022
|
+
if (result.runId !== void 0) console.log(`Run: ${result.runId}`);
|
|
11023
|
+
console.log(
|
|
11024
|
+
`Summary: ${result.summary.findings} finding(s), ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
11025
|
+
);
|
|
11026
|
+
for (const diagnostic3 of result.diagnostics) {
|
|
11027
|
+
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
11028
|
+
}
|
|
11029
|
+
for (const finding of result.findings) {
|
|
11030
|
+
const path12 = finding.evidence[0]?.path;
|
|
11031
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path12 ? ` (${path12})` : ""}`);
|
|
11032
|
+
}
|
|
11033
|
+
console.log(`Note: ${result.note}`);
|
|
11034
|
+
}
|
|
11035
|
+
async function safetyCommand(command, target, options, stdin) {
|
|
11036
|
+
let result;
|
|
11037
|
+
try {
|
|
11038
|
+
const rules = buildSafetyRules(options);
|
|
11039
|
+
const input3 = await inputFromTarget(target, options, stdin);
|
|
11040
|
+
const read = await openTrace(input3, {
|
|
11041
|
+
...options.format !== void 0 ? { format: options.format } : {}
|
|
11042
|
+
});
|
|
11043
|
+
const checkResult = runTraceChecks(
|
|
11044
|
+
{ read },
|
|
11045
|
+
{
|
|
11046
|
+
rules,
|
|
11047
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
11048
|
+
}
|
|
11049
|
+
);
|
|
11050
|
+
result = resultFromParts({
|
|
11051
|
+
command,
|
|
11052
|
+
format: checkResult.format,
|
|
11053
|
+
runId: checkResult.runId,
|
|
11054
|
+
findings: checkResult.findings,
|
|
11055
|
+
diagnostics: [
|
|
11056
|
+
...checkResult.diagnostics.map(diagnosticFromCheck),
|
|
11057
|
+
...warningDiagnostics(read.warnings, read.unsupportedFields)
|
|
11058
|
+
],
|
|
11059
|
+
warnings: read.warnings,
|
|
11060
|
+
unsupportedFields: read.unsupportedFields
|
|
11061
|
+
});
|
|
11062
|
+
} catch (error) {
|
|
11063
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
11064
|
+
result = message.startsWith("--") ? invalidArgumentResult(command, error) : readErrorResult2(command, error);
|
|
11065
|
+
}
|
|
11066
|
+
process.exitCode = exitCodeFor2(result);
|
|
11067
|
+
if (options.json) printJson2(result);
|
|
11068
|
+
else printHuman2(result);
|
|
11069
|
+
}
|
|
11070
|
+
function scanCommand(target, options = {}, stdin = process.stdin) {
|
|
11071
|
+
return safetyCommand("scan", target, options, stdin);
|
|
11072
|
+
}
|
|
11073
|
+
function verifySafeCommand(target, options = {}, stdin = process.stdin) {
|
|
11074
|
+
return safetyCommand("verify-safe", target, options, stdin);
|
|
11075
|
+
}
|
|
11076
|
+
var NOTE = "Generated locally by AgentInspect. Artifacts are best-effort summaries, not compliance or security certification.";
|
|
11077
|
+
var SAFETY_RULES = [
|
|
11078
|
+
createSafetyRawContentRule(),
|
|
11079
|
+
createSafetyRedactionRule(),
|
|
11080
|
+
createSafetySecretPatternRule(),
|
|
11081
|
+
createSafetyOversizedAttributeRule({
|
|
11082
|
+
maxStringLength: 16384,
|
|
11083
|
+
maxArrayLength: 1e3,
|
|
11084
|
+
maxObjectKeys: 200,
|
|
11085
|
+
maxSerializedBytes: 128 * 1024
|
|
11086
|
+
})
|
|
11087
|
+
];
|
|
11088
|
+
function stable3(value) {
|
|
11089
|
+
if (Array.isArray(value)) return value.map(stable3);
|
|
11090
|
+
if (value === null || typeof value !== "object") return value;
|
|
11091
|
+
const record = value;
|
|
11092
|
+
return Object.fromEntries(
|
|
11093
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable3(record[key])])
|
|
11094
|
+
);
|
|
11095
|
+
}
|
|
11096
|
+
function writeJson3(value) {
|
|
11097
|
+
return `${JSON.stringify(stable3(value), null, 2)}
|
|
11098
|
+
`;
|
|
11099
|
+
}
|
|
11100
|
+
function escapeHtml2(value) {
|
|
11101
|
+
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """);
|
|
11102
|
+
}
|
|
11103
|
+
function markdownTable(rows) {
|
|
11104
|
+
const lines = ["| Field | Value |", "| --- | --- |"];
|
|
11105
|
+
for (const [key, value] of rows) {
|
|
11106
|
+
lines.push(`| ${key} | ${value ?? "unknown"} |`);
|
|
11107
|
+
}
|
|
11108
|
+
return lines.join("\n");
|
|
11109
|
+
}
|
|
11110
|
+
function increment(record, key) {
|
|
11111
|
+
const label = key && key.trim() !== "" ? key : "unknown";
|
|
11112
|
+
record[label] = (record[label] ?? 0) + 1;
|
|
11113
|
+
}
|
|
11114
|
+
function selectRun3(read, runId) {
|
|
11115
|
+
if (runId !== void 0) {
|
|
11116
|
+
return read.runs.find((run) => run.runId === runId);
|
|
11117
|
+
}
|
|
11118
|
+
return read.runs.length === 1 ? read.runs[0] : void 0;
|
|
11119
|
+
}
|
|
11120
|
+
function summarizeTrace(read, run) {
|
|
11121
|
+
const runId = run?.runId;
|
|
11122
|
+
const scopedEvents = runId === void 0 ? read.events : read.events.filter((event) => event.runId === runId);
|
|
11123
|
+
const eventsByKind = {};
|
|
11124
|
+
const eventsByStatus = {};
|
|
11125
|
+
for (const event of scopedEvents) {
|
|
11126
|
+
increment(eventsByKind, event.kind);
|
|
11127
|
+
increment(eventsByStatus, event.status);
|
|
11128
|
+
}
|
|
11129
|
+
return {
|
|
11130
|
+
format: read.format,
|
|
11131
|
+
...runId !== void 0 ? { runId } : {},
|
|
11132
|
+
...run?.status !== void 0 ? { runStatus: run.status } : {},
|
|
11133
|
+
...run?.durationMs !== void 0 ? { runDurationMs: run.durationMs } : {},
|
|
11134
|
+
runCount: read.runs.length,
|
|
11135
|
+
eventCount: scopedEvents.length,
|
|
11136
|
+
eventsByKind: Object.fromEntries(Object.entries(eventsByKind).sort()),
|
|
11137
|
+
eventsByStatus: Object.fromEntries(Object.entries(eventsByStatus).sort()),
|
|
11138
|
+
readerWarnings: read.warnings.length,
|
|
11139
|
+
unsupportedFields: read.unsupportedFields.length
|
|
11140
|
+
};
|
|
11141
|
+
}
|
|
11142
|
+
function renderCheckSection(result) {
|
|
11143
|
+
const lines = [
|
|
11144
|
+
`Status: ${result.status}`,
|
|
11145
|
+
`Findings: ${result.findings.length}`,
|
|
11146
|
+
`Diagnostics: ${result.diagnostics.length}`
|
|
11147
|
+
];
|
|
11148
|
+
for (const finding of result.findings.slice(0, 10)) {
|
|
11149
|
+
const path12 = finding.evidence[0]?.path ?? "(run)";
|
|
11150
|
+
lines.push(`- ${finding.ruleId}: ${finding.message} (${path12})`);
|
|
11151
|
+
}
|
|
11152
|
+
for (const diagnostic3 of result.diagnostics.slice(0, 10)) {
|
|
11153
|
+
lines.push(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
11154
|
+
}
|
|
11155
|
+
return lines.join("\n");
|
|
11156
|
+
}
|
|
11157
|
+
function renderMarkdown(trace, check, diff) {
|
|
11158
|
+
const lines = [
|
|
11159
|
+
"# AgentInspect CI Artifacts",
|
|
11160
|
+
"",
|
|
11161
|
+
NOTE,
|
|
11162
|
+
"",
|
|
11163
|
+
"## Trace",
|
|
11164
|
+
"",
|
|
11165
|
+
markdownTable([
|
|
11166
|
+
["Format", trace.format],
|
|
11167
|
+
["Run", trace.runId],
|
|
11168
|
+
["Run status", trace.runStatus],
|
|
11169
|
+
["Run duration ms", trace.runDurationMs],
|
|
11170
|
+
["Runs", trace.runCount],
|
|
11171
|
+
["Events", trace.eventCount],
|
|
11172
|
+
["Reader warnings", trace.readerWarnings],
|
|
11173
|
+
["Unsupported fields", trace.unsupportedFields]
|
|
11174
|
+
]),
|
|
11175
|
+
"",
|
|
11176
|
+
"## Safety check",
|
|
11177
|
+
"",
|
|
11178
|
+
"```text",
|
|
11179
|
+
renderCheckSection(check),
|
|
11180
|
+
"```",
|
|
11181
|
+
"",
|
|
11182
|
+
"## Baseline diff",
|
|
11183
|
+
""
|
|
11184
|
+
];
|
|
11185
|
+
if (diff) {
|
|
11186
|
+
lines.push("```text", renderCheckSection(diff), "```", "");
|
|
11187
|
+
} else {
|
|
11188
|
+
lines.push("No baseline was supplied.", "");
|
|
11189
|
+
}
|
|
11190
|
+
return lines.join("\n");
|
|
11191
|
+
}
|
|
11192
|
+
function renderHtml(trace, check, diff) {
|
|
11193
|
+
const rows = [
|
|
11194
|
+
["Format", trace.format],
|
|
11195
|
+
["Run", trace.runId],
|
|
11196
|
+
["Run status", trace.runStatus],
|
|
11197
|
+
["Run duration ms", trace.runDurationMs],
|
|
11198
|
+
["Runs", trace.runCount],
|
|
11199
|
+
["Events", trace.eventCount],
|
|
11200
|
+
["Reader warnings", trace.readerWarnings],
|
|
11201
|
+
["Unsupported fields", trace.unsupportedFields]
|
|
11202
|
+
];
|
|
11203
|
+
const table = rows.map(
|
|
11204
|
+
([key, value]) => `<tr><th>${escapeHtml2(key)}</th><td>${escapeHtml2(String(value ?? "unknown"))}</td></tr>`
|
|
11205
|
+
).join("");
|
|
11206
|
+
return `<!doctype html>
|
|
11207
|
+
<html lang="en">
|
|
11208
|
+
<head>
|
|
11209
|
+
<meta charset="utf-8"/>
|
|
11210
|
+
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
|
11211
|
+
<title>AgentInspect CI Artifacts</title>
|
|
11212
|
+
<style>body{font-family:system-ui,sans-serif;line-height:1.5;margin:1.5rem;max-width:960px;color:#111}table{border-collapse:collapse}th,td{border:1px solid #ddd;padding:0.35rem 0.5rem;text-align:left}pre{white-space:pre-wrap;background:#f8f8f8;padding:0.75rem;overflow:auto}</style>
|
|
11213
|
+
</head>
|
|
11214
|
+
<body>
|
|
11215
|
+
<h1>AgentInspect CI Artifacts</h1>
|
|
11216
|
+
<p>${escapeHtml2(NOTE)}</p>
|
|
11217
|
+
<h2>Trace</h2>
|
|
11218
|
+
<table><tbody>${table}</tbody></table>
|
|
11219
|
+
<h2>Safety check</h2>
|
|
11220
|
+
<pre>${escapeHtml2(renderCheckSection(check))}</pre>
|
|
11221
|
+
<h2>Baseline diff</h2>
|
|
11222
|
+
<pre>${escapeHtml2(diff ? renderCheckSection(diff) : "No baseline was supplied.")}</pre>
|
|
11223
|
+
</body>
|
|
11224
|
+
</html>
|
|
11225
|
+
`;
|
|
11226
|
+
}
|
|
11227
|
+
async function writeArtifact(outputDir, relativePath, content, files) {
|
|
11228
|
+
const outPath = path10__default.default.join(outputDir, relativePath);
|
|
11229
|
+
await promises.mkdir(path10__default.default.dirname(outPath), { recursive: true });
|
|
11230
|
+
await promises.writeFile(outPath, content, "utf-8");
|
|
11231
|
+
files.push(relativePath);
|
|
11232
|
+
}
|
|
11233
|
+
function readErrorMessage(error) {
|
|
11234
|
+
if (error instanceof TraceReadError) return error.message;
|
|
11235
|
+
return error instanceof Error ? error.message : String(error);
|
|
11236
|
+
}
|
|
11237
|
+
function manifestStatus(check, diff) {
|
|
11238
|
+
if (check.status === "error" || diff?.status === "error") return "unknown";
|
|
11239
|
+
if (check.status === "fail") return "unsafe";
|
|
11240
|
+
if (diff?.status === "fail") return "regression";
|
|
11241
|
+
if (check.summary.warnings > 0 || (diff?.summary.warnings ?? 0) > 0) return "warning";
|
|
11242
|
+
return "ok";
|
|
11243
|
+
}
|
|
11244
|
+
async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
11245
|
+
const outputDir = options.outputDir !== void 0 && options.outputDir.trim() !== "" ? path10__default.default.resolve(options.outputDir.trim()) : "";
|
|
11246
|
+
if (outputDir === "") {
|
|
11247
|
+
console.error("--output-dir is required.");
|
|
11248
|
+
process.exitCode = 1;
|
|
11249
|
+
return;
|
|
11250
|
+
}
|
|
11251
|
+
let read;
|
|
11252
|
+
try {
|
|
11253
|
+
const input3 = await inputFromTarget(target, options, stdin);
|
|
11254
|
+
read = await openTrace(input3, {
|
|
11255
|
+
...options.format !== void 0 ? { format: options.format } : {}
|
|
11256
|
+
});
|
|
11257
|
+
} catch (error) {
|
|
11258
|
+
console.error(`[AgentInspect] artifacts failed: ${readErrorMessage(error)}`);
|
|
11259
|
+
process.exitCode = 1;
|
|
11260
|
+
return;
|
|
11261
|
+
}
|
|
11262
|
+
const selectedRun = selectRun3(read, options.run);
|
|
11263
|
+
const check = runTraceChecks(
|
|
11264
|
+
{ read },
|
|
11265
|
+
{
|
|
11266
|
+
rules: SAFETY_RULES,
|
|
11267
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
11268
|
+
}
|
|
11269
|
+
);
|
|
11270
|
+
const trace = summarizeTrace(read, selectedRun);
|
|
11271
|
+
let diff;
|
|
11272
|
+
if (options.baseline !== void 0 && options.baseline.trim() !== "") {
|
|
11273
|
+
try {
|
|
11274
|
+
const baselineInput = await inputFromTarget(options.baseline, options, stdin);
|
|
11275
|
+
const baselineRead = await openTrace(baselineInput, {
|
|
11276
|
+
...options.format !== void 0 ? { format: options.format } : {}
|
|
11277
|
+
});
|
|
11278
|
+
diff = runTraceChecks(
|
|
11279
|
+
{ read },
|
|
11280
|
+
{
|
|
11281
|
+
rules: [
|
|
11282
|
+
createBaselineRegressionRule({
|
|
11283
|
+
baseline: { read: baselineRead },
|
|
11284
|
+
...options.baselineRun !== void 0 ? { baselineRunId: options.baselineRun } : {},
|
|
11285
|
+
compareFormat: true
|
|
11286
|
+
})
|
|
11287
|
+
],
|
|
11288
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
11289
|
+
}
|
|
11290
|
+
);
|
|
11291
|
+
} catch (error) {
|
|
11292
|
+
console.error(`[AgentInspect] baseline diff failed: ${readErrorMessage(error)}`);
|
|
11293
|
+
process.exitCode = 1;
|
|
11294
|
+
return;
|
|
11295
|
+
}
|
|
11296
|
+
}
|
|
11297
|
+
const files = [];
|
|
11298
|
+
await promises.mkdir(outputDir, { recursive: true });
|
|
11299
|
+
await writeArtifact(outputDir, "trace.json", writeJson3(trace), files);
|
|
11300
|
+
await writeArtifact(outputDir, "check.json", writeJson3(check), files);
|
|
11301
|
+
await writeArtifact(
|
|
11302
|
+
outputDir,
|
|
11303
|
+
"diff.json",
|
|
11304
|
+
writeJson3(diff ?? { status: "not_requested", findings: [], diagnostics: [] }),
|
|
11305
|
+
files
|
|
11306
|
+
);
|
|
11307
|
+
await writeArtifact(outputDir, "summary.md", renderMarkdown(trace, check, diff), files);
|
|
11308
|
+
await writeArtifact(outputDir, "report.html", renderHtml(trace, check, diff), files);
|
|
11309
|
+
const summaryTarget = options.githubSummary ?? process.env.GITHUB_STEP_SUMMARY;
|
|
11310
|
+
if (summaryTarget !== void 0 && summaryTarget.trim() !== "") {
|
|
11311
|
+
await promises.mkdir(path10__default.default.dirname(path10__default.default.resolve(summaryTarget)), { recursive: true });
|
|
11312
|
+
await promises.appendFile(path10__default.default.resolve(summaryTarget), `
|
|
11313
|
+
${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
11314
|
+
}
|
|
11315
|
+
const manifestFiles = [...files, "manifest.json"].sort((a, b) => a.localeCompare(b));
|
|
11316
|
+
const manifest = {
|
|
11317
|
+
status: manifestStatus(check, diff),
|
|
11318
|
+
outputDir,
|
|
11319
|
+
files: manifestFiles,
|
|
11320
|
+
trace,
|
|
11321
|
+
check: {
|
|
11322
|
+
status: check.status,
|
|
11323
|
+
findings: check.findings.length,
|
|
11324
|
+
diagnostics: check.diagnostics.length
|
|
11325
|
+
},
|
|
11326
|
+
diff: {
|
|
11327
|
+
status: diff?.status ?? "not_requested",
|
|
11328
|
+
findings: diff?.findings.length ?? 0,
|
|
11329
|
+
diagnostics: diff?.diagnostics.length ?? 0
|
|
11330
|
+
},
|
|
11331
|
+
...summaryTarget !== void 0 && summaryTarget.trim() !== "" ? { githubSummary: path10__default.default.resolve(summaryTarget) } : {},
|
|
11332
|
+
note: NOTE
|
|
11333
|
+
};
|
|
11334
|
+
await promises.writeFile(path10__default.default.join(outputDir, "manifest.json"), writeJson3(manifest), "utf-8");
|
|
11335
|
+
if (options.json === true) {
|
|
11336
|
+
console.log(writeJson3(manifest).trimEnd());
|
|
11337
|
+
} else {
|
|
11338
|
+
console.log(`Wrote AgentInspect artifacts to ${outputDir}`);
|
|
11339
|
+
console.log(`Status: ${manifest.status}`);
|
|
11340
|
+
for (const file of manifest.files) {
|
|
11341
|
+
console.log(`- ${file}`);
|
|
11342
|
+
}
|
|
11343
|
+
}
|
|
11344
|
+
}
|
|
11345
|
+
|
|
11346
|
+
// packages/cli/src/index.ts
|
|
11347
|
+
function runCommand(action) {
|
|
11348
|
+
void action().catch((error) => {
|
|
11349
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
11350
|
+
console.error(`[AgentInspect] ${msg}`);
|
|
11351
|
+
process.exitCode = 1;
|
|
11352
|
+
});
|
|
11353
|
+
}
|
|
11354
|
+
function createCliProgram() {
|
|
11355
|
+
const program = new commander.Command("agent-inspect").description("Local-first execution-tree debugger for AI agents").version(version);
|
|
11356
|
+
program.command("list").description("List recent AgentInspect runs").option("--dir <path>", "trace directory").option("--limit <number>", "max runs to show (default 20, max 100)").addOption(
|
|
11357
|
+
new commander.Option("--status <status>", "filter by run status").choices([
|
|
11358
|
+
"running",
|
|
11359
|
+
"success",
|
|
11360
|
+
"error",
|
|
11361
|
+
"unknown"
|
|
11362
|
+
])
|
|
11363
|
+
).option("--name <query>", "filter by run name or id (substring match)").option(
|
|
11364
|
+
"--since <duration>",
|
|
11365
|
+
"only include runs since a duration (e.g. 30s, 5m, 2h, 7d)"
|
|
11366
|
+
).option("--json", "print runs as JSON").action(
|
|
11367
|
+
(opts) => {
|
|
11368
|
+
runCommand(() => list(opts));
|
|
11369
|
+
}
|
|
11370
|
+
);
|
|
11371
|
+
program.command("view").description("View a single run trace").argument("<run-id>", "run id (e.g. from list output)").option("--dir <path>", "trace directory").option("--summary", "print a run summary (counts, duration, max depth)").option("--metadata", "print trace metadata (file path/size, timestamps)").option("--errors-only", "show only error events / failed steps").option("--verbose", "show extra detail (types, metadata, error stacks)").option("--json", "print raw trace events as JSON").option(
|
|
11372
|
+
"--tui",
|
|
11373
|
+
"open optional interactive TUI viewer (requires @agent-inspect/tui)"
|
|
11374
|
+
).action(
|
|
11375
|
+
(runId, opts) => {
|
|
11376
|
+
runCommand(() => view(runId, opts));
|
|
11377
|
+
}
|
|
11378
|
+
);
|
|
11379
|
+
program.command("clean").description("Safely delete old AgentInspect run traces").option("--dir <path>", "trace directory").option(
|
|
11380
|
+
"--older-than <duration>",
|
|
11381
|
+
"delete runs older than a duration (e.g. 30s, 5m, 2h, 7d)"
|
|
11382
|
+
).option("--keep <count>", "keep N most recent runs (delete the rest)").option("--dry-run", "print what would be deleted (no changes)").option("--yes", "skip confirmation prompt").action(
|
|
11383
|
+
(opts) => {
|
|
11384
|
+
runCommand(() => clean(opts));
|
|
11385
|
+
}
|
|
11386
|
+
);
|
|
11387
|
+
program.command("logs").description("Parse structured logs into execution trees").argument("<file>", "path to log file").addOption(
|
|
11388
|
+
new commander.Option("--format <format>", "log format").choices([
|
|
11389
|
+
"auto",
|
|
11390
|
+
"json",
|
|
11391
|
+
"log4js"
|
|
11392
|
+
])
|
|
11393
|
+
).option("--config <path>", "path to log ingest config (JSON)").option(
|
|
11394
|
+
"--run-id-key <keys>",
|
|
11395
|
+
"override run id keys (comma-separated, e.g. decisionId,requestId,jobId)"
|
|
11396
|
+
).option("--event-key <key>", "override event key").option("--timestamp-key <key>", "override timestamp key").option("--message-key <key>", "override message key").option("--level-key <key>", "override level key").option("--parent-id-key <key>", "override parent id key").option("--duration-key <key>", "override duration key").option("--status-key <key>", "override status key").option("--json", "print result as JSON").option("--summary", "include summary section in human output").addOption(
|
|
11397
|
+
new commander.Option("--warnings <mode>", "warning output mode").choices([
|
|
11398
|
+
"summary",
|
|
11399
|
+
"all",
|
|
11400
|
+
"none"
|
|
11401
|
+
])
|
|
11402
|
+
).option("--verbose", "show more detail (reserved for future)").option("--no-color", "disable color output").action((file, opts) => {
|
|
11403
|
+
runCommand(() => logs(file, opts));
|
|
11404
|
+
});
|
|
11405
|
+
program.command("tail").description("Live tail structured logs into execution trees").option("--file <path>", "tail a log file (default: read from stdin)").addOption(
|
|
11406
|
+
new commander.Option("--format <format>", "log format").choices([
|
|
11407
|
+
"auto",
|
|
11408
|
+
"json",
|
|
11409
|
+
"log4js"
|
|
11410
|
+
])
|
|
11411
|
+
).option("--config <path>", "path to log ingest config (JSON)").option(
|
|
11412
|
+
"--run-id-key <keys>",
|
|
11413
|
+
"override run id keys (comma-separated, e.g. decisionId,requestId,jobId)"
|
|
11414
|
+
).option("--event-key <key>", "override event key").option("--timestamp-key <key>", "override timestamp key").option("--message-key <key>", "override message key").option("--level-key <key>", "override level key").option("--parent-id-key <key>", "override parent id key").option("--duration-key <key>", "override duration key").option("--status-key <key>", "override status key").addOption(
|
|
11415
|
+
new commander.Option("--warnings <mode>", "warning output mode").choices([
|
|
11416
|
+
"summary",
|
|
11417
|
+
"all",
|
|
11418
|
+
"none"
|
|
11419
|
+
])
|
|
11420
|
+
).option("--refresh <ms>", "minimum time between renders (ms)").option("--once", "read once and exit (for --file)").option("--json", "print newline-delimited JSON updates").option("--no-clear", "do not clear screen between renders").option("--verbose", "show more detail (reserved for future)").option("--no-color", "disable color output").action((opts) => {
|
|
11421
|
+
runCommand(() => tail(opts));
|
|
11422
|
+
});
|
|
11423
|
+
program.command("export").description("Export a manual trace run (Markdown, HTML, OpenInference-compatible JSON, OTLP JSON)").argument("<run-id>", "run id (e.g. from list output)").option("--dir <path>", "trace directory").addOption(
|
|
11424
|
+
new commander.Option("--format <format>", "export format (default: markdown)").choices([
|
|
11425
|
+
"markdown",
|
|
11426
|
+
"html",
|
|
11427
|
+
"openinference",
|
|
11428
|
+
"otlp-json"
|
|
11429
|
+
])
|
|
11430
|
+
).option("-o, --output <path>", "write export to file (creates parent dirs)").option("--json", "emit JSON wrapper about the export (includes content when writing to stdout)").option("--validate", "validate exported payload shape after generation").option("--include-attributes", "include bounded attributes (review before sharing)").option("--no-metadata", "omit summary / metadata sections").option("--no-errors", "omit error sections").addOption(
|
|
11431
|
+
new commander.Option(
|
|
11432
|
+
"--redaction-profile <profile>",
|
|
11433
|
+
"redaction profile for exported copies: local, share, strict (default: local)"
|
|
11434
|
+
).choices(["local", "share", "strict"])
|
|
11435
|
+
).action((runId, opts) => {
|
|
11436
|
+
runCommand(() => exportCommand(runId, opts));
|
|
11437
|
+
});
|
|
11438
|
+
program.command("open").description("Open any supported local trace through the reader pipeline").argument("[input]", "trace file, directory, or - for stdin").addOption(
|
|
11439
|
+
new commander.Option("--format <format>", "trace input format").choices([
|
|
11440
|
+
"agent-inspect-jsonl",
|
|
11441
|
+
"openinference-json",
|
|
11442
|
+
"otlp-json"
|
|
11443
|
+
])
|
|
11444
|
+
).option("--json", "print result as JSON").option("--diagnostics", "print reader warnings and unsupported fields").option("--run <run-id>", "select a run when the trace contains multiple runs").action((input3, opts) => {
|
|
11445
|
+
runCommand(() => openCommand(input3, opts));
|
|
11446
|
+
});
|
|
11447
|
+
program.command("check").description("Run deterministic checks against a local trace").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
11448
|
+
new commander.Option("--format <format>", "trace input format").choices([
|
|
11449
|
+
"agent-inspect-jsonl",
|
|
11450
|
+
"openinference-json",
|
|
11451
|
+
"otlp-json"
|
|
11452
|
+
])
|
|
11453
|
+
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--config <path>", "path to check config (.json, .js, .mjs, .cjs)").option("--json", "print deterministic JSON check result").option("--rule <id>", "select a rule id (repeatable)", (value, previous = []) => [
|
|
11454
|
+
...previous,
|
|
11455
|
+
value
|
|
11456
|
+
]).option("--max-duration-ms <number>", "add run.duration with a max duration").option("--required-tool <name>", "require a tool name (repeatable)", (value, previous = []) => [
|
|
11457
|
+
...previous,
|
|
11458
|
+
value
|
|
11459
|
+
]).option("--forbidden-tool <name>", "forbid a tool name (repeatable)", (value, previous = []) => [
|
|
11460
|
+
...previous,
|
|
11461
|
+
value
|
|
11462
|
+
]).option("--allowed-model <model>", "allow an LLM model (repeatable)", (value, previous = []) => [
|
|
11463
|
+
...previous,
|
|
11464
|
+
value
|
|
11465
|
+
]).option("--max-total-tokens <number>", "add llm.usage with a max total-token budget").action((target, opts) => {
|
|
11466
|
+
runCommand(() => checkCommand(target, opts));
|
|
11467
|
+
});
|
|
11468
|
+
program.command("scan").description("Best-effort local safety scan for trace capture risks").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
11469
|
+
new commander.Option("--format <format>", "trace input format").choices([
|
|
11470
|
+
"agent-inspect-jsonl",
|
|
11471
|
+
"openinference-json",
|
|
11472
|
+
"otlp-json"
|
|
11473
|
+
])
|
|
11474
|
+
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").action((target, opts) => {
|
|
11475
|
+
runCommand(() => scanCommand(target, opts));
|
|
11476
|
+
});
|
|
11477
|
+
program.command("verify-safe").description("Best-effort local trace safety verification").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
11478
|
+
new commander.Option("--format <format>", "trace input format").choices([
|
|
11479
|
+
"agent-inspect-jsonl",
|
|
11480
|
+
"openinference-json",
|
|
11481
|
+
"otlp-json"
|
|
11482
|
+
])
|
|
11483
|
+
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").action((target, opts) => {
|
|
11484
|
+
runCommand(() => verifySafeCommand(target, opts));
|
|
11485
|
+
});
|
|
11486
|
+
program.command("artifacts").description("Create safe local CI trace artifacts").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").requiredOption("--output-dir <path>", "directory for generated artifacts").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
11487
|
+
new commander.Option("--format <format>", "trace input format").choices([
|
|
11488
|
+
"agent-inspect-jsonl",
|
|
11489
|
+
"openinference-json",
|
|
11490
|
+
"otlp-json"
|
|
11491
|
+
])
|
|
11492
|
+
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--baseline <trace-path-or-run-id>", "optional baseline trace for diff artifacts").option("--baseline-run <run-id>", "select a run from the baseline trace").option("--github-summary <path>", "append a safe summary to this file, e.g. GITHUB_STEP_SUMMARY").option("--json", "print deterministic JSON manifest").action((target, opts) => {
|
|
11493
|
+
runCommand(() => artifactsCommand(target, opts));
|
|
11494
|
+
});
|
|
11495
|
+
program.command("diff").description("Compare two local AgentInspect JSONL traces (read-only)").argument("<left-run-id>", "first run id").argument("<right-run-id>", "second run id").option("--dir <path>", "trace directory").option("--json", "print diff result as JSON").option("--ignore-duration", "omit duration comparisons").option(
|
|
9199
11496
|
"--duration-threshold <duration>",
|
|
9200
11497
|
"ignore duration deltas at or below this (e.g. 500ms, 2s, 1m)"
|
|
9201
11498
|
).addOption(
|
|
@@ -9255,6 +11552,23 @@ function createCliProgram() {
|
|
|
9255
11552
|
).action((runId, opts) => {
|
|
9256
11553
|
runCommand(() => reportCommand(runId, opts));
|
|
9257
11554
|
});
|
|
11555
|
+
program.command("explain").description("Explain a local trace with deterministic facts (no provider calls)").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
11556
|
+
new commander.Option("--format <format>", "trace input format").choices([
|
|
11557
|
+
"agent-inspect-jsonl",
|
|
11558
|
+
"openinference-json",
|
|
11559
|
+
"otlp-json"
|
|
11560
|
+
])
|
|
11561
|
+
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--dry-run", "emit only the local facts payload that a provider could receive").option(
|
|
11562
|
+
"--provider <provider>",
|
|
11563
|
+
"reserved for explicit provider explain; currently rejected without network calls"
|
|
11564
|
+
).option("--json", "print deterministic JSON explanation result").addOption(
|
|
11565
|
+
new commander.Option(
|
|
11566
|
+
"--redaction-profile <profile>",
|
|
11567
|
+
"redaction profile for explanation payload: local, share, strict (default: local)"
|
|
11568
|
+
).choices(["local", "share", "strict"])
|
|
11569
|
+
).action((target, opts) => {
|
|
11570
|
+
runCommand(() => explainCommand(target, opts));
|
|
11571
|
+
});
|
|
9258
11572
|
return program;
|
|
9259
11573
|
}
|
|
9260
11574
|
function isPrimaryModule() {
|
|
@@ -9262,9 +11576,9 @@ function isPrimaryModule() {
|
|
|
9262
11576
|
if (!entry) return false;
|
|
9263
11577
|
const selfPath = url.fileURLToPath((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('index.cjs', document.baseURI).href)));
|
|
9264
11578
|
try {
|
|
9265
|
-
return fs.realpathSync(
|
|
11579
|
+
return fs.realpathSync(path10__default.default.resolve(entry)) === fs.realpathSync(path10__default.default.resolve(selfPath));
|
|
9266
11580
|
} catch {
|
|
9267
|
-
return
|
|
11581
|
+
return path10__default.default.resolve(entry) === path10__default.default.resolve(selfPath);
|
|
9268
11582
|
}
|
|
9269
11583
|
}
|
|
9270
11584
|
if (isPrimaryModule()) {
|