agent-inspect 6.17.7 → 6.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +1 -1
- package/SECURITY.md +5 -0
- package/docs/ADAPTERS.md +34 -4
- package/docs/AI-SDK-ADOPTION.md +25 -0
- package/docs/API.md +11 -9
- package/docs/CLI.md +6 -0
- package/docs/COMPARE.md +6 -0
- package/docs/FIRST-TRACE-IN-5-MINUTES.md +3 -1
- package/docs/LIMITATIONS.md +1 -0
- package/docs/OPENAI-AGENTS-LOCAL.md +1 -0
- package/docs/SAFE-TRACE-SHARING.md +2 -2
- package/docs/STANDARDS.md +1 -0
- package/docs/SUPPORT-LEVELS.md +1 -0
- package/docs/USE-CASES.md +1 -1
- package/docs/VSCODE.md +8 -0
- package/package.json +2 -2
- package/packages/cli/dist/{chunk-25LUDS4T.mjs → chunk-S4TLTLXN.mjs} +2 -2
- package/packages/cli/dist/{chunk-25LUDS4T.mjs.map → chunk-S4TLTLXN.mjs.map} +1 -1
- package/packages/cli/dist/index.cjs +1327 -930
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +1245 -857
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-2FS6BLR7.mjs → src-R6V5EZZH.mjs} +3 -3
- package/packages/cli/dist/{src-2FS6BLR7.mjs.map → src-R6V5EZZH.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +180 -0
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +81 -2
- package/packages/core/dist/advanced.d.ts +81 -2
- package/packages/core/dist/advanced.mjs +176 -1
- package/packages/core/dist/advanced.mjs.map +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createSafetyOversizedAttributeRule, resolveTraceDir, parseDuration, buildSessionIndex, resolveBundleRunIds, getTraceFilePath, openTrace, sha256Hex, collectTraceSchemaVersions, aggregateBundleSafeStatus, bundleFailsOnSafety, bundleRunAssetRelativePath, buildPlaceholderArtifact, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceHtmlShell, buildEvidenceProvenanceViewHtml, EVIDENCE_FORMAT_VERSION, inferEvidenceFileRole, buildEvidenceSafetyViewHtml, buildEvidenceDiffViewHtml, buildEvidenceCircuitViewHtml, buildEvidenceOutcomesViewHtml, buildEvidenceContractsViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceCausalFailureViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceTreeViewHtml, buildEvidenceManifest, serializeEvidenceManifest, buildZipArchive, assertBundlePathContained, enrichSessionRunRecord, normalizeBundleOutputPath, sanitizeBundleRunId, defaultBundleOutputPath, runTraceChecks, assertEvidenceRelativePath, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, TraceReadError, resolveRedactionProfile, applyProfileMetadataCaps, Redactor, flattenTree, escapeHtml, safeString, compactAttributes, stableJson, escapeMarkdown, truncateStringForProfile, extractMetadata, filterTraces, truncateName, buildRunSummary, formatDuration, formatTimestamp, isAgentInspectTrace, filterMetasBySessionScope, aggregateSessionCheckResults, createBaselineRegressionRule, summarizeSemanticParity, buildEvidenceCiPackage, verifyEvidenceDirectory, diffTraceEvents, renderRunDiff, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, formatStepLabel, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, resolveSuiteTemplate, defaultSuiteConfigTemplate, loadSuiteConfig, validateSuiteConfig, runSuite, renderSuiteReport, analyzeCohort, renderCohortReport, gateHasThresholds, runGate, renderGateReport, persistedInspectEventsToTraceEvents, renderStepLine,
|
|
2
|
+
import { createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createSafetyOversizedAttributeRule, resolveTraceDir, parseDuration, buildSessionIndex, resolveBundleRunIds, getTraceFilePath, openTrace, sha256Hex, collectTraceSchemaVersions, aggregateBundleSafeStatus, bundleFailsOnSafety, bundleRunAssetRelativePath, buildPlaceholderArtifact, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceHtmlShell, buildEvidenceProvenanceViewHtml, EVIDENCE_FORMAT_VERSION, inferEvidenceFileRole, buildEvidenceSafetyViewHtml, buildEvidenceDiffViewHtml, buildEvidenceCircuitViewHtml, buildEvidenceOutcomesViewHtml, buildEvidenceContractsViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceCausalFailureViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceTreeViewHtml, buildEvidenceManifest, serializeEvidenceManifest, buildZipArchive, assertBundlePathContained, enrichSessionRunRecord, normalizeBundleOutputPath, sanitizeBundleRunId, defaultBundleOutputPath, runTraceChecks, assertEvidenceRelativePath, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, TraceReadError, resolveRedactionProfile, applyProfileMetadataCaps, Redactor, flattenTree, escapeHtml, safeString, compactAttributes, stableJson, escapeMarkdown, truncateStringForProfile, extractMetadata, filterTraces, truncateName, buildRunSummary, formatDuration, formatTimestamp, renderErrorLine, isAgentInspectTrace, filterMetasBySessionScope, aggregateSessionCheckResults, createBaselineRegressionRule, summarizeSemanticParity, buildEvidenceCiPackage, verifyEvidenceDirectory, diffTraceEvents, renderRunDiff, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, formatStepLabel, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, resolveSuiteTemplate, defaultSuiteConfigTemplate, loadSuiteConfig, validateSuiteConfig, runSuite, renderSuiteReport, analyzeCohort, renderCohortReport, gateHasThresholds, runGate, renderGateReport, persistedInspectEventsToTraceEvents, renderStepLine, getIndent, TreeBuilder, zeroKinds, validateEvent, traceEventToPersistedInspectEvent, isPersistedInspectEvent, createRunStatusRule, createStructureOrphanRule, createStructureCycleRule, createRunDurationRule, createMaxStepDurationRule, createRequireCompletedRule, createStallDetectionRule, createObservedOutcomeRule, createRunDepthRule, createToolUsageRule, createLlmUsageRule, createStructureRelationshipRule, createStructureParallelWidthRule, summarizeObservedOutcomes, extractOutcomesFromTraceEvents, renderObservedOutcomesMarkdown, renderObservedOutcomesHtml, DEFAULT_SUITE_ARTIFACTS_DIR, parseCohortMetricList, COHORT_METRIC_IDS, parseGateList, nanoid, diffRuns, manualTraceEventsToComparableRun } from './chunk-S4TLTLXN.mjs';
|
|
3
3
|
import { realpathSync, constants as constants$1, existsSync, readFileSync } from 'fs';
|
|
4
|
-
import
|
|
4
|
+
import path19 from 'path';
|
|
5
5
|
import { fileURLToPath, pathToFileURL } from 'url';
|
|
6
6
|
import { Command, Option } from 'commander';
|
|
7
7
|
import { readFile, stat, mkdir, writeFile, unlink, appendFile, access, rm, open, readdir, constants } from 'fs/promises';
|
|
@@ -13,7 +13,7 @@ import { createServer } from 'http';
|
|
|
13
13
|
import { createRequire } from 'module';
|
|
14
14
|
|
|
15
15
|
// package.json
|
|
16
|
-
var version = "6.
|
|
16
|
+
var version = "6.18.0";
|
|
17
17
|
|
|
18
18
|
// packages/cli/src/trace-dir-scale.ts
|
|
19
19
|
var TRACE_COUNT_WARN = 1e3;
|
|
@@ -162,13 +162,14 @@ async function list(options = {}) {
|
|
|
162
162
|
process.exitCode = 1;
|
|
163
163
|
}
|
|
164
164
|
}
|
|
165
|
+
var DECIMAL_INTEGER_PATTERN = /^\d+$/;
|
|
165
166
|
function parseKeep(raw) {
|
|
166
167
|
const trimmed = typeof raw === "string" ? raw.trim() : "";
|
|
167
|
-
if (trimmed
|
|
168
|
+
if (!DECIMAL_INTEGER_PATTERN.test(trimmed)) {
|
|
168
169
|
throw new Error(`Invalid --keep value: ${raw}. Provide a positive integer.`);
|
|
169
170
|
}
|
|
170
|
-
const n = Number
|
|
171
|
-
if (!Number.
|
|
171
|
+
const n = Number(trimmed);
|
|
172
|
+
if (!Number.isSafeInteger(n) || n <= 0) {
|
|
172
173
|
throw new Error(`Invalid --keep value: ${raw}. Provide a positive integer.`);
|
|
173
174
|
}
|
|
174
175
|
return n;
|
|
@@ -480,6 +481,16 @@ function filterErrorEvents(events) {
|
|
|
480
481
|
return false;
|
|
481
482
|
});
|
|
482
483
|
}
|
|
484
|
+
function pruneErrorTree(nodes) {
|
|
485
|
+
const pruned = [];
|
|
486
|
+
for (const node of nodes) {
|
|
487
|
+
const children = pruneErrorTree(node.children);
|
|
488
|
+
if (node.status === "error" || children.length > 0) {
|
|
489
|
+
pruned.push({ ...node, children });
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
return pruned;
|
|
493
|
+
}
|
|
483
494
|
async function view(runId, options = {}) {
|
|
484
495
|
try {
|
|
485
496
|
const id = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "";
|
|
@@ -552,11 +563,44 @@ async function view(runId, options = {}) {
|
|
|
552
563
|
const errEvents = filterErrorEvents(events);
|
|
553
564
|
if (options.json) {
|
|
554
565
|
console.log(JSON.stringify(errEvents, null, 2));
|
|
555
|
-
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
if (errEvents.length === 0) {
|
|
556
569
|
console.log("No errors found in trace");
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
const started2 = events.find(
|
|
573
|
+
(e) => e.event === "run_started"
|
|
574
|
+
);
|
|
575
|
+
const completed2 = events.filter(
|
|
576
|
+
(e) => e.event === "run_completed"
|
|
577
|
+
);
|
|
578
|
+
const last2 = completed2[completed2.length - 1];
|
|
579
|
+
const status2 = last2 ? last2.status : "running";
|
|
580
|
+
const durationLine2 = last2 !== void 0 && Number.isFinite(last2.durationMs) ? formatDuration(last2.durationMs) : "-";
|
|
581
|
+
const startedTs2 = started2 !== void 0 && Number.isFinite(started2.startTime) ? started2.startTime : started2?.timestamp;
|
|
582
|
+
const startedLabel2 = startedTs2 !== void 0 ? formatTimestamp(startedTs2) : "-";
|
|
583
|
+
console.log(`AgentInspect Run: ${started2?.name ?? id}`);
|
|
584
|
+
console.log(`ID: ${id}`);
|
|
585
|
+
console.log(`Status: ${status2}`);
|
|
586
|
+
console.log(`Duration: ${durationLine2}`);
|
|
587
|
+
console.log(`Started: ${startedLabel2}`);
|
|
588
|
+
console.log("");
|
|
589
|
+
console.log("Error Tree:");
|
|
590
|
+
const pruned = pruneErrorTree(buildStepTree(events));
|
|
591
|
+
if (pruned.length === 0) {
|
|
592
|
+
if (last2?.status === "error") {
|
|
593
|
+
console.log(
|
|
594
|
+
renderErrorLine(
|
|
595
|
+
last2.error ?? { message: "Run completed with error status" },
|
|
596
|
+
0
|
|
597
|
+
)
|
|
598
|
+
);
|
|
599
|
+
} else {
|
|
600
|
+
console.log("No error steps recorded");
|
|
601
|
+
}
|
|
557
602
|
} else {
|
|
558
|
-
|
|
559
|
-
console.log(JSON.stringify(errEvents, null, 2));
|
|
603
|
+
printStepTree(pruned, 0, options.verbose === true);
|
|
560
604
|
}
|
|
561
605
|
return;
|
|
562
606
|
}
|
|
@@ -2978,9 +3022,9 @@ Trace directory: ${traceDir}`);
|
|
|
2978
3022
|
process.exitCode = 1;
|
|
2979
3023
|
}
|
|
2980
3024
|
const resolvedOutput = resolveOutputOption(options);
|
|
2981
|
-
const outPath = resolvedOutput !== void 0 ?
|
|
3025
|
+
const outPath = resolvedOutput !== void 0 ? path19.resolve(resolvedOutput) : void 0;
|
|
2982
3026
|
if (outPath !== void 0) {
|
|
2983
|
-
await mkdir(
|
|
3027
|
+
await mkdir(path19.dirname(outPath), { recursive: true });
|
|
2984
3028
|
await writeFile(outPath, result.content, "utf-8");
|
|
2985
3029
|
const vlabel = validation !== void 0 ? validation.ok ? "ok" : "failed" : "skipped";
|
|
2986
3030
|
console.log(`Wrote ${result.fileExtension} export to ${outPath} (validation: ${vlabel})`);
|
|
@@ -3324,7 +3368,7 @@ function indexedToMetadata(row, traceDir) {
|
|
|
3324
3368
|
endedAt: row.endedAt ?? void 0,
|
|
3325
3369
|
durationMs: row.durationMs ?? void 0,
|
|
3326
3370
|
eventCount: 0,
|
|
3327
|
-
filePath:
|
|
3371
|
+
filePath: path19.join(traceDir, row.file),
|
|
3328
3372
|
fileSize: 0,
|
|
3329
3373
|
createdAt: new Date(row.mtimeMs)
|
|
3330
3374
|
};
|
|
@@ -3353,7 +3397,7 @@ async function newestTraceMtimeMs(traceDir) {
|
|
|
3353
3397
|
}
|
|
3354
3398
|
async function loadSessionRuns(traceDir) {
|
|
3355
3399
|
try {
|
|
3356
|
-
const mod = await import('./src-
|
|
3400
|
+
const mod = await import('./src-R6V5EZZH.mjs');
|
|
3357
3401
|
const dbPath = mod.resolveIndexDbPath(traceDir);
|
|
3358
3402
|
const status = mod.indexStatus(dbPath);
|
|
3359
3403
|
if (!status.healthy) {
|
|
@@ -3802,9 +3846,9 @@ async function reportCommand(runId, options = {}) {
|
|
|
3802
3846
|
...options.section ? { section: options.section } : {}
|
|
3803
3847
|
});
|
|
3804
3848
|
const resolvedOutput = resolveOutputOption(options);
|
|
3805
|
-
const outPath = resolvedOutput !== void 0 ?
|
|
3849
|
+
const outPath = resolvedOutput !== void 0 ? path19.resolve(resolvedOutput) : void 0;
|
|
3806
3850
|
if (outPath !== void 0) {
|
|
3807
|
-
await mkdir(
|
|
3851
|
+
await mkdir(path19.dirname(outPath), { recursive: true });
|
|
3808
3852
|
await writeFile(outPath, result.content, "utf-8");
|
|
3809
3853
|
console.log(`Wrote ${result.fileExtension} report to ${outPath}`);
|
|
3810
3854
|
}
|
|
@@ -3999,8 +4043,8 @@ function passesLuhn(value) {
|
|
|
3999
4043
|
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
4000
4044
|
var EPOCH_MS_RE = /^1[0-9]{12}$/;
|
|
4001
4045
|
var EMAIL_RE = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi;
|
|
4002
|
-
function pathSuggestsNonCard(
|
|
4003
|
-
const normalized =
|
|
4046
|
+
function pathSuggestsNonCard(path30) {
|
|
4047
|
+
const normalized = path30.toLowerCase().replace(/[^a-z0-9._]/g, "");
|
|
4004
4048
|
return /(^|\.)(tokenusage|usage)(\.|$)/.test(normalized) || /(startedat|endedat|durationms|timestamp|createdat|updatedat)(\.|$)/.test(normalized) || /(^|\.)(runid|traceid|spanid|eventid|sessionid|userid|parentid|requestid|correlationid)(\.|$)/.test(
|
|
4005
4049
|
normalized
|
|
4006
4050
|
) || /(^|\.)ts(\.|$)/.test(normalized);
|
|
@@ -4196,17 +4240,17 @@ function applyRule(rule, value, replacement) {
|
|
|
4196
4240
|
}
|
|
4197
4241
|
return value;
|
|
4198
4242
|
}
|
|
4199
|
-
function childPath(
|
|
4243
|
+
function childPath(path30, key) {
|
|
4200
4244
|
if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)) {
|
|
4201
|
-
return
|
|
4245
|
+
return path30 ? `${path30}.${key}` : key;
|
|
4202
4246
|
}
|
|
4203
|
-
return `${
|
|
4247
|
+
return `${path30 || "$"}[${JSON.stringify(key)}]`;
|
|
4204
4248
|
}
|
|
4205
|
-
function indexPath(
|
|
4206
|
-
return `${
|
|
4249
|
+
function indexPath(path30, index) {
|
|
4250
|
+
return `${path30 || "$"}[${index}]`;
|
|
4207
4251
|
}
|
|
4208
|
-
function makeFinding(
|
|
4209
|
-
return preview === void 0 ? { path:
|
|
4252
|
+
function makeFinding(path30, detector, action, matchKind, severity = "warning", preview) {
|
|
4253
|
+
return preview === void 0 ? { path: path30, detector, action, severity, matchKind } : { path: path30, detector, action, severity, matchKind, preview };
|
|
4210
4254
|
}
|
|
4211
4255
|
function createRedactionProfile(profile = "local") {
|
|
4212
4256
|
switch (profile) {
|
|
@@ -4275,11 +4319,11 @@ var Redactor2 = class {
|
|
|
4275
4319
|
#recordFinding(state, finding) {
|
|
4276
4320
|
if (this.#collectFindings) state.findings.push(finding);
|
|
4277
4321
|
}
|
|
4278
|
-
#redactValue(value, key,
|
|
4322
|
+
#redactValue(value, key, path30, depth, state) {
|
|
4279
4323
|
if (depth > this.#maxDepth) {
|
|
4280
4324
|
this.#recordFinding(
|
|
4281
4325
|
state,
|
|
4282
|
-
makeFinding(
|
|
4326
|
+
makeFinding(path30, "structure.maxDepth", "truncate", "value", "warning")
|
|
4283
4327
|
);
|
|
4284
4328
|
return "[Truncated]";
|
|
4285
4329
|
}
|
|
@@ -4288,19 +4332,19 @@ var Redactor2 = class {
|
|
|
4288
4332
|
if (rule) {
|
|
4289
4333
|
this.#recordFinding(
|
|
4290
4334
|
state,
|
|
4291
|
-
makeFinding(
|
|
4335
|
+
makeFinding(path30, `key.${rule.key}`, actionForRule(rule), "key", "warning")
|
|
4292
4336
|
);
|
|
4293
4337
|
return applyRule(rule, value, this.#replacement);
|
|
4294
4338
|
}
|
|
4295
4339
|
}
|
|
4296
4340
|
for (const detector of this.#detectors) {
|
|
4297
|
-
const detections = detector.detect({ path:
|
|
4341
|
+
const detections = detector.detect({ path: path30, key, value });
|
|
4298
4342
|
for (const detection of detections) {
|
|
4299
4343
|
const action = detection.action ?? "replace";
|
|
4300
4344
|
this.#recordFinding(
|
|
4301
4345
|
state,
|
|
4302
4346
|
makeFinding(
|
|
4303
|
-
|
|
4347
|
+
path30,
|
|
4304
4348
|
detector.id,
|
|
4305
4349
|
action,
|
|
4306
4350
|
detection.matchKind ?? detector.matchKind ?? "custom",
|
|
@@ -4318,7 +4362,7 @@ var Redactor2 = class {
|
|
|
4318
4362
|
const out = [];
|
|
4319
4363
|
state.seen.set(value, out);
|
|
4320
4364
|
value.forEach((item, index) => {
|
|
4321
|
-
out[index] = this.#redactValue(item, void 0, indexPath(
|
|
4365
|
+
out[index] = this.#redactValue(item, void 0, indexPath(path30, index), depth + 1, state);
|
|
4322
4366
|
});
|
|
4323
4367
|
return out;
|
|
4324
4368
|
}
|
|
@@ -4330,7 +4374,7 @@ var Redactor2 = class {
|
|
|
4330
4374
|
out[entryKey] = this.#redactValue(
|
|
4331
4375
|
entryValue,
|
|
4332
4376
|
entryKey,
|
|
4333
|
-
childPath(
|
|
4377
|
+
childPath(path30 === "$" ? "" : path30, entryKey),
|
|
4334
4378
|
depth + 1,
|
|
4335
4379
|
state
|
|
4336
4380
|
);
|
|
@@ -4346,6 +4390,230 @@ function createRedactor(options) {
|
|
|
4346
4390
|
function redact(value, options) {
|
|
4347
4391
|
return createRedactor(options).redact(value);
|
|
4348
4392
|
}
|
|
4393
|
+
var REDACTION_POLICY_LIMITS = {
|
|
4394
|
+
maxExtraKeys: 64,
|
|
4395
|
+
maxKeyLength: 64,
|
|
4396
|
+
maxPatterns: 32,
|
|
4397
|
+
maxPatternLength: 128,
|
|
4398
|
+
maxPatternIdLength: 64,
|
|
4399
|
+
maxTypedQuantifier: 64
|
|
4400
|
+
};
|
|
4401
|
+
var SAFE_TYPED_PATTERN = /^[A-Za-z0-9_@./:=<>\-[\]()|+*?{},\\\s^$]+$/;
|
|
4402
|
+
function isRecord6(value) {
|
|
4403
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4404
|
+
}
|
|
4405
|
+
function escapeRegExp(value) {
|
|
4406
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4407
|
+
}
|
|
4408
|
+
function rejectRemotePolicyPath(policyPath) {
|
|
4409
|
+
const trimmed = policyPath.trim();
|
|
4410
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(trimmed)) {
|
|
4411
|
+
throw new Error(
|
|
4412
|
+
`--policy must be a local JSON file path (got a URL-like value). Remote policy fetch is not supported.`
|
|
4413
|
+
);
|
|
4414
|
+
}
|
|
4415
|
+
}
|
|
4416
|
+
function parseSeverity(value, label) {
|
|
4417
|
+
if (value === void 0) return "error";
|
|
4418
|
+
if (value === "info" || value === "warning" || value === "error") return value;
|
|
4419
|
+
throw new Error(`${label} severity must be info, warning, or error.`);
|
|
4420
|
+
}
|
|
4421
|
+
function validateExtraKey(key, index) {
|
|
4422
|
+
const label = `extraKeys[${index}]`;
|
|
4423
|
+
if (key.length === 0) throw new Error(`${label} must be a non-empty string.`);
|
|
4424
|
+
if (key.length > REDACTION_POLICY_LIMITS.maxKeyLength) {
|
|
4425
|
+
throw new Error(
|
|
4426
|
+
`${label} exceeds max length ${REDACTION_POLICY_LIMITS.maxKeyLength}.`
|
|
4427
|
+
);
|
|
4428
|
+
}
|
|
4429
|
+
if (!/^[A-Za-z_][A-Za-z0-9_.-]*$/.test(key)) {
|
|
4430
|
+
throw new Error(
|
|
4431
|
+
`${label} must be an identifier-like key (letters, digits, _, ., -).`
|
|
4432
|
+
);
|
|
4433
|
+
}
|
|
4434
|
+
return key;
|
|
4435
|
+
}
|
|
4436
|
+
function assertBoundedTypedPattern(pattern, label) {
|
|
4437
|
+
if (pattern.length === 0) throw new Error(`${label} must be non-empty.`);
|
|
4438
|
+
if (pattern.length > REDACTION_POLICY_LIMITS.maxPatternLength) {
|
|
4439
|
+
throw new Error(
|
|
4440
|
+
`${label} exceeds max length ${REDACTION_POLICY_LIMITS.maxPatternLength}.`
|
|
4441
|
+
);
|
|
4442
|
+
}
|
|
4443
|
+
if (!SAFE_TYPED_PATTERN.test(pattern)) {
|
|
4444
|
+
throw new Error(
|
|
4445
|
+
`${label} contains unsupported characters for bounded typed patterns.`
|
|
4446
|
+
);
|
|
4447
|
+
}
|
|
4448
|
+
if (/\(\?/.test(pattern) || /\\[1-9]/.test(pattern)) {
|
|
4449
|
+
throw new Error(
|
|
4450
|
+
`${label} rejects lookaround, backreferences, and nested quantifiers.`
|
|
4451
|
+
);
|
|
4452
|
+
}
|
|
4453
|
+
if (/\([^)]*[+*{][^)]*\)[+*{]/.test(pattern) || /\[[^\]]*[+*{][^\]]*\][+*{]/.test(pattern)) {
|
|
4454
|
+
throw new Error(`${label} rejects nested quantifiers.`);
|
|
4455
|
+
}
|
|
4456
|
+
if (/\.[*+]/.test(pattern) || /[*+]\{/.test(pattern)) {
|
|
4457
|
+
throw new Error(`${label} rejects unbounded .* / .+ style quantifiers.`);
|
|
4458
|
+
}
|
|
4459
|
+
for (const match of pattern.matchAll(/\{(\d+)(?:,(\d*))?\}/g)) {
|
|
4460
|
+
const min = Number(match[1]);
|
|
4461
|
+
const maxRaw = match[2];
|
|
4462
|
+
const max = maxRaw === void 0 || maxRaw === "" ? min : Number(maxRaw);
|
|
4463
|
+
if (!Number.isFinite(min) || !Number.isFinite(max) || min > REDACTION_POLICY_LIMITS.maxTypedQuantifier || max > REDACTION_POLICY_LIMITS.maxTypedQuantifier) {
|
|
4464
|
+
throw new Error(
|
|
4465
|
+
`${label} quantifiers must be <= ${REDACTION_POLICY_LIMITS.maxTypedQuantifier}.`
|
|
4466
|
+
);
|
|
4467
|
+
}
|
|
4468
|
+
}
|
|
4469
|
+
}
|
|
4470
|
+
function compilePatternDetector(input3, index) {
|
|
4471
|
+
const label = `patterns[${index}]`;
|
|
4472
|
+
if (!isRecord6(input3)) throw new Error(`${label} must be an object.`);
|
|
4473
|
+
const idRaw = input3.id;
|
|
4474
|
+
if (typeof idRaw !== "string" || idRaw.trim() === "") {
|
|
4475
|
+
throw new Error(`${label}.id must be a non-empty string.`);
|
|
4476
|
+
}
|
|
4477
|
+
const id = idRaw.trim();
|
|
4478
|
+
if (id.length > REDACTION_POLICY_LIMITS.maxPatternIdLength) {
|
|
4479
|
+
throw new Error(
|
|
4480
|
+
`${label}.id exceeds max length ${REDACTION_POLICY_LIMITS.maxPatternIdLength}.`
|
|
4481
|
+
);
|
|
4482
|
+
}
|
|
4483
|
+
if (!/^[A-Za-z][A-Za-z0-9._-]*$/.test(id)) {
|
|
4484
|
+
throw new Error(`${label}.id must be an identifier-like detector id.`);
|
|
4485
|
+
}
|
|
4486
|
+
const type = input3.type;
|
|
4487
|
+
if (type !== "literal" && type !== "prefix" && type !== "typed") {
|
|
4488
|
+
throw new Error(`${label}.type must be literal, prefix, or typed.`);
|
|
4489
|
+
}
|
|
4490
|
+
const severity = parseSeverity(input3.severity, label);
|
|
4491
|
+
let source;
|
|
4492
|
+
if (type === "typed") {
|
|
4493
|
+
if (typeof input3.pattern !== "string") {
|
|
4494
|
+
throw new Error(`${label}.pattern must be a string for typed patterns.`);
|
|
4495
|
+
}
|
|
4496
|
+
assertBoundedTypedPattern(input3.pattern, `${label}.pattern`);
|
|
4497
|
+
source = input3.pattern;
|
|
4498
|
+
} else {
|
|
4499
|
+
if (typeof input3.value !== "string") {
|
|
4500
|
+
throw new Error(`${label}.value must be a string for ${type} patterns.`);
|
|
4501
|
+
}
|
|
4502
|
+
if (input3.value.length === 0) {
|
|
4503
|
+
throw new Error(`${label}.value must be non-empty.`);
|
|
4504
|
+
}
|
|
4505
|
+
if (input3.value.length > REDACTION_POLICY_LIMITS.maxPatternLength) {
|
|
4506
|
+
throw new Error(
|
|
4507
|
+
`${label}.value exceeds max length ${REDACTION_POLICY_LIMITS.maxPatternLength}.`
|
|
4508
|
+
);
|
|
4509
|
+
}
|
|
4510
|
+
const escaped = escapeRegExp(input3.value);
|
|
4511
|
+
source = type === "prefix" ? `^${escaped}` : escaped;
|
|
4512
|
+
}
|
|
4513
|
+
let regex;
|
|
4514
|
+
try {
|
|
4515
|
+
regex = new RegExp(source);
|
|
4516
|
+
} catch (error) {
|
|
4517
|
+
throw new Error(
|
|
4518
|
+
`${label} failed to compile: ${error instanceof Error ? error.message : String(error)}`
|
|
4519
|
+
);
|
|
4520
|
+
}
|
|
4521
|
+
return {
|
|
4522
|
+
id: `policy.${id}`,
|
|
4523
|
+
severity,
|
|
4524
|
+
matchKind: "custom",
|
|
4525
|
+
detect({ value }) {
|
|
4526
|
+
if (typeof value !== "string") return [];
|
|
4527
|
+
regex.lastIndex = 0;
|
|
4528
|
+
return regex.test(value) ? [{ action: "replace", severity, matchKind: "custom" }] : [];
|
|
4529
|
+
}
|
|
4530
|
+
};
|
|
4531
|
+
}
|
|
4532
|
+
function compileRedactionPolicy(raw, policyPath) {
|
|
4533
|
+
if (!isRecord6(raw)) {
|
|
4534
|
+
throw new Error(`Redaction policy must be a JSON object (${policyPath}).`);
|
|
4535
|
+
}
|
|
4536
|
+
const diagnostics = [];
|
|
4537
|
+
if (raw.version !== void 0 && raw.version !== 1) {
|
|
4538
|
+
throw new Error(`Unsupported redaction policy version (supported: 1).`);
|
|
4539
|
+
}
|
|
4540
|
+
const extraKeysRaw = raw.extraKeys;
|
|
4541
|
+
const extraKeys = [];
|
|
4542
|
+
if (extraKeysRaw !== void 0) {
|
|
4543
|
+
if (!Array.isArray(extraKeysRaw)) {
|
|
4544
|
+
throw new Error(`extraKeys must be an array of strings.`);
|
|
4545
|
+
}
|
|
4546
|
+
if (extraKeysRaw.length > REDACTION_POLICY_LIMITS.maxExtraKeys) {
|
|
4547
|
+
throw new Error(
|
|
4548
|
+
`extraKeys exceeds max count ${REDACTION_POLICY_LIMITS.maxExtraKeys}.`
|
|
4549
|
+
);
|
|
4550
|
+
}
|
|
4551
|
+
for (let i = 0; i < extraKeysRaw.length; i += 1) {
|
|
4552
|
+
const key = extraKeysRaw[i];
|
|
4553
|
+
if (typeof key !== "string") {
|
|
4554
|
+
throw new Error(`extraKeys[${i}] must be a string.`);
|
|
4555
|
+
}
|
|
4556
|
+
extraKeys.push(validateExtraKey(key, i));
|
|
4557
|
+
}
|
|
4558
|
+
}
|
|
4559
|
+
const patternsRaw = raw.patterns;
|
|
4560
|
+
const detectors = [];
|
|
4561
|
+
if (patternsRaw !== void 0) {
|
|
4562
|
+
if (!Array.isArray(patternsRaw)) {
|
|
4563
|
+
throw new Error(`patterns must be an array.`);
|
|
4564
|
+
}
|
|
4565
|
+
if (patternsRaw.length > REDACTION_POLICY_LIMITS.maxPatterns) {
|
|
4566
|
+
throw new Error(
|
|
4567
|
+
`patterns exceeds max count ${REDACTION_POLICY_LIMITS.maxPatterns}.`
|
|
4568
|
+
);
|
|
4569
|
+
}
|
|
4570
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
4571
|
+
for (let i = 0; i < patternsRaw.length; i += 1) {
|
|
4572
|
+
const detector = compilePatternDetector(
|
|
4573
|
+
patternsRaw[i],
|
|
4574
|
+
i
|
|
4575
|
+
);
|
|
4576
|
+
if (seenIds.has(detector.id)) {
|
|
4577
|
+
throw new Error(`Duplicate pattern id "${detector.id}".`);
|
|
4578
|
+
}
|
|
4579
|
+
seenIds.add(detector.id);
|
|
4580
|
+
detectors.push(detector);
|
|
4581
|
+
}
|
|
4582
|
+
}
|
|
4583
|
+
if (extraKeys.length === 0 && detectors.length === 0) {
|
|
4584
|
+
diagnostics.push({
|
|
4585
|
+
code: "AI_POLICY_EMPTY",
|
|
4586
|
+
message: "Redaction policy contained no extraKeys or patterns."
|
|
4587
|
+
});
|
|
4588
|
+
}
|
|
4589
|
+
return {
|
|
4590
|
+
path: policyPath,
|
|
4591
|
+
extraKeys,
|
|
4592
|
+
detectors,
|
|
4593
|
+
diagnostics
|
|
4594
|
+
};
|
|
4595
|
+
}
|
|
4596
|
+
async function loadRedactionPolicy(policyPath) {
|
|
4597
|
+
rejectRemotePolicyPath(policyPath);
|
|
4598
|
+
const resolved = path19.resolve(policyPath);
|
|
4599
|
+
let text;
|
|
4600
|
+
try {
|
|
4601
|
+
text = await readFile(resolved, "utf-8");
|
|
4602
|
+
} catch (error) {
|
|
4603
|
+
throw new Error(
|
|
4604
|
+
`Failed to read --policy file "${resolved}": ${error instanceof Error ? error.message : String(error)}`
|
|
4605
|
+
);
|
|
4606
|
+
}
|
|
4607
|
+
let parsed;
|
|
4608
|
+
try {
|
|
4609
|
+
parsed = JSON.parse(text);
|
|
4610
|
+
} catch (error) {
|
|
4611
|
+
throw new Error(
|
|
4612
|
+
`Invalid JSON in --policy file "${resolved}": ${error instanceof Error ? error.message : String(error)}`
|
|
4613
|
+
);
|
|
4614
|
+
}
|
|
4615
|
+
return compileRedactionPolicy(parsed, resolved);
|
|
4616
|
+
}
|
|
4349
4617
|
async function readStdin(stdin) {
|
|
4350
4618
|
stdin.setEncoding("utf8");
|
|
4351
4619
|
let content = "";
|
|
@@ -4374,12 +4642,19 @@ async function inputFromTarget(target, options, stdin) {
|
|
|
4374
4642
|
return { type: "file", path: runPath };
|
|
4375
4643
|
}
|
|
4376
4644
|
|
|
4377
|
-
// packages/cli/src/
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
|
|
4645
|
+
// packages/cli/src/safety.ts
|
|
4646
|
+
var BEST_EFFORT_NOTE = "Best-effort local safety verification only; not a compliance, privacy, security, or regulatory certification.";
|
|
4647
|
+
var DEFAULT_MAX_STRING_LENGTH = 16384;
|
|
4648
|
+
var DEFAULT_MAX_ARRAY_LENGTH = 1e3;
|
|
4649
|
+
var DEFAULT_MAX_OBJECT_KEYS = 200;
|
|
4650
|
+
var DEFAULT_MAX_SERIALIZED_BYTES = 128 * 1024;
|
|
4651
|
+
function parseLimit3(value, label) {
|
|
4652
|
+
if (value === void 0) return void 0;
|
|
4653
|
+
const parsed = Number(value);
|
|
4654
|
+
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
4655
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
4381
4656
|
}
|
|
4382
|
-
|
|
4657
|
+
return parsed;
|
|
4383
4658
|
}
|
|
4384
4659
|
function stable(value) {
|
|
4385
4660
|
if (Array.isArray(value)) return value.map(stable);
|
|
@@ -4389,103 +4664,585 @@ function stable(value) {
|
|
|
4389
4664
|
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable(record[key])])
|
|
4390
4665
|
);
|
|
4391
4666
|
}
|
|
4392
|
-
function
|
|
4393
|
-
return
|
|
4667
|
+
function safetyDiagnostic(code, message, severity = "error") {
|
|
4668
|
+
return { code, message, severity };
|
|
4394
4669
|
}
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
}
|
|
4413
|
-
return { content: await readFile(runPath, "utf-8"), source: runPath };
|
|
4670
|
+
function warningDiagnostics(warnings, unsupportedFields) {
|
|
4671
|
+
return [
|
|
4672
|
+
...warnings.map(
|
|
4673
|
+
(warning) => safetyDiagnostic(
|
|
4674
|
+
warning.code,
|
|
4675
|
+
warning.message,
|
|
4676
|
+
warning.severity === "error" ? "error" : "warning"
|
|
4677
|
+
)
|
|
4678
|
+
),
|
|
4679
|
+
...unsupportedFields.map(
|
|
4680
|
+
(field) => safetyDiagnostic(
|
|
4681
|
+
"unsupported_field",
|
|
4682
|
+
`Reader reported unsupported field: ${field}`,
|
|
4683
|
+
"warning"
|
|
4684
|
+
)
|
|
4685
|
+
)
|
|
4686
|
+
];
|
|
4414
4687
|
}
|
|
4415
|
-
function
|
|
4416
|
-
|
|
4417
|
-
|
|
4688
|
+
function diagnosticFromCheck(item) {
|
|
4689
|
+
return safetyDiagnostic(item.code, item.message, item.severity);
|
|
4690
|
+
}
|
|
4691
|
+
function statusFrom(findings, diagnostics) {
|
|
4692
|
+
if (diagnostics.some((item) => item.severity === "error")) return "UNKNOWN";
|
|
4693
|
+
if (findings.some((item) => item.severity === "error")) return "UNSAFE";
|
|
4694
|
+
if (diagnostics.some((item) => item.severity === "warning")) return "SAFE WITH WARNINGS";
|
|
4695
|
+
if (findings.some((item) => item.severity === "warning")) return "SAFE WITH WARNINGS";
|
|
4696
|
+
return "SAFE";
|
|
4697
|
+
}
|
|
4698
|
+
function resultFromParts(parts) {
|
|
4699
|
+
const findings = [...parts.findings ?? []];
|
|
4700
|
+
const diagnostics = [...parts.diagnostics ?? []];
|
|
4701
|
+
const warnings = [...parts.warnings ?? []];
|
|
4702
|
+
const unsupportedFields = [...parts.unsupportedFields ?? []];
|
|
4703
|
+
const status = parts.status ?? statusFrom(findings, diagnostics);
|
|
4418
4704
|
return {
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4705
|
+
ok: status === "SAFE" || status === "SAFE WITH WARNINGS",
|
|
4706
|
+
command: parts.command,
|
|
4707
|
+
status,
|
|
4708
|
+
format: parts.format,
|
|
4709
|
+
...parts.runId !== void 0 ? { runId: parts.runId } : {},
|
|
4710
|
+
summary: {
|
|
4711
|
+
findings: findings.length,
|
|
4712
|
+
warnings: diagnostics.filter((item) => item.severity === "warning").length + findings.filter((item) => item.severity === "warning").length,
|
|
4713
|
+
errors: diagnostics.filter((item) => item.severity === "error").length + findings.filter((item) => item.severity === "error").length
|
|
4714
|
+
},
|
|
4715
|
+
findings,
|
|
4716
|
+
diagnostics,
|
|
4717
|
+
warnings,
|
|
4718
|
+
unsupportedFields,
|
|
4719
|
+
note: BEST_EFFORT_NOTE,
|
|
4720
|
+
...parts.sourceAssessment !== void 0 ? { sourceAssessment: parts.sourceAssessment } : {},
|
|
4721
|
+
...parts.artifactAssessment !== void 0 ? { artifactAssessment: parts.artifactAssessment } : {},
|
|
4722
|
+
...parts.redactionSummary !== void 0 ? { redactionSummary: parts.redactionSummary } : {}
|
|
4422
4723
|
};
|
|
4423
4724
|
}
|
|
4424
|
-
function
|
|
4425
|
-
const lines = content.split(/\r?\n/);
|
|
4426
|
-
const out = [];
|
|
4427
|
-
const findings = [];
|
|
4428
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
4429
|
-
const line = lines[index] ?? "";
|
|
4430
|
-
if (line.trim() === "") continue;
|
|
4431
|
-
let parsed;
|
|
4432
|
-
try {
|
|
4433
|
-
parsed = JSON.parse(line);
|
|
4434
|
-
} catch {
|
|
4435
|
-
throw new Error(`Input is not valid JSON or JSONL at line ${index + 1}.`);
|
|
4436
|
-
}
|
|
4437
|
-
const result = redact(parsed, { profile });
|
|
4438
|
-
out.push(JSON.stringify(result.value));
|
|
4439
|
-
findings.push(
|
|
4440
|
-
...result.findings.map((finding) => ({
|
|
4441
|
-
...finding,
|
|
4442
|
-
path: `line:${index + 1}:${finding.path}`
|
|
4443
|
-
}))
|
|
4444
|
-
);
|
|
4445
|
-
}
|
|
4725
|
+
function layerFromResult(result) {
|
|
4446
4726
|
return {
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
findings
|
|
4727
|
+
status: result.status,
|
|
4728
|
+
summary: result.summary,
|
|
4729
|
+
findings: result.findings
|
|
4450
4730
|
};
|
|
4451
4731
|
}
|
|
4452
|
-
function
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
}
|
|
4461
|
-
}
|
|
4462
|
-
}
|
|
4463
|
-
try {
|
|
4464
|
-
return redactJsonText(content, profile);
|
|
4465
|
-
} catch {
|
|
4466
|
-
return redactJsonlText(content, profile);
|
|
4732
|
+
function readErrorResult(command, error) {
|
|
4733
|
+
if (error instanceof TraceReadError) {
|
|
4734
|
+
const code = error.code === "unsupported_format" ? "AI_SAFETY_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_SAFETY_AMBIGUOUS_FORMAT" : "AI_SAFETY_TRACE_UNREADABLE";
|
|
4735
|
+
return resultFromParts({
|
|
4736
|
+
command,
|
|
4737
|
+
format: "unknown",
|
|
4738
|
+
diagnostics: [safetyDiagnostic(code, error.message)],
|
|
4739
|
+
warnings: error.warnings
|
|
4740
|
+
});
|
|
4467
4741
|
}
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4742
|
+
return resultFromParts({
|
|
4743
|
+
command,
|
|
4744
|
+
format: "unknown",
|
|
4745
|
+
diagnostics: [
|
|
4746
|
+
safetyDiagnostic(
|
|
4747
|
+
"AI_SAFETY_TRACE_UNREADABLE",
|
|
4748
|
+
error instanceof Error ? error.message : String(error)
|
|
4749
|
+
)
|
|
4750
|
+
]
|
|
4751
|
+
});
|
|
4752
|
+
}
|
|
4753
|
+
function invalidArgumentResult(command, error) {
|
|
4754
|
+
return resultFromParts({
|
|
4755
|
+
command,
|
|
4756
|
+
format: "unknown",
|
|
4757
|
+
diagnostics: [
|
|
4758
|
+
safetyDiagnostic(
|
|
4759
|
+
"AI_SAFETY_INVALID_ARGUMENTS",
|
|
4760
|
+
error instanceof Error ? error.message : String(error)
|
|
4761
|
+
)
|
|
4762
|
+
]
|
|
4763
|
+
});
|
|
4764
|
+
}
|
|
4765
|
+
function buildSafetyRules(options) {
|
|
4766
|
+
const maxStringLength2 = parseLimit3(options.maxStringLength, "--max-string-length") ?? DEFAULT_MAX_STRING_LENGTH;
|
|
4767
|
+
const maxArrayLength = parseLimit3(options.maxArrayLength, "--max-array-length") ?? DEFAULT_MAX_ARRAY_LENGTH;
|
|
4768
|
+
const maxObjectKeys = parseLimit3(options.maxObjectKeys, "--max-object-keys") ?? DEFAULT_MAX_OBJECT_KEYS;
|
|
4769
|
+
const maxSerializedBytes = parseLimit3(options.maxSerializedBytes, "--max-serialized-bytes") ?? DEFAULT_MAX_SERIALIZED_BYTES;
|
|
4770
|
+
return [
|
|
4771
|
+
createSafetyRawContentRule(),
|
|
4772
|
+
createSafetyRedactionRule(),
|
|
4773
|
+
createSafetySecretPatternRule(),
|
|
4774
|
+
createSafetyOversizedAttributeRule({
|
|
4775
|
+
maxStringLength: maxStringLength2,
|
|
4776
|
+
maxArrayLength,
|
|
4777
|
+
maxObjectKeys,
|
|
4778
|
+
maxSerializedBytes
|
|
4779
|
+
})
|
|
4780
|
+
];
|
|
4781
|
+
}
|
|
4782
|
+
function flattenNodes(nodes) {
|
|
4783
|
+
return nodes.flatMap((node) => [
|
|
4784
|
+
node,
|
|
4785
|
+
...flattenNodes(
|
|
4786
|
+
node.children
|
|
4787
|
+
)
|
|
4788
|
+
]);
|
|
4789
|
+
}
|
|
4790
|
+
function detectorSeverity(finding) {
|
|
4791
|
+
return finding.severity;
|
|
4792
|
+
}
|
|
4793
|
+
function classifyRedactionDetector(detector) {
|
|
4794
|
+
if (detector === "value.creditCard" || detector === "value.email" || detector === "value.phone") {
|
|
4795
|
+
return { category: "personal-data", confidence: "high" };
|
|
4796
|
+
}
|
|
4797
|
+
if (detector === "value.ipv4" || detector === "value.ipv6") {
|
|
4798
|
+
return { category: "identifier", confidence: "medium" };
|
|
4799
|
+
}
|
|
4800
|
+
if (detector.startsWith("value.") && (detector.includes("Token") || detector.includes("Key") || detector.includes("jwt") || detector.includes("authorization") || detector.includes("bearer") || detector.includes("cookie") || detector.includes("privateKey") || detector.includes("github") || detector.includes("aws") || detector.includes("provider"))) {
|
|
4801
|
+
return { category: "credential", confidence: "high" };
|
|
4802
|
+
}
|
|
4803
|
+
if (detector.startsWith("key.")) {
|
|
4804
|
+
return { category: "credential", confidence: "medium" };
|
|
4805
|
+
}
|
|
4806
|
+
return { category: "credential", confidence: "medium" };
|
|
4807
|
+
}
|
|
4808
|
+
function redactionOptionsFromPolicy(policy) {
|
|
4809
|
+
if (policy === void 0) return {};
|
|
4810
|
+
return {
|
|
4811
|
+
...policy.extraKeys.length > 0 ? { extraKeys: [...policy.extraKeys] } : {},
|
|
4812
|
+
...policy.detectors.length > 0 ? { detectors: [...policy.detectors] } : {}
|
|
4813
|
+
};
|
|
4814
|
+
}
|
|
4815
|
+
function redactionDetectorFindings(read, runId, policy) {
|
|
4816
|
+
const runs = runId === void 0 ? read.runs : read.runs.filter((run) => run.runId === runId);
|
|
4817
|
+
const out = [];
|
|
4818
|
+
const policyOptions = redactionOptionsFromPolicy(policy);
|
|
4819
|
+
for (const run of runs) {
|
|
4820
|
+
for (const node of flattenNodes(run.children)) {
|
|
4821
|
+
const attrs = node.event.attributes;
|
|
4822
|
+
if (attrs === void 0) continue;
|
|
4823
|
+
const result = createRedactor({
|
|
4824
|
+
profile: "share",
|
|
4825
|
+
...policyOptions
|
|
4826
|
+
}).redact(attrs);
|
|
4827
|
+
for (const finding of result.findings) {
|
|
4828
|
+
if (finding.action === "keep") continue;
|
|
4829
|
+
const taxonomy = classifyRedactionDetector(finding.detector);
|
|
4830
|
+
out.push({
|
|
4831
|
+
ruleId: "safety.redactDetector",
|
|
4832
|
+
severity: detectorSeverity(finding),
|
|
4833
|
+
status: finding.severity === "error" ? "fail" : "warning",
|
|
4834
|
+
message: `Redaction detector ${finding.detector} matched ${finding.matchKind} at ${finding.path}.`,
|
|
4835
|
+
expected: "redacted trace content",
|
|
4836
|
+
actual: finding.detector,
|
|
4837
|
+
evidence: [
|
|
4838
|
+
{
|
|
4839
|
+
runId: node.event.runId,
|
|
4840
|
+
eventId: node.event.eventId,
|
|
4841
|
+
...node.event.parentId !== void 0 ? { parentId: node.event.parentId } : {},
|
|
4842
|
+
kind: node.event.kind,
|
|
4843
|
+
name: node.event.name,
|
|
4844
|
+
...node.event.status !== void 0 ? { status: node.event.status } : {},
|
|
4845
|
+
path: `attributes.${finding.path.replace(/^\$\.?/, "")}`
|
|
4846
|
+
}
|
|
4847
|
+
],
|
|
4848
|
+
category: taxonomy.category,
|
|
4849
|
+
confidence: taxonomy.confidence,
|
|
4850
|
+
detector: finding.detector,
|
|
4851
|
+
action: finding.action
|
|
4852
|
+
});
|
|
4853
|
+
}
|
|
4854
|
+
}
|
|
4855
|
+
}
|
|
4856
|
+
return out.sort((a, b) => {
|
|
4857
|
+
const aEvidence = a.evidence[0];
|
|
4858
|
+
const bEvidence = b.evidence[0];
|
|
4859
|
+
return (aEvidence?.runId ?? "").localeCompare(bEvidence?.runId ?? "") || (aEvidence?.eventId ?? "").localeCompare(bEvidence?.eventId ?? "") || (aEvidence?.path ?? "").localeCompare(bEvidence?.path ?? "") || a.message.localeCompare(b.message);
|
|
4860
|
+
});
|
|
4861
|
+
}
|
|
4862
|
+
function exitCodeFor(result) {
|
|
4863
|
+
if (result.status === "SAFE" || result.status === "SAFE WITH WARNINGS") return 0;
|
|
4864
|
+
if (result.status === "UNSAFE") return 1;
|
|
4865
|
+
return 2;
|
|
4866
|
+
}
|
|
4867
|
+
function toResidualStatus(status) {
|
|
4868
|
+
if (status === "SAFE WITH WARNINGS") return "SAFE_WITH_WARNINGS";
|
|
4869
|
+
return status;
|
|
4870
|
+
}
|
|
4871
|
+
function toResidualSafetyAssessment(result) {
|
|
4872
|
+
const codes = [
|
|
4873
|
+
.../* @__PURE__ */ new Set([
|
|
4874
|
+
...result.diagnostics.map((item) => item.code),
|
|
4875
|
+
...result.findings.map((item) => item.ruleId)
|
|
4876
|
+
])
|
|
4877
|
+
].sort((a, b) => a.localeCompare(b));
|
|
4878
|
+
return {
|
|
4879
|
+
status: toResidualStatus(result.status),
|
|
4880
|
+
findings: result.summary.findings,
|
|
4881
|
+
warnings: result.summary.warnings,
|
|
4882
|
+
errors: result.summary.errors,
|
|
4883
|
+
codes,
|
|
4884
|
+
note: BEST_EFFORT_NOTE
|
|
4885
|
+
};
|
|
4886
|
+
}
|
|
4887
|
+
async function assessResidualFromContent(content, options = {}) {
|
|
4888
|
+
try {
|
|
4889
|
+
const policy = options.compiledPolicy ?? (options.policy !== void 0 ? await loadRedactionPolicy(options.policy) : void 0);
|
|
4890
|
+
const read = await openTrace(
|
|
4891
|
+
{ type: "string", content },
|
|
4892
|
+
{ format: options.format ?? "agent-inspect-jsonl" }
|
|
4893
|
+
);
|
|
4894
|
+
const result = assessOpenedTrace(read, {
|
|
4895
|
+
...options,
|
|
4896
|
+
...policy !== void 0 ? { compiledPolicy: policy } : {}
|
|
4897
|
+
});
|
|
4898
|
+
return toResidualSafetyAssessment(result);
|
|
4899
|
+
} catch (error) {
|
|
4900
|
+
if (error instanceof TraceReadError) {
|
|
4901
|
+
return toResidualSafetyAssessment(readErrorResult("verify-safe", error));
|
|
4902
|
+
}
|
|
4903
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4904
|
+
return {
|
|
4905
|
+
status: "UNKNOWN",
|
|
4906
|
+
findings: 0,
|
|
4907
|
+
warnings: 0,
|
|
4908
|
+
errors: 1,
|
|
4909
|
+
codes: ["AI_SAFETY_TRACE_UNREADABLE"],
|
|
4910
|
+
note: `${BEST_EFFORT_NOTE} Residual assessment requires a supported AgentInspect trace; ${message}`
|
|
4911
|
+
};
|
|
4912
|
+
}
|
|
4913
|
+
}
|
|
4914
|
+
function explainFinding(finding, blocksBundle) {
|
|
4915
|
+
const path30 = finding.evidence[0]?.path ?? "(unknown path)";
|
|
4916
|
+
const category = finding.category ?? "structure";
|
|
4917
|
+
const confidence = finding.confidence ?? "medium";
|
|
4918
|
+
const detector = finding.detector ?? finding.ruleId;
|
|
4919
|
+
const action = finding.action ?? "review";
|
|
4920
|
+
const redactionHint = category === "credential" || category === "personal-data" || category === "raw-content" || action.includes("redact") ? "Usually removable by share/strict redaction before bundling." : "May require omitting the field, lowering limits, or an explicit local override.";
|
|
4921
|
+
return [
|
|
4922
|
+
` Matched: detector=${detector}; path=${path30}; category=${category}`,
|
|
4923
|
+
` Why: ${finding.message}`,
|
|
4924
|
+
` Confidence: ${confidence}`,
|
|
4925
|
+
` Redaction: ${redactionHint}`,
|
|
4926
|
+
` Override: configure a custom redaction/detector rule locally (see docs/SAFETY-POLICY.md); do not weaken defaults globally.`,
|
|
4927
|
+
` Bundle gate: ${blocksBundle ? "blocks share-safe bundle unless --allow-unsafe" : "does not block by itself (warning/info)"}`
|
|
4928
|
+
];
|
|
4929
|
+
}
|
|
4930
|
+
function findingExplanation(finding) {
|
|
4931
|
+
const blocks = finding.severity === "error" || finding.status === "fail";
|
|
4932
|
+
return {
|
|
4933
|
+
ruleId: finding.ruleId,
|
|
4934
|
+
detector: finding.detector ?? finding.ruleId,
|
|
4935
|
+
path: finding.evidence[0]?.path,
|
|
4936
|
+
category: finding.category,
|
|
4937
|
+
confidence: finding.confidence,
|
|
4938
|
+
action: finding.action,
|
|
4939
|
+
blocksBundle: blocks,
|
|
4940
|
+
// Never include matched secret/PII values.
|
|
4941
|
+
message: finding.message
|
|
4942
|
+
};
|
|
4943
|
+
}
|
|
4944
|
+
function printHuman(result, explain = false) {
|
|
4945
|
+
console.log(`Safety status: ${result.status}`);
|
|
4946
|
+
console.log(`Format: ${result.format}`);
|
|
4947
|
+
if (result.runId !== void 0) console.log(`Run: ${result.runId}`);
|
|
4948
|
+
if (result.sourceAssessment !== void 0 && result.artifactAssessment !== void 0) {
|
|
4949
|
+
console.log(`Source assessment: ${result.sourceAssessment.status}`);
|
|
4950
|
+
console.log(`Artifact assessment: ${result.artifactAssessment.status}`);
|
|
4951
|
+
if (result.redactionSummary !== void 0) {
|
|
4952
|
+
console.log(
|
|
4953
|
+
`Redaction: profile=${result.redactionSummary.profile}, findings=${result.redactionSummary.findings}`
|
|
4954
|
+
);
|
|
4955
|
+
}
|
|
4956
|
+
}
|
|
4957
|
+
console.log(
|
|
4958
|
+
`Summary: ${result.summary.findings} finding(s), ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
4959
|
+
);
|
|
4960
|
+
for (const diagnostic3 of result.diagnostics) {
|
|
4961
|
+
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
4962
|
+
}
|
|
4963
|
+
for (const finding of result.findings) {
|
|
4964
|
+
const path30 = finding.evidence[0]?.path;
|
|
4965
|
+
const taxonomy = finding.category !== void 0 || finding.confidence !== void 0 ? ` [${[finding.category, finding.confidence].filter(Boolean).join("/")}]` : "";
|
|
4966
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path30 ? ` (${path30})` : ""}${taxonomy}`);
|
|
4967
|
+
if (explain) {
|
|
4968
|
+
const blocks = finding.severity === "error" || finding.status === "fail";
|
|
4969
|
+
for (const line of explainFinding(finding, blocks)) {
|
|
4970
|
+
console.log(line);
|
|
4971
|
+
}
|
|
4972
|
+
}
|
|
4973
|
+
}
|
|
4974
|
+
console.log(`Note: ${result.note}`);
|
|
4975
|
+
}
|
|
4976
|
+
function printJson(result, explain = false) {
|
|
4977
|
+
const payload = explain === true ? { ...result, explanations: result.findings.map(findingExplanation) } : result;
|
|
4978
|
+
console.log(JSON.stringify(stable(payload), null, 2));
|
|
4979
|
+
}
|
|
4980
|
+
async function safetyCommand(command, target, options, stdin) {
|
|
4981
|
+
let result;
|
|
4982
|
+
try {
|
|
4983
|
+
const policy = options.compiledPolicy ?? (options.policy !== void 0 ? await loadRedactionPolicy(options.policy) : void 0);
|
|
4984
|
+
const optionsWithPolicy = {
|
|
4985
|
+
...options,
|
|
4986
|
+
...policy !== void 0 ? { compiledPolicy: policy } : {}
|
|
4987
|
+
};
|
|
4988
|
+
const input3 = await inputFromTarget(target, optionsWithPolicy, stdin);
|
|
4989
|
+
const read = await openTrace(input3, {
|
|
4990
|
+
...options.format !== void 0 ? { format: options.format } : {}
|
|
4991
|
+
});
|
|
4992
|
+
const source = assessOpenedTrace(read, {
|
|
4993
|
+
...optionsWithPolicy,
|
|
4994
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
4995
|
+
});
|
|
4996
|
+
if (command === "scan") {
|
|
4997
|
+
result = { ...source, command: "scan" };
|
|
4998
|
+
} else {
|
|
4999
|
+
const profile = options.redactionProfile ?? "share";
|
|
5000
|
+
const rawContent = input3.type === "string" ? input3.content : input3.type === "file" ? await readFile(input3.path, "utf-8") : void 0;
|
|
5001
|
+
if (rawContent === void 0) {
|
|
5002
|
+
result = {
|
|
5003
|
+
...source,
|
|
5004
|
+
command: "verify-safe",
|
|
5005
|
+
sourceAssessment: layerFromResult(source)
|
|
5006
|
+
};
|
|
5007
|
+
} else {
|
|
5008
|
+
const redacted = redactTraceContent(rawContent, profile, policy);
|
|
5009
|
+
const artifactRead = await openTrace(
|
|
5010
|
+
{ type: "string", content: redacted.content },
|
|
5011
|
+
{ format: options.format ?? "agent-inspect-jsonl" }
|
|
5012
|
+
);
|
|
5013
|
+
const artifact = assessOpenedTrace(artifactRead, {
|
|
5014
|
+
...optionsWithPolicy,
|
|
5015
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
5016
|
+
});
|
|
5017
|
+
const detectors = [
|
|
5018
|
+
...new Set(redacted.findings.map((finding) => finding.detector))
|
|
5019
|
+
].sort((a, b) => a.localeCompare(b));
|
|
5020
|
+
result = resultFromParts({
|
|
5021
|
+
command: "verify-safe",
|
|
5022
|
+
format: artifact.format,
|
|
5023
|
+
runId: artifact.runId ?? source.runId,
|
|
5024
|
+
findings: artifact.findings,
|
|
5025
|
+
diagnostics: artifact.diagnostics,
|
|
5026
|
+
warnings: artifact.warnings,
|
|
5027
|
+
unsupportedFields: artifact.unsupportedFields,
|
|
5028
|
+
status: artifact.status,
|
|
5029
|
+
sourceAssessment: layerFromResult(source),
|
|
5030
|
+
artifactAssessment: layerFromResult(artifact),
|
|
5031
|
+
redactionSummary: {
|
|
5032
|
+
profile,
|
|
5033
|
+
findings: redacted.findings.length,
|
|
5034
|
+
detectors
|
|
5035
|
+
}
|
|
5036
|
+
});
|
|
5037
|
+
}
|
|
5038
|
+
}
|
|
5039
|
+
} catch (error) {
|
|
5040
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5041
|
+
result = message.startsWith("--") || message.includes("--policy") ? invalidArgumentResult(command, error) : readErrorResult(command, error);
|
|
5042
|
+
}
|
|
5043
|
+
process.exitCode = exitCodeFor(result);
|
|
5044
|
+
if (options.json) printJson(result, options.explain === true);
|
|
5045
|
+
else printHuman(result, options.explain === true);
|
|
5046
|
+
}
|
|
5047
|
+
function scanCommand(target, options = {}, stdin = process.stdin) {
|
|
5048
|
+
return safetyCommand("scan", target, options, stdin);
|
|
5049
|
+
}
|
|
5050
|
+
function verifySafeCommand(target, options = {}, stdin = process.stdin) {
|
|
5051
|
+
return safetyCommand("verify-safe", target, options, stdin);
|
|
5052
|
+
}
|
|
5053
|
+
function assessOpenedTrace(read, options = {}) {
|
|
5054
|
+
try {
|
|
5055
|
+
const rules = buildSafetyRules(options);
|
|
5056
|
+
const checkResult = runTraceChecks(
|
|
5057
|
+
{ read },
|
|
5058
|
+
{
|
|
5059
|
+
rules,
|
|
5060
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
5061
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
5062
|
+
}
|
|
5063
|
+
);
|
|
5064
|
+
const detectorFindings = checkResult.diagnostics.length === 0 ? redactionDetectorFindings(read, checkResult.runId, options.compiledPolicy) : [];
|
|
5065
|
+
return resultFromParts({
|
|
5066
|
+
command: "verify-safe",
|
|
5067
|
+
format: checkResult.format,
|
|
5068
|
+
runId: checkResult.runId,
|
|
5069
|
+
findings: [...checkResult.findings, ...detectorFindings],
|
|
5070
|
+
diagnostics: [
|
|
5071
|
+
...checkResult.diagnostics.map(diagnosticFromCheck),
|
|
5072
|
+
...warningDiagnostics(read.warnings, read.unsupportedFields)
|
|
5073
|
+
],
|
|
5074
|
+
warnings: read.warnings,
|
|
5075
|
+
unsupportedFields: read.unsupportedFields
|
|
5076
|
+
});
|
|
5077
|
+
} catch (error) {
|
|
5078
|
+
return messageStartsWithDash(error) ? invalidArgumentResult("verify-safe", error) : readErrorResult("verify-safe", error);
|
|
5079
|
+
}
|
|
5080
|
+
}
|
|
5081
|
+
function messageStartsWithDash(error) {
|
|
5082
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5083
|
+
return message.startsWith("--");
|
|
5084
|
+
}
|
|
5085
|
+
|
|
5086
|
+
// packages/cli/src/redact.ts
|
|
5087
|
+
function parseRedactionProfile3(value) {
|
|
5088
|
+
if (value === void 0 || value === "local" || value === "share" || value === "strict") {
|
|
5089
|
+
return value ?? "share";
|
|
5090
|
+
}
|
|
5091
|
+
throw new Error(`Unsupported --profile "${value}". Use local, share, or strict.`);
|
|
5092
|
+
}
|
|
5093
|
+
function stable2(value) {
|
|
5094
|
+
if (Array.isArray(value)) return value.map(stable2);
|
|
5095
|
+
if (value === null || typeof value !== "object") return value;
|
|
5096
|
+
const record = value;
|
|
5097
|
+
return Object.fromEntries(
|
|
5098
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable2(record[key])])
|
|
5099
|
+
);
|
|
5100
|
+
}
|
|
5101
|
+
function isMissingFileError3(error) {
|
|
5102
|
+
return error !== null && typeof error === "object" && "code" in error && error.code === "ENOENT";
|
|
5103
|
+
}
|
|
5104
|
+
async function contentFromTarget(target, options, stdin) {
|
|
5105
|
+
if (target === "-") {
|
|
5106
|
+
return { content: await readStdin(stdin), source: "stdin" };
|
|
5107
|
+
}
|
|
5108
|
+
try {
|
|
5109
|
+
const stats2 = await stat(target);
|
|
5110
|
+
if (stats2.isDirectory()) {
|
|
5111
|
+
throw new Error("redact requires a trace file, JSON file, stdin, or run id.");
|
|
5112
|
+
}
|
|
5113
|
+
return { content: await readFile(target, "utf-8"), source: target };
|
|
5114
|
+
} catch (error) {
|
|
5115
|
+
if (!isMissingFileError3(error)) throw error;
|
|
5116
|
+
}
|
|
5117
|
+
const runPath = getTraceFilePath(target, resolveTraceDir({ dir: options.dir }));
|
|
5118
|
+
const stats = await stat(runPath);
|
|
5119
|
+
if (stats.isDirectory()) {
|
|
5120
|
+
throw new Error("redact requires a trace file, JSON file, stdin, or run id.");
|
|
5121
|
+
}
|
|
5122
|
+
return { content: await readFile(runPath, "utf-8"), source: runPath };
|
|
5123
|
+
}
|
|
5124
|
+
function applyRedact(value, profile, policy) {
|
|
5125
|
+
const result = createRedactor({
|
|
5126
|
+
profile,
|
|
5127
|
+
...policy?.extraKeys.length ? { extraKeys: [...policy.extraKeys] } : {},
|
|
5128
|
+
...policy?.detectors.length ? { detectors: [...policy.detectors] } : {}
|
|
5129
|
+
}).redact(value);
|
|
5130
|
+
return { value: result.value, findings: result.findings };
|
|
5131
|
+
}
|
|
5132
|
+
function redactJsonText(content, profile, policy) {
|
|
5133
|
+
const parsed = JSON.parse(content);
|
|
5134
|
+
const result = applyRedact(parsed, profile, policy);
|
|
5135
|
+
return {
|
|
5136
|
+
content: `${JSON.stringify(result.value, null, 2)}
|
|
5137
|
+
`,
|
|
5138
|
+
findings: result.findings
|
|
5139
|
+
};
|
|
5140
|
+
}
|
|
5141
|
+
function redactJsonlText(content, profile, policy) {
|
|
5142
|
+
const lines = content.split(/\r?\n/);
|
|
5143
|
+
const out = [];
|
|
5144
|
+
const findings = [];
|
|
5145
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
5146
|
+
const line = lines[index] ?? "";
|
|
5147
|
+
if (line.trim() === "") continue;
|
|
5148
|
+
let parsed;
|
|
5149
|
+
try {
|
|
5150
|
+
parsed = JSON.parse(line);
|
|
5151
|
+
} catch {
|
|
5152
|
+
throw new Error(`Input is not valid JSON or JSONL at line ${index + 1}.`);
|
|
5153
|
+
}
|
|
5154
|
+
const result = applyRedact(parsed, profile, policy);
|
|
5155
|
+
out.push(JSON.stringify(result.value));
|
|
5156
|
+
findings.push(
|
|
5157
|
+
...result.findings.map((finding) => ({
|
|
5158
|
+
...finding,
|
|
5159
|
+
path: `line:${index + 1}:${finding.path}`
|
|
5160
|
+
}))
|
|
5161
|
+
);
|
|
5162
|
+
}
|
|
5163
|
+
return {
|
|
5164
|
+
content: out.length === 0 ? "" : `${out.join("\n")}
|
|
5165
|
+
`,
|
|
5166
|
+
findings
|
|
5167
|
+
};
|
|
5168
|
+
}
|
|
5169
|
+
function redactDocument(content, profile, policy) {
|
|
5170
|
+
const trimmed = content.trim();
|
|
5171
|
+
if (trimmed.startsWith("{")) {
|
|
5172
|
+
try {
|
|
5173
|
+
const parsed = JSON.parse(trimmed);
|
|
5174
|
+
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) && ("schemaVersion" in parsed || "eventId" in parsed || "runId" in parsed)) {
|
|
5175
|
+
return redactJsonlText(content, profile, policy);
|
|
5176
|
+
}
|
|
5177
|
+
} catch {
|
|
5178
|
+
}
|
|
5179
|
+
}
|
|
5180
|
+
try {
|
|
5181
|
+
return redactJsonText(content, profile, policy);
|
|
5182
|
+
} catch {
|
|
5183
|
+
return redactJsonlText(content, profile, policy);
|
|
5184
|
+
}
|
|
5185
|
+
}
|
|
5186
|
+
function redactTraceContent(content, profile, policy) {
|
|
5187
|
+
return redactDocument(content, profile, policy);
|
|
5188
|
+
}
|
|
5189
|
+
function looksLikeAgentInspectTrace(content) {
|
|
5190
|
+
const trimmed = content.trim();
|
|
5191
|
+
if (!trimmed.startsWith("{")) return false;
|
|
5192
|
+
try {
|
|
5193
|
+
const firstLine = trimmed.split(/\r?\n/).find((line) => line.trim() !== "") ?? trimmed;
|
|
5194
|
+
const parsed = JSON.parse(firstLine);
|
|
5195
|
+
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) && ("schemaVersion" in parsed || "eventId" in parsed || "runId" in parsed);
|
|
5196
|
+
} catch {
|
|
5197
|
+
return false;
|
|
5198
|
+
}
|
|
5199
|
+
}
|
|
5200
|
+
function printResidualWarning(assessment, sourceLooksLikeTrace) {
|
|
5201
|
+
if (assessment.status === "SAFE") return;
|
|
5202
|
+
if (assessment.status === "UNKNOWN" && !sourceLooksLikeTrace) return;
|
|
5203
|
+
const codes = assessment.codes.length > 0 ? ` codes=${assessment.codes.join(",")}` : "";
|
|
5204
|
+
console.error(
|
|
5205
|
+
`Residual safety: ${assessment.status} (findings=${assessment.findings}, warnings=${assessment.warnings}, errors=${assessment.errors})${codes}. Redact does not certify safe sharing; run verify-safe before publishing.`
|
|
5206
|
+
);
|
|
5207
|
+
}
|
|
5208
|
+
function residualExitCode(assessment) {
|
|
5209
|
+
if (assessment.status === "UNSAFE") return 1;
|
|
5210
|
+
if (assessment.status === "UNKNOWN") return 2;
|
|
5211
|
+
return 0;
|
|
5212
|
+
}
|
|
5213
|
+
async function redactCommand(target, options = {}, stdin = process.stdin) {
|
|
5214
|
+
const profile = parseRedactionProfile3(resolveRedactionProfileOption(options));
|
|
5215
|
+
const policy = options.policy !== void 0 ? await loadRedactionPolicy(options.policy) : void 0;
|
|
5216
|
+
const source = await contentFromTarget(target, options, stdin);
|
|
5217
|
+
const redacted = redactDocument(source.content, profile, policy);
|
|
4476
5218
|
const outputPath = resolveOutputOption(options);
|
|
5219
|
+
const residualAssessment = await assessResidualFromContent(redacted.content, {
|
|
5220
|
+
compiledPolicy: policy
|
|
5221
|
+
});
|
|
4477
5222
|
if (outputPath !== void 0) {
|
|
4478
5223
|
await writeFile(outputPath, redacted.content, "utf-8");
|
|
4479
5224
|
}
|
|
5225
|
+
if (options.failOnResidual === true) {
|
|
5226
|
+
process.exitCode = residualExitCode(residualAssessment);
|
|
5227
|
+
}
|
|
4480
5228
|
if (options.json) {
|
|
4481
5229
|
console.log(
|
|
4482
5230
|
JSON.stringify(
|
|
4483
|
-
|
|
5231
|
+
stable2({
|
|
4484
5232
|
ok: true,
|
|
4485
5233
|
profile,
|
|
4486
5234
|
source: source.source,
|
|
4487
5235
|
output: outputPath,
|
|
4488
5236
|
findings: redacted.findings,
|
|
5237
|
+
residualAssessment,
|
|
5238
|
+
...policy !== void 0 ? {
|
|
5239
|
+
policy: {
|
|
5240
|
+
path: policy.path,
|
|
5241
|
+
extraKeys: policy.extraKeys.length,
|
|
5242
|
+
patterns: policy.detectors.length,
|
|
5243
|
+
diagnostics: policy.diagnostics
|
|
5244
|
+
}
|
|
5245
|
+
} : {},
|
|
4489
5246
|
content: outputPath === void 0 ? redacted.content : void 0
|
|
4490
5247
|
}),
|
|
4491
5248
|
null,
|
|
@@ -4494,6 +5251,7 @@ async function redactCommand(target, options = {}, stdin = process.stdin) {
|
|
|
4494
5251
|
);
|
|
4495
5252
|
return;
|
|
4496
5253
|
}
|
|
5254
|
+
printResidualWarning(residualAssessment, looksLikeAgentInspectTrace(redacted.content));
|
|
4497
5255
|
if (outputPath === void 0) {
|
|
4498
5256
|
process.stdout.write(redacted.content);
|
|
4499
5257
|
}
|
|
@@ -4769,7 +5527,7 @@ async function openCommand(input3, options = {}, stdin = process.stdin) {
|
|
|
4769
5527
|
}
|
|
4770
5528
|
}
|
|
4771
5529
|
}
|
|
4772
|
-
function
|
|
5530
|
+
function isRecord7(value) {
|
|
4773
5531
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4774
5532
|
}
|
|
4775
5533
|
function parseTarget(value) {
|
|
@@ -4778,7 +5536,7 @@ function parseTarget(value) {
|
|
|
4778
5536
|
throw new Error('Unsupported migration target. Use "--to 1.0".');
|
|
4779
5537
|
}
|
|
4780
5538
|
function formatOf(value) {
|
|
4781
|
-
if (!
|
|
5539
|
+
if (!isRecord7(value)) return "unknown";
|
|
4782
5540
|
if (value.schemaVersion === "0.1") return "0.1";
|
|
4783
5541
|
if (value.schemaVersion === "0.2") return "0.2";
|
|
4784
5542
|
if (value.schemaVersion === "1.0") return "1.0";
|
|
@@ -4791,14 +5549,14 @@ function uniqueSorted(values) {
|
|
|
4791
5549
|
return [...new Set(values)].sort();
|
|
4792
5550
|
}
|
|
4793
5551
|
function isWithinDirectory(child, parent) {
|
|
4794
|
-
const relative =
|
|
4795
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
5552
|
+
const relative = path19.relative(parent, child);
|
|
5553
|
+
return relative === "" || !relative.startsWith("..") && !path19.isAbsolute(relative);
|
|
4796
5554
|
}
|
|
4797
5555
|
async function resolveOutputPath(inputPath, output2, force) {
|
|
4798
5556
|
if (output2 === void 0 || output2.trim() === "") return void 0;
|
|
4799
|
-
const inputAbs =
|
|
4800
|
-
const outputAbs =
|
|
4801
|
-
const inputDir =
|
|
5557
|
+
const inputAbs = path19.resolve(inputPath);
|
|
5558
|
+
const outputAbs = path19.resolve(output2.trim());
|
|
5559
|
+
const inputDir = path19.dirname(inputAbs);
|
|
4802
5560
|
if (!isWithinDirectory(outputAbs, inputDir)) {
|
|
4803
5561
|
throw new Error("Refusing to write migrated output outside the input directory.");
|
|
4804
5562
|
}
|
|
@@ -4919,7 +5677,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
4919
5677
|
process.exitCode = 1;
|
|
4920
5678
|
return;
|
|
4921
5679
|
}
|
|
4922
|
-
const inputPath =
|
|
5680
|
+
const inputPath = path19.resolve(input3.trim());
|
|
4923
5681
|
const dryRun = options.dryRun === true;
|
|
4924
5682
|
if (!dryRun && (options.output === void 0 || options.output.trim() === "")) {
|
|
4925
5683
|
console.error("migrate requires --dry-run or --output <path>.");
|
|
@@ -4938,7 +5696,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
4938
5696
|
);
|
|
4939
5697
|
const result = await buildMigration(inputPath, outputPath);
|
|
4940
5698
|
if (!dryRun && outputPath !== void 0) {
|
|
4941
|
-
await mkdir(
|
|
5699
|
+
await mkdir(path19.dirname(outputPath), { recursive: true });
|
|
4942
5700
|
await writeFile(outputPath, result.content, "utf-8");
|
|
4943
5701
|
}
|
|
4944
5702
|
printSummary2(result, dryRun);
|
|
@@ -4949,7 +5707,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
4949
5707
|
}
|
|
4950
5708
|
}
|
|
4951
5709
|
function validateReporterArtifactPath(options) {
|
|
4952
|
-
const outputDir =
|
|
5710
|
+
const outputDir = path19.resolve(options.outputDir);
|
|
4953
5711
|
const diagnostics = [];
|
|
4954
5712
|
const rawPath = options.relativePath;
|
|
4955
5713
|
if (rawPath.length === 0) {
|
|
@@ -4969,7 +5727,7 @@ function validateReporterArtifactPath(options) {
|
|
|
4969
5727
|
});
|
|
4970
5728
|
return { ok: false, outputDir, diagnostics };
|
|
4971
5729
|
}
|
|
4972
|
-
if (
|
|
5730
|
+
if (path19.isAbsolute(rawPath) || path19.win32.isAbsolute(rawPath)) {
|
|
4973
5731
|
diagnostics.push({
|
|
4974
5732
|
code: "artifact_path_absolute",
|
|
4975
5733
|
severity: "error",
|
|
@@ -4978,7 +5736,7 @@ function validateReporterArtifactPath(options) {
|
|
|
4978
5736
|
});
|
|
4979
5737
|
return { ok: false, outputDir, diagnostics };
|
|
4980
5738
|
}
|
|
4981
|
-
const normalized =
|
|
5739
|
+
const normalized = path19.posix.normalize(rawPath.replace(/\\/g, "/"));
|
|
4982
5740
|
const segments = normalized.split("/");
|
|
4983
5741
|
if (normalized === "." || normalized.startsWith("../") || segments.some((segment) => segment === "..")) {
|
|
4984
5742
|
diagnostics.push({
|
|
@@ -4989,9 +5747,9 @@ function validateReporterArtifactPath(options) {
|
|
|
4989
5747
|
});
|
|
4990
5748
|
return { ok: false, outputDir, diagnostics };
|
|
4991
5749
|
}
|
|
4992
|
-
const absolutePath =
|
|
4993
|
-
const relFromOutput =
|
|
4994
|
-
if (relFromOutput.length === 0 || relFromOutput.startsWith("..") ||
|
|
5750
|
+
const absolutePath = path19.resolve(outputDir, normalized);
|
|
5751
|
+
const relFromOutput = path19.relative(outputDir, absolutePath);
|
|
5752
|
+
if (relFromOutput.length === 0 || relFromOutput.startsWith("..") || path19.isAbsolute(relFromOutput)) {
|
|
4995
5753
|
diagnostics.push({
|
|
4996
5754
|
code: "artifact_path_escape",
|
|
4997
5755
|
severity: "error",
|
|
@@ -5017,7 +5775,7 @@ var EVIDENCE_CI_ARTIFACT_FILES = [
|
|
|
5017
5775
|
function createEvidenceCiArtifacts(options) {
|
|
5018
5776
|
const profile = options.redactionProfile ?? "share";
|
|
5019
5777
|
const prefix = options.relativeDir !== void 0 && options.relativeDir.trim() !== "" ? options.relativeDir.replace(/\\/g, "/").replace(/^\/+|\/+$/g, "") : "";
|
|
5020
|
-
const join = (name) => prefix === "" ? name :
|
|
5778
|
+
const join = (name) => prefix === "" ? name : path19.posix.join(prefix, name);
|
|
5021
5779
|
const formatFor = (name) => {
|
|
5022
5780
|
if (name.endsWith(".html")) return "html";
|
|
5023
5781
|
if (name.endsWith(".jsonl")) return "jsonl";
|
|
@@ -5066,9 +5824,9 @@ function stableJson2(value) {
|
|
|
5066
5824
|
async function writeLocalEvidence(input3) {
|
|
5067
5825
|
const profile = input3.redactionProfile ?? "share";
|
|
5068
5826
|
const format = input3.format ?? "directory";
|
|
5069
|
-
const baseDir =
|
|
5827
|
+
const baseDir = path19.resolve(input3.outputDir);
|
|
5070
5828
|
await mkdir(
|
|
5071
|
-
format === "zip" && baseDir.toLowerCase().endsWith(".zip") ?
|
|
5829
|
+
format === "zip" && baseDir.toLowerCase().endsWith(".zip") ? path19.dirname(baseDir) : baseDir,
|
|
5072
5830
|
{ recursive: true }
|
|
5073
5831
|
);
|
|
5074
5832
|
const sourceContents = /* @__PURE__ */ new Map();
|
|
@@ -5124,7 +5882,7 @@ async function writeLocalEvidence(input3) {
|
|
|
5124
5882
|
}
|
|
5125
5883
|
if (format === "zip") {
|
|
5126
5884
|
const zipPath = baseDir.toLowerCase().endsWith(".zip") ? baseDir : `${baseDir}.zip`;
|
|
5127
|
-
await mkdir(
|
|
5885
|
+
await mkdir(path19.dirname(zipPath), { recursive: true });
|
|
5128
5886
|
const archive = buildZipArchive(
|
|
5129
5887
|
files.map(([relativePath, content]) => ({
|
|
5130
5888
|
path: relativePath,
|
|
@@ -5139,16 +5897,16 @@ async function writeLocalEvidence(input3) {
|
|
|
5139
5897
|
let sidecarDir = baseDir;
|
|
5140
5898
|
if (baseDir.toLowerCase().endsWith(".html")) {
|
|
5141
5899
|
htmlPath = baseDir;
|
|
5142
|
-
sidecarDir =
|
|
5900
|
+
sidecarDir = path19.dirname(baseDir);
|
|
5143
5901
|
} else {
|
|
5144
5902
|
await mkdir(baseDir, { recursive: true });
|
|
5145
|
-
htmlPath =
|
|
5903
|
+
htmlPath = path19.join(baseDir, "evidence.html");
|
|
5146
5904
|
sidecarDir = baseDir;
|
|
5147
5905
|
}
|
|
5148
5906
|
await mkdir(sidecarDir, { recursive: true });
|
|
5149
5907
|
await writeFile(htmlPath, evidencePackage["evidence.html"], "utf-8");
|
|
5150
5908
|
await writeFile(
|
|
5151
|
-
|
|
5909
|
+
path19.join(sidecarDir, "evidence.json"),
|
|
5152
5910
|
evidencePackage["evidence.json"],
|
|
5153
5911
|
"utf-8"
|
|
5154
5912
|
);
|
|
@@ -5156,7 +5914,7 @@ async function writeLocalEvidence(input3) {
|
|
|
5156
5914
|
}
|
|
5157
5915
|
await mkdir(baseDir, { recursive: true });
|
|
5158
5916
|
for (const [name, content] of files) {
|
|
5159
|
-
await writeFile(
|
|
5917
|
+
await writeFile(path19.join(baseDir, name), content, "utf-8");
|
|
5160
5918
|
}
|
|
5161
5919
|
return baseDir;
|
|
5162
5920
|
}
|
|
@@ -5180,11 +5938,11 @@ function checkResultToEvidenceJson(result, runIds) {
|
|
|
5180
5938
|
}
|
|
5181
5939
|
function defaultEvidenceDir(label) {
|
|
5182
5940
|
const safe = label.replace(/[^a-zA-Z0-9._-]+/g, "_").slice(0, 80) || "evidence";
|
|
5183
|
-
return
|
|
5941
|
+
return path19.join(".agent-inspect", "evidence", safe);
|
|
5184
5942
|
}
|
|
5185
5943
|
function resolveEvidenceOutputDir(evidenceDir, label) {
|
|
5186
5944
|
if (evidenceDir !== void 0 && evidenceDir.trim() !== "") {
|
|
5187
|
-
return
|
|
5945
|
+
return path19.resolve(evidenceDir.trim());
|
|
5188
5946
|
}
|
|
5189
5947
|
return defaultEvidenceDir(label);
|
|
5190
5948
|
}
|
|
@@ -5508,23 +6266,23 @@ function evaluatePromptInjection(text, options = {}) {
|
|
|
5508
6266
|
}
|
|
5509
6267
|
return fail(ruleId, `Matched ${evidence.length} injection pattern(s).`, evidence, "warning");
|
|
5510
6268
|
}
|
|
5511
|
-
function validateSchemaField(value, field,
|
|
6269
|
+
function validateSchemaField(value, field, path30, evidence) {
|
|
5512
6270
|
const ruleId = "guardrail.structured-output";
|
|
5513
6271
|
if (field.type) {
|
|
5514
6272
|
const actual = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
|
|
5515
6273
|
if (actual !== field.type) {
|
|
5516
|
-
evidence.push({ ruleId, path:
|
|
6274
|
+
evidence.push({ ruleId, path: path30, preview: `expected ${field.type}, got ${actual}` });
|
|
5517
6275
|
return;
|
|
5518
6276
|
}
|
|
5519
6277
|
}
|
|
5520
6278
|
if (field.enum && !field.enum.some((item) => Object.is(item, value))) {
|
|
5521
|
-
evidence.push({ ruleId, path:
|
|
6279
|
+
evidence.push({ ruleId, path: path30, preview: "value not in enum" });
|
|
5522
6280
|
}
|
|
5523
6281
|
if (field.type === "object" && field.required && value && typeof value === "object" && !Array.isArray(value)) {
|
|
5524
6282
|
const record = value;
|
|
5525
6283
|
for (const key of field.required) {
|
|
5526
6284
|
if (!(key in record)) {
|
|
5527
|
-
evidence.push({ ruleId, path: `${
|
|
6285
|
+
evidence.push({ ruleId, path: `${path30}.${key}`, preview: "missing required key" });
|
|
5528
6286
|
}
|
|
5529
6287
|
}
|
|
5530
6288
|
}
|
|
@@ -6253,7 +7011,7 @@ function parseCheckConfig(value) {
|
|
|
6253
7011
|
}
|
|
6254
7012
|
async function loadConfig(configPath) {
|
|
6255
7013
|
if (configPath === void 0) return {};
|
|
6256
|
-
const extension =
|
|
7014
|
+
const extension = path19.extname(configPath);
|
|
6257
7015
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
6258
7016
|
throw new CheckConfigError(
|
|
6259
7017
|
"AI_CHECK_CONFIG_LOAD_FAILED",
|
|
@@ -6266,7 +7024,7 @@ async function loadConfig(configPath) {
|
|
|
6266
7024
|
"Unsupported check config extension. Use .json, .js, .mjs, or .cjs."
|
|
6267
7025
|
);
|
|
6268
7026
|
}
|
|
6269
|
-
const absolute =
|
|
7027
|
+
const absolute = path19.resolve(configPath);
|
|
6270
7028
|
try {
|
|
6271
7029
|
if (extension === ".json") {
|
|
6272
7030
|
const raw = await readFile(absolute, "utf-8");
|
|
@@ -6460,7 +7218,7 @@ function buildRules(config, options, presetSelect = []) {
|
|
|
6460
7218
|
diagnostics
|
|
6461
7219
|
};
|
|
6462
7220
|
}
|
|
6463
|
-
function
|
|
7221
|
+
function exitCodeFor2(result) {
|
|
6464
7222
|
if (result.status === "pass") return 0;
|
|
6465
7223
|
if (result.status === "fail") return 1;
|
|
6466
7224
|
const codes = result.diagnostics.map((item) => item.code);
|
|
@@ -6481,16 +7239,16 @@ function exitCodeFor(result) {
|
|
|
6481
7239
|
}
|
|
6482
7240
|
return 1;
|
|
6483
7241
|
}
|
|
6484
|
-
function
|
|
6485
|
-
if (Array.isArray(value)) return value.map(
|
|
7242
|
+
function stable3(value) {
|
|
7243
|
+
if (Array.isArray(value)) return value.map(stable3);
|
|
6486
7244
|
if (value === null || typeof value !== "object") return value;
|
|
6487
7245
|
const record = value;
|
|
6488
7246
|
return Object.fromEntries(
|
|
6489
|
-
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key,
|
|
7247
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable3(record[key])])
|
|
6490
7248
|
);
|
|
6491
7249
|
}
|
|
6492
|
-
function
|
|
6493
|
-
console.log(JSON.stringify(
|
|
7250
|
+
function printJson2(result) {
|
|
7251
|
+
console.log(JSON.stringify(stable3(result), null, 2));
|
|
6494
7252
|
}
|
|
6495
7253
|
function isSafetyFinding(ruleId) {
|
|
6496
7254
|
return ruleId.startsWith("safety.") || ruleId.startsWith("guardrail.") || ruleId.includes("pii") || ruleId.includes("secret");
|
|
@@ -6527,7 +7285,7 @@ function printPresetClassSummary(result, preset) {
|
|
|
6527
7285
|
`Share safety: ${hasSafetyFindings || result.status === "fail" ? "FAIL" : result.status === "pass" ? "PASS" : "FAIL"}`
|
|
6528
7286
|
);
|
|
6529
7287
|
}
|
|
6530
|
-
function
|
|
7288
|
+
function printHuman2(result, options = {}) {
|
|
6531
7289
|
const scoped = result;
|
|
6532
7290
|
if (scoped.scopeLabel) {
|
|
6533
7291
|
console.log(`Scope: ${scoped.scopeKind} ${scoped.scopeLabel}`);
|
|
@@ -6547,13 +7305,13 @@ function printHuman(result, options = {}) {
|
|
|
6547
7305
|
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
6548
7306
|
}
|
|
6549
7307
|
for (const finding of result.findings) {
|
|
6550
|
-
const
|
|
7308
|
+
const path30 = finding.evidence[0]?.path;
|
|
6551
7309
|
const run = finding.evidence[0]?.runId;
|
|
6552
7310
|
const runPrefix = run ? `[${run}] ` : "";
|
|
6553
|
-
console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${
|
|
7311
|
+
console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${path30 ? ` (${path30})` : ""}`);
|
|
6554
7312
|
}
|
|
6555
7313
|
}
|
|
6556
|
-
function
|
|
7314
|
+
function readErrorResult2(error) {
|
|
6557
7315
|
if (error instanceof TraceReadError) {
|
|
6558
7316
|
const code = error.code === "unsupported_format" ? "AI_CHECK_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_CHECK_AMBIGUOUS_FORMAT" : "AI_CHECK_TRACE_UNREADABLE";
|
|
6559
7317
|
return errorResult(code, error.message);
|
|
@@ -6580,9 +7338,9 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
6580
7338
|
"AI_CHECK_CONFIG_NO_EFFECTIVE_RULES",
|
|
6581
7339
|
"Explicit --config has no effective check rules. Configure checks.select, checks.run, checks.tool, checks.llm, checks.structure, or checks.safety."
|
|
6582
7340
|
);
|
|
6583
|
-
if (options.json)
|
|
6584
|
-
else
|
|
6585
|
-
process.exitCode =
|
|
7341
|
+
if (options.json) printJson2(result);
|
|
7342
|
+
else printHuman2(result, options);
|
|
7343
|
+
process.exitCode = exitCodeFor2(result);
|
|
6586
7344
|
return;
|
|
6587
7345
|
}
|
|
6588
7346
|
let effectiveOptions = options;
|
|
@@ -6712,10 +7470,10 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
6712
7470
|
result = errorResult(code, message);
|
|
6713
7471
|
}
|
|
6714
7472
|
} else {
|
|
6715
|
-
result =
|
|
7473
|
+
result = readErrorResult2(error);
|
|
6716
7474
|
}
|
|
6717
7475
|
}
|
|
6718
|
-
process.exitCode =
|
|
7476
|
+
process.exitCode = exitCodeFor2(result);
|
|
6719
7477
|
const failed = result.status !== "pass";
|
|
6720
7478
|
if (shouldEmitEvidence(options.evidenceOn, failed)) {
|
|
6721
7479
|
try {
|
|
@@ -6755,8 +7513,8 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
6755
7513
|
);
|
|
6756
7514
|
}
|
|
6757
7515
|
}
|
|
6758
|
-
if (options.json)
|
|
6759
|
-
else
|
|
7516
|
+
if (options.json) printJson2(result);
|
|
7517
|
+
else printHuman2(result, options);
|
|
6760
7518
|
}
|
|
6761
7519
|
|
|
6762
7520
|
// packages/viewer/src/html.ts
|
|
@@ -6907,7 +7665,7 @@ async function loadSuiteViewerData(options) {
|
|
|
6907
7665
|
for (const suiteCase of result.cases) {
|
|
6908
7666
|
cases.push(await enrichCase(suiteCase, baselinePath));
|
|
6909
7667
|
}
|
|
6910
|
-
const artifactsDir =
|
|
7668
|
+
const artifactsDir = path19.join(path19.dirname(result.configPath), ".agent-inspect/suite-runs");
|
|
6911
7669
|
return {
|
|
6912
7670
|
suiteName: result.suiteName,
|
|
6913
7671
|
configPath: result.configPath,
|
|
@@ -7100,19 +7858,19 @@ function serializeWorkspaceManifest(manifest) {
|
|
|
7100
7858
|
}
|
|
7101
7859
|
var INDEX_DIR_NAME = "index";
|
|
7102
7860
|
function resolveWorkspaceLocation(cwd = process.cwd()) {
|
|
7103
|
-
const projectRoot =
|
|
7104
|
-
const workspaceDir =
|
|
7861
|
+
const projectRoot = path19.resolve(cwd);
|
|
7862
|
+
const workspaceDir = path19.join(projectRoot, WORKSPACE_DIR_NAME);
|
|
7105
7863
|
return {
|
|
7106
7864
|
projectRoot,
|
|
7107
7865
|
workspaceDir,
|
|
7108
|
-
manifestPath:
|
|
7866
|
+
manifestPath: path19.join(workspaceDir, WORKSPACE_MANIFEST_FILENAME)
|
|
7109
7867
|
};
|
|
7110
7868
|
}
|
|
7111
7869
|
function resolveInsideWorkspace(workspaceDir, relative) {
|
|
7112
|
-
const base =
|
|
7113
|
-
const resolved =
|
|
7114
|
-
const rel =
|
|
7115
|
-
if (rel === "" || rel === "." || !rel.startsWith("..") && !
|
|
7870
|
+
const base = path19.resolve(workspaceDir);
|
|
7871
|
+
const resolved = path19.resolve(base, relative);
|
|
7872
|
+
const rel = path19.relative(base, resolved);
|
|
7873
|
+
if (rel === "" || rel === "." || !rel.startsWith("..") && !path19.isAbsolute(rel)) {
|
|
7116
7874
|
return resolved;
|
|
7117
7875
|
}
|
|
7118
7876
|
throw new Error(
|
|
@@ -7181,7 +7939,7 @@ async function createWorkspace(options = {}) {
|
|
|
7181
7939
|
created = false;
|
|
7182
7940
|
adopted = true;
|
|
7183
7941
|
} else {
|
|
7184
|
-
const project = options.project?.trim() ||
|
|
7942
|
+
const project = options.project?.trim() || path19.basename(location.projectRoot) || "workspace";
|
|
7185
7943
|
const traceDirs = detectedExistingTraces ? ["runs", "."] : ["runs"];
|
|
7186
7944
|
manifest = createDefaultWorkspaceManifest({
|
|
7187
7945
|
project,
|
|
@@ -7320,7 +8078,7 @@ async function doctorWorkspace(location) {
|
|
|
7320
8078
|
const abs = resolveInsideWorkspace(location.workspaceDir, rel);
|
|
7321
8079
|
for (const file of await listJsonl(abs)) {
|
|
7322
8080
|
try {
|
|
7323
|
-
const s = await stat(
|
|
8081
|
+
const s = await stat(path19.join(abs, file));
|
|
7324
8082
|
newestTraceMtime = Math.max(newestTraceMtime, s.mtimeMs);
|
|
7325
8083
|
} catch {
|
|
7326
8084
|
checks2.push({ id: "trace-readability", status: "warn", message: `cannot stat ${rel}/${file}` });
|
|
@@ -7368,7 +8126,7 @@ async function cleanWorkspace(location, manifest, options = {}) {
|
|
|
7368
8126
|
const relPath = `${rel}/${entry}`;
|
|
7369
8127
|
removed.push(relPath);
|
|
7370
8128
|
if (!dryRun) {
|
|
7371
|
-
await rm(
|
|
8129
|
+
await rm(path19.join(abs, entry), { recursive: true, force: true });
|
|
7372
8130
|
}
|
|
7373
8131
|
}
|
|
7374
8132
|
}
|
|
@@ -7377,7 +8135,7 @@ async function cleanWorkspace(location, manifest, options = {}) {
|
|
|
7377
8135
|
|
|
7378
8136
|
// packages/viewer/src/workspace-data.ts
|
|
7379
8137
|
async function loadWorkspaceViewerData(options) {
|
|
7380
|
-
const cwd =
|
|
8138
|
+
const cwd = path19.resolve(options.cwd ?? process.cwd());
|
|
7381
8139
|
const location = resolveWorkspaceLocation(cwd);
|
|
7382
8140
|
const manifestRead = await readWorkspaceManifestFile(location);
|
|
7383
8141
|
if (!manifestRead.ok || manifestRead.manifest === void 0) {
|
|
@@ -7390,7 +8148,7 @@ async function loadWorkspaceViewerData(options) {
|
|
|
7390
8148
|
const runs = [];
|
|
7391
8149
|
for (const rel of manifestRead.manifest.traceDirs) {
|
|
7392
8150
|
const traceDir = resolveTraceDir({
|
|
7393
|
-
dir:
|
|
8151
|
+
dir: path19.join(location.workspaceDir, rel)
|
|
7394
8152
|
});
|
|
7395
8153
|
const td = new TraceDirectory({ dir: traceDir });
|
|
7396
8154
|
const files = await td.list();
|
|
@@ -7404,7 +8162,7 @@ async function loadWorkspaceViewerData(options) {
|
|
|
7404
8162
|
runId: meta.runId,
|
|
7405
8163
|
...meta.name !== void 0 ? { name: meta.name } : {},
|
|
7406
8164
|
status: meta.status,
|
|
7407
|
-
file:
|
|
8165
|
+
file: path19.basename(meta.filePath)
|
|
7408
8166
|
});
|
|
7409
8167
|
}
|
|
7410
8168
|
}
|
|
@@ -7415,8 +8173,8 @@ async function loadWorkspaceViewerData(options) {
|
|
|
7415
8173
|
doctor,
|
|
7416
8174
|
runs,
|
|
7417
8175
|
bundleDirs: [
|
|
7418
|
-
|
|
7419
|
-
|
|
8176
|
+
path19.join(location.workspaceDir, manifestRead.manifest.bundlesDir),
|
|
8177
|
+
path19.join(location.workspaceDir, manifestRead.manifest.artifactsDir)
|
|
7420
8178
|
]
|
|
7421
8179
|
};
|
|
7422
8180
|
}
|
|
@@ -7479,7 +8237,7 @@ function createViewerServer(options = {}) {
|
|
|
7479
8237
|
ok: true,
|
|
7480
8238
|
readOnly: true,
|
|
7481
8239
|
mode,
|
|
7482
|
-
traceDir:
|
|
8240
|
+
traceDir: path19.resolve(traceDir)
|
|
7483
8241
|
});
|
|
7484
8242
|
}
|
|
7485
8243
|
if (pathname === "/api/suite" && mode === "suite") {
|
|
@@ -7508,7 +8266,7 @@ function createViewerServer(options = {}) {
|
|
|
7508
8266
|
runId: meta.runId,
|
|
7509
8267
|
name: meta.name,
|
|
7510
8268
|
status: meta.status,
|
|
7511
|
-
file:
|
|
8269
|
+
file: path19.basename(meta.filePath),
|
|
7512
8270
|
startedAt: meta.startedAt,
|
|
7513
8271
|
durationMs: meta.durationMs
|
|
7514
8272
|
}))
|
|
@@ -7625,7 +8383,7 @@ function startViewerServer(options = {}) {
|
|
|
7625
8383
|
resolve({
|
|
7626
8384
|
host,
|
|
7627
8385
|
port: resolvedPort,
|
|
7628
|
-
traceDir:
|
|
8386
|
+
traceDir: path19.resolve(traceDir),
|
|
7629
8387
|
url: `http://${host}:${resolvedPort}/${modeQuery}`,
|
|
7630
8388
|
mode
|
|
7631
8389
|
});
|
|
@@ -8036,10 +8794,10 @@ async function evalRun(input3, options = {}) {
|
|
|
8036
8794
|
diagnostics: []
|
|
8037
8795
|
};
|
|
8038
8796
|
}
|
|
8039
|
-
function evidenceForRun(run,
|
|
8040
|
-
return [{ runId: run.runId, ...
|
|
8797
|
+
function evidenceForRun(run, path30) {
|
|
8798
|
+
return [{ runId: run.runId, ...path30 !== void 0 ? { path: path30 } : {} }];
|
|
8041
8799
|
}
|
|
8042
|
-
function evidenceForEvent(event,
|
|
8800
|
+
function evidenceForEvent(event, path30) {
|
|
8043
8801
|
return [
|
|
8044
8802
|
{
|
|
8045
8803
|
runId: event.runId,
|
|
@@ -8047,7 +8805,7 @@ function evidenceForEvent(event, path29) {
|
|
|
8047
8805
|
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
8048
8806
|
kind: event.kind,
|
|
8049
8807
|
name: event.name,
|
|
8050
|
-
...
|
|
8808
|
+
...path30 !== void 0 ? { path: path30 } : {}
|
|
8051
8809
|
}
|
|
8052
8810
|
];
|
|
8053
8811
|
}
|
|
@@ -8205,9 +8963,9 @@ function collectTextFields(nodes, keys, preferredKinds = []) {
|
|
|
8205
8963
|
function tokenize(text) {
|
|
8206
8964
|
return [...text.toLowerCase().matchAll(/[a-z0-9][a-z0-9'-]{2,}/g)].map((match) => match[0].replace(/^['-]+|['-]+$/g, "")).filter((token) => token.length > 2 && !STOP_WORDS.has(token));
|
|
8207
8965
|
}
|
|
8208
|
-
function firstEvidence(fields, run,
|
|
8966
|
+
function firstEvidence(fields, run, path30) {
|
|
8209
8967
|
const first = fields[0];
|
|
8210
|
-
return first === void 0 ? evidenceForRun(run,
|
|
8968
|
+
return first === void 0 ? evidenceForRun(run, path30) : evidenceForEvent(first.node.event, first.path);
|
|
8211
8969
|
}
|
|
8212
8970
|
function collectSourceIds(nodes, keys) {
|
|
8213
8971
|
const wanted = keySet(keys);
|
|
@@ -8560,273 +9318,55 @@ var checks = {
|
|
|
8560
9318
|
"Answer contained banned unsupported-answer phrasing.",
|
|
8561
9319
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
8562
9320
|
{ bannedPhraseCount: banned.length },
|
|
8563
|
-
{ matchedPhraseCount: matches.length }
|
|
8564
|
-
)
|
|
8565
|
-
] : [];
|
|
8566
|
-
});
|
|
8567
|
-
}
|
|
8568
|
-
};
|
|
8569
|
-
function renderEvalMarkdown(result) {
|
|
8570
|
-
const lines = [
|
|
8571
|
-
`# AgentInspect Eval`,
|
|
8572
|
-
"",
|
|
8573
|
-
`Status: ${result.status}`,
|
|
8574
|
-
`Format: ${result.format}`,
|
|
8575
|
-
...result.runId !== void 0 ? [`Run: ${result.runId}`] : [],
|
|
8576
|
-
`Summary: ${result.summary.passed} passed, ${result.summary.failed} failed, ${result.summary.warnings} warnings, ${result.summary.errors} errors`
|
|
8577
|
-
];
|
|
8578
|
-
if (result.diagnostics.length > 0) {
|
|
8579
|
-
lines.push("", "## Diagnostics");
|
|
8580
|
-
for (const diagnostic3 of result.diagnostics) {
|
|
8581
|
-
lines.push(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
8582
|
-
}
|
|
8583
|
-
}
|
|
8584
|
-
if (result.findings.length > 0) {
|
|
8585
|
-
lines.push("", "## Findings");
|
|
8586
|
-
for (const finding of result.findings) {
|
|
8587
|
-
const path29 = finding.evidence[0]?.path;
|
|
8588
|
-
lines.push(`- ${finding.ruleId}: ${finding.message}${path29 ? ` (${path29})` : ""}`);
|
|
8589
|
-
}
|
|
8590
|
-
}
|
|
8591
|
-
return `${lines.join("\n")}
|
|
8592
|
-
`;
|
|
8593
|
-
}
|
|
8594
|
-
|
|
8595
|
-
// packages/cli/src/eval.ts
|
|
8596
|
-
var CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
8597
|
-
var TS_CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
8598
|
-
function diagnostic2(code, message, severity = "error") {
|
|
8599
|
-
return { code, message, severity };
|
|
8600
|
-
}
|
|
8601
|
-
function errorResult3(code, message, format = "unknown") {
|
|
8602
|
-
return {
|
|
8603
|
-
ok: false,
|
|
8604
|
-
status: "error",
|
|
8605
|
-
format,
|
|
8606
|
-
summary: { passed: 0, failed: 0, warnings: 0, errors: 1 },
|
|
8607
|
-
findings: [],
|
|
8608
|
-
diagnostics: [diagnostic2(code, message)]
|
|
8609
|
-
};
|
|
8610
|
-
}
|
|
8611
|
-
function parseNumber2(value, label) {
|
|
8612
|
-
if (value === void 0) return void 0;
|
|
8613
|
-
const parsed = Number(value);
|
|
8614
|
-
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
8615
|
-
throw new Error(`${label} must be a non-negative number.`);
|
|
8616
|
-
}
|
|
8617
|
-
return parsed;
|
|
8618
|
-
}
|
|
8619
|
-
function asStringArray2(value, label) {
|
|
8620
|
-
if (value === void 0) return void 0;
|
|
8621
|
-
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
8622
|
-
throw new Error(`${label} must be an array of strings.`);
|
|
8623
|
-
}
|
|
8624
|
-
return value;
|
|
8625
|
-
}
|
|
8626
|
-
function asConfig(value) {
|
|
8627
|
-
if (value === void 0 || value === null) return {};
|
|
8628
|
-
if (typeof value !== "object" || Array.isArray(value)) {
|
|
8629
|
-
throw new Error("Config must export an object.");
|
|
8630
|
-
}
|
|
8631
|
-
return value;
|
|
8632
|
-
}
|
|
8633
|
-
async function loadConfig2(configPath) {
|
|
8634
|
-
if (configPath === void 0) return {};
|
|
8635
|
-
const extension = path18.extname(configPath);
|
|
8636
|
-
if (TS_CONFIG_EXTENSIONS2.has(extension)) {
|
|
8637
|
-
throw new Error(
|
|
8638
|
-
"TypeScript eval configs require an explicit precompiled JavaScript config or future --config-loader support."
|
|
8639
|
-
);
|
|
8640
|
-
}
|
|
8641
|
-
if (!CONFIG_EXTENSIONS2.has(extension)) {
|
|
8642
|
-
throw new Error("Unsupported eval config extension. Use .json, .js, .mjs, or .cjs.");
|
|
8643
|
-
}
|
|
8644
|
-
const absolute = path18.resolve(configPath);
|
|
8645
|
-
if (extension === ".json") {
|
|
8646
|
-
const raw = await readFile(absolute, "utf-8");
|
|
8647
|
-
return asConfig(JSON.parse(raw));
|
|
8648
|
-
}
|
|
8649
|
-
const mod = await import(pathToFileURL(absolute).href);
|
|
8650
|
-
return asConfig("default" in mod ? mod.default : mod);
|
|
8651
|
-
}
|
|
8652
|
-
function normalizeConfig2(config) {
|
|
8653
|
-
if (config.eval === void 0) return {};
|
|
8654
|
-
if (typeof config.eval !== "object" || Array.isArray(config.eval)) {
|
|
8655
|
-
throw new Error("eval config must be an object.");
|
|
8656
|
-
}
|
|
8657
|
-
return config.eval;
|
|
8658
|
-
}
|
|
8659
|
-
function maybeAdd(rules, value, makeRule) {
|
|
8660
|
-
if (value !== void 0) rules.push(makeRule(value));
|
|
8661
|
-
}
|
|
8662
|
-
function optionObject(value) {
|
|
8663
|
-
if (value === void 0 || value === false) return void 0;
|
|
8664
|
-
if (value === true) return {};
|
|
8665
|
-
if (typeof value !== "object" || Array.isArray(value)) {
|
|
8666
|
-
throw new Error("Eval heuristic config must be a boolean or object.");
|
|
8667
|
-
}
|
|
8668
|
-
return value;
|
|
8669
|
-
}
|
|
8670
|
-
function sourceIdsFromConfig(value) {
|
|
8671
|
-
if (value === void 0) return void 0;
|
|
8672
|
-
if (Array.isArray(value)) return { ids: asStringArray2(value, "eval.requiredSourceIds") ?? [] };
|
|
8673
|
-
if (typeof value !== "object" || Array.isArray(value)) {
|
|
8674
|
-
throw new Error("eval.requiredSourceIds must be an array or object.");
|
|
8675
|
-
}
|
|
8676
|
-
const ids = asStringArray2(value.ids, "eval.requiredSourceIds.ids") ?? [];
|
|
8677
|
-
return { ids, options: { sourceIdKeys: value.sourceIdKeys } };
|
|
8678
|
-
}
|
|
8679
|
-
function buildRules2(config, options) {
|
|
8680
|
-
const evalConfig = normalizeConfig2(config);
|
|
8681
|
-
const rules = [];
|
|
8682
|
-
if (evalConfig.requireSuccess || options.requireSuccess) rules.push(checks.requireSuccess());
|
|
8683
|
-
const requiredTools = [
|
|
8684
|
-
...asStringArray2(evalConfig.requiredTools, "eval.requiredTools") ?? [],
|
|
8685
|
-
...options.requiredTool ?? []
|
|
8686
|
-
];
|
|
8687
|
-
if (requiredTools.length > 0) rules.push(checks.requiredTools(requiredTools));
|
|
8688
|
-
const forbiddenTools = [
|
|
8689
|
-
...asStringArray2(evalConfig.forbiddenTools, "eval.forbiddenTools") ?? [],
|
|
8690
|
-
...options.forbidTool ?? [],
|
|
8691
|
-
...options.forbiddenTool ?? []
|
|
8692
|
-
];
|
|
8693
|
-
if (forbiddenTools.length > 0) rules.push(checks.forbiddenTools(forbiddenTools));
|
|
8694
|
-
const maxDurationMs = parseNumber2(options.maxDurationMs, "--max-duration-ms") ?? evalConfig.maxDurationMs;
|
|
8695
|
-
maybeAdd(rules, maxDurationMs, checks.maxDurationMs);
|
|
8696
|
-
const maxDepth = parseNumber2(options.maxDepth, "--max-depth") ?? evalConfig.maxDepth;
|
|
8697
|
-
maybeAdd(rules, maxDepth, checks.maxDepth);
|
|
8698
|
-
const maxRetries = parseNumber2(options.maxRetries, "--max-retries") ?? evalConfig.maxRetries;
|
|
8699
|
-
maybeAdd(rules, maxRetries, checks.maxRetries);
|
|
8700
|
-
const maxTotalTokens = parseNumber2(options.maxTotalTokens, "--max-total-tokens") ?? evalConfig.maxTotalTokens;
|
|
8701
|
-
maybeAdd(rules, maxTotalTokens, checks.maxTotalTokens);
|
|
8702
|
-
if (evalConfig.requiredRetrievalBeforeGeneration || options.requireRetrievalBeforeGeneration) {
|
|
8703
|
-
rules.push(checks.requiredRetrievalBeforeGeneration());
|
|
8704
|
-
}
|
|
8705
|
-
const requiredDecisionMetadata = [
|
|
8706
|
-
...asStringArray2(
|
|
8707
|
-
evalConfig.requiredDecisionMetadata,
|
|
8708
|
-
"eval.requiredDecisionMetadata"
|
|
8709
|
-
) ?? [],
|
|
8710
|
-
...options.requiredDecisionMetadata ?? []
|
|
8711
|
-
];
|
|
8712
|
-
if (requiredDecisionMetadata.length > 0) {
|
|
8713
|
-
rules.push(checks.requiredDecisionMetadata(requiredDecisionMetadata));
|
|
8714
|
-
}
|
|
8715
|
-
const contextOverlap = optionObject(evalConfig.contextOverlap);
|
|
8716
|
-
const minOverlap = parseNumber2(options.minContextOverlap, "--min-context-overlap");
|
|
8717
|
-
const minSharedTerms = parseNumber2(options.minSharedTerms, "--min-shared-terms");
|
|
8718
|
-
if (contextOverlap !== void 0 || options.contextOverlap || minOverlap !== void 0 || minSharedTerms !== void 0) {
|
|
8719
|
-
rules.push(checks.contextOverlap({ ...contextOverlap, minOverlap, minSharedTerms }));
|
|
8720
|
-
}
|
|
8721
|
-
const quoteOverlap = optionObject(evalConfig.quoteOverlap);
|
|
8722
|
-
if (quoteOverlap !== void 0 || options.quoteOverlap) {
|
|
8723
|
-
rules.push(checks.quoteOverlap(quoteOverlap));
|
|
8724
|
-
}
|
|
8725
|
-
const citationPresence = optionObject(evalConfig.citationPresence);
|
|
8726
|
-
if (citationPresence !== void 0 || options.citationPresence) {
|
|
8727
|
-
rules.push(checks.citationPresence(citationPresence));
|
|
8728
|
-
}
|
|
8729
|
-
const sourceIds = sourceIdsFromConfig(evalConfig.requiredSourceIds);
|
|
8730
|
-
const requiredSourceIds = [...sourceIds?.ids ?? [], ...options.requiredSourceId ?? []];
|
|
8731
|
-
if (requiredSourceIds.length > 0) {
|
|
8732
|
-
rules.push(checks.requiredSourceIds(requiredSourceIds, sourceIds?.options));
|
|
8733
|
-
}
|
|
8734
|
-
const cliAnswerLength = {
|
|
8735
|
-
minCharacters: parseNumber2(options.minAnswerCharacters, "--min-answer-characters"),
|
|
8736
|
-
maxCharacters: parseNumber2(options.maxAnswerCharacters, "--max-answer-characters"),
|
|
8737
|
-
minWords: parseNumber2(options.minAnswerWords, "--min-answer-words"),
|
|
8738
|
-
maxWords: parseNumber2(options.maxAnswerWords, "--max-answer-words")
|
|
8739
|
-
};
|
|
8740
|
-
const hasCliAnswerLength = Object.values(cliAnswerLength).some((value) => value !== void 0);
|
|
8741
|
-
if (evalConfig.answerLengthBounds !== void 0 || hasCliAnswerLength) {
|
|
8742
|
-
rules.push(checks.answerLengthBounds({ ...evalConfig.answerLengthBounds, ...cliAnswerLength }));
|
|
8743
|
-
}
|
|
8744
|
-
const bannedPhrases = [
|
|
8745
|
-
...asStringArray2(
|
|
8746
|
-
evalConfig.bannedUnsupportedPhrases,
|
|
8747
|
-
"eval.bannedUnsupportedPhrases"
|
|
8748
|
-
) ?? [],
|
|
8749
|
-
...options.bannedPhrase ?? []
|
|
8750
|
-
];
|
|
8751
|
-
if (bannedPhrases.length > 0) rules.push(checks.bannedUnsupportedPhrases(bannedPhrases));
|
|
8752
|
-
return rules.length > 0 ? rules : void 0;
|
|
8753
|
-
}
|
|
8754
|
-
function exitCodeFor2(result) {
|
|
8755
|
-
if (result.status === "pass") return 0;
|
|
8756
|
-
if (result.status === "fail") return 1;
|
|
8757
|
-
return 2;
|
|
8758
|
-
}
|
|
8759
|
-
function stable3(value) {
|
|
8760
|
-
if (Array.isArray(value)) return value.map(stable3);
|
|
8761
|
-
if (value === null || typeof value !== "object") return value;
|
|
8762
|
-
const record = value;
|
|
8763
|
-
return Object.fromEntries(
|
|
8764
|
-
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable3(record[key])])
|
|
8765
|
-
);
|
|
8766
|
-
}
|
|
8767
|
-
function printJson2(result) {
|
|
8768
|
-
console.log(JSON.stringify(stable3(result), null, 2));
|
|
8769
|
-
}
|
|
8770
|
-
function printHuman2(result) {
|
|
8771
|
-
console.log(`Eval status: ${result.status}`);
|
|
8772
|
-
console.log(`Format: ${result.format}`);
|
|
8773
|
-
if (result.runId !== void 0) console.log(`Run: ${result.runId}`);
|
|
8774
|
-
console.log(
|
|
8775
|
-
`Summary: ${result.summary.failed} failed, ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
8776
|
-
);
|
|
8777
|
-
for (const diagnostic3 of result.diagnostics) {
|
|
8778
|
-
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
8779
|
-
}
|
|
8780
|
-
for (const finding of result.findings) {
|
|
8781
|
-
const path29 = finding.evidence[0]?.path;
|
|
8782
|
-
console.log(`- ${finding.ruleId}: ${finding.message}${path29 ? ` (${path29})` : ""}`);
|
|
8783
|
-
}
|
|
8784
|
-
}
|
|
8785
|
-
function readErrorResult2(error) {
|
|
8786
|
-
if (error instanceof TraceReadError) {
|
|
8787
|
-
const code = error.code === "unsupported_format" ? "AI_EVAL_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_EVAL_AMBIGUOUS_FORMAT" : "AI_EVAL_TRACE_UNREADABLE";
|
|
8788
|
-
return errorResult3(code, error.message);
|
|
8789
|
-
}
|
|
8790
|
-
return errorResult3(
|
|
8791
|
-
"AI_EVAL_TRACE_UNREADABLE",
|
|
8792
|
-
error instanceof Error ? error.message : String(error)
|
|
8793
|
-
);
|
|
8794
|
-
}
|
|
8795
|
-
async function evalCommand(target, options = {}, stdin = process.stdin) {
|
|
8796
|
-
let result;
|
|
8797
|
-
let phase = "config";
|
|
8798
|
-
try {
|
|
8799
|
-
const config = await loadConfig2(options.config);
|
|
8800
|
-
const rules = buildRules2(config, options);
|
|
8801
|
-
phase = "read";
|
|
8802
|
-
const input3 = await inputFromTarget(target, options, stdin);
|
|
8803
|
-
const read = await openTrace(input3, {
|
|
8804
|
-
...options.format !== void 0 ? { format: options.format } : {}
|
|
8805
|
-
});
|
|
8806
|
-
result = await evalRun(read, {
|
|
8807
|
-
...rules !== void 0 ? { checks: rules } : {},
|
|
8808
|
-
...options.run !== void 0 ? { runId: options.run } : {}
|
|
9321
|
+
{ matchedPhraseCount: matches.length }
|
|
9322
|
+
)
|
|
9323
|
+
] : [];
|
|
8809
9324
|
});
|
|
8810
|
-
}
|
|
8811
|
-
|
|
8812
|
-
|
|
8813
|
-
|
|
8814
|
-
|
|
8815
|
-
|
|
8816
|
-
|
|
9325
|
+
}
|
|
9326
|
+
};
|
|
9327
|
+
function renderEvalMarkdown(result) {
|
|
9328
|
+
const lines = [
|
|
9329
|
+
`# AgentInspect Eval`,
|
|
9330
|
+
"",
|
|
9331
|
+
`Status: ${result.status}`,
|
|
9332
|
+
`Format: ${result.format}`,
|
|
9333
|
+
...result.runId !== void 0 ? [`Run: ${result.runId}`] : [],
|
|
9334
|
+
`Summary: ${result.summary.passed} passed, ${result.summary.failed} failed, ${result.summary.warnings} warnings, ${result.summary.errors} errors`
|
|
9335
|
+
];
|
|
9336
|
+
if (result.diagnostics.length > 0) {
|
|
9337
|
+
lines.push("", "## Diagnostics");
|
|
9338
|
+
for (const diagnostic3 of result.diagnostics) {
|
|
9339
|
+
lines.push(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
8817
9340
|
}
|
|
8818
9341
|
}
|
|
8819
|
-
|
|
8820
|
-
|
|
8821
|
-
|
|
8822
|
-
|
|
9342
|
+
if (result.findings.length > 0) {
|
|
9343
|
+
lines.push("", "## Findings");
|
|
9344
|
+
for (const finding of result.findings) {
|
|
9345
|
+
const path30 = finding.evidence[0]?.path;
|
|
9346
|
+
lines.push(`- ${finding.ruleId}: ${finding.message}${path30 ? ` (${path30})` : ""}`);
|
|
9347
|
+
}
|
|
9348
|
+
}
|
|
9349
|
+
return `${lines.join("\n")}
|
|
9350
|
+
`;
|
|
8823
9351
|
}
|
|
8824
|
-
|
|
8825
|
-
|
|
8826
|
-
var
|
|
8827
|
-
var
|
|
8828
|
-
|
|
8829
|
-
|
|
9352
|
+
|
|
9353
|
+
// packages/cli/src/eval.ts
|
|
9354
|
+
var CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
9355
|
+
var TS_CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
9356
|
+
function diagnostic2(code, message, severity = "error") {
|
|
9357
|
+
return { code, message, severity };
|
|
9358
|
+
}
|
|
9359
|
+
function errorResult3(code, message, format = "unknown") {
|
|
9360
|
+
return {
|
|
9361
|
+
ok: false,
|
|
9362
|
+
status: "error",
|
|
9363
|
+
format,
|
|
9364
|
+
summary: { passed: 0, failed: 0, warnings: 0, errors: 1 },
|
|
9365
|
+
findings: [],
|
|
9366
|
+
diagnostics: [diagnostic2(code, message)]
|
|
9367
|
+
};
|
|
9368
|
+
}
|
|
9369
|
+
function parseNumber2(value, label) {
|
|
8830
9370
|
if (value === void 0) return void 0;
|
|
8831
9371
|
const parsed = Number(value);
|
|
8832
9372
|
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
@@ -8834,368 +9374,210 @@ function parseLimit3(value, label) {
|
|
|
8834
9374
|
}
|
|
8835
9375
|
return parsed;
|
|
8836
9376
|
}
|
|
8837
|
-
function
|
|
8838
|
-
if (
|
|
8839
|
-
if (value
|
|
8840
|
-
|
|
8841
|
-
|
|
8842
|
-
|
|
8843
|
-
);
|
|
8844
|
-
}
|
|
8845
|
-
function safetyDiagnostic(code, message, severity = "error") {
|
|
8846
|
-
return { code, message, severity };
|
|
8847
|
-
}
|
|
8848
|
-
function warningDiagnostics(warnings, unsupportedFields) {
|
|
8849
|
-
return [
|
|
8850
|
-
...warnings.map(
|
|
8851
|
-
(warning) => safetyDiagnostic(
|
|
8852
|
-
warning.code,
|
|
8853
|
-
warning.message,
|
|
8854
|
-
warning.severity === "error" ? "error" : "warning"
|
|
8855
|
-
)
|
|
8856
|
-
),
|
|
8857
|
-
...unsupportedFields.map(
|
|
8858
|
-
(field) => safetyDiagnostic(
|
|
8859
|
-
"unsupported_field",
|
|
8860
|
-
`Reader reported unsupported field: ${field}`,
|
|
8861
|
-
"warning"
|
|
8862
|
-
)
|
|
8863
|
-
)
|
|
8864
|
-
];
|
|
9377
|
+
function asStringArray2(value, label) {
|
|
9378
|
+
if (value === void 0) return void 0;
|
|
9379
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
9380
|
+
throw new Error(`${label} must be an array of strings.`);
|
|
9381
|
+
}
|
|
9382
|
+
return value;
|
|
8865
9383
|
}
|
|
8866
|
-
function
|
|
8867
|
-
|
|
9384
|
+
function asConfig(value) {
|
|
9385
|
+
if (value === void 0 || value === null) return {};
|
|
9386
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
9387
|
+
throw new Error("Config must export an object.");
|
|
9388
|
+
}
|
|
9389
|
+
return value;
|
|
8868
9390
|
}
|
|
8869
|
-
function
|
|
8870
|
-
if (
|
|
8871
|
-
|
|
8872
|
-
if (
|
|
8873
|
-
|
|
8874
|
-
|
|
9391
|
+
async function loadConfig2(configPath) {
|
|
9392
|
+
if (configPath === void 0) return {};
|
|
9393
|
+
const extension = path19.extname(configPath);
|
|
9394
|
+
if (TS_CONFIG_EXTENSIONS2.has(extension)) {
|
|
9395
|
+
throw new Error(
|
|
9396
|
+
"TypeScript eval configs require an explicit precompiled JavaScript config or future --config-loader support."
|
|
9397
|
+
);
|
|
9398
|
+
}
|
|
9399
|
+
if (!CONFIG_EXTENSIONS2.has(extension)) {
|
|
9400
|
+
throw new Error("Unsupported eval config extension. Use .json, .js, .mjs, or .cjs.");
|
|
9401
|
+
}
|
|
9402
|
+
const absolute = path19.resolve(configPath);
|
|
9403
|
+
if (extension === ".json") {
|
|
9404
|
+
const raw = await readFile(absolute, "utf-8");
|
|
9405
|
+
return asConfig(JSON.parse(raw));
|
|
9406
|
+
}
|
|
9407
|
+
const mod = await import(pathToFileURL(absolute).href);
|
|
9408
|
+
return asConfig("default" in mod ? mod.default : mod);
|
|
8875
9409
|
}
|
|
8876
|
-
function
|
|
8877
|
-
|
|
8878
|
-
|
|
8879
|
-
|
|
8880
|
-
|
|
8881
|
-
|
|
8882
|
-
return {
|
|
8883
|
-
ok: status === "SAFE" || status === "SAFE WITH WARNINGS",
|
|
8884
|
-
command: parts.command,
|
|
8885
|
-
status,
|
|
8886
|
-
format: parts.format,
|
|
8887
|
-
...parts.runId !== void 0 ? { runId: parts.runId } : {},
|
|
8888
|
-
summary: {
|
|
8889
|
-
findings: findings.length,
|
|
8890
|
-
warnings: diagnostics.filter((item) => item.severity === "warning").length + findings.filter((item) => item.severity === "warning").length,
|
|
8891
|
-
errors: diagnostics.filter((item) => item.severity === "error").length + findings.filter((item) => item.severity === "error").length
|
|
8892
|
-
},
|
|
8893
|
-
findings,
|
|
8894
|
-
diagnostics,
|
|
8895
|
-
warnings,
|
|
8896
|
-
unsupportedFields,
|
|
8897
|
-
note: BEST_EFFORT_NOTE,
|
|
8898
|
-
...parts.sourceAssessment !== void 0 ? { sourceAssessment: parts.sourceAssessment } : {},
|
|
8899
|
-
...parts.artifactAssessment !== void 0 ? { artifactAssessment: parts.artifactAssessment } : {},
|
|
8900
|
-
...parts.redactionSummary !== void 0 ? { redactionSummary: parts.redactionSummary } : {}
|
|
8901
|
-
};
|
|
9410
|
+
function normalizeConfig2(config) {
|
|
9411
|
+
if (config.eval === void 0) return {};
|
|
9412
|
+
if (typeof config.eval !== "object" || Array.isArray(config.eval)) {
|
|
9413
|
+
throw new Error("eval config must be an object.");
|
|
9414
|
+
}
|
|
9415
|
+
return config.eval;
|
|
8902
9416
|
}
|
|
8903
|
-
function
|
|
8904
|
-
|
|
8905
|
-
status: result.status,
|
|
8906
|
-
summary: result.summary,
|
|
8907
|
-
findings: result.findings
|
|
8908
|
-
};
|
|
9417
|
+
function maybeAdd(rules, value, makeRule) {
|
|
9418
|
+
if (value !== void 0) rules.push(makeRule(value));
|
|
8909
9419
|
}
|
|
8910
|
-
function
|
|
8911
|
-
if (
|
|
8912
|
-
|
|
8913
|
-
|
|
8914
|
-
|
|
8915
|
-
format: "unknown",
|
|
8916
|
-
diagnostics: [safetyDiagnostic(code, error.message)],
|
|
8917
|
-
warnings: error.warnings
|
|
8918
|
-
});
|
|
9420
|
+
function optionObject(value) {
|
|
9421
|
+
if (value === void 0 || value === false) return void 0;
|
|
9422
|
+
if (value === true) return {};
|
|
9423
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
9424
|
+
throw new Error("Eval heuristic config must be a boolean or object.");
|
|
8919
9425
|
}
|
|
8920
|
-
return
|
|
8921
|
-
command,
|
|
8922
|
-
format: "unknown",
|
|
8923
|
-
diagnostics: [
|
|
8924
|
-
safetyDiagnostic(
|
|
8925
|
-
"AI_SAFETY_TRACE_UNREADABLE",
|
|
8926
|
-
error instanceof Error ? error.message : String(error)
|
|
8927
|
-
)
|
|
8928
|
-
]
|
|
8929
|
-
});
|
|
9426
|
+
return value;
|
|
8930
9427
|
}
|
|
8931
|
-
function
|
|
8932
|
-
return
|
|
8933
|
-
|
|
8934
|
-
|
|
8935
|
-
|
|
8936
|
-
|
|
8937
|
-
|
|
8938
|
-
|
|
8939
|
-
)
|
|
8940
|
-
]
|
|
8941
|
-
});
|
|
9428
|
+
function sourceIdsFromConfig(value) {
|
|
9429
|
+
if (value === void 0) return void 0;
|
|
9430
|
+
if (Array.isArray(value)) return { ids: asStringArray2(value, "eval.requiredSourceIds") ?? [] };
|
|
9431
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
9432
|
+
throw new Error("eval.requiredSourceIds must be an array or object.");
|
|
9433
|
+
}
|
|
9434
|
+
const ids = asStringArray2(value.ids, "eval.requiredSourceIds.ids") ?? [];
|
|
9435
|
+
return { ids, options: { sourceIdKeys: value.sourceIdKeys } };
|
|
8942
9436
|
}
|
|
8943
|
-
function
|
|
8944
|
-
const
|
|
8945
|
-
const
|
|
8946
|
-
|
|
8947
|
-
const
|
|
8948
|
-
|
|
8949
|
-
|
|
8950
|
-
createSafetyRedactionRule(),
|
|
8951
|
-
createSafetySecretPatternRule(),
|
|
8952
|
-
createSafetyOversizedAttributeRule({
|
|
8953
|
-
maxStringLength: maxStringLength2,
|
|
8954
|
-
maxArrayLength,
|
|
8955
|
-
maxObjectKeys,
|
|
8956
|
-
maxSerializedBytes
|
|
8957
|
-
})
|
|
9437
|
+
function buildRules2(config, options) {
|
|
9438
|
+
const evalConfig = normalizeConfig2(config);
|
|
9439
|
+
const rules = [];
|
|
9440
|
+
if (evalConfig.requireSuccess || options.requireSuccess) rules.push(checks.requireSuccess());
|
|
9441
|
+
const requiredTools = [
|
|
9442
|
+
...asStringArray2(evalConfig.requiredTools, "eval.requiredTools") ?? [],
|
|
9443
|
+
...options.requiredTool ?? []
|
|
8958
9444
|
];
|
|
8959
|
-
|
|
8960
|
-
|
|
8961
|
-
|
|
8962
|
-
|
|
8963
|
-
...
|
|
8964
|
-
|
|
8965
|
-
|
|
8966
|
-
|
|
8967
|
-
|
|
8968
|
-
|
|
8969
|
-
|
|
8970
|
-
|
|
8971
|
-
|
|
8972
|
-
|
|
8973
|
-
|
|
9445
|
+
if (requiredTools.length > 0) rules.push(checks.requiredTools(requiredTools));
|
|
9446
|
+
const forbiddenTools = [
|
|
9447
|
+
...asStringArray2(evalConfig.forbiddenTools, "eval.forbiddenTools") ?? [],
|
|
9448
|
+
...options.forbidTool ?? [],
|
|
9449
|
+
...options.forbiddenTool ?? []
|
|
9450
|
+
];
|
|
9451
|
+
if (forbiddenTools.length > 0) rules.push(checks.forbiddenTools(forbiddenTools));
|
|
9452
|
+
const maxDurationMs = parseNumber2(options.maxDurationMs, "--max-duration-ms") ?? evalConfig.maxDurationMs;
|
|
9453
|
+
maybeAdd(rules, maxDurationMs, checks.maxDurationMs);
|
|
9454
|
+
const maxDepth = parseNumber2(options.maxDepth, "--max-depth") ?? evalConfig.maxDepth;
|
|
9455
|
+
maybeAdd(rules, maxDepth, checks.maxDepth);
|
|
9456
|
+
const maxRetries = parseNumber2(options.maxRetries, "--max-retries") ?? evalConfig.maxRetries;
|
|
9457
|
+
maybeAdd(rules, maxRetries, checks.maxRetries);
|
|
9458
|
+
const maxTotalTokens = parseNumber2(options.maxTotalTokens, "--max-total-tokens") ?? evalConfig.maxTotalTokens;
|
|
9459
|
+
maybeAdd(rules, maxTotalTokens, checks.maxTotalTokens);
|
|
9460
|
+
if (evalConfig.requiredRetrievalBeforeGeneration || options.requireRetrievalBeforeGeneration) {
|
|
9461
|
+
rules.push(checks.requiredRetrievalBeforeGeneration());
|
|
8974
9462
|
}
|
|
8975
|
-
|
|
8976
|
-
|
|
9463
|
+
const requiredDecisionMetadata = [
|
|
9464
|
+
...asStringArray2(
|
|
9465
|
+
evalConfig.requiredDecisionMetadata,
|
|
9466
|
+
"eval.requiredDecisionMetadata"
|
|
9467
|
+
) ?? [],
|
|
9468
|
+
...options.requiredDecisionMetadata ?? []
|
|
9469
|
+
];
|
|
9470
|
+
if (requiredDecisionMetadata.length > 0) {
|
|
9471
|
+
rules.push(checks.requiredDecisionMetadata(requiredDecisionMetadata));
|
|
8977
9472
|
}
|
|
8978
|
-
|
|
8979
|
-
|
|
9473
|
+
const contextOverlap = optionObject(evalConfig.contextOverlap);
|
|
9474
|
+
const minOverlap = parseNumber2(options.minContextOverlap, "--min-context-overlap");
|
|
9475
|
+
const minSharedTerms = parseNumber2(options.minSharedTerms, "--min-shared-terms");
|
|
9476
|
+
if (contextOverlap !== void 0 || options.contextOverlap || minOverlap !== void 0 || minSharedTerms !== void 0) {
|
|
9477
|
+
rules.push(checks.contextOverlap({ ...contextOverlap, minOverlap, minSharedTerms }));
|
|
8980
9478
|
}
|
|
8981
|
-
|
|
8982
|
-
|
|
9479
|
+
const quoteOverlap = optionObject(evalConfig.quoteOverlap);
|
|
9480
|
+
if (quoteOverlap !== void 0 || options.quoteOverlap) {
|
|
9481
|
+
rules.push(checks.quoteOverlap(quoteOverlap));
|
|
8983
9482
|
}
|
|
8984
|
-
|
|
8985
|
-
|
|
8986
|
-
|
|
8987
|
-
const runs = runId === void 0 ? read.runs : read.runs.filter((run) => run.runId === runId);
|
|
8988
|
-
const out = [];
|
|
8989
|
-
for (const run of runs) {
|
|
8990
|
-
for (const node of flattenNodes(run.children)) {
|
|
8991
|
-
const attrs = node.event.attributes;
|
|
8992
|
-
if (attrs === void 0) continue;
|
|
8993
|
-
const result = redact(attrs, { profile: "share" });
|
|
8994
|
-
for (const finding of result.findings) {
|
|
8995
|
-
if (finding.action === "keep") continue;
|
|
8996
|
-
const taxonomy = classifyRedactionDetector(finding.detector);
|
|
8997
|
-
out.push({
|
|
8998
|
-
ruleId: "safety.redactDetector",
|
|
8999
|
-
severity: detectorSeverity(finding),
|
|
9000
|
-
status: finding.severity === "error" ? "fail" : "warning",
|
|
9001
|
-
message: `Redaction detector ${finding.detector} matched ${finding.matchKind} at ${finding.path}.`,
|
|
9002
|
-
expected: "redacted trace content",
|
|
9003
|
-
actual: finding.detector,
|
|
9004
|
-
evidence: [
|
|
9005
|
-
{
|
|
9006
|
-
runId: node.event.runId,
|
|
9007
|
-
eventId: node.event.eventId,
|
|
9008
|
-
...node.event.parentId !== void 0 ? { parentId: node.event.parentId } : {},
|
|
9009
|
-
kind: node.event.kind,
|
|
9010
|
-
name: node.event.name,
|
|
9011
|
-
...node.event.status !== void 0 ? { status: node.event.status } : {},
|
|
9012
|
-
path: `attributes.${finding.path.replace(/^\$\.?/, "")}`
|
|
9013
|
-
}
|
|
9014
|
-
],
|
|
9015
|
-
category: taxonomy.category,
|
|
9016
|
-
confidence: taxonomy.confidence,
|
|
9017
|
-
detector: finding.detector,
|
|
9018
|
-
action: finding.action
|
|
9019
|
-
});
|
|
9020
|
-
}
|
|
9021
|
-
}
|
|
9483
|
+
const citationPresence = optionObject(evalConfig.citationPresence);
|
|
9484
|
+
if (citationPresence !== void 0 || options.citationPresence) {
|
|
9485
|
+
rules.push(checks.citationPresence(citationPresence));
|
|
9022
9486
|
}
|
|
9023
|
-
|
|
9024
|
-
|
|
9025
|
-
|
|
9026
|
-
|
|
9027
|
-
}
|
|
9487
|
+
const sourceIds = sourceIdsFromConfig(evalConfig.requiredSourceIds);
|
|
9488
|
+
const requiredSourceIds = [...sourceIds?.ids ?? [], ...options.requiredSourceId ?? []];
|
|
9489
|
+
if (requiredSourceIds.length > 0) {
|
|
9490
|
+
rules.push(checks.requiredSourceIds(requiredSourceIds, sourceIds?.options));
|
|
9491
|
+
}
|
|
9492
|
+
const cliAnswerLength = {
|
|
9493
|
+
minCharacters: parseNumber2(options.minAnswerCharacters, "--min-answer-characters"),
|
|
9494
|
+
maxCharacters: parseNumber2(options.maxAnswerCharacters, "--max-answer-characters"),
|
|
9495
|
+
minWords: parseNumber2(options.minAnswerWords, "--min-answer-words"),
|
|
9496
|
+
maxWords: parseNumber2(options.maxAnswerWords, "--max-answer-words")
|
|
9497
|
+
};
|
|
9498
|
+
const hasCliAnswerLength = Object.values(cliAnswerLength).some((value) => value !== void 0);
|
|
9499
|
+
if (evalConfig.answerLengthBounds !== void 0 || hasCliAnswerLength) {
|
|
9500
|
+
rules.push(checks.answerLengthBounds({ ...evalConfig.answerLengthBounds, ...cliAnswerLength }));
|
|
9501
|
+
}
|
|
9502
|
+
const bannedPhrases = [
|
|
9503
|
+
...asStringArray2(
|
|
9504
|
+
evalConfig.bannedUnsupportedPhrases,
|
|
9505
|
+
"eval.bannedUnsupportedPhrases"
|
|
9506
|
+
) ?? [],
|
|
9507
|
+
...options.bannedPhrase ?? []
|
|
9508
|
+
];
|
|
9509
|
+
if (bannedPhrases.length > 0) rules.push(checks.bannedUnsupportedPhrases(bannedPhrases));
|
|
9510
|
+
return rules.length > 0 ? rules : void 0;
|
|
9028
9511
|
}
|
|
9029
9512
|
function exitCodeFor3(result) {
|
|
9030
|
-
if (result.status === "
|
|
9031
|
-
if (result.status === "
|
|
9513
|
+
if (result.status === "pass") return 0;
|
|
9514
|
+
if (result.status === "fail") return 1;
|
|
9032
9515
|
return 2;
|
|
9033
9516
|
}
|
|
9034
|
-
function
|
|
9035
|
-
|
|
9036
|
-
|
|
9037
|
-
const
|
|
9038
|
-
|
|
9039
|
-
|
|
9040
|
-
|
|
9041
|
-
return [
|
|
9042
|
-
` Matched: detector=${detector}; path=${path29}; category=${category}`,
|
|
9043
|
-
` Why: ${finding.message}`,
|
|
9044
|
-
` Confidence: ${confidence}`,
|
|
9045
|
-
` Redaction: ${redactionHint}`,
|
|
9046
|
-
` Override: configure a custom redaction/detector rule locally (see docs/SAFETY-POLICY.md); do not weaken defaults globally.`,
|
|
9047
|
-
` Bundle gate: ${blocksBundle ? "blocks share-safe bundle unless --allow-unsafe" : "does not block by itself (warning/info)"}`
|
|
9048
|
-
];
|
|
9517
|
+
function stable4(value) {
|
|
9518
|
+
if (Array.isArray(value)) return value.map(stable4);
|
|
9519
|
+
if (value === null || typeof value !== "object") return value;
|
|
9520
|
+
const record = value;
|
|
9521
|
+
return Object.fromEntries(
|
|
9522
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable4(record[key])])
|
|
9523
|
+
);
|
|
9049
9524
|
}
|
|
9050
|
-
function
|
|
9051
|
-
|
|
9052
|
-
return {
|
|
9053
|
-
ruleId: finding.ruleId,
|
|
9054
|
-
detector: finding.detector ?? finding.ruleId,
|
|
9055
|
-
path: finding.evidence[0]?.path,
|
|
9056
|
-
category: finding.category,
|
|
9057
|
-
confidence: finding.confidence,
|
|
9058
|
-
action: finding.action,
|
|
9059
|
-
blocksBundle: blocks,
|
|
9060
|
-
// Never include matched secret/PII values.
|
|
9061
|
-
message: finding.message
|
|
9062
|
-
};
|
|
9525
|
+
function printJson3(result) {
|
|
9526
|
+
console.log(JSON.stringify(stable4(result), null, 2));
|
|
9063
9527
|
}
|
|
9064
|
-
function printHuman3(result
|
|
9065
|
-
console.log(`
|
|
9528
|
+
function printHuman3(result) {
|
|
9529
|
+
console.log(`Eval status: ${result.status}`);
|
|
9066
9530
|
console.log(`Format: ${result.format}`);
|
|
9067
9531
|
if (result.runId !== void 0) console.log(`Run: ${result.runId}`);
|
|
9068
|
-
if (result.sourceAssessment !== void 0 && result.artifactAssessment !== void 0) {
|
|
9069
|
-
console.log(`Source assessment: ${result.sourceAssessment.status}`);
|
|
9070
|
-
console.log(`Artifact assessment: ${result.artifactAssessment.status}`);
|
|
9071
|
-
if (result.redactionSummary !== void 0) {
|
|
9072
|
-
console.log(
|
|
9073
|
-
`Redaction: profile=${result.redactionSummary.profile}, findings=${result.redactionSummary.findings}`
|
|
9074
|
-
);
|
|
9075
|
-
}
|
|
9076
|
-
}
|
|
9077
9532
|
console.log(
|
|
9078
|
-
`Summary: ${result.summary.
|
|
9533
|
+
`Summary: ${result.summary.failed} failed, ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
9079
9534
|
);
|
|
9080
9535
|
for (const diagnostic3 of result.diagnostics) {
|
|
9081
9536
|
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
9082
9537
|
}
|
|
9083
9538
|
for (const finding of result.findings) {
|
|
9084
|
-
const
|
|
9085
|
-
|
|
9086
|
-
console.log(`- ${finding.ruleId}: ${finding.message}${path29 ? ` (${path29})` : ""}${taxonomy}`);
|
|
9087
|
-
if (explain) {
|
|
9088
|
-
const blocks = finding.severity === "error" || finding.status === "fail";
|
|
9089
|
-
for (const line of explainFinding(finding, blocks)) {
|
|
9090
|
-
console.log(line);
|
|
9091
|
-
}
|
|
9092
|
-
}
|
|
9539
|
+
const path30 = finding.evidence[0]?.path;
|
|
9540
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path30 ? ` (${path30})` : ""}`);
|
|
9093
9541
|
}
|
|
9094
|
-
console.log(`Note: ${result.note}`);
|
|
9095
9542
|
}
|
|
9096
|
-
function
|
|
9097
|
-
|
|
9098
|
-
|
|
9543
|
+
function readErrorResult3(error) {
|
|
9544
|
+
if (error instanceof TraceReadError) {
|
|
9545
|
+
const code = error.code === "unsupported_format" ? "AI_EVAL_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_EVAL_AMBIGUOUS_FORMAT" : "AI_EVAL_TRACE_UNREADABLE";
|
|
9546
|
+
return errorResult3(code, error.message);
|
|
9547
|
+
}
|
|
9548
|
+
return errorResult3(
|
|
9549
|
+
"AI_EVAL_TRACE_UNREADABLE",
|
|
9550
|
+
error instanceof Error ? error.message : String(error)
|
|
9551
|
+
);
|
|
9099
9552
|
}
|
|
9100
|
-
async function
|
|
9553
|
+
async function evalCommand(target, options = {}, stdin = process.stdin) {
|
|
9101
9554
|
let result;
|
|
9555
|
+
let phase = "config";
|
|
9102
9556
|
try {
|
|
9557
|
+
const config = await loadConfig2(options.config);
|
|
9558
|
+
const rules = buildRules2(config, options);
|
|
9559
|
+
phase = "read";
|
|
9103
9560
|
const input3 = await inputFromTarget(target, options, stdin);
|
|
9104
9561
|
const read = await openTrace(input3, {
|
|
9105
9562
|
...options.format !== void 0 ? { format: options.format } : {}
|
|
9106
9563
|
});
|
|
9107
|
-
|
|
9108
|
-
...
|
|
9564
|
+
result = await evalRun(read, {
|
|
9565
|
+
...rules !== void 0 ? { checks: rules } : {},
|
|
9109
9566
|
...options.run !== void 0 ? { runId: options.run } : {}
|
|
9110
9567
|
});
|
|
9111
|
-
|
|
9112
|
-
|
|
9568
|
+
} catch (error) {
|
|
9569
|
+
if (phase === "config") {
|
|
9570
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
9571
|
+
const code = message.startsWith("--") ? "AI_EVAL_INVALID_ARGUMENTS" : error instanceof SyntaxError || message.includes("Unsupported eval config extension") || message.includes("TypeScript eval configs") || message.includes("Config must") || message.includes("eval config") || message.includes("must be an array") ? "AI_EVAL_INVALID_CONFIG" : "AI_EVAL_CONFIG_LOAD_FAILED";
|
|
9572
|
+
result = errorResult3(code, message);
|
|
9113
9573
|
} else {
|
|
9114
|
-
|
|
9115
|
-
const rawContent = input3.type === "string" ? input3.content : input3.type === "file" ? await readFile(input3.path, "utf-8") : void 0;
|
|
9116
|
-
if (rawContent === void 0) {
|
|
9117
|
-
result = {
|
|
9118
|
-
...source,
|
|
9119
|
-
command: "verify-safe",
|
|
9120
|
-
sourceAssessment: layerFromResult(source)
|
|
9121
|
-
};
|
|
9122
|
-
} else {
|
|
9123
|
-
const redacted = redactTraceContent(rawContent, profile);
|
|
9124
|
-
const artifactRead = await openTrace(
|
|
9125
|
-
{ type: "string", content: redacted.content },
|
|
9126
|
-
{ format: options.format ?? "agent-inspect-jsonl" }
|
|
9127
|
-
);
|
|
9128
|
-
const artifact = assessOpenedTrace(artifactRead, {
|
|
9129
|
-
...options,
|
|
9130
|
-
...options.run !== void 0 ? { runId: options.run } : {}
|
|
9131
|
-
});
|
|
9132
|
-
const detectors = [
|
|
9133
|
-
...new Set(redacted.findings.map((finding) => finding.detector))
|
|
9134
|
-
].sort((a, b) => a.localeCompare(b));
|
|
9135
|
-
result = resultFromParts({
|
|
9136
|
-
command: "verify-safe",
|
|
9137
|
-
format: artifact.format,
|
|
9138
|
-
runId: artifact.runId ?? source.runId,
|
|
9139
|
-
findings: artifact.findings,
|
|
9140
|
-
diagnostics: artifact.diagnostics,
|
|
9141
|
-
warnings: artifact.warnings,
|
|
9142
|
-
unsupportedFields: artifact.unsupportedFields,
|
|
9143
|
-
status: artifact.status,
|
|
9144
|
-
sourceAssessment: layerFromResult(source),
|
|
9145
|
-
artifactAssessment: layerFromResult(artifact),
|
|
9146
|
-
redactionSummary: {
|
|
9147
|
-
profile,
|
|
9148
|
-
findings: redacted.findings.length,
|
|
9149
|
-
detectors
|
|
9150
|
-
}
|
|
9151
|
-
});
|
|
9152
|
-
}
|
|
9574
|
+
result = readErrorResult3(error);
|
|
9153
9575
|
}
|
|
9154
|
-
} catch (error) {
|
|
9155
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
9156
|
-
result = message.startsWith("--") ? invalidArgumentResult(command, error) : readErrorResult3(command, error);
|
|
9157
9576
|
}
|
|
9158
9577
|
process.exitCode = exitCodeFor3(result);
|
|
9159
|
-
if (options.json) printJson3(result
|
|
9160
|
-
else
|
|
9161
|
-
|
|
9162
|
-
function scanCommand(target, options = {}, stdin = process.stdin) {
|
|
9163
|
-
return safetyCommand("scan", target, options, stdin);
|
|
9164
|
-
}
|
|
9165
|
-
function verifySafeCommand(target, options = {}, stdin = process.stdin) {
|
|
9166
|
-
return safetyCommand("verify-safe", target, options, stdin);
|
|
9167
|
-
}
|
|
9168
|
-
function assessOpenedTrace(read, options = {}) {
|
|
9169
|
-
try {
|
|
9170
|
-
const rules = buildSafetyRules(options);
|
|
9171
|
-
const checkResult = runTraceChecks(
|
|
9172
|
-
{ read },
|
|
9173
|
-
{
|
|
9174
|
-
rules,
|
|
9175
|
-
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
9176
|
-
...options.run !== void 0 ? { runId: options.run } : {}
|
|
9177
|
-
}
|
|
9178
|
-
);
|
|
9179
|
-
const detectorFindings = checkResult.diagnostics.length === 0 ? redactionDetectorFindings(read, checkResult.runId) : [];
|
|
9180
|
-
return resultFromParts({
|
|
9181
|
-
command: "verify-safe",
|
|
9182
|
-
format: checkResult.format,
|
|
9183
|
-
runId: checkResult.runId,
|
|
9184
|
-
findings: [...checkResult.findings, ...detectorFindings],
|
|
9185
|
-
diagnostics: [
|
|
9186
|
-
...checkResult.diagnostics.map(diagnosticFromCheck),
|
|
9187
|
-
...warningDiagnostics(read.warnings, read.unsupportedFields)
|
|
9188
|
-
],
|
|
9189
|
-
warnings: read.warnings,
|
|
9190
|
-
unsupportedFields: read.unsupportedFields
|
|
9191
|
-
});
|
|
9192
|
-
} catch (error) {
|
|
9193
|
-
return messageStartsWithDash(error) ? invalidArgumentResult("verify-safe", error) : readErrorResult3("verify-safe", error);
|
|
9194
|
-
}
|
|
9195
|
-
}
|
|
9196
|
-
function messageStartsWithDash(error) {
|
|
9197
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
9198
|
-
return message.startsWith("--");
|
|
9578
|
+
if (options.json) printJson3(result);
|
|
9579
|
+
else if (options.markdown) console.log(renderEvalMarkdown(result).trimEnd());
|
|
9580
|
+
else printHuman3(result);
|
|
9199
9581
|
}
|
|
9200
9582
|
var NOTE = "Generated locally by AgentInspect. Artifacts are best-effort summaries, not compliance or security certification.";
|
|
9201
9583
|
var SAFETY_RULES = [
|
|
@@ -9270,8 +9652,8 @@ function renderCheckSection(result) {
|
|
|
9270
9652
|
`Diagnostics: ${result.diagnostics.length}`
|
|
9271
9653
|
];
|
|
9272
9654
|
for (const finding of result.findings.slice(0, 10)) {
|
|
9273
|
-
const
|
|
9274
|
-
lines.push(`- ${finding.ruleId}: ${finding.message} (${
|
|
9655
|
+
const path30 = finding.evidence[0]?.path ?? "(run)";
|
|
9656
|
+
lines.push(`- ${finding.ruleId}: ${finding.message} (${path30})`);
|
|
9275
9657
|
}
|
|
9276
9658
|
for (const diagnostic3 of result.diagnostics.slice(0, 10)) {
|
|
9277
9659
|
lines.push(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
@@ -9349,8 +9731,8 @@ function renderHtml(trace, check, diff) {
|
|
|
9349
9731
|
`;
|
|
9350
9732
|
}
|
|
9351
9733
|
async function writeArtifact(outputDir, relativePath, content, files) {
|
|
9352
|
-
const outPath =
|
|
9353
|
-
await mkdir(
|
|
9734
|
+
const outPath = path19.join(outputDir, relativePath);
|
|
9735
|
+
await mkdir(path19.dirname(outPath), { recursive: true });
|
|
9354
9736
|
await writeFile(outPath, content, "utf-8");
|
|
9355
9737
|
files.push(relativePath);
|
|
9356
9738
|
}
|
|
@@ -9379,7 +9761,7 @@ function shouldWriteEvidence(options, status) {
|
|
|
9379
9761
|
return status === "unsafe" || status === "regression" || status === "unknown" || status === "warning";
|
|
9380
9762
|
}
|
|
9381
9763
|
async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
9382
|
-
const outputDir = options.outputDir !== void 0 && options.outputDir.trim() !== "" ?
|
|
9764
|
+
const outputDir = options.outputDir !== void 0 && options.outputDir.trim() !== "" ? path19.resolve(options.outputDir.trim()) : "";
|
|
9383
9765
|
if (outputDir === "") {
|
|
9384
9766
|
console.error("--output-dir is required.");
|
|
9385
9767
|
process.exitCode = 1;
|
|
@@ -9519,8 +9901,8 @@ async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
|
9519
9901
|
}
|
|
9520
9902
|
const summaryTarget = options.githubSummary ?? process.env.GITHUB_STEP_SUMMARY;
|
|
9521
9903
|
if (summaryTarget !== void 0 && summaryTarget.trim() !== "") {
|
|
9522
|
-
await mkdir(
|
|
9523
|
-
await appendFile(
|
|
9904
|
+
await mkdir(path19.dirname(path19.resolve(summaryTarget)), { recursive: true });
|
|
9905
|
+
await appendFile(path19.resolve(summaryTarget), `
|
|
9524
9906
|
${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
9525
9907
|
}
|
|
9526
9908
|
const manifestFiles = [...files, "manifest.json"].sort((a, b) => a.localeCompare(b));
|
|
@@ -9539,10 +9921,10 @@ ${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
|
9539
9921
|
findings: diff?.findings.length ?? 0,
|
|
9540
9922
|
diagnostics: diff?.diagnostics.length ?? 0
|
|
9541
9923
|
},
|
|
9542
|
-
...summaryTarget !== void 0 && summaryTarget.trim() !== "" ? { githubSummary:
|
|
9924
|
+
...summaryTarget !== void 0 && summaryTarget.trim() !== "" ? { githubSummary: path19.resolve(summaryTarget) } : {},
|
|
9543
9925
|
note: NOTE
|
|
9544
9926
|
};
|
|
9545
|
-
await writeFile(
|
|
9927
|
+
await writeFile(path19.join(outputDir, "manifest.json"), writeJson3(manifest), "utf-8");
|
|
9546
9928
|
if (options.json === true) {
|
|
9547
9929
|
console.log(writeJson3(manifest).trimEnd());
|
|
9548
9930
|
} else {
|
|
@@ -9600,8 +9982,8 @@ async function resolveOutputDir(options, runIds, cwd, format) {
|
|
|
9600
9982
|
const location = resolveWorkspaceLocation(cwd);
|
|
9601
9983
|
const manifest = await readWorkspaceManifestFile(location);
|
|
9602
9984
|
if (manifest.ok && manifest.manifest) {
|
|
9603
|
-
const rel =
|
|
9604
|
-
if (!rel.startsWith("..") && !
|
|
9985
|
+
const rel = path19.relative(location.workspaceDir, normalized);
|
|
9986
|
+
if (!rel.startsWith("..") && !path19.isAbsolute(rel)) {
|
|
9605
9987
|
return resolveInsideWorkspace(location.workspaceDir, rel);
|
|
9606
9988
|
}
|
|
9607
9989
|
}
|
|
@@ -9617,7 +9999,7 @@ async function resolveOutputDir(options, runIds, cwd, format) {
|
|
|
9617
9999
|
const label = runIds.length === 1 ? sanitizeBundleRunId(runIds[0]) : `multi-${runIds.length}`;
|
|
9618
10000
|
const base2 = resolveInsideWorkspace(
|
|
9619
10001
|
location.workspaceDir,
|
|
9620
|
-
|
|
10002
|
+
path19.join(manifest.manifest.bundlesDir, `bundle-${label}-${stamp}`)
|
|
9621
10003
|
);
|
|
9622
10004
|
return format === "zip" ? `${base2}.zip` : base2;
|
|
9623
10005
|
}
|
|
@@ -10022,19 +10404,19 @@ async function bundleCommand(runIdArg, options = {}) {
|
|
|
10022
10404
|
let sidecarDir = outputDir;
|
|
10023
10405
|
if (outputDir.toLowerCase().endsWith(".html")) {
|
|
10024
10406
|
htmlPath = outputDir;
|
|
10025
|
-
sidecarDir =
|
|
10407
|
+
sidecarDir = path19.dirname(outputDir);
|
|
10026
10408
|
} else {
|
|
10027
10409
|
await mkdir(outputDir, { recursive: true });
|
|
10028
|
-
htmlPath =
|
|
10410
|
+
htmlPath = path19.join(outputDir, EVIDENCE_HTML_FILENAME);
|
|
10029
10411
|
sidecarDir = outputDir;
|
|
10030
10412
|
}
|
|
10031
10413
|
await mkdir(sidecarDir, { recursive: true });
|
|
10032
10414
|
await writeFile(htmlPath, htmlContent, "utf-8");
|
|
10033
|
-
await writeFile(
|
|
10415
|
+
await writeFile(path19.join(sidecarDir, EVIDENCE_MANIFEST_FILENAME), evidenceJson, "utf-8");
|
|
10034
10416
|
outputPath = htmlPath;
|
|
10035
10417
|
} else if (format === "zip") {
|
|
10036
10418
|
const zipPath = outputDir.toLowerCase().endsWith(".zip") ? outputDir : `${outputDir}.zip`;
|
|
10037
|
-
const zipParent =
|
|
10419
|
+
const zipParent = path19.dirname(zipPath);
|
|
10038
10420
|
await mkdir(zipParent, { recursive: true });
|
|
10039
10421
|
const entries = [
|
|
10040
10422
|
...[...packaged.entries()].map(([relativePath, content]) => ({
|
|
@@ -10050,10 +10432,10 @@ async function bundleCommand(runIdArg, options = {}) {
|
|
|
10050
10432
|
await mkdir(outputDir, { recursive: true });
|
|
10051
10433
|
for (const [relativePath, content] of packaged.entries()) {
|
|
10052
10434
|
const outPath = assertBundlePathContained(outputDir, relativePath);
|
|
10053
|
-
await mkdir(
|
|
10435
|
+
await mkdir(path19.dirname(outPath), { recursive: true });
|
|
10054
10436
|
await writeFile(outPath, content, "utf-8");
|
|
10055
10437
|
}
|
|
10056
|
-
await writeFile(
|
|
10438
|
+
await writeFile(path19.join(outputDir, EVIDENCE_MANIFEST_FILENAME), evidenceJson, "utf-8");
|
|
10057
10439
|
if (!files.includes(EVIDENCE_MANIFEST_FILENAME)) {
|
|
10058
10440
|
files.push(EVIDENCE_MANIFEST_FILENAME);
|
|
10059
10441
|
}
|
|
@@ -10099,7 +10481,7 @@ function writeJson5(value) {
|
|
|
10099
10481
|
`;
|
|
10100
10482
|
}
|
|
10101
10483
|
async function bundleVerifyCommand(targetPath, options = {}) {
|
|
10102
|
-
const root =
|
|
10484
|
+
const root = path19.resolve(targetPath.trim() || ".");
|
|
10103
10485
|
const result = await verifyEvidenceDirectory(root, {
|
|
10104
10486
|
unexpectedFiles: options.unexpected ?? "fail"
|
|
10105
10487
|
});
|
|
@@ -10138,7 +10520,7 @@ function writeJson6(value) {
|
|
|
10138
10520
|
}
|
|
10139
10521
|
async function resolveHtmlPath(root) {
|
|
10140
10522
|
const candidates = [
|
|
10141
|
-
|
|
10523
|
+
path19.join(root, "evidence.html"),
|
|
10142
10524
|
root.toLowerCase().endsWith(".html") ? root : ""
|
|
10143
10525
|
].filter(Boolean);
|
|
10144
10526
|
for (const candidate of candidates) {
|
|
@@ -10153,7 +10535,7 @@ async function resolveHtmlPath(root) {
|
|
|
10153
10535
|
);
|
|
10154
10536
|
}
|
|
10155
10537
|
async function bundleOpenCommand(targetPath, options = {}) {
|
|
10156
|
-
const root =
|
|
10538
|
+
const root = path19.resolve(targetPath.trim() || ".");
|
|
10157
10539
|
try {
|
|
10158
10540
|
await access(root);
|
|
10159
10541
|
} catch {
|
|
@@ -10170,7 +10552,7 @@ async function bundleOpenCommand(targetPath, options = {}) {
|
|
|
10170
10552
|
try {
|
|
10171
10553
|
const info = await stat(root);
|
|
10172
10554
|
if (info.isFile() && root.toLowerCase().endsWith(".html")) {
|
|
10173
|
-
verifyRoot =
|
|
10555
|
+
verifyRoot = path19.dirname(root);
|
|
10174
10556
|
}
|
|
10175
10557
|
} catch {
|
|
10176
10558
|
}
|
|
@@ -10202,7 +10584,7 @@ async function bundleOpenCommand(targetPath, options = {}) {
|
|
|
10202
10584
|
let htmlPath;
|
|
10203
10585
|
try {
|
|
10204
10586
|
htmlPath = await resolveHtmlPath(
|
|
10205
|
-
root.toLowerCase().endsWith(".html") ?
|
|
10587
|
+
root.toLowerCase().endsWith(".html") ? path19.dirname(root) : root
|
|
10206
10588
|
);
|
|
10207
10589
|
if (root.toLowerCase().endsWith(".html")) {
|
|
10208
10590
|
htmlPath = root;
|
|
@@ -10218,7 +10600,7 @@ async function bundleOpenCommand(targetPath, options = {}) {
|
|
|
10218
10600
|
return;
|
|
10219
10601
|
}
|
|
10220
10602
|
try {
|
|
10221
|
-
await readFile(
|
|
10603
|
+
await readFile(path19.join(path19.dirname(htmlPath), "evidence.json"), "utf-8");
|
|
10222
10604
|
} catch {
|
|
10223
10605
|
}
|
|
10224
10606
|
const fileUrl = pathToFileURL(htmlPath).href;
|
|
@@ -10304,24 +10686,24 @@ function resolveTargetPath(client, projectLocal) {
|
|
|
10304
10686
|
if (projectLocal) {
|
|
10305
10687
|
switch (client) {
|
|
10306
10688
|
case "cursor":
|
|
10307
|
-
return
|
|
10689
|
+
return path19.join(".cursor", "mcp.json");
|
|
10308
10690
|
case "claude-code":
|
|
10309
|
-
return
|
|
10691
|
+
return path19.join(".mcp.json");
|
|
10310
10692
|
case "codex":
|
|
10311
|
-
return
|
|
10693
|
+
return path19.join(".codex", "config.toml.json");
|
|
10312
10694
|
case "gemini":
|
|
10313
|
-
return
|
|
10695
|
+
return path19.join(".gemini", "settings.json");
|
|
10314
10696
|
}
|
|
10315
10697
|
}
|
|
10316
10698
|
switch (client) {
|
|
10317
10699
|
case "cursor":
|
|
10318
|
-
return
|
|
10700
|
+
return path19.join("~", ".cursor", "mcp.json");
|
|
10319
10701
|
case "claude-code":
|
|
10320
|
-
return
|
|
10702
|
+
return path19.join("~", ".claude.json");
|
|
10321
10703
|
case "codex":
|
|
10322
|
-
return
|
|
10704
|
+
return path19.join("~", ".codex", "config.toml");
|
|
10323
10705
|
case "gemini":
|
|
10324
|
-
return
|
|
10706
|
+
return path19.join("~", ".gemini", "settings.json");
|
|
10325
10707
|
}
|
|
10326
10708
|
}
|
|
10327
10709
|
async function mcpConfigureCommand(options) {
|
|
@@ -10345,8 +10727,8 @@ async function mcpConfigureCommand(options) {
|
|
|
10345
10727
|
];
|
|
10346
10728
|
let wrote = false;
|
|
10347
10729
|
if (!dryRun && projectLocal) {
|
|
10348
|
-
const abs =
|
|
10349
|
-
await mkdir(
|
|
10730
|
+
const abs = path19.resolve(targetPath);
|
|
10731
|
+
await mkdir(path19.dirname(abs), { recursive: true });
|
|
10350
10732
|
await writeFile(abs, `${JSON.stringify(config, null, 2)}
|
|
10351
10733
|
`, "utf8");
|
|
10352
10734
|
wrote = true;
|
|
@@ -10506,23 +10888,23 @@ function readManifestDocument(value) {
|
|
|
10506
10888
|
};
|
|
10507
10889
|
}
|
|
10508
10890
|
function cwdRelative(filePath) {
|
|
10509
|
-
const relative =
|
|
10510
|
-
if (relative === "" || relative.startsWith("../") ||
|
|
10511
|
-
return
|
|
10891
|
+
const relative = path19.relative(process.cwd(), path19.resolve(filePath)).replace(/\\/g, "/");
|
|
10892
|
+
if (relative === "" || relative.startsWith("../") || path19.isAbsolute(relative)) {
|
|
10893
|
+
return path19.basename(filePath);
|
|
10512
10894
|
}
|
|
10513
10895
|
return relative;
|
|
10514
10896
|
}
|
|
10515
10897
|
async function readReporterManifest(filePath) {
|
|
10516
|
-
const absolute =
|
|
10898
|
+
const absolute = path19.resolve(filePath);
|
|
10517
10899
|
const raw = await readFile(absolute, "utf-8");
|
|
10518
10900
|
const document = readManifestDocument(JSON.parse(raw));
|
|
10519
10901
|
const manifest = document.manifest;
|
|
10520
10902
|
const results = manifest.results.map((result) => ({
|
|
10521
10903
|
testId: safeText(result.testId),
|
|
10522
10904
|
name: safeText(result.name),
|
|
10523
|
-
...result.file === void 0 ? {} : { file: safeText(
|
|
10905
|
+
...result.file === void 0 ? {} : { file: safeText(path19.basename(result.file)) },
|
|
10524
10906
|
status: result.status,
|
|
10525
|
-
...result.tracePath === void 0 ? {} : { tracePath: safeText(
|
|
10907
|
+
...result.tracePath === void 0 ? {} : { tracePath: safeText(path19.basename(result.tracePath)) },
|
|
10526
10908
|
artifacts: result.artifacts,
|
|
10527
10909
|
diagnostics: result.diagnostics
|
|
10528
10910
|
}));
|
|
@@ -10661,15 +11043,15 @@ async function ciSummaryCommand(manifestPaths, options = {}) {
|
|
|
10661
11043
|
return;
|
|
10662
11044
|
}
|
|
10663
11045
|
const markdown = renderMarkdown2(result);
|
|
10664
|
-
const outputPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
11046
|
+
const outputPath = options.output !== void 0 && options.output.trim() !== "" ? path19.resolve(options.output.trim()) : void 0;
|
|
10665
11047
|
if (outputPath !== void 0) {
|
|
10666
|
-
await mkdir(
|
|
11048
|
+
await mkdir(path19.dirname(outputPath), { recursive: true });
|
|
10667
11049
|
await writeFile(outputPath, markdown, "utf-8");
|
|
10668
11050
|
}
|
|
10669
11051
|
const summaryTarget = options.githubSummary ?? process.env.GITHUB_STEP_SUMMARY;
|
|
10670
11052
|
if (summaryTarget !== void 0 && summaryTarget.trim() !== "") {
|
|
10671
|
-
const summaryPath =
|
|
10672
|
-
await mkdir(
|
|
11053
|
+
const summaryPath = path19.resolve(summaryTarget);
|
|
11054
|
+
await mkdir(path19.dirname(summaryPath), { recursive: true });
|
|
10673
11055
|
await appendFile(summaryPath, `
|
|
10674
11056
|
${markdown}`, "utf-8");
|
|
10675
11057
|
}
|
|
@@ -10854,7 +11236,7 @@ jobs:
|
|
|
10854
11236
|
}
|
|
10855
11237
|
async function planInit(options = {}) {
|
|
10856
11238
|
const framework = normalizeFramework(options.framework);
|
|
10857
|
-
const cwd =
|
|
11239
|
+
const cwd = path19.resolve(options.cwd ?? process.cwd());
|
|
10858
11240
|
const demoPath = framework === "custom" ? "examples/agent-inspect-demo.mjs" : `examples/agent-inspect-${framework}-demo.mjs`;
|
|
10859
11241
|
const candidates = [
|
|
10860
11242
|
{ rel: CONFIG_FILE, content: configTemplate(framework) },
|
|
@@ -10869,7 +11251,7 @@ async function planInit(options = {}) {
|
|
|
10869
11251
|
}
|
|
10870
11252
|
const files = [];
|
|
10871
11253
|
for (const candidate of candidates) {
|
|
10872
|
-
const abs =
|
|
11254
|
+
const abs = path19.join(cwd, candidate.rel);
|
|
10873
11255
|
try {
|
|
10874
11256
|
await access(abs);
|
|
10875
11257
|
files.push({
|
|
@@ -10890,12 +11272,12 @@ async function writePlannedFiles(plan, cwd, options) {
|
|
|
10890
11272
|
if (entry.action === "skip") {
|
|
10891
11273
|
continue;
|
|
10892
11274
|
}
|
|
10893
|
-
const abs =
|
|
11275
|
+
const abs = path19.join(cwd, entry.path);
|
|
10894
11276
|
if (options.dryRun) {
|
|
10895
11277
|
written.push(entry.path);
|
|
10896
11278
|
continue;
|
|
10897
11279
|
}
|
|
10898
|
-
await mkdir(
|
|
11280
|
+
await mkdir(path19.dirname(abs), { recursive: true });
|
|
10899
11281
|
const demoPath = plan.framework === "custom" ? "examples/agent-inspect-demo.mjs" : `examples/agent-inspect-${plan.framework}-demo.mjs`;
|
|
10900
11282
|
const content = entry.path === CONFIG_FILE ? configTemplate(plan.framework) : entry.path === GITKEEP ? "" : entry.path.endsWith(".yml") ? githubWorkflowTemplate(demoPath) : demoTemplate(plan.framework);
|
|
10901
11283
|
await writeFile(abs, content, "utf-8");
|
|
@@ -10904,7 +11286,7 @@ async function writePlannedFiles(plan, cwd, options) {
|
|
|
10904
11286
|
return written;
|
|
10905
11287
|
}
|
|
10906
11288
|
async function initCommand(options = {}) {
|
|
10907
|
-
const cwd =
|
|
11289
|
+
const cwd = path19.resolve(options.cwd ?? process.cwd());
|
|
10908
11290
|
try {
|
|
10909
11291
|
const plan = await planInit({ ...options, cwd });
|
|
10910
11292
|
const toWrite = plan.files.filter((file) => file.action === "create").map((f) => f.path);
|
|
@@ -10953,6 +11335,9 @@ async function initCommand(options = {}) {
|
|
|
10953
11335
|
process.exitCode = 1;
|
|
10954
11336
|
}
|
|
10955
11337
|
}
|
|
11338
|
+
var DOCS_BASE = "https://github.com/rajudandigam/agent-inspect/blob/main/docs";
|
|
11339
|
+
var AGENT_INSPECT_NOT_RESOLVABLE_MESSAGE = "agent-inspect does not resolve from this project. A global CLI install is not enough \u2014 the package must be a dependency here.";
|
|
11340
|
+
var AGENT_INSPECT_NOT_RESOLVABLE_REMEDIATION = `Install it in this project: npm install agent-inspect (or pnpm add agent-inspect). See ${DOCS_BASE}/FIRST-TRACE-IN-5-MINUTES.md`;
|
|
10956
11341
|
var OPTIONAL_PACKAGES = {
|
|
10957
11342
|
custom: [],
|
|
10958
11343
|
"ai-sdk": ["@agent-inspect/ai-sdk"],
|
|
@@ -10967,7 +11352,7 @@ function nodeVersionCheck() {
|
|
|
10967
11352
|
id: "node-version",
|
|
10968
11353
|
status: "fail",
|
|
10969
11354
|
message: `Node ${process2.versions.node} is below the supported minimum (20).`,
|
|
10970
|
-
remediation:
|
|
11355
|
+
remediation: `Upgrade to Node 20 LTS or newer, then reinstall. See ${DOCS_BASE}/GETTING-STARTED.md`,
|
|
10971
11356
|
evidence: process2.versions.node
|
|
10972
11357
|
};
|
|
10973
11358
|
}
|
|
@@ -10996,7 +11381,7 @@ function envCheck(name, optional = true) {
|
|
|
10996
11381
|
};
|
|
10997
11382
|
}
|
|
10998
11383
|
async function traceDirWritable(traceDir) {
|
|
10999
|
-
const resolved =
|
|
11384
|
+
const resolved = path19.resolve(traceDir);
|
|
11000
11385
|
try {
|
|
11001
11386
|
await mkdir(resolved, { recursive: true });
|
|
11002
11387
|
await access(resolved, constants.W_OK);
|
|
@@ -11011,16 +11396,16 @@ async function traceDirWritable(traceDir) {
|
|
|
11011
11396
|
id: "trace-dir-writable",
|
|
11012
11397
|
status: "fail",
|
|
11013
11398
|
message: `Trace directory is not writable: ${resolved}`,
|
|
11014
|
-
remediation:
|
|
11399
|
+
remediation: `Create the directory or set AGENT_INSPECT_TRACE_DIR to a writable path. See ${DOCS_BASE}/GETTING-STARTED.md`,
|
|
11015
11400
|
evidence: error instanceof Error ? error.message : String(error)
|
|
11016
11401
|
};
|
|
11017
11402
|
}
|
|
11018
11403
|
}
|
|
11019
11404
|
function readPackageVersionNearEntry(entryPath, packageName) {
|
|
11020
|
-
let dir =
|
|
11021
|
-
const { root } =
|
|
11405
|
+
let dir = path19.dirname(path19.resolve(entryPath));
|
|
11406
|
+
const { root } = path19.parse(dir);
|
|
11022
11407
|
while (true) {
|
|
11023
|
-
const candidate =
|
|
11408
|
+
const candidate = path19.join(dir, "package.json");
|
|
11024
11409
|
if (existsSync(candidate)) {
|
|
11025
11410
|
try {
|
|
11026
11411
|
const pkg = JSON.parse(readFileSync(candidate, "utf8"));
|
|
@@ -11031,14 +11416,14 @@ function readPackageVersionNearEntry(entryPath, packageName) {
|
|
|
11031
11416
|
}
|
|
11032
11417
|
}
|
|
11033
11418
|
if (dir === root) break;
|
|
11034
|
-
const parent =
|
|
11419
|
+
const parent = path19.dirname(dir);
|
|
11035
11420
|
if (parent === dir) break;
|
|
11036
11421
|
dir = parent;
|
|
11037
11422
|
}
|
|
11038
11423
|
return void 0;
|
|
11039
11424
|
}
|
|
11040
11425
|
function resolveInstalledPackage(cwd, name) {
|
|
11041
|
-
const require2 = createRequire(
|
|
11426
|
+
const require2 = createRequire(path19.join(cwd, "package.json"));
|
|
11042
11427
|
try {
|
|
11043
11428
|
const entry = require2.resolve(name);
|
|
11044
11429
|
let version2;
|
|
@@ -11074,8 +11459,8 @@ function importSmoke(cwd) {
|
|
|
11074
11459
|
results.push({
|
|
11075
11460
|
id: "import-agent-inspect",
|
|
11076
11461
|
status: "warn",
|
|
11077
|
-
message:
|
|
11078
|
-
remediation:
|
|
11462
|
+
message: AGENT_INSPECT_NOT_RESOLVABLE_MESSAGE,
|
|
11463
|
+
remediation: AGENT_INSPECT_NOT_RESOLVABLE_REMEDIATION
|
|
11079
11464
|
});
|
|
11080
11465
|
results.push({
|
|
11081
11466
|
id: "import-agent-inspect-cjs",
|
|
@@ -11124,8 +11509,8 @@ function versionMismatchCheck(cwd) {
|
|
|
11124
11509
|
return {
|
|
11125
11510
|
id: "version-alignment",
|
|
11126
11511
|
status: "warn",
|
|
11127
|
-
message: `CLI version ${version} differs from local agent-inspect@${root.version}.`,
|
|
11128
|
-
remediation:
|
|
11512
|
+
message: `CLI version ${version} differs from local agent-inspect@${root.version}. A packed consumer can hit this when npx uses a cached global CLI.`,
|
|
11513
|
+
remediation: `Align versions with npm install agent-inspect@${version}, or run via the local binary. See ${DOCS_BASE}/KNOWN-ISSUES.md`,
|
|
11129
11514
|
evidence: `cli=${version};local=${root.version}`
|
|
11130
11515
|
};
|
|
11131
11516
|
}
|
|
@@ -11136,7 +11521,7 @@ function versionMismatchCheck(cwd) {
|
|
|
11136
11521
|
};
|
|
11137
11522
|
}
|
|
11138
11523
|
async function runDoctorChecks(options = {}) {
|
|
11139
|
-
const cwd =
|
|
11524
|
+
const cwd = path19.resolve(options.cwd ?? process2.cwd());
|
|
11140
11525
|
const traceDir = options.traceDir?.trim() || process2.env.AGENT_INSPECT_TRACE_DIR?.trim() || ".agent-inspect";
|
|
11141
11526
|
const checks2 = [
|
|
11142
11527
|
nodeVersionCheck(),
|
|
@@ -11376,7 +11761,7 @@ function parsePluginManifest(input3) {
|
|
|
11376
11761
|
};
|
|
11377
11762
|
}
|
|
11378
11763
|
async function readPluginManifestFile(packageDir) {
|
|
11379
|
-
const manifestPath =
|
|
11764
|
+
const manifestPath = path19.join(packageDir, PLUGIN_MANIFEST_FILENAME);
|
|
11380
11765
|
try {
|
|
11381
11766
|
const raw = await readFile(manifestPath, "utf8");
|
|
11382
11767
|
const parsed = parsePluginManifest(JSON.parse(raw));
|
|
@@ -11450,7 +11835,7 @@ function createTraceDirectoryIndexer() {
|
|
|
11450
11835
|
// packages/cli/src/plugins.ts
|
|
11451
11836
|
async function readPackageName(packageDir) {
|
|
11452
11837
|
try {
|
|
11453
|
-
const raw = await readFile(
|
|
11838
|
+
const raw = await readFile(path19.join(packageDir, "package.json"), "utf8");
|
|
11454
11839
|
const parsed = JSON.parse(raw);
|
|
11455
11840
|
return typeof parsed.name === "string" ? parsed.name : void 0;
|
|
11456
11841
|
} catch {
|
|
@@ -11458,7 +11843,7 @@ async function readPackageName(packageDir) {
|
|
|
11458
11843
|
}
|
|
11459
11844
|
}
|
|
11460
11845
|
async function discoverPlugins(cwd = process.cwd()) {
|
|
11461
|
-
const nodeModules =
|
|
11846
|
+
const nodeModules = path19.join(cwd, "node_modules");
|
|
11462
11847
|
const found = [];
|
|
11463
11848
|
let entries;
|
|
11464
11849
|
try {
|
|
@@ -11469,7 +11854,7 @@ async function discoverPlugins(cwd = process.cwd()) {
|
|
|
11469
11854
|
for (const entry of entries.sort()) {
|
|
11470
11855
|
if (entry.startsWith("@")) continue;
|
|
11471
11856
|
if (!isPluginPackageName(entry)) continue;
|
|
11472
|
-
const packageDir =
|
|
11857
|
+
const packageDir = path19.join(nodeModules, entry);
|
|
11473
11858
|
const manifestRead = await readPluginManifestFile(packageDir);
|
|
11474
11859
|
found.push({
|
|
11475
11860
|
packageName: entry,
|
|
@@ -11482,7 +11867,7 @@ async function discoverPlugins(cwd = process.cwd()) {
|
|
|
11482
11867
|
return found;
|
|
11483
11868
|
}
|
|
11484
11869
|
async function validatePluginPackage(packageRef, cwd = process.cwd()) {
|
|
11485
|
-
const packageDir =
|
|
11870
|
+
const packageDir = path19.isAbsolute(packageRef) ? packageRef : path19.join(cwd, "node_modules", packageRef);
|
|
11486
11871
|
const packageName = await readPackageName(packageDir) ?? packageRef;
|
|
11487
11872
|
const errors = [];
|
|
11488
11873
|
const warnings = [];
|
|
@@ -11558,7 +11943,7 @@ async function pluginsValidateCommand(packageRef, cwd) {
|
|
|
11558
11943
|
}
|
|
11559
11944
|
var INDEX_FILENAME = ".agent-inspect-index.json";
|
|
11560
11945
|
function traceIndexPath(traceDir) {
|
|
11561
|
-
return
|
|
11946
|
+
return path19.join(traceDir, INDEX_FILENAME);
|
|
11562
11947
|
}
|
|
11563
11948
|
function parseMaxEntries(raw) {
|
|
11564
11949
|
if (raw === void 0 || raw.trim() === "") return void 0;
|
|
@@ -11671,7 +12056,7 @@ function isModuleNotFound4(e) {
|
|
|
11671
12056
|
}
|
|
11672
12057
|
async function loadIndexSqlite() {
|
|
11673
12058
|
try {
|
|
11674
|
-
return await import('./src-
|
|
12059
|
+
return await import('./src-R6V5EZZH.mjs');
|
|
11675
12060
|
} catch (e) {
|
|
11676
12061
|
if (isModuleNotFound4(e)) {
|
|
11677
12062
|
console.error(
|
|
@@ -11701,7 +12086,7 @@ async function newestTraceMtimeMs2(traceDir) {
|
|
|
11701
12086
|
for (const file of files) {
|
|
11702
12087
|
if (!file.endsWith(".jsonl")) continue;
|
|
11703
12088
|
try {
|
|
11704
|
-
const s = await stat(
|
|
12089
|
+
const s = await stat(path19.join(traceDir, file));
|
|
11705
12090
|
if (s.mtimeMs > newest) newest = s.mtimeMs;
|
|
11706
12091
|
} catch {
|
|
11707
12092
|
}
|
|
@@ -11953,11 +12338,11 @@ function printJson5(value) {
|
|
|
11953
12338
|
console.log(JSON.stringify(value, null, 2));
|
|
11954
12339
|
}
|
|
11955
12340
|
function resolveCwd(options) {
|
|
11956
|
-
return
|
|
12341
|
+
return path19.resolve(options.cwd ?? process.cwd());
|
|
11957
12342
|
}
|
|
11958
12343
|
async function suiteInitCommand(options = {}) {
|
|
11959
12344
|
const cwd = resolveCwd(options);
|
|
11960
|
-
const configPath =
|
|
12345
|
+
const configPath = path19.join(cwd, DEFAULT_CONFIG_FILENAME);
|
|
11961
12346
|
const template = options.template?.trim();
|
|
11962
12347
|
const suiteConfig = template !== void 0 && template !== "" ? resolveSuiteTemplate(template) : defaultSuiteConfigTemplate();
|
|
11963
12348
|
if (options.dryRun) {
|
|
@@ -12076,13 +12461,13 @@ async function suiteListCommand(options = {}) {
|
|
|
12076
12461
|
}
|
|
12077
12462
|
async function writeSuiteArtifact(result, configOutputDir, options) {
|
|
12078
12463
|
const cwd = resolveCwd(options);
|
|
12079
|
-
const outputDir =
|
|
12464
|
+
const outputDir = path19.resolve(
|
|
12080
12465
|
cwd,
|
|
12081
12466
|
options.output ?? configOutputDir ?? DEFAULT_SUITE_ARTIFACTS_DIR
|
|
12082
12467
|
);
|
|
12083
12468
|
await mkdir(outputDir, { recursive: true });
|
|
12084
12469
|
const stamp = result.startedAt.replace(/[:.]/g, "-");
|
|
12085
|
-
const filePath =
|
|
12470
|
+
const filePath = path19.join(outputDir, `${result.suiteName}-${stamp}.json`);
|
|
12086
12471
|
await writeFile(filePath, `${JSON.stringify(result, null, 2)}
|
|
12087
12472
|
`, "utf-8");
|
|
12088
12473
|
return filePath;
|
|
@@ -12140,7 +12525,7 @@ async function loadSuiteResultFromInput(options) {
|
|
|
12140
12525
|
if (options.input === void 0 || options.input.trim() === "") {
|
|
12141
12526
|
throw new Error("Pass --input <suite-run.json> from a prior suite run.");
|
|
12142
12527
|
}
|
|
12143
|
-
const inputPath =
|
|
12528
|
+
const inputPath = path19.resolve(cwd, options.input.trim());
|
|
12144
12529
|
const raw = await readFile(inputPath, "utf-8");
|
|
12145
12530
|
return JSON.parse(raw);
|
|
12146
12531
|
}
|
|
@@ -12180,9 +12565,9 @@ function normalizeMetrics(raw) {
|
|
|
12180
12565
|
}
|
|
12181
12566
|
async function writeArtifacts(result, outputDir) {
|
|
12182
12567
|
await mkdir(outputDir, { recursive: true });
|
|
12183
|
-
const jsonPath =
|
|
12184
|
-
const markdownPath =
|
|
12185
|
-
const htmlPath =
|
|
12568
|
+
const jsonPath = path19.join(outputDir, "cohort-results.json");
|
|
12569
|
+
const markdownPath = path19.join(outputDir, "cohort-summary.md");
|
|
12570
|
+
const htmlPath = path19.join(outputDir, "cohort-report.html");
|
|
12186
12571
|
await writeFile(jsonPath, `${renderCohortReport(result, { format: "json" })}
|
|
12187
12572
|
`, "utf-8");
|
|
12188
12573
|
await writeFile(
|
|
@@ -12211,7 +12596,7 @@ async function cohortCommand(options = {}) {
|
|
|
12211
12596
|
const format = options.format ?? (options.json ? "json" : "markdown");
|
|
12212
12597
|
let artifacts;
|
|
12213
12598
|
if (options.output !== void 0 && options.output.trim() !== "") {
|
|
12214
|
-
artifacts = await writeArtifacts(result,
|
|
12599
|
+
artifacts = await writeArtifacts(result, path19.resolve(options.output.trim()));
|
|
12215
12600
|
}
|
|
12216
12601
|
if (options.json || format === "json") {
|
|
12217
12602
|
console.log(
|
|
@@ -12281,11 +12666,11 @@ function parseDuration2(value) {
|
|
|
12281
12666
|
async function writeArtifacts2(result, outputDir) {
|
|
12282
12667
|
await mkdir(outputDir, { recursive: true });
|
|
12283
12668
|
const paths = {
|
|
12284
|
-
jsonPath:
|
|
12285
|
-
markdownPath:
|
|
12286
|
-
htmlPath:
|
|
12287
|
-
junitPath:
|
|
12288
|
-
githubPath:
|
|
12669
|
+
jsonPath: path19.join(outputDir, "gate-results.json"),
|
|
12670
|
+
markdownPath: path19.join(outputDir, "gate-summary.md"),
|
|
12671
|
+
htmlPath: path19.join(outputDir, "gate-report.html"),
|
|
12672
|
+
junitPath: path19.join(outputDir, "junit.xml"),
|
|
12673
|
+
githubPath: path19.join(outputDir, "github-step-summary.md")
|
|
12289
12674
|
};
|
|
12290
12675
|
await writeFile(
|
|
12291
12676
|
paths.jsonPath,
|
|
@@ -12353,7 +12738,7 @@ async function gateCommand(options = {}) {
|
|
|
12353
12738
|
const result = await runGate(runs, gateOptions);
|
|
12354
12739
|
let artifacts;
|
|
12355
12740
|
if (options.output !== void 0 && options.output.trim() !== "") {
|
|
12356
|
-
artifacts = await writeArtifacts2(result,
|
|
12741
|
+
artifacts = await writeArtifacts2(result, path19.resolve(options.output.trim()));
|
|
12357
12742
|
}
|
|
12358
12743
|
const failed = !result.ok;
|
|
12359
12744
|
if (shouldEmitEvidence(options.evidenceOn, failed)) {
|
|
@@ -12660,7 +13045,7 @@ function createCliProgram() {
|
|
|
12660
13045
|
"openinference-json",
|
|
12661
13046
|
"otlp-json"
|
|
12662
13047
|
])
|
|
12663
|
-
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--explain", "explain each finding (path, confidence, redaction, override, bundle gate)").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").action((target, opts) => {
|
|
13048
|
+
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--explain", "explain each finding (path, confidence, redaction, override, bundle gate)").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").option("--policy <path>", "local JSON redaction policy (extraKeys + bounded patterns)").action((target, opts) => {
|
|
12664
13049
|
runCommand(() => scanCommand(target, opts));
|
|
12665
13050
|
});
|
|
12666
13051
|
program.command("verify-safe").description("Best-effort local trace safety verification").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
@@ -12669,7 +13054,7 @@ function createCliProgram() {
|
|
|
12669
13054
|
"openinference-json",
|
|
12670
13055
|
"otlp-json"
|
|
12671
13056
|
])
|
|
12672
|
-
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--explain", "explain each finding (path, confidence, redaction, override, bundle gate)").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").action((target, opts) => {
|
|
13057
|
+
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--explain", "explain each finding (path, confidence, redaction, override, bundle gate)").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").option("--policy <path>", "local JSON redaction policy (extraKeys + bounded patterns)").action((target, opts) => {
|
|
12673
13058
|
runCommand(() => verifySafeCommand(target, opts));
|
|
12674
13059
|
});
|
|
12675
13060
|
program.command("artifacts").description("Create safe local CI trace artifacts").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").requiredOption("--output-dir <path>", "directory for generated artifacts").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
@@ -12829,7 +13214,10 @@ function createCliProgram() {
|
|
|
12829
13214
|
"--redaction-profile <profile>",
|
|
12830
13215
|
"alias for --profile (canonical spelling)"
|
|
12831
13216
|
).choices(["local", "share", "strict"])
|
|
12832
|
-
).option("-o, --output <path>", "write redacted content to a file").option("--out <path>", "alias for --output").option("--json", "print deterministic JSON wrapper with findings").
|
|
13217
|
+
).option("-o, --output <path>", "write redacted content to a file").option("--out <path>", "alias for --output").option("--json", "print deterministic JSON wrapper with findings").option("--policy <path>", "local JSON redaction policy (extraKeys + bounded patterns)").option(
|
|
13218
|
+
"--fail-on-residual",
|
|
13219
|
+
"exit non-zero when residual safety is UNSAFE or UNKNOWN (opt-in)"
|
|
13220
|
+
).action((target, opts) => {
|
|
12833
13221
|
runCommand(() => redactCommand(target, opts));
|
|
12834
13222
|
});
|
|
12835
13223
|
program.command("explain").description("Explain a local trace with deterministic facts (no provider calls)").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
@@ -13011,9 +13399,9 @@ function isPrimaryModule() {
|
|
|
13011
13399
|
if (!entry) return false;
|
|
13012
13400
|
const selfPath = fileURLToPath(import.meta.url);
|
|
13013
13401
|
try {
|
|
13014
|
-
return realpathSync(
|
|
13402
|
+
return realpathSync(path19.resolve(entry)) === realpathSync(path19.resolve(selfPath));
|
|
13015
13403
|
} catch {
|
|
13016
|
-
return
|
|
13404
|
+
return path19.resolve(entry) === path19.resolve(selfPath);
|
|
13017
13405
|
}
|
|
13018
13406
|
}
|
|
13019
13407
|
if (isPrimaryModule()) {
|