agent-inspect 6.17.8 → 6.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +1 -1
- package/docs/ADAPTERS.md +34 -4
- package/docs/AI-SDK-ADOPTION.md +25 -0
- package/docs/API.md +13 -10
- package/docs/CLI.md +6 -0
- package/docs/COMPARE.md +6 -0
- package/docs/FIRST-TRACE-IN-5-MINUTES.md +3 -1
- package/docs/LIMITATIONS.md +1 -0
- package/docs/OPENAI-AGENTS-LOCAL.md +1 -0
- package/docs/SAFE-TRACE-SHARING.md +2 -2
- package/docs/SUPPORT-LEVELS.md +1 -0
- package/docs/USE-CASES.md +1 -1
- package/docs/VSCODE.md +8 -0
- package/package.json +2 -2
- package/packages/cli/dist/{chunk-25LUDS4T.mjs → chunk-R7Y5SGH5.mjs} +348 -27
- package/packages/cli/dist/chunk-R7Y5SGH5.mjs.map +1 -0
- package/packages/cli/dist/index.cjs +1622 -900
- package/packages/cli/dist/index.cjs.map +1 -1
- package/packages/cli/dist/index.mjs +1235 -850
- package/packages/cli/dist/index.mjs.map +1 -1
- package/packages/cli/dist/{src-2FS6BLR7.mjs → src-T4EZERCX.mjs} +3 -3
- package/packages/cli/dist/{src-2FS6BLR7.mjs.map → src-T4EZERCX.mjs.map} +1 -1
- package/packages/core/dist/advanced.cjs +180 -0
- package/packages/core/dist/advanced.cjs.map +1 -1
- package/packages/core/dist/advanced.d.cts +94 -4
- package/packages/core/dist/advanced.d.ts +94 -4
- package/packages/core/dist/advanced.mjs +177 -2
- package/packages/core/dist/advanced.mjs.map +1 -1
- package/packages/core/dist/checks.cjs +334 -5
- package/packages/core/dist/checks.cjs.map +1 -1
- package/packages/core/dist/checks.d.cts +81 -4
- package/packages/core/dist/checks.d.ts +81 -4
- package/packages/core/dist/checks.mjs +1 -1
- package/packages/core/dist/{chunk-LQC7IED3.mjs → chunk-NABMW5DX.mjs} +336 -8
- package/packages/core/dist/chunk-NABMW5DX.mjs.map +1 -0
- package/packages/core/dist/{index-B9ZGvUVL.d.ts → index-AMAGuO_i.d.ts} +1 -1
- package/packages/core/dist/{index-DWu54Y28.d.cts → index-DCR816Yt.d.cts} +2 -2
- package/packages/core/dist/{index-DlwbVqEs.d.ts → index-DKwuGSe-.d.ts} +2 -2
- package/packages/core/dist/{index-B259NKkH.d.cts → index-vqr-g6ev.d.cts} +1 -1
- package/packages/core/dist/readers.d.cts +1 -1
- package/packages/core/dist/readers.d.ts +1 -1
- package/packages/cli/dist/chunk-25LUDS4T.mjs.map +0 -1
- package/packages/core/dist/chunk-LQC7IED3.mjs.map +0 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createSafetyOversizedAttributeRule, resolveTraceDir, parseDuration, buildSessionIndex, resolveBundleRunIds, getTraceFilePath, openTrace, sha256Hex, collectTraceSchemaVersions, aggregateBundleSafeStatus, bundleFailsOnSafety, bundleRunAssetRelativePath, buildPlaceholderArtifact, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceHtmlShell, buildEvidenceProvenanceViewHtml, EVIDENCE_FORMAT_VERSION, inferEvidenceFileRole, buildEvidenceSafetyViewHtml, buildEvidenceDiffViewHtml, buildEvidenceCircuitViewHtml, buildEvidenceOutcomesViewHtml, buildEvidenceContractsViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceCausalFailureViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceTreeViewHtml, buildEvidenceManifest, serializeEvidenceManifest, buildZipArchive, assertBundlePathContained, enrichSessionRunRecord, normalizeBundleOutputPath, sanitizeBundleRunId, defaultBundleOutputPath, runTraceChecks, assertEvidenceRelativePath, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, TraceReadError, resolveRedactionProfile, applyProfileMetadataCaps, Redactor, flattenTree, escapeHtml, safeString, compactAttributes, stableJson, escapeMarkdown, truncateStringForProfile, extractMetadata, filterTraces, truncateName, buildRunSummary, formatDuration, formatTimestamp, isAgentInspectTrace, filterMetasBySessionScope, aggregateSessionCheckResults, createBaselineRegressionRule, summarizeSemanticParity, buildEvidenceCiPackage, verifyEvidenceDirectory, diffTraceEvents, renderRunDiff, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, formatStepLabel, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, resolveSuiteTemplate, defaultSuiteConfigTemplate, loadSuiteConfig, validateSuiteConfig, runSuite, renderSuiteReport, analyzeCohort, renderCohortReport, gateHasThresholds, runGate, renderGateReport, persistedInspectEventsToTraceEvents, renderStepLine,
|
|
2
|
+
import { createSafetyRawContentRule, createSafetyRedactionRule, createSafetySecretPatternRule, createSafetyOversizedAttributeRule, resolveTraceDir, parseDuration, buildSessionIndex, resolveBundleRunIds, getTraceFilePath, openTrace, sha256Hex, collectTraceSchemaVersions, aggregateBundleSafeStatus, bundleFailsOnSafety, bundleRunAssetRelativePath, buildPlaceholderArtifact, EVIDENCE_HTML_FILENAME, EVIDENCE_MANIFEST_FILENAME, buildBundleMetadata, buildBundleSummaryMarkdown, buildEvidenceHtmlShell, buildEvidenceProvenanceViewHtml, EVIDENCE_FORMAT_VERSION, inferEvidenceFileRole, buildEvidenceSafetyViewHtml, buildEvidenceDiffViewHtml, buildEvidenceCircuitViewHtml, buildEvidenceOutcomesViewHtml, buildEvidenceContractsViewHtml, buildEvidenceToolsLlmViewHtml, buildEvidenceCausalFailureViewHtml, buildEvidenceTimelineViewHtml, buildEvidenceTreeViewHtml, buildEvidenceManifest, serializeEvidenceManifest, buildZipArchive, assertBundlePathContained, enrichSessionRunRecord, normalizeBundleOutputPath, sanitizeBundleRunId, defaultBundleOutputPath, runTraceChecks, assertEvidenceRelativePath, TraceDirectory, loadTraceMetadataList, loadSessionRunRecords, TraceReadError, resolveRedactionProfile, applyProfileMetadataCaps, Redactor, flattenTree, escapeHtml, safeString, compactAttributes, stableJson, escapeMarkdown, truncateStringForProfile, extractMetadata, filterTraces, truncateName, buildRunSummary, formatDuration, formatTimestamp, renderErrorLine, isAgentInspectTrace, filterMetasBySessionScope, aggregateSessionCheckResults, createBaselineRegressionRule, summarizeSemanticParity, buildEvidenceCiPackage, verifyEvidenceDirectory, diffTraceEvents, renderRunDiff, buildRunTimeline, renderTimeline, buildTraceStats, renderTraceStats, parseDurationFilter, searchTraces, formatStepLabel, buildActivitySummary, renderActivitySummaryHuman, buildRunWhatSummary, renderRunWhat, buildLocalExplanation, resolveSuiteTemplate, defaultSuiteConfigTemplate, loadSuiteConfig, validateSuiteConfig, runSuite, renderSuiteReport, analyzeCohort, renderCohortReport, gateHasThresholds, runGate, renderGateReport, persistedInspectEventsToTraceEvents, renderStepLine, getIndent, TreeBuilder, zeroKinds, validateEvent, traceEventToPersistedInspectEvent, isPersistedInspectEvent, createRunStatusRule, createStructureOrphanRule, createStructureCycleRule, createRunDurationRule, createMaxStepDurationRule, createRequireCompletedRule, createStallDetectionRule, createObservedOutcomeRule, createRunDepthRule, createToolUsageRule, createLlmUsageRule, createStructureRelationshipRule, createStructureParallelWidthRule, summarizeObservedOutcomes, extractOutcomesFromTraceEvents, renderObservedOutcomesMarkdown, renderObservedOutcomesHtml, DEFAULT_SUITE_ARTIFACTS_DIR, parseCohortMetricList, COHORT_METRIC_IDS, parseGateList, nanoid, diffRuns, manualTraceEventsToComparableRun } from './chunk-R7Y5SGH5.mjs';
|
|
3
3
|
import { realpathSync, constants as constants$1, existsSync, readFileSync } from 'fs';
|
|
4
|
-
import
|
|
4
|
+
import path19 from 'path';
|
|
5
5
|
import { fileURLToPath, pathToFileURL } from 'url';
|
|
6
6
|
import { Command, Option } from 'commander';
|
|
7
7
|
import { readFile, stat, mkdir, writeFile, unlink, appendFile, access, rm, open, readdir, constants } from 'fs/promises';
|
|
@@ -13,7 +13,7 @@ import { createServer } from 'http';
|
|
|
13
13
|
import { createRequire } from 'module';
|
|
14
14
|
|
|
15
15
|
// package.json
|
|
16
|
-
var version = "6.
|
|
16
|
+
var version = "6.19.0";
|
|
17
17
|
|
|
18
18
|
// packages/cli/src/trace-dir-scale.ts
|
|
19
19
|
var TRACE_COUNT_WARN = 1e3;
|
|
@@ -481,6 +481,16 @@ function filterErrorEvents(events) {
|
|
|
481
481
|
return false;
|
|
482
482
|
});
|
|
483
483
|
}
|
|
484
|
+
function pruneErrorTree(nodes) {
|
|
485
|
+
const pruned = [];
|
|
486
|
+
for (const node of nodes) {
|
|
487
|
+
const children = pruneErrorTree(node.children);
|
|
488
|
+
if (node.status === "error" || children.length > 0) {
|
|
489
|
+
pruned.push({ ...node, children });
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
return pruned;
|
|
493
|
+
}
|
|
484
494
|
async function view(runId, options = {}) {
|
|
485
495
|
try {
|
|
486
496
|
const id = typeof runId === "string" && runId.trim() !== "" ? runId.trim() : "";
|
|
@@ -553,11 +563,44 @@ async function view(runId, options = {}) {
|
|
|
553
563
|
const errEvents = filterErrorEvents(events);
|
|
554
564
|
if (options.json) {
|
|
555
565
|
console.log(JSON.stringify(errEvents, null, 2));
|
|
556
|
-
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
if (errEvents.length === 0) {
|
|
557
569
|
console.log("No errors found in trace");
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
const started2 = events.find(
|
|
573
|
+
(e) => e.event === "run_started"
|
|
574
|
+
);
|
|
575
|
+
const completed2 = events.filter(
|
|
576
|
+
(e) => e.event === "run_completed"
|
|
577
|
+
);
|
|
578
|
+
const last2 = completed2[completed2.length - 1];
|
|
579
|
+
const status2 = last2 ? last2.status : "running";
|
|
580
|
+
const durationLine2 = last2 !== void 0 && Number.isFinite(last2.durationMs) ? formatDuration(last2.durationMs) : "-";
|
|
581
|
+
const startedTs2 = started2 !== void 0 && Number.isFinite(started2.startTime) ? started2.startTime : started2?.timestamp;
|
|
582
|
+
const startedLabel2 = startedTs2 !== void 0 ? formatTimestamp(startedTs2) : "-";
|
|
583
|
+
console.log(`AgentInspect Run: ${started2?.name ?? id}`);
|
|
584
|
+
console.log(`ID: ${id}`);
|
|
585
|
+
console.log(`Status: ${status2}`);
|
|
586
|
+
console.log(`Duration: ${durationLine2}`);
|
|
587
|
+
console.log(`Started: ${startedLabel2}`);
|
|
588
|
+
console.log("");
|
|
589
|
+
console.log("Error Tree:");
|
|
590
|
+
const pruned = pruneErrorTree(buildStepTree(events));
|
|
591
|
+
if (pruned.length === 0) {
|
|
592
|
+
if (last2?.status === "error") {
|
|
593
|
+
console.log(
|
|
594
|
+
renderErrorLine(
|
|
595
|
+
last2.error ?? { message: "Run completed with error status" },
|
|
596
|
+
0
|
|
597
|
+
)
|
|
598
|
+
);
|
|
599
|
+
} else {
|
|
600
|
+
console.log("No error steps recorded");
|
|
601
|
+
}
|
|
558
602
|
} else {
|
|
559
|
-
|
|
560
|
-
console.log(JSON.stringify(errEvents, null, 2));
|
|
603
|
+
printStepTree(pruned, 0, options.verbose === true);
|
|
561
604
|
}
|
|
562
605
|
return;
|
|
563
606
|
}
|
|
@@ -2979,9 +3022,9 @@ Trace directory: ${traceDir}`);
|
|
|
2979
3022
|
process.exitCode = 1;
|
|
2980
3023
|
}
|
|
2981
3024
|
const resolvedOutput = resolveOutputOption(options);
|
|
2982
|
-
const outPath = resolvedOutput !== void 0 ?
|
|
3025
|
+
const outPath = resolvedOutput !== void 0 ? path19.resolve(resolvedOutput) : void 0;
|
|
2983
3026
|
if (outPath !== void 0) {
|
|
2984
|
-
await mkdir(
|
|
3027
|
+
await mkdir(path19.dirname(outPath), { recursive: true });
|
|
2985
3028
|
await writeFile(outPath, result.content, "utf-8");
|
|
2986
3029
|
const vlabel = validation !== void 0 ? validation.ok ? "ok" : "failed" : "skipped";
|
|
2987
3030
|
console.log(`Wrote ${result.fileExtension} export to ${outPath} (validation: ${vlabel})`);
|
|
@@ -3325,7 +3368,7 @@ function indexedToMetadata(row, traceDir) {
|
|
|
3325
3368
|
endedAt: row.endedAt ?? void 0,
|
|
3326
3369
|
durationMs: row.durationMs ?? void 0,
|
|
3327
3370
|
eventCount: 0,
|
|
3328
|
-
filePath:
|
|
3371
|
+
filePath: path19.join(traceDir, row.file),
|
|
3329
3372
|
fileSize: 0,
|
|
3330
3373
|
createdAt: new Date(row.mtimeMs)
|
|
3331
3374
|
};
|
|
@@ -3354,7 +3397,7 @@ async function newestTraceMtimeMs(traceDir) {
|
|
|
3354
3397
|
}
|
|
3355
3398
|
async function loadSessionRuns(traceDir) {
|
|
3356
3399
|
try {
|
|
3357
|
-
const mod = await import('./src-
|
|
3400
|
+
const mod = await import('./src-T4EZERCX.mjs');
|
|
3358
3401
|
const dbPath = mod.resolveIndexDbPath(traceDir);
|
|
3359
3402
|
const status = mod.indexStatus(dbPath);
|
|
3360
3403
|
if (!status.healthy) {
|
|
@@ -3803,9 +3846,9 @@ async function reportCommand(runId, options = {}) {
|
|
|
3803
3846
|
...options.section ? { section: options.section } : {}
|
|
3804
3847
|
});
|
|
3805
3848
|
const resolvedOutput = resolveOutputOption(options);
|
|
3806
|
-
const outPath = resolvedOutput !== void 0 ?
|
|
3849
|
+
const outPath = resolvedOutput !== void 0 ? path19.resolve(resolvedOutput) : void 0;
|
|
3807
3850
|
if (outPath !== void 0) {
|
|
3808
|
-
await mkdir(
|
|
3851
|
+
await mkdir(path19.dirname(outPath), { recursive: true });
|
|
3809
3852
|
await writeFile(outPath, result.content, "utf-8");
|
|
3810
3853
|
console.log(`Wrote ${result.fileExtension} report to ${outPath}`);
|
|
3811
3854
|
}
|
|
@@ -4000,8 +4043,8 @@ function passesLuhn(value) {
|
|
|
4000
4043
|
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
4001
4044
|
var EPOCH_MS_RE = /^1[0-9]{12}$/;
|
|
4002
4045
|
var EMAIL_RE = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi;
|
|
4003
|
-
function pathSuggestsNonCard(
|
|
4004
|
-
const normalized =
|
|
4046
|
+
function pathSuggestsNonCard(path30) {
|
|
4047
|
+
const normalized = path30.toLowerCase().replace(/[^a-z0-9._]/g, "");
|
|
4005
4048
|
return /(^|\.)(tokenusage|usage)(\.|$)/.test(normalized) || /(startedat|endedat|durationms|timestamp|createdat|updatedat)(\.|$)/.test(normalized) || /(^|\.)(runid|traceid|spanid|eventid|sessionid|userid|parentid|requestid|correlationid)(\.|$)/.test(
|
|
4006
4049
|
normalized
|
|
4007
4050
|
) || /(^|\.)ts(\.|$)/.test(normalized);
|
|
@@ -4197,17 +4240,17 @@ function applyRule(rule, value, replacement) {
|
|
|
4197
4240
|
}
|
|
4198
4241
|
return value;
|
|
4199
4242
|
}
|
|
4200
|
-
function childPath(
|
|
4243
|
+
function childPath(path30, key) {
|
|
4201
4244
|
if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)) {
|
|
4202
|
-
return
|
|
4245
|
+
return path30 ? `${path30}.${key}` : key;
|
|
4203
4246
|
}
|
|
4204
|
-
return `${
|
|
4247
|
+
return `${path30 || "$"}[${JSON.stringify(key)}]`;
|
|
4205
4248
|
}
|
|
4206
|
-
function indexPath(
|
|
4207
|
-
return `${
|
|
4249
|
+
function indexPath(path30, index) {
|
|
4250
|
+
return `${path30 || "$"}[${index}]`;
|
|
4208
4251
|
}
|
|
4209
|
-
function makeFinding(
|
|
4210
|
-
return preview === void 0 ? { path:
|
|
4252
|
+
function makeFinding(path30, detector, action, matchKind, severity = "warning", preview) {
|
|
4253
|
+
return preview === void 0 ? { path: path30, detector, action, severity, matchKind } : { path: path30, detector, action, severity, matchKind, preview };
|
|
4211
4254
|
}
|
|
4212
4255
|
function createRedactionProfile(profile = "local") {
|
|
4213
4256
|
switch (profile) {
|
|
@@ -4276,11 +4319,11 @@ var Redactor2 = class {
|
|
|
4276
4319
|
#recordFinding(state, finding) {
|
|
4277
4320
|
if (this.#collectFindings) state.findings.push(finding);
|
|
4278
4321
|
}
|
|
4279
|
-
#redactValue(value, key,
|
|
4322
|
+
#redactValue(value, key, path30, depth, state) {
|
|
4280
4323
|
if (depth > this.#maxDepth) {
|
|
4281
4324
|
this.#recordFinding(
|
|
4282
4325
|
state,
|
|
4283
|
-
makeFinding(
|
|
4326
|
+
makeFinding(path30, "structure.maxDepth", "truncate", "value", "warning")
|
|
4284
4327
|
);
|
|
4285
4328
|
return "[Truncated]";
|
|
4286
4329
|
}
|
|
@@ -4289,19 +4332,19 @@ var Redactor2 = class {
|
|
|
4289
4332
|
if (rule) {
|
|
4290
4333
|
this.#recordFinding(
|
|
4291
4334
|
state,
|
|
4292
|
-
makeFinding(
|
|
4335
|
+
makeFinding(path30, `key.${rule.key}`, actionForRule(rule), "key", "warning")
|
|
4293
4336
|
);
|
|
4294
4337
|
return applyRule(rule, value, this.#replacement);
|
|
4295
4338
|
}
|
|
4296
4339
|
}
|
|
4297
4340
|
for (const detector of this.#detectors) {
|
|
4298
|
-
const detections = detector.detect({ path:
|
|
4341
|
+
const detections = detector.detect({ path: path30, key, value });
|
|
4299
4342
|
for (const detection of detections) {
|
|
4300
4343
|
const action = detection.action ?? "replace";
|
|
4301
4344
|
this.#recordFinding(
|
|
4302
4345
|
state,
|
|
4303
4346
|
makeFinding(
|
|
4304
|
-
|
|
4347
|
+
path30,
|
|
4305
4348
|
detector.id,
|
|
4306
4349
|
action,
|
|
4307
4350
|
detection.matchKind ?? detector.matchKind ?? "custom",
|
|
@@ -4319,7 +4362,7 @@ var Redactor2 = class {
|
|
|
4319
4362
|
const out = [];
|
|
4320
4363
|
state.seen.set(value, out);
|
|
4321
4364
|
value.forEach((item, index) => {
|
|
4322
|
-
out[index] = this.#redactValue(item, void 0, indexPath(
|
|
4365
|
+
out[index] = this.#redactValue(item, void 0, indexPath(path30, index), depth + 1, state);
|
|
4323
4366
|
});
|
|
4324
4367
|
return out;
|
|
4325
4368
|
}
|
|
@@ -4331,7 +4374,7 @@ var Redactor2 = class {
|
|
|
4331
4374
|
out[entryKey] = this.#redactValue(
|
|
4332
4375
|
entryValue,
|
|
4333
4376
|
entryKey,
|
|
4334
|
-
childPath(
|
|
4377
|
+
childPath(path30 === "$" ? "" : path30, entryKey),
|
|
4335
4378
|
depth + 1,
|
|
4336
4379
|
state
|
|
4337
4380
|
);
|
|
@@ -4347,6 +4390,230 @@ function createRedactor(options) {
|
|
|
4347
4390
|
function redact(value, options) {
|
|
4348
4391
|
return createRedactor(options).redact(value);
|
|
4349
4392
|
}
|
|
4393
|
+
var REDACTION_POLICY_LIMITS = {
|
|
4394
|
+
maxExtraKeys: 64,
|
|
4395
|
+
maxKeyLength: 64,
|
|
4396
|
+
maxPatterns: 32,
|
|
4397
|
+
maxPatternLength: 128,
|
|
4398
|
+
maxPatternIdLength: 64,
|
|
4399
|
+
maxTypedQuantifier: 64
|
|
4400
|
+
};
|
|
4401
|
+
var SAFE_TYPED_PATTERN = /^[A-Za-z0-9_@./:=<>\-[\]()|+*?{},\\\s^$]+$/;
|
|
4402
|
+
function isRecord6(value) {
|
|
4403
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4404
|
+
}
|
|
4405
|
+
function escapeRegExp(value) {
|
|
4406
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
4407
|
+
}
|
|
4408
|
+
function rejectRemotePolicyPath(policyPath) {
|
|
4409
|
+
const trimmed = policyPath.trim();
|
|
4410
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(trimmed)) {
|
|
4411
|
+
throw new Error(
|
|
4412
|
+
`--policy must be a local JSON file path (got a URL-like value). Remote policy fetch is not supported.`
|
|
4413
|
+
);
|
|
4414
|
+
}
|
|
4415
|
+
}
|
|
4416
|
+
function parseSeverity(value, label) {
|
|
4417
|
+
if (value === void 0) return "error";
|
|
4418
|
+
if (value === "info" || value === "warning" || value === "error") return value;
|
|
4419
|
+
throw new Error(`${label} severity must be info, warning, or error.`);
|
|
4420
|
+
}
|
|
4421
|
+
function validateExtraKey(key, index) {
|
|
4422
|
+
const label = `extraKeys[${index}]`;
|
|
4423
|
+
if (key.length === 0) throw new Error(`${label} must be a non-empty string.`);
|
|
4424
|
+
if (key.length > REDACTION_POLICY_LIMITS.maxKeyLength) {
|
|
4425
|
+
throw new Error(
|
|
4426
|
+
`${label} exceeds max length ${REDACTION_POLICY_LIMITS.maxKeyLength}.`
|
|
4427
|
+
);
|
|
4428
|
+
}
|
|
4429
|
+
if (!/^[A-Za-z_][A-Za-z0-9_.-]*$/.test(key)) {
|
|
4430
|
+
throw new Error(
|
|
4431
|
+
`${label} must be an identifier-like key (letters, digits, _, ., -).`
|
|
4432
|
+
);
|
|
4433
|
+
}
|
|
4434
|
+
return key;
|
|
4435
|
+
}
|
|
4436
|
+
function assertBoundedTypedPattern(pattern, label) {
|
|
4437
|
+
if (pattern.length === 0) throw new Error(`${label} must be non-empty.`);
|
|
4438
|
+
if (pattern.length > REDACTION_POLICY_LIMITS.maxPatternLength) {
|
|
4439
|
+
throw new Error(
|
|
4440
|
+
`${label} exceeds max length ${REDACTION_POLICY_LIMITS.maxPatternLength}.`
|
|
4441
|
+
);
|
|
4442
|
+
}
|
|
4443
|
+
if (!SAFE_TYPED_PATTERN.test(pattern)) {
|
|
4444
|
+
throw new Error(
|
|
4445
|
+
`${label} contains unsupported characters for bounded typed patterns.`
|
|
4446
|
+
);
|
|
4447
|
+
}
|
|
4448
|
+
if (/\(\?/.test(pattern) || /\\[1-9]/.test(pattern)) {
|
|
4449
|
+
throw new Error(
|
|
4450
|
+
`${label} rejects lookaround, backreferences, and nested quantifiers.`
|
|
4451
|
+
);
|
|
4452
|
+
}
|
|
4453
|
+
if (/\([^)]*[+*{][^)]*\)[+*{]/.test(pattern) || /\[[^\]]*[+*{][^\]]*\][+*{]/.test(pattern)) {
|
|
4454
|
+
throw new Error(`${label} rejects nested quantifiers.`);
|
|
4455
|
+
}
|
|
4456
|
+
if (/\.[*+]/.test(pattern) || /[*+]\{/.test(pattern)) {
|
|
4457
|
+
throw new Error(`${label} rejects unbounded .* / .+ style quantifiers.`);
|
|
4458
|
+
}
|
|
4459
|
+
for (const match of pattern.matchAll(/\{(\d+)(?:,(\d*))?\}/g)) {
|
|
4460
|
+
const min = Number(match[1]);
|
|
4461
|
+
const maxRaw = match[2];
|
|
4462
|
+
const max = maxRaw === void 0 || maxRaw === "" ? min : Number(maxRaw);
|
|
4463
|
+
if (!Number.isFinite(min) || !Number.isFinite(max) || min > REDACTION_POLICY_LIMITS.maxTypedQuantifier || max > REDACTION_POLICY_LIMITS.maxTypedQuantifier) {
|
|
4464
|
+
throw new Error(
|
|
4465
|
+
`${label} quantifiers must be <= ${REDACTION_POLICY_LIMITS.maxTypedQuantifier}.`
|
|
4466
|
+
);
|
|
4467
|
+
}
|
|
4468
|
+
}
|
|
4469
|
+
}
|
|
4470
|
+
function compilePatternDetector(input3, index) {
|
|
4471
|
+
const label = `patterns[${index}]`;
|
|
4472
|
+
if (!isRecord6(input3)) throw new Error(`${label} must be an object.`);
|
|
4473
|
+
const idRaw = input3.id;
|
|
4474
|
+
if (typeof idRaw !== "string" || idRaw.trim() === "") {
|
|
4475
|
+
throw new Error(`${label}.id must be a non-empty string.`);
|
|
4476
|
+
}
|
|
4477
|
+
const id = idRaw.trim();
|
|
4478
|
+
if (id.length > REDACTION_POLICY_LIMITS.maxPatternIdLength) {
|
|
4479
|
+
throw new Error(
|
|
4480
|
+
`${label}.id exceeds max length ${REDACTION_POLICY_LIMITS.maxPatternIdLength}.`
|
|
4481
|
+
);
|
|
4482
|
+
}
|
|
4483
|
+
if (!/^[A-Za-z][A-Za-z0-9._-]*$/.test(id)) {
|
|
4484
|
+
throw new Error(`${label}.id must be an identifier-like detector id.`);
|
|
4485
|
+
}
|
|
4486
|
+
const type = input3.type;
|
|
4487
|
+
if (type !== "literal" && type !== "prefix" && type !== "typed") {
|
|
4488
|
+
throw new Error(`${label}.type must be literal, prefix, or typed.`);
|
|
4489
|
+
}
|
|
4490
|
+
const severity = parseSeverity(input3.severity, label);
|
|
4491
|
+
let source;
|
|
4492
|
+
if (type === "typed") {
|
|
4493
|
+
if (typeof input3.pattern !== "string") {
|
|
4494
|
+
throw new Error(`${label}.pattern must be a string for typed patterns.`);
|
|
4495
|
+
}
|
|
4496
|
+
assertBoundedTypedPattern(input3.pattern, `${label}.pattern`);
|
|
4497
|
+
source = input3.pattern;
|
|
4498
|
+
} else {
|
|
4499
|
+
if (typeof input3.value !== "string") {
|
|
4500
|
+
throw new Error(`${label}.value must be a string for ${type} patterns.`);
|
|
4501
|
+
}
|
|
4502
|
+
if (input3.value.length === 0) {
|
|
4503
|
+
throw new Error(`${label}.value must be non-empty.`);
|
|
4504
|
+
}
|
|
4505
|
+
if (input3.value.length > REDACTION_POLICY_LIMITS.maxPatternLength) {
|
|
4506
|
+
throw new Error(
|
|
4507
|
+
`${label}.value exceeds max length ${REDACTION_POLICY_LIMITS.maxPatternLength}.`
|
|
4508
|
+
);
|
|
4509
|
+
}
|
|
4510
|
+
const escaped = escapeRegExp(input3.value);
|
|
4511
|
+
source = type === "prefix" ? `^${escaped}` : escaped;
|
|
4512
|
+
}
|
|
4513
|
+
let regex;
|
|
4514
|
+
try {
|
|
4515
|
+
regex = new RegExp(source);
|
|
4516
|
+
} catch (error) {
|
|
4517
|
+
throw new Error(
|
|
4518
|
+
`${label} failed to compile: ${error instanceof Error ? error.message : String(error)}`
|
|
4519
|
+
);
|
|
4520
|
+
}
|
|
4521
|
+
return {
|
|
4522
|
+
id: `policy.${id}`,
|
|
4523
|
+
severity,
|
|
4524
|
+
matchKind: "custom",
|
|
4525
|
+
detect({ value }) {
|
|
4526
|
+
if (typeof value !== "string") return [];
|
|
4527
|
+
regex.lastIndex = 0;
|
|
4528
|
+
return regex.test(value) ? [{ action: "replace", severity, matchKind: "custom" }] : [];
|
|
4529
|
+
}
|
|
4530
|
+
};
|
|
4531
|
+
}
|
|
4532
|
+
function compileRedactionPolicy(raw, policyPath) {
|
|
4533
|
+
if (!isRecord6(raw)) {
|
|
4534
|
+
throw new Error(`Redaction policy must be a JSON object (${policyPath}).`);
|
|
4535
|
+
}
|
|
4536
|
+
const diagnostics = [];
|
|
4537
|
+
if (raw.version !== void 0 && raw.version !== 1) {
|
|
4538
|
+
throw new Error(`Unsupported redaction policy version (supported: 1).`);
|
|
4539
|
+
}
|
|
4540
|
+
const extraKeysRaw = raw.extraKeys;
|
|
4541
|
+
const extraKeys = [];
|
|
4542
|
+
if (extraKeysRaw !== void 0) {
|
|
4543
|
+
if (!Array.isArray(extraKeysRaw)) {
|
|
4544
|
+
throw new Error(`extraKeys must be an array of strings.`);
|
|
4545
|
+
}
|
|
4546
|
+
if (extraKeysRaw.length > REDACTION_POLICY_LIMITS.maxExtraKeys) {
|
|
4547
|
+
throw new Error(
|
|
4548
|
+
`extraKeys exceeds max count ${REDACTION_POLICY_LIMITS.maxExtraKeys}.`
|
|
4549
|
+
);
|
|
4550
|
+
}
|
|
4551
|
+
for (let i = 0; i < extraKeysRaw.length; i += 1) {
|
|
4552
|
+
const key = extraKeysRaw[i];
|
|
4553
|
+
if (typeof key !== "string") {
|
|
4554
|
+
throw new Error(`extraKeys[${i}] must be a string.`);
|
|
4555
|
+
}
|
|
4556
|
+
extraKeys.push(validateExtraKey(key, i));
|
|
4557
|
+
}
|
|
4558
|
+
}
|
|
4559
|
+
const patternsRaw = raw.patterns;
|
|
4560
|
+
const detectors = [];
|
|
4561
|
+
if (patternsRaw !== void 0) {
|
|
4562
|
+
if (!Array.isArray(patternsRaw)) {
|
|
4563
|
+
throw new Error(`patterns must be an array.`);
|
|
4564
|
+
}
|
|
4565
|
+
if (patternsRaw.length > REDACTION_POLICY_LIMITS.maxPatterns) {
|
|
4566
|
+
throw new Error(
|
|
4567
|
+
`patterns exceeds max count ${REDACTION_POLICY_LIMITS.maxPatterns}.`
|
|
4568
|
+
);
|
|
4569
|
+
}
|
|
4570
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
4571
|
+
for (let i = 0; i < patternsRaw.length; i += 1) {
|
|
4572
|
+
const detector = compilePatternDetector(
|
|
4573
|
+
patternsRaw[i],
|
|
4574
|
+
i
|
|
4575
|
+
);
|
|
4576
|
+
if (seenIds.has(detector.id)) {
|
|
4577
|
+
throw new Error(`Duplicate pattern id "${detector.id}".`);
|
|
4578
|
+
}
|
|
4579
|
+
seenIds.add(detector.id);
|
|
4580
|
+
detectors.push(detector);
|
|
4581
|
+
}
|
|
4582
|
+
}
|
|
4583
|
+
if (extraKeys.length === 0 && detectors.length === 0) {
|
|
4584
|
+
diagnostics.push({
|
|
4585
|
+
code: "AI_POLICY_EMPTY",
|
|
4586
|
+
message: "Redaction policy contained no extraKeys or patterns."
|
|
4587
|
+
});
|
|
4588
|
+
}
|
|
4589
|
+
return {
|
|
4590
|
+
path: policyPath,
|
|
4591
|
+
extraKeys,
|
|
4592
|
+
detectors,
|
|
4593
|
+
diagnostics
|
|
4594
|
+
};
|
|
4595
|
+
}
|
|
4596
|
+
async function loadRedactionPolicy(policyPath) {
|
|
4597
|
+
rejectRemotePolicyPath(policyPath);
|
|
4598
|
+
const resolved = path19.resolve(policyPath);
|
|
4599
|
+
let text;
|
|
4600
|
+
try {
|
|
4601
|
+
text = await readFile(resolved, "utf-8");
|
|
4602
|
+
} catch (error) {
|
|
4603
|
+
throw new Error(
|
|
4604
|
+
`Failed to read --policy file "${resolved}": ${error instanceof Error ? error.message : String(error)}`
|
|
4605
|
+
);
|
|
4606
|
+
}
|
|
4607
|
+
let parsed;
|
|
4608
|
+
try {
|
|
4609
|
+
parsed = JSON.parse(text);
|
|
4610
|
+
} catch (error) {
|
|
4611
|
+
throw new Error(
|
|
4612
|
+
`Invalid JSON in --policy file "${resolved}": ${error instanceof Error ? error.message : String(error)}`
|
|
4613
|
+
);
|
|
4614
|
+
}
|
|
4615
|
+
return compileRedactionPolicy(parsed, resolved);
|
|
4616
|
+
}
|
|
4350
4617
|
async function readStdin(stdin) {
|
|
4351
4618
|
stdin.setEncoding("utf8");
|
|
4352
4619
|
let content = "";
|
|
@@ -4375,12 +4642,19 @@ async function inputFromTarget(target, options, stdin) {
|
|
|
4375
4642
|
return { type: "file", path: runPath };
|
|
4376
4643
|
}
|
|
4377
4644
|
|
|
4378
|
-
// packages/cli/src/
|
|
4379
|
-
|
|
4380
|
-
|
|
4381
|
-
|
|
4645
|
+
// packages/cli/src/safety.ts
|
|
4646
|
+
var BEST_EFFORT_NOTE = "Best-effort local safety verification only; not a compliance, privacy, security, or regulatory certification.";
|
|
4647
|
+
var DEFAULT_MAX_STRING_LENGTH = 16384;
|
|
4648
|
+
var DEFAULT_MAX_ARRAY_LENGTH = 1e3;
|
|
4649
|
+
var DEFAULT_MAX_OBJECT_KEYS = 200;
|
|
4650
|
+
var DEFAULT_MAX_SERIALIZED_BYTES = 128 * 1024;
|
|
4651
|
+
function parseLimit3(value, label) {
|
|
4652
|
+
if (value === void 0) return void 0;
|
|
4653
|
+
const parsed = Number(value);
|
|
4654
|
+
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
4655
|
+
throw new Error(`${label} must be a non-negative number.`);
|
|
4382
4656
|
}
|
|
4383
|
-
|
|
4657
|
+
return parsed;
|
|
4384
4658
|
}
|
|
4385
4659
|
function stable(value) {
|
|
4386
4660
|
if (Array.isArray(value)) return value.map(stable);
|
|
@@ -4390,103 +4664,585 @@ function stable(value) {
|
|
|
4390
4664
|
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable(record[key])])
|
|
4391
4665
|
);
|
|
4392
4666
|
}
|
|
4393
|
-
function
|
|
4394
|
-
return
|
|
4667
|
+
function safetyDiagnostic(code, message, severity = "error") {
|
|
4668
|
+
return { code, message, severity };
|
|
4395
4669
|
}
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
}
|
|
4414
|
-
return { content: await readFile(runPath, "utf-8"), source: runPath };
|
|
4670
|
+
function warningDiagnostics(warnings, unsupportedFields) {
|
|
4671
|
+
return [
|
|
4672
|
+
...warnings.map(
|
|
4673
|
+
(warning) => safetyDiagnostic(
|
|
4674
|
+
warning.code,
|
|
4675
|
+
warning.message,
|
|
4676
|
+
warning.severity === "error" ? "error" : "warning"
|
|
4677
|
+
)
|
|
4678
|
+
),
|
|
4679
|
+
...unsupportedFields.map(
|
|
4680
|
+
(field) => safetyDiagnostic(
|
|
4681
|
+
"unsupported_field",
|
|
4682
|
+
`Reader reported unsupported field: ${field}`,
|
|
4683
|
+
"warning"
|
|
4684
|
+
)
|
|
4685
|
+
)
|
|
4686
|
+
];
|
|
4415
4687
|
}
|
|
4416
|
-
function
|
|
4417
|
-
|
|
4418
|
-
|
|
4688
|
+
function diagnosticFromCheck(item) {
|
|
4689
|
+
return safetyDiagnostic(item.code, item.message, item.severity);
|
|
4690
|
+
}
|
|
4691
|
+
function statusFrom(findings, diagnostics) {
|
|
4692
|
+
if (diagnostics.some((item) => item.severity === "error")) return "UNKNOWN";
|
|
4693
|
+
if (findings.some((item) => item.severity === "error")) return "UNSAFE";
|
|
4694
|
+
if (diagnostics.some((item) => item.severity === "warning")) return "SAFE WITH WARNINGS";
|
|
4695
|
+
if (findings.some((item) => item.severity === "warning")) return "SAFE WITH WARNINGS";
|
|
4696
|
+
return "SAFE";
|
|
4697
|
+
}
|
|
4698
|
+
function resultFromParts(parts) {
|
|
4699
|
+
const findings = [...parts.findings ?? []];
|
|
4700
|
+
const diagnostics = [...parts.diagnostics ?? []];
|
|
4701
|
+
const warnings = [...parts.warnings ?? []];
|
|
4702
|
+
const unsupportedFields = [...parts.unsupportedFields ?? []];
|
|
4703
|
+
const status = parts.status ?? statusFrom(findings, diagnostics);
|
|
4419
4704
|
return {
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4705
|
+
ok: status === "SAFE" || status === "SAFE WITH WARNINGS",
|
|
4706
|
+
command: parts.command,
|
|
4707
|
+
status,
|
|
4708
|
+
format: parts.format,
|
|
4709
|
+
...parts.runId !== void 0 ? { runId: parts.runId } : {},
|
|
4710
|
+
summary: {
|
|
4711
|
+
findings: findings.length,
|
|
4712
|
+
warnings: diagnostics.filter((item) => item.severity === "warning").length + findings.filter((item) => item.severity === "warning").length,
|
|
4713
|
+
errors: diagnostics.filter((item) => item.severity === "error").length + findings.filter((item) => item.severity === "error").length
|
|
4714
|
+
},
|
|
4715
|
+
findings,
|
|
4716
|
+
diagnostics,
|
|
4717
|
+
warnings,
|
|
4718
|
+
unsupportedFields,
|
|
4719
|
+
note: BEST_EFFORT_NOTE,
|
|
4720
|
+
...parts.sourceAssessment !== void 0 ? { sourceAssessment: parts.sourceAssessment } : {},
|
|
4721
|
+
...parts.artifactAssessment !== void 0 ? { artifactAssessment: parts.artifactAssessment } : {},
|
|
4722
|
+
...parts.redactionSummary !== void 0 ? { redactionSummary: parts.redactionSummary } : {}
|
|
4423
4723
|
};
|
|
4424
4724
|
}
|
|
4425
|
-
function
|
|
4426
|
-
const lines = content.split(/\r?\n/);
|
|
4427
|
-
const out = [];
|
|
4428
|
-
const findings = [];
|
|
4429
|
-
for (let index = 0; index < lines.length; index += 1) {
|
|
4430
|
-
const line = lines[index] ?? "";
|
|
4431
|
-
if (line.trim() === "") continue;
|
|
4432
|
-
let parsed;
|
|
4433
|
-
try {
|
|
4434
|
-
parsed = JSON.parse(line);
|
|
4435
|
-
} catch {
|
|
4436
|
-
throw new Error(`Input is not valid JSON or JSONL at line ${index + 1}.`);
|
|
4437
|
-
}
|
|
4438
|
-
const result = redact(parsed, { profile });
|
|
4439
|
-
out.push(JSON.stringify(result.value));
|
|
4440
|
-
findings.push(
|
|
4441
|
-
...result.findings.map((finding) => ({
|
|
4442
|
-
...finding,
|
|
4443
|
-
path: `line:${index + 1}:${finding.path}`
|
|
4444
|
-
}))
|
|
4445
|
-
);
|
|
4446
|
-
}
|
|
4725
|
+
function layerFromResult(result) {
|
|
4447
4726
|
return {
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
findings
|
|
4727
|
+
status: result.status,
|
|
4728
|
+
summary: result.summary,
|
|
4729
|
+
findings: result.findings
|
|
4451
4730
|
};
|
|
4452
4731
|
}
|
|
4453
|
-
function
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
}
|
|
4462
|
-
}
|
|
4463
|
-
}
|
|
4464
|
-
try {
|
|
4465
|
-
return redactJsonText(content, profile);
|
|
4466
|
-
} catch {
|
|
4467
|
-
return redactJsonlText(content, profile);
|
|
4732
|
+
function readErrorResult(command, error) {
|
|
4733
|
+
if (error instanceof TraceReadError) {
|
|
4734
|
+
const code = error.code === "unsupported_format" ? "AI_SAFETY_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_SAFETY_AMBIGUOUS_FORMAT" : "AI_SAFETY_TRACE_UNREADABLE";
|
|
4735
|
+
return resultFromParts({
|
|
4736
|
+
command,
|
|
4737
|
+
format: "unknown",
|
|
4738
|
+
diagnostics: [safetyDiagnostic(code, error.message)],
|
|
4739
|
+
warnings: error.warnings
|
|
4740
|
+
});
|
|
4468
4741
|
}
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4742
|
+
return resultFromParts({
|
|
4743
|
+
command,
|
|
4744
|
+
format: "unknown",
|
|
4745
|
+
diagnostics: [
|
|
4746
|
+
safetyDiagnostic(
|
|
4747
|
+
"AI_SAFETY_TRACE_UNREADABLE",
|
|
4748
|
+
error instanceof Error ? error.message : String(error)
|
|
4749
|
+
)
|
|
4750
|
+
]
|
|
4751
|
+
});
|
|
4752
|
+
}
|
|
4753
|
+
function invalidArgumentResult(command, error) {
|
|
4754
|
+
return resultFromParts({
|
|
4755
|
+
command,
|
|
4756
|
+
format: "unknown",
|
|
4757
|
+
diagnostics: [
|
|
4758
|
+
safetyDiagnostic(
|
|
4759
|
+
"AI_SAFETY_INVALID_ARGUMENTS",
|
|
4760
|
+
error instanceof Error ? error.message : String(error)
|
|
4761
|
+
)
|
|
4762
|
+
]
|
|
4763
|
+
});
|
|
4764
|
+
}
|
|
4765
|
+
function buildSafetyRules(options) {
|
|
4766
|
+
const maxStringLength2 = parseLimit3(options.maxStringLength, "--max-string-length") ?? DEFAULT_MAX_STRING_LENGTH;
|
|
4767
|
+
const maxArrayLength = parseLimit3(options.maxArrayLength, "--max-array-length") ?? DEFAULT_MAX_ARRAY_LENGTH;
|
|
4768
|
+
const maxObjectKeys = parseLimit3(options.maxObjectKeys, "--max-object-keys") ?? DEFAULT_MAX_OBJECT_KEYS;
|
|
4769
|
+
const maxSerializedBytes = parseLimit3(options.maxSerializedBytes, "--max-serialized-bytes") ?? DEFAULT_MAX_SERIALIZED_BYTES;
|
|
4770
|
+
return [
|
|
4771
|
+
createSafetyRawContentRule(),
|
|
4772
|
+
createSafetyRedactionRule(),
|
|
4773
|
+
createSafetySecretPatternRule(),
|
|
4774
|
+
createSafetyOversizedAttributeRule({
|
|
4775
|
+
maxStringLength: maxStringLength2,
|
|
4776
|
+
maxArrayLength,
|
|
4777
|
+
maxObjectKeys,
|
|
4778
|
+
maxSerializedBytes
|
|
4779
|
+
})
|
|
4780
|
+
];
|
|
4781
|
+
}
|
|
4782
|
+
function flattenNodes(nodes) {
|
|
4783
|
+
return nodes.flatMap((node) => [
|
|
4784
|
+
node,
|
|
4785
|
+
...flattenNodes(
|
|
4786
|
+
node.children
|
|
4787
|
+
)
|
|
4788
|
+
]);
|
|
4789
|
+
}
|
|
4790
|
+
function detectorSeverity(finding) {
|
|
4791
|
+
return finding.severity;
|
|
4792
|
+
}
|
|
4793
|
+
function classifyRedactionDetector(detector) {
|
|
4794
|
+
if (detector === "value.creditCard" || detector === "value.email" || detector === "value.phone") {
|
|
4795
|
+
return { category: "personal-data", confidence: "high" };
|
|
4796
|
+
}
|
|
4797
|
+
if (detector === "value.ipv4" || detector === "value.ipv6") {
|
|
4798
|
+
return { category: "identifier", confidence: "medium" };
|
|
4799
|
+
}
|
|
4800
|
+
if (detector.startsWith("value.") && (detector.includes("Token") || detector.includes("Key") || detector.includes("jwt") || detector.includes("authorization") || detector.includes("bearer") || detector.includes("cookie") || detector.includes("privateKey") || detector.includes("github") || detector.includes("aws") || detector.includes("provider"))) {
|
|
4801
|
+
return { category: "credential", confidence: "high" };
|
|
4802
|
+
}
|
|
4803
|
+
if (detector.startsWith("key.")) {
|
|
4804
|
+
return { category: "credential", confidence: "medium" };
|
|
4805
|
+
}
|
|
4806
|
+
return { category: "credential", confidence: "medium" };
|
|
4807
|
+
}
|
|
4808
|
+
function redactionOptionsFromPolicy(policy) {
|
|
4809
|
+
if (policy === void 0) return {};
|
|
4810
|
+
return {
|
|
4811
|
+
...policy.extraKeys.length > 0 ? { extraKeys: [...policy.extraKeys] } : {},
|
|
4812
|
+
...policy.detectors.length > 0 ? { detectors: [...policy.detectors] } : {}
|
|
4813
|
+
};
|
|
4814
|
+
}
|
|
4815
|
+
function redactionDetectorFindings(read, runId, policy) {
|
|
4816
|
+
const runs = runId === void 0 ? read.runs : read.runs.filter((run) => run.runId === runId);
|
|
4817
|
+
const out = [];
|
|
4818
|
+
const policyOptions = redactionOptionsFromPolicy(policy);
|
|
4819
|
+
for (const run of runs) {
|
|
4820
|
+
for (const node of flattenNodes(run.children)) {
|
|
4821
|
+
const attrs = node.event.attributes;
|
|
4822
|
+
if (attrs === void 0) continue;
|
|
4823
|
+
const result = createRedactor({
|
|
4824
|
+
profile: "share",
|
|
4825
|
+
...policyOptions
|
|
4826
|
+
}).redact(attrs);
|
|
4827
|
+
for (const finding of result.findings) {
|
|
4828
|
+
if (finding.action === "keep") continue;
|
|
4829
|
+
const taxonomy = classifyRedactionDetector(finding.detector);
|
|
4830
|
+
out.push({
|
|
4831
|
+
ruleId: "safety.redactDetector",
|
|
4832
|
+
severity: detectorSeverity(finding),
|
|
4833
|
+
status: finding.severity === "error" ? "fail" : "warning",
|
|
4834
|
+
message: `Redaction detector ${finding.detector} matched ${finding.matchKind} at ${finding.path}.`,
|
|
4835
|
+
expected: "redacted trace content",
|
|
4836
|
+
actual: finding.detector,
|
|
4837
|
+
evidence: [
|
|
4838
|
+
{
|
|
4839
|
+
runId: node.event.runId,
|
|
4840
|
+
eventId: node.event.eventId,
|
|
4841
|
+
...node.event.parentId !== void 0 ? { parentId: node.event.parentId } : {},
|
|
4842
|
+
kind: node.event.kind,
|
|
4843
|
+
name: node.event.name,
|
|
4844
|
+
...node.event.status !== void 0 ? { status: node.event.status } : {},
|
|
4845
|
+
path: `attributes.${finding.path.replace(/^\$\.?/, "")}`
|
|
4846
|
+
}
|
|
4847
|
+
],
|
|
4848
|
+
category: taxonomy.category,
|
|
4849
|
+
confidence: taxonomy.confidence,
|
|
4850
|
+
detector: finding.detector,
|
|
4851
|
+
action: finding.action
|
|
4852
|
+
});
|
|
4853
|
+
}
|
|
4854
|
+
}
|
|
4855
|
+
}
|
|
4856
|
+
return out.sort((a, b) => {
|
|
4857
|
+
const aEvidence = a.evidence[0];
|
|
4858
|
+
const bEvidence = b.evidence[0];
|
|
4859
|
+
return (aEvidence?.runId ?? "").localeCompare(bEvidence?.runId ?? "") || (aEvidence?.eventId ?? "").localeCompare(bEvidence?.eventId ?? "") || (aEvidence?.path ?? "").localeCompare(bEvidence?.path ?? "") || a.message.localeCompare(b.message);
|
|
4860
|
+
});
|
|
4861
|
+
}
|
|
4862
|
+
function exitCodeFor(result) {
|
|
4863
|
+
if (result.status === "SAFE" || result.status === "SAFE WITH WARNINGS") return 0;
|
|
4864
|
+
if (result.status === "UNSAFE") return 1;
|
|
4865
|
+
return 2;
|
|
4866
|
+
}
|
|
4867
|
+
function toResidualStatus(status) {
|
|
4868
|
+
if (status === "SAFE WITH WARNINGS") return "SAFE_WITH_WARNINGS";
|
|
4869
|
+
return status;
|
|
4870
|
+
}
|
|
4871
|
+
function toResidualSafetyAssessment(result) {
|
|
4872
|
+
const codes = [
|
|
4873
|
+
.../* @__PURE__ */ new Set([
|
|
4874
|
+
...result.diagnostics.map((item) => item.code),
|
|
4875
|
+
...result.findings.map((item) => item.ruleId)
|
|
4876
|
+
])
|
|
4877
|
+
].sort((a, b) => a.localeCompare(b));
|
|
4878
|
+
return {
|
|
4879
|
+
status: toResidualStatus(result.status),
|
|
4880
|
+
findings: result.summary.findings,
|
|
4881
|
+
warnings: result.summary.warnings,
|
|
4882
|
+
errors: result.summary.errors,
|
|
4883
|
+
codes,
|
|
4884
|
+
note: BEST_EFFORT_NOTE
|
|
4885
|
+
};
|
|
4886
|
+
}
|
|
4887
|
+
async function assessResidualFromContent(content, options = {}) {
|
|
4888
|
+
try {
|
|
4889
|
+
const policy = options.compiledPolicy ?? (options.policy !== void 0 ? await loadRedactionPolicy(options.policy) : void 0);
|
|
4890
|
+
const read = await openTrace(
|
|
4891
|
+
{ type: "string", content },
|
|
4892
|
+
{ format: options.format ?? "agent-inspect-jsonl" }
|
|
4893
|
+
);
|
|
4894
|
+
const result = assessOpenedTrace(read, {
|
|
4895
|
+
...options,
|
|
4896
|
+
...policy !== void 0 ? { compiledPolicy: policy } : {}
|
|
4897
|
+
});
|
|
4898
|
+
return toResidualSafetyAssessment(result);
|
|
4899
|
+
} catch (error) {
|
|
4900
|
+
if (error instanceof TraceReadError) {
|
|
4901
|
+
return toResidualSafetyAssessment(readErrorResult("verify-safe", error));
|
|
4902
|
+
}
|
|
4903
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4904
|
+
return {
|
|
4905
|
+
status: "UNKNOWN",
|
|
4906
|
+
findings: 0,
|
|
4907
|
+
warnings: 0,
|
|
4908
|
+
errors: 1,
|
|
4909
|
+
codes: ["AI_SAFETY_TRACE_UNREADABLE"],
|
|
4910
|
+
note: `${BEST_EFFORT_NOTE} Residual assessment requires a supported AgentInspect trace; ${message}`
|
|
4911
|
+
};
|
|
4912
|
+
}
|
|
4913
|
+
}
|
|
4914
|
+
function explainFinding(finding, blocksBundle) {
|
|
4915
|
+
const path30 = finding.evidence[0]?.path ?? "(unknown path)";
|
|
4916
|
+
const category = finding.category ?? "structure";
|
|
4917
|
+
const confidence = finding.confidence ?? "medium";
|
|
4918
|
+
const detector = finding.detector ?? finding.ruleId;
|
|
4919
|
+
const action = finding.action ?? "review";
|
|
4920
|
+
const redactionHint = category === "credential" || category === "personal-data" || category === "raw-content" || action.includes("redact") ? "Usually removable by share/strict redaction before bundling." : "May require omitting the field, lowering limits, or an explicit local override.";
|
|
4921
|
+
return [
|
|
4922
|
+
` Matched: detector=${detector}; path=${path30}; category=${category}`,
|
|
4923
|
+
` Why: ${finding.message}`,
|
|
4924
|
+
` Confidence: ${confidence}`,
|
|
4925
|
+
` Redaction: ${redactionHint}`,
|
|
4926
|
+
` Override: configure a custom redaction/detector rule locally (see docs/SAFETY-POLICY.md); do not weaken defaults globally.`,
|
|
4927
|
+
` Bundle gate: ${blocksBundle ? "blocks share-safe bundle unless --allow-unsafe" : "does not block by itself (warning/info)"}`
|
|
4928
|
+
];
|
|
4929
|
+
}
|
|
4930
|
+
function findingExplanation(finding) {
|
|
4931
|
+
const blocks = finding.severity === "error" || finding.status === "fail";
|
|
4932
|
+
return {
|
|
4933
|
+
ruleId: finding.ruleId,
|
|
4934
|
+
detector: finding.detector ?? finding.ruleId,
|
|
4935
|
+
path: finding.evidence[0]?.path,
|
|
4936
|
+
category: finding.category,
|
|
4937
|
+
confidence: finding.confidence,
|
|
4938
|
+
action: finding.action,
|
|
4939
|
+
blocksBundle: blocks,
|
|
4940
|
+
// Never include matched secret/PII values.
|
|
4941
|
+
message: finding.message
|
|
4942
|
+
};
|
|
4943
|
+
}
|
|
4944
|
+
function printHuman(result, explain = false) {
|
|
4945
|
+
console.log(`Safety status: ${result.status}`);
|
|
4946
|
+
console.log(`Format: ${result.format}`);
|
|
4947
|
+
if (result.runId !== void 0) console.log(`Run: ${result.runId}`);
|
|
4948
|
+
if (result.sourceAssessment !== void 0 && result.artifactAssessment !== void 0) {
|
|
4949
|
+
console.log(`Source assessment: ${result.sourceAssessment.status}`);
|
|
4950
|
+
console.log(`Artifact assessment: ${result.artifactAssessment.status}`);
|
|
4951
|
+
if (result.redactionSummary !== void 0) {
|
|
4952
|
+
console.log(
|
|
4953
|
+
`Redaction: profile=${result.redactionSummary.profile}, findings=${result.redactionSummary.findings}`
|
|
4954
|
+
);
|
|
4955
|
+
}
|
|
4956
|
+
}
|
|
4957
|
+
console.log(
|
|
4958
|
+
`Summary: ${result.summary.findings} finding(s), ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
4959
|
+
);
|
|
4960
|
+
for (const diagnostic3 of result.diagnostics) {
|
|
4961
|
+
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
4962
|
+
}
|
|
4963
|
+
for (const finding of result.findings) {
|
|
4964
|
+
const path30 = finding.evidence[0]?.path;
|
|
4965
|
+
const taxonomy = finding.category !== void 0 || finding.confidence !== void 0 ? ` [${[finding.category, finding.confidence].filter(Boolean).join("/")}]` : "";
|
|
4966
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path30 ? ` (${path30})` : ""}${taxonomy}`);
|
|
4967
|
+
if (explain) {
|
|
4968
|
+
const blocks = finding.severity === "error" || finding.status === "fail";
|
|
4969
|
+
for (const line of explainFinding(finding, blocks)) {
|
|
4970
|
+
console.log(line);
|
|
4971
|
+
}
|
|
4972
|
+
}
|
|
4973
|
+
}
|
|
4974
|
+
console.log(`Note: ${result.note}`);
|
|
4975
|
+
}
|
|
4976
|
+
function printJson(result, explain = false) {
|
|
4977
|
+
const payload = explain === true ? { ...result, explanations: result.findings.map(findingExplanation) } : result;
|
|
4978
|
+
console.log(JSON.stringify(stable(payload), null, 2));
|
|
4979
|
+
}
|
|
4980
|
+
async function safetyCommand(command, target, options, stdin) {
|
|
4981
|
+
let result;
|
|
4982
|
+
try {
|
|
4983
|
+
const policy = options.compiledPolicy ?? (options.policy !== void 0 ? await loadRedactionPolicy(options.policy) : void 0);
|
|
4984
|
+
const optionsWithPolicy = {
|
|
4985
|
+
...options,
|
|
4986
|
+
...policy !== void 0 ? { compiledPolicy: policy } : {}
|
|
4987
|
+
};
|
|
4988
|
+
const input3 = await inputFromTarget(target, optionsWithPolicy, stdin);
|
|
4989
|
+
const read = await openTrace(input3, {
|
|
4990
|
+
...options.format !== void 0 ? { format: options.format } : {}
|
|
4991
|
+
});
|
|
4992
|
+
const source = assessOpenedTrace(read, {
|
|
4993
|
+
...optionsWithPolicy,
|
|
4994
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
4995
|
+
});
|
|
4996
|
+
if (command === "scan") {
|
|
4997
|
+
result = { ...source, command: "scan" };
|
|
4998
|
+
} else {
|
|
4999
|
+
const profile = options.redactionProfile ?? "share";
|
|
5000
|
+
const rawContent = input3.type === "string" ? input3.content : input3.type === "file" ? await readFile(input3.path, "utf-8") : void 0;
|
|
5001
|
+
if (rawContent === void 0) {
|
|
5002
|
+
result = {
|
|
5003
|
+
...source,
|
|
5004
|
+
command: "verify-safe",
|
|
5005
|
+
sourceAssessment: layerFromResult(source)
|
|
5006
|
+
};
|
|
5007
|
+
} else {
|
|
5008
|
+
const redacted = redactTraceContent(rawContent, profile, policy);
|
|
5009
|
+
const artifactRead = await openTrace(
|
|
5010
|
+
{ type: "string", content: redacted.content },
|
|
5011
|
+
{ format: options.format ?? "agent-inspect-jsonl" }
|
|
5012
|
+
);
|
|
5013
|
+
const artifact = assessOpenedTrace(artifactRead, {
|
|
5014
|
+
...optionsWithPolicy,
|
|
5015
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
5016
|
+
});
|
|
5017
|
+
const detectors = [
|
|
5018
|
+
...new Set(redacted.findings.map((finding) => finding.detector))
|
|
5019
|
+
].sort((a, b) => a.localeCompare(b));
|
|
5020
|
+
result = resultFromParts({
|
|
5021
|
+
command: "verify-safe",
|
|
5022
|
+
format: artifact.format,
|
|
5023
|
+
runId: artifact.runId ?? source.runId,
|
|
5024
|
+
findings: artifact.findings,
|
|
5025
|
+
diagnostics: artifact.diagnostics,
|
|
5026
|
+
warnings: artifact.warnings,
|
|
5027
|
+
unsupportedFields: artifact.unsupportedFields,
|
|
5028
|
+
status: artifact.status,
|
|
5029
|
+
sourceAssessment: layerFromResult(source),
|
|
5030
|
+
artifactAssessment: layerFromResult(artifact),
|
|
5031
|
+
redactionSummary: {
|
|
5032
|
+
profile,
|
|
5033
|
+
findings: redacted.findings.length,
|
|
5034
|
+
detectors
|
|
5035
|
+
}
|
|
5036
|
+
});
|
|
5037
|
+
}
|
|
5038
|
+
}
|
|
5039
|
+
} catch (error) {
|
|
5040
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5041
|
+
result = message.startsWith("--") || message.includes("--policy") ? invalidArgumentResult(command, error) : readErrorResult(command, error);
|
|
5042
|
+
}
|
|
5043
|
+
process.exitCode = exitCodeFor(result);
|
|
5044
|
+
if (options.json) printJson(result, options.explain === true);
|
|
5045
|
+
else printHuman(result, options.explain === true);
|
|
5046
|
+
}
|
|
5047
|
+
function scanCommand(target, options = {}, stdin = process.stdin) {
|
|
5048
|
+
return safetyCommand("scan", target, options, stdin);
|
|
5049
|
+
}
|
|
5050
|
+
function verifySafeCommand(target, options = {}, stdin = process.stdin) {
|
|
5051
|
+
return safetyCommand("verify-safe", target, options, stdin);
|
|
5052
|
+
}
|
|
5053
|
+
function assessOpenedTrace(read, options = {}) {
|
|
5054
|
+
try {
|
|
5055
|
+
const rules = buildSafetyRules(options);
|
|
5056
|
+
const checkResult = runTraceChecks(
|
|
5057
|
+
{ read },
|
|
5058
|
+
{
|
|
5059
|
+
rules,
|
|
5060
|
+
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
5061
|
+
...options.run !== void 0 ? { runId: options.run } : {}
|
|
5062
|
+
}
|
|
5063
|
+
);
|
|
5064
|
+
const detectorFindings = checkResult.diagnostics.length === 0 ? redactionDetectorFindings(read, checkResult.runId, options.compiledPolicy) : [];
|
|
5065
|
+
return resultFromParts({
|
|
5066
|
+
command: "verify-safe",
|
|
5067
|
+
format: checkResult.format,
|
|
5068
|
+
runId: checkResult.runId,
|
|
5069
|
+
findings: [...checkResult.findings, ...detectorFindings],
|
|
5070
|
+
diagnostics: [
|
|
5071
|
+
...checkResult.diagnostics.map(diagnosticFromCheck),
|
|
5072
|
+
...warningDiagnostics(read.warnings, read.unsupportedFields)
|
|
5073
|
+
],
|
|
5074
|
+
warnings: read.warnings,
|
|
5075
|
+
unsupportedFields: read.unsupportedFields
|
|
5076
|
+
});
|
|
5077
|
+
} catch (error) {
|
|
5078
|
+
return messageStartsWithDash(error) ? invalidArgumentResult("verify-safe", error) : readErrorResult("verify-safe", error);
|
|
5079
|
+
}
|
|
5080
|
+
}
|
|
5081
|
+
function messageStartsWithDash(error) {
|
|
5082
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5083
|
+
return message.startsWith("--");
|
|
5084
|
+
}
|
|
5085
|
+
|
|
5086
|
+
// packages/cli/src/redact.ts
|
|
5087
|
+
function parseRedactionProfile3(value) {
|
|
5088
|
+
if (value === void 0 || value === "local" || value === "share" || value === "strict") {
|
|
5089
|
+
return value ?? "share";
|
|
5090
|
+
}
|
|
5091
|
+
throw new Error(`Unsupported --profile "${value}". Use local, share, or strict.`);
|
|
5092
|
+
}
|
|
5093
|
+
function stable2(value) {
|
|
5094
|
+
if (Array.isArray(value)) return value.map(stable2);
|
|
5095
|
+
if (value === null || typeof value !== "object") return value;
|
|
5096
|
+
const record = value;
|
|
5097
|
+
return Object.fromEntries(
|
|
5098
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable2(record[key])])
|
|
5099
|
+
);
|
|
5100
|
+
}
|
|
5101
|
+
function isMissingFileError3(error) {
|
|
5102
|
+
return error !== null && typeof error === "object" && "code" in error && error.code === "ENOENT";
|
|
5103
|
+
}
|
|
5104
|
+
async function contentFromTarget(target, options, stdin) {
|
|
5105
|
+
if (target === "-") {
|
|
5106
|
+
return { content: await readStdin(stdin), source: "stdin" };
|
|
5107
|
+
}
|
|
5108
|
+
try {
|
|
5109
|
+
const stats2 = await stat(target);
|
|
5110
|
+
if (stats2.isDirectory()) {
|
|
5111
|
+
throw new Error("redact requires a trace file, JSON file, stdin, or run id.");
|
|
5112
|
+
}
|
|
5113
|
+
return { content: await readFile(target, "utf-8"), source: target };
|
|
5114
|
+
} catch (error) {
|
|
5115
|
+
if (!isMissingFileError3(error)) throw error;
|
|
5116
|
+
}
|
|
5117
|
+
const runPath = getTraceFilePath(target, resolveTraceDir({ dir: options.dir }));
|
|
5118
|
+
const stats = await stat(runPath);
|
|
5119
|
+
if (stats.isDirectory()) {
|
|
5120
|
+
throw new Error("redact requires a trace file, JSON file, stdin, or run id.");
|
|
5121
|
+
}
|
|
5122
|
+
return { content: await readFile(runPath, "utf-8"), source: runPath };
|
|
5123
|
+
}
|
|
5124
|
+
function applyRedact(value, profile, policy) {
|
|
5125
|
+
const result = createRedactor({
|
|
5126
|
+
profile,
|
|
5127
|
+
...policy?.extraKeys.length ? { extraKeys: [...policy.extraKeys] } : {},
|
|
5128
|
+
...policy?.detectors.length ? { detectors: [...policy.detectors] } : {}
|
|
5129
|
+
}).redact(value);
|
|
5130
|
+
return { value: result.value, findings: result.findings };
|
|
5131
|
+
}
|
|
5132
|
+
function redactJsonText(content, profile, policy) {
|
|
5133
|
+
const parsed = JSON.parse(content);
|
|
5134
|
+
const result = applyRedact(parsed, profile, policy);
|
|
5135
|
+
return {
|
|
5136
|
+
content: `${JSON.stringify(result.value, null, 2)}
|
|
5137
|
+
`,
|
|
5138
|
+
findings: result.findings
|
|
5139
|
+
};
|
|
5140
|
+
}
|
|
5141
|
+
function redactJsonlText(content, profile, policy) {
|
|
5142
|
+
const lines = content.split(/\r?\n/);
|
|
5143
|
+
const out = [];
|
|
5144
|
+
const findings = [];
|
|
5145
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
5146
|
+
const line = lines[index] ?? "";
|
|
5147
|
+
if (line.trim() === "") continue;
|
|
5148
|
+
let parsed;
|
|
5149
|
+
try {
|
|
5150
|
+
parsed = JSON.parse(line);
|
|
5151
|
+
} catch {
|
|
5152
|
+
throw new Error(`Input is not valid JSON or JSONL at line ${index + 1}.`);
|
|
5153
|
+
}
|
|
5154
|
+
const result = applyRedact(parsed, profile, policy);
|
|
5155
|
+
out.push(JSON.stringify(result.value));
|
|
5156
|
+
findings.push(
|
|
5157
|
+
...result.findings.map((finding) => ({
|
|
5158
|
+
...finding,
|
|
5159
|
+
path: `line:${index + 1}:${finding.path}`
|
|
5160
|
+
}))
|
|
5161
|
+
);
|
|
5162
|
+
}
|
|
5163
|
+
return {
|
|
5164
|
+
content: out.length === 0 ? "" : `${out.join("\n")}
|
|
5165
|
+
`,
|
|
5166
|
+
findings
|
|
5167
|
+
};
|
|
5168
|
+
}
|
|
5169
|
+
function redactDocument(content, profile, policy) {
|
|
5170
|
+
const trimmed = content.trim();
|
|
5171
|
+
if (trimmed.startsWith("{")) {
|
|
5172
|
+
try {
|
|
5173
|
+
const parsed = JSON.parse(trimmed);
|
|
5174
|
+
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) && ("schemaVersion" in parsed || "eventId" in parsed || "runId" in parsed)) {
|
|
5175
|
+
return redactJsonlText(content, profile, policy);
|
|
5176
|
+
}
|
|
5177
|
+
} catch {
|
|
5178
|
+
}
|
|
5179
|
+
}
|
|
5180
|
+
try {
|
|
5181
|
+
return redactJsonText(content, profile, policy);
|
|
5182
|
+
} catch {
|
|
5183
|
+
return redactJsonlText(content, profile, policy);
|
|
5184
|
+
}
|
|
5185
|
+
}
|
|
5186
|
+
function redactTraceContent(content, profile, policy) {
|
|
5187
|
+
return redactDocument(content, profile, policy);
|
|
5188
|
+
}
|
|
5189
|
+
function looksLikeAgentInspectTrace(content) {
|
|
5190
|
+
const trimmed = content.trim();
|
|
5191
|
+
if (!trimmed.startsWith("{")) return false;
|
|
5192
|
+
try {
|
|
5193
|
+
const firstLine = trimmed.split(/\r?\n/).find((line) => line.trim() !== "") ?? trimmed;
|
|
5194
|
+
const parsed = JSON.parse(firstLine);
|
|
5195
|
+
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) && ("schemaVersion" in parsed || "eventId" in parsed || "runId" in parsed);
|
|
5196
|
+
} catch {
|
|
5197
|
+
return false;
|
|
5198
|
+
}
|
|
5199
|
+
}
|
|
5200
|
+
function printResidualWarning(assessment, sourceLooksLikeTrace) {
|
|
5201
|
+
if (assessment.status === "SAFE") return;
|
|
5202
|
+
if (assessment.status === "UNKNOWN" && !sourceLooksLikeTrace) return;
|
|
5203
|
+
const codes = assessment.codes.length > 0 ? ` codes=${assessment.codes.join(",")}` : "";
|
|
5204
|
+
console.error(
|
|
5205
|
+
`Residual safety: ${assessment.status} (findings=${assessment.findings}, warnings=${assessment.warnings}, errors=${assessment.errors})${codes}. Redact does not certify safe sharing; run verify-safe before publishing.`
|
|
5206
|
+
);
|
|
5207
|
+
}
|
|
5208
|
+
function residualExitCode(assessment) {
|
|
5209
|
+
if (assessment.status === "UNSAFE") return 1;
|
|
5210
|
+
if (assessment.status === "UNKNOWN") return 2;
|
|
5211
|
+
return 0;
|
|
5212
|
+
}
|
|
5213
|
+
async function redactCommand(target, options = {}, stdin = process.stdin) {
|
|
5214
|
+
const profile = parseRedactionProfile3(resolveRedactionProfileOption(options));
|
|
5215
|
+
const policy = options.policy !== void 0 ? await loadRedactionPolicy(options.policy) : void 0;
|
|
5216
|
+
const source = await contentFromTarget(target, options, stdin);
|
|
5217
|
+
const redacted = redactDocument(source.content, profile, policy);
|
|
5218
|
+
const outputPath = resolveOutputOption(options);
|
|
5219
|
+
const residualAssessment = await assessResidualFromContent(redacted.content, {
|
|
5220
|
+
compiledPolicy: policy
|
|
5221
|
+
});
|
|
4478
5222
|
if (outputPath !== void 0) {
|
|
4479
5223
|
await writeFile(outputPath, redacted.content, "utf-8");
|
|
4480
5224
|
}
|
|
5225
|
+
if (options.failOnResidual === true) {
|
|
5226
|
+
process.exitCode = residualExitCode(residualAssessment);
|
|
5227
|
+
}
|
|
4481
5228
|
if (options.json) {
|
|
4482
5229
|
console.log(
|
|
4483
5230
|
JSON.stringify(
|
|
4484
|
-
|
|
5231
|
+
stable2({
|
|
4485
5232
|
ok: true,
|
|
4486
5233
|
profile,
|
|
4487
5234
|
source: source.source,
|
|
4488
5235
|
output: outputPath,
|
|
4489
5236
|
findings: redacted.findings,
|
|
5237
|
+
residualAssessment,
|
|
5238
|
+
...policy !== void 0 ? {
|
|
5239
|
+
policy: {
|
|
5240
|
+
path: policy.path,
|
|
5241
|
+
extraKeys: policy.extraKeys.length,
|
|
5242
|
+
patterns: policy.detectors.length,
|
|
5243
|
+
diagnostics: policy.diagnostics
|
|
5244
|
+
}
|
|
5245
|
+
} : {},
|
|
4490
5246
|
content: outputPath === void 0 ? redacted.content : void 0
|
|
4491
5247
|
}),
|
|
4492
5248
|
null,
|
|
@@ -4495,6 +5251,7 @@ async function redactCommand(target, options = {}, stdin = process.stdin) {
|
|
|
4495
5251
|
);
|
|
4496
5252
|
return;
|
|
4497
5253
|
}
|
|
5254
|
+
printResidualWarning(residualAssessment, looksLikeAgentInspectTrace(redacted.content));
|
|
4498
5255
|
if (outputPath === void 0) {
|
|
4499
5256
|
process.stdout.write(redacted.content);
|
|
4500
5257
|
}
|
|
@@ -4770,7 +5527,7 @@ async function openCommand(input3, options = {}, stdin = process.stdin) {
|
|
|
4770
5527
|
}
|
|
4771
5528
|
}
|
|
4772
5529
|
}
|
|
4773
|
-
function
|
|
5530
|
+
function isRecord7(value) {
|
|
4774
5531
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4775
5532
|
}
|
|
4776
5533
|
function parseTarget(value) {
|
|
@@ -4779,7 +5536,7 @@ function parseTarget(value) {
|
|
|
4779
5536
|
throw new Error('Unsupported migration target. Use "--to 1.0".');
|
|
4780
5537
|
}
|
|
4781
5538
|
function formatOf(value) {
|
|
4782
|
-
if (!
|
|
5539
|
+
if (!isRecord7(value)) return "unknown";
|
|
4783
5540
|
if (value.schemaVersion === "0.1") return "0.1";
|
|
4784
5541
|
if (value.schemaVersion === "0.2") return "0.2";
|
|
4785
5542
|
if (value.schemaVersion === "1.0") return "1.0";
|
|
@@ -4792,14 +5549,14 @@ function uniqueSorted(values) {
|
|
|
4792
5549
|
return [...new Set(values)].sort();
|
|
4793
5550
|
}
|
|
4794
5551
|
function isWithinDirectory(child, parent) {
|
|
4795
|
-
const relative =
|
|
4796
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
5552
|
+
const relative = path19.relative(parent, child);
|
|
5553
|
+
return relative === "" || !relative.startsWith("..") && !path19.isAbsolute(relative);
|
|
4797
5554
|
}
|
|
4798
5555
|
async function resolveOutputPath(inputPath, output2, force) {
|
|
4799
5556
|
if (output2 === void 0 || output2.trim() === "") return void 0;
|
|
4800
|
-
const inputAbs =
|
|
4801
|
-
const outputAbs =
|
|
4802
|
-
const inputDir =
|
|
5557
|
+
const inputAbs = path19.resolve(inputPath);
|
|
5558
|
+
const outputAbs = path19.resolve(output2.trim());
|
|
5559
|
+
const inputDir = path19.dirname(inputAbs);
|
|
4803
5560
|
if (!isWithinDirectory(outputAbs, inputDir)) {
|
|
4804
5561
|
throw new Error("Refusing to write migrated output outside the input directory.");
|
|
4805
5562
|
}
|
|
@@ -4920,7 +5677,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
4920
5677
|
process.exitCode = 1;
|
|
4921
5678
|
return;
|
|
4922
5679
|
}
|
|
4923
|
-
const inputPath =
|
|
5680
|
+
const inputPath = path19.resolve(input3.trim());
|
|
4924
5681
|
const dryRun = options.dryRun === true;
|
|
4925
5682
|
if (!dryRun && (options.output === void 0 || options.output.trim() === "")) {
|
|
4926
5683
|
console.error("migrate requires --dry-run or --output <path>.");
|
|
@@ -4939,7 +5696,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
4939
5696
|
);
|
|
4940
5697
|
const result = await buildMigration(inputPath, outputPath);
|
|
4941
5698
|
if (!dryRun && outputPath !== void 0) {
|
|
4942
|
-
await mkdir(
|
|
5699
|
+
await mkdir(path19.dirname(outputPath), { recursive: true });
|
|
4943
5700
|
await writeFile(outputPath, result.content, "utf-8");
|
|
4944
5701
|
}
|
|
4945
5702
|
printSummary2(result, dryRun);
|
|
@@ -4950,7 +5707,7 @@ async function migrateCommand(input3, options = {}) {
|
|
|
4950
5707
|
}
|
|
4951
5708
|
}
|
|
4952
5709
|
function validateReporterArtifactPath(options) {
|
|
4953
|
-
const outputDir =
|
|
5710
|
+
const outputDir = path19.resolve(options.outputDir);
|
|
4954
5711
|
const diagnostics = [];
|
|
4955
5712
|
const rawPath = options.relativePath;
|
|
4956
5713
|
if (rawPath.length === 0) {
|
|
@@ -4970,7 +5727,7 @@ function validateReporterArtifactPath(options) {
|
|
|
4970
5727
|
});
|
|
4971
5728
|
return { ok: false, outputDir, diagnostics };
|
|
4972
5729
|
}
|
|
4973
|
-
if (
|
|
5730
|
+
if (path19.isAbsolute(rawPath) || path19.win32.isAbsolute(rawPath)) {
|
|
4974
5731
|
diagnostics.push({
|
|
4975
5732
|
code: "artifact_path_absolute",
|
|
4976
5733
|
severity: "error",
|
|
@@ -4979,7 +5736,7 @@ function validateReporterArtifactPath(options) {
|
|
|
4979
5736
|
});
|
|
4980
5737
|
return { ok: false, outputDir, diagnostics };
|
|
4981
5738
|
}
|
|
4982
|
-
const normalized =
|
|
5739
|
+
const normalized = path19.posix.normalize(rawPath.replace(/\\/g, "/"));
|
|
4983
5740
|
const segments = normalized.split("/");
|
|
4984
5741
|
if (normalized === "." || normalized.startsWith("../") || segments.some((segment) => segment === "..")) {
|
|
4985
5742
|
diagnostics.push({
|
|
@@ -4990,9 +5747,9 @@ function validateReporterArtifactPath(options) {
|
|
|
4990
5747
|
});
|
|
4991
5748
|
return { ok: false, outputDir, diagnostics };
|
|
4992
5749
|
}
|
|
4993
|
-
const absolutePath =
|
|
4994
|
-
const relFromOutput =
|
|
4995
|
-
if (relFromOutput.length === 0 || relFromOutput.startsWith("..") ||
|
|
5750
|
+
const absolutePath = path19.resolve(outputDir, normalized);
|
|
5751
|
+
const relFromOutput = path19.relative(outputDir, absolutePath);
|
|
5752
|
+
if (relFromOutput.length === 0 || relFromOutput.startsWith("..") || path19.isAbsolute(relFromOutput)) {
|
|
4996
5753
|
diagnostics.push({
|
|
4997
5754
|
code: "artifact_path_escape",
|
|
4998
5755
|
severity: "error",
|
|
@@ -5018,7 +5775,7 @@ var EVIDENCE_CI_ARTIFACT_FILES = [
|
|
|
5018
5775
|
function createEvidenceCiArtifacts(options) {
|
|
5019
5776
|
const profile = options.redactionProfile ?? "share";
|
|
5020
5777
|
const prefix = options.relativeDir !== void 0 && options.relativeDir.trim() !== "" ? options.relativeDir.replace(/\\/g, "/").replace(/^\/+|\/+$/g, "") : "";
|
|
5021
|
-
const join = (name) => prefix === "" ? name :
|
|
5778
|
+
const join = (name) => prefix === "" ? name : path19.posix.join(prefix, name);
|
|
5022
5779
|
const formatFor = (name) => {
|
|
5023
5780
|
if (name.endsWith(".html")) return "html";
|
|
5024
5781
|
if (name.endsWith(".jsonl")) return "jsonl";
|
|
@@ -5067,9 +5824,9 @@ function stableJson2(value) {
|
|
|
5067
5824
|
async function writeLocalEvidence(input3) {
|
|
5068
5825
|
const profile = input3.redactionProfile ?? "share";
|
|
5069
5826
|
const format = input3.format ?? "directory";
|
|
5070
|
-
const baseDir =
|
|
5827
|
+
const baseDir = path19.resolve(input3.outputDir);
|
|
5071
5828
|
await mkdir(
|
|
5072
|
-
format === "zip" && baseDir.toLowerCase().endsWith(".zip") ?
|
|
5829
|
+
format === "zip" && baseDir.toLowerCase().endsWith(".zip") ? path19.dirname(baseDir) : baseDir,
|
|
5073
5830
|
{ recursive: true }
|
|
5074
5831
|
);
|
|
5075
5832
|
const sourceContents = /* @__PURE__ */ new Map();
|
|
@@ -5125,7 +5882,7 @@ async function writeLocalEvidence(input3) {
|
|
|
5125
5882
|
}
|
|
5126
5883
|
if (format === "zip") {
|
|
5127
5884
|
const zipPath = baseDir.toLowerCase().endsWith(".zip") ? baseDir : `${baseDir}.zip`;
|
|
5128
|
-
await mkdir(
|
|
5885
|
+
await mkdir(path19.dirname(zipPath), { recursive: true });
|
|
5129
5886
|
const archive = buildZipArchive(
|
|
5130
5887
|
files.map(([relativePath, content]) => ({
|
|
5131
5888
|
path: relativePath,
|
|
@@ -5140,16 +5897,16 @@ async function writeLocalEvidence(input3) {
|
|
|
5140
5897
|
let sidecarDir = baseDir;
|
|
5141
5898
|
if (baseDir.toLowerCase().endsWith(".html")) {
|
|
5142
5899
|
htmlPath = baseDir;
|
|
5143
|
-
sidecarDir =
|
|
5900
|
+
sidecarDir = path19.dirname(baseDir);
|
|
5144
5901
|
} else {
|
|
5145
5902
|
await mkdir(baseDir, { recursive: true });
|
|
5146
|
-
htmlPath =
|
|
5903
|
+
htmlPath = path19.join(baseDir, "evidence.html");
|
|
5147
5904
|
sidecarDir = baseDir;
|
|
5148
5905
|
}
|
|
5149
5906
|
await mkdir(sidecarDir, { recursive: true });
|
|
5150
5907
|
await writeFile(htmlPath, evidencePackage["evidence.html"], "utf-8");
|
|
5151
5908
|
await writeFile(
|
|
5152
|
-
|
|
5909
|
+
path19.join(sidecarDir, "evidence.json"),
|
|
5153
5910
|
evidencePackage["evidence.json"],
|
|
5154
5911
|
"utf-8"
|
|
5155
5912
|
);
|
|
@@ -5157,7 +5914,7 @@ async function writeLocalEvidence(input3) {
|
|
|
5157
5914
|
}
|
|
5158
5915
|
await mkdir(baseDir, { recursive: true });
|
|
5159
5916
|
for (const [name, content] of files) {
|
|
5160
|
-
await writeFile(
|
|
5917
|
+
await writeFile(path19.join(baseDir, name), content, "utf-8");
|
|
5161
5918
|
}
|
|
5162
5919
|
return baseDir;
|
|
5163
5920
|
}
|
|
@@ -5181,11 +5938,11 @@ function checkResultToEvidenceJson(result, runIds) {
|
|
|
5181
5938
|
}
|
|
5182
5939
|
function defaultEvidenceDir(label) {
|
|
5183
5940
|
const safe = label.replace(/[^a-zA-Z0-9._-]+/g, "_").slice(0, 80) || "evidence";
|
|
5184
|
-
return
|
|
5941
|
+
return path19.join(".agent-inspect", "evidence", safe);
|
|
5185
5942
|
}
|
|
5186
5943
|
function resolveEvidenceOutputDir(evidenceDir, label) {
|
|
5187
5944
|
if (evidenceDir !== void 0 && evidenceDir.trim() !== "") {
|
|
5188
|
-
return
|
|
5945
|
+
return path19.resolve(evidenceDir.trim());
|
|
5189
5946
|
}
|
|
5190
5947
|
return defaultEvidenceDir(label);
|
|
5191
5948
|
}
|
|
@@ -5509,23 +6266,23 @@ function evaluatePromptInjection(text, options = {}) {
|
|
|
5509
6266
|
}
|
|
5510
6267
|
return fail(ruleId, `Matched ${evidence.length} injection pattern(s).`, evidence, "warning");
|
|
5511
6268
|
}
|
|
5512
|
-
function validateSchemaField(value, field,
|
|
6269
|
+
function validateSchemaField(value, field, path30, evidence) {
|
|
5513
6270
|
const ruleId = "guardrail.structured-output";
|
|
5514
6271
|
if (field.type) {
|
|
5515
6272
|
const actual = value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
|
|
5516
6273
|
if (actual !== field.type) {
|
|
5517
|
-
evidence.push({ ruleId, path:
|
|
6274
|
+
evidence.push({ ruleId, path: path30, preview: `expected ${field.type}, got ${actual}` });
|
|
5518
6275
|
return;
|
|
5519
6276
|
}
|
|
5520
6277
|
}
|
|
5521
6278
|
if (field.enum && !field.enum.some((item) => Object.is(item, value))) {
|
|
5522
|
-
evidence.push({ ruleId, path:
|
|
6279
|
+
evidence.push({ ruleId, path: path30, preview: "value not in enum" });
|
|
5523
6280
|
}
|
|
5524
6281
|
if (field.type === "object" && field.required && value && typeof value === "object" && !Array.isArray(value)) {
|
|
5525
6282
|
const record = value;
|
|
5526
6283
|
for (const key of field.required) {
|
|
5527
6284
|
if (!(key in record)) {
|
|
5528
|
-
evidence.push({ ruleId, path: `${
|
|
6285
|
+
evidence.push({ ruleId, path: `${path30}.${key}`, preview: "missing required key" });
|
|
5529
6286
|
}
|
|
5530
6287
|
}
|
|
5531
6288
|
}
|
|
@@ -6254,7 +7011,7 @@ function parseCheckConfig(value) {
|
|
|
6254
7011
|
}
|
|
6255
7012
|
async function loadConfig(configPath) {
|
|
6256
7013
|
if (configPath === void 0) return {};
|
|
6257
|
-
const extension =
|
|
7014
|
+
const extension = path19.extname(configPath);
|
|
6258
7015
|
if (TS_CONFIG_EXTENSIONS.has(extension)) {
|
|
6259
7016
|
throw new CheckConfigError(
|
|
6260
7017
|
"AI_CHECK_CONFIG_LOAD_FAILED",
|
|
@@ -6267,7 +7024,7 @@ async function loadConfig(configPath) {
|
|
|
6267
7024
|
"Unsupported check config extension. Use .json, .js, .mjs, or .cjs."
|
|
6268
7025
|
);
|
|
6269
7026
|
}
|
|
6270
|
-
const absolute =
|
|
7027
|
+
const absolute = path19.resolve(configPath);
|
|
6271
7028
|
try {
|
|
6272
7029
|
if (extension === ".json") {
|
|
6273
7030
|
const raw = await readFile(absolute, "utf-8");
|
|
@@ -6461,7 +7218,7 @@ function buildRules(config, options, presetSelect = []) {
|
|
|
6461
7218
|
diagnostics
|
|
6462
7219
|
};
|
|
6463
7220
|
}
|
|
6464
|
-
function
|
|
7221
|
+
function exitCodeFor2(result) {
|
|
6465
7222
|
if (result.status === "pass") return 0;
|
|
6466
7223
|
if (result.status === "fail") return 1;
|
|
6467
7224
|
const codes = result.diagnostics.map((item) => item.code);
|
|
@@ -6482,16 +7239,16 @@ function exitCodeFor(result) {
|
|
|
6482
7239
|
}
|
|
6483
7240
|
return 1;
|
|
6484
7241
|
}
|
|
6485
|
-
function
|
|
6486
|
-
if (Array.isArray(value)) return value.map(
|
|
7242
|
+
function stable3(value) {
|
|
7243
|
+
if (Array.isArray(value)) return value.map(stable3);
|
|
6487
7244
|
if (value === null || typeof value !== "object") return value;
|
|
6488
7245
|
const record = value;
|
|
6489
7246
|
return Object.fromEntries(
|
|
6490
|
-
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key,
|
|
7247
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable3(record[key])])
|
|
6491
7248
|
);
|
|
6492
7249
|
}
|
|
6493
|
-
function
|
|
6494
|
-
console.log(JSON.stringify(
|
|
7250
|
+
function printJson2(result) {
|
|
7251
|
+
console.log(JSON.stringify(stable3(result), null, 2));
|
|
6495
7252
|
}
|
|
6496
7253
|
function isSafetyFinding(ruleId) {
|
|
6497
7254
|
return ruleId.startsWith("safety.") || ruleId.startsWith("guardrail.") || ruleId.includes("pii") || ruleId.includes("secret");
|
|
@@ -6528,7 +7285,7 @@ function printPresetClassSummary(result, preset) {
|
|
|
6528
7285
|
`Share safety: ${hasSafetyFindings || result.status === "fail" ? "FAIL" : result.status === "pass" ? "PASS" : "FAIL"}`
|
|
6529
7286
|
);
|
|
6530
7287
|
}
|
|
6531
|
-
function
|
|
7288
|
+
function printHuman2(result, options = {}) {
|
|
6532
7289
|
const scoped = result;
|
|
6533
7290
|
if (scoped.scopeLabel) {
|
|
6534
7291
|
console.log(`Scope: ${scoped.scopeKind} ${scoped.scopeLabel}`);
|
|
@@ -6548,13 +7305,13 @@ function printHuman(result, options = {}) {
|
|
|
6548
7305
|
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
6549
7306
|
}
|
|
6550
7307
|
for (const finding of result.findings) {
|
|
6551
|
-
const
|
|
7308
|
+
const path30 = finding.evidence[0]?.path;
|
|
6552
7309
|
const run = finding.evidence[0]?.runId;
|
|
6553
7310
|
const runPrefix = run ? `[${run}] ` : "";
|
|
6554
|
-
console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${
|
|
7311
|
+
console.log(`- ${runPrefix}${finding.ruleId}: ${finding.message}${path30 ? ` (${path30})` : ""}`);
|
|
6555
7312
|
}
|
|
6556
7313
|
}
|
|
6557
|
-
function
|
|
7314
|
+
function readErrorResult2(error) {
|
|
6558
7315
|
if (error instanceof TraceReadError) {
|
|
6559
7316
|
const code = error.code === "unsupported_format" ? "AI_CHECK_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_CHECK_AMBIGUOUS_FORMAT" : "AI_CHECK_TRACE_UNREADABLE";
|
|
6560
7317
|
return errorResult(code, error.message);
|
|
@@ -6581,9 +7338,9 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
6581
7338
|
"AI_CHECK_CONFIG_NO_EFFECTIVE_RULES",
|
|
6582
7339
|
"Explicit --config has no effective check rules. Configure checks.select, checks.run, checks.tool, checks.llm, checks.structure, or checks.safety."
|
|
6583
7340
|
);
|
|
6584
|
-
if (options.json)
|
|
6585
|
-
else
|
|
6586
|
-
process.exitCode =
|
|
7341
|
+
if (options.json) printJson2(result);
|
|
7342
|
+
else printHuman2(result, options);
|
|
7343
|
+
process.exitCode = exitCodeFor2(result);
|
|
6587
7344
|
return;
|
|
6588
7345
|
}
|
|
6589
7346
|
let effectiveOptions = options;
|
|
@@ -6713,10 +7470,10 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
6713
7470
|
result = errorResult(code, message);
|
|
6714
7471
|
}
|
|
6715
7472
|
} else {
|
|
6716
|
-
result =
|
|
7473
|
+
result = readErrorResult2(error);
|
|
6717
7474
|
}
|
|
6718
7475
|
}
|
|
6719
|
-
process.exitCode =
|
|
7476
|
+
process.exitCode = exitCodeFor2(result);
|
|
6720
7477
|
const failed = result.status !== "pass";
|
|
6721
7478
|
if (shouldEmitEvidence(options.evidenceOn, failed)) {
|
|
6722
7479
|
try {
|
|
@@ -6756,8 +7513,8 @@ async function checkCommand(target, options = {}, stdin = process.stdin) {
|
|
|
6756
7513
|
);
|
|
6757
7514
|
}
|
|
6758
7515
|
}
|
|
6759
|
-
if (options.json)
|
|
6760
|
-
else
|
|
7516
|
+
if (options.json) printJson2(result);
|
|
7517
|
+
else printHuman2(result, options);
|
|
6761
7518
|
}
|
|
6762
7519
|
|
|
6763
7520
|
// packages/viewer/src/html.ts
|
|
@@ -6908,7 +7665,7 @@ async function loadSuiteViewerData(options) {
|
|
|
6908
7665
|
for (const suiteCase of result.cases) {
|
|
6909
7666
|
cases.push(await enrichCase(suiteCase, baselinePath));
|
|
6910
7667
|
}
|
|
6911
|
-
const artifactsDir =
|
|
7668
|
+
const artifactsDir = path19.join(path19.dirname(result.configPath), ".agent-inspect/suite-runs");
|
|
6912
7669
|
return {
|
|
6913
7670
|
suiteName: result.suiteName,
|
|
6914
7671
|
configPath: result.configPath,
|
|
@@ -7101,19 +7858,19 @@ function serializeWorkspaceManifest(manifest) {
|
|
|
7101
7858
|
}
|
|
7102
7859
|
var INDEX_DIR_NAME = "index";
|
|
7103
7860
|
function resolveWorkspaceLocation(cwd = process.cwd()) {
|
|
7104
|
-
const projectRoot =
|
|
7105
|
-
const workspaceDir =
|
|
7861
|
+
const projectRoot = path19.resolve(cwd);
|
|
7862
|
+
const workspaceDir = path19.join(projectRoot, WORKSPACE_DIR_NAME);
|
|
7106
7863
|
return {
|
|
7107
7864
|
projectRoot,
|
|
7108
7865
|
workspaceDir,
|
|
7109
|
-
manifestPath:
|
|
7866
|
+
manifestPath: path19.join(workspaceDir, WORKSPACE_MANIFEST_FILENAME)
|
|
7110
7867
|
};
|
|
7111
7868
|
}
|
|
7112
7869
|
function resolveInsideWorkspace(workspaceDir, relative) {
|
|
7113
|
-
const base =
|
|
7114
|
-
const resolved =
|
|
7115
|
-
const rel =
|
|
7116
|
-
if (rel === "" || rel === "." || !rel.startsWith("..") && !
|
|
7870
|
+
const base = path19.resolve(workspaceDir);
|
|
7871
|
+
const resolved = path19.resolve(base, relative);
|
|
7872
|
+
const rel = path19.relative(base, resolved);
|
|
7873
|
+
if (rel === "" || rel === "." || !rel.startsWith("..") && !path19.isAbsolute(rel)) {
|
|
7117
7874
|
return resolved;
|
|
7118
7875
|
}
|
|
7119
7876
|
throw new Error(
|
|
@@ -7182,7 +7939,7 @@ async function createWorkspace(options = {}) {
|
|
|
7182
7939
|
created = false;
|
|
7183
7940
|
adopted = true;
|
|
7184
7941
|
} else {
|
|
7185
|
-
const project = options.project?.trim() ||
|
|
7942
|
+
const project = options.project?.trim() || path19.basename(location.projectRoot) || "workspace";
|
|
7186
7943
|
const traceDirs = detectedExistingTraces ? ["runs", "."] : ["runs"];
|
|
7187
7944
|
manifest = createDefaultWorkspaceManifest({
|
|
7188
7945
|
project,
|
|
@@ -7321,7 +8078,7 @@ async function doctorWorkspace(location) {
|
|
|
7321
8078
|
const abs = resolveInsideWorkspace(location.workspaceDir, rel);
|
|
7322
8079
|
for (const file of await listJsonl(abs)) {
|
|
7323
8080
|
try {
|
|
7324
|
-
const s = await stat(
|
|
8081
|
+
const s = await stat(path19.join(abs, file));
|
|
7325
8082
|
newestTraceMtime = Math.max(newestTraceMtime, s.mtimeMs);
|
|
7326
8083
|
} catch {
|
|
7327
8084
|
checks2.push({ id: "trace-readability", status: "warn", message: `cannot stat ${rel}/${file}` });
|
|
@@ -7369,7 +8126,7 @@ async function cleanWorkspace(location, manifest, options = {}) {
|
|
|
7369
8126
|
const relPath = `${rel}/${entry}`;
|
|
7370
8127
|
removed.push(relPath);
|
|
7371
8128
|
if (!dryRun) {
|
|
7372
|
-
await rm(
|
|
8129
|
+
await rm(path19.join(abs, entry), { recursive: true, force: true });
|
|
7373
8130
|
}
|
|
7374
8131
|
}
|
|
7375
8132
|
}
|
|
@@ -7378,7 +8135,7 @@ async function cleanWorkspace(location, manifest, options = {}) {
|
|
|
7378
8135
|
|
|
7379
8136
|
// packages/viewer/src/workspace-data.ts
|
|
7380
8137
|
async function loadWorkspaceViewerData(options) {
|
|
7381
|
-
const cwd =
|
|
8138
|
+
const cwd = path19.resolve(options.cwd ?? process.cwd());
|
|
7382
8139
|
const location = resolveWorkspaceLocation(cwd);
|
|
7383
8140
|
const manifestRead = await readWorkspaceManifestFile(location);
|
|
7384
8141
|
if (!manifestRead.ok || manifestRead.manifest === void 0) {
|
|
@@ -7391,7 +8148,7 @@ async function loadWorkspaceViewerData(options) {
|
|
|
7391
8148
|
const runs = [];
|
|
7392
8149
|
for (const rel of manifestRead.manifest.traceDirs) {
|
|
7393
8150
|
const traceDir = resolveTraceDir({
|
|
7394
|
-
dir:
|
|
8151
|
+
dir: path19.join(location.workspaceDir, rel)
|
|
7395
8152
|
});
|
|
7396
8153
|
const td = new TraceDirectory({ dir: traceDir });
|
|
7397
8154
|
const files = await td.list();
|
|
@@ -7405,7 +8162,7 @@ async function loadWorkspaceViewerData(options) {
|
|
|
7405
8162
|
runId: meta.runId,
|
|
7406
8163
|
...meta.name !== void 0 ? { name: meta.name } : {},
|
|
7407
8164
|
status: meta.status,
|
|
7408
|
-
file:
|
|
8165
|
+
file: path19.basename(meta.filePath)
|
|
7409
8166
|
});
|
|
7410
8167
|
}
|
|
7411
8168
|
}
|
|
@@ -7416,8 +8173,8 @@ async function loadWorkspaceViewerData(options) {
|
|
|
7416
8173
|
doctor,
|
|
7417
8174
|
runs,
|
|
7418
8175
|
bundleDirs: [
|
|
7419
|
-
|
|
7420
|
-
|
|
8176
|
+
path19.join(location.workspaceDir, manifestRead.manifest.bundlesDir),
|
|
8177
|
+
path19.join(location.workspaceDir, manifestRead.manifest.artifactsDir)
|
|
7421
8178
|
]
|
|
7422
8179
|
};
|
|
7423
8180
|
}
|
|
@@ -7480,7 +8237,7 @@ function createViewerServer(options = {}) {
|
|
|
7480
8237
|
ok: true,
|
|
7481
8238
|
readOnly: true,
|
|
7482
8239
|
mode,
|
|
7483
|
-
traceDir:
|
|
8240
|
+
traceDir: path19.resolve(traceDir)
|
|
7484
8241
|
});
|
|
7485
8242
|
}
|
|
7486
8243
|
if (pathname === "/api/suite" && mode === "suite") {
|
|
@@ -7509,7 +8266,7 @@ function createViewerServer(options = {}) {
|
|
|
7509
8266
|
runId: meta.runId,
|
|
7510
8267
|
name: meta.name,
|
|
7511
8268
|
status: meta.status,
|
|
7512
|
-
file:
|
|
8269
|
+
file: path19.basename(meta.filePath),
|
|
7513
8270
|
startedAt: meta.startedAt,
|
|
7514
8271
|
durationMs: meta.durationMs
|
|
7515
8272
|
}))
|
|
@@ -7626,7 +8383,7 @@ function startViewerServer(options = {}) {
|
|
|
7626
8383
|
resolve({
|
|
7627
8384
|
host,
|
|
7628
8385
|
port: resolvedPort,
|
|
7629
|
-
traceDir:
|
|
8386
|
+
traceDir: path19.resolve(traceDir),
|
|
7630
8387
|
url: `http://${host}:${resolvedPort}/${modeQuery}`,
|
|
7631
8388
|
mode
|
|
7632
8389
|
});
|
|
@@ -8037,10 +8794,10 @@ async function evalRun(input3, options = {}) {
|
|
|
8037
8794
|
diagnostics: []
|
|
8038
8795
|
};
|
|
8039
8796
|
}
|
|
8040
|
-
function evidenceForRun(run,
|
|
8041
|
-
return [{ runId: run.runId, ...
|
|
8797
|
+
function evidenceForRun(run, path30) {
|
|
8798
|
+
return [{ runId: run.runId, ...path30 !== void 0 ? { path: path30 } : {} }];
|
|
8042
8799
|
}
|
|
8043
|
-
function evidenceForEvent(event,
|
|
8800
|
+
function evidenceForEvent(event, path30) {
|
|
8044
8801
|
return [
|
|
8045
8802
|
{
|
|
8046
8803
|
runId: event.runId,
|
|
@@ -8048,7 +8805,7 @@ function evidenceForEvent(event, path29) {
|
|
|
8048
8805
|
...event.parentId !== void 0 ? { parentId: event.parentId } : {},
|
|
8049
8806
|
kind: event.kind,
|
|
8050
8807
|
name: event.name,
|
|
8051
|
-
...
|
|
8808
|
+
...path30 !== void 0 ? { path: path30 } : {}
|
|
8052
8809
|
}
|
|
8053
8810
|
];
|
|
8054
8811
|
}
|
|
@@ -8206,9 +8963,9 @@ function collectTextFields(nodes, keys, preferredKinds = []) {
|
|
|
8206
8963
|
function tokenize(text) {
|
|
8207
8964
|
return [...text.toLowerCase().matchAll(/[a-z0-9][a-z0-9'-]{2,}/g)].map((match) => match[0].replace(/^['-]+|['-]+$/g, "")).filter((token) => token.length > 2 && !STOP_WORDS.has(token));
|
|
8208
8965
|
}
|
|
8209
|
-
function firstEvidence(fields, run,
|
|
8966
|
+
function firstEvidence(fields, run, path30) {
|
|
8210
8967
|
const first = fields[0];
|
|
8211
|
-
return first === void 0 ? evidenceForRun(run,
|
|
8968
|
+
return first === void 0 ? evidenceForRun(run, path30) : evidenceForEvent(first.node.event, first.path);
|
|
8212
8969
|
}
|
|
8213
8970
|
function collectSourceIds(nodes, keys) {
|
|
8214
8971
|
const wanted = keySet(keys);
|
|
@@ -8561,273 +9318,55 @@ var checks = {
|
|
|
8561
9318
|
"Answer contained banned unsupported-answer phrasing.",
|
|
8562
9319
|
firstEvidence(answers, context.run, "attributes.answer"),
|
|
8563
9320
|
{ bannedPhraseCount: banned.length },
|
|
8564
|
-
{ matchedPhraseCount: matches.length }
|
|
8565
|
-
)
|
|
8566
|
-
] : [];
|
|
8567
|
-
});
|
|
8568
|
-
}
|
|
8569
|
-
};
|
|
8570
|
-
function renderEvalMarkdown(result) {
|
|
8571
|
-
const lines = [
|
|
8572
|
-
`# AgentInspect Eval`,
|
|
8573
|
-
"",
|
|
8574
|
-
`Status: ${result.status}`,
|
|
8575
|
-
`Format: ${result.format}`,
|
|
8576
|
-
...result.runId !== void 0 ? [`Run: ${result.runId}`] : [],
|
|
8577
|
-
`Summary: ${result.summary.passed} passed, ${result.summary.failed} failed, ${result.summary.warnings} warnings, ${result.summary.errors} errors`
|
|
8578
|
-
];
|
|
8579
|
-
if (result.diagnostics.length > 0) {
|
|
8580
|
-
lines.push("", "## Diagnostics");
|
|
8581
|
-
for (const diagnostic3 of result.diagnostics) {
|
|
8582
|
-
lines.push(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
8583
|
-
}
|
|
8584
|
-
}
|
|
8585
|
-
if (result.findings.length > 0) {
|
|
8586
|
-
lines.push("", "## Findings");
|
|
8587
|
-
for (const finding of result.findings) {
|
|
8588
|
-
const path29 = finding.evidence[0]?.path;
|
|
8589
|
-
lines.push(`- ${finding.ruleId}: ${finding.message}${path29 ? ` (${path29})` : ""}`);
|
|
8590
|
-
}
|
|
8591
|
-
}
|
|
8592
|
-
return `${lines.join("\n")}
|
|
8593
|
-
`;
|
|
8594
|
-
}
|
|
8595
|
-
|
|
8596
|
-
// packages/cli/src/eval.ts
|
|
8597
|
-
var CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
8598
|
-
var TS_CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
8599
|
-
function diagnostic2(code, message, severity = "error") {
|
|
8600
|
-
return { code, message, severity };
|
|
8601
|
-
}
|
|
8602
|
-
function errorResult3(code, message, format = "unknown") {
|
|
8603
|
-
return {
|
|
8604
|
-
ok: false,
|
|
8605
|
-
status: "error",
|
|
8606
|
-
format,
|
|
8607
|
-
summary: { passed: 0, failed: 0, warnings: 0, errors: 1 },
|
|
8608
|
-
findings: [],
|
|
8609
|
-
diagnostics: [diagnostic2(code, message)]
|
|
8610
|
-
};
|
|
8611
|
-
}
|
|
8612
|
-
function parseNumber2(value, label) {
|
|
8613
|
-
if (value === void 0) return void 0;
|
|
8614
|
-
const parsed = Number(value);
|
|
8615
|
-
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
8616
|
-
throw new Error(`${label} must be a non-negative number.`);
|
|
8617
|
-
}
|
|
8618
|
-
return parsed;
|
|
8619
|
-
}
|
|
8620
|
-
function asStringArray2(value, label) {
|
|
8621
|
-
if (value === void 0) return void 0;
|
|
8622
|
-
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
8623
|
-
throw new Error(`${label} must be an array of strings.`);
|
|
8624
|
-
}
|
|
8625
|
-
return value;
|
|
8626
|
-
}
|
|
8627
|
-
function asConfig(value) {
|
|
8628
|
-
if (value === void 0 || value === null) return {};
|
|
8629
|
-
if (typeof value !== "object" || Array.isArray(value)) {
|
|
8630
|
-
throw new Error("Config must export an object.");
|
|
8631
|
-
}
|
|
8632
|
-
return value;
|
|
8633
|
-
}
|
|
8634
|
-
async function loadConfig2(configPath) {
|
|
8635
|
-
if (configPath === void 0) return {};
|
|
8636
|
-
const extension = path18.extname(configPath);
|
|
8637
|
-
if (TS_CONFIG_EXTENSIONS2.has(extension)) {
|
|
8638
|
-
throw new Error(
|
|
8639
|
-
"TypeScript eval configs require an explicit precompiled JavaScript config or future --config-loader support."
|
|
8640
|
-
);
|
|
8641
|
-
}
|
|
8642
|
-
if (!CONFIG_EXTENSIONS2.has(extension)) {
|
|
8643
|
-
throw new Error("Unsupported eval config extension. Use .json, .js, .mjs, or .cjs.");
|
|
8644
|
-
}
|
|
8645
|
-
const absolute = path18.resolve(configPath);
|
|
8646
|
-
if (extension === ".json") {
|
|
8647
|
-
const raw = await readFile(absolute, "utf-8");
|
|
8648
|
-
return asConfig(JSON.parse(raw));
|
|
8649
|
-
}
|
|
8650
|
-
const mod = await import(pathToFileURL(absolute).href);
|
|
8651
|
-
return asConfig("default" in mod ? mod.default : mod);
|
|
8652
|
-
}
|
|
8653
|
-
function normalizeConfig2(config) {
|
|
8654
|
-
if (config.eval === void 0) return {};
|
|
8655
|
-
if (typeof config.eval !== "object" || Array.isArray(config.eval)) {
|
|
8656
|
-
throw new Error("eval config must be an object.");
|
|
8657
|
-
}
|
|
8658
|
-
return config.eval;
|
|
8659
|
-
}
|
|
8660
|
-
function maybeAdd(rules, value, makeRule) {
|
|
8661
|
-
if (value !== void 0) rules.push(makeRule(value));
|
|
8662
|
-
}
|
|
8663
|
-
function optionObject(value) {
|
|
8664
|
-
if (value === void 0 || value === false) return void 0;
|
|
8665
|
-
if (value === true) return {};
|
|
8666
|
-
if (typeof value !== "object" || Array.isArray(value)) {
|
|
8667
|
-
throw new Error("Eval heuristic config must be a boolean or object.");
|
|
8668
|
-
}
|
|
8669
|
-
return value;
|
|
8670
|
-
}
|
|
8671
|
-
function sourceIdsFromConfig(value) {
|
|
8672
|
-
if (value === void 0) return void 0;
|
|
8673
|
-
if (Array.isArray(value)) return { ids: asStringArray2(value, "eval.requiredSourceIds") ?? [] };
|
|
8674
|
-
if (typeof value !== "object" || Array.isArray(value)) {
|
|
8675
|
-
throw new Error("eval.requiredSourceIds must be an array or object.");
|
|
8676
|
-
}
|
|
8677
|
-
const ids = asStringArray2(value.ids, "eval.requiredSourceIds.ids") ?? [];
|
|
8678
|
-
return { ids, options: { sourceIdKeys: value.sourceIdKeys } };
|
|
8679
|
-
}
|
|
8680
|
-
function buildRules2(config, options) {
|
|
8681
|
-
const evalConfig = normalizeConfig2(config);
|
|
8682
|
-
const rules = [];
|
|
8683
|
-
if (evalConfig.requireSuccess || options.requireSuccess) rules.push(checks.requireSuccess());
|
|
8684
|
-
const requiredTools = [
|
|
8685
|
-
...asStringArray2(evalConfig.requiredTools, "eval.requiredTools") ?? [],
|
|
8686
|
-
...options.requiredTool ?? []
|
|
8687
|
-
];
|
|
8688
|
-
if (requiredTools.length > 0) rules.push(checks.requiredTools(requiredTools));
|
|
8689
|
-
const forbiddenTools = [
|
|
8690
|
-
...asStringArray2(evalConfig.forbiddenTools, "eval.forbiddenTools") ?? [],
|
|
8691
|
-
...options.forbidTool ?? [],
|
|
8692
|
-
...options.forbiddenTool ?? []
|
|
8693
|
-
];
|
|
8694
|
-
if (forbiddenTools.length > 0) rules.push(checks.forbiddenTools(forbiddenTools));
|
|
8695
|
-
const maxDurationMs = parseNumber2(options.maxDurationMs, "--max-duration-ms") ?? evalConfig.maxDurationMs;
|
|
8696
|
-
maybeAdd(rules, maxDurationMs, checks.maxDurationMs);
|
|
8697
|
-
const maxDepth = parseNumber2(options.maxDepth, "--max-depth") ?? evalConfig.maxDepth;
|
|
8698
|
-
maybeAdd(rules, maxDepth, checks.maxDepth);
|
|
8699
|
-
const maxRetries = parseNumber2(options.maxRetries, "--max-retries") ?? evalConfig.maxRetries;
|
|
8700
|
-
maybeAdd(rules, maxRetries, checks.maxRetries);
|
|
8701
|
-
const maxTotalTokens = parseNumber2(options.maxTotalTokens, "--max-total-tokens") ?? evalConfig.maxTotalTokens;
|
|
8702
|
-
maybeAdd(rules, maxTotalTokens, checks.maxTotalTokens);
|
|
8703
|
-
if (evalConfig.requiredRetrievalBeforeGeneration || options.requireRetrievalBeforeGeneration) {
|
|
8704
|
-
rules.push(checks.requiredRetrievalBeforeGeneration());
|
|
8705
|
-
}
|
|
8706
|
-
const requiredDecisionMetadata = [
|
|
8707
|
-
...asStringArray2(
|
|
8708
|
-
evalConfig.requiredDecisionMetadata,
|
|
8709
|
-
"eval.requiredDecisionMetadata"
|
|
8710
|
-
) ?? [],
|
|
8711
|
-
...options.requiredDecisionMetadata ?? []
|
|
8712
|
-
];
|
|
8713
|
-
if (requiredDecisionMetadata.length > 0) {
|
|
8714
|
-
rules.push(checks.requiredDecisionMetadata(requiredDecisionMetadata));
|
|
8715
|
-
}
|
|
8716
|
-
const contextOverlap = optionObject(evalConfig.contextOverlap);
|
|
8717
|
-
const minOverlap = parseNumber2(options.minContextOverlap, "--min-context-overlap");
|
|
8718
|
-
const minSharedTerms = parseNumber2(options.minSharedTerms, "--min-shared-terms");
|
|
8719
|
-
if (contextOverlap !== void 0 || options.contextOverlap || minOverlap !== void 0 || minSharedTerms !== void 0) {
|
|
8720
|
-
rules.push(checks.contextOverlap({ ...contextOverlap, minOverlap, minSharedTerms }));
|
|
8721
|
-
}
|
|
8722
|
-
const quoteOverlap = optionObject(evalConfig.quoteOverlap);
|
|
8723
|
-
if (quoteOverlap !== void 0 || options.quoteOverlap) {
|
|
8724
|
-
rules.push(checks.quoteOverlap(quoteOverlap));
|
|
8725
|
-
}
|
|
8726
|
-
const citationPresence = optionObject(evalConfig.citationPresence);
|
|
8727
|
-
if (citationPresence !== void 0 || options.citationPresence) {
|
|
8728
|
-
rules.push(checks.citationPresence(citationPresence));
|
|
8729
|
-
}
|
|
8730
|
-
const sourceIds = sourceIdsFromConfig(evalConfig.requiredSourceIds);
|
|
8731
|
-
const requiredSourceIds = [...sourceIds?.ids ?? [], ...options.requiredSourceId ?? []];
|
|
8732
|
-
if (requiredSourceIds.length > 0) {
|
|
8733
|
-
rules.push(checks.requiredSourceIds(requiredSourceIds, sourceIds?.options));
|
|
8734
|
-
}
|
|
8735
|
-
const cliAnswerLength = {
|
|
8736
|
-
minCharacters: parseNumber2(options.minAnswerCharacters, "--min-answer-characters"),
|
|
8737
|
-
maxCharacters: parseNumber2(options.maxAnswerCharacters, "--max-answer-characters"),
|
|
8738
|
-
minWords: parseNumber2(options.minAnswerWords, "--min-answer-words"),
|
|
8739
|
-
maxWords: parseNumber2(options.maxAnswerWords, "--max-answer-words")
|
|
8740
|
-
};
|
|
8741
|
-
const hasCliAnswerLength = Object.values(cliAnswerLength).some((value) => value !== void 0);
|
|
8742
|
-
if (evalConfig.answerLengthBounds !== void 0 || hasCliAnswerLength) {
|
|
8743
|
-
rules.push(checks.answerLengthBounds({ ...evalConfig.answerLengthBounds, ...cliAnswerLength }));
|
|
8744
|
-
}
|
|
8745
|
-
const bannedPhrases = [
|
|
8746
|
-
...asStringArray2(
|
|
8747
|
-
evalConfig.bannedUnsupportedPhrases,
|
|
8748
|
-
"eval.bannedUnsupportedPhrases"
|
|
8749
|
-
) ?? [],
|
|
8750
|
-
...options.bannedPhrase ?? []
|
|
8751
|
-
];
|
|
8752
|
-
if (bannedPhrases.length > 0) rules.push(checks.bannedUnsupportedPhrases(bannedPhrases));
|
|
8753
|
-
return rules.length > 0 ? rules : void 0;
|
|
8754
|
-
}
|
|
8755
|
-
function exitCodeFor2(result) {
|
|
8756
|
-
if (result.status === "pass") return 0;
|
|
8757
|
-
if (result.status === "fail") return 1;
|
|
8758
|
-
return 2;
|
|
8759
|
-
}
|
|
8760
|
-
function stable3(value) {
|
|
8761
|
-
if (Array.isArray(value)) return value.map(stable3);
|
|
8762
|
-
if (value === null || typeof value !== "object") return value;
|
|
8763
|
-
const record = value;
|
|
8764
|
-
return Object.fromEntries(
|
|
8765
|
-
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable3(record[key])])
|
|
8766
|
-
);
|
|
8767
|
-
}
|
|
8768
|
-
function printJson2(result) {
|
|
8769
|
-
console.log(JSON.stringify(stable3(result), null, 2));
|
|
8770
|
-
}
|
|
8771
|
-
function printHuman2(result) {
|
|
8772
|
-
console.log(`Eval status: ${result.status}`);
|
|
8773
|
-
console.log(`Format: ${result.format}`);
|
|
8774
|
-
if (result.runId !== void 0) console.log(`Run: ${result.runId}`);
|
|
8775
|
-
console.log(
|
|
8776
|
-
`Summary: ${result.summary.failed} failed, ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
8777
|
-
);
|
|
8778
|
-
for (const diagnostic3 of result.diagnostics) {
|
|
8779
|
-
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
8780
|
-
}
|
|
8781
|
-
for (const finding of result.findings) {
|
|
8782
|
-
const path29 = finding.evidence[0]?.path;
|
|
8783
|
-
console.log(`- ${finding.ruleId}: ${finding.message}${path29 ? ` (${path29})` : ""}`);
|
|
8784
|
-
}
|
|
8785
|
-
}
|
|
8786
|
-
function readErrorResult2(error) {
|
|
8787
|
-
if (error instanceof TraceReadError) {
|
|
8788
|
-
const code = error.code === "unsupported_format" ? "AI_EVAL_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_EVAL_AMBIGUOUS_FORMAT" : "AI_EVAL_TRACE_UNREADABLE";
|
|
8789
|
-
return errorResult3(code, error.message);
|
|
8790
|
-
}
|
|
8791
|
-
return errorResult3(
|
|
8792
|
-
"AI_EVAL_TRACE_UNREADABLE",
|
|
8793
|
-
error instanceof Error ? error.message : String(error)
|
|
8794
|
-
);
|
|
8795
|
-
}
|
|
8796
|
-
async function evalCommand(target, options = {}, stdin = process.stdin) {
|
|
8797
|
-
let result;
|
|
8798
|
-
let phase = "config";
|
|
8799
|
-
try {
|
|
8800
|
-
const config = await loadConfig2(options.config);
|
|
8801
|
-
const rules = buildRules2(config, options);
|
|
8802
|
-
phase = "read";
|
|
8803
|
-
const input3 = await inputFromTarget(target, options, stdin);
|
|
8804
|
-
const read = await openTrace(input3, {
|
|
8805
|
-
...options.format !== void 0 ? { format: options.format } : {}
|
|
8806
|
-
});
|
|
8807
|
-
result = await evalRun(read, {
|
|
8808
|
-
...rules !== void 0 ? { checks: rules } : {},
|
|
8809
|
-
...options.run !== void 0 ? { runId: options.run } : {}
|
|
9321
|
+
{ matchedPhraseCount: matches.length }
|
|
9322
|
+
)
|
|
9323
|
+
] : [];
|
|
8810
9324
|
});
|
|
8811
|
-
}
|
|
8812
|
-
|
|
8813
|
-
|
|
8814
|
-
|
|
8815
|
-
|
|
8816
|
-
|
|
8817
|
-
|
|
9325
|
+
}
|
|
9326
|
+
};
|
|
9327
|
+
function renderEvalMarkdown(result) {
|
|
9328
|
+
const lines = [
|
|
9329
|
+
`# AgentInspect Eval`,
|
|
9330
|
+
"",
|
|
9331
|
+
`Status: ${result.status}`,
|
|
9332
|
+
`Format: ${result.format}`,
|
|
9333
|
+
...result.runId !== void 0 ? [`Run: ${result.runId}`] : [],
|
|
9334
|
+
`Summary: ${result.summary.passed} passed, ${result.summary.failed} failed, ${result.summary.warnings} warnings, ${result.summary.errors} errors`
|
|
9335
|
+
];
|
|
9336
|
+
if (result.diagnostics.length > 0) {
|
|
9337
|
+
lines.push("", "## Diagnostics");
|
|
9338
|
+
for (const diagnostic3 of result.diagnostics) {
|
|
9339
|
+
lines.push(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
8818
9340
|
}
|
|
8819
9341
|
}
|
|
8820
|
-
|
|
8821
|
-
|
|
8822
|
-
|
|
8823
|
-
|
|
9342
|
+
if (result.findings.length > 0) {
|
|
9343
|
+
lines.push("", "## Findings");
|
|
9344
|
+
for (const finding of result.findings) {
|
|
9345
|
+
const path30 = finding.evidence[0]?.path;
|
|
9346
|
+
lines.push(`- ${finding.ruleId}: ${finding.message}${path30 ? ` (${path30})` : ""}`);
|
|
9347
|
+
}
|
|
9348
|
+
}
|
|
9349
|
+
return `${lines.join("\n")}
|
|
9350
|
+
`;
|
|
8824
9351
|
}
|
|
8825
|
-
|
|
8826
|
-
|
|
8827
|
-
var
|
|
8828
|
-
var
|
|
8829
|
-
|
|
8830
|
-
|
|
9352
|
+
|
|
9353
|
+
// packages/cli/src/eval.ts
|
|
9354
|
+
var CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".json", ".js", ".mjs", ".cjs"]);
|
|
9355
|
+
var TS_CONFIG_EXTENSIONS2 = /* @__PURE__ */ new Set([".ts", ".mts", ".cts"]);
|
|
9356
|
+
function diagnostic2(code, message, severity = "error") {
|
|
9357
|
+
return { code, message, severity };
|
|
9358
|
+
}
|
|
9359
|
+
function errorResult3(code, message, format = "unknown") {
|
|
9360
|
+
return {
|
|
9361
|
+
ok: false,
|
|
9362
|
+
status: "error",
|
|
9363
|
+
format,
|
|
9364
|
+
summary: { passed: 0, failed: 0, warnings: 0, errors: 1 },
|
|
9365
|
+
findings: [],
|
|
9366
|
+
diagnostics: [diagnostic2(code, message)]
|
|
9367
|
+
};
|
|
9368
|
+
}
|
|
9369
|
+
function parseNumber2(value, label) {
|
|
8831
9370
|
if (value === void 0) return void 0;
|
|
8832
9371
|
const parsed = Number(value);
|
|
8833
9372
|
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
@@ -8835,368 +9374,210 @@ function parseLimit3(value, label) {
|
|
|
8835
9374
|
}
|
|
8836
9375
|
return parsed;
|
|
8837
9376
|
}
|
|
8838
|
-
function
|
|
8839
|
-
if (
|
|
8840
|
-
if (value
|
|
8841
|
-
|
|
8842
|
-
|
|
8843
|
-
|
|
8844
|
-
);
|
|
8845
|
-
}
|
|
8846
|
-
function safetyDiagnostic(code, message, severity = "error") {
|
|
8847
|
-
return { code, message, severity };
|
|
8848
|
-
}
|
|
8849
|
-
function warningDiagnostics(warnings, unsupportedFields) {
|
|
8850
|
-
return [
|
|
8851
|
-
...warnings.map(
|
|
8852
|
-
(warning) => safetyDiagnostic(
|
|
8853
|
-
warning.code,
|
|
8854
|
-
warning.message,
|
|
8855
|
-
warning.severity === "error" ? "error" : "warning"
|
|
8856
|
-
)
|
|
8857
|
-
),
|
|
8858
|
-
...unsupportedFields.map(
|
|
8859
|
-
(field) => safetyDiagnostic(
|
|
8860
|
-
"unsupported_field",
|
|
8861
|
-
`Reader reported unsupported field: ${field}`,
|
|
8862
|
-
"warning"
|
|
8863
|
-
)
|
|
8864
|
-
)
|
|
8865
|
-
];
|
|
9377
|
+
function asStringArray2(value, label) {
|
|
9378
|
+
if (value === void 0) return void 0;
|
|
9379
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
|
|
9380
|
+
throw new Error(`${label} must be an array of strings.`);
|
|
9381
|
+
}
|
|
9382
|
+
return value;
|
|
8866
9383
|
}
|
|
8867
|
-
function
|
|
8868
|
-
|
|
9384
|
+
function asConfig(value) {
|
|
9385
|
+
if (value === void 0 || value === null) return {};
|
|
9386
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
9387
|
+
throw new Error("Config must export an object.");
|
|
9388
|
+
}
|
|
9389
|
+
return value;
|
|
8869
9390
|
}
|
|
8870
|
-
function
|
|
8871
|
-
if (
|
|
8872
|
-
|
|
8873
|
-
if (
|
|
8874
|
-
|
|
8875
|
-
|
|
9391
|
+
async function loadConfig2(configPath) {
|
|
9392
|
+
if (configPath === void 0) return {};
|
|
9393
|
+
const extension = path19.extname(configPath);
|
|
9394
|
+
if (TS_CONFIG_EXTENSIONS2.has(extension)) {
|
|
9395
|
+
throw new Error(
|
|
9396
|
+
"TypeScript eval configs require an explicit precompiled JavaScript config or future --config-loader support."
|
|
9397
|
+
);
|
|
9398
|
+
}
|
|
9399
|
+
if (!CONFIG_EXTENSIONS2.has(extension)) {
|
|
9400
|
+
throw new Error("Unsupported eval config extension. Use .json, .js, .mjs, or .cjs.");
|
|
9401
|
+
}
|
|
9402
|
+
const absolute = path19.resolve(configPath);
|
|
9403
|
+
if (extension === ".json") {
|
|
9404
|
+
const raw = await readFile(absolute, "utf-8");
|
|
9405
|
+
return asConfig(JSON.parse(raw));
|
|
9406
|
+
}
|
|
9407
|
+
const mod = await import(pathToFileURL(absolute).href);
|
|
9408
|
+
return asConfig("default" in mod ? mod.default : mod);
|
|
8876
9409
|
}
|
|
8877
|
-
function
|
|
8878
|
-
|
|
8879
|
-
|
|
8880
|
-
|
|
8881
|
-
|
|
8882
|
-
|
|
8883
|
-
return {
|
|
8884
|
-
ok: status === "SAFE" || status === "SAFE WITH WARNINGS",
|
|
8885
|
-
command: parts.command,
|
|
8886
|
-
status,
|
|
8887
|
-
format: parts.format,
|
|
8888
|
-
...parts.runId !== void 0 ? { runId: parts.runId } : {},
|
|
8889
|
-
summary: {
|
|
8890
|
-
findings: findings.length,
|
|
8891
|
-
warnings: diagnostics.filter((item) => item.severity === "warning").length + findings.filter((item) => item.severity === "warning").length,
|
|
8892
|
-
errors: diagnostics.filter((item) => item.severity === "error").length + findings.filter((item) => item.severity === "error").length
|
|
8893
|
-
},
|
|
8894
|
-
findings,
|
|
8895
|
-
diagnostics,
|
|
8896
|
-
warnings,
|
|
8897
|
-
unsupportedFields,
|
|
8898
|
-
note: BEST_EFFORT_NOTE,
|
|
8899
|
-
...parts.sourceAssessment !== void 0 ? { sourceAssessment: parts.sourceAssessment } : {},
|
|
8900
|
-
...parts.artifactAssessment !== void 0 ? { artifactAssessment: parts.artifactAssessment } : {},
|
|
8901
|
-
...parts.redactionSummary !== void 0 ? { redactionSummary: parts.redactionSummary } : {}
|
|
8902
|
-
};
|
|
9410
|
+
function normalizeConfig2(config) {
|
|
9411
|
+
if (config.eval === void 0) return {};
|
|
9412
|
+
if (typeof config.eval !== "object" || Array.isArray(config.eval)) {
|
|
9413
|
+
throw new Error("eval config must be an object.");
|
|
9414
|
+
}
|
|
9415
|
+
return config.eval;
|
|
8903
9416
|
}
|
|
8904
|
-
function
|
|
8905
|
-
|
|
8906
|
-
status: result.status,
|
|
8907
|
-
summary: result.summary,
|
|
8908
|
-
findings: result.findings
|
|
8909
|
-
};
|
|
9417
|
+
function maybeAdd(rules, value, makeRule) {
|
|
9418
|
+
if (value !== void 0) rules.push(makeRule(value));
|
|
8910
9419
|
}
|
|
8911
|
-
function
|
|
8912
|
-
if (
|
|
8913
|
-
|
|
8914
|
-
|
|
8915
|
-
|
|
8916
|
-
format: "unknown",
|
|
8917
|
-
diagnostics: [safetyDiagnostic(code, error.message)],
|
|
8918
|
-
warnings: error.warnings
|
|
8919
|
-
});
|
|
9420
|
+
function optionObject(value) {
|
|
9421
|
+
if (value === void 0 || value === false) return void 0;
|
|
9422
|
+
if (value === true) return {};
|
|
9423
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
9424
|
+
throw new Error("Eval heuristic config must be a boolean or object.");
|
|
8920
9425
|
}
|
|
8921
|
-
return
|
|
8922
|
-
command,
|
|
8923
|
-
format: "unknown",
|
|
8924
|
-
diagnostics: [
|
|
8925
|
-
safetyDiagnostic(
|
|
8926
|
-
"AI_SAFETY_TRACE_UNREADABLE",
|
|
8927
|
-
error instanceof Error ? error.message : String(error)
|
|
8928
|
-
)
|
|
8929
|
-
]
|
|
8930
|
-
});
|
|
9426
|
+
return value;
|
|
8931
9427
|
}
|
|
8932
|
-
function
|
|
8933
|
-
return
|
|
8934
|
-
|
|
8935
|
-
|
|
8936
|
-
|
|
8937
|
-
|
|
8938
|
-
|
|
8939
|
-
|
|
8940
|
-
)
|
|
8941
|
-
]
|
|
8942
|
-
});
|
|
9428
|
+
function sourceIdsFromConfig(value) {
|
|
9429
|
+
if (value === void 0) return void 0;
|
|
9430
|
+
if (Array.isArray(value)) return { ids: asStringArray2(value, "eval.requiredSourceIds") ?? [] };
|
|
9431
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
9432
|
+
throw new Error("eval.requiredSourceIds must be an array or object.");
|
|
9433
|
+
}
|
|
9434
|
+
const ids = asStringArray2(value.ids, "eval.requiredSourceIds.ids") ?? [];
|
|
9435
|
+
return { ids, options: { sourceIdKeys: value.sourceIdKeys } };
|
|
8943
9436
|
}
|
|
8944
|
-
function
|
|
8945
|
-
const
|
|
8946
|
-
const
|
|
8947
|
-
|
|
8948
|
-
const
|
|
8949
|
-
|
|
8950
|
-
|
|
8951
|
-
createSafetyRedactionRule(),
|
|
8952
|
-
createSafetySecretPatternRule(),
|
|
8953
|
-
createSafetyOversizedAttributeRule({
|
|
8954
|
-
maxStringLength: maxStringLength2,
|
|
8955
|
-
maxArrayLength,
|
|
8956
|
-
maxObjectKeys,
|
|
8957
|
-
maxSerializedBytes
|
|
8958
|
-
})
|
|
9437
|
+
function buildRules2(config, options) {
|
|
9438
|
+
const evalConfig = normalizeConfig2(config);
|
|
9439
|
+
const rules = [];
|
|
9440
|
+
if (evalConfig.requireSuccess || options.requireSuccess) rules.push(checks.requireSuccess());
|
|
9441
|
+
const requiredTools = [
|
|
9442
|
+
...asStringArray2(evalConfig.requiredTools, "eval.requiredTools") ?? [],
|
|
9443
|
+
...options.requiredTool ?? []
|
|
8959
9444
|
];
|
|
8960
|
-
|
|
8961
|
-
|
|
8962
|
-
|
|
8963
|
-
|
|
8964
|
-
...
|
|
8965
|
-
|
|
8966
|
-
|
|
8967
|
-
|
|
8968
|
-
|
|
8969
|
-
|
|
8970
|
-
|
|
8971
|
-
|
|
8972
|
-
|
|
8973
|
-
|
|
8974
|
-
|
|
9445
|
+
if (requiredTools.length > 0) rules.push(checks.requiredTools(requiredTools));
|
|
9446
|
+
const forbiddenTools = [
|
|
9447
|
+
...asStringArray2(evalConfig.forbiddenTools, "eval.forbiddenTools") ?? [],
|
|
9448
|
+
...options.forbidTool ?? [],
|
|
9449
|
+
...options.forbiddenTool ?? []
|
|
9450
|
+
];
|
|
9451
|
+
if (forbiddenTools.length > 0) rules.push(checks.forbiddenTools(forbiddenTools));
|
|
9452
|
+
const maxDurationMs = parseNumber2(options.maxDurationMs, "--max-duration-ms") ?? evalConfig.maxDurationMs;
|
|
9453
|
+
maybeAdd(rules, maxDurationMs, checks.maxDurationMs);
|
|
9454
|
+
const maxDepth = parseNumber2(options.maxDepth, "--max-depth") ?? evalConfig.maxDepth;
|
|
9455
|
+
maybeAdd(rules, maxDepth, checks.maxDepth);
|
|
9456
|
+
const maxRetries = parseNumber2(options.maxRetries, "--max-retries") ?? evalConfig.maxRetries;
|
|
9457
|
+
maybeAdd(rules, maxRetries, checks.maxRetries);
|
|
9458
|
+
const maxTotalTokens = parseNumber2(options.maxTotalTokens, "--max-total-tokens") ?? evalConfig.maxTotalTokens;
|
|
9459
|
+
maybeAdd(rules, maxTotalTokens, checks.maxTotalTokens);
|
|
9460
|
+
if (evalConfig.requiredRetrievalBeforeGeneration || options.requireRetrievalBeforeGeneration) {
|
|
9461
|
+
rules.push(checks.requiredRetrievalBeforeGeneration());
|
|
8975
9462
|
}
|
|
8976
|
-
|
|
8977
|
-
|
|
9463
|
+
const requiredDecisionMetadata = [
|
|
9464
|
+
...asStringArray2(
|
|
9465
|
+
evalConfig.requiredDecisionMetadata,
|
|
9466
|
+
"eval.requiredDecisionMetadata"
|
|
9467
|
+
) ?? [],
|
|
9468
|
+
...options.requiredDecisionMetadata ?? []
|
|
9469
|
+
];
|
|
9470
|
+
if (requiredDecisionMetadata.length > 0) {
|
|
9471
|
+
rules.push(checks.requiredDecisionMetadata(requiredDecisionMetadata));
|
|
8978
9472
|
}
|
|
8979
|
-
|
|
8980
|
-
|
|
9473
|
+
const contextOverlap = optionObject(evalConfig.contextOverlap);
|
|
9474
|
+
const minOverlap = parseNumber2(options.minContextOverlap, "--min-context-overlap");
|
|
9475
|
+
const minSharedTerms = parseNumber2(options.minSharedTerms, "--min-shared-terms");
|
|
9476
|
+
if (contextOverlap !== void 0 || options.contextOverlap || minOverlap !== void 0 || minSharedTerms !== void 0) {
|
|
9477
|
+
rules.push(checks.contextOverlap({ ...contextOverlap, minOverlap, minSharedTerms }));
|
|
8981
9478
|
}
|
|
8982
|
-
|
|
8983
|
-
|
|
9479
|
+
const quoteOverlap = optionObject(evalConfig.quoteOverlap);
|
|
9480
|
+
if (quoteOverlap !== void 0 || options.quoteOverlap) {
|
|
9481
|
+
rules.push(checks.quoteOverlap(quoteOverlap));
|
|
8984
9482
|
}
|
|
8985
|
-
|
|
8986
|
-
|
|
8987
|
-
|
|
8988
|
-
const runs = runId === void 0 ? read.runs : read.runs.filter((run) => run.runId === runId);
|
|
8989
|
-
const out = [];
|
|
8990
|
-
for (const run of runs) {
|
|
8991
|
-
for (const node of flattenNodes(run.children)) {
|
|
8992
|
-
const attrs = node.event.attributes;
|
|
8993
|
-
if (attrs === void 0) continue;
|
|
8994
|
-
const result = redact(attrs, { profile: "share" });
|
|
8995
|
-
for (const finding of result.findings) {
|
|
8996
|
-
if (finding.action === "keep") continue;
|
|
8997
|
-
const taxonomy = classifyRedactionDetector(finding.detector);
|
|
8998
|
-
out.push({
|
|
8999
|
-
ruleId: "safety.redactDetector",
|
|
9000
|
-
severity: detectorSeverity(finding),
|
|
9001
|
-
status: finding.severity === "error" ? "fail" : "warning",
|
|
9002
|
-
message: `Redaction detector ${finding.detector} matched ${finding.matchKind} at ${finding.path}.`,
|
|
9003
|
-
expected: "redacted trace content",
|
|
9004
|
-
actual: finding.detector,
|
|
9005
|
-
evidence: [
|
|
9006
|
-
{
|
|
9007
|
-
runId: node.event.runId,
|
|
9008
|
-
eventId: node.event.eventId,
|
|
9009
|
-
...node.event.parentId !== void 0 ? { parentId: node.event.parentId } : {},
|
|
9010
|
-
kind: node.event.kind,
|
|
9011
|
-
name: node.event.name,
|
|
9012
|
-
...node.event.status !== void 0 ? { status: node.event.status } : {},
|
|
9013
|
-
path: `attributes.${finding.path.replace(/^\$\.?/, "")}`
|
|
9014
|
-
}
|
|
9015
|
-
],
|
|
9016
|
-
category: taxonomy.category,
|
|
9017
|
-
confidence: taxonomy.confidence,
|
|
9018
|
-
detector: finding.detector,
|
|
9019
|
-
action: finding.action
|
|
9020
|
-
});
|
|
9021
|
-
}
|
|
9022
|
-
}
|
|
9483
|
+
const citationPresence = optionObject(evalConfig.citationPresence);
|
|
9484
|
+
if (citationPresence !== void 0 || options.citationPresence) {
|
|
9485
|
+
rules.push(checks.citationPresence(citationPresence));
|
|
9023
9486
|
}
|
|
9024
|
-
|
|
9025
|
-
|
|
9026
|
-
|
|
9027
|
-
|
|
9028
|
-
}
|
|
9487
|
+
const sourceIds = sourceIdsFromConfig(evalConfig.requiredSourceIds);
|
|
9488
|
+
const requiredSourceIds = [...sourceIds?.ids ?? [], ...options.requiredSourceId ?? []];
|
|
9489
|
+
if (requiredSourceIds.length > 0) {
|
|
9490
|
+
rules.push(checks.requiredSourceIds(requiredSourceIds, sourceIds?.options));
|
|
9491
|
+
}
|
|
9492
|
+
const cliAnswerLength = {
|
|
9493
|
+
minCharacters: parseNumber2(options.minAnswerCharacters, "--min-answer-characters"),
|
|
9494
|
+
maxCharacters: parseNumber2(options.maxAnswerCharacters, "--max-answer-characters"),
|
|
9495
|
+
minWords: parseNumber2(options.minAnswerWords, "--min-answer-words"),
|
|
9496
|
+
maxWords: parseNumber2(options.maxAnswerWords, "--max-answer-words")
|
|
9497
|
+
};
|
|
9498
|
+
const hasCliAnswerLength = Object.values(cliAnswerLength).some((value) => value !== void 0);
|
|
9499
|
+
if (evalConfig.answerLengthBounds !== void 0 || hasCliAnswerLength) {
|
|
9500
|
+
rules.push(checks.answerLengthBounds({ ...evalConfig.answerLengthBounds, ...cliAnswerLength }));
|
|
9501
|
+
}
|
|
9502
|
+
const bannedPhrases = [
|
|
9503
|
+
...asStringArray2(
|
|
9504
|
+
evalConfig.bannedUnsupportedPhrases,
|
|
9505
|
+
"eval.bannedUnsupportedPhrases"
|
|
9506
|
+
) ?? [],
|
|
9507
|
+
...options.bannedPhrase ?? []
|
|
9508
|
+
];
|
|
9509
|
+
if (bannedPhrases.length > 0) rules.push(checks.bannedUnsupportedPhrases(bannedPhrases));
|
|
9510
|
+
return rules.length > 0 ? rules : void 0;
|
|
9029
9511
|
}
|
|
9030
9512
|
function exitCodeFor3(result) {
|
|
9031
|
-
if (result.status === "
|
|
9032
|
-
if (result.status === "
|
|
9513
|
+
if (result.status === "pass") return 0;
|
|
9514
|
+
if (result.status === "fail") return 1;
|
|
9033
9515
|
return 2;
|
|
9034
9516
|
}
|
|
9035
|
-
function
|
|
9036
|
-
|
|
9037
|
-
|
|
9038
|
-
const
|
|
9039
|
-
|
|
9040
|
-
|
|
9041
|
-
|
|
9042
|
-
return [
|
|
9043
|
-
` Matched: detector=${detector}; path=${path29}; category=${category}`,
|
|
9044
|
-
` Why: ${finding.message}`,
|
|
9045
|
-
` Confidence: ${confidence}`,
|
|
9046
|
-
` Redaction: ${redactionHint}`,
|
|
9047
|
-
` Override: configure a custom redaction/detector rule locally (see docs/SAFETY-POLICY.md); do not weaken defaults globally.`,
|
|
9048
|
-
` Bundle gate: ${blocksBundle ? "blocks share-safe bundle unless --allow-unsafe" : "does not block by itself (warning/info)"}`
|
|
9049
|
-
];
|
|
9517
|
+
function stable4(value) {
|
|
9518
|
+
if (Array.isArray(value)) return value.map(stable4);
|
|
9519
|
+
if (value === null || typeof value !== "object") return value;
|
|
9520
|
+
const record = value;
|
|
9521
|
+
return Object.fromEntries(
|
|
9522
|
+
Object.keys(record).sort((a, b) => a.localeCompare(b)).map((key) => [key, stable4(record[key])])
|
|
9523
|
+
);
|
|
9050
9524
|
}
|
|
9051
|
-
function
|
|
9052
|
-
|
|
9053
|
-
return {
|
|
9054
|
-
ruleId: finding.ruleId,
|
|
9055
|
-
detector: finding.detector ?? finding.ruleId,
|
|
9056
|
-
path: finding.evidence[0]?.path,
|
|
9057
|
-
category: finding.category,
|
|
9058
|
-
confidence: finding.confidence,
|
|
9059
|
-
action: finding.action,
|
|
9060
|
-
blocksBundle: blocks,
|
|
9061
|
-
// Never include matched secret/PII values.
|
|
9062
|
-
message: finding.message
|
|
9063
|
-
};
|
|
9525
|
+
function printJson3(result) {
|
|
9526
|
+
console.log(JSON.stringify(stable4(result), null, 2));
|
|
9064
9527
|
}
|
|
9065
|
-
function printHuman3(result
|
|
9066
|
-
console.log(`
|
|
9528
|
+
function printHuman3(result) {
|
|
9529
|
+
console.log(`Eval status: ${result.status}`);
|
|
9067
9530
|
console.log(`Format: ${result.format}`);
|
|
9068
9531
|
if (result.runId !== void 0) console.log(`Run: ${result.runId}`);
|
|
9069
|
-
if (result.sourceAssessment !== void 0 && result.artifactAssessment !== void 0) {
|
|
9070
|
-
console.log(`Source assessment: ${result.sourceAssessment.status}`);
|
|
9071
|
-
console.log(`Artifact assessment: ${result.artifactAssessment.status}`);
|
|
9072
|
-
if (result.redactionSummary !== void 0) {
|
|
9073
|
-
console.log(
|
|
9074
|
-
`Redaction: profile=${result.redactionSummary.profile}, findings=${result.redactionSummary.findings}`
|
|
9075
|
-
);
|
|
9076
|
-
}
|
|
9077
|
-
}
|
|
9078
9532
|
console.log(
|
|
9079
|
-
`Summary: ${result.summary.
|
|
9533
|
+
`Summary: ${result.summary.failed} failed, ${result.summary.warnings} warning(s), ${result.summary.errors} error(s)`
|
|
9080
9534
|
);
|
|
9081
9535
|
for (const diagnostic3 of result.diagnostics) {
|
|
9082
9536
|
console.log(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
9083
9537
|
}
|
|
9084
9538
|
for (const finding of result.findings) {
|
|
9085
|
-
const
|
|
9086
|
-
|
|
9087
|
-
console.log(`- ${finding.ruleId}: ${finding.message}${path29 ? ` (${path29})` : ""}${taxonomy}`);
|
|
9088
|
-
if (explain) {
|
|
9089
|
-
const blocks = finding.severity === "error" || finding.status === "fail";
|
|
9090
|
-
for (const line of explainFinding(finding, blocks)) {
|
|
9091
|
-
console.log(line);
|
|
9092
|
-
}
|
|
9093
|
-
}
|
|
9539
|
+
const path30 = finding.evidence[0]?.path;
|
|
9540
|
+
console.log(`- ${finding.ruleId}: ${finding.message}${path30 ? ` (${path30})` : ""}`);
|
|
9094
9541
|
}
|
|
9095
|
-
console.log(`Note: ${result.note}`);
|
|
9096
9542
|
}
|
|
9097
|
-
function
|
|
9098
|
-
|
|
9099
|
-
|
|
9543
|
+
function readErrorResult3(error) {
|
|
9544
|
+
if (error instanceof TraceReadError) {
|
|
9545
|
+
const code = error.code === "unsupported_format" ? "AI_EVAL_UNSUPPORTED_FORMAT" : error.code === "ambiguous_format" ? "AI_EVAL_AMBIGUOUS_FORMAT" : "AI_EVAL_TRACE_UNREADABLE";
|
|
9546
|
+
return errorResult3(code, error.message);
|
|
9547
|
+
}
|
|
9548
|
+
return errorResult3(
|
|
9549
|
+
"AI_EVAL_TRACE_UNREADABLE",
|
|
9550
|
+
error instanceof Error ? error.message : String(error)
|
|
9551
|
+
);
|
|
9100
9552
|
}
|
|
9101
|
-
async function
|
|
9553
|
+
async function evalCommand(target, options = {}, stdin = process.stdin) {
|
|
9102
9554
|
let result;
|
|
9555
|
+
let phase = "config";
|
|
9103
9556
|
try {
|
|
9557
|
+
const config = await loadConfig2(options.config);
|
|
9558
|
+
const rules = buildRules2(config, options);
|
|
9559
|
+
phase = "read";
|
|
9104
9560
|
const input3 = await inputFromTarget(target, options, stdin);
|
|
9105
9561
|
const read = await openTrace(input3, {
|
|
9106
9562
|
...options.format !== void 0 ? { format: options.format } : {}
|
|
9107
9563
|
});
|
|
9108
|
-
|
|
9109
|
-
...
|
|
9564
|
+
result = await evalRun(read, {
|
|
9565
|
+
...rules !== void 0 ? { checks: rules } : {},
|
|
9110
9566
|
...options.run !== void 0 ? { runId: options.run } : {}
|
|
9111
9567
|
});
|
|
9112
|
-
|
|
9113
|
-
|
|
9568
|
+
} catch (error) {
|
|
9569
|
+
if (phase === "config") {
|
|
9570
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
9571
|
+
const code = message.startsWith("--") ? "AI_EVAL_INVALID_ARGUMENTS" : error instanceof SyntaxError || message.includes("Unsupported eval config extension") || message.includes("TypeScript eval configs") || message.includes("Config must") || message.includes("eval config") || message.includes("must be an array") ? "AI_EVAL_INVALID_CONFIG" : "AI_EVAL_CONFIG_LOAD_FAILED";
|
|
9572
|
+
result = errorResult3(code, message);
|
|
9114
9573
|
} else {
|
|
9115
|
-
|
|
9116
|
-
const rawContent = input3.type === "string" ? input3.content : input3.type === "file" ? await readFile(input3.path, "utf-8") : void 0;
|
|
9117
|
-
if (rawContent === void 0) {
|
|
9118
|
-
result = {
|
|
9119
|
-
...source,
|
|
9120
|
-
command: "verify-safe",
|
|
9121
|
-
sourceAssessment: layerFromResult(source)
|
|
9122
|
-
};
|
|
9123
|
-
} else {
|
|
9124
|
-
const redacted = redactTraceContent(rawContent, profile);
|
|
9125
|
-
const artifactRead = await openTrace(
|
|
9126
|
-
{ type: "string", content: redacted.content },
|
|
9127
|
-
{ format: options.format ?? "agent-inspect-jsonl" }
|
|
9128
|
-
);
|
|
9129
|
-
const artifact = assessOpenedTrace(artifactRead, {
|
|
9130
|
-
...options,
|
|
9131
|
-
...options.run !== void 0 ? { runId: options.run } : {}
|
|
9132
|
-
});
|
|
9133
|
-
const detectors = [
|
|
9134
|
-
...new Set(redacted.findings.map((finding) => finding.detector))
|
|
9135
|
-
].sort((a, b) => a.localeCompare(b));
|
|
9136
|
-
result = resultFromParts({
|
|
9137
|
-
command: "verify-safe",
|
|
9138
|
-
format: artifact.format,
|
|
9139
|
-
runId: artifact.runId ?? source.runId,
|
|
9140
|
-
findings: artifact.findings,
|
|
9141
|
-
diagnostics: artifact.diagnostics,
|
|
9142
|
-
warnings: artifact.warnings,
|
|
9143
|
-
unsupportedFields: artifact.unsupportedFields,
|
|
9144
|
-
status: artifact.status,
|
|
9145
|
-
sourceAssessment: layerFromResult(source),
|
|
9146
|
-
artifactAssessment: layerFromResult(artifact),
|
|
9147
|
-
redactionSummary: {
|
|
9148
|
-
profile,
|
|
9149
|
-
findings: redacted.findings.length,
|
|
9150
|
-
detectors
|
|
9151
|
-
}
|
|
9152
|
-
});
|
|
9153
|
-
}
|
|
9574
|
+
result = readErrorResult3(error);
|
|
9154
9575
|
}
|
|
9155
|
-
} catch (error) {
|
|
9156
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
9157
|
-
result = message.startsWith("--") ? invalidArgumentResult(command, error) : readErrorResult3(command, error);
|
|
9158
9576
|
}
|
|
9159
9577
|
process.exitCode = exitCodeFor3(result);
|
|
9160
|
-
if (options.json) printJson3(result
|
|
9161
|
-
else
|
|
9162
|
-
|
|
9163
|
-
function scanCommand(target, options = {}, stdin = process.stdin) {
|
|
9164
|
-
return safetyCommand("scan", target, options, stdin);
|
|
9165
|
-
}
|
|
9166
|
-
function verifySafeCommand(target, options = {}, stdin = process.stdin) {
|
|
9167
|
-
return safetyCommand("verify-safe", target, options, stdin);
|
|
9168
|
-
}
|
|
9169
|
-
function assessOpenedTrace(read, options = {}) {
|
|
9170
|
-
try {
|
|
9171
|
-
const rules = buildSafetyRules(options);
|
|
9172
|
-
const checkResult = runTraceChecks(
|
|
9173
|
-
{ read },
|
|
9174
|
-
{
|
|
9175
|
-
rules,
|
|
9176
|
-
...options.runId !== void 0 ? { runId: options.runId } : {},
|
|
9177
|
-
...options.run !== void 0 ? { runId: options.run } : {}
|
|
9178
|
-
}
|
|
9179
|
-
);
|
|
9180
|
-
const detectorFindings = checkResult.diagnostics.length === 0 ? redactionDetectorFindings(read, checkResult.runId) : [];
|
|
9181
|
-
return resultFromParts({
|
|
9182
|
-
command: "verify-safe",
|
|
9183
|
-
format: checkResult.format,
|
|
9184
|
-
runId: checkResult.runId,
|
|
9185
|
-
findings: [...checkResult.findings, ...detectorFindings],
|
|
9186
|
-
diagnostics: [
|
|
9187
|
-
...checkResult.diagnostics.map(diagnosticFromCheck),
|
|
9188
|
-
...warningDiagnostics(read.warnings, read.unsupportedFields)
|
|
9189
|
-
],
|
|
9190
|
-
warnings: read.warnings,
|
|
9191
|
-
unsupportedFields: read.unsupportedFields
|
|
9192
|
-
});
|
|
9193
|
-
} catch (error) {
|
|
9194
|
-
return messageStartsWithDash(error) ? invalidArgumentResult("verify-safe", error) : readErrorResult3("verify-safe", error);
|
|
9195
|
-
}
|
|
9196
|
-
}
|
|
9197
|
-
function messageStartsWithDash(error) {
|
|
9198
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
9199
|
-
return message.startsWith("--");
|
|
9578
|
+
if (options.json) printJson3(result);
|
|
9579
|
+
else if (options.markdown) console.log(renderEvalMarkdown(result).trimEnd());
|
|
9580
|
+
else printHuman3(result);
|
|
9200
9581
|
}
|
|
9201
9582
|
var NOTE = "Generated locally by AgentInspect. Artifacts are best-effort summaries, not compliance or security certification.";
|
|
9202
9583
|
var SAFETY_RULES = [
|
|
@@ -9271,8 +9652,8 @@ function renderCheckSection(result) {
|
|
|
9271
9652
|
`Diagnostics: ${result.diagnostics.length}`
|
|
9272
9653
|
];
|
|
9273
9654
|
for (const finding of result.findings.slice(0, 10)) {
|
|
9274
|
-
const
|
|
9275
|
-
lines.push(`- ${finding.ruleId}: ${finding.message} (${
|
|
9655
|
+
const path30 = finding.evidence[0]?.path ?? "(run)";
|
|
9656
|
+
lines.push(`- ${finding.ruleId}: ${finding.message} (${path30})`);
|
|
9276
9657
|
}
|
|
9277
9658
|
for (const diagnostic3 of result.diagnostics.slice(0, 10)) {
|
|
9278
9659
|
lines.push(`- ${diagnostic3.code}: ${diagnostic3.message}`);
|
|
@@ -9350,8 +9731,8 @@ function renderHtml(trace, check, diff) {
|
|
|
9350
9731
|
`;
|
|
9351
9732
|
}
|
|
9352
9733
|
async function writeArtifact(outputDir, relativePath, content, files) {
|
|
9353
|
-
const outPath =
|
|
9354
|
-
await mkdir(
|
|
9734
|
+
const outPath = path19.join(outputDir, relativePath);
|
|
9735
|
+
await mkdir(path19.dirname(outPath), { recursive: true });
|
|
9355
9736
|
await writeFile(outPath, content, "utf-8");
|
|
9356
9737
|
files.push(relativePath);
|
|
9357
9738
|
}
|
|
@@ -9380,7 +9761,7 @@ function shouldWriteEvidence(options, status) {
|
|
|
9380
9761
|
return status === "unsafe" || status === "regression" || status === "unknown" || status === "warning";
|
|
9381
9762
|
}
|
|
9382
9763
|
async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
9383
|
-
const outputDir = options.outputDir !== void 0 && options.outputDir.trim() !== "" ?
|
|
9764
|
+
const outputDir = options.outputDir !== void 0 && options.outputDir.trim() !== "" ? path19.resolve(options.outputDir.trim()) : "";
|
|
9384
9765
|
if (outputDir === "") {
|
|
9385
9766
|
console.error("--output-dir is required.");
|
|
9386
9767
|
process.exitCode = 1;
|
|
@@ -9494,7 +9875,8 @@ async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
|
9494
9875
|
finishedToolNames: [...parity.finishedToolNames],
|
|
9495
9876
|
pairedCount: parity.pairedCount,
|
|
9496
9877
|
parentRemapCount: parity.parentRemapCount,
|
|
9497
|
-
contractStatus: check.status === "pass" ? "pass" : check.status === "fail" ? "fail" : "error"
|
|
9878
|
+
contractStatus: check.status === "pass" ? "pass" : check.status === "fail" ? "fail" : "error",
|
|
9879
|
+
...parity.failureRoleCounts !== void 0 ? { failureRoleCounts: { ...parity.failureRoleCounts } } : {}
|
|
9498
9880
|
}
|
|
9499
9881
|
});
|
|
9500
9882
|
await writeArtifact(outputDir, "evidence.html", evidencePackage["evidence.html"], files);
|
|
@@ -9520,8 +9902,8 @@ async function artifactsCommand(target, options = {}, stdin = process.stdin) {
|
|
|
9520
9902
|
}
|
|
9521
9903
|
const summaryTarget = options.githubSummary ?? process.env.GITHUB_STEP_SUMMARY;
|
|
9522
9904
|
if (summaryTarget !== void 0 && summaryTarget.trim() !== "") {
|
|
9523
|
-
await mkdir(
|
|
9524
|
-
await appendFile(
|
|
9905
|
+
await mkdir(path19.dirname(path19.resolve(summaryTarget)), { recursive: true });
|
|
9906
|
+
await appendFile(path19.resolve(summaryTarget), `
|
|
9525
9907
|
${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
9526
9908
|
}
|
|
9527
9909
|
const manifestFiles = [...files, "manifest.json"].sort((a, b) => a.localeCompare(b));
|
|
@@ -9540,10 +9922,10 @@ ${renderMarkdown(trace, check, diff)}`, "utf-8");
|
|
|
9540
9922
|
findings: diff?.findings.length ?? 0,
|
|
9541
9923
|
diagnostics: diff?.diagnostics.length ?? 0
|
|
9542
9924
|
},
|
|
9543
|
-
...summaryTarget !== void 0 && summaryTarget.trim() !== "" ? { githubSummary:
|
|
9925
|
+
...summaryTarget !== void 0 && summaryTarget.trim() !== "" ? { githubSummary: path19.resolve(summaryTarget) } : {},
|
|
9544
9926
|
note: NOTE
|
|
9545
9927
|
};
|
|
9546
|
-
await writeFile(
|
|
9928
|
+
await writeFile(path19.join(outputDir, "manifest.json"), writeJson3(manifest), "utf-8");
|
|
9547
9929
|
if (options.json === true) {
|
|
9548
9930
|
console.log(writeJson3(manifest).trimEnd());
|
|
9549
9931
|
} else {
|
|
@@ -9601,8 +9983,8 @@ async function resolveOutputDir(options, runIds, cwd, format) {
|
|
|
9601
9983
|
const location = resolveWorkspaceLocation(cwd);
|
|
9602
9984
|
const manifest = await readWorkspaceManifestFile(location);
|
|
9603
9985
|
if (manifest.ok && manifest.manifest) {
|
|
9604
|
-
const rel =
|
|
9605
|
-
if (!rel.startsWith("..") && !
|
|
9986
|
+
const rel = path19.relative(location.workspaceDir, normalized);
|
|
9987
|
+
if (!rel.startsWith("..") && !path19.isAbsolute(rel)) {
|
|
9606
9988
|
return resolveInsideWorkspace(location.workspaceDir, rel);
|
|
9607
9989
|
}
|
|
9608
9990
|
}
|
|
@@ -9618,7 +10000,7 @@ async function resolveOutputDir(options, runIds, cwd, format) {
|
|
|
9618
10000
|
const label = runIds.length === 1 ? sanitizeBundleRunId(runIds[0]) : `multi-${runIds.length}`;
|
|
9619
10001
|
const base2 = resolveInsideWorkspace(
|
|
9620
10002
|
location.workspaceDir,
|
|
9621
|
-
|
|
10003
|
+
path19.join(manifest.manifest.bundlesDir, `bundle-${label}-${stamp}`)
|
|
9622
10004
|
);
|
|
9623
10005
|
return format === "zip" ? `${base2}.zip` : base2;
|
|
9624
10006
|
}
|
|
@@ -10023,19 +10405,19 @@ async function bundleCommand(runIdArg, options = {}) {
|
|
|
10023
10405
|
let sidecarDir = outputDir;
|
|
10024
10406
|
if (outputDir.toLowerCase().endsWith(".html")) {
|
|
10025
10407
|
htmlPath = outputDir;
|
|
10026
|
-
sidecarDir =
|
|
10408
|
+
sidecarDir = path19.dirname(outputDir);
|
|
10027
10409
|
} else {
|
|
10028
10410
|
await mkdir(outputDir, { recursive: true });
|
|
10029
|
-
htmlPath =
|
|
10411
|
+
htmlPath = path19.join(outputDir, EVIDENCE_HTML_FILENAME);
|
|
10030
10412
|
sidecarDir = outputDir;
|
|
10031
10413
|
}
|
|
10032
10414
|
await mkdir(sidecarDir, { recursive: true });
|
|
10033
10415
|
await writeFile(htmlPath, htmlContent, "utf-8");
|
|
10034
|
-
await writeFile(
|
|
10416
|
+
await writeFile(path19.join(sidecarDir, EVIDENCE_MANIFEST_FILENAME), evidenceJson, "utf-8");
|
|
10035
10417
|
outputPath = htmlPath;
|
|
10036
10418
|
} else if (format === "zip") {
|
|
10037
10419
|
const zipPath = outputDir.toLowerCase().endsWith(".zip") ? outputDir : `${outputDir}.zip`;
|
|
10038
|
-
const zipParent =
|
|
10420
|
+
const zipParent = path19.dirname(zipPath);
|
|
10039
10421
|
await mkdir(zipParent, { recursive: true });
|
|
10040
10422
|
const entries = [
|
|
10041
10423
|
...[...packaged.entries()].map(([relativePath, content]) => ({
|
|
@@ -10051,10 +10433,10 @@ async function bundleCommand(runIdArg, options = {}) {
|
|
|
10051
10433
|
await mkdir(outputDir, { recursive: true });
|
|
10052
10434
|
for (const [relativePath, content] of packaged.entries()) {
|
|
10053
10435
|
const outPath = assertBundlePathContained(outputDir, relativePath);
|
|
10054
|
-
await mkdir(
|
|
10436
|
+
await mkdir(path19.dirname(outPath), { recursive: true });
|
|
10055
10437
|
await writeFile(outPath, content, "utf-8");
|
|
10056
10438
|
}
|
|
10057
|
-
await writeFile(
|
|
10439
|
+
await writeFile(path19.join(outputDir, EVIDENCE_MANIFEST_FILENAME), evidenceJson, "utf-8");
|
|
10058
10440
|
if (!files.includes(EVIDENCE_MANIFEST_FILENAME)) {
|
|
10059
10441
|
files.push(EVIDENCE_MANIFEST_FILENAME);
|
|
10060
10442
|
}
|
|
@@ -10100,7 +10482,7 @@ function writeJson5(value) {
|
|
|
10100
10482
|
`;
|
|
10101
10483
|
}
|
|
10102
10484
|
async function bundleVerifyCommand(targetPath, options = {}) {
|
|
10103
|
-
const root =
|
|
10485
|
+
const root = path19.resolve(targetPath.trim() || ".");
|
|
10104
10486
|
const result = await verifyEvidenceDirectory(root, {
|
|
10105
10487
|
unexpectedFiles: options.unexpected ?? "fail"
|
|
10106
10488
|
});
|
|
@@ -10139,7 +10521,7 @@ function writeJson6(value) {
|
|
|
10139
10521
|
}
|
|
10140
10522
|
async function resolveHtmlPath(root) {
|
|
10141
10523
|
const candidates = [
|
|
10142
|
-
|
|
10524
|
+
path19.join(root, "evidence.html"),
|
|
10143
10525
|
root.toLowerCase().endsWith(".html") ? root : ""
|
|
10144
10526
|
].filter(Boolean);
|
|
10145
10527
|
for (const candidate of candidates) {
|
|
@@ -10154,7 +10536,7 @@ async function resolveHtmlPath(root) {
|
|
|
10154
10536
|
);
|
|
10155
10537
|
}
|
|
10156
10538
|
async function bundleOpenCommand(targetPath, options = {}) {
|
|
10157
|
-
const root =
|
|
10539
|
+
const root = path19.resolve(targetPath.trim() || ".");
|
|
10158
10540
|
try {
|
|
10159
10541
|
await access(root);
|
|
10160
10542
|
} catch {
|
|
@@ -10171,7 +10553,7 @@ async function bundleOpenCommand(targetPath, options = {}) {
|
|
|
10171
10553
|
try {
|
|
10172
10554
|
const info = await stat(root);
|
|
10173
10555
|
if (info.isFile() && root.toLowerCase().endsWith(".html")) {
|
|
10174
|
-
verifyRoot =
|
|
10556
|
+
verifyRoot = path19.dirname(root);
|
|
10175
10557
|
}
|
|
10176
10558
|
} catch {
|
|
10177
10559
|
}
|
|
@@ -10203,7 +10585,7 @@ async function bundleOpenCommand(targetPath, options = {}) {
|
|
|
10203
10585
|
let htmlPath;
|
|
10204
10586
|
try {
|
|
10205
10587
|
htmlPath = await resolveHtmlPath(
|
|
10206
|
-
root.toLowerCase().endsWith(".html") ?
|
|
10588
|
+
root.toLowerCase().endsWith(".html") ? path19.dirname(root) : root
|
|
10207
10589
|
);
|
|
10208
10590
|
if (root.toLowerCase().endsWith(".html")) {
|
|
10209
10591
|
htmlPath = root;
|
|
@@ -10219,7 +10601,7 @@ async function bundleOpenCommand(targetPath, options = {}) {
|
|
|
10219
10601
|
return;
|
|
10220
10602
|
}
|
|
10221
10603
|
try {
|
|
10222
|
-
await readFile(
|
|
10604
|
+
await readFile(path19.join(path19.dirname(htmlPath), "evidence.json"), "utf-8");
|
|
10223
10605
|
} catch {
|
|
10224
10606
|
}
|
|
10225
10607
|
const fileUrl = pathToFileURL(htmlPath).href;
|
|
@@ -10305,24 +10687,24 @@ function resolveTargetPath(client, projectLocal) {
|
|
|
10305
10687
|
if (projectLocal) {
|
|
10306
10688
|
switch (client) {
|
|
10307
10689
|
case "cursor":
|
|
10308
|
-
return
|
|
10690
|
+
return path19.join(".cursor", "mcp.json");
|
|
10309
10691
|
case "claude-code":
|
|
10310
|
-
return
|
|
10692
|
+
return path19.join(".mcp.json");
|
|
10311
10693
|
case "codex":
|
|
10312
|
-
return
|
|
10694
|
+
return path19.join(".codex", "config.toml.json");
|
|
10313
10695
|
case "gemini":
|
|
10314
|
-
return
|
|
10696
|
+
return path19.join(".gemini", "settings.json");
|
|
10315
10697
|
}
|
|
10316
10698
|
}
|
|
10317
10699
|
switch (client) {
|
|
10318
10700
|
case "cursor":
|
|
10319
|
-
return
|
|
10701
|
+
return path19.join("~", ".cursor", "mcp.json");
|
|
10320
10702
|
case "claude-code":
|
|
10321
|
-
return
|
|
10703
|
+
return path19.join("~", ".claude.json");
|
|
10322
10704
|
case "codex":
|
|
10323
|
-
return
|
|
10705
|
+
return path19.join("~", ".codex", "config.toml");
|
|
10324
10706
|
case "gemini":
|
|
10325
|
-
return
|
|
10707
|
+
return path19.join("~", ".gemini", "settings.json");
|
|
10326
10708
|
}
|
|
10327
10709
|
}
|
|
10328
10710
|
async function mcpConfigureCommand(options) {
|
|
@@ -10346,8 +10728,8 @@ async function mcpConfigureCommand(options) {
|
|
|
10346
10728
|
];
|
|
10347
10729
|
let wrote = false;
|
|
10348
10730
|
if (!dryRun && projectLocal) {
|
|
10349
|
-
const abs =
|
|
10350
|
-
await mkdir(
|
|
10731
|
+
const abs = path19.resolve(targetPath);
|
|
10732
|
+
await mkdir(path19.dirname(abs), { recursive: true });
|
|
10351
10733
|
await writeFile(abs, `${JSON.stringify(config, null, 2)}
|
|
10352
10734
|
`, "utf8");
|
|
10353
10735
|
wrote = true;
|
|
@@ -10507,23 +10889,23 @@ function readManifestDocument(value) {
|
|
|
10507
10889
|
};
|
|
10508
10890
|
}
|
|
10509
10891
|
function cwdRelative(filePath) {
|
|
10510
|
-
const relative =
|
|
10511
|
-
if (relative === "" || relative.startsWith("../") ||
|
|
10512
|
-
return
|
|
10892
|
+
const relative = path19.relative(process.cwd(), path19.resolve(filePath)).replace(/\\/g, "/");
|
|
10893
|
+
if (relative === "" || relative.startsWith("../") || path19.isAbsolute(relative)) {
|
|
10894
|
+
return path19.basename(filePath);
|
|
10513
10895
|
}
|
|
10514
10896
|
return relative;
|
|
10515
10897
|
}
|
|
10516
10898
|
async function readReporterManifest(filePath) {
|
|
10517
|
-
const absolute =
|
|
10899
|
+
const absolute = path19.resolve(filePath);
|
|
10518
10900
|
const raw = await readFile(absolute, "utf-8");
|
|
10519
10901
|
const document = readManifestDocument(JSON.parse(raw));
|
|
10520
10902
|
const manifest = document.manifest;
|
|
10521
10903
|
const results = manifest.results.map((result) => ({
|
|
10522
10904
|
testId: safeText(result.testId),
|
|
10523
10905
|
name: safeText(result.name),
|
|
10524
|
-
...result.file === void 0 ? {} : { file: safeText(
|
|
10906
|
+
...result.file === void 0 ? {} : { file: safeText(path19.basename(result.file)) },
|
|
10525
10907
|
status: result.status,
|
|
10526
|
-
...result.tracePath === void 0 ? {} : { tracePath: safeText(
|
|
10908
|
+
...result.tracePath === void 0 ? {} : { tracePath: safeText(path19.basename(result.tracePath)) },
|
|
10527
10909
|
artifacts: result.artifacts,
|
|
10528
10910
|
diagnostics: result.diagnostics
|
|
10529
10911
|
}));
|
|
@@ -10662,15 +11044,15 @@ async function ciSummaryCommand(manifestPaths, options = {}) {
|
|
|
10662
11044
|
return;
|
|
10663
11045
|
}
|
|
10664
11046
|
const markdown = renderMarkdown2(result);
|
|
10665
|
-
const outputPath = options.output !== void 0 && options.output.trim() !== "" ?
|
|
11047
|
+
const outputPath = options.output !== void 0 && options.output.trim() !== "" ? path19.resolve(options.output.trim()) : void 0;
|
|
10666
11048
|
if (outputPath !== void 0) {
|
|
10667
|
-
await mkdir(
|
|
11049
|
+
await mkdir(path19.dirname(outputPath), { recursive: true });
|
|
10668
11050
|
await writeFile(outputPath, markdown, "utf-8");
|
|
10669
11051
|
}
|
|
10670
11052
|
const summaryTarget = options.githubSummary ?? process.env.GITHUB_STEP_SUMMARY;
|
|
10671
11053
|
if (summaryTarget !== void 0 && summaryTarget.trim() !== "") {
|
|
10672
|
-
const summaryPath =
|
|
10673
|
-
await mkdir(
|
|
11054
|
+
const summaryPath = path19.resolve(summaryTarget);
|
|
11055
|
+
await mkdir(path19.dirname(summaryPath), { recursive: true });
|
|
10674
11056
|
await appendFile(summaryPath, `
|
|
10675
11057
|
${markdown}`, "utf-8");
|
|
10676
11058
|
}
|
|
@@ -10855,7 +11237,7 @@ jobs:
|
|
|
10855
11237
|
}
|
|
10856
11238
|
async function planInit(options = {}) {
|
|
10857
11239
|
const framework = normalizeFramework(options.framework);
|
|
10858
|
-
const cwd =
|
|
11240
|
+
const cwd = path19.resolve(options.cwd ?? process.cwd());
|
|
10859
11241
|
const demoPath = framework === "custom" ? "examples/agent-inspect-demo.mjs" : `examples/agent-inspect-${framework}-demo.mjs`;
|
|
10860
11242
|
const candidates = [
|
|
10861
11243
|
{ rel: CONFIG_FILE, content: configTemplate(framework) },
|
|
@@ -10870,7 +11252,7 @@ async function planInit(options = {}) {
|
|
|
10870
11252
|
}
|
|
10871
11253
|
const files = [];
|
|
10872
11254
|
for (const candidate of candidates) {
|
|
10873
|
-
const abs =
|
|
11255
|
+
const abs = path19.join(cwd, candidate.rel);
|
|
10874
11256
|
try {
|
|
10875
11257
|
await access(abs);
|
|
10876
11258
|
files.push({
|
|
@@ -10891,12 +11273,12 @@ async function writePlannedFiles(plan, cwd, options) {
|
|
|
10891
11273
|
if (entry.action === "skip") {
|
|
10892
11274
|
continue;
|
|
10893
11275
|
}
|
|
10894
|
-
const abs =
|
|
11276
|
+
const abs = path19.join(cwd, entry.path);
|
|
10895
11277
|
if (options.dryRun) {
|
|
10896
11278
|
written.push(entry.path);
|
|
10897
11279
|
continue;
|
|
10898
11280
|
}
|
|
10899
|
-
await mkdir(
|
|
11281
|
+
await mkdir(path19.dirname(abs), { recursive: true });
|
|
10900
11282
|
const demoPath = plan.framework === "custom" ? "examples/agent-inspect-demo.mjs" : `examples/agent-inspect-${plan.framework}-demo.mjs`;
|
|
10901
11283
|
const content = entry.path === CONFIG_FILE ? configTemplate(plan.framework) : entry.path === GITKEEP ? "" : entry.path.endsWith(".yml") ? githubWorkflowTemplate(demoPath) : demoTemplate(plan.framework);
|
|
10902
11284
|
await writeFile(abs, content, "utf-8");
|
|
@@ -10905,7 +11287,7 @@ async function writePlannedFiles(plan, cwd, options) {
|
|
|
10905
11287
|
return written;
|
|
10906
11288
|
}
|
|
10907
11289
|
async function initCommand(options = {}) {
|
|
10908
|
-
const cwd =
|
|
11290
|
+
const cwd = path19.resolve(options.cwd ?? process.cwd());
|
|
10909
11291
|
try {
|
|
10910
11292
|
const plan = await planInit({ ...options, cwd });
|
|
10911
11293
|
const toWrite = plan.files.filter((file) => file.action === "create").map((f) => f.path);
|
|
@@ -11000,7 +11382,7 @@ function envCheck(name, optional = true) {
|
|
|
11000
11382
|
};
|
|
11001
11383
|
}
|
|
11002
11384
|
async function traceDirWritable(traceDir) {
|
|
11003
|
-
const resolved =
|
|
11385
|
+
const resolved = path19.resolve(traceDir);
|
|
11004
11386
|
try {
|
|
11005
11387
|
await mkdir(resolved, { recursive: true });
|
|
11006
11388
|
await access(resolved, constants.W_OK);
|
|
@@ -11021,10 +11403,10 @@ async function traceDirWritable(traceDir) {
|
|
|
11021
11403
|
}
|
|
11022
11404
|
}
|
|
11023
11405
|
function readPackageVersionNearEntry(entryPath, packageName) {
|
|
11024
|
-
let dir =
|
|
11025
|
-
const { root } =
|
|
11406
|
+
let dir = path19.dirname(path19.resolve(entryPath));
|
|
11407
|
+
const { root } = path19.parse(dir);
|
|
11026
11408
|
while (true) {
|
|
11027
|
-
const candidate =
|
|
11409
|
+
const candidate = path19.join(dir, "package.json");
|
|
11028
11410
|
if (existsSync(candidate)) {
|
|
11029
11411
|
try {
|
|
11030
11412
|
const pkg = JSON.parse(readFileSync(candidate, "utf8"));
|
|
@@ -11035,14 +11417,14 @@ function readPackageVersionNearEntry(entryPath, packageName) {
|
|
|
11035
11417
|
}
|
|
11036
11418
|
}
|
|
11037
11419
|
if (dir === root) break;
|
|
11038
|
-
const parent =
|
|
11420
|
+
const parent = path19.dirname(dir);
|
|
11039
11421
|
if (parent === dir) break;
|
|
11040
11422
|
dir = parent;
|
|
11041
11423
|
}
|
|
11042
11424
|
return void 0;
|
|
11043
11425
|
}
|
|
11044
11426
|
function resolveInstalledPackage(cwd, name) {
|
|
11045
|
-
const require2 = createRequire(
|
|
11427
|
+
const require2 = createRequire(path19.join(cwd, "package.json"));
|
|
11046
11428
|
try {
|
|
11047
11429
|
const entry = require2.resolve(name);
|
|
11048
11430
|
let version2;
|
|
@@ -11140,7 +11522,7 @@ function versionMismatchCheck(cwd) {
|
|
|
11140
11522
|
};
|
|
11141
11523
|
}
|
|
11142
11524
|
async function runDoctorChecks(options = {}) {
|
|
11143
|
-
const cwd =
|
|
11525
|
+
const cwd = path19.resolve(options.cwd ?? process2.cwd());
|
|
11144
11526
|
const traceDir = options.traceDir?.trim() || process2.env.AGENT_INSPECT_TRACE_DIR?.trim() || ".agent-inspect";
|
|
11145
11527
|
const checks2 = [
|
|
11146
11528
|
nodeVersionCheck(),
|
|
@@ -11380,7 +11762,7 @@ function parsePluginManifest(input3) {
|
|
|
11380
11762
|
};
|
|
11381
11763
|
}
|
|
11382
11764
|
async function readPluginManifestFile(packageDir) {
|
|
11383
|
-
const manifestPath =
|
|
11765
|
+
const manifestPath = path19.join(packageDir, PLUGIN_MANIFEST_FILENAME);
|
|
11384
11766
|
try {
|
|
11385
11767
|
const raw = await readFile(manifestPath, "utf8");
|
|
11386
11768
|
const parsed = parsePluginManifest(JSON.parse(raw));
|
|
@@ -11454,7 +11836,7 @@ function createTraceDirectoryIndexer() {
|
|
|
11454
11836
|
// packages/cli/src/plugins.ts
|
|
11455
11837
|
async function readPackageName(packageDir) {
|
|
11456
11838
|
try {
|
|
11457
|
-
const raw = await readFile(
|
|
11839
|
+
const raw = await readFile(path19.join(packageDir, "package.json"), "utf8");
|
|
11458
11840
|
const parsed = JSON.parse(raw);
|
|
11459
11841
|
return typeof parsed.name === "string" ? parsed.name : void 0;
|
|
11460
11842
|
} catch {
|
|
@@ -11462,7 +11844,7 @@ async function readPackageName(packageDir) {
|
|
|
11462
11844
|
}
|
|
11463
11845
|
}
|
|
11464
11846
|
async function discoverPlugins(cwd = process.cwd()) {
|
|
11465
|
-
const nodeModules =
|
|
11847
|
+
const nodeModules = path19.join(cwd, "node_modules");
|
|
11466
11848
|
const found = [];
|
|
11467
11849
|
let entries;
|
|
11468
11850
|
try {
|
|
@@ -11473,7 +11855,7 @@ async function discoverPlugins(cwd = process.cwd()) {
|
|
|
11473
11855
|
for (const entry of entries.sort()) {
|
|
11474
11856
|
if (entry.startsWith("@")) continue;
|
|
11475
11857
|
if (!isPluginPackageName(entry)) continue;
|
|
11476
|
-
const packageDir =
|
|
11858
|
+
const packageDir = path19.join(nodeModules, entry);
|
|
11477
11859
|
const manifestRead = await readPluginManifestFile(packageDir);
|
|
11478
11860
|
found.push({
|
|
11479
11861
|
packageName: entry,
|
|
@@ -11486,7 +11868,7 @@ async function discoverPlugins(cwd = process.cwd()) {
|
|
|
11486
11868
|
return found;
|
|
11487
11869
|
}
|
|
11488
11870
|
async function validatePluginPackage(packageRef, cwd = process.cwd()) {
|
|
11489
|
-
const packageDir =
|
|
11871
|
+
const packageDir = path19.isAbsolute(packageRef) ? packageRef : path19.join(cwd, "node_modules", packageRef);
|
|
11490
11872
|
const packageName = await readPackageName(packageDir) ?? packageRef;
|
|
11491
11873
|
const errors = [];
|
|
11492
11874
|
const warnings = [];
|
|
@@ -11562,7 +11944,7 @@ async function pluginsValidateCommand(packageRef, cwd) {
|
|
|
11562
11944
|
}
|
|
11563
11945
|
var INDEX_FILENAME = ".agent-inspect-index.json";
|
|
11564
11946
|
function traceIndexPath(traceDir) {
|
|
11565
|
-
return
|
|
11947
|
+
return path19.join(traceDir, INDEX_FILENAME);
|
|
11566
11948
|
}
|
|
11567
11949
|
function parseMaxEntries(raw) {
|
|
11568
11950
|
if (raw === void 0 || raw.trim() === "") return void 0;
|
|
@@ -11675,7 +12057,7 @@ function isModuleNotFound4(e) {
|
|
|
11675
12057
|
}
|
|
11676
12058
|
async function loadIndexSqlite() {
|
|
11677
12059
|
try {
|
|
11678
|
-
return await import('./src-
|
|
12060
|
+
return await import('./src-T4EZERCX.mjs');
|
|
11679
12061
|
} catch (e) {
|
|
11680
12062
|
if (isModuleNotFound4(e)) {
|
|
11681
12063
|
console.error(
|
|
@@ -11705,7 +12087,7 @@ async function newestTraceMtimeMs2(traceDir) {
|
|
|
11705
12087
|
for (const file of files) {
|
|
11706
12088
|
if (!file.endsWith(".jsonl")) continue;
|
|
11707
12089
|
try {
|
|
11708
|
-
const s = await stat(
|
|
12090
|
+
const s = await stat(path19.join(traceDir, file));
|
|
11709
12091
|
if (s.mtimeMs > newest) newest = s.mtimeMs;
|
|
11710
12092
|
} catch {
|
|
11711
12093
|
}
|
|
@@ -11957,11 +12339,11 @@ function printJson5(value) {
|
|
|
11957
12339
|
console.log(JSON.stringify(value, null, 2));
|
|
11958
12340
|
}
|
|
11959
12341
|
function resolveCwd(options) {
|
|
11960
|
-
return
|
|
12342
|
+
return path19.resolve(options.cwd ?? process.cwd());
|
|
11961
12343
|
}
|
|
11962
12344
|
async function suiteInitCommand(options = {}) {
|
|
11963
12345
|
const cwd = resolveCwd(options);
|
|
11964
|
-
const configPath =
|
|
12346
|
+
const configPath = path19.join(cwd, DEFAULT_CONFIG_FILENAME);
|
|
11965
12347
|
const template = options.template?.trim();
|
|
11966
12348
|
const suiteConfig = template !== void 0 && template !== "" ? resolveSuiteTemplate(template) : defaultSuiteConfigTemplate();
|
|
11967
12349
|
if (options.dryRun) {
|
|
@@ -12080,13 +12462,13 @@ async function suiteListCommand(options = {}) {
|
|
|
12080
12462
|
}
|
|
12081
12463
|
async function writeSuiteArtifact(result, configOutputDir, options) {
|
|
12082
12464
|
const cwd = resolveCwd(options);
|
|
12083
|
-
const outputDir =
|
|
12465
|
+
const outputDir = path19.resolve(
|
|
12084
12466
|
cwd,
|
|
12085
12467
|
options.output ?? configOutputDir ?? DEFAULT_SUITE_ARTIFACTS_DIR
|
|
12086
12468
|
);
|
|
12087
12469
|
await mkdir(outputDir, { recursive: true });
|
|
12088
12470
|
const stamp = result.startedAt.replace(/[:.]/g, "-");
|
|
12089
|
-
const filePath =
|
|
12471
|
+
const filePath = path19.join(outputDir, `${result.suiteName}-${stamp}.json`);
|
|
12090
12472
|
await writeFile(filePath, `${JSON.stringify(result, null, 2)}
|
|
12091
12473
|
`, "utf-8");
|
|
12092
12474
|
return filePath;
|
|
@@ -12144,7 +12526,7 @@ async function loadSuiteResultFromInput(options) {
|
|
|
12144
12526
|
if (options.input === void 0 || options.input.trim() === "") {
|
|
12145
12527
|
throw new Error("Pass --input <suite-run.json> from a prior suite run.");
|
|
12146
12528
|
}
|
|
12147
|
-
const inputPath =
|
|
12529
|
+
const inputPath = path19.resolve(cwd, options.input.trim());
|
|
12148
12530
|
const raw = await readFile(inputPath, "utf-8");
|
|
12149
12531
|
return JSON.parse(raw);
|
|
12150
12532
|
}
|
|
@@ -12184,9 +12566,9 @@ function normalizeMetrics(raw) {
|
|
|
12184
12566
|
}
|
|
12185
12567
|
async function writeArtifacts(result, outputDir) {
|
|
12186
12568
|
await mkdir(outputDir, { recursive: true });
|
|
12187
|
-
const jsonPath =
|
|
12188
|
-
const markdownPath =
|
|
12189
|
-
const htmlPath =
|
|
12569
|
+
const jsonPath = path19.join(outputDir, "cohort-results.json");
|
|
12570
|
+
const markdownPath = path19.join(outputDir, "cohort-summary.md");
|
|
12571
|
+
const htmlPath = path19.join(outputDir, "cohort-report.html");
|
|
12190
12572
|
await writeFile(jsonPath, `${renderCohortReport(result, { format: "json" })}
|
|
12191
12573
|
`, "utf-8");
|
|
12192
12574
|
await writeFile(
|
|
@@ -12215,7 +12597,7 @@ async function cohortCommand(options = {}) {
|
|
|
12215
12597
|
const format = options.format ?? (options.json ? "json" : "markdown");
|
|
12216
12598
|
let artifacts;
|
|
12217
12599
|
if (options.output !== void 0 && options.output.trim() !== "") {
|
|
12218
|
-
artifacts = await writeArtifacts(result,
|
|
12600
|
+
artifacts = await writeArtifacts(result, path19.resolve(options.output.trim()));
|
|
12219
12601
|
}
|
|
12220
12602
|
if (options.json || format === "json") {
|
|
12221
12603
|
console.log(
|
|
@@ -12285,11 +12667,11 @@ function parseDuration2(value) {
|
|
|
12285
12667
|
async function writeArtifacts2(result, outputDir) {
|
|
12286
12668
|
await mkdir(outputDir, { recursive: true });
|
|
12287
12669
|
const paths = {
|
|
12288
|
-
jsonPath:
|
|
12289
|
-
markdownPath:
|
|
12290
|
-
htmlPath:
|
|
12291
|
-
junitPath:
|
|
12292
|
-
githubPath:
|
|
12670
|
+
jsonPath: path19.join(outputDir, "gate-results.json"),
|
|
12671
|
+
markdownPath: path19.join(outputDir, "gate-summary.md"),
|
|
12672
|
+
htmlPath: path19.join(outputDir, "gate-report.html"),
|
|
12673
|
+
junitPath: path19.join(outputDir, "junit.xml"),
|
|
12674
|
+
githubPath: path19.join(outputDir, "github-step-summary.md")
|
|
12293
12675
|
};
|
|
12294
12676
|
await writeFile(
|
|
12295
12677
|
paths.jsonPath,
|
|
@@ -12357,7 +12739,7 @@ async function gateCommand(options = {}) {
|
|
|
12357
12739
|
const result = await runGate(runs, gateOptions);
|
|
12358
12740
|
let artifacts;
|
|
12359
12741
|
if (options.output !== void 0 && options.output.trim() !== "") {
|
|
12360
|
-
artifacts = await writeArtifacts2(result,
|
|
12742
|
+
artifacts = await writeArtifacts2(result, path19.resolve(options.output.trim()));
|
|
12361
12743
|
}
|
|
12362
12744
|
const failed = !result.ok;
|
|
12363
12745
|
if (shouldEmitEvidence(options.evidenceOn, failed)) {
|
|
@@ -12664,7 +13046,7 @@ function createCliProgram() {
|
|
|
12664
13046
|
"openinference-json",
|
|
12665
13047
|
"otlp-json"
|
|
12666
13048
|
])
|
|
12667
|
-
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--explain", "explain each finding (path, confidence, redaction, override, bundle gate)").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").action((target, opts) => {
|
|
13049
|
+
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--explain", "explain each finding (path, confidence, redaction, override, bundle gate)").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").option("--policy <path>", "local JSON redaction policy (extraKeys + bounded patterns)").action((target, opts) => {
|
|
12668
13050
|
runCommand(() => scanCommand(target, opts));
|
|
12669
13051
|
});
|
|
12670
13052
|
program.command("verify-safe").description("Best-effort local trace safety verification").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
@@ -12673,7 +13055,7 @@ function createCliProgram() {
|
|
|
12673
13055
|
"openinference-json",
|
|
12674
13056
|
"otlp-json"
|
|
12675
13057
|
])
|
|
12676
|
-
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--explain", "explain each finding (path, confidence, redaction, override, bundle gate)").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").action((target, opts) => {
|
|
13058
|
+
).option("--run <run-id>", "select a run when the trace contains multiple runs").option("--json", "print deterministic JSON safety result").option("--explain", "explain each finding (path, confidence, redaction, override, bundle gate)").option("--max-string-length <number>", "unsafe threshold for string values").option("--max-array-length <number>", "unsafe threshold for array values").option("--max-object-keys <number>", "unsafe threshold for object key counts").option("--max-serialized-bytes <number>", "unsafe threshold for serialized values").option("--policy <path>", "local JSON redaction policy (extraKeys + bounded patterns)").action((target, opts) => {
|
|
12677
13059
|
runCommand(() => verifySafeCommand(target, opts));
|
|
12678
13060
|
});
|
|
12679
13061
|
program.command("artifacts").description("Create safe local CI trace artifacts").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").requiredOption("--output-dir <path>", "directory for generated artifacts").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
@@ -12833,7 +13215,10 @@ function createCliProgram() {
|
|
|
12833
13215
|
"--redaction-profile <profile>",
|
|
12834
13216
|
"alias for --profile (canonical spelling)"
|
|
12835
13217
|
).choices(["local", "share", "strict"])
|
|
12836
|
-
).option("-o, --output <path>", "write redacted content to a file").option("--out <path>", "alias for --output").option("--json", "print deterministic JSON wrapper with findings").
|
|
13218
|
+
).option("-o, --output <path>", "write redacted content to a file").option("--out <path>", "alias for --output").option("--json", "print deterministic JSON wrapper with findings").option("--policy <path>", "local JSON redaction policy (extraKeys + bounded patterns)").option(
|
|
13219
|
+
"--fail-on-residual",
|
|
13220
|
+
"exit non-zero when residual safety is UNSAFE or UNKNOWN (opt-in)"
|
|
13221
|
+
).action((target, opts) => {
|
|
12837
13222
|
runCommand(() => redactCommand(target, opts));
|
|
12838
13223
|
});
|
|
12839
13224
|
program.command("explain").description("Explain a local trace with deterministic facts (no provider calls)").argument("<trace-path-or-run-id>", "trace file, directory, stdin -, or run id").option("--dir <path>", "trace directory for run-id lookup").addOption(
|
|
@@ -13015,9 +13400,9 @@ function isPrimaryModule() {
|
|
|
13015
13400
|
if (!entry) return false;
|
|
13016
13401
|
const selfPath = fileURLToPath(import.meta.url);
|
|
13017
13402
|
try {
|
|
13018
|
-
return realpathSync(
|
|
13403
|
+
return realpathSync(path19.resolve(entry)) === realpathSync(path19.resolve(selfPath));
|
|
13019
13404
|
} catch {
|
|
13020
|
-
return
|
|
13405
|
+
return path19.resolve(entry) === path19.resolve(selfPath);
|
|
13021
13406
|
}
|
|
13022
13407
|
}
|
|
13023
13408
|
if (isPrimaryModule()) {
|