@dailephd/my-dev-kit-lab 0.9.4 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +4 -2
- package/dist/src/evaluation/benchmarkSandbox/createBenchmarkSandbox.js +9 -1
- package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js +91 -1
- package/dist/src/evaluation/runMyDevKitRetrieval.js +137 -4
- package/dist/src/evaluation/syntheticRepository/materialize.js +103 -7
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +10 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +9 -1
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +41 -5
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +4 -4
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +4 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +1 -1
- package/dist/src/securityValidation/attackScenarios/boundedSourceScan.js +112 -25
- package/dist/src/securityValidation/attackScenarios/scenarios/networkAssumptionScenario.js +58 -6
- package/dist/src/securityValidation/cliAdversarial/pathBoundaryChecks.js +108 -70
- package/dist/src/securityValidation/cliAdversarial/tempWorkspace.js +57 -8
- package/dist/src/securityValidation/cliAdversarial/traversalContainment.js +161 -0
- package/dist/src/securityValidation/commandRunner.js +52 -26
- package/dist/src/securityValidation/config.js +10 -0
- package/dist/src/securityValidation/validate/runCliSecuritySuiteCheck.js +21 -5
- package/dist/src/securityValidation/validate/runSecurityValidation.js +2 -2
- package/dist/src/tutorial/tutorialAssertions.js +7 -0
- package/docs/METRICS.md +11 -1
- package/package.json +1 -1
|
@@ -1,18 +1,28 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, statSync, writeFileSync, } from "node:fs";
|
|
2
|
+
import { lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, statSync, writeFileSync, } from "node:fs";
|
|
3
3
|
import { rm } from "node:fs/promises";
|
|
4
4
|
import os from "node:os";
|
|
5
5
|
import path from "node:path";
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
6
|
+
export function createTempWorkspace(prefix = "adv-cli-", options = {}) {
|
|
7
|
+
let enclosureDir;
|
|
8
|
+
let root;
|
|
9
|
+
let outsideDir;
|
|
10
|
+
if (options.nestDepth === undefined) {
|
|
11
|
+
root = mkdtempSync(path.join(os.tmpdir(), prefix));
|
|
12
|
+
// Create outside dir as a sibling of root (not inside it).
|
|
13
|
+
outsideDir = mkdtempSync(path.join(os.tmpdir(), `${prefix}outside-`));
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
enclosureDir = mkdtempSync(path.join(os.tmpdir(), `${prefix}encl-`));
|
|
17
|
+
const nest = Array.from({ length: options.nestDepth }, (_, i) => `d${i + 1}`);
|
|
18
|
+
root = path.join(enclosureDir, ...nest, "ws");
|
|
19
|
+
outsideDir = path.join(enclosureDir, "outside");
|
|
20
|
+
mkdirSync(root, { recursive: true });
|
|
21
|
+
mkdirSync(outsideDir, { recursive: true });
|
|
22
|
+
}
|
|
11
23
|
const sourceDir = path.join(root, "source");
|
|
12
24
|
const outputDir = path.join(root, "output");
|
|
13
25
|
const indexDir = path.join(root, "index");
|
|
14
|
-
// Create outside dir as a sibling of root (not inside it).
|
|
15
|
-
const outsideDir = mkdtempSync(path.join(os.tmpdir(), `${prefix}outside-`));
|
|
16
26
|
mkdirSync(sourceDir, { recursive: true });
|
|
17
27
|
mkdirSync(outputDir, { recursive: true });
|
|
18
28
|
mkdirSync(indexDir, { recursive: true });
|
|
@@ -28,7 +38,12 @@ export function createTempWorkspace(prefix = "adv-cli-") {
|
|
|
28
38
|
outputDir,
|
|
29
39
|
indexDir,
|
|
30
40
|
outsideDir,
|
|
41
|
+
enclosureDir,
|
|
31
42
|
cleanup: async () => {
|
|
43
|
+
if (enclosureDir) {
|
|
44
|
+
await rm(enclosureDir, { recursive: true, force: true });
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
32
47
|
await Promise.all([
|
|
33
48
|
rm(root, { recursive: true, force: true }),
|
|
34
49
|
rm(outsideDir, { recursive: true, force: true }),
|
|
@@ -82,6 +97,40 @@ function collectSnapshots(baseDir, currentDir, out) {
|
|
|
82
97
|
}
|
|
83
98
|
}
|
|
84
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* Strict snapshot of a whole tree: records directories and links as well as
|
|
102
|
+
* files, never follows links, and THROWS when any entry cannot be inspected.
|
|
103
|
+
* Used where an unreadable tree must not be mistaken for an unchanged one.
|
|
104
|
+
*/
|
|
105
|
+
export function snapshotTreeStrict(dir) {
|
|
106
|
+
const snapshots = [];
|
|
107
|
+
collectStrict(dir, dir, snapshots);
|
|
108
|
+
return snapshots.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
|
|
109
|
+
}
|
|
110
|
+
function collectStrict(baseDir, currentDir, out) {
|
|
111
|
+
for (const entry of readdirSync(currentDir)) {
|
|
112
|
+
const abs = path.join(currentDir, entry);
|
|
113
|
+
const stat = lstatSync(abs);
|
|
114
|
+
const relativePath = path.relative(baseDir, abs);
|
|
115
|
+
if (stat.isSymbolicLink()) {
|
|
116
|
+
out.push({ relativePath, absolutePath: abs, contentHash: `<link:${readlinkSync(abs)}>` });
|
|
117
|
+
}
|
|
118
|
+
else if (stat.isDirectory()) {
|
|
119
|
+
out.push({ relativePath, absolutePath: abs, contentHash: "<dir>" });
|
|
120
|
+
collectStrict(baseDir, abs, out);
|
|
121
|
+
}
|
|
122
|
+
else if (stat.isFile()) {
|
|
123
|
+
out.push({
|
|
124
|
+
relativePath,
|
|
125
|
+
absolutePath: abs,
|
|
126
|
+
contentHash: createHash("sha256").update(readFileSync(abs)).digest("hex"),
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
out.push({ relativePath, absolutePath: abs, contentHash: "<special>" });
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
85
134
|
export function diffSnapshots(before, after) {
|
|
86
135
|
const beforeMap = new Map(before.map((s) => [s.relativePath, s.contentHash]));
|
|
87
136
|
const afterMap = new Map(after.map((s) => [s.relativePath, s.contentHash]));
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { lstatSync, realpathSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// Containment policy for deliberately traversing --out / --index values.
|
|
5
|
+
//
|
|
6
|
+
// A traversing value may only be handed to a CLI when its full destination is
|
|
7
|
+
// proven to stay inside a NEW task-owned disposable enclosure. The enclosure is
|
|
8
|
+
// not the declared output directory: the value may leave the declared
|
|
9
|
+
// directory, but never the enclosure. Anything that cannot be proven is refused
|
|
10
|
+
// before launch.
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
/** Largest number of net parent components the harness will accommodate. */
|
|
13
|
+
export const MAX_TRAVERSAL_DEPTH = 10;
|
|
14
|
+
/** Longest destination (excluding the file a CLI adds) allowed on Windows. */
|
|
15
|
+
const MAX_WINDOWS_DESTINATION_LENGTH = 200;
|
|
16
|
+
const MAX_POSIX_DESTINATION_LENGTH = 3000;
|
|
17
|
+
function pathFor(platform) {
|
|
18
|
+
return platform === "win32" ? path.win32 : path.posix;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Pure classification of a traversal value according to how the given host
|
|
22
|
+
* platform resolves it. Reports how many parent components the value needs
|
|
23
|
+
* room for, or refuses values that are absolute or ambiguous on that host.
|
|
24
|
+
*/
|
|
25
|
+
export function classifyTraversalValue(value, platform = process.platform) {
|
|
26
|
+
if (value.length === 0) {
|
|
27
|
+
return { kind: "refused", reason: "empty path value" };
|
|
28
|
+
}
|
|
29
|
+
if (value.includes("\0")) {
|
|
30
|
+
return { kind: "refused", reason: "path value contains a NUL character" };
|
|
31
|
+
}
|
|
32
|
+
const host = pathFor(platform);
|
|
33
|
+
if (host.isAbsolute(value)) {
|
|
34
|
+
return { kind: "refused", reason: `absolute path on ${platform}: destination is not confined to the enclosure` };
|
|
35
|
+
}
|
|
36
|
+
if (platform === "win32") {
|
|
37
|
+
if (/^[A-Za-z]:/.test(value)) {
|
|
38
|
+
return { kind: "refused", reason: "drive-qualified path on win32: drive-relative resolution is ambiguous" };
|
|
39
|
+
}
|
|
40
|
+
if (/[<>:"|?*]/.test(value)) {
|
|
41
|
+
return { kind: "refused", reason: "win32 path contains reserved characters (stream or device ambiguity)" };
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
const segments = value.split(platform === "win32" ? /[\\/]+/ : /\/+/);
|
|
45
|
+
let depth = 0;
|
|
46
|
+
let lowest = 0;
|
|
47
|
+
for (const segment of segments) {
|
|
48
|
+
if (segment === "" || segment === ".")
|
|
49
|
+
continue;
|
|
50
|
+
if (segment === "..") {
|
|
51
|
+
depth -= 1;
|
|
52
|
+
lowest = Math.min(lowest, depth);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
if (platform === "win32" && /[. ]$/.test(segment)) {
|
|
56
|
+
return { kind: "refused", reason: "win32 path component ends in a dot or space and would be rewritten by the OS" };
|
|
57
|
+
}
|
|
58
|
+
depth += 1;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
const nestDepth = lowest === 0 ? 0 : -lowest;
|
|
62
|
+
if (nestDepth > MAX_TRAVERSAL_DEPTH) {
|
|
63
|
+
return {
|
|
64
|
+
kind: "refused",
|
|
65
|
+
reason: `traversal depth ${nestDepth} exceeds the supported maximum of ${MAX_TRAVERSAL_DEPTH}`,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
return { kind: "relative", nestDepth };
|
|
69
|
+
}
|
|
70
|
+
/** True when `candidate` equals or is below `parent` under the host's path semantics. */
|
|
71
|
+
export function isPathInside(parent, candidate, platform = process.platform) {
|
|
72
|
+
const host = pathFor(platform);
|
|
73
|
+
const relative = host.relative(parent, candidate);
|
|
74
|
+
if (relative === "")
|
|
75
|
+
return true;
|
|
76
|
+
return !relative.startsWith("..") && !host.isAbsolute(relative);
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Resolves `value` against `baseDir` and proves, using the real filesystem,
|
|
80
|
+
* that the destination, its nearest existing parent and every not-yet-created
|
|
81
|
+
* descendant stay inside `enclosureDir`. Never creates anything.
|
|
82
|
+
*/
|
|
83
|
+
export function proveContainment(options) {
|
|
84
|
+
const { value, baseDir, enclosureDir, declaredDir } = options;
|
|
85
|
+
const platform = process.platform;
|
|
86
|
+
const host = pathFor(platform);
|
|
87
|
+
const plan = classifyTraversalValue(value, platform);
|
|
88
|
+
if (plan.kind === "refused")
|
|
89
|
+
return { ok: false, reason: plan.reason };
|
|
90
|
+
let enclosure;
|
|
91
|
+
let realBase;
|
|
92
|
+
let realDeclared;
|
|
93
|
+
try {
|
|
94
|
+
enclosure = realpathSync.native(enclosureDir);
|
|
95
|
+
realBase = realpathSync.native(baseDir);
|
|
96
|
+
realDeclared = realpathSync.native(declaredDir);
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
return { ok: false, reason: `enclosure, base or declared directory is not inspectable: ${errorMessage(error)}` };
|
|
100
|
+
}
|
|
101
|
+
if (!isPathInside(enclosure, realBase, platform) || !isPathInside(enclosure, realDeclared, platform)) {
|
|
102
|
+
return { ok: false, reason: "base or declared directory is not inside the enclosure" };
|
|
103
|
+
}
|
|
104
|
+
const destination = host.resolve(realBase, value);
|
|
105
|
+
if (!isPathInside(enclosure, destination, platform)) {
|
|
106
|
+
return { ok: false, reason: `destination ${destination} resolves outside the enclosure ${enclosure}` };
|
|
107
|
+
}
|
|
108
|
+
const limit = platform === "win32" ? MAX_WINDOWS_DESTINATION_LENGTH : MAX_POSIX_DESTINATION_LENGTH;
|
|
109
|
+
if (destination.length > limit) {
|
|
110
|
+
return { ok: false, reason: `destination length ${destination.length} exceeds the supported ${limit}` };
|
|
111
|
+
}
|
|
112
|
+
// Find the nearest existing physical ancestor; anything else is inspectable
|
|
113
|
+
// only if lstat answers either "exists" or "does not exist".
|
|
114
|
+
let nearest = destination;
|
|
115
|
+
for (;;) {
|
|
116
|
+
try {
|
|
117
|
+
lstatSync(nearest);
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
if (error.code !== "ENOENT") {
|
|
122
|
+
return { ok: false, reason: `cannot inspect ${nearest}: ${errorMessage(error)}` };
|
|
123
|
+
}
|
|
124
|
+
const parent = host.dirname(nearest);
|
|
125
|
+
if (parent === nearest || !isPathInside(enclosure, parent, platform)) {
|
|
126
|
+
return { ok: false, reason: "no existing ancestor found inside the enclosure" };
|
|
127
|
+
}
|
|
128
|
+
nearest = parent;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
// Existing part must be a plain directory chain: no links, junctions or
|
|
132
|
+
// case/short-name aliases (the real path must equal the lexical path).
|
|
133
|
+
let nearestReal;
|
|
134
|
+
try {
|
|
135
|
+
if (lstatSync(nearest).isSymbolicLink()) {
|
|
136
|
+
return { ok: false, reason: `${nearest} is a symbolic link or junction` };
|
|
137
|
+
}
|
|
138
|
+
nearestReal = realpathSync.native(nearest);
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
return { ok: false, reason: `cannot resolve ${nearest}: ${errorMessage(error)}` };
|
|
142
|
+
}
|
|
143
|
+
if (nearestReal !== nearest) {
|
|
144
|
+
return { ok: false, reason: `${nearest} is an alias of ${nearestReal} (link, junction, case or short-name ambiguity)` };
|
|
145
|
+
}
|
|
146
|
+
if (!isPathInside(enclosure, nearestReal, platform)) {
|
|
147
|
+
return { ok: false, reason: `${nearest} resolves outside the enclosure` };
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
ok: true,
|
|
151
|
+
destination,
|
|
152
|
+
enclosure,
|
|
153
|
+
nearestExisting: nearest,
|
|
154
|
+
relativeToEnclosure: host.relative(enclosure, destination).split(host.sep).join("/"),
|
|
155
|
+
declared: realDeclared,
|
|
156
|
+
insideDeclared: isPathInside(realDeclared, destination, platform),
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
function errorMessage(error) {
|
|
160
|
+
return error instanceof Error ? error.message : String(error);
|
|
161
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { resolveCommand } from "../core/resolveCommand.js";
|
|
3
|
+
import { terminateProcessTree } from "../core/processTree.js";
|
|
3
4
|
// Prefer the unqualified npm command and let the shared resolver locate
|
|
4
5
|
// npm.cmd/.ps1 shims or direct executables per platform.
|
|
5
6
|
export function resolveNpmCommand() {
|
|
@@ -13,6 +14,8 @@ export function buildSecurityCommandSpawnOptions(options) {
|
|
|
13
14
|
return {
|
|
14
15
|
cwd: options.cwd,
|
|
15
16
|
shell: false,
|
|
17
|
+
windowsHide: true,
|
|
18
|
+
detached: process.platform !== "win32",
|
|
16
19
|
stdio: ["ignore", "pipe", "pipe"],
|
|
17
20
|
env: options.env,
|
|
18
21
|
};
|
|
@@ -53,6 +56,8 @@ export async function runSecurityCommand(options) {
|
|
|
53
56
|
stdout: "",
|
|
54
57
|
stderr: resolved.warnings.join("\n"),
|
|
55
58
|
timedOut: false,
|
|
59
|
+
resolutionKind: resolved.resolutionKind,
|
|
60
|
+
timeoutMs: options.timeoutMs,
|
|
56
61
|
skipped: false,
|
|
57
62
|
};
|
|
58
63
|
}
|
|
@@ -62,6 +67,12 @@ export async function runSecurityCommand(options) {
|
|
|
62
67
|
let timedOut = false;
|
|
63
68
|
let settled = false;
|
|
64
69
|
let timeout;
|
|
70
|
+
let closeGrace;
|
|
71
|
+
let termination;
|
|
72
|
+
let treeTerminated;
|
|
73
|
+
let signal;
|
|
74
|
+
let exitCode = null;
|
|
75
|
+
let spawnError;
|
|
65
76
|
let child;
|
|
66
77
|
try {
|
|
67
78
|
child = spawn(command, args, buildSecurityCommandSpawnOptions({ cwd: options.cwd, env }));
|
|
@@ -76,6 +87,9 @@ export async function runSecurityCommand(options) {
|
|
|
76
87
|
stdout: "",
|
|
77
88
|
stderr: spawnErr instanceof Error ? spawnErr.message : String(spawnErr),
|
|
78
89
|
timedOut: false,
|
|
90
|
+
spawnError: spawnErr instanceof Error ? spawnErr.message : String(spawnErr),
|
|
91
|
+
resolutionKind: resolved.resolutionKind,
|
|
92
|
+
timeoutMs: options.timeoutMs,
|
|
79
93
|
skipped: false,
|
|
80
94
|
});
|
|
81
95
|
return;
|
|
@@ -86,52 +100,64 @@ export async function runSecurityCommand(options) {
|
|
|
86
100
|
child.stderr.on("data", (chunk) => {
|
|
87
101
|
stderr += chunk.toString("utf8");
|
|
88
102
|
});
|
|
89
|
-
|
|
103
|
+
const finish = async () => {
|
|
90
104
|
if (settled)
|
|
91
105
|
return;
|
|
92
106
|
settled = true;
|
|
93
107
|
if (timeout)
|
|
94
108
|
clearTimeout(timeout);
|
|
109
|
+
if (termination)
|
|
110
|
+
await termination;
|
|
111
|
+
if (closeGrace)
|
|
112
|
+
clearTimeout(closeGrace);
|
|
95
113
|
resolve({
|
|
96
114
|
command,
|
|
97
115
|
args,
|
|
98
116
|
cwd: options.cwd,
|
|
99
|
-
exitCode
|
|
117
|
+
exitCode,
|
|
100
118
|
durationMs: Date.now() - started,
|
|
101
119
|
stdout,
|
|
102
|
-
stderr
|
|
120
|
+
stderr,
|
|
103
121
|
timedOut,
|
|
122
|
+
signal,
|
|
123
|
+
spawnError,
|
|
124
|
+
resolutionKind: resolved.resolutionKind,
|
|
125
|
+
timeoutMs: options.timeoutMs,
|
|
126
|
+
treeTerminated,
|
|
104
127
|
skipped: false,
|
|
105
128
|
});
|
|
129
|
+
};
|
|
130
|
+
child.on("error", (error) => {
|
|
131
|
+
spawnError = error.message;
|
|
132
|
+
stderr += `${stderr ? "\n" : ""}${error.message}`;
|
|
133
|
+
void finish();
|
|
106
134
|
});
|
|
107
135
|
if (options.timeoutMs > 0) {
|
|
108
136
|
timeout = setTimeout(() => {
|
|
109
137
|
timedOut = true;
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
}
|
|
138
|
+
// Keep the parent alive until its entire owned tree is addressed.
|
|
139
|
+
// Killing only npm's cmd parent can strand its children and their pipes.
|
|
140
|
+
termination = terminateProcessTree(child.pid, { waitTimeoutMs: 5000 }).then((terminated) => {
|
|
141
|
+
treeTerminated = terminated;
|
|
142
|
+
return terminated;
|
|
143
|
+
});
|
|
144
|
+
void termination.then(() => {
|
|
145
|
+
if (settled)
|
|
146
|
+
return;
|
|
147
|
+
// A failed tree operation or a pipe that never closes must not make
|
|
148
|
+
// the command wait indefinitely beyond its deadline and kill grace.
|
|
149
|
+
closeGrace = setTimeout(() => {
|
|
150
|
+
child.stdout.destroy();
|
|
151
|
+
child.stderr.destroy();
|
|
152
|
+
void finish();
|
|
153
|
+
}, 1000);
|
|
154
|
+
});
|
|
116
155
|
}, options.timeoutMs);
|
|
117
156
|
}
|
|
118
|
-
child.on("close", (
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
if (timeout)
|
|
123
|
-
clearTimeout(timeout);
|
|
124
|
-
resolve({
|
|
125
|
-
command,
|
|
126
|
-
args,
|
|
127
|
-
cwd: options.cwd,
|
|
128
|
-
exitCode,
|
|
129
|
-
durationMs: Date.now() - started,
|
|
130
|
-
stdout,
|
|
131
|
-
stderr,
|
|
132
|
-
timedOut,
|
|
133
|
-
skipped: false,
|
|
134
|
-
});
|
|
157
|
+
child.on("close", (code, exitSignal) => {
|
|
158
|
+
exitCode = code;
|
|
159
|
+
signal = exitSignal;
|
|
160
|
+
void finish();
|
|
135
161
|
});
|
|
136
162
|
});
|
|
137
163
|
}
|
|
@@ -37,3 +37,13 @@ export const DEFAULT_SECURITY_CONFIG = {
|
|
|
37
37
|
commandTimeoutMs: 60_000,
|
|
38
38
|
requireOsvScanner: false,
|
|
39
39
|
};
|
|
40
|
+
// Windows native Semgrep startup can exceed the shared one-minute deadline.
|
|
41
|
+
// Keep this allowance local to Semgrep, bounded, and independently cancellable.
|
|
42
|
+
export function resolveSemgrepCommandTimeoutMs(config, platform = process.platform) {
|
|
43
|
+
const timeoutMs = config.semgrepCommandTimeoutMs ??
|
|
44
|
+
(platform === "win32" ? 300_000 : config.commandTimeoutMs);
|
|
45
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
46
|
+
throw new Error("Semgrep process timeout must be a positive finite duration.");
|
|
47
|
+
}
|
|
48
|
+
return platform === "win32" ? Math.min(timeoutMs, 300_000) : timeoutMs;
|
|
49
|
+
}
|
|
@@ -65,10 +65,17 @@ export async function runCliSecuritySuiteCheck(options) {
|
|
|
65
65
|
timeoutMs: Math.min(options.timeoutMs * 3, 180_000),
|
|
66
66
|
});
|
|
67
67
|
const finishedAt = new Date().toISOString();
|
|
68
|
-
const passed = cmd.exitCode === 0;
|
|
68
|
+
const passed = cmd.exitCode === 0 && !cmd.timedOut && !cmd.spawnError && cmd.treeTerminated !== false;
|
|
69
69
|
const name = isSelf
|
|
70
70
|
? "CLI adversarial test suite (tool self-tests)"
|
|
71
71
|
: "Target security test suite";
|
|
72
|
+
const outcome = cmd.timedOut
|
|
73
|
+
? `exceeded its ${cmd.timeoutMs ?? "configured"} ms deadline (exit code ${String(cmd.exitCode)}).`
|
|
74
|
+
: cmd.spawnError
|
|
75
|
+
? "could not be started; see process error evidence."
|
|
76
|
+
: cmd.treeTerminated === false
|
|
77
|
+
? "did not confirm process-tree cleanup."
|
|
78
|
+
: `exited with code ${String(cmd.exitCode)}.`;
|
|
72
79
|
return {
|
|
73
80
|
id: "cli-adversarial-suite",
|
|
74
81
|
name,
|
|
@@ -87,8 +94,8 @@ export async function runCliSecuritySuiteCheck(options) {
|
|
|
87
94
|
severity: "major",
|
|
88
95
|
category: "cli-adversarial",
|
|
89
96
|
description: isSelf
|
|
90
|
-
? `Tool-root test:security
|
|
91
|
-
: `Target-root test:security
|
|
97
|
+
? `Tool-root test:security ${outcome}`
|
|
98
|
+
: `Target-root test:security ${outcome}`,
|
|
92
99
|
evidence: buildEvidence(cmd),
|
|
93
100
|
recommendation: isSelf
|
|
94
101
|
? "Fix failing tool security tests before release."
|
|
@@ -104,14 +111,23 @@ export async function runCliSecuritySuiteCheck(options) {
|
|
|
104
111
|
};
|
|
105
112
|
}
|
|
106
113
|
function summarizeOutput(value) {
|
|
107
|
-
const normalized = value.replace(/\
|
|
114
|
+
const normalized = value.replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
115
|
+
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "")
|
|
116
|
+
.replace(/\r\n/g, "\n").trim();
|
|
108
117
|
if (!normalized) {
|
|
109
118
|
return undefined;
|
|
110
119
|
}
|
|
111
120
|
return normalized.slice(0, 500);
|
|
112
121
|
}
|
|
113
122
|
function buildEvidence(cmd) {
|
|
114
|
-
const parts = [
|
|
123
|
+
const parts = [
|
|
124
|
+
`cwd=${cmd.cwd}`, `exitCode=${String(cmd.exitCode)}`,
|
|
125
|
+
`timedOut=${cmd.timedOut}`, `signal=${cmd.signal === undefined ? "unknown" : String(cmd.signal)}`,
|
|
126
|
+
`timeoutMs=${cmd.timeoutMs ?? "unknown"}`, `durationMs=${cmd.durationMs}`,
|
|
127
|
+
`resolutionKind=${cmd.resolutionKind ?? "unknown"}`, `treeTerminated=${cmd.treeTerminated ?? "not-requested"}`,
|
|
128
|
+
];
|
|
129
|
+
if (cmd.spawnError)
|
|
130
|
+
parts.push(`spawnError=${summarizeOutput(cmd.spawnError)}`);
|
|
115
131
|
const stdoutSummary = summarizeOutput(cmd.stdout);
|
|
116
132
|
const stderrSummary = summarizeOutput(cmd.stderr);
|
|
117
133
|
if (stdoutSummary) {
|
|
@@ -8,7 +8,7 @@ import { runAllFuzzTargets } from "../fuzz/fuzzHarness.js";
|
|
|
8
8
|
import { ALL_FUZZ_TARGETS } from "../fuzz/fuzzTargets.js";
|
|
9
9
|
import { calculateVerdict, summarizeVerdictReasoning } from "./verdict.js";
|
|
10
10
|
import { resolveValidationTarget, targetDescription as describeTarget } from "./resolveTarget.js";
|
|
11
|
-
import { DEFAULT_SECURITY_CONFIG } from "../config.js";
|
|
11
|
+
import { DEFAULT_SECURITY_CONFIG, resolveSemgrepCommandTimeoutMs } from "../config.js";
|
|
12
12
|
import { runCliSecuritySuiteCheck } from "./runCliSecuritySuiteCheck.js";
|
|
13
13
|
import { DEFAULT_SECURITY_CHECKS, DEFAULT_SECURITY_PROFILE, IMPLEMENTED_SECURITY_CHECK_IDS, PLANNED_SECURITY_CHECK_IDS, } from "./cliOptions.js";
|
|
14
14
|
import { runAttackScenarios } from "../attackScenarios/attackRunner.js";
|
|
@@ -102,7 +102,7 @@ export async function runSecurityValidation(options) {
|
|
|
102
102
|
collect(await runSemgrepCheck({
|
|
103
103
|
targetRoot: target.targetRoot,
|
|
104
104
|
toolRoot,
|
|
105
|
-
timeoutMs: config
|
|
105
|
+
timeoutMs: resolveSemgrepCommandTimeoutMs(config),
|
|
106
106
|
}));
|
|
107
107
|
}
|
|
108
108
|
catch (err) {
|
|
@@ -66,6 +66,13 @@ export function resolveAssertionRequestUrl(applicationUrl, suppliedPath) {
|
|
|
66
66
|
throw new Error(`HTTP assertion path must be a root-relative path beginning with "/" and must not be protocol-relative; received ${JSON.stringify(suppliedPath)}.`);
|
|
67
67
|
}
|
|
68
68
|
const base = new URL(applicationUrl);
|
|
69
|
+
// Defend the exported assertion boundary as well as the validated tutorial
|
|
70
|
+
// caller: same-origin alone does not make an unchecked remote origin local.
|
|
71
|
+
const hostname = base.hostname.replace(/^\[/, "").replace(/\]$/, "").toLowerCase();
|
|
72
|
+
if ((base.protocol !== "http:" && base.protocol !== "https:") ||
|
|
73
|
+
!["localhost", "127.0.0.1", "::1"].includes(hostname)) {
|
|
74
|
+
throw new Error("HTTP assertion application origin must use HTTP(S) on loopback.");
|
|
75
|
+
}
|
|
69
76
|
const resolved = new URL(suppliedPath, base);
|
|
70
77
|
if (resolved.protocol !== base.protocol || resolved.hostname !== base.hostname || resolved.port !== base.port) {
|
|
71
78
|
throw new Error(`HTTP assertion path ${JSON.stringify(suppliedPath)} resolves to ${resolved.origin}, which leaves the application origin ${base.origin}.`);
|
package/docs/METRICS.md
CHANGED
|
@@ -646,7 +646,7 @@ The `incremental-change-staleness` plugin reuses the six existing v0.6.1 affecte
|
|
|
646
646
|
**Required-file relation** (per scenario, comparing the two treatments' required-file status):
|
|
647
647
|
|
|
648
648
|
- `stale-worse`: `stale-index` is `missing` while `full-refresh` is `present`.
|
|
649
|
-
- `same`: both treatments have the same status
|
|
649
|
+
- `same`: both treatments have the same status, except that `missing`/`missing` is `unknown` (see the reference-adequacy gate below).
|
|
650
650
|
- `stale-better`: `stale-index` is `present` while `full-refresh` is `missing`.
|
|
651
651
|
- `unknown`: either treatment's status is `unknown`.
|
|
652
652
|
|
|
@@ -690,6 +690,16 @@ Availability: the first three are always present for a validated incremental tre
|
|
|
690
690
|
|
|
691
691
|
`no-observed-regression-relative-to-full` means only that the executed scenario showed no regression under those two evidence dimensions. It does not establish equivalence or safety and does not show that full refresh is unnecessary. `not-comparable-as-partial-refresh` says the run gives no evidence about the requested partial behavior. There is no winner, ranking, composite score, safety score, safe-to-skip-refresh flag, or stale-risk percentage. Ordinary per-treatment correctness `score` remains the existing deterministic fake-agent evidence.
|
|
692
692
|
|
|
693
|
+
**Evidence families and the reference-adequacy gate (v0.9.5).** The staleness experiment keeps five kinds of evidence distinct and never converts one into another:
|
|
694
|
+
|
|
695
|
+
- *Source read*: a file whose own `source` command succeeded under the bounded multi-source retrieval protocol (`bounded-multi-source-v1`: one `search` with an explicit limit, then at most 8 distinct candidate files in search rank order, each with one `source` command). `retrieval.filesRead` and `requiredFileEvidence.observedFiles` carry only these. Candidates come only from the actual search output; the answer key never selects them.
|
|
696
|
+
- *Surfaced*: a file or symbol that search, lookup or slice merely returned. It is not credited as read.
|
|
697
|
+
- *Expected but not observed*: `requiredFileEvidence.missingFiles`. `missing` is relative to the fixed candidate budget and is not an exhaustive source-set assessment; its reason states the budget and truncation flags. Unsourced required files are `unknown` (not `missing`) when the protocol cannot source as many files as are required, a bounded source command failed, or candidate identities were rejected.
|
|
698
|
+
- *Synthetic agent correctness*: the deterministic fake agent reports only two expected facts, so scenarios requiring three or more facts score 0.75 (0.70 for T1) and never pass. This is not measured my-dev-kit answer correctness, is identical across treatments, and cannot discriminate between them.
|
|
699
|
+
- *Candidate relative to full refresh*: the relation and classification fields. Full refresh is the reference, not an automatically adequate treatment: an equal required-file status supports `same` only when the full-refresh reference itself is `present`. `missing`/`missing` is reported as `unknown`, so an all-equal but insufficient experiment is `inconclusive`, never `no-observed-regression`.
|
|
700
|
+
|
|
701
|
+
Fact-context-target coverage from `retrievalQuality` is not used here: five of the six frozen scenario answer keys carry no `factIds` mappings, so that metric would be unavailable.
|
|
702
|
+
|
|
693
703
|
**Report presentation.** `report.incrementalChangeStaleness` (schema `my-dev-kit-lab-incremental-change-staleness-report-v2`) holds the summary counts (including per-treatment applied-partial, fallback, and classification counts), per-scenario lifecycle, four treatments and three comparisons identified by explicit treatment IDs, and fixed limitations text. Unavailable evidence is written as `unavailable` or `not applicable`, never zero. Historical V1 reports keep their original two-treatment presentation.
|
|
694
704
|
|
|
695
705
|
## Retrieval precision/recall evidence (v0.8.0)
|