@dailephd/my-dev-kit-lab 0.9.3 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +7 -3
- package/dist/src/evaluation/affectedNeighborhood.js +140 -32
- package/dist/src/evaluation/benchmarkSandbox/createBenchmarkSandbox.js +9 -1
- package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js +91 -1
- package/dist/src/evaluation/runMyDevKitRetrieval.js +137 -4
- package/dist/src/evaluation/syntheticRepository/materialize.js +103 -7
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +10 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +9 -1
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +41 -5
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +4 -4
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +4 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +1 -1
- package/dist/src/securityValidation/attackScenarios/boundedSourceScan.js +112 -25
- package/dist/src/securityValidation/attackScenarios/scenarios/networkAssumptionScenario.js +58 -6
- package/dist/src/securityValidation/cliAdversarial/pathBoundaryChecks.js +108 -70
- package/dist/src/securityValidation/cliAdversarial/tempWorkspace.js +57 -8
- package/dist/src/securityValidation/cliAdversarial/traversalContainment.js +161 -0
- package/dist/src/securityValidation/commandRunner.js +52 -26
- package/dist/src/securityValidation/config.js +10 -0
- package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +46 -21
- package/dist/src/securityValidation/validate/runCliSecuritySuiteCheck.js +21 -5
- package/dist/src/securityValidation/validate/runSecurityValidation.js +2 -2
- package/dist/src/tutorial/tutorialAssertions.js +7 -0
- package/docs/METRICS.md +11 -1
- package/package.json +3 -3
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { lstatSync, realpathSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// Containment policy for deliberately traversing --out / --index values.
|
|
5
|
+
//
|
|
6
|
+
// A traversing value may only be handed to a CLI when its full destination is
|
|
7
|
+
// proven to stay inside a NEW task-owned disposable enclosure. The enclosure is
|
|
8
|
+
// not the declared output directory: the value may leave the declared
|
|
9
|
+
// directory, but never the enclosure. Anything that cannot be proven is refused
|
|
10
|
+
// before launch.
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
/** Largest number of net parent components the harness will accommodate. */
|
|
13
|
+
export const MAX_TRAVERSAL_DEPTH = 10;
|
|
14
|
+
/** Longest destination (excluding the file a CLI adds) allowed on Windows. */
|
|
15
|
+
const MAX_WINDOWS_DESTINATION_LENGTH = 200;
|
|
16
|
+
const MAX_POSIX_DESTINATION_LENGTH = 3000;
|
|
17
|
+
function pathFor(platform) {
|
|
18
|
+
return platform === "win32" ? path.win32 : path.posix;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Pure classification of a traversal value according to how the given host
|
|
22
|
+
* platform resolves it. Reports how many parent components the value needs
|
|
23
|
+
* room for, or refuses values that are absolute or ambiguous on that host.
|
|
24
|
+
*/
|
|
25
|
+
export function classifyTraversalValue(value, platform = process.platform) {
|
|
26
|
+
if (value.length === 0) {
|
|
27
|
+
return { kind: "refused", reason: "empty path value" };
|
|
28
|
+
}
|
|
29
|
+
if (value.includes("\0")) {
|
|
30
|
+
return { kind: "refused", reason: "path value contains a NUL character" };
|
|
31
|
+
}
|
|
32
|
+
const host = pathFor(platform);
|
|
33
|
+
if (host.isAbsolute(value)) {
|
|
34
|
+
return { kind: "refused", reason: `absolute path on ${platform}: destination is not confined to the enclosure` };
|
|
35
|
+
}
|
|
36
|
+
if (platform === "win32") {
|
|
37
|
+
if (/^[A-Za-z]:/.test(value)) {
|
|
38
|
+
return { kind: "refused", reason: "drive-qualified path on win32: drive-relative resolution is ambiguous" };
|
|
39
|
+
}
|
|
40
|
+
if (/[<>:"|?*]/.test(value)) {
|
|
41
|
+
return { kind: "refused", reason: "win32 path contains reserved characters (stream or device ambiguity)" };
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
const segments = value.split(platform === "win32" ? /[\\/]+/ : /\/+/);
|
|
45
|
+
let depth = 0;
|
|
46
|
+
let lowest = 0;
|
|
47
|
+
for (const segment of segments) {
|
|
48
|
+
if (segment === "" || segment === ".")
|
|
49
|
+
continue;
|
|
50
|
+
if (segment === "..") {
|
|
51
|
+
depth -= 1;
|
|
52
|
+
lowest = Math.min(lowest, depth);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
if (platform === "win32" && /[. ]$/.test(segment)) {
|
|
56
|
+
return { kind: "refused", reason: "win32 path component ends in a dot or space and would be rewritten by the OS" };
|
|
57
|
+
}
|
|
58
|
+
depth += 1;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
const nestDepth = lowest === 0 ? 0 : -lowest;
|
|
62
|
+
if (nestDepth > MAX_TRAVERSAL_DEPTH) {
|
|
63
|
+
return {
|
|
64
|
+
kind: "refused",
|
|
65
|
+
reason: `traversal depth ${nestDepth} exceeds the supported maximum of ${MAX_TRAVERSAL_DEPTH}`,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
return { kind: "relative", nestDepth };
|
|
69
|
+
}
|
|
70
|
+
/** True when `candidate` equals or is below `parent` under the host's path semantics. */
|
|
71
|
+
export function isPathInside(parent, candidate, platform = process.platform) {
|
|
72
|
+
const host = pathFor(platform);
|
|
73
|
+
const relative = host.relative(parent, candidate);
|
|
74
|
+
if (relative === "")
|
|
75
|
+
return true;
|
|
76
|
+
return !relative.startsWith("..") && !host.isAbsolute(relative);
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Resolves `value` against `baseDir` and proves, using the real filesystem,
|
|
80
|
+
* that the destination, its nearest existing parent and every not-yet-created
|
|
81
|
+
* descendant stay inside `enclosureDir`. Never creates anything.
|
|
82
|
+
*/
|
|
83
|
+
export function proveContainment(options) {
|
|
84
|
+
const { value, baseDir, enclosureDir, declaredDir } = options;
|
|
85
|
+
const platform = process.platform;
|
|
86
|
+
const host = pathFor(platform);
|
|
87
|
+
const plan = classifyTraversalValue(value, platform);
|
|
88
|
+
if (plan.kind === "refused")
|
|
89
|
+
return { ok: false, reason: plan.reason };
|
|
90
|
+
let enclosure;
|
|
91
|
+
let realBase;
|
|
92
|
+
let realDeclared;
|
|
93
|
+
try {
|
|
94
|
+
enclosure = realpathSync.native(enclosureDir);
|
|
95
|
+
realBase = realpathSync.native(baseDir);
|
|
96
|
+
realDeclared = realpathSync.native(declaredDir);
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
return { ok: false, reason: `enclosure, base or declared directory is not inspectable: ${errorMessage(error)}` };
|
|
100
|
+
}
|
|
101
|
+
if (!isPathInside(enclosure, realBase, platform) || !isPathInside(enclosure, realDeclared, platform)) {
|
|
102
|
+
return { ok: false, reason: "base or declared directory is not inside the enclosure" };
|
|
103
|
+
}
|
|
104
|
+
const destination = host.resolve(realBase, value);
|
|
105
|
+
if (!isPathInside(enclosure, destination, platform)) {
|
|
106
|
+
return { ok: false, reason: `destination ${destination} resolves outside the enclosure ${enclosure}` };
|
|
107
|
+
}
|
|
108
|
+
const limit = platform === "win32" ? MAX_WINDOWS_DESTINATION_LENGTH : MAX_POSIX_DESTINATION_LENGTH;
|
|
109
|
+
if (destination.length > limit) {
|
|
110
|
+
return { ok: false, reason: `destination length ${destination.length} exceeds the supported ${limit}` };
|
|
111
|
+
}
|
|
112
|
+
// Find the nearest existing physical ancestor; anything else is inspectable
|
|
113
|
+
// only if lstat answers either "exists" or "does not exist".
|
|
114
|
+
let nearest = destination;
|
|
115
|
+
for (;;) {
|
|
116
|
+
try {
|
|
117
|
+
lstatSync(nearest);
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
if (error.code !== "ENOENT") {
|
|
122
|
+
return { ok: false, reason: `cannot inspect ${nearest}: ${errorMessage(error)}` };
|
|
123
|
+
}
|
|
124
|
+
const parent = host.dirname(nearest);
|
|
125
|
+
if (parent === nearest || !isPathInside(enclosure, parent, platform)) {
|
|
126
|
+
return { ok: false, reason: "no existing ancestor found inside the enclosure" };
|
|
127
|
+
}
|
|
128
|
+
nearest = parent;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
// Existing part must be a plain directory chain: no links, junctions or
|
|
132
|
+
// case/short-name aliases (the real path must equal the lexical path).
|
|
133
|
+
let nearestReal;
|
|
134
|
+
try {
|
|
135
|
+
if (lstatSync(nearest).isSymbolicLink()) {
|
|
136
|
+
return { ok: false, reason: `${nearest} is a symbolic link or junction` };
|
|
137
|
+
}
|
|
138
|
+
nearestReal = realpathSync.native(nearest);
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
return { ok: false, reason: `cannot resolve ${nearest}: ${errorMessage(error)}` };
|
|
142
|
+
}
|
|
143
|
+
if (nearestReal !== nearest) {
|
|
144
|
+
return { ok: false, reason: `${nearest} is an alias of ${nearestReal} (link, junction, case or short-name ambiguity)` };
|
|
145
|
+
}
|
|
146
|
+
if (!isPathInside(enclosure, nearestReal, platform)) {
|
|
147
|
+
return { ok: false, reason: `${nearest} resolves outside the enclosure` };
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
ok: true,
|
|
151
|
+
destination,
|
|
152
|
+
enclosure,
|
|
153
|
+
nearestExisting: nearest,
|
|
154
|
+
relativeToEnclosure: host.relative(enclosure, destination).split(host.sep).join("/"),
|
|
155
|
+
declared: realDeclared,
|
|
156
|
+
insideDeclared: isPathInside(realDeclared, destination, platform),
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
function errorMessage(error) {
|
|
160
|
+
return error instanceof Error ? error.message : String(error);
|
|
161
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { resolveCommand } from "../core/resolveCommand.js";
|
|
3
|
+
import { terminateProcessTree } from "../core/processTree.js";
|
|
3
4
|
// Prefer the unqualified npm command and let the shared resolver locate
|
|
4
5
|
// npm.cmd/.ps1 shims or direct executables per platform.
|
|
5
6
|
export function resolveNpmCommand() {
|
|
@@ -13,6 +14,8 @@ export function buildSecurityCommandSpawnOptions(options) {
|
|
|
13
14
|
return {
|
|
14
15
|
cwd: options.cwd,
|
|
15
16
|
shell: false,
|
|
17
|
+
windowsHide: true,
|
|
18
|
+
detached: process.platform !== "win32",
|
|
16
19
|
stdio: ["ignore", "pipe", "pipe"],
|
|
17
20
|
env: options.env,
|
|
18
21
|
};
|
|
@@ -53,6 +56,8 @@ export async function runSecurityCommand(options) {
|
|
|
53
56
|
stdout: "",
|
|
54
57
|
stderr: resolved.warnings.join("\n"),
|
|
55
58
|
timedOut: false,
|
|
59
|
+
resolutionKind: resolved.resolutionKind,
|
|
60
|
+
timeoutMs: options.timeoutMs,
|
|
56
61
|
skipped: false,
|
|
57
62
|
};
|
|
58
63
|
}
|
|
@@ -62,6 +67,12 @@ export async function runSecurityCommand(options) {
|
|
|
62
67
|
let timedOut = false;
|
|
63
68
|
let settled = false;
|
|
64
69
|
let timeout;
|
|
70
|
+
let closeGrace;
|
|
71
|
+
let termination;
|
|
72
|
+
let treeTerminated;
|
|
73
|
+
let signal;
|
|
74
|
+
let exitCode = null;
|
|
75
|
+
let spawnError;
|
|
65
76
|
let child;
|
|
66
77
|
try {
|
|
67
78
|
child = spawn(command, args, buildSecurityCommandSpawnOptions({ cwd: options.cwd, env }));
|
|
@@ -76,6 +87,9 @@ export async function runSecurityCommand(options) {
|
|
|
76
87
|
stdout: "",
|
|
77
88
|
stderr: spawnErr instanceof Error ? spawnErr.message : String(spawnErr),
|
|
78
89
|
timedOut: false,
|
|
90
|
+
spawnError: spawnErr instanceof Error ? spawnErr.message : String(spawnErr),
|
|
91
|
+
resolutionKind: resolved.resolutionKind,
|
|
92
|
+
timeoutMs: options.timeoutMs,
|
|
79
93
|
skipped: false,
|
|
80
94
|
});
|
|
81
95
|
return;
|
|
@@ -86,52 +100,64 @@ export async function runSecurityCommand(options) {
|
|
|
86
100
|
child.stderr.on("data", (chunk) => {
|
|
87
101
|
stderr += chunk.toString("utf8");
|
|
88
102
|
});
|
|
89
|
-
|
|
103
|
+
const finish = async () => {
|
|
90
104
|
if (settled)
|
|
91
105
|
return;
|
|
92
106
|
settled = true;
|
|
93
107
|
if (timeout)
|
|
94
108
|
clearTimeout(timeout);
|
|
109
|
+
if (termination)
|
|
110
|
+
await termination;
|
|
111
|
+
if (closeGrace)
|
|
112
|
+
clearTimeout(closeGrace);
|
|
95
113
|
resolve({
|
|
96
114
|
command,
|
|
97
115
|
args,
|
|
98
116
|
cwd: options.cwd,
|
|
99
|
-
exitCode
|
|
117
|
+
exitCode,
|
|
100
118
|
durationMs: Date.now() - started,
|
|
101
119
|
stdout,
|
|
102
|
-
stderr
|
|
120
|
+
stderr,
|
|
103
121
|
timedOut,
|
|
122
|
+
signal,
|
|
123
|
+
spawnError,
|
|
124
|
+
resolutionKind: resolved.resolutionKind,
|
|
125
|
+
timeoutMs: options.timeoutMs,
|
|
126
|
+
treeTerminated,
|
|
104
127
|
skipped: false,
|
|
105
128
|
});
|
|
129
|
+
};
|
|
130
|
+
child.on("error", (error) => {
|
|
131
|
+
spawnError = error.message;
|
|
132
|
+
stderr += `${stderr ? "\n" : ""}${error.message}`;
|
|
133
|
+
void finish();
|
|
106
134
|
});
|
|
107
135
|
if (options.timeoutMs > 0) {
|
|
108
136
|
timeout = setTimeout(() => {
|
|
109
137
|
timedOut = true;
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
}
|
|
138
|
+
// Keep the parent alive until its entire owned tree is addressed.
|
|
139
|
+
// Killing only npm's cmd parent can strand its children and their pipes.
|
|
140
|
+
termination = terminateProcessTree(child.pid, { waitTimeoutMs: 5000 }).then((terminated) => {
|
|
141
|
+
treeTerminated = terminated;
|
|
142
|
+
return terminated;
|
|
143
|
+
});
|
|
144
|
+
void termination.then(() => {
|
|
145
|
+
if (settled)
|
|
146
|
+
return;
|
|
147
|
+
// A failed tree operation or a pipe that never closes must not make
|
|
148
|
+
// the command wait indefinitely beyond its deadline and kill grace.
|
|
149
|
+
closeGrace = setTimeout(() => {
|
|
150
|
+
child.stdout.destroy();
|
|
151
|
+
child.stderr.destroy();
|
|
152
|
+
void finish();
|
|
153
|
+
}, 1000);
|
|
154
|
+
});
|
|
116
155
|
}, options.timeoutMs);
|
|
117
156
|
}
|
|
118
|
-
child.on("close", (
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
if (timeout)
|
|
123
|
-
clearTimeout(timeout);
|
|
124
|
-
resolve({
|
|
125
|
-
command,
|
|
126
|
-
args,
|
|
127
|
-
cwd: options.cwd,
|
|
128
|
-
exitCode,
|
|
129
|
-
durationMs: Date.now() - started,
|
|
130
|
-
stdout,
|
|
131
|
-
stderr,
|
|
132
|
-
timedOut,
|
|
133
|
-
skipped: false,
|
|
134
|
-
});
|
|
157
|
+
child.on("close", (code, exitSignal) => {
|
|
158
|
+
exitCode = code;
|
|
159
|
+
signal = exitSignal;
|
|
160
|
+
void finish();
|
|
135
161
|
});
|
|
136
162
|
});
|
|
137
163
|
}
|
|
@@ -37,3 +37,13 @@ export const DEFAULT_SECURITY_CONFIG = {
|
|
|
37
37
|
commandTimeoutMs: 60_000,
|
|
38
38
|
requireOsvScanner: false,
|
|
39
39
|
};
|
|
40
|
+
// Windows native Semgrep startup can exceed the shared one-minute deadline.
|
|
41
|
+
// Keep this allowance local to Semgrep, bounded, and independently cancellable.
|
|
42
|
+
export function resolveSemgrepCommandTimeoutMs(config, platform = process.platform) {
|
|
43
|
+
const timeoutMs = config.semgrepCommandTimeoutMs ??
|
|
44
|
+
(platform === "win32" ? 300_000 : config.commandTimeoutMs);
|
|
45
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
46
|
+
throw new Error("Semgrep process timeout must be a positive finite duration.");
|
|
47
|
+
}
|
|
48
|
+
return platform === "win32" ? Math.min(timeoutMs, 300_000) : timeoutMs;
|
|
49
|
+
}
|
|
@@ -14,27 +14,13 @@ export function parseNpmPackDryRun(stdout) {
|
|
|
14
14
|
return { files: [], parseError: "npm pack --dry-run produced no output" };
|
|
15
15
|
}
|
|
16
16
|
if (/^[\[{]/.test(stdout.trim())) {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
for (const file of inventory[0].files) {
|
|
25
|
-
if (!file || typeof file.path !== "string" || !safeInventoryPath(file.path)) {
|
|
26
|
-
return { files: [], parseError: "npm pack JSON contains a malformed or unsafe file path" };
|
|
27
|
-
}
|
|
28
|
-
// JSON paths are package-relative, unlike text tarball paths. Add the
|
|
29
|
-
// archive root exactly once so a real nested package/ directory cannot
|
|
30
|
-
// masquerade as a root entrypoint in the existing detectors.
|
|
31
|
-
files.push(`package/${file.path.replace(/\\/g, "/").replace(/^(\.\/)+/, "")}`);
|
|
32
|
-
}
|
|
33
|
-
return { files };
|
|
34
|
-
}
|
|
35
|
-
catch {
|
|
36
|
-
return { files: [], parseError: "npm pack JSON is unparseable" };
|
|
37
|
-
}
|
|
17
|
+
const parsed = parseNpmPackJsonEntry(stdout);
|
|
18
|
+
if (!parsed.entry)
|
|
19
|
+
return { files: [], parseError: parsed.parseError };
|
|
20
|
+
// JSON paths are package-relative, unlike text tarball paths. Add the
|
|
21
|
+
// archive root exactly once so a real nested package/ directory cannot
|
|
22
|
+
// masquerade as a root entrypoint in the existing detectors.
|
|
23
|
+
return { files: parsed.entry.files.map((file) => `package/${file}`) };
|
|
38
24
|
}
|
|
39
25
|
const files = [];
|
|
40
26
|
let inContents = false;
|
|
@@ -78,6 +64,45 @@ export function parseNpmPackDryRun(stdout) {
|
|
|
78
64
|
}
|
|
79
65
|
return { files, totalSize };
|
|
80
66
|
}
|
|
67
|
+
// Validated single-package entry from `npm pack --json` (or --dry-run --json) output, accepting both
|
|
68
|
+
// the npm 11 array shape and the npm 12 identity-keyed object shape. Exactly one package with a
|
|
69
|
+
// nonempty, safe file list is required; anything else is an error, never an empty success.
|
|
70
|
+
export function parseNpmPackJsonEntry(stdout) {
|
|
71
|
+
try {
|
|
72
|
+
const packages = packageInventories(JSON.parse(stdout));
|
|
73
|
+
const inventory = packages?.length === 1 ? packages[0] : null;
|
|
74
|
+
if (!isPlainObject(inventory) || !Array.isArray(inventory.files) || !inventory.files.length) {
|
|
75
|
+
return { parseError: "npm pack JSON must contain exactly one nonempty package inventory" };
|
|
76
|
+
}
|
|
77
|
+
const files = [];
|
|
78
|
+
for (const file of inventory.files) {
|
|
79
|
+
if (!file || typeof file.path !== "string" || !safeInventoryPath(file.path)) {
|
|
80
|
+
return { parseError: "npm pack JSON contains a malformed or unsafe file path" };
|
|
81
|
+
}
|
|
82
|
+
files.push(file.path.replace(/\\/g, "/").replace(/^(\.\/)+/, ""));
|
|
83
|
+
}
|
|
84
|
+
const text = (value) => (typeof value === "string" ? value : undefined);
|
|
85
|
+
return { entry: { name: text(inventory.name), version: text(inventory.version), filename: text(inventory.filename), files } };
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
return { parseError: "npm pack JSON is unparseable" };
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
function isPlainObject(value) {
|
|
92
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
93
|
+
}
|
|
94
|
+
// The package inventories in `npm pack --json` output. npm 11 and earlier print an array of package
|
|
95
|
+
// objects; npm 12 prints one object keyed by package identity whose values are the package objects.
|
|
96
|
+
// Anything else (including a bare package object, or a keyed object holding non-objects) is not a
|
|
97
|
+
// recognized shape and yields null, so it can never be read as an empty or successful package.
|
|
98
|
+
function packageInventories(output) {
|
|
99
|
+
if (Array.isArray(output))
|
|
100
|
+
return output;
|
|
101
|
+
if (!isPlainObject(output))
|
|
102
|
+
return null;
|
|
103
|
+
const packages = Object.values(output);
|
|
104
|
+
return packages.every(isPlainObject) ? packages : null;
|
|
105
|
+
}
|
|
81
106
|
function safeInventoryPath(file) {
|
|
82
107
|
const normalized = file.replace(/\\/g, "/").replace(/^(\.\/)+/, "");
|
|
83
108
|
return Boolean(normalized) && !normalized.startsWith("/") &&
|
|
@@ -65,10 +65,17 @@ export async function runCliSecuritySuiteCheck(options) {
|
|
|
65
65
|
timeoutMs: Math.min(options.timeoutMs * 3, 180_000),
|
|
66
66
|
});
|
|
67
67
|
const finishedAt = new Date().toISOString();
|
|
68
|
-
const passed = cmd.exitCode === 0;
|
|
68
|
+
const passed = cmd.exitCode === 0 && !cmd.timedOut && !cmd.spawnError && cmd.treeTerminated !== false;
|
|
69
69
|
const name = isSelf
|
|
70
70
|
? "CLI adversarial test suite (tool self-tests)"
|
|
71
71
|
: "Target security test suite";
|
|
72
|
+
const outcome = cmd.timedOut
|
|
73
|
+
? `exceeded its ${cmd.timeoutMs ?? "configured"} ms deadline (exit code ${String(cmd.exitCode)}).`
|
|
74
|
+
: cmd.spawnError
|
|
75
|
+
? "could not be started; see process error evidence."
|
|
76
|
+
: cmd.treeTerminated === false
|
|
77
|
+
? "did not confirm process-tree cleanup."
|
|
78
|
+
: `exited with code ${String(cmd.exitCode)}.`;
|
|
72
79
|
return {
|
|
73
80
|
id: "cli-adversarial-suite",
|
|
74
81
|
name,
|
|
@@ -87,8 +94,8 @@ export async function runCliSecuritySuiteCheck(options) {
|
|
|
87
94
|
severity: "major",
|
|
88
95
|
category: "cli-adversarial",
|
|
89
96
|
description: isSelf
|
|
90
|
-
? `Tool-root test:security
|
|
91
|
-
: `Target-root test:security
|
|
97
|
+
? `Tool-root test:security ${outcome}`
|
|
98
|
+
: `Target-root test:security ${outcome}`,
|
|
92
99
|
evidence: buildEvidence(cmd),
|
|
93
100
|
recommendation: isSelf
|
|
94
101
|
? "Fix failing tool security tests before release."
|
|
@@ -104,14 +111,23 @@ export async function runCliSecuritySuiteCheck(options) {
|
|
|
104
111
|
};
|
|
105
112
|
}
|
|
106
113
|
function summarizeOutput(value) {
|
|
107
|
-
const normalized = value.replace(/\
|
|
114
|
+
const normalized = value.replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "")
|
|
115
|
+
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "")
|
|
116
|
+
.replace(/\r\n/g, "\n").trim();
|
|
108
117
|
if (!normalized) {
|
|
109
118
|
return undefined;
|
|
110
119
|
}
|
|
111
120
|
return normalized.slice(0, 500);
|
|
112
121
|
}
|
|
113
122
|
function buildEvidence(cmd) {
|
|
114
|
-
const parts = [
|
|
123
|
+
const parts = [
|
|
124
|
+
`cwd=${cmd.cwd}`, `exitCode=${String(cmd.exitCode)}`,
|
|
125
|
+
`timedOut=${cmd.timedOut}`, `signal=${cmd.signal === undefined ? "unknown" : String(cmd.signal)}`,
|
|
126
|
+
`timeoutMs=${cmd.timeoutMs ?? "unknown"}`, `durationMs=${cmd.durationMs}`,
|
|
127
|
+
`resolutionKind=${cmd.resolutionKind ?? "unknown"}`, `treeTerminated=${cmd.treeTerminated ?? "not-requested"}`,
|
|
128
|
+
];
|
|
129
|
+
if (cmd.spawnError)
|
|
130
|
+
parts.push(`spawnError=${summarizeOutput(cmd.spawnError)}`);
|
|
115
131
|
const stdoutSummary = summarizeOutput(cmd.stdout);
|
|
116
132
|
const stderrSummary = summarizeOutput(cmd.stderr);
|
|
117
133
|
if (stdoutSummary) {
|
|
@@ -8,7 +8,7 @@ import { runAllFuzzTargets } from "../fuzz/fuzzHarness.js";
|
|
|
8
8
|
import { ALL_FUZZ_TARGETS } from "../fuzz/fuzzTargets.js";
|
|
9
9
|
import { calculateVerdict, summarizeVerdictReasoning } from "./verdict.js";
|
|
10
10
|
import { resolveValidationTarget, targetDescription as describeTarget } from "./resolveTarget.js";
|
|
11
|
-
import { DEFAULT_SECURITY_CONFIG } from "../config.js";
|
|
11
|
+
import { DEFAULT_SECURITY_CONFIG, resolveSemgrepCommandTimeoutMs } from "../config.js";
|
|
12
12
|
import { runCliSecuritySuiteCheck } from "./runCliSecuritySuiteCheck.js";
|
|
13
13
|
import { DEFAULT_SECURITY_CHECKS, DEFAULT_SECURITY_PROFILE, IMPLEMENTED_SECURITY_CHECK_IDS, PLANNED_SECURITY_CHECK_IDS, } from "./cliOptions.js";
|
|
14
14
|
import { runAttackScenarios } from "../attackScenarios/attackRunner.js";
|
|
@@ -102,7 +102,7 @@ export async function runSecurityValidation(options) {
|
|
|
102
102
|
collect(await runSemgrepCheck({
|
|
103
103
|
targetRoot: target.targetRoot,
|
|
104
104
|
toolRoot,
|
|
105
|
-
timeoutMs: config
|
|
105
|
+
timeoutMs: resolveSemgrepCommandTimeoutMs(config),
|
|
106
106
|
}));
|
|
107
107
|
}
|
|
108
108
|
catch (err) {
|
|
@@ -66,6 +66,13 @@ export function resolveAssertionRequestUrl(applicationUrl, suppliedPath) {
|
|
|
66
66
|
throw new Error(`HTTP assertion path must be a root-relative path beginning with "/" and must not be protocol-relative; received ${JSON.stringify(suppliedPath)}.`);
|
|
67
67
|
}
|
|
68
68
|
const base = new URL(applicationUrl);
|
|
69
|
+
// Defend the exported assertion boundary as well as the validated tutorial
|
|
70
|
+
// caller: same-origin alone does not make an unchecked remote origin local.
|
|
71
|
+
const hostname = base.hostname.replace(/^\[/, "").replace(/\]$/, "").toLowerCase();
|
|
72
|
+
if ((base.protocol !== "http:" && base.protocol !== "https:") ||
|
|
73
|
+
!["localhost", "127.0.0.1", "::1"].includes(hostname)) {
|
|
74
|
+
throw new Error("HTTP assertion application origin must use HTTP(S) on loopback.");
|
|
75
|
+
}
|
|
69
76
|
const resolved = new URL(suppliedPath, base);
|
|
70
77
|
if (resolved.protocol !== base.protocol || resolved.hostname !== base.hostname || resolved.port !== base.port) {
|
|
71
78
|
throw new Error(`HTTP assertion path ${JSON.stringify(suppliedPath)} resolves to ${resolved.origin}, which leaves the application origin ${base.origin}.`);
|
package/docs/METRICS.md
CHANGED
|
@@ -646,7 +646,7 @@ The `incremental-change-staleness` plugin reuses the six existing v0.6.1 affecte
|
|
|
646
646
|
**Required-file relation** (per scenario, comparing the two treatments' required-file status):
|
|
647
647
|
|
|
648
648
|
- `stale-worse`: `stale-index` is `missing` while `full-refresh` is `present`.
|
|
649
|
-
- `same`: both treatments have the same status
|
|
649
|
+
- `same`: both treatments have the same status, except that `missing`/`missing` is `unknown` (see the reference-adequacy gate below).
|
|
650
650
|
- `stale-better`: `stale-index` is `present` while `full-refresh` is `missing`.
|
|
651
651
|
- `unknown`: either treatment's status is `unknown`.
|
|
652
652
|
|
|
@@ -690,6 +690,16 @@ Availability: the first three are always present for a validated incremental tre
|
|
|
690
690
|
|
|
691
691
|
`no-observed-regression-relative-to-full` means only that the executed scenario showed no regression under those two evidence dimensions. It does not establish equivalence or safety and does not show that full refresh is unnecessary. `not-comparable-as-partial-refresh` says the run gives no evidence about the requested partial behavior. There is no winner, ranking, composite score, safety score, safe-to-skip-refresh flag, or stale-risk percentage. Ordinary per-treatment correctness `score` remains the existing deterministic fake-agent evidence.
|
|
692
692
|
|
|
693
|
+
**Evidence families and the reference-adequacy gate (v0.9.5).** The staleness experiment keeps five kinds of evidence distinct and never converts one into another:
|
|
694
|
+
|
|
695
|
+
- *Source read*: a file whose own `source` command succeeded under the bounded multi-source retrieval protocol (`bounded-multi-source-v1`: one `search` with an explicit limit, then at most 8 distinct candidate files in search rank order, each with one `source` command). `retrieval.filesRead` and `requiredFileEvidence.observedFiles` carry only these. Candidates come only from the actual search output; the answer key never selects them.
|
|
696
|
+
- *Surfaced*: a file or symbol that search, lookup or slice merely returned. It is not credited as read.
|
|
697
|
+
- *Expected but not observed*: `requiredFileEvidence.missingFiles`. `missing` is relative to the fixed candidate budget and is not an exhaustive source-set assessment; its reason states the budget and truncation flags. Unsourced required files are `unknown` (not `missing`) when the protocol cannot source as many files as are required, a bounded source command failed, or candidate identities were rejected.
|
|
698
|
+
- *Synthetic agent correctness*: the deterministic fake agent reports only two expected facts, so scenarios requiring three or more facts score 0.75 (0.70 for T1) and never pass. This is not measured my-dev-kit answer correctness, is identical across treatments, and cannot discriminate between them.
|
|
699
|
+
- *Candidate relative to full refresh*: the relation and classification fields. Full refresh is the reference, not an automatically adequate treatment: an equal required-file status supports `same` only when the full-refresh reference itself is `present`. `missing`/`missing` is reported as `unknown`, so an all-equal but insufficient experiment is `inconclusive`, never `no-observed-regression`.
|
|
700
|
+
|
|
701
|
+
Fact-context-target coverage from `retrievalQuality` is not used here: five of the six frozen scenario answer keys carry no `factIds` mappings, so that metric would be unavailable.
|
|
702
|
+
|
|
693
703
|
**Report presentation.** `report.incrementalChangeStaleness` (schema `my-dev-kit-lab-incremental-change-staleness-report-v2`) holds the summary counts (including per-treatment applied-partial, fallback, and classification counts), per-scenario lifecycle, four treatments and three comparisons identified by explicit treatment IDs, and fixed limitations text. Unavailable evidence is written as `unavailable` or `not applicable`, never zero. Historical V1 reports keep their original two-treatment presentation.
|
|
694
704
|
|
|
695
705
|
## Retrieval precision/recall evidence (v0.8.0)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dailephd/my-dev-kit-lab",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.5",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
|
|
6
6
|
"bin": {
|
|
@@ -93,10 +93,10 @@
|
|
|
93
93
|
"@types/node": "^24.0.0",
|
|
94
94
|
"tsx": "^4.20.3",
|
|
95
95
|
"typescript": "^5.8.3",
|
|
96
|
-
"vitest": "
|
|
96
|
+
"vitest": "4.1.12"
|
|
97
97
|
},
|
|
98
98
|
"overrides": {
|
|
99
|
-
"vite": "7.3.
|
|
99
|
+
"vite": "7.3.7"
|
|
100
100
|
},
|
|
101
101
|
"engines": {
|
|
102
102
|
"node": ">=24"
|