@dailephd/my-dev-kit-lab 0.9.3 → 0.9.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/README.md +7 -3
  3. package/dist/src/evaluation/affectedNeighborhood.js +140 -32
  4. package/dist/src/evaluation/benchmarkSandbox/createBenchmarkSandbox.js +9 -1
  5. package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js +91 -1
  6. package/dist/src/evaluation/runMyDevKitRetrieval.js +137 -4
  7. package/dist/src/evaluation/syntheticRepository/materialize.js +103 -7
  8. package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +10 -0
  9. package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +9 -1
  10. package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +41 -5
  11. package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +4 -4
  12. package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +4 -0
  13. package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +1 -1
  14. package/dist/src/securityValidation/attackScenarios/boundedSourceScan.js +112 -25
  15. package/dist/src/securityValidation/attackScenarios/scenarios/networkAssumptionScenario.js +58 -6
  16. package/dist/src/securityValidation/cliAdversarial/pathBoundaryChecks.js +108 -70
  17. package/dist/src/securityValidation/cliAdversarial/tempWorkspace.js +57 -8
  18. package/dist/src/securityValidation/cliAdversarial/traversalContainment.js +161 -0
  19. package/dist/src/securityValidation/commandRunner.js +52 -26
  20. package/dist/src/securityValidation/config.js +10 -0
  21. package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +46 -21
  22. package/dist/src/securityValidation/validate/runCliSecuritySuiteCheck.js +21 -5
  23. package/dist/src/securityValidation/validate/runSecurityValidation.js +2 -2
  24. package/dist/src/tutorial/tutorialAssertions.js +7 -0
  25. package/docs/METRICS.md +11 -1
  26. package/package.json +3 -3
@@ -0,0 +1,161 @@
1
+ import { lstatSync, realpathSync } from "node:fs";
2
+ import path from "node:path";
3
+ // ---------------------------------------------------------------------------
4
+ // Containment policy for deliberately traversing --out / --index values.
5
+ //
6
+ // A traversing value may only be handed to a CLI when its full destination is
7
+ // proven to stay inside a NEW task-owned disposable enclosure. The enclosure is
8
+ // not the declared output directory: the value may leave the declared
9
+ // directory, but never the enclosure. Anything that cannot be proven is refused
10
+ // before launch.
11
+ // ---------------------------------------------------------------------------
12
+ /** Largest number of net parent components the harness will accommodate. */
13
+ export const MAX_TRAVERSAL_DEPTH = 10;
14
+ /** Longest destination (excluding the file a CLI adds) allowed on Windows. */
15
+ const MAX_WINDOWS_DESTINATION_LENGTH = 200;
16
+ const MAX_POSIX_DESTINATION_LENGTH = 3000;
17
+ function pathFor(platform) {
18
+ return platform === "win32" ? path.win32 : path.posix;
19
+ }
20
+ /**
21
+ * Pure classification of a traversal value according to how the given host
22
+ * platform resolves it. Reports how many parent components the value needs
23
+ * room for, or refuses values that are absolute or ambiguous on that host.
24
+ */
25
+ export function classifyTraversalValue(value, platform = process.platform) {
26
+ if (value.length === 0) {
27
+ return { kind: "refused", reason: "empty path value" };
28
+ }
29
+ if (value.includes("\0")) {
30
+ return { kind: "refused", reason: "path value contains a NUL character" };
31
+ }
32
+ const host = pathFor(platform);
33
+ if (host.isAbsolute(value)) {
34
+ return { kind: "refused", reason: `absolute path on ${platform}: destination is not confined to the enclosure` };
35
+ }
36
+ if (platform === "win32") {
37
+ if (/^[A-Za-z]:/.test(value)) {
38
+ return { kind: "refused", reason: "drive-qualified path on win32: drive-relative resolution is ambiguous" };
39
+ }
40
+ if (/[<>:"|?*]/.test(value)) {
41
+ return { kind: "refused", reason: "win32 path contains reserved characters (stream or device ambiguity)" };
42
+ }
43
+ }
44
+ const segments = value.split(platform === "win32" ? /[\\/]+/ : /\/+/);
45
+ let depth = 0;
46
+ let lowest = 0;
47
+ for (const segment of segments) {
48
+ if (segment === "" || segment === ".")
49
+ continue;
50
+ if (segment === "..") {
51
+ depth -= 1;
52
+ lowest = Math.min(lowest, depth);
53
+ }
54
+ else {
55
+ if (platform === "win32" && /[. ]$/.test(segment)) {
56
+ return { kind: "refused", reason: "win32 path component ends in a dot or space and would be rewritten by the OS" };
57
+ }
58
+ depth += 1;
59
+ }
60
+ }
61
+ const nestDepth = lowest === 0 ? 0 : -lowest;
62
+ if (nestDepth > MAX_TRAVERSAL_DEPTH) {
63
+ return {
64
+ kind: "refused",
65
+ reason: `traversal depth ${nestDepth} exceeds the supported maximum of ${MAX_TRAVERSAL_DEPTH}`,
66
+ };
67
+ }
68
+ return { kind: "relative", nestDepth };
69
+ }
70
+ /** True when `candidate` equals or is below `parent` under the host's path semantics. */
71
+ export function isPathInside(parent, candidate, platform = process.platform) {
72
+ const host = pathFor(platform);
73
+ const relative = host.relative(parent, candidate);
74
+ if (relative === "")
75
+ return true;
76
+ return !relative.startsWith("..") && !host.isAbsolute(relative);
77
+ }
78
+ /**
79
+ * Resolves `value` against `baseDir` and proves, using the real filesystem,
80
+ * that the destination, its nearest existing parent and every not-yet-created
81
+ * descendant stay inside `enclosureDir`. Never creates anything.
82
+ */
83
+ export function proveContainment(options) {
84
+ const { value, baseDir, enclosureDir, declaredDir } = options;
85
+ const platform = process.platform;
86
+ const host = pathFor(platform);
87
+ const plan = classifyTraversalValue(value, platform);
88
+ if (plan.kind === "refused")
89
+ return { ok: false, reason: plan.reason };
90
+ let enclosure;
91
+ let realBase;
92
+ let realDeclared;
93
+ try {
94
+ enclosure = realpathSync.native(enclosureDir);
95
+ realBase = realpathSync.native(baseDir);
96
+ realDeclared = realpathSync.native(declaredDir);
97
+ }
98
+ catch (error) {
99
+ return { ok: false, reason: `enclosure, base or declared directory is not inspectable: ${errorMessage(error)}` };
100
+ }
101
+ if (!isPathInside(enclosure, realBase, platform) || !isPathInside(enclosure, realDeclared, platform)) {
102
+ return { ok: false, reason: "base or declared directory is not inside the enclosure" };
103
+ }
104
+ const destination = host.resolve(realBase, value);
105
+ if (!isPathInside(enclosure, destination, platform)) {
106
+ return { ok: false, reason: `destination ${destination} resolves outside the enclosure ${enclosure}` };
107
+ }
108
+ const limit = platform === "win32" ? MAX_WINDOWS_DESTINATION_LENGTH : MAX_POSIX_DESTINATION_LENGTH;
109
+ if (destination.length > limit) {
110
+ return { ok: false, reason: `destination length ${destination.length} exceeds the supported ${limit}` };
111
+ }
112
+ // Find the nearest existing physical ancestor; anything else is inspectable
113
+ // only if lstat answers either "exists" or "does not exist".
114
+ let nearest = destination;
115
+ for (;;) {
116
+ try {
117
+ lstatSync(nearest);
118
+ break;
119
+ }
120
+ catch (error) {
121
+ if (error.code !== "ENOENT") {
122
+ return { ok: false, reason: `cannot inspect ${nearest}: ${errorMessage(error)}` };
123
+ }
124
+ const parent = host.dirname(nearest);
125
+ if (parent === nearest || !isPathInside(enclosure, parent, platform)) {
126
+ return { ok: false, reason: "no existing ancestor found inside the enclosure" };
127
+ }
128
+ nearest = parent;
129
+ }
130
+ }
131
+ // Existing part must be a plain directory chain: no links, junctions or
132
+ // case/short-name aliases (the real path must equal the lexical path).
133
+ let nearestReal;
134
+ try {
135
+ if (lstatSync(nearest).isSymbolicLink()) {
136
+ return { ok: false, reason: `${nearest} is a symbolic link or junction` };
137
+ }
138
+ nearestReal = realpathSync.native(nearest);
139
+ }
140
+ catch (error) {
141
+ return { ok: false, reason: `cannot resolve ${nearest}: ${errorMessage(error)}` };
142
+ }
143
+ if (nearestReal !== nearest) {
144
+ return { ok: false, reason: `${nearest} is an alias of ${nearestReal} (link, junction, case or short-name ambiguity)` };
145
+ }
146
+ if (!isPathInside(enclosure, nearestReal, platform)) {
147
+ return { ok: false, reason: `${nearest} resolves outside the enclosure` };
148
+ }
149
+ return {
150
+ ok: true,
151
+ destination,
152
+ enclosure,
153
+ nearestExisting: nearest,
154
+ relativeToEnclosure: host.relative(enclosure, destination).split(host.sep).join("/"),
155
+ declared: realDeclared,
156
+ insideDeclared: isPathInside(realDeclared, destination, platform),
157
+ };
158
+ }
159
+ function errorMessage(error) {
160
+ return error instanceof Error ? error.message : String(error);
161
+ }
@@ -1,5 +1,6 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { resolveCommand } from "../core/resolveCommand.js";
3
+ import { terminateProcessTree } from "../core/processTree.js";
3
4
  // Prefer the unqualified npm command and let the shared resolver locate
4
5
  // npm.cmd/.ps1 shims or direct executables per platform.
5
6
  export function resolveNpmCommand() {
@@ -13,6 +14,8 @@ export function buildSecurityCommandSpawnOptions(options) {
13
14
  return {
14
15
  cwd: options.cwd,
15
16
  shell: false,
17
+ windowsHide: true,
18
+ detached: process.platform !== "win32",
16
19
  stdio: ["ignore", "pipe", "pipe"],
17
20
  env: options.env,
18
21
  };
@@ -53,6 +56,8 @@ export async function runSecurityCommand(options) {
53
56
  stdout: "",
54
57
  stderr: resolved.warnings.join("\n"),
55
58
  timedOut: false,
59
+ resolutionKind: resolved.resolutionKind,
60
+ timeoutMs: options.timeoutMs,
56
61
  skipped: false,
57
62
  };
58
63
  }
@@ -62,6 +67,12 @@ export async function runSecurityCommand(options) {
62
67
  let timedOut = false;
63
68
  let settled = false;
64
69
  let timeout;
70
+ let closeGrace;
71
+ let termination;
72
+ let treeTerminated;
73
+ let signal;
74
+ let exitCode = null;
75
+ let spawnError;
65
76
  let child;
66
77
  try {
67
78
  child = spawn(command, args, buildSecurityCommandSpawnOptions({ cwd: options.cwd, env }));
@@ -76,6 +87,9 @@ export async function runSecurityCommand(options) {
76
87
  stdout: "",
77
88
  stderr: spawnErr instanceof Error ? spawnErr.message : String(spawnErr),
78
89
  timedOut: false,
90
+ spawnError: spawnErr instanceof Error ? spawnErr.message : String(spawnErr),
91
+ resolutionKind: resolved.resolutionKind,
92
+ timeoutMs: options.timeoutMs,
79
93
  skipped: false,
80
94
  });
81
95
  return;
@@ -86,52 +100,64 @@ export async function runSecurityCommand(options) {
86
100
  child.stderr.on("data", (chunk) => {
87
101
  stderr += chunk.toString("utf8");
88
102
  });
89
- child.on("error", (error) => {
103
+ const finish = async () => {
90
104
  if (settled)
91
105
  return;
92
106
  settled = true;
93
107
  if (timeout)
94
108
  clearTimeout(timeout);
109
+ if (termination)
110
+ await termination;
111
+ if (closeGrace)
112
+ clearTimeout(closeGrace);
95
113
  resolve({
96
114
  command,
97
115
  args,
98
116
  cwd: options.cwd,
99
- exitCode: null,
117
+ exitCode,
100
118
  durationMs: Date.now() - started,
101
119
  stdout,
102
- stderr: error.message,
120
+ stderr,
103
121
  timedOut,
122
+ signal,
123
+ spawnError,
124
+ resolutionKind: resolved.resolutionKind,
125
+ timeoutMs: options.timeoutMs,
126
+ treeTerminated,
104
127
  skipped: false,
105
128
  });
129
+ };
130
+ child.on("error", (error) => {
131
+ spawnError = error.message;
132
+ stderr += `${stderr ? "\n" : ""}${error.message}`;
133
+ void finish();
106
134
  });
107
135
  if (options.timeoutMs > 0) {
108
136
  timeout = setTimeout(() => {
109
137
  timedOut = true;
110
- try {
111
- child.kill("SIGTERM");
112
- }
113
- catch {
114
- // Process may have already exited.
115
- }
138
+ // Keep the parent alive until its entire owned tree is addressed.
139
+ // Killing only npm's cmd parent can strand its children and their pipes.
140
+ termination = terminateProcessTree(child.pid, { waitTimeoutMs: 5000 }).then((terminated) => {
141
+ treeTerminated = terminated;
142
+ return terminated;
143
+ });
144
+ void termination.then(() => {
145
+ if (settled)
146
+ return;
147
+ // A failed tree operation or a pipe that never closes must not make
148
+ // the command wait indefinitely beyond its deadline and kill grace.
149
+ closeGrace = setTimeout(() => {
150
+ child.stdout.destroy();
151
+ child.stderr.destroy();
152
+ void finish();
153
+ }, 1000);
154
+ });
116
155
  }, options.timeoutMs);
117
156
  }
118
- child.on("close", (exitCode) => {
119
- if (settled)
120
- return;
121
- settled = true;
122
- if (timeout)
123
- clearTimeout(timeout);
124
- resolve({
125
- command,
126
- args,
127
- cwd: options.cwd,
128
- exitCode,
129
- durationMs: Date.now() - started,
130
- stdout,
131
- stderr,
132
- timedOut,
133
- skipped: false,
134
- });
157
+ child.on("close", (code, exitSignal) => {
158
+ exitCode = code;
159
+ signal = exitSignal;
160
+ void finish();
135
161
  });
136
162
  });
137
163
  }
@@ -37,3 +37,13 @@ export const DEFAULT_SECURITY_CONFIG = {
37
37
  commandTimeoutMs: 60_000,
38
38
  requireOsvScanner: false,
39
39
  };
40
+ // Windows native Semgrep startup can exceed the shared one-minute deadline.
41
+ // Keep this allowance local to Semgrep, bounded, and independently cancellable.
42
+ export function resolveSemgrepCommandTimeoutMs(config, platform = process.platform) {
43
+ const timeoutMs = config.semgrepCommandTimeoutMs ??
44
+ (platform === "win32" ? 300_000 : config.commandTimeoutMs);
45
+ if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
46
+ throw new Error("Semgrep process timeout must be a positive finite duration.");
47
+ }
48
+ return platform === "win32" ? Math.min(timeoutMs, 300_000) : timeoutMs;
49
+ }
@@ -14,27 +14,13 @@ export function parseNpmPackDryRun(stdout) {
14
14
  return { files: [], parseError: "npm pack --dry-run produced no output" };
15
15
  }
16
16
  if (/^[\[{]/.test(stdout.trim())) {
17
- try {
18
- const inventory = JSON.parse(stdout);
19
- if (!Array.isArray(inventory) || inventory.length !== 1 ||
20
- !inventory[0] || !Array.isArray(inventory[0].files) || !inventory[0].files.length) {
21
- return { files: [], parseError: "npm pack JSON must contain exactly one nonempty package inventory" };
22
- }
23
- const files = [];
24
- for (const file of inventory[0].files) {
25
- if (!file || typeof file.path !== "string" || !safeInventoryPath(file.path)) {
26
- return { files: [], parseError: "npm pack JSON contains a malformed or unsafe file path" };
27
- }
28
- // JSON paths are package-relative, unlike text tarball paths. Add the
29
- // archive root exactly once so a real nested package/ directory cannot
30
- // masquerade as a root entrypoint in the existing detectors.
31
- files.push(`package/${file.path.replace(/\\/g, "/").replace(/^(\.\/)+/, "")}`);
32
- }
33
- return { files };
34
- }
35
- catch {
36
- return { files: [], parseError: "npm pack JSON is unparseable" };
37
- }
17
+ const parsed = parseNpmPackJsonEntry(stdout);
18
+ if (!parsed.entry)
19
+ return { files: [], parseError: parsed.parseError };
20
+ // JSON paths are package-relative, unlike text tarball paths. Add the
21
+ // archive root exactly once so a real nested package/ directory cannot
22
+ // masquerade as a root entrypoint in the existing detectors.
23
+ return { files: parsed.entry.files.map((file) => `package/${file}`) };
38
24
  }
39
25
  const files = [];
40
26
  let inContents = false;
@@ -78,6 +64,45 @@ export function parseNpmPackDryRun(stdout) {
78
64
  }
79
65
  return { files, totalSize };
80
66
  }
67
+ // Validated single-package entry from `npm pack --json` (or --dry-run --json) output, accepting both
68
+ // the npm 11 array shape and the npm 12 identity-keyed object shape. Exactly one package with a
69
+ // nonempty, safe file list is required; anything else is an error, never an empty success.
70
+ export function parseNpmPackJsonEntry(stdout) {
71
+ try {
72
+ const packages = packageInventories(JSON.parse(stdout));
73
+ const inventory = packages?.length === 1 ? packages[0] : null;
74
+ if (!isPlainObject(inventory) || !Array.isArray(inventory.files) || !inventory.files.length) {
75
+ return { parseError: "npm pack JSON must contain exactly one nonempty package inventory" };
76
+ }
77
+ const files = [];
78
+ for (const file of inventory.files) {
79
+ if (!file || typeof file.path !== "string" || !safeInventoryPath(file.path)) {
80
+ return { parseError: "npm pack JSON contains a malformed or unsafe file path" };
81
+ }
82
+ files.push(file.path.replace(/\\/g, "/").replace(/^(\.\/)+/, ""));
83
+ }
84
+ const text = (value) => (typeof value === "string" ? value : undefined);
85
+ return { entry: { name: text(inventory.name), version: text(inventory.version), filename: text(inventory.filename), files } };
86
+ }
87
+ catch {
88
+ return { parseError: "npm pack JSON is unparseable" };
89
+ }
90
+ }
91
+ function isPlainObject(value) {
92
+ return typeof value === "object" && value !== null && !Array.isArray(value);
93
+ }
94
+ // The package inventories in `npm pack --json` output. npm 11 and earlier print an array of package
95
+ // objects; npm 12 prints one object keyed by package identity whose values are the package objects.
96
+ // Anything else (including a bare package object, or a keyed object holding non-objects) is not a
97
+ // recognized shape and yields null, so it can never be read as an empty or successful package.
98
+ function packageInventories(output) {
99
+ if (Array.isArray(output))
100
+ return output;
101
+ if (!isPlainObject(output))
102
+ return null;
103
+ const packages = Object.values(output);
104
+ return packages.every(isPlainObject) ? packages : null;
105
+ }
81
106
  function safeInventoryPath(file) {
82
107
  const normalized = file.replace(/\\/g, "/").replace(/^(\.\/)+/, "");
83
108
  return Boolean(normalized) && !normalized.startsWith("/") &&
@@ -65,10 +65,17 @@ export async function runCliSecuritySuiteCheck(options) {
65
65
  timeoutMs: Math.min(options.timeoutMs * 3, 180_000),
66
66
  });
67
67
  const finishedAt = new Date().toISOString();
68
- const passed = cmd.exitCode === 0;
68
+ const passed = cmd.exitCode === 0 && !cmd.timedOut && !cmd.spawnError && cmd.treeTerminated !== false;
69
69
  const name = isSelf
70
70
  ? "CLI adversarial test suite (tool self-tests)"
71
71
  : "Target security test suite";
72
+ const outcome = cmd.timedOut
73
+ ? `exceeded its ${cmd.timeoutMs ?? "configured"} ms deadline (exit code ${String(cmd.exitCode)}).`
74
+ : cmd.spawnError
75
+ ? "could not be started; see process error evidence."
76
+ : cmd.treeTerminated === false
77
+ ? "did not confirm process-tree cleanup."
78
+ : `exited with code ${String(cmd.exitCode)}.`;
72
79
  return {
73
80
  id: "cli-adversarial-suite",
74
81
  name,
@@ -87,8 +94,8 @@ export async function runCliSecuritySuiteCheck(options) {
87
94
  severity: "major",
88
95
  category: "cli-adversarial",
89
96
  description: isSelf
90
- ? `Tool-root test:security exited with code ${String(cmd.exitCode)}.`
91
- : `Target-root test:security exited with code ${String(cmd.exitCode)}.`,
97
+ ? `Tool-root test:security ${outcome}`
98
+ : `Target-root test:security ${outcome}`,
92
99
  evidence: buildEvidence(cmd),
93
100
  recommendation: isSelf
94
101
  ? "Fix failing tool security tests before release."
@@ -104,14 +111,23 @@ export async function runCliSecuritySuiteCheck(options) {
104
111
  };
105
112
  }
106
113
  function summarizeOutput(value) {
107
- const normalized = value.replace(/\r\n/g, "\n").trim();
114
+ const normalized = value.replace(/\x1b\[[0-?]*[ -/]*[@-~]/g, "")
115
+ .replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "")
116
+ .replace(/\r\n/g, "\n").trim();
108
117
  if (!normalized) {
109
118
  return undefined;
110
119
  }
111
120
  return normalized.slice(0, 500);
112
121
  }
113
122
  function buildEvidence(cmd) {
114
- const parts = [`cwd=${cmd.cwd}`, `exitCode=${String(cmd.exitCode)}`];
123
+ const parts = [
124
+ `cwd=${cmd.cwd}`, `exitCode=${String(cmd.exitCode)}`,
125
+ `timedOut=${cmd.timedOut}`, `signal=${cmd.signal === undefined ? "unknown" : String(cmd.signal)}`,
126
+ `timeoutMs=${cmd.timeoutMs ?? "unknown"}`, `durationMs=${cmd.durationMs}`,
127
+ `resolutionKind=${cmd.resolutionKind ?? "unknown"}`, `treeTerminated=${cmd.treeTerminated ?? "not-requested"}`,
128
+ ];
129
+ if (cmd.spawnError)
130
+ parts.push(`spawnError=${summarizeOutput(cmd.spawnError)}`);
115
131
  const stdoutSummary = summarizeOutput(cmd.stdout);
116
132
  const stderrSummary = summarizeOutput(cmd.stderr);
117
133
  if (stdoutSummary) {
@@ -8,7 +8,7 @@ import { runAllFuzzTargets } from "../fuzz/fuzzHarness.js";
8
8
  import { ALL_FUZZ_TARGETS } from "../fuzz/fuzzTargets.js";
9
9
  import { calculateVerdict, summarizeVerdictReasoning } from "./verdict.js";
10
10
  import { resolveValidationTarget, targetDescription as describeTarget } from "./resolveTarget.js";
11
- import { DEFAULT_SECURITY_CONFIG } from "../config.js";
11
+ import { DEFAULT_SECURITY_CONFIG, resolveSemgrepCommandTimeoutMs } from "../config.js";
12
12
  import { runCliSecuritySuiteCheck } from "./runCliSecuritySuiteCheck.js";
13
13
  import { DEFAULT_SECURITY_CHECKS, DEFAULT_SECURITY_PROFILE, IMPLEMENTED_SECURITY_CHECK_IDS, PLANNED_SECURITY_CHECK_IDS, } from "./cliOptions.js";
14
14
  import { runAttackScenarios } from "../attackScenarios/attackRunner.js";
@@ -102,7 +102,7 @@ export async function runSecurityValidation(options) {
102
102
  collect(await runSemgrepCheck({
103
103
  targetRoot: target.targetRoot,
104
104
  toolRoot,
105
- timeoutMs: config.commandTimeoutMs,
105
+ timeoutMs: resolveSemgrepCommandTimeoutMs(config),
106
106
  }));
107
107
  }
108
108
  catch (err) {
@@ -66,6 +66,13 @@ export function resolveAssertionRequestUrl(applicationUrl, suppliedPath) {
66
66
  throw new Error(`HTTP assertion path must be a root-relative path beginning with "/" and must not be protocol-relative; received ${JSON.stringify(suppliedPath)}.`);
67
67
  }
68
68
  const base = new URL(applicationUrl);
69
+ // Defend the exported assertion boundary as well as the validated tutorial
70
+ // caller: same-origin alone does not make an unchecked remote origin local.
71
+ const hostname = base.hostname.replace(/^\[/, "").replace(/\]$/, "").toLowerCase();
72
+ if ((base.protocol !== "http:" && base.protocol !== "https:") ||
73
+ !["localhost", "127.0.0.1", "::1"].includes(hostname)) {
74
+ throw new Error("HTTP assertion application origin must use HTTP(S) on loopback.");
75
+ }
69
76
  const resolved = new URL(suppliedPath, base);
70
77
  if (resolved.protocol !== base.protocol || resolved.hostname !== base.hostname || resolved.port !== base.port) {
71
78
  throw new Error(`HTTP assertion path ${JSON.stringify(suppliedPath)} resolves to ${resolved.origin}, which leaves the application origin ${base.origin}.`);
package/docs/METRICS.md CHANGED
@@ -646,7 +646,7 @@ The `incremental-change-staleness` plugin reuses the six existing v0.6.1 affecte
646
646
  **Required-file relation** (per scenario, comparing the two treatments' required-file status):
647
647
 
648
648
  - `stale-worse`: `stale-index` is `missing` while `full-refresh` is `present`.
649
- - `same`: both treatments have the same status (including `missing`/`missing`).
649
+ - `same`: both treatments have the same status, except that `missing`/`missing` is `unknown` (see the reference-adequacy gate below).
650
650
  - `stale-better`: `stale-index` is `present` while `full-refresh` is `missing`.
651
651
  - `unknown`: either treatment's status is `unknown`.
652
652
 
@@ -690,6 +690,16 @@ Availability: the first three are always present for a validated incremental tre
690
690
 
691
691
  `no-observed-regression-relative-to-full` means only that the executed scenario showed no regression under those two evidence dimensions. It does not establish equivalence or safety and does not show that full refresh is unnecessary. `not-comparable-as-partial-refresh` says the run gives no evidence about the requested partial behavior. There is no winner, ranking, composite score, safety score, safe-to-skip-refresh flag, or stale-risk percentage. Ordinary per-treatment correctness `score` remains the existing deterministic fake-agent evidence.
692
692
 
693
+ **Evidence families and the reference-adequacy gate (v0.9.5).** The staleness experiment keeps five kinds of evidence distinct and never converts one into another:
694
+
695
+ - *Source read*: a file whose own `source` command succeeded under the bounded multi-source retrieval protocol (`bounded-multi-source-v1`: one `search` with an explicit limit, then at most 8 distinct candidate files in search rank order, each with one `source` command). `retrieval.filesRead` and `requiredFileEvidence.observedFiles` carry only these. Candidates come only from the actual search output; the answer key never selects them.
696
+ - *Surfaced*: a file or symbol that search, lookup or slice merely returned. It is not credited as read.
697
+ - *Expected but not observed*: `requiredFileEvidence.missingFiles`. `missing` is relative to the fixed candidate budget and is not an exhaustive source-set assessment; its reason states the budget and truncation flags. Unsourced required files are `unknown` (not `missing`) when the protocol cannot source as many files as are required, a bounded source command failed, or candidate identities were rejected.
698
+ - *Synthetic agent correctness*: the deterministic fake agent reports only two expected facts, so scenarios requiring three or more facts score 0.75 (0.70 for T1) and never pass. This is not measured my-dev-kit answer correctness, is identical across treatments, and cannot discriminate between them.
699
+ - *Candidate relative to full refresh*: the relation and classification fields. Full refresh is the reference, not an automatically adequate treatment: an equal required-file status supports `same` only when the full-refresh reference itself is `present`. `missing`/`missing` is reported as `unknown`, so an all-equal but insufficient experiment is `inconclusive`, never `no-observed-regression`.
700
+
701
+ Fact-context-target coverage from `retrievalQuality` is not used here: five of the six frozen scenario answer keys carry no `factIds` mappings, so that metric would be unavailable.
702
+
693
703
  **Report presentation.** `report.incrementalChangeStaleness` (schema `my-dev-kit-lab-incremental-change-staleness-report-v2`) holds the summary counts (including per-treatment applied-partial, fallback, and classification counts), per-scenario lifecycle, four treatments and three comparisons identified by explicit treatment IDs, and fixed limitations text. Unavailable evidence is written as `unavailable` or `not applicable`, never zero. Historical V1 reports keep their original two-treatment presentation.
694
704
 
695
705
  ## Retrieval precision/recall evidence (v0.8.0)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dailephd/my-dev-kit-lab",
3
- "version": "0.9.3",
3
+ "version": "0.9.5",
4
4
  "type": "module",
5
5
  "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
6
6
  "bin": {
@@ -93,10 +93,10 @@
93
93
  "@types/node": "^24.0.0",
94
94
  "tsx": "^4.20.3",
95
95
  "typescript": "^5.8.3",
96
- "vitest": "^4.1.11"
96
+ "vitest": "4.1.12"
97
97
  },
98
98
  "overrides": {
99
- "vite": "7.3.6"
99
+ "vite": "7.3.7"
100
100
  },
101
101
  "engines": {
102
102
  "node": ">=24"