@dailephd/my-dev-kit-lab 0.9.4 → 0.9.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/README.md +4 -2
  3. package/dist/src/evaluation/benchmarkSandbox/createBenchmarkSandbox.js +9 -1
  4. package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js +91 -1
  5. package/dist/src/evaluation/runMyDevKitRetrieval.js +137 -4
  6. package/dist/src/evaluation/syntheticRepository/materialize.js +103 -7
  7. package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +10 -0
  8. package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +9 -1
  9. package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +41 -5
  10. package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +4 -4
  11. package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +4 -0
  12. package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +1 -1
  13. package/dist/src/securityValidation/attackScenarios/boundedSourceScan.js +112 -25
  14. package/dist/src/securityValidation/attackScenarios/scenarios/networkAssumptionScenario.js +58 -6
  15. package/dist/src/securityValidation/cliAdversarial/pathBoundaryChecks.js +108 -70
  16. package/dist/src/securityValidation/cliAdversarial/tempWorkspace.js +57 -8
  17. package/dist/src/securityValidation/cliAdversarial/traversalContainment.js +161 -0
  18. package/dist/src/securityValidation/commandRunner.js +52 -26
  19. package/dist/src/securityValidation/config.js +10 -0
  20. package/dist/src/securityValidation/validate/runCliSecuritySuiteCheck.js +21 -5
  21. package/dist/src/securityValidation/validate/runSecurityValidation.js +2 -2
  22. package/dist/src/tutorial/tutorialAssertions.js +7 -0
  23. package/docs/METRICS.md +11 -1
  24. package/package.json +1 -1
@@ -1,7 +1,7 @@
1
1
  import path from "node:path";
2
2
  import { resolveWithinRoot } from "../../../core/pathSafety.js";
3
- import { classifyStaleRisk, compareCorrectness, compareRequiredFileEvidence } from "./comparison.js";
4
- import { compareCandidateCorrectness, compareCandidateRequiredFileEvidence, classifyPartialRefreshAgainstFull } from "./comparisonV2.js";
3
+ import { classifyStaleRisk, compareCorrectness, compareRequiredFileEvidenceAgainstReference } from "./comparison.js";
4
+ import { compareCandidateCorrectness, compareCandidateRequiredFileEvidenceAgainstReference, classifyPartialRefreshAgainstFull } from "./comparisonV2.js";
5
5
  import { INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS } from "./disposableTarget.js";
6
6
  import { assessIncrementalChangeStalenessAffectedNeighborhood, checkIncrementalChangeStalenessAffectedNeighborhoodSymmetry, correctnessComparable, defaultIncrementalChangeStalenessExecutionDeps, executeTreatmentEvaluation, INCONCLUSIVE_NO_EXECUTION, resolveIncrementalChangeStalenessScenarioTask } from "./execution.js";
7
7
  /**
@@ -153,7 +153,7 @@ export function buildReferenceComparisons(treatments) {
153
153
  throw new Error("Reference comparisons require the stale-index and full-refresh treatment executions.");
154
154
  // Historical v0.6.2 semantics, unchanged.
155
155
  const staleCorrectness = compareCorrectness(correctnessComparable(stale.fakeAgent), correctnessComparable(reference.fakeAgent));
156
- const staleRequired = compareRequiredFileEvidence(stale.requiredFileEvidence, reference.requiredFileEvidence);
156
+ const staleRequired = compareRequiredFileEvidenceAgainstReference(stale.requiredFileEvidence, reference.requiredFileEvidence);
157
157
  const staleRisk = classifyStaleRisk(staleCorrectness, staleRequired);
158
158
  const comparisons = [
159
159
  {
@@ -174,7 +174,7 @@ export function buildReferenceComparisons(treatments) {
174
174
  throw new Error(`${candidateTreatmentId} has no incremental refresh execution to compare.`);
175
175
  }
176
176
  const correctnessRelation = compareCandidateCorrectness(correctnessComparable(candidate.fakeAgent), correctnessComparable(reference.fakeAgent));
177
- const requiredFileEvidenceRelation = compareCandidateRequiredFileEvidence(candidate.requiredFileEvidence, reference.requiredFileEvidence);
177
+ const requiredFileEvidenceRelation = compareCandidateRequiredFileEvidenceAgainstReference(candidate.requiredFileEvidence, reference.requiredFileEvidence);
178
178
  const classified = classifyPartialRefreshAgainstFull({
179
179
  realization: candidate.refreshExecution.realization,
180
180
  correctnessRelation,
@@ -14,6 +14,10 @@ export const LIMITATIONS_V2 = [
14
14
  "\"no-observed-regression-relative-to-full\" means only that the executed scenario did not show a regression under the frozen correctness and required-file evidence dimensions. It does not prove general equivalence or safety, and does not show that full refresh is unnecessary.",
15
15
  "Required-file evidence is a bounded answer-key file-presence check, not retrieval precision/recall.",
16
16
  "Correctness remains deterministic fake-agent/harness evidence in v0.6.3.",
17
+ "Fake-agent correctness is a synthetic score computed from the answer key (the fake agent deliberately reports only two expected facts, so scenarios requiring three or more facts cannot pass). It is not measured my-dev-kit answer correctness and does not discriminate between treatments; a failed synthetic flag is not a my-dev-kit regression and a passing one would not be evidence of Kit correctness.",
18
+ "Files Read and Observed Files count a file only when the bounded retrieval protocol ran a successful source command for it; a file merely surfaced by search, lookup or slice was not read. Artifacts recorded before v0.9.5 name only the single selected search candidate.",
19
+ "A Missing required-file result is relative to the fixed bounded candidate budget and is not an exhaustive source-set assessment; evidence the protocol could not observe (coverage limits, failed source commands, rejected candidates) is reported as unknown. Equal required-file evidence supports a same relation only when the full-refresh reference itself observed every required file; two equally insufficient treatments are reported as unknown/inconclusive.",
20
+ "Command log paths in persisted retrieval telemetry name run-owned files that the mandatory run cleanup removes after the artifact is written; they are not retained and must not be relied on as links.",
17
21
  "Lab affected-neighborhood evidence (changed baseline neighborhood relative to the task) is distinct from upstream refresh execution evidence (what my-dev-kit actually refreshed).",
18
22
  "\"reindexRecommendation\" remains observational evidence; it did not select or trigger any treatment.",
19
23
  "\"changedSymbolCount\" does not mean source-level symbol-diff proof; it retains the released v0.6.1 baseline-symbol-identity definition.",
@@ -27,7 +27,7 @@ export function treatmentBlock(scenario, treatmentId, label) {
27
27
  ["Estimated context tokens", retrieval.status === "not-run" ? "unavailable" : String(retrieval.totalEstimatedTokens)],
28
28
  ["Retrieval duration (ms)", retrieval.status === "not-run" ? "unavailable" : String(retrieval.durationMs)],
29
29
  ["Fake-agent status", display(treatment.fakeAgent?.status, "not run")],
30
- ["Correctness score", correctness && correctness.available && correctness.score !== null ? String(correctness.score) : "unavailable"],
30
+ ["Synthetic fake-agent correctness score", correctness && correctness.available && correctness.score !== null ? String(correctness.score) : "unavailable"],
31
31
  ["Required-file evidence", treatment.requiredFileEvidence.status],
32
32
  ["Required-file reason", display(treatment.requiredFileEvidence.reason, "not applicable")],
33
33
  ["Affected-neighborhood relationship", treatment.affectedNeighborhood.relationship],
@@ -1,14 +1,13 @@
1
1
  import fs from "node:fs";
2
- import { collectFilesForGlobs } from "../../core/fileGlobs.js";
2
+ import path from "node:path";
3
3
  // ---------------------------------------------------------------------------
4
4
  // v0.2.2 Batch 4 — shared bounded, deterministic source-file scan helper.
5
5
  //
6
6
  // Reused by secretLeakageScenario and networkAssumptionScenario. Both need
7
7
  // "read a bounded, deterministic set of source-ish files, skip large/binary
8
- // content" and nothing more — collectFilesForGlobs() already provides
9
- // deterministic ordering and standard exclusions (node_modules, dist, build,
10
- // coverage, .git, lab-output, .my-dev-kit); this layer adds a size cap and a
11
- // small file-content allowlist filter for scenario-specific exclusions.
8
+ // content". Discovery here must refuse links BEFORE traversal or content reads;
9
+ // the general fileGlobs walker follows a linked glob base. Preserve its standard
10
+ // directory/temp exclusions, with a size cap and synthetic-fixture exclusion.
12
11
  // ---------------------------------------------------------------------------
13
12
  // Default bounded glob set: project manifests/config plus first-party source
14
13
  // and scripts. Deliberately excludes tests/, benchmarks/, examples/ — those
@@ -37,28 +36,108 @@ const MAX_SCANNABLE_FILE_BYTES = 1_000_000;
37
36
  // Built via fromCharCode rather than a "\0" string escape to avoid any
38
37
  // literal control byte ending up in this source file.
39
38
  const NUL_CHAR = String.fromCharCode(0);
39
+ const EXCLUDED_SOURCE_DIRECTORIES = new Set([
40
+ "node_modules", "dist", "build", "coverage", ".git", "lab-output",
41
+ ".my-dev-kit", ".my-dev-kit-v1", ".my-dev-kit-lab", "__pycache__"
42
+ ]);
43
+ const samePhysicalPath = (a, b) => process.platform === "win32"
44
+ ? a.toLowerCase() === b.toLowerCase() : a === b;
40
45
  // Returns file contents for the bounded glob set, skipping known synthetic
41
46
  // fixtures, oversized files, and anything unreadable as UTF-8 text (treated
42
47
  // as binary and skipped, not an error). Stable, deterministic order.
43
- export function collectBoundedSourceFiles(targetRoot, globs = DEFAULT_SOURCE_SCAN_GLOBS) {
44
- // collectFilesForGlobs() throws for the *entire* call if any single glob's
45
- // base path doesn't exist (e.g. no .env file, no tsconfig.*.json match) —
46
- // resolve each glob independently so one missing/inapplicable glob doesn't
47
- // silently zero out results for all the others.
48
- const candidateMap = new Map();
49
- for (const glob of globs) {
50
- let matches;
51
- try {
52
- matches = collectFilesForGlobs(targetRoot, [glob]);
48
+ export function collectBoundedSourceFiles(targetRoot, globs = DEFAULT_SOURCE_SCAN_GLOBS, provenanceIssues = [], reviewedPaths = []) {
49
+ const root = path.resolve(targetRoot);
50
+ const issueMap = new Map();
51
+ const inspectedPaths = new Map();
52
+ const relative = (file) => path.relative(root, file).replace(/\\/g, "/") || ".";
53
+ const refuse = (file, reason) => {
54
+ issueMap.set(relative(file), reason);
55
+ return undefined;
56
+ };
57
+ // lstat every component, including the scan root, before realpath or reads.
58
+ // This rejects junctions/symlinks, physical aliases and multiply-linked files.
59
+ // Rechecking before a read narrows races; it is not a race-free OS sandbox.
60
+ function inspect(file) {
61
+ const rel = path.relative(root, file);
62
+ if (rel === ".." || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel)) {
63
+ return refuse(root, "path-outside-scan-root");
53
64
  }
54
- catch {
55
- // This glob's base path doesn't exist for this target — a legitimate
56
- // "nothing to scan for this pattern" outcome, not an error.
57
- continue;
65
+ let current = root;
66
+ let stat;
67
+ for (const segment of ["", ...rel.split(path.sep).filter(Boolean)]) {
68
+ if (segment)
69
+ current = path.join(current, segment);
70
+ try {
71
+ stat = fs.lstatSync(current);
72
+ if (stat.isSymbolicLink())
73
+ return refuse(current, "symlink-or-junction");
74
+ if (!stat.isDirectory() && !stat.isFile())
75
+ return refuse(current, "non-regular-path");
76
+ if (stat.isFile() && stat.nlink > 1)
77
+ return refuse(current, "multiply-linked-source");
78
+ const physical = fs.realpathSync.native(current);
79
+ if (!samePhysicalPath(physical, current)) {
80
+ return refuse(current, "physical-path-alias");
81
+ }
82
+ inspectedPaths.set(current, physical);
83
+ }
84
+ catch (error) {
85
+ // Missing optional files are normal for generic external targets.
86
+ if (error.code !== "ENOENT")
87
+ refuse(current, "unreadable-path");
88
+ return undefined;
89
+ }
58
90
  }
59
- for (const m of matches) {
60
- candidateMap.set(m.relativePath, m.absolutePath);
91
+ return stat;
92
+ }
93
+ // Absence is normal in unrelated targets; an existing reviewed path that is
94
+ // not a direct regular file must not disappear into an empty directory scan.
95
+ for (const reviewed of reviewedPaths) {
96
+ const file = path.resolve(root, reviewed);
97
+ const stat = inspect(file);
98
+ if (stat && !stat.isFile())
99
+ refuse(file, "non-regular-reviewed-source");
100
+ }
101
+ const candidateMap = new Map();
102
+ for (const glob of globs) {
103
+ const pattern = glob.replace(/\\/g, "/");
104
+ const wildcard = pattern.search(/[*?]/);
105
+ const prefix = wildcard < 0 ? pattern : pattern.slice(0, wildcard);
106
+ const base = wildcard < 0 ? prefix : prefix.slice(0, prefix.lastIndexOf("/") + 1) || ".";
107
+ const recursive = pattern.slice(base === "." ? 0 : base.length).includes("/");
108
+ function visit(file) {
109
+ const stat = inspect(file);
110
+ if (!stat)
111
+ return;
112
+ const rel = relative(file);
113
+ if (stat.isDirectory()) {
114
+ let entries;
115
+ try {
116
+ entries = fs.readdirSync(file).sort();
117
+ }
118
+ catch {
119
+ refuse(file, "unreadable-directory");
120
+ return;
121
+ }
122
+ for (const name of entries) {
123
+ const child = path.join(file, name);
124
+ // Root-level wildcard globs must not traverse unrelated directories.
125
+ if (recursive || path.matchesGlob(relative(child), pattern)) {
126
+ // Exclusions name generated directories, never ordinary files.
127
+ // Inspect first so a link with an excluded name stays reportable.
128
+ if (EXCLUDED_SOURCE_DIRECTORIES.has(name) && inspect(child)?.isDirectory())
129
+ continue;
130
+ visit(child);
131
+ }
132
+ }
133
+ }
134
+ else if (path.matchesGlob(rel, pattern) && !/\.(?:tmp|temp|log|pyc)$|~$/.test(rel)) {
135
+ // Windows identity comparisons may ignore case for containment, but
136
+ // reviewed trust must use observed spelling, including the glob base.
137
+ candidateMap.set(relative(inspectedPaths.get(file)), file);
138
+ }
61
139
  }
140
+ visit(path.resolve(root, base));
62
141
  }
63
142
  const candidates = [...candidateMap.entries()]
64
143
  .sort(([a], [b]) => a.localeCompare(b))
@@ -70,23 +149,31 @@ export function collectBoundedSourceFiles(targetRoot, globs = DEFAULT_SOURCE_SCA
70
149
  continue;
71
150
  let stat;
72
151
  try {
73
- stat = fs.statSync(candidate.absolutePath);
152
+ stat = inspect(candidate.absolutePath);
74
153
  }
75
154
  catch {
76
155
  continue;
77
156
  }
78
- if (!stat.isFile() || stat.size > MAX_SCANNABLE_FILE_BYTES)
157
+ if (!stat?.isFile())
79
158
  continue;
159
+ if (stat.size > MAX_SCANNABLE_FILE_BYTES) {
160
+ refuse(candidate.absolutePath, "oversized-source");
161
+ continue;
162
+ }
80
163
  try {
81
164
  const content = fs.readFileSync(candidate.absolutePath, "utf8");
82
- if (content.indexOf(NUL_CHAR) !== -1)
165
+ if (content.indexOf(NUL_CHAR) !== -1) {
166
+ refuse(candidate.absolutePath, "binary-source");
83
167
  continue;
168
+ }
84
169
  files.push({ relativePath: normalizedRel, absolutePath: candidate.absolutePath, content });
85
170
  }
86
171
  catch {
87
- continue;
172
+ refuse(candidate.absolutePath, "unreadable-source");
88
173
  }
89
174
  }
175
+ provenanceIssues.push(...[...issueMap.entries()].sort(([a], [b]) => a.localeCompare(b))
176
+ .map(([relativePath, reason]) => ({ relativePath, reason })));
90
177
  return files;
91
178
  }
92
179
  // Returns the 1-based line number containing `index` within `content`.
@@ -1,5 +1,6 @@
1
1
  import { makeEvidence } from "../exploitEvidence.js";
2
2
  import { collectBoundedSourceFiles, lineNumberAt } from "../boundedSourceScan.js";
3
+ import { createHash } from "node:crypto";
3
4
  // ---------------------------------------------------------------------------
4
5
  // v0.2.2 Batch 4 — network / local-first assumption scenario.
5
6
  //
@@ -31,6 +32,21 @@ const KNOWN_NETWORK_INTENTIONAL_PREFIXES = [
31
32
  function isKnownIntentionalPath(relativePath) {
32
33
  return KNOWN_NETWORK_INTENTIONAL_PREFIXES.some((p) => relativePath.startsWith(p));
33
34
  }
35
+ // Reviewed source identities, not file exemptions. A changed site, extra match,
36
+ // missing dependency, or changed caller precondition invalidates recognition.
37
+ const REVIEWED_LOCAL_HTTP_SITES = [
38
+ { file: "src/runtime/managedProcess.ts", symbol: "waitForReadiness", sha256: "978dacaf401cac3c3f6fdc96ca92163b766b14bead7f3a7c49b1ca28d8ac3900", dependencies: {} },
39
+ { file: "src/tutorial/tutorialAssertions.ts", symbol: "http-json-equals / resolveAssertionRequestUrl", sha256: "eba83bd94342d108e1ecc8491e09e584360095823332dffe2df28e49ecf011b3", dependencies: {
40
+ "src/tutorial/runTutorial.ts": "ded69670f0aec6ad31e714e77c6a8e769abcb28fcdc68b81584fd582e593e298",
41
+ "src/tutorial/tutorialSession.ts": "08e0317600754a033ebf8022e984e941c60786bbc3c9baafc3fa7be3ccdd90ce",
42
+ "src/tutorial/targetContractValidation.ts": "da6c51c36827bf3d9b7375cdd71e488d35732f389f989a7dbc462b21063a96a6",
43
+ "src/tutorial/loadTutorialContracts.ts": "10e17f851febf783076d2d7ad5de70f6e950308819ec30edaecf749ba4b11c7a",
44
+ "src/tutorial/validationPrimitives.ts": "e49258d33d0a88d051fa783d8e2236827940ed5895fd1ce844b9f58b4ec08cb1"
45
+ } }
46
+ ];
47
+ // Normalize only checkout EOLs for these exact reviewed-content comparisons.
48
+ // The tutorialSession identity is the verified LF form of the reviewed CRLF text.
49
+ const sourceDigest = (content) => createHash("sha256").update(content.replace(/\r\n/g, "\n"), "utf8").digest("hex");
34
50
  function scanFileForNetworkApis(relativePath, content) {
35
51
  const matches = [];
36
52
  const intentional = isKnownIntentionalPath(relativePath);
@@ -58,7 +74,12 @@ export const NETWORK_ASSUMPTION_SCENARIO = {
58
74
  expectedSafeBehavior: "No obvious unexpected network API usage is found in the bounded local-first source paths inspected.",
59
75
  evidenceRequirements: ["observation"],
60
76
  run: async (ctx) => {
61
- const files = collectBoundedSourceFiles(ctx.target.targetRoot);
77
+ const provenanceIssues = [];
78
+ const reviewedPaths = new Set(REVIEWED_LOCAL_HTTP_SITES.flatMap(site => [site.file, ...Object.keys(site.dependencies)]));
79
+ const files = collectBoundedSourceFiles(ctx.target.targetRoot, undefined, provenanceIssues, [...reviewedPaths]);
80
+ // Size/binary exclusions remain bounded-scan policy for ordinary sources;
81
+ // they can never establish the identity of a reviewed site or precondition.
82
+ const refusals = provenanceIssues.filter(issue => !["oversized-source", "binary-source"].includes(issue.reason) || reviewedPaths.has(issue.relativePath));
62
83
  const isolationCaveat = makeEvidence({
63
84
  kind: "observation",
64
85
  source: "scope statement",
@@ -66,7 +87,7 @@ export const NETWORK_ASSUMPTION_SCENARIO = {
66
87
  expectedBehavior: "This is a bounded static check of local-first source paths only. It does not prove runtime network isolation of subprocess-invoked tools (npm, CodeQL, Semgrep, OSV-Scanner) or any other process spawned during validation.",
67
88
  observedBehavior: "Static-source-only claim; no runtime network instrumentation was performed.",
68
89
  });
69
- if (files.length === 0) {
90
+ if (files.length === 0 && refusals.length === 0) {
70
91
  return {
71
92
  status: "passed",
72
93
  confidence: "low",
@@ -82,10 +103,33 @@ export const NETWORK_ASSUMPTION_SCENARIO = {
82
103
  ],
83
104
  };
84
105
  }
85
- const allMatches = [];
106
+ const allMatches = refusals.map(issue => ({
107
+ id: "untrusted-source-provenance", file: issue.relativePath, line: 1,
108
+ intentional: false, provenanceReason: issue.reason
109
+ }));
86
110
  for (const file of files) {
87
111
  allMatches.push(...scanFileForNetworkApis(file.relativePath, file.content));
88
112
  }
113
+ const byPath = new Map(files.map(file => [file.relativePath, file.content]));
114
+ for (const site of REVIEWED_LOCAL_HTTP_SITES) {
115
+ const content = byPath.get(site.file);
116
+ if (content === undefined)
117
+ continue;
118
+ const siteMatches = allMatches.filter(match => match.file === site.file);
119
+ const recognized = sourceDigest(content) === site.sha256 &&
120
+ siteMatches.length === 1 && siteMatches[0]?.id === "fetch-call" &&
121
+ Object.entries(site.dependencies).every(([file, sha256]) => {
122
+ const dependency = byPath.get(file);
123
+ return dependency !== undefined && sourceDigest(dependency) === sha256;
124
+ });
125
+ if (recognized) {
126
+ siteMatches[0].intentional = true;
127
+ siteMatches[0].reviewedSite = `${site.symbol}; LF-normalized source SHA-256 ${site.sha256}`;
128
+ }
129
+ else if (siteMatches.length === 0) {
130
+ allMatches.push({ id: "reviewed-local-http-source-mismatch", file: site.file, line: 1, intentional: false });
131
+ }
132
+ }
89
133
  allMatches.sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line);
90
134
  const unexpected = allMatches.filter((m) => !m.intentional);
91
135
  const intentional = allMatches.filter((m) => m.intentional);
@@ -103,16 +147,24 @@ export const NETWORK_ASSUMPTION_SCENARIO = {
103
147
  source: "known network-using validation commands",
104
148
  confidence: "high",
105
149
  expectedBehavior: "npm audit / npm outdated (deps check) may contact the npm registry; OSV-Scanner may contact the OSV API; Semgrep's npx fallback may contact the npm registry. These are expected when those check groups are selected via --checks and are not local-first source violations.",
106
- observedBehavior: `${intentional.length} match(es) found under known network-using paths (${KNOWN_NETWORK_INTENTIONAL_PREFIXES.join(", ")}).`,
150
+ observedBehavior: `${intentional.filter(m => !m.reviewedSite).length} match(es) found under known network-using paths (${KNOWN_NETWORK_INTENTIONAL_PREFIXES.join(", ")}).`,
107
151
  }),
152
+ ...intentional.filter(m => m.reviewedSite).map(m => makeEvidence({
153
+ kind: "observation", source: "known-intentional local HTTP use", filePath: m.file, line: m.line,
154
+ confidence: "medium",
155
+ expectedBehavior: "Exactly one reviewed fetch-call with unchanged local HTTP guards and caller preconditions. Bounded static findings do not prove full runtime network isolation.",
156
+ observedBehavior: `Recognized '${m.id}' in ${m.file}:${m.line} (${m.reviewedSite}).`
157
+ })),
108
158
  ...unexpected.slice(0, 20).map((m) => makeEvidence({
109
159
  kind: "observation",
110
- source: `pattern '${m.id}'`,
160
+ source: m.provenanceReason ? "source provenance refusal" : `pattern '${m.id}'`,
111
161
  filePath: m.file,
112
162
  line: m.line,
113
163
  confidence: "medium",
114
164
  expectedBehavior: "No unexpected direct network API usage in local-first source paths.",
115
- observedBehavior: `Detected '${m.id}' usage in ${m.file}:${m.line}.`,
165
+ observedBehavior: m.provenanceReason
166
+ ? `Untrusted source path ${m.file}: ${m.provenanceReason}; source content was not accepted.`
167
+ : `Detected '${m.id}' usage in ${m.file}:${m.line}.`,
116
168
  })),
117
169
  ];
118
170
  if (unexpected.length > 0) {
@@ -2,7 +2,8 @@ import { spawn } from "node:child_process";
2
2
  import path from "node:path";
3
3
  import { buildCliCommand } from "./adversarialCliConfig.js";
4
4
  import { makeFinding, runAdversarialCheck, skippedCheck } from "./runAdversarialCheck.js";
5
- import { createTempWorkspace, diffSnapshots, snapshotDir } from "./tempWorkspace.js";
5
+ import { createTempWorkspace, diffSnapshots, snapshotDir, snapshotTreeStrict } from "./tempWorkspace.js";
6
+ import { classifyTraversalValue, proveContainment } from "./traversalContainment.js";
6
7
  // ---------------------------------------------------------------------------
7
8
  // Path traversal checks
8
9
  // ---------------------------------------------------------------------------
@@ -58,27 +59,68 @@ export async function checkRootPathTraversal(target, testInput) {
58
59
  await workspace.cleanup();
59
60
  }
60
61
  }
61
- /**
62
- * Checks that a path traversal input to --out does not write outside the workspace.
63
- */
64
- export async function checkOutPathTraversal(target, testInput) {
65
- const workspace = createTempWorkspace("p4-out-");
62
+ const OUT_TRAVERSAL = {
63
+ flag: "--out",
64
+ workspacePrefix: "p4-out-",
65
+ idStem: "path-traversal-out",
66
+ nameStem: "Path traversal via --out",
67
+ title: "Write escaped workspace via --out traversal",
68
+ recommendation: "Validate --out argument is within an allowed prefix before writing.",
69
+ declaredDir: (workspace) => workspace.outputDir,
70
+ };
71
+ const INDEX_TRAVERSAL = {
72
+ flag: "--index",
73
+ workspacePrefix: "p4-idx-",
74
+ idStem: "path-traversal-index",
75
+ nameStem: "Path traversal via --index",
76
+ title: "Write escaped workspace via --index traversal",
77
+ recommendation: "Validate --index argument is within an allowed prefix.",
78
+ declaredDir: (workspace) => workspace.indexDir,
79
+ };
80
+ function isUnder(relativePath, relativeDir) {
81
+ return relativePath === relativeDir || relativePath.startsWith(relativeDir + path.sep);
82
+ }
83
+ async function checkContainedTraversal(config, target, testInput) {
84
+ const id = `${config.idStem}-${testInput.id}`;
85
+ const name = `${config.nameStem}: ${testInput.description}`;
86
+ const blocked = (reason) => skippedCheck({
87
+ id,
88
+ name,
89
+ category: "cli-adversarial",
90
+ reason: `BLOCKED_BEFORE_LAUNCH: ${reason}`,
91
+ });
92
+ const plan = classifyTraversalValue(testInput.value);
93
+ if (plan.kind === "refused")
94
+ return blocked(plan.reason);
95
+ const workspace = createTempWorkspace(config.workspacePrefix, { nestDepth: plan.nestDepth });
66
96
  try {
67
- const beforeOutside = snapshotDir(workspace.outsideDir);
68
- // Build an escaped --out path that points outside the workspace.
69
- // We use the actual outsideDir path so we can check for writes there.
70
- const escapedOut = testInput.value.startsWith("/") || /^[A-Za-z]:\\/.test(testInput.value)
71
- ? testInput.value // absolute path given in test input
72
- : path.resolve(workspace.outputDir, testInput.value); // resolve relative from output
97
+ const enclosureDir = workspace.enclosureDir;
98
+ const declaredDir = config.declaredDir(workspace);
99
+ const proof = proveContainment({
100
+ value: testInput.value,
101
+ baseDir: declaredDir,
102
+ enclosureDir,
103
+ declaredDir,
104
+ });
105
+ if (!proof.ok)
106
+ return blocked(proof.reason);
107
+ let before;
108
+ try {
109
+ before = snapshotTreeStrict(enclosureDir);
110
+ }
111
+ catch (error) {
112
+ return blocked(`enclosure is not inspectable before launch: ${error instanceof Error ? error.message : String(error)}`);
113
+ }
73
114
  const { command, args } = buildCliCommand(target, [
74
115
  "--root",
75
116
  workspace.sourceDir,
76
- "--out",
77
- escapedOut,
117
+ config.flag,
118
+ proof.destination,
78
119
  ]);
79
- const result = await runAdversarialCheck({
80
- id: `path-traversal-out-${testInput.id}`,
81
- name: `Path traversal via --out: ${testInput.description}`,
120
+ const allowedRelative = path.relative(proof.enclosure, proof.declared);
121
+ return await runAdversarialCheck({
122
+ id,
123
+ name,
82
124
  category: "cli-adversarial",
83
125
  severity: "blocker",
84
126
  command,
@@ -87,77 +129,73 @@ export async function checkOutPathTraversal(target, testInput) {
87
129
  timeoutMs: target.timeoutMs,
88
130
  evaluate: (cmdResult) => {
89
131
  const findings = [];
90
- const afterOutside = snapshotDir(workspace.outsideDir);
91
- const outsideDiff = diffSnapshots(beforeOutside, afterOutside);
92
- if (outsideDiff.added.length > 0 || outsideDiff.modified.length > 0) {
132
+ // A child that never produced an exit code (and did not time out) did
133
+ // not execute; the runner reports asynchronous spawn failures this way.
134
+ if (cmdResult.spawnError || (cmdResult.exitCode === null && !cmdResult.timedOut)) {
93
135
  findings.push(makeFinding({
94
- id: `path-traversal-out-${testInput.id}-escape`,
95
- title: "Write escaped workspace via --out traversal",
136
+ id: `${id}-not-executed`,
137
+ title: `CLI did not run for ${config.flag} traversal check`,
138
+ severity: "major",
139
+ category: "cli-adversarial",
140
+ description: `Spawn error: ${cmdResult.spawnError ?? cmdResult.stderr.slice(0, 200)}`,
141
+ evidence: `${config.flag} value: ${proof.destination}`,
142
+ recommendation: "Fix the CLI invocation so the traversal check actually executes.",
143
+ }));
144
+ }
145
+ let after;
146
+ try {
147
+ after = snapshotTreeStrict(enclosureDir);
148
+ }
149
+ catch (error) {
150
+ findings.push(makeFinding({
151
+ id: `${id}-inspection-lost`,
152
+ title: `Cannot inspect enclosure after ${config.flag} traversal`,
96
153
  severity: "blocker",
97
154
  category: "cli-adversarial",
98
- description: `CLI wrote outside workspace: ${outsideDiff.added.join(", ")}`,
99
- evidence: `--out value: ${escapedOut}, stdout: ${cmdResult.stdout.slice(0, 200)}`,
100
- recommendation: "Validate --out argument is within an allowed prefix before writing.",
155
+ description: `Post-run snapshot failed: ${error instanceof Error ? error.message : String(error)}`,
156
+ evidence: `${config.flag} value: ${proof.destination}`,
157
+ recommendation: "Treat the traversal outcome as unknown; investigate before release.",
101
158
  }));
159
+ return findings;
102
160
  }
103
- return findings;
104
- },
105
- });
106
- return result;
107
- }
108
- finally {
109
- await workspace.cleanup();
110
- }
111
- }
112
- /**
113
- * Checks that a path traversal input to --index does not allow writes outside workspace.
114
- */
115
- export async function checkIndexPathTraversal(target, testInput) {
116
- const workspace = createTempWorkspace("p4-idx-");
117
- try {
118
- const beforeOutside = snapshotDir(workspace.outsideDir);
119
- const escapedIndex = testInput.value.startsWith("/") || /^[A-Za-z]:\\/.test(testInput.value)
120
- ? testInput.value
121
- : path.resolve(workspace.indexDir, testInput.value);
122
- const { command, args } = buildCliCommand(target, [
123
- "--root",
124
- workspace.sourceDir,
125
- "--index",
126
- escapedIndex,
127
- ]);
128
- const result = await runAdversarialCheck({
129
- id: `path-traversal-index-${testInput.id}`,
130
- name: `Path traversal via --index: ${testInput.description}`,
131
- category: "cli-adversarial",
132
- severity: "blocker",
133
- command,
134
- args,
135
- cwd: workspace.root,
136
- timeoutMs: target.timeoutMs,
137
- evaluate: (cmdResult) => {
138
- const findings = [];
139
- const afterOutside = snapshotDir(workspace.outsideDir);
140
- const outsideDiff = diffSnapshots(beforeOutside, afterOutside);
141
- if (outsideDiff.added.length > 0 || outsideDiff.modified.length > 0) {
161
+ const diff = diffSnapshots(before, after);
162
+ const outside = (list) => list.filter((rel) => !isUnder(rel, allowedRelative));
163
+ const added = outside(diff.added);
164
+ const modified = outside(diff.modified);
165
+ const removed = outside(diff.removed);
166
+ if (added.length + modified.length + removed.length > 0) {
167
+ const changed = [...added, ...modified, ...removed].map((rel) => path.join(proof.enclosure, rel));
142
168
  findings.push(makeFinding({
143
- id: `path-traversal-index-${testInput.id}-escape`,
144
- title: "Write escaped workspace via --index traversal",
169
+ id: `${id}-escape`,
170
+ title: config.title,
145
171
  severity: "blocker",
146
172
  category: "cli-adversarial",
147
- description: `CLI wrote outside workspace: ${outsideDiff.added.join(", ")}`,
148
- evidence: `--index value: ${escapedIndex}, stdout: ${cmdResult.stdout.slice(0, 200)}`,
149
- recommendation: "Validate --index argument is within an allowed prefix.",
173
+ description: `CLI wrote outside workspace: added [${added.join(", ")}], modified [${modified.join(", ")}], removed [${removed.join(", ")}]`,
174
+ evidence: `${config.flag} value: ${proof.destination}, stdout: ${cmdResult.stdout.slice(0, 200)}`,
175
+ affectedFiles: changed,
176
+ recommendation: config.recommendation,
150
177
  }));
151
178
  }
152
179
  return findings;
153
180
  },
154
181
  });
155
- return result;
156
182
  }
157
183
  finally {
158
184
  await workspace.cleanup();
159
185
  }
160
186
  }
187
+ /**
188
+ * Checks that a path traversal input to --out does not write outside the declared output directory.
189
+ */
190
+ export async function checkOutPathTraversal(target, testInput) {
191
+ return checkContainedTraversal(OUT_TRAVERSAL, target, testInput);
192
+ }
193
+ /**
194
+ * Checks that a path traversal input to --index does not write outside the declared index directory.
195
+ */
196
+ export async function checkIndexPathTraversal(target, testInput) {
197
+ return checkContainedTraversal(INDEX_TRAVERSAL, target, testInput);
198
+ }
161
199
  // ---------------------------------------------------------------------------
162
200
  // Safe path handling checks
163
201
  // ---------------------------------------------------------------------------