@dailephd/my-dev-kit-lab 0.9.4 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +4 -2
- package/dist/src/evaluation/benchmarkSandbox/createBenchmarkSandbox.js +9 -1
- package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js +91 -1
- package/dist/src/evaluation/runMyDevKitRetrieval.js +137 -4
- package/dist/src/evaluation/syntheticRepository/materialize.js +103 -7
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +10 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +9 -1
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +41 -5
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +4 -4
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +4 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +1 -1
- package/dist/src/securityValidation/attackScenarios/boundedSourceScan.js +112 -25
- package/dist/src/securityValidation/attackScenarios/scenarios/networkAssumptionScenario.js +58 -6
- package/dist/src/securityValidation/cliAdversarial/pathBoundaryChecks.js +108 -70
- package/dist/src/securityValidation/cliAdversarial/tempWorkspace.js +57 -8
- package/dist/src/securityValidation/cliAdversarial/traversalContainment.js +161 -0
- package/dist/src/securityValidation/commandRunner.js +52 -26
- package/dist/src/securityValidation/config.js +10 -0
- package/dist/src/securityValidation/validate/runCliSecuritySuiteCheck.js +21 -5
- package/dist/src/securityValidation/validate/runSecurityValidation.js +2 -2
- package/dist/src/tutorial/tutorialAssertions.js +7 -0
- package/docs/METRICS.md +11 -1
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { resolveWithinRoot } from "../../../core/pathSafety.js";
|
|
3
|
-
import { classifyStaleRisk, compareCorrectness,
|
|
4
|
-
import { compareCandidateCorrectness,
|
|
3
|
+
import { classifyStaleRisk, compareCorrectness, compareRequiredFileEvidenceAgainstReference } from "./comparison.js";
|
|
4
|
+
import { compareCandidateCorrectness, compareCandidateRequiredFileEvidenceAgainstReference, classifyPartialRefreshAgainstFull } from "./comparisonV2.js";
|
|
5
5
|
import { INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS } from "./disposableTarget.js";
|
|
6
6
|
import { assessIncrementalChangeStalenessAffectedNeighborhood, checkIncrementalChangeStalenessAffectedNeighborhoodSymmetry, correctnessComparable, defaultIncrementalChangeStalenessExecutionDeps, executeTreatmentEvaluation, INCONCLUSIVE_NO_EXECUTION, resolveIncrementalChangeStalenessScenarioTask } from "./execution.js";
|
|
7
7
|
/**
|
|
@@ -153,7 +153,7 @@ export function buildReferenceComparisons(treatments) {
|
|
|
153
153
|
throw new Error("Reference comparisons require the stale-index and full-refresh treatment executions.");
|
|
154
154
|
// Historical v0.6.2 semantics, unchanged.
|
|
155
155
|
const staleCorrectness = compareCorrectness(correctnessComparable(stale.fakeAgent), correctnessComparable(reference.fakeAgent));
|
|
156
|
-
const staleRequired =
|
|
156
|
+
const staleRequired = compareRequiredFileEvidenceAgainstReference(stale.requiredFileEvidence, reference.requiredFileEvidence);
|
|
157
157
|
const staleRisk = classifyStaleRisk(staleCorrectness, staleRequired);
|
|
158
158
|
const comparisons = [
|
|
159
159
|
{
|
|
@@ -174,7 +174,7 @@ export function buildReferenceComparisons(treatments) {
|
|
|
174
174
|
throw new Error(`${candidateTreatmentId} has no incremental refresh execution to compare.`);
|
|
175
175
|
}
|
|
176
176
|
const correctnessRelation = compareCandidateCorrectness(correctnessComparable(candidate.fakeAgent), correctnessComparable(reference.fakeAgent));
|
|
177
|
-
const requiredFileEvidenceRelation =
|
|
177
|
+
const requiredFileEvidenceRelation = compareCandidateRequiredFileEvidenceAgainstReference(candidate.requiredFileEvidence, reference.requiredFileEvidence);
|
|
178
178
|
const classified = classifyPartialRefreshAgainstFull({
|
|
179
179
|
realization: candidate.refreshExecution.realization,
|
|
180
180
|
correctnessRelation,
|
|
@@ -14,6 +14,10 @@ export const LIMITATIONS_V2 = [
|
|
|
14
14
|
"\"no-observed-regression-relative-to-full\" means only that the executed scenario did not show a regression under the frozen correctness and required-file evidence dimensions. It does not prove general equivalence or safety, and does not show that full refresh is unnecessary.",
|
|
15
15
|
"Required-file evidence is a bounded answer-key file-presence check, not retrieval precision/recall.",
|
|
16
16
|
"Correctness remains deterministic fake-agent/harness evidence in v0.6.3.",
|
|
17
|
+
"Fake-agent correctness is a synthetic score computed from the answer key (the fake agent deliberately reports only two expected facts, so scenarios requiring three or more facts cannot pass). It is not measured my-dev-kit answer correctness and does not discriminate between treatments; a failed synthetic flag is not a my-dev-kit regression and a passing one would not be evidence of Kit correctness.",
|
|
18
|
+
"Files Read and Observed Files count a file only when the bounded retrieval protocol ran a successful source command for it; a file merely surfaced by search, lookup or slice was not read. Artifacts recorded before v0.9.5 name only the single selected search candidate.",
|
|
19
|
+
"A Missing required-file result is relative to the fixed bounded candidate budget and is not an exhaustive source-set assessment; evidence the protocol could not observe (coverage limits, failed source commands, rejected candidates) is reported as unknown. Equal required-file evidence supports a same relation only when the full-refresh reference itself observed every required file; two equally insufficient treatments are reported as unknown/inconclusive.",
|
|
20
|
+
"Command log paths in persisted retrieval telemetry name run-owned files that the mandatory run cleanup removes after the artifact is written; they are not retained and must not be relied on as links.",
|
|
17
21
|
"Lab affected-neighborhood evidence (changed baseline neighborhood relative to the task) is distinct from upstream refresh execution evidence (what my-dev-kit actually refreshed).",
|
|
18
22
|
"\"reindexRecommendation\" remains observational evidence; it did not select or trigger any treatment.",
|
|
19
23
|
"\"changedSymbolCount\" does not mean source-level symbol-diff proof; it retains the released v0.6.1 baseline-symbol-identity definition.",
|
|
@@ -27,7 +27,7 @@ export function treatmentBlock(scenario, treatmentId, label) {
|
|
|
27
27
|
["Estimated context tokens", retrieval.status === "not-run" ? "unavailable" : String(retrieval.totalEstimatedTokens)],
|
|
28
28
|
["Retrieval duration (ms)", retrieval.status === "not-run" ? "unavailable" : String(retrieval.durationMs)],
|
|
29
29
|
["Fake-agent status", display(treatment.fakeAgent?.status, "not run")],
|
|
30
|
-
["
|
|
30
|
+
["Synthetic fake-agent correctness score", correctness && correctness.available && correctness.score !== null ? String(correctness.score) : "unavailable"],
|
|
31
31
|
["Required-file evidence", treatment.requiredFileEvidence.status],
|
|
32
32
|
["Required-file reason", display(treatment.requiredFileEvidence.reason, "not applicable")],
|
|
33
33
|
["Affected-neighborhood relationship", treatment.affectedNeighborhood.relationship],
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
|
-
import
|
|
2
|
+
import path from "node:path";
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
4
|
// v0.2.2 Batch 4 — shared bounded, deterministic source-file scan helper.
|
|
5
5
|
//
|
|
6
6
|
// Reused by secretLeakageScenario and networkAssumptionScenario. Both need
|
|
7
7
|
// "read a bounded, deterministic set of source-ish files, skip large/binary
|
|
8
|
-
// content"
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
// small file-content allowlist filter for scenario-specific exclusions.
|
|
8
|
+
// content". Discovery here must refuse links BEFORE traversal or content reads;
|
|
9
|
+
// the general fileGlobs walker follows a linked glob base. Preserve its standard
|
|
10
|
+
// directory/temp exclusions, with a size cap and synthetic-fixture exclusion.
|
|
12
11
|
// ---------------------------------------------------------------------------
|
|
13
12
|
// Default bounded glob set: project manifests/config plus first-party source
|
|
14
13
|
// and scripts. Deliberately excludes tests/, benchmarks/, examples/ — those
|
|
@@ -37,28 +36,108 @@ const MAX_SCANNABLE_FILE_BYTES = 1_000_000;
|
|
|
37
36
|
// Built via fromCharCode rather than a "\0" string escape to avoid any
|
|
38
37
|
// literal control byte ending up in this source file.
|
|
39
38
|
const NUL_CHAR = String.fromCharCode(0);
|
|
39
|
+
const EXCLUDED_SOURCE_DIRECTORIES = new Set([
|
|
40
|
+
"node_modules", "dist", "build", "coverage", ".git", "lab-output",
|
|
41
|
+
".my-dev-kit", ".my-dev-kit-v1", ".my-dev-kit-lab", "__pycache__"
|
|
42
|
+
]);
|
|
43
|
+
const samePhysicalPath = (a, b) => process.platform === "win32"
|
|
44
|
+
? a.toLowerCase() === b.toLowerCase() : a === b;
|
|
40
45
|
// Returns file contents for the bounded glob set, skipping known synthetic
|
|
41
46
|
// fixtures, oversized files, and anything unreadable as UTF-8 text (treated
|
|
42
47
|
// as binary and skipped, not an error). Stable, deterministic order.
|
|
43
|
-
export function collectBoundedSourceFiles(targetRoot, globs = DEFAULT_SOURCE_SCAN_GLOBS) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
48
|
+
export function collectBoundedSourceFiles(targetRoot, globs = DEFAULT_SOURCE_SCAN_GLOBS, provenanceIssues = [], reviewedPaths = []) {
|
|
49
|
+
const root = path.resolve(targetRoot);
|
|
50
|
+
const issueMap = new Map();
|
|
51
|
+
const inspectedPaths = new Map();
|
|
52
|
+
const relative = (file) => path.relative(root, file).replace(/\\/g, "/") || ".";
|
|
53
|
+
const refuse = (file, reason) => {
|
|
54
|
+
issueMap.set(relative(file), reason);
|
|
55
|
+
return undefined;
|
|
56
|
+
};
|
|
57
|
+
// lstat every component, including the scan root, before realpath or reads.
|
|
58
|
+
// This rejects junctions/symlinks, physical aliases and multiply-linked files.
|
|
59
|
+
// Rechecking before a read narrows races; it is not a race-free OS sandbox.
|
|
60
|
+
function inspect(file) {
|
|
61
|
+
const rel = path.relative(root, file);
|
|
62
|
+
if (rel === ".." || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel)) {
|
|
63
|
+
return refuse(root, "path-outside-scan-root");
|
|
53
64
|
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
65
|
+
let current = root;
|
|
66
|
+
let stat;
|
|
67
|
+
for (const segment of ["", ...rel.split(path.sep).filter(Boolean)]) {
|
|
68
|
+
if (segment)
|
|
69
|
+
current = path.join(current, segment);
|
|
70
|
+
try {
|
|
71
|
+
stat = fs.lstatSync(current);
|
|
72
|
+
if (stat.isSymbolicLink())
|
|
73
|
+
return refuse(current, "symlink-or-junction");
|
|
74
|
+
if (!stat.isDirectory() && !stat.isFile())
|
|
75
|
+
return refuse(current, "non-regular-path");
|
|
76
|
+
if (stat.isFile() && stat.nlink > 1)
|
|
77
|
+
return refuse(current, "multiply-linked-source");
|
|
78
|
+
const physical = fs.realpathSync.native(current);
|
|
79
|
+
if (!samePhysicalPath(physical, current)) {
|
|
80
|
+
return refuse(current, "physical-path-alias");
|
|
81
|
+
}
|
|
82
|
+
inspectedPaths.set(current, physical);
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
// Missing optional files are normal for generic external targets.
|
|
86
|
+
if (error.code !== "ENOENT")
|
|
87
|
+
refuse(current, "unreadable-path");
|
|
88
|
+
return undefined;
|
|
89
|
+
}
|
|
58
90
|
}
|
|
59
|
-
|
|
60
|
-
|
|
91
|
+
return stat;
|
|
92
|
+
}
|
|
93
|
+
// Absence is normal in unrelated targets; an existing reviewed path that is
|
|
94
|
+
// not a direct regular file must not disappear into an empty directory scan.
|
|
95
|
+
for (const reviewed of reviewedPaths) {
|
|
96
|
+
const file = path.resolve(root, reviewed);
|
|
97
|
+
const stat = inspect(file);
|
|
98
|
+
if (stat && !stat.isFile())
|
|
99
|
+
refuse(file, "non-regular-reviewed-source");
|
|
100
|
+
}
|
|
101
|
+
const candidateMap = new Map();
|
|
102
|
+
for (const glob of globs) {
|
|
103
|
+
const pattern = glob.replace(/\\/g, "/");
|
|
104
|
+
const wildcard = pattern.search(/[*?]/);
|
|
105
|
+
const prefix = wildcard < 0 ? pattern : pattern.slice(0, wildcard);
|
|
106
|
+
const base = wildcard < 0 ? prefix : prefix.slice(0, prefix.lastIndexOf("/") + 1) || ".";
|
|
107
|
+
const recursive = pattern.slice(base === "." ? 0 : base.length).includes("/");
|
|
108
|
+
function visit(file) {
|
|
109
|
+
const stat = inspect(file);
|
|
110
|
+
if (!stat)
|
|
111
|
+
return;
|
|
112
|
+
const rel = relative(file);
|
|
113
|
+
if (stat.isDirectory()) {
|
|
114
|
+
let entries;
|
|
115
|
+
try {
|
|
116
|
+
entries = fs.readdirSync(file).sort();
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
refuse(file, "unreadable-directory");
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
for (const name of entries) {
|
|
123
|
+
const child = path.join(file, name);
|
|
124
|
+
// Root-level wildcard globs must not traverse unrelated directories.
|
|
125
|
+
if (recursive || path.matchesGlob(relative(child), pattern)) {
|
|
126
|
+
// Exclusions name generated directories, never ordinary files.
|
|
127
|
+
// Inspect first so a link with an excluded name stays reportable.
|
|
128
|
+
if (EXCLUDED_SOURCE_DIRECTORIES.has(name) && inspect(child)?.isDirectory())
|
|
129
|
+
continue;
|
|
130
|
+
visit(child);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
else if (path.matchesGlob(rel, pattern) && !/\.(?:tmp|temp|log|pyc)$|~$/.test(rel)) {
|
|
135
|
+
// Windows identity comparisons may ignore case for containment, but
|
|
136
|
+
// reviewed trust must use observed spelling, including the glob base.
|
|
137
|
+
candidateMap.set(relative(inspectedPaths.get(file)), file);
|
|
138
|
+
}
|
|
61
139
|
}
|
|
140
|
+
visit(path.resolve(root, base));
|
|
62
141
|
}
|
|
63
142
|
const candidates = [...candidateMap.entries()]
|
|
64
143
|
.sort(([a], [b]) => a.localeCompare(b))
|
|
@@ -70,23 +149,31 @@ export function collectBoundedSourceFiles(targetRoot, globs = DEFAULT_SOURCE_SCA
|
|
|
70
149
|
continue;
|
|
71
150
|
let stat;
|
|
72
151
|
try {
|
|
73
|
-
stat =
|
|
152
|
+
stat = inspect(candidate.absolutePath);
|
|
74
153
|
}
|
|
75
154
|
catch {
|
|
76
155
|
continue;
|
|
77
156
|
}
|
|
78
|
-
if (!stat
|
|
157
|
+
if (!stat?.isFile())
|
|
79
158
|
continue;
|
|
159
|
+
if (stat.size > MAX_SCANNABLE_FILE_BYTES) {
|
|
160
|
+
refuse(candidate.absolutePath, "oversized-source");
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
80
163
|
try {
|
|
81
164
|
const content = fs.readFileSync(candidate.absolutePath, "utf8");
|
|
82
|
-
if (content.indexOf(NUL_CHAR) !== -1)
|
|
165
|
+
if (content.indexOf(NUL_CHAR) !== -1) {
|
|
166
|
+
refuse(candidate.absolutePath, "binary-source");
|
|
83
167
|
continue;
|
|
168
|
+
}
|
|
84
169
|
files.push({ relativePath: normalizedRel, absolutePath: candidate.absolutePath, content });
|
|
85
170
|
}
|
|
86
171
|
catch {
|
|
87
|
-
|
|
172
|
+
refuse(candidate.absolutePath, "unreadable-source");
|
|
88
173
|
}
|
|
89
174
|
}
|
|
175
|
+
provenanceIssues.push(...[...issueMap.entries()].sort(([a], [b]) => a.localeCompare(b))
|
|
176
|
+
.map(([relativePath, reason]) => ({ relativePath, reason })));
|
|
90
177
|
return files;
|
|
91
178
|
}
|
|
92
179
|
// Returns the 1-based line number containing `index` within `content`.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { makeEvidence } from "../exploitEvidence.js";
|
|
2
2
|
import { collectBoundedSourceFiles, lineNumberAt } from "../boundedSourceScan.js";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
3
4
|
// ---------------------------------------------------------------------------
|
|
4
5
|
// v0.2.2 Batch 4 — network / local-first assumption scenario.
|
|
5
6
|
//
|
|
@@ -31,6 +32,21 @@ const KNOWN_NETWORK_INTENTIONAL_PREFIXES = [
|
|
|
31
32
|
function isKnownIntentionalPath(relativePath) {
|
|
32
33
|
return KNOWN_NETWORK_INTENTIONAL_PREFIXES.some((p) => relativePath.startsWith(p));
|
|
33
34
|
}
|
|
35
|
+
// Reviewed source identities, not file exemptions. A changed site, extra match,
|
|
36
|
+
// missing dependency, or changed caller precondition invalidates recognition.
|
|
37
|
+
const REVIEWED_LOCAL_HTTP_SITES = [
|
|
38
|
+
{ file: "src/runtime/managedProcess.ts", symbol: "waitForReadiness", sha256: "978dacaf401cac3c3f6fdc96ca92163b766b14bead7f3a7c49b1ca28d8ac3900", dependencies: {} },
|
|
39
|
+
{ file: "src/tutorial/tutorialAssertions.ts", symbol: "http-json-equals / resolveAssertionRequestUrl", sha256: "eba83bd94342d108e1ecc8491e09e584360095823332dffe2df28e49ecf011b3", dependencies: {
|
|
40
|
+
"src/tutorial/runTutorial.ts": "ded69670f0aec6ad31e714e77c6a8e769abcb28fcdc68b81584fd582e593e298",
|
|
41
|
+
"src/tutorial/tutorialSession.ts": "08e0317600754a033ebf8022e984e941c60786bbc3c9baafc3fa7be3ccdd90ce",
|
|
42
|
+
"src/tutorial/targetContractValidation.ts": "da6c51c36827bf3d9b7375cdd71e488d35732f389f989a7dbc462b21063a96a6",
|
|
43
|
+
"src/tutorial/loadTutorialContracts.ts": "10e17f851febf783076d2d7ad5de70f6e950308819ec30edaecf749ba4b11c7a",
|
|
44
|
+
"src/tutorial/validationPrimitives.ts": "e49258d33d0a88d051fa783d8e2236827940ed5895fd1ce844b9f58b4ec08cb1"
|
|
45
|
+
} }
|
|
46
|
+
];
|
|
47
|
+
// Normalize only checkout EOLs for these exact reviewed-content comparisons.
|
|
48
|
+
// The tutorialSession identity is the verified LF form of the reviewed CRLF text.
|
|
49
|
+
const sourceDigest = (content) => createHash("sha256").update(content.replace(/\r\n/g, "\n"), "utf8").digest("hex");
|
|
34
50
|
function scanFileForNetworkApis(relativePath, content) {
|
|
35
51
|
const matches = [];
|
|
36
52
|
const intentional = isKnownIntentionalPath(relativePath);
|
|
@@ -58,7 +74,12 @@ export const NETWORK_ASSUMPTION_SCENARIO = {
|
|
|
58
74
|
expectedSafeBehavior: "No obvious unexpected network API usage is found in the bounded local-first source paths inspected.",
|
|
59
75
|
evidenceRequirements: ["observation"],
|
|
60
76
|
run: async (ctx) => {
|
|
61
|
-
const
|
|
77
|
+
const provenanceIssues = [];
|
|
78
|
+
const reviewedPaths = new Set(REVIEWED_LOCAL_HTTP_SITES.flatMap(site => [site.file, ...Object.keys(site.dependencies)]));
|
|
79
|
+
const files = collectBoundedSourceFiles(ctx.target.targetRoot, undefined, provenanceIssues, [...reviewedPaths]);
|
|
80
|
+
// Size/binary exclusions remain bounded-scan policy for ordinary sources;
|
|
81
|
+
// they can never establish the identity of a reviewed site or precondition.
|
|
82
|
+
const refusals = provenanceIssues.filter(issue => !["oversized-source", "binary-source"].includes(issue.reason) || reviewedPaths.has(issue.relativePath));
|
|
62
83
|
const isolationCaveat = makeEvidence({
|
|
63
84
|
kind: "observation",
|
|
64
85
|
source: "scope statement",
|
|
@@ -66,7 +87,7 @@ export const NETWORK_ASSUMPTION_SCENARIO = {
|
|
|
66
87
|
expectedBehavior: "This is a bounded static check of local-first source paths only. It does not prove runtime network isolation of subprocess-invoked tools (npm, CodeQL, Semgrep, OSV-Scanner) or any other process spawned during validation.",
|
|
67
88
|
observedBehavior: "Static-source-only claim; no runtime network instrumentation was performed.",
|
|
68
89
|
});
|
|
69
|
-
if (files.length === 0) {
|
|
90
|
+
if (files.length === 0 && refusals.length === 0) {
|
|
70
91
|
return {
|
|
71
92
|
status: "passed",
|
|
72
93
|
confidence: "low",
|
|
@@ -82,10 +103,33 @@ export const NETWORK_ASSUMPTION_SCENARIO = {
|
|
|
82
103
|
],
|
|
83
104
|
};
|
|
84
105
|
}
|
|
85
|
-
const allMatches =
|
|
106
|
+
const allMatches = refusals.map(issue => ({
|
|
107
|
+
id: "untrusted-source-provenance", file: issue.relativePath, line: 1,
|
|
108
|
+
intentional: false, provenanceReason: issue.reason
|
|
109
|
+
}));
|
|
86
110
|
for (const file of files) {
|
|
87
111
|
allMatches.push(...scanFileForNetworkApis(file.relativePath, file.content));
|
|
88
112
|
}
|
|
113
|
+
const byPath = new Map(files.map(file => [file.relativePath, file.content]));
|
|
114
|
+
for (const site of REVIEWED_LOCAL_HTTP_SITES) {
|
|
115
|
+
const content = byPath.get(site.file);
|
|
116
|
+
if (content === undefined)
|
|
117
|
+
continue;
|
|
118
|
+
const siteMatches = allMatches.filter(match => match.file === site.file);
|
|
119
|
+
const recognized = sourceDigest(content) === site.sha256 &&
|
|
120
|
+
siteMatches.length === 1 && siteMatches[0]?.id === "fetch-call" &&
|
|
121
|
+
Object.entries(site.dependencies).every(([file, sha256]) => {
|
|
122
|
+
const dependency = byPath.get(file);
|
|
123
|
+
return dependency !== undefined && sourceDigest(dependency) === sha256;
|
|
124
|
+
});
|
|
125
|
+
if (recognized) {
|
|
126
|
+
siteMatches[0].intentional = true;
|
|
127
|
+
siteMatches[0].reviewedSite = `${site.symbol}; LF-normalized source SHA-256 ${site.sha256}`;
|
|
128
|
+
}
|
|
129
|
+
else if (siteMatches.length === 0) {
|
|
130
|
+
allMatches.push({ id: "reviewed-local-http-source-mismatch", file: site.file, line: 1, intentional: false });
|
|
131
|
+
}
|
|
132
|
+
}
|
|
89
133
|
allMatches.sort((a, b) => a.file.localeCompare(b.file) || a.line - b.line);
|
|
90
134
|
const unexpected = allMatches.filter((m) => !m.intentional);
|
|
91
135
|
const intentional = allMatches.filter((m) => m.intentional);
|
|
@@ -103,16 +147,24 @@ export const NETWORK_ASSUMPTION_SCENARIO = {
|
|
|
103
147
|
source: "known network-using validation commands",
|
|
104
148
|
confidence: "high",
|
|
105
149
|
expectedBehavior: "npm audit / npm outdated (deps check) may contact the npm registry; OSV-Scanner may contact the OSV API; Semgrep's npx fallback may contact the npm registry. These are expected when those check groups are selected via --checks and are not local-first source violations.",
|
|
106
|
-
observedBehavior: `${intentional.length} match(es) found under known network-using paths (${KNOWN_NETWORK_INTENTIONAL_PREFIXES.join(", ")}).`,
|
|
150
|
+
observedBehavior: `${intentional.filter(m => !m.reviewedSite).length} match(es) found under known network-using paths (${KNOWN_NETWORK_INTENTIONAL_PREFIXES.join(", ")}).`,
|
|
107
151
|
}),
|
|
152
|
+
...intentional.filter(m => m.reviewedSite).map(m => makeEvidence({
|
|
153
|
+
kind: "observation", source: "known-intentional local HTTP use", filePath: m.file, line: m.line,
|
|
154
|
+
confidence: "medium",
|
|
155
|
+
expectedBehavior: "Exactly one reviewed fetch-call with unchanged local HTTP guards and caller preconditions. Bounded static findings do not prove full runtime network isolation.",
|
|
156
|
+
observedBehavior: `Recognized '${m.id}' in ${m.file}:${m.line} (${m.reviewedSite}).`
|
|
157
|
+
})),
|
|
108
158
|
...unexpected.slice(0, 20).map((m) => makeEvidence({
|
|
109
159
|
kind: "observation",
|
|
110
|
-
source: `pattern '${m.id}'`,
|
|
160
|
+
source: m.provenanceReason ? "source provenance refusal" : `pattern '${m.id}'`,
|
|
111
161
|
filePath: m.file,
|
|
112
162
|
line: m.line,
|
|
113
163
|
confidence: "medium",
|
|
114
164
|
expectedBehavior: "No unexpected direct network API usage in local-first source paths.",
|
|
115
|
-
observedBehavior:
|
|
165
|
+
observedBehavior: m.provenanceReason
|
|
166
|
+
? `Untrusted source path ${m.file}: ${m.provenanceReason}; source content was not accepted.`
|
|
167
|
+
: `Detected '${m.id}' usage in ${m.file}:${m.line}.`,
|
|
116
168
|
})),
|
|
117
169
|
];
|
|
118
170
|
if (unexpected.length > 0) {
|
|
@@ -2,7 +2,8 @@ import { spawn } from "node:child_process";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { buildCliCommand } from "./adversarialCliConfig.js";
|
|
4
4
|
import { makeFinding, runAdversarialCheck, skippedCheck } from "./runAdversarialCheck.js";
|
|
5
|
-
import { createTempWorkspace, diffSnapshots, snapshotDir } from "./tempWorkspace.js";
|
|
5
|
+
import { createTempWorkspace, diffSnapshots, snapshotDir, snapshotTreeStrict } from "./tempWorkspace.js";
|
|
6
|
+
import { classifyTraversalValue, proveContainment } from "./traversalContainment.js";
|
|
6
7
|
// ---------------------------------------------------------------------------
|
|
7
8
|
// Path traversal checks
|
|
8
9
|
// ---------------------------------------------------------------------------
|
|
@@ -58,27 +59,68 @@ export async function checkRootPathTraversal(target, testInput) {
|
|
|
58
59
|
await workspace.cleanup();
|
|
59
60
|
}
|
|
60
61
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
62
|
+
const OUT_TRAVERSAL = {
|
|
63
|
+
flag: "--out",
|
|
64
|
+
workspacePrefix: "p4-out-",
|
|
65
|
+
idStem: "path-traversal-out",
|
|
66
|
+
nameStem: "Path traversal via --out",
|
|
67
|
+
title: "Write escaped workspace via --out traversal",
|
|
68
|
+
recommendation: "Validate --out argument is within an allowed prefix before writing.",
|
|
69
|
+
declaredDir: (workspace) => workspace.outputDir,
|
|
70
|
+
};
|
|
71
|
+
const INDEX_TRAVERSAL = {
|
|
72
|
+
flag: "--index",
|
|
73
|
+
workspacePrefix: "p4-idx-",
|
|
74
|
+
idStem: "path-traversal-index",
|
|
75
|
+
nameStem: "Path traversal via --index",
|
|
76
|
+
title: "Write escaped workspace via --index traversal",
|
|
77
|
+
recommendation: "Validate --index argument is within an allowed prefix.",
|
|
78
|
+
declaredDir: (workspace) => workspace.indexDir,
|
|
79
|
+
};
|
|
80
|
+
function isUnder(relativePath, relativeDir) {
|
|
81
|
+
return relativePath === relativeDir || relativePath.startsWith(relativeDir + path.sep);
|
|
82
|
+
}
|
|
83
|
+
async function checkContainedTraversal(config, target, testInput) {
|
|
84
|
+
const id = `${config.idStem}-${testInput.id}`;
|
|
85
|
+
const name = `${config.nameStem}: ${testInput.description}`;
|
|
86
|
+
const blocked = (reason) => skippedCheck({
|
|
87
|
+
id,
|
|
88
|
+
name,
|
|
89
|
+
category: "cli-adversarial",
|
|
90
|
+
reason: `BLOCKED_BEFORE_LAUNCH: ${reason}`,
|
|
91
|
+
});
|
|
92
|
+
const plan = classifyTraversalValue(testInput.value);
|
|
93
|
+
if (plan.kind === "refused")
|
|
94
|
+
return blocked(plan.reason);
|
|
95
|
+
const workspace = createTempWorkspace(config.workspacePrefix, { nestDepth: plan.nestDepth });
|
|
66
96
|
try {
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
97
|
+
const enclosureDir = workspace.enclosureDir;
|
|
98
|
+
const declaredDir = config.declaredDir(workspace);
|
|
99
|
+
const proof = proveContainment({
|
|
100
|
+
value: testInput.value,
|
|
101
|
+
baseDir: declaredDir,
|
|
102
|
+
enclosureDir,
|
|
103
|
+
declaredDir,
|
|
104
|
+
});
|
|
105
|
+
if (!proof.ok)
|
|
106
|
+
return blocked(proof.reason);
|
|
107
|
+
let before;
|
|
108
|
+
try {
|
|
109
|
+
before = snapshotTreeStrict(enclosureDir);
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
return blocked(`enclosure is not inspectable before launch: ${error instanceof Error ? error.message : String(error)}`);
|
|
113
|
+
}
|
|
73
114
|
const { command, args } = buildCliCommand(target, [
|
|
74
115
|
"--root",
|
|
75
116
|
workspace.sourceDir,
|
|
76
|
-
|
|
77
|
-
|
|
117
|
+
config.flag,
|
|
118
|
+
proof.destination,
|
|
78
119
|
]);
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
|
|
120
|
+
const allowedRelative = path.relative(proof.enclosure, proof.declared);
|
|
121
|
+
return await runAdversarialCheck({
|
|
122
|
+
id,
|
|
123
|
+
name,
|
|
82
124
|
category: "cli-adversarial",
|
|
83
125
|
severity: "blocker",
|
|
84
126
|
command,
|
|
@@ -87,77 +129,73 @@ export async function checkOutPathTraversal(target, testInput) {
|
|
|
87
129
|
timeoutMs: target.timeoutMs,
|
|
88
130
|
evaluate: (cmdResult) => {
|
|
89
131
|
const findings = [];
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
if (
|
|
132
|
+
// A child that never produced an exit code (and did not time out) did
|
|
133
|
+
// not execute; the runner reports asynchronous spawn failures this way.
|
|
134
|
+
if (cmdResult.spawnError || (cmdResult.exitCode === null && !cmdResult.timedOut)) {
|
|
93
135
|
findings.push(makeFinding({
|
|
94
|
-
id:
|
|
95
|
-
title:
|
|
136
|
+
id: `${id}-not-executed`,
|
|
137
|
+
title: `CLI did not run for ${config.flag} traversal check`,
|
|
138
|
+
severity: "major",
|
|
139
|
+
category: "cli-adversarial",
|
|
140
|
+
description: `Spawn error: ${cmdResult.spawnError ?? cmdResult.stderr.slice(0, 200)}`,
|
|
141
|
+
evidence: `${config.flag} value: ${proof.destination}`,
|
|
142
|
+
recommendation: "Fix the CLI invocation so the traversal check actually executes.",
|
|
143
|
+
}));
|
|
144
|
+
}
|
|
145
|
+
let after;
|
|
146
|
+
try {
|
|
147
|
+
after = snapshotTreeStrict(enclosureDir);
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
findings.push(makeFinding({
|
|
151
|
+
id: `${id}-inspection-lost`,
|
|
152
|
+
title: `Cannot inspect enclosure after ${config.flag} traversal`,
|
|
96
153
|
severity: "blocker",
|
|
97
154
|
category: "cli-adversarial",
|
|
98
|
-
description: `
|
|
99
|
-
evidence:
|
|
100
|
-
recommendation: "
|
|
155
|
+
description: `Post-run snapshot failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
156
|
+
evidence: `${config.flag} value: ${proof.destination}`,
|
|
157
|
+
recommendation: "Treat the traversal outcome as unknown; investigate before release.",
|
|
101
158
|
}));
|
|
159
|
+
return findings;
|
|
102
160
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
/**
|
|
113
|
-
* Checks that a path traversal input to --index does not allow writes outside workspace.
|
|
114
|
-
*/
|
|
115
|
-
export async function checkIndexPathTraversal(target, testInput) {
|
|
116
|
-
const workspace = createTempWorkspace("p4-idx-");
|
|
117
|
-
try {
|
|
118
|
-
const beforeOutside = snapshotDir(workspace.outsideDir);
|
|
119
|
-
const escapedIndex = testInput.value.startsWith("/") || /^[A-Za-z]:\\/.test(testInput.value)
|
|
120
|
-
? testInput.value
|
|
121
|
-
: path.resolve(workspace.indexDir, testInput.value);
|
|
122
|
-
const { command, args } = buildCliCommand(target, [
|
|
123
|
-
"--root",
|
|
124
|
-
workspace.sourceDir,
|
|
125
|
-
"--index",
|
|
126
|
-
escapedIndex,
|
|
127
|
-
]);
|
|
128
|
-
const result = await runAdversarialCheck({
|
|
129
|
-
id: `path-traversal-index-${testInput.id}`,
|
|
130
|
-
name: `Path traversal via --index: ${testInput.description}`,
|
|
131
|
-
category: "cli-adversarial",
|
|
132
|
-
severity: "blocker",
|
|
133
|
-
command,
|
|
134
|
-
args,
|
|
135
|
-
cwd: workspace.root,
|
|
136
|
-
timeoutMs: target.timeoutMs,
|
|
137
|
-
evaluate: (cmdResult) => {
|
|
138
|
-
const findings = [];
|
|
139
|
-
const afterOutside = snapshotDir(workspace.outsideDir);
|
|
140
|
-
const outsideDiff = diffSnapshots(beforeOutside, afterOutside);
|
|
141
|
-
if (outsideDiff.added.length > 0 || outsideDiff.modified.length > 0) {
|
|
161
|
+
const diff = diffSnapshots(before, after);
|
|
162
|
+
const outside = (list) => list.filter((rel) => !isUnder(rel, allowedRelative));
|
|
163
|
+
const added = outside(diff.added);
|
|
164
|
+
const modified = outside(diff.modified);
|
|
165
|
+
const removed = outside(diff.removed);
|
|
166
|
+
if (added.length + modified.length + removed.length > 0) {
|
|
167
|
+
const changed = [...added, ...modified, ...removed].map((rel) => path.join(proof.enclosure, rel));
|
|
142
168
|
findings.push(makeFinding({
|
|
143
|
-
id:
|
|
144
|
-
title:
|
|
169
|
+
id: `${id}-escape`,
|
|
170
|
+
title: config.title,
|
|
145
171
|
severity: "blocker",
|
|
146
172
|
category: "cli-adversarial",
|
|
147
|
-
description: `CLI wrote outside workspace: ${
|
|
148
|
-
evidence:
|
|
149
|
-
|
|
173
|
+
description: `CLI wrote outside workspace: added [${added.join(", ")}], modified [${modified.join(", ")}], removed [${removed.join(", ")}]`,
|
|
174
|
+
evidence: `${config.flag} value: ${proof.destination}, stdout: ${cmdResult.stdout.slice(0, 200)}`,
|
|
175
|
+
affectedFiles: changed,
|
|
176
|
+
recommendation: config.recommendation,
|
|
150
177
|
}));
|
|
151
178
|
}
|
|
152
179
|
return findings;
|
|
153
180
|
},
|
|
154
181
|
});
|
|
155
|
-
return result;
|
|
156
182
|
}
|
|
157
183
|
finally {
|
|
158
184
|
await workspace.cleanup();
|
|
159
185
|
}
|
|
160
186
|
}
|
|
187
|
+
/**
|
|
188
|
+
* Checks that a path traversal input to --out does not write outside the declared output directory.
|
|
189
|
+
*/
|
|
190
|
+
export async function checkOutPathTraversal(target, testInput) {
|
|
191
|
+
return checkContainedTraversal(OUT_TRAVERSAL, target, testInput);
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Checks that a path traversal input to --index does not write outside the declared index directory.
|
|
195
|
+
*/
|
|
196
|
+
export async function checkIndexPathTraversal(target, testInput) {
|
|
197
|
+
return checkContainedTraversal(INDEX_TRAVERSAL, target, testInput);
|
|
198
|
+
}
|
|
161
199
|
// ---------------------------------------------------------------------------
|
|
162
200
|
// Safe path handling checks
|
|
163
201
|
// ---------------------------------------------------------------------------
|