@cosyte/script-utils 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +174 -2
- package/README.md +423 -17
- package/attw.d.ts +36 -0
- package/attw.js +1679 -0
- package/check-no-emdash.sh +874 -0
- package/internal-refs.d.ts +223 -0
- package/internal-refs.js +1363 -0
- package/package.json +28 -1
- package/phi-scan.d.ts +271 -0
- package/phi-scan.js +1936 -0
package/phi-scan.js
ADDED
|
@@ -0,0 +1,1936 @@
|
|
|
1
|
+
// @cosyte/script-utils/phi-scan
|
|
2
|
+
//
|
|
3
|
+
// THE SHARED MACHINERY OF THE `@cosyte/*` PHI COMMIT-GATE, PARAMETERISED.
|
|
4
|
+
//
|
|
5
|
+
// WHY THIS FILE EXISTS, IN ONE SENTENCE: `scripts/parser-template/` is a SCAFFOLD, not a
|
|
6
|
+
// dependency, so `scripts/phi-scan.ts` was COPIED into every parser repo, and a newly-found escape
|
|
7
|
+
// therefore cost one pull request and one adversarial review PER REPO. Three escape classes have
|
|
8
|
+
// been closed that way already (the index union, the completeness rule, and the read filters), and
|
|
9
|
+
// each one was paid for thirteen times. The machinery lives here so the next one is paid for once.
|
|
10
|
+
//
|
|
11
|
+
// WHAT IS SHARED AND WHAT IS NOT. This module owns the ENGINE: argument parsing, the allow-list and
|
|
12
|
+
// override log, target enumeration on all three routes, the union of the working-tree walk with the
|
|
13
|
+
// bytes git carries, content deduplication under git's own `blob <len>\0` framing, the completeness
|
|
14
|
+
// rule, the refusals, and the cross-cutting SSN/email FLOOR. It does NOT own the per-standard,
|
|
15
|
+
// field-level detectors (names, DOB, MRN / member id, address, phone), which genuinely differ per
|
|
16
|
+
// healthcare standard and are supplied by the caller through `detect`.
|
|
17
|
+
//
|
|
18
|
+
// THE FIVE PER-REPO AXES ARE PARAMETERS, NOT FORKS. They differ between repos, which is an argument
|
|
19
|
+
// for parameters and not against sharing:
|
|
20
|
+
//
|
|
21
|
+
// 1. EXIT CODES `exitCodes`, REQUIRED. The siblings deliberately disagree; a default here
|
|
22
|
+
// would be the porting mistake this whole campaign exists to catch.
|
|
23
|
+
// 2. ROOTS+EXCLUSIONS `scanRoots` (REQUIRED), `excludedPaths` and `isWalkReadable` (both
|
|
24
|
+
// DEFAULTED, so a change to the shared default reaches every consumer
|
|
25
|
+
// through a version bump instead of thirteen edits).
|
|
26
|
+
// 3. `--staged` SCOPE `isStagedReadable`, REQUIRED. Widening it changes what a COMMIT is blocked
|
|
27
|
+
// on, which is a hook decision each repo takes for itself.
|
|
28
|
+
// 4. GITLINKS `regularBlobModes`, DEFAULTED to git's two regular-blob modes.
|
|
29
|
+
// 5. EOL NORMALIZATION No parameter. The walk/index deduplication is BY CONTENT, so where a `text`
|
|
30
|
+
// attribute or `core.autocrlf` makes the index carry LF and the working tree
|
|
31
|
+
// CRLF, BOTH forms are scanned rather than one being assumed to stand for the
|
|
32
|
+
// other. It is listed because a port must CHECK it, not skip it.
|
|
33
|
+
//
|
|
34
|
+
// ZERO DEPENDENCIES AND NO BUILD STEP, the same constraint the rest of this package carries. `git`
|
|
35
|
+
// is the only subprocess, always through `execFileSync` with array args, never shell-form.
|
|
36
|
+
//
|
|
37
|
+
// PHI DISCIPLINE INSIDE THIS FILE ITSELF: no literal identifier-shaped value appears anywhere in
|
|
38
|
+
// this source, not even as an example in a comment. A repo that widens its scan roots to the whole
|
|
39
|
+
// repository reads its own `node_modules` copy of this file only if it is not gitignored, and a
|
|
40
|
+
// diagnostic ABOUT a PHI leak is itself a PHI surface. Describe the shape; never write one down.
|
|
41
|
+
|
|
42
|
+
import { readFileSync, statSync, lstatSync, existsSync, readdirSync } from "node:fs";
|
|
43
|
+
import { execFileSync } from "node:child_process";
|
|
44
|
+
import { createHash } from "node:crypto";
|
|
45
|
+
import { join, resolve, relative, sep, isAbsolute } from "node:path";
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Every state in which the scan cannot account for something. Raised throughout, caught in `run`,
|
|
49
|
+
* and turned into the caller's REFUSE code there.
|
|
50
|
+
*
|
|
51
|
+
* IT IS NOT EXPORTED, DELIBERATELY. A consumer that could catch it could also swallow it, and every
|
|
52
|
+
* one of these states must reach a non-zero exit.
|
|
53
|
+
*/
|
|
54
|
+
class InvocationError extends Error {
|
|
55
|
+
/** @param {string} message */
|
|
56
|
+
constructor(message) {
|
|
57
|
+
super(message);
|
|
58
|
+
this.name = "InvocationError";
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* A target whose bytes could not be read, carrying the errno the filesystem gave.
|
|
64
|
+
*
|
|
65
|
+
* IT IS AN `InvocationError`, SO THE DEFAULT IS STILL REFUSAL. The subclass exists for exactly one
|
|
66
|
+
* decision, the ENUMERATION TOCTOU WINDOW in `run`: between the walk listing a path and the sweep
|
|
67
|
+
* opening it, an editor, a build or a test can remove it. The errno is what tells "the file is
|
|
68
|
+
* gone" from "the file is there and something else failed", and nothing else in this engine may
|
|
69
|
+
* branch on it. Every other read failure keeps taking the refuse path unchanged.
|
|
70
|
+
*/
|
|
71
|
+
class TargetReadError extends InvocationError {
|
|
72
|
+
/**
|
|
73
|
+
* @param {string} message
|
|
74
|
+
* @param {string | undefined} errnoCode The `code` the filesystem error carried, when it had one.
|
|
75
|
+
*/
|
|
76
|
+
constructor(message, errnoCode) {
|
|
77
|
+
super(message);
|
|
78
|
+
this.name = "TargetReadError";
|
|
79
|
+
this.errnoCode = errnoCode;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** git's file modes for a regular blob. Every other mode names something with no bytes to read. */
|
|
84
|
+
const DEFAULT_REGULAR_BLOB_MODES = new Set(["100644", "100755"]);
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* AXIS 2's shared default read filter for the two SWEEPING routes: Markdown is documentation rather
|
|
88
|
+
* than fixture data and may legitimately describe a violator value.
|
|
89
|
+
*
|
|
90
|
+
* IT IS A DEFAULT RATHER THAN A CONSTANT SO THAT MOVING IT IS ONE CHANGE. Every `@cosyte/*` scanner
|
|
91
|
+
* carries this exemption today, and a tracked `.md` is therefore read by NEITHER sweeping route
|
|
92
|
+
* (the union inherits this filter on purpose, so `all` mode's verdict cannot depend on which copy of
|
|
93
|
+
* a file it reached). Whether that boundary should move is a real question with a real cost on both
|
|
94
|
+
* sides: `README.md` and `CHANGELOG.md` ship inside the npm tarball, and a `.md` named explicitly on
|
|
95
|
+
* argv IS scanned today, so the escape is route-dependent rather than file-dependent. Moving it is a
|
|
96
|
+
* deliberate change to THIS line, with its own measurement, and it then reaches every consumer
|
|
97
|
+
* through a version bump.
|
|
98
|
+
*
|
|
99
|
+
* @param {string} relPath A repo-relative, forward-slashed path.
|
|
100
|
+
* @returns {boolean} `true` when the sweeping routes may read this path's bytes.
|
|
101
|
+
*/
|
|
102
|
+
export function exemptsMarkdown(relPath) {
|
|
103
|
+
return !relPath.toLowerCase().endsWith(".md");
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Normalise and CHECK the caller's configuration.
|
|
108
|
+
*
|
|
109
|
+
* IT THROWS A `TypeError` RATHER THAN RETURNING A CODE, because at the point a required axis is
|
|
110
|
+
* missing there is no trustworthy code to return: `exitCodes` is itself the thing that was not
|
|
111
|
+
* supplied. That escapes to node's own exit 1 with a stack, which is loud and lands on the author's
|
|
112
|
+
* very first run of their own scanner.
|
|
113
|
+
*
|
|
114
|
+
* WHAT IT CHECKS AND WHAT IT DOES NOT, SCOPED TO WHAT IS ENFORCED HERE RATHER THAN TO AN INTENT.
|
|
115
|
+
* An earlier draft of this docblock said a misconfigured scanner "must not be able to report clean",
|
|
116
|
+
* and a reviewer falsified that sentence twice in four lines. Both reproductions were the same
|
|
117
|
+
* shape: a configuration this function ACCEPTED, whose roots then matched nothing, so every
|
|
118
|
+
* index-keyed rule went silently empty and the run printed `OK: no hits`. So the two are named:
|
|
119
|
+
*
|
|
120
|
+
* 1. A ROOT IS NORMALISED THE SAME WAY A PATH IS. `"./src"` is a spelling the type describes as
|
|
121
|
+
* valid, and it walked correctly while `isUnderScanRoot` compared it against the NORMALIZED
|
|
122
|
+
* index path `src/...` and never matched. Measured: `["src"]` refused a tracked mode-120000
|
|
123
|
+
* entry at exit 2 while `["./src"]` reported clean at exit 0 over the same repository.
|
|
124
|
+
* 2. A ROOT THAT RESOLVES OUTSIDE THE REPOSITORY IS REFUSED, because no index path can ever be
|
|
125
|
+
* under it, which is the same silently-empty state arrived at by another spelling.
|
|
126
|
+
*
|
|
127
|
+
* IT IS STILL NOT AN EXHAUSTIVE GUARANTEE, and the residual is named rather than implied: the
|
|
128
|
+
* CONTENT of a caller's predicates is not checkable here. `isStagedReadable` admitting a path
|
|
129
|
+
* outside every scan root is the one containment that matters, and it is enforced where it can be
|
|
130
|
+
* observed, in `buildTargetsForStaged`, not here.
|
|
131
|
+
*
|
|
132
|
+
* @param {import("./phi-scan.js").PhiScanConfig} config
|
|
133
|
+
* @returns {Required<Omit<import("./phi-scan.js").PhiScanConfig, "detect">> & { detect?: import("./phi-scan.js").DetectFn }}
|
|
134
|
+
*/
|
|
135
|
+
function normalizeConfig(config) {
|
|
136
|
+
if (typeof config !== "object" || config === null) {
|
|
137
|
+
throw new TypeError("runPhiScan(config) expects a configuration object.");
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const repoRoot = config.repoRoot ?? process.cwd();
|
|
141
|
+
if (typeof repoRoot !== "string" || repoRoot === "") {
|
|
142
|
+
throw new TypeError("runPhiScan: `repoRoot` must be a non-empty string when given.");
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const codes = config.exitCodes;
|
|
146
|
+
if (typeof codes !== "object" || codes === null) {
|
|
147
|
+
throw new TypeError(
|
|
148
|
+
"runPhiScan: `exitCodes` is REQUIRED ({ clean, hits, refuse }). There is deliberately no " +
|
|
149
|
+
"default: the sibling @cosyte/* scanners do not agree on these numbers, and a caller that " +
|
|
150
|
+
"branches on the code (CI does) must read this repo's own contract, never an inherited one.",
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
for (const key of /** @type {const} */ (["clean", "hits", "refuse"])) {
|
|
154
|
+
const value = codes[key];
|
|
155
|
+
if (!Number.isInteger(value) || value < 0 || value > 125) {
|
|
156
|
+
throw new TypeError(
|
|
157
|
+
`runPhiScan: exitCodes.${key} must be an integer in 0..125, got ${String(value)}.`,
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (codes.clean === codes.hits || codes.clean === codes.refuse || codes.hits === codes.refuse) {
|
|
162
|
+
throw new TypeError(
|
|
163
|
+
"runPhiScan: exitCodes.clean, .hits and .refuse must be three DIFFERENT numbers. A caller " +
|
|
164
|
+
"has to be able to tell `this corpus contains something that looks like PHI` from `this " +
|
|
165
|
+
"scan is not trustworthy`: those need different human responses, and collapsing them makes " +
|
|
166
|
+
"the second read as the first.",
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const rawRoots = config.scanRoots;
|
|
171
|
+
if (!Array.isArray(rawRoots) || rawRoots.length === 0) {
|
|
172
|
+
throw new TypeError(
|
|
173
|
+
'runPhiScan: `scanRoots` is REQUIRED and must name at least one root. Use `["."]` for the ' +
|
|
174
|
+
"whole repository.",
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
/** @type {Set<string>} */
|
|
178
|
+
const scanRootSet = new Set();
|
|
179
|
+
for (const root of rawRoots) {
|
|
180
|
+
if (typeof root !== "string" || root === "") {
|
|
181
|
+
throw new TypeError("runPhiScan: every entry in `scanRoots` must be a non-empty string.");
|
|
182
|
+
}
|
|
183
|
+
// THE SAME NORMALIZATION EVERY OTHER PATH GETS, and it is the fix for reproduction 1 above:
|
|
184
|
+
// `isUnderScanRoot` compares against repo-relative, forward-slashed paths, so a root must be one
|
|
185
|
+
// too or it matches nothing while still walking correctly. This maps `./src`, `src/` and
|
|
186
|
+
// `<repoRoot>/src` onto `src`, and the repository root itself onto `.`.
|
|
187
|
+
const rel = relative(repoRoot, isAbsolute(root) ? root : resolve(repoRoot, root))
|
|
188
|
+
.split(sep)
|
|
189
|
+
.join("/");
|
|
190
|
+
if (rel === ".." || rel.startsWith("../")) {
|
|
191
|
+
throw new TypeError(
|
|
192
|
+
`runPhiScan: scanRoots entry ${JSON.stringify(root)} resolves outside the repository ` +
|
|
193
|
+
`(${rel}). No path git can name is under it, so every index-keyed rule would go silently ` +
|
|
194
|
+
`empty and the sweep would report clean over a corpus it never had in scope.`,
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
scanRootSet.add(rel === "" ? "." : rel);
|
|
198
|
+
}
|
|
199
|
+
// Deduped, so two spellings of one root cannot walk the same tree twice.
|
|
200
|
+
const scanRoots = [...scanRootSet];
|
|
201
|
+
|
|
202
|
+
if (typeof config.isStagedReadable !== "function") {
|
|
203
|
+
throw new TypeError(
|
|
204
|
+
"runPhiScan: `isStagedReadable` is REQUIRED. It decides which staged blobs a COMMIT is " +
|
|
205
|
+
"blocked on, which is a per-repo hook decision and must not be inherited silently.",
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// THE OPTIONAL AXES ARE SHAPE-CHECKED TOO, AND THAT IS NOT TIDINESS. A reviewer measured
|
|
210
|
+
// `excludedPaths: ["a"]`, which is a plausible reading of "repo-relative paths": it survived
|
|
211
|
+
// normalization, reached `.has(...)` inside enumeration, threw an uncaught `TypeError` from there,
|
|
212
|
+
// and the run took node's exit 1, which the exit contract reserves for HITS FOUND. Checking the
|
|
213
|
+
// shape here turns that into a throw that says what to fix, before any scan begins.
|
|
214
|
+
if (config.excludedPaths !== undefined && typeof config.excludedPaths.has !== "function") {
|
|
215
|
+
throw new TypeError(
|
|
216
|
+
"runPhiScan: `excludedPaths` must be a Set (or anything with `.has(path)`), not an array.",
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
if (config.regularBlobModes !== undefined && typeof config.regularBlobModes.has !== "function") {
|
|
220
|
+
throw new TypeError(
|
|
221
|
+
"runPhiScan: `regularBlobModes` must be a Set (or anything with `.has(mode)`).",
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
for (const key of /** @type {const} */ (["isWalkReadable", "detect"])) {
|
|
225
|
+
if (config[key] !== undefined && typeof config[key] !== "function") {
|
|
226
|
+
throw new TypeError(`runPhiScan: \`${key}\` must be a function when given.`);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
for (const key of /** @type {const} */ (["allowListPath", "overrideLogPath"])) {
|
|
230
|
+
const value = config[key];
|
|
231
|
+
if (value !== undefined && (typeof value !== "string" || value === "")) {
|
|
232
|
+
throw new TypeError(`runPhiScan: \`${key}\` must be a non-empty string when given.`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
if (config.argv !== undefined && !Array.isArray(config.argv)) {
|
|
236
|
+
throw new TypeError("runPhiScan: `argv` must be an array of strings when given.");
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
return {
|
|
240
|
+
repoRoot,
|
|
241
|
+
argv: config.argv ?? process.argv.slice(2),
|
|
242
|
+
exitCodes: { clean: codes.clean, hits: codes.hits, refuse: codes.refuse },
|
|
243
|
+
scanRoots,
|
|
244
|
+
excludedPaths: config.excludedPaths ?? new Set(),
|
|
245
|
+
isWalkReadable: config.isWalkReadable ?? exemptsMarkdown,
|
|
246
|
+
isStagedReadable: config.isStagedReadable,
|
|
247
|
+
regularBlobModes: config.regularBlobModes ?? DEFAULT_REGULAR_BLOB_MODES,
|
|
248
|
+
allowListPath: config.allowListPath ?? join(repoRoot, "scripts", "phi-allow-list.txt"),
|
|
249
|
+
overrideLogPath: config.overrideLogPath ?? join(repoRoot, "phi-scan-overrides.md"),
|
|
250
|
+
detect: config.detect,
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Run the PHI scan and RETURN an exit code. Nothing here calls `process.exit`, so a test can drive
|
|
256
|
+
* the engine in-process; the scanner script is the one that exits.
|
|
257
|
+
*
|
|
258
|
+
* ===========================================================================================
|
|
259
|
+
* MODES
|
|
260
|
+
* `--staged` scan only the blobs `git diff --cached` names.
|
|
261
|
+
* `<path> [<path>...]` scan specific paths.
|
|
262
|
+
* (no args) `all` mode: sweep the scan roots, as a union with the bytes git carries.
|
|
263
|
+
*
|
|
264
|
+
* `--allow-fixture <path>` IS A MODIFIER, NOT A MODE. A bypass is subtractive, so it must not also
|
|
265
|
+
* decide what gets scanned: a lone `--allow-fixture X` used to select `paths` mode over exactly
|
|
266
|
+
* `X`, then withdraw `X`, then report a clean whole run having opened nothing. The mode is chosen by
|
|
267
|
+
* positional paths alone, and in `paths` mode the flag is UNCONDITIONALLY UNIONED into the target
|
|
268
|
+
* list, deduped by repo-relative path, so it means the same thing in every argv.
|
|
269
|
+
* ===========================================================================================
|
|
270
|
+
* THE COMPLETENESS RULE: a target this run ENUMERATED and NEVER READ refuses, IN EVERY MODE, NAMING
|
|
271
|
+
* THE PATHS. The comparison is a SET DIFFERENCE, never a size: a count counts the targets that DID
|
|
272
|
+
* get read, so a plausible-looking total hides exactly the paths that did not.
|
|
273
|
+
*
|
|
274
|
+
* ENUMERATION IS THE RUN'S OWN DECLARATION OF WHAT IT WILL READ, so the read filters upstream of it
|
|
275
|
+
* are not weakened by the rule: a path the read filter dropped, a gitignored entry, an excluded
|
|
276
|
+
* path and a staged path outside `isStagedReadable` never become targets at all. What the rule
|
|
277
|
+
* catches is a path that BECAME a target and then did not get opened.
|
|
278
|
+
*
|
|
279
|
+
* A BYPASS NAMING A PATH THIS RUN DOES NOT ENUMERATE ALSO REFUSES: such a flag subtracts nothing, so
|
|
280
|
+
* honouring it silently would let a developer believe a file was acknowledged when the run never had
|
|
281
|
+
* it in scope.
|
|
282
|
+
*
|
|
283
|
+
* WHAT IT COSTS, STATED RATHER THAN LEFT TO BE DISCOVERED: `--allow-fixture` CANNOT REACH THE CLEAN
|
|
284
|
+
* CODE IN ANY MODE. The flag, the override log and the rejection gate are all kept, so an attempt is
|
|
285
|
+
* RECORDED AND REFUSED rather than silently honoured.
|
|
286
|
+
* ===========================================================================================
|
|
287
|
+
* `all` MODE READS THE BYTES GIT CARRIES AS A UNION WITH THE WALK.
|
|
288
|
+
*
|
|
289
|
+
* The walk answers "what is on disk under the scan roots", which is not the question "what does this
|
|
290
|
+
* repository carry". Three states were measured in which the walk alone reported clean at exit 0
|
|
291
|
+
* over a TRACKED file carrying a live, detectable hit: the path OCCUPIED BY A DIRECTORY (a path-SET
|
|
292
|
+
* reconciliation cannot see this one, because the path IS present: only reading the OBJECT closes
|
|
293
|
+
* it), the working tree SHORT of a tracked file, and the two copies simply DIFFERING.
|
|
294
|
+
*
|
|
295
|
+
* So `git ls-files -s -z` is read for the whole index, and every in-scope tracked path whose bytes
|
|
296
|
+
* the walk did not already read VERBATIM is scanned through `git cat-file blob <sha>`. It is a union
|
|
297
|
+
* and never a replacement: the walk still runs first and still reaches UNTRACKED files, which git
|
|
298
|
+
* cannot name at all.
|
|
299
|
+
*
|
|
300
|
+
* DEDUPLICATION IS BY CONTENT, NOT BY PATH, AND THAT IS THE EOL AXIS. A walk target is skipped by
|
|
301
|
+
* the union only when the bytes it read hash to the index entry's own object id, so on a clean
|
|
302
|
+
* checkout the union adds ZERO reads and NEVER invokes `git cat-file`. The exact fixed cost is ONE
|
|
303
|
+
* `git rev-parse --show-object-format` per `all`-mode run, because the deduplication needs the
|
|
304
|
+
* algorithm before it can compare anything. Where the two copies of a path DIFFER, BOTH are scanned.
|
|
305
|
+
*
|
|
306
|
+
* A HIT FROM THE UNION IS LABELLED `(git index)` OR `(git index; the working tree differs)`, ON THE
|
|
307
|
+
* REPORTED LOCUS ONLY, and a footer counts how many of the reported hits came from those bytes. A
|
|
308
|
+
* hit naming the bare path would send a developer to open a file that is clean, or not there at all,
|
|
309
|
+
* and a single label would hide that re-staging is part of the remedy for the second kind. The
|
|
310
|
+
* target's `path` stays undecorated because every filter, exclusion and completeness tier keys on it.
|
|
311
|
+
* The `detect` callback is handed the LOCUS as `path`, so a caller cannot get this wrong.
|
|
312
|
+
*
|
|
313
|
+
* THE INDEX HALF IS NOT NARROWED BY `scanRoots`: every path the index carries is considered, because
|
|
314
|
+
* a root list is the WALK's scope and not a statement about what the repository carries. `--staged`
|
|
315
|
+
* is untouched by that and keeps its own scope, which is what a COMMIT is blocked on.
|
|
316
|
+
*
|
|
317
|
+
* `all` MODE REFUSES WHEN GIT CANNOT READ THE INDEX, OR READS IT EMPTY, and the two arrive through
|
|
318
|
+
* DIFFERENT branches and carry DIFFERENT sentences, because the remedies differ. Measured on git 2.39.5: a directory that is no repository at all FATALS (exit
|
|
319
|
+
* 128), so the `catch` is what turns it into a refusal, and WITHOUT that catch the throw escapes and
|
|
320
|
+
* the run takes node's own exit 1, which this contract reserves for HITS FOUND. A repository whose
|
|
321
|
+
* index is empty, and a directory inside a repository with nothing tracked under it, both print
|
|
322
|
+
* nothing and exit 0, which is what the size check is for. A scaffolded repo must therefore
|
|
323
|
+
* `git init` and commit before an `all`-mode run means anything.
|
|
324
|
+
* ===========================================================================================
|
|
325
|
+
*
|
|
326
|
+
* @param {import("./phi-scan.js").PhiScanConfig} config
|
|
327
|
+
* @returns {number} The exit code, drawn from `config.exitCodes`.
|
|
328
|
+
* @example
|
|
329
|
+
* process.exit(
|
|
330
|
+
* runPhiScan({
|
|
331
|
+
* exitCodes: { clean: 0, hits: 1, refuse: 2 },
|
|
332
|
+
* scanRoots: ["."],
|
|
333
|
+
* isStagedReadable: (p) => p.startsWith("test/fixtures/"),
|
|
334
|
+
* }),
|
|
335
|
+
* );
|
|
336
|
+
*/
|
|
337
|
+
export function runPhiScan(config) {
|
|
338
|
+
return new PhiScan(normalizeConfig(config)).run();
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
class PhiScan {
|
|
342
|
+
/** @param {ReturnType<typeof normalizeConfig>} cfg */
|
|
343
|
+
constructor(cfg) {
|
|
344
|
+
this.cfg = cfg;
|
|
345
|
+
/** Does any scan root name the repository root itself? Then everything is in scope. */
|
|
346
|
+
this.wholeRepo = cfg.scanRoots.includes(".");
|
|
347
|
+
/** Reported loci whose bytes came from the index, for the report's per-origin footer. */
|
|
348
|
+
/** @type {Set<string>} */
|
|
349
|
+
this.indexLoci = new Set();
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// -------------------------------------------------------------------------
|
|
353
|
+
// Paths
|
|
354
|
+
// -------------------------------------------------------------------------
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* A repo-relative, forward-slashed path: the key every filter, exclusion, dedupe and completeness
|
|
358
|
+
* tier uses, and the only spelling that appears in a diagnostic.
|
|
359
|
+
*
|
|
360
|
+
* @param {string} p
|
|
361
|
+
* @returns {string}
|
|
362
|
+
*/
|
|
363
|
+
normalizePath(p) {
|
|
364
|
+
const abs = isAbsolute(p) ? p : resolve(this.cfg.repoRoot, p);
|
|
365
|
+
return relative(this.cfg.repoRoot, abs).split(sep).join("/");
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* AXIS 2, the ROOT half of scope: is this entry the scan's BUSINESS at all?
|
|
370
|
+
*
|
|
371
|
+
* THERE ARE TWO SCOPE PREDICATES AND COLLAPSING THEM REOPENS A MEASURED HOLE. This one decides
|
|
372
|
+
* whether an entry is in scope; the READ filters (`isWalkReadable`, `isStagedReadable`) decide
|
|
373
|
+
* whether a REGULAR FILE's bytes get read. Every non-regular and non-blob check keys on THIS one.
|
|
374
|
+
* Two sibling ports independently shipped a single shared predicate and both had the routes
|
|
375
|
+
* disagree about the same entry: a `.md`-named link fell out through the read filter on one route
|
|
376
|
+
* while the other refused it. A link's NAME is no evidence at all about what is on the other side
|
|
377
|
+
* of it, which is exactly what a read filter is entitled to assume about a file and is not
|
|
378
|
+
* entitled to assume about a link.
|
|
379
|
+
*
|
|
380
|
+
* A bare root name is in scope because git records no index entry for a directory. A scan root
|
|
381
|
+
* appearing as an index entry therefore means it is not a directory, and AT LEAST these readings
|
|
382
|
+
* exist: the root has been replaced by a blob or a link; it is a FILE ROOT, which this engine
|
|
383
|
+
* admits, since `scanRoots` derives a root's kind from the filesystem; or it is a gitlink, a
|
|
384
|
+
* nested repository, which the index refusal names in those words. No enumeration is attempted.
|
|
385
|
+
* An earlier draft predated file roots and said the first reading was the only one.
|
|
386
|
+
*
|
|
387
|
+
* @param {string} relPath
|
|
388
|
+
* @returns {boolean}
|
|
389
|
+
*/
|
|
390
|
+
isUnderScanRoot(relPath) {
|
|
391
|
+
if (this.wholeRepo) return true;
|
|
392
|
+
return this.cfg.scanRoots.some((root) => relPath === root || relPath.startsWith(`${root}/`));
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// -------------------------------------------------------------------------
|
|
396
|
+
// Argument parsing
|
|
397
|
+
// -------------------------------------------------------------------------
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* @param {string[]} argv
|
|
401
|
+
* @returns {{ mode: "all" | "staged" | "paths", paths: string[], allowFixtures: string[] }}
|
|
402
|
+
*/
|
|
403
|
+
parseArgs(argv) {
|
|
404
|
+
let staged = false;
|
|
405
|
+
/** @type {string[]} */
|
|
406
|
+
const paths = [];
|
|
407
|
+
/** @type {string[]} */
|
|
408
|
+
const allowFixtures = [];
|
|
409
|
+
let i = 0;
|
|
410
|
+
while (i < argv.length) {
|
|
411
|
+
const a = argv[i];
|
|
412
|
+
if (a === "--") {
|
|
413
|
+
for (let j = i + 1; j < argv.length; j += 1) {
|
|
414
|
+
const v = argv[j];
|
|
415
|
+
if (v !== undefined) paths.push(v);
|
|
416
|
+
}
|
|
417
|
+
break;
|
|
418
|
+
} else if (a === "--staged") {
|
|
419
|
+
staged = true;
|
|
420
|
+
i += 1;
|
|
421
|
+
} else if (a === "--allow-fixture") {
|
|
422
|
+
const next = argv[i + 1];
|
|
423
|
+
if (next === undefined) {
|
|
424
|
+
throw new InvocationError("--allow-fixture requires a path argument");
|
|
425
|
+
}
|
|
426
|
+
allowFixtures.push(next);
|
|
427
|
+
i += 2;
|
|
428
|
+
} else if (a !== undefined && a.startsWith("--")) {
|
|
429
|
+
throw new InvocationError(`Unknown flag: ${a}`);
|
|
430
|
+
} else if (a !== undefined) {
|
|
431
|
+
paths.push(a);
|
|
432
|
+
i += 1;
|
|
433
|
+
} else {
|
|
434
|
+
i += 1;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
if (staged && paths.length > 0) {
|
|
439
|
+
throw new InvocationError("--staged cannot be combined with positional paths");
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
// THE MODE IS CHOSEN BY POSITIONAL PATHS ALONE. Letting a bypass select the mode made
|
|
443
|
+
// `--allow-fixture X` scan exactly `X`, then withdraw it, then report a clean whole run over a
|
|
444
|
+
// corpus it never touched.
|
|
445
|
+
/** @type {"all" | "staged" | "paths"} */
|
|
446
|
+
let mode;
|
|
447
|
+
if (staged) mode = "staged";
|
|
448
|
+
else if (paths.length > 0) mode = "paths";
|
|
449
|
+
else mode = "all";
|
|
450
|
+
|
|
451
|
+
// A BYPASS IS NOT A TARGET LIST, AND WHICH TIER REFUSES IT IS THE CONTRACT (AC-17). In `paths`
|
|
452
|
+
// mode the run's declared scope is argv, so a bypass naming something else subtracts nothing and
|
|
453
|
+
// is refused by the UNMATCHED tier, which says so and names it. In `all` mode the declared scope
|
|
454
|
+
// is the whole corpus, so a bypass declares a path this run would have read and is refused by the
|
|
455
|
+
// COMPLETENESS tier: `run` adds it to the enumerated set for exactly that reason. Both refuse,
|
|
456
|
+
// and neither can reach the clean code.
|
|
457
|
+
//
|
|
458
|
+
// WHAT THIS IS NOT: the flag still does not choose the MODE. Letting it made
|
|
459
|
+
// `--allow-fixture X` scan exactly `X`, then withdraw it, then report a clean whole run over a
|
|
460
|
+
// corpus it never touched, and that is the hole the mode rule below closes. Dedupe is by
|
|
461
|
+
// repo-relative path, so one file named twice on argv is one target, not two.
|
|
462
|
+
const seed = [...paths];
|
|
463
|
+
const scanPaths = mode === "paths" ? this.dedupeByRepoPath(seed) : paths;
|
|
464
|
+
|
|
465
|
+
return { mode, paths: scanPaths, allowFixtures };
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/**
|
|
469
|
+
* Dedupe argument paths by the repo-relative path each resolves to, keeping the caller's original
|
|
470
|
+
* spelling for the first occurrence: that spelling is what gets resolved and echoed back.
|
|
471
|
+
*
|
|
472
|
+
* @param {string[]} paths
|
|
473
|
+
* @returns {string[]}
|
|
474
|
+
*/
|
|
475
|
+
dedupeByRepoPath(paths) {
|
|
476
|
+
const seen = new Set();
|
|
477
|
+
/** @type {string[]} */
|
|
478
|
+
const out = [];
|
|
479
|
+
for (const p of paths) {
|
|
480
|
+
const key = this.normalizePath(p);
|
|
481
|
+
if (seen.has(key)) continue;
|
|
482
|
+
seen.add(key);
|
|
483
|
+
out.push(p);
|
|
484
|
+
}
|
|
485
|
+
return out;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// -------------------------------------------------------------------------
|
|
489
|
+
// Allow-list + override log
|
|
490
|
+
// -------------------------------------------------------------------------
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* The positive declaration that specific identifiers are synthetic.
|
|
494
|
+
*
|
|
495
|
+
* ONE ESCAPE IS DISCLOSED RATHER THAN CLOSED: an allow-list that EXISTS but cannot be READ (a
|
|
496
|
+
* directory at that path, or mode 000) makes `readFileSync` throw a plain `Error`, which is not an
|
|
497
|
+
* `InvocationError` and is therefore rethrown, and the run takes node's own exit 1 with a stack. A
|
|
498
|
+
* caller reads that as "hits found". `loadOverrideLog` has the identical shape. Both are left
|
|
499
|
+
* alone deliberately rather than papered over by widening a catch or enumerating `EACCES`/
|
|
500
|
+
* `EISDIR`: a deny-list of spellings buys exactly one more evasion per round, and this repo
|
|
501
|
+
* retired that shape once already on the `attw` gate. The exit contract below therefore says
|
|
502
|
+
* MISSING rather than "missing or unreadable", and says 1 is reserved but NOT exclusive.
|
|
503
|
+
*
|
|
504
|
+
* @returns {import("./phi-scan.js").AllowList}
|
|
505
|
+
*/
|
|
506
|
+
loadAllowList() {
|
|
507
|
+
if (!existsSync(this.cfg.allowListPath)) {
|
|
508
|
+
throw new InvocationError(`allow-list not found at ${this.cfg.allowListPath}`);
|
|
509
|
+
}
|
|
510
|
+
const raw = readFileSync(this.cfg.allowListPath, "utf8");
|
|
511
|
+
/** @type {Set<string>} */ const names = new Set();
|
|
512
|
+
/** @type {Set<string>} */ const dobs = new Set();
|
|
513
|
+
/** @type {Set<string>} */ const ids = new Set();
|
|
514
|
+
/** @type {Set<string>} */ const emailDomains = new Set();
|
|
515
|
+
for (const lineRaw of raw.split(/\r?\n/)) {
|
|
516
|
+
const line = lineRaw.trim();
|
|
517
|
+
if (line.length === 0 || line.startsWith("#")) continue;
|
|
518
|
+
const sp = line.indexOf(" ");
|
|
519
|
+
if (sp < 0) continue;
|
|
520
|
+
const tag = line.slice(0, sp);
|
|
521
|
+
const value = line.slice(sp + 1).trim();
|
|
522
|
+
if (value.length === 0) continue;
|
|
523
|
+
switch (tag) {
|
|
524
|
+
case "NAME":
|
|
525
|
+
names.add(value.toUpperCase());
|
|
526
|
+
break;
|
|
527
|
+
case "DOB":
|
|
528
|
+
dobs.add(value);
|
|
529
|
+
break;
|
|
530
|
+
case "ID":
|
|
531
|
+
ids.add(value.toUpperCase());
|
|
532
|
+
break;
|
|
533
|
+
case "EMAILDOMAIN":
|
|
534
|
+
emailDomains.add(value.toLowerCase());
|
|
535
|
+
break;
|
|
536
|
+
default:
|
|
537
|
+
break;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
return { names, dobs, ids, emailDomains };
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/** @returns {Set<string>} Every path the override log records, repo-relative. */
|
|
544
|
+
loadOverrideLog() {
|
|
545
|
+
if (!existsSync(this.cfg.overrideLogPath)) return new Set();
|
|
546
|
+
const raw = readFileSync(this.cfg.overrideLogPath, "utf8");
|
|
547
|
+
/** @type {Set<string>} */
|
|
548
|
+
const out = new Set();
|
|
549
|
+
for (const lineRaw of raw.split(/\r?\n/)) {
|
|
550
|
+
const m = /^###\s+(.+?)\s*$/.exec(lineRaw);
|
|
551
|
+
if (m && m[1] !== undefined) out.add(this.normalizePath(m[1]));
|
|
552
|
+
}
|
|
553
|
+
return out;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/** @param {string[]} allowFixtures */
|
|
557
|
+
validateAllowFixtures(allowFixtures) {
|
|
558
|
+
if (allowFixtures.length === 0) return;
|
|
559
|
+
const overrides = this.loadOverrideLog();
|
|
560
|
+
const missing = allowFixtures
|
|
561
|
+
.map((p) => this.normalizePath(p))
|
|
562
|
+
.filter((p) => !overrides.has(p));
|
|
563
|
+
if (missing.length > 0) {
|
|
564
|
+
const lines = missing.map((p) => ` - ${p}`).join("\n");
|
|
565
|
+
throw new InvocationError(
|
|
566
|
+
`--allow-fixture rejected: no matching entry in ${this.relOverrideLog()} for:\n${lines}\n` +
|
|
567
|
+
`Add a "### <path>" subsection to ${this.relOverrideLog()} and commit it.`,
|
|
568
|
+
);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/** @returns {string} The override log's repo-relative path, for a diagnostic. */
|
|
573
|
+
relOverrideLog() {
|
|
574
|
+
return this.normalizePath(this.cfg.overrideLogPath);
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
/** @returns {string} The allow-list's repo-relative path, for a diagnostic. */
|
|
578
|
+
relAllowList() {
|
|
579
|
+
return this.normalizePath(this.cfg.allowListPath);
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
// -------------------------------------------------------------------------
|
|
583
|
+
// git
|
|
584
|
+
// -------------------------------------------------------------------------
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* Which of these paths git considers ignored.
|
|
588
|
+
*
|
|
589
|
+
* ONE BOUNDARY, NOT TWO: an ignored entry is already out of scope for the file route, so applying
|
|
590
|
+
* the same rule to a link keeps a single boundary rather than inventing a second, stricter one for
|
|
591
|
+
* links alone. `git check-ignore` is INDEX-AWARE, so `git add -f` on an ignored link does not buy
|
|
592
|
+
* a bypass: once tracked it is no longer reported ignored.
|
|
593
|
+
*
|
|
594
|
+
* @param {string[]} paths Repo-relative or absolute paths.
|
|
595
|
+
* @returns {Set<string>} The repo-relative paths git reports ignored.
|
|
596
|
+
*/
|
|
597
|
+
gitIgnored(paths) {
|
|
598
|
+
/** @type {Set<string>} */
|
|
599
|
+
const ignored = new Set();
|
|
600
|
+
if (paths.length === 0) return ignored;
|
|
601
|
+
try {
|
|
602
|
+
// SECURITY: array-form execFileSync, no shell. Default (Buffer) encoding, because
|
|
603
|
+
// `encoding: "buffer"` together with `input` is rejected by Node.
|
|
604
|
+
const out = execFileSync("git", ["check-ignore", "--stdin", "-z"], {
|
|
605
|
+
cwd: this.cfg.repoRoot,
|
|
606
|
+
input: paths.map((p) => this.normalizePath(p)).join("\0"),
|
|
607
|
+
stdio: ["pipe", "pipe", "ignore"],
|
|
608
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
609
|
+
});
|
|
610
|
+
for (const p of out.toString("utf8").split("\0")) {
|
|
611
|
+
if (p.length > 0) ignored.add(p);
|
|
612
|
+
}
|
|
613
|
+
} catch {
|
|
614
|
+
// `git check-ignore` exits 1 when nothing matches, and fatals outside a repository. Treat both
|
|
615
|
+
// as "none ignored": in `all` mode the missing index is refused separately and loudly, and
|
|
616
|
+
// pruning nothing can only ever make the sweep read MORE.
|
|
617
|
+
}
|
|
618
|
+
return ignored;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* The repository's object format as a Node hash name, or `null` when git says something we do not
|
|
623
|
+
* recognise. `null` disables the union's content deduplication, which scans MORE, never less.
|
|
624
|
+
*
|
|
625
|
+
* WHEN GIT WILL NOT SAY AT ALL THE ANSWER IS `sha1`, NOT `null`, and the two cases are stated
|
|
626
|
+
* apart. A git too old to know `--show-object-format` predates sha256 repositories entirely, so
|
|
627
|
+
* the fallback is a derivation rather than a guess; an answer we do not recognise is a git NEWER
|
|
628
|
+
* than this file, and there the honest move is to stop deduplicating.
|
|
629
|
+
*
|
|
630
|
+
* @returns {string | null}
|
|
631
|
+
*/
|
|
632
|
+
gitObjectHash() {
|
|
633
|
+
let answer;
|
|
634
|
+
try {
|
|
635
|
+
// SECURITY: array-form execFileSync, no shell.
|
|
636
|
+
answer = execFileSync("git", ["rev-parse", "--show-object-format"], {
|
|
637
|
+
cwd: this.cfg.repoRoot,
|
|
638
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
639
|
+
})
|
|
640
|
+
.toString("utf8")
|
|
641
|
+
.trim();
|
|
642
|
+
} catch {
|
|
643
|
+
return "sha1";
|
|
644
|
+
}
|
|
645
|
+
if (answer === "sha1") return "sha1";
|
|
646
|
+
if (answer === "sha256") return "sha256";
|
|
647
|
+
return null;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* The object id git would record for these bytes, under its own `blob <len>\0` framing.
|
|
652
|
+
*
|
|
653
|
+
* Used only to answer "did the walk already read EXACTLY the bytes the index carries here", so a
|
|
654
|
+
* wrong answer can only ever cost a second scan of the same content. THIS IS THE EOL AXIS: where a
|
|
655
|
+
* `text` attribute or `core.autocrlf` makes the index carry LF and the working tree CRLF, the two
|
|
656
|
+
* ids differ and BOTH copies are scanned.
|
|
657
|
+
*
|
|
658
|
+
* @param {string} algorithm
|
|
659
|
+
* @param {Buffer} bytes
|
|
660
|
+
* @returns {string | null}
|
|
661
|
+
*/
|
|
662
|
+
blobOid(algorithm, bytes) {
|
|
663
|
+
try {
|
|
664
|
+
return createHash(algorithm)
|
|
665
|
+
.update(`blob ${String(bytes.length)}\0`)
|
|
666
|
+
.update(bytes)
|
|
667
|
+
.digest("hex");
|
|
668
|
+
} catch {
|
|
669
|
+
return null;
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Every stage-0 index entry keyed by repo-relative path, plus the paths that have a record but NO
|
|
675
|
+
* stage-0 record.
|
|
676
|
+
*
|
|
677
|
+
* THREE OUTCOMES, NOT TWO, AND THE SPLIT IS THE POINT (AC-18). `unreadable` means git could not
|
|
678
|
+
* say what this repository carries; `empty` means it said, and the answer is nothing. Both refuse
|
|
679
|
+
* the sweep, and an empty answer still counts as no corpus: an empty map would make every tracked
|
|
680
|
+
* path untracked, which is the one state in which the union silently stops existing. What differs
|
|
681
|
+
* is the REMEDY, so the two states carry different sentences rather than one that covers both.
|
|
682
|
+
*
|
|
683
|
+
* 🛑 THE STAGE DIGIT IS READ, AND KEYING ON IT IS NOT OPTIONAL. THE RULE IS THE ABSENCE OF STAGE 0.
|
|
684
|
+
* Do NOT re-derive it from a record count or from a mode, and do NOT port it from the `--staged`
|
|
685
|
+
* route: that route spots an unmerged path from `--raw`'s status `U` and a destination mode of
|
|
686
|
+
* `000000`, and NOTHING IN `ls-files -s` LOOKS LIKE THAT. An unmerged path is reported here only at
|
|
687
|
+
* stages 1, 2 and/or 3, with ORDINARY BLOB MODES, so the mode rule cannot see it. A sibling's draft
|
|
688
|
+
* took the FIRST record per path and never looked at the stage: it scanned STAGE 1, THE MERGE BASE,
|
|
689
|
+
* labelled it as the bytes git carries, and printed a clean line over a marker living only in
|
|
690
|
+
* stage 3.
|
|
691
|
+
*
|
|
692
|
+
* @returns {{ status: "read", entries: Map<string, import("./phi-scan.js").IndexEntry>, unmerged: string[] } | { status: "unreadable" | "empty" }}
|
|
693
|
+
*/
|
|
694
|
+
gitIndexEntries() {
|
|
695
|
+
let out;
|
|
696
|
+
try {
|
|
697
|
+
// SECURITY: array-form execFileSync, no shell. `-z` is NUL-separated and unquoted, so it
|
|
698
|
+
// matches the walk's forward-slash relative paths exactly. `maxBuffer` is raised because a
|
|
699
|
+
// TRUNCATED list is a SHORT list, and a short list is the unscanned corpus this whole rule is
|
|
700
|
+
// about; Node throws `ENOBUFS` rather than truncating, so the bound refuses either way.
|
|
701
|
+
out = execFileSync("git", ["ls-files", "-s", "-z"], {
|
|
702
|
+
cwd: this.cfg.repoRoot,
|
|
703
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
704
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
705
|
+
});
|
|
706
|
+
} catch {
|
|
707
|
+
// LOAD-BEARING. A directory that is no repository FATALS at 128; without this catch the throw
|
|
708
|
+
// escapes and the run takes node's exit 1, which this contract reserves for HITS FOUND.
|
|
709
|
+
return { status: "unreadable" };
|
|
710
|
+
}
|
|
711
|
+
/** @type {Map<string, import("./phi-scan.js").IndexEntry>} */
|
|
712
|
+
const entries = new Map();
|
|
713
|
+
/** @type {Set<string>} */
|
|
714
|
+
const higherStages = new Set();
|
|
715
|
+
for (const rec of out.toString("utf8").split("\0")) {
|
|
716
|
+
if (rec.length === 0) continue;
|
|
717
|
+
// `<mode> <oid> <stage>\t<path>`; a path may contain anything but NUL.
|
|
718
|
+
const m = /^(\d{6}) ([0-9a-f]+) (\d)\t([\s\S]+)$/.exec(rec);
|
|
719
|
+
const mode = m?.[1];
|
|
720
|
+
const oid = m?.[2];
|
|
721
|
+
const stage = m?.[3];
|
|
722
|
+
const path = m?.[4];
|
|
723
|
+
if (mode === undefined || oid === undefined || stage === undefined || path === undefined) {
|
|
724
|
+
// An unparseable record means the list may be SHORT in a way we cannot see, which is the one
|
|
725
|
+
// thing this sweep must never scan past. It is UNREADABLE rather than empty: git answered,
|
|
726
|
+
// and the answer is one this engine cannot account for.
|
|
727
|
+
return { status: "unreadable" };
|
|
728
|
+
}
|
|
729
|
+
if (stage === "0") entries.set(path, { mode, oid });
|
|
730
|
+
else higherStages.add(path);
|
|
731
|
+
}
|
|
732
|
+
// A path is unmerged when it has a record and none of them is stage 0. The set difference is
|
|
733
|
+
// taken rather than assuming the two are disjoint.
|
|
734
|
+
const unmerged = [...higherStages].filter((p) => !entries.has(p));
|
|
735
|
+
if (entries.size === 0 && unmerged.length === 0) return { status: "empty" };
|
|
736
|
+
return { status: "read", entries, unmerged };
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
// -------------------------------------------------------------------------
|
|
740
|
+
// Enumeration: the walk
|
|
741
|
+
// -------------------------------------------------------------------------
|
|
742
|
+
|
|
743
|
+
/**
|
|
744
|
+
* Enumerate the scan roots.
|
|
745
|
+
*
|
|
746
|
+
* `Dirent`'s predicates are lstat answers and are not exhaustive: an entry that is neither a
|
|
747
|
+
* directory nor a regular file is collected into `unscannable` rather than dropped, so the caller
|
|
748
|
+
* can refuse instead of reporting clean over it.
|
|
749
|
+
*
|
|
750
|
+
* IGNORED DIRECTORIES ARE PRUNED DURING DESCENT, ONE `git check-ignore` PER LEVEL. That is not an
|
|
751
|
+
* optimisation bolted onto a filter: `scanRoots: ["."]` is the only honest default for a freshly
|
|
752
|
+
* scaffolded repo, and without pruning such a sweep descends into `node_modules`.
|
|
753
|
+
*
|
|
754
|
+
* WHY IT IS EQUIVALENT TO FILTERING AFTERWARDS, NAMED AS THE RULE THAT ACTUALLY CARRIES IT. An
|
|
755
|
+
* earlier draft said "git cannot re-include a path under an excluded directory", which is a
|
|
756
|
+
* gitignore-PATTERN rule and does not settle this on its own, because the filter it replaces
|
|
757
|
+
* asked `check-ignore` too. The load-bearing property is that `git check-ignore` IS INDEX-AWARE AT
|
|
758
|
+
* DIRECTORY GRANULARITY. Measured on git 2.39.5, with `node_modules/` in `.gitignore`:
|
|
759
|
+
*
|
|
760
|
+
* - nothing tracked underneath -> `check-ignore node_modules` exits 0 and the directory is
|
|
761
|
+
* pruned, which is right, because no file under it could have survived the file-level filter
|
|
762
|
+
* either;
|
|
763
|
+
* - one file force-added underneath -> `check-ignore node_modules` exits 1, the directory is NOT
|
|
764
|
+
* pruned, and the walk descends and reads the tracked file exactly as before.
|
|
765
|
+
*
|
|
766
|
+
* The file-level filter still runs below, so pruning only ever removes work the filter would have
|
|
767
|
+
* removed anyway. `test/phi-scan-engine.test.ts` pins BOTH directions, because a test that only
|
|
768
|
+
* plants a violator in an ignored directory measures that pruning happens, not that it is safe.
|
|
769
|
+
*
|
|
770
|
+
* A DIRECTORY NAMED `.git` IS SKIPPED BY NAME, at any depth. It is git's own object store rather
|
|
771
|
+
* than the corpus, git does not report it ignored, and the union already reads what the repository
|
|
772
|
+
* carries. This is a literal name, never a predicate over content.
|
|
773
|
+
*
|
|
774
|
+
* A SCAN ROOT MAY NAME A REGULAR FILE, AND THE KIND IS DERIVED FROM THE FILESYSTEM RATHER THAN
|
|
775
|
+
* DECLARED. That is a deliberate API decision, and it was made against a measurement rather than a
|
|
776
|
+
* preference: one sibling declares its roots as `{ rel, shape: "directory" | "file" }` and lists a
|
|
777
|
+
* single file (`README.md`) among them, and an earlier draft of this walk fed every root straight
|
|
778
|
+
* to `readdirSync`, so such a root threw `ENOTDIR` from here, uncaught, and the run took node's own
|
|
779
|
+
* exit 1, which this contract reserves for HITS FOUND. Deriving the kind keeps the parameter a
|
|
780
|
+
* plain `string[]` and still expresses that shape.
|
|
781
|
+
*
|
|
782
|
+
* THAT PARTICULAR ROOT READS NOTHING UNDER THE DEFAULT `isWalkReadable`, and the example is
|
|
783
|
+
* caveated rather than left to be copied. A file root goes through the same read filter every
|
|
784
|
+
* other file does, so with the shared Markdown exemption in place `scanRoots: ["README.md"]` was
|
|
785
|
+
* measured returning `OK: no hits` at exit 0 over a live dashed identifier. A repo porting a `.md`
|
|
786
|
+
* root has to override that filter as well.
|
|
787
|
+
*
|
|
788
|
+
* WHAT DERIVING GIVES UP, STATED RATHER THAN LEFT TO BE FOUND: a declaration can notice that a root
|
|
789
|
+
* is not the KIND it was meant to be, and derivation cannot. A root that was a file and became a
|
|
790
|
+
* directory is descended, and one that was a directory and became a file is read as a file, in both
|
|
791
|
+
* cases silently. What is NOT given up is the non-regular case: a root that is neither is collected
|
|
792
|
+
* as unscannable and refused, exactly as an entry inside one would be.
|
|
793
|
+
*
|
|
794
|
+
* A MISSING ROOT IS SKIPPED BY THE WALK, AND THAT IS UNCHANGED FROM THE COPIED SCANNERS rather
|
|
795
|
+
* than chosen here. Two root states are refused or normalised away in `normalizeConfig` (a
|
|
796
|
+
* spelling that matches no index path is normalised away and the run proceeds; a root outside the
|
|
797
|
+
* repository throws), and this one is neither.
|
|
798
|
+
*
|
|
799
|
+
* WHAT IT NO LONGER DOES IS CONTRIBUTE NOTHING IN SILENCE, and that is a change. A draft of this
|
|
800
|
+
* paragraph claimed the states in which a root contributes nothing were not enumerable, and named
|
|
801
|
+
* three anyway: a MISSING root; an UNREADABLE one, which `lstatOrNull`'s bare catch reports the
|
|
802
|
+
* same way (an `EACCES` on a parent directory, or an `ELOOP`); and a root whose every file the
|
|
803
|
+
* read filter drops, which is what a `.md` file root does under the default `isWalkReadable`. The
|
|
804
|
+
* set still is not enumerated, because nothing here has measured it to be complete - and it no
|
|
805
|
+
* longer has to be. `run`'s PER-ROOT OBSERVATION RULE asks the question from the other end, of
|
|
806
|
+
* the paths actually READ, so a root that yielded nothing refuses whatever put it in that state.
|
|
807
|
+
*
|
|
808
|
+
* A DIRECTORY THE WALK CANNOT LIST IS COLLECTED AND REFUSED, NOT THROWN OUT OF. `readdirSync` used
|
|
809
|
+
* to run bare here, so an `EACCES` on a root (or on any directory under one) escaped uncaught and
|
|
810
|
+
* the run took node's own exit 1, the code this contract reserves for HITS FOUND. It is the third
|
|
811
|
+
* of the three root KINDS this engine settles (AC-10): a regular file root is scanned as one
|
|
812
|
+
* target, a root naming a symbolic link is refused rather than followed, and a directory that
|
|
813
|
+
* cannot be enumerated is refused at the caller's `refuse` code, naming the path.
|
|
814
|
+
*
|
|
815
|
+
* The result is SORTED by repo-relative path, so a report and a refusal read the same way twice.
|
|
816
|
+
*
|
|
817
|
+
* @returns {{ files: string[], unscannable: import("./phi-scan.js").Unscannable[], unenumerable: import("./phi-scan.js").Unscannable[] }}
|
|
818
|
+
*/
|
|
819
|
+
walkRoots() {
|
|
820
|
+
/** @type {string[]} */
|
|
821
|
+
const files = [];
|
|
822
|
+
/** @type {import("./phi-scan.js").Unscannable[]} */
|
|
823
|
+
const unscannable = [];
|
|
824
|
+
/** Directories the walk reached and could not list. `kind` is the errno, never a path. */
|
|
825
|
+
/** @type {import("./phi-scan.js").Unscannable[]} */
|
|
826
|
+
const unenumerable = [];
|
|
827
|
+
|
|
828
|
+
// Split the roots by what is actually THERE, before the descent. A root naming a regular file is
|
|
829
|
+
// one target; a root naming a directory is a frontier; a root naming anything else is refused
|
|
830
|
+
// through the same unscannable path an entry inside one would take.
|
|
831
|
+
/** @type {string[]} */
|
|
832
|
+
const rootDirs = [];
|
|
833
|
+
for (const root of this.cfg.scanRoots) {
|
|
834
|
+
const abs = root === "." ? this.cfg.repoRoot : join(this.cfg.repoRoot, ...root.split("/"));
|
|
835
|
+
const stats = lstatOrNull(abs);
|
|
836
|
+
if (stats === null) continue; // A missing root, skipped. See the docblock above.
|
|
837
|
+
if (stats.isDirectory()) rootDirs.push(abs);
|
|
838
|
+
else if (stats.isFile()) {
|
|
839
|
+
if (this.cfg.isWalkReadable(this.normalizePath(abs))) files.push(abs);
|
|
840
|
+
} else {
|
|
841
|
+
unscannable.push({ path: this.normalizePath(abs), kind: statsKind(stats) });
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
let frontier = rootDirs;
|
|
846
|
+
const visited = new Set(frontier);
|
|
847
|
+
|
|
848
|
+
while (frontier.length > 0) {
|
|
849
|
+
/** @type {string[]} */
|
|
850
|
+
const nextDirs = [];
|
|
851
|
+
for (const dir of frontier) {
|
|
852
|
+
if (!existsSync(dir)) continue;
|
|
853
|
+
let listing;
|
|
854
|
+
try {
|
|
855
|
+
listing = readdirSync(dir, { withFileTypes: true });
|
|
856
|
+
} catch (err) {
|
|
857
|
+
// A DIRECTORY THAT CANNOT BE LISTED IS A REFUSAL, NEVER A CRASH. The errno is recorded
|
|
858
|
+
// and nothing else: a message off the filesystem is text this engine cannot vouch for.
|
|
859
|
+
unenumerable.push({
|
|
860
|
+
path: this.normalizePath(dir),
|
|
861
|
+
kind:
|
|
862
|
+
err !== null && typeof err === "object" && "code" in err
|
|
863
|
+
? String(err.code)
|
|
864
|
+
: "unreadable",
|
|
865
|
+
});
|
|
866
|
+
continue;
|
|
867
|
+
}
|
|
868
|
+
for (const e of listing) {
|
|
869
|
+
const full = join(dir, e.name);
|
|
870
|
+
if (e.isDirectory()) {
|
|
871
|
+
if (e.name === ".git") continue;
|
|
872
|
+
if (visited.has(full)) continue;
|
|
873
|
+
visited.add(full);
|
|
874
|
+
nextDirs.push(full);
|
|
875
|
+
} else if (e.isFile()) {
|
|
876
|
+
// A READ filter. The branch below is deliberately NOT subject to it: that exemption is a
|
|
877
|
+
// judgement about a file whose bytes the walk could have read, and a link's name is no
|
|
878
|
+
// evidence at all about what is on the other side.
|
|
879
|
+
if (!this.cfg.isWalkReadable(this.normalizePath(full))) continue;
|
|
880
|
+
files.push(full);
|
|
881
|
+
} else {
|
|
882
|
+
unscannable.push({ path: this.normalizePath(full), kind: direntKind(e) });
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
frontier = this.pruneIgnoredDirs(nextDirs);
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
files.sort((a, b) => (this.normalizePath(a) < this.normalizePath(b) ? -1 : 1));
|
|
890
|
+
unscannable.sort((a, b) => (a.path < b.path ? -1 : 1));
|
|
891
|
+
unenumerable.sort((a, b) => (a.path < b.path ? -1 : 1));
|
|
892
|
+
return { files, unscannable, unenumerable };
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/**
|
|
896
|
+
* @param {string[]} dirs
|
|
897
|
+
* @returns {string[]} The subset git does not report ignored.
|
|
898
|
+
*/
|
|
899
|
+
pruneIgnoredDirs(dirs) {
|
|
900
|
+
if (dirs.length === 0) return dirs;
|
|
901
|
+
const ignored = this.gitIgnored(dirs);
|
|
902
|
+
return dirs.filter((d) => !ignored.has(this.normalizePath(d)));
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
// -------------------------------------------------------------------------
|
|
906
|
+
// Enumeration: the three routes
|
|
907
|
+
// -------------------------------------------------------------------------
|
|
908
|
+
|
|
909
|
+
/**
|
|
910
|
+
* `all` mode's enumeration: the walk, PLUS the index the union half reads.
|
|
911
|
+
*
|
|
912
|
+
* THE TWO INDEX-ROUTE REFUSALS ARE RETURNED RATHER THAN THROWN, AND THAT IS A CHANGE (AC-15).
|
|
913
|
+
* They used to fire here, BEFORE the sweep, so an unmerged entry or a tracked link discarded every
|
|
914
|
+
* hit the walk was about to find and a consumer saw a refusal with no indication that PHI had
|
|
915
|
+
* already been found. The caller raises them after the walk sweep, printing the hits first. The
|
|
916
|
+
* refusal still wins the exit code and the clean line is still unreachable from there; what
|
|
917
|
+
* changes is that a finding already made is not thrown away. The cost is stated rather than
|
|
918
|
+
* hidden: a developer now waits out the walk before being told the index is unreadable at one
|
|
919
|
+
* path.
|
|
920
|
+
*
|
|
921
|
+
* @returns {{ targets: import("./phi-scan.js").Target[], index: Map<string, import("./phi-scan.js").IndexEntry>, indexRefusals: string[] }}
|
|
922
|
+
*/
|
|
923
|
+
buildTargetsForAll() {
|
|
924
|
+
const { files, unscannable, unenumerable } = this.walkRoots();
|
|
925
|
+
|
|
926
|
+
// One `git check-ignore` over all three lists, so a link, a file and an unlistable directory get
|
|
927
|
+
// the same boundary.
|
|
928
|
+
const ignored = this.gitIgnored([
|
|
929
|
+
...files.map((f) => this.normalizePath(f)),
|
|
930
|
+
...unscannable.map((u) => u.path),
|
|
931
|
+
...unenumerable.map((u) => u.path),
|
|
932
|
+
]);
|
|
933
|
+
|
|
934
|
+
const blocked = unenumerable.filter(
|
|
935
|
+
(u) => !ignored.has(u.path) && !this.cfg.excludedPaths.has(u.path),
|
|
936
|
+
);
|
|
937
|
+
if (blocked.length > 0) {
|
|
938
|
+
// AC-10, the third root kind. Named one by one rather than listed under a count, because the
|
|
939
|
+
// remedy is per path and a developer re-running the gate once per directory learns to
|
|
940
|
+
// distrust it.
|
|
941
|
+
const named = blocked.map((u) => `${u.path} (${u.kind})`).join(", ");
|
|
942
|
+
throw new InvocationError(
|
|
943
|
+
`refusing the scan: could not enumerate ${named}. A directory the walk cannot list ` +
|
|
944
|
+
`contributes nothing while looking exactly like one that is empty, so the sweep would ` +
|
|
945
|
+
`cover less than its own configuration says. Make it readable, or drop it from the scan ` +
|
|
946
|
+
`roots.`,
|
|
947
|
+
);
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
this.refuseUnscannable(
|
|
951
|
+
unscannable.filter((u) => !ignored.has(u.path) && !this.cfg.excludedPaths.has(u.path)),
|
|
952
|
+
"The walk can neither read such an entry nor vouch for what is on the other side of it.",
|
|
953
|
+
"Remove it, replace it with a regular file, or (if it is genuinely not part of the corpus) " +
|
|
954
|
+
"untrack it and add it to .gitignore.",
|
|
955
|
+
);
|
|
956
|
+
|
|
957
|
+
const listed = this.gitIndexEntries();
|
|
958
|
+
if (listed.status !== "read") {
|
|
959
|
+
// AC-18: TWO STATES, TWO MESSAGES, because the remedies differ. "Run this inside a git
|
|
960
|
+
// repository with a readable index" is no help to somebody whose repository is fine and whose
|
|
961
|
+
// index is empty, and one sentence for both was measured sending a reader to the wrong one.
|
|
962
|
+
throw new InvocationError(
|
|
963
|
+
listed.status === "empty"
|
|
964
|
+
? "refusing the sweep: the git index holds no entries, so every check the sweep makes " +
|
|
965
|
+
"against what git carries would pass vacuously and every path would look untracked. " +
|
|
966
|
+
"Stage and commit this repository's corpus, then re-run."
|
|
967
|
+
: "refusing the sweep: could not read this repository's git index, so the sweep would be " +
|
|
968
|
+
"the working-tree walk's word alone and could report clean over tracked bytes it " +
|
|
969
|
+
"never opened. Run it inside a git repository with a readable index.",
|
|
970
|
+
);
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
/** @type {string[]} */
|
|
974
|
+
const indexRefusals = [];
|
|
975
|
+
|
|
976
|
+
// THE INDEX ROUTE IS NOT FILTERED TO `scanRoots` (AC-12). `scanRoots` is the WALK's scope: it
|
|
977
|
+
// answers "what is on disk under the roots this repo declared". The index answers "what does
|
|
978
|
+
// this repository CARRY", which is not a question a root list is entitled to narrow. Measured on
|
|
979
|
+
// the first consumer: three real messages under an undeclared top-level directory were read by
|
|
980
|
+
// neither route, and a tracked link outside every root was reached by neither. So every path the
|
|
981
|
+
// index carries is considered here, and `--staged` is untouched by that: it keeps its own scope,
|
|
982
|
+
// which is what a COMMIT is blocked on.
|
|
983
|
+
//
|
|
984
|
+
// Unmerged first, and under its OWN sentence: an unmerged path is not a link and not a gitlink,
|
|
985
|
+
// and reporting it as one sends a developer looking for something that is not there.
|
|
986
|
+
const unmergedMessage = this.unscannableMessage(
|
|
987
|
+
listed.unmerged
|
|
988
|
+
.filter((p) => !this.cfg.excludedPaths.has(p))
|
|
989
|
+
.map((p) => ({ path: p, kind: "no stage-0 blob" })),
|
|
990
|
+
"An unmerged path has no single merged blob, so there is no one set of bytes git carries here " +
|
|
991
|
+
"for the sweep to read, only the conflicting sides and, when there is one, their base.",
|
|
992
|
+
"Resolve the conflict and stage the result, then re-run.",
|
|
993
|
+
{ one: "path is unmerged", many: "paths are unmerged" },
|
|
994
|
+
);
|
|
995
|
+
if (unmergedMessage !== null) indexRefusals.push(unmergedMessage);
|
|
996
|
+
|
|
997
|
+
// The index's own non-blob entries. `120000` is a symbolic link, whose blob is its TARGET PATH
|
|
998
|
+
// and not any content; `160000` is a gitlink, which carries another repository's commit id and
|
|
999
|
+
// no bytes at this path at all.
|
|
1000
|
+
const nonBlobMessage = this.unscannableMessage(
|
|
1001
|
+
[...listed.entries]
|
|
1002
|
+
.filter(
|
|
1003
|
+
([p, e]) => !this.cfg.regularBlobModes.has(e.mode) && !this.cfg.excludedPaths.has(p),
|
|
1004
|
+
)
|
|
1005
|
+
.map(([p, e]) => ({ path: p, kind: gitModeKind(e.mode) })),
|
|
1006
|
+
"Git records no readable content at such a path: a symbolic link's blob is its target path, " +
|
|
1007
|
+
"and a gitlink is another repository's commit id, never bytes here.",
|
|
1008
|
+
"Untrack it, or replace it with a regular file.",
|
|
1009
|
+
// Its own noun: the offender is an INDEX RECORD, and a gitlink's working tree may not exist at
|
|
1010
|
+
// all, so "not a regular file" would send a developer to a path where there is nothing to see.
|
|
1011
|
+
{ one: "index entry is not a regular blob", many: "index entries are not regular blobs" },
|
|
1012
|
+
);
|
|
1013
|
+
if (nonBlobMessage !== null) indexRefusals.push(nonBlobMessage);
|
|
1014
|
+
|
|
1015
|
+
const targets = files
|
|
1016
|
+
.map((abs) => ({ abs, rel: this.normalizePath(abs) }))
|
|
1017
|
+
.filter(({ rel }) => !ignored.has(rel) && !this.cfg.excludedPaths.has(rel))
|
|
1018
|
+
.map(({ abs, rel }) => ({ path: rel, read: () => readFileSync(abs) }));
|
|
1019
|
+
return { targets, index: listed.entries, indexRefusals };
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
/**
|
|
1023
|
+
* The tracked paths the union half is entitled to read: EVERY regular blob the index carries, not
|
|
1024
|
+
* only the ones under a scan root (AC-12).
|
|
1025
|
+
*
|
|
1026
|
+
* `scanRoots` IS THE WALK'S SCOPE AND NOT THE INDEX'S. The walk answers "what is on disk under the
|
|
1027
|
+
* roots this repo declared"; the index answers "what does this repository CARRY", and narrowing
|
|
1028
|
+
* the second by the first was measured hiding three real messages under an undeclared top-level
|
|
1029
|
+
* directory from BOTH routes. The two READ filters still apply, so the `.md` exemption and
|
|
1030
|
+
* `excludedPaths` mean the same thing on both sweeping routes: one boundary, not two.
|
|
1031
|
+
*
|
|
1032
|
+
* IT IS COMPUTED BEFORE THE FIRST BYTE IS READ, AND THAT IS LOAD-BEARING. This set is part of what
|
|
1033
|
+
* `all` mode ENUMERATES, so both completeness tiers see it: a bypass naming a tracked-but-absent
|
|
1034
|
+
* path subtracts something real rather than being refused as naming nothing, and a target that
|
|
1035
|
+
* ends up unread is named by the unread refusal whichever route would have read it.
|
|
1036
|
+
*
|
|
1037
|
+
* @param {Map<string, import("./phi-scan.js").IndexEntry>} index
|
|
1038
|
+
* @returns {string[]}
|
|
1039
|
+
*/
|
|
1040
|
+
unionCandidatePaths(index) {
|
|
1041
|
+
return [...index]
|
|
1042
|
+
.filter(([p, e]) => this.cfg.regularBlobModes.has(e.mode) && this.cfg.isWalkReadable(p))
|
|
1043
|
+
.filter(([p]) => !this.cfg.excludedPaths.has(p))
|
|
1044
|
+
.map(([p]) => p);
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
/**
|
|
1048
|
+
* THE UNION HALF of `all` mode: the bytes git carries at every in-scope tracked path whose bytes
|
|
1049
|
+
* the walk did not already read VERBATIM.
|
|
1050
|
+
*
|
|
1051
|
+
* `readOids` maps a path the walk actually READ to the object id of what it read. A path absent
|
|
1052
|
+
* from it was never opened, whatever the reason, so its blob is scanned; a path present with a
|
|
1053
|
+
* DIFFERENT id had a different copy read, so its blob is scanned too. That second case is the EOL
|
|
1054
|
+
* axis.
|
|
1055
|
+
*
|
|
1056
|
+
* THOSE TWO CASES CARRY DIFFERENT ORIGIN LABELS, AND THE REMEDY IS WHY (AC-14). `(git index)` is a
|
|
1057
|
+
* path whose working-tree copy this run did not read at all: it is absent, occupied by a
|
|
1058
|
+
* directory, or outside the walk's roots, so there is nothing on disk to open. `(git index; the
|
|
1059
|
+
* working tree differs)` is a path whose disk copy WAS read and carries other bytes, so the file a
|
|
1060
|
+
* developer opens is clean and RE-STAGING is part of the fix. One label for both hid that
|
|
1061
|
+
* difference behind the same sentence.
|
|
1062
|
+
*
|
|
1063
|
+
* @param {Map<string, import("./phi-scan.js").IndexEntry>} index
|
|
1064
|
+
* @param {Map<string, string>} readOids
|
|
1065
|
+
* @returns {import("./phi-scan.js").Target[]}
|
|
1066
|
+
*/
|
|
1067
|
+
buildTargetsForGitIndex(index, readOids) {
|
|
1068
|
+
/** @type {import("./phi-scan.js").Target[]} */
|
|
1069
|
+
const targets = [];
|
|
1070
|
+
for (const path of this.unionCandidatePaths(index)) {
|
|
1071
|
+
const entry = index.get(path);
|
|
1072
|
+
if (entry === undefined) continue;
|
|
1073
|
+
if (readOids.get(path) === entry.oid) continue;
|
|
1074
|
+
targets.push({
|
|
1075
|
+
path,
|
|
1076
|
+
origin: readOids.has(path) ? "git index; the working tree differs" : "git index",
|
|
1077
|
+
// SECURITY: array-form execFileSync, no shell. The object id is git's own output, and naming
|
|
1078
|
+
// the OBJECT rather than the path is the whole point: it cannot be redirected by whatever
|
|
1079
|
+
// the working tree currently holds. `maxBuffer` defaults to 1 MiB, so a larger tracked blob
|
|
1080
|
+
// fails the read and REFUSES rather than reporting a truncated scan clean.
|
|
1081
|
+
read: () =>
|
|
1082
|
+
execFileSync("git", ["cat-file", "blob", entry.oid], {
|
|
1083
|
+
cwd: this.cfg.repoRoot,
|
|
1084
|
+
encoding: "buffer",
|
|
1085
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1086
|
+
}),
|
|
1087
|
+
});
|
|
1088
|
+
}
|
|
1089
|
+
return targets;
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
/**
|
|
1093
|
+
* @param {string[]} paths
|
|
1094
|
+
* @returns {import("./phi-scan.js").Target[]}
|
|
1095
|
+
*/
|
|
1096
|
+
buildTargetsForPaths(paths) {
|
|
1097
|
+
return paths.map((p) => {
|
|
1098
|
+
const abs = isAbsolute(p) ? p : resolve(this.cfg.repoRoot, p);
|
|
1099
|
+
if (!existsSync(abs)) throw new InvocationError(`File not found: ${p}`);
|
|
1100
|
+
if (!statSync(abs).isFile()) throw new InvocationError(`Not a regular file: ${p}`);
|
|
1101
|
+
return { path: this.normalizePath(abs), read: () => readFileSync(abs) };
|
|
1102
|
+
});
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
/**
|
|
1106
|
+
* `--staged`: exactly the blobs a commit would carry.
|
|
1107
|
+
*
|
|
1108
|
+
* `--raw` rather than `--name-only` because the DESTINATION MODE is the only thing that
|
|
1109
|
+
* distinguishes a staged regular file from a staged symlink or gitlink. `git show :<path>` does not
|
|
1110
|
+
* stand in for it: for a symbolic link it hands back the target path as if it were content, and it
|
|
1111
|
+
* is the mode, not the answer, that says so.
|
|
1112
|
+
*
|
|
1113
|
+
* `--diff-filter=d` IS AN EXCLUSION ("everything EXCEPT deletions"), NOT AN ALLOW-LIST OF STATUS
|
|
1114
|
+
* LETTERS, AND THE POLARITY IS THE WHOLE POINT. An allow-list drops every letter it does not name,
|
|
1115
|
+
* silently; that polarity is what made sibling scanners miss `R` (rename) and then `T`
|
|
1116
|
+
* (typechange), each found by a separate refuter pass one round apart. An unfamiliar or future
|
|
1117
|
+
* status letter can now only ever cost a wasted scan or a loud refusal, never a missed file.
|
|
1118
|
+
* Against an `AMT` allow-list, what `d` newly enumerates is `U` (unmerged), `X` (unknown) and `B`
|
|
1119
|
+
* (broken pairing). Measured on git 2.39.5, a conflicted path lists as one record with destination
|
|
1120
|
+
* mode `000000`, so the stride is unaffected and the mode is not a regular blob, which puts it
|
|
1121
|
+
* through the refusal below instead of past it.
|
|
1122
|
+
*
|
|
1123
|
+
* `T` (TYPECHANGE) IS ENUMERATED, AND LEAVING IT OUT MAKES THE MODE CHECK UNREACHABLE WHENEVER THE
|
|
1124
|
+
* FILE IS ALREADY TRACKED: replacing a TRACKED regular file with a link is not an add and not a
|
|
1125
|
+
* modify, so an `AM` allow-list deletes the record before any mode can be read.
|
|
1126
|
+
*
|
|
1127
|
+
* `--no-renames` IS SEPARATELY LOAD-BEARING. Rename detection is on by default, so `git mv <link>
|
|
1128
|
+
* <scan root>/<name>` stages as one record carrying TWO paths, which desyncs the two-field stride.
|
|
1129
|
+
* Turning detection off makes the destination arrive as an ordinary single-path `A` and the source
|
|
1130
|
+
* a `D` the filter drops, which makes the stride STRUCTURAL rather than conditional on the
|
|
1131
|
+
* caller's `diff.renames` setting.
|
|
1132
|
+
*
|
|
1133
|
+
* @returns {import("./phi-scan.js").Target[]}
|
|
1134
|
+
*/
|
|
1135
|
+
buildTargetsForStaged() {
|
|
1136
|
+
let listBuf;
|
|
1137
|
+
try {
|
|
1138
|
+
// SECURITY: array-form execFileSync, no shell.
|
|
1139
|
+
listBuf = execFileSync(
|
|
1140
|
+
"git",
|
|
1141
|
+
["diff", "--cached", "--raw", "-z", "--no-renames", "--diff-filter=d"],
|
|
1142
|
+
{
|
|
1143
|
+
cwd: this.cfg.repoRoot,
|
|
1144
|
+
encoding: "buffer",
|
|
1145
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1146
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
1147
|
+
},
|
|
1148
|
+
);
|
|
1149
|
+
} catch (err) {
|
|
1150
|
+
throw new InvocationError(
|
|
1151
|
+
`git diff --cached failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1152
|
+
);
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
// `--raw -z` emits `<info>\0<path>\0` per record. `R` and `C` are the only statuses carrying a
|
|
1156
|
+
// SECOND path and `--no-renames` means git cannot emit either, so the stride is two fields. A
|
|
1157
|
+
// record that does not parse REFUSES rather than being skipped: a silently shortened list is
|
|
1158
|
+
// exactly the shape this scan must never report clean over.
|
|
1159
|
+
const fields = listBuf.toString("utf8").split("\0");
|
|
1160
|
+
/** @type {{ path: string, mode: string, status: string }[]} */
|
|
1161
|
+
const staged = [];
|
|
1162
|
+
let i = 0;
|
|
1163
|
+
while (i < fields.length) {
|
|
1164
|
+
const info = fields[i];
|
|
1165
|
+
if (info === undefined || info.length === 0) {
|
|
1166
|
+
i += 1;
|
|
1167
|
+
continue;
|
|
1168
|
+
}
|
|
1169
|
+
const m = RAW_RECORD.exec(info);
|
|
1170
|
+
const mode = m?.[1];
|
|
1171
|
+
const status = m?.[2];
|
|
1172
|
+
const path = fields[i + 1];
|
|
1173
|
+
if (mode === undefined || status === undefined || path === undefined || path.length === 0) {
|
|
1174
|
+
throw new InvocationError(
|
|
1175
|
+
"could not read the output of `git diff --cached --raw -z`: unrecognized record. " +
|
|
1176
|
+
"Refusing rather than scanning a list that may be short.",
|
|
1177
|
+
);
|
|
1178
|
+
}
|
|
1179
|
+
staged.push({ path, mode, status });
|
|
1180
|
+
i += 2;
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
// THIS ROUTE'S REFUSALS KEY ON `isStagedReadable`, WHICH IS THE CALLER'S OWN SCOPE FOR IT
|
|
1184
|
+
// (AC-16). They used to key on the ROOT half instead, and for a caller whose staged scope is
|
|
1185
|
+
// narrower than its roots that REFUSED a commit the repo's own gate had always let through:
|
|
1186
|
+
// measured on the first consumer, a staged `src/notes.txt` symbolic link is outside a
|
|
1187
|
+
// `src/**.ts` staged scope and was refused at the refuse code where the repo's own scanner
|
|
1188
|
+
// exited 0. What a commit is blocked on is a hook decision each repo takes for itself, and
|
|
1189
|
+
// widening it from inside a shared engine takes that decision away.
|
|
1190
|
+
//
|
|
1191
|
+
// 🛑 THIS NARROWS WHAT THIS ROUTE REFUSES, AND THE COMPENSATING CONTROL IS NAMED RATHER THAN
|
|
1192
|
+
// ASSUMED. A staged non-regular entry the caller's staged scope declines is no longer refused
|
|
1193
|
+
// HERE. It is still refused by `all` mode twice over: the walk classifies it with
|
|
1194
|
+
// `Dirent.isSymbolicLink()` when it sits under a scan root, and the index route refuses the
|
|
1195
|
+
// tracked mode-120000 record wherever it sits, now that that route is not narrowed to the roots
|
|
1196
|
+
// either. So the sweep a repository runs in CI still refuses it; the pre-commit hook no longer
|
|
1197
|
+
// does. See `documentation/decisions/0003-the-phi-scan-engine-gap-settlements.md`.
|
|
1198
|
+
//
|
|
1199
|
+
// AN UNMERGED PATH IS NOT ONE OF THESE. Its destination mode is `000000`, which the mode rule
|
|
1200
|
+
// would otherwise describe as "a git mode-000000 entry" beside a sentence about content git
|
|
1201
|
+
// holds at the path, and both halves are false for a conflicted regular file.
|
|
1202
|
+
const unmerged = staged.filter((s) => s.status === "U" || s.mode === "000000");
|
|
1203
|
+
const stagedReadable = (/** @type {{ path: string, mode: string, status: string }} */ s) =>
|
|
1204
|
+
this.cfg.isStagedReadable(s.path) && !this.cfg.excludedPaths.has(s.path);
|
|
1205
|
+
|
|
1206
|
+
// THE CONTAINMENT IS ENFORCED, NOT ASSUMED, AND IT USED TO BE THE WORD "by construction".
|
|
1207
|
+
// `isStagedReadable` and `scanRoots` are two independent keys a repo fills in, and nothing
|
|
1208
|
+
// relates them. A reviewer measured what that costs: with roots narrowed to `src` and this
|
|
1209
|
+
// filter left at the shared Markdown exemption, a STAGED mode-120000 entry under
|
|
1210
|
+
// `test/fixtures/` was outside every scan root, so the non-regular refusal above never saw it,
|
|
1211
|
+
// and the route then ENUMERATED it, READ it, handed the link's TARGET PATH to the detector as
|
|
1212
|
+
// if it were content, counted the scan complete, and printed `OK: no hits` at exit 0. That is
|
|
1213
|
+
// exactly the hole the two-predicate rule exists to close, arriving through configuration
|
|
1214
|
+
// instead of through code.
|
|
1215
|
+
//
|
|
1216
|
+
// So a staged path this filter admits and `isUnderScanRoot` does not is REFUSED. Narrowing it
|
|
1217
|
+
// silently to the intersection would be the wrong repair: it would hide a misconfiguration that
|
|
1218
|
+
// the author needs to see, in the one place the gate is a commit blocker.
|
|
1219
|
+
this.refuseUnscannable(
|
|
1220
|
+
staged
|
|
1221
|
+
.filter(
|
|
1222
|
+
(s) =>
|
|
1223
|
+
this.cfg.isStagedReadable(s.path) &&
|
|
1224
|
+
!this.isUnderScanRoot(s.path) &&
|
|
1225
|
+
!this.cfg.excludedPaths.has(s.path),
|
|
1226
|
+
)
|
|
1227
|
+
.map((s) => ({ path: s.path, kind: "outside every scan root" })),
|
|
1228
|
+
"`isStagedReadable` admits a path no scan root covers, so the checks that key on the ROOT " +
|
|
1229
|
+
"half of scope never ran for it: its bytes would be read without anything having " +
|
|
1230
|
+
"established that there are bytes there to read.",
|
|
1231
|
+
"Widen `scanRoots` to cover it, or narrow `isStagedReadable` to stay inside them.",
|
|
1232
|
+
{
|
|
1233
|
+
one: "staged path is readable but outside every scan root",
|
|
1234
|
+
many: "staged paths are readable but outside every scan root",
|
|
1235
|
+
},
|
|
1236
|
+
);
|
|
1237
|
+
|
|
1238
|
+
// THE MISCONFIGURATION REFUSAL ABOVE COMES FIRST ON PURPOSE. Both it and the two below can fire
|
|
1239
|
+
// over the same entry, and a developer whose `isStagedReadable` admits a path no root covers
|
|
1240
|
+
// needs to hear that rather than a sentence about this one entry's kind.
|
|
1241
|
+
this.refuseUnscannable(
|
|
1242
|
+
unmerged.filter(stagedReadable).map((s) => ({ path: s.path, kind: "no stage-0 blob" })),
|
|
1243
|
+
"An unmerged path has no stage-0 blob, so the index holds no single set of bytes a commit " +
|
|
1244
|
+
"would carry here, only the conflicting sides and, when there is one, their base.",
|
|
1245
|
+
"Resolve the conflict and stage the result, then re-run.",
|
|
1246
|
+
{ one: "staged path is unmerged", many: "staged paths are unmerged" },
|
|
1247
|
+
);
|
|
1248
|
+
|
|
1249
|
+
this.refuseUnscannable(
|
|
1250
|
+
staged
|
|
1251
|
+
.filter(
|
|
1252
|
+
(s) =>
|
|
1253
|
+
stagedReadable(s) && !unmerged.includes(s) && !this.cfg.regularBlobModes.has(s.mode),
|
|
1254
|
+
)
|
|
1255
|
+
.map((s) => ({ path: s.path, kind: gitModeKind(s.mode) })),
|
|
1256
|
+
"The index holds no file content for such an entry, so scanning it would prove nothing about " +
|
|
1257
|
+
"what it refers to.",
|
|
1258
|
+
"Unstage it, or replace it with a regular file.",
|
|
1259
|
+
);
|
|
1260
|
+
|
|
1261
|
+
// Every remaining readable record is a regular blob: anything non-regular was refused above,
|
|
1262
|
+
// anything unmerged was refused above it, and anything outside the roots before that.
|
|
1263
|
+
return staged
|
|
1264
|
+
.filter((s) => this.cfg.isStagedReadable(s.path) && !this.cfg.excludedPaths.has(s.path))
|
|
1265
|
+
.map((s) => s.path)
|
|
1266
|
+
.map((relPath) => ({
|
|
1267
|
+
path: relPath,
|
|
1268
|
+
// SECURITY: array-form execFileSync, no shell. `:<path>` is a git pathspec.
|
|
1269
|
+
read: () =>
|
|
1270
|
+
execFileSync("git", ["show", `:${relPath}`], {
|
|
1271
|
+
cwd: this.cfg.repoRoot,
|
|
1272
|
+
encoding: "buffer",
|
|
1273
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1274
|
+
}),
|
|
1275
|
+
}));
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
/**
|
|
1279
|
+
* The absolute path of a repo-relative one, for a question about the filesystem rather than
|
|
1280
|
+
* about the corpus. Only the TOCTOU re-check asks one.
|
|
1281
|
+
*
|
|
1282
|
+
* @param {string} relPath
|
|
1283
|
+
* @returns {string}
|
|
1284
|
+
*/
|
|
1285
|
+
absolutePath(relPath) {
|
|
1286
|
+
return relPath === "." ? this.cfg.repoRoot : join(this.cfg.repoRoot, ...relPath.split("/"));
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
/**
|
|
1290
|
+
* Did this root yield at least one file whose bytes were ACTUALLY READ?
|
|
1291
|
+
*
|
|
1292
|
+
* IT IS ASKED OF `read`, WHICH IS EVIDENCE OF OBSERVATION RATHER THAN A PLAN TO OBSERVE, and it
|
|
1293
|
+
* is asked per root rather than of the union, because the union is exactly what hides a starved
|
|
1294
|
+
* root: one productive root makes the whole sweep look productive.
|
|
1295
|
+
*
|
|
1296
|
+
* @param {string} root A normalised scan root.
|
|
1297
|
+
* @param {Set<string>} read The repo-relative paths this run opened.
|
|
1298
|
+
* @returns {boolean}
|
|
1299
|
+
*/
|
|
1300
|
+
anyReadUnder(root, read) {
|
|
1301
|
+
for (const p of read) {
|
|
1302
|
+
if (root === "." || p === root || p.startsWith(`${root}/`)) return true;
|
|
1303
|
+
}
|
|
1304
|
+
return false;
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
/**
|
|
1308
|
+
* Is this read failure the ENUMERATION TOCTOU WINDOW rather than a state the run must refuse?
|
|
1309
|
+
*
|
|
1310
|
+
* FOUR CONDITIONS, ALL OF THEM NECESSARY, because each one is a different question the run can
|
|
1311
|
+
* still answer honestly afterwards:
|
|
1312
|
+
*
|
|
1313
|
+
* 1. `all` MODE. The other two routes were asked for a path BY NAME - on argv, or by git's
|
|
1314
|
+
* own index - so a caller is owed an answer about that path rather than a shrug.
|
|
1315
|
+
* 2. THE WALK'S OWN TARGET, never the union's. The union reads an OBJECT by id, which cannot
|
|
1316
|
+
* go missing between enumeration and read.
|
|
1317
|
+
* 3. `ENOENT`, exactly. Every other errno says the entry is still there and something else
|
|
1318
|
+
* went wrong, which is the state a scan must never report past.
|
|
1319
|
+
* 4. UNTRACKED. Git carries bytes at a tracked path, so a tracked file that vanished still
|
|
1320
|
+
* leaves this run with content it declared it would read and did not.
|
|
1321
|
+
*
|
|
1322
|
+
* @param {unknown} err
|
|
1323
|
+
* @param {import("./phi-scan.js").Target} target
|
|
1324
|
+
* @param {Map<string, import("./phi-scan.js").IndexEntry> | null} index
|
|
1325
|
+
* @returns {boolean}
|
|
1326
|
+
*/
|
|
1327
|
+
isVanishedUntracked(err, target, index) {
|
|
1328
|
+
return (
|
|
1329
|
+
err instanceof TargetReadError &&
|
|
1330
|
+
err.errnoCode === "ENOENT" &&
|
|
1331
|
+
index !== null &&
|
|
1332
|
+
target.origin === undefined &&
|
|
1333
|
+
!index.has(target.path)
|
|
1334
|
+
);
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
/**
|
|
1338
|
+
* Refuse over entries the enumeration reached and cannot scan. EVERY offender IN THE GROUP is
|
|
1339
|
+
* named, not just the first: a developer who has to re-run the gate once per link learns to
|
|
1340
|
+
* distrust it.
|
|
1341
|
+
*
|
|
1342
|
+
* A refusal names the entry's own repo-relative path and an engine-owned token for its kind. IT
|
|
1343
|
+
* NEVER REPORTS A LINK TARGET, which is text off the working tree and can itself carry PHI.
|
|
1344
|
+
*
|
|
1345
|
+
* `noun` is overridable because the refusal must say something TRUE about what it refused: an
|
|
1346
|
+
* unmerged path is not a non-regular file, it is a path with no single blob.
|
|
1347
|
+
*
|
|
1348
|
+
* @param {import("./phi-scan.js").Unscannable[]} entries
|
|
1349
|
+
* @param {string} why
|
|
1350
|
+
* @param {string} remedy
|
|
1351
|
+
* @param {{ one: string, many: string }} [noun]
|
|
1352
|
+
*/
|
|
1353
|
+
refuseUnscannable(entries, why, remedy, noun) {
|
|
1354
|
+
const message = this.unscannableMessage(entries, why, remedy, noun);
|
|
1355
|
+
if (message !== null) throw new InvocationError(message);
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
/**
|
|
1359
|
+
* The refusal message for a group of unscannable entries, or `null` when the group is empty.
|
|
1360
|
+
*
|
|
1361
|
+
* SPLIT FROM THE THROW so a refusal can be DEFERRED rather than raised where it is computed. The
|
|
1362
|
+
* index route's two refusals are built before the sweep and raised after it, so the hits the walk
|
|
1363
|
+
* found are printed first; every other caller throws immediately, as before.
|
|
1364
|
+
*
|
|
1365
|
+
* @param {import("./phi-scan.js").Unscannable[]} entries
|
|
1366
|
+
* @param {string} why
|
|
1367
|
+
* @param {string} remedy
|
|
1368
|
+
* @param {{ one: string, many: string }} [noun]
|
|
1369
|
+
* @returns {string | null}
|
|
1370
|
+
*/
|
|
1371
|
+
unscannableMessage(
|
|
1372
|
+
entries,
|
|
1373
|
+
why,
|
|
1374
|
+
remedy,
|
|
1375
|
+
noun = { one: "entry is not a regular file", many: "entries are not regular files" },
|
|
1376
|
+
) {
|
|
1377
|
+
if (entries.length === 0) return null;
|
|
1378
|
+
const lines = entries.map((u) => ` - ${u.path} (${u.kind})`).join("\n");
|
|
1379
|
+
const phrase = entries.length === 1 ? noun.one : noun.many;
|
|
1380
|
+
return `refusing the scan: ${String(entries.length)} ${phrase}:\n${lines}\n${why} ${remedy}`;
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
// -------------------------------------------------------------------------
|
|
1384
|
+
// Detection
|
|
1385
|
+
// -------------------------------------------------------------------------
|
|
1386
|
+
|
|
1387
|
+
/**
|
|
1388
|
+
* The format-agnostic FLOOR: a dashed Social Security Number shape, and an email at a domain the
|
|
1389
|
+
* allow-list does not declare.
|
|
1390
|
+
*
|
|
1391
|
+
* BOTH BRANCHES CONSULT THE ALLOW-LIST, AND THE SSN BRANCH DOING SO IS A CORRECTION. A sibling's
|
|
1392
|
+
* refuter measured that its dashed-SSN branch consulted nothing, so the footer's claim that the
|
|
1393
|
+
* token allow-list "is the only remedy that reaches a clean run" was FALSE for exactly that
|
|
1394
|
+
* branch: a developer meeting it had no remedy at all, the bypass having been closed. Declaring an
|
|
1395
|
+
* identifier is a reviewed, committed act, which is the mechanism this gate is built on; a
|
|
1396
|
+
* detector that cannot be answered is a detector people route around.
|
|
1397
|
+
*
|
|
1398
|
+
* The declared form is matched BOTH as written and with its separators removed, so one `ID` entry
|
|
1399
|
+
* covers both renderings rather than requiring a repo to guess which one a fixture uses.
|
|
1400
|
+
*
|
|
1401
|
+
* THE MATCHED TOKEN IS A LOCAL, AND IT STAYS ONE. It decides whether the allow-list already
|
|
1402
|
+
* declares this identifier and is then dropped: the hit that leaves this function carries a
|
|
1403
|
+
* position and a rule, so there is no record for a later reporting path to print. See
|
|
1404
|
+
* `reportHits` for why that is structural rather than a call-site decision.
|
|
1405
|
+
*
|
|
1406
|
+
* @param {string} path The reported LOCUS.
|
|
1407
|
+
* @param {string} content
|
|
1408
|
+
* @param {import("./phi-scan.js").AllowList} allow
|
|
1409
|
+
* @param {import("./phi-scan.js").Hit[]} hits
|
|
1410
|
+
*/
|
|
1411
|
+
scanCommonShapes(path, content, allow, hits) {
|
|
1412
|
+
for (const m of content.matchAll(/\b\d{3}-\d{2}-\d{4}\b/g)) {
|
|
1413
|
+
const value = m[0];
|
|
1414
|
+
if (allow.ids.has(value.toUpperCase())) continue;
|
|
1415
|
+
if (allow.ids.has(value.replace(/\D/g, ""))) continue;
|
|
1416
|
+
hits.push({ path, segment: "(ssn)", reason: "dashed SSN pattern" });
|
|
1417
|
+
}
|
|
1418
|
+
for (const m of content.matchAll(/\b[A-Za-z0-9._%+-]+@([A-Za-z0-9.-]+\.[A-Za-z]{2,})\b/g)) {
|
|
1419
|
+
const domain = (m[1] ?? "").toLowerCase();
|
|
1420
|
+
if (!allow.emailDomains.has(domain)) {
|
|
1421
|
+
hits.push({ path, segment: "(email)", reason: "email with non-test domain" });
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
/**
|
|
1427
|
+
* Scan one target and RETURN THE BYTES IT OBSERVED. The bytes are returned rather than a boolean
|
|
1428
|
+
* so `all` mode can ask whether the walk already read exactly what the index carries at this path.
|
|
1429
|
+
*
|
|
1430
|
+
* THE CALLER'S DETECTOR IS HANDED THE LOCUS, NOT THE TARGET PATH, so a hit from the union half
|
|
1431
|
+
* cannot be reported against a bare path a developer would open and find clean. That used to be a
|
|
1432
|
+
* sentence in a comment; here it is the only path a caller can reach.
|
|
1433
|
+
*
|
|
1434
|
+
* A DETECTOR THAT THROWS REFUSES THE SCAN rather than escaping to node's own exit code. A
|
|
1435
|
+
* per-standard parser meeting input it cannot handle is an ordinary event, and the code node would
|
|
1436
|
+
* pick is the one this contract reserves for HITS FOUND.
|
|
1437
|
+
*
|
|
1438
|
+
* 🛑 THE REFUSAL PRINTS THE DETECTOR'S OWN MESSAGE VERBATIM, AND THAT IS A DISCLOSED RESIDUAL
|
|
1439
|
+
* RATHER THAN A CLOSED ONE. Everywhere else this engine prints only a repo-relative path and a
|
|
1440
|
+
* token from a closed set, precisely because a diagnostic ABOUT a PHI leak is itself a PHI
|
|
1441
|
+
* surface. A caller's error message is text this engine cannot vouch for, and a parser that
|
|
1442
|
+
* interpolates the record it choked on will put that record on stderr, which is CI logs. It is not
|
|
1443
|
+
* suppressed, because a refusal nobody can diagnose is its own defect, and it is not new: before
|
|
1444
|
+
* this engine existed the same string reached the same place through node's default handler. What
|
|
1445
|
+
* is new is that it is written down, here and on `DetectFn`. Throw a message that names the
|
|
1446
|
+
* position, never the content.
|
|
1447
|
+
*
|
|
1448
|
+
* @param {import("./phi-scan.js").Target} target
|
|
1449
|
+
* @param {import("./phi-scan.js").AllowList} allow
|
|
1450
|
+
* @param {import("./phi-scan.js").Hit[]} hits
|
|
1451
|
+
* @returns {Buffer}
|
|
1452
|
+
*/
|
|
1453
|
+
scanTarget(target, allow, hits) {
|
|
1454
|
+
let buf;
|
|
1455
|
+
try {
|
|
1456
|
+
buf = target.read();
|
|
1457
|
+
} catch (err) {
|
|
1458
|
+
throw new TargetReadError(
|
|
1459
|
+
`could not read ${target.path}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1460
|
+
err !== null && typeof err === "object" && "code" in err ? String(err.code) : undefined,
|
|
1461
|
+
);
|
|
1462
|
+
}
|
|
1463
|
+
const text = buf.toString("utf8");
|
|
1464
|
+
// Scope is decided on the target's own path; only the REPORTED locus carries the origin label,
|
|
1465
|
+
// so a labelled target is never a differently-scoped one.
|
|
1466
|
+
const locus = target.origin === undefined ? target.path : `${target.path} (${target.origin})`;
|
|
1467
|
+
// Recorded rather than re-derived from the locus string: the footer counts hits in bytes git
|
|
1468
|
+
// carries, and reading that back off the text of a path is a parse of the engine's own prose.
|
|
1469
|
+
if (target.origin !== undefined) this.indexLoci.add(locus);
|
|
1470
|
+
|
|
1471
|
+
this.scanCommonShapes(locus, text, allow, hits);
|
|
1472
|
+
|
|
1473
|
+
const detect = this.cfg.detect;
|
|
1474
|
+
if (detect !== undefined) {
|
|
1475
|
+
try {
|
|
1476
|
+
detect({
|
|
1477
|
+
path: locus,
|
|
1478
|
+
text,
|
|
1479
|
+
bytes: buf,
|
|
1480
|
+
allow,
|
|
1481
|
+
hit: (h) => {
|
|
1482
|
+
// THE LOCUS AND THE RULE ARE TAKEN; ANY `value` A DETECTOR PASSES IS DROPPED HERE. A
|
|
1483
|
+
// per-standard detector holds the record it parsed, so it is the caller most likely to
|
|
1484
|
+
// hand over a token that is real PHI, and the engine is the side that can refuse to
|
|
1485
|
+
// carry it. Dropping it at the boundary is what makes "no matched value reaches a
|
|
1486
|
+
// report" a property of this engine rather than of every detector's discipline.
|
|
1487
|
+
hits.push({ path: locus, segment: h.segment, reason: h.reason });
|
|
1488
|
+
},
|
|
1489
|
+
});
|
|
1490
|
+
} catch (err) {
|
|
1491
|
+
throw new InvocationError(
|
|
1492
|
+
`the field detector threw on ${locus}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1493
|
+
);
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
return buf;
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
// -------------------------------------------------------------------------
|
|
1501
|
+
// Reporting
|
|
1502
|
+
// -------------------------------------------------------------------------
|
|
1503
|
+
|
|
1504
|
+
/**
|
|
1505
|
+
* Print the hits.
|
|
1506
|
+
*
|
|
1507
|
+
* SPLIT FROM THE CLEAN LINE ON PURPOSE. Every refusal that can follow a read prints the hits
|
|
1508
|
+
* FIRST, so a run that is both incomplete and carrying hits prints both. The clean line is printed
|
|
1509
|
+
* only once every tier has passed, so it can never appear beside a refusal.
|
|
1510
|
+
*
|
|
1511
|
+
* IT NAMES A POSITION AND STOPS THERE: the repo-relative locus, the locator inside it, and the
|
|
1512
|
+
* rule that fired. THE MATCHED TOKEN IS NEVER PRINTED. This is the one surface that would have
|
|
1513
|
+
* the token in hand, which is exactly why the rule is stated on it: stderr is a CI log, a CI log
|
|
1514
|
+
* is a transcript nobody retains deliberately, and a diagnostic ABOUT a PHI leak that quotes the
|
|
1515
|
+
* leak is a second copy of it in a worse place (`phi-safety` P4, `observability` B1). Nothing is
|
|
1516
|
+
* redacted here either, because the value never reaches this function: a seam a call site has to
|
|
1517
|
+
* remember is a seam the next call site forgets.
|
|
1518
|
+
*
|
|
1519
|
+
* WHAT A DEVELOPER DOES WITH A HIT THEY CANNOT READ THE VALUE OF, since an error owes an action
|
|
1520
|
+
* (`observability` B2): open the named path, find the shape the named rule describes, and either
|
|
1521
|
+
* remove it or declare it in the allow-list the footer names if it is genuinely synthetic. The
|
|
1522
|
+
* position is what locates the token; the token itself was never the remedy.
|
|
1523
|
+
*
|
|
1524
|
+
* THE FOOTER IS SCOPED TO WHAT THIS ENGINE KNOWS. It does not claim the allow-list reaches a clean
|
|
1525
|
+
* run for every hit, because a per-repo detector supplied through `detect` may raise one without
|
|
1526
|
+
* consulting the allow-list at all, and a sibling shipped exactly that claim and had it refuted.
|
|
1527
|
+
* What is true, and all that is said: the two branches of the cross-cutting floor consult it, a
|
|
1528
|
+
* whole-file bypass is recorded and then refused, and a detector that consults nothing has to be
|
|
1529
|
+
* changed rather than argued with.
|
|
1530
|
+
*
|
|
1531
|
+
* @param {import("./phi-scan.js").Hit[]} hits
|
|
1532
|
+
*/
|
|
1533
|
+
reportHits(hits) {
|
|
1534
|
+
if (hits.length === 0) return;
|
|
1535
|
+
/** @type {Map<string, import("./phi-scan.js").Hit[]>} */
|
|
1536
|
+
const byPath = new Map();
|
|
1537
|
+
for (const h of hits) {
|
|
1538
|
+
const arr = byPath.get(h.path);
|
|
1539
|
+
if (arr) arr.push(h);
|
|
1540
|
+
else byPath.set(h.path, [h]);
|
|
1541
|
+
}
|
|
1542
|
+
for (const [path, group] of byPath) {
|
|
1543
|
+
process.stderr.write(`[phi-scan] HIT: ${path}\n`);
|
|
1544
|
+
for (const h of group) {
|
|
1545
|
+
process.stderr.write(` segment=${h.segment} (${h.reason})\n`);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
// THE PER-ORIGIN FOOTER, AND IT IS A REMEDY RATHER THAN A STATISTIC (AC-14). A hit in bytes git
|
|
1549
|
+
// carries is not answered by editing the file on disk: the identifier is in the index, so
|
|
1550
|
+
// re-staging (or removing the blob) is part of the fix, and a reader counting `HIT:` lines has
|
|
1551
|
+
// no way to tell how many of them are in that state.
|
|
1552
|
+
const fromIndex = hits.filter((h) => this.indexLoci.has(h.path)).length;
|
|
1553
|
+
if (fromIndex > 0) {
|
|
1554
|
+
process.stderr.write(
|
|
1555
|
+
`[phi-scan] ${String(fromIndex)} of those are in bytes git carries at that path, not in ` +
|
|
1556
|
+
`the file on disk: fix the content and RE-STAGE it, or the next sweep reports the same ` +
|
|
1557
|
+
`hit over a working tree that reads clean.\n`,
|
|
1558
|
+
);
|
|
1559
|
+
}
|
|
1560
|
+
process.stderr.write(
|
|
1561
|
+
`[phi-scan] ${String(hits.length)} hit(s) across ${String(byPath.size)} file(s). ` +
|
|
1562
|
+
`The cross-cutting floor (SSN and email shapes) consults ${this.relAllowList()}, so a ` +
|
|
1563
|
+
`genuinely synthetic identifier is declared there: a token-level, reviewed declaration. A ` +
|
|
1564
|
+
`hit raised by this repo's own field detectors is answerable that way only if that detector ` +
|
|
1565
|
+
`consults the allow-list; one that does not has to be changed rather than declared around. ` +
|
|
1566
|
+
`A whole-file --allow-fixture bypass is recorded and then REFUSED, because a scan that ` +
|
|
1567
|
+
`never opened a file has no clean verdict to give about it.\n`,
|
|
1568
|
+
);
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
// -------------------------------------------------------------------------
|
|
1572
|
+
// The run
|
|
1573
|
+
// -------------------------------------------------------------------------
|
|
1574
|
+
|
|
1575
|
+
/**
|
|
1576
|
+
* ===========================================================================================
|
|
1577
|
+
* THE EXIT CONTRACT IS THE CALLER'S, NOT THIS FILE'S. The three codes come from `exitCodes`, and
|
|
1578
|
+
* their meanings are:
|
|
1579
|
+
*
|
|
1580
|
+
* clean the scan ran, READ EVERY TARGET IT ENUMERATED, and found nothing. THE ONE
|
|
1581
|
+
* SUBTRACTION IS THE ENUMERATION TOCTOU WINDOW, and it is bounded rather than
|
|
1582
|
+
* described: an UNTRACKED file the walk itself listed, in `all` mode, whose read
|
|
1583
|
+
* failed with `ENOENT`, and which is still absent at the end of the run, is reported
|
|
1584
|
+
* SKIPPED and does not hold the clean code back. Git carries no bytes at such a path,
|
|
1585
|
+
* so nothing this repository holds went unread. Every other shape of the same failure
|
|
1586
|
+
* is in `refuse` below.
|
|
1587
|
+
* hits this corpus contains something that looks like PHI. Nothing this engine RAISES ever
|
|
1588
|
+
* takes it. It is NOT exclusive, and the escapes are named rather than left to be
|
|
1589
|
+
* discovered: an allow-list or an override log that EXISTS but cannot be READ throws a
|
|
1590
|
+
* plain `Error`, which is rethrown, and the run takes node's own exit 1.
|
|
1591
|
+
* refuse every state this engine RAISES in which the scan cannot account for something: a bad
|
|
1592
|
+
* argument, a MISSING allow-list, an unlogged bypass, a bypass naming a path this run
|
|
1593
|
+
* does not enumerate, an in-scope entry that is not a regular file, an unparseable
|
|
1594
|
+
* `git diff --cached` record, an index git cannot name or names empty, an in-scope index
|
|
1595
|
+
* entry that is not a regular blob, an in-scope path with no stage-0 blob (unmerged), a
|
|
1596
|
+
* target whose bytes cannot be read, a field detector that threw, a target enumerated
|
|
1597
|
+
* but never read, a target that was absent at read time and PRESENT AGAIN at the end
|
|
1598
|
+
* of the run, and (`all` mode) a SCAN ROOT THAT YIELDED NO FILE THAT WAS READ.
|
|
1599
|
+
* ===========================================================================================
|
|
1600
|
+
* THE PER-ROOT OBSERVATION RULE, `all` MODE ONLY. The completeness rule asks whether every
|
|
1601
|
+
* target was read. It cannot ask whether a root produced a target at all, and a root that
|
|
1602
|
+
* produces none is a sweep that is narrower than its own configuration says: a typo, a directory
|
|
1603
|
+
* that moved or was never created, a root whose every file the read filter drops. ONE
|
|
1604
|
+
* PRODUCTIVE ROOT MAKES THE WHOLE RUN LOOK PRODUCTIVE, which is why the question is asked per
|
|
1605
|
+
* root and answered from the set of paths actually READ, and why the refusal names every starved
|
|
1606
|
+
* root rather than the first.
|
|
1607
|
+
*
|
|
1608
|
+
* IT APPLIES TO A MISSING ROOT TOO, AND THAT DOES NOT CONTRADICT "A MISSING ROOT IS SKIPPED".
|
|
1609
|
+
* The walk still skips it: it is not reported as an unscannable KIND and it does not crash the
|
|
1610
|
+
* enumeration, which is what a missing root used to do. What it no longer does is contribute
|
|
1611
|
+
* nothing in silence.
|
|
1612
|
+
* ===========================================================================================
|
|
1613
|
+
*
|
|
1614
|
+
* @returns {number}
|
|
1615
|
+
*/
|
|
1616
|
+
run() {
|
|
1617
|
+
const { clean: EXIT_CLEAN, hits: EXIT_HITS, refuse: EXIT_REFUSE } = this.cfg.exitCodes;
|
|
1618
|
+
|
|
1619
|
+
/** @type {{ mode: "all" | "staged" | "paths", paths: string[], allowFixtures: string[] }} */
|
|
1620
|
+
let args;
|
|
1621
|
+
try {
|
|
1622
|
+
args = this.parseArgs(this.cfg.argv);
|
|
1623
|
+
this.validateAllowFixtures(args.allowFixtures);
|
|
1624
|
+
} catch (err) {
|
|
1625
|
+
if (err instanceof InvocationError) {
|
|
1626
|
+
process.stderr.write(`[phi-scan] ${err.message}\n`);
|
|
1627
|
+
return EXIT_REFUSE;
|
|
1628
|
+
}
|
|
1629
|
+
throw err;
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
const allowed = new Set(args.allowFixtures.map((p) => this.normalizePath(p)));
|
|
1633
|
+
|
|
1634
|
+
/** @type {import("./phi-scan.js").AllowList} */
|
|
1635
|
+
let allow;
|
|
1636
|
+
/** @type {import("./phi-scan.js").Target[]} */
|
|
1637
|
+
let targets;
|
|
1638
|
+
/** `all` mode's index, read once: it is the union half's whole enumeration. */
|
|
1639
|
+
/** @type {Map<string, import("./phi-scan.js").IndexEntry> | null} */
|
|
1640
|
+
let index = null;
|
|
1641
|
+
/** The index route's refusals, raised AFTER the walk sweep so no hit is swallowed. */
|
|
1642
|
+
/** @type {string[]} */
|
|
1643
|
+
let indexRefusals = [];
|
|
1644
|
+
try {
|
|
1645
|
+
// `loadAllowList()` IS INSIDE THIS HANDLER, AND THAT PLACEMENT IS THE POINT. Outside it, a
|
|
1646
|
+
// missing allow-list escaped as an uncaught throw and took node's exit 1, which this contract
|
|
1647
|
+
// reserves for "hits found".
|
|
1648
|
+
allow = this.loadAllowList();
|
|
1649
|
+
if (args.mode === "staged") targets = this.buildTargetsForStaged();
|
|
1650
|
+
else if (args.mode === "paths") targets = this.buildTargetsForPaths(args.paths);
|
|
1651
|
+
else {
|
|
1652
|
+
const built = this.buildTargetsForAll();
|
|
1653
|
+
targets = built.targets;
|
|
1654
|
+
index = built.index;
|
|
1655
|
+
indexRefusals = built.indexRefusals;
|
|
1656
|
+
}
|
|
1657
|
+
} catch (err) {
|
|
1658
|
+
if (err instanceof InvocationError) {
|
|
1659
|
+
process.stderr.write(`[phi-scan] ${err.message}\n`);
|
|
1660
|
+
return EXIT_REFUSE;
|
|
1661
|
+
}
|
|
1662
|
+
throw err;
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
// ENUMERATED: the set of paths this run declared it would read. Everything the read filters
|
|
1666
|
+
// dropped upstream never became a target and is not in here, which is why the completeness rule
|
|
1667
|
+
// does not fire on them. In `all` mode it is the walk's targets UNION the in-scope tracked paths.
|
|
1668
|
+
const enumerated = new Set(targets.map((t) => t.path));
|
|
1669
|
+
if (index !== null) for (const p of this.unionCandidatePaths(index)) enumerated.add(p);
|
|
1670
|
+
// IN `all` MODE A BYPASS IS ITSELF AN ENUMERATION (AC-17). The run's declared scope is the whole
|
|
1671
|
+
// corpus, so naming a path in `--allow-fixture` says "this run would have read that", and the
|
|
1672
|
+
// honest tier for withdrawing it is the completeness rule rather than the unmatched one. It
|
|
1673
|
+
// holds whatever the sweep happens to reach: a path the walk never listed and the index never
|
|
1674
|
+
// carried is still refused HERE, naming it, instead of being reported as subtracting nothing.
|
|
1675
|
+
if (args.mode === "all") for (const p of allowed) enumerated.add(p);
|
|
1676
|
+
|
|
1677
|
+
// TIER: A BYPASS MUST NAME A PATH THIS RUN ENUMERATES. Otherwise it subtracts nothing, and a
|
|
1678
|
+
// flag that subtracts nothing lets a developer believe a file was acknowledged when the run
|
|
1679
|
+
// never had it in scope. Compared by DIFFERENCE, and every offender is named. In `all` mode the
|
|
1680
|
+
// line above makes this tier unreachable, deliberately: there it is the completeness rule's job.
|
|
1681
|
+
//
|
|
1682
|
+
// THIS TIER FIRES BEFORE ANY TARGET IS READ, so no hit exists for it to swallow. That is a
|
|
1683
|
+
// narrower guarantee than the unread tier's and is stated as such rather than generalised.
|
|
1684
|
+
const unmatched = [...allowed].filter((p) => !enumerated.has(p));
|
|
1685
|
+
if (unmatched.length > 0) {
|
|
1686
|
+
process.stderr.write(
|
|
1687
|
+
`[phi-scan] --allow-fixture names ${String(unmatched.length)} path(s) this run does not ` +
|
|
1688
|
+
`enumerate, so the flag subtracts nothing:\n${unmatched.map((p) => ` - ${p}`).join("\n")}\n` +
|
|
1689
|
+
`Scan a corpus that contains the path, or drop the flag.\n`,
|
|
1690
|
+
);
|
|
1691
|
+
return EXIT_REFUSE;
|
|
1692
|
+
}
|
|
1693
|
+
|
|
1694
|
+
/** @type {import("./phi-scan.js").Hit[]} */
|
|
1695
|
+
const hits = [];
|
|
1696
|
+
// READ: filled in only after a target's bytes have actually been through `scanTarget`. This is
|
|
1697
|
+
// evidence of observation, never a plan to observe.
|
|
1698
|
+
/** @type {Set<string>} */
|
|
1699
|
+
const read = new Set();
|
|
1700
|
+
/** Path -> object id of the bytes the walk actually read, so the union can skip a re-scan. */
|
|
1701
|
+
/** @type {Map<string, string>} */
|
|
1702
|
+
const readOids = new Map();
|
|
1703
|
+
const objectHash = index === null ? null : this.gitObjectHash();
|
|
1704
|
+
/**
|
|
1705
|
+
* Paths the walk listed and that were GONE by read time. WHAT THIS MEANS IS DECIDED AFTER THE
|
|
1706
|
+
* SWEEP, not here: see the resolution below for why the window is left open as long as the
|
|
1707
|
+
* run lasts.
|
|
1708
|
+
*/
|
|
1709
|
+
/** @type {Set<string>} */
|
|
1710
|
+
const vanished = new Set();
|
|
1711
|
+
|
|
1712
|
+
/**
|
|
1713
|
+
* @param {import("./phi-scan.js").Target[]} batch
|
|
1714
|
+
* @returns {number | null} A refusal code, or `null` when the batch completed.
|
|
1715
|
+
*/
|
|
1716
|
+
const sweep = (batch) => {
|
|
1717
|
+
for (const t of batch) {
|
|
1718
|
+
if (allowed.has(t.path)) continue;
|
|
1719
|
+
let bytes;
|
|
1720
|
+
try {
|
|
1721
|
+
bytes = this.scanTarget(t, allow, hits);
|
|
1722
|
+
} catch (err) {
|
|
1723
|
+
// THE ENUMERATION TOCTOU WINDOW: the ONE read failure that is not settled here. It is
|
|
1724
|
+
// recorded and judged after the sweep, because whether the path came back is part of
|
|
1725
|
+
// the answer and this is the earliest the run could possibly know.
|
|
1726
|
+
if (this.isVanishedUntracked(err, t, index)) {
|
|
1727
|
+
vanished.add(t.path);
|
|
1728
|
+
continue;
|
|
1729
|
+
}
|
|
1730
|
+
if (err instanceof InvocationError) {
|
|
1731
|
+
// HITS FOUND SO FAR ARE PRINTED BEFORE THIS REFUSAL, DELIBERATELY, AND THIS IS A CHANGE
|
|
1732
|
+
// FROM THE COPIED SCANNERS. A refuter measured the old ordering: a fatal partway through
|
|
1733
|
+
// the sweep discarded every hit found before it, so a consumer saw a refusal with no
|
|
1734
|
+
// indication that PHI had already been found. The refusal still wins the exit code, and
|
|
1735
|
+
// the clean line is still unreachable from here, so nothing is reported as accounted for
|
|
1736
|
+
// that is not; what changes is that a finding already made is not thrown away.
|
|
1737
|
+
this.reportHits(hits);
|
|
1738
|
+
process.stderr.write(`[phi-scan] ${err.message}\n`);
|
|
1739
|
+
return EXIT_REFUSE;
|
|
1740
|
+
}
|
|
1741
|
+
throw err;
|
|
1742
|
+
}
|
|
1743
|
+
read.add(t.path);
|
|
1744
|
+
if (objectHash !== null && t.origin === undefined) {
|
|
1745
|
+
const oid = this.blobOid(objectHash, bytes);
|
|
1746
|
+
if (oid !== null) readOids.set(t.path, oid);
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
return null;
|
|
1750
|
+
};
|
|
1751
|
+
|
|
1752
|
+
const walkFailure = sweep(targets);
|
|
1753
|
+
if (walkFailure !== null) return walkFailure;
|
|
1754
|
+
|
|
1755
|
+
// THE INDEX ROUTE'S OWN REFUSALS, RAISED HERE RATHER THAN WHERE THEY ARE COMPUTED (AC-15). An
|
|
1756
|
+
// unmerged entry or a tracked link anywhere in the index refuses the whole sweep, and the walk
|
|
1757
|
+
// may have found PHI under a root that yielded perfectly well. Refusing at enumeration time
|
|
1758
|
+
// discarded those findings: measured, exit 2 with the refusal alone over a repository whose walk
|
|
1759
|
+
// had five hits in hand. The exit code is still the refusal's, because an incomplete sweep is
|
|
1760
|
+
// not a verdict whatever it found.
|
|
1761
|
+
if (indexRefusals.length > 0) {
|
|
1762
|
+
this.reportHits(hits);
|
|
1763
|
+
for (const message of indexRefusals) process.stderr.write(`[phi-scan] ${message}\n`);
|
|
1764
|
+
return EXIT_REFUSE;
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
// THE UNION. It runs AFTER the walk, not instead of it, and only over the paths the walk did not
|
|
1768
|
+
// already read verbatim.
|
|
1769
|
+
if (index !== null) {
|
|
1770
|
+
const unionFailure = sweep(this.buildTargetsForGitIndex(index, readOids));
|
|
1771
|
+
if (unionFailure !== null) return unionFailure;
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
// THE TOCTOU WINDOW IS RESOLVED AT THE END OF THE RUN, NOT AT THE MOMENT OF THE FAILURE, AND
|
|
1775
|
+
// THE LATE ANSWER IS THE STRONGER ONE. A path that was absent when the sweep reached it and is
|
|
1776
|
+
// present now was not a file being deleted: something wrote there while this run was looking
|
|
1777
|
+
// away, and no verdict this run can give covers the bytes that lived at it in between. Asking
|
|
1778
|
+
// immediately would answer within microseconds of the failure and miss almost all of that.
|
|
1779
|
+
/** @type {string[]} */
|
|
1780
|
+
const reappeared = [];
|
|
1781
|
+
/** @type {Set<string>} */
|
|
1782
|
+
const skipped = new Set();
|
|
1783
|
+
for (const p of vanished) {
|
|
1784
|
+
if (lstatOrNull(this.absolutePath(p)) === null) skipped.add(p);
|
|
1785
|
+
else reappeared.push(p);
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
// THE COMPLETENESS RULE. A SET DIFFERENCE, NEVER A SIZE COMPARISON: a count counts the targets
|
|
1789
|
+
// that DID get read, so `n read of n targets` is exactly the arithmetic that hides which ones did
|
|
1790
|
+
// not. Names every offender.
|
|
1791
|
+
//
|
|
1792
|
+
// THE SKIPPED SET IS THE ONE SUBTRACTION, AND IT IS BOUNDED BY `isVanishedUntracked` RATHER
|
|
1793
|
+
// THAN BY A JUDGEMENT CALL: `all` mode, the walk's own target, `ENOENT`, untracked, and still
|
|
1794
|
+
// absent at the end of the run. Nothing git carries can enter it, so the rule keeps its whole
|
|
1795
|
+
// meaning over the repository and gives up only what the filesystem took away mid-run.
|
|
1796
|
+
const unread = [...enumerated].filter((p) => !read.has(p) && !skipped.has(p));
|
|
1797
|
+
|
|
1798
|
+
// THE PER-ROOT OBSERVATION RULE, `all` MODE ONLY. The completeness rule asks whether every
|
|
1799
|
+
// target was read; it cannot ask whether a root produced a target in the first place, and a
|
|
1800
|
+
// root that produces none is the silently-narrowed sweep: a typo, a directory that moved, a
|
|
1801
|
+
// root whose every file the read filter drops. One productive root makes the whole run look
|
|
1802
|
+
// productive, so the question is asked per root and answered from `read`.
|
|
1803
|
+
const starved =
|
|
1804
|
+
index === null ? [] : this.cfg.scanRoots.filter((root) => !this.anyReadUnder(root, read));
|
|
1805
|
+
|
|
1806
|
+
// Hits FIRST, so nothing below can swallow one.
|
|
1807
|
+
this.reportHits(hits);
|
|
1808
|
+
|
|
1809
|
+
if (skipped.size > 0) {
|
|
1810
|
+
process.stderr.write(
|
|
1811
|
+
`[phi-scan] skipped ${String(skipped.size)} untracked file(s) gone between enumeration ` +
|
|
1812
|
+
`and read:\n${[...skipped].map((p) => ` - ${p}`).join("\n")}\n` +
|
|
1813
|
+
`Nothing this repository carries was left unread: git has no bytes at such a path. ` +
|
|
1814
|
+
`Re-run the scan if the file was meant to stay.\n`,
|
|
1815
|
+
);
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
if (reappeared.length > 0) {
|
|
1819
|
+
process.stderr.write(
|
|
1820
|
+
`[phi-scan] refusing the scan: ${String(reappeared.length)} target(s) were absent when ` +
|
|
1821
|
+
`the sweep reached them and present again at the end of the run:\n` +
|
|
1822
|
+
`${reappeared.map((p) => ` - ${p}`).join("\n")}\n` +
|
|
1823
|
+
`A path that came back carried bytes this run never opened, so the scan has no verdict ` +
|
|
1824
|
+
`about it. Re-run the scan on a tree nothing else is writing to.\n`,
|
|
1825
|
+
);
|
|
1826
|
+
return EXIT_REFUSE;
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
if (unread.length > 0) {
|
|
1830
|
+
process.stderr.write(
|
|
1831
|
+
`[phi-scan] refusing the scan: ${String(unread.length)} target(s) were enumerated and ` +
|
|
1832
|
+
`never read:\n${unread.map((p) => ` - ${p}`).join("\n")}\n` +
|
|
1833
|
+
`A scan that did not open a file has no clean verdict to give about it. If the file is ` +
|
|
1834
|
+
`genuinely synthetic, declare its identifiers in ${this.relAllowList()} rather than ` +
|
|
1835
|
+
`withdrawing the file from the scan.\n`,
|
|
1836
|
+
);
|
|
1837
|
+
return EXIT_REFUSE;
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
if (starved.length > 0) {
|
|
1841
|
+
process.stderr.write(
|
|
1842
|
+
`[phi-scan] refusing the sweep: the all-mode sweep observed no files under ` +
|
|
1843
|
+
`${String(starved.length)} of its ${String(this.cfg.scanRoots.length)} scan roots ` +
|
|
1844
|
+
`(${starved.join(", ")}).\n` +
|
|
1845
|
+
`A root that contributes nothing leaves this sweep covering less than its own ` +
|
|
1846
|
+
`configuration says, and no other tier can see that: the index route reads what git ` +
|
|
1847
|
+
`carries and cannot vouch for a directory on disk. Point the root at a directory or ` +
|
|
1848
|
+
`a file this run can read, widen the read filter that drops its contents, or drop the ` +
|
|
1849
|
+
`root.\n`,
|
|
1850
|
+
);
|
|
1851
|
+
return EXIT_REFUSE;
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
if (hits.length > 0) return EXIT_HITS;
|
|
1855
|
+
// THE CLEAN LINE CARRIES THE SUBTRACTION (AC-13). A reader watching stdout alone must not be
|
|
1856
|
+
// able to take an unqualified OK from a sweep that skipped a file: the skip is on stderr, and
|
|
1857
|
+
// stderr is where a CI log buries it. The unqualified line is byte-for-byte what it was when
|
|
1858
|
+
// nothing was skipped, so a consumer matching it exactly still matches.
|
|
1859
|
+
process.stdout.write(
|
|
1860
|
+
skipped.size > 0
|
|
1861
|
+
? `[phi-scan] OK: no hits (${String(skipped.size)} untracked file(s) skipped, see stderr)\n`
|
|
1862
|
+
: "[phi-scan] OK: no hits\n",
|
|
1863
|
+
);
|
|
1864
|
+
return EXIT_CLEAN;
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1868
|
+
/**
|
|
1869
|
+
* `:<srcmode> <dstmode> <srcsha> <dstsha> <status>`: the info half of a `--raw -z` record.
|
|
1870
|
+
*
|
|
1871
|
+
* THE STATUS LETTER IS CAPTURED, not merely matched, because `U` (unmerged) is the one status whose
|
|
1872
|
+
* refusal has to say something different: its destination mode is `000000`, and calling that "a git
|
|
1873
|
+
* mode-000000 entry" beside a sentence about content git holds at the path is false twice over for a
|
|
1874
|
+
* conflicted regular file.
|
|
1875
|
+
*/
|
|
1876
|
+
const RAW_RECORD = /^:(?:\d{6}) (\d{6}) [0-9a-f]+ [0-9a-f]+ ([A-Z])\d*$/;
|
|
1877
|
+
|
|
1878
|
+
/**
|
|
1879
|
+
* `lstatSync` that reports a missing path as `null` rather than throwing. LSTAT, not stat, so a
|
|
1880
|
+
* symbolic link named as a scan root is seen as a link rather than as whatever it points at: the
|
|
1881
|
+
* walk refuses such an entry everywhere else and a root must not be the one place it follows one.
|
|
1882
|
+
*
|
|
1883
|
+
* @param {string} path
|
|
1884
|
+
* @returns {import("node:fs").Stats | null}
|
|
1885
|
+
*/
|
|
1886
|
+
function lstatOrNull(path) {
|
|
1887
|
+
try {
|
|
1888
|
+
return lstatSync(path);
|
|
1889
|
+
} catch {
|
|
1890
|
+
return null;
|
|
1891
|
+
}
|
|
1892
|
+
}
|
|
1893
|
+
|
|
1894
|
+
/**
|
|
1895
|
+
* Closed-set, engine-owned description of a `Stats` kind, for a scan root that is neither a
|
|
1896
|
+
* directory nor a regular file. Same closed set as `direntKind`, off the same predicates.
|
|
1897
|
+
*
|
|
1898
|
+
* @param {import("node:fs").Stats} s
|
|
1899
|
+
* @returns {string}
|
|
1900
|
+
*/
|
|
1901
|
+
function statsKind(s) {
|
|
1902
|
+
if (s.isSymbolicLink()) return "a symbolic link";
|
|
1903
|
+
if (s.isFIFO()) return "a FIFO";
|
|
1904
|
+
if (s.isSocket()) return "a socket";
|
|
1905
|
+
if (s.isBlockDevice()) return "a block device";
|
|
1906
|
+
if (s.isCharacterDevice()) return "a character device";
|
|
1907
|
+
return "not a regular file";
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
/**
|
|
1911
|
+
* Closed-set, engine-owned description of a directory entry's kind. Nothing off the other side of a
|
|
1912
|
+
* link is ever recorded.
|
|
1913
|
+
*
|
|
1914
|
+
* @param {import("node:fs").Dirent} e
|
|
1915
|
+
* @returns {string}
|
|
1916
|
+
*/
|
|
1917
|
+
function direntKind(e) {
|
|
1918
|
+
if (e.isSymbolicLink()) return "a symbolic link";
|
|
1919
|
+
if (e.isFIFO()) return "a FIFO";
|
|
1920
|
+
if (e.isSocket()) return "a socket";
|
|
1921
|
+
if (e.isBlockDevice()) return "a block device";
|
|
1922
|
+
if (e.isCharacterDevice()) return "a character device";
|
|
1923
|
+
return "not a regular file";
|
|
1924
|
+
}
|
|
1925
|
+
|
|
1926
|
+
/**
|
|
1927
|
+
* Closed-set, engine-owned description of a git file mode.
|
|
1928
|
+
*
|
|
1929
|
+
* @param {string} mode
|
|
1930
|
+
* @returns {string}
|
|
1931
|
+
*/
|
|
1932
|
+
function gitModeKind(mode) {
|
|
1933
|
+
if (mode === "120000") return "a symbolic link";
|
|
1934
|
+
if (mode === "160000") return "a gitlink (a nested repository)";
|
|
1935
|
+
return `a git mode-${mode} entry`;
|
|
1936
|
+
}
|