@cosyte/script-utils 0.0.2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -2
- package/README.md +365 -30
- package/attw.d.ts +36 -0
- package/attw.js +1679 -0
- package/check-no-emdash.sh +874 -0
- package/internal-refs.d.ts +223 -0
- package/internal-refs.js +1363 -0
- package/package.json +21 -1
- package/phi-scan.d.ts +59 -12
- package/phi-scan.js +500 -107
package/phi-scan.js
CHANGED
|
@@ -59,6 +59,27 @@ class InvocationError extends Error {
|
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
+
/**
|
|
63
|
+
* A target whose bytes could not be read, carrying the errno the filesystem gave.
|
|
64
|
+
*
|
|
65
|
+
* IT IS AN `InvocationError`, SO THE DEFAULT IS STILL REFUSAL. The subclass exists for exactly one
|
|
66
|
+
* decision, the ENUMERATION TOCTOU WINDOW in `run`: between the walk listing a path and the sweep
|
|
67
|
+
* opening it, an editor, a build or a test can remove it. The errno is what tells "the file is
|
|
68
|
+
* gone" from "the file is there and something else failed", and nothing else in this engine may
|
|
69
|
+
* branch on it. Every other read failure keeps taking the refuse path unchanged.
|
|
70
|
+
*/
|
|
71
|
+
class TargetReadError extends InvocationError {
|
|
72
|
+
/**
|
|
73
|
+
* @param {string} message
|
|
74
|
+
* @param {string | undefined} errnoCode The `code` the filesystem error carried, when it had one.
|
|
75
|
+
*/
|
|
76
|
+
constructor(message, errnoCode) {
|
|
77
|
+
super(message);
|
|
78
|
+
this.name = "TargetReadError";
|
|
79
|
+
this.errnoCode = errnoCode;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
62
83
|
/** git's file modes for a regular blob. Every other mode names something with no bytes to read. */
|
|
63
84
|
const DEFAULT_REGULAR_BLOB_MODES = new Set(["100644", "100755"]);
|
|
64
85
|
|
|
@@ -282,13 +303,19 @@ function normalizeConfig(config) {
|
|
|
282
303
|
* `git rev-parse --show-object-format` per `all`-mode run, because the deduplication needs the
|
|
283
304
|
* algorithm before it can compare anything. Where the two copies of a path DIFFER, BOTH are scanned.
|
|
284
305
|
*
|
|
285
|
-
* A HIT FROM THE UNION IS LABELLED `(
|
|
286
|
-
*
|
|
306
|
+
* A HIT FROM THE UNION IS LABELLED `(git index)` OR `(git index; the working tree differs)`, ON THE
|
|
307
|
+
* REPORTED LOCUS ONLY, and a footer counts how many of the reported hits came from those bytes. A
|
|
308
|
+
* hit naming the bare path would send a developer to open a file that is clean, or not there at all,
|
|
309
|
+
* and a single label would hide that re-staging is part of the remedy for the second kind. The
|
|
287
310
|
* target's `path` stays undecorated because every filter, exclusion and completeness tier keys on it.
|
|
288
311
|
* The `detect` callback is handed the LOCUS as `path`, so a caller cannot get this wrong.
|
|
289
312
|
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
313
|
+
* THE INDEX HALF IS NOT NARROWED BY `scanRoots`: every path the index carries is considered, because
|
|
314
|
+
* a root list is the WALK's scope and not a statement about what the repository carries. `--staged`
|
|
315
|
+
* is untouched by that and keeps its own scope, which is what a COMMIT is blocked on.
|
|
316
|
+
*
|
|
317
|
+
* `all` MODE REFUSES WHEN GIT CANNOT READ THE INDEX, OR READS IT EMPTY, and the two arrive through
|
|
318
|
+
* DIFFERENT branches and carry DIFFERENT sentences, because the remedies differ. Measured on git 2.39.5: a directory that is no repository at all FATALS (exit
|
|
292
319
|
* 128), so the `catch` is what turns it into a refusal, and WITHOUT that catch the throw escapes and
|
|
293
320
|
* the run takes node's own exit 1, which this contract reserves for HITS FOUND. A repository whose
|
|
294
321
|
* index is empty, and a directory inside a repository with nothing tracked under it, both print
|
|
@@ -317,6 +344,9 @@ class PhiScan {
|
|
|
317
344
|
this.cfg = cfg;
|
|
318
345
|
/** Does any scan root name the repository root itself? Then everything is in scope. */
|
|
319
346
|
this.wholeRepo = cfg.scanRoots.includes(".");
|
|
347
|
+
/** Reported loci whose bytes came from the index, for the report's per-origin footer. */
|
|
348
|
+
/** @type {Set<string>} */
|
|
349
|
+
this.indexLoci = new Set();
|
|
320
350
|
}
|
|
321
351
|
|
|
322
352
|
// -------------------------------------------------------------------------
|
|
@@ -418,14 +448,18 @@ class PhiScan {
|
|
|
418
448
|
else if (paths.length > 0) mode = "paths";
|
|
419
449
|
else mode = "all";
|
|
420
450
|
|
|
421
|
-
//
|
|
422
|
-
//
|
|
423
|
-
//
|
|
424
|
-
//
|
|
425
|
-
//
|
|
426
|
-
//
|
|
427
|
-
//
|
|
428
|
-
|
|
451
|
+
// A BYPASS IS NOT A TARGET LIST, AND WHICH TIER REFUSES IT IS THE CONTRACT (AC-17). In `paths`
|
|
452
|
+
// mode the run's declared scope is argv, so a bypass naming something else subtracts nothing and
|
|
453
|
+
// is refused by the UNMATCHED tier, which says so and names it. In `all` mode the declared scope
|
|
454
|
+
// is the whole corpus, so a bypass declares a path this run would have read and is refused by the
|
|
455
|
+
// COMPLETENESS tier: `run` adds it to the enumerated set for exactly that reason. Both refuse,
|
|
456
|
+
// and neither can reach the clean code.
|
|
457
|
+
//
|
|
458
|
+
// WHAT THIS IS NOT: the flag still does not choose the MODE. Letting it made
|
|
459
|
+
// `--allow-fixture X` scan exactly `X`, then withdraw it, then report a clean whole run over a
|
|
460
|
+
// corpus it never touched, and that is the hole the mode rule below closes. Dedupe is by
|
|
461
|
+
// repo-relative path, so one file named twice on argv is one target, not two.
|
|
462
|
+
const seed = [...paths];
|
|
429
463
|
const scanPaths = mode === "paths" ? this.dedupeByRepoPath(seed) : paths;
|
|
430
464
|
|
|
431
465
|
return { mode, paths: scanPaths, allowFixtures };
|
|
@@ -638,10 +672,13 @@ class PhiScan {
|
|
|
638
672
|
|
|
639
673
|
/**
|
|
640
674
|
* Every stage-0 index entry keyed by repo-relative path, plus the paths that have a record but NO
|
|
641
|
-
* stage-0 record
|
|
675
|
+
* stage-0 record.
|
|
642
676
|
*
|
|
643
|
-
*
|
|
644
|
-
*
|
|
677
|
+
* THREE OUTCOMES, NOT TWO, AND THE SPLIT IS THE POINT (AC-18). `unreadable` means git could not
|
|
678
|
+
* say what this repository carries; `empty` means it said, and the answer is nothing. Both refuse
|
|
679
|
+
* the sweep, and an empty answer still counts as no corpus: an empty map would make every tracked
|
|
680
|
+
* path untracked, which is the one state in which the union silently stops existing. What differs
|
|
681
|
+
* is the REMEDY, so the two states carry different sentences rather than one that covers both.
|
|
645
682
|
*
|
|
646
683
|
* 🛑 THE STAGE DIGIT IS READ, AND KEYING ON IT IS NOT OPTIONAL. THE RULE IS THE ABSENCE OF STAGE 0.
|
|
647
684
|
* Do NOT re-derive it from a record count or from a mode, and do NOT port it from the `--staged`
|
|
@@ -652,7 +689,7 @@ class PhiScan {
|
|
|
652
689
|
* labelled it as the bytes git carries, and printed a clean line over a marker living only in
|
|
653
690
|
* stage 3.
|
|
654
691
|
*
|
|
655
|
-
* @returns {{ entries: Map<string, import("./phi-scan.js").IndexEntry>, unmerged: string[] } |
|
|
692
|
+
* @returns {{ status: "read", entries: Map<string, import("./phi-scan.js").IndexEntry>, unmerged: string[] } | { status: "unreadable" | "empty" }}
|
|
656
693
|
*/
|
|
657
694
|
gitIndexEntries() {
|
|
658
695
|
let out;
|
|
@@ -669,7 +706,7 @@ class PhiScan {
|
|
|
669
706
|
} catch {
|
|
670
707
|
// LOAD-BEARING. A directory that is no repository FATALS at 128; without this catch the throw
|
|
671
708
|
// escapes and the run takes node's exit 1, which this contract reserves for HITS FOUND.
|
|
672
|
-
return
|
|
709
|
+
return { status: "unreadable" };
|
|
673
710
|
}
|
|
674
711
|
/** @type {Map<string, import("./phi-scan.js").IndexEntry>} */
|
|
675
712
|
const entries = new Map();
|
|
@@ -685,8 +722,9 @@ class PhiScan {
|
|
|
685
722
|
const path = m?.[4];
|
|
686
723
|
if (mode === undefined || oid === undefined || stage === undefined || path === undefined) {
|
|
687
724
|
// An unparseable record means the list may be SHORT in a way we cannot see, which is the one
|
|
688
|
-
// thing this sweep must never scan past.
|
|
689
|
-
|
|
725
|
+
// thing this sweep must never scan past. It is UNREADABLE rather than empty: git answered,
|
|
726
|
+
// and the answer is one this engine cannot account for.
|
|
727
|
+
return { status: "unreadable" };
|
|
690
728
|
}
|
|
691
729
|
if (stage === "0") entries.set(path, { mode, oid });
|
|
692
730
|
else higherStages.add(path);
|
|
@@ -694,8 +732,8 @@ class PhiScan {
|
|
|
694
732
|
// A path is unmerged when it has a record and none of them is stage 0. The set difference is
|
|
695
733
|
// taken rather than assuming the two are disjoint.
|
|
696
734
|
const unmerged = [...higherStages].filter((p) => !entries.has(p));
|
|
697
|
-
if (entries.size === 0 && unmerged.length === 0) return
|
|
698
|
-
return { entries, unmerged };
|
|
735
|
+
if (entries.size === 0 && unmerged.length === 0) return { status: "empty" };
|
|
736
|
+
return { status: "read", entries, unmerged };
|
|
699
737
|
}
|
|
700
738
|
|
|
701
739
|
// -------------------------------------------------------------------------
|
|
@@ -753,27 +791,39 @@ class PhiScan {
|
|
|
753
791
|
* cases silently. What is NOT given up is the non-regular case: a root that is neither is collected
|
|
754
792
|
* as unscannable and refused, exactly as an entry inside one would be.
|
|
755
793
|
*
|
|
756
|
-
* A MISSING ROOT IS SKIPPED, AND THAT IS UNCHANGED FROM THE COPIED SCANNERS rather
|
|
757
|
-
* here. Two root states are refused or normalised away in `normalizeConfig` (a
|
|
758
|
-
* matches no index path is normalised away and the run proceeds; a root outside the
|
|
759
|
-
* throws), and this one is neither.
|
|
794
|
+
* A MISSING ROOT IS SKIPPED BY THE WALK, AND THAT IS UNCHANGED FROM THE COPIED SCANNERS rather
|
|
795
|
+
* than chosen here. Two root states are refused or normalised away in `normalizeConfig` (a
|
|
796
|
+
* spelling that matches no index path is normalised away and the run proceeds; a root outside the
|
|
797
|
+
* repository throws), and this one is neither.
|
|
760
798
|
*
|
|
761
|
-
*
|
|
762
|
-
*
|
|
763
|
-
*
|
|
764
|
-
*
|
|
765
|
-
*
|
|
766
|
-
*
|
|
799
|
+
* WHAT IT NO LONGER DOES IS CONTRIBUTE NOTHING IN SILENCE, and that is a change. A draft of this
|
|
800
|
+
* paragraph claimed the states in which a root contributes nothing were not enumerable, and named
|
|
801
|
+
* three anyway: a MISSING root; an UNREADABLE one, which `lstatOrNull`'s bare catch reports the
|
|
802
|
+
* same way (an `EACCES` on a parent directory, or an `ELOOP`); and a root whose every file the
|
|
803
|
+
* read filter drops, which is what a `.md` file root does under the default `isWalkReadable`. The
|
|
804
|
+
* set still is not enumerated, because nothing here has measured it to be complete - and it no
|
|
805
|
+
* longer has to be. `run`'s PER-ROOT OBSERVATION RULE asks the question from the other end, of
|
|
806
|
+
* the paths actually READ, so a root that yielded nothing refuses whatever put it in that state.
|
|
807
|
+
*
|
|
808
|
+
* A DIRECTORY THE WALK CANNOT LIST IS COLLECTED AND REFUSED, NOT THROWN OUT OF. `readdirSync` used
|
|
809
|
+
* to run bare here, so an `EACCES` on a root (or on any directory under one) escaped uncaught and
|
|
810
|
+
* the run took node's own exit 1, the code this contract reserves for HITS FOUND. It is the third
|
|
811
|
+
* of the three root KINDS this engine settles (AC-10): a regular file root is scanned as one
|
|
812
|
+
* target, a root naming a symbolic link is refused rather than followed, and a directory that
|
|
813
|
+
* cannot be enumerated is refused at the caller's `refuse` code, naming the path.
|
|
767
814
|
*
|
|
768
815
|
* The result is SORTED by repo-relative path, so a report and a refusal read the same way twice.
|
|
769
816
|
*
|
|
770
|
-
* @returns {{ files: string[], unscannable: import("./phi-scan.js").Unscannable[] }}
|
|
817
|
+
* @returns {{ files: string[], unscannable: import("./phi-scan.js").Unscannable[], unenumerable: import("./phi-scan.js").Unscannable[] }}
|
|
771
818
|
*/
|
|
772
819
|
walkRoots() {
|
|
773
820
|
/** @type {string[]} */
|
|
774
821
|
const files = [];
|
|
775
822
|
/** @type {import("./phi-scan.js").Unscannable[]} */
|
|
776
823
|
const unscannable = [];
|
|
824
|
+
/** Directories the walk reached and could not list. `kind` is the errno, never a path. */
|
|
825
|
+
/** @type {import("./phi-scan.js").Unscannable[]} */
|
|
826
|
+
const unenumerable = [];
|
|
777
827
|
|
|
778
828
|
// Split the roots by what is actually THERE, before the descent. A root naming a regular file is
|
|
779
829
|
// one target; a root naming a directory is a frontier; a root naming anything else is refused
|
|
@@ -800,7 +850,22 @@ class PhiScan {
|
|
|
800
850
|
const nextDirs = [];
|
|
801
851
|
for (const dir of frontier) {
|
|
802
852
|
if (!existsSync(dir)) continue;
|
|
803
|
-
|
|
853
|
+
let listing;
|
|
854
|
+
try {
|
|
855
|
+
listing = readdirSync(dir, { withFileTypes: true });
|
|
856
|
+
} catch (err) {
|
|
857
|
+
// A DIRECTORY THAT CANNOT BE LISTED IS A REFUSAL, NEVER A CRASH. The errno is recorded
|
|
858
|
+
// and nothing else: a message off the filesystem is text this engine cannot vouch for.
|
|
859
|
+
unenumerable.push({
|
|
860
|
+
path: this.normalizePath(dir),
|
|
861
|
+
kind:
|
|
862
|
+
err !== null && typeof err === "object" && "code" in err
|
|
863
|
+
? String(err.code)
|
|
864
|
+
: "unreadable",
|
|
865
|
+
});
|
|
866
|
+
continue;
|
|
867
|
+
}
|
|
868
|
+
for (const e of listing) {
|
|
804
869
|
const full = join(dir, e.name);
|
|
805
870
|
if (e.isDirectory()) {
|
|
806
871
|
if (e.name === ".git") continue;
|
|
@@ -823,7 +888,8 @@ class PhiScan {
|
|
|
823
888
|
|
|
824
889
|
files.sort((a, b) => (this.normalizePath(a) < this.normalizePath(b) ? -1 : 1));
|
|
825
890
|
unscannable.sort((a, b) => (a.path < b.path ? -1 : 1));
|
|
826
|
-
|
|
891
|
+
unenumerable.sort((a, b) => (a.path < b.path ? -1 : 1));
|
|
892
|
+
return { files, unscannable, unenumerable };
|
|
827
893
|
}
|
|
828
894
|
|
|
829
895
|
/**
|
|
@@ -841,19 +907,46 @@ class PhiScan {
|
|
|
841
907
|
// -------------------------------------------------------------------------
|
|
842
908
|
|
|
843
909
|
/**
|
|
844
|
-
* `all` mode's enumeration: the walk, PLUS the
|
|
910
|
+
* `all` mode's enumeration: the walk, PLUS the index the union half reads.
|
|
911
|
+
*
|
|
912
|
+
* THE TWO INDEX-ROUTE REFUSALS ARE RETURNED RATHER THAN THROWN, AND THAT IS A CHANGE (AC-15).
|
|
913
|
+
* They used to fire here, BEFORE the sweep, so an unmerged entry or a tracked link discarded every
|
|
914
|
+
* hit the walk was about to find and a consumer saw a refusal with no indication that PHI had
|
|
915
|
+
* already been found. The caller raises them after the walk sweep, printing the hits first. The
|
|
916
|
+
* refusal still wins the exit code and the clean line is still unreachable from there; what
|
|
917
|
+
* changes is that a finding already made is not thrown away. The cost is stated rather than
|
|
918
|
+
* hidden: a developer now waits out the walk before being told the index is unreadable at one
|
|
919
|
+
* path.
|
|
845
920
|
*
|
|
846
|
-
* @returns {{ targets: import("./phi-scan.js").Target[], index: Map<string, import("./phi-scan.js").IndexEntry
|
|
921
|
+
* @returns {{ targets: import("./phi-scan.js").Target[], index: Map<string, import("./phi-scan.js").IndexEntry>, indexRefusals: string[] }}
|
|
847
922
|
*/
|
|
848
923
|
buildTargetsForAll() {
|
|
849
|
-
const { files, unscannable } = this.walkRoots();
|
|
924
|
+
const { files, unscannable, unenumerable } = this.walkRoots();
|
|
850
925
|
|
|
851
|
-
// One `git check-ignore` over
|
|
926
|
+
// One `git check-ignore` over all three lists, so a link, a file and an unlistable directory get
|
|
927
|
+
// the same boundary.
|
|
852
928
|
const ignored = this.gitIgnored([
|
|
853
929
|
...files.map((f) => this.normalizePath(f)),
|
|
854
930
|
...unscannable.map((u) => u.path),
|
|
931
|
+
...unenumerable.map((u) => u.path),
|
|
855
932
|
]);
|
|
856
933
|
|
|
934
|
+
const blocked = unenumerable.filter(
|
|
935
|
+
(u) => !ignored.has(u.path) && !this.cfg.excludedPaths.has(u.path),
|
|
936
|
+
);
|
|
937
|
+
if (blocked.length > 0) {
|
|
938
|
+
// AC-10, the third root kind. Named one by one rather than listed under a count, because the
|
|
939
|
+
// remedy is per path and a developer re-running the gate once per directory learns to
|
|
940
|
+
// distrust it.
|
|
941
|
+
const named = blocked.map((u) => `${u.path} (${u.kind})`).join(", ");
|
|
942
|
+
throw new InvocationError(
|
|
943
|
+
`refusing the scan: could not enumerate ${named}. A directory the walk cannot list ` +
|
|
944
|
+
`contributes nothing while looking exactly like one that is empty, so the sweep would ` +
|
|
945
|
+
`cover less than its own configuration says. Make it readable, or drop it from the scan ` +
|
|
946
|
+
`roots.`,
|
|
947
|
+
);
|
|
948
|
+
}
|
|
949
|
+
|
|
857
950
|
this.refuseUnscannable(
|
|
858
951
|
unscannable.filter((u) => !ignored.has(u.path) && !this.cfg.excludedPaths.has(u.path)),
|
|
859
952
|
"The walk can neither read such an entry nor vouch for what is on the other side of it.",
|
|
@@ -862,56 +955,79 @@ class PhiScan {
|
|
|
862
955
|
);
|
|
863
956
|
|
|
864
957
|
const listed = this.gitIndexEntries();
|
|
865
|
-
if (listed
|
|
958
|
+
if (listed.status !== "read") {
|
|
959
|
+
// AC-18: TWO STATES, TWO MESSAGES, because the remedies differ. "Run this inside a git
|
|
960
|
+
// repository with a readable index" is no help to somebody whose repository is fine and whose
|
|
961
|
+
// index is empty, and one sentence for both was measured sending a reader to the wrong one.
|
|
866
962
|
throw new InvocationError(
|
|
867
|
-
|
|
868
|
-
"
|
|
869
|
-
|
|
963
|
+
listed.status === "empty"
|
|
964
|
+
? "refusing the sweep: the git index holds no entries, so every check the sweep makes " +
|
|
965
|
+
"against what git carries would pass vacuously and every path would look untracked. " +
|
|
966
|
+
"Stage and commit this repository's corpus, then re-run."
|
|
967
|
+
: "refusing the sweep: could not read this repository's git index, so the sweep would be " +
|
|
968
|
+
"the working-tree walk's word alone and could report clean over tracked bytes it " +
|
|
969
|
+
"never opened. Run it inside a git repository with a readable index.",
|
|
870
970
|
);
|
|
871
971
|
}
|
|
872
972
|
|
|
973
|
+
/** @type {string[]} */
|
|
974
|
+
const indexRefusals = [];
|
|
975
|
+
|
|
976
|
+
// THE INDEX ROUTE IS NOT FILTERED TO `scanRoots` (AC-12). `scanRoots` is the WALK's scope: it
|
|
977
|
+
// answers "what is on disk under the roots this repo declared". The index answers "what does
|
|
978
|
+
// this repository CARRY", which is not a question a root list is entitled to narrow. Measured on
|
|
979
|
+
// the first consumer: three real messages under an undeclared top-level directory were read by
|
|
980
|
+
// neither route, and a tracked link outside every root was reached by neither. So every path the
|
|
981
|
+
// index carries is considered here, and `--staged` is untouched by that: it keeps its own scope,
|
|
982
|
+
// which is what a COMMIT is blocked on.
|
|
983
|
+
//
|
|
873
984
|
// Unmerged first, and under its OWN sentence: an unmerged path is not a link and not a gitlink,
|
|
874
985
|
// and reporting it as one sends a developer looking for something that is not there.
|
|
875
|
-
this.
|
|
986
|
+
const unmergedMessage = this.unscannableMessage(
|
|
876
987
|
listed.unmerged
|
|
877
|
-
.filter((p) =>
|
|
988
|
+
.filter((p) => !this.cfg.excludedPaths.has(p))
|
|
878
989
|
.map((p) => ({ path: p, kind: "no stage-0 blob" })),
|
|
879
990
|
"An unmerged path has no single merged blob, so there is no one set of bytes git carries here " +
|
|
880
991
|
"for the sweep to read, only the conflicting sides and, when there is one, their base.",
|
|
881
992
|
"Resolve the conflict and stage the result, then re-run.",
|
|
882
993
|
{ one: "path is unmerged", many: "paths are unmerged" },
|
|
883
994
|
);
|
|
995
|
+
if (unmergedMessage !== null) indexRefusals.push(unmergedMessage);
|
|
884
996
|
|
|
885
|
-
// The index's own non-blob entries
|
|
886
|
-
//
|
|
887
|
-
//
|
|
888
|
-
|
|
889
|
-
this.refuseUnscannable(
|
|
997
|
+
// The index's own non-blob entries. `120000` is a symbolic link, whose blob is its TARGET PATH
|
|
998
|
+
// and not any content; `160000` is a gitlink, which carries another repository's commit id and
|
|
999
|
+
// no bytes at this path at all.
|
|
1000
|
+
const nonBlobMessage = this.unscannableMessage(
|
|
890
1001
|
[...listed.entries]
|
|
891
1002
|
.filter(
|
|
892
|
-
([p, e]) =>
|
|
893
|
-
this.isUnderScanRoot(p) &&
|
|
894
|
-
!this.cfg.regularBlobModes.has(e.mode) &&
|
|
895
|
-
!this.cfg.excludedPaths.has(p),
|
|
1003
|
+
([p, e]) => !this.cfg.regularBlobModes.has(e.mode) && !this.cfg.excludedPaths.has(p),
|
|
896
1004
|
)
|
|
897
1005
|
.map(([p, e]) => ({ path: p, kind: gitModeKind(e.mode) })),
|
|
898
|
-
"Git records no readable content at such a path
|
|
899
|
-
"
|
|
1006
|
+
"Git records no readable content at such a path: a symbolic link's blob is its target path, " +
|
|
1007
|
+
"and a gitlink is another repository's commit id, never bytes here.",
|
|
900
1008
|
"Untrack it, or replace it with a regular file.",
|
|
901
1009
|
// Its own noun: the offender is an INDEX RECORD, and a gitlink's working tree may not exist at
|
|
902
1010
|
// all, so "not a regular file" would send a developer to a path where there is nothing to see.
|
|
903
1011
|
{ one: "index entry is not a regular blob", many: "index entries are not regular blobs" },
|
|
904
1012
|
);
|
|
1013
|
+
if (nonBlobMessage !== null) indexRefusals.push(nonBlobMessage);
|
|
905
1014
|
|
|
906
1015
|
const targets = files
|
|
907
1016
|
.map((abs) => ({ abs, rel: this.normalizePath(abs) }))
|
|
908
1017
|
.filter(({ rel }) => !ignored.has(rel) && !this.cfg.excludedPaths.has(rel))
|
|
909
1018
|
.map(({ abs, rel }) => ({ path: rel, read: () => readFileSync(abs) }));
|
|
910
|
-
return { targets, index: listed.entries };
|
|
1019
|
+
return { targets, index: listed.entries, indexRefusals };
|
|
911
1020
|
}
|
|
912
1021
|
|
|
913
1022
|
/**
|
|
914
|
-
* The
|
|
1023
|
+
* The tracked paths the union half is entitled to read: EVERY regular blob the index carries, not
|
|
1024
|
+
* only the ones under a scan root (AC-12).
|
|
1025
|
+
*
|
|
1026
|
+
* `scanRoots` IS THE WALK'S SCOPE AND NOT THE INDEX'S. The walk answers "what is on disk under the
|
|
1027
|
+
* roots this repo declared"; the index answers "what does this repository CARRY", and narrowing
|
|
1028
|
+
* the second by the first was measured hiding three real messages under an undeclared top-level
|
|
1029
|
+
* directory from BOTH routes. The two READ filters still apply, so the `.md` exemption and
|
|
1030
|
+
* `excludedPaths` mean the same thing on both sweeping routes: one boundary, not two.
|
|
915
1031
|
*
|
|
916
1032
|
* IT IS COMPUTED BEFORE THE FIRST BYTE IS READ, AND THAT IS LOAD-BEARING. This set is part of what
|
|
917
1033
|
* `all` mode ENUMERATES, so both completeness tiers see it: a bypass naming a tracked-but-absent
|
|
@@ -923,12 +1039,7 @@ class PhiScan {
|
|
|
923
1039
|
*/
|
|
924
1040
|
unionCandidatePaths(index) {
|
|
925
1041
|
return [...index]
|
|
926
|
-
.filter(
|
|
927
|
-
([p, e]) =>
|
|
928
|
-
this.cfg.regularBlobModes.has(e.mode) &&
|
|
929
|
-
this.isUnderScanRoot(p) &&
|
|
930
|
-
this.cfg.isWalkReadable(p),
|
|
931
|
-
)
|
|
1042
|
+
.filter(([p, e]) => this.cfg.regularBlobModes.has(e.mode) && this.cfg.isWalkReadable(p))
|
|
932
1043
|
.filter(([p]) => !this.cfg.excludedPaths.has(p))
|
|
933
1044
|
.map(([p]) => p);
|
|
934
1045
|
}
|
|
@@ -942,6 +1053,13 @@ class PhiScan {
|
|
|
942
1053
|
* DIFFERENT id had a different copy read, so its blob is scanned too. That second case is the EOL
|
|
943
1054
|
* axis.
|
|
944
1055
|
*
|
|
1056
|
+
* THOSE TWO CASES CARRY DIFFERENT ORIGIN LABELS, AND THE REMEDY IS WHY (AC-14). `(git index)` is a
|
|
1057
|
+
* path whose working-tree copy this run did not read at all: it is absent, occupied by a
|
|
1058
|
+
* directory, or outside the walk's roots, so there is nothing on disk to open. `(git index; the
|
|
1059
|
+
* working tree differs)` is a path whose disk copy WAS read and carries other bytes, so the file a
|
|
1060
|
+
* developer opens is clean and RE-STAGING is part of the fix. One label for both hid that
|
|
1061
|
+
* difference behind the same sentence.
|
|
1062
|
+
*
|
|
945
1063
|
* @param {Map<string, import("./phi-scan.js").IndexEntry>} index
|
|
946
1064
|
* @param {Map<string, string>} readOids
|
|
947
1065
|
* @returns {import("./phi-scan.js").Target[]}
|
|
@@ -955,7 +1073,7 @@ class PhiScan {
|
|
|
955
1073
|
if (readOids.get(path) === entry.oid) continue;
|
|
956
1074
|
targets.push({
|
|
957
1075
|
path,
|
|
958
|
-
origin: "
|
|
1076
|
+
origin: readOids.has(path) ? "git index; the working tree differs" : "git index",
|
|
959
1077
|
// SECURITY: array-form execFileSync, no shell. The object id is git's own output, and naming
|
|
960
1078
|
// the OBJECT rather than the path is the whole point: it cannot be redirected by whatever
|
|
961
1079
|
// the working tree currently holds. `maxBuffer` defaults to 1 MiB, so a larger tracked blob
|
|
@@ -1039,7 +1157,7 @@ class PhiScan {
|
|
|
1039
1157
|
// record that does not parse REFUSES rather than being skipped: a silently shortened list is
|
|
1040
1158
|
// exactly the shape this scan must never report clean over.
|
|
1041
1159
|
const fields = listBuf.toString("utf8").split("\0");
|
|
1042
|
-
/** @type {{ path: string, mode: string }[]} */
|
|
1160
|
+
/** @type {{ path: string, mode: string, status: string }[]} */
|
|
1043
1161
|
const staged = [];
|
|
1044
1162
|
let i = 0;
|
|
1045
1163
|
while (i < fields.length) {
|
|
@@ -1050,33 +1168,40 @@ class PhiScan {
|
|
|
1050
1168
|
}
|
|
1051
1169
|
const m = RAW_RECORD.exec(info);
|
|
1052
1170
|
const mode = m?.[1];
|
|
1171
|
+
const status = m?.[2];
|
|
1053
1172
|
const path = fields[i + 1];
|
|
1054
|
-
if (mode === undefined || path === undefined || path.length === 0) {
|
|
1173
|
+
if (mode === undefined || status === undefined || path === undefined || path.length === 0) {
|
|
1055
1174
|
throw new InvocationError(
|
|
1056
1175
|
"could not read the output of `git diff --cached --raw -z`: unrecognized record. " +
|
|
1057
1176
|
"Refusing rather than scanning a list that may be short.",
|
|
1058
1177
|
);
|
|
1059
1178
|
}
|
|
1060
|
-
staged.push({ path, mode });
|
|
1179
|
+
staged.push({ path, mode, status });
|
|
1061
1180
|
i += 2;
|
|
1062
1181
|
}
|
|
1063
1182
|
|
|
1064
|
-
//
|
|
1065
|
-
//
|
|
1066
|
-
//
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1183
|
+
// THIS ROUTE'S REFUSALS KEY ON `isStagedReadable`, WHICH IS THE CALLER'S OWN SCOPE FOR IT
|
|
1184
|
+
// (AC-16). They used to key on the ROOT half instead, and for a caller whose staged scope is
|
|
1185
|
+
// narrower than its roots that REFUSED a commit the repo's own gate had always let through:
|
|
1186
|
+
// measured on the first consumer, a staged `src/notes.txt` symbolic link is outside a
|
|
1187
|
+
// `src/**.ts` staged scope and was refused at the refuse code where the repo's own scanner
|
|
1188
|
+
// exited 0. What a commit is blocked on is a hook decision each repo takes for itself, and
|
|
1189
|
+
// widening it from inside a shared engine takes that decision away.
|
|
1190
|
+
//
|
|
1191
|
+
// 🛑 THIS NARROWS WHAT THIS ROUTE REFUSES, AND THE COMPENSATING CONTROL IS NAMED RATHER THAN
|
|
1192
|
+
// ASSUMED. A staged non-regular entry the caller's staged scope declines is no longer refused
|
|
1193
|
+
// HERE. It is still refused by `all` mode twice over: the walk classifies it with
|
|
1194
|
+
// `Dirent.isSymbolicLink()` when it sits under a scan root, and the index route refuses the
|
|
1195
|
+
// tracked mode-120000 record wherever it sits, now that that route is not narrowed to the roots
|
|
1196
|
+
// either. So the sweep a repository runs in CI still refuses it; the pre-commit hook no longer
|
|
1197
|
+
// does. See `documentation/decisions/0003-the-phi-scan-engine-gap-settlements.md`.
|
|
1198
|
+
//
|
|
1199
|
+
// AN UNMERGED PATH IS NOT ONE OF THESE. Its destination mode is `000000`, which the mode rule
|
|
1200
|
+
// would otherwise describe as "a git mode-000000 entry" beside a sentence about content git
|
|
1201
|
+
// holds at the path, and both halves are false for a conflicted regular file.
|
|
1202
|
+
const unmerged = staged.filter((s) => s.status === "U" || s.mode === "000000");
|
|
1203
|
+
const stagedReadable = (/** @type {{ path: string, mode: string, status: string }} */ s) =>
|
|
1204
|
+
this.cfg.isStagedReadable(s.path) && !this.cfg.excludedPaths.has(s.path);
|
|
1080
1205
|
|
|
1081
1206
|
// THE CONTAINMENT IS ENFORCED, NOT ASSUMED, AND IT USED TO BE THE WORD "by construction".
|
|
1082
1207
|
// `isStagedReadable` and `scanRoots` are two independent keys a repo fills in, and nothing
|
|
@@ -1110,8 +1235,31 @@ class PhiScan {
|
|
|
1110
1235
|
},
|
|
1111
1236
|
);
|
|
1112
1237
|
|
|
1113
|
-
//
|
|
1114
|
-
//
|
|
1238
|
+
// THE MISCONFIGURATION REFUSAL ABOVE COMES FIRST ON PURPOSE. Both it and the two below can fire
|
|
1239
|
+
// over the same entry, and a developer whose `isStagedReadable` admits a path no root covers
|
|
1240
|
+
// needs to hear that rather than a sentence about this one entry's kind.
|
|
1241
|
+
this.refuseUnscannable(
|
|
1242
|
+
unmerged.filter(stagedReadable).map((s) => ({ path: s.path, kind: "no stage-0 blob" })),
|
|
1243
|
+
"An unmerged path has no stage-0 blob, so the index holds no single set of bytes a commit " +
|
|
1244
|
+
"would carry here, only the conflicting sides and, when there is one, their base.",
|
|
1245
|
+
"Resolve the conflict and stage the result, then re-run.",
|
|
1246
|
+
{ one: "staged path is unmerged", many: "staged paths are unmerged" },
|
|
1247
|
+
);
|
|
1248
|
+
|
|
1249
|
+
this.refuseUnscannable(
|
|
1250
|
+
staged
|
|
1251
|
+
.filter(
|
|
1252
|
+
(s) =>
|
|
1253
|
+
stagedReadable(s) && !unmerged.includes(s) && !this.cfg.regularBlobModes.has(s.mode),
|
|
1254
|
+
)
|
|
1255
|
+
.map((s) => ({ path: s.path, kind: gitModeKind(s.mode) })),
|
|
1256
|
+
"The index holds no file content for such an entry, so scanning it would prove nothing about " +
|
|
1257
|
+
"what it refers to.",
|
|
1258
|
+
"Unstage it, or replace it with a regular file.",
|
|
1259
|
+
);
|
|
1260
|
+
|
|
1261
|
+
// Every remaining readable record is a regular blob: anything non-regular was refused above,
|
|
1262
|
+
// anything unmerged was refused above it, and anything outside the roots before that.
|
|
1115
1263
|
return staged
|
|
1116
1264
|
.filter((s) => this.cfg.isStagedReadable(s.path) && !this.cfg.excludedPaths.has(s.path))
|
|
1117
1265
|
.map((s) => s.path)
|
|
@@ -1127,6 +1275,65 @@ class PhiScan {
|
|
|
1127
1275
|
}));
|
|
1128
1276
|
}
|
|
1129
1277
|
|
|
1278
|
+
/**
|
|
1279
|
+
* The absolute path of a repo-relative one, for a question about the filesystem rather than
|
|
1280
|
+
* about the corpus. Only the TOCTOU re-check asks one.
|
|
1281
|
+
*
|
|
1282
|
+
* @param {string} relPath
|
|
1283
|
+
* @returns {string}
|
|
1284
|
+
*/
|
|
1285
|
+
absolutePath(relPath) {
|
|
1286
|
+
return relPath === "." ? this.cfg.repoRoot : join(this.cfg.repoRoot, ...relPath.split("/"));
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
/**
|
|
1290
|
+
* Did this root yield at least one file whose bytes were ACTUALLY READ?
|
|
1291
|
+
*
|
|
1292
|
+
* IT IS ASKED OF `read`, WHICH IS EVIDENCE OF OBSERVATION RATHER THAN A PLAN TO OBSERVE, and it
|
|
1293
|
+
* is asked per root rather than of the union, because the union is exactly what hides a starved
|
|
1294
|
+
* root: one productive root makes the whole sweep look productive.
|
|
1295
|
+
*
|
|
1296
|
+
* @param {string} root A normalised scan root.
|
|
1297
|
+
* @param {Set<string>} read The repo-relative paths this run opened.
|
|
1298
|
+
* @returns {boolean}
|
|
1299
|
+
*/
|
|
1300
|
+
anyReadUnder(root, read) {
|
|
1301
|
+
for (const p of read) {
|
|
1302
|
+
if (root === "." || p === root || p.startsWith(`${root}/`)) return true;
|
|
1303
|
+
}
|
|
1304
|
+
return false;
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
/**
|
|
1308
|
+
* Is this read failure the ENUMERATION TOCTOU WINDOW rather than a state the run must refuse?
|
|
1309
|
+
*
|
|
1310
|
+
* FOUR CONDITIONS, ALL OF THEM NECESSARY, because each one is a different question the run can
|
|
1311
|
+
* still answer honestly afterwards:
|
|
1312
|
+
*
|
|
1313
|
+
* 1. `all` MODE. The other two routes were asked for a path BY NAME - on argv, or by git's
|
|
1314
|
+
* own index - so a caller is owed an answer about that path rather than a shrug.
|
|
1315
|
+
* 2. THE WALK'S OWN TARGET, never the union's. The union reads an OBJECT by id, which cannot
|
|
1316
|
+
* go missing between enumeration and read.
|
|
1317
|
+
* 3. `ENOENT`, exactly. Every other errno says the entry is still there and something else
|
|
1318
|
+
* went wrong, which is the state a scan must never report past.
|
|
1319
|
+
* 4. UNTRACKED. Git carries bytes at a tracked path, so a tracked file that vanished still
|
|
1320
|
+
* leaves this run with content it declared it would read and did not.
|
|
1321
|
+
*
|
|
1322
|
+
* @param {unknown} err
|
|
1323
|
+
* @param {import("./phi-scan.js").Target} target
|
|
1324
|
+
* @param {Map<string, import("./phi-scan.js").IndexEntry> | null} index
|
|
1325
|
+
* @returns {boolean}
|
|
1326
|
+
*/
|
|
1327
|
+
isVanishedUntracked(err, target, index) {
|
|
1328
|
+
return (
|
|
1329
|
+
err instanceof TargetReadError &&
|
|
1330
|
+
err.errnoCode === "ENOENT" &&
|
|
1331
|
+
index !== null &&
|
|
1332
|
+
target.origin === undefined &&
|
|
1333
|
+
!index.has(target.path)
|
|
1334
|
+
);
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1130
1337
|
/**
|
|
1131
1338
|
* Refuse over entries the enumeration reached and cannot scan. EVERY offender IN THE GROUP is
|
|
1132
1339
|
* named, not just the first: a developer who has to re-run the gate once per link learns to
|
|
@@ -1143,18 +1350,34 @@ class PhiScan {
|
|
|
1143
1350
|
* @param {string} remedy
|
|
1144
1351
|
* @param {{ one: string, many: string }} [noun]
|
|
1145
1352
|
*/
|
|
1146
|
-
refuseUnscannable(
|
|
1353
|
+
refuseUnscannable(entries, why, remedy, noun) {
|
|
1354
|
+
const message = this.unscannableMessage(entries, why, remedy, noun);
|
|
1355
|
+
if (message !== null) throw new InvocationError(message);
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
/**
|
|
1359
|
+
* The refusal message for a group of unscannable entries, or `null` when the group is empty.
|
|
1360
|
+
*
|
|
1361
|
+
* SPLIT FROM THE THROW so a refusal can be DEFERRED rather than raised where it is computed. The
|
|
1362
|
+
* index route's two refusals are built before the sweep and raised after it, so the hits the walk
|
|
1363
|
+
* found are printed first; every other caller throws immediately, as before.
|
|
1364
|
+
*
|
|
1365
|
+
* @param {import("./phi-scan.js").Unscannable[]} entries
|
|
1366
|
+
* @param {string} why
|
|
1367
|
+
* @param {string} remedy
|
|
1368
|
+
* @param {{ one: string, many: string }} [noun]
|
|
1369
|
+
* @returns {string | null}
|
|
1370
|
+
*/
|
|
1371
|
+
unscannableMessage(
|
|
1147
1372
|
entries,
|
|
1148
1373
|
why,
|
|
1149
1374
|
remedy,
|
|
1150
1375
|
noun = { one: "entry is not a regular file", many: "entries are not regular files" },
|
|
1151
1376
|
) {
|
|
1152
|
-
if (entries.length === 0) return;
|
|
1377
|
+
if (entries.length === 0) return null;
|
|
1153
1378
|
const lines = entries.map((u) => ` - ${u.path} (${u.kind})`).join("\n");
|
|
1154
1379
|
const phrase = entries.length === 1 ? noun.one : noun.many;
|
|
1155
|
-
|
|
1156
|
-
`refusing the scan: ${String(entries.length)} ${phrase}:\n${lines}\n${why} ${remedy}`,
|
|
1157
|
-
);
|
|
1380
|
+
return `refusing the scan: ${String(entries.length)} ${phrase}:\n${lines}\n${why} ${remedy}`;
|
|
1158
1381
|
}
|
|
1159
1382
|
|
|
1160
1383
|
// -------------------------------------------------------------------------
|
|
@@ -1175,6 +1398,11 @@ class PhiScan {
|
|
|
1175
1398
|
* The declared form is matched BOTH as written and with its separators removed, so one `ID` entry
|
|
1176
1399
|
* covers both renderings rather than requiring a repo to guess which one a fixture uses.
|
|
1177
1400
|
*
|
|
1401
|
+
* THE MATCHED TOKEN IS A LOCAL, AND IT STAYS ONE. It decides whether the allow-list already
|
|
1402
|
+
* declares this identifier and is then dropped: the hit that leaves this function carries a
|
|
1403
|
+
* position and a rule, so there is no record for a later reporting path to print. See
|
|
1404
|
+
* `reportHits` for why that is structural rather than a call-site decision.
|
|
1405
|
+
*
|
|
1178
1406
|
* @param {string} path The reported LOCUS.
|
|
1179
1407
|
* @param {string} content
|
|
1180
1408
|
* @param {import("./phi-scan.js").AllowList} allow
|
|
@@ -1185,12 +1413,12 @@ class PhiScan {
|
|
|
1185
1413
|
const value = m[0];
|
|
1186
1414
|
if (allow.ids.has(value.toUpperCase())) continue;
|
|
1187
1415
|
if (allow.ids.has(value.replace(/\D/g, ""))) continue;
|
|
1188
|
-
hits.push({ path, segment: "(ssn)",
|
|
1416
|
+
hits.push({ path, segment: "(ssn)", reason: "dashed SSN pattern" });
|
|
1189
1417
|
}
|
|
1190
1418
|
for (const m of content.matchAll(/\b[A-Za-z0-9._%+-]+@([A-Za-z0-9.-]+\.[A-Za-z]{2,})\b/g)) {
|
|
1191
1419
|
const domain = (m[1] ?? "").toLowerCase();
|
|
1192
1420
|
if (!allow.emailDomains.has(domain)) {
|
|
1193
|
-
hits.push({ path, segment: "(email)",
|
|
1421
|
+
hits.push({ path, segment: "(email)", reason: "email with non-test domain" });
|
|
1194
1422
|
}
|
|
1195
1423
|
}
|
|
1196
1424
|
}
|
|
@@ -1227,14 +1455,18 @@ class PhiScan {
|
|
|
1227
1455
|
try {
|
|
1228
1456
|
buf = target.read();
|
|
1229
1457
|
} catch (err) {
|
|
1230
|
-
throw new
|
|
1458
|
+
throw new TargetReadError(
|
|
1231
1459
|
`could not read ${target.path}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1460
|
+
err !== null && typeof err === "object" && "code" in err ? String(err.code) : undefined,
|
|
1232
1461
|
);
|
|
1233
1462
|
}
|
|
1234
1463
|
const text = buf.toString("utf8");
|
|
1235
1464
|
// Scope is decided on the target's own path; only the REPORTED locus carries the origin label,
|
|
1236
1465
|
// so a labelled target is never a differently-scoped one.
|
|
1237
1466
|
const locus = target.origin === undefined ? target.path : `${target.path} (${target.origin})`;
|
|
1467
|
+
// Recorded rather than re-derived from the locus string: the footer counts hits in bytes git
|
|
1468
|
+
// carries, and reading that back off the text of a path is a parse of the engine's own prose.
|
|
1469
|
+
if (target.origin !== undefined) this.indexLoci.add(locus);
|
|
1238
1470
|
|
|
1239
1471
|
this.scanCommonShapes(locus, text, allow, hits);
|
|
1240
1472
|
|
|
@@ -1247,7 +1479,12 @@ class PhiScan {
|
|
|
1247
1479
|
bytes: buf,
|
|
1248
1480
|
allow,
|
|
1249
1481
|
hit: (h) => {
|
|
1250
|
-
|
|
1482
|
+
// THE LOCUS AND THE RULE ARE TAKEN; ANY `value` A DETECTOR PASSES IS DROPPED HERE. A
|
|
1483
|
+
// per-standard detector holds the record it parsed, so it is the caller most likely to
|
|
1484
|
+
// hand over a token that is real PHI, and the engine is the side that can refuse to
|
|
1485
|
+
// carry it. Dropping it at the boundary is what makes "no matched value reaches a
|
|
1486
|
+
// report" a property of this engine rather than of every detector's discipline.
|
|
1487
|
+
hits.push({ path: locus, segment: h.segment, reason: h.reason });
|
|
1251
1488
|
},
|
|
1252
1489
|
});
|
|
1253
1490
|
} catch (err) {
|
|
@@ -1271,6 +1508,19 @@ class PhiScan {
|
|
|
1271
1508
|
* FIRST, so a run that is both incomplete and carrying hits prints both. The clean line is printed
|
|
1272
1509
|
* only once every tier has passed, so it can never appear beside a refusal.
|
|
1273
1510
|
*
|
|
1511
|
+
* IT NAMES A POSITION AND STOPS THERE: the repo-relative locus, the locator inside it, and the
|
|
1512
|
+
* rule that fired. THE MATCHED TOKEN IS NEVER PRINTED. This is the one surface that would have
|
|
1513
|
+
* the token in hand, which is exactly why the rule is stated on it: stderr is a CI log, a CI log
|
|
1514
|
+
* is a transcript nobody retains deliberately, and a diagnostic ABOUT a PHI leak that quotes the
|
|
1515
|
+
* leak is a second copy of it in a worse place (`phi-safety` P4, `observability` B1). Nothing is
|
|
1516
|
+
* redacted here either, because the value never reaches this function: a seam a call site has to
|
|
1517
|
+
* remember is a seam the next call site forgets.
|
|
1518
|
+
*
|
|
1519
|
+
* WHAT A DEVELOPER DOES WITH A HIT THEY CANNOT READ THE VALUE OF, since an error owes an action
|
|
1520
|
+
* (`observability` B2): open the named path, find the shape the named rule describes, and either
|
|
1521
|
+
* remove it or declare it in the allow-list the footer names if it is genuinely synthetic. The
|
|
1522
|
+
* position is what locates the token; the token itself was never the remedy.
|
|
1523
|
+
*
|
|
1274
1524
|
* THE FOOTER IS SCOPED TO WHAT THIS ENGINE KNOWS. It does not claim the allow-list reaches a clean
|
|
1275
1525
|
* run for every hit, because a per-repo detector supplied through `detect` may raise one without
|
|
1276
1526
|
* consulting the allow-list at all, and a sibling shipped exactly that claim and had it refuted.
|
|
@@ -1292,11 +1542,21 @@ class PhiScan {
|
|
|
1292
1542
|
for (const [path, group] of byPath) {
|
|
1293
1543
|
process.stderr.write(`[phi-scan] HIT: ${path}\n`);
|
|
1294
1544
|
for (const h of group) {
|
|
1295
|
-
process.stderr.write(
|
|
1296
|
-
` segment=${h.segment} value=${JSON.stringify(h.value)} (${h.reason})\n`,
|
|
1297
|
-
);
|
|
1545
|
+
process.stderr.write(` segment=${h.segment} (${h.reason})\n`);
|
|
1298
1546
|
}
|
|
1299
1547
|
}
|
|
1548
|
+
// THE PER-ORIGIN FOOTER, AND IT IS A REMEDY RATHER THAN A STATISTIC (AC-14). A hit in bytes git
|
|
1549
|
+
// carries is not answered by editing the file on disk: the identifier is in the index, so
|
|
1550
|
+
// re-staging (or removing the blob) is part of the fix, and a reader counting `HIT:` lines has
|
|
1551
|
+
// no way to tell how many of them are in that state.
|
|
1552
|
+
const fromIndex = hits.filter((h) => this.indexLoci.has(h.path)).length;
|
|
1553
|
+
if (fromIndex > 0) {
|
|
1554
|
+
process.stderr.write(
|
|
1555
|
+
`[phi-scan] ${String(fromIndex)} of those are in bytes git carries at that path, not in ` +
|
|
1556
|
+
`the file on disk: fix the content and RE-STAGE it, or the next sweep reports the same ` +
|
|
1557
|
+
`hit over a working tree that reads clean.\n`,
|
|
1558
|
+
);
|
|
1559
|
+
}
|
|
1300
1560
|
process.stderr.write(
|
|
1301
1561
|
`[phi-scan] ${String(hits.length)} hit(s) across ${String(byPath.size)} file(s). ` +
|
|
1302
1562
|
`The cross-cutting floor (SSN and email shapes) consults ${this.relAllowList()}, so a ` +
|
|
@@ -1317,7 +1577,13 @@ class PhiScan {
|
|
|
1317
1577
|
* THE EXIT CONTRACT IS THE CALLER'S, NOT THIS FILE'S. The three codes come from `exitCodes`, and
|
|
1318
1578
|
* their meanings are:
|
|
1319
1579
|
*
|
|
1320
|
-
* clean the scan ran, READ EVERY TARGET IT ENUMERATED, and found nothing.
|
|
1580
|
+
* clean the scan ran, READ EVERY TARGET IT ENUMERATED, and found nothing. THE ONE
|
|
1581
|
+
* SUBTRACTION IS THE ENUMERATION TOCTOU WINDOW, and it is bounded rather than
|
|
1582
|
+
* described: an UNTRACKED file the walk itself listed, in `all` mode, whose read
|
|
1583
|
+
* failed with `ENOENT`, and which is still absent at the end of the run, is reported
|
|
1584
|
+
* SKIPPED and does not hold the clean code back. Git carries no bytes at such a path,
|
|
1585
|
+
* so nothing this repository holds went unread. Every other shape of the same failure
|
|
1586
|
+
* is in `refuse` below.
|
|
1321
1587
|
* hits this corpus contains something that looks like PHI. Nothing this engine RAISES ever
|
|
1322
1588
|
* takes it. It is NOT exclusive, and the escapes are named rather than left to be
|
|
1323
1589
|
* discovered: an allow-list or an override log that EXISTS but cannot be READ throws a
|
|
@@ -1327,8 +1593,22 @@ class PhiScan {
|
|
|
1327
1593
|
* does not enumerate, an in-scope entry that is not a regular file, an unparseable
|
|
1328
1594
|
* `git diff --cached` record, an index git cannot name or names empty, an in-scope index
|
|
1329
1595
|
* entry that is not a regular blob, an in-scope path with no stage-0 blob (unmerged), a
|
|
1330
|
-
* target whose bytes cannot be read, a field detector that threw,
|
|
1331
|
-
*
|
|
1596
|
+
* target whose bytes cannot be read, a field detector that threw, a target enumerated
|
|
1597
|
+
* but never read, a target that was absent at read time and PRESENT AGAIN at the end
|
|
1598
|
+
* of the run, and (`all` mode) a SCAN ROOT THAT YIELDED NO FILE THAT WAS READ.
|
|
1599
|
+
* ===========================================================================================
|
|
1600
|
+
* THE PER-ROOT OBSERVATION RULE, `all` MODE ONLY. The completeness rule asks whether every
|
|
1601
|
+
* target was read. It cannot ask whether a root produced a target at all, and a root that
|
|
1602
|
+
* produces none is a sweep that is narrower than its own configuration says: a typo, a directory
|
|
1603
|
+
* that moved or was never created, a root whose every file the read filter drops. ONE
|
|
1604
|
+
* PRODUCTIVE ROOT MAKES THE WHOLE RUN LOOK PRODUCTIVE, which is why the question is asked per
|
|
1605
|
+
* root and answered from the set of paths actually READ, and why the refusal names every starved
|
|
1606
|
+
* root rather than the first.
|
|
1607
|
+
*
|
|
1608
|
+
* IT APPLIES TO A MISSING ROOT TOO, AND THAT DOES NOT CONTRADICT "A MISSING ROOT IS SKIPPED".
|
|
1609
|
+
* The walk still skips it: it is not reported as an unscannable KIND and it does not crash the
|
|
1610
|
+
* enumeration, which is what a missing root used to do. What it no longer does is contribute
|
|
1611
|
+
* nothing in silence.
|
|
1332
1612
|
* ===========================================================================================
|
|
1333
1613
|
*
|
|
1334
1614
|
* @returns {number}
|
|
@@ -1358,6 +1638,9 @@ class PhiScan {
|
|
|
1358
1638
|
/** `all` mode's index, read once: it is the union half's whole enumeration. */
|
|
1359
1639
|
/** @type {Map<string, import("./phi-scan.js").IndexEntry> | null} */
|
|
1360
1640
|
let index = null;
|
|
1641
|
+
/** The index route's refusals, raised AFTER the walk sweep so no hit is swallowed. */
|
|
1642
|
+
/** @type {string[]} */
|
|
1643
|
+
let indexRefusals = [];
|
|
1361
1644
|
try {
|
|
1362
1645
|
// `loadAllowList()` IS INSIDE THIS HANDLER, AND THAT PLACEMENT IS THE POINT. Outside it, a
|
|
1363
1646
|
// missing allow-list escaped as an uncaught throw and took node's exit 1, which this contract
|
|
@@ -1369,6 +1652,7 @@ class PhiScan {
|
|
|
1369
1652
|
const built = this.buildTargetsForAll();
|
|
1370
1653
|
targets = built.targets;
|
|
1371
1654
|
index = built.index;
|
|
1655
|
+
indexRefusals = built.indexRefusals;
|
|
1372
1656
|
}
|
|
1373
1657
|
} catch (err) {
|
|
1374
1658
|
if (err instanceof InvocationError) {
|
|
@@ -1383,10 +1667,17 @@ class PhiScan {
|
|
|
1383
1667
|
// does not fire on them. In `all` mode it is the walk's targets UNION the in-scope tracked paths.
|
|
1384
1668
|
const enumerated = new Set(targets.map((t) => t.path));
|
|
1385
1669
|
if (index !== null) for (const p of this.unionCandidatePaths(index)) enumerated.add(p);
|
|
1670
|
+
// IN `all` MODE A BYPASS IS ITSELF AN ENUMERATION (AC-17). The run's declared scope is the whole
|
|
1671
|
+
// corpus, so naming a path in `--allow-fixture` says "this run would have read that", and the
|
|
1672
|
+
// honest tier for withdrawing it is the completeness rule rather than the unmatched one. It
|
|
1673
|
+
// holds whatever the sweep happens to reach: a path the walk never listed and the index never
|
|
1674
|
+
// carried is still refused HERE, naming it, instead of being reported as subtracting nothing.
|
|
1675
|
+
if (args.mode === "all") for (const p of allowed) enumerated.add(p);
|
|
1386
1676
|
|
|
1387
1677
|
// TIER: A BYPASS MUST NAME A PATH THIS RUN ENUMERATES. Otherwise it subtracts nothing, and a
|
|
1388
1678
|
// flag that subtracts nothing lets a developer believe a file was acknowledged when the run
|
|
1389
|
-
// never had it in scope. Compared by DIFFERENCE, and every offender is named.
|
|
1679
|
+
// never had it in scope. Compared by DIFFERENCE, and every offender is named. In `all` mode the
|
|
1680
|
+
// line above makes this tier unreachable, deliberately: there it is the completeness rule's job.
|
|
1390
1681
|
//
|
|
1391
1682
|
// THIS TIER FIRES BEFORE ANY TARGET IS READ, so no hit exists for it to swallow. That is a
|
|
1392
1683
|
// narrower guarantee than the unread tier's and is stated as such rather than generalised.
|
|
@@ -1410,6 +1701,13 @@ class PhiScan {
|
|
|
1410
1701
|
/** @type {Map<string, string>} */
|
|
1411
1702
|
const readOids = new Map();
|
|
1412
1703
|
const objectHash = index === null ? null : this.gitObjectHash();
|
|
1704
|
+
/**
|
|
1705
|
+
* Paths the walk listed and that were GONE by read time. WHAT THIS MEANS IS DECIDED AFTER THE
|
|
1706
|
+
* SWEEP, not here: see the resolution below for why the window is left open as long as the
|
|
1707
|
+
* run lasts.
|
|
1708
|
+
*/
|
|
1709
|
+
/** @type {Set<string>} */
|
|
1710
|
+
const vanished = new Set();
|
|
1413
1711
|
|
|
1414
1712
|
/**
|
|
1415
1713
|
* @param {import("./phi-scan.js").Target[]} batch
|
|
@@ -1422,6 +1720,13 @@ class PhiScan {
|
|
|
1422
1720
|
try {
|
|
1423
1721
|
bytes = this.scanTarget(t, allow, hits);
|
|
1424
1722
|
} catch (err) {
|
|
1723
|
+
// THE ENUMERATION TOCTOU WINDOW: the ONE read failure that is not settled here. It is
|
|
1724
|
+
// recorded and judged after the sweep, because whether the path came back is part of
|
|
1725
|
+
// the answer and this is the earliest the run could possibly know.
|
|
1726
|
+
if (this.isVanishedUntracked(err, t, index)) {
|
|
1727
|
+
vanished.add(t.path);
|
|
1728
|
+
continue;
|
|
1729
|
+
}
|
|
1425
1730
|
if (err instanceof InvocationError) {
|
|
1426
1731
|
// HITS FOUND SO FAR ARE PRINTED BEFORE THIS REFUSAL, DELIBERATELY, AND THIS IS A CHANGE
|
|
1427
1732
|
// FROM THE COPIED SCANNERS. A refuter measured the old ordering: a fatal partway through
|
|
@@ -1447,6 +1752,18 @@ class PhiScan {
|
|
|
1447
1752
|
const walkFailure = sweep(targets);
|
|
1448
1753
|
if (walkFailure !== null) return walkFailure;
|
|
1449
1754
|
|
|
1755
|
+
// THE INDEX ROUTE'S OWN REFUSALS, RAISED HERE RATHER THAN WHERE THEY ARE COMPUTED (AC-15). An
|
|
1756
|
+
// unmerged entry or a tracked link anywhere in the index refuses the whole sweep, and the walk
|
|
1757
|
+
// may have found PHI under a root that yielded perfectly well. Refusing at enumeration time
|
|
1758
|
+
// discarded those findings: measured, exit 2 with the refusal alone over a repository whose walk
|
|
1759
|
+
// had five hits in hand. The exit code is still the refusal's, because an incomplete sweep is
|
|
1760
|
+
// not a verdict whatever it found.
|
|
1761
|
+
if (indexRefusals.length > 0) {
|
|
1762
|
+
this.reportHits(hits);
|
|
1763
|
+
for (const message of indexRefusals) process.stderr.write(`[phi-scan] ${message}\n`);
|
|
1764
|
+
return EXIT_REFUSE;
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1450
1767
|
// THE UNION. It runs AFTER the walk, not instead of it, and only over the paths the walk did not
|
|
1451
1768
|
// already read verbatim.
|
|
1452
1769
|
if (index !== null) {
|
|
@@ -1454,14 +1771,61 @@ class PhiScan {
|
|
|
1454
1771
|
if (unionFailure !== null) return unionFailure;
|
|
1455
1772
|
}
|
|
1456
1773
|
|
|
1774
|
+
// THE TOCTOU WINDOW IS RESOLVED AT THE END OF THE RUN, NOT AT THE MOMENT OF THE FAILURE, AND
|
|
1775
|
+
// THE LATE ANSWER IS THE STRONGER ONE. A path that was absent when the sweep reached it and is
|
|
1776
|
+
// present now was not a file being deleted: something wrote there while this run was looking
|
|
1777
|
+
// away, and no verdict this run can give covers the bytes that lived at it in between. Asking
|
|
1778
|
+
// immediately would answer within microseconds of the failure and miss almost all of that.
|
|
1779
|
+
/** @type {string[]} */
|
|
1780
|
+
const reappeared = [];
|
|
1781
|
+
/** @type {Set<string>} */
|
|
1782
|
+
const skipped = new Set();
|
|
1783
|
+
for (const p of vanished) {
|
|
1784
|
+
if (lstatOrNull(this.absolutePath(p)) === null) skipped.add(p);
|
|
1785
|
+
else reappeared.push(p);
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1457
1788
|
// THE COMPLETENESS RULE. A SET DIFFERENCE, NEVER A SIZE COMPARISON: a count counts the targets
|
|
1458
1789
|
// that DID get read, so `n read of n targets` is exactly the arithmetic that hides which ones did
|
|
1459
1790
|
// not. Names every offender.
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
//
|
|
1791
|
+
//
|
|
1792
|
+
// THE SKIPPED SET IS THE ONE SUBTRACTION, AND IT IS BOUNDED BY `isVanishedUntracked` RATHER
|
|
1793
|
+
// THAN BY A JUDGEMENT CALL: `all` mode, the walk's own target, `ENOENT`, untracked, and still
|
|
1794
|
+
// absent at the end of the run. Nothing git carries can enter it, so the rule keeps its whole
|
|
1795
|
+
// meaning over the repository and gives up only what the filesystem took away mid-run.
|
|
1796
|
+
const unread = [...enumerated].filter((p) => !read.has(p) && !skipped.has(p));
|
|
1797
|
+
|
|
1798
|
+
// THE PER-ROOT OBSERVATION RULE, `all` MODE ONLY. The completeness rule asks whether every
|
|
1799
|
+
// target was read; it cannot ask whether a root produced a target in the first place, and a
|
|
1800
|
+
// root that produces none is the silently-narrowed sweep: a typo, a directory that moved, a
|
|
1801
|
+
// root whose every file the read filter drops. One productive root makes the whole run look
|
|
1802
|
+
// productive, so the question is asked per root and answered from `read`.
|
|
1803
|
+
const starved =
|
|
1804
|
+
index === null ? [] : this.cfg.scanRoots.filter((root) => !this.anyReadUnder(root, read));
|
|
1805
|
+
|
|
1806
|
+
// Hits FIRST, so nothing below can swallow one.
|
|
1463
1807
|
this.reportHits(hits);
|
|
1464
1808
|
|
|
1809
|
+
if (skipped.size > 0) {
|
|
1810
|
+
process.stderr.write(
|
|
1811
|
+
`[phi-scan] skipped ${String(skipped.size)} untracked file(s) gone between enumeration ` +
|
|
1812
|
+
`and read:\n${[...skipped].map((p) => ` - ${p}`).join("\n")}\n` +
|
|
1813
|
+
`Nothing this repository carries was left unread: git has no bytes at such a path. ` +
|
|
1814
|
+
`Re-run the scan if the file was meant to stay.\n`,
|
|
1815
|
+
);
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
if (reappeared.length > 0) {
|
|
1819
|
+
process.stderr.write(
|
|
1820
|
+
`[phi-scan] refusing the scan: ${String(reappeared.length)} target(s) were absent when ` +
|
|
1821
|
+
`the sweep reached them and present again at the end of the run:\n` +
|
|
1822
|
+
`${reappeared.map((p) => ` - ${p}`).join("\n")}\n` +
|
|
1823
|
+
`A path that came back carried bytes this run never opened, so the scan has no verdict ` +
|
|
1824
|
+
`about it. Re-run the scan on a tree nothing else is writing to.\n`,
|
|
1825
|
+
);
|
|
1826
|
+
return EXIT_REFUSE;
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1465
1829
|
if (unread.length > 0) {
|
|
1466
1830
|
process.stderr.write(
|
|
1467
1831
|
`[phi-scan] refusing the scan: ${String(unread.length)} target(s) were enumerated and ` +
|
|
@@ -1473,14 +1837,43 @@ class PhiScan {
|
|
|
1473
1837
|
return EXIT_REFUSE;
|
|
1474
1838
|
}
|
|
1475
1839
|
|
|
1840
|
+
if (starved.length > 0) {
|
|
1841
|
+
process.stderr.write(
|
|
1842
|
+
`[phi-scan] refusing the sweep: the all-mode sweep observed no files under ` +
|
|
1843
|
+
`${String(starved.length)} of its ${String(this.cfg.scanRoots.length)} scan roots ` +
|
|
1844
|
+
`(${starved.join(", ")}).\n` +
|
|
1845
|
+
`A root that contributes nothing leaves this sweep covering less than its own ` +
|
|
1846
|
+
`configuration says, and no other tier can see that: the index route reads what git ` +
|
|
1847
|
+
`carries and cannot vouch for a directory on disk. Point the root at a directory or ` +
|
|
1848
|
+
`a file this run can read, widen the read filter that drops its contents, or drop the ` +
|
|
1849
|
+
`root.\n`,
|
|
1850
|
+
);
|
|
1851
|
+
return EXIT_REFUSE;
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1476
1854
|
if (hits.length > 0) return EXIT_HITS;
|
|
1477
|
-
|
|
1855
|
+
// THE CLEAN LINE CARRIES THE SUBTRACTION (AC-13). A reader watching stdout alone must not be
|
|
1856
|
+
// able to take an unqualified OK from a sweep that skipped a file: the skip is on stderr, and
|
|
1857
|
+
// stderr is where a CI log buries it. The unqualified line is byte-for-byte what it was when
|
|
1858
|
+
// nothing was skipped, so a consumer matching it exactly still matches.
|
|
1859
|
+
process.stdout.write(
|
|
1860
|
+
skipped.size > 0
|
|
1861
|
+
? `[phi-scan] OK: no hits (${String(skipped.size)} untracked file(s) skipped, see stderr)\n`
|
|
1862
|
+
: "[phi-scan] OK: no hits\n",
|
|
1863
|
+
);
|
|
1478
1864
|
return EXIT_CLEAN;
|
|
1479
1865
|
}
|
|
1480
1866
|
}
|
|
1481
1867
|
|
|
1482
|
-
/**
|
|
1483
|
-
|
|
1868
|
+
/**
|
|
1869
|
+
* `:<srcmode> <dstmode> <srcsha> <dstsha> <status>`: the info half of a `--raw -z` record.
|
|
1870
|
+
*
|
|
1871
|
+
* THE STATUS LETTER IS CAPTURED, not merely matched, because `U` (unmerged) is the one status whose
|
|
1872
|
+
* refusal has to say something different: its destination mode is `000000`, and calling that "a git
|
|
1873
|
+
* mode-000000 entry" beside a sentence about content git holds at the path is false twice over for a
|
|
1874
|
+
* conflicted regular file.
|
|
1875
|
+
*/
|
|
1876
|
+
const RAW_RECORD = /^:(?:\d{6}) (\d{6}) [0-9a-f]+ [0-9a-f]+ ([A-Z])\d*$/;
|
|
1484
1877
|
|
|
1485
1878
|
/**
|
|
1486
1879
|
* `lstatSync` that reports a missing path as `null` rather than throwing. LSTAT, not stat, so a
|