@crewhaus/tool-secure 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +273 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.js +1006 -0
- package/dist/lib/entropy.d.ts +65 -0
- package/dist/lib/entropy.js +99 -0
- package/dist/lib/evidence.d.ts +55 -0
- package/dist/lib/evidence.js +134 -0
- package/dist/lib/files.d.ts +71 -0
- package/dist/lib/files.js +227 -0
- package/dist/lib/injection.d.ts +38 -0
- package/dist/lib/injection.js +396 -0
- package/dist/lib/pii.d.ts +91 -0
- package/dist/lib/pii.js +757 -0
- package/dist/lib/policy.d.ts +42 -0
- package/dist/lib/policy.js +138 -0
- package/dist/lib/redact.d.ts +41 -0
- package/dist/lib/redact.js +121 -0
- package/dist/lib/secrets.d.ts +111 -0
- package/dist/lib/secrets.js +329 -0
- package/dist/lib/text.d.ts +129 -0
- package/dist/lib/text.js +191 -0
- package/dist/lib/unicode.d.ts +106 -0
- package/dist/lib/unicode.js +431 -0
- package/dist/lib/url.d.ts +35 -0
- package/dist/lib/url.js +363 -0
- package/dist/paths.d.ts +42 -0
- package/dist/paths.js +158 -0
- package/package.json +45 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shannon entropy over the symbols a string actually contains.
|
|
3
|
+
*
|
|
4
|
+
* ## What this measures
|
|
5
|
+
*
|
|
6
|
+
* `H = -Σ p(c) · log2 p(c)`, where `p(c)` is the frequency of character `c`
|
|
7
|
+
* in THIS string. Units are bits per character. It is a property of the
|
|
8
|
+
* string's own symbol distribution and nothing else — it has no model of
|
|
9
|
+
* English, of base64, or of what a key looks like.
|
|
10
|
+
*
|
|
11
|
+
* ## What it does not measure
|
|
12
|
+
*
|
|
13
|
+
* Not randomness, and not secrecy. `"abcdefgh"` scores a perfect 3.0 bits
|
|
14
|
+
* per character over its own alphabet while being entirely predictable, and
|
|
15
|
+
* a genuine 128-bit key rendered as 32 hex characters cannot exceed 4.0
|
|
16
|
+
* because hex has only 16 symbols. Short strings score low by construction:
|
|
17
|
+
* a 6-character string cannot exceed log2(6) ≈ 2.58.
|
|
18
|
+
*
|
|
19
|
+
* That is why every entropy threshold in this package is paired with a
|
|
20
|
+
* minimum length and a charset check, and why the threshold is reported
|
|
21
|
+
* alongside the finding rather than hidden inside it.
|
|
22
|
+
*/
|
|
23
|
+
/** Entropy in bits per character, plus the observed alphabet size. */
|
|
24
|
+
export declare function shannonEntropy(value: string): {
|
|
25
|
+
bits: number;
|
|
26
|
+
alphabet: number;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Round to 4 decimal places so the same input serializes to the same bytes.
|
|
30
|
+
*
|
|
31
|
+
* The convention is HALF-UP TOWARDS POSITIVE INFINITY — `Math.round`'s rule,
|
|
32
|
+
* so 0.00005 rounds to 0.0001 and -0.00005 rounds to -0. Entropy is never
|
|
33
|
+
* negative, so the asymmetry is unreachable here, but the rule is named
|
|
34
|
+
* because an unstated rounding mode is how two hosts disagree about a value
|
|
35
|
+
* they both computed correctly.
|
|
36
|
+
*/
|
|
37
|
+
export declare function roundBits(bits: number): number;
|
|
38
|
+
export type Charset = "hex" | "base64" | "base64url" | "alphanumeric" | "printable" | "mixed";
|
|
39
|
+
/**
|
|
40
|
+
* Classify the alphabet a string is drawn from. The order matters: the
|
|
41
|
+
* narrowest matching class wins, because the narrower the alphabet the lower
|
|
42
|
+
* the entropy ceiling, and thresholds are per-class for exactly that reason.
|
|
43
|
+
*/
|
|
44
|
+
export declare function classifyCharset(value: string): Charset;
|
|
45
|
+
/**
|
|
46
|
+
* The bits-per-character floor above which a string of this charset is worth
|
|
47
|
+
* a look. Derived from each alphabet's ceiling (log2 of its size) with room
|
|
48
|
+
* left for the uneven symbol distribution of a real 32-character sample:
|
|
49
|
+
* hex tops out at 4.0, base64 at 6.0.
|
|
50
|
+
*
|
|
51
|
+
* These are heuristics, stated here so a caller can see and override them.
|
|
52
|
+
*/
|
|
53
|
+
export declare const DEFAULT_ENTROPY_THRESHOLDS: Readonly<Record<Charset, number>>;
|
|
54
|
+
/** The shortest run that the high-entropy rule will consider at all. */
|
|
55
|
+
export declare const MIN_HIGH_ENTROPY_LENGTH = 20;
|
|
56
|
+
/**
|
|
57
|
+
* Is this string a plausible opaque credential? Length, charset and entropy
|
|
58
|
+
* together — any one of the three alone is a false-positive machine.
|
|
59
|
+
*/
|
|
60
|
+
export declare function looksHighEntropy(value: string, override?: number): {
|
|
61
|
+
high: boolean;
|
|
62
|
+
bits: number;
|
|
63
|
+
charset: Charset;
|
|
64
|
+
threshold: number;
|
|
65
|
+
};
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shannon entropy over the symbols a string actually contains.
|
|
3
|
+
*
|
|
4
|
+
* ## What this measures
|
|
5
|
+
*
|
|
6
|
+
* `H = -Σ p(c) · log2 p(c)`, where `p(c)` is the frequency of character `c`
|
|
7
|
+
* in THIS string. Units are bits per character. It is a property of the
|
|
8
|
+
* string's own symbol distribution and nothing else — it has no model of
|
|
9
|
+
* English, of base64, or of what a key looks like.
|
|
10
|
+
*
|
|
11
|
+
* ## What it does not measure
|
|
12
|
+
*
|
|
13
|
+
* Not randomness, and not secrecy. `"abcdefgh"` scores a perfect 3.0 bits
|
|
14
|
+
* per character over its own alphabet while being entirely predictable, and
|
|
15
|
+
* a genuine 128-bit key rendered as 32 hex characters cannot exceed 4.0
|
|
16
|
+
* because hex has only 16 symbols. Short strings score low by construction:
|
|
17
|
+
* a 6-character string cannot exceed log2(6) ≈ 2.58.
|
|
18
|
+
*
|
|
19
|
+
* That is why every entropy threshold in this package is paired with a
|
|
20
|
+
* minimum length and a charset check, and why the threshold is reported
|
|
21
|
+
* alongside the finding rather than hidden inside it.
|
|
22
|
+
*/
|
|
23
|
+
/** Entropy in bits per character, plus the observed alphabet size. */
|
|
24
|
+
export function shannonEntropy(value) {
|
|
25
|
+
if (value.length === 0)
|
|
26
|
+
return { bits: 0, alphabet: 0 };
|
|
27
|
+
const counts = new Map();
|
|
28
|
+
for (const ch of value)
|
|
29
|
+
counts.set(ch, (counts.get(ch) ?? 0) + 1);
|
|
30
|
+
const n = [...value].length;
|
|
31
|
+
let bits = 0;
|
|
32
|
+
for (const count of counts.values()) {
|
|
33
|
+
const p = count / n;
|
|
34
|
+
bits -= p * Math.log2(p);
|
|
35
|
+
}
|
|
36
|
+
// Clamp the -0 that a single-symbol string produces.
|
|
37
|
+
return { bits: bits === 0 ? 0 : bits, alphabet: counts.size };
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Round to 4 decimal places so the same input serializes to the same bytes.
|
|
41
|
+
*
|
|
42
|
+
* The convention is HALF-UP TOWARDS POSITIVE INFINITY — `Math.round`'s rule,
|
|
43
|
+
* so 0.00005 rounds to 0.0001 and -0.00005 rounds to -0. Entropy is never
|
|
44
|
+
* negative, so the asymmetry is unreachable here, but the rule is named
|
|
45
|
+
* because an unstated rounding mode is how two hosts disagree about a value
|
|
46
|
+
* they both computed correctly.
|
|
47
|
+
*/
|
|
48
|
+
export function roundBits(bits) {
|
|
49
|
+
return Math.round(bits * 10_000) / 10_000;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Classify the alphabet a string is drawn from. The order matters: the
|
|
53
|
+
* narrowest matching class wins, because the narrower the alphabet the lower
|
|
54
|
+
* the entropy ceiling, and thresholds are per-class for exactly that reason.
|
|
55
|
+
*/
|
|
56
|
+
export function classifyCharset(value) {
|
|
57
|
+
if (value.length === 0)
|
|
58
|
+
return "mixed";
|
|
59
|
+
if (/^[0-9a-fA-F]+$/.test(value))
|
|
60
|
+
return "hex";
|
|
61
|
+
if (/^[A-Za-z0-9+/]+={0,2}$/.test(value))
|
|
62
|
+
return "base64";
|
|
63
|
+
if (/^[A-Za-z0-9_-]+$/.test(value))
|
|
64
|
+
return "base64url";
|
|
65
|
+
if (/^[A-Za-z0-9]+$/.test(value))
|
|
66
|
+
return "alphanumeric";
|
|
67
|
+
if (/^[\x20-\x7e]+$/.test(value))
|
|
68
|
+
return "printable";
|
|
69
|
+
return "mixed";
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* The bits-per-character floor above which a string of this charset is worth
|
|
73
|
+
* a look. Derived from each alphabet's ceiling (log2 of its size) with room
|
|
74
|
+
* left for the uneven symbol distribution of a real 32-character sample:
|
|
75
|
+
* hex tops out at 4.0, base64 at 6.0.
|
|
76
|
+
*
|
|
77
|
+
* These are heuristics, stated here so a caller can see and override them.
|
|
78
|
+
*/
|
|
79
|
+
export const DEFAULT_ENTROPY_THRESHOLDS = {
|
|
80
|
+
hex: 3.2,
|
|
81
|
+
base64: 4.2,
|
|
82
|
+
base64url: 4.2,
|
|
83
|
+
alphanumeric: 4.0,
|
|
84
|
+
printable: 4.0,
|
|
85
|
+
mixed: 4.0,
|
|
86
|
+
};
|
|
87
|
+
/** The shortest run that the high-entropy rule will consider at all. */
|
|
88
|
+
export const MIN_HIGH_ENTROPY_LENGTH = 20;
|
|
89
|
+
/**
|
|
90
|
+
* Is this string a plausible opaque credential? Length, charset and entropy
|
|
91
|
+
* together — any one of the three alone is a false-positive machine.
|
|
92
|
+
*/
|
|
93
|
+
export function looksHighEntropy(value, override) {
|
|
94
|
+
const charset = classifyCharset(value);
|
|
95
|
+
const threshold = override ?? DEFAULT_ENTROPY_THRESHOLDS[charset];
|
|
96
|
+
const { bits } = shannonEntropy(value);
|
|
97
|
+
const high = value.length >= MIN_HIGH_ENTROPY_LENGTH && bits >= threshold;
|
|
98
|
+
return { high, bits: roundBits(bits), charset, threshold };
|
|
99
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/** Digest algorithms this package will use, and why each is here. */
|
|
2
|
+
export declare const HASH_ALGORITHMS: readonly ["sha256", "sha384", "sha512", "sha1"];
|
|
3
|
+
export type HashAlgorithm = (typeof HASH_ALGORITHMS)[number];
|
|
4
|
+
export declare const SIGNATURE_ENCODINGS: readonly ["hex", "base64url"];
|
|
5
|
+
export type SignatureEncoding = (typeof SIGNATURE_ENCODINGS)[number];
|
|
6
|
+
/** Bounds a chain: verification is linear, but the result is not. */
|
|
7
|
+
export declare const MAX_CHAIN_RECORDS = 50000;
|
|
8
|
+
export declare class EvidenceError extends Error {
|
|
9
|
+
readonly name = "EvidenceError";
|
|
10
|
+
}
|
|
11
|
+
export type ChainRecord = {
|
|
12
|
+
readonly data: string;
|
|
13
|
+
readonly prevHash: string;
|
|
14
|
+
readonly hash: string;
|
|
15
|
+
};
|
|
16
|
+
export type ChainOptions = {
|
|
17
|
+
readonly algorithm?: HashAlgorithm;
|
|
18
|
+
readonly separator?: string;
|
|
19
|
+
readonly genesisPrevHash?: string;
|
|
20
|
+
};
|
|
21
|
+
export type ChainBreak = {
|
|
22
|
+
readonly index: number;
|
|
23
|
+
readonly kind: "link.mismatch" | "hash.mismatch";
|
|
24
|
+
readonly expected: string;
|
|
25
|
+
readonly actual: string;
|
|
26
|
+
readonly explanation: string;
|
|
27
|
+
};
|
|
28
|
+
export type ChainResult = {
|
|
29
|
+
readonly ok: boolean;
|
|
30
|
+
readonly length: number;
|
|
31
|
+
readonly algorithm: HashAlgorithm;
|
|
32
|
+
/** Records checked before stopping. Equals `length` for an intact chain. */
|
|
33
|
+
readonly verified: number;
|
|
34
|
+
readonly headHash?: string;
|
|
35
|
+
readonly firstBreak?: ChainBreak;
|
|
36
|
+
};
|
|
37
|
+
/** One link, by the convention documented above. */
|
|
38
|
+
export declare function computeLink(prevHash: string, data: string, algorithm?: HashAlgorithm, separator?: string): string;
|
|
39
|
+
/**
|
|
40
|
+
* Walk the chain and stop at the first break.
|
|
41
|
+
*
|
|
42
|
+
* Stopping is the point: after a break every later hash is computed over
|
|
43
|
+
* altered material, so continuing would report a cascade of failures that
|
|
44
|
+
* all have one cause. The index reported is where to look.
|
|
45
|
+
*/
|
|
46
|
+
export declare function verifyChain(records: ReadonlyArray<ChainRecord>, options?: ChainOptions): ChainResult;
|
|
47
|
+
/** HMAC over a payload. The key never appears in the result. */
|
|
48
|
+
export declare function signPayload(key: string, payload: string, algorithm?: HashAlgorithm, encoding?: SignatureEncoding): string;
|
|
49
|
+
/**
|
|
50
|
+
* Constant-time verification via the double-HMAC comparison described above.
|
|
51
|
+
* A malformed signature is a `false`, not a throw.
|
|
52
|
+
*/
|
|
53
|
+
export declare function verifyPayload(key: string, payload: string, signature: string, algorithm?: HashAlgorithm, encoding?: SignatureEncoding): boolean;
|
|
54
|
+
/** SHA-256 of a string, hex. Used for evidence records about a document. */
|
|
55
|
+
export declare function sha256Hex(value: string): string;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evidence that a record was not altered: hash chains and HMAC signatures.
|
|
3
|
+
*
|
|
4
|
+
* ## The hash-chain convention, stated exactly
|
|
5
|
+
*
|
|
6
|
+
* A record is `{ data, prevHash, hash }`, all strings, and the link is
|
|
7
|
+
*
|
|
8
|
+
* hash = hex( H( prevHash + separator + data ) )
|
|
9
|
+
*
|
|
10
|
+
* with `separator` defaulting to `"\n"`, `H` defaulting to SHA-256, and the
|
|
11
|
+
* first record's `prevHash` defaulting to the empty string. Every part is an
|
|
12
|
+
* input because chains in the wild differ on all of them, and a verifier
|
|
13
|
+
* that guesses is a verifier that returns a confidently wrong "valid".
|
|
14
|
+
*
|
|
15
|
+
* `data` must be a string the producer already canonicalized. This module
|
|
16
|
+
* deliberately does not serialize objects for you: two JSON encoders
|
|
17
|
+
* disagree about key order and number formatting, and a chain that verifies
|
|
18
|
+
* under one and not the other is worse than no chain at all.
|
|
19
|
+
*
|
|
20
|
+
* ## What a valid chain proves
|
|
21
|
+
*
|
|
22
|
+
* That these records are internally consistent — nobody edited one without
|
|
23
|
+
* recomputing every hash after it. It does NOT prove who wrote them or when,
|
|
24
|
+
* and an attacker who can rewrite the whole sequence can produce a perfectly
|
|
25
|
+
* valid chain. For that you need a signature over the head, which is what
|
|
26
|
+
* `SignPayload` is for, or an external anchor.
|
|
27
|
+
*
|
|
28
|
+
* ## Signature comparison
|
|
29
|
+
*
|
|
30
|
+
* Verification compares `HMAC(key, expectedSignature)` with
|
|
31
|
+
* `HMAC(key, providedSignature)` rather than the signatures themselves. The
|
|
32
|
+
* two HMACs are always the same length, so `timingSafeEqual` can be used
|
|
33
|
+
* even when the caller supplies a signature of the wrong length, and an
|
|
34
|
+
* attacker cannot learn the expected bytes from how long the comparison
|
|
35
|
+
* took.
|
|
36
|
+
*/
|
|
37
|
+
import { createHash, createHmac, timingSafeEqual } from "node:crypto";
|
|
38
|
+
/** Digest algorithms this package will use, and why each is here. */
|
|
39
|
+
export const HASH_ALGORITHMS = ["sha256", "sha384", "sha512", "sha1"];
|
|
40
|
+
export const SIGNATURE_ENCODINGS = ["hex", "base64url"];
|
|
41
|
+
/** Bounds a chain: verification is linear, but the result is not. */
|
|
42
|
+
export const MAX_CHAIN_RECORDS = 50_000;
|
|
43
|
+
export class EvidenceError extends Error {
|
|
44
|
+
name = "EvidenceError";
|
|
45
|
+
}
|
|
46
|
+
/** One link, by the convention documented above. */
|
|
47
|
+
export function computeLink(prevHash, data, algorithm = "sha256", separator = "\n") {
|
|
48
|
+
return createHash(algorithm).update(`${prevHash}${separator}${data}`, "utf8").digest("hex");
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Walk the chain and stop at the first break.
|
|
52
|
+
*
|
|
53
|
+
* Stopping is the point: after a break every later hash is computed over
|
|
54
|
+
* altered material, so continuing would report a cascade of failures that
|
|
55
|
+
* all have one cause. The index reported is where to look.
|
|
56
|
+
*/
|
|
57
|
+
export function verifyChain(records, options = {}) {
|
|
58
|
+
if (records.length > MAX_CHAIN_RECORDS) {
|
|
59
|
+
throw new EvidenceError(`${records.length} records, over the ${MAX_CHAIN_RECORDS} limit — verify the chain in segments`);
|
|
60
|
+
}
|
|
61
|
+
const algorithm = options.algorithm ?? "sha256";
|
|
62
|
+
const separator = options.separator ?? "\n";
|
|
63
|
+
const genesis = options.genesisPrevHash ?? "";
|
|
64
|
+
let previousHash = genesis;
|
|
65
|
+
for (let i = 0; i < records.length; i++) {
|
|
66
|
+
const record = records[i];
|
|
67
|
+
if (!record)
|
|
68
|
+
continue;
|
|
69
|
+
if (record.prevHash !== previousHash) {
|
|
70
|
+
return {
|
|
71
|
+
ok: false,
|
|
72
|
+
length: records.length,
|
|
73
|
+
algorithm,
|
|
74
|
+
verified: i,
|
|
75
|
+
firstBreak: {
|
|
76
|
+
index: i,
|
|
77
|
+
kind: "link.mismatch",
|
|
78
|
+
expected: previousHash,
|
|
79
|
+
actual: record.prevHash,
|
|
80
|
+
explanation: i === 0
|
|
81
|
+
? "the first record's prevHash is not the genesis value — pass genesisPrevHash if this chain starts somewhere else"
|
|
82
|
+
: "this record does not point at the previous record's hash: a record was inserted, removed or reordered",
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
const expected = computeLink(record.prevHash, record.data, algorithm, separator);
|
|
87
|
+
if (expected !== record.hash) {
|
|
88
|
+
return {
|
|
89
|
+
ok: false,
|
|
90
|
+
length: records.length,
|
|
91
|
+
algorithm,
|
|
92
|
+
verified: i,
|
|
93
|
+
firstBreak: {
|
|
94
|
+
index: i,
|
|
95
|
+
kind: "hash.mismatch",
|
|
96
|
+
expected,
|
|
97
|
+
actual: record.hash,
|
|
98
|
+
explanation: "the stored hash does not match this record's own data: the data was altered, or it was hashed under a different algorithm or separator",
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
previousHash = record.hash;
|
|
103
|
+
}
|
|
104
|
+
return {
|
|
105
|
+
ok: true,
|
|
106
|
+
length: records.length,
|
|
107
|
+
algorithm,
|
|
108
|
+
verified: records.length,
|
|
109
|
+
...(records.length > 0 ? { headHash: previousHash } : {}),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/** Encode a digest in the requested form. */
|
|
113
|
+
function encode(buffer, encoding) {
|
|
114
|
+
return encoding === "hex" ? buffer.toString("hex") : buffer.toString("base64url");
|
|
115
|
+
}
|
|
116
|
+
/** HMAC over a payload. The key never appears in the result. */
|
|
117
|
+
export function signPayload(key, payload, algorithm = "sha256", encoding = "hex") {
|
|
118
|
+
return encode(createHmac(algorithm, key).update(payload, "utf8").digest(), encoding);
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Constant-time verification via the double-HMAC comparison described above.
|
|
122
|
+
* A malformed signature is a `false`, not a throw.
|
|
123
|
+
*/
|
|
124
|
+
export function verifyPayload(key, payload, signature, algorithm = "sha256", encoding = "hex") {
|
|
125
|
+
const expected = signPayload(key, payload, algorithm, encoding);
|
|
126
|
+
const blind = (value) => createHmac(algorithm, key).update(value, "utf8").digest();
|
|
127
|
+
const a = blind(expected);
|
|
128
|
+
const b = blind(signature);
|
|
129
|
+
return a.length === b.length && timingSafeEqual(a, b);
|
|
130
|
+
}
|
|
131
|
+
/** SHA-256 of a string, hex. Used for evidence records about a document. */
|
|
132
|
+
export function sha256Hex(value) {
|
|
133
|
+
return createHash("sha256").update(value, "utf8").digest("hex");
|
|
134
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { type SafePath } from "../paths";
|
|
2
|
+
/** Default per-file ceiling. Generous for source, refuses a database dump. */
|
|
3
|
+
export declare const DEFAULT_MAX_FILE_BYTES: number;
|
|
4
|
+
/** Hard ceiling a caller cannot raise past. */
|
|
5
|
+
export declare const MAX_FILE_BYTES_LIMIT: number;
|
|
6
|
+
/** Directories skipped unless the caller names them explicitly. */
|
|
7
|
+
export declare const DEFAULT_SKIP_DIRS: ReadonlyArray<string>;
|
|
8
|
+
/**
|
|
9
|
+
* Why a path in the tree was not scanned.
|
|
10
|
+
*
|
|
11
|
+
* The first three are read failures. The last three are DELIBERATE
|
|
12
|
+
* exclusions, and they are listed for the same reason the others are: a
|
|
13
|
+
* secret scanner whose walk quietly drops `.env`, `node_modules/` and every
|
|
14
|
+
* symlink, and then returns `skipped: []`, is telling the caller it covered
|
|
15
|
+
* a tree it did not open.
|
|
16
|
+
*/
|
|
17
|
+
export type SkipReason = "too-large" | "binary" | "unreadable" | "symlink" | "hidden" | "excluded-directory";
|
|
18
|
+
export type ScannedFile = {
|
|
19
|
+
/** Slash-separated, relative to the scan root. */
|
|
20
|
+
readonly rel: string;
|
|
21
|
+
readonly abs: string;
|
|
22
|
+
readonly bytes: number;
|
|
23
|
+
readonly text: string;
|
|
24
|
+
};
|
|
25
|
+
export type SkippedFile = {
|
|
26
|
+
readonly rel: string;
|
|
27
|
+
readonly reason: SkipReason;
|
|
28
|
+
readonly detail: string;
|
|
29
|
+
};
|
|
30
|
+
export type WalkLimits = {
|
|
31
|
+
readonly maxFiles: number;
|
|
32
|
+
readonly maxDepth: number;
|
|
33
|
+
readonly maxFileBytes: number;
|
|
34
|
+
readonly includeHidden: boolean;
|
|
35
|
+
readonly skipDirs: ReadonlyArray<string>;
|
|
36
|
+
};
|
|
37
|
+
export declare const DEFAULT_WALK_LIMITS: WalkLimits;
|
|
38
|
+
export type ReadOutcome = {
|
|
39
|
+
readonly ok: true;
|
|
40
|
+
readonly text: string;
|
|
41
|
+
readonly bytes: number;
|
|
42
|
+
} | {
|
|
43
|
+
readonly ok: false;
|
|
44
|
+
readonly reason: SkipReason;
|
|
45
|
+
readonly detail: string;
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Read a file as UTF-8, refusing it when it is too large or not text.
|
|
49
|
+
*
|
|
50
|
+
* The size check happens against the open descriptor before a single byte is
|
|
51
|
+
* buffered, which is what makes the limit a memory bound rather than a
|
|
52
|
+
* post-hoc complaint.
|
|
53
|
+
*/
|
|
54
|
+
export declare function readTextBounded(toolName: string, abs: string, maxBytes: number): ReadOutcome;
|
|
55
|
+
export type WalkOutcome = {
|
|
56
|
+
readonly files: ReadonlyArray<ScannedFile>;
|
|
57
|
+
readonly skipped: ReadonlyArray<SkippedFile>;
|
|
58
|
+
/** True when the entry cap or the depth cap cut the walk short. */
|
|
59
|
+
readonly truncated: boolean;
|
|
60
|
+
readonly depthLimited: boolean;
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* Collect the readable text files under `root`, depth-first in sorted order.
|
|
64
|
+
*
|
|
65
|
+
* Symlinks are named in `skipped` but never followed, for containment and to
|
|
66
|
+
* make cycles impossible rather than merely unlikely. So are dot-entries and
|
|
67
|
+
* the default-excluded directories: what was not opened is always said.
|
|
68
|
+
*/
|
|
69
|
+
export declare function walkTextFiles(toolName: string, root: SafePath, limits?: WalkLimits): WalkOutcome;
|
|
70
|
+
/** Relative, slash-separated label for a path inside the workspace. */
|
|
71
|
+
export declare function relLabel(safe: SafePath): string;
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The only filesystem access in this package, kept in one place so the
|
|
3
|
+
* bounds are in one place too.
|
|
4
|
+
*
|
|
5
|
+
* Three properties hold for every read here:
|
|
6
|
+
*
|
|
7
|
+
* 1. CONTAINMENT. The caller's path has already been through
|
|
8
|
+
* `resolveSafe`; directories are walked without following symlinks, so
|
|
9
|
+
* a link inside the workspace cannot walk the scan out of it.
|
|
10
|
+
* 2. BOUNDED MEMORY. A file's size is taken from its open descriptor and
|
|
11
|
+
* compared with the limit BEFORE any bytes are read. A limit applied
|
|
12
|
+
* after buffering is not a limit.
|
|
13
|
+
* 3. BOUNDED WORK. The walk has an entry cap and a depth cap, and both are
|
|
14
|
+
* reported when they bite, so a truncated scan never looks complete.
|
|
15
|
+
* 4. NO SILENT EXCLUSION. Everything the walk declines to open — a symlink,
|
|
16
|
+
* a dot-file, an excluded directory, a binary, an oversized file — is
|
|
17
|
+
* named in `skipped` with the reason. A scanner that drops `.env` and
|
|
18
|
+
* `node_modules/` and then reports `skipped: []` is claiming coverage it
|
|
19
|
+
* does not have, which is the only failure here that gets someone hurt.
|
|
20
|
+
*
|
|
21
|
+
* Determinism: entries are sorted with plain string comparison, so the same
|
|
22
|
+
* tree produces the same listing and the same truncation point on any host.
|
|
23
|
+
*/
|
|
24
|
+
import { closeSync, constants as fsConstants, fstatSync, openSync, readSync, readdirSync, } from "node:fs";
|
|
25
|
+
import * as path from "node:path";
|
|
26
|
+
import { ToolPermissionError, toPosix } from "../paths";
|
|
27
|
+
import { compareStrings } from "./text";
|
|
28
|
+
/** Default per-file ceiling. Generous for source, refuses a database dump. */
|
|
29
|
+
export const DEFAULT_MAX_FILE_BYTES = 1024 * 1024;
|
|
30
|
+
/** Hard ceiling a caller cannot raise past. */
|
|
31
|
+
export const MAX_FILE_BYTES_LIMIT = 16 * 1024 * 1024;
|
|
32
|
+
/** How much of a file is inspected for NUL bytes before calling it binary. */
|
|
33
|
+
const BINARY_PROBE_BYTES = 8192;
|
|
34
|
+
/** Directories skipped unless the caller names them explicitly. */
|
|
35
|
+
export const DEFAULT_SKIP_DIRS = [
|
|
36
|
+
".git",
|
|
37
|
+
".hg",
|
|
38
|
+
".svn",
|
|
39
|
+
"node_modules",
|
|
40
|
+
"dist",
|
|
41
|
+
"build",
|
|
42
|
+
"coverage",
|
|
43
|
+
".next",
|
|
44
|
+
".turbo",
|
|
45
|
+
".venv",
|
|
46
|
+
"__pycache__",
|
|
47
|
+
];
|
|
48
|
+
export const DEFAULT_WALK_LIMITS = {
|
|
49
|
+
maxFiles: 2000,
|
|
50
|
+
maxDepth: 12,
|
|
51
|
+
maxFileBytes: DEFAULT_MAX_FILE_BYTES,
|
|
52
|
+
includeHidden: false,
|
|
53
|
+
skipDirs: DEFAULT_SKIP_DIRS,
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* Open without following a final symlink. `resolveSafe` proved containment,
|
|
57
|
+
* but the leaf could be swapped between the check and the open (CWE-367);
|
|
58
|
+
* refusing to follow it closes that window.
|
|
59
|
+
*/
|
|
60
|
+
function openNoFollow(toolName, abs) {
|
|
61
|
+
try {
|
|
62
|
+
return openSync(abs, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
if (err.code === "ELOOP") {
|
|
66
|
+
throw new ToolPermissionError(toolName, abs);
|
|
67
|
+
}
|
|
68
|
+
throw err;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Read a file as UTF-8, refusing it when it is too large or not text.
|
|
73
|
+
*
|
|
74
|
+
* The size check happens against the open descriptor before a single byte is
|
|
75
|
+
* buffered, which is what makes the limit a memory bound rather than a
|
|
76
|
+
* post-hoc complaint.
|
|
77
|
+
*/
|
|
78
|
+
export function readTextBounded(toolName, abs, maxBytes) {
|
|
79
|
+
let fd;
|
|
80
|
+
try {
|
|
81
|
+
fd = openNoFollow(toolName, abs);
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
if (err instanceof ToolPermissionError)
|
|
85
|
+
throw err;
|
|
86
|
+
return { ok: false, reason: "unreadable", detail: err.message };
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
const stat = fstatSync(fd);
|
|
90
|
+
// A directory is not an oversized file, and the order of these two checks
|
|
91
|
+
// is load-bearing. A directory's st_size is 4096 on ext4 and a handful of
|
|
92
|
+
// bytes on APFS, so comparing it against the per-file cap first made
|
|
93
|
+
// `SecretScan({path: ".", maxFileBytes: 100})` walk the tree on macOS and
|
|
94
|
+
// refuse it outright on Linux — reported as a skipped oversized file
|
|
95
|
+
// rather than as a scan that covered nothing.
|
|
96
|
+
if (stat.isDirectory()) {
|
|
97
|
+
return { ok: false, reason: "unreadable", detail: "is a directory" };
|
|
98
|
+
}
|
|
99
|
+
const { size } = stat;
|
|
100
|
+
if (size > maxBytes) {
|
|
101
|
+
return {
|
|
102
|
+
ok: false,
|
|
103
|
+
reason: "too-large",
|
|
104
|
+
detail: `${size} bytes, over the ${maxBytes}-byte per-file limit`,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
const probeLength = Math.min(size, BINARY_PROBE_BYTES);
|
|
108
|
+
const probe = Buffer.allocUnsafe(probeLength);
|
|
109
|
+
if (probeLength > 0)
|
|
110
|
+
readSync(fd, probe, 0, probeLength, 0);
|
|
111
|
+
if (probe.includes(0)) {
|
|
112
|
+
return { ok: false, reason: "binary", detail: "a NUL byte appears in the first 8 KiB" };
|
|
113
|
+
}
|
|
114
|
+
const buffer = Buffer.allocUnsafe(size);
|
|
115
|
+
let offset = 0;
|
|
116
|
+
while (offset < size) {
|
|
117
|
+
const read = readSync(fd, buffer, offset, size - offset, offset);
|
|
118
|
+
if (read <= 0)
|
|
119
|
+
break;
|
|
120
|
+
offset += read;
|
|
121
|
+
}
|
|
122
|
+
return { ok: true, text: buffer.subarray(0, offset).toString("utf8"), bytes: offset };
|
|
123
|
+
}
|
|
124
|
+
catch (err) {
|
|
125
|
+
return { ok: false, reason: "unreadable", detail: err.message };
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
closeSync(fd);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Collect the readable text files under `root`, depth-first in sorted order.
|
|
133
|
+
*
|
|
134
|
+
* Symlinks are named in `skipped` but never followed, for containment and to
|
|
135
|
+
* make cycles impossible rather than merely unlikely. So are dot-entries and
|
|
136
|
+
* the default-excluded directories: what was not opened is always said.
|
|
137
|
+
*/
|
|
138
|
+
export function walkTextFiles(toolName, root, limits = DEFAULT_WALK_LIMITS) {
|
|
139
|
+
const files = [];
|
|
140
|
+
const skipped = [];
|
|
141
|
+
let truncated = false;
|
|
142
|
+
let depthLimited = false;
|
|
143
|
+
const skipDirs = new Set(limits.skipDirs);
|
|
144
|
+
const visit = (dir, rel, depth) => {
|
|
145
|
+
if (files.length >= limits.maxFiles) {
|
|
146
|
+
truncated = true;
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
if (depth > limits.maxDepth) {
|
|
150
|
+
depthLimited = true;
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
let entries;
|
|
154
|
+
try {
|
|
155
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
156
|
+
}
|
|
157
|
+
catch (err) {
|
|
158
|
+
skipped.push({
|
|
159
|
+
rel: rel === "" ? "." : rel,
|
|
160
|
+
reason: "unreadable",
|
|
161
|
+
detail: err.message,
|
|
162
|
+
});
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
entries.sort((a, b) => compareStrings(a.name, b.name));
|
|
166
|
+
for (const entry of entries) {
|
|
167
|
+
if (files.length >= limits.maxFiles) {
|
|
168
|
+
truncated = true;
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
const childRel = rel === "" ? entry.name : `${rel}/${entry.name}`;
|
|
172
|
+
const childAbs = path.join(dir, entry.name);
|
|
173
|
+
if (!limits.includeHidden && entry.name.startsWith(".")) {
|
|
174
|
+
skipped.push({
|
|
175
|
+
rel: childRel,
|
|
176
|
+
reason: "hidden",
|
|
177
|
+
detail: "a dot-file or dot-directory; pass includeHidden to scan it",
|
|
178
|
+
});
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
if (entry.isSymbolicLink()) {
|
|
182
|
+
// Never followed, for containment and to make cycles impossible —
|
|
183
|
+
// but named, so the caller knows this path went unread.
|
|
184
|
+
skipped.push({
|
|
185
|
+
rel: childRel,
|
|
186
|
+
reason: "symlink",
|
|
187
|
+
detail: "a symbolic link; links are never followed, so its target was not scanned",
|
|
188
|
+
});
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
if (entry.isDirectory()) {
|
|
192
|
+
if (skipDirs.has(entry.name)) {
|
|
193
|
+
skipped.push({
|
|
194
|
+
rel: childRel,
|
|
195
|
+
reason: "excluded-directory",
|
|
196
|
+
detail: `"${entry.name}" is excluded by default; nothing under it was scanned`,
|
|
197
|
+
});
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
visit(childAbs, childRel, depth + 1);
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
if (!entry.isFile()) {
|
|
204
|
+
skipped.push({
|
|
205
|
+
rel: childRel,
|
|
206
|
+
reason: "unreadable",
|
|
207
|
+
detail: "not a regular file (socket, fifo or device); nothing was read from it",
|
|
208
|
+
});
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
const outcome = readTextBounded(toolName, childAbs, limits.maxFileBytes);
|
|
212
|
+
if (!outcome.ok) {
|
|
213
|
+
skipped.push({ rel: childRel, reason: outcome.reason, detail: outcome.detail });
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
files.push({ rel: childRel, abs: childAbs, bytes: outcome.bytes, text: outcome.text });
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
visit(root.real, "", 1);
|
|
220
|
+
files.sort((a, b) => compareStrings(a.rel, b.rel));
|
|
221
|
+
skipped.sort((a, b) => compareStrings(a.rel, b.rel) || compareStrings(a.reason, b.reason));
|
|
222
|
+
return { files, skipped, truncated, depthLimited };
|
|
223
|
+
}
|
|
224
|
+
/** Relative, slash-separated label for a path inside the workspace. */
|
|
225
|
+
export function relLabel(safe) {
|
|
226
|
+
return safe.rel === "" ? "." : toPosix(safe.rel);
|
|
227
|
+
}
|