dsh-dbhub-live 5.1.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +95 -233
- package/README.en.md +96 -11
- package/README.md +97 -12
- package/doc/ARCHITECTURE.md +108 -0
- package/doc/CLIENT.md +124 -0
- package/doc/DESENSITIZE.md +56 -0
- package/doc/DEVELOPMENT.md +139 -0
- package/doc/IMPACT-MATRIX.md +36 -0
- package/doc/RELEASE.md +93 -0
- package/doc/REQUIREMENTS.md +41 -3
- package/lib/client.js +704 -27
- package/lib/config.mjs +3 -0
- package/lib/i18n.mjs +14 -4
- package/lib/index.mjs +107 -0
- package/lib/meta.mjs +267 -0
- package/lib/options.mjs +16 -1
- package/lib/policy.mjs +92 -0
- package/lib/tools.mjs +4 -1
- package/lib/update.mjs +244 -0
- package/package.json +32 -9
- package/scripts/desensitize-check.mjs +331 -324
- package/scripts/desensitize.config.mjs +18 -10
- package/scripts/docs-check.mjs +214 -0
- package/scripts/release-preflight.mjs +180 -0
- package/scripts/render-check.mjs +286 -2
- package/test/client-format.test.mjs +89 -5
- package/test/meta.test.mjs +234 -0
- package/test/options.test.mjs +28 -15
- package/test/policy.test.mjs +132 -0
- package/test/update.test.mjs +270 -0
|
@@ -1,324 +1,331 @@
|
|
|
1
|
-
// Desensitization gate for dsh-dbhub-live.
|
|
2
|
-
//
|
|
3
|
-
// Runs two rule sets, both defined by scripts/desensitize.config.mjs:
|
|
4
|
-
// * a BACKSTOP over every scanned file (IP literals, secret formats, entropy)
|
|
5
|
-
// so an unregistered new surface is still caught;
|
|
6
|
-
// * the registered SLOTS' targeted rules (credentialed URLs, secret-keyword
|
|
7
|
-
// assignments, internal host suffixes, local
|
|
8
|
-
// values are actually plausible.
|
|
9
|
-
//
|
|
10
|
-
// The repository is treated as content that ships publicly (npm tarball and the
|
|
11
|
-
// public repository), so anything that looks like a real host, account,
|
|
12
|
-
// password, database or project name fails the gate. Real values belong in
|
|
13
|
-
// $DSH_HOME/storages/dsh-dbhub-live/credentials.json (outside the repository)
|
|
14
|
-
// and in the local, gitignored .env denylist.
|
|
15
|
-
//
|
|
16
|
-
// Usage: npm run check:secrets (exit 0 = clean, exit 1 = hit or broken config)
|
|
17
|
-
|
|
18
|
-
import { readdirSync, readFileSync, statSync } from 'node:fs'
|
|
19
|
-
import { dirname, join, relative, sep } from 'node:path'
|
|
20
|
-
import { fileURLToPath } from 'node:url'
|
|
21
|
-
import {
|
|
22
|
-
ALL_RULES, BACKSTOP_RULES, DENYLIST_VAR, ENTROPY, IP_ALLOWLIST, INTERNAL_HOST_SUFFIXES,
|
|
23
|
-
MAX_FILE_BYTES, PLACEHOLDER_HOSTS, PLACEHOLDER_PASSWORDS, PLACEHOLDER_VALUE_RE, PRAGMA,
|
|
24
|
-
SECRET_FORMATS, SECRET_KEYWORDS, SKIP_DIRS, SKIP_EXT_RE, SKIP_FILES, SLOTS,
|
|
25
|
-
} from './desensitize.config.mjs'
|
|
26
|
-
|
|
27
|
-
const ROOT = dirname(dirname(fileURLToPath(import.meta.url)))
|
|
28
|
-
const ENV_FILE = join(ROOT, '.env')
|
|
29
|
-
|
|
30
|
-
// ── local denylist (supplementary: bare names no pattern can see) ──────────
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* Union every `DSH_DBHUB_DESENSITIZE_RE` assignment in the local .env.
|
|
34
|
-
* @returns `{ matcher, count, broken }`; matcher is undefined when unconfigured.
|
|
35
|
-
*/
|
|
36
|
-
function readDenylist() {
|
|
37
|
-
let text
|
|
38
|
-
try {
|
|
39
|
-
text = readFileSync(ENV_FILE, 'utf8')
|
|
40
|
-
} catch {
|
|
41
|
-
return { matcher: undefined, count: 0 }
|
|
42
|
-
}
|
|
43
|
-
const values = []
|
|
44
|
-
for (const raw of text.split(/\r?\n/)) {
|
|
45
|
-
const line = raw.trim()
|
|
46
|
-
if (!line || line.startsWith('#')) continue
|
|
47
|
-
const eq = line.indexOf('=')
|
|
48
|
-
if (eq < 0) continue
|
|
49
|
-
if (line.slice(0, eq).trim() !== DENYLIST_VAR) continue
|
|
50
|
-
const value = line.slice(eq + 1).trim().replace(/^(['"])(.*)\1$/, '$2')
|
|
51
|
-
if (value) values.push(value)
|
|
52
|
-
}
|
|
53
|
-
if (values.length === 0) return { matcher: undefined, count: 0 }
|
|
54
|
-
try {
|
|
55
|
-
return { matcher: new RegExp(values.join('|'), 'i'), count: values.length }
|
|
56
|
-
} catch (e) {
|
|
57
|
-
console.error(`[check:secrets] ${DENYLIST_VAR} in .env is not a valid regular expression: ${String((e && e.message) || e)}`)
|
|
58
|
-
return { matcher: undefined, count: values.length, broken: true }
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
// ── glob matching (no dependencies) ────────────────────────────────────────
|
|
63
|
-
|
|
64
|
-
/** Compile a repository-relative glob (`**`, `*`, `?`) into a RegExp. */
|
|
65
|
-
function globToRegExp(glob) {
|
|
66
|
-
let out = '^'
|
|
67
|
-
for (let i = 0; i < glob.length; i++) {
|
|
68
|
-
const ch = glob[i]
|
|
69
|
-
if (ch === '*') {
|
|
70
|
-
if (glob[i + 1] === '*') {
|
|
71
|
-
if (glob[i + 2] === '/') {
|
|
72
|
-
out += '(?:[^/]+/)*'
|
|
73
|
-
i += 2
|
|
74
|
-
} else {
|
|
75
|
-
out += '.*'
|
|
76
|
-
i += 1
|
|
77
|
-
}
|
|
78
|
-
} else {
|
|
79
|
-
out += '[^/]*'
|
|
80
|
-
}
|
|
81
|
-
continue
|
|
82
|
-
}
|
|
83
|
-
if (ch === '?') {
|
|
84
|
-
out += '[^/]'
|
|
85
|
-
continue
|
|
86
|
-
}
|
|
87
|
-
out += ch.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
|
88
|
-
}
|
|
89
|
-
return new RegExp(out + '$')
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
// ── rule helpers ───────────────────────────────────────────────────────────
|
|
93
|
-
|
|
94
|
-
const IPV4_RE = /\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b/g
|
|
95
|
-
const CREDENTIAL_URL_RE = /\b([a-z][a-z0-9+.-]*):\/\/([^/@\s:]+):([^@\s]*)@([A-Za-z0-9._:-]+)/gi
|
|
96
|
-
const URL_HOST_RE = /\b[a-z][a-z0-9+.-]*:\/\/([A-Za-z0-9._-]+)/gi
|
|
97
|
-
const SECRET_ASSIGN_RE = new RegExp(
|
|
98
|
-
`\\b(${SECRET_KEYWORDS.join('|')})\\b\\s*[:=]\\s*(['"]?)([^\\s'"]+)\\2`,
|
|
99
|
-
'gi',
|
|
100
|
-
)
|
|
101
|
-
const HOSTNAME_RE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$/
|
|
102
|
-
const QUOTED_TOKEN_RE = /(['"`])([A-Za-z0-9_+/=.-]{8,})\1/g
|
|
103
|
-
|
|
104
|
-
/**
|
|
105
|
-
* Whether an assigned value looks like a written-down credential rather than a
|
|
106
|
-
* code expression (identifier, call, index, template, escape sequence). Only the
|
|
107
|
-
* former is a leak; the latter is how source code reads values at runtime.
|
|
108
|
-
*/
|
|
109
|
-
function looksLikeSecretLiteral(raw) {
|
|
110
|
-
const value = raw.replace(/[,;)\]}"']+$/, '')
|
|
111
|
-
if (value.length < 4) return false
|
|
112
|
-
if (PLACEHOLDER_VALUE_RE.test(value) || PLACEHOLDER_PASSWORDS.includes(value)) return false
|
|
113
|
-
if (/[([{$\\|`]/.test(value)) return false // call / index / template / escape / regex alternation
|
|
114
|
-
if (/^[A-Za-z_$][A-Za-z0-9_$.]*$/.test(value)) return false // bare identifier or member path
|
|
115
|
-
return true
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/** Host-shaped quoted token (rejects config keys like `.env.local`, code like `process.env.HOME`). */
|
|
119
|
-
function isHostShaped(token) {
|
|
120
|
-
return token === token.toLowerCase() && HOSTNAME_RE.test(token) && !token.startsWith('.') && !token.includes('..')
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/** Shannon entropy in bits per character. */
|
|
124
|
-
function entropyBits(value) {
|
|
125
|
-
const counts = new Map()
|
|
126
|
-
for (const ch of value) counts.set(ch, (counts.get(ch) || 0) + 1)
|
|
127
|
-
let bits = 0
|
|
128
|
-
for (const count of counts.values()) {
|
|
129
|
-
const p = count / value.length
|
|
130
|
-
bits -= p * Math.log2(p)
|
|
131
|
-
}
|
|
132
|
-
return bits
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const ipAllowed = (ip) => IP_ALLOWLIST.some((entry) => entry.test.test(ip))
|
|
136
|
-
|
|
137
|
-
function hostAllowed(host) {
|
|
138
|
-
const bare = host.replace(/:\d+$/, '').toLowerCase()
|
|
139
|
-
return ipAllowed(bare) || PLACEHOLDER_HOSTS.includes(bare)
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
/**
|
|
143
|
-
* Evaluate the enabled rules on one line.
|
|
144
|
-
* @param line - the source line.
|
|
145
|
-
* @param rules - rule ids enabled for this file (backstop + slot rules).
|
|
146
|
-
* @param denylist - compiled local denylist, or undefined.
|
|
147
|
-
* @returns findings as `{ rule, message }`.
|
|
148
|
-
*/
|
|
149
|
-
function checkLine(line, rules, denylist) {
|
|
150
|
-
const found = []
|
|
151
|
-
const add = (rule, message) => found.push({ rule, message })
|
|
152
|
-
|
|
153
|
-
if (rules.includes('ip')) {
|
|
154
|
-
for (const match of line.matchAll(IPV4_RE)) {
|
|
155
|
-
if (!ipAllowed(match[0])) {
|
|
156
|
-
add('ip', `${match[0]} is not loopback or an RFC 5737 documentation address`)
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
if (rules.includes('secret')) {
|
|
162
|
-
for (const { label, re } of SECRET_FORMATS) {
|
|
163
|
-
if (re.test(line)) add('secret', `looks like a real ${label}`)
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
if (rules.includes('password')) {
|
|
168
|
-
for (const match of line.matchAll(CREDENTIAL_URL_RE)) {
|
|
169
|
-
const password = match[3]
|
|
170
|
-
if (!password) continue
|
|
171
|
-
if (!PLACEHOLDER_PASSWORDS.includes(password) && !PLACEHOLDER_VALUE_RE.test(password)) {
|
|
172
|
-
add('password', `credentialed URL password "${password}" is not a placeholder (use CHANGE_ME)`)
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
for (const match of line.matchAll(SECRET_ASSIGN_RE)) {
|
|
176
|
-
if (!looksLikeSecretLiteral(match[3])) continue
|
|
177
|
-
add('password', `${match[1]} is assigned the literal "${match[3]}" — use a placeholder or a reference`)
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
if (rules.includes('dsn')) {
|
|
182
|
-
for (const match of line.matchAll(CREDENTIAL_URL_RE)) {
|
|
183
|
-
if (!hostAllowed(match[4])) {
|
|
184
|
-
add('dsn', `example host "${match[4]}" is not loopback, an RFC 5737 address or a placeholder host`)
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
if (rules.includes('host')) {
|
|
190
|
-
// Only URL hosts and host-shaped quoted tokens are candidates: a raw line is
|
|
191
|
-
// full of identifiers (`process.env.HOME` ends with `.home` but is code).
|
|
192
|
-
const candidates = []
|
|
193
|
-
for (const match of line.matchAll(URL_HOST_RE)) candidates.push(match[1])
|
|
194
|
-
for (const match of line.matchAll(QUOTED_TOKEN_RE)) {
|
|
195
|
-
if (isHostShaped(match[2])) candidates.push(match[2])
|
|
196
|
-
}
|
|
197
|
-
for (const host of candidates) {
|
|
198
|
-
const bare = host.replace(/:\d+$/, '').toLowerCase()
|
|
199
|
-
if (INTERNAL_HOST_SUFFIXES.some((suffix) => bare.endsWith(suffix)) && !hostAllowed(bare)) {
|
|
200
|
-
add('host', `hostname "${host}" looks internal`)
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
if (rules.includes('name') && denylist) {
|
|
206
|
-
const hit = line.match(denylist)
|
|
207
|
-
if (hit) add('name', `matches the local denylist ("${hit[0]}")`)
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
if (rules.includes('
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
if (
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
console.
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
console.
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
1
|
+
// Desensitization gate for dsh-dbhub-live.
|
|
2
|
+
//
|
|
3
|
+
// Runs two rule sets, both defined by scripts/desensitize.config.mjs:
|
|
4
|
+
// * a BACKSTOP over every scanned file (IP literals, secret formats, entropy)
|
|
5
|
+
// so an unregistered new surface is still caught;
|
|
6
|
+
// * the registered SLOTS' targeted rules (credentialed URLs, secret-keyword
|
|
7
|
+
// assignments, internal host suffixes, absolute local paths, local denylist
|
|
8
|
+
// names) where real values are actually plausible.
|
|
9
|
+
//
|
|
10
|
+
// The repository is treated as content that ships publicly (npm tarball and the
|
|
11
|
+
// public repository), so anything that looks like a real host, account,
|
|
12
|
+
// password, database or project name fails the gate. Real values belong in
|
|
13
|
+
// $DSH_HOME/storages/dsh-dbhub-live/credentials.json (outside the repository)
|
|
14
|
+
// and in the local, gitignored .env denylist.
|
|
15
|
+
//
|
|
16
|
+
// Usage: npm run check:secrets (exit 0 = clean, exit 1 = hit or broken config)
|
|
17
|
+
|
|
18
|
+
import { readdirSync, readFileSync, statSync } from 'node:fs'
|
|
19
|
+
import { dirname, join, relative, sep } from 'node:path'
|
|
20
|
+
import { fileURLToPath } from 'node:url'
|
|
21
|
+
import {
|
|
22
|
+
ALL_RULES, BACKSTOP_RULES, DENYLIST_VAR, ENTROPY, IP_ALLOWLIST, INTERNAL_HOST_SUFFIXES,
|
|
23
|
+
LOCAL_PATH_RE, MAX_FILE_BYTES, PLACEHOLDER_HOSTS, PLACEHOLDER_PASSWORDS, PLACEHOLDER_VALUE_RE, PRAGMA,
|
|
24
|
+
SECRET_FORMATS, SECRET_KEYWORDS, SKIP_DIRS, SKIP_EXT_RE, SKIP_FILES, SLOTS,
|
|
25
|
+
} from './desensitize.config.mjs'
|
|
26
|
+
|
|
27
|
+
const ROOT = dirname(dirname(fileURLToPath(import.meta.url)))
|
|
28
|
+
const ENV_FILE = join(ROOT, '.env')
|
|
29
|
+
|
|
30
|
+
// ── local denylist (supplementary: bare names no pattern can see) ──────────
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Union every `DSH_DBHUB_DESENSITIZE_RE` assignment in the local .env.
|
|
34
|
+
* @returns `{ matcher, count, broken }`; matcher is undefined when unconfigured.
|
|
35
|
+
*/
|
|
36
|
+
function readDenylist() {
|
|
37
|
+
let text
|
|
38
|
+
try {
|
|
39
|
+
text = readFileSync(ENV_FILE, 'utf8')
|
|
40
|
+
} catch {
|
|
41
|
+
return { matcher: undefined, count: 0 }
|
|
42
|
+
}
|
|
43
|
+
const values = []
|
|
44
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
45
|
+
const line = raw.trim()
|
|
46
|
+
if (!line || line.startsWith('#')) continue
|
|
47
|
+
const eq = line.indexOf('=')
|
|
48
|
+
if (eq < 0) continue
|
|
49
|
+
if (line.slice(0, eq).trim() !== DENYLIST_VAR) continue
|
|
50
|
+
const value = line.slice(eq + 1).trim().replace(/^(['"])(.*)\1$/, '$2')
|
|
51
|
+
if (value) values.push(value)
|
|
52
|
+
}
|
|
53
|
+
if (values.length === 0) return { matcher: undefined, count: 0 }
|
|
54
|
+
try {
|
|
55
|
+
return { matcher: new RegExp(values.join('|'), 'i'), count: values.length }
|
|
56
|
+
} catch (e) {
|
|
57
|
+
console.error(`[check:secrets] ${DENYLIST_VAR} in .env is not a valid regular expression: ${String((e && e.message) || e)}`)
|
|
58
|
+
return { matcher: undefined, count: values.length, broken: true }
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// ── glob matching (no dependencies) ────────────────────────────────────────
|
|
63
|
+
|
|
64
|
+
/** Compile a repository-relative glob (`**`, `*`, `?`) into a RegExp. */
|
|
65
|
+
function globToRegExp(glob) {
|
|
66
|
+
let out = '^'
|
|
67
|
+
for (let i = 0; i < glob.length; i++) {
|
|
68
|
+
const ch = glob[i]
|
|
69
|
+
if (ch === '*') {
|
|
70
|
+
if (glob[i + 1] === '*') {
|
|
71
|
+
if (glob[i + 2] === '/') {
|
|
72
|
+
out += '(?:[^/]+/)*'
|
|
73
|
+
i += 2
|
|
74
|
+
} else {
|
|
75
|
+
out += '.*'
|
|
76
|
+
i += 1
|
|
77
|
+
}
|
|
78
|
+
} else {
|
|
79
|
+
out += '[^/]*'
|
|
80
|
+
}
|
|
81
|
+
continue
|
|
82
|
+
}
|
|
83
|
+
if (ch === '?') {
|
|
84
|
+
out += '[^/]'
|
|
85
|
+
continue
|
|
86
|
+
}
|
|
87
|
+
out += ch.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
|
88
|
+
}
|
|
89
|
+
return new RegExp(out + '$')
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// ── rule helpers ───────────────────────────────────────────────────────────
|
|
93
|
+
|
|
94
|
+
const IPV4_RE = /\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b/g
|
|
95
|
+
const CREDENTIAL_URL_RE = /\b([a-z][a-z0-9+.-]*):\/\/([^/@\s:]+):([^@\s]*)@([A-Za-z0-9._:-]+)/gi
|
|
96
|
+
const URL_HOST_RE = /\b[a-z][a-z0-9+.-]*:\/\/([A-Za-z0-9._-]+)/gi
|
|
97
|
+
const SECRET_ASSIGN_RE = new RegExp(
|
|
98
|
+
`\\b(${SECRET_KEYWORDS.join('|')})\\b\\s*[:=]\\s*(['"]?)([^\\s'"]+)\\2`,
|
|
99
|
+
'gi',
|
|
100
|
+
)
|
|
101
|
+
const HOSTNAME_RE = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)+$/
|
|
102
|
+
const QUOTED_TOKEN_RE = /(['"`])([A-Za-z0-9_+/=.-]{8,})\1/g
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Whether an assigned value looks like a written-down credential rather than a
|
|
106
|
+
* code expression (identifier, call, index, template, escape sequence). Only the
|
|
107
|
+
* former is a leak; the latter is how source code reads values at runtime.
|
|
108
|
+
*/
|
|
109
|
+
function looksLikeSecretLiteral(raw) {
|
|
110
|
+
const value = raw.replace(/[,;)\]}"']+$/, '')
|
|
111
|
+
if (value.length < 4) return false
|
|
112
|
+
if (PLACEHOLDER_VALUE_RE.test(value) || PLACEHOLDER_PASSWORDS.includes(value)) return false
|
|
113
|
+
if (/[([{$\\|`]/.test(value)) return false // call / index / template / escape / regex alternation
|
|
114
|
+
if (/^[A-Za-z_$][A-Za-z0-9_$.]*$/.test(value)) return false // bare identifier or member path
|
|
115
|
+
return true
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Host-shaped quoted token (rejects config keys like `.env.local`, code like `process.env.HOME`). */
|
|
119
|
+
function isHostShaped(token) {
|
|
120
|
+
return token === token.toLowerCase() && HOSTNAME_RE.test(token) && !token.startsWith('.') && !token.includes('..')
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Shannon entropy in bits per character. */
|
|
124
|
+
function entropyBits(value) {
|
|
125
|
+
const counts = new Map()
|
|
126
|
+
for (const ch of value) counts.set(ch, (counts.get(ch) || 0) + 1)
|
|
127
|
+
let bits = 0
|
|
128
|
+
for (const count of counts.values()) {
|
|
129
|
+
const p = count / value.length
|
|
130
|
+
bits -= p * Math.log2(p)
|
|
131
|
+
}
|
|
132
|
+
return bits
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const ipAllowed = (ip) => IP_ALLOWLIST.some((entry) => entry.test.test(ip))
|
|
136
|
+
|
|
137
|
+
function hostAllowed(host) {
|
|
138
|
+
const bare = host.replace(/:\d+$/, '').toLowerCase()
|
|
139
|
+
return ipAllowed(bare) || PLACEHOLDER_HOSTS.includes(bare)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Evaluate the enabled rules on one line.
|
|
144
|
+
* @param line - the source line.
|
|
145
|
+
* @param rules - rule ids enabled for this file (backstop + slot rules).
|
|
146
|
+
* @param denylist - compiled local denylist, or undefined.
|
|
147
|
+
* @returns findings as `{ rule, message }`.
|
|
148
|
+
*/
|
|
149
|
+
function checkLine(line, rules, denylist) {
|
|
150
|
+
const found = []
|
|
151
|
+
const add = (rule, message) => found.push({ rule, message })
|
|
152
|
+
|
|
153
|
+
if (rules.includes('ip')) {
|
|
154
|
+
for (const match of line.matchAll(IPV4_RE)) {
|
|
155
|
+
if (!ipAllowed(match[0])) {
|
|
156
|
+
add('ip', `${match[0]} is not loopback or an RFC 5737 documentation address`)
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (rules.includes('secret')) {
|
|
162
|
+
for (const { label, re } of SECRET_FORMATS) {
|
|
163
|
+
if (re.test(line)) add('secret', `looks like a real ${label}`)
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
if (rules.includes('password')) {
|
|
168
|
+
for (const match of line.matchAll(CREDENTIAL_URL_RE)) {
|
|
169
|
+
const password = match[3]
|
|
170
|
+
if (!password) continue
|
|
171
|
+
if (!PLACEHOLDER_PASSWORDS.includes(password) && !PLACEHOLDER_VALUE_RE.test(password)) {
|
|
172
|
+
add('password', `credentialed URL password "${password}" is not a placeholder (use CHANGE_ME)`)
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
for (const match of line.matchAll(SECRET_ASSIGN_RE)) {
|
|
176
|
+
if (!looksLikeSecretLiteral(match[3])) continue
|
|
177
|
+
add('password', `${match[1]} is assigned the literal "${match[3]}" — use a placeholder or a reference`)
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (rules.includes('dsn')) {
|
|
182
|
+
for (const match of line.matchAll(CREDENTIAL_URL_RE)) {
|
|
183
|
+
if (!hostAllowed(match[4])) {
|
|
184
|
+
add('dsn', `example host "${match[4]}" is not loopback, an RFC 5737 address or a placeholder host`)
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
if (rules.includes('host')) {
|
|
190
|
+
// Only URL hosts and host-shaped quoted tokens are candidates: a raw line is
|
|
191
|
+
// full of identifiers (`process.env.HOME` ends with `.home` but is code).
|
|
192
|
+
const candidates = []
|
|
193
|
+
for (const match of line.matchAll(URL_HOST_RE)) candidates.push(match[1])
|
|
194
|
+
for (const match of line.matchAll(QUOTED_TOKEN_RE)) {
|
|
195
|
+
if (isHostShaped(match[2])) candidates.push(match[2])
|
|
196
|
+
}
|
|
197
|
+
for (const host of candidates) {
|
|
198
|
+
const bare = host.replace(/:\d+$/, '').toLowerCase()
|
|
199
|
+
if (INTERNAL_HOST_SUFFIXES.some((suffix) => bare.endsWith(suffix)) && !hostAllowed(bare)) {
|
|
200
|
+
add('host', `hostname "${host}" looks internal`)
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
if (rules.includes('name') && denylist) {
|
|
206
|
+
const hit = line.match(denylist)
|
|
207
|
+
if (hit) add('name', `matches the local denylist ("${hit[0]}")`)
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
if (rules.includes('localpath')) {
|
|
211
|
+
const hit = line.match(LOCAL_PATH_RE)
|
|
212
|
+
if (hit) {
|
|
213
|
+
add('localpath', `"${hit[0]}" looks like an absolute local path — use a placeholder such as <repo>`)
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if (rules.includes('entropy')) {
|
|
218
|
+
for (const match of line.matchAll(QUOTED_TOKEN_RE)) {
|
|
219
|
+
const token = match[2]
|
|
220
|
+
if (token.length < ENTROPY.minLength) continue
|
|
221
|
+
if (ENTROPY.requireMixed && !(/[a-z]/.test(token) && (/[A-Z]/.test(token) || /[0-9]/.test(token)))) continue
|
|
222
|
+
if (ENTROPY.allowPatterns.some((re) => re.test(token))) continue
|
|
223
|
+
if (PLACEHOLDER_VALUE_RE.test(token) || PLACEHOLDER_PASSWORDS.includes(token)) continue
|
|
224
|
+
const bits = entropyBits(token)
|
|
225
|
+
if (bits >= ENTROPY.minBitsPerChar) {
|
|
226
|
+
add('entropy', `"${token}" looks like a random secret (${bits.toFixed(1)} bits/char)`)
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
return found
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// ── walk ───────────────────────────────────────────────────────────────────
|
|
235
|
+
|
|
236
|
+
function walk(dir, out = []) {
|
|
237
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
238
|
+
if (entry.isDirectory()) {
|
|
239
|
+
if (!SKIP_DIRS.includes(entry.name)) walk(join(dir, entry.name), out)
|
|
240
|
+
continue
|
|
241
|
+
}
|
|
242
|
+
if (!entry.isFile()) continue
|
|
243
|
+
if (SKIP_FILES.includes(entry.name) || SKIP_EXT_RE.test(entry.name)) continue
|
|
244
|
+
const full = join(dir, entry.name)
|
|
245
|
+
try {
|
|
246
|
+
if (statSync(full).size <= MAX_FILE_BYTES) out.push(full)
|
|
247
|
+
} catch {
|
|
248
|
+
/* unreadable: skip */
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return out
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// ── run ────────────────────────────────────────────────────────────────────
|
|
255
|
+
|
|
256
|
+
const badRule = SLOTS.find((slot) => slot.rules.some((rule) => !ALL_RULES.includes(rule)))
|
|
257
|
+
if (badRule) {
|
|
258
|
+
const unknown = badRule.rules.find((rule) => !ALL_RULES.includes(rule))
|
|
259
|
+
console.error(`[check:secrets] slot "${badRule.id}" declares unknown rule "${unknown}" — fix scripts/desensitize.config.mjs`)
|
|
260
|
+
process.exitCode = 1
|
|
261
|
+
} else {
|
|
262
|
+
const { matcher: denylist, count, broken } = readDenylist()
|
|
263
|
+
if (broken) {
|
|
264
|
+
console.error('[check:secrets] .env denylist could not be compiled — fix it before continuing')
|
|
265
|
+
} else if (!denylist) {
|
|
266
|
+
console.warn(
|
|
267
|
+
`[check:secrets] note: no local denylist (${DENYLIST_VAR} in .env) — bare project/database names are\n` +
|
|
268
|
+
' not pattern-detectable. Create .env (gitignored) to enable that layer.',
|
|
269
|
+
)
|
|
270
|
+
} else if (count > 1) {
|
|
271
|
+
console.warn(`[check:secrets] note: ${count} ${DENYLIST_VAR} assignments in .env — all applied (union).`)
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const files = walk(ROOT)
|
|
275
|
+
const rel = (file) => relative(ROOT, file).split(sep).join('/')
|
|
276
|
+
const slotMatchers = SLOTS.map((slot) => ({ slot, res: slot.globs.map(globToRegExp) }))
|
|
277
|
+
const slotFileCounts = new Map(SLOTS.map((slot) => [slot.id, 0]))
|
|
278
|
+
const findings = []
|
|
279
|
+
|
|
280
|
+
for (const file of files) {
|
|
281
|
+
const path = rel(file)
|
|
282
|
+
const rules = new Set(BACKSTOP_RULES)
|
|
283
|
+
for (const { slot, res } of slotMatchers) {
|
|
284
|
+
if (!res.some((re) => re.test(path))) continue
|
|
285
|
+
slotFileCounts.set(slot.id, slotFileCounts.get(slot.id) + 1)
|
|
286
|
+
for (const rule of slot.rules) rules.add(rule)
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
let text
|
|
290
|
+
try {
|
|
291
|
+
text = readFileSync(file, 'utf8')
|
|
292
|
+
} catch {
|
|
293
|
+
continue
|
|
294
|
+
}
|
|
295
|
+
const lines = text.split(/\r?\n/)
|
|
296
|
+
const ruleList = [...rules]
|
|
297
|
+
for (let i = 0; i < lines.length; i++) {
|
|
298
|
+
const line = lines[i]
|
|
299
|
+
if (line.includes(PRAGMA)) continue
|
|
300
|
+
for (const { rule, message } of checkLine(line, ruleList, denylist)) {
|
|
301
|
+
findings.push(`${path}:${i + 1}: [${rule}] ${message}`)
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
for (const slot of SLOTS) {
|
|
307
|
+
if ((slotFileCounts.get(slot.id) || 0) === 0) {
|
|
308
|
+
findings.push(`scripts/desensitize.config.mjs: [slot-stale] slot "${slot.id}" matches no file — update or remove it`)
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const activeRules = [
|
|
313
|
+
...BACKSTOP_RULES.map((rule) => `${rule}(backstop)`),
|
|
314
|
+
...[...new Set(SLOTS.flatMap((slot) => slot.rules))].filter((rule) => !BACKSTOP_RULES.includes(rule)),
|
|
315
|
+
denylist ? 'denylist' : 'denylist(inactive)',
|
|
316
|
+
]
|
|
317
|
+
|
|
318
|
+
if (findings.length > 0) {
|
|
319
|
+
console.error(`[check:secrets] ${findings.length} finding(s) across ${files.length} file(s):`)
|
|
320
|
+
for (const finding of findings) console.error(' ' + finding)
|
|
321
|
+
console.error(
|
|
322
|
+
'[check:secrets] replace with loopback/RFC 5737 hosts + CHANGE_ME passwords, or waive a false\n' +
|
|
323
|
+
` positive with the inline marker "${PRAGMA}".`,
|
|
324
|
+
)
|
|
325
|
+
process.exitCode = 1
|
|
326
|
+
} else {
|
|
327
|
+
console.log(
|
|
328
|
+
`[check:secrets] clean: ${files.length} file(s), ${SLOTS.length} slot(s) covered — rules: ${activeRules.join(', ')}`,
|
|
329
|
+
)
|
|
330
|
+
}
|
|
331
|
+
}
|