@vieko/pi-redact 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/NOTICE +14 -0
- package/README.md +72 -2
- package/extensions/pi-redact.ts +329 -0
- package/package.json +26 -4
- package/scripts/scrub-sessions.ts +209 -0
- package/src/detect.ts +1636 -0
package/src/detect.ts
ADDED
|
@@ -0,0 +1,1636 @@
|
|
|
1
|
+
// Vendored from github.com/ray-amjad/claude-code-mods
|
|
2
|
+
// plugins/transcript-redactor/hooks/detect.ts @ 812a5e9 (MIT, (c) 2026 Ray Amjad).
|
|
3
|
+
// Local changes are marked "pi-redact:". See NOTICE.
|
|
4
|
+
|
|
5
|
+
// The pure half of transcript-redactor: the config, the vault, the detectors,
|
|
6
|
+
// and the walkers. Nothing in here touches `$` or registers a hook, so another
|
|
7
|
+
// mod can vendor this file as it is and the tests can run it under plain node.
|
|
8
|
+
//
|
|
9
|
+
// The value is not deleted. It is put in a vault that lives in this module,
|
|
10
|
+
// in memory, for the session, and the transcript gets a placeholder in its
|
|
11
|
+
// place: `[REDACTED-SECRET-1a2b3c4d]`. Within one session the same value always
|
|
12
|
+
// mints the same placeholder, so the model can still tell one customer from
|
|
13
|
+
// another, and the placeholder is swapped back for the real value on the way
|
|
14
|
+
// into a tool that runs on this machine, so a `Bash` command or a `Write` still
|
|
15
|
+
// carries the real key.
|
|
16
|
+
//
|
|
17
|
+
// Nothing is written to disk. The vault dies with the session.
|
|
18
|
+
|
|
19
|
+
export type Kind = 'SECRET' | 'EMAIL' | 'IP' | 'LITERAL'
|
|
20
|
+
|
|
21
|
+
export type Config = {
|
|
22
|
+
secrets: boolean
|
|
23
|
+
pii: boolean
|
|
24
|
+
emails: boolean
|
|
25
|
+
privateIps: boolean
|
|
26
|
+
restore: boolean
|
|
27
|
+
restoreTools: ReadonlySet<string>
|
|
28
|
+
notify: boolean
|
|
29
|
+
minEntropy: number
|
|
30
|
+
minLength: number
|
|
31
|
+
contextMinEntropy: number
|
|
32
|
+
allow: ReadonlySet<string>
|
|
33
|
+
allowEmails: readonly string[]
|
|
34
|
+
allowPrefix: readonly string[]
|
|
35
|
+
denyPrefix: readonly string[]
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// The tools that run on this machine. A placeholder is only swapped back for
|
|
39
|
+
// its real value in these: a tool that hands its input to another model or to
|
|
40
|
+
// the network (Agent, WebFetch, WebSearch, SendMessage, an MCP tool) gets the
|
|
41
|
+
// placeholder as written.
|
|
42
|
+
export const DEFAULT_RESTORE_TOOLS = 'Bash,Write,Edit,MultiEdit,NotebookEdit'
|
|
43
|
+
|
|
44
|
+
// ---------------------------------------------------------------- the vault
|
|
45
|
+
|
|
46
|
+
const byValue = new Map<string, string>()
|
|
47
|
+
const byTag = new Map<string, string>()
|
|
48
|
+
let hidden = 0
|
|
49
|
+
let minted: Set<string> | null = null
|
|
50
|
+
|
|
51
|
+
const TAG = /\[REDACTED-(?:SECRET|EMAIL|IP|LITERAL)-[0-9a-f]{8}\]/g
|
|
52
|
+
const WHOLE_TAG = /^\[REDACTED-(?:SECRET|EMAIL|IP|LITERAL)-[0-9a-f]{8}\]$/
|
|
53
|
+
|
|
54
|
+
function randomHex8(): string {
|
|
55
|
+
const bytes = new Uint8Array(4)
|
|
56
|
+
const crypto = (globalThis as { crypto?: { getRandomValues?: (values: Uint8Array) => Uint8Array } }).crypto
|
|
57
|
+
if (crypto?.getRandomValues) crypto.getRandomValues(bytes)
|
|
58
|
+
else for (let index = 0; index < bytes.length; index++) bytes[index] = Math.floor(Math.random() * 256)
|
|
59
|
+
return Array.from(bytes, byte => byte.toString(16).padStart(2, '0')).join('')
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Swaps a value for its placeholder and remembers both. A value that already
|
|
63
|
+
// holds a placeholder from an earlier pass (a key inside a URL password) is
|
|
64
|
+
// stored with that placeholder put back, so one restore gives the real text.
|
|
65
|
+
export function mint(kind: Kind, raw: string): string {
|
|
66
|
+
hidden++
|
|
67
|
+
const value = kind === 'LITERAL' ? raw : restore(unpark(raw))
|
|
68
|
+
const seen = byValue.get(value)
|
|
69
|
+
if (seen) {
|
|
70
|
+
if (kind === 'SECRET') remember(value)
|
|
71
|
+
minted?.add(seen)
|
|
72
|
+
return seen
|
|
73
|
+
}
|
|
74
|
+
let tag: string
|
|
75
|
+
do tag = `[REDACTED-${kind}-${randomHex8()}]`
|
|
76
|
+
while (byTag.has(tag))
|
|
77
|
+
byValue.set(value, tag)
|
|
78
|
+
byTag.set(tag, value)
|
|
79
|
+
if (kind === 'SECRET') remember(value)
|
|
80
|
+
minted?.add(tag)
|
|
81
|
+
return tag
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// A secret is hidden again wherever it turns up later: across the session
|
|
85
|
+
// when it is distinctive enough (echoable), and within the text being
|
|
86
|
+
// scanned right now when it is shorter but still not a plain word.
|
|
87
|
+
function remember(value: string): void {
|
|
88
|
+
const echoes = echoable(value)
|
|
89
|
+
if (echoes && !echo.has(value)) {
|
|
90
|
+
echo.add(value)
|
|
91
|
+
echoIndex = null
|
|
92
|
+
}
|
|
93
|
+
if (local && (echoes || localWorthy(value))) local.add(value)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// One pass, on purpose: a LITERAL placeholder gives back the placeholder text
|
|
97
|
+
// it stands for, and that text must not then turn into the secret.
|
|
98
|
+
export function restore(text: string): string {
|
|
99
|
+
if (byTag.size === 0 || !text.includes('[REDACTED-')) return text
|
|
100
|
+
return text.replace(TAG, (tag) => byTag.get(tag) ?? tag)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Hides a value the caller knows is secret wherever it turns up for the rest
|
|
104
|
+
// of the session, even one too short or too plain for the echo rule (a word
|
|
105
|
+
// password from .env). Values under PREFIX characters can't be indexed.
|
|
106
|
+
export function keepSecret(value: string): string {
|
|
107
|
+
const tag = mint('SECRET', value)
|
|
108
|
+
if (value.length >= PREFIX && !echo.has(value)) {
|
|
109
|
+
echo.add(value)
|
|
110
|
+
echoIndex = null
|
|
111
|
+
}
|
|
112
|
+
return tag
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// How many times a value has been hidden this session. A hook reads it before
|
|
116
|
+
// and after a scrub to learn whether anything changed.
|
|
117
|
+
export function hiddenCount(): number {
|
|
118
|
+
return hidden
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Runs `fn` and hands back every placeholder it minted or reused, so a hook
|
|
122
|
+
// can count the values it hid without counting one value twice.
|
|
123
|
+
export function collect<T>(fn: () => T): { value: T; tags: Set<string> } {
|
|
124
|
+
const outer = minted
|
|
125
|
+
const tags = new Set<string>()
|
|
126
|
+
minted = tags
|
|
127
|
+
try {
|
|
128
|
+
return { value: fn(), tags }
|
|
129
|
+
} finally {
|
|
130
|
+
minted = outer
|
|
131
|
+
if (outer) for (const t of tags) outer.add(t)
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// A placeholder read back from a file or a log is quoted text, not a handle.
|
|
136
|
+
// Left alone it would turn into the real secret the next time the model
|
|
137
|
+
// copied that text into a command. So a live placeholder found in a tool's
|
|
138
|
+
// output is swapped for a LITERAL one, which restores to the placeholder text
|
|
139
|
+
// and nothing more. Run this before the detectors, so a key the tool printed
|
|
140
|
+
// still comes back as its live placeholder.
|
|
141
|
+
export function escapeLiteralTags(text: string): string {
|
|
142
|
+
if (byTag.size === 0 || !text.includes('[REDACTED-')) return text
|
|
143
|
+
return text.replace(TAG, (t) => (byTag.has(t) && !t.startsWith('[REDACTED-LITERAL-') ? mint('LITERAL', t) : t))
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ------------------------------------------------------- known value echoes
|
|
147
|
+
|
|
148
|
+
// A value hidden once because of where it sat (after `API_KEY=`, inside a
|
|
149
|
+
// URL) is hidden again wherever it turns up, even bare, where no detector
|
|
150
|
+
// would fire on it: a tool that echoes a restored key must not print it.
|
|
151
|
+
const echo = new Set<string>()
|
|
152
|
+
let echoIndex: EchoIndex | null = null
|
|
153
|
+
// The values minted during the current scan, to hide at their other spots.
|
|
154
|
+
let local: Set<string> | null = null
|
|
155
|
+
|
|
156
|
+
// Short or all-letter values (`postgres`) are left out: hiding every later
|
|
157
|
+
// `postgres` in unrelated output would make the session unreadable.
|
|
158
|
+
function echoable(value: string): boolean {
|
|
159
|
+
if (countsUp(value)) return false
|
|
160
|
+
// A slug or a dated name hidden once beside a key name is not worth hiding
|
|
161
|
+
// everywhere: it would turn every path that holds it into a placeholder.
|
|
162
|
+
if (value.split(/[^A-Za-z0-9]+/).every((p) => p === '' || isSlugPart(p))) return false
|
|
163
|
+
if (value.includes('\n')) return value.length >= 40
|
|
164
|
+
return value.length >= 20 || (value.length >= 10 && /[0-9]/.test(value) && /[A-Za-z]/.test(value))
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// A value too short to hide across the session is still hidden at its other
|
|
168
|
+
// spots in the same text, unless it is a plain word or slug (`postgres`
|
|
169
|
+
// hidden once in a URL must not turn every `postgres://` into a placeholder).
|
|
170
|
+
function localWorthy(value: string): boolean {
|
|
171
|
+
if (value.length < PREFIX || value.length > 400 || value.includes('\n') || countsUp(value)) return false
|
|
172
|
+
if (value.split(/[^A-Za-z0-9]+/).every((p) => p === '' || isSlugPart(p))) return false
|
|
173
|
+
return /[0-9]/.test(value) || /[^A-Za-z0-9]/.test(value) || value.length >= 12
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// The known values are found with a rolling hash over their first PREFIX
|
|
177
|
+
// characters, so the cost of the scan does not grow with the vault. A regex
|
|
178
|
+
// of every value joined by `|` took 40 s on 200 KB once a long session had
|
|
179
|
+
// hidden a few thousand values.
|
|
180
|
+
const PREFIX = 8
|
|
181
|
+
// Under each prefix the values are kept by length, longest first, so a
|
|
182
|
+
// prefix that thousands of values share costs one set lookup per length,
|
|
183
|
+
// not a walk of the whole list.
|
|
184
|
+
type ByLength = { lengths: number[]; values: Set<string> }
|
|
185
|
+
type EchoIndex = { hashes: Set<number>; byPrefix: Map<string, ByLength>; top: number }
|
|
186
|
+
|
|
187
|
+
function prefixHash(text: string, at: number): number {
|
|
188
|
+
let h = 0
|
|
189
|
+
for (let i = 0; i < PREFIX; i++) h = (Math.imul(h, 31) + text.charCodeAt(at + i)) >>> 0
|
|
190
|
+
return h
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function buildEchoIndex(values: Iterable<string>): EchoIndex {
|
|
194
|
+
const byPrefix = new Map<string, ByLength>()
|
|
195
|
+
for (const v of values) {
|
|
196
|
+
const p = v.slice(0, PREFIX)
|
|
197
|
+
let entry = byPrefix.get(p)
|
|
198
|
+
if (!entry) byPrefix.set(p, (entry = { lengths: [], values: new Set() }))
|
|
199
|
+
if (!entry.lengths.includes(v.length)) entry.lengths.push(v.length)
|
|
200
|
+
entry.values.add(v)
|
|
201
|
+
}
|
|
202
|
+
for (const entry of byPrefix.values()) entry.lengths.sort((a, b) => b - a)
|
|
203
|
+
const hashes = new Set<number>()
|
|
204
|
+
for (const p of byPrefix.keys()) hashes.add(prefixHash(p, 0))
|
|
205
|
+
// 31 to the power PREFIX - 1, to take the oldest character out of the hash.
|
|
206
|
+
let top = 1
|
|
207
|
+
for (let i = 0; i < PREFIX - 1; i++) top = Math.imul(top, 31) >>> 0
|
|
208
|
+
return { hashes, byPrefix, top }
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function rehideKnown(text: string): string {
|
|
212
|
+
if (echo.size === 0 || text.length < PREFIX) return text
|
|
213
|
+
return rehideWith(text, (echoIndex ??= buildEchoIndex(echo)))
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// The values minted in one scan, hidden at their other spots in it. A few
|
|
217
|
+
// values are found faster one by one than by the rolling hash.
|
|
218
|
+
function rehideLocal(text: string, values: Set<string>): string {
|
|
219
|
+
if (values.size > 64) return rehideWith(text, buildEchoIndex(values))
|
|
220
|
+
let out = text
|
|
221
|
+
for (const v of [...values].sort((a, b) => b.length - a.length)) if (out.includes(v)) out = out.split(v).join(mint('SECRET', v))
|
|
222
|
+
return out
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function rehideWith(text: string, ix: EchoIndex): string {
|
|
226
|
+
if (text.length < PREFIX) return text
|
|
227
|
+
const parts: string[] = []
|
|
228
|
+
let last = 0
|
|
229
|
+
let h = prefixHash(text, 0)
|
|
230
|
+
for (let i = 0; i + PREFIX <= text.length; ) {
|
|
231
|
+
if (ix.hashes.has(h)) {
|
|
232
|
+
const entry = ix.byPrefix.get(text.slice(i, i + PREFIX))
|
|
233
|
+
let found: string | undefined
|
|
234
|
+
if (entry) {
|
|
235
|
+
for (const n of entry.lengths) {
|
|
236
|
+
const v = text.slice(i, i + n)
|
|
237
|
+
if (v.length === n && entry.values.has(v)) {
|
|
238
|
+
found = v
|
|
239
|
+
break
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
if (found) {
|
|
244
|
+
parts.push(text.slice(last, i), mint('SECRET', found))
|
|
245
|
+
i += found.length
|
|
246
|
+
last = i
|
|
247
|
+
if (i + PREFIX <= text.length) h = prefixHash(text, i)
|
|
248
|
+
continue
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (i + PREFIX >= text.length) break
|
|
252
|
+
h = (Math.imul((h - Math.imul(text.charCodeAt(i), ix.top)) >>> 0, 31) + text.charCodeAt(i + PREFIX)) >>> 0
|
|
253
|
+
i++
|
|
254
|
+
}
|
|
255
|
+
if (last === 0) return text
|
|
256
|
+
parts.push(text.slice(last))
|
|
257
|
+
return parts.join('')
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// ------------------------------------------------------------------ parking
|
|
261
|
+
|
|
262
|
+
// Some spans are known not to be secrets and are big enough to drown the
|
|
263
|
+
// transcript in placeholders: an image in a data: URI, a lockfile hash, a
|
|
264
|
+
// thinking-block signature, a base64 attachment. They are parked behind a
|
|
265
|
+
// marker for the scan and put back after it. The marker is a character the
|
|
266
|
+
// text does not contain; if every candidate is taken, nothing is parked and
|
|
267
|
+
// the scan simply hides more.
|
|
268
|
+
type Parked = { mark: string; spans: string[]; re: RegExp }
|
|
269
|
+
let parked: Parked | null = null
|
|
270
|
+
const MARKS = ['\u0000', '\uE000', '\uE001', '\uFDD0']
|
|
271
|
+
|
|
272
|
+
function startParking(text: string): Parked | null {
|
|
273
|
+
const mark = MARKS.find((m) => !text.includes(m))
|
|
274
|
+
return mark ? { mark, spans: [], re: new RegExp(`${mark}(\\d+)${mark}`, 'g') } : null
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function park(span: string): string {
|
|
278
|
+
if (!parked) return span
|
|
279
|
+
parked.spans.push(span)
|
|
280
|
+
return `${parked.mark}${parked.spans.length - 1}${parked.mark}`
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function unpark(text: string): string {
|
|
284
|
+
const p = parked
|
|
285
|
+
if (!p || !text.includes(p.mark)) return text
|
|
286
|
+
let out = text
|
|
287
|
+
for (let i = 0; i < 3 && out.includes(p.mark); i++) out = out.replace(p.re, (_m, n: string) => p.spans[Number(n)] ?? '')
|
|
288
|
+
return out
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// ------------------------------------------------------------- small tools
|
|
292
|
+
|
|
293
|
+
// Bits per character. Counted in a reused array for plain ASCII, since the
|
|
294
|
+
// scan calls this for every candidate token.
|
|
295
|
+
const COUNTS = new Int32Array(128)
|
|
296
|
+
|
|
297
|
+
function entropy(text: string): number {
|
|
298
|
+
let ascii = true
|
|
299
|
+
for (let i = 0; i < text.length; i++) {
|
|
300
|
+
const c = text.charCodeAt(i)
|
|
301
|
+
if (c >= 128) {
|
|
302
|
+
ascii = false
|
|
303
|
+
break
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
if (!ascii) {
|
|
307
|
+
const counts = new Map<string, number>()
|
|
308
|
+
for (const ch of text) counts.set(ch, (counts.get(ch) ?? 0) + 1)
|
|
309
|
+
let h = 0
|
|
310
|
+
for (const n of counts.values()) h -= (n / text.length) * Math.log2(n / text.length)
|
|
311
|
+
return h
|
|
312
|
+
}
|
|
313
|
+
COUNTS.fill(0)
|
|
314
|
+
for (let i = 0; i < text.length; i++) COUNTS[text.charCodeAt(i)]!++
|
|
315
|
+
let h = 0
|
|
316
|
+
for (let c = 0; c < 128; c++) {
|
|
317
|
+
const n = COUNTS[c]!
|
|
318
|
+
if (n > 0) h -= (n / text.length) * Math.log2(n / text.length)
|
|
319
|
+
}
|
|
320
|
+
return h
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
const B64_TABLE = (() => {
|
|
324
|
+
const t = new Int16Array(128).fill(-1)
|
|
325
|
+
const abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
|
|
326
|
+
for (let i = 0; i < abc.length; i++) t[abc.charCodeAt(i)] = i
|
|
327
|
+
t['-'.charCodeAt(0)] = 62
|
|
328
|
+
t['_'.charCodeAt(0)] = 63
|
|
329
|
+
return t
|
|
330
|
+
})()
|
|
331
|
+
|
|
332
|
+
// Decodes the start of a base64 text into one character per byte. The hooks
|
|
333
|
+
// run with no Buffer and no atob, so this is done by hand.
|
|
334
|
+
function decodeBase64Head(text: string, max = 8192): string {
|
|
335
|
+
let out = ''
|
|
336
|
+
let acc = 0
|
|
337
|
+
let bits = 0
|
|
338
|
+
for (let i = 0; i < text.length && out.length < max; i++) {
|
|
339
|
+
const c = text.charCodeAt(i)
|
|
340
|
+
if (c === 61) break
|
|
341
|
+
const v = c < 128 ? B64_TABLE[c]! : -1
|
|
342
|
+
if (v < 0) continue
|
|
343
|
+
acc = ((acc << 6) | v) & 0xffffff
|
|
344
|
+
bits += 6
|
|
345
|
+
if (bits >= 8) {
|
|
346
|
+
bits -= 8
|
|
347
|
+
out += String.fromCharCode((acc >> bits) & 0xff)
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
return out
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// Ranges of the text that the token scan skips. A `soft` range is skipped
|
|
354
|
+
// only when no key name stands in front of the token.
|
|
355
|
+
type Span = [start: number, end: number, soft: boolean]
|
|
356
|
+
|
|
357
|
+
class Spans {
|
|
358
|
+
private list: Span[] = []
|
|
359
|
+
private i = 0
|
|
360
|
+
add(start: number, end: number, soft = false): void {
|
|
361
|
+
if (end > start) this.list.push([start, end, soft])
|
|
362
|
+
}
|
|
363
|
+
seal(): this {
|
|
364
|
+
this.list.sort((a, b) => a[0] - b[0])
|
|
365
|
+
return this
|
|
366
|
+
}
|
|
367
|
+
// Offsets are asked in ascending order, so a cursor keeps this linear.
|
|
368
|
+
at(offset: number): 'hard' | 'soft' | null {
|
|
369
|
+
const l = this.list
|
|
370
|
+
while (this.i < l.length && l[this.i]![1] <= offset) this.i++
|
|
371
|
+
let found: 'hard' | 'soft' | null = null
|
|
372
|
+
for (let j = this.i; j < l.length && l[j]![0] <= offset; j++) {
|
|
373
|
+
if (l[j]![1] > offset) {
|
|
374
|
+
if (!l[j]![2]) return 'hard'
|
|
375
|
+
found = 'soft'
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return found
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// ------------------------------------------------------------ secret tests
|
|
383
|
+
|
|
384
|
+
// Escapes that end in a letter or digit and may sit right before a key:
|
|
385
|
+
// quoted-printable `=3D`, URL `%3D`, and the `\n` or `"` of escaped JSON.
|
|
386
|
+
const ESC = String.raw`[%=]3D|\\[nrtfbv]|\\u[0-9A-Fa-f]{4}`
|
|
387
|
+
const UUID = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
|
|
388
|
+
|
|
389
|
+
// A vendor prefix only counts at the start of a word, or right after one of
|
|
390
|
+
// those escapes. Without this, `task-runner-for-the-build` reads as `sk-`.
|
|
391
|
+
// The prefix is written first and the boundary checked behind it, so the
|
|
392
|
+
// regex engine can jump between occurrences of the prefix instead of
|
|
393
|
+
// testing the boundary at every character of the text.
|
|
394
|
+
function word(prefix: string, body: string, edge = 'A-Za-z0-9'): string {
|
|
395
|
+
return String.raw`${prefix}(?<=(?:^|[^${edge}]|${ESC})${prefix})${body}`
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// Shapes that belong to one vendor and mean one thing. These are hidden on
|
|
399
|
+
// sight, whatever their entropy. Each has a hint, a cheap test that the text
|
|
400
|
+
// could hold one, so a 10 MB log pays only for the shapes it might contain.
|
|
401
|
+
// Every quantifier is bounded: an open `{n,}` over a run of millions of
|
|
402
|
+
// characters overflows the regex stack.
|
|
403
|
+
type Shape = { hint: string | RegExp; re: string }
|
|
404
|
+
const VENDOR_SHAPES: readonly Shape[] = [
|
|
405
|
+
{ hint: 'sk-ant-', re: word('sk-ant-', '[A-Za-z0-9_-]{16,4096}') },
|
|
406
|
+
// A real OpenAI key has a digit or a capital early on; a kebab word does not.
|
|
407
|
+
{ hint: 'sk-', re: word('sk-', '(?=[a-z_-]{0,200}[0-9A-Z])[A-Za-z0-9_-]{20,4096}') },
|
|
408
|
+
{ hint: /gh[pousr]_/, re: word('gh[pousr]_', '[A-Za-z0-9]{20,4096}') },
|
|
409
|
+
{ hint: 'github_pat_', re: word('github_pat_', '[A-Za-z0-9_]{20,4096}') },
|
|
410
|
+
{ hint: 'xox', re: word('xox[baprse]-', '[A-Za-z0-9-]{10,4096}') },
|
|
411
|
+
{ hint: 'xapp-', re: word('xapp-', '[0-9]-[A-Za-z0-9-]{10,4096}') },
|
|
412
|
+
{ hint: /A[KS]IA/, re: word('A(?:KIA|SIA)', '[0-9A-Z]{16}(?![A-Za-z0-9])') },
|
|
413
|
+
{ hint: /[sr]k_(?:live|test)_/, re: word('[sr]k_(?:live|test)_', '[A-Za-z0-9]{16,4096}') },
|
|
414
|
+
{ hint: 'whsec_', re: word('whsec_', '[A-Za-z0-9]{16,4096}') },
|
|
415
|
+
{ hint: 'AIza', re: word('AIza', '[0-9A-Za-z_-]{30,4096}') },
|
|
416
|
+
{ hint: 'ya29.', re: word(String.raw`ya29\.`, '[A-Za-z0-9_-]{20,4096}') },
|
|
417
|
+
{ hint: 'npm_', re: word('npm_', '[A-Za-z0-9]{30,4096}') },
|
|
418
|
+
{ hint: 'dop_v1_', re: word('dop_v1_', '[a-f0-9]{40,4096}') },
|
|
419
|
+
{ hint: 'glpat-', re: word('glpat-', '[A-Za-z0-9_-]{16,4096}') },
|
|
420
|
+
{ hint: 'shpat_', re: word('shpat_', '[a-f0-9]{32,4096}') },
|
|
421
|
+
{ hint: 'SG.', re: word(String.raw`SG\.`, String.raw`[A-Za-z0-9_-]{16,4096}\.[A-Za-z0-9_-]{16,4096}`) },
|
|
422
|
+
{ hint: 'hooks.slack.com', re: String.raw`https://hooks\.slack\.com/services/[A-Za-z0-9/+_-]{16,4096}` },
|
|
423
|
+
{ hint: '/api/webhooks/', re: String.raw`https://discord(?:app)?\.com/api/webhooks/[0-9]{1,30}/[A-Za-z0-9_-]{16,4096}` },
|
|
424
|
+
// Prefixes built from words, which the name test would otherwise excuse.
|
|
425
|
+
{ hint: 'vercel_blob_rw_', re: word('vercel_blob_rw_', '[A-Za-z0-9]{16}_[A-Za-z0-9]{30}') },
|
|
426
|
+
{ hint: 'tskey-', re: word('tskey-', '[a-z]{1,20}-[A-Za-z0-9_-]{20,4096}') },
|
|
427
|
+
{ hint: 'pscale_', re: word('pscale_', '(?:tkn|pw|oauth)_[A-Za-z0-9_-]{32,4096}') },
|
|
428
|
+
{ hint: 'pdl_', re: word('pdl_', '(?:live|sdbx)_apikey_[A-Za-z0-9_-]{20,4096}') },
|
|
429
|
+
{ hint: 'pypi-AgE', re: word('pypi-AgE', '[A-Za-z0-9_-]{50,4096}') },
|
|
430
|
+
{ hint: 'sntrys_eyJ', re: word('sntrys_eyJ', '[A-Za-z0-9+/=_-]{50,4096}') },
|
|
431
|
+
{ hint: 'pcsk_', re: word('pcsk_', '[A-Za-z0-9_-]{40,4096}') },
|
|
432
|
+
{ hint: 'HRKU-AA', re: word('HRKU-AA', '[A-Za-z0-9_-]{40,4096}') },
|
|
433
|
+
{ hint: 'dckr_', re: word('dckr_', '(?:pat|oat)_[A-Za-z0-9_-]{20,4096}') },
|
|
434
|
+
{ hint: 'figd_', re: word('figd_', '[A-Za-z0-9_-]{30,4096}') },
|
|
435
|
+
{ hint: 'GOCSPX-', re: word('GOCSPX-', '[A-Za-z0-9_-]{20,4096}') },
|
|
436
|
+
{ hint: /gl(?:rt|dt|ptt|cbt|imt|ft|soat|oas|agent)-/, re: word('gl(?:rt|dt|ptt|cbt|imt|ft|soat|oas|agent)-', '[A-Za-z0-9_-]{20,4096}') },
|
|
437
|
+
{ hint: 'bb_live_', re: word('bb_live_', '[A-Za-z0-9_-]{20,4096}') },
|
|
438
|
+
{ hint: 'sb_secret_', re: word('sb_secret_', '[A-Za-z0-9_-]{20,4096}') },
|
|
439
|
+
{ hint: 'ATATT3', re: word('ATATT3', '[A-Za-z0-9_=-]{100,4096}') },
|
|
440
|
+
// Keys whose secret half sits after a `.`, `:` or `/`.
|
|
441
|
+
{ hint: 'pat', re: word('pat', String.raw`[A-Za-z0-9]{14}\.[a-f0-9]{64}`) },
|
|
442
|
+
{ hint: /[st]k\.eyJ/, re: word(String.raw`[st]k\.eyJ`, String.raw`[A-Za-z0-9_-]{10,4096}\.[A-Za-z0-9_-]{20,4096}`) },
|
|
443
|
+
{ hint: 'st.', re: word(String.raw`st\.`, String.raw`(?:[0-9a-f]{24}|${UUID})\.[0-9a-f]{32}\.[0-9a-f]{32}`) },
|
|
444
|
+
{ hint: /\/[0-9]{8}[0-9/]*:[0-9a-f]{32}/, re: String.raw`(?<![0-9])[0-9]/[0-9]{8,30}(?:/[0-9]{8,30})?:[0-9a-f]{32}` },
|
|
445
|
+
{ hint: /-[0-9a-f]{12}:[0-9a-f]{32}/, re: String.raw`(?<![0-9A-Fa-f])${UUID}:[0-9a-f]{32}` },
|
|
446
|
+
// Hugging Face tokens are letters only, so no entropy rule can see them.
|
|
447
|
+
{ hint: /(?:hf|api_org)_/, re: word('(?:hf|api_org)_', '[A-Za-z0-9]{34}(?![A-Za-z0-9_])') },
|
|
448
|
+
{ hint: 'ops_eyJ', re: word('ops_eyJ', '[A-Za-z0-9+/=_-]{50,4096}') },
|
|
449
|
+
{ hint: /fm[12][ar]?_/, re: word('fm[12][ar]?_', '[A-Za-z0-9+/=,_-]{50,4096}') },
|
|
450
|
+
{ hint: 'sig=', re: String.raw`(?<=[?&]sig=)[A-Za-z0-9%+/]{30,4096}` },
|
|
451
|
+
{ hint: 'AAAAAAAAAAAAAAAAAAAAA', re: word('AAAAAAAAAAAAAAAAAAAAA', '[A-Za-z0-9]{0,200}%[A-Za-z0-9%]{30,4096}') },
|
|
452
|
+
{ hint: 'Q~', re: String.raw`[A-Za-z0-9_~.-]{3}[0-9][A-Za-z0-9_~.-]Q~[A-Za-z0-9_~.-]{31,34}` },
|
|
453
|
+
// Lowercase and hex shapes that one vendor owns.
|
|
454
|
+
{ hint: 'e2b_', re: word('e2b_', String.raw`[a-f0-9]{40}(?![\w-])`) },
|
|
455
|
+
{ hint: 'lsv2_', re: word('lsv2_', String.raw`(?:pt|sk)_[a-f0-9]{32}_[a-f0-9]{10}(?![\w-])`) },
|
|
456
|
+
{ hint: /[0-9a-f]-us[0-9]/, re: String.raw`(?:(?<=${ESC})|(?<![A-Za-z0-9]))[a-f0-9]{32}-us[0-9]{1,2}(?![\w-])` },
|
|
457
|
+
{ hint: /-[a-f0-9]{8}-[a-f0-9]{8}(?![\w-])/, re: String.raw`(?:(?<=${ESC})|(?<![A-Za-z0-9]))[a-f0-9]{32}-[a-f0-9]{8}-[a-f0-9]{8}(?![\w-])` },
|
|
458
|
+
{ hint: /xa[ap]t-/, re: word('xa[ap]t-', String.raw`${UUID}(?![\w-])`) },
|
|
459
|
+
{ hint: 'SK', re: word('SK', String.raw`[0-9a-f]{32}(?![\w-])`) },
|
|
460
|
+
{ hint: 'tvly-', re: word('tvly-', '(?:dev-|prod-)?[A-Za-z0-9]{32,4096}') },
|
|
461
|
+
{ hint: 'tr_', re: word('tr_', '(?:dev|prod|stg|pat)_[A-Za-z0-9]{20,4096}', 'A-Za-z0-9_') },
|
|
462
|
+
{ hint: ':AA', re: String.raw`(?<![0-9])[0-9]{8,10}:AA[A-Za-z0-9_-]{33}(?![A-Za-z0-9_-])` },
|
|
463
|
+
{ hint: /a[ks]-/, re: word('a[ks]-', '[A-Za-z0-9]{22}(?![A-Za-z0-9_-])', 'A-Za-z0-9_-') },
|
|
464
|
+
// Keys that reuse a Stripe id prefix: Resend `re_`, Stripe client secrets.
|
|
465
|
+
{ hint: 're_', re: word('re_', '[A-Za-z0-9]{8}_[A-Za-z0-9]{24}(?![A-Za-z0-9_])', 'A-Za-z0-9_') },
|
|
466
|
+
{ hint: '_secret_', re: word('(?:pi|seti)_', '[A-Za-z0-9]{14,64}_secret_[A-Za-z0-9]{16,4096}', 'A-Za-z0-9_') },
|
|
467
|
+
// A Klaviyo private key is `pk_` and 34 alnum. Stripe and Clerk publishable
|
|
468
|
+
// keys always carry `live_` or `test_` after `pk_`, so they do not match.
|
|
469
|
+
{ hint: 'pk_', re: word('pk_', '[A-Za-z0-9]{34}(?![A-Za-z0-9_])', 'A-Za-z0-9_') },
|
|
470
|
+
// An age identity (SOPS reads it from keys.txt) is upper case and digits.
|
|
471
|
+
{ hint: 'AGE-SECRET-KEY-1', re: word('AGE-SECRET-KEY-1', '[0-9A-Z]{58}') },
|
|
472
|
+
{ hint: 'AGE-PLUGIN-', re: word('AGE-PLUGIN-[A-Z0-9]{1,40}-1', '[0-9A-Z]{20,4096}') },
|
|
473
|
+
// Tokens that are not mixed case, so no entropy rule sees them alone.
|
|
474
|
+
{ hint: /NR(?:AK|II|JS|AA|RA)-/, re: word('NR(?:AK|II|JS|AA|RA)-', '[A-Za-z0-9_-]{27,40}') },
|
|
475
|
+
{ hint: 'dt0c01.', re: word(String.raw`dt0c01\.`, String.raw`[A-Z0-9]{24}\.[A-Z0-9]{64}`) },
|
|
476
|
+
{ hint: /pat-(?:na|eu|ap)[0-9]-/, re: word('pat-(?:na|eu|ap)[0-9]-', String.raw`${UUID}(?![\w-])`) },
|
|
477
|
+
{ hint: /(?:sdk|api)-[0-9a-f]{8}-/, re: word('(?:sdk|api)-', String.raw`${UUID}(?![\w-])`) },
|
|
478
|
+
{ hint: 'lin_oauth_', re: word('lin_oauth_', '[a-f0-9]{64}') },
|
|
479
|
+
// pi-redact: Linear API keys and Vercel AI Gateway keys. Entropy already
|
|
480
|
+
// catches both; naming them makes the history scan see them as vendor keys.
|
|
481
|
+
{ hint: 'lin_api_', re: word('lin_api_', '[A-Za-z0-9]{32,4096}') },
|
|
482
|
+
{ hint: 'vck_', re: word('vck_', '[A-Za-z0-9]{40,4096}') },
|
|
483
|
+
{ hint: 'shippo_', re: word('shippo_(?:live|test)_', '[a-f0-9]{40}') },
|
|
484
|
+
{ hint: 'dapi', re: word('dapi', String.raw`[a-f0-9]{32}(?:-[0-9]{1,2})?(?![\w-])`) },
|
|
485
|
+
{ hint: ':fx', re: String.raw`(?<![0-9a-f-])${UUID}:fx(?!\w)` },
|
|
486
|
+
{ hint: 'access_token$', re: String.raw`access_token\$(?:production|sandbox)\$[a-z0-9]{16}\$[a-f0-9]{32}` },
|
|
487
|
+
// A Twilio auth token is plain hex; it is known by the account SID beside it.
|
|
488
|
+
{ hint: /AC[0-9a-f]{32}["']?[ \t]*[,:]/, re: String.raw`(?<=\bAC[0-9a-f]{32}["']?[ \t]*[,:][ \t]*["']?)[0-9a-f]{32}(?![0-9A-Za-z])` },
|
|
489
|
+
]
|
|
490
|
+
|
|
491
|
+
// Each shape also takes in the rest of its token, so a match that stops at a
|
|
492
|
+
// `-` or `_` inside a key cannot leave the tail of the key on screen.
|
|
493
|
+
const VENDOR = VENDOR_SHAPES.map((s) => ({ hint: s.hint, re: new RegExp(`(?:${s.re})[A-Za-z0-9_-]{0,4096}`, 'g') }))
|
|
494
|
+
|
|
495
|
+
const GLUED = /(lin_api_(?=[A-Za-z0-9]{32})|vck_(?=[A-Za-z0-9]{40})|npm_(?=[A-Za-z0-9]{36}(?![A-Za-z0-9_]))|gh[pousr]_(?=[A-Za-z0-9]{36}(?![A-Za-z0-9_]))|sk-ant-(?=[A-Za-z0-9_-]{80})|xox[bp]-(?=[0-9]{6,}-))[A-Za-z0-9_-]{0,4096}/g
|
|
496
|
+
|
|
497
|
+
// pi-redact: every vendor-shaped key and private key block in `text`, with
|
|
498
|
+
// the shape's hint as its family. Nothing is minted; the history scan uses it
|
|
499
|
+
// to rewrite old session files with high-confidence matches only.
|
|
500
|
+
export function findVendorKeys(text: string): { value: string; family: string }[] {
|
|
501
|
+
const found: { value: string; family: string }[] = []
|
|
502
|
+
for (const { hint, re } of VENDOR) {
|
|
503
|
+
if (typeof hint === 'string' ? !text.includes(hint) : !hint.test(text)) continue
|
|
504
|
+
for (const m of text.matchAll(re)) if (!m[0].includes('[REDACTED-')) found.push({ value: m[0], family: typeof hint === 'string' ? hint : hint.source })
|
|
505
|
+
}
|
|
506
|
+
if (text.includes('PRIVATE KEY')) for (const m of text.matchAll(PRIVATE_KEY)) found.push({ value: m[0], family: 'PRIVATE KEY' })
|
|
507
|
+
// Prefixes distinctive enough to trust mid-word, as in `token=set<key>`
|
|
508
|
+
// where shell output glued a key to a word. Live scrubbing catches these by
|
|
509
|
+
// entropy; the history scan only trusts shapes.
|
|
510
|
+
for (const m of text.matchAll(GLUED)) if (!found.some((f) => f.value.includes(m[0]))) found.push({ value: m[0], family: m[1]! })
|
|
511
|
+
return found
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
function scrubVendors(text: string, cfg: Config): string {
|
|
515
|
+
let out = text
|
|
516
|
+
for (const { hint, re } of VENDOR) {
|
|
517
|
+
if (typeof hint === 'string' ? !out.includes(hint) : !hint.test(out)) continue
|
|
518
|
+
out = out.replace(re, (m) => (cfg.allow.has(m) || m.includes('[REDACTED-') ? m : mint('SECRET', m)))
|
|
519
|
+
}
|
|
520
|
+
return out
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// A private key block, up to its END line (never past the next BEGIN, so a
|
|
524
|
+
// file of BEGIN lines is not scanned again for each one). When the END line
|
|
525
|
+
// is missing (`head key.pem`), the base64 lines that follow the BEGIN line
|
|
526
|
+
// are still taken, and only those.
|
|
527
|
+
const PRIVATE_KEY =
|
|
528
|
+
/-----BEGIN[^\n-]{0,40}PRIVATE KEY(?: BLOCK)?-----(?:(?:(?!-----BEGIN)[\s\S]){0,65536}?-----END[^\n-]{0,40}PRIVATE KEY(?: BLOCK)?-----|(?:\r?\n(?:[A-Za-z][A-Za-z-]{0,40}: [^\n]*|[ \t]*))*(?:\r?\n[A-Za-z0-9+/=]{4}[A-Za-z0-9+/=]*(?=\r?\n|$))+)/g
|
|
529
|
+
const PUTTY_PRIVATE = /(Private-Lines: \d+\r?\n)((?:[A-Za-z0-9+/=]{1,80}(?:\r?\n|$))+)/g
|
|
530
|
+
|
|
531
|
+
// A run of dot-separated parts, where a JWT may hide. It has one starting
|
|
532
|
+
// point per run, so a long run is scanned once. A JWT regex of its own went
|
|
533
|
+
// quadratic on base64 JSON, where every row starts with `eyJ`.
|
|
534
|
+
const DOTTED = /(?<![A-Za-z0-9_-])[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)(?:\.[A-Za-z0-9_-]+)+/g
|
|
535
|
+
|
|
536
|
+
// The password inside a connection string: postgres://user:PASSWORD@host.
|
|
537
|
+
// The user may be empty (redis://:pw@host), and the password may hold `/` or
|
|
538
|
+
// `@`: it ends at the `@` that is followed by a host.
|
|
539
|
+
const URL_PASSWORD =
|
|
540
|
+
/\b([a-z][a-z0-9+.-]{1,20}:\/\/[^\s/@:[\]]{0,80}):(?!\d{1,5}(?:[/?#]|$))([^\s"'`<>]{3,200}?)@(?=[A-Za-z0-9.-]{1,253}(?::\d{1,5})?(?:[/?#\s"'`<>]|$))/gi
|
|
541
|
+
|
|
542
|
+
// A token used alone as the userinfo: `https://<token>@dev.azure.com/...`,
|
|
543
|
+
// the documented clone form for Azure DevOps, GitHub and Bitbucket tokens.
|
|
544
|
+
const URL_USERINFO = /\b([a-z][a-z0-9+.-]{1,20}:\/\/)([A-Za-z0-9_-]{20,200})@(?=[A-Za-z0-9.-]{1,253}(?::\d{1,5})?[/:])/g
|
|
545
|
+
|
|
546
|
+
// A credential in a URL query: ?api_key=, &sig=, ?accessToken=. The value may
|
|
547
|
+
// be percent-encoded, so it is decoded before it is judged.
|
|
548
|
+
const URL_KEYWORD =
|
|
549
|
+
'(?:[Aa]pi[_-]?[Kk]ey|API[_-]?KEY|[Aa]pikey|[Kk]ey|KEY|[Tt]oken|TOKEN|[Ss]ecret|SECRET|[Ss]ig|SIG|[Ss]ignature|SIGNATURE|[Pp]assword|PASSWORD|[Pp]asswd|[Pp]wd|PWD|[Aa]uth|AUTH|[Cc]redential|CREDENTIAL)'
|
|
550
|
+
const URL_PARAM = new RegExp(
|
|
551
|
+
String.raw`([?&;](?:[A-Za-z0-9._-]*[_.-]|[a-z0-9]+(?=[A-Z]))?${URL_KEYWORD}=)((?:[A-Za-z0-9+_~-]|%[0-9A-Fa-f]{2}){8,400}={0,2})(?=[&#\s"'<>)\]]|$)`,
|
|
552
|
+
'g',
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
// Values after a password-like key, judged by the key and not by entropy:
|
|
556
|
+
// a human password (`Summer2024!`, `hunter2hunter2`) has too little.
|
|
557
|
+
const PW_WORD = String.raw`(?:password|passwd|passphrase|secret|secret[_-]?key|client[_-]?secret|api[_-]?key|apikey|private[_-]?key|secret[_-]?access[_-]?key|access[_-]?key|auth[_-]?token)`
|
|
558
|
+
// Short password names (`pass`, `pw`, `pwd`, `passcode`). They count only as
|
|
559
|
+
// the whole key or after a separator, so `compass` and `bypass` are left.
|
|
560
|
+
const SHORT_PW = String.raw`(?:pass(?:[_-]?code)?|pwd?)`
|
|
561
|
+
const SHORT_KEY = new RegExp(String.raw`(?:^|[_.-])${SHORT_PW}$`, 'i')
|
|
562
|
+
// A short key (`pass:`) with a number or a short word after it is a count
|
|
563
|
+
// or a status, not a password.
|
|
564
|
+
const shortKeyStatus = (key: string, v: string): boolean => SHORT_KEY.test(key) && (/^[0-9]+$/.test(v) || /^[a-z]{1,8}$/.test(v))
|
|
565
|
+
const STRONG_QUOTED = new RegExp(
|
|
566
|
+
String.raw`((?:^|[\s{,;(\[])(\\?["']?)((?:[A-Za-z0-9._-]*[_.-]|pg)?${PW_WORD}|(?:[A-Za-z0-9._-]*[_.-])?${SHORT_PW})\2[ \t]*[:=][ \t]*)(\\?["'])([^\n]{3,200}?)\4`,
|
|
567
|
+
'gim',
|
|
568
|
+
)
|
|
569
|
+
// A YAML list item (`- password: x`, compose `- DB_PASS=x`) counts too.
|
|
570
|
+
const STRONG_LINE = new RegExp(
|
|
571
|
+
String.raw`^([ \t]*(?:-[ \t]+)?(?:export[ \t]+)?([A-Za-z0-9._-]*${PW_WORD}|(?:[A-Za-z0-9._-]*[_.-])?${SHORT_PW})[ \t]*[:=][ \t]*)([^\s"'#;][^\s"';]{2,199})[ \t]*$`,
|
|
572
|
+
'gim',
|
|
573
|
+
)
|
|
574
|
+
// `DB_PASS=`: upper case with a separator only, so prose `Pass:` is left.
|
|
575
|
+
const ENV_PASS = /^([ \t]*(?:-[ \t]+)?(?:export[ \t]+)?()[A-Z0-9_]*_PASS[ \t]*[:=][ \t]*)([^\s"'#;][^\s"';]{2,199})[ \t]*$/gm
|
|
576
|
+
// A Gmail app password written unquoted: four groups of four letters.
|
|
577
|
+
const APP_PASSWORD = new RegExp(
|
|
578
|
+
String.raw`^([ \t]*(?:-[ \t]+)?(?:export[ \t]+)?[A-Za-z0-9._-]*${PW_WORD}[ \t]*[:=][ \t]*)([a-z]{4}(?: [a-z]{4}){3})[ \t]*$`,
|
|
579
|
+
'gim',
|
|
580
|
+
)
|
|
581
|
+
// The line after a YAML block indicator: `password: |` then the value.
|
|
582
|
+
const BLOCK_PASSWORD = new RegExp(
|
|
583
|
+
String.raw`^([ \t]*(?:-[ \t]+)?["']?[A-Za-z0-9._-]*${PW_WORD}["']?[ \t]*:[ \t]*[|>][-+]?[0-9]?[ \t]*\r?\n[ \t]+)(\S[^\n]{2,199}?)[ \t]*(?=\r?$)`,
|
|
584
|
+
'gim',
|
|
585
|
+
)
|
|
586
|
+
// SQL: `CREATE USER x WITH PASSWORD '...'`, MySQL `IDENTIFIED BY '...'`.
|
|
587
|
+
const SQL_PASSWORD =
|
|
588
|
+
/(\b(?:(?:with|encrypted|unencrypted)[ \t]+password|identified[ \t]+(?:with[ \t]+\S+[ \t]+)?by(?:[ \t]+password)?)[ \t]+)'([^'\n]{3,200})'/gi
|
|
589
|
+
// Terraform: the default of a variable whose name says it is a password.
|
|
590
|
+
const HCL_DEFAULT = new RegExp(String.raw`(\bvariable[ \t]+"[\w-]*${PW_WORD}"[ \t]*\{[^{}]{0,400}?\bdefault[ \t]*=[ \t]*")([^"\n]{3,200})"`, 'gi')
|
|
591
|
+
// Terraform state: the result of a random_password (usually a database
|
|
592
|
+
// master password), and an output that sits under a password-like name.
|
|
593
|
+
const TF_RESULT = /("result"\s*:\s*")((?:[^"\\\n]|\\.){3,400})"/g
|
|
594
|
+
const TF_OUTPUT = new RegExp(
|
|
595
|
+
String.raw`("([\w.-]{1,100})"\s*:\s*\{\s*(?:"(?:sensitive|type)"\s*:\s*[^,{}]{1,40},\s*){0,4}"value"\s*:\s*")((?:[^"\\\n]|\\.){3,400})"`,
|
|
596
|
+
'g',
|
|
597
|
+
)
|
|
598
|
+
// A name and its value as sibling fields: a HAR header or form field, a
|
|
599
|
+
// Kubernetes env entry. The name decides; the value sits apart from it.
|
|
600
|
+
const JSON_NAME_VALUE = /("name"\s*:\s*"([^"\n]{1,100})"\s*,\s*"value"\s*:\s*")((?:[^"\\\n]|\\.){3,400})"/g
|
|
601
|
+
const JSON_VALUE_NAME = /("value"\s*:\s*")((?:[^"\\\n]|\\.){3,400})("\s*,\s*"name"\s*:\s*"([^"\n]{1,100})")/g
|
|
602
|
+
const YAML_NAME_VALUE =
|
|
603
|
+
/^([ \t]*)(-[ \t]+name:[ \t]*["']?([\w.-]{1,100})["']?[ \t]*\r?\n\1[ \t]+value:[ \t]*(["']?))([^\n"']{3,200}?)\4[ \t]*(?=\r?$)/gm
|
|
604
|
+
const PW_NAME = new RegExp(String.raw`^[\w.-]*${PW_WORD}$|^(?:[\w.-]*[_.-])?${SHORT_PW}$`, 'i')
|
|
605
|
+
// SQL: an INSERT whose column list names a key or a password.
|
|
606
|
+
const SQL_INSERT = /\bINSERT[ \t]+INTO[ \t]+[\w."`[\]]{1,100}[ \t]*\(([^()\n]{1,1000})\)\s*VALUES\s*/gi
|
|
607
|
+
// `PGPASSWORD=x psql` is the one common env name with no separator.
|
|
608
|
+
const KV_PASSWORD = /((?:^|[\s;&?,])-{0,2}(?:[A-Za-z0-9._-]*[_.-]|pg)?(?:password|passwd)=|[;&?]pwd=)([^\s"'&;,]{3,200})/gim
|
|
609
|
+
|
|
610
|
+
// A generated password in quotes, with no name beside it: upper, lower,
|
|
611
|
+
// digit and a symbol from the Terraform and password-manager sets, no space,
|
|
612
|
+
// no path or regex characters. The token scan cuts such a value at each
|
|
613
|
+
// symbol, so it is judged whole here.
|
|
614
|
+
// The opening quote must follow a space, `=`, `:`, `(`, `,`, `[` or `{`, so
|
|
615
|
+
// the text between two strings (`"0"=20cellpadding=3D"`) is not taken.
|
|
616
|
+
const QUOTED_VALUE = /(?<=^|[\s:=(,[{])(["'])([^\s"'\\/.,;`]{12,128})\1/gm
|
|
617
|
+
const PW_SYMBOL = /[!#$%&*()=+[\]{}<>:?@^~]/
|
|
618
|
+
|
|
619
|
+
function looksGenerated(v: string): boolean {
|
|
620
|
+
if (!/[A-Z]/.test(v) || !/[a-z]/.test(v) || !/[0-9]/.test(v) || !PW_SYMBOL.test(v)) return false
|
|
621
|
+
if (/\$\{|\{\{|%\(|^\^|\$$|[A-Za-z]-[A-Za-z]\]|=>|::|\(\)|<\/?[a-z]|=[0-9A-F]{2}|[0-9]{6}/.test(v)) return false
|
|
622
|
+
// Real words in it: code, a format, a shortcut (`Ctrl+Alt+F12`), a payee
|
|
623
|
+
// with an order number (`Merchant#a12345`).
|
|
624
|
+
const words = v.match(/[A-Za-z]{4,}/g)?.filter((w) => WORDY.test(w)) ?? []
|
|
625
|
+
if (words.length >= 2 || words.some((w) => w.length >= 6)) return false
|
|
626
|
+
// Mostly letters and digits: a run of symbols is binary or markup.
|
|
627
|
+
if ((v.match(/[A-Za-z0-9]/g)?.length ?? 0) * 5 < v.length * 3) return false
|
|
628
|
+
return entropy(v) >= 3.5
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
// Things that stand in for a value without being one: a variable, a call, a
|
|
632
|
+
// template, a placeholder, a type, the key's own name, a path.
|
|
633
|
+
// A call has to close (`getSecret("x")`), and a YAML tag is a word
|
|
634
|
+
// (`!Ref DbPassword`), so a generated password that starts `ab(` or `!` is
|
|
635
|
+
// not taken for one.
|
|
636
|
+
const REFERENCE =
|
|
637
|
+
/^[A-Za-z_$][\w$.]*\(.*\)[,;]?$|^\$|^\{\{|^<.*>$|^%\(|^!!?[A-Za-z][\w:]*(?:[ \t]|$)|^\*+$|^(?:true|false|null|none|nil|undefined|required|optional|string|str|bool|int|number)$|^(?:process\.env|os\.environ|env\.|config\.|settings\.)|^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+$/i
|
|
638
|
+
const SELF_REF =
|
|
639
|
+
/^[{`(]?(?:[a-z][A-Za-z0-9]*)?(?:Password|Passwd|Secret|Key|Token|Pwd)[A-Za-z0-9]*[)},`]*$|^(?:password|passwd|secret|apikey|api_key|pwd|token)[),}]*,?$/
|
|
640
|
+
const PLACEHOLDER = /^(?:x{3,}|\.{3,}|…|(?:your|my|<)[_ -]?[a-z_ -]*(?:key|password|secret|token)[a-z_ -]*>?)$/i
|
|
641
|
+
|
|
642
|
+
function isStandIn(v: string): boolean {
|
|
643
|
+
return (
|
|
644
|
+
REFERENCE.test(v) ||
|
|
645
|
+
SELF_REF.test(v) ||
|
|
646
|
+
PLACEHOLDER.test(v) ||
|
|
647
|
+
/^(?:[/~]|\.\.?\/)/.test(v) ||
|
|
648
|
+
/^\{[\w$.]+\}$/.test(v) ||
|
|
649
|
+
// A test fixture made of words: `test-internal-key`, `fake_secret`.
|
|
650
|
+
(/^(?:[a-z]+[_-])*(?:test|fake|dummy|mock|sample|example|placeholder|changeme)(?:[_-][a-z]+)*$/.test(v) && /[_-]/.test(v)) ||
|
|
651
|
+
// `password: pw,` in an object literal is code, not a value.
|
|
652
|
+
/^[A-Za-z_$][\w$]*,$/.test(v)
|
|
653
|
+
)
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
// Credentials on a command line: `--password x`, `curl -u user:pw`,
|
|
657
|
+
// `mysql -pPW`, a .pgpass line, a .netrc entry, `gh secret set --body`.
|
|
658
|
+
const CLI_FLAG =
|
|
659
|
+
/((?:^|\s)--?(?:password|passwd|pass|api-key|apikey|api_key|token|client-secret|auth-token|access-token)(?:=|[ \t]+)(["']?))(?![$<{%])([^\s"'=-][^\s"']{2,200})\2/g
|
|
660
|
+
const CLI_USERPASS =
|
|
661
|
+
/(\b(?:curl|wget|http|https|xh|httpie|lftp|ftp)\b[^\n|;&]*?\s(?:-u|-U|--user|--proxy-user)(?:[ \t]+|=)["']?[^\s:"'$]+:)(?!\$)([^\s"'`]{3,200})/g
|
|
662
|
+
const MYSQL_P =
|
|
663
|
+
/(\b(?:mysql(?:dump|admin|import|sh|check|slap|binlog)?|mariadb(?:-dump|-admin)?)\b[^\n|;&]*?\s-p)(?!\$)([^\s`-][^\s`]{2,200})/g
|
|
664
|
+
const PGPASS =
|
|
665
|
+
/^((?:\*|[A-Za-z0-9.-]+|\[[0-9A-Fa-f:]+\]):(?:\d{1,5}|\*):(?![0-9]+:)[A-Za-z0-9_$*.-]+:[A-Za-z0-9_$*.@-]+:)(?!\/\/)(\S{3,200})$/gm
|
|
666
|
+
const NETRC =
|
|
667
|
+
/((?:^|\n)[ \t]*(?:machine[ \t]+\S+|default)(?:\s+(?:login|account|port)[ \t]+\S+)*\s+password[ \t]+)(\S{3,200})(?=[ \t]*(?:\r?\n|$)|\s+(?:machine|default|login|account|macdef)\b)/g
|
|
668
|
+
const GH_SECRET = /(\bgh[ \t]+secret[ \t]+set\b[^\n;&|]*?[ \t](?:-b|--body)(?:[ \t]+|=)(["']?))([^\s"']{3,400})(?=\2)/g
|
|
669
|
+
// Short password flags that only mean a password for one command:
|
|
670
|
+
// `sshpass -p`, `docker login -p`, `mongosh -p`, `7z -pPW`, `sqlcmd -P`,
|
|
671
|
+
// `zip -P`, `redis-cli -a`, `ldapsearch -w`. Each is anchored at its command.
|
|
672
|
+
const CMD_PW_TOOLS: [cmd: string, flag: string][] = [
|
|
673
|
+
[String.raw`sshpass|(?:docker|podman|nerdctl|helm[ \t]+registry)[ \t]+login|mongo(?:sh|dump|restore|import|export)?`, String.raw`-p[ \t]*`],
|
|
674
|
+
[String.raw`7za?|7zz`, '-p'],
|
|
675
|
+
[String.raw`sqlcmd|zip`, String.raw`-P[ \t]+`],
|
|
676
|
+
[String.raw`redis-cli|valkey-cli`, String.raw`-a[ \t]+`],
|
|
677
|
+
[String.raw`ldap[a-z]+`, String.raw`-w[ \t]+`],
|
|
678
|
+
]
|
|
679
|
+
const CMD_PW = CMD_PW_TOOLS.map(
|
|
680
|
+
([cmd, flag]) => new RegExp(String.raw`(\b(?:${cmd})\b[^\n|;&]*?[ \t]${flag}(["']?))(?![$-])([^\s"']{3,200})\2`, 'g'),
|
|
681
|
+
)
|
|
682
|
+
const CMD_PW_GATE = /sshpass|login|mongo|7z|sqlcmd|zip|redis-cli|valkey-cli|ldap/
|
|
683
|
+
// `openssl ... -passout pass:PW`, `smbclient -U user%PW`, `htpasswd -b file
|
|
684
|
+
// user PW`, `rabbitmqctl add_user user PW`.
|
|
685
|
+
const OPENSSL_PASS = /(-pass(?:in|out)?[ \t]+pass:)([^\s"']{3,200})/g
|
|
686
|
+
const SMB_USER = /(\bsmbclient\b[^\n|;&]*?[ \t]-U[ \t]+["']?[^\s%"']+%)([^\s"']{3,200})/g
|
|
687
|
+
const HTPASSWD = /(\bhtpasswd[ \t]+-[a-zA-Z]*b[a-zA-Z]*[ \t]+(?:-[a-zA-Z]+[ \t]+)*\S+[ \t]+\S+[ \t]+)(["']?)([^\s"']{3,200})\2/g
|
|
688
|
+
const NET_USER = /(\bnet[ \t]+user[ \t]+(?:"[^"\n]{1,64}"|\S+)[ \t]+)(["']?)((?!\/)[^\s"'*]{3,200})\2/gi
|
|
689
|
+
const RABBIT_USER = /(\brabbitmqctl[ \t]+(?:add_user|change_password)[ \t]+\S+[ \t]+)(["']?)([^\s"']{3,200})\2/g
|
|
690
|
+
|
|
691
|
+
// Cookies: every value in a Cookie header, curl -b, or cookies.txt.
|
|
692
|
+
const COOKIE = /((?:^|["'\s])(?:(?:set-)?cookie["']?[ \t]*:|-b|--cookie)[ \t]*["']?)([^"'\n]*)/gim
|
|
693
|
+
const COOKIE_PAIR = /(^|;[ \t]*)([^=;\s]+=)([^;\s]{16}[^;\s]*)/g
|
|
694
|
+
const COOKIE_ATTR = /^(?:path|domain|expires|max-age|samesite|priority)=$/i
|
|
695
|
+
const COOKIE_VALUE = /^(?:s%3A)?([A-Za-z0-9_+%-]{16}[A-Za-z0-9_+%-]*)(?:\.[A-Za-z0-9_+%/-]{16}[A-Za-z0-9_+%/-]*)?={0,2}$/i
|
|
696
|
+
const COOKIES_TXT =
|
|
697
|
+
/^((?:#HttpOnly_)?\.?[\w.-]+\t(?:TRUE|FALSE)\t\S+\t(?:TRUE|FALSE)\t\d+\t[^\t\n]+\t)([^\t\r\n]{16}[^\t\r\n]*)$/gm
|
|
698
|
+
const HAR_COOKIE = /("name"\s*:\s*"[^"]*(?:sess|sid|token|auth)[^"]*"\s*,\s*"value"\s*:\s*")([^"]{16}[^"]*)"/gi
|
|
699
|
+
|
|
700
|
+
// Basic auth and Docker `auth`: base64 of `user:password`, often short.
|
|
701
|
+
const BASIC_AUTH = /((?:["']auth["'][ \t]*:[ \t]*["']|\b[Bb]asic[ \t]+))([A-Za-z0-9+/]{8,200}={0,2})(?![A-Za-z0-9+/=])/g
|
|
702
|
+
// A Kubernetes bootstrap token: `abcdef.0123456789abcdef`.
|
|
703
|
+
const BOOTSTRAP = /\b([a-z0-9]{6})\.([a-z0-9]{16})\b/g
|
|
704
|
+
|
|
705
|
+
// Inline media in a data: URI. Only media types: a key put in a JSON or text
|
|
706
|
+
// data URI is still scanned. The length floor stops a short key placed after
|
|
707
|
+
// a fake image prefix from being excused.
|
|
708
|
+
const DATA_URI =
|
|
709
|
+
/data:(?:(?:image|font|audio|video)\/[\w.+-]+|application\/(?:x-)?font-[\w.+-]+|application\/vnd\.ms-fontobject)(?:;[\w-]+=[\w.-]+)*;base64,[A-Za-z0-9+/]{64}[A-Za-z0-9+/]*={0,2}/g
|
|
710
|
+
|
|
711
|
+
// A base64 blob: one long run, or three or more wrapped lines (a PEM bundle,
|
|
712
|
+
// a MIME part). A trailing short line is only taken when a blank line, a
|
|
713
|
+
// `--boundary` or the end of the text follows it.
|
|
714
|
+
const B64_LINE = String.raw`[A-Za-z0-9+/]{40}[A-Za-z0-9+/]*={0,2}\r?\n`
|
|
715
|
+
const BLOB = new RegExp(
|
|
716
|
+
String.raw`(?<![A-Za-z0-9+/=])(?:(?:${B64_LINE}){3}(?:${B64_LINE})*(?:[A-Za-z0-9+/]+={0,2}(?=\r?\n[\r\n-]|\r?\n?$))?|[A-Za-z0-9+/]{256}[A-Za-z0-9+/]*={0,2})`,
|
|
717
|
+
'g',
|
|
718
|
+
)
|
|
719
|
+
const BLOB_NAME =
|
|
720
|
+
/(?:key|token|secret|password|passwd|credential|auth|private|signature|session|cookie|dsn|cert[_-]?key)[\w.-]*["'\]\s]{0,4}[:=]{1,2}\s*["'`]?\s*$/i
|
|
721
|
+
const BLOB_INNER = /PRIVATE KEY|OPENSSH|"private_key"|"client_secret"|"refresh_token"|"password"|"auths"/
|
|
722
|
+
|
|
723
|
+
// A Claude thinking-block signature, and a redacted thinking block: opaque
|
|
724
|
+
// to everyone and useless as a credential. Only the value is parked; the
|
|
725
|
+
// thinking text beside it is still scanned.
|
|
726
|
+
const THINKING_SIG =
|
|
727
|
+
/("type"\s*:\s*"thinking"\s*,\s*"thinking"\s*:\s*"[^"\\]*(?:\\.[^"\\]*)*"\s*,\s*"signature"\s*:\s*")([A-Za-z0-9+/]{200}[A-Za-z0-9+/]*={0,2})(?=")/g
|
|
728
|
+
const REDACTED_THINKING = /("type"\s*:\s*"redacted_thinking"\s*,\s*"data"\s*:\s*")([A-Za-z0-9+/]{100}[A-Za-z0-9+/]*={0,2})(?=")/g
|
|
729
|
+
|
|
730
|
+
// A lockfile or SRI integrity hash: exact digest lengths only, so a key
|
|
731
|
+
// glued after `sha256-` is still judged.
|
|
732
|
+
const SRI = /\bsha(?:1-[A-Za-z0-9+/]{27}=|256-[A-Za-z0-9+/]{43}=|384-[A-Za-z0-9+/]{64}|512-[A-Za-z0-9+/]{86}==)(?![A-Za-z0-9+/=])/g
|
|
733
|
+
|
|
734
|
+
// Public font files: the file name is a hash, not a secret. Path characters
|
|
735
|
+
// only, so a query string on the same URL is still scanned.
|
|
736
|
+
const PUBLIC_URL = /https:\/\/fonts\.gstatic\.com\/[A-Za-z0-9_/.-]+/g
|
|
737
|
+
|
|
738
|
+
// Mail: an optional `cat -n`, Read or `grep -n` prefix on each line.
|
|
739
|
+
const PFX = String.raw`(?:[ \t]*\d+\t|[^\s:]+:\d+[:-])?`
|
|
740
|
+
const PFX_RE = new RegExp(`^${PFX}`)
|
|
741
|
+
const CTE = new RegExp(String.raw`^${PFX}Content-Transfer-Encoding:[ \t]*base64[ \t]*\r?$`, 'gim')
|
|
742
|
+
const CONTENT_TYPE = new RegExp(String.raw`^${PFX}Content-Type:[ \t]*(?:\r?\n${PFX}[ \t]+)?([^;\s]+)`, 'im')
|
|
743
|
+
const MAIL_HEADER_NAMES =
|
|
744
|
+
'DKIM-Signature|X-Google-DKIM-Signature|ARC-Seal|ARC-Message-Signature|ARC-Authentication-Results|Authentication-Results|Message-I[Dd]|References|In-Reply-To|Feedback-ID|X-SG-EID|X-SG-ID|X-MSFBL|X-Gm-Message-State|X-Google-Smtp-Source|X-Gm-Gg|X-Gm-Features|X-Received|Received-SPF|Return-Path|Received'
|
|
745
|
+
const MAIL_HEADER = new RegExp(String.raw`^${PFX}(?:${MAIL_HEADER_NAMES}):[^\n]*(?:\r?\n${PFX}[ \t][^\n]*){0,12}`, 'gm')
|
|
746
|
+
const HEADER_LINE = new RegExp(String.raw`^${PFX}[A-Z][A-Za-z0-9-]*:`)
|
|
747
|
+
const ENCODED_WORD = /=\?[\w-]+\?[Bb]\?[A-Za-z0-9+/=]{4}[A-Za-z0-9+/=]*\?=/g
|
|
748
|
+
const BOUNDARY = /Content-Type:[ \t]*multipart\/[^\n]*?(?:\r?\n[ \t][^\n]*?)?\bboundary="?([^"\s;]+)"?/gi
|
|
749
|
+
|
|
750
|
+
// The generic scan. A slash-bearing base64 run (an AWS secret, `openssl rand
|
|
751
|
+
// -base64`) is judged whole first; anything else is cut into candidate
|
|
752
|
+
// tokens. `/` and `.` are left out of a candidate on purpose: with them in,
|
|
753
|
+
// every long file path becomes one. A token never starts inside a `%XX`
|
|
754
|
+
// escape, and `=` only counts as padding when nothing follows it.
|
|
755
|
+
const CAND_BODY = String.raw`[A-Za-z0-9+_-]{12,200}(?:={1,2}(?![A-Za-z0-9=]))?`
|
|
756
|
+
const CANDIDATE_SRC = String.raw`(?<=%[0-9A-Fa-f]{2})(?<!%25)${CAND_BODY}|(?<=%25[0-9A-Fa-f]{2})${CAND_BODY}|(?<!%[0-9A-Fa-f]?|%25[0-9A-Fa-f]?)${CAND_BODY}`
|
|
757
|
+
const SLASHED_SRC = String.raw`(?<![A-Za-z0-9+/_.%~-])[A-Za-z0-9+/]{20,300}={0,2}(?![A-Za-z0-9+/=_.%~-])`
|
|
758
|
+
const CANDIDATE = new RegExp(CANDIDATE_SRC, 'g')
|
|
759
|
+
const SCAN = new RegExp(`(${SLASHED_SRC})|${CANDIDATE_SRC}`, 'g')
|
|
760
|
+
|
|
761
|
+
// A name to the left of the candidate that says the candidate is a key:
|
|
762
|
+
// `API_KEY=`, `"token":`, an escaped `\"token\":`, `"token":`,
|
|
763
|
+
// `Authorization: Bearer `, `Authorization: <any scheme> `, `--api-key `, a
|
|
764
|
+
// table cell, a YAML block, a Go header map `X-Api-Key:[`, a Python header
|
|
765
|
+
// tuple `('X-Api-Key', '`. `PAT` is a personal access token, not `PATH`.
|
|
766
|
+
const NAME_WORDS =
|
|
767
|
+
'key|token|secret|password|passwd|pwd|credential|auth|bearer|private|signature|session|cookie|dsn|salt|nonce|otp|sess(?:ion)?[_-]?id|(?<![A-Za-z])sid|(?<![A-Za-z])sig|(?<![A-Za-z])pat(?![a-z])'
|
|
768
|
+
const Q = String.raw`(?:\\{0,3}["'${'`'}]|&(?:quot|apos|#0*3[49]|#x0*2[27]);|\\u00(?:22|27))`
|
|
769
|
+
const NAMED = new RegExp(
|
|
770
|
+
'(?:' +
|
|
771
|
+
String.raw`(?:${NAME_WORDS})(?:(?<=key)[_-]?id)?(?:${Q}|\]|\s){0,6}[:=]{1,2}(?:${Q}|\s){0,4}\[?` +
|
|
772
|
+
String.raw`|(?:${NAME_WORDS})[\w-]{0,20}${Q}[ \t]*,[ \t]*${Q}` +
|
|
773
|
+
String.raw`|\bbearer[ \t]+["']?` +
|
|
774
|
+
String.raw`|authorization["']?[ \t]*[:=][ \t]*["']?(?:[A-Za-z][\w-]{0,31}[ \t]+)?` +
|
|
775
|
+
String.raw`|(?:^|\s)--?(?![a-z_-]*(?:public|pub|cache|idempotency|signing|recv)[_-]?key)[a-z_-]*(?:key|token|secret|password|passwd|pw|auth)[ \t]+["']?` +
|
|
776
|
+
String.raw`|\|[ \t]*[\w.-]*(?:key|token|secret|password)[ \t]*\|[ \t]*` +
|
|
777
|
+
String.raw`|(?:key|token|secret|password|passwd|pwd|credential|auth)[\w-]*["']?[ \t]*:[ \t]*[>|][-+1-9]{0,2}[ \t]*\r?\n[ \t]+` +
|
|
778
|
+
')$',
|
|
779
|
+
'i',
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
// A sibling field name that makes its value a key: the value is then
|
|
783
|
+
// judged as if the name stood right before it.
|
|
784
|
+
const KEY_NAME = new RegExp(String.raw`(?:${NAME_WORDS}|authorization)`, 'i')
|
|
785
|
+
|
|
786
|
+
const HEX_ONLY = /^[0-9a-f]+$/i
|
|
787
|
+
const PREFIXED_KEY = /^[a-z]{2,12}(?:_v[0-9])?[_-]([a-z0-9]{32,})$/
|
|
788
|
+
const DIGITS_ONLY = /^[0-9]+$/
|
|
789
|
+
const LETTERS_ONLY = /^[A-Za-z]+$/
|
|
790
|
+
|
|
791
|
+
// Public ids with a shape rather than a prefix: a YouTube channel or
|
|
792
|
+
// playlist, a Monzo transaction, a Vercel deployment, team or project, an
|
|
793
|
+
// OpenAI org, a Claude Code session, environment or server tool call. Exact
|
|
794
|
+
// lengths only, and the Anthropic ids start `01`, so any other token after
|
|
795
|
+
// the same prefix is still judged.
|
|
796
|
+
const PUBLIC_SHAPE =
|
|
797
|
+
/^(?:UC[A-Za-z0-9_-]{22}|PL[A-Za-z0-9_-]{16,32}|(?:tx|mm|acc|pot|merch|user)_0000[A-Za-z0-9]{18}|dpl_[A-Za-z0-9]{24,32}|org-[A-Za-z0-9]{24}|team_[A-Za-z0-9]{24}|prj_[A-Za-z0-9]{28}|(?:session|env|srvtoolu)_01[A-Za-z0-9]{22})$/
|
|
798
|
+
|
|
799
|
+
// Prefixes that name a PUBLIC object id, not a key. Stripe hands these out in
|
|
800
|
+
// dashboards, invoices and source code; hiding them makes a session useless
|
|
801
|
+
// and protects nothing. `pk_live_` is Stripe's publishable key, also public.
|
|
802
|
+
const PUBLIC_PREFIX =
|
|
803
|
+
/^(?:price|prod|cus|sub|sched|in|ch|pi|cs|py|re|txn|il|si|seti|evt|acct|promo|coupon|plan|card|ba|src|dp|du|iv|ii|rcpt|file|link|pm|tok|pk|test|toolu|msg|req|run|wf)_/i
|
|
804
|
+
// A public id is the prefix, maybe `live_` or `test_`, then one alnum body.
|
|
805
|
+
// `prod_api_key_<random>` is not that shape, so it is judged like any token.
|
|
806
|
+
const PUBLIC_ID_BODY = /^(?:(?:live|test)_)?[A-Za-z0-9]+$/
|
|
807
|
+
// A WooCommerce consumer secret reuses Stripe's `cs_`; Stripe's never is
|
|
808
|
+
// 40 lowercase hex.
|
|
809
|
+
// A Klaviyo private key reuses `pk_`, with no `live_` or `test_` after it.
|
|
810
|
+
const NOT_PUBLIC = /^cs_[a-f0-9]{40}$|^pk_(?!live_|test_)[A-Za-z0-9]{34}$/
|
|
811
|
+
// Generic prefixes a key name in front may override (`API_KEY=test_...`).
|
|
812
|
+
const SOFT_PUBLIC = /^(?:test|tok)_/i
|
|
813
|
+
|
|
814
|
+
// A name written in code (`archived-modules-marker`, `handleCheckoutSession`,
|
|
815
|
+
// `lesson_article_outline_open_v1`) reads as high entropy but is a word list.
|
|
816
|
+
// A key is not built out of words.
|
|
817
|
+
const WORD_PART = /^(?:[a-z]+[0-9]{0,3}|[A-Z][a-z]+[0-9]{0,3}|[A-Z]{2,})$/
|
|
818
|
+
const ALNUM_ONLY = /^[A-Za-z0-9]+$/
|
|
819
|
+
const SEGMENTS = /[A-Z]+(?![a-z])|[A-Z]?[a-z]+|[0-9]+/g
|
|
820
|
+
const WORDY = /^[A-Za-z]?[a-z]{2,}$/
|
|
821
|
+
const SLASH_WORD = /^(?:[a-z]+[0-9]{0,3}|[A-Z][a-z]+[0-9]{0,3})$/
|
|
822
|
+
const STRICT_CAMEL = /^(?:[a-z]{2,}|[A-Z][a-z]{2,})(?:[A-Z][a-z]{2,})+[0-9]{0,3}$/
|
|
823
|
+
|
|
824
|
+
// `playerEventBatchV1Schema` splits into six segments of which four are
|
|
825
|
+
// words; a random key splits into many segments of which almost none are.
|
|
826
|
+
// A camel name has a capital in it: an all-lowercase token with a digit run
|
|
827
|
+
// in the middle is a random value more often than a name.
|
|
828
|
+
function isCamelName(token: string): boolean {
|
|
829
|
+
if (!ALNUM_ONLY.test(token) || !/[A-Z]/.test(token)) return false
|
|
830
|
+
const segs = token.match(SEGMENTS)
|
|
831
|
+
if (!segs || segs.join('') !== token) return false
|
|
832
|
+
const words = segs.filter((seg) => WORDY.test(seg)).length
|
|
833
|
+
return words >= 2 && words >= segs.length - 2
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
// Every character used once AND mostly in alphabet order: a charset constant.
|
|
837
|
+
// The order test matters: a random 24-character key has all distinct
|
|
838
|
+
// characters about 1% of the time.
|
|
839
|
+
function isAlphabet(token: string): boolean {
|
|
840
|
+
if (token.length < 20) return false
|
|
841
|
+
let runs = 0
|
|
842
|
+
for (let i = 1; i < token.length; i++) if (token.charCodeAt(i) === token.charCodeAt(i - 1) + 1) runs++
|
|
843
|
+
return runs * 2 >= token.length && new Set(token).size === token.length
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
// Half or more of the characters follow the one before in code order:
|
|
847
|
+
// `0123456789abcdef`, the base64 alphabet. A sample in docs, never a key.
|
|
848
|
+
function countsUp(s: string): boolean {
|
|
849
|
+
let runs = 0
|
|
850
|
+
for (let i = 1; i < s.length; i++) if (s.charCodeAt(i) === s.charCodeAt(i - 1) + 1) runs++
|
|
851
|
+
return runs * 2 >= s.length
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
function isMixed(s: string): boolean {
|
|
855
|
+
return /[a-z]/.test(s) && /[A-Z]/.test(s) && /[0-9]/.test(s)
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
// One word, or a number of up to four digits (a year, a month, a day).
|
|
859
|
+
function isSlugPart(part: string): boolean {
|
|
860
|
+
return WORD_PART.test(part) || /^[0-9]{1,4}$/.test(part)
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// A part made only of words, short acronyms and short numbers: `expenseID`,
|
|
864
|
+
// `2026`, `v2Final`. It names something; it is not a random remainder.
|
|
865
|
+
function isWordish(part: string): boolean {
|
|
866
|
+
const segs = part.match(SEGMENTS)
|
|
867
|
+
if (!segs || segs.join('') !== part) return false
|
|
868
|
+
return segs.every((seg) => WORDY.test(seg) || /^[A-Z]{2,4}$/.test(seg) || /^[0-9]{1,4}$/.test(seg))
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
function looksLikeName(token: string, named: boolean): boolean {
|
|
872
|
+
if (isCamelName(token)) return true
|
|
873
|
+
// A name beside the token outweighs the alphabet rule.
|
|
874
|
+
if (!named && isAlphabet(token)) return true
|
|
875
|
+
const parts = token.split(/[_-]/)
|
|
876
|
+
if (parts.length < 2) return false
|
|
877
|
+
// Beside a key's name, one key-like part makes the whole token a key:
|
|
878
|
+
// `django-insecure-<random>`, `live-secret-<random>`.
|
|
879
|
+
if (named && parts.some((p) => p.length >= 16 && /[0-9]/.test(p) && /[A-Za-z]/.test(p) && entropy(p) >= 3.0)) return false
|
|
880
|
+
// A long random remainder outweighs any words in front of it
|
|
881
|
+
// (`vercel_blob_rw_<random>`, `dckr_pat_<random>`). Mixed case is asked
|
|
882
|
+
// for, so a hex cache key or a timestamp is not a remainder.
|
|
883
|
+
const residue = parts.filter((p) => !WORD_PART.test(p) && !isCamelName(p) && !isWordish(p)).join('')
|
|
884
|
+
// Words plus one long number (a LinkedIn post slug, a Zoom recording, an
|
|
885
|
+
// Amazon ref tag): the number is an id, not part of a random remainder.
|
|
886
|
+
// Only the parts that are neither words, numbers nor date stamps count,
|
|
887
|
+
// and camel parts are not excused here, since a random string can pass
|
|
888
|
+
// the camel test.
|
|
889
|
+
const longNumber = !named && parts.some((p) => /^[A-Z]{0,4}[0-9]{6,}$/.test(p))
|
|
890
|
+
const randomPart = longNumber ? parts.filter((p) => !WORD_PART.test(p) && !/^[0-9]{1,4}$/.test(p) && !DIGITS_ONLY.test(p) && !DATE_STAMP.test(p)).join('') : ''
|
|
891
|
+
if ((longNumber ? (randomPart.length >= 8 ? 16 : 0) : residue.length) >= 16 && isMixed(residue)) return false
|
|
892
|
+
// Words, dates and short numbers only (`2025-08-26-lesson`): a slug.
|
|
893
|
+
if (parts.every(isSlugPart)) return true
|
|
894
|
+
// Three or more separated parts, two of them plain words: a naming
|
|
895
|
+
// convention (`META_Conv_Lookalike-Customers_FreeTrial_2024Q1`), not a key.
|
|
896
|
+
if (parts.length >= 3 && parts.filter((p) => /^[A-Za-z]{4,}$/.test(p)).length >= 2) return true
|
|
897
|
+
return longNumber && randomPart.length < 8
|
|
898
|
+
}
|
|
899
|
+
const DATE_STAMP = /^[A-Z]{0,4}(?:19|20)[0-9]{6}$/
|
|
900
|
+
|
|
901
|
+
function isPublicId(token: string, named: boolean, cfg: Config): boolean {
|
|
902
|
+
if (PUBLIC_SHAPE.test(token)) return true
|
|
903
|
+
if (cfg.allowPrefix.some((p) => token.startsWith(p))) return true
|
|
904
|
+
if (!PUBLIC_PREFIX.test(token) || NOT_PUBLIC.test(token)) return false
|
|
905
|
+
if (cfg.denyPrefix.some((p) => token.startsWith(p))) return false
|
|
906
|
+
if (named && SOFT_PUBLIC.test(token)) return false
|
|
907
|
+
return PUBLIC_ID_BODY.test(token.replace(PUBLIC_PREFIX, ''))
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
// A letters-only key (no digit at all) splits into many short camel
|
|
911
|
+
// segments; a camel name of the same length splits into a few long ones.
|
|
912
|
+
function lettersLookRandom(token: string, named: boolean, h: number): boolean {
|
|
913
|
+
if (!LETTERS_ONLY.test(token) || !/[a-z]/.test(token) || !/[A-Z]/.test(token)) return false
|
|
914
|
+
const segs = token.match(SEGMENTS) ?? []
|
|
915
|
+
if (named && token.length >= 20 && h >= 3.6 && segs.length * 4 >= token.length) return true
|
|
916
|
+
return token.length >= 24 && h >= 4.0 && segs.length * 3 >= token.length
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
// Claude Code transcripts and most analytics put a `sessionId` UUID on every
|
|
920
|
+
// line. A UUID beside that name is an identifier, not a credential; a real
|
|
921
|
+
// session cookie is random text, and the cookie rules hide it on their own.
|
|
922
|
+
const SESSION_UUID = new RegExp(`^${UUID}$`, 'i')
|
|
923
|
+
const SESSION_ID_NAME = new RegExp(String.raw`sess(?:ion)?[_-]?id(?:${Q}|\s){0,6}[:=]{1,2}(?:${Q}|\s){0,4}$`, 'i')
|
|
924
|
+
|
|
925
|
+
// pi-redact: pi names session files `<ISO time with dashes>Z_<uuid>`. The
|
|
926
|
+
// T and Z make the run mixed case, but it is a file name, not a key.
|
|
927
|
+
const PI_SESSION_FILE = new RegExp(`^(?:19|20)[0-9]{2}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}(?:-[0-9]{3})?Z_${UUID}(?:\\.jsonl)?$`, 'i')
|
|
928
|
+
|
|
929
|
+
function looksSecret(token: string, before: string, cfg: Config): boolean {
|
|
930
|
+
if (token.startsWith('REDACTED-')) return false
|
|
931
|
+
if (PI_SESSION_FILE.test(token)) return false
|
|
932
|
+
if (cfg.allow.has(token) || DIGITS_ONLY.test(token)) return false
|
|
933
|
+
const named = NAMED.test(before)
|
|
934
|
+
if (isPublicId(token, named, cfg)) return false
|
|
935
|
+
// `invoice+statements+acct_<id>`: a mail sender address made of words and
|
|
936
|
+
// a public id. Each `+` part has to be one, so base64 is still judged.
|
|
937
|
+
if (token.includes('+') && token.split('+').every((p) => p === '' || WORD_PART.test(p) || isPublicId(p, named, cfg))) return false
|
|
938
|
+
if (named && SESSION_UUID.test(token) && SESSION_ID_NAME.test(before)) return false
|
|
939
|
+
if (looksLikeName(token, named)) return false
|
|
940
|
+
|
|
941
|
+
const h = entropy(token)
|
|
942
|
+
|
|
943
|
+
// A bare hex run is a git SHA or a checksum far more often than it is a
|
|
944
|
+
// key, so hex needs a name beside it before it is hidden.
|
|
945
|
+
if (HEX_ONLY.test(token)) return named && token.length >= 24 && h >= cfg.contextMinEntropy
|
|
946
|
+
|
|
947
|
+
// A short lowercase prefix and a long random body (imp_ and 40 hex): a
|
|
948
|
+
// vendor key no rule names. A git SHA has no prefix, so it is not caught here.
|
|
949
|
+
const prefixed = token.match(PREFIXED_KEY)
|
|
950
|
+
if (prefixed && /[0-9]/.test(prefixed[1]!) && /[a-z]/.test(prefixed[1]!) && entropy(prefixed[1]!) >= cfg.contextMinEntropy) return true
|
|
951
|
+
|
|
952
|
+
if (isMixed(token) && token.length >= cfg.minLength && h >= cfg.minEntropy) return true
|
|
953
|
+
if (lettersLookRandom(token, named, h)) return true
|
|
954
|
+
// Next to a key's name the bar is lower, but the token still has to look
|
|
955
|
+
// like a key: letters and digits together, not a word.
|
|
956
|
+
const alnum = /[0-9]/.test(token) && /[A-Za-z]/.test(token)
|
|
957
|
+
return named && alnum && token.length >= 16 && h >= cfg.contextMinEntropy
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
// A slash-bearing base64 run, judged on the parts that are not words, so a
|
|
961
|
+
// path like `uploads/2026/10/12/Report2Final` stays a path.
|
|
962
|
+
// pi-redact: the macOS per-user temp dir (`/var/folders/1s/<random>/T/`).
|
|
963
|
+
// Its random segment is a path every tool prints, not a key.
|
|
964
|
+
const MAC_TEMP_DIR = /^\/*(?:private\/)?var\/folders\/[A-Za-z0-9_+-]{2}\/[A-Za-z0-9_+-]{8,64}\/[TCX](?:\/|$)/
|
|
965
|
+
|
|
966
|
+
function slashedSecret(run: string, named: boolean, cfg: Config): boolean {
|
|
967
|
+
if (cfg.allow.has(run)) return false
|
|
968
|
+
if (MAC_TEMP_DIR.test(run)) return false
|
|
969
|
+
const body = run.replace(/=+$/, '').replace(/^\/+/, '')
|
|
970
|
+
if (!body.includes('/') || !isMixed(body) || countsUp(body)) return false
|
|
971
|
+
const parts = body.split('/').filter(Boolean)
|
|
972
|
+
const isWord = (p: string): boolean => SLASH_WORD.test(p) || STRICT_CAMEL.test(p) || DIGITS_ONLY.test(p)
|
|
973
|
+
const words = parts.filter(isWord).length
|
|
974
|
+
if (words * 2 >= parts.length) return false
|
|
975
|
+
if (!isMixed(parts.filter((p) => !isWord(p)).join(''))) return false
|
|
976
|
+
const h = entropy(body)
|
|
977
|
+
return (body.length >= 32 && h >= 4.3) || (named && body.length >= 16 && h >= 3.6)
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
// -------------------------------------------------------------- the passes
|
|
981
|
+
|
|
982
|
+
// Mints `value` unless it is already a placeholder, allowed, or a stand-in.
|
|
983
|
+
function pairOut(cfg: Config, head: string, value: string, tail = ''): string {
|
|
984
|
+
if (WHOLE_TAG.test(value) || cfg.allow.has(value) || isStandIn(value)) return head + value + tail
|
|
985
|
+
return head + mint('SECRET', value) + tail
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
function parkOpaque(text: string): string {
|
|
989
|
+
let out = text
|
|
990
|
+
if (out.includes('"signature"')) out = out.replace(THINKING_SIG, (_m, head: string, sig: string) => head + park(sig))
|
|
991
|
+
if (out.includes('redacted_thinking')) out = out.replace(REDACTED_THINKING, (_m, head: string, data: string) => head + park(data))
|
|
992
|
+
if (out.includes('sha')) out = out.replace(SRI, (m) => park(m))
|
|
993
|
+
return out
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
function scrubKeyBlocks(text: string, cfg: Config): string {
|
|
997
|
+
let out = text
|
|
998
|
+
if (out.includes('PRIVATE KEY')) out = out.replace(PRIVATE_KEY, (m) => (cfg.allow.has(m) ? m : mint('SECRET', m)))
|
|
999
|
+
if (out.includes('Private-Lines:')) out = out.replace(PUTTY_PRIVATE, (_m, head: string, body: string) => head + mint('SECRET', body))
|
|
1000
|
+
return out
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
// Walks the dot-separated parts of a run: a part holding `eyJ` and the two
|
|
1004
|
+
// parts after it are a JWT, when each is long enough.
|
|
1005
|
+
function scrubJwtChain(chain: string, cfg: Config): string {
|
|
1006
|
+
if (!chain.includes('eyJ')) return chain
|
|
1007
|
+
const segs = chain.split('.')
|
|
1008
|
+
const out: string[] = []
|
|
1009
|
+
let k = 0
|
|
1010
|
+
while (k < segs.length) {
|
|
1011
|
+
const i = k + 2 < segs.length ? segs[k]!.indexOf('eyJ') : -1
|
|
1012
|
+
if (i >= 0 && segs[k]!.length - i >= 11 && segs[k + 1]!.length >= 8 && segs[k + 2]!.length >= 8) {
|
|
1013
|
+
const jwt = `${segs[k]!.slice(i)}.${segs[k + 1]}.${segs[k + 2]}`
|
|
1014
|
+
out.push(segs[k]!.slice(0, i) + (cfg.allow.has(jwt) ? jwt : mint('SECRET', jwt)))
|
|
1015
|
+
k += 3
|
|
1016
|
+
} else {
|
|
1017
|
+
out.push(segs[k]!)
|
|
1018
|
+
k += 1
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
return out.join('.')
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
// `postgres://user:PASSWORD@host` in docs: the word itself stands in for one.
|
|
1025
|
+
const URL_PASSWORD_WORD = /^(?:password|passwd|pass|pw|pwd|secret|<[^>]*>)$/i
|
|
1026
|
+
|
|
1027
|
+
function scrubUrls(text: string, cfg: Config): string {
|
|
1028
|
+
let out = text
|
|
1029
|
+
if (out.includes('://')) {
|
|
1030
|
+
out = out.replace(URL_PASSWORD, (m, head: string, password: string) =>
|
|
1031
|
+
WHOLE_TAG.test(password) || cfg.allow.has(password) || URL_PASSWORD_WORD.test(password) ? m : `${head}:${mint('SECRET', password)}@`,
|
|
1032
|
+
)
|
|
1033
|
+
out = out.replace(URL_USERINFO, (m, head: string, user: string) => {
|
|
1034
|
+
if (cfg.allow.has(user) || !/[0-9]/.test(user) || !/[A-Za-z]/.test(user) || looksLikeName(user, true)) return m
|
|
1035
|
+
return entropy(user) >= cfg.contextMinEntropy ? `${head}${mint('SECRET', user)}@` : m
|
|
1036
|
+
})
|
|
1037
|
+
}
|
|
1038
|
+
if (out.includes('=')) {
|
|
1039
|
+
out = out.replace(URL_PARAM, (m, head: string, value: string) => (urlParamSecret(value, cfg) ? head + mint('SECRET', value) : m))
|
|
1040
|
+
}
|
|
1041
|
+
return out
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
function urlParamSecret(raw: string, cfg: Config): boolean {
|
|
1045
|
+
if (cfg.allow.has(raw)) return false
|
|
1046
|
+
let v = raw
|
|
1047
|
+
try {
|
|
1048
|
+
v = decodeURIComponent(raw)
|
|
1049
|
+
} catch {
|
|
1050
|
+
// A broken escape: judge it as it stands.
|
|
1051
|
+
}
|
|
1052
|
+
if (cfg.allow.has(v) || v.length < 12) return false
|
|
1053
|
+
if ([/[a-z]/, /[A-Z]/, /[0-9]/, /[^A-Za-z0-9]/].filter((r) => r.test(v)).length < 2) return false
|
|
1054
|
+
if (looksLikeName(v, true)) return false
|
|
1055
|
+
return entropy(v) >= cfg.contextMinEntropy
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
function scrubCommandLines(text: string, cfg: Config): string {
|
|
1059
|
+
let out = text
|
|
1060
|
+
if (/-(?:pass|api|token|client|auth|access)/.test(out)) {
|
|
1061
|
+
out = out.replace(CLI_FLAG, (m, head: string, quote: string, v: string) => {
|
|
1062
|
+
if (/^[A-Z][A-Z0-9_]*$/.test(v) || /^[A-Za-z_]+=/.test(v) || /^[A-Za-z]+(?:[-_][A-Za-z]+)+$/.test(v)) return m
|
|
1063
|
+
return pairOut(cfg, head, v, quote)
|
|
1064
|
+
})
|
|
1065
|
+
}
|
|
1066
|
+
if (/\s-(?:u|U|-user|-proxy-user)\b/.test(out)) {
|
|
1067
|
+
out = out.replace(CLI_USERPASS, (m, head: string, v: string) => (DIGITS_ONLY.test(v) ? m : pairOut(cfg, head, v)))
|
|
1068
|
+
}
|
|
1069
|
+
if (/mysql|mariadb/.test(out)) out = out.replace(MYSQL_P, (_m, head: string, v: string) => pairOut(cfg, head, v))
|
|
1070
|
+
if (out.includes(':')) out = out.replace(PGPASS, (_m, head: string, v: string) => pairOut(cfg, head, v))
|
|
1071
|
+
if (out.includes('password')) out = out.replace(NETRC, (_m, head: string, v: string) => pairOut(cfg, head, v))
|
|
1072
|
+
if (out.includes('gh')) out = out.replace(GH_SECRET, (_m, head: string, _q: string, v: string) => pairOut(cfg, head, v))
|
|
1073
|
+
if (CMD_PW_GATE.test(out)) {
|
|
1074
|
+
for (const re of CMD_PW) out = out.replace(re, (_m, head: string, q: string, v: string) => pairOut(cfg, head, v, q))
|
|
1075
|
+
}
|
|
1076
|
+
if (out.includes('pass:')) out = out.replace(OPENSSL_PASS, (_m, head: string, v: string) => pairOut(cfg, head, v))
|
|
1077
|
+
if (out.includes('smbclient')) out = out.replace(SMB_USER, (_m, head: string, v: string) => pairOut(cfg, head, v))
|
|
1078
|
+
if (out.includes('htpasswd')) out = out.replace(HTPASSWD, (_m, head: string, q: string, v: string) => pairOut(cfg, head, v, q))
|
|
1079
|
+
if (/net[ \t]+user/i.test(out)) out = out.replace(NET_USER, (_m, head: string, q: string, v: string) => pairOut(cfg, head, v, q))
|
|
1080
|
+
if (out.includes('rabbitmqctl')) out = out.replace(RABBIT_USER, (_m, head: string, q: string, v: string) => pairOut(cfg, head, v, q))
|
|
1081
|
+
return out
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
function scrubAssignments(text: string, cfg: Config): string {
|
|
1085
|
+
let out = text
|
|
1086
|
+
// A value under a key that says it is public is left: a publishable key.
|
|
1087
|
+
const value = (head: string, key: string, v: string, tail = ''): string => {
|
|
1088
|
+
if (/public|publishable/i.test(key)) return head + v + tail
|
|
1089
|
+
if (PUBLIC_PREFIX.test(v) && !/secret|private/i.test(key) && isPublicId(v, false, cfg)) return head + v + tail
|
|
1090
|
+
if (shortKeyStatus(key, v)) return head + v + tail
|
|
1091
|
+
return pairOut(cfg, head, v, tail)
|
|
1092
|
+
}
|
|
1093
|
+
// A value whose name sits in a sibling field.
|
|
1094
|
+
const sibling = (head: string, name: string, v: string, tail = ''): string => {
|
|
1095
|
+
if (PW_NAME.test(name)) return value(head, name, v, tail)
|
|
1096
|
+
if (KEY_NAME.test(name) && !/public|publishable/i.test(name) && looksSecret(v, 'key=', cfg)) return head + mint('SECRET', v) + tail
|
|
1097
|
+
return head + v + tail
|
|
1098
|
+
}
|
|
1099
|
+
if (/pass|secret|key|token|pw/i.test(out)) {
|
|
1100
|
+
out = out.replace(STRONG_QUOTED, (_m, head: string, _q: string, key: string, quote: string, v: string) => value(head + quote, key, v, quote))
|
|
1101
|
+
out = out.replace(APP_PASSWORD, (_m, head: string, v: string) => pairOut(cfg, head, v))
|
|
1102
|
+
out = out.replace(STRONG_LINE, (_m, head: string, key: string, v: string) => value(head, key, v))
|
|
1103
|
+
out = out.replace(ENV_PASS, (_m, head: string, _k: string, v: string) => value(head, '', v))
|
|
1104
|
+
out = out.replace(BLOCK_PASSWORD, (_m, head: string, v: string) => pairOut(cfg, head, v))
|
|
1105
|
+
out = out.replace(KV_PASSWORD, (_m, head: string, v: string) => pairOut(cfg, head, v))
|
|
1106
|
+
}
|
|
1107
|
+
if (PW_SYMBOL.test(out)) {
|
|
1108
|
+
out = out.replace(QUOTED_VALUE, (m, q: string, v: string, offset: number, whole: string) => {
|
|
1109
|
+
if (!looksGenerated(v) || isStandIn(v) || cfg.allow.has(v)) return m
|
|
1110
|
+
// A MIME boundary is random by design and public.
|
|
1111
|
+
if (/boundary=$/i.test(whole.slice(Math.max(0, offset - 9), offset))) return m
|
|
1112
|
+
return q + mint('SECRET', v) + q
|
|
1113
|
+
})
|
|
1114
|
+
}
|
|
1115
|
+
if (/password|identified/i.test(out)) out = out.replace(SQL_PASSWORD, (_m, head: string, v: string) => pairOut(cfg, head + "'", v, "'"))
|
|
1116
|
+
if (out.includes('variable')) out = out.replace(HCL_DEFAULT, (_m, head: string, v: string) => pairOut(cfg, head, v, '"'))
|
|
1117
|
+
if (out.includes('"value"')) {
|
|
1118
|
+
out = out.replace(JSON_NAME_VALUE, (_m, head: string, name: string, v: string) => sibling(head, name, v, '"'))
|
|
1119
|
+
out = out.replace(JSON_VALUE_NAME, (_m, head: string, v: string, tail: string, name: string) => sibling(head, name, v, tail))
|
|
1120
|
+
out = out.replace(TF_OUTPUT, (_m, head: string, name: string, v: string) => (PW_NAME.test(name) ? value(head, name, v, '"') : _m))
|
|
1121
|
+
}
|
|
1122
|
+
if (/"random_(?:password|string)"/.test(out)) {
|
|
1123
|
+
out = out.replace(TF_RESULT, (m, head: string, v: string) => (WHOLE_TAG.test(v) || cfg.allow.has(v) ? m : `${head}${mint('SECRET', v)}"`))
|
|
1124
|
+
}
|
|
1125
|
+
if (/-[ \t]+name:/.test(out)) {
|
|
1126
|
+
out = out.replace(YAML_NAME_VALUE, (_m, indent: string, head: string, name: string, q: string, v: string) => sibling(indent + head, name, v, q))
|
|
1127
|
+
}
|
|
1128
|
+
if (/INSERT[ \t]+INTO/i.test(out)) out = scrubSqlInserts(out, cfg)
|
|
1129
|
+
return out
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
// The VALUES rows after an INSERT whose columns name a key or a password:
|
|
1133
|
+
// the value in that column is hidden. Bounded to 100 rows and 20,000
|
|
1134
|
+
// characters per INSERT. Quoted values only; a number is never a key.
|
|
1135
|
+
function scrubSqlInserts(text: string, cfg: Config): string {
|
|
1136
|
+
const ranges: [number, number][] = []
|
|
1137
|
+
for (const m of text.matchAll(SQL_INSERT)) {
|
|
1138
|
+
const names = m[1]!.split(',').map((c) => c.trim().replace(/^[`"[]|[`"\]]$/g, ''))
|
|
1139
|
+
const kinds = names.map((n) => (PW_NAME.test(n) ? 'pw' : KEY_NAME.test(n) && !/(?:^|_)id$|public/i.test(n) ? 'key' : ''))
|
|
1140
|
+
if (!kinds.some(Boolean)) continue
|
|
1141
|
+
const limit = Math.min(text.length, m.index! + m[0].length + 20_000)
|
|
1142
|
+
let i = m.index! + m[0].length
|
|
1143
|
+
const skipSpace = (): void => {
|
|
1144
|
+
while (i < limit && /\s/.test(text[i]!)) i++
|
|
1145
|
+
}
|
|
1146
|
+
for (let row = 0; row < 100 && text[i] === '('; row++) {
|
|
1147
|
+
i++
|
|
1148
|
+
for (let col = 0; i < limit && text[i] !== ')'; col++) {
|
|
1149
|
+
skipSpace()
|
|
1150
|
+
if (text[i] === "'") {
|
|
1151
|
+
const from = ++i
|
|
1152
|
+
while (i < limit && !(text[i] === "'" && text[i + 1] !== "'")) i += text[i] === "'" ? 2 : 1
|
|
1153
|
+
const v = text.slice(from, i)
|
|
1154
|
+
i++
|
|
1155
|
+
const kind = kinds[col]
|
|
1156
|
+
if (kind && v.length >= 3 && !cfg.allow.has(v) && (kind === 'pw' ? !isStandIn(v) : looksSecret(v, 'key=', cfg))) ranges.push([from, from + v.length])
|
|
1157
|
+
} else {
|
|
1158
|
+
while (i < limit && text[i] !== ',' && text[i] !== ')') i++
|
|
1159
|
+
}
|
|
1160
|
+
skipSpace()
|
|
1161
|
+
if (text[i] === ',') i++
|
|
1162
|
+
else break
|
|
1163
|
+
}
|
|
1164
|
+
if (text[i] !== ')') break
|
|
1165
|
+
i++
|
|
1166
|
+
skipSpace()
|
|
1167
|
+
if (text[i] !== ',') break
|
|
1168
|
+
i++
|
|
1169
|
+
skipSpace()
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
if (ranges.length === 0) return text
|
|
1173
|
+
const parts: string[] = []
|
|
1174
|
+
let last = 0
|
|
1175
|
+
for (const [from, to] of ranges) {
|
|
1176
|
+
if (from < last) continue
|
|
1177
|
+
parts.push(text.slice(last, from), mint('SECRET', text.slice(from, to)))
|
|
1178
|
+
last = to
|
|
1179
|
+
}
|
|
1180
|
+
parts.push(text.slice(last))
|
|
1181
|
+
return parts.join('')
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
function cookieSecret(raw: string, cfg: Config): boolean {
|
|
1185
|
+
if (cfg.allow.has(raw) || raw.includes('[REDACTED-')) return false
|
|
1186
|
+
const m = raw.match(COOKIE_VALUE)
|
|
1187
|
+
if (!m) return false
|
|
1188
|
+
const body = m[1]!
|
|
1189
|
+
if (looksLikeName(body, true)) return false
|
|
1190
|
+
return /[0-9]/.test(body) && /[A-Za-z]/.test(body) && entropy(body) >= cfg.contextMinEntropy
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
function scrubCookies(text: string, cfg: Config): string {
|
|
1194
|
+
let out = text
|
|
1195
|
+
if (/cookie|-b\s/i.test(out)) {
|
|
1196
|
+
out = out.replace(COOKIE, (_m, head: string, header: string) =>
|
|
1197
|
+
head +
|
|
1198
|
+
header.replace(COOKIE_PAIR, (pair, sep: string, name: string, v: string) =>
|
|
1199
|
+
!COOKIE_ATTR.test(name) && cookieSecret(v, cfg) ? sep + name + mint('SECRET', v) : pair,
|
|
1200
|
+
),
|
|
1201
|
+
)
|
|
1202
|
+
}
|
|
1203
|
+
if (out.includes('\t')) out = out.replace(COOKIES_TXT, (m, head: string, v: string) => (cookieSecret(v, cfg) ? head + mint('SECRET', v) : m))
|
|
1204
|
+
if (out.includes('"value"')) out = out.replace(HAR_COOKIE, (m, head: string, v: string) => (cookieSecret(v, cfg) ? `${head}${mint('SECRET', v)}"` : m))
|
|
1205
|
+
return out
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
function scrubAuthBlobs(text: string, cfg: Config): string {
|
|
1209
|
+
let out = text
|
|
1210
|
+
if (/asic|"auth"|'auth'/.test(out)) {
|
|
1211
|
+
out = out.replace(BASIC_AUTH, (m, head: string, v: string) => {
|
|
1212
|
+
if (v.length % 4 !== 0 || cfg.allow.has(v)) return m
|
|
1213
|
+
return /^[\x21-\x7e]+:[\x20-\x7e]+$/.test(decodeBase64Head(v)) ? head + mint('SECRET', v) : m
|
|
1214
|
+
})
|
|
1215
|
+
}
|
|
1216
|
+
if (out.includes('.')) {
|
|
1217
|
+
out = out.replace(BOOTSTRAP, (m, id: string, secret: string, offset: number, whole: string) => {
|
|
1218
|
+
if (!/[0-9]/.test(secret) || !/[a-z]/.test(secret) || countsUp(secret)) return m
|
|
1219
|
+
return NAMED.test(whole.slice(Math.max(0, offset - 64), offset)) ? `${id}.${mint('SECRET', secret)}` : m
|
|
1220
|
+
})
|
|
1221
|
+
}
|
|
1222
|
+
return out
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
// The decoded start of a base64 text is text, not media: almost no control
|
|
1226
|
+
// bytes. An image or random bytes fail this at once, and a PDF or a zip
|
|
1227
|
+
// (docx, xlsx) is known by its first bytes, so the full decode below only
|
|
1228
|
+
// runs on encoded text.
|
|
1229
|
+
function decodesToText(b64: string): boolean {
|
|
1230
|
+
const head = decodeBase64Head(b64, 512)
|
|
1231
|
+
if (head.length < 8 || head.startsWith('%PDF') || head.startsWith('PK\x03\x04')) return false
|
|
1232
|
+
let control = 0
|
|
1233
|
+
for (let i = 0; i < head.length; i++) {
|
|
1234
|
+
const c = head.charCodeAt(i)
|
|
1235
|
+
if ((c < 32 && c !== 9 && c !== 10 && c !== 13) || c === 127) control++
|
|
1236
|
+
}
|
|
1237
|
+
return control * 50 <= head.length
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
const firstHit = (re: RegExp, text: string): RegExpExecArray | null => {
|
|
1241
|
+
re.lastIndex = 0
|
|
1242
|
+
const m = re.exec(text)
|
|
1243
|
+
re.lastIndex = 0
|
|
1244
|
+
return m
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
// Decoded text that holds a secret by the hard rules: a vendor shape, a
|
|
1248
|
+
// private key, a credentials file, a password-like key with a real value,
|
|
1249
|
+
// or a URL password. These need no entropy, so they stay quiet on mail.
|
|
1250
|
+
function decodedHoldsSecret(d: string): boolean {
|
|
1251
|
+
if (BLOB_INNER.test(d) || d.includes('PRIVATE KEY')) return true
|
|
1252
|
+
for (const { hint, re } of VENDOR) {
|
|
1253
|
+
if (typeof hint === 'string' ? !d.includes(hint) : !hint.test(d)) continue
|
|
1254
|
+
if (firstHit(re, d)) return true
|
|
1255
|
+
}
|
|
1256
|
+
if (/pass|secret|key|token/i.test(d)) {
|
|
1257
|
+
for (const m of d.matchAll(STRONG_LINE)) if (!isStandIn(m[3]!) && !WHOLE_TAG.test(m[3]!) && !shortKeyStatus(m[2]!, m[3]!)) return true
|
|
1258
|
+
for (const m of d.matchAll(STRONG_QUOTED)) if (!isStandIn(m[5]!) && !WHOLE_TAG.test(m[5]!) && !shortKeyStatus(m[3]!, m[5]!)) return true
|
|
1259
|
+
}
|
|
1260
|
+
if (d.includes('://')) for (const m of d.matchAll(URL_PASSWORD)) if (!URL_PASSWORD_WORD.test(m[2]!)) return true
|
|
1261
|
+
return false
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
// A base64 text that decodes to text holding a secret. Up to 1 MB of it is
|
|
1265
|
+
// decoded; the model could decode it as easily.
|
|
1266
|
+
function base64HoldsSecret(b64: string): boolean {
|
|
1267
|
+
if (!decodesToText(b64)) return false
|
|
1268
|
+
return decodedHoldsSecret(decodeBase64Head(b64, 1 << 20))
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
// Data URIs are parked whole. A blob is hidden as ONE placeholder when a key
|
|
1272
|
+
// name stands in front of it, or it decodes to text that holds a key or a
|
|
1273
|
+
// credentials file; otherwise it is parked, never shredded into thousands.
|
|
1274
|
+
function parkMedia(text: string, cfg: Config): string {
|
|
1275
|
+
let out = text
|
|
1276
|
+
if (out.includes(';base64,')) out = out.replace(DATA_URI, (m) => park(m))
|
|
1277
|
+
if (/Content-Transfer-Encoding/i.test(out)) out = mintMimeSecrets(out, cfg)
|
|
1278
|
+
out = out.replace(BLOB, (m, offset: number, whole: string) => {
|
|
1279
|
+
if (cfg.allow.has(m)) return m
|
|
1280
|
+
if (m.includes('\n') && !evenlyWrapped(m)) return m
|
|
1281
|
+
// The text before it on the same line, at most 120 characters.
|
|
1282
|
+
const near = whole.slice(Math.max(0, offset - 120), offset)
|
|
1283
|
+
const before = near.slice(near.lastIndexOf('\n') + 1)
|
|
1284
|
+
if (BLOB_NAME.test(before) || BLOB_INNER.test(decodeBase64Head(m)) || base64HoldsSecret(m)) return mint('SECRET', m)
|
|
1285
|
+
return park(m)
|
|
1286
|
+
})
|
|
1287
|
+
return out
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
// A base64 MIME body of any type (a text part, an attachment) that decodes
|
|
1291
|
+
// to a secret is hidden whole. The width test of a blob does not apply, so
|
|
1292
|
+
// a narrow attachment is caught too.
|
|
1293
|
+
function mintMimeSecrets(text: string, cfg: Config): string {
|
|
1294
|
+
const ranges: [number, number][] = []
|
|
1295
|
+
for (const body of mimeBodies(text)) {
|
|
1296
|
+
const b64 = text.slice(body.start, body.end).split(/\r?\n/).map(stripPfx).join('')
|
|
1297
|
+
if (base64HoldsSecret(b64)) ranges.push([body.start, body.end])
|
|
1298
|
+
}
|
|
1299
|
+
if (ranges.length === 0) return text
|
|
1300
|
+
const parts: string[] = []
|
|
1301
|
+
let last = 0
|
|
1302
|
+
for (const [from, to] of ranges) {
|
|
1303
|
+
const raw = text.slice(from, to)
|
|
1304
|
+
parts.push(text.slice(last, from), cfg.allow.has(raw) ? raw : mint('SECRET', raw))
|
|
1305
|
+
last = to
|
|
1306
|
+
}
|
|
1307
|
+
parts.push(text.slice(last))
|
|
1308
|
+
return parts.join('')
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
// Wrapped base64 (PEM, MIME, `base64` output) has one line width, 60 or
|
|
1312
|
+
// more, on every line but the last. A list of keys one per line is not
|
|
1313
|
+
// excused as a blob unless it looks exactly like that.
|
|
1314
|
+
function evenlyWrapped(blob: string): boolean {
|
|
1315
|
+
const lines = blob.split(/\r?\n/).filter((l) => l.length > 0)
|
|
1316
|
+
if (lines.length < 3) return false
|
|
1317
|
+
const width = lines[0]!.length
|
|
1318
|
+
if (width < 60) return false
|
|
1319
|
+
for (let i = 0; i < lines.length - 1; i++) if (lines[i]!.length !== width) return false
|
|
1320
|
+
return lines[lines.length - 1]!.length <= width
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
function lineAt(text: string, start: number): [string, number] {
|
|
1324
|
+
const nl = text.indexOf('\n', start)
|
|
1325
|
+
const end = nl < 0 ? text.length : nl
|
|
1326
|
+
return [text.slice(start, end).replace(/\r$/, ''), end]
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
const stripPfx = (line: string): string => line.replace(PFX_RE, '')
|
|
1330
|
+
|
|
1331
|
+
type MimeBody = { start: number; end: number; lines: [number, number][]; text: boolean }
|
|
1332
|
+
|
|
1333
|
+
// The base64 body of every MIME part, found from its
|
|
1334
|
+
// Content-Transfer-Encoding header, with or without a line prefix. Each line
|
|
1335
|
+
// is a [start, end] range; a last short line counts only when a blank line
|
|
1336
|
+
// or a boundary follows it.
|
|
1337
|
+
function mimeBodies(text: string): MimeBody[] {
|
|
1338
|
+
const found: MimeBody[] = []
|
|
1339
|
+
if (!/Content-Transfer-Encoding/i.test(text)) return found
|
|
1340
|
+
for (const m of text.matchAll(CTE)) {
|
|
1341
|
+
const at = m.index!
|
|
1342
|
+
// The header block ends at the next blank line; a part's headers are
|
|
1343
|
+
// short, so a block that runs on for 20 lines is not a MIME part.
|
|
1344
|
+
let pos = text.indexOf('\n', at) + 1
|
|
1345
|
+
let bodyStart = -1
|
|
1346
|
+
for (let n = 0; pos > 0 && pos < text.length && n < 20; n++) {
|
|
1347
|
+
const [line, end] = lineAt(text, pos)
|
|
1348
|
+
if (stripPfx(line) === '') {
|
|
1349
|
+
bodyStart = end + 1
|
|
1350
|
+
break
|
|
1351
|
+
}
|
|
1352
|
+
pos = end + 1
|
|
1353
|
+
}
|
|
1354
|
+
if (bodyStart < 0) continue
|
|
1355
|
+
// It starts after a blank line or a `--boundary` line.
|
|
1356
|
+
let start = text.lastIndexOf('\n', at - 1) + 1
|
|
1357
|
+
for (let n = 0; start > 0 && n < 20; n++) {
|
|
1358
|
+
const prev = text.lastIndexOf('\n', start - 2) + 1
|
|
1359
|
+
const line = stripPfx(text.slice(prev, start - 1).replace(/\r$/, ''))
|
|
1360
|
+
if (line === '' || line.startsWith('--')) break
|
|
1361
|
+
start = prev
|
|
1362
|
+
}
|
|
1363
|
+
const type = text.slice(start, bodyStart).match(CONTENT_TYPE)
|
|
1364
|
+
const isText = !type || /^(?:text|message|multipart)\//i.test(type[1]!)
|
|
1365
|
+
const lines: [number, number][] = []
|
|
1366
|
+
let width = 0
|
|
1367
|
+
for (let p = bodyStart; p < text.length; ) {
|
|
1368
|
+
const [line, end] = lineAt(text, p)
|
|
1369
|
+
const body = stripPfx(line)
|
|
1370
|
+
if (!/^[A-Za-z0-9+/]+={0,2}$/.test(body)) break
|
|
1371
|
+
if (width === 0) {
|
|
1372
|
+
width = body.length
|
|
1373
|
+
if (width < 40) break
|
|
1374
|
+
}
|
|
1375
|
+
if (body.length === width && !body.endsWith('=')) {
|
|
1376
|
+
lines.push([p, end])
|
|
1377
|
+
p = end + 1
|
|
1378
|
+
continue
|
|
1379
|
+
}
|
|
1380
|
+
const next = end < text.length ? stripPfx(lineAt(text, end + 1)[0]) : ''
|
|
1381
|
+
if (next === '' || next.startsWith('--')) lines.push([p, end])
|
|
1382
|
+
break
|
|
1383
|
+
}
|
|
1384
|
+
if (lines.length > 0) found.push({ start: lines[0]![0], end: lines[lines.length - 1]![1], lines, text: isText })
|
|
1385
|
+
}
|
|
1386
|
+
return found
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
// The body lines of a non-text base64 MIME part (an attachment) are skipped
|
|
1390
|
+
// by the token scan. A body that held a secret was already hidden whole.
|
|
1391
|
+
function mimeBodySpans(text: string, spans: Spans): void {
|
|
1392
|
+
for (const body of mimeBodies(text)) if (!body.text) for (const [from, to] of body.lines) spans.add(from, to)
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
// Mail transport headers (DKIM, ARC, Message-ID, Received ...) inside a real
|
|
1396
|
+
// header block, encoded-word subjects, and multipart boundaries. These are
|
|
1397
|
+
// soft: a token after `token=` inside one is still judged.
|
|
1398
|
+
function mailSpans(text: string, spans: Spans, boundaries: Set<string>): void {
|
|
1399
|
+
if (/(?:Signature|Seal|Message-I[Dd]|References|Received|Feedback-ID|X-SG-|X-MSFBL|X-Gm-|X-Google-)/.test(text)) {
|
|
1400
|
+
for (const m of text.matchAll(MAIL_HEADER)) {
|
|
1401
|
+
const at = m.index!
|
|
1402
|
+
const end = at + m[0].length
|
|
1403
|
+
const prevStart = text.lastIndexOf('\n', at - 2) + 1
|
|
1404
|
+
const prev = at > 0 ? text.slice(prevStart, at - 1).replace(/\r$/, '') : ''
|
|
1405
|
+
const next = end < text.length ? lineAt(text, end + 1)[0] : ''
|
|
1406
|
+
const inBlock = HEADER_LINE.test(prev) || /^[ \t]/.test(stripPfx(prev)) || HEADER_LINE.test(next)
|
|
1407
|
+
if (inBlock) spans.add(at, end, true)
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
if (text.includes('=?')) for (const m of text.matchAll(ENCODED_WORD)) spans.add(m.index!, m.index! + m[0].length, true)
|
|
1411
|
+
if (/multipart\//i.test(text)) for (const m of text.matchAll(BOUNDARY)) boundaries.add(m[1]!.replace(/^-+|[-=]+$/g, ''))
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
function protectedSpans(text: string, boundaries: Set<string>): Spans {
|
|
1415
|
+
const spans = new Spans()
|
|
1416
|
+
mimeBodySpans(text, spans)
|
|
1417
|
+
mailSpans(text, spans, boundaries)
|
|
1418
|
+
if (text.includes('fonts.gstatic.com')) for (const m of text.matchAll(PUBLIC_URL)) spans.add(m.index!, m.index! + m[0].length)
|
|
1419
|
+
// The Cloudflare Web Analytics beacon token is public, on every page.
|
|
1420
|
+
if (text.includes('data-cf-beacon')) for (const m of text.matchAll(CF_BEACON)) spans.add(m.index!, m.index! + m[0].length)
|
|
1421
|
+
return spans.seal()
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
const CF_BEACON = /data-cf-beacon=(?:3D)?'[^'<>]{0,800}'/g
|
|
1425
|
+
// A content-hashed asset name: words, then a hex hash, then a file type.
|
|
1426
|
+
const ASSET_EXT = /^\.(?:png|jpe?g|gif|webp|avif|svg|ico|mp3|mp4|m4a|wav|webm|mov|js|mjs|cjs|css|map|woff2?|ttf|otf)(?![A-Za-z0-9])/i
|
|
1427
|
+
const HASHED_ASSET = /^[a-z][a-z0-9]*(?:[-_][a-z0-9]+)*[-_][0-9a-f]{8,64}$/
|
|
1428
|
+
|
|
1429
|
+
// A hashed class name in a <style> block: `.rio-card-header-<hash> {`. It
|
|
1430
|
+
// has to be kebab case with two lowercase words before the hash, so a key
|
|
1431
|
+
// built from words and `_` (`acme_live_<random>`) is still judged there.
|
|
1432
|
+
const CSS_HASHED = /^[a-z]{2,}(?:-[a-z]{2,})+-[A-Za-z0-9]{6,}$/
|
|
1433
|
+
|
|
1434
|
+
function cssClass(token: string, offset: number, whole: string, after: string): boolean {
|
|
1435
|
+
if (!CSS_HASHED.test(token) || !/^[ \t]*(?:\{|,|\.[A-Za-z])/.test(after)) return false
|
|
1436
|
+
if (!/[{},\n]\.$/.test(whole.slice(Math.max(0, offset - 64), offset))) return false
|
|
1437
|
+
return whole.lastIndexOf('<style', offset) > whole.lastIndexOf('</style', offset)
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
function scrubTokens(text: string, cfg: Config): string {
|
|
1441
|
+
const boundaries = new Set<string>()
|
|
1442
|
+
const spans = protectedSpans(text, boundaries)
|
|
1443
|
+
const beforeAt = (whole: string, offset: number): string => whole.slice(Math.max(0, offset - 64), offset)
|
|
1444
|
+
const isBoundary = (t: string): boolean => {
|
|
1445
|
+
const core = t.replace(/^-+|[-=]+$/g, '')
|
|
1446
|
+
for (const b of boundaries) if (core.length >= 8 && b.includes(core)) return true
|
|
1447
|
+
return false
|
|
1448
|
+
}
|
|
1449
|
+
const judge = (token: string, offset: number, whole: string): string => {
|
|
1450
|
+
const span = spans.at(offset)
|
|
1451
|
+
if (span === 'hard') return token
|
|
1452
|
+
const after = whole.slice(offset + token.length, offset + token.length + 8)
|
|
1453
|
+
if (ASSET_EXT.test(after) && HASHED_ASSET.test(token) && !NAMED.test(beforeAt(whole, offset))) return token
|
|
1454
|
+
if (cssClass(token, offset, whole, after)) return token
|
|
1455
|
+
if (span === 'soft' || (boundaries.size > 0 && isBoundary(token))) {
|
|
1456
|
+
if (!NAMED.test(beforeAt(whole, offset))) return token
|
|
1457
|
+
}
|
|
1458
|
+
return looksSecret(token, beforeAt(whole, offset), cfg) ? mint('SECRET', token) : token
|
|
1459
|
+
}
|
|
1460
|
+
return text.replace(SCAN, (m: string, slashed: string | undefined, offset: number, whole: string) => {
|
|
1461
|
+
if (slashed === undefined) return judge(m, offset, whole)
|
|
1462
|
+
if (m.includes('/')) {
|
|
1463
|
+
const span = spans.at(offset)
|
|
1464
|
+
const named = NAMED.test(beforeAt(whole, offset))
|
|
1465
|
+
if (span !== 'hard' && (span !== 'soft' || named) && slashedSecret(m, named, cfg)) return mint('SECRET', m)
|
|
1466
|
+
}
|
|
1467
|
+
return m.replace(CANDIDATE, (t: string, o: number) => judge(t, offset + o, whole))
|
|
1468
|
+
})
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
export function scrubSecrets(text: string, cfg: Config): string {
|
|
1472
|
+
if (text.length < 8) return text
|
|
1473
|
+
// A whole value that is base64 (or base64url) of text holding a secret.
|
|
1474
|
+
if (text.length >= 256 && BASE64_VALUE.test(text) && !cfg.allow.has(text) && base64HoldsSecret(text)) return mint('SECRET', text)
|
|
1475
|
+
const outer = parked
|
|
1476
|
+
const outerLocal = local
|
|
1477
|
+
parked = startParking(text)
|
|
1478
|
+
local = new Set()
|
|
1479
|
+
try {
|
|
1480
|
+
let out = rehideKnown(text)
|
|
1481
|
+
out = parkOpaque(out)
|
|
1482
|
+
out = scrubKeyBlocks(out, cfg)
|
|
1483
|
+
if (out.includes('eyJ')) out = out.replace(DOTTED, (m) => scrubJwtChain(m, cfg))
|
|
1484
|
+
out = scrubVendors(out, cfg)
|
|
1485
|
+
out = scrubUrls(out, cfg)
|
|
1486
|
+
out = scrubCommandLines(out, cfg)
|
|
1487
|
+
out = scrubAssignments(out, cfg)
|
|
1488
|
+
out = scrubCookies(out, cfg)
|
|
1489
|
+
out = scrubAuthBlobs(out, cfg)
|
|
1490
|
+
out = parkMedia(out, cfg)
|
|
1491
|
+
out = scrubTokens(out, cfg)
|
|
1492
|
+
// A value hidden at one spot in this text is hidden at its other spots
|
|
1493
|
+
// too: a HAR repeats each query parameter in `url` and `queryString`.
|
|
1494
|
+
if (local.size > 0) out = rehideLocal(out, local)
|
|
1495
|
+
return unpark(out)
|
|
1496
|
+
} finally {
|
|
1497
|
+
parked = outer
|
|
1498
|
+
local = outerLocal
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
// --------------------------------------------------------------- PII tests
|
|
1503
|
+
|
|
1504
|
+
// The local part may not start in the middle of a run: without that the
|
|
1505
|
+
// scan restarts at every character of a long run and goes quadratic.
|
|
1506
|
+
const EMAIL = /(?<![A-Za-z0-9._%+-])[A-Za-z0-9._%+-]+@[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,24}/g
|
|
1507
|
+
|
|
1508
|
+
// Addresses that only ever stand in for a real one: form placeholders, docs
|
|
1509
|
+
// and test fixtures. Hiding them is noise.
|
|
1510
|
+
const EXAMPLE_DOMAIN =
|
|
1511
|
+
/@(?:example\.(?:com|org|net)|examples?\.[a-z]+|test|invalid|localhost|acme\.com|(?:your)?domain\.com|company\.com|email\.com|mail\.com|foo\.com|bar\.com|sample\.com|placeholder\.[a-z]+)$/i
|
|
1512
|
+
const EXAMPLE_LOCAL =
|
|
1513
|
+
/^(?:you|your|user|username|name|email|someone|test|example|placeholder|first\.last|jane|john|alice|bob|teammate|colleague|member|admin)(?:[.+_-]?[a-z0-9]{0,12})?@/i
|
|
1514
|
+
|
|
1515
|
+
function allowedEmail(address: string, cfg: Config): boolean {
|
|
1516
|
+
const low = address.toLowerCase()
|
|
1517
|
+
if (low.endsWith('@users.noreply.github.com')) return true
|
|
1518
|
+
if (low.startsWith('noreply@') || low.startsWith('no-reply@')) return true
|
|
1519
|
+
if (EXAMPLE_DOMAIN.test(low) || EXAMPLE_LOCAL.test(low)) return true
|
|
1520
|
+
return cfg.allowEmails.some((a) => {
|
|
1521
|
+
const rule = a.toLowerCase().trim()
|
|
1522
|
+
return rule.startsWith('@') ? low.endsWith(rule) : low === rule
|
|
1523
|
+
})
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
const OCTET = '(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])'
|
|
1527
|
+
const IPV4 = new RegExp('(?<![0-9.])(?:' + OCTET + '\\.){3}' + OCTET + '(?![0-9.])', 'g')
|
|
1528
|
+
const IPV6 = /(?<![0-9A-Za-z:])(?:[0-9A-Fa-f]{1,4}:){4,7}[0-9A-Fa-f]{1,4}(?![0-9A-Za-z:])/g
|
|
1529
|
+
|
|
1530
|
+
function isPrivateV4(ip: string): boolean {
|
|
1531
|
+
const parts = ip.split('.').map(Number)
|
|
1532
|
+
if (parts[0] === 10 || parts[0] === 127 || parts[0] === 0) return true
|
|
1533
|
+
if (parts[0] === 172 && parts[1]! >= 16 && parts[1]! <= 31) return true
|
|
1534
|
+
if (parts[0] === 192 && parts[1] === 168) return true
|
|
1535
|
+
if (parts[0] === 169 && parts[1] === 254) return true
|
|
1536
|
+
if (parts[0] === 100 && parts[1]! >= 64 && parts[1]! <= 127) return true
|
|
1537
|
+
if (parts[0]! >= 224) return true
|
|
1538
|
+
if (parts[0] === 198 && (parts[1] === 18 || parts[1] === 19)) return true
|
|
1539
|
+
if (parts[0] === 192 && parts[1] === 0 && parts[2] === 2) return true
|
|
1540
|
+
if (parts[0] === 198 && parts[1] === 51 && parts[2] === 100) return true
|
|
1541
|
+
if (parts[0] === 203 && parts[1] === 0 && parts[2] === 113) return true
|
|
1542
|
+
return false
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
export function scrubPii(text: string, cfg: Config): string {
|
|
1546
|
+
let out = text
|
|
1547
|
+
if (cfg.emails && out.includes('@')) {
|
|
1548
|
+
out = out.replace(EMAIL, address => (allowedEmail(address, cfg) || cfg.allow.has(address) ? address : mint('EMAIL', address)))
|
|
1549
|
+
}
|
|
1550
|
+
out = out.replace(IPV4, address => {
|
|
1551
|
+
if (cfg.allow.has(address) || (!cfg.privateIps && isPrivateV4(address))) return address
|
|
1552
|
+
return mint('IP', address)
|
|
1553
|
+
})
|
|
1554
|
+
return out.replace(IPV6, address => {
|
|
1555
|
+
if (cfg.allow.has(address)) return address
|
|
1556
|
+
const lower = address.toLowerCase()
|
|
1557
|
+
if (!cfg.privateIps && (lower.startsWith('fe80:') || lower.startsWith('fc') || lower.startsWith('fd'))) return address
|
|
1558
|
+
return mint('IP', address)
|
|
1559
|
+
})
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
// ------------------------------------------------------------- the walkers
|
|
1563
|
+
|
|
1564
|
+
// A string under one of these keys is media (an image, a file body) when it
|
|
1565
|
+
// is one long base64 run; then it is passed through untouched. Anything else
|
|
1566
|
+
// under the same key is text and is scanned, and so is a run that decodes to
|
|
1567
|
+
// text holding a secret (a Gmail API `body.data`): scrubSecrets then hides it
|
|
1568
|
+
// whole.
|
|
1569
|
+
const MEDIA_KEYS = new Set(['data', 'base64', 'b64_json', 'imageData', 'thumbnail', 'blob'])
|
|
1570
|
+
const BASE64_VALUE = /^[A-Za-z0-9+/_-]{256}[A-Za-z0-9+/_-]*={0,2}$/
|
|
1571
|
+
const MAX_DEPTH = 64
|
|
1572
|
+
|
|
1573
|
+
// Set when the walk met a part it could not rewrite: deeper than MAX_DEPTH,
|
|
1574
|
+
// or an object that is not plain data. A hook that finds a secret elsewhere
|
|
1575
|
+
// in the same result then withholds it rather than hand the part on raw.
|
|
1576
|
+
export type WalkState = { incomplete: boolean }
|
|
1577
|
+
|
|
1578
|
+
export function walk(value: unknown, fn: (s: string) => string, st: WalkState = { incomplete: false }, depth = 0, key = ''): unknown {
|
|
1579
|
+
if (typeof value === 'string') return MEDIA_KEYS.has(key) && BASE64_VALUE.test(value) && !base64HoldsSecret(value) ? value : fn(value)
|
|
1580
|
+
if (value === null || typeof value !== 'object') return value
|
|
1581
|
+
if (ArrayBuffer.isView(value) || value instanceof ArrayBuffer) return value
|
|
1582
|
+
if (depth >= MAX_DEPTH) {
|
|
1583
|
+
st.incomplete = true
|
|
1584
|
+
return value
|
|
1585
|
+
}
|
|
1586
|
+
// Array items inherit the key, so `data: [<base64>, ...]` stays media.
|
|
1587
|
+
if (Array.isArray(value)) return value.map((v) => walk(v, fn, st, depth + 1, key))
|
|
1588
|
+
const proto = Object.getPrototypeOf(value)
|
|
1589
|
+
if (proto !== Object.prototype && proto !== null) {
|
|
1590
|
+
st.incomplete = true
|
|
1591
|
+
return value
|
|
1592
|
+
}
|
|
1593
|
+
const out: Record<string, unknown> = {}
|
|
1594
|
+
for (const [k, v] of Object.entries(value as Record<string, unknown>)) out[k] = walk(v, fn, st, depth + 1, k)
|
|
1595
|
+
return out
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
// ------------------------------------------------------------------ config
|
|
1599
|
+
|
|
1600
|
+
function bool(v: unknown, fallback: boolean): boolean {
|
|
1601
|
+
if (typeof v === 'boolean') return v
|
|
1602
|
+
if (v === 'true') return true
|
|
1603
|
+
if (v === 'false') return false
|
|
1604
|
+
return fallback
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
function num(v: unknown, fallback: number): number {
|
|
1608
|
+
const n = Number(v)
|
|
1609
|
+
return v !== '' && v !== null && v !== undefined && Number.isFinite(n) ? n : fallback
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
function strings(v: unknown): string[] {
|
|
1613
|
+
if (Array.isArray(v)) return v.map(String)
|
|
1614
|
+
if (typeof v === 'string' && v.trim()) return v.split(',').map((s) => s.trim()).filter(Boolean)
|
|
1615
|
+
return []
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
export function makeConfig(options: Record<string, unknown>): Config {
|
|
1619
|
+
const tools = options.restoreTools === undefined ? DEFAULT_RESTORE_TOOLS : options.restoreTools
|
|
1620
|
+
return {
|
|
1621
|
+
secrets: bool(options.secrets, true),
|
|
1622
|
+
pii: bool(options.pii, false),
|
|
1623
|
+
emails: bool(options.hideEmails, true),
|
|
1624
|
+
privateIps: bool(options.redactPrivateIps, false),
|
|
1625
|
+
restore: bool(options.restoreInToolInputs, true),
|
|
1626
|
+
restoreTools: new Set(strings(tools)),
|
|
1627
|
+
notify: bool(options.notify, true),
|
|
1628
|
+
minEntropy: num(options.minEntropy, 3.6),
|
|
1629
|
+
minLength: Math.max(8, num(options.minLength, 24)),
|
|
1630
|
+
contextMinEntropy: num(options.contextMinEntropy, 3.0),
|
|
1631
|
+
allow: new Set(strings(options.allow)),
|
|
1632
|
+
allowEmails: strings(options.allowEmails),
|
|
1633
|
+
allowPrefix: strings(options.allowPrefixes),
|
|
1634
|
+
denyPrefix: strings(options.denyPrefixes),
|
|
1635
|
+
}
|
|
1636
|
+
}
|