@kaynooo/cli-env 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/pins.ts ADDED
@@ -0,0 +1,268 @@
1
+ import type { Dirent } from 'node:fs'
2
+ import { readdir, readFile } from 'node:fs/promises'
3
+ import { join, relative } from 'node:path'
4
+ import { fail } from '@kaynooo/cli-kit'
5
+ import { parseDotenv } from './dotenv'
6
+
7
+ export interface PinFinding {
8
+ // A test file that renders a page or starts the app, relative to the app folder.
9
+ file: string
10
+ unpinned: string[]
11
+ }
12
+
13
+ export interface AppPins {
14
+ app: string
15
+ schema: string
16
+ // The optional variables of the schema whose name says they reach outside.
17
+ reaching: string[]
18
+ // What no test env entry, preload or test file pins.
19
+ unpinned: string[]
20
+ // The rendering test files leaving some of the reaching variables to the developer's env.
21
+ files: PinFinding[]
22
+ // The local preload that drops what `.env` sets and `.env.test` does not, which pins every variable.
23
+ preload?: string
24
+ }
25
+
26
+ const OPTIONAL = /^\s*([A-Z][A-Z0-9_]*)\s*:\s*s\.Optional\(/gm
27
+ // A URL, key, token, secret, password, DSN or non-bare host, or the file holding one; the app's own public address only links back to itself.
28
+ const REACHING = /(?:URL|URI|TOKEN|KEY|SECRET|PASSWORD|DSN|_HOST)(?:_FILE)?$/
29
+ const SELF = /(?:^|_)(?:PUBLIC|SITE|BASE|APP)_URL$/
30
+ const RENDERS = /\b(?:testSite|proxyKeyTests)\s*\(/
31
+ // A test file that renders pages reaching none of the variables says so with this marker and a reason.
32
+ const NOT_NEEDED = /\/\/\s*ev pins:\s*none reached\b/
33
+ const OVERRIDE = /\boverrideEnv\s*\(/g
34
+ const KEY = /(?:^|[{,])\s*([A-Z_]\w*)\s*:/gi
35
+ const TEST_FILE = /\.test\.[cm]?[jt]sx?$/
36
+ const ENV_TEST = /^\.env\.test(?:\.local)?$/
37
+ const TEST_ROOTS = ['tests', 'test', '__tests__', 'e2e', 'src']
38
+ const SKIPPED = new Set(['node_modules', '.git', '.kay', 'dist', 'vendor'])
39
+ const SCHEMA_CANDIDATES = ['src/env.ts', 'env.ts', 'src/env/index.ts']
40
+ const OPENERS = new Set(['(', '[', '{'])
41
+ const CLOSERS = new Set([')', ']', '}'])
42
+ const QUOTES = new Set(['\'', '"', '`'])
43
+
44
+ async function exists(path: string): Promise<boolean> {
45
+ return Bun.file(path).exists()
46
+ }
47
+
48
+ async function walk(folder: string, keep: (name: string) => boolean): Promise<string[]> {
49
+ const found: string[] = []
50
+ let entries: Dirent[] = []
51
+
52
+ try {
53
+ entries = await readdir(folder, { withFileTypes: true })
54
+ }
55
+ catch {
56
+ return found
57
+ }
58
+
59
+ for (const entry of entries) {
60
+ if (SKIPPED.has(entry.name))
61
+ continue
62
+
63
+ const path = join(folder, entry.name)
64
+
65
+ if (entry.isDirectory())
66
+ found.push(...await walk(path, keep))
67
+ else if (entry.isFile() && keep(entry.name))
68
+ found.push(path)
69
+ }
70
+
71
+ return found.sort()
72
+ }
73
+
74
+ // The test files under the folders tests live in, which leaves an app's data folders alone.
75
+ async function testFiles(folder: string): Promise<string[]> {
76
+ const found: string[] = []
77
+
78
+ for (const root of TEST_ROOTS) {
79
+ found.push(...await walk(join(folder, root), name => TEST_FILE.test(name)))
80
+ }
81
+
82
+ return found
83
+ }
84
+
85
+ // The text between the parenthesis opening at `open` and its match, skipping string literals.
86
+ function balanced(text: string, open: number): string {
87
+ let depth = 0
88
+ let quote = ''
89
+
90
+ for (let index = open; index < text.length; index++) {
91
+ const char = text[index] ?? ''
92
+
93
+ if (quote !== '') {
94
+ if (char === '\\')
95
+ index++
96
+ else if (char === quote)
97
+ quote = ''
98
+ }
99
+ else if (QUOTES.has(char)) {
100
+ quote = char
101
+ }
102
+ else if (OPENERS.has(char)) {
103
+ depth++
104
+ }
105
+ else if (CLOSERS.has(char) && --depth === 0) {
106
+ return text.slice(open + 1, index)
107
+ }
108
+ }
109
+
110
+ return text.slice(open + 1)
111
+ }
112
+
113
+ // What is written at the top level of a text: nested brackets and string literals are blanked, so only the keys of the object itself remain.
114
+ function topLevel(text: string): string {
115
+ let depth = 0
116
+ let quote = ''
117
+ let out = ''
118
+
119
+ for (let index = 0; index < text.length; index++) {
120
+ const char = text[index] ?? ''
121
+
122
+ if (quote !== '') {
123
+ if (char === '\\')
124
+ index++
125
+ else if (char === quote)
126
+ quote = ''
127
+ }
128
+ else if (QUOTES.has(char)) {
129
+ quote = char
130
+ }
131
+ else if (OPENERS.has(char)) {
132
+ out += depth === 0 ? char : ''
133
+ depth++
134
+ }
135
+ else if (CLOSERS.has(char)) {
136
+ depth--
137
+ out += depth === 0 ? char : ''
138
+ }
139
+ else if (depth <= 1) {
140
+ out += char
141
+ }
142
+ }
143
+
144
+ return out
145
+ }
146
+
147
+ // The variable names a test file gives another value through `overrideEnv(env, { NAME: ... })`; values are never looked at.
148
+ export function overriddenNames(source: string): Set<string> {
149
+ const names = new Set<string>()
150
+
151
+ for (const match of source.matchAll(OVERRIDE)) {
152
+ const args = topLevel(balanced(source, match.index + match[0].length - 1))
153
+ const object = args.slice(args.indexOf(',') + 1)
154
+
155
+ for (const key of object.matchAll(KEY))
156
+ names.add(key[1] ?? '')
157
+ }
158
+
159
+ return names
160
+ }
161
+
162
+ // The optional variables of a `defineEnv` schema, by name.
163
+ export function optionalNames(source: string): string[] {
164
+ const start = source.indexOf('defineEnv(')
165
+
166
+ return start === -1 ? [] : [...source.slice(start).matchAll(OPTIONAL)].map(match => match[1] ?? '')
167
+ }
168
+
169
+ export function reachesOutside(name: string): boolean {
170
+ return REACHING.test(name) && !SELF.test(name)
171
+ }
172
+
173
+ async function schemaFile(folder: string): Promise<string | undefined> {
174
+ for (const candidate of SCHEMA_CANDIDATES) {
175
+ const path = join(folder, candidate)
176
+
177
+ if (await exists(path) && (await readFile(path, 'utf8')).includes('defineEnv('))
178
+ return path
179
+ }
180
+
181
+ return undefined
182
+ }
183
+
184
+ // The keys `.env.test` and `.env.test.local` set, even to an empty value; the values are dropped as parsed.
185
+ async function testEnvKeys(folder: string): Promise<Set<string>> {
186
+ const keys = new Set<string>()
187
+
188
+ for (const entry of await readdir(folder, { withFileTypes: true })) {
189
+ if (!entry.isFile() || !ENV_TEST.test(entry.name))
190
+ continue
191
+
192
+ for (const line of parseDotenv(await readFile(join(folder, entry.name), 'utf8')).lines) {
193
+ if (line.key !== undefined)
194
+ keys.add(line.key)
195
+ }
196
+ }
197
+
198
+ return keys
199
+ }
200
+
201
+ // A local test preload reading `.env` and writing process.env drops what the files set that `.env.test` does not, so no developer value reaches a test.
202
+ async function pinningPreload(folder: string): Promise<string | undefined> {
203
+ const path = join(folder, 'bunfig.toml')
204
+
205
+ if (!await exists(path))
206
+ return undefined
207
+
208
+ let config: { preload?: unknown, test?: { preload?: unknown } }
209
+
210
+ try {
211
+ config = Bun.TOML.parse(await readFile(path, 'utf8'))
212
+ }
213
+ catch {
214
+ fail(`${path} is not valid TOML`)
215
+ }
216
+
217
+ const preloads = [config.test?.preload, config.preload].flat().filter((entry): entry is string => typeof entry === 'string' && entry.startsWith('.'))
218
+
219
+ for (const preload of preloads) {
220
+ const file = join(folder, preload)
221
+
222
+ if (!await exists(file))
223
+ continue
224
+
225
+ const source = await readFile(file, 'utf8')
226
+
227
+ if (source.includes('.env') && source.includes('process.env'))
228
+ return preload
229
+ }
230
+
231
+ return undefined
232
+ }
233
+
234
+ // Reads one app folder, or undefined when it declares no env schema.
235
+ export async function checkApp(folder: string, except: ReadonlySet<string> = new Set()): Promise<AppPins | undefined> {
236
+ const schema = await schemaFile(folder)
237
+
238
+ if (schema === undefined)
239
+ return undefined
240
+
241
+ const reaching = optionalNames(await readFile(schema, 'utf8')).filter(name => reachesOutside(name) && !except.has(name))
242
+ const preload = await pinningPreload(folder)
243
+ const app: AppPins = { app: folder, schema: relative(folder, schema), reaching, unpinned: [], files: [], ...preload !== undefined && { preload } }
244
+
245
+ if (preload !== undefined)
246
+ return app
247
+
248
+ const fromEnv = await testEnvKeys(folder)
249
+ const open = reaching.filter(name => !fromEnv.has(name))
250
+ const anywhere = new Set<string>()
251
+
252
+ for (const path of await testFiles(folder)) {
253
+ const source = await readFile(path, 'utf8')
254
+ const pinned = overriddenNames(source)
255
+
256
+ for (const name of pinned)
257
+ anywhere.add(name)
258
+
259
+ const unpinned = open.filter(name => !pinned.has(name))
260
+
261
+ if (RENDERS.test(source) && !NOT_NEEDED.test(source) && unpinned.length > 0)
262
+ app.files.push({ file: relative(folder, path), unpinned })
263
+ }
264
+
265
+ app.unpinned = open.filter(name => !anywhere.has(name))
266
+
267
+ return app
268
+ }
package/src/redact.ts ADDED
@@ -0,0 +1,43 @@
1
+ #!/usr/bin/env bun
2
+
3
+ import process from 'node:process'
4
+ import { parse, readInput, runMain } from '@kaynooo/cli-kit'
5
+ import { loadKnown } from './files'
6
+ import { check, Redactor } from './redactor'
7
+
8
+ const usage = `redact [--check]
9
+
10
+ Copies stdin to stdout line by line with secrets masked: known token formats (GitHub, GitLab, Slack, AWS, npm, Stripe, Google and SendGrid keys, sk-, JWTs, Authorization headers, password= and secret= pairs), private key blocks, the password and the tokens of any URL and the values of secret-looking keys in this folder's .env* files.
11
+ --check prints nothing of the input and lists the file and line of each secret found, exiting 1 when there is any: \`git diff --cached | redact --check\`.\n\nExits 0 when it copied the input or found no secret, 1 when --check finds one, 2 on a usage error.`
12
+
13
+ const options = {
14
+ 'check': { type: 'boolean', description: 'list where secrets are instead of masking them, exit 1 when found' },
15
+ 'no-env': { type: 'boolean', description: 'do not search for the values of the .env* files' },
16
+ } as const
17
+
18
+ async function run(args: string[]): Promise<number> {
19
+ const parsed = parse(args, usage, options, { replaces: 'printing a command\'s secrets into a log or a transcript', examples: [['docker compose config | redact', 'the resolved compose file, secrets masked'], ['git diff --cached | redact --check', 'refuse a commit carrying a secret']] })
20
+
21
+ const known = parsed.values['no-env'] === true ? undefined : await loadKnown()
22
+
23
+ if (parsed.values.check === true) {
24
+ const findings = check(await readInput('-'), known)
25
+
26
+ for (const { file, line, kind } of findings)
27
+ console.log(`${file}:${line}: ${kind}`)
28
+
29
+ return findings.length === 0 ? 0 : 1
30
+ }
31
+
32
+ const redactor = new Redactor(known)
33
+ const decoder = new TextDecoder()
34
+
35
+ for await (const chunk of Bun.stdin.stream())
36
+ process.stdout.write(redactor.push(decoder.decode(chunk, { stream: true })))
37
+
38
+ process.stdout.write(redactor.push(decoder.decode()) + redactor.end())
39
+
40
+ return 0
41
+ }
42
+
43
+ process.exitCode = await runMain({ name: 'redact', run })
@@ -0,0 +1,166 @@
1
+ import type { KnownSecrets } from './mask'
2
+ import { findSecrets, KEY_MASK, maskLine, openKeyBody, PEM_END } from './mask'
3
+
4
+ export interface Finding {
5
+ file: string
6
+ line: number
7
+ kind: string
8
+ }
9
+
10
+ const DIFF_START = /^(?:diff --git |@@ )/
11
+ const NEW_FILE = /^\+\+\+ (?:b\/)?(.+)$/
12
+ const HUNK = /^@@ -\d+(?:,\d+)? \+(\d+)/
13
+
14
+ // A line longer than this with no line break is cut, keeping its last OVERLAP characters to be read with what follows.
15
+ const MAX_PENDING = 1 << 18
16
+ const OVERLAP = 1 << 13
17
+
18
+ // Masks text pushed in chunks of any size: only whole lines are emitted, so a secret split across two chunks is seen whole, and the lines of a private key block are replaced by one marker.
19
+ export class Redactor {
20
+ private pending = ''
21
+ private inKey = false
22
+ private hidden = false
23
+
24
+ constructor(private readonly known?: KnownSecrets) {}
25
+
26
+ push(chunk: string): string {
27
+ const end = chunk.lastIndexOf('\n')
28
+
29
+ this.pending += chunk
30
+
31
+ if (end === -1)
32
+ return this.pending.length > MAX_PENDING ? this.cut() : ''
33
+
34
+ const split = this.pending.length - chunk.length + end + 1
35
+ const complete = this.pending.slice(0, split)
36
+
37
+ this.pending = this.pending.slice(split)
38
+
39
+ return complete.split('\n').slice(0, -1).map(line => this.line(line)).filter(line => line !== undefined).map(line => `${line}\n`).join('')
40
+ }
41
+
42
+ // Emits a line that never ends, up to the last space before the overlap when there is one so a secret is not cut in two.
43
+ private cut(): string {
44
+ const limit = this.pending.length - OVERLAP
45
+ const space = this.pending.lastIndexOf(' ', limit)
46
+ const at = space > 0 ? space + 1 : limit
47
+ const head = this.pending.slice(0, at)
48
+
49
+ this.pending = this.pending.slice(at)
50
+
51
+ return this.line(head) ?? ''
52
+ }
53
+
54
+ end(): string {
55
+ const rest = this.pending
56
+
57
+ this.pending = ''
58
+
59
+ return rest === '' ? '' : this.line(rest) ?? ''
60
+ }
61
+
62
+ private line(line: string): string | undefined {
63
+ if (this.inKey) {
64
+ const end = PEM_END.exec(line)
65
+
66
+ if (end === null) {
67
+ if (this.hidden)
68
+ return undefined
69
+
70
+ this.hidden = true
71
+
72
+ return KEY_MASK
73
+ }
74
+
75
+ this.inKey = false
76
+
77
+ // The text before the END marker is the last of the key, the text after it is read as any line.
78
+ return (end.index > 0 && !this.hidden ? KEY_MASK : '') + this.line(line.slice(end.index))
79
+ }
80
+
81
+ const body = openKeyBody(line)
82
+
83
+ if (body !== undefined) {
84
+ this.inKey = true
85
+ this.hidden = body !== ''
86
+ }
87
+
88
+ return maskLine(line, this.known)
89
+ }
90
+ }
91
+
92
+ // Lists the secrets in text without printing any. A unified diff is read on its added lines, with the new file's name and line numbers.
93
+ export function check(text: string, known?: KnownSecrets): Finding[] {
94
+ const findings: Finding[] = []
95
+ const isDiff = text.split('\n').some(line => DIFF_START.test(line))
96
+ let file = 'stdin'
97
+ let number = 0
98
+ let inKey = false
99
+
100
+ for (const [index, raw] of text.split('\n').entries()) {
101
+ let line = raw
102
+
103
+ if (isDiff) {
104
+ const hunk = HUNK.exec(raw)
105
+
106
+ if (hunk !== null) {
107
+ number = Number(hunk[1]) - 1
108
+
109
+ continue
110
+ }
111
+
112
+ const named = NEW_FILE.exec(raw)
113
+
114
+ if (named !== null) {
115
+ file = named[1] ?? file
116
+
117
+ continue
118
+ }
119
+
120
+ if (raw.startsWith('diff --git ')) {
121
+ file = '(unnamed)'
122
+
123
+ continue
124
+ }
125
+
126
+ if (raw.startsWith('-') || raw.startsWith('\\'))
127
+ continue
128
+
129
+ number++
130
+
131
+ if (!raw.startsWith('+'))
132
+ continue
133
+
134
+ line = raw.slice(1)
135
+ }
136
+ else {
137
+ number = index + 1
138
+ }
139
+
140
+ if (inKey) {
141
+ const end = PEM_END.exec(line)
142
+
143
+ inKey = end === null
144
+
145
+ if (end === null)
146
+ continue
147
+
148
+ line = line.slice(end.index)
149
+ }
150
+
151
+ const found = findSecrets(line, known).map(match => match.kind)
152
+ const body = openKeyBody(line)
153
+
154
+ if (body !== undefined) {
155
+ inKey = true
156
+
157
+ if (body === '')
158
+ found.push('private key')
159
+ }
160
+
161
+ for (const kind of found)
162
+ findings.push({ file, line: number, kind })
163
+ }
164
+
165
+ return findings
166
+ }