@nightloom/isy 1.0.4 → 1.0.6-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/dist/agents/claude.js +3 -2
- package/dist/agents/claude.js.map +1 -1
- package/dist/agents/codex.js +14 -5
- package/dist/agents/codex.js.map +1 -1
- package/dist/agents/index.d.ts +14 -10
- package/dist/agents/index.js +65 -15
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/kimi.js +26 -8
- package/dist/agents/kimi.js.map +1 -1
- package/dist/agents/types.d.ts +12 -3
- package/dist/api.d.ts +1 -0
- package/dist/api.js +1 -1
- package/dist/api.js.map +1 -1
- package/dist/commands/commit.d.ts +6 -0
- package/dist/commands/commit.js +326 -16
- package/dist/commands/commit.js.map +1 -1
- package/dist/commands/health.js +1 -1
- package/dist/commands/health.js.map +1 -1
- package/dist/commands/init.js +40 -9
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/status.d.ts +2 -0
- package/dist/commands/status.js +7 -4
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/upload.d.ts +8 -1
- package/dist/commands/upload.js +65 -48
- package/dist/commands/upload.js.map +1 -1
- package/dist/detectors.d.ts +32 -0
- package/dist/detectors.js +279 -72
- package/dist/detectors.js.map +1 -1
- package/dist/git.d.ts +52 -0
- package/dist/git.js +166 -9
- package/dist/git.js.map +1 -1
- package/dist/githook.d.ts +18 -7
- package/dist/githook.js +147 -17
- package/dist/githook.js.map +1 -1
- package/dist/hook.d.ts +22 -2
- package/dist/hook.js +28 -3
- package/dist/hook.js.map +1 -1
- package/dist/index.js +23 -7
- package/dist/index.js.map +1 -1
- package/dist/mask-paths.d.ts +4 -3
- package/dist/mask-paths.js +4 -3
- package/dist/mask-paths.js.map +1 -1
- package/dist/parser.d.ts +58 -9
- package/dist/parser.js +337 -58
- package/dist/parser.js.map +1 -1
- package/dist/paths.d.ts +19 -1
- package/dist/paths.js +28 -3
- package/dist/paths.js.map +1 -1
- package/dist/redact.d.ts +2 -0
- package/dist/redact.js +987 -95
- package/dist/redact.js.map +1 -1
- package/dist/stage0.js +27 -8
- package/dist/stage0.js.map +1 -1
- package/dist/state.d.ts +15 -0
- package/dist/state.js +8 -0
- package/dist/state.js.map +1 -1
- package/dist/test/check.test.js +1 -1
- package/dist/test/check.test.js.map +1 -1
- package/dist/test/cli.test.js +11 -1
- package/dist/test/cli.test.js.map +1 -1
- package/dist/test/codex.test.js +27 -4
- package/dist/test/codex.test.js.map +1 -1
- package/dist/test/commit.test.js +1466 -6
- package/dist/test/commit.test.js.map +1 -1
- package/dist/test/e2e-harness.test.d.ts +1 -0
- package/dist/test/e2e-harness.test.js +53 -0
- package/dist/test/e2e-harness.test.js.map +1 -0
- package/dist/test/git.test.js +72 -2
- package/dist/test/git.test.js.map +1 -1
- package/dist/test/hook.test.js +138 -3
- package/dist/test/hook.test.js.map +1 -1
- package/dist/test/init.test.js +61 -2
- package/dist/test/init.test.js.map +1 -1
- package/dist/test/kimi-config.test.js +27 -5
- package/dist/test/kimi-config.test.js.map +1 -1
- package/dist/test/kimi-hook.test.js +1 -1
- package/dist/test/kimi-hook.test.js.map +1 -1
- package/dist/test/kimi-listing.test.js +42 -2
- package/dist/test/kimi-listing.test.js.map +1 -1
- package/dist/test/kimi.test.js +26 -2
- package/dist/test/kimi.test.js.map +1 -1
- package/dist/test/parser.test.js +527 -1
- package/dist/test/parser.test.js.map +1 -1
- package/dist/test/redact.test.js +2472 -14
- package/dist/test/redact.test.js.map +1 -1
- package/dist/test/stage0.test.js +617 -1
- package/dist/test/stage0.test.js.map +1 -1
- package/dist/test/sweep.test.js +110 -7
- package/dist/test/sweep.test.js.map +1 -1
- package/package.json +1 -1
- package/src/agents/claude.ts +3 -2
- package/src/agents/codex.ts +14 -5
- package/src/agents/index.ts +71 -16
- package/src/agents/kimi.ts +28 -8
- package/src/agents/types.ts +12 -3
- package/src/api.ts +4 -2
- package/src/commands/commit.ts +354 -18
- package/src/commands/health.ts +1 -1
- package/src/commands/init.ts +45 -10
- package/src/commands/status.ts +9 -4
- package/src/commands/upload.ts +73 -63
- package/src/detectors.ts +284 -75
- package/src/git.ts +175 -9
- package/src/githook.ts +151 -19
- package/src/hook.ts +30 -3
- package/src/index.ts +25 -7
- package/src/mask-paths.ts +5 -4
- package/src/parser.ts +355 -51
- package/src/paths.ts +30 -3
- package/src/redact.ts +991 -98
- package/src/stage0.ts +28 -8
- package/src/state.ts +18 -0
package/src/redact.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { homedir } from "node:os";
|
|
2
|
-
import { basename } from "node:path";
|
|
2
|
+
import { basename, resolve } from "node:path";
|
|
3
3
|
import { fileLines } from "./lines.js";
|
|
4
|
-
import { HEREDOC, commandSegments, withoutHeredocs } from "./parser.js";
|
|
4
|
+
import { type Budget, HEREDOC, STRUCTURES, commandSegments, mayParse, shellEditWords, spend, withoutHeredocs } from "./parser.js";
|
|
5
5
|
import {
|
|
6
6
|
PATH_KEYS,
|
|
7
7
|
REDACTION_MARKER,
|
|
8
|
+
looksAbsolute,
|
|
8
9
|
marker,
|
|
9
10
|
maskPath,
|
|
10
11
|
maskPaths,
|
|
@@ -28,24 +29,65 @@ function anyCase(source: string): string {
|
|
|
28
29
|
return source.replace(/[a-z]/g, (letter) => `[${letter}${letter.toUpperCase()}]`);
|
|
29
30
|
}
|
|
30
31
|
|
|
31
|
-
|
|
32
|
+
/**
|
|
33
|
+
* `pass` and `pwd` count only as a word of their own, as `compass`, `bypass`, and the
|
|
34
|
+
* shell's `PWD` and `OLDPWD` are all over a transcript: `DB_PASS`, `db-pass`, `dbPass`,
|
|
35
|
+
* `DB_PWD`, `jdbc.pwd`, `Uid=sa;Pwd=…`, and `Pwd=…` as only a connection string spells it.
|
|
36
|
+
*/
|
|
37
|
+
const SECRET_WORD = [
|
|
38
|
+
anyCase("pass(?:word|wd|phrase)|secret|token|(?:api|private|access|account)[-_]?key|credentials?|auth(?:orization)?|cookie"),
|
|
39
|
+
`(?:(?<![A-Za-z0-9])|(?<=[a-z0-9])(?<![bB][yY])(?=[A-Z]))${anyCase("pass")}`,
|
|
40
|
+
`(?:(?<=[;_]|[A-Za-z0-9][.-])|(?<=[a-z0-9])(?=[A-Z]))${anyCase("pwd")}`,
|
|
41
|
+
"(?<![A-Za-z0-9])Pwd",
|
|
42
|
+
].join("|");
|
|
43
|
+
|
|
44
|
+
/** What is still the secret itself: its value, and the environment it is for. */
|
|
45
|
+
const SAME_SECRET = "value|prod(?:uction)?|dev|stag(?:e|ing)|test|live|old|new|local";
|
|
46
|
+
const KEY_OR_BASE = "(?:[-_]?key)?(?:[-_]?base)?";
|
|
47
|
+
/** Its version too, short of a year: `token_2022` names a chain's program and holds nothing. */
|
|
48
|
+
const VERSION = "[0-9]{1,3}";
|
|
49
|
+
/** `TOKEN_PROD`, `secret-old`, `API_KEY_2`. */
|
|
50
|
+
const PLAIN_TAIL = anyCase(`${KEY_OR_BASE}(?:[-_](?:${SAME_SECRET}))?(?:[-_]${VERSION})?`);
|
|
51
|
+
/** `tokenValue`, `apiKeyProd`, and `accessToken2`, as `token0` and `oauth2` are no secrets. The cap keeps a run of `token1` linear. */
|
|
52
|
+
const HUMP_TAIL = anyCase(`${KEY_OR_BASE}(?:[-_]?(?:${SAME_SECRET}))?`) + `(?:(?:[-_]|(?<=[a-z0-9][A-Z][A-Za-z0-9]{0,64}))${VERSION})?`;
|
|
32
53
|
|
|
33
54
|
/**
|
|
34
55
|
* A name that holds a secret: one of the words, wherever it sits in an
|
|
35
56
|
* identifier — `PGPASSWORD`, `STRIPE_SECRET_KEY`, `aws_secret_access_key`,
|
|
36
|
-
* `x-api-key` — and followed by nothing but `KEY` and `BASE` (`SECRET_KEY_BASE`)
|
|
37
|
-
* That tail is the line between the secret and what
|
|
38
|
-
* `tokenHash`, `max_tokens`, `TOKENS_PER_ANALYSIS` and
|
|
39
|
-
* in a real corpus as often as the secrets do, and none of
|
|
57
|
+
* `x-api-key` — and followed by nothing but `KEY` and `BASE` (`SECRET_KEY_BASE`)
|
|
58
|
+
* and what is still the secret. That tail is the line between the secret and what
|
|
59
|
+
* is said about it: `tokenHash`, `max_tokens`, `TOKENS_PER_ANALYSIS` and
|
|
60
|
+
* `PASSWORD_FILE` turn up in a real corpus as often as the secrets do, and none of
|
|
61
|
+
* them is one.
|
|
40
62
|
*/
|
|
41
|
-
function secretName(atWord = ""): string {
|
|
42
|
-
|
|
63
|
+
function secretName(atWord = "", tail = HUMP_TAIL): string {
|
|
64
|
+
// From the front this runs at every character of a word, so the letters the words start with go first.
|
|
65
|
+
return String.raw`[A-Za-z0-9_.-]*?(?=[aAcCpPsStT])${atWord}(?:${SECRET_WORD})${tail}(?![A-Za-z0-9_.-])`;
|
|
43
66
|
}
|
|
44
67
|
|
|
68
|
+
/** `adminPassword`, `_authToken`, `tokenValue`: as often a variable in code as a key in a file, and told by a hump. */
|
|
69
|
+
const CAMEL_NAME = secretName() + "(?<=[a-z0-9][A-Z][A-Za-z0-9_.-]*)";
|
|
70
|
+
/** `accessToken: …`, which a value in no quotes leaves to be a type as often as data. */
|
|
71
|
+
const CAMEL_ENDING = String.raw`[A-Za-z0-9_.-]*?(?<=[a-z0-9])(?:Token|Secret|Password)`;
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* `First pass: everything compiles`, `--- PASS: TestParser`, `auth: requireAuth`,
|
|
75
|
+
* `const cookie = parseCookie(req)`: alone, these words are all over prose, code and
|
|
76
|
+
* test output, so alone and spaced from a value with no digit and no quote they name
|
|
77
|
+
* nothing. A secret has a name in front (`SMTP_PASS: …`), a quote or a digit, or
|
|
78
|
+
* takes its `=` with no space.
|
|
79
|
+
*/
|
|
80
|
+
const NOT_SAID_IN_PASSING = String.raw`(?!PASS:[ \t]|(?:${anyCase("pass|auth|credentials?|cookie")})["']?\s*[=:]\s+(?!["'])[^\d\s]*(?:\s|$))`;
|
|
45
81
|
/** The word glued to what came before, but not as a camelCase hump. */
|
|
46
|
-
const PLAIN_NAME = secretName("(?!(?<=[a-z0-9])[A-Z])");
|
|
47
|
-
|
|
48
|
-
const
|
|
82
|
+
const PLAIN_NAME = NOT_SAID_IN_PASSING + secretName("(?!(?<=[a-z0-9])[A-Z])", PLAIN_TAIL);
|
|
83
|
+
|
|
84
|
+
const VALUE_CHAR = String.raw`[^\s"'{}()[\]<>$]`;
|
|
85
|
+
/**
|
|
86
|
+
* `hash(input)`, `foo.bar(baz).qux(),`, `f(g(x))`: a call, which in quotes is a password
|
|
87
|
+
* with a bracket in it, and so is `abc(def)ghi1` anywhere, as code puts nothing like that
|
|
88
|
+
* after a closing bracket.
|
|
89
|
+
*/
|
|
90
|
+
const CALL = String.raw`(?<!["'])[A-Za-z_][\w.]*\((?![^\s"'{}[\]<>]{0,256}\)[\w$#@~\\])`;
|
|
49
91
|
|
|
50
92
|
/**
|
|
51
93
|
* What an assigned secret looks like: long enough to be one, and not code. A
|
|
@@ -53,8 +95,152 @@ const CAMEL_NAME = secretName("(?<=[a-z0-9])(?=[A-Z])");
|
|
|
53
95
|
* secret would be a string; anywhere else a number is a password like any other
|
|
54
96
|
* — `DB_PASSWORD=1234567890123`, `MYSQL_ROOT_PASSWORD: 12345678`. The digit is
|
|
55
97
|
* looked for first so the lookbehind runs only where a number starts.
|
|
98
|
+
*
|
|
99
|
+
* A `$` or a bracket inside belongs to the value (`abc$REST123`, `abc(def)ghi1`), as
|
|
100
|
+
* a password takes any character. They end it only where they start something
|
|
101
|
+
* else: `$PASSWORD` and `$(cat f)` at the front, `prefix${X}`, and a call, of which
|
|
102
|
+
* only a name long enough to be a value goes.
|
|
103
|
+
*
|
|
104
|
+
* How much more a value takes goes by what is around it. In quotes it runs to the
|
|
105
|
+
* closing one, the other quote and `{}[]<>` included, though what opens with a
|
|
106
|
+
* bracket is a placeholder; `$` at the front is literal in single quotes, and in
|
|
107
|
+
* double ones where no shell would expand it: a hash, or anything with a digit in it
|
|
108
|
+
* that no all-caps name spells. With no space after an `=` it takes `{}[]<>` too,
|
|
109
|
+
* short of one closing around the assignment. Spaced from its name and unquoted it is
|
|
110
|
+
* code as often as not (`token = tokens[index]`), so the limits above hold there.
|
|
111
|
+
*
|
|
112
|
+
* A token's rule runs before the rules that go by a name and ends at the token's shape, so a
|
|
113
|
+
* value with a token in it (`api_key=glpat-…,…`, `secret: "ghp_…&…"`) reaches them as markers
|
|
114
|
+
* with the rest of the value around them, and nothing else has a name for that rest. It is the
|
|
115
|
+
* value still, whatever its length, as long as some of it follows a marker: a value that ends
|
|
116
|
+
* on a marker was cut whole. RUNS_ON reads a value so too.
|
|
117
|
+
*/
|
|
118
|
+
const MARKED_VALUE = String.raw`(?:${VALUE_CHAR}*\[ISY_REDACTED:\w+\])+${VALUE_CHAR}+`;
|
|
119
|
+
const INNER_DOLLAR_OR_PAREN = String.raw`[$()](?=[^\s"'{}[\]<>])`;
|
|
120
|
+
const LITERAL_DOLLAR = String.raw`\$(?=[^\s"]{0,256}\d)(?![{(])(?![A-Z_][A-Z0-9_]*\b)`;
|
|
121
|
+
const SECRET_VALUE = String.raw`(?:${MARKED_VALUE}|(?!\[ISY_REDACTED)(?!(?=-?\d)(?<=["']\s*:\s*)-?\d[\d.,eE+-]*(?![^\s"'{}()[\]<>$]))(?:(?<=")(?:${VALUE_CHAR}|${LITERAL_DOLLAR})(?:[^\s"$()]|${INNER_DOLLAR_OR_PAREN}){7,}|(?<=')(?:${VALUE_CHAR}|\$)(?:[^\s'$()]|${INNER_DOLLAR_OR_PAREN}){7,}|(?<=[\w.-]=)(?!=|${CALL})${VALUE_CHAR}(?:${VALUE_CHAR}|${INNER_DOLLAR_OR_PAREN}|[{}[\]<>](?=[^\s"'])){7,}|(?!${CALL})${VALUE_CHAR}(?:${VALUE_CHAR}|${INNER_DOLLAR_OR_PAREN}){7,}|${VALUE_CHAR}{8,}))`;
|
|
122
|
+
/**
|
|
123
|
+
* A value read to a space or a quote can end on the name of the next one: `Set-Cookie:
|
|
124
|
+
* session_token="…"`, `DB_PASS=DB_PASSWORD="…"`, `auth-cookie: api_key= …`. That name is then
|
|
125
|
+
* inside the match, where no rule after it looks, so the match goes on through the name's
|
|
126
|
+
* separator and over what it holds, as the name's own would have, closing quote and all, and
|
|
127
|
+
* again where that ends on a name. The rules after the plain assignment find such a name cut
|
|
128
|
+
* already, by its own match there, so only that rule goes on, with those ahead of it that can take a
|
|
129
|
+
* name (the authorization header's, a bearer token's, which takes a separator doubled (`secret==`) too,
|
|
130
|
+
* and a token's, with the rest of a name glued on to it) and the camelCase one after it, whose name the
|
|
131
|
+
* plain rule does not read (`apiKey: 'accessToken": '…'`).
|
|
132
|
+
* A quote right after a separator opens what the name holds rather than closing the step before:
|
|
133
|
+
* taken as the close, it left the next step nothing to start from (`password=api_token: "…,api_key="…"`).
|
|
134
|
+
*
|
|
135
|
+
* ponytail: the name is told by the word it ends with, as a copy of secretName() in both would
|
|
136
|
+
* take the patterns past the size a commit compiles in time (the size test in redact.test.ts),
|
|
137
|
+
* so what a name ending otherwise holds (`DB_PASS`, `TOKEN_PROD`) stays. Telling the name
|
|
138
|
+
* outside the pattern, once a match is found, is the way up.
|
|
139
|
+
*/
|
|
140
|
+
const NAME_END = anyCase("(?:passw(?:or)?d|secret|token|key|base)");
|
|
141
|
+
const RUNS_ON = String.raw`(?:(?<=${NAME_END}["']?[=:]*)(?:(?<=[=:])|["']?\s*[=:])\s*(?<q>["']?)(?:${MARKED_VALUE}|${VALUE_CHAR}{8,})(?:(?<![=:])\k<q>)?)*`;
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* The rest of a name glued on to a token whose shape stops inside it (`ghp_…2api-key==…`), so that
|
|
145
|
+
* the token's match reaches the name's end and goes on over what the name holds. Only with that
|
|
146
|
+
* value: a name taken with no value after it is one the assignment rule no longer sees. Any name, not
|
|
147
|
+
* only one RUNS_ON reads: the shape takes the name it is glued to whatever the name ends with
|
|
148
|
+
* (`glpat-…2auth=…`, `xapp-…TOKEN_PROD:=…`), so a value left after it had nothing to cut it.
|
|
149
|
+
* The value stops short of `Bearer` and a space, which a token's rule, ahead of the bearer
|
|
150
|
+
* token's, would otherwise take from it (`…-accessToken:…&Proxy-Authorization=Bearer …`).
|
|
151
|
+
*/
|
|
152
|
+
const NAME_TAIL = String.raw`(?:[\w.-]*["']?\s*[=:]\s*["']?(?:(?!${anyCase("bearer")}\s)${VALUE_CHAR}){8,})?${RUNS_ON}`;
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* A separator with a name before it, looked for from the separator back: from the front,
|
|
156
|
+
* a name costs a look at every letter of every word.
|
|
157
|
+
*/
|
|
158
|
+
const afterName = (name: string, between: string, separator: string): string => String.raw`${separator}(?<=(?<![A-Za-z0-9_.-])${name}${between}${separator})`;
|
|
159
|
+
/** The space or quote a name may have before its separator. */
|
|
160
|
+
const CLOSED = String.raw`["']?\s*`;
|
|
161
|
+
/**
|
|
162
|
+
* `=>`, `:=`, a comparison and a subscript's `"] =`: what is code as often as data
|
|
163
|
+
* (`password => password.trim()`), unless what follows is in quotes. A run of `=` is
|
|
164
|
+
* looked back from only where it starts, as no name stands right before the rest of it.
|
|
165
|
+
*/
|
|
166
|
+
const beforeQuoted = (name: string): string => afterName(name, String.raw`["']?\]?\s*`, "(?:=>|:=|(?<!=)={1,3}|!={1,2})") + String.raw`\s*["']`;
|
|
167
|
+
|
|
168
|
+
/** What nobody types by hand: twelve letters and digits in a row, both among them, as `documentation` and `202410201230` are not. */
|
|
169
|
+
const GENERATED_RUN = String.raw`(?=[A-Za-z]{0,11}\d)(?=\d{0,11}[A-Za-z])[A-Za-z0-9]{12}`;
|
|
170
|
+
const GENERATED = new RegExp(GENERATED_RUN);
|
|
171
|
+
/** `STRIPE_KEY`, `HMAC_KEY`: as often a sort order as a secret (`SORT_KEY=created_at`), so what it holds has to look generated too. */
|
|
172
|
+
const KEY_NAME = String.raw`(?:[A-Z][A-Z0-9_]{0,64}_)?KEY`;
|
|
173
|
+
const KEY_FIELD = new RegExp(`^${KEY_NAME}$`);
|
|
174
|
+
const KEY_END = String.raw`KEY(?<=(?<![A-Za-z0-9_])${KEY_NAME})`;
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* `curl -u admin:…`, its flag alone or last among others (`-fsSLu`). A walk to the flag
|
|
178
|
+
* stops at the next `curl`, which finds the same from there, so the walks of a line come
|
|
179
|
+
* to about its length however often it says `curl`. The cap says how far a flag is still
|
|
180
|
+
* this client's.
|
|
181
|
+
*/
|
|
182
|
+
const CURL_USER_FLAG = String.raw`curl\b(?:(?!\bcurl\b)[^\n;|]){0,200}?\s(?:-[a-zA-Z]{0,16}[uU]\s*|--(?:proxy-)?user[=\s]\s*)`;
|
|
183
|
+
/** A name may be a marker already: an address is one by now. */
|
|
184
|
+
const CURL_USER = String.raw`(?:\[ISY_REDACTED:\w+\]|(?!\[ISY_REDACTED)[^\s:"']+):`;
|
|
185
|
+
/**
|
|
186
|
+
* A client, whatever comes between, and its flag: stopped and capped as above, at a start
|
|
187
|
+
* as `PASSWORD_FLAG` has it.
|
|
188
|
+
*
|
|
189
|
+
* ponytail: the walks come to a line's length for each client it names, so a line naming
|
|
190
|
+
* every one of them in turn costs ten times a plain one. Finding each client's flags once,
|
|
191
|
+
* outside the pattern, is the way up if a stranger's session makes that matter.
|
|
56
192
|
*/
|
|
57
|
-
const
|
|
193
|
+
const onClient = (client: string, flag: string): string => String.raw`(?:${client})\b(?:(?!\b(?<![.-])(?:${client})\b)[^\n;|]){0,200}?\s${flag}`;
|
|
194
|
+
/** The next word looks generated, for where what stands in front of it says too little. */
|
|
195
|
+
const GENERATED_WORD = String.raw`(?=["']?[^\s"'/]{0,256}?${GENERATED_RUN})`;
|
|
196
|
+
/**
|
|
197
|
+
* Where a client takes a password, which says nothing anywhere else: `-p` is a port to
|
|
198
|
+
* `ssh`, a directory's parents to `mkdir`, and to `mysql` with a space after it only a
|
|
199
|
+
* prompt. A name after `.` or `-` is a file's or another tool's (`wget a.zip -P out`),
|
|
200
|
+
* and `--name mongo -p 27017:27017` is a container, so to that family `-p` counts
|
|
201
|
+
* behind `-u`. What a here-string feeds is anyone's, so that word has to look generated.
|
|
202
|
+
*
|
|
203
|
+
* A walk that a text can start again and again stops where the next one starts, which
|
|
204
|
+
* finds the same from there: a text of `mongo -u a ` is otherwise thousands of steps for
|
|
205
|
+
* each word in it, and a text of `<` hundreds for each character. A `-u` with `-p` for its
|
|
206
|
+
* name finds nothing from there, so that one is walked through.
|
|
207
|
+
*
|
|
208
|
+
* ponytail: the clients people use. Another tool's own flag stays, as it does anywhere,
|
|
209
|
+
* unless a project's own script (`./x`) gets something generated on one of these flags.
|
|
210
|
+
*/
|
|
211
|
+
const USER_FIRST = "mongosh|mongo|mongodump|mongorestore|mongoexport|mongoimport|cqlsh";
|
|
212
|
+
const USER_FLAG = String.raw`\s(?:-u|--username)[=\s]`;
|
|
213
|
+
const PASSWORD_FLAG = String.raw`(?:\b(?<![.-])(?:${[
|
|
214
|
+
onClient("sshpass", String.raw`-p\s*`),
|
|
215
|
+
onClient(String.raw`(?:docker|podman)\s+login|helm\s+registry\s+login|az\s+login|twine\s+upload`, String.raw`-p\s+`),
|
|
216
|
+
String.raw`(?:${USER_FIRST})\b(?:(?!\b(?<![.-])(?:${USER_FIRST})\b)[^\n;|]){0,200}?${USER_FLAG}\s*\S+(?!\S)(?:(?!${USER_FLAG}(?!\s*-p\s))[^\n;|]){0,200}?\s-p\s+(?!\d+:\d)`,
|
|
217
|
+
onClient("redis-cli", String.raw`-a\s+`),
|
|
218
|
+
onClient("mysql|mariadb|mysqldump|mysqladmin|7z|7za|7zz", String.raw`-p(?=[^\s=])`),
|
|
219
|
+
onClient("sqlcmd|bcp|zip|unzip", String.raw`-P\s*`),
|
|
220
|
+
onClient("plink|pscp|psftp", String.raw`-pw\s+`),
|
|
221
|
+
onClient("ssh-keygen", String.raw`-N\s+`),
|
|
222
|
+
onClient("lftp", String.raw`-u\s+["']?[^\s,"']+,`),
|
|
223
|
+
onClient("smbclient", String.raw`-U\s+["']?[^\s%"']+%`),
|
|
224
|
+
onClient(String.raw`(?:https?|xh)(?=\s)`, String.raw`(?:-a|--auth)[=\s]\s*["']?[^\s:"']+:`),
|
|
225
|
+
onClient("curl", String.raw`(?:-[a-zA-Z]{0,16}b|--cookie)[=\s]\s*(?=["']?[^\s"'=]{1,256}=)`),
|
|
226
|
+
onClient("rabbitmqctl", String.raw`(?:add_user|change_password)\s+\S+\s+`),
|
|
227
|
+
onClient(String.raw`gh\s+secret\s+set`, String.raw`(?:-b|--body)[=\s]\s*`),
|
|
228
|
+
onClient(String.raw`doctl\s+auth\s+init`, String.raw`(?:-t|--access-token)[=\s]\s*`),
|
|
229
|
+
onClient(String.raw`security\s+add-\w+-password`, String.raw`-w\s+`),
|
|
230
|
+
String.raw`htpasswd\s+(?=-[a-zA-Z]*b)(?:-[a-zA-Z]*n[a-zA-Z]*\s+\S+\s+|-[a-zA-Z]+\s+\S+\s+\S+\s+)`,
|
|
231
|
+
String.raw`sqlplus\s+(?:-\S+\s+)*[^\s/@"'-][^\s/@"']*\/`,
|
|
232
|
+
// Not a second `smbclient`, which is the next command's: `apt install smbclient && smbclient -U a%b`.
|
|
233
|
+
String.raw`smbclient\s+[^\s-]\S*\s+(?!smbclient\b)`,
|
|
234
|
+
String.raw`net\s+use\s+(?:\S+\s+){1,2}\/user:\S+\s+`,
|
|
235
|
+
String.raw`vault\s+login\s+${GENERATED_WORD}`,
|
|
236
|
+
String.raw`ngrok\s+(?:config\s+add-)?authtoken\s+`,
|
|
237
|
+
String.raw`(?:poetry|composer)\s+config\s+(?:--\S+\s+)*(?:http-basic\.\S+\s+\S+|pypi-token\.\S+)\s+`,
|
|
238
|
+
String.raw`(?:aws\s+configure\s+set|(?:npm|yarn|wp)\s+config\s+set|setx|setenv|set\s+-[a-z]*x[a-z]*)\s+(?:\S*[/:])?${secretName()}\s+`,
|
|
239
|
+
String.raw`SetEnvironmentVariable\(\s*["']${secretName()}["']\s*,\s*`,
|
|
240
|
+
String.raw`echo\s+(?:-n\s+)?(?=["']?[^\s"']{1,400}["']?\s*\|[^\n;|]{0,200}?--(?:password-stdin|with-token)\b)`,
|
|
241
|
+
].join("|")})|-(?<=\s-)(?:(?:src|dest)?(?:store|key)pass\s+|Dsonar\.login=)|<<<\s*(?=["']?(?:(?!<<<)[^\s"'/]){0,256}?${GENERATED_RUN})|\.\/[\w.-](?:(?!\.\/[\w.-])[^\n;|]){0,263}?\s(?:-p|--key|--pwd)\s+${GENERATED_WORD})`;
|
|
242
|
+
/** To a closing quote named by the backreference `quote`, spaces and the other quote included. */
|
|
243
|
+
const quotedTo = (quote: string): string => String.raw`(?!\[ISY_REDACTED|\$)(?:(?!${quote})[^\n]){1,400}(?=${quote})`;
|
|
58
244
|
|
|
59
245
|
export const BUILTIN_RULES: RedactionRule[] = [
|
|
60
246
|
{
|
|
@@ -79,8 +265,10 @@ export const BUILTIN_RULES: RedactionRule[] = [
|
|
|
79
265
|
replacement: marker("openai_key"),
|
|
80
266
|
},
|
|
81
267
|
{
|
|
268
|
+
// Open at the end for a refresh token's length, so it also takes a name glued on
|
|
269
|
+
// (`ghp_…passwd=…`): the rest of that name goes too, and what it holds (NAME_TAIL).
|
|
82
270
|
type: "github_token",
|
|
83
|
-
pattern:
|
|
271
|
+
pattern: new RegExp(String.raw`\b(?:ghp|gho|ghs|ghu|ghr)_[A-Za-z0-9]{36,}\b` + NAME_TAIL, "g"),
|
|
84
272
|
replacement: marker("github_token"),
|
|
85
273
|
},
|
|
86
274
|
{
|
|
@@ -114,20 +302,59 @@ export const BUILTIN_RULES: RedactionRule[] = [
|
|
|
114
302
|
pattern: /\bnpm_[A-Za-z0-9]{36}\b/g,
|
|
115
303
|
replacement: marker("npm_token"),
|
|
116
304
|
},
|
|
305
|
+
{
|
|
306
|
+
// One pass for all of them, as a rule each is one more scan of every string: GitLab, Slack,
|
|
307
|
+
// Discord, Groq, xAI, Google's OAuth, Stripe's webhooks, SendGrid, Mailgun, Twilio, Shopify,
|
|
308
|
+
// DigitalOcean, Hugging Face, PyPI, RubyGems, Docker, Supabase, Vercel, Atlassian, Linear,
|
|
309
|
+
// Notion, Figma, Replicate, ElevenLabs, age. A fal.ai key, and a Telegram or a Discord bot
|
|
310
|
+
// token, open with an id that is public and whose shape would be read at every word of
|
|
311
|
+
// every string, so they are found from what comes after it and the id stays. A shape open at
|
|
312
|
+
// its end also takes a name glued on (`glpat-…-apikey=`), so the rest of it goes too (NAME_TAIL).
|
|
313
|
+
type: "service_token",
|
|
314
|
+
pattern: new RegExp(
|
|
315
|
+
String.raw`(?:\b(?:gl(?:pat|dt)-[\w-]{20,}|xapp-\d-[A-Za-z0-9-]{20,}|hooks\.slack\.com\/\w+\/[\w/]{20,}|discord(?:app)?\.com\/api\/webhooks\/\d+\/[\w-]{40,}` +
|
|
316
|
+
String.raw`|gsk_[A-Za-z0-9]{40,}|xai-[A-Za-z0-9]{40,}|ya29\.[\w-]{40,}|1\/\/0[\w-]{40,}|whsec_[A-Za-z0-9+/]{24,}|SG\.[\w-]{16,}\.[\w-]{16,}` +
|
|
317
|
+
String.raw`|key-[0-9a-f]{32}\b|SK[0-9a-f]{32}\b|shp(?:at|ss|ca|pa)_[0-9a-fA-F]{32}\b|do[pors]_v1_[0-9a-f]{64}\b|hf_[A-Za-z0-9]{30,}|pypi-[\w-]{50,}` +
|
|
318
|
+
String.raw`|rubygems_[0-9a-f]{48}\b|dckr_pat_[\w-]{20,}|sbp_[0-9a-f]{40}\b|vercel_[A-Za-z0-9]{20,}|ATATT[\w-]{50,}|lin_api_[A-Za-z0-9]{30,}` +
|
|
319
|
+
String.raw`|ntn_[A-Za-z0-9]{40,}|secret_[A-Za-z0-9]{40,}|figd_[\w-]{30,}|r8_[A-Za-z0-9]{30,}|sk_[0-9a-f]{48}\b|AGE-SECRET-KEY-1[A-Z0-9]{50,})` +
|
|
320
|
+
String.raw`|:(?<=\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}:)[0-9a-f]{32}\b|:(?<=\b\d{8,12}:)AA[\w-]{30,}` +
|
|
321
|
+
String.raw`|\.(?<=[\w-]{24}\.)[\w-]{6}\.(?=[\w-]{0,26}?${GENERATED_RUN})[\w-]{27,})${NAME_TAIL}`,
|
|
322
|
+
"g",
|
|
323
|
+
),
|
|
324
|
+
replacement: marker("service_token"),
|
|
325
|
+
},
|
|
117
326
|
{
|
|
118
327
|
// After every shape a bearer token can have, so a JWT or a GitHub token
|
|
119
|
-
// keeps its own label and its marker is not marked again.
|
|
328
|
+
// keeps its own label and its marker is not marked again. Twenty characters
|
|
329
|
+
// for a word alone, or eight with a digit: `Bearer authentication` is prose.
|
|
330
|
+
// Eight with a digit also take a name glued on or on the next line
|
|
331
|
+
// (`Bearer x1_password="…"`, `TOKEN_TYPE=Bearer` over `AWS_S3_SECRET=…`), so the
|
|
332
|
+
// token goes on over what that name holds.
|
|
120
333
|
type: "bearer_token",
|
|
121
|
-
pattern: /\b(Bearer\s+)(?!\[ISY_REDACTED)[A-Za-z0-9._~+/-]{20,}
|
|
334
|
+
pattern: new RegExp(/\b(Bearer\s+)(?!\[ISY_REDACTED)(?:(?=[A-Za-z._~+/-]*\d)[A-Za-z0-9._~+/-]{8,}|[A-Za-z0-9._~+/-]{20,})=*/.source + RUNS_ON, "gi"),
|
|
122
335
|
replacement: `$1${marker("bearer_token")}`,
|
|
123
336
|
},
|
|
337
|
+
{
|
|
338
|
+
// Whatever the scheme (`Basic`, `Token`, `Digest`): the header says what the
|
|
339
|
+
// word after it is, so it needs no shape. Before the assignment rule, which
|
|
340
|
+
// would stop at the scheme. A variable or a placeholder is left as it is. A
|
|
341
|
+
// backtick right after a separator opens a value (Basic secret=:credentials=`…`):
|
|
342
|
+
// stopped there, the credential was the names and left the value. That value
|
|
343
|
+
// runs on as an assigned one does, over backticks too (`…,auth=`…``), and where
|
|
344
|
+
// it ends on a name, before or after the name's separator (`Pwd="…"`), over what
|
|
345
|
+
// the name holds, as no rule after this one sees the name inside it.
|
|
346
|
+
type: "authorization",
|
|
347
|
+
pattern: new RegExp(/\b(authorization["'`]?[ \t]*[:=][ \t]*["'`]?[A-Za-z][A-Za-z0-9-]*[ \t]+)(?!\[ISY_REDACTED)[^\s"'`$<>{}()[\]]{8,}(?:(?<=[=:])`[^\s"'{}()[\]<>$]+(?:(?:(?<=[=:])|["']?\s*[=:])\s*["']?[^\s"'{}()[\]<>$]{8,})?)?/.source + RUNS_ON, "gi"),
|
|
348
|
+
replacement: `$1${marker("authorization")}`,
|
|
349
|
+
},
|
|
124
350
|
{
|
|
125
351
|
// Any scheme: a password in a URL is one whatever speaks it — rediss,
|
|
126
352
|
// amqps, https to a bare IP. The scheme is capped because `\b` lets a match
|
|
127
353
|
// start inside a dotted or dashed run: unbounded, every start rescans the
|
|
128
|
-
// rest of the run, and a long one costs minutes.
|
|
354
|
+
// rest of the run, and a long one costs minutes. The user may be empty, as
|
|
355
|
+
// Redis URLs usually write it: `redis://:password@host`.
|
|
129
356
|
type: "connection_string",
|
|
130
|
-
pattern: /\b([A-Za-z][A-Za-z0-9+.-]{0,31}):\/\/([^\s:/@]
|
|
357
|
+
pattern: /\b([A-Za-z][A-Za-z0-9+.-]{0,31}):\/\/([^\s:/@]*):([^\s/@]+)@/g,
|
|
131
358
|
replacement: `$1://$2:${marker("connection_string")}@`,
|
|
132
359
|
},
|
|
133
360
|
{
|
|
@@ -142,19 +369,88 @@ export const BUILTIN_RULES: RedactionRule[] = [
|
|
|
142
369
|
{
|
|
143
370
|
// The name may close a quote of its own: `"password": "…"`.
|
|
144
371
|
type: "assignment",
|
|
145
|
-
pattern: new RegExp(String.raw`(
|
|
146
|
-
replacement: `$1$
|
|
372
|
+
pattern: new RegExp(String.raw`(${afterName(PLAIN_NAME, CLOSED, "[=:]")}\s*["']?|${beforeQuoted(PLAIN_NAME)})${SECRET_VALUE}${RUNS_ON}`, "g"),
|
|
373
|
+
replacement: `$1${marker("assignment")}`,
|
|
147
374
|
},
|
|
148
375
|
{
|
|
149
|
-
// A camelCase name counts where it is plainly data — a quoted
|
|
150
|
-
//
|
|
151
|
-
//
|
|
376
|
+
// A camelCase name counts where it is plainly data — with a quoted value
|
|
377
|
+
// (`accessToken: "…"`), as `name=value` with no space, as `.npmrc` writes
|
|
378
|
+
// `_authToken=`, or as a log prints one when what it holds looks generated
|
|
379
|
+
// (`accessToken: aB3…`) — and not in `adminPassword = someVariable`. One look for
|
|
380
|
+
// `=` and `:`, as a commit compiles every copy of a name in here again.
|
|
381
|
+
type: "assignment",
|
|
382
|
+
pattern: new RegExp(String.raw`(${afterName(CAMEL_NAME, String.raw`(?:(?=[=])|${CLOSED}(?=:))`, "[=:]")}(?:(?<==)|\s*["'])|${beforeQuoted(CAMEL_NAME)}|${afterName(CAMEL_ENDING, "", ":")}[ \t]+(?=[^\s"']{0,256}?${GENERATED_RUN}))${SECRET_VALUE}${RUNS_ON}`, "g"),
|
|
383
|
+
replacement: `$1${marker("assignment")}`,
|
|
384
|
+
},
|
|
385
|
+
{
|
|
386
|
+
// What SQL, `.netrc` and XML have for a separator. `password '…'` only in single quotes,
|
|
387
|
+
// as SQL writes it, since `grep password "src/auth"` names none; a tag only with its
|
|
388
|
+
// close, since `bot<token>/getUpdates` is a placeholder.
|
|
152
389
|
type: "assignment",
|
|
153
390
|
pattern: new RegExp(
|
|
154
|
-
String.raw`(
|
|
391
|
+
String.raw`(\b${anyCase("password")}\s+(?:${anyCase("for")}\s+\S{1,256}\s*=\s*)?'|\b${anyCase("identified")}\s+(?:${anyCase("with")}\s+\S{1,64}\s+)?${anyCase("by")}\s+["']|\bmachine\s+\S{1,256}\s+login\s+\S{1,256}\s+password\s+["']?)${SECRET_VALUE}` +
|
|
392
|
+
String.raw`|(<${secretName()}>)${SECRET_VALUE}(?=<\/)`,
|
|
155
393
|
"g",
|
|
156
394
|
),
|
|
157
|
-
replacement: `$1$2$
|
|
395
|
+
replacement: `$1$2${marker("assignment")}`,
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
// What signs a URL, up to its next parameter.
|
|
399
|
+
type: "assignment",
|
|
400
|
+
pattern: new RegExp(String.raw`([?&](?:[xX]-(?:${anyCase("amz")}|${anyCase("goog")})-)?${anyCase("sig")}(?:${anyCase("nature")})?=)(?!\[ISY_REDACTED|\$)[^\s&"'#]{8,}`, "g"),
|
|
401
|
+
replacement: `$1${marker("assignment")}`,
|
|
402
|
+
},
|
|
403
|
+
{
|
|
404
|
+
// `--password hunter2000`, `--api-key "…"`: a name like any other, with a
|
|
405
|
+
// space where `=` goes. `--password-file /x` has a tail no secret's name has,
|
|
406
|
+
// and what starts with a dash is the next flag.
|
|
407
|
+
type: "assignment",
|
|
408
|
+
pattern: new RegExp(String.raw`(?<![A-Za-z0-9_.-])(-${secretName()})([ \t]+)(["']?)(?!-)${SECRET_VALUE}`, "g"),
|
|
409
|
+
replacement: `$1$2$3${marker("assignment")}`,
|
|
410
|
+
},
|
|
411
|
+
{
|
|
412
|
+
// From the word back to the name, as the name from its front is a scan at every capital.
|
|
413
|
+
// The word is looked through as far as the next such name, which looks on from there.
|
|
414
|
+
type: "assignment",
|
|
415
|
+
pattern: new RegExp(String.raw`${KEY_END}(["']?\]?\s*[=:]\s*["']?)(?=(?:(?!${KEY_END}\]?[=:])[^\s"']){0,256}?${GENERATED_RUN})${SECRET_VALUE}`, "g"),
|
|
416
|
+
replacement: `KEY$1${marker("assignment")}`,
|
|
417
|
+
},
|
|
418
|
+
{
|
|
419
|
+
// A cookie header's value runs to the end of the line, spaces and `;` included, or to a quote.
|
|
420
|
+
// It opens with a name and `=`, which `cookie == null` and `cookie = jar.get()` do not.
|
|
421
|
+
//
|
|
422
|
+
// After every rule that goes by a name, as this one takes whatever stands in its way:
|
|
423
|
+
// ahead of them, `--cookie=k=v --password='…'` lost the name and kept what it named. So a
|
|
424
|
+
// marker may stand for its first pair, or for that pair's name, and the rest still goes where
|
|
425
|
+
// it is a pair. A flag, a URL or the next command is none (`--cookie=cookies.txt -u "bob:…"`),
|
|
426
|
+
// and is left to the rules after this one.
|
|
427
|
+
type: "cookie",
|
|
428
|
+
pattern: /\b(cookie["'`]?[ \t]*[:=][ \t]*["'`]?)(?:(?!\[ISY_REDACTED)(?=[^\s"'`=;,$<{]{1,256}=[^=])|(?=\[ISY_REDACTED:\w+\](?:=[^=]|[ \t;]*(?!-)[^\s"'`=;,$<{\/:?]{1,256}=[^=])))[^"'`\r\n]{8,}/gi,
|
|
429
|
+
replacement: `$1${marker("cookie")}`,
|
|
430
|
+
},
|
|
431
|
+
{
|
|
432
|
+
// The quote opens before the name or after its colon, and only one of the two
|
|
433
|
+
// groups takes part, so together they are that one quote.
|
|
434
|
+
type: "assignment",
|
|
435
|
+
pattern: new RegExp(String.raw`\b(${CURL_USER_FLAG}(?:(["'])${CURL_USER}|${CURL_USER}(["'])))${quotedTo(String.raw`\2\3`)}`, "g"),
|
|
436
|
+
replacement: `$1${marker("assignment")}`,
|
|
437
|
+
},
|
|
438
|
+
{
|
|
439
|
+
type: "assignment",
|
|
440
|
+
pattern: new RegExp(String.raw`\b(${CURL_USER_FLAG}["']?${CURL_USER})(?!\[ISY_REDACTED)[^\s"'$][^\s"']*`, "g"),
|
|
441
|
+
replacement: `$1${marker("assignment")}`,
|
|
442
|
+
},
|
|
443
|
+
{
|
|
444
|
+
// One pass for both spellings, as each is a scan of every word for every client.
|
|
445
|
+
//
|
|
446
|
+
// ponytail: one pass over the text, as a second doubles what any text with a password
|
|
447
|
+
// costs and goes over all it grew by. So a flag's second use stays (`curl -b k=a -b j=b`),
|
|
448
|
+
// and so does a flag a match of another kind took the way to: `ssh-keygen -t rsa && curl
|
|
449
|
+
// -N https://x -b k=b` loses `curl` to `-N`, `plink a mysql plink -pw b` loses `-pw` to
|
|
450
|
+
// `-p`. Tracking each client's flags outside the pattern is the way up.
|
|
451
|
+
type: "assignment",
|
|
452
|
+
pattern: new RegExp(String.raw`(${PASSWORD_FLAG})(?:(["'])${quotedTo(String.raw`\2`)}|(["']?)(?!\[ISY_REDACTED|(?<=\s)-)[^\s"'$][^\s"']*)`, "g"),
|
|
453
|
+
replacement: `$1$2$3${marker("assignment")}`,
|
|
158
454
|
},
|
|
159
455
|
];
|
|
160
456
|
|
|
@@ -175,6 +471,13 @@ interface RedactContext {
|
|
|
175
471
|
rules: RedactionRule[];
|
|
176
472
|
counts: Record<string, number>;
|
|
177
473
|
envToolUseIds: Set<string>;
|
|
474
|
+
/** The last `!` command typed in Claude Code read or wrote a dotenv file. */
|
|
475
|
+
bashInputTouchedDotenv: boolean;
|
|
476
|
+
/** What dotenv files held, as far as the session showed them (`learnValues`). */
|
|
477
|
+
learned: Set<string>;
|
|
478
|
+
structures: Budget;
|
|
479
|
+
dotenvCommands: Budget;
|
|
480
|
+
added: Budget;
|
|
178
481
|
/**
|
|
179
482
|
* Mutable: a transcript names its own working directory on every record, so a
|
|
180
483
|
* caller that did not pass one still gets repository-relative paths. The
|
|
@@ -182,6 +485,8 @@ interface RedactContext {
|
|
|
182
485
|
* directories is judged from where it started.
|
|
183
486
|
*/
|
|
184
487
|
roots: PreparedRoots;
|
|
488
|
+
/** What a relative name in a path field is relative to, when not the root. */
|
|
489
|
+
base?: string;
|
|
185
490
|
/** Whether the working directory was given rather than read off a record. */
|
|
186
491
|
rootGiven: boolean;
|
|
187
492
|
}
|
|
@@ -224,42 +529,346 @@ export function applyRules(
|
|
|
224
529
|
return result;
|
|
225
530
|
}
|
|
226
531
|
|
|
532
|
+
/** `NAME=value` as a dotenv file or a shell write spells it; code puts spaces round its `=`. */
|
|
533
|
+
const DOTENV_PAIR = /(?<![\w.$-])([A-Za-z_][A-Za-z0-9_]*)=("[^"\n]*"|'[^'\n]*'|[^\s"']+)/g;
|
|
534
|
+
/** A line that is one value and nothing else: `cut -d= -f2 .env`, `node -p process.env.X`. */
|
|
535
|
+
const BARE_VALUE = /^[^\s"'{}[\]<>=]+=*$/;
|
|
536
|
+
/** A word or a number: `password`, `changeme`, `production` and `12345678` are all through normal text. */
|
|
537
|
+
const PLAIN_WORD = /^(?:[A-Z]?[a-z]+|[A-Z]+|\d+)$/;
|
|
538
|
+
const MIN_LEARNED_VALUE = 12;
|
|
539
|
+
/** ponytail: a session names far fewer. The first ones win. */
|
|
540
|
+
const MAX_LEARNED_VALUES = 500;
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* A value worth cutting wherever it turns up. Under a secret's name it only has to
|
|
544
|
+
* be no plain word. Under any other, it has to be long and not plainly public:
|
|
545
|
+
* `BASE_URL=http://localhost:3000`, `APP_DIR=/srv/app`, `HOST=docs.example.com`, a name:
|
|
546
|
+
* `MODEL=whisper-large-v3`, `APP_NAME=MyCoolApplication`, a list: `METHODS=GET,POST,PUT`,
|
|
547
|
+
* or words and spaces: `START_CMD="npm run build"`, whose going would blind whatever
|
|
548
|
+
* reads the commands. A generated value gives itself away with a sign none of those
|
|
549
|
+
* has, or twelve letters and digits in a row, both among them.
|
|
550
|
+
*
|
|
551
|
+
* ponytail: a passphrase in words, or generated in short groups (`ABCD1-EFGH2-IJKL3`),
|
|
552
|
+
* under a name nothing knows stays where it is quoted outside the read itself; telling
|
|
553
|
+
* it from a command or a name takes more than its shape.
|
|
554
|
+
*/
|
|
555
|
+
function worthLearning(name: string | undefined, value: string): boolean {
|
|
556
|
+
if (value.length < MIN_SENSITIVE_VALUE || value.startsWith("$")) return false;
|
|
557
|
+
if (value.includes(REDACTION_MARKER) || PLAIN_WORD.test(value)) return false;
|
|
558
|
+
if (name !== undefined && SENSITIVE_KEY.test(name)) return true;
|
|
559
|
+
if (value.length < MIN_LEARNED_VALUE || /\s/.test(value)) return false;
|
|
560
|
+
if (/^[~.]{0,2}\/[\w./-]*$/.test(value)) return false;
|
|
561
|
+
return !/^[\w.:/,=-]*$/.test(value) || GENERATED.test(value);
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/**
|
|
565
|
+
* What a dotenv file holds is a secret by where it sits, whatever it looks like
|
|
566
|
+
* and whatever it is called, so the rules cannot be asked to know it: `MAILER_PHRASE`
|
|
567
|
+
* names nothing they know, and the value quoted in a later sentence has no name
|
|
568
|
+
* at all. What a dotenv call read or wrote is searched for values here, and they go
|
|
569
|
+
* from the whole session at the end (`cutLearnedValues`). Only names written the way
|
|
570
|
+
* such files write them count, so that a test run under `--env-file` does not teach
|
|
571
|
+
* its log's `id=…`.
|
|
572
|
+
*
|
|
573
|
+
* ponytail: only what the session showed. A value printed by a script that read
|
|
574
|
+
* the file itself, or typed in a prompt, is known to nothing here; so is one under
|
|
575
|
+
* the floors above, which keep short values from shredding the text.
|
|
576
|
+
*/
|
|
577
|
+
function learnValues(value: unknown, context: RedactContext): void {
|
|
578
|
+
// A loop and not a call per level: what a dotenv read returned goes whatever it holds,
|
|
579
|
+
// and a result nested deeper than the stack must not take the session with it.
|
|
580
|
+
const pending = [value];
|
|
581
|
+
while (pending.length > 0) {
|
|
582
|
+
const item = pending.pop();
|
|
583
|
+
if (Array.isArray(item)) for (const inner of item) pending.push(inner);
|
|
584
|
+
else if (isObject(item)) for (const inner of Object.values(item)) pending.push(inner);
|
|
585
|
+
else if (typeof item === "string") learnText(item, context);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
function learnText(value: string, context: RedactContext): void {
|
|
590
|
+
// `printf 'A=1\nB=2\n' > .env` holds its lines as written, and `echo K=\"v\" > .env` a quoted value.
|
|
591
|
+
const text = value.replace(/\\n/g, "\n").replace(/\\(["'])/g, "$1");
|
|
592
|
+
const learn = (name: string | undefined, found: string): void => {
|
|
593
|
+
if (context.learned.size < MAX_LEARNED_VALUES && worthLearning(name, found)) context.learned.add(found);
|
|
594
|
+
};
|
|
595
|
+
for (const [, name, written] of text.matchAll(DOTENV_PAIR)) {
|
|
596
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(name!) && !SENSITIVE_KEY.test(name!)) continue;
|
|
597
|
+
if (/^["']/.test(written!)) {
|
|
598
|
+
learn(name, written!.slice(1, -1));
|
|
599
|
+
continue;
|
|
600
|
+
}
|
|
601
|
+
// A shell ends a word at an operator the value does not hold: `VALUE=x; echo "K=$VALUE"`,
|
|
602
|
+
// `echo K=x>.env`. A dotenv file may hold one in its value, so the whole goes as well.
|
|
603
|
+
const whole = written!.replace(/;$/, "");
|
|
604
|
+
learn(name, whole);
|
|
605
|
+
const end = whole.search(/[;|&<>)\\]/);
|
|
606
|
+
if (end > 0) learn(name, whole.slice(0, end));
|
|
607
|
+
}
|
|
608
|
+
for (const line of text.split("\n")) {
|
|
609
|
+
const bare = line.trim();
|
|
610
|
+
if (BARE_VALUE.test(bare) && /[A-Za-z]/.test(bare) && /\d/.test(bare)) learn(undefined, bare);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/** What says what a record is, not what anybody said: the parse reads these, and they stay. */
|
|
615
|
+
const NOT_SAID = new Set([
|
|
616
|
+
...PATH_KEYS,
|
|
617
|
+
"type",
|
|
618
|
+
"role",
|
|
619
|
+
"uuid",
|
|
620
|
+
"parentUuid",
|
|
621
|
+
"sessionId",
|
|
622
|
+
"id",
|
|
623
|
+
"tool_use_id",
|
|
624
|
+
"name",
|
|
625
|
+
"model",
|
|
626
|
+
"timestamp",
|
|
627
|
+
"version",
|
|
628
|
+
"gitBranch",
|
|
629
|
+
]);
|
|
630
|
+
|
|
631
|
+
type FindValue = (text: string, from: number, walks: Budget) => [at: number, value: string] | undefined;
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* What may be walked along values that then are not there, over a text of some length:
|
|
635
|
+
* one of 100 KB of a letter took a minute over 200 KB of that letter. Past this the text
|
|
636
|
+
* goes whole, since what could not be looked through may hold one.
|
|
637
|
+
*
|
|
638
|
+
* ponytail: a text held up against a value that keeps almost matching, like thousands of
|
|
639
|
+
* zeros against one that starts with dozens, is lost with it. Failure links if a real one is.
|
|
640
|
+
*/
|
|
641
|
+
const walksOver = (length: number): Budget => ({ left: 4096 + 8 * length });
|
|
642
|
+
|
|
643
|
+
/** Two bits up per character, so that one falls off the top after `MIN_SENSITIVE_VALUE`, which no value is shorter than. */
|
|
644
|
+
const hashedOn = (hash: number, code: number): number => ((hash << 2) ^ code) & 0xffff;
|
|
645
|
+
|
|
646
|
+
/** What some values still have in common, and how they part after it. */
|
|
647
|
+
type Branch = { common: string; value: string | undefined; parts: Map<number, Branch> };
|
|
648
|
+
|
|
649
|
+
/** Of values in sorted order, so that what the first and the last share, all do. */
|
|
650
|
+
function branchOf(values: readonly string[], depth: number): Branch {
|
|
651
|
+
const first = values[0]!;
|
|
652
|
+
const last = values.at(-1)!;
|
|
653
|
+
let end = depth;
|
|
654
|
+
while (end < first.length && first[end] === last[end]) end += 1;
|
|
655
|
+
const groups = new Map<number, string[]>();
|
|
656
|
+
for (const value of values) {
|
|
657
|
+
if (value.length > end) groups.set(value.charCodeAt(end), [...(groups.get(value.charCodeAt(end)) ?? []), value]);
|
|
658
|
+
}
|
|
659
|
+
const parts = new Map<number, Branch>();
|
|
660
|
+
for (const [code, group] of groups) parts.set(code, branchOf(group, end + 1));
|
|
661
|
+
return { common: first.slice(depth, end), value: first.length === end ? first : undefined, parts };
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
/**
|
|
665
|
+
* Finds the next of some values, the longest of those that start together, in one pass
|
|
666
|
+
* over the text whatever their number. A loop over them costs that number times the
|
|
667
|
+
* session's size; one alternation of them is slower than that below a few hundred, and
|
|
668
|
+
* does not compile when they are long.
|
|
669
|
+
*/
|
|
670
|
+
function valueFinder(values: readonly string[]): FindValue {
|
|
671
|
+
const seen = new Uint8Array(1 << 16);
|
|
672
|
+
for (const value of values) {
|
|
673
|
+
let hash = 0;
|
|
674
|
+
for (let i = 0; i < MIN_SENSITIVE_VALUE; i += 1) hash = hashedOn(hash, value.charCodeAt(i));
|
|
675
|
+
seen[hash] = 1;
|
|
676
|
+
}
|
|
677
|
+
const root = branchOf([...values].sort(), 0);
|
|
678
|
+
return (text, from, walks) => {
|
|
679
|
+
let hash = 0;
|
|
680
|
+
for (let end = from; end < text.length; end += 1) {
|
|
681
|
+
hash = hashedOn(hash, text.charCodeAt(end));
|
|
682
|
+
if (seen[hash] === 0 || end - from < MIN_SENSITIVE_VALUE - 1) continue;
|
|
683
|
+
const at = end - MIN_SENSITIVE_VALUE + 1;
|
|
684
|
+
let found: string | undefined;
|
|
685
|
+
let next = at;
|
|
686
|
+
for (let branch: Branch | undefined = root; branch !== undefined; next += 1) {
|
|
687
|
+
const { common } = branch;
|
|
688
|
+
let same = 0;
|
|
689
|
+
while (same < common.length && text.charCodeAt(next + same) === common.charCodeAt(same)) same += 1;
|
|
690
|
+
next += same;
|
|
691
|
+
if (same < common.length) break;
|
|
692
|
+
found = branch.value ?? found;
|
|
693
|
+
branch = branch.parts.get(text.charCodeAt(next));
|
|
694
|
+
}
|
|
695
|
+
walks.left -= next - at - (found?.length ?? 0);
|
|
696
|
+
if (walks.left < 0) return undefined;
|
|
697
|
+
if (found !== undefined) return [at, found];
|
|
698
|
+
}
|
|
699
|
+
return undefined;
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* What a session may cut before a text holding a learned value goes whole, as one too
|
|
705
|
+
* dear to look through does. Each cut makes the text longer and holds a string of its
|
|
706
|
+
* own until the text is done, and finding one spends no walks, so that a value repeated
|
|
707
|
+
* through a few gzipped kilobytes is what the heap has to hold. The most in a real
|
|
708
|
+
* session is 5271.
|
|
709
|
+
*
|
|
710
|
+
* ponytail: one count for a session, so that past it a text with a single hit is lost
|
|
711
|
+
* with the rest. Per-text limits if a real session ever comes near.
|
|
712
|
+
*/
|
|
713
|
+
const MOST_CUTS = 1_000_000;
|
|
714
|
+
|
|
715
|
+
function withoutValues(value: unknown, find: FindValue, context: RedactContext, key?: string): unknown {
|
|
716
|
+
if (typeof value === "string") {
|
|
717
|
+
if (key !== undefined && NOT_SAID.has(key)) return value;
|
|
718
|
+
let text = "";
|
|
719
|
+
let from = 0;
|
|
720
|
+
const walks = walksOver(value.length);
|
|
721
|
+
for (let hit = find(value, 0, walks); hit !== undefined; hit = find(value, from, walks)) {
|
|
722
|
+
if ((context.counts.dotenv ?? 0) >= MOST_CUTS) {
|
|
723
|
+
count(context, "oversized", 1);
|
|
724
|
+
return marker("oversized");
|
|
725
|
+
}
|
|
726
|
+
text += value.slice(from, hit[0]) + marker("dotenv");
|
|
727
|
+
from = hit[0] + hit[1].length;
|
|
728
|
+
count(context, "dotenv", 1);
|
|
729
|
+
}
|
|
730
|
+
if (walks.left < 0) {
|
|
731
|
+
count(context, "oversized", 1);
|
|
732
|
+
return marker("oversized");
|
|
733
|
+
}
|
|
734
|
+
return text + value.slice(from);
|
|
735
|
+
}
|
|
736
|
+
if (Array.isArray(value)) return value.map((item) => withoutValues(item, find, context));
|
|
737
|
+
if (!isObject(value)) return value;
|
|
738
|
+
const result: Record<string, unknown> = {};
|
|
739
|
+
for (const [entryKey, entry] of Object.entries(value)) result[entryKey] = withoutValues(entry, find, context, entryKey);
|
|
740
|
+
return result;
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
/**
|
|
744
|
+
* After the whole session, because a value can be quoted long before the read that
|
|
745
|
+
* shows what it is. A line is parsed only if it holds one, and the cut is made on
|
|
746
|
+
* its strings, as one made in the JSON could land across an escape.
|
|
747
|
+
*/
|
|
748
|
+
function cutLearnedValues(lines: string[], context: RedactContext): string[] {
|
|
749
|
+
if (context.learned.size === 0) return lines;
|
|
750
|
+
// Longest first: a value inside another must not leave the rest of that one.
|
|
751
|
+
const values = [...context.learned].sort((a, b) => b.length - a.length);
|
|
752
|
+
const inText = valueFinder(values);
|
|
753
|
+
const inLine = valueFinder(values.map((value) => JSON.stringify(value).slice(1, -1)));
|
|
754
|
+
return lines.map((line) => {
|
|
755
|
+
const walks = walksOver(line.length);
|
|
756
|
+
if (inLine(line, 0, walks) === undefined && walks.left >= 0) return line;
|
|
757
|
+
try {
|
|
758
|
+
return JSON.stringify(withoutValues(JSON.parse(line), inText, context));
|
|
759
|
+
} catch {
|
|
760
|
+
return withoutValues(line, inText, context) as string;
|
|
761
|
+
}
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
/**
|
|
766
|
+
* How code reaches the environment, which reads like a file called `process.env`.
|
|
767
|
+
*
|
|
768
|
+
* ponytail: the objects it is usually read from. Any other's `.env` in a one-liner
|
|
769
|
+
* costs that one command's result.
|
|
770
|
+
*/
|
|
771
|
+
const CODE_ENV = /(?:^|\.)(?:process|meta|this|self|globalThis|window|Deno|Bun|ctx|c|os|context|event|req|request)\.env$/;
|
|
772
|
+
|
|
773
|
+
/** `.env`, `.env.local`, and the other way of naming one: `prod.env`, `.x.env`, `$STAGE.env`. */
|
|
227
774
|
function isDotenvPath(filePath: unknown): boolean {
|
|
228
|
-
|
|
775
|
+
if (typeof filePath !== "string") return false;
|
|
776
|
+
const name = basename(filePath);
|
|
777
|
+
return name.startsWith(".env") || (/^[\w.*?${}-]*\.env$/.test(name) && !CODE_ENV.test(name));
|
|
229
778
|
}
|
|
230
779
|
|
|
231
|
-
/**
|
|
232
|
-
|
|
233
|
-
|
|
780
|
+
/**
|
|
781
|
+
* Commands that can name a dotenv file without printing any of it. The list runs
|
|
782
|
+
* this way round because the other one has no end: `cut`, `node -e`, `python -c`,
|
|
783
|
+
* `docker exec … cat`, anything behind `ssh` or `timeout`.
|
|
784
|
+
*/
|
|
785
|
+
const SHELL_NO_READ =
|
|
786
|
+
/^\s*(?:(?:sudo|if|then|elif|else|while|until|do)\s+|[!{(]\s*)*(?:ls|ll|stat|file|wc|test|test-path|\[\[?|echo|printf|touch|chmod|chown|mkdir|rm|mv|cp|ln|cd|git\s+(?:add|rm|mv|status|commit|check-ignore|ls-files))(?:\s|$)/i;
|
|
787
|
+
|
|
788
|
+
/** Where a shell line, or the code it hands an interpreter, ends one word and starts the next. */
|
|
789
|
+
const WORD_BREAK = /[\s"'`(){}=,;|&<>]+/;
|
|
234
790
|
|
|
235
791
|
/**
|
|
236
|
-
* `cat .env`, `
|
|
237
|
-
*
|
|
238
|
-
*
|
|
239
|
-
* what is in it.
|
|
792
|
+
* `cat .env`, `cut -d= -f2 config/.env.local`, `node -e "…readFileSync('.env')…"`,
|
|
793
|
+
* `source .env`: a shell call that names a dotenv file is as much a read of it as
|
|
794
|
+
* `Read {file_path}` is, and what it prints may be any of the file. The command
|
|
795
|
+
* itself stays — it names the file, not what is in it. A revision goes with its
|
|
796
|
+
* word: `git show origin/main:.env`.
|
|
797
|
+
*
|
|
798
|
+
* ponytail: by the name in the command. `node scripts/x.js` opening the file
|
|
799
|
+
* itself, or `load_dotenv()` with no path, still goes up; scrubbing the file's
|
|
800
|
+
* values at upload is the way up.
|
|
240
801
|
*/
|
|
241
802
|
function readsDotenv(command: unknown): boolean {
|
|
242
803
|
if (typeof command !== "string" || !command.includes(".env")) return false;
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
804
|
+
const names = (text: string): boolean => text.split(WORD_BREAK).some((word) => isDotenvPath(word.replace(/^.*:/, "")));
|
|
805
|
+
const text = withoutHeredocs(command);
|
|
806
|
+
// What a substitution or `xargs` turns up goes to whatever takes it: `cat $(ls .env*)`.
|
|
807
|
+
if ((text.match(/\$\([^()]*\)/g) ?? []).some(names) || (/\|\s*xargs\b/.test(text) && names(text))) return true;
|
|
808
|
+
// A backtick or a `$(` left in a segment still runs what it holds.
|
|
809
|
+
return commandSegments(text).some((segment) => (!SHELL_NO_READ.test(segment) || /`|\$\(/.test(segment)) && names(segment));
|
|
248
810
|
}
|
|
249
811
|
|
|
250
|
-
/** Commands that write the file they are given: `tee
|
|
251
|
-
const SHELL_WRITER = /^\s*(?:sudo\s+)?(?:tee|set-content|sc|add-content|ac|out-file)(?:\s|$)/i;
|
|
812
|
+
/** Commands that write the file they are given: `tee`, PowerShell's own, `dd of=`, and `cp` from stdin. */
|
|
813
|
+
const SHELL_WRITER = /^\s*(?:sudo\s+)?(?:tee|set-content|sc|add-content|ac|out-file|new-item|dd|(?:cp|install)(?=\s.*\/dev\/stdin))(?:\s|$)/i;
|
|
814
|
+
/** `tee` behind whatever runs it — `sudo -u app`, `doas`, `ssh host 'tee …'`, `/usr/bin/` — still writes. */
|
|
815
|
+
const TEE = /(?:^|[\s/"'])(?:tee|tee-object)(?=\s|$)/i;
|
|
252
816
|
|
|
253
|
-
|
|
817
|
+
/** A word as the shell hands it over: quotes gone, `\ ` a plain space; a Windows `\` stays. */
|
|
818
|
+
const unquoted = (word: string): string => word.replace(/\\( )|["'`]/g, "$1");
|
|
254
819
|
|
|
255
|
-
/**
|
|
820
|
+
/** A quoted or escaped word keeps its spaces: `> "My Projects/.env"`. */
|
|
821
|
+
const REDIRECT = String.raw`>\s*((?:"[^"]*"|'[^']*'|\\.|[^\s;&|<>"'\\])+)`;
|
|
822
|
+
const ANY_REDIRECT = new RegExp(REDIRECT, "g");
|
|
823
|
+
/**
|
|
824
|
+
* Quotes go first and run to the end if they never close, so nothing is scanned twice;
|
|
825
|
+
* ahead of them what a backslash escapes, which opens nothing: `echo don\'t > .env`.
|
|
826
|
+
*/
|
|
827
|
+
const REDIRECT_PAST_QUOTES = new RegExp(String.raw`\\.|'[^']*(?:'|$)|"(?:[^"\\]|\\[\s\S]?)*(?:"|$)|${REDIRECT}`, "g");
|
|
828
|
+
/**
|
|
829
|
+
* Commands whose quoted arguments are only text: `git commit -m "x > .env"` writes
|
|
830
|
+
* nothing. What runs its argument is not here, as `sh -c`, `ssh` and `sudo` hold a
|
|
831
|
+
* real write in those quotes.
|
|
832
|
+
*/
|
|
833
|
+
const TEXT_ARGUMENTS = /^\s*(?:echo|printf|grep|egrep|fgrep|rg|git\s+(?:commit|tag|log|grep)|gh)(?:\s|$)/i;
|
|
834
|
+
|
|
835
|
+
/** `git commit -m "$(cat <<'EOF'`: what such a heredoc holds is a message. */
|
|
836
|
+
const TAKES_MESSAGE = /^\s*(?:git\s+(?:commit|tag)|gh)\s/;
|
|
837
|
+
|
|
838
|
+
/** A path as a call's first argument, in quotes a shell may have had to escape. */
|
|
839
|
+
const PATH_ARGUMENT = String.raw`\(\s{0,8}\\?["'\`]([^"'\`\\\n]{1,256})\\?["'\`]`;
|
|
840
|
+
/**
|
|
841
|
+
* What a one-liner calls to write a file it names: `fs.writeFileSync('.env', …)`,
|
|
842
|
+
* `Path('.env').write_text(…)`, and `open('.env', 'w')`, as `open` alone only reads.
|
|
843
|
+
*
|
|
844
|
+
* ponytail: a literal path, in the usual calls. A path in the script's own variable,
|
|
845
|
+
* or a call not listed, leaves the one-liner as it was; parsing each language is the
|
|
846
|
+
* way up.
|
|
847
|
+
*/
|
|
848
|
+
const WRITE_CALL = new RegExp(
|
|
849
|
+
String.raw`\b(?:(?:write|append)File(?:Sync)?|createWriteStream|file_put_contents|(?:File|IO)\.write)${PATH_ARGUMENT}` +
|
|
850
|
+
String.raw`|\bf?open${PATH_ARGUMENT}\s{0,8},\s{0,8}(?:mode\s{0,8}=\s{0,8})?\\?["'][a-z]{0,2}[wax+]` +
|
|
851
|
+
String.raw`|\bPath${PATH_ARGUMENT}\s{0,8}\)\.write_`,
|
|
852
|
+
"g",
|
|
853
|
+
);
|
|
854
|
+
|
|
855
|
+
/** The dotenv file a statement redirects, tees or opens to write, if it writes one. */
|
|
256
856
|
function dotenvTarget(statement: string): string | undefined {
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
857
|
+
// `>|` overrides `noclobber` and is no pipe.
|
|
858
|
+
for (const segment of commandSegments(statement.replace(/>\|/g, ">"))) {
|
|
859
|
+
const text = TEXT_ARGUMENTS.test(segment);
|
|
860
|
+
for (const redirect of segment.matchAll(text ? REDIRECT_PAST_QUOTES : ANY_REDIRECT)) {
|
|
861
|
+
if (redirect[1] !== undefined && isDotenvPath(unquoted(redirect[1]))) return unquoted(redirect[1]);
|
|
862
|
+
}
|
|
863
|
+
if (text) continue;
|
|
864
|
+
for (const call of segment.matchAll(WRITE_CALL)) {
|
|
865
|
+
const path = call[1] ?? call[2] ?? call[3];
|
|
866
|
+
if (isDotenvPath(path)) return path;
|
|
260
867
|
}
|
|
261
|
-
|
|
262
|
-
|
|
868
|
+
const writer = SHELL_WRITER.exec(segment) ?? TEE.exec(segment);
|
|
869
|
+
if (!writer) continue;
|
|
870
|
+
const after = segment.slice(writer.index + writer[0].length);
|
|
871
|
+
const named = after.trim().split(/\s+/).map((word) => unquoted(word).replace(/^of=/, "")).find(isDotenvPath);
|
|
263
872
|
if (named) return named;
|
|
264
873
|
}
|
|
265
874
|
return undefined;
|
|
@@ -275,8 +884,10 @@ function statements(line: string): { parts: string[]; open: boolean } {
|
|
|
275
884
|
let quote: string | undefined;
|
|
276
885
|
for (let i = 0; i < line.length; i += 1) {
|
|
277
886
|
const char = line[i]!;
|
|
278
|
-
if (quote !==
|
|
279
|
-
|
|
887
|
+
if (char === "\\" && quote !== "'") {
|
|
888
|
+
i += 1;
|
|
889
|
+
} else if (quote !== undefined) {
|
|
890
|
+
if (char === quote) quote = undefined;
|
|
280
891
|
} else if (char === '"' || char === "'" || char === "`") {
|
|
281
892
|
quote = char;
|
|
282
893
|
} else if (char === ";" || ((char === "&" || char === "|") && line[i + 1] === char)) {
|
|
@@ -290,8 +901,68 @@ function statements(line: string): { parts: string[]; open: boolean } {
|
|
|
290
901
|
return { parts, open: quote !== undefined };
|
|
291
902
|
}
|
|
292
903
|
|
|
293
|
-
/**
|
|
294
|
-
|
|
904
|
+
/** `VALUE=x`, then `echo "K=$VALUE" > .env`: the value sits where the write only names it. */
|
|
905
|
+
function expandsAssigned(statement: string, assigned: ReadonlySet<string>): boolean {
|
|
906
|
+
for (const match of statement.matchAll(/\$\{?([A-Za-z_]\w*)/g)) if (assigned.has(match[1]!)) return true;
|
|
907
|
+
return false;
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
/** A write that closes a group or a substitution: `{ …; } > .env`, `(…) | tee .env`, `done > .env`. */
|
|
911
|
+
const GROUP_WRITE = /(?:[})]|\b(?:done|fi|esac))\s*(?:\d?>|&>|\|)/;
|
|
912
|
+
/** A write whose values come later or from around it: `exec 3>.env; echo K=v >&3`, `export K=v; envsubst < t > .env`. */
|
|
913
|
+
const WRITES_FROM_AROUND = /^\s*exec\b|\benvsubst\b/;
|
|
914
|
+
|
|
915
|
+
/**
|
|
916
|
+
* `F=.env`, and a loop over a list: both say what `$F` is, and that `F` was set. A quote
|
|
917
|
+
* right behind the `=` makes the value one word, one anywhere else closes something.
|
|
918
|
+
*/
|
|
919
|
+
const ASSIGNMENT = /(?<![\w$])([A-Za-z_]\w*)=("[^"]*"|'[^']*'|[^\s;&|]*)/g;
|
|
920
|
+
const FOR_LOOP = /\bfor\s+([A-Za-z_]\w*)\s+in\s([^\n;]{0,2000})/g;
|
|
921
|
+
const VARIABLE = /\$\{?([A-Za-z_]\w*)\}?/g;
|
|
922
|
+
/** `${F:-.env}`, `${F:=.env}`: nothing else in the command says what `F` is. */
|
|
923
|
+
const VARIABLE_DEFAULT = /\$\{\w+:?[-=]([^{}]*)\}/g;
|
|
924
|
+
/** What says only where a path starts or how it is named: `$HOME/.env`, `$STAGE.env`. */
|
|
925
|
+
const VARIABLE_DIR = /\$\{?\w+\}?(?:\/|(?=\.env\b))/g;
|
|
926
|
+
/** `F="$(pwd)/.env"`: what a substitution turns up says as little. */
|
|
927
|
+
const SUBSTITUTED_DIR = /\$\([^()]*\)\/?/g;
|
|
928
|
+
|
|
929
|
+
/** What sed's `a`, `i` and `c` open with, right against the text: `$aK=v`, `1iK=v`, `/^K=/cK=v`. */
|
|
930
|
+
const SED_TEXT_COMMAND = /(?<=^|['"\s])(?:\$|\d{1,12}|\/[^/\n]{0,256}\/)[ \t]{0,8}[aic]\\?[ \t]{0,8}/g;
|
|
931
|
+
|
|
932
|
+
/**
|
|
933
|
+
* An in-place editor wherever a statement runs it: behind `time`, `/usr/bin/`,
|
|
934
|
+
* `docker exec app`, in the quotes of `ssh host "…"`. Only where to read from, as the
|
|
935
|
+
* parse says whether it edits and what.
|
|
936
|
+
*
|
|
937
|
+
* ponytail: up to 16 flags ahead of its script, as any run of them is tried again from each
|
|
938
|
+
* word in it, and the usual editors. `time sed 's/a/b/' -i .env` or `sh -c "ex .env"` keeps
|
|
939
|
+
* its script, as a command nothing recognises does.
|
|
940
|
+
*/
|
|
941
|
+
const EDITOR_CALL = /(?<![\w.-])(?:g?sed|g?awk|perl|ruby)(?:\s+-\S+){0,16}?\s+(?:-[a-zA-Z0-9]*[iI]|--in-place|--include)/;
|
|
942
|
+
/** What hands an editor its files from in front of it: `find … -exec sed -i`, `… | xargs sed -i`. */
|
|
943
|
+
const FEEDS_FILES = /\s-(?:exec|ok)(?:dir)?\s|\bxargs\s/;
|
|
944
|
+
|
|
945
|
+
/**
|
|
946
|
+
* The text of an edit's script: `s|a|b|` and `s#a#b#` split where `s/a/b/` does, and
|
|
947
|
+
* `sub("a","b")`, whose values have no name to go by, at its quotes.
|
|
948
|
+
*/
|
|
949
|
+
function learnEdit(statement: string, context: RedactContext): void {
|
|
950
|
+
const script = statement.replace(SED_TEXT_COMMAND, "\n");
|
|
951
|
+
const lines = script.replaceAll(/(?<![\w-])s([^\w\s\\])/.exec(script)?.[1] ?? "/", "\n");
|
|
952
|
+
learnValues(lines, context);
|
|
953
|
+
learnValues(lines.replace(/["']/g, "\n"), context);
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
/**
|
|
957
|
+
* What a session's commands naming a dotenv file may come to, past which one goes
|
|
958
|
+
* whole. Each statement of them is held up to a dozen patterns, several times what a
|
|
959
|
+
* byte of any other text costs, on the thread that serves requests. The most in a real
|
|
960
|
+
* session is 3.1 MB, out of 552 MB, and in any other 67 KB.
|
|
961
|
+
*
|
|
962
|
+
* ponytail: what goes whole reads as no write to the parse, and teaches nothing of what
|
|
963
|
+
* it held. Raise it if sessions come near, or take redaction off that thread.
|
|
964
|
+
*/
|
|
965
|
+
const DOTENV_COMMAND_CHARS = 4 * 1024 * 1024;
|
|
295
966
|
|
|
296
967
|
/**
|
|
297
968
|
* `cat > .env << 'EOF'`, `echo "KEY=…" >> .env.local`, `… | tee .env`: a shell
|
|
@@ -301,60 +972,211 @@ const GROUP_WRITE = /[})]\s*(?:\d?>|&>|\|)/;
|
|
|
301
972
|
* itself (`python - <<PY` opening `.env`) — and the statement keeps its target,
|
|
302
973
|
* so `sed -i` beside it is still an edit and the parse still sees a write.
|
|
303
974
|
*/
|
|
304
|
-
function cutDotenvWrites(command: string): string {
|
|
975
|
+
function cutDotenvWrites(command: string, context: RedactContext): string {
|
|
305
976
|
if (!command.includes(".env")) return command;
|
|
977
|
+
if (!spend(context.dotenvCommands, command.length)) {
|
|
978
|
+
count(context, "oversized", 1);
|
|
979
|
+
return marker("oversized");
|
|
980
|
+
}
|
|
981
|
+
// Once, as a statement asking for itself would rescan the command for each one.
|
|
982
|
+
const assigned = new Set<string>();
|
|
983
|
+
const files = new Map<string, string>();
|
|
984
|
+
// ponytail: one budget for the command, as a long name used in every statement otherwise
|
|
985
|
+
// grows it past what memory holds. A write past the budget reads as no write, as one
|
|
986
|
+
// through a variable nothing knows does; a real command spends a few hundred bytes of it.
|
|
987
|
+
let room = 1 << 16;
|
|
988
|
+
const resolved = (text: string): string =>
|
|
989
|
+
text.replace(VARIABLE_DEFAULT, "$1").replace(VARIABLE, (whole, name: string) => {
|
|
990
|
+
const file = files.get(name);
|
|
991
|
+
if (file === undefined || file.length > room) return whole;
|
|
992
|
+
room -= file.length;
|
|
993
|
+
return file;
|
|
994
|
+
});
|
|
995
|
+
// ponytail: a substitution inside one, the deepest a path is usually built.
|
|
996
|
+
const unsubstituted = command.replace(SUBSTITUTED_DIR, "").replace(SUBSTITUTED_DIR, "");
|
|
997
|
+
// In the order they run, for `A=.env; B=$A`.
|
|
998
|
+
for (const [, name, list] of [...unsubstituted.matchAll(ASSIGNMENT), ...command.matchAll(FOR_LOOP)]) {
|
|
999
|
+
assigned.add(name!);
|
|
1000
|
+
const file = resolved(list!).split(/\s+/).map(unquoted).find(isDotenvPath);
|
|
1001
|
+
// A glob in the list is that file as far as a write goes.
|
|
1002
|
+
if (file !== undefined) files.set(name!, file.replace(/[*?]/g, ""));
|
|
1003
|
+
}
|
|
1004
|
+
const parsedEdit = (text: string): string | undefined =>
|
|
1005
|
+
shellEditWords(resolved(text).replace(VARIABLE_DIR, "").replace(/\\\n/g, " ")).find(isDotenvPath);
|
|
1006
|
+
const editedFile = (text: string): string | undefined => {
|
|
1007
|
+
const call = EDITOR_CALL.exec(text);
|
|
1008
|
+
const before = text.slice(0, call?.index ?? 0);
|
|
1009
|
+
// `git commit -m "sed -i …"` only speaks of one.
|
|
1010
|
+
if (call === null || TEXT_ARGUMENTS.test(before.slice(before.lastIndexOf("|") + 1))) return parsedEdit(text);
|
|
1011
|
+
return parsedEdit(text.slice(call.index)) ?? (FEEDS_FILES.test(before) ? resolved(before).split(WORD_BREAK).find(isDotenvPath) : undefined);
|
|
1012
|
+
};
|
|
306
1013
|
const lines = command.split("\n");
|
|
307
1014
|
const kept: string[] = [];
|
|
308
1015
|
let written: string | undefined;
|
|
309
1016
|
let spread = false;
|
|
310
1017
|
for (let i = 0; i < lines.length; i += 1) {
|
|
311
1018
|
let writes = false;
|
|
312
|
-
|
|
313
|
-
|
|
1019
|
+
// A backslash joins them, to the shell and so here.
|
|
1020
|
+
const first = i;
|
|
1021
|
+
while (lines[i]!.endsWith("\\") && i + 1 < lines.length) i += 1;
|
|
1022
|
+
const line = lines.slice(first, i + 1).join("\n");
|
|
1023
|
+
const { parts, open } = statements(line);
|
|
1024
|
+
// A line that goes on in the next (a trailing pipe, an open quote) is one
|
|
1025
|
+
// statement with it, and the values may sit on either.
|
|
1026
|
+
if (open || /[\\|]\s*$/.test(line)) spread = true;
|
|
314
1027
|
kept.push(
|
|
315
1028
|
parts
|
|
316
1029
|
.map((part) => {
|
|
317
|
-
|
|
318
|
-
if (
|
|
1030
|
+
// Once per statement of what can be a few hundred thousand, so no check for one that names no file.
|
|
1031
|
+
if (!part.includes(".env") && (files.size === 0 || !part.includes("$"))) return part;
|
|
1032
|
+
const indent = /^\s*/.exec(part)![0];
|
|
1033
|
+
const target = dotenvTarget(resolved(part));
|
|
1034
|
+
if (target === undefined) {
|
|
1035
|
+
// The script holds the value, and the parse needs only to see an edit.
|
|
1036
|
+
const edited = editedFile(part);
|
|
1037
|
+
if (edited === undefined) return part;
|
|
1038
|
+
writes = true;
|
|
1039
|
+
learnEdit(part, context);
|
|
1040
|
+
const edit = `sed -i ${marker("dotenv")} ${edited}`;
|
|
1041
|
+
if (expandsAssigned(part, assigned)) {
|
|
1042
|
+
spread = true;
|
|
1043
|
+
written ??= edit;
|
|
1044
|
+
}
|
|
1045
|
+
return `${indent}${edit}${part.slice(part.trimEnd().length)}`;
|
|
1046
|
+
}
|
|
1047
|
+
const write = `${marker("dotenv")} > ${target}`;
|
|
319
1048
|
writes = true;
|
|
320
|
-
written ??=
|
|
321
|
-
|
|
322
|
-
|
|
1049
|
+
written ??= write;
|
|
1050
|
+
learnValues(part, context);
|
|
1051
|
+
if (GROUP_WRITE.test(part) || WRITES_FROM_AROUND.test(part) || expandsAssigned(part, assigned)) spread = true;
|
|
1052
|
+
return `${indent}${write}`;
|
|
323
1053
|
})
|
|
324
1054
|
.join(""),
|
|
325
1055
|
);
|
|
326
|
-
const terminator = HEREDOC.exec(
|
|
1056
|
+
const terminator = HEREDOC.exec(line)?.[2];
|
|
327
1057
|
if (terminator === undefined) continue;
|
|
328
|
-
let end =
|
|
329
|
-
|
|
1058
|
+
let end = i + 1;
|
|
1059
|
+
while (end < lines.length && lines[end]!.trim() !== terminator) end += 1;
|
|
330
1060
|
const body = lines.slice(i + 1, end);
|
|
331
1061
|
if (writes) {
|
|
1062
|
+
learnValues(body.join("\n"), context);
|
|
1063
|
+
if (expandsAssigned(body.join("\n"), assigned)) spread = true;
|
|
332
1064
|
i = end;
|
|
333
1065
|
continue;
|
|
334
1066
|
}
|
|
335
|
-
const
|
|
336
|
-
|
|
1067
|
+
const message = commandSegments(parts.find((part) => HEREDOC.test(part))!).every((segment) => TAKES_MESSAGE.test(segment));
|
|
1068
|
+
// A shell or an interpreter fed a script that writes the file: what the script
|
|
1069
|
+
// holds is written as surely as what `cat > .env` is fed.
|
|
1070
|
+
let bodyWrites: string | undefined;
|
|
1071
|
+
for (const line of message ? [] : body) {
|
|
1072
|
+
const edited = editedFile(line);
|
|
1073
|
+
if (edited !== undefined) learnEdit(line, context);
|
|
1074
|
+
bodyWrites ??= dotenvTarget(resolved(line)) ?? edited;
|
|
1075
|
+
}
|
|
1076
|
+
if (bodyWrites !== undefined) {
|
|
1077
|
+
learnValues(body.join("\n"), context);
|
|
1078
|
+
if (expandsAssigned(body.join("\n"), assigned)) {
|
|
1079
|
+
spread = true;
|
|
1080
|
+
written ??= `${marker("dotenv")} > ${bodyWrites}`;
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
// A body is code as often as not, where `cfg.env` is a property and not a file.
|
|
1084
|
+
const namesDotenv =
|
|
1085
|
+
bodyWrites !== undefined || (!message && body.some((line) => line.split(WORD_BREAK).some((word) => basename(word).startsWith(".env"))));
|
|
1086
|
+
// One at a time: spread, a body of a few hundred thousand lines overflows the stack.
|
|
1087
|
+
for (const line of namesDotenv ? [marker("dotenv")] : body) kept.push(line);
|
|
1088
|
+
if (end < lines.length) kept.push(lines[end]!);
|
|
337
1089
|
i = end;
|
|
338
1090
|
}
|
|
1091
|
+
// A quoted script over several lines is an edit that no one of them shows.
|
|
1092
|
+
const editAcross = written === undefined && spread ? parsedEdit(command) : undefined;
|
|
1093
|
+
if (editAcross !== undefined) {
|
|
1094
|
+
learnEdit(command, context);
|
|
1095
|
+
written = `sed -i ${marker("dotenv")} ${editAcross}`;
|
|
1096
|
+
}
|
|
339
1097
|
// A group or a quoted string over several lines holds the values outside the
|
|
340
1098
|
// statement that names the file (`{ echo A=1; echo B=2; } > .env`), so the
|
|
341
1099
|
// whole command goes, and only the write is left for the parse to see.
|
|
342
|
-
if (written !== undefined && spread)
|
|
1100
|
+
if (written !== undefined && spread) {
|
|
1101
|
+
learnValues(command, context);
|
|
1102
|
+
return written;
|
|
1103
|
+
}
|
|
343
1104
|
return kept.join("\n");
|
|
344
1105
|
}
|
|
345
1106
|
|
|
1107
|
+
/** A multi-edit keeps its steps: each is one edit to the parse, and only their strings go. */
|
|
346
1108
|
function cutDotenvInput(input: Record<string, unknown>, context: RedactContext): void {
|
|
347
1109
|
for (const field of EDIT_INPUT_FIELDS) {
|
|
348
1110
|
if (typeof input[field] === "string") {
|
|
1111
|
+
learnValues(input[field], context);
|
|
349
1112
|
input[field] = marker("dotenv");
|
|
350
1113
|
count(context, "dotenv", 1);
|
|
351
1114
|
}
|
|
352
1115
|
}
|
|
1116
|
+
if (!Array.isArray(input.edits)) return;
|
|
1117
|
+
for (const edit of input.edits) if (isObject(edit)) cutDotenvInput(edit, context);
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
const FILE_ARGUMENT = /^(?:file_?path|filePath|notebook_path|(?:relative_)?path|paths|file|files|filename|target_file)$/;
|
|
1121
|
+
|
|
1122
|
+
/**
|
|
1123
|
+
* Whether a call that is no shell works on a dotenv file, by the argument that
|
|
1124
|
+
* says what it works on: `Read {file_path}`, `Grep {path}`, a filesystem
|
|
1125
|
+
* server's `read_file {path}`. A glob reads a gitignored file as well as a named
|
|
1126
|
+
* one, however it is spelled (`{.env,.env.local}`, `*.env`), so any mention
|
|
1127
|
+
* counts, and the worst of that is one search's result on `*.envelope.ts`. Code
|
|
1128
|
+
* a tool runs (`REPL {code}`, `exec {script}`) is read as a shell line is.
|
|
1129
|
+
*
|
|
1130
|
+
* ponytail: by what the call names. A search that names no file (`Grep
|
|
1131
|
+
* {pattern}`, `grep -r KEY .`) still goes up when `.env` is not ignored; reading
|
|
1132
|
+
* its result's lines for a dotenv path is the way up.
|
|
1133
|
+
*/
|
|
1134
|
+
function callNamesDotenv(name: unknown, input: Record<string, unknown>): boolean {
|
|
1135
|
+
if (typeof input.glob === "string" && input.glob.includes(".env")) return true;
|
|
1136
|
+
// A workflow's script hands out prompts: it reads nothing by itself.
|
|
1137
|
+
if (readsDotenv(input.code) || readsDotenv(input.cmd) || (name !== "Workflow" && readsDotenv(input.script))) return true;
|
|
1138
|
+
return Object.entries(input).some(([key, value]) => FILE_ARGUMENT.test(key) && [value].flat().some(isDotenvPath));
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
/**
|
|
1142
|
+
* `! cat .env` typed in Claude Code is no tool call: the command comes as one
|
|
1143
|
+
* user turn, `<bash-input>…</bash-input>`, and what it printed as the next,
|
|
1144
|
+
* `<bash-stdout>…`. The cuts of a shell call apply across the pair.
|
|
1145
|
+
*/
|
|
1146
|
+
function scanBashMode(message: Record<string, unknown>, content: string, context: RedactContext): void {
|
|
1147
|
+
const input = /^\s*<bash-input>([\s\S]*)<\/bash-input>\s*$/.exec(content)?.[1];
|
|
1148
|
+
if (input !== undefined) {
|
|
1149
|
+
const cut = cutDotenvWrites(input, context);
|
|
1150
|
+
context.bashInputTouchedDotenv = cut !== input || readsDotenv(input);
|
|
1151
|
+
if (cut !== input) {
|
|
1152
|
+
message.content = `<bash-input>${cut}</bash-input>`;
|
|
1153
|
+
count(context, "dotenv", 1);
|
|
1154
|
+
}
|
|
1155
|
+
return;
|
|
1156
|
+
}
|
|
1157
|
+
if (!context.bashInputTouchedDotenv || !content.trimStart().startsWith("<bash-stdout>")) return;
|
|
1158
|
+
context.bashInputTouchedDotenv = false;
|
|
1159
|
+
learnValues(content.replace(/<\/?bash-std(?:out|err)>/g, "\n"), context);
|
|
1160
|
+
message.content = marker("dotenv");
|
|
1161
|
+
count(context, "dotenv", 1);
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
/** Where a background command's output also lands: `tail -f …/tasks/<id>.output`, `Read {file_path}`. */
|
|
1165
|
+
const TASK_OUTPUT = /(?<![\w-])([\w-]+)\.output\b/g;
|
|
1166
|
+
|
|
1167
|
+
/** Whether a call goes on with a cut background command, by stopping it or reading what it printed. */
|
|
1168
|
+
function followsCutTask(input: Record<string, unknown>, context: RedactContext): boolean {
|
|
1169
|
+
const tasks = [input.task_id, input.shell_id, input.bash_id];
|
|
1170
|
+
for (const text of [input.file_path, input.command]) {
|
|
1171
|
+
if (typeof text === "string") tasks.push(...[...text.matchAll(TASK_OUTPUT)].map((match) => match[1]));
|
|
1172
|
+
}
|
|
1173
|
+
return tasks.some((task) => typeof task === "string" && context.envToolUseIds.has(task));
|
|
353
1174
|
}
|
|
354
1175
|
|
|
355
1176
|
function scanDotenv(record: Record<string, unknown>, context: RedactContext): void {
|
|
356
1177
|
const message = record.message;
|
|
357
1178
|
const content = isObject(message) ? message.content : undefined;
|
|
1179
|
+
if (isObject(message) && typeof content === "string") return scanBashMode(message, content, context);
|
|
358
1180
|
if (!Array.isArray(content)) return;
|
|
359
1181
|
|
|
360
1182
|
for (const block of content) {
|
|
@@ -363,28 +1185,33 @@ function scanDotenv(record: Record<string, unknown>, context: RedactContext): vo
|
|
|
363
1185
|
if (block.type === "tool_use") {
|
|
364
1186
|
const input = isObject(block.input) ? block.input : undefined;
|
|
365
1187
|
if (!input) continue;
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
1188
|
+
const command = input.command;
|
|
1189
|
+
const cut = typeof command === "string" ? cutDotenvWrites(command, context) : command;
|
|
1190
|
+
if (cut !== command) {
|
|
1191
|
+
input.command = cut;
|
|
1192
|
+
count(context, "dotenv", 1);
|
|
1193
|
+
}
|
|
1194
|
+
const named = callNamesDotenv(block.name, input);
|
|
1195
|
+
if (named) cutDotenvInput(input, context);
|
|
1196
|
+
if ((named || cut !== command || readsDotenv(command) || followsCutTask(input, context)) && typeof block.id === "string") {
|
|
1197
|
+
context.envToolUseIds.add(block.id);
|
|
376
1198
|
}
|
|
377
|
-
if (!isDotenvPath(input.file_path) && !isDotenvPath(input.notebook_path)) continue;
|
|
378
|
-
if (typeof block.id === "string") context.envToolUseIds.add(block.id);
|
|
379
|
-
cutDotenvInput(input, context);
|
|
380
1199
|
}
|
|
381
1200
|
|
|
382
1201
|
if (block.type === "tool_result") {
|
|
383
1202
|
const id = block.tool_use_id;
|
|
384
1203
|
if (typeof id !== "string" || !context.envToolUseIds.has(id)) continue;
|
|
1204
|
+
// A command sent to the background goes on as a task: stopping that repeats
|
|
1205
|
+
// the command, and reading it prints what the command printed.
|
|
1206
|
+
const task = isObject(record.toolUseResult) ? record.toolUseResult.backgroundTaskId : undefined;
|
|
1207
|
+
if (typeof task === "string") context.envToolUseIds.add(task);
|
|
1208
|
+
learnValues(block.content, context);
|
|
1209
|
+
// A tool can say "done" in its text and return the file as data.
|
|
1210
|
+
learnValues(record.mcpMeta, context);
|
|
385
1211
|
block.content = marker("dotenv");
|
|
386
1212
|
count(context, "dotenv", 1);
|
|
387
1213
|
if (record.toolUseResult !== undefined) {
|
|
1214
|
+
learnValues(record.toolUseResult, context);
|
|
388
1215
|
record.toolUseResult = marker("dotenv");
|
|
389
1216
|
count(context, "dotenv", 1);
|
|
390
1217
|
}
|
|
@@ -392,20 +1219,64 @@ function scanDotenv(record: Record<string, unknown>, context: RedactContext): vo
|
|
|
392
1219
|
}
|
|
393
1220
|
}
|
|
394
1221
|
|
|
1222
|
+
/**
|
|
1223
|
+
* Codex and Kimi pass a tool the name the agent typed, relative to where the
|
|
1224
|
+
* session stood. Read from a root above that it would be another file, so it is
|
|
1225
|
+
* made whole here and masking spells it from the root like any other.
|
|
1226
|
+
*/
|
|
1227
|
+
function fromBase(value: string, base: string | undefined): string {
|
|
1228
|
+
if (base === undefined || value.length === 0) return value;
|
|
1229
|
+
if (looksAbsolute(value) || value.startsWith("~") || value.includes("://")) return value;
|
|
1230
|
+
return resolve(base, value);
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
/**
|
|
1234
|
+
* What redaction may add to a session, in characters, net of what it takes off. A marker
|
|
1235
|
+
* is longer than a short value, so `7z -pa ` comes to four times its length, and an upload
|
|
1236
|
+
* of 400 KB outgrew the heap, on the cut itself as well as on all it kept.
|
|
1237
|
+
*
|
|
1238
|
+
* ponytail: one count for a session, so past it any text with a hit goes whole, however
|
|
1239
|
+
* little it holds. A limit per text if a real session ever comes near.
|
|
1240
|
+
*/
|
|
1241
|
+
const ADDED_CHARS = 16 * 1024 * 1024;
|
|
1242
|
+
/** One, as millions of short texts past the limit would each pay for a stack trace. */
|
|
1243
|
+
const ADDS_TOO_MUCH = new RangeError("adds more than is left");
|
|
1244
|
+
|
|
1245
|
+
/**
|
|
1246
|
+
* The rules, then the paths. A pattern that runs out of backtrack stack on a run of
|
|
1247
|
+
* millions of characters reports it as a RangeError, and then nothing says how much of
|
|
1248
|
+
* the text is a secret: all of it goes, as the session around it can still go out. The
|
|
1249
|
+
* same goes for a text that would take the session past `ADDED_CHARS`.
|
|
1250
|
+
*/
|
|
1251
|
+
function scrubbed(text: string, context: RedactContext, key?: string): string {
|
|
1252
|
+
try {
|
|
1253
|
+
const secretsGone = applyRules(text, context.rules, (type, amount) => {
|
|
1254
|
+
count(context, type, amount);
|
|
1255
|
+
// Ahead of the cut, whose result is what the heap runs out on. A match adds at most its marker.
|
|
1256
|
+
if (amount * marker(type).length > context.added.left) throw ADDS_TOO_MUCH;
|
|
1257
|
+
});
|
|
1258
|
+
// Paths last: a secret already replaced by its marker must not then be read
|
|
1259
|
+
// as a directory, and `[ISY_REDACTED:…]` carries no slash to be read as one.
|
|
1260
|
+
const onMask = (amount: number): void => count(context, "path", amount);
|
|
1261
|
+
const clean = key && PATH_KEYS.has(key)
|
|
1262
|
+
? maskPath(fromBase(secretsGone, context.base), context.roots, onMask)
|
|
1263
|
+
: maskPaths(secretsGone, context.roots, onMask);
|
|
1264
|
+
if (spend(context.added, clean.length - text.length)) return clean;
|
|
1265
|
+
} catch (error) {
|
|
1266
|
+
if (!(error instanceof RangeError)) throw error;
|
|
1267
|
+
}
|
|
1268
|
+
count(context, "oversized", 1);
|
|
1269
|
+
return marker("oversized");
|
|
1270
|
+
}
|
|
1271
|
+
|
|
395
1272
|
function redactValue(value: unknown, context: RedactContext, key?: string): unknown {
|
|
396
1273
|
if (typeof value === "string") {
|
|
397
|
-
if (key && SENSITIVE_KEY.test(key)
|
|
1274
|
+
if (key && value.length >= MIN_SENSITIVE_VALUE && (SENSITIVE_KEY.test(key) || (KEY_FIELD.test(key) && GENERATED.test(value)))) {
|
|
398
1275
|
if (value.startsWith(`[${REDACTION_MARKER}`)) return value;
|
|
399
1276
|
count(context, "assignment", 1);
|
|
400
1277
|
return marker("assignment");
|
|
401
1278
|
}
|
|
402
|
-
|
|
403
|
-
count(context, type, amount),
|
|
404
|
-
);
|
|
405
|
-
// Paths last: a secret already replaced by its marker must not then be read
|
|
406
|
-
// as a directory, and `[ISY_REDACTED:…]` carries no slash to be read as one.
|
|
407
|
-
const mask = key && PATH_KEYS.has(key) ? maskPath : maskPaths;
|
|
408
|
-
return mask(secretsGone, context.roots, (amount) => count(context, "path", amount));
|
|
1279
|
+
return scrubbed(value, context, key);
|
|
409
1280
|
}
|
|
410
1281
|
|
|
411
1282
|
if (Array.isArray(value)) return value.map((item) => redactValue(item, context));
|
|
@@ -494,7 +1365,8 @@ function stripAttachment(record: Record<string, unknown>, context: RedactContext
|
|
|
494
1365
|
* `serverClassifierRequest` and `serverClassifierContext` are what a permission
|
|
495
1366
|
* classifier was shown, the working tree's git state included;
|
|
496
1367
|
* `wireIngestContext` and `permissionDecision` say where a call ran and what
|
|
497
|
-
* allowed it
|
|
1368
|
+
* allowed it; `mcpMeta` repeats what an MCP tool returned, as out of reach of the
|
|
1369
|
+
* cut as `wireToolInputs` is.
|
|
498
1370
|
*/
|
|
499
1371
|
const HARNESS_FIELDS = [
|
|
500
1372
|
"wireToolInputs",
|
|
@@ -502,6 +1374,7 @@ const HARNESS_FIELDS = [
|
|
|
502
1374
|
"serverClassifierContext",
|
|
503
1375
|
"wireIngestContext",
|
|
504
1376
|
"permissionDecision",
|
|
1377
|
+
"mcpMeta",
|
|
505
1378
|
];
|
|
506
1379
|
|
|
507
1380
|
function stripHarnessFields(record: Record<string, unknown>, context: RedactContext): void {
|
|
@@ -542,6 +1415,8 @@ function redactRecord(
|
|
|
542
1415
|
|
|
543
1416
|
export interface RedactOptions extends MaskRoots {
|
|
544
1417
|
extraPatterns?: readonly string[];
|
|
1418
|
+
/** Where the session stood, when that is below `cwd`: `git.ts:maskRootsFor`. */
|
|
1419
|
+
base?: string;
|
|
545
1420
|
}
|
|
546
1421
|
|
|
547
1422
|
function createContext(options: RedactOptions, invalid: string[]): RedactContext {
|
|
@@ -549,10 +1424,16 @@ function createContext(options: RedactOptions, invalid: string[]): RedactContext
|
|
|
549
1424
|
rules: [...BUILTIN_RULES, ...compileExtraRules(options.extraPatterns, invalid)],
|
|
550
1425
|
counts: {},
|
|
551
1426
|
envToolUseIds: new Set(),
|
|
1427
|
+
bashInputTouchedDotenv: false,
|
|
1428
|
+
learned: new Set(),
|
|
1429
|
+
structures: { left: STRUCTURES },
|
|
1430
|
+
dotenvCommands: { left: DOTENV_COMMAND_CHARS },
|
|
1431
|
+
added: { left: ADDED_CHARS },
|
|
552
1432
|
// The home directory is this machine's and is never in the transcript's
|
|
553
1433
|
// gift, so it is read here rather than passed by every caller.
|
|
554
1434
|
roots: resolveRoots({ cwd: options.cwd, home: options.home ?? homedir() }),
|
|
555
1435
|
rootGiven: Boolean(options.cwd),
|
|
1436
|
+
base: options.base,
|
|
556
1437
|
};
|
|
557
1438
|
}
|
|
558
1439
|
|
|
@@ -570,17 +1451,17 @@ function redactOneLine(line: string, context: RedactContext): { line: string; un
|
|
|
570
1451
|
const trimmed = line.trim();
|
|
571
1452
|
if (trimmed.length === 0) return { line: trimmed, unparsed: false };
|
|
572
1453
|
|
|
1454
|
+
// Parsing and copying it would hold twice what the parser allows itself to keep.
|
|
1455
|
+
if (!mayParse(trimmed, context.structures)) {
|
|
1456
|
+
count(context, "oversized", 1);
|
|
1457
|
+
return { line: marker("oversized"), unparsed: true };
|
|
1458
|
+
}
|
|
1459
|
+
|
|
573
1460
|
let parsed: unknown;
|
|
574
1461
|
try {
|
|
575
1462
|
parsed = JSON.parse(trimmed);
|
|
576
1463
|
} catch {
|
|
577
|
-
|
|
578
|
-
count(context, type, amount),
|
|
579
|
-
);
|
|
580
|
-
return {
|
|
581
|
-
line: maskPaths(secretsGone, context.roots, (amount) => count(context, "path", amount)),
|
|
582
|
-
unparsed: true,
|
|
583
|
-
};
|
|
1464
|
+
return { line: scrubbed(trimmed, context), unparsed: true };
|
|
584
1465
|
}
|
|
585
1466
|
|
|
586
1467
|
if (!isObject(parsed)) {
|
|
@@ -593,6 +1474,16 @@ function redactOneLine(line: string, context: RedactContext): { line: string; un
|
|
|
593
1474
|
return { line: JSON.stringify(redactRecord(parsed, context)), unparsed: false };
|
|
594
1475
|
}
|
|
595
1476
|
|
|
1477
|
+
function chargedLine(line: string, context: RedactContext): { line: string; unparsed: boolean } {
|
|
1478
|
+
const left = context.added.left;
|
|
1479
|
+
const result = redactOneLine(line, context);
|
|
1480
|
+
// What its texts did not pay for: a cut by key name, and a marker past the limit, which
|
|
1481
|
+
// is more than a text shorter than it, so that a line of such texts still grew threefold.
|
|
1482
|
+
if (spend(context.added, result.line.length - line.length - (left - context.added.left))) return result;
|
|
1483
|
+
count(context, "oversized", 1);
|
|
1484
|
+
return { line: marker("oversized"), unparsed: true };
|
|
1485
|
+
}
|
|
1486
|
+
|
|
596
1487
|
/**
|
|
597
1488
|
* A transcript's lines, redacted. With no `cwd`, paths are rooted at the first
|
|
598
1489
|
* record that names one; with no `home`, at the home that directory implies,
|
|
@@ -612,13 +1503,14 @@ export function redactLines(
|
|
|
612
1503
|
|
|
613
1504
|
for (const line of lines) {
|
|
614
1505
|
total += 1;
|
|
615
|
-
const result =
|
|
1506
|
+
const result = chargedLine(line, context);
|
|
616
1507
|
if (result.line.length === 0) continue;
|
|
617
1508
|
if (result.unparsed) unparsed += 1;
|
|
618
1509
|
output.push(result.line);
|
|
619
1510
|
}
|
|
620
1511
|
|
|
621
|
-
|
|
1512
|
+
const clean = cutLearnedValues(output, context);
|
|
1513
|
+
return { lines: clean, summary: summarize(context, total, unparsed, invalid) };
|
|
622
1514
|
}
|
|
623
1515
|
|
|
624
1516
|
/**
|
|
@@ -639,13 +1531,14 @@ export async function redactLineStream(
|
|
|
639
1531
|
|
|
640
1532
|
for await (const line of lines) {
|
|
641
1533
|
total += 1;
|
|
642
|
-
const result =
|
|
1534
|
+
const result = chargedLine(line, context);
|
|
643
1535
|
if (result.line.length === 0) continue;
|
|
644
1536
|
if (result.unparsed) unparsed += 1;
|
|
645
1537
|
output.push(result.line);
|
|
646
1538
|
}
|
|
647
1539
|
|
|
648
|
-
|
|
1540
|
+
const clean = cutLearnedValues(output, context);
|
|
1541
|
+
return { lines: clean, summary: summarize(context, total, unparsed, invalid) };
|
|
649
1542
|
}
|
|
650
1543
|
|
|
651
1544
|
export function redactTranscriptFile(
|