@getmarrow/mcp 3.9.101 → 3.9.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -29
- package/dist/cli.js +64 -1005
- package/dist/cli.js.map +1 -1
- package/dist/codex-plugin-pins.d.ts +19 -0
- package/dist/codex-plugin-pins.d.ts.map +1 -0
- package/dist/codex-plugin-pins.js +43 -0
- package/dist/codex-plugin-pins.js.map +1 -0
- package/dist/codex-plugins.d.ts +34 -0
- package/dist/codex-plugins.d.ts.map +1 -0
- package/dist/codex-plugins.js +240 -0
- package/dist/codex-plugins.js.map +1 -0
- package/dist/env.d.ts +2 -0
- package/dist/env.d.ts.map +1 -1
- package/dist/env.js +229 -39
- package/dist/env.js.map +1 -1
- package/dist/hook-contract.d.ts +41 -3
- package/dist/hook-contract.d.ts.map +1 -1
- package/dist/hook-contract.js +126 -18
- package/dist/hook-contract.js.map +1 -1
- package/dist/hook-pre-action.d.ts +48 -2
- package/dist/hook-pre-action.d.ts.map +1 -1
- package/dist/hook-pre-action.js +199 -23
- package/dist/hook-pre-action.js.map +1 -1
- package/dist/hook-tool-policy.d.ts +3 -3
- package/dist/hook-tool-policy.d.ts.map +1 -1
- package/dist/hook-tool-policy.js +6 -22
- package/dist/hook-tool-policy.js.map +1 -1
- package/dist/hook.d.ts +2 -2
- package/dist/hook.d.ts.map +1 -1
- package/dist/hook.js +13 -6
- package/dist/hook.js.map +1 -1
- package/dist/host-approval-state.d.ts +9 -0
- package/dist/host-approval-state.d.ts.map +1 -1
- package/dist/host-approval-state.js +4 -0
- package/dist/host-approval-state.js.map +1 -1
- package/dist/host-approval.d.ts +6 -0
- package/dist/host-approval.d.ts.map +1 -1
- package/dist/host-approval.js +3 -1
- package/dist/host-approval.js.map +1 -1
- package/dist/host-session.d.ts +11 -0
- package/dist/host-session.d.ts.map +1 -1
- package/dist/host-session.js +49 -0
- package/dist/host-session.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +27 -19
- package/dist/index.js.map +1 -1
- package/dist/normalized-action.d.ts +29 -6
- package/dist/normalized-action.d.ts.map +1 -1
- package/dist/normalized-action.js +497 -103
- package/dist/normalized-action.js.map +1 -1
- package/dist/own-server-identity.d.ts +123 -0
- package/dist/own-server-identity.d.ts.map +1 -0
- package/dist/own-server-identity.js +1052 -0
- package/dist/own-server-identity.js.map +1 -0
- package/dist/own-tool.d.ts +87 -0
- package/dist/own-tool.d.ts.map +1 -0
- package/dist/own-tool.js +169 -0
- package/dist/own-tool.js.map +1 -0
- package/dist/redact.d.ts +23 -0
- package/dist/redact.d.ts.map +1 -1
- package/dist/redact.js +181 -8
- package/dist/redact.js.map +1 -1
- package/dist/seatbelt/classifier-pin.d.ts +15 -0
- package/dist/seatbelt/classifier-pin.d.ts.map +1 -0
- package/dist/seatbelt/classifier-pin.js +26 -0
- package/dist/seatbelt/classifier-pin.js.map +1 -0
- package/dist/seatbelt/guards.d.ts +10 -0
- package/dist/seatbelt/guards.d.ts.map +1 -0
- package/dist/seatbelt/guards.js +509 -0
- package/dist/seatbelt/guards.js.map +1 -0
- package/dist/seatbelt/hook.d.ts +72 -0
- package/dist/seatbelt/hook.d.ts.map +1 -0
- package/dist/seatbelt/hook.js +362 -0
- package/dist/seatbelt/hook.js.map +1 -0
- package/dist/seatbelt/longopts.d.ts +68 -0
- package/dist/seatbelt/longopts.d.ts.map +1 -0
- package/dist/seatbelt/longopts.js +377 -0
- package/dist/seatbelt/longopts.js.map +1 -0
- package/dist/seatbelt/mapper.d.ts +83 -0
- package/dist/seatbelt/mapper.d.ts.map +1 -0
- package/dist/seatbelt/mapper.js +10248 -0
- package/dist/seatbelt/mapper.js.map +1 -0
- package/dist/seatbelt/record.d.ts +94 -0
- package/dist/seatbelt/record.d.ts.map +1 -0
- package/dist/seatbelt/record.js +776 -0
- package/dist/seatbelt/record.js.map +1 -0
- package/dist/seatbelt/tables.d.ts +82 -0
- package/dist/seatbelt/tables.d.ts.map +1 -0
- package/dist/seatbelt/tables.js +359 -0
- package/dist/seatbelt/tables.js.map +1 -0
- package/dist/seatbelt-deploy.d.ts +100 -0
- package/dist/seatbelt-deploy.d.ts.map +1 -0
- package/dist/seatbelt-deploy.js +393 -0
- package/dist/seatbelt-deploy.js.map +1 -0
- package/dist/seatbelt-fs.d.ts +30 -0
- package/dist/seatbelt-fs.d.ts.map +1 -0
- package/dist/seatbelt-fs.js +94 -0
- package/dist/seatbelt-fs.js.map +1 -0
- package/dist/seatbelt-git.d.ts +212 -0
- package/dist/seatbelt-git.d.ts.map +1 -0
- package/dist/seatbelt-git.js +844 -0
- package/dist/seatbelt-git.js.map +1 -0
- package/dist/seatbelt-mode.d.ts +77 -0
- package/dist/seatbelt-mode.d.ts.map +1 -0
- package/dist/seatbelt-mode.js +496 -0
- package/dist/seatbelt-mode.js.map +1 -0
- package/dist/seatbelt-path.d.ts +123 -0
- package/dist/seatbelt-path.d.ts.map +1 -0
- package/dist/seatbelt-path.js +482 -0
- package/dist/seatbelt-path.js.map +1 -0
- package/dist/seatbelt-scripts.d.ts +109 -0
- package/dist/seatbelt-scripts.d.ts.map +1 -0
- package/dist/seatbelt-scripts.js +1060 -0
- package/dist/seatbelt-scripts.js.map +1 -0
- package/dist/tool-catalog.d.ts +6852 -0
- package/dist/tool-catalog.d.ts.map +1 -0
- package/dist/tool-catalog.js +999 -0
- package/dist/tool-catalog.js.map +1 -0
- package/dist/types.d.ts +2 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -1
- package/server.json +10 -3
- package/vendor/classifier/MANIFEST.json +25 -0
- package/vendor/classifier/classifier.js +1356 -0
- package/vendor/classifier/policy-baseline.js +187 -0
- package/vendor/classifier/redact.js +59 -0
- package/vendor/classifier/shell-parse.js +455 -0
|
@@ -10,7 +10,7 @@ const node_crypto_1 = require("node:crypto");
|
|
|
10
10
|
const node_os_1 = require("node:os");
|
|
11
11
|
const hook_tool_policy_1 = require("./hook-tool-policy");
|
|
12
12
|
const SECRET = '[secret]';
|
|
13
|
-
const HASH_VERSION = 'marrow-normalized-action-
|
|
13
|
+
const HASH_VERSION = 'marrow-normalized-action-v3';
|
|
14
14
|
const EDIT_TOOLS = /^(?:edit|write|multiedit|apply_patch|notebookedit|replace|write_file|edit_file|delete_file|search_replace|delete|create_file)$/i;
|
|
15
15
|
const MAX_DEPTH = 3;
|
|
16
16
|
/** A word, data text or input string longer than this is not scanned: it is withheld and the action marked truncated. */
|
|
@@ -27,8 +27,16 @@ const DEFAULT_NORMALIZE_MS = 1_000;
|
|
|
27
27
|
/** Thrown when normalization runs out of its budget: the caller sends a truncated placeholder instead. */
|
|
28
28
|
class NormalizationLimit extends Error {
|
|
29
29
|
}
|
|
30
|
-
function newExactness(deadline = Date.now() + DEFAULT_NORMALIZE_MS) {
|
|
31
|
-
return { truncated: false, budget: { scanned: 0, deadline } };
|
|
30
|
+
function newExactness(deadline = Date.now() + DEFAULT_NORMALIZE_MS, exact = false) {
|
|
31
|
+
return { truncated: false, budget: { scanned: 0, deadline }, exact, sigs: [] };
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* A value withheld only because it MAY be a secret (it may also be an id, a commit or a name that decides the
|
|
35
|
+
* action): replaced and the action marked truncated; the exact pass keeps it as it is.
|
|
36
|
+
*/
|
|
37
|
+
function withheld(state, original, replacement = SECRET) {
|
|
38
|
+
state.truncated = true; // exact: kept
|
|
39
|
+
return state.exact ? original : replacement;
|
|
32
40
|
}
|
|
33
41
|
/** Counts text about to be scanned against the shared budget; past it, normalization stops. */
|
|
34
42
|
function spend(state, chars) {
|
|
@@ -44,6 +52,8 @@ function sha256(text) {
|
|
|
44
52
|
// ---------------------------------------------------------------------------
|
|
45
53
|
const KNOWN_KEY_PREFIX = /^(?:AKIA|ASIA|AGPA|AIDA|AROA)[0-9A-Z]{12,}$|^(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}$|^github_pat_[A-Za-z0-9_]{20,}$|^(?:sk|pk|rk)[-_](?:live|test|proj|ant)?[-_]?[A-Za-z0-9_-]{12,}$|^xox[abprs]-[A-Za-z0-9-]{10,}$|^AIza[0-9A-Za-z_-]{30,}$|^glpat-[A-Za-z0-9_-]{16,}$|^npm_[A-Za-z0-9]{20,}$|^mrw_[A-Za-z0-9_-]{8,}$|^cfut_[A-Za-z0-9_-]{12,}$|^eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}$/;
|
|
46
54
|
const PEM = /-----BEGIN [A-Z ]*(?:PRIVATE KEY|CERTIFICATE)-----/;
|
|
55
|
+
/** Known formats that only name a key or are public (AWS key, user, group and role ids; publishable keys): the exact pass keeps them. */
|
|
56
|
+
const PUBLIC_KNOWN_KEY = /^(?:AKIA|ASIA|AGPA|AIDA|AROA)[0-9A-Z]{12,}$|^pk[-_]/;
|
|
47
57
|
/** A value shaped like a key, token or password hash (mixed-case random text, long hex, known prefixes). */
|
|
48
58
|
function looksLikeKey(value) {
|
|
49
59
|
const text = value.trim();
|
|
@@ -72,18 +82,19 @@ function redactKeyShapes(text, state) {
|
|
|
72
82
|
const whole = text.trim();
|
|
73
83
|
if (whole && !/\s/.test(whole) && looksLikeKey(whole)) {
|
|
74
84
|
if (!KNOWN_KEY_PREFIX.test(whole) && !PEM.test(whole))
|
|
75
|
-
state
|
|
76
|
-
return SECRET;
|
|
85
|
+
return withheld(state, text);
|
|
86
|
+
return state.exact && PUBLIC_KNOWN_KEY.test(whole) ? text : SECRET;
|
|
77
87
|
}
|
|
78
88
|
if (PEM.test(text)) {
|
|
79
|
-
|
|
89
|
+
// The exact pass replaces only a private key block; a certificate or a public key block is kept.
|
|
90
|
+
return text.replace(/-----BEGIN ([A-Z ]*)-----[\s\S]*?(?:-----END [A-Z ]*-----|$)/g, (block, label) => (state.exact && !/PRIVATE/.test(label) ? block : SECRET));
|
|
80
91
|
}
|
|
81
92
|
return text.split(KEY_SPLIT).map((part) => {
|
|
82
93
|
if (!looksLikeKey(part))
|
|
83
94
|
return part;
|
|
84
95
|
if (!KNOWN_KEY_PREFIX.test(part))
|
|
85
|
-
state
|
|
86
|
-
return SECRET;
|
|
96
|
+
return withheld(state, part);
|
|
97
|
+
return state.exact && PUBLIC_KNOWN_KEY.test(part) ? part : SECRET;
|
|
87
98
|
}).join('');
|
|
88
99
|
}
|
|
89
100
|
// ---------------------------------------------------------------------------
|
|
@@ -129,6 +140,24 @@ function holdsSecret(name) {
|
|
|
129
140
|
const kind = classifySecretName(name);
|
|
130
141
|
return kind === 'credential' || kind === 'ambiguous';
|
|
131
142
|
}
|
|
143
|
+
/** A name part that ends in a credential word (DB_PASSWORD, PGPASSWORD, GITHUB_TOKEN, ADMINPASS), not one that only starts with it. */
|
|
144
|
+
const CERTAIN_END = /(?:password|passwd|passphrase|passcode|secret|token|apikey|accesskey|privatekey|signingkey|masterkey|encryptionkey|accountkey|credential|creds|bearer|cookie|authorization|requirepass|masterauth|storepass|keypass|newpass|oldpass|pwd)$/;
|
|
145
|
+
/**
|
|
146
|
+
* classifySecretName, except that the exact pass (exact_hmac, never sent) reads a credential name as certain only
|
|
147
|
+
* when a part ends in a credential word, is one (otp, pat, jwt), or the name holds a pair (api_key): a value under
|
|
148
|
+
* `passwordless`, `SECRETNAME`, `TOKENID`, `max_tokens` or `credentials` may name or count something that decides
|
|
149
|
+
* the action, so the exact pass keeps it (as an ambiguous value). Pass 1 is unchanged.
|
|
150
|
+
*/
|
|
151
|
+
function nameClass(name, state) {
|
|
152
|
+
const kind = classifySecretName(name);
|
|
153
|
+
if (!state.exact || kind !== 'credential')
|
|
154
|
+
return kind;
|
|
155
|
+
const parts = nameParts(name);
|
|
156
|
+
const joined = parts.join('_');
|
|
157
|
+
const pair = CREDENTIAL_PAIRS.some((item) => joined === item || joined.startsWith(`${item}_`) || joined.endsWith(`_${item}`) || joined.includes(`_${item}_`));
|
|
158
|
+
const certain = pair || parts.some((part) => CREDENTIAL_WORDS.has(part) || CERTAIN_END.test(part) || (holdsPass(part) && part.endsWith('pass')));
|
|
159
|
+
return certain ? 'credential' : 'ambiguous';
|
|
160
|
+
}
|
|
132
161
|
// ---------------------------------------------------------------------------
|
|
133
162
|
// Secrets inside a text
|
|
134
163
|
// ---------------------------------------------------------------------------
|
|
@@ -195,10 +224,10 @@ function cutCredential(value, state) {
|
|
|
195
224
|
const at = credentialValueEnd(value, 0, true);
|
|
196
225
|
if (at >= value.length) {
|
|
197
226
|
if (holdsMoreThanCredential(value))
|
|
198
|
-
state
|
|
227
|
+
return withheld(state, value);
|
|
199
228
|
return SECRET;
|
|
200
229
|
}
|
|
201
|
-
state.truncated = true;
|
|
230
|
+
state.truncated = true; // exact: certain (the credential before the field; the rest is scanned in the same pass)
|
|
202
231
|
return SECRET + redactText(value.slice(at), state);
|
|
203
232
|
}
|
|
204
233
|
/** One-value auth header (`Authorization: Bearer X`, `X-Api-Key: K`): the whole value is the credential. */
|
|
@@ -209,21 +238,29 @@ const SINGLE_VALUE_HEADER = /^[^:\s]{1,128}:[ \t]*(?:(?:Bearer|Basic|Token|Bot)[
|
|
|
209
238
|
* names mark the action truncated) and key-shaped values.
|
|
210
239
|
*/
|
|
211
240
|
function redactText(text, state) {
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
return
|
|
215
|
-
}
|
|
241
|
+
// Not scanned (the exact pass keeps it as it is).
|
|
242
|
+
if (text.length > exports.MAX_SCAN_CHARS)
|
|
243
|
+
return withheld(state, text);
|
|
216
244
|
spend(state, text.length);
|
|
217
245
|
// A short word without a separator cannot hold any of these (keys need 12 or more characters).
|
|
218
246
|
if (text.length < 12 && !/[:=?&@\s]/.test(text))
|
|
219
247
|
return text;
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
248
|
+
// M2-R1: a `#` or `?` in the password ends the URL authority for URL parsers (`https://deploy:443#@host`
|
|
249
|
+
// goes to host `deploy`), while libpq, mysql and redis clients take the whole span as the password. It is
|
|
250
|
+
// hidden either way, but it is not a certain credential: the action is marked truncated and the exact pass
|
|
251
|
+
// keeps the span, so it never shares a key with `https://deploy:<pw>@host`.
|
|
252
|
+
let out = text.replace(URL_CREDENTIALS, (_match, scheme, user, password) => (`${scheme}${user}:${/[#?]/.test(password) ? withheld(state, password) : SECRET}@`));
|
|
253
|
+
// The exact pass keeps a `Token`/`Bot` value that is not key-shaped (`git commit -m "Token rotation"`).
|
|
254
|
+
out = replaceValuesAfter(out, AUTH_SCHEME, state.exact ? (scheme, value) => /^(?:Token|Bot)$/.test(scheme) && !looksLikeKey(value) : null);
|
|
255
|
+
// The exact pass keeps a URL `code` value: it may be an OAuth code, but it may as well be a country or product code.
|
|
256
|
+
out = replaceValuesAfter(out, URL_CODE, state.exact ? () => true : null);
|
|
223
257
|
return redactKeyShapes(redactAssignments(out, state), state);
|
|
224
258
|
}
|
|
225
|
-
/**
|
|
226
|
-
|
|
259
|
+
/**
|
|
260
|
+
* Replaces the value after each match of `pattern` (an auth scheme, an OAuth code), ended by the shared rule.
|
|
261
|
+
* `keep` (exact pass only): a value it returns true for stays as it is.
|
|
262
|
+
*/
|
|
263
|
+
function replaceValuesAfter(text, pattern, keep) {
|
|
227
264
|
pattern.lastIndex = 0;
|
|
228
265
|
let out = '';
|
|
229
266
|
let copied = 0;
|
|
@@ -234,6 +271,10 @@ function replaceValuesAfter(text, pattern) {
|
|
|
234
271
|
pattern.lastIndex = start + 1;
|
|
235
272
|
continue;
|
|
236
273
|
}
|
|
274
|
+
if (keep && keep(match[1], text.slice(start, end))) {
|
|
275
|
+
pattern.lastIndex = end;
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
237
278
|
out += text.slice(copied, start) + SECRET;
|
|
238
279
|
copied = end;
|
|
239
280
|
pattern.lastIndex = end;
|
|
@@ -254,11 +295,10 @@ function redactAssignments(text, state) {
|
|
|
254
295
|
let matches = 0;
|
|
255
296
|
for (let match = pattern.exec(text); match; match = pattern.exec(text)) {
|
|
256
297
|
matches += 1;
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
return
|
|
260
|
-
|
|
261
|
-
const kind = classifySecretName(match[2]);
|
|
298
|
+
// Too many names to scan: withheld (the exact pass keeps the text as it is).
|
|
299
|
+
if (matches > MAX_ASSIGNMENTS)
|
|
300
|
+
return withheld(state, text);
|
|
301
|
+
const kind = nameClass(match[2], state);
|
|
262
302
|
if (kind !== 'credential' && kind !== 'ambiguous')
|
|
263
303
|
continue;
|
|
264
304
|
let start = match.index + match[0].length;
|
|
@@ -275,34 +315,63 @@ function redactAssignments(text, state) {
|
|
|
275
315
|
continue;
|
|
276
316
|
const value = text.slice(start, end);
|
|
277
317
|
if (!value.startsWith('[secret') && !value.startsWith('[data:')) {
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
out += text.slice(copied, start) + SECRET;
|
|
318
|
+
const replacement = kind === 'ambiguous' || holdsMoreThanCredential(value) ? withheld(state, value) : SECRET;
|
|
319
|
+
out += text.slice(copied, start) + replacement;
|
|
281
320
|
copied = end;
|
|
282
321
|
}
|
|
283
322
|
pattern.lastIndex = end;
|
|
284
323
|
}
|
|
285
324
|
return out + text.slice(copied);
|
|
286
325
|
}
|
|
287
|
-
/** Whether a text carries anything secret-shaped (it is then withheld, never hashed). */
|
|
288
|
-
function secretShaped(text, state) {
|
|
289
|
-
const probe = { truncated: false, budget: state.budget };
|
|
290
|
-
return redactText(text, probe) !== text;
|
|
291
|
-
}
|
|
292
326
|
/**
|
|
293
|
-
* Data a command reads: its SHA-256 in the hashed form
|
|
294
|
-
* truncated when it feeds a secrets command or file
|
|
327
|
+
* Data a command reads: its SHA-256 in the hashed form; `[secret]` and
|
|
328
|
+
* truncated when it feeds a secrets command or file (all of it is the
|
|
329
|
+
* secret); the SHA-256 of its redacted form and truncated when it looks secret.
|
|
295
330
|
*/
|
|
296
|
-
function dataMarker(text, feedsSecrets, state) {
|
|
297
|
-
if (
|
|
298
|
-
|
|
331
|
+
function dataMarker(text, feedsSecrets, state, words = null) {
|
|
332
|
+
if (state.exact) {
|
|
333
|
+
// The exact pass: only the certain credentials inside the data are replaced and the rest is hashed, also for
|
|
334
|
+
// data a secrets command or file reads (a vault policy, an authorized_keys line, a kubeconfig differ outside
|
|
335
|
+
// any secret), so data that differs outside a certain credential never shares exact material.
|
|
336
|
+
const probe = { truncated: false, budget: state.budget, exact: true, sigs: [] };
|
|
337
|
+
return `[data:${sha256(`${HASH_VERSION}\nexact-data\n${redactText(text, probe)}`)}]`;
|
|
338
|
+
}
|
|
339
|
+
if (feedsSecrets) {
|
|
340
|
+
state.truncated = true; // exact: kept (hashed above, certain credentials replaced)
|
|
299
341
|
return SECRET;
|
|
300
342
|
}
|
|
301
|
-
|
|
343
|
+
// Secret-shaped data (`'{"api_key":"S","env":"production"}'`): the secrets are
|
|
344
|
+
// replaced and the rest is hashed, so data that differs outside a secret gives
|
|
345
|
+
// a different key, data that differs only in a secret the same one. Withheld:
|
|
346
|
+
// the action is truncated and never binds an approval.
|
|
347
|
+
const probe = { truncated: false, budget: state.budget, exact: false, sigs: [] };
|
|
348
|
+
const redacted = redactText(text, probe);
|
|
349
|
+
if (redacted !== text) {
|
|
350
|
+
state.truncated = true; // exact: kept (hashed above, certain credentials replaced)
|
|
351
|
+
return `[secret-data:${sha256(`${HASH_VERSION}\nsecret-data\n${redacted}`)}]`;
|
|
352
|
+
}
|
|
353
|
+
// Printed words (`words`): hashed as their list, so word boundaries count.
|
|
354
|
+
return words === null ? `[data:${sha256(`${HASH_VERSION}\ndata\n${text}`)}]` : `[data:${sha256(`${HASH_VERSION}\nwords\n${JSON.stringify(words)}`)}]`;
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Text echoed or printed into a pipe or a file: probed for secrets as the shell joins it (one space), as
|
|
358
|
+
* before; hashed as the list of printed words, so word boundaries and flags count (`echo -n X` is not
|
|
359
|
+
* `echo "-n X"`, `printf '%s\n' a b` is not `printf '%s\n' 'a b'`). The exact pass replaces the certain
|
|
360
|
+
* credentials word by word and keeps the boundaries too.
|
|
361
|
+
*/
|
|
362
|
+
function printedData(words, feedsSecrets, state) {
|
|
363
|
+
if (state.exact) {
|
|
364
|
+
const probe = { truncated: false, budget: state.budget, exact: true, sigs: [] };
|
|
365
|
+
return `[data:${sha256(`${HASH_VERSION}\nexact-words\n${JSON.stringify(words.map((word) => redactText(word, probe)))}`)}]`;
|
|
366
|
+
}
|
|
367
|
+
return dataMarker(words.join(' '), feedsSecrets, state, words);
|
|
302
368
|
}
|
|
303
369
|
const OPERATORS = ['<<<', '&&', '||', '|&', '>>', '<<', ';;', '|', '&', ';', '<', '>', '(', ')', '\n'];
|
|
304
|
-
/**
|
|
305
|
-
|
|
370
|
+
/**
|
|
371
|
+
* Splits a command into words and operators, honoring quotes and escapes (an approximation of the shell).
|
|
372
|
+
* `crIsWord` (the exact pass): a CR is a word character, as it is to the shell.
|
|
373
|
+
*/
|
|
374
|
+
function tokenize(command, crIsWord = false) {
|
|
306
375
|
const tokens = [];
|
|
307
376
|
let current = '';
|
|
308
377
|
let started = false;
|
|
@@ -328,7 +397,7 @@ function tokenize(command) {
|
|
|
328
397
|
i = end < 0 ? command.length : end;
|
|
329
398
|
continue;
|
|
330
399
|
}
|
|
331
|
-
if (ch === ' ' || ch === '\t' || ch === '\r') {
|
|
400
|
+
if (ch === ' ' || ch === '\t' || (ch === '\r' && !crIsWord)) {
|
|
332
401
|
push();
|
|
333
402
|
continue;
|
|
334
403
|
}
|
|
@@ -489,10 +558,10 @@ function takesSecrets(words, parsed) {
|
|
|
489
558
|
return parsed.targets.some(isSecretFile);
|
|
490
559
|
}
|
|
491
560
|
function assignedValue(name, value, state) {
|
|
492
|
-
const kind =
|
|
561
|
+
const kind = nameClass(name, state);
|
|
493
562
|
if (kind === 'credential' || kind === 'ambiguous') {
|
|
494
563
|
if (kind === 'ambiguous' || holdsMoreThanCredential(value))
|
|
495
|
-
state.
|
|
564
|
+
return `${name}=${withheld(state, state.exact ? dataMarker(value, false, state) : value)}`;
|
|
496
565
|
return `${name}=${SECRET}`;
|
|
497
566
|
}
|
|
498
567
|
return value ? `${name}=${dataMarker(value, false, state)}` : `${name}=`;
|
|
@@ -539,7 +608,7 @@ function redactSegment(words, ctx) {
|
|
|
539
608
|
if ((program === 'echo' || program === 'printf') && (ctx.pipedOut || targets.length)) {
|
|
540
609
|
const printed = parsed.plain;
|
|
541
610
|
if (printed.length) {
|
|
542
|
-
out[printed[0]] =
|
|
611
|
+
out[printed[0]] = printedData(printed.map((i) => out[i]), downstreamTakes || secretTarget, state);
|
|
543
612
|
handled.add(printed[0]);
|
|
544
613
|
for (const i of printed.slice(1))
|
|
545
614
|
removed.add(i);
|
|
@@ -560,8 +629,7 @@ function redactSegment(words, ctx) {
|
|
|
560
629
|
// Commands inside the command: `bash -c CMD`, `ssh HOST CMD …`, `eval …`, `su -c CMD`.
|
|
561
630
|
const embed = (at, text, also = []) => {
|
|
562
631
|
if (ctx.depth >= MAX_DEPTH) {
|
|
563
|
-
out[at] =
|
|
564
|
-
state.truncated = true;
|
|
632
|
+
out[at] = withheld(state, text);
|
|
565
633
|
}
|
|
566
634
|
else {
|
|
567
635
|
out[at] = normalizeInner(text, state, ctx.depth + 1).text;
|
|
@@ -620,23 +688,31 @@ function redactSegment(words, ctx) {
|
|
|
620
688
|
const envStore = storeContext(lowered.slice(0, 4));
|
|
621
689
|
const stem = programStem(program);
|
|
622
690
|
const redactAt = (i, ambiguous = false) => {
|
|
623
|
-
|
|
624
|
-
state.truncated = true;
|
|
625
|
-
out[i] = SECRET;
|
|
691
|
+
out[i] = ambiguous || holdsMoreThanCredential(out[i]) ? withheld(state, out[i]) : SECRET;
|
|
626
692
|
handled.add(i);
|
|
627
693
|
};
|
|
628
694
|
// An auth header: a one-value header is the credential; any other (`Cookie: a=1; env=prod`)
|
|
629
695
|
// keeps its other fields, with the credentials inside replaced.
|
|
630
696
|
const redactHeaderValue = (header) => {
|
|
631
697
|
if (SINGLE_VALUE_HEADER.test(header)) {
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
698
|
+
const colon = header.indexOf(':');
|
|
699
|
+
const value = header.slice(colon + 1);
|
|
700
|
+
if (holdsMoreThanCredential(value))
|
|
701
|
+
return withheld(state, header);
|
|
702
|
+
if (!state.exact)
|
|
703
|
+
return SECRET;
|
|
704
|
+
// The exact pass: the value of a credential header (or a Bearer/Basic value) is certain; any other
|
|
705
|
+
// one-value header (`X-Idempotency-Key: k`) and the header name are kept.
|
|
706
|
+
const name = header.slice(0, colon);
|
|
707
|
+
if (nameClass(name, state) !== 'credential' && !/^[ \t]*(?:Bearer|Basic)[ \t]/i.test(value))
|
|
708
|
+
return header;
|
|
709
|
+
// Basic USER:PASSWORD names who acts, as `curl -u` does: only the password is certain.
|
|
710
|
+
const basic = /^[ \t]*Basic[ \t]+([A-Za-z0-9+/]+={0,2})[ \t]*$/i.exec(value);
|
|
711
|
+
const decoded = basic ? Buffer.from(basic[1], 'base64').toString('latin1') : '';
|
|
712
|
+
return /^[\x20-\x7e]*:/.test(decoded) ? `${name}:Basic ${JSON.stringify(decoded.slice(0, decoded.indexOf(':')))}:${SECRET}` : `${name}:${SECRET}`;
|
|
635
713
|
}
|
|
636
714
|
const redacted = redactText(header, state);
|
|
637
|
-
|
|
638
|
-
state.truncated = true;
|
|
639
|
-
return redacted === header ? SECRET : redacted;
|
|
715
|
+
return redacted === header ? withheld(state, header) : redacted;
|
|
640
716
|
};
|
|
641
717
|
// curl -b/--cookie NAME=VALUE[; …] is a Cookie header (without `=` it names a cookie file).
|
|
642
718
|
const redactCookieValue = (value) => {
|
|
@@ -651,7 +727,11 @@ function redactSegment(words, ctx) {
|
|
|
651
727
|
return redactText(`${name}=${value}`, state);
|
|
652
728
|
return `${name}=${cutCredential(value, state)}`;
|
|
653
729
|
};
|
|
654
|
-
//
|
|
730
|
+
// The exact pass: `curl -u USER:PASSWORD` names who acts; only the password is certain.
|
|
731
|
+
const exactUser = (value) => (value.includes(':') ? `${value.slice(0, value.indexOf(':'))}:${SECRET}` : value);
|
|
732
|
+
// The exact pass: openssl `pass:X` and a bare value are the password; `env:`, `file:`, `fd:` name where it is.
|
|
733
|
+
const exactOpensslPass = (value) => (/^(?:env|file|fd):/.test(value) ? value : /^pass:/.test(value) ? `pass:${SECRET}` : SECRET);
|
|
734
|
+
// A program named for a secret (set-password.sh): its arguments cannot be told apart (the exact pass keeps them).
|
|
655
735
|
if (!SECRET_PROGRAMS.has(stem) && classifySecretName(stem) === 'credential') {
|
|
656
736
|
for (const i of positionals)
|
|
657
737
|
redactAt(i, true);
|
|
@@ -666,12 +746,14 @@ function redactSegment(words, ctx) {
|
|
|
666
746
|
const j = args[k + 1];
|
|
667
747
|
return j !== undefined && !handled.has(j) && !out[j].startsWith('-') ? j : null;
|
|
668
748
|
};
|
|
669
|
-
|
|
749
|
+
// `exactForm` (exact pass only): the certain part of a value that also names something (a user, a file).
|
|
750
|
+
const next = (ambiguous = false, exactForm = null) => {
|
|
670
751
|
const j = valueAt();
|
|
671
752
|
if (j !== null) {
|
|
672
753
|
if (ambiguous)
|
|
673
|
-
state.
|
|
674
|
-
|
|
754
|
+
out[j] = state.exact ? withheld(state, out[j]) : withheld(state, out[j], cutCredential(out[j], state));
|
|
755
|
+
else
|
|
756
|
+
out[j] = state.exact && exactForm ? exactForm(out[j]) : cutCredential(out[j], state);
|
|
675
757
|
handled.add(j);
|
|
676
758
|
k += 1;
|
|
677
759
|
}
|
|
@@ -686,7 +768,7 @@ function redactSegment(words, ctx) {
|
|
|
686
768
|
}
|
|
687
769
|
};
|
|
688
770
|
const byName = (name) => {
|
|
689
|
-
const kind =
|
|
771
|
+
const kind = nameClass(name, state);
|
|
690
772
|
if (kind === 'credential')
|
|
691
773
|
next();
|
|
692
774
|
else if (kind === 'ambiguous')
|
|
@@ -697,13 +779,12 @@ function redactSegment(words, ctx) {
|
|
|
697
779
|
if (long) {
|
|
698
780
|
const name = long[1].replace(/^-+/, '');
|
|
699
781
|
if (name === 'from-literal' || name === 'from-env-literal') {
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
out[i] = `${long[1]}=${long[2].replace(/^([^=]*)=.*$/s, `$1=${SECRET}`)}`;
|
|
782
|
+
const replaced = `${long[1]}=${long[2].replace(/^([^=]*)=.*$/s, `$1=${SECRET}`)}`;
|
|
783
|
+
out[i] = holdsMoreThanCredential(long[2].replace(/^[^=]*=/, '')) ? withheld(state, arg, replaced) : replaced;
|
|
703
784
|
handled.add(i);
|
|
704
785
|
}
|
|
705
786
|
else if (CURL_FAMILY.has(program) && /^(?:u|user|proxy-user)$/.test(name)) {
|
|
706
|
-
out[i] = `${long[1]}=${SECRET}`;
|
|
787
|
+
out[i] = `${long[1]}=${state.exact ? exactUser(long[2]) : SECRET}`;
|
|
707
788
|
handled.add(i);
|
|
708
789
|
}
|
|
709
790
|
else if (CURL_FAMILY.has(program) && name === 'header' && AUTH_HEADER.test(long[2])) {
|
|
@@ -715,9 +796,11 @@ function redactSegment(words, ctx) {
|
|
|
715
796
|
handled.add(i);
|
|
716
797
|
}
|
|
717
798
|
else if (holdsSecret(name)) {
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
799
|
+
const kind = nameClass(name, state);
|
|
800
|
+
if (kind === 'ambiguous')
|
|
801
|
+
out[i] = withheld(state, arg, `${long[1]}=${cutCredential(long[2], state)}`);
|
|
802
|
+
else if (kind === 'credential')
|
|
803
|
+
out[i] = `${long[1]}=${cutCredential(long[2], state)}`;
|
|
721
804
|
handled.add(i);
|
|
722
805
|
}
|
|
723
806
|
continue;
|
|
@@ -726,7 +809,7 @@ function redactSegment(words, ctx) {
|
|
|
726
809
|
if (/^--[A-Za-z0-9]/.test(arg)) {
|
|
727
810
|
const name = arg.slice(2);
|
|
728
811
|
if (CURL_FAMILY.has(program) && /^(?:user|proxy-user)$/.test(name))
|
|
729
|
-
next();
|
|
812
|
+
next(false, exactUser);
|
|
730
813
|
else if (CURL_FAMILY.has(program) && name === 'header') {
|
|
731
814
|
const j = valueAt();
|
|
732
815
|
if (j !== null && AUTH_HEADER.test(out[j])) {
|
|
@@ -764,7 +847,7 @@ function redactSegment(words, ctx) {
|
|
|
764
847
|
else if (REDIS_FAMILY.has(program) && /^-a/.test(arg))
|
|
765
848
|
attachedOrNext('-a');
|
|
766
849
|
else if (CURL_FAMILY.has(program) && (arg === '-u' || arg === '-U'))
|
|
767
|
-
next();
|
|
850
|
+
next(false, exactUser);
|
|
768
851
|
else if (CURL_FAMILY.has(program) && arg === '-H') {
|
|
769
852
|
const j = valueAt();
|
|
770
853
|
if (j !== null && AUTH_HEADER.test(out[j])) {
|
|
@@ -782,7 +865,7 @@ function redactSegment(words, ctx) {
|
|
|
782
865
|
}
|
|
783
866
|
}
|
|
784
867
|
else if (program === 'openssl' && OPENSSL_PASS_FLAGS.has(arg))
|
|
785
|
-
next();
|
|
868
|
+
next(false, arg === '-kfile' ? (value) => value : exactOpensslPass);
|
|
786
869
|
else if (program === 'gh' && secretContext && arg === '-b')
|
|
787
870
|
next();
|
|
788
871
|
else if (/^-[A-Za-z][A-Za-z0-9_-]{2,}$/.test(arg))
|
|
@@ -795,15 +878,17 @@ function redactSegment(words, ctx) {
|
|
|
795
878
|
const [, name, value] = assignment;
|
|
796
879
|
if (secretContext) {
|
|
797
880
|
// In a secrets command every value is a secret: withheld whole, and truncated if it held more fields.
|
|
798
|
-
|
|
799
|
-
state.truncated = true;
|
|
800
|
-
out[i] = `${name}=${SECRET}`;
|
|
881
|
+
out[i] = /[&#]/.test(value) || holdsMoreThanCredential(value) ? withheld(state, arg, `${name}=${SECRET}`) : `${name}=${SECRET}`;
|
|
801
882
|
handled.add(i);
|
|
802
883
|
}
|
|
803
884
|
else if (holdsSecret(name)) {
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
885
|
+
const kind = nameClass(name, state);
|
|
886
|
+
if (kind === 'ambiguous')
|
|
887
|
+
out[i] = withheld(state, arg, credentialArgument(name, value));
|
|
888
|
+
else if (kind === 'credential')
|
|
889
|
+
out[i] = credentialArgument(name, value);
|
|
890
|
+
else
|
|
891
|
+
out[i] = redactText(arg, state);
|
|
807
892
|
handled.add(i);
|
|
808
893
|
}
|
|
809
894
|
else if (envStore) {
|
|
@@ -813,7 +898,7 @@ function redactSegment(words, ctx) {
|
|
|
813
898
|
continue;
|
|
814
899
|
}
|
|
815
900
|
if (program === 'openssl' && /^(?:pass|env|file|fd):/.test(arg)) {
|
|
816
|
-
out[i] = arg.replace(/:.*/s, `:${SECRET}`);
|
|
901
|
+
out[i] = state.exact ? exactOpensslPass(arg) : arg.replace(/:.*/s, `:${SECRET}`);
|
|
817
902
|
handled.add(i);
|
|
818
903
|
continue;
|
|
819
904
|
}
|
|
@@ -830,7 +915,7 @@ function redactSegment(words, ctx) {
|
|
|
830
915
|
if (envStore && position > 0 && /^(?:env|config|variables?|vars?):(?:set|add|push|import|create|update|put)$|^(?:set|add|put|push|import|create|update)$/.test(previous)) {
|
|
831
916
|
const j = valueAt();
|
|
832
917
|
if (j !== null) {
|
|
833
|
-
const kind =
|
|
918
|
+
const kind = nameClass(arg, state);
|
|
834
919
|
if (kind === 'credential')
|
|
835
920
|
redactAt(j);
|
|
836
921
|
else if (kind === 'ambiguous')
|
|
@@ -880,12 +965,95 @@ function redactSegment(words, ctx) {
|
|
|
880
965
|
function quoteWord(word) {
|
|
881
966
|
return /[\s'"\\|&;<>()$`*?]/.test(word) ? `'${word.replace(/'/g, `'\\''`)}'` : word;
|
|
882
967
|
}
|
|
968
|
+
const SIG_SPECIAL = '$`*?[{~';
|
|
969
|
+
const HEREDOC_OPEN = /(?<!<)<<(-?)[ \t]*(['"]?)([A-Za-z_][A-Za-z0-9_]*)\2/g;
|
|
970
|
+
/**
|
|
971
|
+
* What quoting does to the characters the shell expands (`$`, backquote, `*`, `?`, `[`, `{`, `~`), in order: `A`
|
|
972
|
+
* when the shell acts on it, `L` when it is literal (single quotes, a backslash, double quotes for all but `$` and
|
|
973
|
+
* backquote, a quoted heredoc). The words the normalizer reads are unquoted, so `rm -rf '*'` and `rm -rf *`,
|
|
974
|
+
* `'$HOME'` and `$HOME`, `<<'EOF'` and `<<EOF` read alike; this signature keeps them apart. `'*'`, `"*"` and `\*`
|
|
975
|
+
* mean the same and sign the same.
|
|
976
|
+
*/
|
|
977
|
+
function quoteSignature(command) {
|
|
978
|
+
const parts = [];
|
|
979
|
+
const lines = command.split('\n');
|
|
980
|
+
const pending = [];
|
|
981
|
+
let quote = '';
|
|
982
|
+
for (let n = 0; n < lines.length; n += 1) {
|
|
983
|
+
if (pending.length && !quote) {
|
|
984
|
+
const doc = pending.shift();
|
|
985
|
+
for (; n < lines.length; n += 1) {
|
|
986
|
+
const line = lines[n];
|
|
987
|
+
if (line.replace(/^[ \t]+|[ \t]+$/g, '') === doc.tag)
|
|
988
|
+
break;
|
|
989
|
+
for (let i = 0; i < line.length; i += 1) {
|
|
990
|
+
const c = line[i];
|
|
991
|
+
if (!doc.quoted && c === '\\' && i + 1 < line.length && '$`\\'.includes(line[i + 1])) {
|
|
992
|
+
if (SIG_SPECIAL.includes(line[i + 1]))
|
|
993
|
+
parts.push(`L${line[i + 1]}`);
|
|
994
|
+
i += 1;
|
|
995
|
+
}
|
|
996
|
+
else if (SIG_SPECIAL.includes(c))
|
|
997
|
+
parts.push(`${!doc.quoted && (c === '$' || c === '`') ? 'A' : 'L'}${c}`);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
parts.push('|');
|
|
1001
|
+
continue;
|
|
1002
|
+
}
|
|
1003
|
+
const line = lines[n];
|
|
1004
|
+
if (!quote)
|
|
1005
|
+
for (const match of line.matchAll(HEREDOC_OPEN))
|
|
1006
|
+
pending.push({ tag: match[3], quoted: match[2] !== '' });
|
|
1007
|
+
for (let i = 0; i < line.length; i += 1) {
|
|
1008
|
+
const c = line[i];
|
|
1009
|
+
if (quote === "'") {
|
|
1010
|
+
if (c === "'")
|
|
1011
|
+
quote = '';
|
|
1012
|
+
else if (SIG_SPECIAL.includes(c))
|
|
1013
|
+
parts.push(`L${c}`);
|
|
1014
|
+
continue;
|
|
1015
|
+
}
|
|
1016
|
+
if (c === '\\' && i + 1 < line.length) {
|
|
1017
|
+
const next = line[i + 1];
|
|
1018
|
+
if (SIG_SPECIAL.includes(next))
|
|
1019
|
+
parts.push(`L${next}`);
|
|
1020
|
+
i += 1;
|
|
1021
|
+
continue;
|
|
1022
|
+
}
|
|
1023
|
+
if (quote === '"') {
|
|
1024
|
+
if (c === '"')
|
|
1025
|
+
quote = '';
|
|
1026
|
+
else if (SIG_SPECIAL.includes(c))
|
|
1027
|
+
parts.push(`${c === '$' || c === '`' ? 'A' : 'L'}${c}`);
|
|
1028
|
+
continue;
|
|
1029
|
+
}
|
|
1030
|
+
if (c === "'" || c === '"') {
|
|
1031
|
+
quote = c;
|
|
1032
|
+
continue;
|
|
1033
|
+
}
|
|
1034
|
+
if (SIG_SPECIAL.includes(c))
|
|
1035
|
+
parts.push(`A${c}`);
|
|
1036
|
+
}
|
|
1037
|
+
if (quote)
|
|
1038
|
+
parts.push('/');
|
|
1039
|
+
}
|
|
1040
|
+
return parts.join('');
|
|
1041
|
+
}
|
|
883
1042
|
function normalizeInner(command, state, depth) {
|
|
884
1043
|
if (command.length > MAX_COMMAND_CHARS)
|
|
885
1044
|
throw new NormalizationLimit();
|
|
886
1045
|
spend(state, 0);
|
|
1046
|
+
// Quoting of the characters the shell expands (kept apart from the unquoted words, see quoteSignature).
|
|
1047
|
+
const signature = quoteSignature(command);
|
|
1048
|
+
if (signature)
|
|
1049
|
+
state.sigs.push(`${depth}:${signature}`);
|
|
1050
|
+
// A lone CR (not before LF) is a word character to the shell; read as a line end here, so the action is
|
|
1051
|
+
// truncated and the exact pass keeps it (`main<CR>` is not `main`, `build<CR>dist` is not `build ; dist`).
|
|
1052
|
+
if (/\r(?!\n)/.test(command))
|
|
1053
|
+
state.truncated = true; // exact: kept (crIsWord)
|
|
887
1054
|
const data = [];
|
|
888
|
-
const
|
|
1055
|
+
const lines = state.exact ? command.replace(/\r\n/g, '\n') : command.replace(/\r\n?/g, '\n');
|
|
1056
|
+
const tokens = tokenize(extractHeredocs(lines, data), state.exact);
|
|
889
1057
|
if (tokens.length > MAX_TOKENS)
|
|
890
1058
|
throw new NormalizationLimit();
|
|
891
1059
|
const segments = [];
|
|
@@ -953,10 +1121,13 @@ function normalizeInner(command, state, depth) {
|
|
|
953
1121
|
* never sent. `truncated` is true when something that may change what the
|
|
954
1122
|
* command does had to be withheld. Exported for tests.
|
|
955
1123
|
*/
|
|
956
|
-
function normalizeShellCommand(command, deadline) {
|
|
957
|
-
const state = newExactness(deadline);
|
|
1124
|
+
function normalizeShellCommand(command, deadline, exact = false) {
|
|
1125
|
+
const state = newExactness(deadline, exact);
|
|
958
1126
|
try {
|
|
959
1127
|
const result = normalizeInner(command, state, 0);
|
|
1128
|
+
// The exact pass: the form exact_hmac keys (the words, the shell references and the quoting).
|
|
1129
|
+
if (exact)
|
|
1130
|
+
return { ...result, text: exactShellForm(result.text, command, state), truncated: state.truncated };
|
|
960
1131
|
return { ...result, truncated: state.truncated };
|
|
961
1132
|
}
|
|
962
1133
|
catch (error) {
|
|
@@ -975,28 +1146,37 @@ function sortedValue(value, depth = 0) {
|
|
|
975
1146
|
}
|
|
976
1147
|
/** A tool input with credentials and key-shaped values replaced; hashed, never sent. */
|
|
977
1148
|
function redactedInput(value, state, depth = 0) {
|
|
1149
|
+
// Past the depth or entry bounds the exact pass stops; exact_hmac then keys the raw canonical input.
|
|
978
1150
|
if (depth > 6) {
|
|
979
|
-
state.
|
|
1151
|
+
if (state.exact)
|
|
1152
|
+
throw new NormalizationLimit();
|
|
1153
|
+
state.truncated = true; // exact: raw (canonical input)
|
|
980
1154
|
return SECRET;
|
|
981
1155
|
}
|
|
982
1156
|
if (typeof value === 'string')
|
|
983
1157
|
return redactText(value, state);
|
|
984
1158
|
if (Array.isArray(value)) {
|
|
1159
|
+
if (value.length > 64 && state.exact)
|
|
1160
|
+
throw new NormalizationLimit();
|
|
985
1161
|
if (value.length > 64)
|
|
986
|
-
state.truncated = true;
|
|
1162
|
+
state.truncated = true; // exact: raw (canonical input)
|
|
987
1163
|
return value.slice(0, 64).map((item) => redactedInput(item, state, depth + 1));
|
|
988
1164
|
}
|
|
989
1165
|
if (value && typeof value === 'object') {
|
|
990
1166
|
const entries = Object.entries(value);
|
|
1167
|
+
if (entries.length > 64 && state.exact)
|
|
1168
|
+
throw new NormalizationLimit();
|
|
991
1169
|
if (entries.length > 64)
|
|
992
|
-
state.truncated = true;
|
|
1170
|
+
state.truncated = true; // exact: raw (canonical input)
|
|
993
1171
|
const out = {};
|
|
994
1172
|
for (const [key, item] of entries.slice(0, 64)) {
|
|
995
|
-
const kind =
|
|
1173
|
+
const kind = nameClass(key, state);
|
|
996
1174
|
if (kind === 'credential' || kind === 'ambiguous') {
|
|
1175
|
+
// The exact pass keeps an ambiguous value, and looks inside an object or array under a credential
|
|
1176
|
+
// name (`credentials: {user, role, password}`): only a value itself is certain.
|
|
997
1177
|
if (kind === 'ambiguous')
|
|
998
|
-
state.truncated = true;
|
|
999
|
-
out[key] = SECRET;
|
|
1178
|
+
state.truncated = true; // exact: kept (below)
|
|
1179
|
+
out[key] = state.exact && (kind === 'ambiguous' || (item !== null && typeof item === 'object')) ? redactedInput(item, state, depth + 1) : SECRET;
|
|
1000
1180
|
}
|
|
1001
1181
|
else {
|
|
1002
1182
|
out[key] = redactedInput(item, state, depth + 1);
|
|
@@ -1006,6 +1186,24 @@ function redactedInput(value, state, depth = 0) {
|
|
|
1006
1186
|
}
|
|
1007
1187
|
return value;
|
|
1008
1188
|
}
|
|
1189
|
+
/**
|
|
1190
|
+
* The host's own shell text when trimming changed what bash runs, else null. The command is read trimmed, but a
|
|
1191
|
+
* lone CR, a vertical tab, a form feed or a non-ASCII space at either end is part of a word in bash (`main<CR>` is
|
|
1192
|
+
* another ref than `main`). ASCII spaces, tabs, newlines and CRLF line ends at the ends change nothing.
|
|
1193
|
+
*/
|
|
1194
|
+
function untrimmedShellText(event, trimmed) {
|
|
1195
|
+
const input = event.tool_input && typeof event.tool_input === 'object' && !Array.isArray(event.tool_input)
|
|
1196
|
+
? event.tool_input : null;
|
|
1197
|
+
const host = typeof event.tool_input === 'string' ? event.tool_input
|
|
1198
|
+
: typeof input?.command === 'string' ? input.command : typeof input?.cmd === 'string' ? input.cmd : null;
|
|
1199
|
+
if (host === null || host === trimmed)
|
|
1200
|
+
return null;
|
|
1201
|
+
const at = host.indexOf(trimmed);
|
|
1202
|
+
if (at < 0)
|
|
1203
|
+
return null;
|
|
1204
|
+
const edges = `${host.slice(0, at)}${host.slice(at + trimmed.length)}`.replace(/\r\n/g, '\n');
|
|
1205
|
+
return /[^ \t\n]/.test(edges) ? host : null;
|
|
1206
|
+
}
|
|
1009
1207
|
/**
|
|
1010
1208
|
* The command a shell tool runs. Codex may send an argument vector
|
|
1011
1209
|
* (["bash", "-lc", "<command>"]): its command is the same text a string form
|
|
@@ -1019,7 +1217,8 @@ function shellCommand(event) {
|
|
|
1019
1217
|
const words = argv;
|
|
1020
1218
|
if (words.length === 3 && /^(?:.*\/)?(?:ba|z|da|k)?sh$/.test(words[0]) && /^-[a-z]*c$/.test(words[1]))
|
|
1021
1219
|
return words[2];
|
|
1022
|
-
|
|
1220
|
+
// An argument vector runs without a shell: every word is literal, so any character the shell reads is quoted.
|
|
1221
|
+
return words.map((word) => (/[\s'"\\$`*?[\]{}~|&;<>()!#]/.test(word) ? `'${word.replace(/'/g, `'\\''`)}'` : word)).join(' ');
|
|
1023
1222
|
}
|
|
1024
1223
|
return (0, hook_tool_policy_1.hookToolCommand)(event);
|
|
1025
1224
|
}
|
|
@@ -1035,7 +1234,7 @@ function withoutUserName(path, state) {
|
|
|
1035
1234
|
const other = /^(\/(?:home|Users))\/[^/]+(?=\/|$)/.exec(path);
|
|
1036
1235
|
if (!other)
|
|
1037
1236
|
return path;
|
|
1038
|
-
state.truncated = true;
|
|
1237
|
+
state.truncated = true; // exact: n/a (paths are not exact material; the whole input is)
|
|
1039
1238
|
return `${other[1]}/[user]${path.slice(other[0].length)}`;
|
|
1040
1239
|
}
|
|
1041
1240
|
// ---------------------------------------------------------------------------
|
|
@@ -1127,12 +1326,181 @@ function inputHash(input, state) {
|
|
|
1127
1326
|
return {};
|
|
1128
1327
|
const text = JSON.stringify(sortedValue(redactedInput(input, state)));
|
|
1129
1328
|
if (text.includes(SECRET))
|
|
1130
|
-
state.truncated = true;
|
|
1329
|
+
state.truncated = true; // exact: n/a (pass 1 only; exactMaterial is the exact form)
|
|
1131
1330
|
return { input_sha256: sha256(`${HASH_VERSION}\ninput\n${text}`) };
|
|
1132
1331
|
}
|
|
1133
1332
|
function withExactness(action, state) {
|
|
1134
1333
|
return state.truncated ? { ...action, truncated: true } : action;
|
|
1135
1334
|
}
|
|
1335
|
+
// ---------------------------------------------------------------------------
|
|
1336
|
+
// Exact material for a truncated action (exact_hmac; M2)
|
|
1337
|
+
// ---------------------------------------------------------------------------
|
|
1338
|
+
const EXACT_DOMAIN = 'marrow-exact-action-v1';
|
|
1339
|
+
/** Raw canonical input past this is never keyed: a nonce instead (it matches nothing). */
|
|
1340
|
+
const MAX_RAW_EXACT_CHARS = 1_048_576;
|
|
1341
|
+
const EXACT_SALT_NOTE = 'Marrow: the private device key for exact action matching is unavailable or unsafe on this machine, so a held action that hides a value is asked again instead of resumed on retry. Owner: run the Marrow installer doctor command in a terminal.';
|
|
1342
|
+
/** Canonical JSON (keys sorted at every depth), or null past 1 MB, past 64 levels, or when it cannot be serialized. */
|
|
1343
|
+
function canonicalJson(value) {
|
|
1344
|
+
const parts = [];
|
|
1345
|
+
let size = 0;
|
|
1346
|
+
const push = (text) => {
|
|
1347
|
+
size += text.length;
|
|
1348
|
+
if (size > MAX_RAW_EXACT_CHARS)
|
|
1349
|
+
throw new NormalizationLimit();
|
|
1350
|
+
parts.push(text);
|
|
1351
|
+
};
|
|
1352
|
+
const walk = (item, depth) => {
|
|
1353
|
+
if (depth > 64)
|
|
1354
|
+
throw new NormalizationLimit();
|
|
1355
|
+
if (Array.isArray(item)) {
|
|
1356
|
+
if (item.length > MAX_RAW_EXACT_CHARS / 2)
|
|
1357
|
+
throw new NormalizationLimit();
|
|
1358
|
+
push('[');
|
|
1359
|
+
item.forEach((entry, index) => { if (index)
|
|
1360
|
+
push(','); walk(entry, depth + 1); });
|
|
1361
|
+
push(']');
|
|
1362
|
+
}
|
|
1363
|
+
else if (item !== null && typeof item === 'object') {
|
|
1364
|
+
const keys = Object.keys(item);
|
|
1365
|
+
if (keys.length > MAX_RAW_EXACT_CHARS / 4)
|
|
1366
|
+
throw new NormalizationLimit();
|
|
1367
|
+
push('{');
|
|
1368
|
+
keys.sort().forEach((key, index) => {
|
|
1369
|
+
if (index)
|
|
1370
|
+
push(',');
|
|
1371
|
+
push(`${JSON.stringify(key)}:`);
|
|
1372
|
+
walk(item[key], depth + 1);
|
|
1373
|
+
});
|
|
1374
|
+
push('}');
|
|
1375
|
+
}
|
|
1376
|
+
else {
|
|
1377
|
+
push(JSON.stringify(item) ?? 'null');
|
|
1378
|
+
}
|
|
1379
|
+
};
|
|
1380
|
+
try {
|
|
1381
|
+
walk(value, 0);
|
|
1382
|
+
return parts.join('');
|
|
1383
|
+
}
|
|
1384
|
+
catch {
|
|
1385
|
+
return null;
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
/**
|
|
1389
|
+
* Every shell reference in a command, in order: `$NAME`, `${...}`, `$(...)` (balanced) and backquoted text,
|
|
1390
|
+
* quotes ignored (so a quoted literal `$` counts as well: stricter). A reference is never a credential value
|
|
1391
|
+
* itself: `API_KEY=$READONLY_KEY` and `API_KEY=$ADMIN_KEY`, or `mysql -p"$(cat ro.pw)"` and `"$(cat admin.pw)"`,
|
|
1392
|
+
* name different credentials. The exact pass replaces a credential position whole, so the references are
|
|
1393
|
+
* added to the exact material; two commands that differ only in a literal credential still match.
|
|
1394
|
+
*/
|
|
1395
|
+
function shellReferences(command) {
|
|
1396
|
+
const refs = [];
|
|
1397
|
+
for (let i = 0; i < command.length; i += 1) {
|
|
1398
|
+
const ch = command[i];
|
|
1399
|
+
if (ch === '`') {
|
|
1400
|
+
const end = command.indexOf('`', i + 1);
|
|
1401
|
+
const stop = end < 0 ? command.length : end + 1;
|
|
1402
|
+
refs.push(command.slice(i, stop));
|
|
1403
|
+
i = stop - 1;
|
|
1404
|
+
}
|
|
1405
|
+
else if (ch === '$' && (command[i + 1] === '(' || command[i + 1] === '{')) {
|
|
1406
|
+
const open = command[i + 1];
|
|
1407
|
+
const close = open === '(' ? ')' : '}';
|
|
1408
|
+
let depth = 0;
|
|
1409
|
+
let j = i + 1;
|
|
1410
|
+
for (; j < command.length; j += 1) {
|
|
1411
|
+
if (command[j] === open)
|
|
1412
|
+
depth += 1;
|
|
1413
|
+
else if (command[j] === close) {
|
|
1414
|
+
depth -= 1;
|
|
1415
|
+
if (depth === 0)
|
|
1416
|
+
break;
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
refs.push(command.slice(i, Math.min(j + 1, command.length)));
|
|
1420
|
+
i = Math.min(j, command.length);
|
|
1421
|
+
}
|
|
1422
|
+
else if (ch === '$') {
|
|
1423
|
+
const name = /^\$(?:[A-Za-z_][A-Za-z0-9_]*|[0-9@*#?$!-])/.exec(command.slice(i, i + 256));
|
|
1424
|
+
if (name) {
|
|
1425
|
+
refs.push(name[0]);
|
|
1426
|
+
i += name[0].length - 1;
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
return refs;
|
|
1431
|
+
}
|
|
1432
|
+
/** The exact form of a command: its words (only certain credentials replaced), its shell references and its quoting. */
|
|
1433
|
+
function exactShellForm(text, command, state) {
|
|
1434
|
+
return `${text}\n${JSON.stringify(shellReferences(command))}\n${JSON.stringify(state.sigs)}`;
|
|
1435
|
+
}
|
|
1436
|
+
/** The raw input as material (a placeholder, or an exact pass that hit a limit): the command, or the canonical input. */
|
|
1437
|
+
function rawExactMaterial(event, kind) {
|
|
1438
|
+
if (kind === 'shell') {
|
|
1439
|
+
try {
|
|
1440
|
+
const trimmed = shellCommand(event);
|
|
1441
|
+
const command = untrimmedShellText(event, trimmed) ?? trimmed;
|
|
1442
|
+
return typeof command === 'string' && command.length <= MAX_RAW_EXACT_CHARS ? `raw-shell\n${command}` : null;
|
|
1443
|
+
}
|
|
1444
|
+
catch {
|
|
1445
|
+
return null;
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
const json = canonicalJson(event.tool_input ?? null);
|
|
1449
|
+
return json === null ? null : `raw-input\n${json}`;
|
|
1450
|
+
}
|
|
1451
|
+
/**
|
|
1452
|
+
* The exact material of an action: the same normalization with `exact` set (only a certain credential replaced,
|
|
1453
|
+
* every other value kept), or the raw input when that pass hits a limit. null: no material (a nonce instead).
|
|
1454
|
+
*/
|
|
1455
|
+
function exactMaterial(event, kind, deadline) {
|
|
1456
|
+
const state = newExactness(deadline, true);
|
|
1457
|
+
// Over 1 MB of material: a nonce (asked again), as for raw material.
|
|
1458
|
+
const bounded = (material) => (material.length <= MAX_RAW_EXACT_CHARS + 16 ? material : null);
|
|
1459
|
+
try {
|
|
1460
|
+
if (kind === 'shell') {
|
|
1461
|
+
const trimmed = shellCommand(event);
|
|
1462
|
+
const host = untrimmedShellText(event, trimmed);
|
|
1463
|
+
if (host !== null)
|
|
1464
|
+
return bounded(`raw-shell\n${host}`);
|
|
1465
|
+
const { text } = normalizeInner(trimmed, state, 0);
|
|
1466
|
+
return bounded(`exact-shell\n${exactShellForm(text, trimmed, state)}`);
|
|
1467
|
+
}
|
|
1468
|
+
const input = event.tool_input;
|
|
1469
|
+
const text = input === undefined || input === null ? 'null' : JSON.stringify(sortedValue(redactedInput(input, state)));
|
|
1470
|
+
return bounded(`exact-input\n${text}`);
|
|
1471
|
+
}
|
|
1472
|
+
catch (error) {
|
|
1473
|
+
return error instanceof NormalizationLimit ? rawExactMaterial(event, kind) : null;
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
let saltNoted = false;
|
|
1477
|
+
/** The device salt (seatbelt store, same checks), or null with one stderr line per process (no path, no value). */
|
|
1478
|
+
function exactSalt() {
|
|
1479
|
+
let salt = null;
|
|
1480
|
+
try {
|
|
1481
|
+
salt = require('./seatbelt/record').deviceSalt();
|
|
1482
|
+
}
|
|
1483
|
+
catch {
|
|
1484
|
+
salt = null;
|
|
1485
|
+
}
|
|
1486
|
+
if (!salt && !saltNoted) {
|
|
1487
|
+
saltNoted = true;
|
|
1488
|
+
try {
|
|
1489
|
+
process.stderr.write(`${EXACT_SALT_NOTE}\n`);
|
|
1490
|
+
}
|
|
1491
|
+
catch { /* best effort */ }
|
|
1492
|
+
}
|
|
1493
|
+
return salt;
|
|
1494
|
+
}
|
|
1495
|
+
/**
|
|
1496
|
+
* exact_hmac: HMAC-SHA256(device salt, domain, kind, the host's full tool name, material), 128 bits. Without a
|
|
1497
|
+
* salt or material it is a fresh random nonce, which matches nothing: never unsalted, never the merged form.
|
|
1498
|
+
*/
|
|
1499
|
+
function exactHmac(salt, kind, toolName, material) {
|
|
1500
|
+
if (!salt || salt.length < 16 || material === null)
|
|
1501
|
+
return (0, node_crypto_1.randomBytes)(16).toString('hex');
|
|
1502
|
+
return (0, node_crypto_1.createHmac)('sha256', salt).update(`${EXACT_DOMAIN}\n${kind}\n${JSON.stringify(toolName)}\n`).update(material).digest('hex').slice(0, 32);
|
|
1503
|
+
}
|
|
1136
1504
|
/**
|
|
1137
1505
|
* The normalized action of a hook event. `deadlineAt` bounds the time it may
|
|
1138
1506
|
* take (default one second from now): past it, or past the size limits, the
|
|
@@ -1140,26 +1508,49 @@ function withExactness(action, state) {
|
|
|
1140
1508
|
* service never binds, so the hook still answers within its host's budget.
|
|
1141
1509
|
* An event whose input a host adapter cut or dropped (`input_truncated`) is
|
|
1142
1510
|
* marked truncated as well.
|
|
1511
|
+
*
|
|
1512
|
+
* A truncated action also carries `tool_input.exact_hmac`: a keyed hash (the
|
|
1513
|
+
* device salt never leaves the machine) of the action with only certain
|
|
1514
|
+
* credentials replaced, so two actions that differ anywhere else never share a
|
|
1515
|
+
* correlation, hold or key, and a retry that differs only in a certain
|
|
1516
|
+
* credential (an OTP) still does. A placeholder keys the raw input (up to
|
|
1517
|
+
* 1 MB). Input a host adapter cut or dropped, input over 1 MB and a missing or
|
|
1518
|
+
* unsafe salt give a random nonce: the action matches nothing (it is asked
|
|
1519
|
+
* again). `salt` is a test seam (undefined: the device salt).
|
|
1143
1520
|
*/
|
|
1144
1521
|
function normalizedHookAction(event, options = {}) {
|
|
1145
|
-
const
|
|
1522
|
+
const deadline = options.deadlineAt ?? Date.now() + DEFAULT_NORMALIZE_MS;
|
|
1523
|
+
const state = newExactness(deadline);
|
|
1146
1524
|
const toolName = (0, hook_tool_policy_1.normalizeHookToolName)(event.tool_name) || 'tool';
|
|
1147
|
-
// The host's own tool name (stable across retries); the policy name decides the kind.
|
|
1148
|
-
|
|
1525
|
+
// The host's own tool name, exactly as sent (stable across retries); the policy name decides the kind.
|
|
1526
|
+
// Not trimmed, so `X ` and `X` get different actions; a name past 128 characters is never bound.
|
|
1527
|
+
const rawToolName = typeof event.tool_name === 'string' && event.tool_name ? event.tool_name : toolName;
|
|
1528
|
+
const hostToolName = rawToolName.slice(0, 128);
|
|
1529
|
+
if (rawToolName.length > 128)
|
|
1530
|
+
state.truncated = true; // exact: kept (the full name is in exact_hmac)
|
|
1149
1531
|
const kind = (0, hook_tool_policy_1.isMcpHookTool)(event.tool_name) || /^MCP:/i.test(hostToolName) ? 'mcp'
|
|
1150
1532
|
: (0, hook_tool_policy_1.isShellGovernedTool)(event) ? 'shell'
|
|
1151
1533
|
: EDIT_TOOLS.test(toolName) ? 'edit'
|
|
1152
1534
|
: 'other';
|
|
1153
|
-
|
|
1154
|
-
|
|
1535
|
+
const inputCut = event.input_truncated === true;
|
|
1536
|
+
if (inputCut)
|
|
1537
|
+
state.truncated = true; // exact: nonce (the cut or dropped input is all that is left)
|
|
1538
|
+
let action;
|
|
1539
|
+
let placeholder = false;
|
|
1155
1540
|
try {
|
|
1156
|
-
|
|
1541
|
+
action = normalizedOfKind(event, kind, hostToolName, state);
|
|
1157
1542
|
}
|
|
1158
1543
|
catch (error) {
|
|
1159
|
-
if (error instanceof NormalizationLimit)
|
|
1160
|
-
|
|
1161
|
-
|
|
1544
|
+
if (!(error instanceof NormalizationLimit))
|
|
1545
|
+
throw error;
|
|
1546
|
+
action = { tool_kind: kind, tool_name: hostToolName, tool_input: {}, truncated: true };
|
|
1547
|
+
placeholder = true;
|
|
1162
1548
|
}
|
|
1549
|
+
if (action.truncated !== true)
|
|
1550
|
+
return action;
|
|
1551
|
+
const material = inputCut ? null : placeholder ? rawExactMaterial(event, kind) : exactMaterial(event, kind, deadline);
|
|
1552
|
+
const salt = options.salt !== undefined ? options.salt : material === null ? null : exactSalt();
|
|
1553
|
+
return { ...action, tool_input: { ...action.tool_input, exact_hmac: exactHmac(salt, kind, rawToolName, material) } };
|
|
1163
1554
|
}
|
|
1164
1555
|
function normalizedOfKind(event, kind, hostToolName, state) {
|
|
1165
1556
|
if (kind === 'mcp') {
|
|
@@ -1167,10 +1558,12 @@ function normalizedOfKind(event, kind, hostToolName, state) {
|
|
|
1167
1558
|
}
|
|
1168
1559
|
if (kind === 'shell') {
|
|
1169
1560
|
const raw = shellCommand(event);
|
|
1561
|
+
if (untrimmedShellText(event, raw) !== null)
|
|
1562
|
+
state.truncated = true; // exact: raw (the host's untrimmed text)
|
|
1170
1563
|
const { text, programs } = normalizeInner(raw, state, 0);
|
|
1171
1564
|
// A secret was replaced: the action is never bound or reused (it asks each time).
|
|
1172
1565
|
if (text.includes(SECRET))
|
|
1173
|
-
state.truncated = true;
|
|
1566
|
+
state.truncated = true; // exact: n/a (pass 1 only; the exact pass decides each value)
|
|
1174
1567
|
// The command text itself only for a deploy the service can judge word for word (see deployCommandText).
|
|
1175
1568
|
// Judged on the host's exact text (a trimmed non-ASCII space must not slip through).
|
|
1176
1569
|
const input = event.tool_input && typeof event.tool_input === 'object' && !Array.isArray(event.tool_input) ? event.tool_input : null;
|
|
@@ -1181,7 +1574,8 @@ function normalizedOfKind(event, kind, hostToolName, state) {
|
|
|
1181
1574
|
tool_name: hostToolName,
|
|
1182
1575
|
programs,
|
|
1183
1576
|
tool_input: {
|
|
1184
|
-
|
|
1577
|
+
// An exact action also hashes its quoting (never for a truncated one: a secret's characters stay out of it).
|
|
1578
|
+
command_sha256: sha256(`${HASH_VERSION}\nshell\n${state.truncated || !state.sigs.length ? text : `${text}\n${JSON.stringify(state.sigs)}`}`),
|
|
1185
1579
|
...(deployText !== null && deployText === text ? { command: deployText } : {}),
|
|
1186
1580
|
},
|
|
1187
1581
|
}, state);
|
|
@@ -1189,11 +1583,11 @@ function normalizedOfKind(event, kind, hostToolName, state) {
|
|
|
1189
1583
|
if (kind === 'edit') {
|
|
1190
1584
|
const targets = (0, hook_tool_policy_1.toolTargetPaths)(event);
|
|
1191
1585
|
if (targets.length > 64)
|
|
1192
|
-
state.truncated = true;
|
|
1586
|
+
state.truncated = true; // exact: n/a (paths are not exact material; the whole input is)
|
|
1193
1587
|
const paths = targets.slice(0, 64).map((path) => {
|
|
1194
1588
|
const redacted = redactText(withoutUserName(path, state), state);
|
|
1195
1589
|
if (redacted.length > 512 || redacted.includes(SECRET))
|
|
1196
|
-
state.truncated = true;
|
|
1590
|
+
state.truncated = true; // exact: n/a (as above)
|
|
1197
1591
|
return redacted.slice(0, 512);
|
|
1198
1592
|
});
|
|
1199
1593
|
return withExactness({ tool_kind: 'edit', tool_name: hostToolName, paths, tool_input: inputHash(event.tool_input, state) }, state);
|