@mossbear/eval-core 0.1.0-alpha.104 → 0.1.0-alpha.105

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -54,9 +54,13 @@ const REDIS_URI = /^(?:rediss?|valkeys?):\/\//i;
54
54
  /**
55
55
  * The tool a word invokes: its basename, lowercased, without a Windows
56
56
  * executable suffix. Windows resolves executable names case-insensitively.
57
+ * Backticks are dropped, so a word a substitution begins or ends (`` `curl ``)
58
+ * still names its tool when quoting hides the substitution from the parser.
57
59
  */
58
60
  function invokedTool(word) {
59
- return (word.split(/[\\/]/).at(-1) ?? word).replace(/\.exe$/i, '').toLowerCase();
61
+ return (word.replaceAll('`', '').split(/[\\/]/).at(-1) ?? word)
62
+ .replace(/\.exe$/i, '')
63
+ .toLowerCase();
60
64
  }
61
65
  const CREDENTIAL_TOOLS = [
62
66
  'curl',
@@ -87,6 +91,68 @@ const WRAPPER_VALUE_FLAGS = {
87
91
  time: /^-[of](.+)?$|^--(?:output|format)(?:=(.*))?$/,
88
92
  xargs: /^-[adEILnPs](.+)?$|^--(?:arg-file|delimiter|eof|replace|max-lines|max-args|max-procs|max-chars)(?:=(.*))?$/,
89
93
  };
94
+ // Wrapper options known to take no value. Any other option, unless its value is
95
+ // attached with `=`, may have consumed the next word.
96
+ const WRAPPER_SWITCHES = {
97
+ sudo: /^-[AbEHIKklnPSsVvie]+$|^--(?:preserve-env|login|shell|set-home|non-interactive|background)$/,
98
+ doas: /^-[nsL]+$/,
99
+ env: /^-[i0v]+$|^-$|^--(?:ignore-environment|null|debug)$/,
100
+ time: /^-[pvqa]+$|^--(?:portability|verbose|quiet|append)$/,
101
+ xargs: /^-[0rtpxo]+$|^--(?:null|no-run-if-empty|verbose|interactive|exit|open-tty)$/,
102
+ };
103
+ // Programs that never run an argument as a command, so a credential tool's name
104
+ // among their arguments is a file (`cat ./bin/curl`), not an invocation. Any
105
+ // program not listed may dispatch (`timeout`, `watch`, `find -exec`, `ssh`), and
106
+ // keeps treating every word as a possible invocation, so an unlisted program
107
+ // errs toward over-redaction.
108
+ const NON_DISPATCHING_TOOLS = new Set([
109
+ 'basename',
110
+ 'cat',
111
+ 'chgrp',
112
+ 'chmod',
113
+ 'chown',
114
+ 'cmp',
115
+ 'cp',
116
+ 'diff',
117
+ 'dirname',
118
+ 'du',
119
+ 'file',
120
+ 'grep',
121
+ 'head',
122
+ 'ln',
123
+ 'ls',
124
+ 'mkdir',
125
+ 'mv',
126
+ 'readlink',
127
+ 'realpath',
128
+ 'rm',
129
+ 'stat',
130
+ 'tail',
131
+ 'touch',
132
+ 'wc',
133
+ 'which',
134
+ ]);
135
+ /**
136
+ * Whether the command word starting at `index` of `text` runs a listed program:
137
+ * bare, or from a system bin directory. A same-named local script
138
+ * (`./tools/diff`) may run anything, and a redirection operand (`>file curl`)
139
+ * names a file, not the command.
140
+ */
141
+ function isNonDispatching(text, index, word) {
142
+ let before = index - 1;
143
+ while (text[before] === ' ' || text[before] === '\t')
144
+ before -= 1;
145
+ // `>&file` and `>|file` redirect too.
146
+ if (text[before] === '&' || text[before] === '|')
147
+ before -= 1;
148
+ if (text[before] === '<' || text[before] === '>')
149
+ return false;
150
+ // A backtick in the command word may start a substitution that runs anything.
151
+ if (word.includes('`'))
152
+ return false;
153
+ return (NON_DISPATCHING_TOOLS.has(invokedTool(word)) &&
154
+ /^(?:(?:\/usr)?\/bin\/)?[^\\/]+$/.test(word));
155
+ }
90
156
  const CURL_AUTH_OPTIONS = `user|proxy-user|${CURL_URL_OPTIONS}|cookie|oauth2-bearer|cert|proxy-cert|pass|proxy-pass`;
91
157
  const CURL_AUTH_FLAG = new RegExp(`^(-${CURL_SHORT_SWITCHES}[uUbEx])(.+)?$|^(--(?:${CURL_AUTH_OPTIONS}))(?:=(.*))?$`);
92
158
  const CURL_EXPANSION = /\{\{([A-Za-z0-9_]+)(?::[^{}]*)?\}\}/g;
@@ -128,22 +194,110 @@ function decodeShellWord(word) {
128
194
  return escaped ?? plain ?? '';
129
195
  });
130
196
  }
131
- /** A program supplying a credential argument carries private produced content. */
132
- function redactCommandSubstitutions(command) {
197
+ /**
198
+ * Undo the escapes a backtick substitution's body carries for `\\` and `\``,
199
+ * and for `\"` when the substitution sits inside double quotes.
200
+ * `\$` is left escaped: the word parser decodes it either way.
201
+ */
202
+ function unescapeBacktickBody(body, doubleQuoted = false) {
203
+ return body.replace(doubleQuoted ? /\\([\\`"])/g : /\\([\\`])/g, '$1');
204
+ }
205
+ /** Spell a body back inside backticks, reversing `unescapeBacktickBody`. */
206
+ function escapeBacktickBody(body, doubleQuoted = false) {
207
+ return body.replace(doubleQuoted ? /\\(?=[\\`"]|$)|[`"]/g : /\\(?=[\\`]|$)|`/g, '\\$&');
208
+ }
209
+ /**
210
+ * The command with each backtick substitution spelled `$(…)`. `(` separates
211
+ * words for the tool parsers and a backtick does not, so tool-input discovery
212
+ * reads this spelling to see the commands inside backticks.
213
+ */
214
+ function backticksAsSubshells(command, depth = 0) {
215
+ if (depth > 8 || !command.includes('`'))
216
+ return command;
217
+ let quote = null;
218
+ let out = '';
219
+ for (let index = 0; index < command.length; index += 1) {
220
+ const char = command[index];
221
+ if (char === '\\' && quote !== "'") {
222
+ out += command.slice(index, index + 2);
223
+ index += 1;
224
+ continue;
225
+ }
226
+ if (char === '`' && quote !== "'") {
227
+ let end = index + 1;
228
+ while (end < command.length && command[end] !== '`')
229
+ end += command[end] === '\\' ? 2 : 1;
230
+ const body = unescapeBacktickBody(command.slice(index + 1, end));
231
+ out += `$(${backticksAsSubshells(body, depth + 1)})`;
232
+ index = end;
233
+ continue;
234
+ }
235
+ if (char === "'" && quote !== '"')
236
+ quote = quote === "'" ? null : "'";
237
+ else if (char === '"' && quote !== "'")
238
+ quote = quote === '"' ? null : '"';
239
+ out += char;
240
+ }
241
+ return out;
242
+ }
243
+ /**
244
+ * Tool inputs read from the command as written and from its `$(…)` spelling.
245
+ * The respelling separates a backtick body from its first word but also
246
+ * splits a word that runs on past the substitution, so neither alone sees
247
+ * every reference.
248
+ */
249
+ function backtickAwareInputs(command) {
250
+ const inputs = getToolInputs(command);
251
+ return command.includes('`')
252
+ ? withInheritedInputs(getToolInputs(backticksAsSubshells(command)), inputs)
253
+ : inputs;
254
+ }
255
+ function withInheritedInputs(inputs, inherited) {
256
+ inputs.curlConfig ||= inherited?.curlConfig ?? false;
257
+ for (const name of inherited?.curlVariables ?? [])
258
+ inputs.curlVariables.add(name);
259
+ for (const key of inherited?.env ?? [])
260
+ inputs.env.add(key);
261
+ for (const source of inherited?.passSources ?? [])
262
+ inputs.passSources.add(source);
263
+ return inputs;
264
+ }
265
+ /**
266
+ * A program supplying a credential argument carries private produced content.
267
+ * Any other backtick substitution runs a command of its own, which a backtick
268
+ * does not separate from its first word the way `(` does, so its body is
269
+ * redacted as a command in its own right. An escaped backtick is literal text;
270
+ * `invokedTool` still reads the tool its word names.
271
+ */
272
+ function redactCommandSubstitutions(command, depth = 0, inheritedInputs, valuesFirst = true) {
273
+ if (depth > 8)
274
+ return REDACTED;
133
275
  const marker = '__SUBSTITUTION_BOUNDARY__';
134
276
  if (command.includes(marker))
135
277
  return REDACTED;
278
+ let contextInputs;
279
+ // A body also feeds, or consumes, the commands around it.
280
+ const surroundingInputs = () => (contextInputs ??= withInheritedInputs(backtickAwareInputs(command), inheritedInputs));
136
281
  const isCredentialInput = (start, end) => {
137
282
  const probe = command.slice(0, start) + marker + command.slice(end + 1);
138
283
  const inputs = getToolInputs(probe);
139
- let redacted = redactCredentialAssignments(redactToolArguments(redactJsonArguments(probe), inputs), inputs);
140
- for (const [pattern, replacement] of COMMAND_REDACTIONS)
141
- redacted = redacted.replace(pattern, replacement);
142
- return !redacted.includes(marker);
284
+ const redacted = redactCredentialAssignments(redactToolArguments(redactJsonArguments(probe), inputs), inputs);
285
+ return !applyCommandRedactions(redacted).includes(marker);
143
286
  };
144
287
  const findEnd = (start, closing, depth) => {
145
288
  if (depth > 8)
146
289
  return command.length;
290
+ // A backtick body ends at the first unescaped backtick, as bash reads it:
291
+ // quotes inside it do not hide one, and `\` escapes even inside `'…'`.
292
+ if (closing === '`') {
293
+ for (let index = start; index < command.length; index += 1) {
294
+ if (command[index] === '\\')
295
+ index += 1;
296
+ else if (command[index] === '`')
297
+ return index;
298
+ }
299
+ return command.length;
300
+ }
147
301
  let quote = null;
148
302
  let groups = 0;
149
303
  for (let index = start; index < command.length; index += 1) {
@@ -152,8 +306,6 @@ function redactCommandSubstitutions(command) {
152
306
  index += 1;
153
307
  continue;
154
308
  }
155
- if (closing === '`' && char === '`')
156
- return index;
157
309
  if (quote === "'") {
158
310
  if (char === "'")
159
311
  quote = null;
@@ -195,8 +347,33 @@ function redactCommandSubstitutions(command) {
195
347
  if (++substitutions > 64)
196
348
  return out + REDACTED;
197
349
  const start = index;
198
- index = findEnd(index + (char === '`' ? 1 : 2), char === '`' ? '`' : ')', 0);
199
- out += isCredentialInput(start, index) ? REDACTED : command.slice(start, index + 1);
350
+ const bodyStart = index + (char === '`' ? 1 : 2);
351
+ index = findEnd(bodyStart, char === '`' ? '`' : ')', 0);
352
+ const original = command.slice(start, index + 1);
353
+ if (isCredentialInput(start, index))
354
+ out += REDACTED;
355
+ else if (char === '`') {
356
+ // In the command as written, an unterminated backtick leaves the
357
+ // body's extent unknown: fail closed. A nested pass sees words the
358
+ // parser split at a backtick (`"a b"\``), so it parses the rest instead.
359
+ const unterminated = command[index] !== '`';
360
+ if (unterminated && depth === 0)
361
+ return `${out}\`${REDACTED}`;
362
+ const body = unescapeBacktickBody(command.slice(bodyStart, index), quote === '"');
363
+ const redacted = redactShellSyntax(body, depth + 1, surroundingInputs(), valuesFirst);
364
+ out +=
365
+ redacted === body
366
+ ? original
367
+ : `\`${escapeBacktickBody(redacted, quote === '"')}${unterminated ? '' : '`'}`;
368
+ }
369
+ else if (original.includes('`')) {
370
+ // `(` already exposes this body to the outer parse; only backticks
371
+ // nested inside it need a pass of their own.
372
+ const body = command.slice(bodyStart, index);
373
+ out += `$(${redactCommandSubstitutions(body, depth + 1, surroundingInputs(), valuesFirst)}${command.slice(index, index + 1)}`;
374
+ }
375
+ else
376
+ out += original;
200
377
  continue;
201
378
  }
202
379
  if (char === "'" && quote !== '"')
@@ -259,6 +436,237 @@ function redactJsonArguments(command) {
259
436
  }
260
437
  });
261
438
  }
439
+ /**
440
+ * Credential values recognised by the name beside them (URL userinfo,
441
+ * Authorization and Cookie headers, query keys) rather than by shell words.
442
+ */
443
+ const VALUE_REDACTIONS = [
444
+ [/\b([a-z][a-z0-9+.-]*:\/\/[^/\s:@]+:)[^@\s]+@/gi, `$1${REDACTED}@`],
445
+ [/\b([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+@/gi, `$1${REDACTED}@`],
446
+ [/\b(authorization\s*:\s*)(?:(?:bearer|basic|token)\s+)?\S+/gi, `$1${REDACTED}`],
447
+ [new RegExp(COOKIE_HEADER_SOURCE, 'gi'), `$1${REDACTED}`],
448
+ [
449
+ new RegExp(`([?&#]${COMMAND_SENSITIVE_KEY_SOURCE}=)[^&\\s"'<>]+`, 'gi'),
450
+ `$1${REDACTED}`,
451
+ ],
452
+ ];
453
+ /**
454
+ * Space-separated credential flags have no assignment operator or token
455
+ * prefix. Keep the flag for evidence, but never its value. The value is the
456
+ * next shell word across any whitespace, newlines included: a word typed on
457
+ * the line after `--password` is very likely the password, even though bash
458
+ * would read it as a new command.
459
+ *
460
+ * That reach means the value can itself be a credential flag
461
+ * (`--password --token <value>`). Taking it as the first flag's value must not
462
+ * spare its own value, so the word after a swallowed credential flag is
463
+ * redacted too, for as long as the chain runs. A chained word that spells a
464
+ * credential key is left for the passes after this one to read (see
465
+ * `isChainedValue`), and one with an attached value ends the chain. Every
466
+ * match the rule makes on its own is still redacted: the chain only adds.
467
+ */
468
+ const CREDENTIAL_FLAG = new RegExp(`(^|[\\s;&|(\\x60])(--${COMMAND_SENSITIVE_KEY_SOURCE}\\s+)(${SHELL_WORD_SOURCE})`, 'gi');
469
+ const NEXT_SHELL_WORD = new RegExp(`(\\s+)(${SHELL_WORD_SOURCE})`, 'y');
470
+ // The words a credential key is spelled with. Keys are tested by scanning a
471
+ // word's `[\w.-]` runs for one rather than by a regex with `[\w.-]*` on both
472
+ // sides of the key, which backtracks quadratically on a long word.
473
+ const CREDENTIAL_KEY_WORD = new RegExp(`${SENSITIVE_WORD_SOURCE}|SSHPASS|REDISCLI_AUTH|VALKEYCLI_AUTH|MYSQL_PWD`, 'i');
474
+ const ASSIGNMENT_OPERATOR = /\s*[=:]/y;
475
+ const LINE_CONTINUATION = /^\\\r?\n$/;
476
+ const KEY_CHARACTER = /[\w.-]/;
477
+ /** `SENSITIVE_KEY`'s test, without its backtracking on a long name. */
478
+ function isCredentialKeyName(name) {
479
+ return ((/^[\w.-]+$/.test(name) && SENSITIVE_CONTENT.test(name)) ||
480
+ /^(?:SSHPASS|REDISCLI_AUTH|VALKEYCLI_AUTH|MYSQL_PWD)$/i.test(name));
481
+ }
482
+ function isBareCredentialFlag(word) {
483
+ const flag = /^--([^=]+)$/.exec(decodeShellWord(word));
484
+ return flag !== null && isCredentialKeyName(flag[1]);
485
+ }
486
+ /**
487
+ * `--token=<value>` carries its own value, which the attached-flag and
488
+ * assignment rules redact while keeping the flag's name readable to the
489
+ * passes after them.
490
+ */
491
+ function isAttachedCredentialFlag(word) {
492
+ const flag = /^--([^=]+)=/.exec(decodeShellWord(word));
493
+ return flag !== null && isCredentialKeyName(flag[1]);
494
+ }
495
+ /**
496
+ * Whether `text` spells a credential key where the other rules read one: a
497
+ * `[\w.-]` run at the start, or after a shell boundary, quote or escape (where
498
+ * a decoded word starts), that is a `--` flag or is followed by `=` or `:`.
499
+ */
500
+ function spellsCredentialKey(text) {
501
+ for (const run of text.matchAll(/[\w.-]+/g)) {
502
+ if (run.index > 0 && !/[\s;&|(\x60'"\\]/.test(text[run.index - 1]))
503
+ continue;
504
+ if (!CREDENTIAL_KEY_WORD.test(run[0]))
505
+ continue;
506
+ if (run[0].startsWith('--'))
507
+ return true;
508
+ ASSIGNMENT_OPERATOR.lastIndex = run.index + run[0].length;
509
+ if (ASSIGNMENT_OPERATOR.test(text))
510
+ return true;
511
+ }
512
+ return false;
513
+ }
514
+ /**
515
+ * Whether `word` ends in a credential flag the space-separated rule reads on
516
+ * its own: it reads boundaries in the raw text, so `\<tab>--token` (an escaped
517
+ * tab) ends in one.
518
+ */
519
+ function endsInCredentialFlag(word) {
520
+ let start = word.length;
521
+ while (start > 0 && KEY_CHARACTER.test(word[start - 1]))
522
+ start -= 1;
523
+ if (!word.startsWith('--', start))
524
+ return false;
525
+ if (start > 0 && !/[\s;&|(\x60]/.test(word[start - 1]))
526
+ return false;
527
+ return CREDENTIAL_KEY_WORD.test(word.slice(start + 2));
528
+ }
529
+ /**
530
+ * Whether to redact a word the chain reached. The other rules read credential
531
+ * keys in the raw text and, for nested commands, in decoded words, so a word
532
+ * that spells one either way (a bare credential flag among them) is left whole
533
+ * for them to read, exactly as it reads without the chain.
534
+ */
535
+ function isChainedValue(word) {
536
+ return ![word, decodeShellWord(word)].some((text) => spellsCredentialKey(text) || startsValueHeader(text));
537
+ }
538
+ // `VALUE_REDACTIONS` without the global flag, for testing a single word.
539
+ const VALUE_PATTERNS = VALUE_REDACTIONS.map(([pattern]) => new RegExp(pattern.source, pattern.flags.replace('g', '')));
540
+ /**
541
+ * Whether `text` begins a match of a value pattern that runs on into the next
542
+ * word, as a header name does (`Cookie: name=value`). Redacting that word
543
+ * would hide the value from the pattern, so the chain leaves it whole. The
544
+ * test is read off `VALUE_REDACTIONS` itself, so a pattern added later is
545
+ * covered without a list of header names here.
546
+ */
547
+ function startsValueHeader(text) {
548
+ return VALUE_PATTERNS.some((pattern) => [' name=value', ' value'].some((next) => {
549
+ const match = pattern.exec(text + next);
550
+ return (match !== null &&
551
+ match.index < text.length &&
552
+ match.index + match[0].length > text.length);
553
+ }));
554
+ }
555
+ /** Whether the word after `word` is a credential flag's value. */
556
+ function continuesCredentialChain(word) {
557
+ return isBareCredentialFlag(word) || endsInCredentialFlag(word);
558
+ }
559
+ /**
560
+ * The words to redact after `value`, a word ending at `end` that a rule has
561
+ * just taken as a credential's value. If `value` is itself a credential flag,
562
+ * the redaction hides the flag the space-separated rule would read, so its
563
+ * value, and the chain after it, are redacted here instead. This is the one
564
+ * rule for every value edit: each rule that redacts a word as a value passes
565
+ * the word here, whatever flag or tool took it. `seen` holds the ends a chain
566
+ * has already left, so a run of flags stays linear.
567
+ */
568
+ function followCredentialChain(value, end, next, seen) {
569
+ const ranges = [];
570
+ for (let word = value; continuesCredentialChain(word);) {
571
+ if (seen.has(end))
572
+ break;
573
+ seen.add(end);
574
+ const following = next(end);
575
+ if (!following)
576
+ break;
577
+ if (isAttachedCredentialFlag(following.word))
578
+ break;
579
+ if (isChainedValue(following.word))
580
+ ranges.push({ start: following.start, end: following.end, text: REDACTED });
581
+ word = following.word;
582
+ end = following.end;
583
+ }
584
+ return ranges;
585
+ }
586
+ /** Steps through `text` by shell word: a chain ends at an operator. */
587
+ function textChainCursor(text) {
588
+ return (end) => {
589
+ NEXT_SHELL_WORD.lastIndex = end;
590
+ let next = NEXT_SHELL_WORD.exec(text);
591
+ // A line continuation left in a nested script is not a word.
592
+ while (next && LINE_CONTINUATION.test(next[2]))
593
+ next = NEXT_SHELL_WORD.exec(text);
594
+ if (!next)
595
+ return undefined;
596
+ return {
597
+ word: next[2],
598
+ start: NEXT_SHELL_WORD.lastIndex - next[2].length,
599
+ end: NEXT_SHELL_WORD.lastIndex,
600
+ };
601
+ };
602
+ }
603
+ /**
604
+ * Replace each range of `text` with its text. Where ranges overlap, the
605
+ * earlier one is written and the text under the later one is dropped, so an
606
+ * overlap never puts back what either range removed.
607
+ */
608
+ function applyRanges(text, ranges) {
609
+ ranges.sort((a, b) => a.start - b.start);
610
+ let out = '';
611
+ let cursor = 0;
612
+ for (const range of ranges) {
613
+ if (range.end <= cursor)
614
+ continue;
615
+ if (range.start >= cursor)
616
+ out += text.slice(cursor, range.start) + range.text;
617
+ cursor = range.end;
618
+ }
619
+ return out + text.slice(cursor);
620
+ }
621
+ /**
622
+ * `text.replace(pattern, replace)` for a pattern whose last group is a value
623
+ * the replacement may redact. A redacted value keeps its credential chain,
624
+ * which may run on into `after`, the text that follows `text`; the words it
625
+ * redacts there are returned as ranges of `after`.
626
+ */
627
+ function replaceValues(text, pattern, replace, after = '') {
628
+ const next = textChainCursor(text + after);
629
+ const seen = new Set();
630
+ const ranges = [];
631
+ const beyond = [];
632
+ for (const match of text.matchAll(pattern)) {
633
+ const end = match.index + match[0].length;
634
+ const replaced = replace(match[0], ...match.slice(1).map((group) => group ?? ''));
635
+ if (replaced === match[0])
636
+ continue;
637
+ ranges.push({ start: match.index, end, text: replaced });
638
+ for (const range of followCredentialChain(match.at(-1) ?? '', end, next, seen)) {
639
+ if (range.start < text.length)
640
+ ranges.push(range);
641
+ else
642
+ beyond.push({
643
+ start: range.start - text.length,
644
+ end: range.end - text.length,
645
+ text: range.text,
646
+ });
647
+ }
648
+ }
649
+ return { text: applyRanges(text, ranges), after: beyond };
650
+ }
651
+ function redactCredentialFlagValues(command) {
652
+ const next = textChainCursor(command);
653
+ const ranges = [];
654
+ // A chain runs the same from any word it reaches, so one that reaches a word
655
+ // another chain already left stops there, keeping a run of flags linear.
656
+ const chained = new Set();
657
+ for (const match of command.matchAll(CREDENTIAL_FLAG)) {
658
+ const end = match.index + match[1].length + match[2].length + match[3].length;
659
+ ranges.push({ start: end - match[3].length, end, text: REDACTED });
660
+ ranges.push(...followCredentialChain(match[3], end, next, chained));
661
+ }
662
+ return applyRanges(command, ranges);
663
+ }
664
+ function applyCommandRedactions(command) {
665
+ let out = command;
666
+ for (const redaction of COMMAND_REDACTIONS)
667
+ out = typeof redaction === 'function' ? redaction(out) : out.replace(...redaction);
668
+ return out;
669
+ }
262
670
  /**
263
671
  * Redactions applied to command text, in order: heredoc bodies (bulk inline
264
672
  * file content), private-key blocks, URL userinfo credentials, Authorization
@@ -281,20 +689,8 @@ const COMMAND_REDACTIONS = [
281
689
  `$1$2$3 ${REDACTED}$4`,
282
690
  ],
283
691
  [new RegExp(PRIVATE_KEY_BLOCK_SOURCE, 'g'), REDACTED],
284
- [/\b([a-z][a-z0-9+.-]*:\/\/[^/\s:@]+:)[^@\s]+@/gi, `$1${REDACTED}@`],
285
- [/\b([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+@/gi, `$1${REDACTED}@`],
286
- [/\b(authorization\s*:\s*)(?:(?:bearer|basic|token)\s+)?\S+/gi, `$1${REDACTED}`],
287
- [new RegExp(COOKIE_HEADER_SOURCE, 'gi'), `$1${REDACTED}`],
288
- [
289
- new RegExp(`([?&#]${COMMAND_SENSITIVE_KEY_SOURCE}=)[^&\\s"'<>]+`, 'gi'),
290
- `$1${REDACTED}`,
291
- ],
292
- // Space-separated credential flags have no assignment operator or token
293
- // prefix. Keep the flag for evidence, but never its value.
294
- [
295
- new RegExp(`(^|[\\s;&|(])(--${COMMAND_SENSITIVE_KEY_SOURCE}\\s+)${SHELL_WORD_SOURCE}`, 'gi'),
296
- `$1$2${REDACTED}`,
297
- ],
692
+ ...VALUE_REDACTIONS,
693
+ redactCredentialFlagValues,
298
694
  // JSON bodies embedded in shell commands are still command text. Quoted
299
695
  // keys evade the shell-assignment scanner below, including escaped values.
300
696
  [
@@ -306,10 +702,11 @@ const COMMAND_REDACTIONS = [
306
702
  // match, not just a bare word) followed by `=`/`:` and a value that is a
307
703
  // double/single-quoted string (spaces and all) or an unquoted run up to the
308
704
  // next whitespace. Anchored at a shell-token boundary ($1, preserved) so it
309
- // never fires mid-word; the whole value — quotes included — is redacted so
705
+ // never fires mid-word (a backtick opens a substitution's first word, as
706
+ // `(` does); the whole value — quotes included — is redacted so
310
707
  // whitespace-containing passphrases can't slip through.
311
708
  [
312
- new RegExp(`(^|[\\s;&|(])(${COMMAND_SENSITIVE_KEY_SOURCE}\\s*[=:]\\s*)${SHELL_WORD_SOURCE}`, 'gi'),
709
+ new RegExp(`(^|[\\s;&|(\\x60])(${COMMAND_SENSITIVE_KEY_SOURCE}\\s*[=:]\\s*)${SHELL_WORD_SOURCE}`, 'gi'),
313
710
  `$1$2${REDACTED}`,
314
711
  ],
315
712
  // Our own CLI token (`API_KEY_PREFIX`, the prefix the dashboard mints CLI
@@ -441,10 +838,16 @@ function redactCredentialAssignments(command, inputs) {
441
838
  // retaining untouched words and also scanning unquoted multiword fields.
442
839
  let text = body.replace(new RegExp(SHELL_WORD_SOURCE, 'g'), (word) => {
443
840
  const decoded = decodeShellWord(word);
444
- const redacted = decoded.replace(fields, redactField);
841
+ const redacted = replaceValues(decoded, fields, redactField).text;
445
842
  return redacted === decoded ? word : JSON.stringify(redacted);
446
843
  });
447
- text = text.replace(fields, redactField);
844
+ // The body ends at the producer's operator, but a field's credential
845
+ // chain reaches on across a newline, as `redactCredentialFlagValues`
846
+ // does, so the chain's words after the body are edited too.
847
+ const fieldEdits = replaceValues(text, fields, redactField, assignments.slice(end));
848
+ text = fieldEdits.text;
849
+ for (const range of fieldEdits.after)
850
+ edits.push({ start: end + range.start, end: end + range.end, text: range.text });
448
851
  // A printf template may take the credential from later arguments. Once
449
852
  // its format is private, drop the argument list rather than expose the
450
853
  // values substituted into it or attempt to interpret printf formatting.
@@ -454,20 +857,33 @@ function redactCredentialAssignments(command, inputs) {
454
857
  edits.push({ start: producerStart, end, text });
455
858
  producerStart = null;
456
859
  };
860
+ // Decided at each command's first word, so a file argument named echo under
861
+ // a program that never runs its arguments is not mistaken for a producer.
862
+ // Wrappers are not looked past here, which only errs toward redaction.
863
+ let commandSeen = false;
864
+ let argumentsInert = false;
457
865
  for (const word of words) {
458
- if (/^[;&|()\n]$/.test(word[0]))
866
+ if (/^[;&|()\n]$/.test(word[0])) {
459
867
  finishProducer(word.index);
460
- else if (producerStart === null &&
461
- ['echo', 'printf'].includes(decodeShellWord(word[0]).split(/[\\/]/).at(-1) ?? '')) {
868
+ commandSeen = false;
869
+ argumentsInert = false;
870
+ continue;
871
+ }
872
+ const decoded = decodeShellWord(word[0]);
873
+ if (!commandSeen && !/^\w+\+?=/.test(decoded)) {
874
+ commandSeen = true;
875
+ argumentsInert = isNonDispatching(assignments, word.index, decoded);
876
+ }
877
+ const tool = invokedTool(decoded);
878
+ // A backtick substitution runs its own command, whatever the outer one is.
879
+ const inert = argumentsInert && !word[0].includes('`');
880
+ if (producerStart === null && !inert && ['echo', 'printf'].includes(tool)) {
462
881
  producerStart = word.index + word[0].length;
463
- producerPrintf = decodeShellWord(word[0]).split(/[\\/]/).at(-1) === 'printf';
882
+ producerPrintf = tool === 'printf';
464
883
  }
465
884
  }
466
885
  finishProducer(assignments.length);
467
- let result = assignments;
468
- for (const edit of edits.reverse())
469
- result = result.slice(0, edit.start) + edit.text + result.slice(edit.end);
470
- return result;
886
+ return applyRanges(assignments, edits);
471
887
  }
472
888
  /** Redact known tool credential flags without matching unrelated short flags. */
473
889
  function redactToolArguments(command, inputs) {
@@ -477,6 +893,11 @@ function redactToolArguments(command, inputs) {
477
893
  let options = true;
478
894
  let opensslPasswd = false;
479
895
  let opensslSubcommandRead = false;
896
+ // Set once a wrapper in the current command receives an option it does not
897
+ // model, which may have taken the next word as its value.
898
+ let wrapperOptionUnknown = false;
899
+ // The current command is a program that never runs its arguments.
900
+ let argumentsInert = false;
480
901
  const variables = new Map();
481
902
  const expansions = [];
482
903
  const resolveVariable = (contents, depth = 0) => {
@@ -485,7 +906,7 @@ function redactToolArguments(command, inputs) {
485
906
  return contents.replace(CURL_EXPANSION, (_, name) => resolveVariable(variables.get(name)?.at(-1)?.contents ?? '', depth + 1));
486
907
  };
487
908
  const finishCurl = () => {
488
- const privateNames = new Set();
909
+ const privateNames = new Set(inputs?.curlVariables);
489
910
  for (const expansion of expansions) {
490
911
  if (CURL_URL_FIELD.test(expansion.field) &&
491
912
  !PROXY_USERINFO.test(expansion.resolved) &&
@@ -511,6 +932,8 @@ function redactToolArguments(command, inputs) {
511
932
  });
512
933
  }
513
934
  }
935
+ for (const name of privateNames)
936
+ inputs?.curlVariables.add(name);
514
937
  variables.clear();
515
938
  expansions.length = 0;
516
939
  };
@@ -524,6 +947,8 @@ function redactToolArguments(command, inputs) {
524
947
  client = null;
525
948
  opensslPasswd = false;
526
949
  opensslSubcommandRead = false;
950
+ wrapperOptionUnknown = false;
951
+ argumentsInert = false;
527
952
  options = true;
528
953
  continue;
529
954
  }
@@ -576,7 +1001,11 @@ function redactToolArguments(command, inputs) {
576
1001
  }
577
1002
  }
578
1003
  const executable = invokedTool(decoded);
579
- if (WRAPPER_VALUE_FLAGS[executable] && !CREDENTIAL_TOOLS.includes(client ?? '')) {
1004
+ // A backtick substitution runs its own command, whatever the outer one is.
1005
+ const inert = argumentsInert && !raw.includes('`');
1006
+ if (!inert &&
1007
+ WRAPPER_VALUE_FLAGS[executable] &&
1008
+ !CREDENTIAL_TOOLS.includes(client ?? '')) {
580
1009
  client = executable;
581
1010
  options = true;
582
1011
  continue;
@@ -584,7 +1013,8 @@ function redactToolArguments(command, inputs) {
584
1013
  // Dispatching tools may take their own options and positional arguments.
585
1014
  // Once a credential-consuming tool appears, its subsequent flags still carry
586
1015
  // credentials; a wrapper's -u or timeout must not hide that client.
587
- if (CREDENTIAL_TOOLS.includes(executable) &&
1016
+ if (!inert &&
1017
+ CREDENTIAL_TOOLS.includes(executable) &&
588
1018
  !CREDENTIAL_TOOLS.includes(client ?? '')) {
589
1019
  client = executable;
590
1020
  opensslPasswd = false;
@@ -592,11 +1022,30 @@ function redactToolArguments(command, inputs) {
592
1022
  options = true;
593
1023
  continue;
594
1024
  }
1025
+ // A wrapper's first operand (its modelled valued options were consumed
1026
+ // above) is the program it runs, which decides whether later words can be
1027
+ // invocations. After an option the table does not model, that operand may
1028
+ // be the option's value instead, so the wrapper keeps dispatching.
1029
+ if (client !== null && WRAPPER_VALUE_FLAGS[client]) {
1030
+ if (options && decoded.startsWith('-')) {
1031
+ if (decoded !== '--' &&
1032
+ !decoded.includes('=') &&
1033
+ !WRAPPER_SWITCHES[client]?.test(decoded))
1034
+ wrapperOptionUnknown = true;
1035
+ }
1036
+ else if (!wrapperOptionUnknown) {
1037
+ client = executable;
1038
+ argumentsInert = isNonDispatching(command, word.index, decoded);
1039
+ options = true;
1040
+ continue;
1041
+ }
1042
+ }
595
1043
  if (client === null) {
596
1044
  if (/^[\w]+=.*/.test(decoded) ||
597
1045
  ['env', 'sudo', 'command', 'exec', 'nohup'].includes(decoded))
598
1046
  continue;
599
1047
  client = invokedTool(decoded);
1048
+ argumentsInert = isNonDispatching(command, word.index, decoded);
600
1049
  continue;
601
1050
  }
602
1051
  if (client === 'openssl' && !opensslSubcommandRead && !decoded.startsWith('-')) {
@@ -804,6 +1253,46 @@ function redactToolArguments(command, inputs) {
804
1253
  }
805
1254
  }
806
1255
  finishCurl();
1256
+ // Every edit above replaces a whole word, and a word an edit replaces is
1257
+ // gone before `redactCredentialFlagValues` reads the text. Where that word
1258
+ // was a credential flag, the flag's value would escape, so the chain is
1259
+ // followed here for every edit, whichever tool or flag made it.
1260
+ const wordAt = new Map(words.map((word, at) => [word.index, at]));
1261
+ const wordEndingAt = new Map(words.map((word, at) => [word.index + word[0].length, at]));
1262
+ const nextWord = (end) => {
1263
+ let next = wordEndingAt.get(end);
1264
+ if (next === undefined)
1265
+ return undefined;
1266
+ next += 1;
1267
+ while (words[next] && /^(?:\n|\\\r?\n)$/.test(words[next][0]))
1268
+ next += 1;
1269
+ const following = words[next];
1270
+ if (!following || /^[;&|()\n]$/.test(following[0]))
1271
+ return undefined;
1272
+ return {
1273
+ word: following[0],
1274
+ start: following.index,
1275
+ end: following.index + following[0].length,
1276
+ };
1277
+ };
1278
+ const chainedWords = new Set();
1279
+ const chainEdits = edits.flatMap((edit) => {
1280
+ const at = wordAt.get(edit.start);
1281
+ const word = at === undefined ? undefined : words[at];
1282
+ if (!word || edit.end !== word.index + word[0].length)
1283
+ return [];
1284
+ return followCredentialChain(word[0], edit.end, nextWord, chainedWords);
1285
+ });
1286
+ // A chain edit either matches an edit's word or misses it; on a match the
1287
+ // whole-word redaction replaces it, so the chain only ever adds.
1288
+ const editAt = new Map(edits.map((edit, at) => [edit.start, at]));
1289
+ for (const chainEdit of chainEdits) {
1290
+ const same = editAt.get(chainEdit.start);
1291
+ if (same === undefined)
1292
+ edits.push(chainEdit);
1293
+ else
1294
+ edits[same] = chainEdit;
1295
+ }
807
1296
  edits.sort((a, b) => a.start - b.start);
808
1297
  for (const edit of edits.reverse())
809
1298
  command = command.slice(0, edit.start) + edit.text + command.slice(edit.end);
@@ -815,6 +1304,7 @@ function getToolInputs(command, depth = 0) {
815
1304
  curlConfig: depth > 8,
816
1305
  env: new Set(),
817
1306
  passSources: new Set(),
1307
+ curlVariables: new Set(),
818
1308
  };
819
1309
  if (depth > 8)
820
1310
  return inputs;
@@ -831,6 +1321,8 @@ function getToolInputs(command, depth = 0) {
831
1321
  inputs.env.add(key);
832
1322
  for (const source of nested.passSources)
833
1323
  inputs.passSources.add(source);
1324
+ for (const name of nested.curlVariables)
1325
+ inputs.curlVariables.add(name);
834
1326
  }
835
1327
  if (inputs.curlConfig) {
836
1328
  for (const word of words) {
@@ -844,21 +1336,97 @@ function getToolInputs(command, depth = 0) {
844
1336
  return inputs;
845
1337
  }
846
1338
  /** Apply the same boundary to scripts carried inside quoted shell words. */
847
- function redactShellSyntax(command, depth = 0, inheritedInputs) {
1339
+ /** The part of `before` that `after` replaced: what a redaction removed. */
1340
+ function changedPart(before, after) {
1341
+ let start = 0;
1342
+ while (start < before.length && before[start] === after[start])
1343
+ start += 1;
1344
+ let end = 0;
1345
+ while (end < before.length - start &&
1346
+ before[before.length - 1 - end] === after[after.length - 1 - end])
1347
+ end += 1;
1348
+ return before.slice(start, before.length - end);
1349
+ }
1350
+ function redactShellSyntax(command, depth = 0, inheritedInputs, valuesFirst = true) {
848
1351
  // Nested shell/eval wrappers cannot create unbounded recursive scanning.
849
1352
  if (depth > 8)
850
1353
  return REDACTED;
851
- command = redactCommandSubstitutions(command);
1354
+ // Redacting a backtick body can erase the reference (`{{VALUE}}`) that
1355
+ // marks an outer assignment private, so read its inputs from the command
1356
+ // as written too.
1357
+ const writtenInputs = command.includes('`')
1358
+ ? withInheritedInputs(backtickAwareInputs(command), inheritedInputs)
1359
+ : inheritedInputs;
1360
+ // Rewriting part of a backtick body can separate a header from its value, so
1361
+ // the value patterns redact such a match before any body is rewritten. Only
1362
+ // such a match: one whose value the later passes remove anyway is left to
1363
+ // them, since the word-based rules see the rest of its word (`"Cookie: a"b`).
1364
+ // Redacting one match early changes what the later passes see, so a value
1365
+ // still in their output is redacted early too and the passes run again.
1366
+ if (valuesFirst && command.includes('`') && !command.includes('\uE000')) {
1367
+ const values = valueRedactionTargets(command);
1368
+ if (values.length > 0) {
1369
+ const withoutValuesFirst = redactShellSyntax(command, depth, inheritedInputs, false);
1370
+ const early = new Set(values.filter((value) => withoutValuesFirst.includes(value)));
1371
+ for (let round = 0;; round += 1) {
1372
+ const out = redactAfterValues(redactValuesFirst(command, round < 3 ? early : new Set(values)), depth, writtenInputs);
1373
+ const stranded = values.filter((value) => !early.has(value) && out.includes(value));
1374
+ if (stranded.length === 0 || round === 3)
1375
+ return out;
1376
+ for (const value of stranded)
1377
+ early.add(value);
1378
+ }
1379
+ }
1380
+ }
1381
+ return redactAfterValues(command, depth, writtenInputs, valuesFirst);
1382
+ }
1383
+ /** The values the `VALUE_REDACTIONS` would remove from text with no backtick in the match. */
1384
+ function valueRedactionTargets(command) {
1385
+ const values = [];
1386
+ for (const [pattern, replacement] of VALUE_REDACTIONS) {
1387
+ const single = new RegExp(pattern.source, pattern.flags.replace('g', ''));
1388
+ for (const match of command.match(pattern) ?? []) {
1389
+ if (/[`\uE000\uE001]/.test(match))
1390
+ continue;
1391
+ const value = changedPart(match, match.replace(single, replacement));
1392
+ if (value !== '' && !values.includes(value))
1393
+ values.push(value);
1394
+ }
1395
+ }
1396
+ return values;
1397
+ }
1398
+ /**
1399
+ * Apply the value patterns whose value is in `early`, holding every other
1400
+ * match aside so no later pattern can match part of it. A match that spans a
1401
+ * backtick is always held: the substitution pass owns that boundary, and an
1402
+ * unterminated one must still fail closed. A reference this erases is still
1403
+ * in the caller's `writtenInputs`.
1404
+ */
1405
+ function redactValuesFirst(command, early) {
1406
+ const held = [];
1407
+ let whole = command;
1408
+ for (const [pattern, replacement] of VALUE_REDACTIONS) {
1409
+ const single = new RegExp(pattern.source, pattern.flags.replace('g', ''));
1410
+ whole = whole.replace(pattern, (match) => {
1411
+ const redacted = match.replace(single, replacement);
1412
+ if (!/[`\uE000\uE001]/.test(match) && early.has(changedPart(match, redacted)))
1413
+ return redacted;
1414
+ held.push(match);
1415
+ return `\uE000${held.length - 1}\uE001`;
1416
+ });
1417
+ }
1418
+ // A held match can contain an earlier one; each refers only to earlier ones.
1419
+ while (whole.includes('\uE000'))
1420
+ whole = whole.replace(/\uE000(\d+)\uE001/g, (_, index) => held[Number(index)]);
1421
+ return whole;
1422
+ }
1423
+ function redactAfterValues(command, depth, writtenInputs, valuesFirst = true) {
1424
+ command = redactCommandSubstitutions(command, depth, writtenInputs, valuesFirst);
852
1425
  // Nested scripts may be dispatched by any tool. Scan decoded compound
853
1426
  // words instead of maintaining another list of interpreter/wrapper flags.
854
1427
  // JSON and sensitive assignments already have structural/whole-word passes;
855
1428
  // treating their values as commands would lose evidence or word boundaries.
856
- const inputs = getToolInputs(command);
857
- inputs.curlConfig ||= inheritedInputs?.curlConfig ?? false;
858
- for (const key of inheritedInputs?.env ?? [])
859
- inputs.env.add(key);
860
- for (const source of inheritedInputs?.passSources ?? [])
861
- inputs.passSources.add(source);
1429
+ const inputs = withInheritedInputs(backtickAwareInputs(command), writtenInputs);
862
1430
  const sensitiveAssignment = new RegExp(`^(${COMMAND_SENSITIVE_KEY_SOURCE})(\\+?=)`, 'i');
863
1431
  // Resolve credential references before nested word redaction can erase the
864
1432
  // expansion marker while leaving its original variable definition behind.
@@ -882,13 +1450,10 @@ function redactShellSyntax(command, depth = 0, inheritedInputs) {
882
1450
  // structural JSON pass instead of the nested-command pass.
883
1451
  }
884
1452
  }
885
- const redacted = redactShellSyntax(decoded, depth + 1, inputs);
1453
+ const redacted = redactShellSyntax(decoded, depth + 1, inputs, valuesFirst);
886
1454
  return redacted === decoded ? word : JSON.stringify(redacted);
887
1455
  });
888
- out = redactCredentialAssignments(out, inputs);
889
- for (const [pattern, replacement] of COMMAND_REDACTIONS)
890
- out = out.replace(pattern, replacement);
891
- return out;
1456
+ return applyCommandRedactions(redactCredentialAssignments(out, inputs));
892
1457
  }
893
1458
  /**
894
1459
  * Redact common secret shapes from a shell command and cap its length so