@smthrs/journal 0.0.0-stage → 1.0.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +149 -0
- package/LICENSE +21 -0
- package/README.md +206 -2
- package/dist/cjs/CallFact.d.ts +129 -0
- package/dist/cjs/CallFact.d.ts.map +1 -0
- package/dist/cjs/CallFact.js +106 -0
- package/dist/cjs/CallFact.js.map +7 -0
- package/dist/cjs/Consensus.d.ts +440 -0
- package/dist/cjs/Consensus.d.ts.map +1 -0
- package/dist/cjs/Consensus.js +251 -0
- package/dist/cjs/Consensus.js.map +7 -0
- package/dist/cjs/EngineEvent.d.ts +890 -0
- package/dist/cjs/EngineEvent.d.ts.map +1 -0
- package/dist/cjs/EngineEvent.js +383 -0
- package/dist/cjs/EngineEvent.js.map +7 -0
- package/dist/cjs/ExecutionFact.d.ts +216 -0
- package/dist/cjs/ExecutionFact.d.ts.map +1 -0
- package/dist/cjs/ExecutionFact.js +303 -0
- package/dist/cjs/ExecutionFact.js.map +7 -0
- package/dist/cjs/Journal.d.ts +617 -0
- package/dist/cjs/Journal.d.ts.map +1 -0
- package/dist/cjs/Journal.js +190 -0
- package/dist/cjs/Journal.js.map +7 -0
- package/dist/cjs/JournalEvent.d.ts +421 -0
- package/dist/cjs/JournalEvent.d.ts.map +1 -0
- package/dist/cjs/JournalEvent.js +281 -0
- package/dist/cjs/JournalEvent.js.map +7 -0
- package/dist/cjs/JournalGeneration.d.ts +34 -0
- package/dist/cjs/JournalGeneration.d.ts.map +1 -0
- package/dist/cjs/JournalGeneration.js +72 -0
- package/dist/cjs/JournalGeneration.js.map +7 -0
- package/dist/cjs/JournalMetrics.d.ts +72 -0
- package/dist/cjs/JournalMetrics.d.ts.map +1 -0
- package/dist/cjs/JournalMetrics.js +55 -0
- package/dist/cjs/JournalMetrics.js.map +7 -0
- package/dist/cjs/Migrations.d.ts +42 -0
- package/dist/cjs/Migrations.d.ts.map +1 -0
- package/dist/cjs/Migrations.js +58 -0
- package/dist/cjs/Migrations.js.map +7 -0
- package/dist/cjs/OwnerId.d.ts +40 -0
- package/dist/cjs/OwnerId.d.ts.map +1 -0
- package/dist/cjs/OwnerId.js +40 -0
- package/dist/cjs/OwnerId.js.map +7 -0
- package/dist/cjs/Projection.d.ts +29 -0
- package/dist/cjs/Projection.d.ts.map +1 -0
- package/dist/cjs/Projection.js +25 -0
- package/dist/cjs/Projection.js.map +7 -0
- package/dist/cjs/RedactedLogger.d.ts +121 -0
- package/dist/cjs/RedactedLogger.d.ts.map +1 -0
- package/dist/cjs/RedactedLogger.js +235 -0
- package/dist/cjs/RedactedLogger.js.map +7 -0
- package/dist/cjs/Redaction.d.ts +350 -0
- package/dist/cjs/Redaction.d.ts.map +1 -0
- package/dist/cjs/Redaction.js +928 -0
- package/dist/cjs/Redaction.js.map +7 -0
- package/dist/cjs/SqlConsensus.d.ts +44 -0
- package/dist/cjs/SqlConsensus.d.ts.map +1 -0
- package/dist/cjs/SqlConsensus.js +243 -0
- package/dist/cjs/SqlConsensus.js.map +7 -0
- package/dist/cjs/SqlJournal.d.ts +173 -0
- package/dist/cjs/SqlJournal.d.ts.map +1 -0
- package/dist/cjs/SqlJournal.js +1568 -0
- package/dist/cjs/SqlJournal.js.map +7 -0
- package/dist/cjs/StepFact.d.ts +66 -0
- package/dist/cjs/StepFact.d.ts.map +1 -0
- package/dist/cjs/StepFact.js +68 -0
- package/dist/cjs/StepFact.js.map +7 -0
- package/dist/cjs/index.d.ts +95 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +63 -0
- package/dist/cjs/index.js.map +7 -0
- package/dist/cjs/internal/JournalInternalFault.d.ts +37 -0
- package/dist/cjs/internal/JournalInternalFault.d.ts.map +1 -0
- package/dist/cjs/internal/JournalInternalFault.js +43 -0
- package/dist/cjs/internal/JournalInternalFault.js.map +7 -0
- package/dist/cjs/migrations/0001_initial.d.ts +15 -0
- package/dist/cjs/migrations/0001_initial.d.ts.map +1 -0
- package/dist/cjs/migrations/0001_initial.js +54 -0
- package/dist/cjs/migrations/0001_initial.js.map +7 -0
- package/dist/cjs/migrations/0002_checkpoints.d.ts +27 -0
- package/dist/cjs/migrations/0002_checkpoints.d.ts.map +1 -0
- package/dist/cjs/migrations/0002_checkpoints.js +48 -0
- package/dist/cjs/migrations/0002_checkpoints.js.map +7 -0
- package/dist/cjs/migrations/0003_startup_index.d.ts +15 -0
- package/dist/cjs/migrations/0003_startup_index.d.ts.map +1 -0
- package/dist/cjs/migrations/0003_startup_index.js +40 -0
- package/dist/cjs/migrations/0003_startup_index.js.map +7 -0
- package/dist/cjs/migrations/0004_dedup.d.ts +15 -0
- package/dist/cjs/migrations/0004_dedup.d.ts.map +1 -0
- package/dist/cjs/migrations/0004_dedup.js +57 -0
- package/dist/cjs/migrations/0004_dedup.js.map +7 -0
- package/dist/cjs/migrations/0005_run_event_type.d.ts +15 -0
- package/dist/cjs/migrations/0005_run_event_type.d.ts.map +1 -0
- package/dist/cjs/migrations/0005_run_event_type.js +40 -0
- package/dist/cjs/migrations/0005_run_event_type.js.map +7 -0
- package/dist/cjs/migrations/0006_consensus.d.ts +22 -0
- package/dist/cjs/migrations/0006_consensus.d.ts.map +1 -0
- package/dist/cjs/migrations/0006_consensus.js +84 -0
- package/dist/cjs/migrations/0006_consensus.js.map +7 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/test/Notifying.d.ts +65 -0
- package/dist/cjs/test/Notifying.d.ts.map +1 -0
- package/dist/cjs/test/Notifying.js +62 -0
- package/dist/cjs/test/Notifying.js.map +7 -0
- package/dist/cjs/test/TestJournal.d.ts +43 -0
- package/dist/cjs/test/TestJournal.d.ts.map +1 -0
- package/dist/cjs/test/TestJournal.js +43 -0
- package/dist/cjs/test/TestJournal.js.map +7 -0
- package/dist/esm/CallFact.d.ts +129 -0
- package/dist/esm/CallFact.d.ts.map +1 -0
- package/dist/esm/CallFact.js +96 -0
- package/dist/esm/CallFact.js.map +1 -0
- package/dist/esm/Consensus.d.ts +440 -0
- package/dist/esm/Consensus.d.ts.map +1 -0
- package/dist/esm/Consensus.js +454 -0
- package/dist/esm/Consensus.js.map +1 -0
- package/dist/esm/EngineEvent.d.ts +890 -0
- package/dist/esm/EngineEvent.d.ts.map +1 -0
- package/dist/esm/EngineEvent.js +614 -0
- package/dist/esm/EngineEvent.js.map +1 -0
- package/dist/esm/ExecutionFact.d.ts +216 -0
- package/dist/esm/ExecutionFact.d.ts.map +1 -0
- package/dist/esm/ExecutionFact.js +374 -0
- package/dist/esm/ExecutionFact.js.map +1 -0
- package/dist/esm/Journal.d.ts +617 -0
- package/dist/esm/Journal.d.ts.map +1 -0
- package/dist/esm/Journal.js +323 -0
- package/dist/esm/Journal.js.map +1 -0
- package/dist/esm/JournalEvent.d.ts +421 -0
- package/dist/esm/JournalEvent.d.ts.map +1 -0
- package/dist/esm/JournalEvent.js +517 -0
- package/dist/esm/JournalEvent.js.map +1 -0
- package/dist/esm/JournalGeneration.d.ts +34 -0
- package/dist/esm/JournalGeneration.d.ts.map +1 -0
- package/dist/esm/JournalGeneration.js +65 -0
- package/dist/esm/JournalGeneration.js.map +1 -0
- package/dist/esm/JournalMetrics.d.ts +72 -0
- package/dist/esm/JournalMetrics.d.ts.map +1 -0
- package/dist/esm/JournalMetrics.js +79 -0
- package/dist/esm/JournalMetrics.js.map +1 -0
- package/dist/esm/Migrations.d.ts +42 -0
- package/dist/esm/Migrations.d.ts.map +1 -0
- package/dist/esm/Migrations.js +59 -0
- package/dist/esm/Migrations.js.map +1 -0
- package/dist/esm/OwnerId.d.ts +40 -0
- package/dist/esm/OwnerId.d.ts.map +1 -0
- package/dist/esm/OwnerId.js +33 -0
- package/dist/esm/OwnerId.js.map +1 -0
- package/dist/esm/Projection.d.ts +29 -0
- package/dist/esm/Projection.d.ts.map +1 -0
- package/dist/esm/Projection.js +13 -0
- package/dist/esm/Projection.js.map +1 -0
- package/dist/esm/RedactedLogger.d.ts +121 -0
- package/dist/esm/RedactedLogger.d.ts.map +1 -0
- package/dist/esm/RedactedLogger.js +445 -0
- package/dist/esm/RedactedLogger.js.map +1 -0
- package/dist/esm/Redaction.d.ts +350 -0
- package/dist/esm/Redaction.d.ts.map +1 -0
- package/dist/esm/Redaction.js +1584 -0
- package/dist/esm/Redaction.js.map +1 -0
- package/dist/esm/SqlConsensus.d.ts +44 -0
- package/dist/esm/SqlConsensus.d.ts.map +1 -0
- package/dist/esm/SqlConsensus.js +252 -0
- package/dist/esm/SqlConsensus.js.map +1 -0
- package/dist/esm/SqlJournal.d.ts +173 -0
- package/dist/esm/SqlJournal.d.ts.map +1 -0
- package/dist/esm/SqlJournal.js +1858 -0
- package/dist/esm/SqlJournal.js.map +1 -0
- package/dist/esm/StepFact.d.ts +66 -0
- package/dist/esm/StepFact.d.ts.map +1 -0
- package/dist/esm/StepFact.js +50 -0
- package/dist/esm/StepFact.js.map +1 -0
- package/dist/esm/index.d.ts +95 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +95 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/internal/JournalInternalFault.d.ts +37 -0
- package/dist/esm/internal/JournalInternalFault.d.ts.map +1 -0
- package/dist/esm/internal/JournalInternalFault.js +28 -0
- package/dist/esm/internal/JournalInternalFault.js.map +1 -0
- package/dist/esm/migrations/0001_initial.d.ts +15 -0
- package/dist/esm/migrations/0001_initial.d.ts.map +1 -0
- package/dist/esm/migrations/0001_initial.js +32 -0
- package/dist/esm/migrations/0001_initial.js.map +1 -0
- package/dist/esm/migrations/0002_checkpoints.d.ts +27 -0
- package/dist/esm/migrations/0002_checkpoints.d.ts.map +1 -0
- package/dist/esm/migrations/0002_checkpoints.js +38 -0
- package/dist/esm/migrations/0002_checkpoints.js.map +1 -0
- package/dist/esm/migrations/0003_startup_index.d.ts +15 -0
- package/dist/esm/migrations/0003_startup_index.d.ts.map +1 -0
- package/dist/esm/migrations/0003_startup_index.js +18 -0
- package/dist/esm/migrations/0003_startup_index.js.map +1 -0
- package/dist/esm/migrations/0004_dedup.d.ts +15 -0
- package/dist/esm/migrations/0004_dedup.d.ts.map +1 -0
- package/dist/esm/migrations/0004_dedup.js +38 -0
- package/dist/esm/migrations/0004_dedup.js.map +1 -0
- package/dist/esm/migrations/0005_run_event_type.d.ts +15 -0
- package/dist/esm/migrations/0005_run_event_type.d.ts.map +1 -0
- package/dist/esm/migrations/0005_run_event_type.js +18 -0
- package/dist/esm/migrations/0005_run_event_type.js.map +1 -0
- package/dist/esm/migrations/0006_consensus.d.ts +22 -0
- package/dist/esm/migrations/0006_consensus.d.ts.map +1 -0
- package/dist/esm/migrations/0006_consensus.js +69 -0
- package/dist/esm/migrations/0006_consensus.js.map +1 -0
- package/dist/esm/test/Notifying.d.ts +65 -0
- package/dist/esm/test/Notifying.d.ts.map +1 -0
- package/dist/esm/test/Notifying.js +62 -0
- package/dist/esm/test/Notifying.js.map +1 -0
- package/dist/esm/test/TestJournal.d.ts +43 -0
- package/dist/esm/test/TestJournal.d.ts.map +1 -0
- package/dist/esm/test/TestJournal.js +27 -0
- package/dist/esm/test/TestJournal.js.map +1 -0
- package/docs/README.md +164 -0
- package/docs/api.md +639 -0
- package/docs/concepts/compaction.md +157 -0
- package/docs/concepts/idempotency.md +127 -0
- package/docs/concepts/owner-fence.md +124 -0
- package/docs/concepts/redaction.md +218 -0
- package/docs/concepts/state-event-authority.md +159 -0
- package/docs/concepts/two-channels.md +167 -0
- package/docs/guides/commit-state-and-entry.md +114 -0
- package/docs/guides/compact-a-run.md +134 -0
- package/docs/guides/fold-a-projection.md +106 -0
- package/docs/guides/read-a-run.md +139 -0
- package/docs/guides/redact-log-output.md +94 -0
- package/docs/guides/testing.md +124 -0
- package/docs/guides/write-lifecycle-events.md +110 -0
- package/docs/installation.md +133 -0
- package/docs/quickstart.md +143 -0
- package/docs/troubleshooting.md +214 -0
- package/package.json +263 -3
- package/src/CallFact.ts +117 -0
- package/src/Consensus.ts +683 -0
- package/src/EngineEvent.ts +823 -0
- package/src/ExecutionFact.ts +433 -0
- package/src/Journal.ts +688 -0
- package/src/JournalEvent.ts +654 -0
- package/src/JournalGeneration.ts +77 -0
- package/src/JournalMetrics.ts +89 -0
- package/src/Migrations.ts +62 -0
- package/src/OwnerId.ts +42 -0
- package/src/Projection.ts +31 -0
- package/src/RedactedLogger.ts +483 -0
- package/src/Redaction.ts +1784 -0
- package/src/SqlConsensus.ts +344 -0
- package/src/SqlJournal.ts +2732 -0
- package/src/StepFact.ts +69 -0
- package/src/index.ts +109 -0
- package/src/internal/JournalInternalFault.ts +40 -0
- package/src/migrations/0001_initial.ts +41 -0
- package/src/migrations/0002_checkpoints.ts +46 -0
- package/src/migrations/0003_startup_index.ts +19 -0
- package/src/migrations/0004_dedup.ts +43 -0
- package/src/migrations/0005_run_event_type.ts +19 -0
- package/src/migrations/0006_consensus.ts +83 -0
- package/src/test/Notifying.ts +101 -0
- package/src/test/TestJournal.ts +52 -0
|
@@ -0,0 +1,1584 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Payload redaction for durable journal writes.
|
|
3
|
+
*
|
|
4
|
+
* The journal is permanent and broadly readable: every entry is replayed to
|
|
5
|
+
* sync subscribers and to time-travel consumers. A secret that reaches
|
|
6
|
+
* `payload_json` is therefore not a transient leak, so redaction happens once,
|
|
7
|
+
* on the journal write path, rather than at each reader.
|
|
8
|
+
*
|
|
9
|
+
* Redaction is an **observability** concern and is confined to journal events
|
|
10
|
+
* and export/display surfaces. It is deliberately not applied to executable
|
|
11
|
+
* state, `flows_runs.state_json`, attempt checkpoints, errors, outcomes, and
|
|
12
|
+
* cache results, because those are decoded and re-entered on resume: a
|
|
13
|
+
* placeholder there resumes the flow with the wrong data, and replacing a
|
|
14
|
+
* non-string value with a placeholder string makes the persisted state fail
|
|
15
|
+
* schema decode outright, leaving the run undrivable (issue #72). A value
|
|
16
|
+
* that must never reach durable executable state is a `Redacted` field in the
|
|
17
|
+
* caller's own schema, not a name-suffix guess made at the storage seam.
|
|
18
|
+
*
|
|
19
|
+
* The rule set is a best-effort textual net over credential shapes seen in
|
|
20
|
+
* real bug reports, plus structural redaction by sensitive field name. It is
|
|
21
|
+
* finite, so a value that must never persist belongs in a `Redacted` field of
|
|
22
|
+
* the caller's own schema. See https://journal.smithers.sh/concepts/redaction/
|
|
23
|
+
* for the rule set and how to replace it.
|
|
24
|
+
*
|
|
25
|
+
* @since 0.1.0
|
|
26
|
+
*/
|
|
27
|
+
import * as Result from "effect/Result";
|
|
28
|
+
import * as Schema from "effect/Schema";
|
|
29
|
+
/** JSON text carrying an arbitrary decoded value. */
|
|
30
|
+
const UnknownFromJsonString = Schema.fromJsonString(Schema.Unknown);
|
|
31
|
+
/**
|
|
32
|
+
* The placeholder written in place of a redacted value.
|
|
33
|
+
*
|
|
34
|
+
* @since 0.1.0
|
|
35
|
+
* @category constants
|
|
36
|
+
*/
|
|
37
|
+
export const placeholder = "[REDACTED]";
|
|
38
|
+
const isQuote = (char) => char === "\"" || char === "'" || char === "`";
|
|
39
|
+
const closers = { "{": "}", "[": "]", "(": ")" };
|
|
40
|
+
/**
|
|
41
|
+
* Whether the last {@link skipQuoted} or {@link skipContainer} found its
|
|
42
|
+
* closer. Both return the end of the text for a value that never closes, and
|
|
43
|
+
* for one that closes on the text's last character, so the end alone cannot
|
|
44
|
+
* say which.
|
|
45
|
+
*/
|
|
46
|
+
let lastClosed = false;
|
|
47
|
+
/**
|
|
48
|
+
* The index just past the quote that closes the string opened at `open`. A
|
|
49
|
+
* quote nested n strings deep sits behind `depth` = 2^n - 1 backslashes, and
|
|
50
|
+
* the quote that closes it has that count modulo 2^(n+1). A string that is
|
|
51
|
+
* never closed, cut by a bound or by its producer, runs to the end of the
|
|
52
|
+
* text. A caller that knows the backslashes were doubled again passes a larger
|
|
53
|
+
* `modulus`.
|
|
54
|
+
*/
|
|
55
|
+
const skipQuoted = (text, open, depth, modulus = 2 * (depth + 1), limit = text.length) => {
|
|
56
|
+
let end = open + 1;
|
|
57
|
+
while (end < limit) {
|
|
58
|
+
if (text[end] === text[open]) {
|
|
59
|
+
let slashes = 0;
|
|
60
|
+
for (let i = end - 1; i > open && text[i] === "\\"; i--)
|
|
61
|
+
slashes++;
|
|
62
|
+
if (slashes % modulus === depth) {
|
|
63
|
+
lastClosed = true;
|
|
64
|
+
return end + 1;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
end++;
|
|
68
|
+
}
|
|
69
|
+
lastClosed = false;
|
|
70
|
+
return end;
|
|
71
|
+
};
|
|
72
|
+
/**
|
|
73
|
+
* The index just past the bracket that balances the one at `open`, skipping
|
|
74
|
+
* quoted strings at any escape depth. An unbalanced container runs to the end
|
|
75
|
+
* of the text.
|
|
76
|
+
*/
|
|
77
|
+
const skipContainer = (text, open, limit = text.length) => {
|
|
78
|
+
const pending = [];
|
|
79
|
+
let end = open;
|
|
80
|
+
while (end < limit) {
|
|
81
|
+
const char = text[end];
|
|
82
|
+
if (isQuote(char)) {
|
|
83
|
+
let depth = 0;
|
|
84
|
+
while (text[end - 1 - depth] === "\\")
|
|
85
|
+
depth++;
|
|
86
|
+
end = skipQuoted(text, end, depth, 2 * (depth + 1), limit);
|
|
87
|
+
if (!lastClosed)
|
|
88
|
+
return end;
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
const closer = closers[char];
|
|
92
|
+
if (closer !== undefined)
|
|
93
|
+
pending.push(closer);
|
|
94
|
+
else if (char === pending.at(-1)) {
|
|
95
|
+
pending.pop();
|
|
96
|
+
if (pending.length === 0) {
|
|
97
|
+
lastClosed = true;
|
|
98
|
+
return end + 1;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
end++;
|
|
102
|
+
}
|
|
103
|
+
lastClosed = false;
|
|
104
|
+
return end;
|
|
105
|
+
};
|
|
106
|
+
/**
|
|
107
|
+
* The escape depth of a quote behind `slashes` backslashes: 2^k - 1 for its k
|
|
108
|
+
* trailing escaped levels, so `\\"` (an escaped backslash, then a quote) is
|
|
109
|
+
* depth 0 and `\\\"` is depth 1.
|
|
110
|
+
*/
|
|
111
|
+
const escapeDepth = (slashes) => (slashes ^ (slashes + 1)) >> 1;
|
|
112
|
+
const innermostString = (stack) => stack.at(-1)?.string ?? -1;
|
|
113
|
+
/**
|
|
114
|
+
* Updates `stack` with the brackets and strings that `text[from, to)` opens
|
|
115
|
+
* and closes. A quote closes the innermost open string it can close at that
|
|
116
|
+
* string's escape depth, and otherwise opens a string at its own depth. The
|
|
117
|
+
* other quote characters inside a string are content (`"don't"`).
|
|
118
|
+
*/
|
|
119
|
+
const track = (stack, text, from, to) => {
|
|
120
|
+
for (let i = from; i < to; i++) {
|
|
121
|
+
const char = text[i];
|
|
122
|
+
if (isQuote(char)) {
|
|
123
|
+
let slashes = 0;
|
|
124
|
+
while (text[i - 1 - slashes] === "\\")
|
|
125
|
+
slashes++;
|
|
126
|
+
const inner = innermostString(stack);
|
|
127
|
+
if (inner === -1) {
|
|
128
|
+
stack.push({ close: char, depth: escapeDepth(slashes), string: stack.length });
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (stack[inner].close !== char)
|
|
132
|
+
continue;
|
|
133
|
+
// Only strings can close, so walk the strings alone, innermost first.
|
|
134
|
+
let closes = inner;
|
|
135
|
+
while (closes !== -1) {
|
|
136
|
+
const depth = stack[closes].depth;
|
|
137
|
+
if (slashes % (2 * (depth + 1)) === depth)
|
|
138
|
+
break;
|
|
139
|
+
closes = closes === 0 ? -1 : stack[closes - 1].string;
|
|
140
|
+
}
|
|
141
|
+
if (closes === -1)
|
|
142
|
+
stack.push({ close: char, depth: escapeDepth(slashes), string: stack.length });
|
|
143
|
+
else
|
|
144
|
+
stack.length = closes;
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
const closer = closers[char];
|
|
148
|
+
if (closer !== undefined)
|
|
149
|
+
stack.push({ close: closer, string: innermostString(stack) });
|
|
150
|
+
else if (stack.at(-1)?.depth === undefined && char === stack.at(-1)?.close)
|
|
151
|
+
stack.pop();
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
/**
|
|
155
|
+
* Whether the closer at `end` closes the bracket open before the name: it and
|
|
156
|
+
* any closers right after it balance the enclosing brackets, and a separator
|
|
157
|
+
* or the end of the text follows (`[{token:abc}]`, not `Tr0ub4dor)]&3`).
|
|
158
|
+
*/
|
|
159
|
+
const closesEnclosing = (text, end, stack) => {
|
|
160
|
+
let at = end;
|
|
161
|
+
for (let k = stack.length - 1; k >= 0 && stack[k].depth === undefined && text[at] === stack[k].close; k--)
|
|
162
|
+
at++;
|
|
163
|
+
return at > end && /^(?:[\s,;"'`]|\\+["'`]|$)/.test(text.slice(at, at + 2));
|
|
164
|
+
};
|
|
165
|
+
/** Credential words a name ends in. */
|
|
166
|
+
const credentialWord = "(?:KEY|TOKEN|SECRET|PASSWORD|PASSPHRASE|CREDENTIAL)S?";
|
|
167
|
+
/**
|
|
168
|
+
* A name's separator: `=`, `:` or `=>`, after any quote that closes the name.
|
|
169
|
+
* In a durable row it stays on the name's line; a diagnostic also reads a
|
|
170
|
+
* value on the next line (`password:` then the value), which is how a stream
|
|
171
|
+
* redactor sees it arrive.
|
|
172
|
+
*/
|
|
173
|
+
const separator = (space) => String.raw `(?:\\*["'\x60])?${space}(?:=>|[=:])${space}`;
|
|
174
|
+
/**
|
|
175
|
+
* A credential name and its separator. Start once per identifier, including
|
|
176
|
+
* leading underscores, so a name containing many underscores cannot force
|
|
177
|
+
* repeated scans of its suffix. The name may close a quote of its own
|
|
178
|
+
* (`'password':`, `\"token\":`), so a key in a Python repr, an inspected
|
|
179
|
+
* `Map` or escaped JSON meets the rule too.
|
|
180
|
+
*/
|
|
181
|
+
const credentialName = String.raw `(?<![A-Za-z0-9_-])([A-Za-z0-9_-]*${credentialWord})`;
|
|
182
|
+
const credentialNames = new RegExp(credentialName + separator("[ \\t]*"), "gi");
|
|
183
|
+
const diagnosticNames = new RegExp(credentialName + separator(String.raw `\s*`), "gi");
|
|
184
|
+
/** Case-insensitive authorization names, without making scheme words prose matches. */
|
|
185
|
+
const authorizationName = "(?:proxy-)?authorization".replace(/[a-z]/g, (char) => `[${char}${char.toUpperCase()}]`);
|
|
186
|
+
/** Complete header names whose whole value is a credential whatever its scheme. */
|
|
187
|
+
const headerWord = String.raw `${authorizationName}|${"(?:set-)?cookies?".replace(/[a-z]/g, (char) => `[${char}${char.toUpperCase()}]`)}`;
|
|
188
|
+
// Apostrophes and backticks open quoted names only outside a header token.
|
|
189
|
+
const headerName = String
|
|
190
|
+
.raw `(?<![A-Za-z0-9!#$%&*+.^_|~-])(?<![A-Za-z0-9!#$%&'*+.^_\x60|~-]['\x60])(${headerWord})`;
|
|
191
|
+
const headerNames = new RegExp(String.raw `((?:proxy-)?authorization|(?:set-)?cookies?)${separator(String.raw `\s*`)}`, "gi");
|
|
192
|
+
/** Whitespace headers require a scheme spelling, so ordinary authorization prose survives. */
|
|
193
|
+
const diagnosticHeaderNames = new RegExp(
|
|
194
|
+
// A trailing token quote requires an opener; double quotes are never token characters.
|
|
195
|
+
`${headerName}(?:` +
|
|
196
|
+
String.raw `(?:(?<=["'\x60](?:${headerWord}))|(?!\\*['\x60]))${separator(String.raw `\s*`)}|` +
|
|
197
|
+
String.raw `(?<=${authorizationName})\s+(?=(?:Token|TOKEN|[Bb]earer|BEARER|Basic|BASIC)[ \t]+\S))`, "g");
|
|
198
|
+
/** Any credential or header name, to stop a bare value before the next pair. */
|
|
199
|
+
const anyName = new RegExp(`${credentialNames.source}|${headerNames.source}`, "gi");
|
|
200
|
+
/** A credential or header name and its separator ending the text before a quote. */
|
|
201
|
+
const nameBefore = new RegExp(`(?:${credentialNames.source}|${headerNames.source})$`, "i");
|
|
202
|
+
/**
|
|
203
|
+
* The index of the next credential or header name at or after `from`; with
|
|
204
|
+
* `quoted`, only a name written right after a quote.
|
|
205
|
+
*/
|
|
206
|
+
const nextName = (text, from, quoted) => {
|
|
207
|
+
// No name starts between a search's start and what it found, so a later
|
|
208
|
+
// search from inside that span has the same answer. Values are read left to
|
|
209
|
+
// right, so this keeps every search over one text linear in total.
|
|
210
|
+
const cached = nameSearches[quoted ? 1 : 0];
|
|
211
|
+
if (cached.text === text && from >= cached.from && from <= cached.found)
|
|
212
|
+
return cached.found;
|
|
213
|
+
let found = text.length;
|
|
214
|
+
anyName.lastIndex = from;
|
|
215
|
+
for (let match = anyName.exec(text); match !== null; match = anyName.exec(text)) {
|
|
216
|
+
if (!quoted || isQuote(text[match.index - 1])) {
|
|
217
|
+
found = match.index;
|
|
218
|
+
break;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
nameSearches[quoted ? 1 : 0] = { text, from, found };
|
|
222
|
+
return found;
|
|
223
|
+
};
|
|
224
|
+
/** The last search for any name and for a quoted name. */
|
|
225
|
+
const nameSearches = [{ text: "", from: 0, found: 0 }, { text: "", from: 0, found: 0 }];
|
|
226
|
+
/** The end of the line `index` is on. */
|
|
227
|
+
const lineEnd = (text, index) => {
|
|
228
|
+
// Every index up to a line's end shares that end, so the last answer serves
|
|
229
|
+
// each later value on the same line.
|
|
230
|
+
if (lastLine.text === text && index >= lastLine.from && index <= lastLine.end)
|
|
231
|
+
return lastLine.end;
|
|
232
|
+
newline.lastIndex = index;
|
|
233
|
+
const end = newline.exec(text)?.index ?? text.length;
|
|
234
|
+
lastLine = { text, from: index, end };
|
|
235
|
+
return end;
|
|
236
|
+
};
|
|
237
|
+
const newline = /[\r\n]/g;
|
|
238
|
+
let lastLine = { text: "", from: 0, end: 0 };
|
|
239
|
+
/**
|
|
240
|
+
* Whether the whitespace at `end` continues the one-token value that began at
|
|
241
|
+
* `start`: `util.inspect` separates a constructor or a marker from its
|
|
242
|
+
* contents (`Foo {`, `Map(1) {`, `[Object: null prototype] {`, `<ref *1> {`,
|
|
243
|
+
* `new Password(`).
|
|
244
|
+
*/
|
|
245
|
+
const continuesAcrossSpace = (text, start, end) => {
|
|
246
|
+
const run = text.slice(start, end);
|
|
247
|
+
let next = end;
|
|
248
|
+
while (text[next] === " " || text[next] === "\t")
|
|
249
|
+
next++;
|
|
250
|
+
if (run === "new")
|
|
251
|
+
return next;
|
|
252
|
+
const prefix = /^[\w$.]+$/.test(run) || /[)\]>]$/.test(run);
|
|
253
|
+
return prefix && closers[text[next] ?? ""] !== undefined ? next : undefined;
|
|
254
|
+
};
|
|
255
|
+
/**
|
|
256
|
+
* Whether the `,`, `;` or `&` at `end` separates pairs rather than sitting
|
|
257
|
+
* inside a value (`Zq7;Syn,thetic`, `Tr0ub(4dor)&3`): a space or another name
|
|
258
|
+
* follows it, and for `&`, a query parameter (`&page=2`).
|
|
259
|
+
*/
|
|
260
|
+
const separatesPairs = (text, end) => text[end] === "&"
|
|
261
|
+
? /^&[A-Za-z_][\w.-]*=/.test(text.slice(end, end + 64))
|
|
262
|
+
: /^[,;](?:\s|$|\s*\\*["'`]?[A-Za-z_][\w.-]*\\*["'`]?\s*(?:=>|[=:]))/.test(text.slice(end, end + 64));
|
|
263
|
+
/**
|
|
264
|
+
* How far past its line every durable container may read, whatever the
|
|
265
|
+
* {@link Budget} has left: a pretty-printed credential object of this size
|
|
266
|
+
* is read whole on any row.
|
|
267
|
+
*/
|
|
268
|
+
const containerWindow = 4096;
|
|
269
|
+
const containerBudget = 4;
|
|
270
|
+
/**
|
|
271
|
+
* The end of the bare value at `start`, read as a run of tokens: words,
|
|
272
|
+
* balanced containers anywhere in it (`{bcrypt}$2a$...`, `Tr0ub(4dor)&3`,
|
|
273
|
+
* `Some("x")`), inspect markers (`<Buffer 73 65>`, `<ref *1>`), and quoted
|
|
274
|
+
* parts. `stack` is what is open before the name: the value stops at a quote
|
|
275
|
+
* that closes the enclosing string, at a closer that closes the enclosing
|
|
276
|
+
* bracket, and before the next credential name.
|
|
277
|
+
*/
|
|
278
|
+
const bareValueEnd = (text, start, scope, stack, budget) => {
|
|
279
|
+
const inner = stack[innermostString(stack)];
|
|
280
|
+
// A durable value never reads an unclosed quote past its line, and one it
|
|
281
|
+
// cannot close stops at its line. A bracket reads past 4 KiB only while
|
|
282
|
+
// the text's scan budget lasts: each unbalanced bracket costs a bounded
|
|
283
|
+
// scan, and all of them together a bounded multiple of the text.
|
|
284
|
+
const bound = scope.diagnostic ? text.length : lineEnd(text, start);
|
|
285
|
+
const window = scope.diagnostic
|
|
286
|
+
? text.length
|
|
287
|
+
: Math.min(text.length, start + Math.max(containerWindow, budget.remaining));
|
|
288
|
+
// Never before the opener: a closed container may already have carried the
|
|
289
|
+
// value past its first line.
|
|
290
|
+
const unclosed = (open, end) => Math.max(open, Math.min(end, bound));
|
|
291
|
+
// A durable container first reads its own line. A pretty-printer only
|
|
292
|
+
// continues one on the next line after the opener or a member's comma
|
|
293
|
+
// (`apiKey: {`, `credentials: { a: 1,`), so only then does the scan read on
|
|
294
|
+
// through the window.
|
|
295
|
+
const container = (open) => {
|
|
296
|
+
if (scope.diagnostic)
|
|
297
|
+
return skipContainer(text, open, window);
|
|
298
|
+
const end = skipContainer(text, open, bound);
|
|
299
|
+
if (lastClosed || !/[[{(,]\s*$/.test(text.slice(open, bound)))
|
|
300
|
+
return end;
|
|
301
|
+
const past = skipContainer(text, open, window);
|
|
302
|
+
budget.remaining -= past - open;
|
|
303
|
+
return past;
|
|
304
|
+
};
|
|
305
|
+
let end = start;
|
|
306
|
+
// The value's own first token is never the next name: in
|
|
307
|
+
// `api_key=lowercase-secret:end` the value only looks like a `name:` pair.
|
|
308
|
+
let limit = nextName(text, start + 1, scope.header);
|
|
309
|
+
// A header value that is one container ends with it: `"cookie":["sid=a"],`.
|
|
310
|
+
if (scope.header && closers[text[start]] !== undefined) {
|
|
311
|
+
end = container(start);
|
|
312
|
+
if (!/^[^\s,;"'`\\)\]}]/.test(text[end] ?? " "))
|
|
313
|
+
return lastClosed ? end : unclosed(start, end);
|
|
314
|
+
}
|
|
315
|
+
while (end < text.length) {
|
|
316
|
+
if (end >= limit) {
|
|
317
|
+
if (end === limit)
|
|
318
|
+
break;
|
|
319
|
+
limit = nextName(text, end, scope.header);
|
|
320
|
+
continue;
|
|
321
|
+
}
|
|
322
|
+
const char = text[end];
|
|
323
|
+
if (closers[char] !== undefined) {
|
|
324
|
+
const open = end;
|
|
325
|
+
end = container(open);
|
|
326
|
+
// An unbalanced container runs on, spaces included.
|
|
327
|
+
if (!lastClosed)
|
|
328
|
+
return unclosed(open, end);
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
331
|
+
if (char === "<") {
|
|
332
|
+
angleClose.lastIndex = end;
|
|
333
|
+
const close = angleClose.exec(text)?.index;
|
|
334
|
+
if (close !== undefined && text[close] === ">") {
|
|
335
|
+
end = close + 1;
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
let depth = 0;
|
|
340
|
+
while (text[end + depth] === "\\")
|
|
341
|
+
depth++;
|
|
342
|
+
const quote = text[end + depth];
|
|
343
|
+
if (isQuote(quote)) {
|
|
344
|
+
// A quote at the enclosing string's depth or shallower closes it,
|
|
345
|
+
// whatever precedes it (`Basic abc=`), unless a credential name and
|
|
346
|
+
// separator precede it (`credential:"`). Inside a bracket, a field's
|
|
347
|
+
// value ends at a quote (`{\"token\":abc\"}`). Any other quote opens a
|
|
348
|
+
// quoted part of the value (`username="admin"`, `it's`).
|
|
349
|
+
const closesString = inner !== undefined && quote === inner.close &&
|
|
350
|
+
depth % (2 * (inner.depth + 1)) <= inner.depth &&
|
|
351
|
+
!nameBefore.test(text.slice(Math.max(start, end - 256), end));
|
|
352
|
+
if (closesString || (!scope.header && inner === undefined && stack.length > 0))
|
|
353
|
+
break;
|
|
354
|
+
// A quote between two word characters is an apostrophe (`it's`).
|
|
355
|
+
if (depth === 0 && /\w/.test(text[end - 1] ?? "") && /\w/.test(text[end + 1] ?? "")) {
|
|
356
|
+
end++;
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
const open = end;
|
|
360
|
+
end = skipQuoted(text, end + depth, depth, 2 * (depth + 1), bound);
|
|
361
|
+
if (!lastClosed)
|
|
362
|
+
return unclosed(open, end);
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
if (char === "\r" || char === "\n")
|
|
366
|
+
break;
|
|
367
|
+
// Inside a bracket a `,` or `;` separates members; in raw text it does
|
|
368
|
+
// when a space or another name follows.
|
|
369
|
+
if (!scope.header &&
|
|
370
|
+
((char === "&" && separatesPairs(text, end)) ||
|
|
371
|
+
((char === "," || char === ";") && (stack.length > 0 || separatesPairs(text, end)))))
|
|
372
|
+
break;
|
|
373
|
+
if (!scope.header && !scope.diagnostic && (char === " " || char === "\t")) {
|
|
374
|
+
const next = continuesAcrossSpace(text, start, end);
|
|
375
|
+
if (next === undefined)
|
|
376
|
+
break;
|
|
377
|
+
end = next;
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
if (closesEnclosing(text, end, stack))
|
|
381
|
+
break;
|
|
382
|
+
end++;
|
|
383
|
+
}
|
|
384
|
+
while (end > start && /\s/.test(text[end - 1]))
|
|
385
|
+
end--;
|
|
386
|
+
return end;
|
|
387
|
+
};
|
|
388
|
+
/**
|
|
389
|
+
* Whether a name ending in `key` or `keys` is not a credential's: it counts
|
|
390
|
+
* only where {@link isSensitiveKey} agrees, as it does for the same name as an
|
|
391
|
+
* object key (`apiKeys` counts; `keys`, `sortKeys`, `idempotencyKey` and
|
|
392
|
+
* `monkey` do not).
|
|
393
|
+
*/
|
|
394
|
+
const plainKeyName = (name) => /keys?$/i.test(name) && !isSensitiveKey(name);
|
|
395
|
+
/**
|
|
396
|
+
* The end of a credential that is a URL's password (`//x-access-token:secret@`):
|
|
397
|
+
* the `@` that closes the userinfo, so the host and path stay readable. Any
|
|
398
|
+
* other bare value keeps its end.
|
|
399
|
+
*/
|
|
400
|
+
const userinfoEnd = (text, start, end) => {
|
|
401
|
+
if (!/\/\/[^\s/@?#]*[:=]$/.test(text.slice(Math.max(0, start - 128), start)))
|
|
402
|
+
return end;
|
|
403
|
+
const value = text.slice(start, end);
|
|
404
|
+
const authority = value.search(/[/?#]/);
|
|
405
|
+
const at = value.lastIndexOf("@", authority < 0 ? value.length : authority);
|
|
406
|
+
return at > 0 ? start + at : end;
|
|
407
|
+
};
|
|
408
|
+
/**
|
|
409
|
+
* Whether a name that matched a credential word names a credential: not a
|
|
410
|
+
* {@link plainKeyName}, and not a count under a plural `tokens` name
|
|
411
|
+
* (`max_tokens: 4096`), which is accounting.
|
|
412
|
+
*/
|
|
413
|
+
const namesCredential = (name, value) => !plainKeyName(name) &&
|
|
414
|
+
!(/tokens$/i.test(name) && /^["'`]?\d+(?![\w.])/.test(value));
|
|
415
|
+
/** The `>` that closes an inspect marker, or the line end that means there is none. */
|
|
416
|
+
const angleClose = /[>\r\n]/g;
|
|
417
|
+
/**
|
|
418
|
+
* The `+` with which `util.inspect` continues a long string on the next line,
|
|
419
|
+
* its newline raw or escaped (`\\n`) when the inspected text was itself
|
|
420
|
+
* written into a string.
|
|
421
|
+
*/
|
|
422
|
+
const continuation = /(?:\s|\\+[nrt])*\+(?:\s|\\+[nrt])*/y;
|
|
423
|
+
/** A value an earlier pass already replaced. */
|
|
424
|
+
const alreadyRedacted = /^\[REDACTED\]$/;
|
|
425
|
+
/**
|
|
426
|
+
* Redacts the value after each name `names` matches. A quoted value is
|
|
427
|
+
* consumed whole at any string-escape depth, with the pieces `util.inspect`
|
|
428
|
+
* joins with `+` across lines, and its quotes are kept around the
|
|
429
|
+
* placeholder; a bare value is consumed as a run of tokens. A name that ends
|
|
430
|
+
* in `key` counts only where {@link isSensitiveKey} agrees, so `sortKey` and
|
|
431
|
+
* `idempotencyKey` keep their values, and a count under a plural `tokens`
|
|
432
|
+
* name, `max_tokens: 4096`, is accounting and is kept too.
|
|
433
|
+
*/
|
|
434
|
+
const redactValues = (text, names, scope) => {
|
|
435
|
+
const keys = new RegExp(names.source, names.flags);
|
|
436
|
+
let output = "";
|
|
437
|
+
let position = 0;
|
|
438
|
+
// What is open before the current value. Values are skipped, so a bracket
|
|
439
|
+
// or quote inside a credential never counts.
|
|
440
|
+
const stack = [];
|
|
441
|
+
const budget = { remaining: containerBudget * text.length };
|
|
442
|
+
let scanned = 0;
|
|
443
|
+
for (let match = keys.exec(text); match !== null; match = keys.exec(text)) {
|
|
444
|
+
const name = match[1];
|
|
445
|
+
if (!scope.header && plainKeyName(name))
|
|
446
|
+
continue;
|
|
447
|
+
track(stack, text, scanned, keys.lastIndex);
|
|
448
|
+
scanned = keys.lastIndex;
|
|
449
|
+
let start = keys.lastIndex;
|
|
450
|
+
let depth = 0;
|
|
451
|
+
while (text[start + depth] === "\\")
|
|
452
|
+
depth++;
|
|
453
|
+
const opener = text[start + depth];
|
|
454
|
+
const quote = isQuote(opener) ? opener : undefined;
|
|
455
|
+
let end;
|
|
456
|
+
if (quote !== undefined) {
|
|
457
|
+
start += depth;
|
|
458
|
+
const inner = stack[innermostString(stack)];
|
|
459
|
+
if (inner !== undefined) {
|
|
460
|
+
// Inside a string written with another quote character, every
|
|
461
|
+
// backslash is doubled again: `inspect` renders `"a\"b"` as
|
|
462
|
+
// `'"a\\"b"'`.
|
|
463
|
+
end = skipQuoted(text, start, depth, (inner.close === quote ? 1 : 2 * (inner.depth + 1)) * 2 * (depth + 1));
|
|
464
|
+
}
|
|
465
|
+
else {
|
|
466
|
+
// No enclosing string is visible: raw text, or a string `track` cannot
|
|
467
|
+
// see. Close no earlier than a doubled string would, but never past the
|
|
468
|
+
// next credential name: at the undoubled close when only separators
|
|
469
|
+
// precede that name (`"abc\\", password=`), else at the name.
|
|
470
|
+
const undoubled = skipQuoted(text, start, depth);
|
|
471
|
+
end = skipQuoted(text, start, depth, Math.max(4, 2 * (depth + 1)));
|
|
472
|
+
const next = nextName(text, undoubled, false);
|
|
473
|
+
if (next < end)
|
|
474
|
+
end = /^[\s,;&]*$/.test(text.slice(undoubled, next)) ? undoubled : next;
|
|
475
|
+
}
|
|
476
|
+
const closed = end - start > 1 && text[end - 1] === quote;
|
|
477
|
+
if (!closed && !scope.diagnostic)
|
|
478
|
+
end = Math.min(end, lineEnd(text, start));
|
|
479
|
+
// `util.inspect` continues a long string on the next line with `+`.
|
|
480
|
+
if (depth === 0) {
|
|
481
|
+
for (continuation.lastIndex = end; continuation.test(text); continuation.lastIndex = end) {
|
|
482
|
+
const piece = continuation.lastIndex;
|
|
483
|
+
if (!isQuote(text[piece]))
|
|
484
|
+
break;
|
|
485
|
+
end = skipQuoted(text, piece, 0);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
if (alreadyRedacted.test(text.slice(start + 1, end - (text[end - 1] === quote ? 1 + depth : 0))))
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
else {
|
|
492
|
+
end = bareValueEnd(text, start, scope, stack, budget);
|
|
493
|
+
end = userinfoEnd(text, start, end);
|
|
494
|
+
// In a journal row a placeholder then a space is a value an earlier pass
|
|
495
|
+
// already bounded: what follows it is the next part of the line,
|
|
496
|
+
// `{"statusCode":401}`. A diagnostic value runs over several words, so
|
|
497
|
+
// there the words after a placeholder are still the value.
|
|
498
|
+
const bounded = !scope.diagnostic && text.startsWith(placeholder, start) &&
|
|
499
|
+
/\s/.test(text[start + placeholder.length] ?? "");
|
|
500
|
+
const kept = end === start || bounded || alreadyRedacted.test(text.slice(start, end)) ||
|
|
501
|
+
(/tokens$/i.test(name) && /^\d+(?![\w.])/.test(text.slice(start, end))) ||
|
|
502
|
+
// A Bearer or Basic value the default rules already replaced keeps its scheme.
|
|
503
|
+
(scope.header && /^(?:bearer|basic)\s+\[REDACTED/i.test(text.slice(start, start + 24)));
|
|
504
|
+
if (kept)
|
|
505
|
+
continue;
|
|
506
|
+
}
|
|
507
|
+
const escape = quote === undefined ? "" : "\\".repeat(depth);
|
|
508
|
+
const closed = quote !== undefined && end - start > 1 && text[end - 1] === quote;
|
|
509
|
+
output += text.slice(position, keys.lastIndex) + escape + (quote ?? "") + placeholder +
|
|
510
|
+
(closed ? escape + quote : "");
|
|
511
|
+
position = end;
|
|
512
|
+
scanned = end;
|
|
513
|
+
keys.lastIndex = end;
|
|
514
|
+
}
|
|
515
|
+
return output + text.slice(position);
|
|
516
|
+
};
|
|
517
|
+
/**
|
|
518
|
+
* What separates a flag from its value: whitespace on a command line, or
|
|
519
|
+
* `", "` between two elements of a JSON or inspected array.
|
|
520
|
+
*/
|
|
521
|
+
const argumentSeparator = String.raw `(?:\s+|["'\x60]\s*,\s*["'\x60])`;
|
|
522
|
+
/** A credential flag and the separator before its value. */
|
|
523
|
+
const credentialFlag = new RegExp(String.raw `(?<![A-Za-z0-9_-])(--?[A-Za-z0-9_-]*${credentialWord})${argumentSeparator}`, "gi");
|
|
524
|
+
/** `-p` and the separator before its value, unless the value is a path or a long option. */
|
|
525
|
+
const shortPasswordFlag = new RegExp(String.raw `(?<![^\s"'\x60,[(])-p${argumentSeparator}(?!(?:\.{0,2}|~)\/|--)`, "g");
|
|
526
|
+
/**
|
|
527
|
+
* A flag that always takes a password, and the separator before it: `plink
|
|
528
|
+
* -pw`, openssl's `-pass`, `-passin` and `-passout` (`pass:secret`), `-passwd`.
|
|
529
|
+
*/
|
|
530
|
+
const passwordFlag = new RegExp(String.raw `(?<![^\s"'\x60,[(])-p(?:w|ass(?:in|out|wd)?)${argumentSeparator}`, "g");
|
|
531
|
+
/** `-p` with a value attached, unless the word is a known single-dash option. */
|
|
532
|
+
const attachedPasswordFlag = new RegExp(String
|
|
533
|
+
.raw `(?<![^\s"'\x60,[(])-p(?!(?:rint\w*|rune|erm|ath|threads?|edantic[\w-]*|ipe|ie|g|retty|lain|rogress|rofile|arents|reserve|ass(?:in|out|wd)?|w)(?![^\s"'\x60,\]]))(?=[^\s,\]])`, "g");
|
|
534
|
+
/** `-u` or `--user` and the separator before its value, or `-u` with its value attached (`-uadmin:pw`). */
|
|
535
|
+
const userFlag = new RegExp(String.raw `(?<![^\s"'\x60,[(])(?:(?:-u|--user)(?:=|${argumentSeparator})|-u(?=[^\s=-]))`, "g");
|
|
536
|
+
/**
|
|
537
|
+
* A flag that takes a password only under one command, and the separator
|
|
538
|
+
* before its value: `zip -P` (a port to `ssh` and `mysql`) and `redis-cli -a`
|
|
539
|
+
* or `--pass`. The command is looked for at most 256 characters back on the
|
|
540
|
+
* same command line.
|
|
541
|
+
*/
|
|
542
|
+
const commandPasswordFlag = new RegExp(String
|
|
543
|
+
.raw `\b(?:(?:(?:un)?zip|zipcloak)\b[^\r\n|;&]{0,256}?[\s"'\x60,[(]-P|redis-cli\b[^\r\n|;&]{0,256}?[\s"'\x60,[(](?:-a|--pass))${argumentSeparator}`, "g");
|
|
544
|
+
/**
|
|
545
|
+
* An HTTP header written as a name and value pair of an array
|
|
546
|
+
* (`[['Authorization', 'Token x']]`), the shape `Headers` entries and many
|
|
547
|
+
* clients log, up to the quote that opens the value.
|
|
548
|
+
*/
|
|
549
|
+
const headerPair = /(?<![A-Za-z0-9_-])(?:(?:proxy-)?authorization|(?:set-)?cookies?)\\*["'`]\s*,\s*(?=\\*["'`])/gi;
|
|
550
|
+
/** Redacts the quoted value of each {@link headerPair}, keeping its quotes. */
|
|
551
|
+
const redactHeaderPairs = (text) => {
|
|
552
|
+
const pattern = new RegExp(headerPair.source, headerPair.flags);
|
|
553
|
+
let output = "";
|
|
554
|
+
let position = 0;
|
|
555
|
+
for (let match = pattern.exec(text); match !== null; match = pattern.exec(text)) {
|
|
556
|
+
let depth = 0;
|
|
557
|
+
while (text[pattern.lastIndex + depth] === "\\")
|
|
558
|
+
depth++;
|
|
559
|
+
const open = pattern.lastIndex + depth;
|
|
560
|
+
const end = skipQuoted(text, open, depth);
|
|
561
|
+
const close = lastClosed ? end - 1 - depth : end;
|
|
562
|
+
if (alreadyRedacted.test(text.slice(open + 1, close)))
|
|
563
|
+
continue;
|
|
564
|
+
output += text.slice(position, open + 1) + placeholder;
|
|
565
|
+
position = close;
|
|
566
|
+
pattern.lastIndex = end;
|
|
567
|
+
}
|
|
568
|
+
return output + text.slice(position);
|
|
569
|
+
};
|
|
570
|
+
/** The placeholder alone, bare, quoted or escaped: a second pass leaves it alone. */
|
|
571
|
+
const redactedArgument = /^(?:\\*(["'`]))?\[REDACTED\](?:\\*\1)?$/;
|
|
572
|
+
/**
|
|
573
|
+
* The end of the unquoted argument at `start`. It runs to whitespace or a
|
|
574
|
+
* quote, and on through a quoted part written against it (`abc'def'`, which a
|
|
575
|
+
* shell reads as one argument) when that part closes before any whitespace
|
|
576
|
+
* and holds none of `,:{}[]`. Any other quote ends it: the close of a string
|
|
577
|
+
* the command line sits in (`{"cmd":"mysql -p abc","next":1}`).
|
|
578
|
+
*/
|
|
579
|
+
const wordEnd = (text, start) => {
|
|
580
|
+
let end = start;
|
|
581
|
+
while (end < text.length && !/\s/.test(text[end])) {
|
|
582
|
+
const quote = text[end];
|
|
583
|
+
if (!isQuote(quote)) {
|
|
584
|
+
end++;
|
|
585
|
+
continue;
|
|
586
|
+
}
|
|
587
|
+
let close = end + 1;
|
|
588
|
+
while (close < text.length && !/[\s"'`,:{}[\]]/.test(text[close]))
|
|
589
|
+
close++;
|
|
590
|
+
if (close === end + 1 || text[close] !== quote)
|
|
591
|
+
break;
|
|
592
|
+
end = close + 1;
|
|
593
|
+
}
|
|
594
|
+
return end;
|
|
595
|
+
};
|
|
596
|
+
/**
|
|
597
|
+
* Redacts the value after each flag `flags` matches: a quoted value, or one
|
|
598
|
+
* argv word, which ends at whitespace or a quote, so the same rule reads a
|
|
599
|
+
* command line joined by spaces, a JSON array and an inspected array. A comma
|
|
600
|
+
* or a `]` is part of a password on a command line (`-p abc,def`); in an
|
|
601
|
+
* array the element's closing quote ends it first.
|
|
602
|
+
*/
|
|
603
|
+
const redactArguments = (text, flags,
|
|
604
|
+
/** Whether this flag and value are a credential at all. */
|
|
605
|
+
accept) => {
|
|
606
|
+
const pattern = new RegExp(flags.source, flags.flags);
|
|
607
|
+
let output = "";
|
|
608
|
+
let position = 0;
|
|
609
|
+
for (let match = pattern.exec(text); match !== null; match = pattern.exec(text)) {
|
|
610
|
+
const start = pattern.lastIndex;
|
|
611
|
+
let depth = 0;
|
|
612
|
+
while (text[start + depth] === "\\")
|
|
613
|
+
depth++;
|
|
614
|
+
let end = start;
|
|
615
|
+
if (isQuote(text[start + depth]))
|
|
616
|
+
end = skipQuoted(text, start + depth, depth);
|
|
617
|
+
else
|
|
618
|
+
end = wordEnd(text, start);
|
|
619
|
+
const value = text.slice(start, end);
|
|
620
|
+
if (end === start || redactedArgument.test(value) || accept?.(match, value) === false)
|
|
621
|
+
continue;
|
|
622
|
+
output += text.slice(position, start) + placeholder;
|
|
623
|
+
position = end;
|
|
624
|
+
pattern.lastIndex = end;
|
|
625
|
+
}
|
|
626
|
+
return output + text.slice(position);
|
|
627
|
+
};
|
|
628
|
+
const durable = { header: false, diagnostic: false };
|
|
629
|
+
/**
|
|
630
|
+
* Best-effort textual rules for credential shapes observed in real reports.
|
|
631
|
+
* Every replacement reaches a fixed point so replaying or exporting an entry
|
|
632
|
+
* cannot mutate it again.
|
|
633
|
+
*
|
|
634
|
+
* @since 0.1.0
|
|
635
|
+
* @category constants
|
|
636
|
+
*/
|
|
637
|
+
export const defaultRules = [
|
|
638
|
+
{
|
|
639
|
+
id: "private-key-block",
|
|
640
|
+
// Stop an unterminated block at the next header so repeated headers cannot
|
|
641
|
+
// each rescan the remaining input. A valid PEM body contains no headers.
|
|
642
|
+
// A block with no footer is a key a bound or a producer truncated, and its
|
|
643
|
+
// body is as secret as a whole one's, so it runs to the next header or to
|
|
644
|
+
// the end of the text. It runs first: a block inside URL userinfo, once
|
|
645
|
+
// replaced, is userinfo the URL rule then reads whole, so a second pass
|
|
646
|
+
// finds nothing left to change.
|
|
647
|
+
pattern: /-----BEGIN[^-]*PRIVATE KEY(?: BLOCK)?-----(?:(?!-----BEGIN)[\s\S])*?(?:-----END[^-]*-----|(?=-----BEGIN)|$)/g
|
|
648
|
+
},
|
|
649
|
+
// Git HTTPS OAuth credentials place the secret in the username. Match
|
|
650
|
+
// before password rewriting removes the x-oauth-basic marker.
|
|
651
|
+
{
|
|
652
|
+
id: "git-oauth-userinfo",
|
|
653
|
+
pattern: /(https?:\/\/)[^\s/@:"'\\]+:x-oauth-basic@/gi,
|
|
654
|
+
replace: "$1[REDACTED]@"
|
|
655
|
+
},
|
|
656
|
+
{
|
|
657
|
+
id: "url-credentials",
|
|
658
|
+
// The scheme is bounded rather than open. `-` is not a word character, so
|
|
659
|
+
// `\b[a-z][a-z0-9+.-]*` starts a fresh scan after every hyphen, and on a
|
|
660
|
+
// long run of that shape the rule rescanned the tail from each one: 400 kB
|
|
661
|
+
// of `aaaaaaaa-` cost 11 seconds, on the path every journal write and every
|
|
662
|
+
// log line takes. A scheme longer than 30 characters is not one this net is
|
|
663
|
+
// for, and the cap makes each start position cost a constant.
|
|
664
|
+
pattern: /\b([a-z][a-z0-9+.-]{0,30}:\/\/[^\s:@/]+):(?!\[REDACTED\]@)[^\s:@/]+@/gi,
|
|
665
|
+
replace: "$1:[REDACTED]@"
|
|
666
|
+
},
|
|
667
|
+
{
|
|
668
|
+
id: "bearer-token",
|
|
669
|
+
pattern: /\bBearer\s+[A-Za-z0-9._~+/-]{4,}=*/gi,
|
|
670
|
+
replace: "Bearer [REDACTED_TOKEN]"
|
|
671
|
+
},
|
|
672
|
+
{
|
|
673
|
+
id: "basic-authorization",
|
|
674
|
+
pattern: /\bBasic\s+[A-Za-z0-9+/]{4,}={0,2}/gi,
|
|
675
|
+
replace: "Basic [REDACTED_TOKEN]"
|
|
676
|
+
},
|
|
677
|
+
{
|
|
678
|
+
id: "jwt",
|
|
679
|
+
// Check the three segments once per token before looking for a header.
|
|
680
|
+
// Starting at every `-eyJ` would rescan a malformed token's entire tail.
|
|
681
|
+
// Preserve any prefix so a JWT following a hyphen still gets redacted.
|
|
682
|
+
pattern: /(?<![A-Za-z0-9_-])(?=[A-Za-z0-9_-]*\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-])([A-Za-z0-9_-]*?)\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g,
|
|
683
|
+
replace: "$1[REDACTED_TOKEN]"
|
|
684
|
+
},
|
|
685
|
+
{
|
|
686
|
+
id: "api-key",
|
|
687
|
+
// Not `\b`, for the reason the assignment rule below gives: an underscore
|
|
688
|
+
// is a word character, so `\bsk` never fires after `ANTHROPIC_` or after
|
|
689
|
+
// Effect's log-span sanitizer folds `token=` into `token_`. The lookbehind
|
|
690
|
+
// excludes only letters and digits, so a key still reads as a key when an
|
|
691
|
+
// underscore or a hyphen runs into it.
|
|
692
|
+
pattern: /(?<![A-Za-z0-9])(?:sk|pk)[-_][A-Za-z0-9][A-Za-z0-9_-]{7,}/g,
|
|
693
|
+
replace: "[REDACTED_API_KEY]"
|
|
694
|
+
},
|
|
695
|
+
{
|
|
696
|
+
id: "github-token",
|
|
697
|
+
pattern: /\bgh[pousr]_[A-Za-z0-9]{20,}(?![A-Za-z0-9])/g,
|
|
698
|
+
replace: "[REDACTED]"
|
|
699
|
+
},
|
|
700
|
+
{
|
|
701
|
+
id: "github-fine-grained-token",
|
|
702
|
+
pattern: /\bgithub_pat_[A-Za-z0-9_]{20,}(?![A-Za-z0-9_])/g,
|
|
703
|
+
replace: "[REDACTED]"
|
|
704
|
+
},
|
|
705
|
+
{
|
|
706
|
+
id: "aws-access-key",
|
|
707
|
+
pattern: /\bAKIA[0-9A-Z]{16}\b/g,
|
|
708
|
+
replace: "[REDACTED]"
|
|
709
|
+
},
|
|
710
|
+
{
|
|
711
|
+
id: "slack-token",
|
|
712
|
+
pattern: /\bxox[baprs]-[A-Za-z0-9-]{10,}(?![A-Za-z0-9-])/g,
|
|
713
|
+
replace: "[REDACTED]"
|
|
714
|
+
},
|
|
715
|
+
{
|
|
716
|
+
id: "google-api-key",
|
|
717
|
+
pattern: /\bAIza[0-9A-Za-z_-]{35,}(?![0-9A-Za-z_-])/g,
|
|
718
|
+
replace: "[REDACTED]"
|
|
719
|
+
},
|
|
720
|
+
{
|
|
721
|
+
id: "assignment",
|
|
722
|
+
// A credential name, `:`, `=` or `=>`, and its whole value: quoted at any
|
|
723
|
+
// escape depth, a container, or several bare words. See `redactValues`.
|
|
724
|
+
pattern: credentialNames,
|
|
725
|
+
rewrite: (text) => redactValues(text, credentialNames, durable)
|
|
726
|
+
},
|
|
727
|
+
{
|
|
728
|
+
id: "credential-flag",
|
|
729
|
+
// `--password hunter2`, `--api-key "a b"`, or the same pair as two argv
|
|
730
|
+
// elements. `--token=x` is an assignment above; this is the spelling a
|
|
731
|
+
// command line separates with a space. The next word is the value even
|
|
732
|
+
// when it starts with `-`: a credential flag always takes one.
|
|
733
|
+
pattern: credentialFlag,
|
|
734
|
+
rewrite: (text) => redactArguments(text, credentialFlag, (match, value) => namesCredential(match[1].replace(/^-+/, ""), value))
|
|
735
|
+
},
|
|
736
|
+
{
|
|
737
|
+
id: "cookie-session-assignment",
|
|
738
|
+
pattern: /(?<![A-Za-z0-9-])((?:COOKIE|SESSION)S?)(\s*=\s*)(?!\[REDACTED\])("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|(?!["'])[^\s;,]+)/gi,
|
|
739
|
+
replace: "$1$2[REDACTED]"
|
|
740
|
+
}
|
|
741
|
+
];
|
|
742
|
+
/**
|
|
743
|
+
* The rules for a diagnostic: {@link defaultRules} and the spellings too broad
|
|
744
|
+
* for a durable row.
|
|
745
|
+
*
|
|
746
|
+
* A log line, a stderr message, and an error handed to a remote caller are
|
|
747
|
+
* read once, so a false positive costs a word of context. A journal row is
|
|
748
|
+
* permanent, so the same false positive destroys data. These rules sit on the
|
|
749
|
+
* diagnostic side of that line: an authorization header in any scheme, bare
|
|
750
|
+
* URL userinfo (`https://token@host`), a signed query parameter, and the value
|
|
751
|
+
* of a `-p` flag, which is a password to `mysql` and `sshpass` and a path to
|
|
752
|
+
* `mkdir`. A path, a port and a lowercase word attached to `-p` (`-print`,
|
|
753
|
+
* `-path`) are left alone.
|
|
754
|
+
*
|
|
755
|
+
* @since 1.0.0
|
|
756
|
+
* @category constants
|
|
757
|
+
*/
|
|
758
|
+
export const diagnosticRules = [
|
|
759
|
+
...defaultRules.map((rule) => rule.id === "assignment"
|
|
760
|
+
? {
|
|
761
|
+
id: "assignment",
|
|
762
|
+
pattern: diagnosticNames,
|
|
763
|
+
rewrite: (text) => redactValues(text, diagnosticNames, { header: false, diagnostic: true })
|
|
764
|
+
}
|
|
765
|
+
: rule),
|
|
766
|
+
{
|
|
767
|
+
id: "authorization-header",
|
|
768
|
+
// The header value is a credential whatever its scheme (`Token`, a cookie
|
|
769
|
+
// list, `Digest` with quoted parameters), so the value runs to the end of
|
|
770
|
+
// its line or the close of the string it was written in.
|
|
771
|
+
// A lookbehind-free superset lets the compiled prefilter skip ordinary prose.
|
|
772
|
+
pattern: /(?:proxy-)?authorization|(?:set-)?cookies?/gi,
|
|
773
|
+
rewrite: (text) => redactValues(text, diagnosticHeaderNames, { header: true, diagnostic: true })
|
|
774
|
+
},
|
|
775
|
+
{
|
|
776
|
+
id: "bearer-or-basic",
|
|
777
|
+
// A token shorter than the default rules' minimum. A lowercase `basic` is
|
|
778
|
+
// prose ("a basic example"), so only the scheme's own spellings count.
|
|
779
|
+
pattern: /(\b(?:[Bb]earer|BEARER|Basic|BASIC)\s+)(?!\[REDACTED)[^\s,;"'\\]+/g,
|
|
780
|
+
replace: "$1[REDACTED]"
|
|
781
|
+
},
|
|
782
|
+
{
|
|
783
|
+
id: "password-flag",
|
|
784
|
+
// The whole next argument, a path or a leading `-` included.
|
|
785
|
+
pattern: passwordFlag,
|
|
786
|
+
rewrite: (text) => redactArguments(text, passwordFlag)
|
|
787
|
+
},
|
|
788
|
+
{
|
|
789
|
+
id: "sshpass-password",
|
|
790
|
+
// `sshpass -p` always takes a value, even one that starts with `--`,
|
|
791
|
+
// which `mysql -p --database x` reads as the next option.
|
|
792
|
+
pattern: /(\bsshpass(?:\s+-[^\sp]\S*){0,8}\s+-p\s+)--(?!\[REDACTED\])[^\s"'`]*/g,
|
|
793
|
+
replace: "$1[REDACTED]"
|
|
794
|
+
},
|
|
795
|
+
{
|
|
796
|
+
id: "user-flag",
|
|
797
|
+
// `curl -u user:pass`, `--user=user:pass`, quoted or not: the whole
|
|
798
|
+
// argument, commas included, when it carries a `:`.
|
|
799
|
+
pattern: userFlag,
|
|
800
|
+
rewrite: (text) => redactArguments(text, userFlag, (_match, value) => value.includes(":"))
|
|
801
|
+
},
|
|
802
|
+
{
|
|
803
|
+
id: "header-pair",
|
|
804
|
+
pattern: headerPair,
|
|
805
|
+
rewrite: redactHeaderPairs
|
|
806
|
+
},
|
|
807
|
+
{
|
|
808
|
+
id: "command-password-flag",
|
|
809
|
+
// The whole next argument, whatever it starts with.
|
|
810
|
+
pattern: commandPasswordFlag,
|
|
811
|
+
rewrite: (text) => redactArguments(text, commandPasswordFlag)
|
|
812
|
+
},
|
|
813
|
+
{
|
|
814
|
+
id: "url-userinfo",
|
|
815
|
+
pattern: /(\/\/)(?!\[REDACTED\]@)(?![^/@\s"'\\]*:\[REDACTED\]@)[^/@\s"'\\]+@/g,
|
|
816
|
+
replace: "$1[REDACTED]@"
|
|
817
|
+
},
|
|
818
|
+
{
|
|
819
|
+
id: "signed-query",
|
|
820
|
+
pattern: /([?&][\w.-]*(?:key|sig|signature|auth|credential)=)(?!\[REDACTED\])[^&#\s"'\\]+/gi,
|
|
821
|
+
replace: "$1[REDACTED]"
|
|
822
|
+
},
|
|
823
|
+
{
|
|
824
|
+
id: "short-password-flag",
|
|
825
|
+
// A path is kept, and so is a flag: `mysql -p --database x` prompts for
|
|
826
|
+
// the password rather than taking one.
|
|
827
|
+
pattern: shortPasswordFlag,
|
|
828
|
+
rewrite: (text) => redactArguments(text, shortPasswordFlag)
|
|
829
|
+
},
|
|
830
|
+
{
|
|
831
|
+
id: "attached-password-flag",
|
|
832
|
+
// `-psecret`. The single-dash words that start with `p` and are not a
|
|
833
|
+
// password, `find -print` and the compilers' `-pthread` among them, are kept.
|
|
834
|
+
pattern: attachedPasswordFlag,
|
|
835
|
+
rewrite: (text) => redactArguments(text, attachedPasswordFlag)
|
|
836
|
+
}
|
|
837
|
+
];
|
|
838
|
+
/**
|
|
839
|
+
* Redacts one diagnostic value with {@link diagnosticRules}: a log line, a
|
|
840
|
+
* stderr message, an error handed back to a caller.
|
|
841
|
+
*
|
|
842
|
+
* Every diagnostic path shares this one function, and it runs on the value
|
|
843
|
+
* before anything renders or bounds it. Redacting a rendering line by line, or
|
|
844
|
+
* after a bound, is how a secret escaped: `util.inspect` splits a long string
|
|
845
|
+
* over several lines, and a bound cuts a quoted value before its closing
|
|
846
|
+
* quote. A value past {@link maxDepth} is named rather than thrown.
|
|
847
|
+
*
|
|
848
|
+
* @since 1.0.0
|
|
849
|
+
* @category redaction
|
|
850
|
+
*/
|
|
851
|
+
export const redactDiagnostic = (value) => redact(value, { rules: diagnosticRules, onTooDeep: "name" });
|
|
852
|
+
/**
|
|
853
|
+
* Credential names this module refuses to persist, matched as suffixes of the
|
|
854
|
+
* separator-free lowercase form of a key.
|
|
855
|
+
*
|
|
856
|
+
* The list is the union of the names `packages/smithers/src/Bug.ts` redacts
|
|
857
|
+
* structurally, because the journal is the PERMANENT side of that pair: a bug
|
|
858
|
+
* report is one upload an operator reviews before it leaves the machine, while
|
|
859
|
+
* `flows_journal_events.payload_json` is replayed verbatim to every sync
|
|
860
|
+
* subscriber and time-travel consumer forever. The journal redacting less than
|
|
861
|
+
* the report inverts the risk, and it did: `credential`, `credentials`, `dsn`,
|
|
862
|
+
* and `connectionString` all round-tripped through a durable row in clear.
|
|
863
|
+
*
|
|
864
|
+
* Two families are spelled out rather than reduced to a shorter word, because
|
|
865
|
+
* these are SUFFIX tests: `connectionString` does not end in `connection`, and
|
|
866
|
+
* `secretKey` does not end in `secret`. A bare `key` suffix is not on the list
|
|
867
|
+
* for the opposite reason: it would redact `monkey` and `turkey`.
|
|
868
|
+
*
|
|
869
|
+
* `signature` stays, and a field it costs is renamed rather than excused. A
|
|
870
|
+
* name test cannot tell a MAC computed with a secret from a digest over public
|
|
871
|
+
* input, and both are called `signature`: `WorkspaceShare` signs its claims
|
|
872
|
+
* with `ShareSigner.signHmac` and that value IS the share's authorization,
|
|
873
|
+
* `GrantStore` carries envelope signatures, and the GitHub, Linear and
|
|
874
|
+
* Telegram webhook readers all verify a header by that name. Against those, a
|
|
875
|
+
* digest whose whole purpose is to be reconciled costs nothing to rename and
|
|
876
|
+
* everything to leave unreadable -- see
|
|
877
|
+
* `AgentEvent.VacuousVerificationObserved.callDigest`, which was `signature`
|
|
878
|
+
* and reached durable rows as the placeholder.
|
|
879
|
+
*/
|
|
880
|
+
const sensitiveKeySuffixes = [
|
|
881
|
+
"auth",
|
|
882
|
+
"authorization",
|
|
883
|
+
"cookie",
|
|
884
|
+
"apikey",
|
|
885
|
+
"token",
|
|
886
|
+
"password",
|
|
887
|
+
"passphrase",
|
|
888
|
+
"secret",
|
|
889
|
+
"secretaccesskey",
|
|
890
|
+
"credential",
|
|
891
|
+
"dsn",
|
|
892
|
+
"connection",
|
|
893
|
+
"connectionstring",
|
|
894
|
+
"connectionuri",
|
|
895
|
+
"connectionurl",
|
|
896
|
+
"secretkey",
|
|
897
|
+
"privatekey",
|
|
898
|
+
"sshkey",
|
|
899
|
+
"signingkey",
|
|
900
|
+
"signature",
|
|
901
|
+
"encryptionkey",
|
|
902
|
+
"session",
|
|
903
|
+
"sessionkey"
|
|
904
|
+
];
|
|
905
|
+
/**
|
|
906
|
+
* A trailing `key` that is a word of its own.
|
|
907
|
+
*
|
|
908
|
+
* Read against the ORIGINAL key, where the separators that make `key` a word
|
|
909
|
+
* still exist: `key`, `api_key`, `x-api-key`, `signing-key`. This is the shape
|
|
910
|
+
* `packages/smithers/src/Bug.ts` matches, and stopping here is deliberate. Treating
|
|
911
|
+
* a camel-case hump as a separator too, `/[a-z0-9]Key$/`, looks equivalent and
|
|
912
|
+
* is not: it redacted `idempotencyKey`, the durable identity an effect boundary
|
|
913
|
+
* replays on, out of `@smthrs/engine-store`'s journal rows. A credential-named
|
|
914
|
+
* key is covered by {@link sensitiveKeySuffixes} instead, where the word before
|
|
915
|
+
* `key` has to be one that names a secret.
|
|
916
|
+
*/
|
|
917
|
+
const trailingKeyWord = /(?:^|[^A-Za-z0-9])key$/i;
|
|
918
|
+
const canonicalKey = (key) => key.toLowerCase().replace(/[^a-z0-9]/g, "").replace(/s$/, "");
|
|
919
|
+
/** Numeric accounting fields that contain counts, never bearer material. */
|
|
920
|
+
const tokenCounterKeys = new Set([
|
|
921
|
+
"inputtoken",
|
|
922
|
+
"outputtoken",
|
|
923
|
+
"cachedinputtoken",
|
|
924
|
+
"reasoningtoken",
|
|
925
|
+
"totaltoken"
|
|
926
|
+
]);
|
|
927
|
+
/**
|
|
928
|
+
* A bare plural `tokens` is a count too: `Envelope.budget.tokens` is the run's
|
|
929
|
+
* token ceiling, and redacting it erased every journaled budget. A singular
|
|
930
|
+
* `token` stays bearer material even when it is numeric.
|
|
931
|
+
*/
|
|
932
|
+
const isTokenCount = (key, value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 &&
|
|
933
|
+
(tokenCounterKeys.has(canonicalKey(key)) || key.toLowerCase() === "tokens");
|
|
934
|
+
/**
|
|
935
|
+
* Whether a field name names a credential.
|
|
936
|
+
*
|
|
937
|
+
* Case and separators are ignored, so `api_key`, `apiKey`, and `x-api-key` are
|
|
938
|
+
* all recognised, one trailing plural `s` is stripped so `credentials` reads as
|
|
939
|
+
* `credential`, and a trailing `key` counts where it is a word of its own or
|
|
940
|
+
* where the word before it names a secret.
|
|
941
|
+
*
|
|
942
|
+
* The rule is a suffix test, not a substring test. `tokenizer`, `secretary`,
|
|
943
|
+
* `monkey`, and `idempotencyKey` are ordinary field names, and replacing their
|
|
944
|
+
* values would destroy data in a permanent row without protecting anything.
|
|
945
|
+
* That is the one place this rule is deliberately narrower than
|
|
946
|
+
* `packages/smithers/src/Bug.ts`'s substring form, which can afford the false
|
|
947
|
+
* positives because its output is a single report an operator reads once.
|
|
948
|
+
*
|
|
949
|
+
* @since 0.1.0
|
|
950
|
+
* @category predicates
|
|
951
|
+
*/
|
|
952
|
+
export const isSensitiveKey = (key) => {
|
|
953
|
+
if (trailingKeyWord.test(key))
|
|
954
|
+
return true;
|
|
955
|
+
const canonical = canonicalKey(key);
|
|
956
|
+
return sensitiveKeySuffixes.some((suffix) => canonical.endsWith(suffix));
|
|
957
|
+
};
|
|
958
|
+
const redactMember = (key, value, walk) => isSensitiveKey(key) && !isTokenCount(key, value) ? placeholder : walk(value);
|
|
959
|
+
/**
|
|
960
|
+
* A field name with a credential in it replaced outright, not rewritten in place.
|
|
961
|
+
*
|
|
962
|
+
* Redacting part of a key changes what the key reads as. `secret=sk-` rewrites
|
|
963
|
+
* to `secret=[REDACTED]`, which now ENDS in a sensitive name, so a second
|
|
964
|
+
* application would replace a value the first one left alone and redaction
|
|
965
|
+
* would stop being a fixed point. Naming the whole key holds the verdict still:
|
|
966
|
+
* `[REDACTED]` matches no rule and names no credential. Two such keys collapse
|
|
967
|
+
* into one and the later member wins, which is the fidelity trade every other
|
|
968
|
+
* marker here makes.
|
|
969
|
+
*/
|
|
970
|
+
const redactKey = (key, rules) => {
|
|
971
|
+
const redacted = redactString(key, rules);
|
|
972
|
+
return redacted === key ? key : placeholder;
|
|
973
|
+
};
|
|
974
|
+
/**
|
|
975
|
+
* How many times {@link redactString} re-applies the rules to reach a fixed
|
|
976
|
+
* point. Rules feed each other: an assignment redacted inside URL userinfo
|
|
977
|
+
* leaves userinfo the URL rule only then matches. Text no rule touches costs
|
|
978
|
+
* one pass, and the default rules settle within three. Text that has not
|
|
979
|
+
* settled after the last pass is replaced whole.
|
|
980
|
+
*/
|
|
981
|
+
const fixedPointPasses = 8;
|
|
982
|
+
/**
|
|
983
|
+
* A copy of `pattern` that matches wherever `pattern` does, with no
|
|
984
|
+
* lookbehind, or `null`. JavaScriptCore runs a pattern that has a lookbehind
|
|
985
|
+
* in its interpreter, 30 to 70 times slower than compiled: 40 kB of prose cost
|
|
986
|
+
* a Bun process 300 ms per diagnostic, where Node took 1 ms. Text the copy
|
|
987
|
+
* never matches is text the rule cannot change, so the rule is skipped.
|
|
988
|
+
*
|
|
989
|
+
* Only a one-character negative lookbehind that starts the pattern is
|
|
990
|
+
* rewritten, as a character the copy consumes, which keeps a name scan
|
|
991
|
+
* starting once per identifier. Anywhere else, after `\b` or inside a
|
|
992
|
+
* repeated group, the consumed character would make the copy stricter than
|
|
993
|
+
* the rule, so any other lookbehind or a backreference leaves the rule
|
|
994
|
+
* unfiltered.
|
|
995
|
+
*/
|
|
996
|
+
const candidate = (pattern) => {
|
|
997
|
+
const cached = candidates.get(pattern);
|
|
998
|
+
if (cached !== undefined)
|
|
999
|
+
return cached;
|
|
1000
|
+
const leading = /^\(\?<!\[(\^?)((?:\\.|[^\]\\])*)\]\)/.exec(pattern.source);
|
|
1001
|
+
const rest = leading === null ? pattern.source : pattern.source.slice(leading[0].length);
|
|
1002
|
+
const compiled = /\(\?<[!=]|\\(?:[1-9]|k<)/.test(rest)
|
|
1003
|
+
? null
|
|
1004
|
+
: new RegExp(leading === null ? rest : `(?:^|[${leading[1] === "" ? "^" : ""}${leading[2]}])${rest}`, pattern.flags.replace(/[gy]/g, ""));
|
|
1005
|
+
candidates.set(pattern, compiled);
|
|
1006
|
+
return compiled;
|
|
1007
|
+
};
|
|
1008
|
+
const candidates = new WeakMap();
|
|
1009
|
+
const applyRule = (text, rule) => {
|
|
1010
|
+
if (candidate(rule.pattern)?.test(text) === false)
|
|
1011
|
+
return text;
|
|
1012
|
+
return rule.rewrite === undefined ? text.replace(rule.pattern, rule.replace ?? placeholder) : rule.rewrite(text);
|
|
1013
|
+
};
|
|
1014
|
+
const redactString = (value, rules) => {
|
|
1015
|
+
let text = value;
|
|
1016
|
+
for (let pass = 0; pass < fixedPointPasses; pass++) {
|
|
1017
|
+
const next = rules.reduce(applyRule, text);
|
|
1018
|
+
if (next === text)
|
|
1019
|
+
return text;
|
|
1020
|
+
text = next;
|
|
1021
|
+
}
|
|
1022
|
+
// Rules that never settle would let an exported row keep changing, and
|
|
1023
|
+
// what they left may still hold a credential. Fail closed.
|
|
1024
|
+
return placeholder;
|
|
1025
|
+
};
|
|
1026
|
+
/**
|
|
1027
|
+
* Maximum number of container edges traversed from a redaction root.
|
|
1028
|
+
*
|
|
1029
|
+
* Journal event schemas never approach 256 nested containers. This bound is
|
|
1030
|
+
* far above practical payloads while keeping traversal safely below Node's
|
|
1031
|
+
* call-stack limit for hostile input.
|
|
1032
|
+
*
|
|
1033
|
+
* @since 1.0.0
|
|
1034
|
+
* @category constants
|
|
1035
|
+
*/
|
|
1036
|
+
export const maxDepth = 256;
|
|
1037
|
+
/**
|
|
1038
|
+
* A payload nested deeper than {@link maxDepth}, under `onTooDeep: "throw"`.
|
|
1039
|
+
* `code` is always `depth_exceeded`; `maxDepth` is the bound that was crossed.
|
|
1040
|
+
*
|
|
1041
|
+
* @since 1.0.0
|
|
1042
|
+
* @category errors
|
|
1043
|
+
*/
|
|
1044
|
+
export class RedactionDepthError extends Schema.TaggedError()("@smthrs/journal/RedactionDepthError", { code: Schema.Literal("depth_exceeded"), maxDepth: Schema.Number, message: Schema.String }) {
|
|
1045
|
+
name = "RedactionDepthError";
|
|
1046
|
+
}
|
|
1047
|
+
/**
|
|
1048
|
+
* How many bytes a view may hold, and how many own members {@link redact} reads
|
|
1049
|
+
* off it, before it stops reading them.
|
|
1050
|
+
*
|
|
1051
|
+
* Enumerating a view's own properties materialises one pair per byte, so the
|
|
1052
|
+
* walk costs the buffer's size even though the rendering does not. Both halves
|
|
1053
|
+
* of the bound carry load, and neither is a bound on its own. A size alone is
|
|
1054
|
+
* not, because `byteLength` was read as an ordinary property and an ordinary
|
|
1055
|
+
* property can be shadowed: a pooled chunk that reports the bytes it has used
|
|
1056
|
+
* rather than the bytes it holds is a plain pattern, not an adversarial one,
|
|
1057
|
+
* and a 4 MB chunk reporting 12 walked one property per byte anyway. A count
|
|
1058
|
+
* alone is not, because a caller can hang a million properties on a ten-byte
|
|
1059
|
+
* view. So the size is read from the value's own internal slot, where nothing
|
|
1060
|
+
* a caller writes can answer for it, and the members are counted as they are
|
|
1061
|
+
* walked.
|
|
1062
|
+
*
|
|
1063
|
+
* @since 0.1.0
|
|
1064
|
+
* @category redaction
|
|
1065
|
+
*/
|
|
1066
|
+
export const binaryWalkLimit = 65_536;
|
|
1067
|
+
/**
|
|
1068
|
+
* The byte-length getters, taken from the prototypes rather than from a value.
|
|
1069
|
+
*
|
|
1070
|
+
* `%TypedArray%.prototype`, `DataView.prototype` and `ArrayBuffer.prototype`
|
|
1071
|
+
* each define `byteLength` as an accessor over an internal slot, and applying
|
|
1072
|
+
* one to a value that has no such slot throws. Reading the size this way is
|
|
1073
|
+
* therefore a brand check as well as a measurement: a caller's own `byteLength`
|
|
1074
|
+
* property, accessor or not, never answers it.
|
|
1075
|
+
*/
|
|
1076
|
+
const byteLengthGetters = [
|
|
1077
|
+
Object.getOwnPropertyDescriptor(Object.getPrototypeOf(Uint8Array.prototype), "byteLength").get,
|
|
1078
|
+
Object.getOwnPropertyDescriptor(DataView.prototype, "byteLength").get,
|
|
1079
|
+
Object.getOwnPropertyDescriptor(ArrayBuffer.prototype, "byteLength").get
|
|
1080
|
+
];
|
|
1081
|
+
/** A value's byte length from its own internal slot, or `undefined` when it has none. */
|
|
1082
|
+
const byteLength = (node) => {
|
|
1083
|
+
for (const getter of byteLengthGetters) {
|
|
1084
|
+
try {
|
|
1085
|
+
return Reflect.apply(getter, node, []);
|
|
1086
|
+
}
|
|
1087
|
+
catch {
|
|
1088
|
+
// Not that kind of view. Another getter answers, or none does and the
|
|
1089
|
+
// bytes are named without the members being read.
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
return undefined;
|
|
1093
|
+
};
|
|
1094
|
+
/** The prototypes every binary view inherits from. */
|
|
1095
|
+
const binaryPrototypes = new Set([
|
|
1096
|
+
Object.getPrototypeOf(Uint8Array.prototype),
|
|
1097
|
+
DataView.prototype,
|
|
1098
|
+
ArrayBuffer.prototype
|
|
1099
|
+
]);
|
|
1100
|
+
/** How many prototypes {@link isBinary} climbs before it stops asking. */
|
|
1101
|
+
const prototypeWalkLimit = 64;
|
|
1102
|
+
/**
|
|
1103
|
+
* Whether `node` is a binary view, including one held behind a proxy.
|
|
1104
|
+
*
|
|
1105
|
+
* `ArrayBuffer.isView` reads an internal slot, and a proxy has none of its own,
|
|
1106
|
+
* so it answers `false` for a proxy over a view. That sent a proxied 2 MB
|
|
1107
|
+
* buffer to the object branch, which rebuilt it one key per byte: 2,000 ms and
|
|
1108
|
+
* 22.9 million characters for one logged value. A proxy forwards
|
|
1109
|
+
* `getPrototypeOf` to its target, so the prototype chain answers where the
|
|
1110
|
+
* brand check cannot. The climb is capped because a proxy is free to hand back
|
|
1111
|
+
* a fresh object every time it is asked, which is an endless chain.
|
|
1112
|
+
*/
|
|
1113
|
+
const isBinary = (node) => {
|
|
1114
|
+
if (ArrayBuffer.isView(node) || node instanceof ArrayBuffer)
|
|
1115
|
+
return true;
|
|
1116
|
+
try {
|
|
1117
|
+
let prototype = Object.getPrototypeOf(node);
|
|
1118
|
+
for (let step = 0; prototype !== null && step < prototypeWalkLimit; step++) {
|
|
1119
|
+
if (binaryPrototypes.has(prototype))
|
|
1120
|
+
return true;
|
|
1121
|
+
prototype = Object.getPrototypeOf(prototype);
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
catch {
|
|
1125
|
+
// A revoked proxy refuses to be asked. It is not a view, and the object
|
|
1126
|
+
// branch names it.
|
|
1127
|
+
}
|
|
1128
|
+
return false;
|
|
1129
|
+
};
|
|
1130
|
+
/** Own keys of a binary view that name one of its bytes rather than a property. */
|
|
1131
|
+
const indexKey = /^(?:0|[1-9][0-9]*)$/;
|
|
1132
|
+
/** `Uint8Array 1024 bytes`, or the bare marker when the value will not say. */
|
|
1133
|
+
const describeBinary = (node, size) => {
|
|
1134
|
+
if (size === undefined)
|
|
1135
|
+
return binaryMarker;
|
|
1136
|
+
try {
|
|
1137
|
+
return `${node.constructor.name} ${size} bytes`;
|
|
1138
|
+
}
|
|
1139
|
+
catch {
|
|
1140
|
+
return binaryMarker;
|
|
1141
|
+
}
|
|
1142
|
+
};
|
|
1143
|
+
/**
|
|
1144
|
+
* The key {@link redact} files a binary view's size under.
|
|
1145
|
+
*
|
|
1146
|
+
* @since 0.1.0
|
|
1147
|
+
* @category redaction
|
|
1148
|
+
*/
|
|
1149
|
+
export const binaryMarker = "[Binary]";
|
|
1150
|
+
/**
|
|
1151
|
+
* What {@link redact} writes in place of a function or a class object.
|
|
1152
|
+
*
|
|
1153
|
+
* @since 0.1.0
|
|
1154
|
+
* @category redaction
|
|
1155
|
+
*/
|
|
1156
|
+
export const functionMarker = "[Function]";
|
|
1157
|
+
/**
|
|
1158
|
+
* What {@link redact} writes in place of a symbol.
|
|
1159
|
+
*
|
|
1160
|
+
* @since 0.1.0
|
|
1161
|
+
* @category redaction
|
|
1162
|
+
*/
|
|
1163
|
+
export const symbolMarker = "[Symbol]";
|
|
1164
|
+
/**
|
|
1165
|
+
* What {@link redact} writes past {@link maxDepth} under `onTooDeep: "name"`.
|
|
1166
|
+
*
|
|
1167
|
+
* @since 0.1.0
|
|
1168
|
+
* @category redaction
|
|
1169
|
+
*/
|
|
1170
|
+
export const depthMarker = "[Deep]";
|
|
1171
|
+
/**
|
|
1172
|
+
* Returns `value` with credentials removed.
|
|
1173
|
+
*
|
|
1174
|
+
* Objects and arrays are rebuilt: a field whose name {@link isSensitiveKey}
|
|
1175
|
+
* is replaced wholesale, every other string is run through the textual rules,
|
|
1176
|
+
* and a cycle is collapsed to `"[Circular]"` so the result always encodes.
|
|
1177
|
+
* A number, a boolean, `null` and `undefined` are returned untouched, because
|
|
1178
|
+
* redacting one destroys data without protecting anything. A function and a
|
|
1179
|
+
* symbol are NAMED, {@link functionMarker} and {@link symbolMarker}: a body, an
|
|
1180
|
+
* own property and a description are all text a renderer prints, and none of it
|
|
1181
|
+
* can be rewritten in place.
|
|
1182
|
+
*
|
|
1183
|
+
* Traversal accepts at most {@link maxDepth} container edges. A deeper value
|
|
1184
|
+
* throws by default, so the journal boundary can report a typed `invalid_event`
|
|
1185
|
+
* instead of overflowing the runtime stack. {@link Options.onTooDeep} set to
|
|
1186
|
+
* `"name"` writes {@link depthMarker} in its place instead, which is what a
|
|
1187
|
+
* logger wants.
|
|
1188
|
+
*
|
|
1189
|
+
* @since 0.1.0
|
|
1190
|
+
* @category redaction
|
|
1191
|
+
*/
|
|
1192
|
+
export const redact = (value, options) => {
|
|
1193
|
+
const onTooDeep = options?.onTooDeep ?? "throw";
|
|
1194
|
+
const rules = globalRules(options?.rules ?? defaultRules);
|
|
1195
|
+
/**
|
|
1196
|
+
* A binary view named by its type and size, with its own properties walked.
|
|
1197
|
+
*
|
|
1198
|
+
* Its bytes are not text the rules can read, and rebuilding it from its
|
|
1199
|
+
* entries wrote one key per byte: a 100 kB buffer rendered 1.4 MB. Handing
|
|
1200
|
+
* the view back untouched is not the answer either, because `redact` is the
|
|
1201
|
+
* journal's own write path, so a caller's `apiKey` property hung on a buffer
|
|
1202
|
+
* went into a durable row in clear. Name the bytes, keep walking the text.
|
|
1203
|
+
*/
|
|
1204
|
+
const binary = (node, ancestors, depth) => {
|
|
1205
|
+
const size = byteLength(node);
|
|
1206
|
+
// The description is text read off the value, so it meets the rules like
|
|
1207
|
+
// any other text: a class name is caller data, and a view whose
|
|
1208
|
+
// constructor is named after a credential wrote it into a durable row.
|
|
1209
|
+
const entries = [[binaryMarker, redactString(describeBinary(node, size), rules)]];
|
|
1210
|
+
// `Object.entries` on a view materialises one pair per byte before the
|
|
1211
|
+
// index filter can discard them, which cost 312 ms for 1 MB on the journal
|
|
1212
|
+
// write path. Past the bound the bytes are named and nothing else is read,
|
|
1213
|
+
// so a member a caller hung on a large view is dropped rather than shown.
|
|
1214
|
+
// A value that will not answer from its internal slot, such as a proxy over
|
|
1215
|
+
// a view, is named and never enumerated at all.
|
|
1216
|
+
if (size !== undefined && size <= binaryWalkLimit) {
|
|
1217
|
+
let walked = 0;
|
|
1218
|
+
for (const [key, field] of Object.entries(node)) {
|
|
1219
|
+
// An index is one of the bytes just named, not a property a caller set.
|
|
1220
|
+
if (indexKey.test(key))
|
|
1221
|
+
continue;
|
|
1222
|
+
// The size bounds the bytes, not the properties: a caller can hang a
|
|
1223
|
+
// million members on a ten-byte view, and each one costs three rule
|
|
1224
|
+
// scans over its name and a walk of its value.
|
|
1225
|
+
if (walked >= binaryWalkLimit)
|
|
1226
|
+
break;
|
|
1227
|
+
walked++;
|
|
1228
|
+
entries.push([
|
|
1229
|
+
redactKey(key, rules),
|
|
1230
|
+
redactMember(key, field, (value) => walk(value, ancestors, depth + 1, key))
|
|
1231
|
+
]);
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
// `Object.fromEntries`, not `named[key] = …`, for the reason the object
|
|
1235
|
+
// branch below gives: a literal `__proto__` key would route through the
|
|
1236
|
+
// inherited setter and the member would vanish from the row.
|
|
1237
|
+
return Object.fromEntries(entries);
|
|
1238
|
+
};
|
|
1239
|
+
const walk = (node, ancestors, depth, key) => {
|
|
1240
|
+
if (depth > maxDepth) {
|
|
1241
|
+
if (onTooDeep === "name")
|
|
1242
|
+
return depthMarker;
|
|
1243
|
+
throw new RedactionDepthError({
|
|
1244
|
+
code: "depth_exceeded",
|
|
1245
|
+
maxDepth,
|
|
1246
|
+
message: `redaction depth exceeds ${maxDepth}`
|
|
1247
|
+
});
|
|
1248
|
+
}
|
|
1249
|
+
if (typeof node === "string")
|
|
1250
|
+
return redactString(node, rules);
|
|
1251
|
+
// A function, a class object or a symbol carries text a walk never reaches
|
|
1252
|
+
// (own properties, a description, a body), and the renderer prints it. None
|
|
1253
|
+
// of it can be redacted in place, so the value is named instead.
|
|
1254
|
+
if (typeof node === "function")
|
|
1255
|
+
return functionMarker;
|
|
1256
|
+
if (typeof node === "symbol")
|
|
1257
|
+
return symbolMarker;
|
|
1258
|
+
if (node === null || typeof node !== "object")
|
|
1259
|
+
return node;
|
|
1260
|
+
if (isBinary(node))
|
|
1261
|
+
return binary(node, ancestors, depth);
|
|
1262
|
+
if (ancestors.has(node))
|
|
1263
|
+
return "[Circular]";
|
|
1264
|
+
// A journal row is bounded by its schema, but a LOGGED value is arbitrary:
|
|
1265
|
+
// the logger sends every non-Error value here, and a chain deep enough to
|
|
1266
|
+
// exhaust the stack would throw while the line is rendering, killing the
|
|
1267
|
+
// run the line describes. Past the cap the value is named, the way a cycle
|
|
1268
|
+
// is named.
|
|
1269
|
+
ancestors.add(node);
|
|
1270
|
+
try {
|
|
1271
|
+
const toJSON = node.toJSON;
|
|
1272
|
+
// JSON.stringify passes the containing key (or "" at the root) to toJSON.
|
|
1273
|
+
// Reuse it when walking the replacement; siblings may share this object.
|
|
1274
|
+
if (typeof toJSON === "function")
|
|
1275
|
+
return walk(toJSON.call(node, key), ancestors, depth, key);
|
|
1276
|
+
if (Array.isArray(node)) {
|
|
1277
|
+
// `map` invokes the input's species constructor, which can restore
|
|
1278
|
+
// credential fields and inspection hooks after the elements are walked.
|
|
1279
|
+
// Rebuild a plain array, preserving the original length and holes.
|
|
1280
|
+
const length = node.length;
|
|
1281
|
+
const result = new Array(length);
|
|
1282
|
+
const elements = new Array(length);
|
|
1283
|
+
for (let index = 0; index < length; index++) {
|
|
1284
|
+
if (!(index in node))
|
|
1285
|
+
continue;
|
|
1286
|
+
elements[index] = node[index];
|
|
1287
|
+
result[index] = walk(elements[index], ancestors, depth + 1, String(index));
|
|
1288
|
+
}
|
|
1289
|
+
// An argv keeps a flag and its value in two elements, so no rule sees
|
|
1290
|
+
// `--password hunter2` whole. The element after a flag is a credential
|
|
1291
|
+
// when its name says so (`--token=x` carries its own, and a count under
|
|
1292
|
+
// `--max-tokens` is kept), or when the active rules redact it read
|
|
1293
|
+
// after the flag and the elements before it (`-p`, `-pw`, `-passin`,
|
|
1294
|
+
// `sshpass -p`) differently than alone.
|
|
1295
|
+
for (let index = 0; index + 1 < length; index++) {
|
|
1296
|
+
// The values the first loop read, never a second read: a proxy may
|
|
1297
|
+
// answer a second `get` differently.
|
|
1298
|
+
const flag = elements[index];
|
|
1299
|
+
const value = elements[index + 1];
|
|
1300
|
+
if (typeof flag !== "string" || typeof value !== "string" || !/^-[^=\s]+$/.test(flag))
|
|
1301
|
+
continue;
|
|
1302
|
+
const name = flag.replace(/^-+/, "");
|
|
1303
|
+
const before = elements.slice(Math.max(0, index - 3), index + 1).filter((element) => typeof element === "string").join(" ");
|
|
1304
|
+
const consumes = (isSensitiveKey(name) && namesCredential(name, value)) ||
|
|
1305
|
+
redactString(`${before} ${value}`, rules) !== `${redactString(before, rules)} ${redactString(value, rules)}`;
|
|
1306
|
+
if (consumes)
|
|
1307
|
+
result[index + 1] = placeholder;
|
|
1308
|
+
}
|
|
1309
|
+
return result;
|
|
1310
|
+
}
|
|
1311
|
+
// `result[key] = …` routes a literal `__proto__` key through the
|
|
1312
|
+
// inherited setter, so the field would silently become the result's
|
|
1313
|
+
// prototype instead of a member: the payload loses data and redaction
|
|
1314
|
+
// stops being a fixed point. `Object.fromEntries` defines own data
|
|
1315
|
+
// properties and has no such hole.
|
|
1316
|
+
return Object.fromEntries(Object.entries(node).map(([key, field]) => [
|
|
1317
|
+
// The KEY is text too. Rewriting only values let a credential used as
|
|
1318
|
+
// a log annotation key reach the operator, since Effect renders an
|
|
1319
|
+
// annotation as `key=value`, and become an OTLP span attribute name.
|
|
1320
|
+
redactKey(key, rules),
|
|
1321
|
+
redactMember(key, field, (value) => walk(value, ancestors, depth + 1, key))
|
|
1322
|
+
]));
|
|
1323
|
+
}
|
|
1324
|
+
finally {
|
|
1325
|
+
ancestors.delete(node);
|
|
1326
|
+
}
|
|
1327
|
+
};
|
|
1328
|
+
return walk(value, new WeakSet(), 0, "");
|
|
1329
|
+
};
|
|
1330
|
+
/**
|
|
1331
|
+
* Builds a redactor over a rule set.
|
|
1332
|
+
*
|
|
1333
|
+
* @since 0.1.0
|
|
1334
|
+
* @category constructors
|
|
1335
|
+
*/
|
|
1336
|
+
export const make = (options) => (value) => redact(value, options);
|
|
1337
|
+
/**
|
|
1338
|
+
* Applies a redactor to an already-encoded JSON string, returning the
|
|
1339
|
+
* re-encoded result.
|
|
1340
|
+
*
|
|
1341
|
+
* For export and display surfaces that hold a column verbatim, a rendered
|
|
1342
|
+
* `state_json`, a support bundle, where the value is already encoded and must
|
|
1343
|
+
* not be decoded into the executable path. A string that does not parse is
|
|
1344
|
+
* returned untouched: validation is the caller's, and rejecting here would
|
|
1345
|
+
* turn a redaction concern into a schema error. Once parsing succeeds, a
|
|
1346
|
+
* throwing redactor or an encoding failure returns the valid JSON string
|
|
1347
|
+
* `"[REDACTED]"`; the original parsed text is never returned.
|
|
1348
|
+
*
|
|
1349
|
+
* @since 0.1.0
|
|
1350
|
+
* @category redaction
|
|
1351
|
+
*/
|
|
1352
|
+
export const redactJsonString = (json, redactor) => {
|
|
1353
|
+
const decoded = Schema.decodeUnknownResult(UnknownFromJsonString)(json);
|
|
1354
|
+
if (Result.isFailure(decoded))
|
|
1355
|
+
return json;
|
|
1356
|
+
const attempted = Result.try(() => Schema.encodeUnknownResult(UnknownFromJsonString)(redactor(decoded.success)));
|
|
1357
|
+
if (Result.isFailure(attempted))
|
|
1358
|
+
return JSON.stringify(placeholder);
|
|
1359
|
+
return Result.isSuccess(attempted.success) ? attempted.success.success : JSON.stringify(placeholder);
|
|
1360
|
+
};
|
|
1361
|
+
/**
|
|
1362
|
+
* Lines a {@link lineRedactor} holds for one open value. Past this, or past
|
|
1363
|
+
* 64 KiB, it stops holding: it emits what it held, redacted, and withholds
|
|
1364
|
+
* every later line until the stream ends, when one placeholder stands for
|
|
1365
|
+
* them. Dropping lines from the middle instead changed which brackets and
|
|
1366
|
+
* quotes were open, and released a later secret in clear.
|
|
1367
|
+
*
|
|
1368
|
+
* @since 1.0.0
|
|
1369
|
+
* @category constants
|
|
1370
|
+
*/
|
|
1371
|
+
export const maxHeldLines = 256;
|
|
1372
|
+
const maxHeldBytes = 64 * 1024;
|
|
1373
|
+
/** Held lines a {@link lineRedactor} asks about one by one before it asks only as the block grows by half. */
|
|
1374
|
+
const eagerHeldLines = 16;
|
|
1375
|
+
/**
|
|
1376
|
+
* The most brackets and strings the dropped middle of an overlong line may
|
|
1377
|
+
* leave open before a {@link lineRedactor} stops reading it and treats the
|
|
1378
|
+
* line as leaving a value open. Without it the middle's state grew by one
|
|
1379
|
+
* entry per bracket: 32 MiB of `[` held 1.7 GB (#3088).
|
|
1380
|
+
*/
|
|
1381
|
+
const maxDroppedDepth = 1024;
|
|
1382
|
+
/**
|
|
1383
|
+
* The longest line a {@link lineRedactor} reads whole. A longer line keeps
|
|
1384
|
+
* its first and last 16 KiB; its middle is scanned for what it leaves open
|
|
1385
|
+
* and dropped.
|
|
1386
|
+
*
|
|
1387
|
+
* @since 1.0.0
|
|
1388
|
+
* @category constants
|
|
1389
|
+
*/
|
|
1390
|
+
export const maxPartialLine = 32 * 1024;
|
|
1391
|
+
/**
|
|
1392
|
+
* What a {@link lineRedactor} emits in place of a line longer than
|
|
1393
|
+
* {@link maxPartialLine}.
|
|
1394
|
+
*
|
|
1395
|
+
* @since 1.0.0
|
|
1396
|
+
* @category constants
|
|
1397
|
+
*/
|
|
1398
|
+
export const omittedLine = "[overlong line omitted]";
|
|
1399
|
+
/**
|
|
1400
|
+
* A next line no rule redacts alone, which an open value swallows. The value
|
|
1401
|
+
* is open when the redacted text no longer ENDS with it: text before it may
|
|
1402
|
+
* contain the same word.
|
|
1403
|
+
*/
|
|
1404
|
+
const continuationProbe = "\n'probe'";
|
|
1405
|
+
/** Rules with the global flag each rule needs to replace every match. */
|
|
1406
|
+
const globalRules = (rules) => rules.map((rule) => rule.pattern.flags.includes("g")
|
|
1407
|
+
? rule
|
|
1408
|
+
: { ...rule, pattern: new RegExp(rule.pattern.source, `${rule.pattern.flags}g`) });
|
|
1409
|
+
/**
|
|
1410
|
+
* A redactor for a stream that must be emitted line by line: a live build
|
|
1411
|
+
* log, a child's stderr.
|
|
1412
|
+
*
|
|
1413
|
+
* Redacting each line alone is the flaw that leaked a private key's body, and
|
|
1414
|
+
* the tail of every string `util.inspect` split over several lines: only the
|
|
1415
|
+
* line holding the name was redacted. This one asks the rules themselves
|
|
1416
|
+
* whether a line leaves a value open, by checking whether the value swallows
|
|
1417
|
+
* a probe placed on the next line, and holds lines until it closes. The held
|
|
1418
|
+
* block is then redacted whole, exactly as {@link redact} redacts a string.
|
|
1419
|
+
*
|
|
1420
|
+
* A line longer than {@link maxPartialLine} is read as its first and last
|
|
1421
|
+
* 16 KiB. Its middle is dropped, but a bracket, quote or key block the middle
|
|
1422
|
+
* leaves open would have opened a value the kept ends cannot show, so then
|
|
1423
|
+
* the rest of the stream is withheld. Such a line is never emitted, only
|
|
1424
|
+
* {@link omittedLine}, and neither is any block it was held in.
|
|
1425
|
+
*
|
|
1426
|
+
* Its state is bounded whatever the stream does: the line still arriving is
|
|
1427
|
+
* a 16 KiB head, a 16 KiB tail and at most 1024 open brackets or strings of
|
|
1428
|
+
* its middle, a middle nested deeper counting as one left open; the held
|
|
1429
|
+
* block is bounded by {@link maxHeldLines} and 64 KiB. Its cost is linear:
|
|
1430
|
+
* past the first 16 held lines the block is re-read only once it has grown
|
|
1431
|
+
* by half, on Node and on Bun alike (#3088).
|
|
1432
|
+
*
|
|
1433
|
+
* `rules` defaults to {@link diagnosticRules}. The array is read on every
|
|
1434
|
+
* line, so a caller may add a rule, a session secret, as the stream runs.
|
|
1435
|
+
*
|
|
1436
|
+
* @since 1.0.0
|
|
1437
|
+
* @category constructors
|
|
1438
|
+
*/
|
|
1439
|
+
export const lineRedactor = (rules = diagnosticRules) => {
|
|
1440
|
+
const held = [];
|
|
1441
|
+
let heldBytes = 0;
|
|
1442
|
+
let askAt = 0;
|
|
1443
|
+
let heldOverlong = false;
|
|
1444
|
+
let withheld = false;
|
|
1445
|
+
// The line still arriving: whole up to the bound, then a head, a rolling
|
|
1446
|
+
// tail, and what the dropped middle leaves open.
|
|
1447
|
+
const edge = maxPartialLine / 2;
|
|
1448
|
+
let head = "";
|
|
1449
|
+
let tail = "";
|
|
1450
|
+
let overlong = false;
|
|
1451
|
+
let middle = [];
|
|
1452
|
+
let middleDeep = false;
|
|
1453
|
+
let middleKey = false;
|
|
1454
|
+
let carry = "";
|
|
1455
|
+
const resetPartial = () => {
|
|
1456
|
+
head = "";
|
|
1457
|
+
tail = "";
|
|
1458
|
+
overlong = false;
|
|
1459
|
+
middle = [];
|
|
1460
|
+
middleDeep = false;
|
|
1461
|
+
middleKey = false;
|
|
1462
|
+
carry = "";
|
|
1463
|
+
};
|
|
1464
|
+
const scanDropped = (dropped) => {
|
|
1465
|
+
// What the middle leaves open is read in slices, and once it is nested
|
|
1466
|
+
// past `maxDroppedDepth` it is not read at all: the line is then already
|
|
1467
|
+
// risky, and the stack held one entry per bracket of a line of any length.
|
|
1468
|
+
for (let from = 0; from < dropped.length && !middleDeep; from += edge) {
|
|
1469
|
+
track(middle, dropped, from, Math.min(dropped.length, from + edge));
|
|
1470
|
+
if (middle.length > maxDroppedDepth) {
|
|
1471
|
+
middleDeep = true;
|
|
1472
|
+
middle = [];
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
const seen = carry + dropped;
|
|
1476
|
+
const begin = seen.search(/-----BEGIN[^-]*PRIVATE KEY/);
|
|
1477
|
+
const lastBegin = seen.lastIndexOf("-----BEGIN");
|
|
1478
|
+
const lastEnd = seen.lastIndexOf("-----END");
|
|
1479
|
+
if (begin !== -1 || lastEnd !== -1)
|
|
1480
|
+
middleKey = lastBegin > lastEnd;
|
|
1481
|
+
carry = seen.slice(-64);
|
|
1482
|
+
};
|
|
1483
|
+
const redacted = (text) => redactString(text, globalRules(rules));
|
|
1484
|
+
const release = () => {
|
|
1485
|
+
const text = redacted(held.join("\n"));
|
|
1486
|
+
const overlongBlock = heldOverlong;
|
|
1487
|
+
held.length = 0;
|
|
1488
|
+
heldBytes = 0;
|
|
1489
|
+
askAt = 0;
|
|
1490
|
+
heldOverlong = false;
|
|
1491
|
+
return overlongBlock ? [omittedLine] : text.split("\n");
|
|
1492
|
+
};
|
|
1493
|
+
const part = (text) => {
|
|
1494
|
+
if (withheld)
|
|
1495
|
+
return;
|
|
1496
|
+
if (!overlong) {
|
|
1497
|
+
head += text;
|
|
1498
|
+
if (head.length <= maxPartialLine)
|
|
1499
|
+
return;
|
|
1500
|
+
overlong = true;
|
|
1501
|
+
tail = head.slice(edge);
|
|
1502
|
+
head = head.slice(0, edge);
|
|
1503
|
+
}
|
|
1504
|
+
else
|
|
1505
|
+
tail += text;
|
|
1506
|
+
if (tail.length > edge) {
|
|
1507
|
+
scanDropped(tail.slice(0, tail.length - edge));
|
|
1508
|
+
tail = tail.slice(-edge);
|
|
1509
|
+
}
|
|
1510
|
+
};
|
|
1511
|
+
const line = (rest) => {
|
|
1512
|
+
if (withheld) {
|
|
1513
|
+
resetPartial();
|
|
1514
|
+
return [];
|
|
1515
|
+
}
|
|
1516
|
+
part(rest);
|
|
1517
|
+
let text = head;
|
|
1518
|
+
if (overlong) {
|
|
1519
|
+
const risky = middle.length > 0 || middleDeep || middleKey;
|
|
1520
|
+
text = `${head} ${tail}`;
|
|
1521
|
+
resetPartial();
|
|
1522
|
+
if (risky) {
|
|
1523
|
+
const before = held.length === 0 ? [] : [placeholder];
|
|
1524
|
+
held.length = 0;
|
|
1525
|
+
heldBytes = 0;
|
|
1526
|
+
askAt = 0;
|
|
1527
|
+
heldOverlong = false;
|
|
1528
|
+
withheld = true;
|
|
1529
|
+
return [...before, omittedLine];
|
|
1530
|
+
}
|
|
1531
|
+
heldOverlong = true;
|
|
1532
|
+
}
|
|
1533
|
+
else
|
|
1534
|
+
resetPartial();
|
|
1535
|
+
held.push(text);
|
|
1536
|
+
heldBytes += text.length + 1;
|
|
1537
|
+
// Asking after every line re-redacted the whole block per line, so a
|
|
1538
|
+
// block of n bytes cost n² (#3088). Past the first lines the question is
|
|
1539
|
+
// asked again only once the block has grown by half, which keeps the
|
|
1540
|
+
// cost linear; a block that closed in between is released with the
|
|
1541
|
+
// lines after it, redacted as one block, which hides no less.
|
|
1542
|
+
if (held.length <= eagerHeldLines || heldBytes >= askAt) {
|
|
1543
|
+
if (redacted(held.join("\n") + continuationProbe).endsWith(continuationProbe))
|
|
1544
|
+
return release();
|
|
1545
|
+
askAt = heldBytes + (heldBytes >> 1);
|
|
1546
|
+
}
|
|
1547
|
+
if (held.length <= maxHeldLines && heldBytes <= maxHeldBytes)
|
|
1548
|
+
return [];
|
|
1549
|
+
withheld = true;
|
|
1550
|
+
return release();
|
|
1551
|
+
};
|
|
1552
|
+
const peek = () => {
|
|
1553
|
+
if (withheld)
|
|
1554
|
+
return [placeholder];
|
|
1555
|
+
const pending = overlong ? [omittedLine] : head === "" ? [] : [head];
|
|
1556
|
+
if (heldOverlong)
|
|
1557
|
+
return [omittedLine];
|
|
1558
|
+
const text = [...held, ...(overlong ? [] : pending)];
|
|
1559
|
+
const out = text.length === 0 ? [] : redacted(text.join("\n")).split("\n");
|
|
1560
|
+
return overlong ? [...out, omittedLine] : out;
|
|
1561
|
+
};
|
|
1562
|
+
return {
|
|
1563
|
+
line,
|
|
1564
|
+
part,
|
|
1565
|
+
peek,
|
|
1566
|
+
flush: () => {
|
|
1567
|
+
const out = head !== "" || overlong ? [...line("")] : [];
|
|
1568
|
+
if (withheld) {
|
|
1569
|
+
withheld = false;
|
|
1570
|
+
return [...out, placeholder];
|
|
1571
|
+
}
|
|
1572
|
+
return held.length === 0 ? out : [...out, ...release()];
|
|
1573
|
+
}
|
|
1574
|
+
};
|
|
1575
|
+
};
|
|
1576
|
+
/**
|
|
1577
|
+
* The identity redactor, for a caller that persists payloads verbatim by
|
|
1578
|
+
* choice: a trusted single-tenant store, or a suite asserting on raw input.
|
|
1579
|
+
*
|
|
1580
|
+
* @since 0.1.0
|
|
1581
|
+
* @category constructors
|
|
1582
|
+
*/
|
|
1583
|
+
export const makeNoop = () => (value) => value;
|
|
1584
|
+
//# sourceMappingURL=Redaction.js.map
|