@memlab/core 2.0.3 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/lib/HeapAnonymizer.test.d.ts +11 -0
- package/dist/__tests__/lib/HeapAnonymizer.test.d.ts.map +1 -0
- package/dist/__tests__/lib/HeapAnonymizer.test.js +377 -0
- package/dist/__tests__/trace-cluster/MLTraceSimilarityStrategy.test.d.ts +11 -0
- package/dist/__tests__/trace-cluster/MLTraceSimilarityStrategy.test.d.ts.map +1 -0
- package/dist/__tests__/trace-cluster/MLTraceSimilarityStrategy.test.js +50 -0
- package/dist/__tests__/utils/Utils.test.js +11 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -1
- package/dist/lib/HeapAnonymizer.d.ts +362 -0
- package/dist/lib/HeapAnonymizer.d.ts.map +1 -0
- package/dist/lib/HeapAnonymizer.js +670 -0
- package/dist/lib/HeapSerializer.d.ts +125 -0
- package/dist/lib/HeapSerializer.d.ts.map +1 -0
- package/dist/lib/HeapSerializer.js +315 -0
- package/dist/lib/Types.d.ts.map +1 -1
- package/dist/lib/Utils.d.ts.map +1 -1
- package/dist/lib/Utils.js +12 -0
- package/dist/lib/leak-filters/BaseLeakFilter.rule.d.ts.map +1 -1
- package/dist/lib/leak-filters/rules/FilterUserTaggedLeaks.rule.d.ts.map +1 -1
- package/dist/lib/leak-filters/rules/FilterUserTaggedLeaks.rule.js +17 -2
- package/dist/lib/trace-filters/rules/FilterAttachedDOMToDetachedDOMTrace.rule.d.ts.map +1 -1
- package/dist/lib/trace-filters/rules/FilterCppRootsToDetachedDOMTrace.rule.d.ts.map +1 -1
- package/dist/paths/TraceFinder.d.ts +1 -3
- package/dist/paths/TraceFinder.d.ts.map +1 -1
- package/dist/paths/TraceFinder.js +205 -254
- package/dist/trace-cluster/strategies/MLTraceSimilarityStrategy.d.ts.map +1 -1
- package/dist/trace-cluster/strategies/MLTraceSimilarityStrategy.js +7 -2
- package/package.json +3 -3
|
@@ -0,0 +1,670 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
3
|
+
*
|
|
4
|
+
* This source code is licensed under the MIT license found in the
|
|
5
|
+
* LICENSE file in the root directory of this source tree.
|
|
6
|
+
*
|
|
7
|
+
* @format
|
|
8
|
+
* @lightSyntaxTransform
|
|
9
|
+
* @oncall memory_lab
|
|
10
|
+
*/
|
|
11
|
+
'use strict';
|
|
12
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
13
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
14
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
15
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
16
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
17
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
18
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
19
|
+
});
|
|
20
|
+
};
|
|
21
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
22
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
23
|
+
};
|
|
24
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
+
exports.anonymizeHeapSnapshot = anonymizeHeapSnapshot;
|
|
26
|
+
exports.anonymizeRawHeapSnapshot = anonymizeRawHeapSnapshot;
|
|
27
|
+
exports.readRawHeapSnapshot = readRawHeapSnapshot;
|
|
28
|
+
exports.anonymizeHeapSnapshotFile = anonymizeHeapSnapshotFile;
|
|
29
|
+
exports.auditHeapSnapshotFile = auditHeapSnapshotFile;
|
|
30
|
+
exports.resolveForComparison = resolveForComparison;
|
|
31
|
+
const crypto_1 = __importDefault(require("crypto"));
|
|
32
|
+
const fs_1 = __importDefault(require("fs"));
|
|
33
|
+
const path_1 = __importDefault(require("path"));
|
|
34
|
+
const StringLoader_1 = __importDefault(require("./StringLoader"));
|
|
35
|
+
const HeapSerializer_1 = require("./HeapSerializer");
|
|
36
|
+
const DEFAULT_MIN_DIGIT_RUN = 9;
|
|
37
|
+
/**
|
|
38
|
+
* Content rules applied to EVERY string table entry, label or not.
|
|
39
|
+
*
|
|
40
|
+
* Deliberately shape-based and application-agnostic: an anchored word list
|
|
41
|
+
* would be a list of the leaks someone already thought of. `\d{9,}` is
|
|
42
|
+
* unanchored on purpose — it is what catches an identifier embedded in a
|
|
43
|
+
* composite key, which is the common shape for a per-contact or per-session
|
|
44
|
+
* map key.
|
|
45
|
+
*
|
|
46
|
+
* URLs are NOT in this list. Script URLs are how `module attribution` and
|
|
47
|
+
* script census name the code that owns memory, so redacting them would remove
|
|
48
|
+
* a primary analysis while catching data that is, as a string VALUE, already
|
|
49
|
+
* redacted by the node-type rule. A URL carrying a token in its query string is
|
|
50
|
+
* a real residual; it is reported as such rather than silently handled.
|
|
51
|
+
*/
|
|
52
|
+
/**
|
|
53
|
+
* A caller's regex may carry `/g` or `/y`, whose `lastIndex` makes `test`
|
|
54
|
+
* alternate between true and false on identical input. Rebuilt without them so
|
|
55
|
+
* the same value always classifies the same way.
|
|
56
|
+
*/
|
|
57
|
+
function withoutStatefulFlags(pattern) {
|
|
58
|
+
return new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, ''));
|
|
59
|
+
}
|
|
60
|
+
function buildContentRules(options) {
|
|
61
|
+
var _a, _b, _c;
|
|
62
|
+
const rules = [];
|
|
63
|
+
if (options.redactDomText !== false) {
|
|
64
|
+
rules.push({
|
|
65
|
+
name: 'dom-text',
|
|
66
|
+
// Blink element descriptions and any serialized markup.
|
|
67
|
+
test: v => /^\s*<[a-zA-Z!/]/.test(v),
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
if (options.redactIdentifierKeys !== false) {
|
|
71
|
+
const minRun = (_a = options.minDigitRunLength) !== null && _a !== void 0 ? _a : DEFAULT_MIN_DIGIT_RUN;
|
|
72
|
+
// Interpolated straight into a quantifier, so an out-of-range value is not
|
|
73
|
+
// a mild misconfiguration: `0` yields `\d{0,}`, which matches EVERY string
|
|
74
|
+
// and would redact the entire label vocabulary -- silently, since the run
|
|
75
|
+
// would look successful. A negative or fractional value throws a
|
|
76
|
+
// SyntaxError from deep inside RegExp construction instead of naming the
|
|
77
|
+
// option that was wrong. Rejecting here fails loudly and early.
|
|
78
|
+
if (!Number.isInteger(minRun) || minRun < 1) {
|
|
79
|
+
throw new Error(`minDigitRunLength must be a positive integer, got: ${String(minRun)}. ` +
|
|
80
|
+
`A value below 1 matches every string and would redact the whole ` +
|
|
81
|
+
`label vocabulary.`);
|
|
82
|
+
}
|
|
83
|
+
const digitRun = new RegExp(`\\d{${minRun},}`);
|
|
84
|
+
rules.push({
|
|
85
|
+
// Named for the shape, not for email: `local@domain.tld` is also the
|
|
86
|
+
// shape of a federated handle, and on one real capture that is what
|
|
87
|
+
// most of the matches were. Calling the rule `email` would have
|
|
88
|
+
// under-reported what it actually removed.
|
|
89
|
+
name: 'email-or-handle',
|
|
90
|
+
test: v => /^[^\s@]+@[^\s@]+\.[A-Za-z]{2,}$/.test(v),
|
|
91
|
+
}, {
|
|
92
|
+
name: 'credential',
|
|
93
|
+
test: v => /^(Bearer\s|eyJ[A-Za-z0-9_-]{8,}\.)/.test(v),
|
|
94
|
+
}, { name: 'data-uri', test: v => /^data:[\w.+-]+\/[\w.+-]+[;,]/i.test(v) }, {
|
|
95
|
+
name: 'long-base64',
|
|
96
|
+
// Mixed case AND a digit are required, not just the base64 character
|
|
97
|
+
// set. Real base64 of real bytes has both; a long single-case run is
|
|
98
|
+
// far more likely to be a module path segment, a minified identifier —
|
|
99
|
+
// or this tool's own replacement text, which must not re-match on a
|
|
100
|
+
// second pass or in an audit.
|
|
101
|
+
test: v => v.length >= 40 &&
|
|
102
|
+
/^[A-Za-z0-9+/]+={0,2}$/.test(v) &&
|
|
103
|
+
/[A-Z]/.test(v) &&
|
|
104
|
+
/[a-z]/.test(v) &&
|
|
105
|
+
/[0-9]/.test(v),
|
|
106
|
+
}, { name: 'digit-run', test: v => digitRun.test(v) });
|
|
107
|
+
}
|
|
108
|
+
for (let i = 0; i < ((_c = (_b = options.extraPatterns) === null || _b === void 0 ? void 0 : _b.length) !== null && _c !== void 0 ? _c : 0); ++i) {
|
|
109
|
+
// Compiled ONCE, here, not inside `test`. `test` runs for every entry in
|
|
110
|
+
// the string table -- 732,455 of them on one real capture -- so compiling
|
|
111
|
+
// inside the closure would recompile the same pattern once per entry per
|
|
112
|
+
// caller pattern.
|
|
113
|
+
const compiled = withoutStatefulFlags(options.extraPatterns[i]);
|
|
114
|
+
rules.push({
|
|
115
|
+
name: `extra-pattern-${i + 1}`,
|
|
116
|
+
test: v => compiled.test(v),
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
return rules;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Length-preserving replacement text.
|
|
123
|
+
*
|
|
124
|
+
* Length is preserved in UTF-16 code units so `self_size` keeps agreeing with
|
|
125
|
+
* the string it describes — V8 sizes a one-byte string at `align4(12 + length)`,
|
|
126
|
+
* and a reader that checks will otherwise see a corrupt capture. The output is
|
|
127
|
+
* ASCII, so it can never be longer in bytes than what it replaces.
|
|
128
|
+
*/
|
|
129
|
+
function redactedText(value, mode, salt) {
|
|
130
|
+
const len = value.length;
|
|
131
|
+
if (len === 0) {
|
|
132
|
+
return value;
|
|
133
|
+
}
|
|
134
|
+
if (mode === 'uniform') {
|
|
135
|
+
return '?'.repeat(len);
|
|
136
|
+
}
|
|
137
|
+
// Lowercase letters only, NOT hex. Replacement text is scanned by the very
|
|
138
|
+
// content rules that decide what is sensitive, so digit-bearing fill makes
|
|
139
|
+
// the tool's own output look like the thing it removes: with hex fill a real
|
|
140
|
+
// capture still reported 17,837 entries carrying a 9-digit run AFTER
|
|
141
|
+
// anonymization, essentially all of them replacements rather than leaks. A
|
|
142
|
+
// digit-free alphabet is what keeps a reported residual meaningful.
|
|
143
|
+
// Filled into a preallocated array and joined once, rather than accumulated
|
|
144
|
+
// with `+=`. The values being replaced include serialized DOM node names that
|
|
145
|
+
// reach tens of kilobytes, so this loop runs per character of the longest
|
|
146
|
+
// string in the capture; it also stops exactly at `len` instead of
|
|
147
|
+
// overshooting by up to a digest and slicing the remainder away.
|
|
148
|
+
const chars = new Array(len);
|
|
149
|
+
let filled = 0;
|
|
150
|
+
let counter = 0;
|
|
151
|
+
while (filled < len) {
|
|
152
|
+
const digest = crypto_1.default
|
|
153
|
+
.createHash('sha256')
|
|
154
|
+
.update(salt)
|
|
155
|
+
.update(' ')
|
|
156
|
+
.update(value)
|
|
157
|
+
.update(` ${counter++}`)
|
|
158
|
+
.digest();
|
|
159
|
+
for (let i = 0; i < digest.length && filled < len; ++i) {
|
|
160
|
+
chars[filled++] =
|
|
161
|
+
REDACTION_ALPHABET[digest[i] % REDACTION_ALPHABET.length];
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return chars.join('');
|
|
165
|
+
}
|
|
166
|
+
const REDACTION_ALPHABET = 'abcdefghijklmnopqrstuvwxyz';
|
|
167
|
+
/**
|
|
168
|
+
* Record width implied by the DATA, not by the declared field list.
|
|
169
|
+
*
|
|
170
|
+
* `HeapSnapshot._buildMetaData` appends `'invisible'` to `meta.edge_fields` in
|
|
171
|
+
* place while parsing, so a parsed snapshot claims one more edge field than its
|
|
172
|
+
* edge array actually carries. Striding by the declared count reads every edge
|
|
173
|
+
* misaligned — and silently, since the numbers still look like numbers.
|
|
174
|
+
*/
|
|
175
|
+
function fittedFieldCount(values, recordCount, declared) {
|
|
176
|
+
if (recordCount <= 0) {
|
|
177
|
+
return declared;
|
|
178
|
+
}
|
|
179
|
+
const width = values.length / recordCount;
|
|
180
|
+
return Number.isInteger(width) && width > 0 ? width : declared;
|
|
181
|
+
}
|
|
182
|
+
/** Rule name reported for entries the caller's own callback selected. */
|
|
183
|
+
const CUSTOM_RULE = 'custom-callback';
|
|
184
|
+
/** Kept distinct from the positional built-in rule indices. */
|
|
185
|
+
const CUSTOM_RULE_ID = 1 << 30;
|
|
186
|
+
const MAX_REPORTED_FAMILIES = 12;
|
|
187
|
+
const MAX_FAMILY_EXAMPLES = 3;
|
|
188
|
+
/**
|
|
189
|
+
* Coarse shape of a string: character classes, with runs collapsed.
|
|
190
|
+
*
|
|
191
|
+
* `7ac91e02-11bd-4c7f` and `0f31ba77-92cd-4e10` both reduce to `dada-dada-dada`
|
|
192
|
+
* -ish, while `enableFastPath` reduces to `aAaAa`. Separators survive
|
|
193
|
+
* literally, because they are what makes a generated format recognizable.
|
|
194
|
+
*/
|
|
195
|
+
function shapeSignature(value) {
|
|
196
|
+
let out = '';
|
|
197
|
+
let last = '';
|
|
198
|
+
const limit = Math.min(value.length, 48);
|
|
199
|
+
for (let i = 0; i < limit; ++i) {
|
|
200
|
+
const c = value[i];
|
|
201
|
+
let cls;
|
|
202
|
+
if (c >= '0' && c <= '9') {
|
|
203
|
+
cls = 'd';
|
|
204
|
+
}
|
|
205
|
+
else if (c >= 'a' && c <= 'z') {
|
|
206
|
+
cls = 'a';
|
|
207
|
+
}
|
|
208
|
+
else if (c >= 'A' && c <= 'Z') {
|
|
209
|
+
cls = 'A';
|
|
210
|
+
}
|
|
211
|
+
else {
|
|
212
|
+
cls = c;
|
|
213
|
+
}
|
|
214
|
+
if (cls !== last) {
|
|
215
|
+
out += cls;
|
|
216
|
+
last = cls;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
return out;
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Group the labels still in the clear into machine-generated-looking families.
|
|
223
|
+
*
|
|
224
|
+
* Two exclusions, both engine-level facts rather than judgements about any
|
|
225
|
+
* application:
|
|
226
|
+
*
|
|
227
|
+
* - **Pure digit strings.** V8 emits an exhaustive run of numeric index names
|
|
228
|
+
* (`"0"`..`"9"`, `"10"`..`"99"`, and so on); they are array indices by
|
|
229
|
+
* construction, and on one capture they were 133,226 of the labels, which
|
|
230
|
+
* would bury everything else.
|
|
231
|
+
* - **Retainer descriptors.** Strings of the form `N / part of key (...) ->
|
|
232
|
+
* value (...)` are written by the snapshot serializer to describe WeakMap
|
|
233
|
+
* entries. They are diagnostics, never application data.
|
|
234
|
+
*
|
|
235
|
+
* What remains is filtered to shapes that carry a digit or a separator. A shape
|
|
236
|
+
* built only from letters, `_` and `$` is the shape of source-code vocabulary
|
|
237
|
+
* in every language, so surfacing those would make the report mostly noise —
|
|
238
|
+
* and unlike a redaction rule, being wrong here only costs a line of output.
|
|
239
|
+
*/
|
|
240
|
+
function collectUnclassifiedLabelFamilies(strings, isLabel, contentRule, limit) {
|
|
241
|
+
const families = new Map();
|
|
242
|
+
const vocabularyShape = /^[aA_$]+$/;
|
|
243
|
+
for (let i = 0; i < isLabel.length; ++i) {
|
|
244
|
+
if (!isLabel[i] || contentRule[i] >= 0) {
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
const value = strings[i];
|
|
248
|
+
if (value.length === 0 || /^\d+$/.test(value) || /^\d+ \/ /.test(value)) {
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
const shape = shapeSignature(value);
|
|
252
|
+
if (vocabularyShape.test(shape)) {
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
let entry = families.get(shape);
|
|
256
|
+
if (entry == null) {
|
|
257
|
+
entry = { count: 0, examples: [] };
|
|
258
|
+
families.set(shape, entry);
|
|
259
|
+
}
|
|
260
|
+
entry.count++;
|
|
261
|
+
if (entry.examples.length < MAX_FAMILY_EXAMPLES) {
|
|
262
|
+
entry.examples.push(value);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return [...families]
|
|
266
|
+
.map(([shape, { count, examples }]) => ({ shape, count, examples }))
|
|
267
|
+
.sort((a, b) => b.count - a.count)
|
|
268
|
+
.slice(0, limit);
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Read the node layout out of the snapshot's own meta rather than assuming it.
|
|
272
|
+
*
|
|
273
|
+
* Field order and the node type list are both snapshot-declared and differ
|
|
274
|
+
* between engines and V8 versions (a browser capture has 6 node fields, a
|
|
275
|
+
* Node.js one has 7).
|
|
276
|
+
*/
|
|
277
|
+
function readLayout(raw) {
|
|
278
|
+
const meta = raw.snapshot.meta;
|
|
279
|
+
const nodeFields = meta.node_fields;
|
|
280
|
+
const nodeTypeOffset = nodeFields.indexOf('type');
|
|
281
|
+
const nodeTypes = meta.node_types[nodeTypeOffset];
|
|
282
|
+
const stringNodeTypes = new Set();
|
|
283
|
+
for (const name of ['string', 'concatenated string', 'sliced string']) {
|
|
284
|
+
const idx = nodeTypes.indexOf(name);
|
|
285
|
+
if (idx >= 0) {
|
|
286
|
+
stringNodeTypes.add(idx);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
const edgeFields = meta.edge_fields;
|
|
290
|
+
const edgeTypeOffset = edgeFields.indexOf('type');
|
|
291
|
+
const edgeTypes = meta.edge_types[edgeTypeOffset];
|
|
292
|
+
// For `element` and `hidden` edges `name_or_index` is an integer index; for
|
|
293
|
+
// every other type it is an index into the string table.
|
|
294
|
+
const indexNamedEdgeTypes = new Set();
|
|
295
|
+
for (const name of ['element', 'hidden']) {
|
|
296
|
+
const idx = edgeTypes.indexOf(name);
|
|
297
|
+
if (idx >= 0) {
|
|
298
|
+
indexNamedEdgeTypes.add(idx);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return {
|
|
302
|
+
nodeFieldsCount: fittedFieldCount(raw.nodes, raw.snapshot.node_count, nodeFields.length),
|
|
303
|
+
nodeTypeOffset,
|
|
304
|
+
nodeNameOffset: nodeFields.indexOf('name'),
|
|
305
|
+
stringNodeTypes,
|
|
306
|
+
edgeFieldsCount: fittedFieldCount(raw.edges, raw.snapshot.edge_count, edgeFields.length),
|
|
307
|
+
edgeTypeOffset,
|
|
308
|
+
edgeNameOffset: edgeFields.indexOf('name_or_index'),
|
|
309
|
+
indexNamedEdgeTypes,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Rewrite a parsed heap snapshot in place so it no longer carries user data.
|
|
314
|
+
*
|
|
315
|
+
* Redacts the content of every string on the heap, plus any string table entry
|
|
316
|
+
* whose text looks like an identifier, a credential or serialized DOM. Class
|
|
317
|
+
* names, function names and ordinary property keys are left alone, so retainer
|
|
318
|
+
* traces, class histograms, dominator trees and shape analyses all still work
|
|
319
|
+
* on the result.
|
|
320
|
+
*
|
|
321
|
+
* The snapshot is modified in place and its parsed view updates with it — node
|
|
322
|
+
* names are read from the string table on each access rather than cached.
|
|
323
|
+
* Persist the result with {@link serializeHeapSnapshot}.
|
|
324
|
+
*
|
|
325
|
+
* This does not defeat a determined attacker who already knows what they are
|
|
326
|
+
* looking for: lengths are preserved exactly (they have to be, or `self_size`
|
|
327
|
+
* stops matching), and in `stable` mode equal values stay equal. It removes the
|
|
328
|
+
* content, not the shape of the content.
|
|
329
|
+
*
|
|
330
|
+
* @param snapshot the parsed heap snapshot to rewrite in place
|
|
331
|
+
* @param options see {@link AnonymizeOptions}
|
|
332
|
+
* @returns a summary of what was redacted, per rule; see
|
|
333
|
+
* {@link AnonymizeReport}
|
|
334
|
+
*
|
|
335
|
+
* * **Examples**:
|
|
336
|
+
* ```typescript
|
|
337
|
+
* import type {IHeapSnapshot} from '@memlab/core';
|
|
338
|
+
* import {
|
|
339
|
+
* dumpNodeHeapSnapshot,
|
|
340
|
+
* anonymizeHeapSnapshot,
|
|
341
|
+
* serializeHeapSnapshot,
|
|
342
|
+
* } from '@memlab/core';
|
|
343
|
+
* import {getFullHeapFromFile} from '@memlab/heap-analysis';
|
|
344
|
+
*
|
|
345
|
+
* (async function () {
|
|
346
|
+
* const file = dumpNodeHeapSnapshot();
|
|
347
|
+
* const heap: IHeapSnapshot = await getFullHeapFromFile(file);
|
|
348
|
+
*
|
|
349
|
+
* const report = anonymizeHeapSnapshot(heap);
|
|
350
|
+
* console.log(`redacted ${report.valuesRedacted} string values`);
|
|
351
|
+
*
|
|
352
|
+
* serializeHeapSnapshot(heap, '/tmp/shareable.heapsnapshot');
|
|
353
|
+
* })();
|
|
354
|
+
* ```
|
|
355
|
+
*/
|
|
356
|
+
function anonymizeHeapSnapshot(snapshot, options = {}) {
|
|
357
|
+
return anonymizeRawHeapSnapshot(snapshot.snapshot, options);
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* The in-place rewrite, against the raw snapshot data.
|
|
361
|
+
*
|
|
362
|
+
* @param raw the raw snapshot data to rewrite
|
|
363
|
+
* @param options see {@link AnonymizeOptions}
|
|
364
|
+
* @returns a summary of what was redacted; see {@link AnonymizeReport}
|
|
365
|
+
*
|
|
366
|
+
* @internal
|
|
367
|
+
*/
|
|
368
|
+
function anonymizeRawHeapSnapshot(raw, options = {}) {
|
|
369
|
+
var _a, _b, _c, _d, _e;
|
|
370
|
+
const mode = (_a = options.mode) !== null && _a !== void 0 ? _a : 'stable';
|
|
371
|
+
const salt = (_b = options.salt) !== null && _b !== void 0 ? _b : '';
|
|
372
|
+
const strings = raw.strings;
|
|
373
|
+
const layout = readLayout(raw);
|
|
374
|
+
const rules = buildContentRules(options);
|
|
375
|
+
// Same reason as the extra patterns: this is consulted once per string table
|
|
376
|
+
// entry, so the sanitized regexes are built here rather than in the loop.
|
|
377
|
+
const keep = ((_c = options.keepPatterns) !== null && _c !== void 0 ? _c : []).map(withoutStatefulFlags);
|
|
378
|
+
const { nodeFieldsCount, nodeTypeOffset, nodeNameOffset, stringNodeTypes, edgeFieldsCount, edgeTypeOffset, edgeNameOffset, indexNamedEdgeTypes, } = layout;
|
|
379
|
+
// Classify every string table entry by HOW IT IS REACHED. The table is
|
|
380
|
+
// deduplicated, so one entry can be reached both ways at once, and the two
|
|
381
|
+
// roles need opposite treatment.
|
|
382
|
+
const nodes = raw.nodes;
|
|
383
|
+
const isStringValue = new Uint8Array(strings.length);
|
|
384
|
+
const isLabel = new Uint8Array(strings.length);
|
|
385
|
+
for (let i = 0; i < nodes.length; i += nodeFieldsCount) {
|
|
386
|
+
const nameIdx = nodes[i + nodeNameOffset];
|
|
387
|
+
if (stringNodeTypes.has(nodes[i + nodeTypeOffset])) {
|
|
388
|
+
// the CONTENT of a JS string on the heap: data
|
|
389
|
+
isStringValue[nameIdx] = 1;
|
|
390
|
+
}
|
|
391
|
+
else {
|
|
392
|
+
// a class name, function name, or engine-internal name: a label
|
|
393
|
+
isLabel[nameIdx] = 1;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
const edges = raw.edges;
|
|
397
|
+
const nameUseCount = new Uint32Array(strings.length);
|
|
398
|
+
for (let i = 0; i < edges.length; i += edgeFieldsCount) {
|
|
399
|
+
if (!indexNamedEdgeTypes.has(edges[i + edgeTypeOffset])) {
|
|
400
|
+
// a property key, closure variable, or context slot name: a label
|
|
401
|
+
const nameIdx = edges[i + edgeNameOffset];
|
|
402
|
+
isLabel[nameIdx] = 1;
|
|
403
|
+
if (nameUseCount[nameIdx] < 0xffffffff) {
|
|
404
|
+
nameUseCount[nameIdx]++;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
const ruleCounts = new Map();
|
|
409
|
+
// -1 = not content-sensitive; CUSTOM_RULE_ID for the caller's callback;
|
|
410
|
+
// otherwise the index of the matching built-in rule.
|
|
411
|
+
const contentRule = new Int32Array(strings.length).fill(-1);
|
|
412
|
+
// Entries the caller explicitly protected. Tracked separately from
|
|
413
|
+
// `contentRule` because protection has to reach the string-VALUE pass below,
|
|
414
|
+
// which does not consult the content rules at all: `keepPatterns` promises
|
|
415
|
+
// an entry is NEVER redacted, and `shouldRedact` returning false promises the
|
|
416
|
+
// same, so honouring either only for content rules would break both
|
|
417
|
+
// contracts for exactly the entries a caller cared enough to name.
|
|
418
|
+
const protectedEntry = new Uint8Array(strings.length);
|
|
419
|
+
const shouldRedact = options.shouldRedact;
|
|
420
|
+
let contentRedacted = 0;
|
|
421
|
+
for (let i = 0; i < strings.length; ++i) {
|
|
422
|
+
const value = strings[i];
|
|
423
|
+
if (value.length === 0) {
|
|
424
|
+
continue;
|
|
425
|
+
}
|
|
426
|
+
if (shouldRedact != null) {
|
|
427
|
+
// Asked first, and its answer is final either way. The caller knows their
|
|
428
|
+
// own application; the built-in rules only know internet formats.
|
|
429
|
+
const verdict = shouldRedact(value, {
|
|
430
|
+
isValue: isStringValue[i] === 1,
|
|
431
|
+
isLabel: isLabel[i] === 1,
|
|
432
|
+
labelUseCount: nameUseCount[i],
|
|
433
|
+
shape: shapeSignature(value),
|
|
434
|
+
});
|
|
435
|
+
if (verdict === true) {
|
|
436
|
+
contentRule[i] = CUSTOM_RULE_ID;
|
|
437
|
+
contentRedacted++;
|
|
438
|
+
ruleCounts.set(CUSTOM_RULE, ((_d = ruleCounts.get(CUSTOM_RULE)) !== null && _d !== void 0 ? _d : 0) + 1);
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
if (verdict === false) {
|
|
442
|
+
protectedEntry[i] = 1;
|
|
443
|
+
continue;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
if (keep.some(p => p.test(value))) {
|
|
447
|
+
protectedEntry[i] = 1;
|
|
448
|
+
continue;
|
|
449
|
+
}
|
|
450
|
+
for (let r = 0; r < rules.length; ++r) {
|
|
451
|
+
if (rules[r].test(value)) {
|
|
452
|
+
contentRule[i] = r;
|
|
453
|
+
contentRedacted++;
|
|
454
|
+
ruleCounts.set(rules[r].name, ((_e = ruleCounts.get(rules[r].name)) !== null && _e !== void 0 ? _e : 0) + 1);
|
|
455
|
+
break;
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
// Content matches are redacted where they stand, because the value is
|
|
460
|
+
// sensitive wherever it appears — including as a property name.
|
|
461
|
+
for (let i = 0; i < strings.length; ++i) {
|
|
462
|
+
if (contentRule[i] >= 0) {
|
|
463
|
+
strings[i] = redactedText(strings[i], mode, salt);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
// Now the string values. Which of two treatments applies turns on whether
|
|
467
|
+
// the same entry is ALSO a label:
|
|
468
|
+
//
|
|
469
|
+
// value only -> redact the entry in place. Splitting here would be a leak,
|
|
470
|
+
// not a safeguard: repointing the node at a redacted twin
|
|
471
|
+
// leaves the original entry sitting in the table with the
|
|
472
|
+
// plaintext still in it. Nothing references it, but the
|
|
473
|
+
// bytes are in the file, which is all an attacker needs.
|
|
474
|
+
// value+label -> append a redacted twin and repoint only the string node,
|
|
475
|
+
// so the label keeps its text. Overwriting in place would
|
|
476
|
+
// destroy a property or class name that merely happens to
|
|
477
|
+
// share the deduplicated entry.
|
|
478
|
+
const twinOf = new Int32Array(strings.length).fill(-1);
|
|
479
|
+
let valuesRedacted = 0;
|
|
480
|
+
let entriesSplit = 0;
|
|
481
|
+
const originalTableSize = strings.length;
|
|
482
|
+
for (let i = 0; i < originalTableSize; ++i) {
|
|
483
|
+
if (!isStringValue[i] ||
|
|
484
|
+
isLabel[i] ||
|
|
485
|
+
contentRule[i] >= 0 ||
|
|
486
|
+
protectedEntry[i]) {
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
if (strings[i].length === 0) {
|
|
490
|
+
continue;
|
|
491
|
+
}
|
|
492
|
+
strings[i] = redactedText(strings[i], mode, salt);
|
|
493
|
+
valuesRedacted++;
|
|
494
|
+
}
|
|
495
|
+
for (let i = 0; i < nodes.length; i += nodeFieldsCount) {
|
|
496
|
+
if (!stringNodeTypes.has(nodes[i + nodeTypeOffset])) {
|
|
497
|
+
continue;
|
|
498
|
+
}
|
|
499
|
+
const nameIdx = nodes[i + nodeNameOffset];
|
|
500
|
+
if (!isLabel[nameIdx] ||
|
|
501
|
+
contentRule[nameIdx] >= 0 ||
|
|
502
|
+
protectedEntry[nameIdx] ||
|
|
503
|
+
strings[nameIdx].length === 0) {
|
|
504
|
+
// redacted in place above, redacted by content, or nothing to redact
|
|
505
|
+
continue;
|
|
506
|
+
}
|
|
507
|
+
if (twinOf[nameIdx] < 0) {
|
|
508
|
+
twinOf[nameIdx] = strings.length;
|
|
509
|
+
strings.push(redactedText(strings[nameIdx], mode, salt));
|
|
510
|
+
valuesRedacted++;
|
|
511
|
+
entriesSplit++;
|
|
512
|
+
}
|
|
513
|
+
nodes[i + nodeNameOffset] = twinOf[nameIdx];
|
|
514
|
+
}
|
|
515
|
+
const contentRedactedByRule = [...ruleCounts]
|
|
516
|
+
.map(([rule, count]) => ({ rule, count }))
|
|
517
|
+
.sort((a, b) => b.count - a.count);
|
|
518
|
+
return {
|
|
519
|
+
mode,
|
|
520
|
+
salted: salt.length > 0,
|
|
521
|
+
stringTableSize: originalTableSize,
|
|
522
|
+
valuesRedacted,
|
|
523
|
+
entriesSplit,
|
|
524
|
+
contentRedacted,
|
|
525
|
+
contentRedactedByRule,
|
|
526
|
+
unclassifiedLabelFamilies: collectUnclassifiedLabelFamilies(strings, isLabel, contentRule, MAX_REPORTED_FAMILIES),
|
|
527
|
+
};
|
|
528
|
+
}
|
|
529
|
+
/**
|
|
530
|
+
* Read a snapshot file into its raw arrays, WITHOUT building the object graph.
|
|
531
|
+
*
|
|
532
|
+
* `HeapParser.parse` additionally computes referrers, dominators, detachedness
|
|
533
|
+
* and node indices — on a 7.1M-node capture that is the part that costs ~15
|
|
534
|
+
* seconds and several gigabytes. Anonymization needs none of it: it works off
|
|
535
|
+
* node types, edge names and the string table. Reading the arrays and stopping
|
|
536
|
+
* there is the difference between "shareable in a moment" and "load the whole
|
|
537
|
+
* heap first".
|
|
538
|
+
*
|
|
539
|
+
* @param file absolute path of the `.heapsnapshot` file to read
|
|
540
|
+
* @returns the snapshot's raw arrays
|
|
541
|
+
*
|
|
542
|
+
* @internal
|
|
543
|
+
*/
|
|
544
|
+
function readRawHeapSnapshot(file) {
|
|
545
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
546
|
+
const [nodes, edges, locations, content] = yield Promise.all([
|
|
547
|
+
StringLoader_1.default.readFileAndExtractTypedArray(file, 'nodes'),
|
|
548
|
+
StringLoader_1.default.readFileAndExtractTypedArray(file, 'edges'),
|
|
549
|
+
StringLoader_1.default.readFileAndExtractTypedArray(file, 'locations'),
|
|
550
|
+
StringLoader_1.default.readFileAndExcludeTypedArray(file, [
|
|
551
|
+
'nodes',
|
|
552
|
+
'edges',
|
|
553
|
+
'locations',
|
|
554
|
+
]),
|
|
555
|
+
]);
|
|
556
|
+
const raw = JSON.parse(content);
|
|
557
|
+
raw.nodes = nodes;
|
|
558
|
+
raw.edges = edges;
|
|
559
|
+
raw.locations = locations;
|
|
560
|
+
return raw;
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* Anonymize a `.heapsnapshot` file and write the result to another file.
|
|
565
|
+
*
|
|
566
|
+
* The file-to-file form of {@link anonymizeHeapSnapshot}, and the one to reach
|
|
567
|
+
* for when the capture is only being shared rather than analyzed here: it skips
|
|
568
|
+
* building the object graph on the way in, and streams on the way out, so
|
|
569
|
+
* neither the input nor the output is ever held as one string.
|
|
570
|
+
*
|
|
571
|
+
* @param inputFile absolute path of the capture to read
|
|
572
|
+
* @param outputFile absolute path to write the anonymized capture to; an
|
|
573
|
+
* existing file at that path is overwritten
|
|
574
|
+
* @param options see {@link AnonymizeOptions}
|
|
575
|
+
* @returns a summary of what was redacted, and what was left in the clear; see
|
|
576
|
+
* {@link AnonymizeReport}
|
|
577
|
+
*
|
|
578
|
+
* * **Examples**:
|
|
579
|
+
* ```typescript
|
|
580
|
+
* import {anonymizeHeapSnapshotFile} from '@memlab/core';
|
|
581
|
+
*
|
|
582
|
+
* (async function () {
|
|
583
|
+
* const report = await anonymizeHeapSnapshotFile(
|
|
584
|
+
* '/tmp/capture.heapsnapshot',
|
|
585
|
+
* '/tmp/shareable.heapsnapshot',
|
|
586
|
+
* );
|
|
587
|
+
* console.log(`redacted ${report.valuesRedacted} string values`);
|
|
588
|
+
* for (const family of report.unclassifiedLabelFamilies) {
|
|
589
|
+
* console.log(`still in the clear: ${family.count} x ${family.shape}`);
|
|
590
|
+
* }
|
|
591
|
+
* })();
|
|
592
|
+
* ```
|
|
593
|
+
*/
|
|
594
|
+
function anonymizeHeapSnapshotFile(inputFile_1, outputFile_1) {
|
|
595
|
+
return __awaiter(this, arguments, void 0, function* (inputFile, outputFile, options = {}) {
|
|
596
|
+
if (resolveForComparison(inputFile) === resolveForComparison(outputFile)) {
|
|
597
|
+
throw new Error(`anonymizeHeapSnapshotFile: outputFile resolves to the same file as ` +
|
|
598
|
+
`inputFile (${inputFile}). Writing there would overwrite the only ` +
|
|
599
|
+
`unredacted copy of the capture.`);
|
|
600
|
+
}
|
|
601
|
+
const raw = yield readRawHeapSnapshot(inputFile);
|
|
602
|
+
const report = anonymizeRawHeapSnapshot(raw, options);
|
|
603
|
+
(0, HeapSerializer_1.serializeRawHeapSnapshot)(raw, outputFile);
|
|
604
|
+
return report;
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* Report what anonymizing a capture WOULD remove, and what it would leave,
|
|
609
|
+
* without writing anything.
|
|
610
|
+
*
|
|
611
|
+
* Point it at a capture someone already anonymized to find out whether they
|
|
612
|
+
* missed something: `unclassifiedLabelFamilies` names the identifier-shaped
|
|
613
|
+
* text still in the clear. Run against one already-anonymized capture, this is
|
|
614
|
+
* what showed its author had removed every string VALUE and left 26,303 account
|
|
615
|
+
* handles behind as property names.
|
|
616
|
+
*
|
|
617
|
+
* @param inputFile absolute path of the capture to inspect
|
|
618
|
+
* @param options see {@link AnonymizeOptions}
|
|
619
|
+
* @returns the same summary {@link anonymizeHeapSnapshotFile} returns, for a
|
|
620
|
+
* run that was not written to disk
|
|
621
|
+
*
|
|
622
|
+
* * **Examples**:
|
|
623
|
+
* ```typescript
|
|
624
|
+
* import {auditHeapSnapshotFile} from '@memlab/core';
|
|
625
|
+
*
|
|
626
|
+
* (async function () {
|
|
627
|
+
* const report = await auditHeapSnapshotFile('/tmp/shared.heapsnapshot');
|
|
628
|
+
* console.log(report.unclassifiedLabelFamilies);
|
|
629
|
+
* })();
|
|
630
|
+
* ```
|
|
631
|
+
*/
|
|
632
|
+
function auditHeapSnapshotFile(inputFile_1) {
|
|
633
|
+
return __awaiter(this, arguments, void 0, function* (inputFile, options = {}) {
|
|
634
|
+
const raw = yield readRawHeapSnapshot(inputFile);
|
|
635
|
+
return anonymizeRawHeapSnapshot(raw, options);
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
/**
|
|
639
|
+
* Canonical form of a path, for deciding whether two of them are the same file.
|
|
640
|
+
*
|
|
641
|
+
* A string comparison is not enough, and the cost of it being wrong here is the
|
|
642
|
+
* original capture: `a.heapsnapshot` and `./a.heapsnapshot`, an absolute and a
|
|
643
|
+
* relative spelling, or a symlink and its target are all distinct strings
|
|
644
|
+
* naming one file. `realpathSync` resolves all three.
|
|
645
|
+
*
|
|
646
|
+
* The output file usually does not exist yet, which makes `realpathSync` throw
|
|
647
|
+
* on it — so its DIRECTORY is canonicalized instead and the basename appended.
|
|
648
|
+
* That still catches a symlinked parent, which a plain `path.resolve` would
|
|
649
|
+
* miss. Falls back to `path.resolve` when even the directory is absent, since
|
|
650
|
+
* at that point the two cannot be the same existing file anyway.
|
|
651
|
+
*
|
|
652
|
+
* @param file the path to canonicalize
|
|
653
|
+
* @returns a path safe to compare against another canonicalized path
|
|
654
|
+
*
|
|
655
|
+
* @internal
|
|
656
|
+
*/
|
|
657
|
+
function resolveForComparison(file) {
|
|
658
|
+
const resolved = path_1.default.resolve(file);
|
|
659
|
+
try {
|
|
660
|
+
return fs_1.default.realpathSync(resolved);
|
|
661
|
+
}
|
|
662
|
+
catch (_a) {
|
|
663
|
+
try {
|
|
664
|
+
return path_1.default.join(fs_1.default.realpathSync(path_1.default.dirname(resolved)), path_1.default.basename(resolved));
|
|
665
|
+
}
|
|
666
|
+
catch (_b) {
|
|
667
|
+
return resolved;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
}
|