@milaboratories/pl-flight-recorder 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -0
- package/dist/analyze.d.ts +133 -0
- package/dist/analyze.d.ts.map +1 -0
- package/dist/analyze.js +525 -0
- package/dist/analyze.js.map +1 -0
- package/dist/data_summary.d.ts +28 -0
- package/dist/data_summary.d.ts.map +1 -0
- package/dist/data_summary.js +73 -0
- package/dist/data_summary.js.map +1 -0
- package/dist/digest.d.ts +28 -0
- package/dist/digest.d.ts.map +1 -0
- package/dist/digest.js +55 -0
- package/dist/digest.js.map +1 -0
- package/dist/events.d.ts +89 -0
- package/dist/events.d.ts.map +1 -0
- package/dist/events.js +12 -0
- package/dist/events.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +13 -0
- package/dist/instrument.d.ts +82 -0
- package/dist/instrument.d.ts.map +1 -0
- package/dist/instrument.js +313 -0
- package/dist/instrument.js.map +1 -0
- package/dist/recorder.d.ts +82 -0
- package/dist/recorder.d.ts.map +1 -0
- package/dist/recorder.js +293 -0
- package/dist/recorder.js.map +1 -0
- package/dist/redact.d.ts +53 -0
- package/dist/redact.d.ts.map +1 -0
- package/dist/redact.js +145 -0
- package/dist/redact.js.map +1 -0
- package/dist/report.d.ts +6 -0
- package/dist/report.d.ts.map +1 -0
- package/dist/report.js +377 -0
- package/dist/report.js.map +1 -0
- package/dist/rules.d.ts +73 -0
- package/dist/rules.d.ts.map +1 -0
- package/dist/rules.js +245 -0
- package/dist/rules.js.map +1 -0
- package/dist/sampler.d.ts +22 -0
- package/dist/sampler.d.ts.map +1 -0
- package/dist/sampler.js +33 -0
- package/dist/sampler.js.map +1 -0
- package/dist/sampler_thread.d.ts +1 -0
- package/dist/sampler_thread.js +41 -0
- package/dist/sampler_thread.js.map +1 -0
- package/dist/session.d.ts +40 -0
- package/dist/session.d.ts.map +1 -0
- package/dist/session.js +50 -0
- package/dist/session.js.map +1 -0
- package/dist/supervisor.d.ts +58 -0
- package/dist/supervisor.d.ts.map +1 -0
- package/dist/supervisor.js +108 -0
- package/dist/supervisor.js.map +1 -0
- package/package.json +43 -0
- package/src/analyze.test.ts +539 -0
- package/src/analyze.ts +795 -0
- package/src/data_summary.ts +110 -0
- package/src/digest.ts +49 -0
- package/src/events.ts +102 -0
- package/src/index.ts +104 -0
- package/src/instrument.ts +442 -0
- package/src/recorder.ts +397 -0
- package/src/redact.test.ts +155 -0
- package/src/redact.ts +213 -0
- package/src/report.ts +512 -0
- package/src/rules.test.ts +182 -0
- package/src/rules.ts +383 -0
- package/src/sampler.ts +40 -0
- package/src/sampler_thread.ts +45 -0
- package/src/session.ts +69 -0
- package/src/supervisor.ts +150 -0
package/dist/recorder.js
ADDED
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { FLIGHT_FILE_PREFIX, MEM_BASELINE_RECORD, SESSION_END_RECORD, SESSION_RECORD } from "./events.js";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import v8 from "node:v8";
|
|
6
|
+
//#region src/recorder.ts
|
|
7
|
+
/**
|
|
8
|
+
* Opens a flight log for this process and writes the session header.
|
|
9
|
+
*
|
|
10
|
+
* Records are appended with a synchronous write rather than through a stream:
|
|
11
|
+
* the process being recorded dies without warning — V8 fatal out-of-memory, a
|
|
12
|
+
* failed native allocation, the OS out-of-memory killer — so no exit hook, no
|
|
13
|
+
* flush and no `finally` block runs. Anything still sitting in a userspace
|
|
14
|
+
* buffer is exactly the part that would have explained the death.
|
|
15
|
+
*/
|
|
16
|
+
function openRecorder(options) {
|
|
17
|
+
const { dir, role = "middle-layer", meta = {}, maxFileBytes = 32 * 1024 * 1024, sessionId = newSessionId() } = options;
|
|
18
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
19
|
+
const file = path.join(dir, `${FLIGHT_FILE_PREFIX}-${sessionId}.ndjson`);
|
|
20
|
+
const state = {
|
|
21
|
+
fd: fs.openSync(file, "a"),
|
|
22
|
+
bytes: 0,
|
|
23
|
+
preambleBytes: 0,
|
|
24
|
+
seq: 0,
|
|
25
|
+
closed: false,
|
|
26
|
+
header: void 0,
|
|
27
|
+
baselineMem: void 0,
|
|
28
|
+
openBegins: /* @__PURE__ */ new Map()
|
|
29
|
+
};
|
|
30
|
+
const memorySnapshot = () => {
|
|
31
|
+
const usage = process.memoryUsage();
|
|
32
|
+
return {
|
|
33
|
+
rss: usage.rss,
|
|
34
|
+
heapUsed: usage.heapUsed,
|
|
35
|
+
heapTotal: usage.heapTotal,
|
|
36
|
+
external: usage.external,
|
|
37
|
+
arrayBuffers: usage.arrayBuffers,
|
|
38
|
+
heapLimit: v8.getHeapStatistics().heap_size_limit
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
const event = (type, payload = {}) => {
|
|
42
|
+
if (state.closed) return -1;
|
|
43
|
+
const seq = ++state.seq;
|
|
44
|
+
const record = {
|
|
45
|
+
seq,
|
|
46
|
+
t: monotonic(),
|
|
47
|
+
wall: Date.now(),
|
|
48
|
+
type,
|
|
49
|
+
...payload
|
|
50
|
+
};
|
|
51
|
+
if (state.baselineMem === void 0 && record.mem) state.baselineMem = record.mem;
|
|
52
|
+
trackOpenOperation(state, record);
|
|
53
|
+
writeLine(state, file, maxFileBytes, record);
|
|
54
|
+
return seq;
|
|
55
|
+
};
|
|
56
|
+
const recorder = {
|
|
57
|
+
sessionId,
|
|
58
|
+
file,
|
|
59
|
+
event,
|
|
60
|
+
memorySnapshot,
|
|
61
|
+
close(reason = "normal") {
|
|
62
|
+
if (state.closed) return;
|
|
63
|
+
event(SESSION_END_RECORD, {
|
|
64
|
+
reason,
|
|
65
|
+
mem: memorySnapshot()
|
|
66
|
+
});
|
|
67
|
+
state.closed = true;
|
|
68
|
+
try {
|
|
69
|
+
fs.closeSync(state.fd);
|
|
70
|
+
} catch {}
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
state.header = {
|
|
74
|
+
role,
|
|
75
|
+
pid: process.pid,
|
|
76
|
+
meta,
|
|
77
|
+
env: describeEnvironment()
|
|
78
|
+
};
|
|
79
|
+
event(SESSION_RECORD, {
|
|
80
|
+
...state.header,
|
|
81
|
+
mem: memorySnapshot()
|
|
82
|
+
});
|
|
83
|
+
return recorder;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Periodic memory record written by the recorded thread itself.
|
|
87
|
+
*
|
|
88
|
+
* Doubles as a stall detector. This timer cannot fire while its thread is inside
|
|
89
|
+
* a long synchronous call, so a gap here that the independent sampler thread
|
|
90
|
+
* does not share means the thread was blocked — which is also why the heap
|
|
91
|
+
* reading nearest a synchronous blow-up is always stale.
|
|
92
|
+
*/
|
|
93
|
+
function startSelfSampler(recorder, intervalMs = 500) {
|
|
94
|
+
let last = Date.now();
|
|
95
|
+
const timer = setInterval(() => {
|
|
96
|
+
const now = Date.now();
|
|
97
|
+
recorder.event("mem-self", {
|
|
98
|
+
mem: recorder.memorySnapshot(),
|
|
99
|
+
stallMs: Math.max(0, now - last - intervalMs)
|
|
100
|
+
});
|
|
101
|
+
last = now;
|
|
102
|
+
}, intervalMs);
|
|
103
|
+
timer.unref();
|
|
104
|
+
return () => clearInterval(timer);
|
|
105
|
+
}
|
|
106
|
+
/** Flight logs in a directory, newest first, each flagged as crashed or clean. */
|
|
107
|
+
function listSessions(dir) {
|
|
108
|
+
let names;
|
|
109
|
+
try {
|
|
110
|
+
names = fs.readdirSync(dir);
|
|
111
|
+
} catch {
|
|
112
|
+
return [];
|
|
113
|
+
}
|
|
114
|
+
return names.filter((name) => name.startsWith(`flight-`) && name.endsWith(".ndjson")).map((name) => {
|
|
115
|
+
const file = path.join(dir, name);
|
|
116
|
+
const stat = fs.statSync(file);
|
|
117
|
+
return {
|
|
118
|
+
file,
|
|
119
|
+
mtimeMs: stat.mtimeMs,
|
|
120
|
+
bytes: stat.size,
|
|
121
|
+
crashed: !hasSessionEnd(file)
|
|
122
|
+
};
|
|
123
|
+
}).sort((lhs, rhs) => rhs.mtimeMs - lhs.mtimeMs);
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Parses a flight log, tolerating a final line cut short by a hard kill.
|
|
127
|
+
*
|
|
128
|
+
* A rotated session spans two files: the parked `.1` segment holds the original
|
|
129
|
+
* header and the earlier operations, the active file holds the tail. Both are
|
|
130
|
+
* read so begin records in one segment can be paired with end records in the
|
|
131
|
+
* other; sequence numbers run across the boundary.
|
|
132
|
+
*/
|
|
133
|
+
function readSession(file) {
|
|
134
|
+
const records = [];
|
|
135
|
+
let truncatedTail = false;
|
|
136
|
+
const parked = `${file}.1`;
|
|
137
|
+
if (fs.existsSync(parked)) for (const line of fs.readFileSync(parked, "utf8").split("\n")) {
|
|
138
|
+
if (line === "") continue;
|
|
139
|
+
try {
|
|
140
|
+
records.push(JSON.parse(line));
|
|
141
|
+
} catch {}
|
|
142
|
+
}
|
|
143
|
+
const lines = fs.readFileSync(file, "utf8").split("\n");
|
|
144
|
+
for (const [index, line] of lines.entries()) {
|
|
145
|
+
if (line === "") continue;
|
|
146
|
+
try {
|
|
147
|
+
records.push(JSON.parse(line));
|
|
148
|
+
} catch {
|
|
149
|
+
if (index >= lines.length - 2) truncatedTail = true;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return {
|
|
153
|
+
file,
|
|
154
|
+
records,
|
|
155
|
+
truncatedTail
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Mints a session id. A parent that supervises a worker calls this, hands the id
|
|
160
|
+
* to the worker, and keeps it for the crash marker, so both sides agree on the
|
|
161
|
+
* session by construction. The leading timestamp is what
|
|
162
|
+
* {@link sessionStartFromId} reads back.
|
|
163
|
+
*/
|
|
164
|
+
function newSessionId() {
|
|
165
|
+
return `${Date.now()}-${process.pid}-${randomTag()}`;
|
|
166
|
+
}
|
|
167
|
+
/** Wall-clock start of a session, taken from the id its file name carries. */
|
|
168
|
+
function sessionStartFromId(sessionId) {
|
|
169
|
+
const start = Number(sessionId.split("-")[0]);
|
|
170
|
+
return Number.isFinite(start) ? start : 0;
|
|
171
|
+
}
|
|
172
|
+
/** Session id embedded in a flight log's file name. */
|
|
173
|
+
function sessionIdFromFile(file) {
|
|
174
|
+
const match = path.basename(file).match(/^flight-(.+)\.ndjson(\.1)?$/);
|
|
175
|
+
return match ? match[1] : path.basename(file);
|
|
176
|
+
}
|
|
177
|
+
/** Cap on carried-forward begins, so a leak cannot make the preamble unbounded. */
|
|
178
|
+
const MAX_CARRIED_BEGINS = 256;
|
|
179
|
+
function writeLine(state, file, maxFileBytes, record) {
|
|
180
|
+
let line;
|
|
181
|
+
try {
|
|
182
|
+
line = `${JSON.stringify(record, bigintSafe)}\n`;
|
|
183
|
+
} catch {
|
|
184
|
+
line = `${JSON.stringify({
|
|
185
|
+
seq: record.seq,
|
|
186
|
+
type: "record-serialization-failed"
|
|
187
|
+
})}\n`;
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
if (state.bytes - state.preambleBytes + line.length > maxFileBytes) {
|
|
191
|
+
rotate(state, file);
|
|
192
|
+
writePreamble(state);
|
|
193
|
+
}
|
|
194
|
+
fs.writeSync(state.fd, line);
|
|
195
|
+
state.bytes += line.length;
|
|
196
|
+
} catch {}
|
|
197
|
+
}
|
|
198
|
+
function rotate(state, file) {
|
|
199
|
+
fs.closeSync(state.fd);
|
|
200
|
+
try {
|
|
201
|
+
fs.renameSync(file, `${file}.1`);
|
|
202
|
+
} catch {}
|
|
203
|
+
state.fd = fs.openSync(file, "a");
|
|
204
|
+
state.bytes = 0;
|
|
205
|
+
state.preambleBytes = 0;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Rewrites into the new segment the three things a report cannot be produced
|
|
209
|
+
* without, so repeated rotation costs only completed operations.
|
|
210
|
+
*
|
|
211
|
+
* Losing an *open* begin record would remove that operation from pairing
|
|
212
|
+
* entirely, and the operation still running at the moment of death is the one
|
|
213
|
+
* the report exists to name. Losing the earliest memory reading would leave the
|
|
214
|
+
* heap series starting mid-session while the sampler's resident series still
|
|
215
|
+
* starts at zero, which biases the classifier toward blaming native memory.
|
|
216
|
+
*/
|
|
217
|
+
function writePreamble(state) {
|
|
218
|
+
if (state.header) emitPreambleRecord(state, {
|
|
219
|
+
seq: ++state.seq,
|
|
220
|
+
t: monotonic(),
|
|
221
|
+
wall: Date.now(),
|
|
222
|
+
type: SESSION_RECORD,
|
|
223
|
+
...state.header,
|
|
224
|
+
continuation: true
|
|
225
|
+
});
|
|
226
|
+
if (state.baselineMem) emitPreambleRecord(state, {
|
|
227
|
+
seq: ++state.seq,
|
|
228
|
+
t: monotonic(),
|
|
229
|
+
wall: Date.now(),
|
|
230
|
+
type: MEM_BASELINE_RECORD,
|
|
231
|
+
mem: state.baselineMem,
|
|
232
|
+
carriedForward: true
|
|
233
|
+
});
|
|
234
|
+
for (const begin of state.openBegins.values()) emitPreambleRecord(state, {
|
|
235
|
+
...begin,
|
|
236
|
+
carriedForward: true
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
function emitPreambleRecord(state, record) {
|
|
240
|
+
try {
|
|
241
|
+
const line = `${JSON.stringify(record, bigintSafe)}\n`;
|
|
242
|
+
fs.writeSync(state.fd, line);
|
|
243
|
+
state.bytes += line.length;
|
|
244
|
+
state.preambleBytes += line.length;
|
|
245
|
+
} catch {}
|
|
246
|
+
}
|
|
247
|
+
function trackOpenOperation(state, record) {
|
|
248
|
+
if (record.type.endsWith("-begin")) {
|
|
249
|
+
if (state.openBegins.size < MAX_CARRIED_BEGINS) state.openBegins.set(record.seq, record);
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
if (record.type.endsWith("-end") || record.type.endsWith("-error")) {
|
|
253
|
+
if (typeof record.begin === "number") state.openBegins.delete(record.begin);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
function hasSessionEnd(file) {
|
|
257
|
+
const size = fs.statSync(file).size;
|
|
258
|
+
if (size === 0) return false;
|
|
259
|
+
const window = Math.min(size, 8192);
|
|
260
|
+
const buffer = Buffer.alloc(window);
|
|
261
|
+
const fd = fs.openSync(file, "r");
|
|
262
|
+
try {
|
|
263
|
+
fs.readSync(fd, buffer, 0, window, size - window);
|
|
264
|
+
} finally {
|
|
265
|
+
fs.closeSync(fd);
|
|
266
|
+
}
|
|
267
|
+
return buffer.toString("utf8").includes(`"type":"${SESSION_END_RECORD}"`);
|
|
268
|
+
}
|
|
269
|
+
function describeEnvironment() {
|
|
270
|
+
const maxOldSpaceFlag = process.execArgv.find((arg) => arg.startsWith("--max-old-space-size"));
|
|
271
|
+
return {
|
|
272
|
+
node: process.version,
|
|
273
|
+
platform: `${process.platform}-${process.arch}`,
|
|
274
|
+
cpus: os.cpus().length,
|
|
275
|
+
totalMemory: os.totalmem(),
|
|
276
|
+
heapLimit: v8.getHeapStatistics().heap_size_limit,
|
|
277
|
+
execArgv: [...process.execArgv],
|
|
278
|
+
maxOldSpaceSize: maxOldSpaceFlag ? Number(maxOldSpaceFlag.split("=")[1]) : void 0
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
function bigintSafe(_key, value) {
|
|
282
|
+
return typeof value === "bigint" ? Number(value) : value;
|
|
283
|
+
}
|
|
284
|
+
function monotonic() {
|
|
285
|
+
return Math.round(performance.now() * 1e3) / 1e3;
|
|
286
|
+
}
|
|
287
|
+
function randomTag() {
|
|
288
|
+
return Math.random().toString(36).slice(2, 8);
|
|
289
|
+
}
|
|
290
|
+
//#endregion
|
|
291
|
+
export { listSessions, newSessionId, openRecorder, readSession, sessionIdFromFile, sessionStartFromId, startSelfSampler };
|
|
292
|
+
|
|
293
|
+
//# sourceMappingURL=recorder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recorder.js","names":[],"sources":["../src/recorder.ts"],"sourcesContent":["import fs from \"node:fs\";\nimport path from \"node:path\";\nimport os from \"node:os\";\nimport v8 from \"node:v8\";\nimport {\n FLIGHT_FILE_PREFIX,\n MEM_BASELINE_RECORD,\n SESSION_END_RECORD,\n SESSION_RECORD,\n type FlightRecord,\n type MemorySnapshot,\n type SessionEnvironment,\n} from \"./events\";\n\nexport type RecorderOptions = {\n /** Directory holding flight logs; created if absent. */\n dir: string;\n /** Which part of the app is recording, e.g. `middle-layer`. */\n role?: string;\n /** Free-form context stored in the session header (app version, project id). */\n meta?: Record<string, unknown>;\n /** Log is rotated past this size so the tail, which explains the crash, survives. */\n maxFileBytes?: number;\n /**\n * Session id assigned by a supervising parent, so the crash marker the parent\n * writes names this session with certainty rather than by inference.\n * Generated when absent.\n */\n sessionId?: string;\n};\n\nexport type Recorder = {\n readonly sessionId: string;\n readonly file: string;\n /** Appends one record and returns its sequence number. Never throws. */\n event(type: string, payload?: Record<string, unknown>): number;\n /** Memory reading for the calling thread; `rss` is process-wide. */\n memorySnapshot(): MemorySnapshot;\n /** Writes the terminating record. Its absence is how a crash is detected. */\n close(reason?: string): void;\n};\n\nexport type SessionFileInfo = {\n file: string;\n mtimeMs: number;\n bytes: number;\n /** True when the log has no terminating record, i.e. the process died. */\n crashed: boolean;\n};\n\nexport type ParsedSession = {\n file: string;\n records: FlightRecord[];\n /** True when the last line was cut mid-write by the kill. */\n truncatedTail: boolean;\n};\n\n/**\n * Opens a flight log for this process and writes the session header.\n *\n * Records are appended with a synchronous write rather than through a stream:\n * the process being recorded dies without warning — V8 fatal out-of-memory, a\n * failed native allocation, the OS out-of-memory killer — so no exit hook, no\n * flush and no `finally` block runs. Anything still sitting in a userspace\n * buffer is exactly the part that would have explained the death.\n */\nexport function openRecorder(options: RecorderOptions): Recorder {\n const {\n dir,\n role = \"middle-layer\",\n meta = {},\n maxFileBytes = 32 * 1024 * 1024,\n sessionId = newSessionId(),\n } = options;\n fs.mkdirSync(dir, { recursive: true });\n\n const file = path.join(dir, `${FLIGHT_FILE_PREFIX}-${sessionId}.ndjson`);\n const state: WriterState = {\n fd: fs.openSync(file, \"a\"),\n bytes: 0,\n preambleBytes: 0,\n seq: 0,\n closed: false,\n header: undefined,\n baselineMem: undefined,\n openBegins: new Map(),\n };\n\n const memorySnapshot = (): MemorySnapshot => {\n const usage = process.memoryUsage();\n return {\n rss: usage.rss,\n heapUsed: usage.heapUsed,\n heapTotal: usage.heapTotal,\n external: usage.external,\n arrayBuffers: usage.arrayBuffers,\n heapLimit: v8.getHeapStatistics().heap_size_limit,\n };\n };\n\n const event = (type: string, payload: Record<string, unknown> = {}): number => {\n if (state.closed) return -1;\n const seq = ++state.seq;\n const record: FlightRecord = {\n seq,\n t: monotonic(),\n wall: Date.now(),\n type,\n ...payload,\n };\n if (state.baselineMem === undefined && record.mem) state.baselineMem = record.mem;\n trackOpenOperation(state, record);\n writeLine(state, file, maxFileBytes, record);\n return seq;\n };\n\n const recorder: Recorder = {\n sessionId,\n file,\n event,\n memorySnapshot,\n close(reason = \"normal\") {\n if (state.closed) return;\n event(SESSION_END_RECORD, { reason, mem: memorySnapshot() });\n state.closed = true;\n try {\n fs.closeSync(state.fd);\n } catch {\n // Closing an already-dead descriptor must not fail shutdown.\n }\n },\n };\n\n // Kept so a rotated log can be given the same header again: the active file\n // must describe its own session even if the parked segment is lost.\n state.header = { role, pid: process.pid, meta, env: describeEnvironment() };\n event(SESSION_RECORD, { ...state.header, mem: memorySnapshot() });\n\n return recorder;\n}\n\n/**\n * Periodic memory record written by the recorded thread itself.\n *\n * Doubles as a stall detector. This timer cannot fire while its thread is inside\n * a long synchronous call, so a gap here that the independent sampler thread\n * does not share means the thread was blocked — which is also why the heap\n * reading nearest a synchronous blow-up is always stale.\n */\nexport function startSelfSampler(recorder: Recorder, intervalMs = 500): () => void {\n let last = Date.now();\n const timer = setInterval(() => {\n const now = Date.now();\n recorder.event(\"mem-self\", {\n mem: recorder.memorySnapshot(),\n stallMs: Math.max(0, now - last - intervalMs),\n });\n last = now;\n }, intervalMs);\n timer.unref();\n return () => clearInterval(timer);\n}\n\n/** Flight logs in a directory, newest first, each flagged as crashed or clean. */\nexport function listSessions(dir: string): SessionFileInfo[] {\n let names: string[];\n try {\n names = fs.readdirSync(dir);\n } catch {\n return [];\n }\n return names\n .filter((name) => name.startsWith(`${FLIGHT_FILE_PREFIX}-`) && name.endsWith(\".ndjson\"))\n .map((name) => {\n const file = path.join(dir, name);\n const stat = fs.statSync(file);\n return { file, mtimeMs: stat.mtimeMs, bytes: stat.size, crashed: !hasSessionEnd(file) };\n })\n .sort((lhs, rhs) => rhs.mtimeMs - lhs.mtimeMs);\n}\n\n/**\n * Parses a flight log, tolerating a final line cut short by a hard kill.\n *\n * A rotated session spans two files: the parked `.1` segment holds the original\n * header and the earlier operations, the active file holds the tail. Both are\n * read so begin records in one segment can be paired with end records in the\n * other; sequence numbers run across the boundary.\n */\nexport function readSession(file: string): ParsedSession {\n const records: FlightRecord[] = [];\n let truncatedTail = false;\n const parked = `${file}.1`;\n if (fs.existsSync(parked)) {\n for (const line of fs.readFileSync(parked, \"utf8\").split(\"\\n\")) {\n if (line === \"\") continue;\n try {\n records.push(JSON.parse(line) as FlightRecord);\n } catch {\n // A damaged line in the parked segment costs one record, not the session.\n }\n }\n }\n const lines = fs.readFileSync(file, \"utf8\").split(\"\\n\");\n for (const [index, line] of lines.entries()) {\n if (line === \"\") continue;\n try {\n records.push(JSON.parse(line) as FlightRecord);\n } catch {\n if (index >= lines.length - 2) truncatedTail = true;\n }\n }\n return { file, records, truncatedTail };\n}\n\n/**\n * Mints a session id. A parent that supervises a worker calls this, hands the id\n * to the worker, and keeps it for the crash marker, so both sides agree on the\n * session by construction. The leading timestamp is what\n * {@link sessionStartFromId} reads back.\n */\nexport function newSessionId(): string {\n return `${Date.now()}-${process.pid}-${randomTag()}`;\n}\n\n/** Wall-clock start of a session, taken from the id its file name carries. */\nexport function sessionStartFromId(sessionId: string): number {\n const start = Number(sessionId.split(\"-\")[0]);\n return Number.isFinite(start) ? start : 0;\n}\n\n/** Session id embedded in a flight log's file name. */\nexport function sessionIdFromFile(file: string): string {\n const match = path.basename(file).match(/^flight-(.+)\\.ndjson(\\.1)?$/);\n return match ? match[1] : path.basename(file);\n}\n\n// Internals\n\ntype WriterState = {\n fd: number;\n bytes: number;\n /** Bytes of the carried-forward preamble, which do not count toward the limit. */\n preambleBytes: number;\n seq: number;\n closed: boolean;\n header: Record<string, unknown> | undefined;\n /** Earliest memory reading of the session, so growth stays measurable. */\n baselineMem: MemorySnapshot | undefined;\n /** Begin records with no end yet, keyed by their sequence number. */\n openBegins: Map<number, FlightRecord>;\n};\n\n/** Cap on carried-forward begins, so a leak cannot make the preamble unbounded. */\nconst MAX_CARRIED_BEGINS = 256;\n\nfunction writeLine(\n state: WriterState,\n file: string,\n maxFileBytes: number,\n record: FlightRecord,\n): void {\n let line: string;\n try {\n line = `${JSON.stringify(record, bigintSafe)}\\n`;\n } catch {\n line = `${JSON.stringify({ seq: record.seq, type: \"record-serialization-failed\" })}\\n`;\n }\n try {\n // The preamble is not charged against the limit, so a large preamble cannot\n // trigger another rotation on the very next write.\n if (state.bytes - state.preambleBytes + line.length > maxFileBytes) {\n rotate(state, file);\n writePreamble(state);\n }\n fs.writeSync(state.fd, line);\n state.bytes += line.length;\n } catch {\n // A recorder that cannot write stays silent rather than cascading into the\n // application it is only supposed to observe.\n }\n}\n\n// The tail is the only part that explains a crash, so the old file is parked\n// beside the new one instead of the new writes being dropped. Exactly one parked\n// segment is kept, which bounds a session's disk use at twice the file limit.\nfunction rotate(state: WriterState, file: string): void {\n fs.closeSync(state.fd);\n try {\n fs.renameSync(file, `${file}.1`);\n } catch {\n // If the parked slot cannot be written, recording simply continues.\n }\n state.fd = fs.openSync(file, \"a\");\n state.bytes = 0;\n state.preambleBytes = 0;\n}\n\n/**\n * Rewrites into the new segment the three things a report cannot be produced\n * without, so repeated rotation costs only completed operations.\n *\n * Losing an *open* begin record would remove that operation from pairing\n * entirely, and the operation still running at the moment of death is the one\n * the report exists to name. Losing the earliest memory reading would leave the\n * heap series starting mid-session while the sampler's resident series still\n * starts at zero, which biases the classifier toward blaming native memory.\n */\nfunction writePreamble(state: WriterState): void {\n if (state.header) {\n emitPreambleRecord(state, {\n seq: ++state.seq,\n t: monotonic(),\n wall: Date.now(),\n type: SESSION_RECORD,\n ...state.header,\n continuation: true,\n });\n }\n if (state.baselineMem) {\n emitPreambleRecord(state, {\n seq: ++state.seq,\n t: monotonic(),\n wall: Date.now(),\n type: MEM_BASELINE_RECORD,\n mem: state.baselineMem,\n carriedForward: true,\n });\n }\n // Original sequence numbers are kept, which is what lets an end record in a\n // later segment pair with a begin first written in an overwritten one.\n for (const begin of state.openBegins.values()) {\n emitPreambleRecord(state, { ...begin, carriedForward: true });\n }\n}\n\nfunction emitPreambleRecord(state: WriterState, record: FlightRecord): void {\n try {\n const line = `${JSON.stringify(record, bigintSafe)}\\n`;\n fs.writeSync(state.fd, line);\n state.bytes += line.length;\n state.preambleBytes += line.length;\n } catch {\n // A preamble that cannot be written must not stop the session.\n }\n}\n\n// Open operations are tracked by the same suffix convention the analyzer pairs\n// on, so the recorder needs no separate vocabulary for them.\nfunction trackOpenOperation(state: WriterState, record: FlightRecord): void {\n if (record.type.endsWith(\"-begin\")) {\n if (state.openBegins.size < MAX_CARRIED_BEGINS) state.openBegins.set(record.seq, record);\n return;\n }\n if (record.type.endsWith(\"-end\") || record.type.endsWith(\"-error\")) {\n if (typeof record.begin === \"number\") state.openBegins.delete(record.begin);\n }\n}\n\nfunction hasSessionEnd(file: string): boolean {\n const size = fs.statSync(file).size;\n if (size === 0) return false;\n const window = Math.min(size, 8192);\n const buffer = Buffer.alloc(window);\n const fd = fs.openSync(file, \"r\");\n try {\n fs.readSync(fd, buffer, 0, window, size - window);\n } finally {\n fs.closeSync(fd);\n }\n return buffer.toString(\"utf8\").includes(`\"type\":\"${SESSION_END_RECORD}\"`);\n}\n\nfunction describeEnvironment(): SessionEnvironment {\n const maxOldSpaceFlag = process.execArgv.find((arg) => arg.startsWith(\"--max-old-space-size\"));\n return {\n node: process.version,\n platform: `${process.platform}-${process.arch}`,\n cpus: os.cpus().length,\n totalMemory: os.totalmem(),\n heapLimit: v8.getHeapStatistics().heap_size_limit,\n execArgv: [...process.execArgv],\n maxOldSpaceSize: maxOldSpaceFlag ? Number(maxOldSpaceFlag.split(\"=\")[1]) : undefined,\n };\n}\n\nfunction bigintSafe(_key: string, value: unknown): unknown {\n return typeof value === \"bigint\" ? Number(value) : value;\n}\n\nfunction monotonic(): number {\n return Math.round(performance.now() * 1000) / 1000;\n}\n\nfunction randomTag(): string {\n return Math.random().toString(36).slice(2, 8);\n}\n"],"mappings":";;;;;;;;;;;;;;;AAkEA,SAAgB,aAAa,SAAoC;CAC/D,MAAM,EACJ,KACA,OAAO,gBACP,OAAO,CAAC,GACR,eAAe,KAAK,OAAO,MAC3B,YAAY,aAAa,MACvB;CACJ,GAAG,UAAU,KAAK,EAAE,WAAW,KAAK,CAAC;CAErC,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,mBAAmB,GAAG,UAAU,QAAQ;CACvE,MAAM,QAAqB;EACzB,IAAI,GAAG,SAAS,MAAM,GAAG;EACzB,OAAO;EACP,eAAe;EACf,KAAK;EACL,QAAQ;EACR,QAAQ,KAAA;EACR,aAAa,KAAA;EACb,4BAAY,IAAI,IAAI;CACtB;CAEA,MAAM,uBAAuC;EAC3C,MAAM,QAAQ,QAAQ,YAAY;EAClC,OAAO;GACL,KAAK,MAAM;GACX,UAAU,MAAM;GAChB,WAAW,MAAM;GACjB,UAAU,MAAM;GAChB,cAAc,MAAM;GACpB,WAAW,GAAG,kBAAkB,CAAC,CAAC;EACpC;CACF;CAEA,MAAM,SAAS,MAAc,UAAmC,CAAC,MAAc;EAC7E,IAAI,MAAM,QAAQ,OAAO;EACzB,MAAM,MAAM,EAAE,MAAM;EACpB,MAAM,SAAuB;GAC3B;GACA,GAAG,UAAU;GACb,MAAM,KAAK,IAAI;GACf;GACA,GAAG;EACL;EACA,IAAI,MAAM,gBAAgB,KAAA,KAAa,OAAO,KAAK,MAAM,cAAc,OAAO;EAC9E,mBAAmB,OAAO,MAAM;EAChC,UAAU,OAAO,MAAM,cAAc,MAAM;EAC3C,OAAO;CACT;CAEA,MAAM,WAAqB;EACzB;EACA;EACA;EACA;EACA,MAAM,SAAS,UAAU;GACvB,IAAI,MAAM,QAAQ;GAClB,MAAM,oBAAoB;IAAE;IAAQ,KAAK,eAAe;GAAE,CAAC;GAC3D,MAAM,SAAS;GACf,IAAI;IACF,GAAG,UAAU,MAAM,EAAE;GACvB,QAAQ,CAER;EACF;CACF;CAIA,MAAM,SAAS;EAAE;EAAM,KAAK,QAAQ;EAAK;EAAM,KAAK,oBAAoB;CAAE;CAC1E,MAAM,gBAAgB;EAAE,GAAG,MAAM;EAAQ,KAAK,eAAe;CAAE,CAAC;CAEhE,OAAO;AACT;;;;;;;;;AAUA,SAAgB,iBAAiB,UAAoB,aAAa,KAAiB;CACjF,IAAI,OAAO,KAAK,IAAI;CACpB,MAAM,QAAQ,kBAAkB;EAC9B,MAAM,MAAM,KAAK,IAAI;EACrB,SAAS,MAAM,YAAY;GACzB,KAAK,SAAS,eAAe;GAC7B,SAAS,KAAK,IAAI,GAAG,MAAM,OAAO,UAAU;EAC9C,CAAC;EACD,OAAO;CACT,GAAG,UAAU;CACb,MAAM,MAAM;CACZ,aAAa,cAAc,KAAK;AAClC;;AAGA,SAAgB,aAAa,KAAgC;CAC3D,IAAI;CACJ,IAAI;EACF,QAAQ,GAAG,YAAY,GAAG;CAC5B,QAAQ;EACN,OAAO,CAAC;CACV;CACA,OAAO,MACJ,QAAQ,SAAS,KAAK,WAAW,SAAwB,KAAK,KAAK,SAAS,SAAS,CAAC,CAAC,CACvF,KAAK,SAAS;EACb,MAAM,OAAO,KAAK,KAAK,KAAK,IAAI;EAChC,MAAM,OAAO,GAAG,SAAS,IAAI;EAC7B,OAAO;GAAE;GAAM,SAAS,KAAK;GAAS,OAAO,KAAK;GAAM,SAAS,CAAC,cAAc,IAAI;EAAE;CACxF,CAAC,CAAC,CACD,MAAM,KAAK,QAAQ,IAAI,UAAU,IAAI,OAAO;AACjD;;;;;;;;;AAUA,SAAgB,YAAY,MAA6B;CACvD,MAAM,UAA0B,CAAC;CACjC,IAAI,gBAAgB;CACpB,MAAM,SAAS,GAAG,KAAK;CACvB,IAAI,GAAG,WAAW,MAAM,GACtB,KAAK,MAAM,QAAQ,GAAG,aAAa,QAAQ,MAAM,CAAC,CAAC,MAAM,IAAI,GAAG;EAC9D,IAAI,SAAS,IAAI;EACjB,IAAI;GACF,QAAQ,KAAK,KAAK,MAAM,IAAI,CAAiB;EAC/C,QAAQ,CAER;CACF;CAEF,MAAM,QAAQ,GAAG,aAAa,MAAM,MAAM,CAAC,CAAC,MAAM,IAAI;CACtD,KAAK,MAAM,CAAC,OAAO,SAAS,MAAM,QAAQ,GAAG;EAC3C,IAAI,SAAS,IAAI;EACjB,IAAI;GACF,QAAQ,KAAK,KAAK,MAAM,IAAI,CAAiB;EAC/C,QAAQ;GACN,IAAI,SAAS,MAAM,SAAS,GAAG,gBAAgB;EACjD;CACF;CACA,OAAO;EAAE;EAAM;EAAS;CAAc;AACxC;;;;;;;AAQA,SAAgB,eAAuB;CACrC,OAAO,GAAG,KAAK,IAAI,EAAE,GAAG,QAAQ,IAAI,GAAG,UAAU;AACnD;;AAGA,SAAgB,mBAAmB,WAA2B;CAC5D,MAAM,QAAQ,OAAO,UAAU,MAAM,GAAG,CAAC,CAAC,EAAE;CAC5C,OAAO,OAAO,SAAS,KAAK,IAAI,QAAQ;AAC1C;;AAGA,SAAgB,kBAAkB,MAAsB;CACtD,MAAM,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,6BAA6B;CACrE,OAAO,QAAQ,MAAM,KAAK,KAAK,SAAS,IAAI;AAC9C;;AAmBA,MAAM,qBAAqB;AAE3B,SAAS,UACP,OACA,MACA,cACA,QACM;CACN,IAAI;CACJ,IAAI;EACF,OAAO,GAAG,KAAK,UAAU,QAAQ,UAAU,EAAE;CAC/C,QAAQ;EACN,OAAO,GAAG,KAAK,UAAU;GAAE,KAAK,OAAO;GAAK,MAAM;EAA8B,CAAC,EAAE;CACrF;CACA,IAAI;EAGF,IAAI,MAAM,QAAQ,MAAM,gBAAgB,KAAK,SAAS,cAAc;GAClE,OAAO,OAAO,IAAI;GAClB,cAAc,KAAK;EACrB;EACA,GAAG,UAAU,MAAM,IAAI,IAAI;EAC3B,MAAM,SAAS,KAAK;CACtB,QAAQ,CAGR;AACF;AAKA,SAAS,OAAO,OAAoB,MAAoB;CACtD,GAAG,UAAU,MAAM,EAAE;CACrB,IAAI;EACF,GAAG,WAAW,MAAM,GAAG,KAAK,GAAG;CACjC,QAAQ,CAER;CACA,MAAM,KAAK,GAAG,SAAS,MAAM,GAAG;CAChC,MAAM,QAAQ;CACd,MAAM,gBAAgB;AACxB;;;;;;;;;;;AAYA,SAAS,cAAc,OAA0B;CAC/C,IAAI,MAAM,QACR,mBAAmB,OAAO;EACxB,KAAK,EAAE,MAAM;EACb,GAAG,UAAU;EACb,MAAM,KAAK,IAAI;EACf,MAAM;EACN,GAAG,MAAM;EACT,cAAc;CAChB,CAAC;CAEH,IAAI,MAAM,aACR,mBAAmB,OAAO;EACxB,KAAK,EAAE,MAAM;EACb,GAAG,UAAU;EACb,MAAM,KAAK,IAAI;EACf,MAAM;EACN,KAAK,MAAM;EACX,gBAAgB;CAClB,CAAC;CAIH,KAAK,MAAM,SAAS,MAAM,WAAW,OAAO,GAC1C,mBAAmB,OAAO;EAAE,GAAG;EAAO,gBAAgB;CAAK,CAAC;AAEhE;AAEA,SAAS,mBAAmB,OAAoB,QAA4B;CAC1E,IAAI;EACF,MAAM,OAAO,GAAG,KAAK,UAAU,QAAQ,UAAU,EAAE;EACnD,GAAG,UAAU,MAAM,IAAI,IAAI;EAC3B,MAAM,SAAS,KAAK;EACpB,MAAM,iBAAiB,KAAK;CAC9B,QAAQ,CAER;AACF;AAIA,SAAS,mBAAmB,OAAoB,QAA4B;CAC1E,IAAI,OAAO,KAAK,SAAS,QAAQ,GAAG;EAClC,IAAI,MAAM,WAAW,OAAO,oBAAoB,MAAM,WAAW,IAAI,OAAO,KAAK,MAAM;EACvF;CACF;CACA,IAAI,OAAO,KAAK,SAAS,MAAM,KAAK,OAAO,KAAK,SAAS,QAAQ;MAC3D,OAAO,OAAO,UAAU,UAAU,MAAM,WAAW,OAAO,OAAO,KAAK;CAAA;AAE9E;AAEA,SAAS,cAAc,MAAuB;CAC5C,MAAM,OAAO,GAAG,SAAS,IAAI,CAAC,CAAC;CAC/B,IAAI,SAAS,GAAG,OAAO;CACvB,MAAM,SAAS,KAAK,IAAI,MAAM,IAAI;CAClC,MAAM,SAAS,OAAO,MAAM,MAAM;CAClC,MAAM,KAAK,GAAG,SAAS,MAAM,GAAG;CAChC,IAAI;EACF,GAAG,SAAS,IAAI,QAAQ,GAAG,QAAQ,OAAO,MAAM;CAClD,UAAU;EACR,GAAG,UAAU,EAAE;CACjB;CACA,OAAO,OAAO,SAAS,MAAM,CAAC,CAAC,SAAS,WAAW,mBAAmB,EAAE;AAC1E;AAEA,SAAS,sBAA0C;CACjD,MAAM,kBAAkB,QAAQ,SAAS,MAAM,QAAQ,IAAI,WAAW,sBAAsB,CAAC;CAC7F,OAAO;EACL,MAAM,QAAQ;EACd,UAAU,GAAG,QAAQ,SAAS,GAAG,QAAQ;EACzC,MAAM,GAAG,KAAK,CAAC,CAAC;EAChB,aAAa,GAAG,SAAS;EACzB,WAAW,GAAG,kBAAkB,CAAC,CAAC;EAClC,UAAU,CAAC,GAAG,QAAQ,QAAQ;EAC9B,iBAAiB,kBAAkB,OAAO,gBAAgB,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,KAAA;CAC7E;AACF;AAEA,SAAS,WAAW,MAAc,OAAyB;CACzD,OAAO,OAAO,UAAU,WAAW,OAAO,KAAK,IAAI;AACrD;AAEA,SAAS,YAAoB;CAC3B,OAAO,KAAK,MAAM,YAAY,IAAI,IAAI,GAAI,IAAI;AAChD;AAEA,SAAS,YAAoB;CAC3B,OAAO,KAAK,OAAO,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;AAC9C"}
|
package/dist/redact.d.ts
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { DataSummary } from "./data_summary.js";
|
|
2
|
+
//#region src/redact.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* Structure-preserving redaction for anything a driver seam is handed.
|
|
5
|
+
*
|
|
6
|
+
* The definition a block model builds is recorded by *shape* rather than by a
|
|
7
|
+
* hand-written digest per definition type. Keys, numbers and the small set of
|
|
8
|
+
* strings that are schema survive verbatim; every other string is replaced by a
|
|
9
|
+
* hash and a length. That keeps the record useful for diagnosis, keeps customer
|
|
10
|
+
* data out of it by default rather than by enumeration, and works unchanged for
|
|
11
|
+
* definition shapes this code has never seen — the V2 query API included.
|
|
12
|
+
*
|
|
13
|
+
* The default for an unrecognised string is to hash it. A new field can
|
|
14
|
+
* therefore make a report less informative, but never make it leak.
|
|
15
|
+
*/
|
|
16
|
+
/** Keys whose string value is schema, kept as written. */
|
|
17
|
+
export declare const SCHEMA_KEYS: Set<string>;
|
|
18
|
+
/** Keys under which every string is schema, at any depth (axis identity). */
|
|
19
|
+
export declare const SCHEMA_SUBTREE_KEYS: Set<string>;
|
|
20
|
+
/** Keys never descended into; summarised by counts instead. */
|
|
21
|
+
export declare const SUMMARISED_KEYS: Set<string>;
|
|
22
|
+
/** Keys reduced to a cardinality, because their contents are values. */
|
|
23
|
+
export declare const COUNTED_KEYS: Set<string>;
|
|
24
|
+
export type RedactionStats = {
|
|
25
|
+
hashedStrings: number;
|
|
26
|
+
truncatedArrays: number;
|
|
27
|
+
omittedItems: number;
|
|
28
|
+
depthCapped: number;
|
|
29
|
+
opaqueObjects: number;
|
|
30
|
+
budgetExhausted: boolean;
|
|
31
|
+
};
|
|
32
|
+
export type RedactOptions = {
|
|
33
|
+
maxDepth?: number;
|
|
34
|
+
maxArrayItems?: number;
|
|
35
|
+
maxStringLength?: number;
|
|
36
|
+
/** Ceiling on emitted values, so one pathological definition cannot fill the log. */
|
|
37
|
+
maxNodes?: number;
|
|
38
|
+
};
|
|
39
|
+
export type HashedString = {
|
|
40
|
+
h: string;
|
|
41
|
+
n: number;
|
|
42
|
+
};
|
|
43
|
+
/** Redacts a definition, returning the new value and what had to be elided. */
|
|
44
|
+
export declare function redact(value: unknown, options?: RedactOptions): {
|
|
45
|
+
value: unknown;
|
|
46
|
+
stats: RedactionStats;
|
|
47
|
+
};
|
|
48
|
+
/** Stable short hash plus the original length. Never reversible to the value. */
|
|
49
|
+
export declare function hashString(value: string): HashedString;
|
|
50
|
+
/** True for the object form produced in place of a redacted string. */
|
|
51
|
+
export declare function isHashedString(value: unknown): value is HashedString;
|
|
52
|
+
//#endregion
|
|
53
|
+
//# sourceMappingURL=redact.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redact.d.ts","names":[],"sources":["../src/redact.ts"],"mappings":";;;;;;;;;;;;;;;;qBAkBa,aAAW;;qBAGX,qBAAmB;;qBAGnB,iBAAe;;qBAGf,cAAY;YAEb;EACV;EACA;EACA;EACA;EACA;EACA;;YAGU;EACV;EACA;EACA;;EAEA;;YAGU;EAAiB;EAAW;;;wBAGxB,OACd,gBACA,UAAS;EACN;EAAgB,OAAO;;;wBAuBZ,WAAW,gBAAgB;;wBAQ3B,eAAe,iBAAiB,SAAS"}
|
package/dist/redact.js
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { summarizeData } from "./data_summary.js";
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
//#region src/redact.ts
|
|
4
|
+
/**
|
|
5
|
+
* Structure-preserving redaction for anything a driver seam is handed.
|
|
6
|
+
*
|
|
7
|
+
* The definition a block model builds is recorded by *shape* rather than by a
|
|
8
|
+
* hand-written digest per definition type. Keys, numbers and the small set of
|
|
9
|
+
* strings that are schema survive verbatim; every other string is replaced by a
|
|
10
|
+
* hash and a length. That keeps the record useful for diagnosis, keeps customer
|
|
11
|
+
* data out of it by default rather than by enumeration, and works unchanged for
|
|
12
|
+
* definition shapes this code has never seen — the V2 query API included.
|
|
13
|
+
*
|
|
14
|
+
* The default for an unrecognised string is to hash it. A new field can
|
|
15
|
+
* therefore make a report less informative, but never make it leak.
|
|
16
|
+
*/
|
|
17
|
+
/** Keys whose string value is schema, kept as written. */
|
|
18
|
+
const SCHEMA_KEYS = /* @__PURE__ */ new Set([
|
|
19
|
+
"type",
|
|
20
|
+
"name",
|
|
21
|
+
"valueType",
|
|
22
|
+
"kind",
|
|
23
|
+
"operator",
|
|
24
|
+
"mode"
|
|
25
|
+
]);
|
|
26
|
+
/** Keys under which every string is schema, at any depth (axis identity). */
|
|
27
|
+
const SCHEMA_SUBTREE_KEYS = /* @__PURE__ */ new Set(["domain", "contextDomain"]);
|
|
28
|
+
/** Keys never descended into; summarised by counts instead. */
|
|
29
|
+
const SUMMARISED_KEYS = /* @__PURE__ */ new Set(["data", "dataInfo"]);
|
|
30
|
+
/** Keys reduced to a cardinality, because their contents are values. */
|
|
31
|
+
const COUNTED_KEYS = /* @__PURE__ */ new Set(["references", "parts"]);
|
|
32
|
+
/** Redacts a definition, returning the new value and what had to be elided. */
|
|
33
|
+
function redact(value, options = {}) {
|
|
34
|
+
const limits = {
|
|
35
|
+
maxDepth: options.maxDepth ?? 32,
|
|
36
|
+
maxArrayItems: options.maxArrayItems ?? 64,
|
|
37
|
+
maxStringLength: options.maxStringLength ?? 128,
|
|
38
|
+
maxNodes: options.maxNodes ?? 2e4
|
|
39
|
+
};
|
|
40
|
+
const stats = {
|
|
41
|
+
hashedStrings: 0,
|
|
42
|
+
truncatedArrays: 0,
|
|
43
|
+
omittedItems: 0,
|
|
44
|
+
depthCapped: 0,
|
|
45
|
+
opaqueObjects: 0,
|
|
46
|
+
budgetExhausted: false
|
|
47
|
+
};
|
|
48
|
+
return {
|
|
49
|
+
value: walk(value, {
|
|
50
|
+
key: void 0,
|
|
51
|
+
schemaSubtree: false,
|
|
52
|
+
depth: 0
|
|
53
|
+
}, limits, stats, {
|
|
54
|
+
nodes: 0,
|
|
55
|
+
seen: /* @__PURE__ */ new WeakSet()
|
|
56
|
+
}),
|
|
57
|
+
stats
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/** Stable short hash plus the original length. Never reversible to the value. */
|
|
61
|
+
function hashString(value) {
|
|
62
|
+
return {
|
|
63
|
+
h: crypto.createHash("sha256").update(value).digest("hex").slice(0, 12),
|
|
64
|
+
n: value.length
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** True for the object form produced in place of a redacted string. */
|
|
68
|
+
function isHashedString(value) {
|
|
69
|
+
return typeof value === "object" && value !== null && "h" in value && "n" in value;
|
|
70
|
+
}
|
|
71
|
+
function walk(value, at, limits, stats, state) {
|
|
72
|
+
if (state.nodes++ > limits.maxNodes) {
|
|
73
|
+
stats.budgetExhausted = true;
|
|
74
|
+
return { $budget: true };
|
|
75
|
+
}
|
|
76
|
+
if (value === null || value === void 0) return value ?? null;
|
|
77
|
+
if (typeof value === "bigint") return Number(value);
|
|
78
|
+
if (typeof value === "number" || typeof value === "boolean") return value;
|
|
79
|
+
if (typeof value === "string") return redactString(value, at, limits, stats);
|
|
80
|
+
if (typeof value !== "object") return { $type: typeof value };
|
|
81
|
+
if (at.depth >= limits.maxDepth) {
|
|
82
|
+
stats.depthCapped++;
|
|
83
|
+
return { $depth: at.depth };
|
|
84
|
+
}
|
|
85
|
+
if (state.seen.has(value)) return { $cycle: true };
|
|
86
|
+
if (Array.isArray(value)) {
|
|
87
|
+
state.seen.add(value);
|
|
88
|
+
return walkArray(value, at, limits, stats, state);
|
|
89
|
+
}
|
|
90
|
+
if (!isPlainObject(value)) {
|
|
91
|
+
stats.opaqueObjects++;
|
|
92
|
+
return { $opaque: className(value) };
|
|
93
|
+
}
|
|
94
|
+
state.seen.add(value);
|
|
95
|
+
const out = {};
|
|
96
|
+
for (const [key, child] of Object.entries(value)) {
|
|
97
|
+
if (SUMMARISED_KEYS.has(key)) {
|
|
98
|
+
out[key] = summarizeData(child);
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
if (COUNTED_KEYS.has(key)) {
|
|
102
|
+
out[key] = { $count: countOf(child) };
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
out[key] = walk(child, {
|
|
106
|
+
key,
|
|
107
|
+
schemaSubtree: at.schemaSubtree || SCHEMA_SUBTREE_KEYS.has(key),
|
|
108
|
+
depth: at.depth + 1
|
|
109
|
+
}, limits, stats, state);
|
|
110
|
+
}
|
|
111
|
+
return out;
|
|
112
|
+
}
|
|
113
|
+
function walkArray(value, at, limits, stats, state) {
|
|
114
|
+
const kept = value.slice(0, limits.maxArrayItems).map((item) => walk(item, {
|
|
115
|
+
key: at.key,
|
|
116
|
+
schemaSubtree: at.schemaSubtree,
|
|
117
|
+
depth: at.depth + 1
|
|
118
|
+
}, limits, stats, state));
|
|
119
|
+
const omitted = value.length - kept.length;
|
|
120
|
+
if (omitted <= 0) return kept;
|
|
121
|
+
stats.truncatedArrays++;
|
|
122
|
+
stats.omittedItems += omitted;
|
|
123
|
+
return [...kept, { $omitted: omitted }];
|
|
124
|
+
}
|
|
125
|
+
function redactString(value, at, limits, stats) {
|
|
126
|
+
if (at.schemaSubtree || at.key !== void 0 && SCHEMA_KEYS.has(at.key)) return value.length > limits.maxStringLength ? `${value.slice(0, limits.maxStringLength)}…` : value;
|
|
127
|
+
stats.hashedStrings++;
|
|
128
|
+
return hashString(value);
|
|
129
|
+
}
|
|
130
|
+
function countOf(value) {
|
|
131
|
+
if (Array.isArray(value)) return value.length;
|
|
132
|
+
if (isPlainObject(value)) return Object.keys(value).length;
|
|
133
|
+
}
|
|
134
|
+
function isPlainObject(value) {
|
|
135
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
136
|
+
const proto = Object.getPrototypeOf(value);
|
|
137
|
+
return proto === Object.prototype || proto === null;
|
|
138
|
+
}
|
|
139
|
+
function className(value) {
|
|
140
|
+
return value.constructor?.name ?? "unknown";
|
|
141
|
+
}
|
|
142
|
+
//#endregion
|
|
143
|
+
export { COUNTED_KEYS, SCHEMA_KEYS, SCHEMA_SUBTREE_KEYS, SUMMARISED_KEYS, hashString, isHashedString, redact };
|
|
144
|
+
|
|
145
|
+
//# sourceMappingURL=redact.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redact.js","names":[],"sources":["../src/redact.ts"],"sourcesContent":["import crypto from \"node:crypto\";\nimport { summarizeData, type DataSummary } from \"./data_summary\";\n\n/**\n * Structure-preserving redaction for anything a driver seam is handed.\n *\n * The definition a block model builds is recorded by *shape* rather than by a\n * hand-written digest per definition type. Keys, numbers and the small set of\n * strings that are schema survive verbatim; every other string is replaced by a\n * hash and a length. That keeps the record useful for diagnosis, keeps customer\n * data out of it by default rather than by enumeration, and works unchanged for\n * definition shapes this code has never seen — the V2 query API included.\n *\n * The default for an unrecognised string is to hash it. A new field can\n * therefore make a report less informative, but never make it leak.\n */\n\n/** Keys whose string value is schema, kept as written. */\nexport const SCHEMA_KEYS = new Set([\"type\", \"name\", \"valueType\", \"kind\", \"operator\", \"mode\"]);\n\n/** Keys under which every string is schema, at any depth (axis identity). */\nexport const SCHEMA_SUBTREE_KEYS = new Set([\"domain\", \"contextDomain\"]);\n\n/** Keys never descended into; summarised by counts instead. */\nexport const SUMMARISED_KEYS = new Set([\"data\", \"dataInfo\"]);\n\n/** Keys reduced to a cardinality, because their contents are values. */\nexport const COUNTED_KEYS = new Set([\"references\", \"parts\"]);\n\nexport type RedactionStats = {\n hashedStrings: number;\n truncatedArrays: number;\n omittedItems: number;\n depthCapped: number;\n opaqueObjects: number;\n budgetExhausted: boolean;\n};\n\nexport type RedactOptions = {\n maxDepth?: number;\n maxArrayItems?: number;\n maxStringLength?: number;\n /** Ceiling on emitted values, so one pathological definition cannot fill the log. */\n maxNodes?: number;\n};\n\nexport type HashedString = { h: string; n: number };\n\n/** Redacts a definition, returning the new value and what had to be elided. */\nexport function redact(\n value: unknown,\n options: RedactOptions = {},\n): { value: unknown; stats: RedactionStats } {\n const limits = {\n maxDepth: options.maxDepth ?? 32,\n maxArrayItems: options.maxArrayItems ?? 64,\n maxStringLength: options.maxStringLength ?? 128,\n maxNodes: options.maxNodes ?? 20_000,\n };\n const stats: RedactionStats = {\n hashedStrings: 0,\n truncatedArrays: 0,\n omittedItems: 0,\n depthCapped: 0,\n opaqueObjects: 0,\n budgetExhausted: false,\n };\n const state = { nodes: 0, seen: new WeakSet<object>() };\n return {\n value: walk(value, { key: undefined, schemaSubtree: false, depth: 0 }, limits, stats, state),\n stats,\n };\n}\n\n/** Stable short hash plus the original length. Never reversible to the value. */\nexport function hashString(value: string): HashedString {\n return {\n h: crypto.createHash(\"sha256\").update(value).digest(\"hex\").slice(0, 12),\n n: value.length,\n };\n}\n\n/** True for the object form produced in place of a redacted string. */\nexport function isHashedString(value: unknown): value is HashedString {\n return typeof value === \"object\" && value !== null && \"h\" in value && \"n\" in value;\n}\n\n// Internals\n\ntype Position = { key: string | undefined; schemaSubtree: boolean; depth: number };\ntype Limits = Required<RedactOptions>;\ntype State = { nodes: number; seen: WeakSet<object> };\n\nfunction walk(\n value: unknown,\n at: Position,\n limits: Limits,\n stats: RedactionStats,\n state: State,\n): unknown {\n if (state.nodes++ > limits.maxNodes) {\n stats.budgetExhausted = true;\n return { $budget: true };\n }\n\n if (value === null || value === undefined) return value ?? null;\n if (typeof value === \"bigint\") return Number(value);\n if (typeof value === \"number\" || typeof value === \"boolean\") return value;\n if (typeof value === \"string\") return redactString(value, at, limits, stats);\n if (typeof value !== \"object\") return { $type: typeof value };\n\n if (at.depth >= limits.maxDepth) {\n stats.depthCapped++;\n return { $depth: at.depth };\n }\n // A definition can carry live accessors and other class instances whose\n // internals reference each other; walking those is neither safe nor useful.\n if (state.seen.has(value)) return { $cycle: true };\n\n if (Array.isArray(value)) {\n state.seen.add(value);\n return walkArray(value, at, limits, stats, state);\n }\n if (!isPlainObject(value)) {\n stats.opaqueObjects++;\n return { $opaque: className(value) };\n }\n\n state.seen.add(value);\n const out: Record<string, unknown> = {};\n for (const [key, child] of Object.entries(value)) {\n if (SUMMARISED_KEYS.has(key)) {\n out[key] = summarizeData(child);\n continue;\n }\n if (COUNTED_KEYS.has(key)) {\n out[key] = { $count: countOf(child) };\n continue;\n }\n out[key] = walk(\n child,\n {\n key,\n schemaSubtree: at.schemaSubtree || SCHEMA_SUBTREE_KEYS.has(key),\n depth: at.depth + 1,\n },\n limits,\n stats,\n state,\n );\n }\n return out;\n}\n\nfunction walkArray(\n value: unknown[],\n at: Position,\n limits: Limits,\n stats: RedactionStats,\n state: State,\n): unknown[] {\n const kept = value\n .slice(0, limits.maxArrayItems)\n .map((item) =>\n walk(\n item,\n { key: at.key, schemaSubtree: at.schemaSubtree, depth: at.depth + 1 },\n limits,\n stats,\n state,\n ),\n );\n const omitted = value.length - kept.length;\n if (omitted <= 0) return kept;\n // Arrays stay arrays so the rules can still walk join entries; the loss is\n // recorded in the array itself rather than in a side channel.\n stats.truncatedArrays++;\n stats.omittedItems += omitted;\n return [...kept, { $omitted: omitted }];\n}\n\nfunction redactString(\n value: string,\n at: Position,\n limits: Limits,\n stats: RedactionStats,\n): string | HashedString {\n if (at.schemaSubtree || (at.key !== undefined && SCHEMA_KEYS.has(at.key))) {\n return value.length > limits.maxStringLength\n ? `${value.slice(0, limits.maxStringLength)}…`\n : value;\n }\n stats.hashedStrings++;\n return hashString(value);\n}\n\nfunction countOf(value: unknown): number | undefined {\n if (Array.isArray(value)) return value.length;\n if (isPlainObject(value)) return Object.keys(value).length;\n return undefined;\n}\n\nfunction isPlainObject(value: unknown): value is Record<string, unknown> {\n if (typeof value !== \"object\" || value === null || Array.isArray(value)) return false;\n const proto = Object.getPrototypeOf(value) as object | null;\n return proto === Object.prototype || proto === null;\n}\n\nfunction className(value: object): string {\n return (value as { constructor?: { name?: string } }).constructor?.name ?? \"unknown\";\n}\n\nexport type { DataSummary };\n"],"mappings":";;;;;;;;;;;;;;;;;AAkBA,MAAa,8BAAc,IAAI,IAAI;CAAC;CAAQ;CAAQ;CAAa;CAAQ;CAAY;AAAM,CAAC;;AAG5F,MAAa,sCAAsB,IAAI,IAAI,CAAC,UAAU,eAAe,CAAC;;AAGtE,MAAa,kCAAkB,IAAI,IAAI,CAAC,QAAQ,UAAU,CAAC;;AAG3D,MAAa,+BAAe,IAAI,IAAI,CAAC,cAAc,OAAO,CAAC;;AAsB3D,SAAgB,OACd,OACA,UAAyB,CAAC,GACiB;CAC3C,MAAM,SAAS;EACb,UAAU,QAAQ,YAAY;EAC9B,eAAe,QAAQ,iBAAiB;EACxC,iBAAiB,QAAQ,mBAAmB;EAC5C,UAAU,QAAQ,YAAY;CAChC;CACA,MAAM,QAAwB;EAC5B,eAAe;EACf,iBAAiB;EACjB,cAAc;EACd,aAAa;EACb,eAAe;EACf,iBAAiB;CACnB;CAEA,OAAO;EACL,OAAO,KAAK,OAAO;GAAE,KAAK,KAAA;GAAW,eAAe;GAAO,OAAO;EAAE,GAAG,QAAQ,OAAO;GAFxE,OAAO;GAAG,sBAAM,IAAI,QAAgB;EAEwC,CAAC;EAC3F;CACF;AACF;;AAGA,SAAgB,WAAW,OAA6B;CACtD,OAAO;EACL,GAAG,OAAO,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,EAAE;EACtE,GAAG,MAAM;CACX;AACF;;AAGA,SAAgB,eAAe,OAAuC;CACpE,OAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAO,SAAS,OAAO;AAC/E;AAQA,SAAS,KACP,OACA,IACA,QACA,OACA,OACS;CACT,IAAI,MAAM,UAAU,OAAO,UAAU;EACnC,MAAM,kBAAkB;EACxB,OAAO,EAAE,SAAS,KAAK;CACzB;CAEA,IAAI,UAAU,QAAQ,UAAU,KAAA,GAAW,OAAO,SAAS;CAC3D,IAAI,OAAO,UAAU,UAAU,OAAO,OAAO,KAAK;CAClD,IAAI,OAAO,UAAU,YAAY,OAAO,UAAU,WAAW,OAAO;CACpE,IAAI,OAAO,UAAU,UAAU,OAAO,aAAa,OAAO,IAAI,QAAQ,KAAK;CAC3E,IAAI,OAAO,UAAU,UAAU,OAAO,EAAE,OAAO,OAAO,MAAM;CAE5D,IAAI,GAAG,SAAS,OAAO,UAAU;EAC/B,MAAM;EACN,OAAO,EAAE,QAAQ,GAAG,MAAM;CAC5B;CAGA,IAAI,MAAM,KAAK,IAAI,KAAK,GAAG,OAAO,EAAE,QAAQ,KAAK;CAEjD,IAAI,MAAM,QAAQ,KAAK,GAAG;EACxB,MAAM,KAAK,IAAI,KAAK;EACpB,OAAO,UAAU,OAAO,IAAI,QAAQ,OAAO,KAAK;CAClD;CACA,IAAI,CAAC,cAAc,KAAK,GAAG;EACzB,MAAM;EACN,OAAO,EAAE,SAAS,UAAU,KAAK,EAAE;CACrC;CAEA,MAAM,KAAK,IAAI,KAAK;CACpB,MAAM,MAA+B,CAAC;CACtC,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,KAAK,GAAG;EAChD,IAAI,gBAAgB,IAAI,GAAG,GAAG;GAC5B,IAAI,OAAO,cAAc,KAAK;GAC9B;EACF;EACA,IAAI,aAAa,IAAI,GAAG,GAAG;GACzB,IAAI,OAAO,EAAE,QAAQ,QAAQ,KAAK,EAAE;GACpC;EACF;EACA,IAAI,OAAO,KACT,OACA;GACE;GACA,eAAe,GAAG,iBAAiB,oBAAoB,IAAI,GAAG;GAC9D,OAAO,GAAG,QAAQ;EACpB,GACA,QACA,OACA,KACF;CACF;CACA,OAAO;AACT;AAEA,SAAS,UACP,OACA,IACA,QACA,OACA,OACW;CACX,MAAM,OAAO,MACV,MAAM,GAAG,OAAO,aAAa,CAAC,CAC9B,KAAK,SACJ,KACE,MACA;EAAE,KAAK,GAAG;EAAK,eAAe,GAAG;EAAe,OAAO,GAAG,QAAQ;CAAE,GACpE,QACA,OACA,KACF,CACF;CACF,MAAM,UAAU,MAAM,SAAS,KAAK;CACpC,IAAI,WAAW,GAAG,OAAO;CAGzB,MAAM;CACN,MAAM,gBAAgB;CACtB,OAAO,CAAC,GAAG,MAAM,EAAE,UAAU,QAAQ,CAAC;AACxC;AAEA,SAAS,aACP,OACA,IACA,QACA,OACuB;CACvB,IAAI,GAAG,iBAAkB,GAAG,QAAQ,KAAA,KAAa,YAAY,IAAI,GAAG,GAAG,GACrE,OAAO,MAAM,SAAS,OAAO,kBACzB,GAAG,MAAM,MAAM,GAAG,OAAO,eAAe,EAAE,KAC1C;CAEN,MAAM;CACN,OAAO,WAAW,KAAK;AACzB;AAEA,SAAS,QAAQ,OAAoC;CACnD,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO,MAAM;CACvC,IAAI,cAAc,KAAK,GAAG,OAAO,OAAO,KAAK,KAAK,CAAC,CAAC;AAEtD;AAEA,SAAS,cAAc,OAAkD;CACvE,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG,OAAO;CAChF,MAAM,QAAQ,OAAO,eAAe,KAAK;CACzC,OAAO,UAAU,OAAO,aAAa,UAAU;AACjD;AAEA,SAAS,UAAU,OAAuB;CACxC,OAAQ,MAA8C,aAAa,QAAQ;AAC7E"}
|
package/dist/report.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { SessionAnalysis } from "./analyze.js";
|
|
2
|
+
//#region src/report.d.ts
|
|
3
|
+
/** Renders an analysis as the markdown report a developer reads. */
|
|
4
|
+
export declare function renderReport(analysis: SessionAnalysis): string;
|
|
5
|
+
//#endregion
|
|
6
|
+
//# sourceMappingURL=report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.d.ts","names":[],"sources":["../src/report.ts"],"mappings":";;;wBAOgB,aAAa,UAAU"}
|