@milaboratories/pl-flight-recorder 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -0
- package/dist/analyze.d.ts +133 -0
- package/dist/analyze.d.ts.map +1 -0
- package/dist/analyze.js +525 -0
- package/dist/analyze.js.map +1 -0
- package/dist/data_summary.d.ts +28 -0
- package/dist/data_summary.d.ts.map +1 -0
- package/dist/data_summary.js +73 -0
- package/dist/data_summary.js.map +1 -0
- package/dist/digest.d.ts +28 -0
- package/dist/digest.d.ts.map +1 -0
- package/dist/digest.js +55 -0
- package/dist/digest.js.map +1 -0
- package/dist/events.d.ts +89 -0
- package/dist/events.d.ts.map +1 -0
- package/dist/events.js +12 -0
- package/dist/events.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +13 -0
- package/dist/instrument.d.ts +82 -0
- package/dist/instrument.d.ts.map +1 -0
- package/dist/instrument.js +313 -0
- package/dist/instrument.js.map +1 -0
- package/dist/recorder.d.ts +82 -0
- package/dist/recorder.d.ts.map +1 -0
- package/dist/recorder.js +293 -0
- package/dist/recorder.js.map +1 -0
- package/dist/redact.d.ts +53 -0
- package/dist/redact.d.ts.map +1 -0
- package/dist/redact.js +145 -0
- package/dist/redact.js.map +1 -0
- package/dist/report.d.ts +6 -0
- package/dist/report.d.ts.map +1 -0
- package/dist/report.js +377 -0
- package/dist/report.js.map +1 -0
- package/dist/rules.d.ts +73 -0
- package/dist/rules.d.ts.map +1 -0
- package/dist/rules.js +245 -0
- package/dist/rules.js.map +1 -0
- package/dist/sampler.d.ts +22 -0
- package/dist/sampler.d.ts.map +1 -0
- package/dist/sampler.js +33 -0
- package/dist/sampler.js.map +1 -0
- package/dist/sampler_thread.d.ts +1 -0
- package/dist/sampler_thread.js +41 -0
- package/dist/sampler_thread.js.map +1 -0
- package/dist/session.d.ts +40 -0
- package/dist/session.d.ts.map +1 -0
- package/dist/session.js +50 -0
- package/dist/session.js.map +1 -0
- package/dist/supervisor.d.ts +58 -0
- package/dist/supervisor.d.ts.map +1 -0
- package/dist/supervisor.js +108 -0
- package/dist/supervisor.js.map +1 -0
- package/package.json +43 -0
- package/src/analyze.test.ts +539 -0
- package/src/analyze.ts +795 -0
- package/src/data_summary.ts +110 -0
- package/src/digest.ts +49 -0
- package/src/events.ts +102 -0
- package/src/index.ts +104 -0
- package/src/instrument.ts +442 -0
- package/src/recorder.ts +397 -0
- package/src/redact.test.ts +155 -0
- package/src/redact.ts +213 -0
- package/src/report.ts +512 -0
- package/src/rules.test.ts +182 -0
- package/src/rules.ts +383 -0
- package/src/sampler.ts +40 -0
- package/src/sampler_thread.ts +45 -0
- package/src/session.ts +69 -0
- package/src/supervisor.ts +150 -0
package/dist/rules.js
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
//#region src/rules.ts
|
|
2
|
+
/** Joins that keep only keys present in every entry; an entry missing part of the key fans out. */
|
|
3
|
+
const INTERSECT_JOINS = /* @__PURE__ */ new Set(["inner", "innerJoin"]);
|
|
4
|
+
/** Joins that keep keys present in any entry, filling the rest with nulls. */
|
|
5
|
+
const UNION_JOINS = /* @__PURE__ */ new Set(["full", "fullJoin"]);
|
|
6
|
+
/** Joins driven by one side: the primary or linker decides which keys exist. */
|
|
7
|
+
const DRIVEN_JOINS = /* @__PURE__ */ new Set([
|
|
8
|
+
"outer",
|
|
9
|
+
"outerJoin",
|
|
10
|
+
"linkerJoin"
|
|
11
|
+
]);
|
|
12
|
+
/** Structural findings for a recorded definition, most specific first. */
|
|
13
|
+
function structuralFindings(def) {
|
|
14
|
+
const findings = [];
|
|
15
|
+
visit(def, "root", (node, path) => collect(node, path, findings));
|
|
16
|
+
return findings;
|
|
17
|
+
}
|
|
18
|
+
/** Shape of every join node in a definition, outermost first. */
|
|
19
|
+
function joinShapes(def) {
|
|
20
|
+
const shapes = [];
|
|
21
|
+
visit(def, "root", (node, path) => shapes.push(shapeOf(node, path)));
|
|
22
|
+
return shapes;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Largest input row count the definition declares, used to judge how much a
|
|
26
|
+
* join amplified. Unknown when no workflow recorded chunk statistics.
|
|
27
|
+
*/
|
|
28
|
+
function inputRowsMax(def) {
|
|
29
|
+
const rows = childRowCounts(def);
|
|
30
|
+
return rows.length > 0 ? Math.max(...rows) : estimateRows(def);
|
|
31
|
+
}
|
|
32
|
+
/** Canonical axis identity, matching how join keys are formed. */
|
|
33
|
+
function axisKey(axis) {
|
|
34
|
+
return `${axis.type}|${axis.name}|${canonicalDomain(axis.domain)}`;
|
|
35
|
+
}
|
|
36
|
+
/** Axis identity ignoring domain, used to spot near-miss axes that fail to join. */
|
|
37
|
+
function axisNameKey(axis) {
|
|
38
|
+
return `${axis.type}|${axis.name}`;
|
|
39
|
+
}
|
|
40
|
+
/** True when a node is a join, by its discriminator. */
|
|
41
|
+
function isJoinNode(node) {
|
|
42
|
+
const type = discriminator(node);
|
|
43
|
+
return type !== void 0 && (INTERSECT_JOINS.has(type) || UNION_JOINS.has(type) || DRIVEN_JOINS.has(type));
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* A join's children, by position. The V2 API wraps each child in `{ entry }`;
|
|
47
|
+
* that wrapper is left in place because every read here descends through it.
|
|
48
|
+
*/
|
|
49
|
+
function joinChildren(node) {
|
|
50
|
+
const record = asRecord(node);
|
|
51
|
+
if (!record) return [];
|
|
52
|
+
if (Array.isArray(record.entries)) return record.entries;
|
|
53
|
+
return [record.primary ?? record.linker, ...toArray(record.secondary)].filter((child) => child !== void 0 && child !== null);
|
|
54
|
+
}
|
|
55
|
+
/** Axis descriptors anywhere beneath a node, deduplicated by identity. */
|
|
56
|
+
function axesUnder(node) {
|
|
57
|
+
const out = /* @__PURE__ */ new Map();
|
|
58
|
+
gatherAxes(node, out, 0);
|
|
59
|
+
return [...out.values()];
|
|
60
|
+
}
|
|
61
|
+
const MAX_WALK_DEPTH = 40;
|
|
62
|
+
function visit(node, path, onJoin, depth = 0) {
|
|
63
|
+
if (depth > MAX_WALK_DEPTH || !isTraversable(node)) return;
|
|
64
|
+
if (isJoinNode(node)) {
|
|
65
|
+
onJoin(node, path);
|
|
66
|
+
const join = discriminator(node) ?? "join";
|
|
67
|
+
for (const [index, child] of joinChildren(node).entries()) visit(child, `${path}/${join}[${index}]`, onJoin, depth + 1);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
for (const child of childValues(node)) visit(child, path, onJoin, depth + 1);
|
|
71
|
+
}
|
|
72
|
+
function shapeOf(node, path) {
|
|
73
|
+
const join = discriminator(node) ?? "join";
|
|
74
|
+
const children = joinChildren(node);
|
|
75
|
+
const keySets = children.map((child) => new Set(axesUnder(child).map(axisKey)));
|
|
76
|
+
const union = /* @__PURE__ */ new Set();
|
|
77
|
+
for (const set of keySets) for (const key of set) union.add(key);
|
|
78
|
+
const disjointPairs = [];
|
|
79
|
+
for (let i = 0; i < keySets.length; i++) for (let j = i + 1; j < keySets.length; j++) {
|
|
80
|
+
if (keySets[i].size === 0 || keySets[j].size === 0) continue;
|
|
81
|
+
if ([...keySets[i]].some((key) => keySets[j].has(key))) continue;
|
|
82
|
+
disjointPairs.push([i, j]);
|
|
83
|
+
}
|
|
84
|
+
const rows = children.map(estimateRows);
|
|
85
|
+
const known = rows.filter((value) => typeof value === "number");
|
|
86
|
+
return {
|
|
87
|
+
join,
|
|
88
|
+
path,
|
|
89
|
+
childCount: children.length,
|
|
90
|
+
axisUnion: [...union],
|
|
91
|
+
sharedAxes: [...union].filter((key) => keySets.every((set) => set.has(key))),
|
|
92
|
+
disjointPairs,
|
|
93
|
+
inputRowsMax: known.length > 0 ? Math.max(...known) : void 0,
|
|
94
|
+
rowsUpperBound: known.length === rows.length && known.length > 0 ? product(known) : void 0
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
function collect(node, path, findings) {
|
|
98
|
+
const shape = shapeOf(node, path);
|
|
99
|
+
const children = joinChildren(node);
|
|
100
|
+
if (shape.disjointPairs.length > 0) findings.push({
|
|
101
|
+
rule: "cross-join",
|
|
102
|
+
severity: "critical",
|
|
103
|
+
path,
|
|
104
|
+
join: shape.join,
|
|
105
|
+
detail: `join siblings share no axis: pairs ${JSON.stringify(shape.disjointPairs)}`,
|
|
106
|
+
rowsUpperBound: shape.rowsUpperBound
|
|
107
|
+
});
|
|
108
|
+
for (const nearMiss of nearMissAxes(children)) findings.push({
|
|
109
|
+
rule: "axis-domain-mismatch",
|
|
110
|
+
severity: "high",
|
|
111
|
+
path,
|
|
112
|
+
join: shape.join,
|
|
113
|
+
detail: `axis ${nearMiss.axis} appears with ${nearMiss.domains.length} different domains`,
|
|
114
|
+
domains: nearMiss.domains
|
|
115
|
+
});
|
|
116
|
+
if (!INTERSECT_JOINS.has(shape.join) || shape.disjointPairs.length > 0) return;
|
|
117
|
+
const missing = children.map((child, index) => {
|
|
118
|
+
const own = new Set(axesUnder(child).map(axisKey));
|
|
119
|
+
return {
|
|
120
|
+
index,
|
|
121
|
+
missing: shape.axisUnion.filter((key) => !own.has(key))
|
|
122
|
+
};
|
|
123
|
+
}).filter((entry) => entry.missing.length > 0);
|
|
124
|
+
if (missing.length === 0) return;
|
|
125
|
+
findings.push({
|
|
126
|
+
rule: "partial-key-fan-out",
|
|
127
|
+
severity: "medium",
|
|
128
|
+
path,
|
|
129
|
+
join: shape.join,
|
|
130
|
+
detail: `${missing.length} sibling(s) lack part of the node's axis union and get replicated`,
|
|
131
|
+
missing
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
function nearMissAxes(children) {
|
|
135
|
+
const byName = /* @__PURE__ */ new Map();
|
|
136
|
+
for (const [index, child] of children.entries()) for (const axis of axesUnder(child)) {
|
|
137
|
+
const nameKey = axisNameKey(axis);
|
|
138
|
+
let perDomain = byName.get(nameKey);
|
|
139
|
+
if (!perDomain) byName.set(nameKey, perDomain = /* @__PURE__ */ new Map());
|
|
140
|
+
const domainKey = canonicalDomain(axis.domain);
|
|
141
|
+
let indices = perDomain.get(domainKey);
|
|
142
|
+
if (!indices) perDomain.set(domainKey, indices = /* @__PURE__ */ new Set());
|
|
143
|
+
indices.add(index);
|
|
144
|
+
}
|
|
145
|
+
const out = [];
|
|
146
|
+
for (const [axis, perDomain] of byName) {
|
|
147
|
+
if (perDomain.size < 2) continue;
|
|
148
|
+
out.push({
|
|
149
|
+
axis,
|
|
150
|
+
domains: [...perDomain.entries()].map(([domain, indices]) => ({
|
|
151
|
+
domain: domain || "(none)",
|
|
152
|
+
children: [...indices]
|
|
153
|
+
}))
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
return out;
|
|
157
|
+
}
|
|
158
|
+
function estimateRows(node, depth = 0) {
|
|
159
|
+
if (depth > MAX_WALK_DEPTH || !isTraversable(node)) return void 0;
|
|
160
|
+
if (isJoinNode(node)) {
|
|
161
|
+
const join = discriminator(node) ?? "";
|
|
162
|
+
const rows = joinChildren(node).map((child) => estimateRows(child, depth + 1)).filter((value) => typeof value === "number");
|
|
163
|
+
if (rows.length === 0) return void 0;
|
|
164
|
+
return INTERSECT_JOINS.has(join) ? Math.max(...rows) : sum(rows);
|
|
165
|
+
}
|
|
166
|
+
const own = ownRows(node);
|
|
167
|
+
if (own !== void 0) return own;
|
|
168
|
+
const rows = childValues(node).map((child) => estimateRows(child, depth + 1)).filter((value) => typeof value === "number");
|
|
169
|
+
return rows.length > 0 ? Math.max(...rows) : void 0;
|
|
170
|
+
}
|
|
171
|
+
function childRowCounts(def) {
|
|
172
|
+
return joinShapes(def).map((shape) => shape.inputRowsMax).filter((value) => typeof value === "number");
|
|
173
|
+
}
|
|
174
|
+
function ownRows(node) {
|
|
175
|
+
const record = asRecord(node);
|
|
176
|
+
if (!record) return void 0;
|
|
177
|
+
for (const key of ["data", "dataInfo"]) {
|
|
178
|
+
const summary = record[key];
|
|
179
|
+
if (!summary || typeof summary !== "object") continue;
|
|
180
|
+
if (typeof summary.rows === "number") return summary.rows;
|
|
181
|
+
if (typeof summary.entries === "number") return summary.entries;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
function gatherAxes(node, out, depth) {
|
|
185
|
+
if (depth > MAX_WALK_DEPTH || !isTraversable(node)) return;
|
|
186
|
+
const record = asRecord(node);
|
|
187
|
+
if (record) for (const key of ["axesSpec", "axes"]) {
|
|
188
|
+
const value = record[key];
|
|
189
|
+
if (!Array.isArray(value)) continue;
|
|
190
|
+
for (const item of value) {
|
|
191
|
+
const axis = asAxis(item);
|
|
192
|
+
if (axis) out.set(axisKey(axis), axis);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
for (const child of childValues(node)) gatherAxes(child, out, depth + 1);
|
|
196
|
+
}
|
|
197
|
+
function asAxis(value) {
|
|
198
|
+
const record = asRecord(value);
|
|
199
|
+
if (!record) return void 0;
|
|
200
|
+
const { name, type, domain } = record;
|
|
201
|
+
if (typeof name !== "string" || typeof type !== "string") return void 0;
|
|
202
|
+
return {
|
|
203
|
+
name,
|
|
204
|
+
type,
|
|
205
|
+
domain: plainStringMap(domain)
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
function plainStringMap(value) {
|
|
209
|
+
const record = asRecord(value);
|
|
210
|
+
if (!record) return void 0;
|
|
211
|
+
const out = {};
|
|
212
|
+
for (const [key, item] of Object.entries(record)) if (typeof item === "string") out[key] = item;
|
|
213
|
+
return Object.keys(out).length > 0 ? out : void 0;
|
|
214
|
+
}
|
|
215
|
+
function canonicalDomain(domain) {
|
|
216
|
+
return Object.entries(domain ?? {}).sort(([lhs], [rhs]) => lhs < rhs ? -1 : 1).map(([key, value]) => `${key}=${value}`).join(",");
|
|
217
|
+
}
|
|
218
|
+
function discriminator(node) {
|
|
219
|
+
const type = asRecord(node)?.type;
|
|
220
|
+
return typeof type === "string" ? type : void 0;
|
|
221
|
+
}
|
|
222
|
+
function childValues(node) {
|
|
223
|
+
if (Array.isArray(node)) return node;
|
|
224
|
+
const record = asRecord(node);
|
|
225
|
+
return record ? Object.values(record) : [];
|
|
226
|
+
}
|
|
227
|
+
function asRecord(value) {
|
|
228
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
|
|
229
|
+
}
|
|
230
|
+
function isTraversable(value) {
|
|
231
|
+
return typeof value === "object" && value !== null;
|
|
232
|
+
}
|
|
233
|
+
function toArray(value) {
|
|
234
|
+
return Array.isArray(value) ? value : value === void 0 || value === null ? [] : [value];
|
|
235
|
+
}
|
|
236
|
+
function sum(values) {
|
|
237
|
+
return values.reduce((acc, value) => acc + value, 0);
|
|
238
|
+
}
|
|
239
|
+
function product(values) {
|
|
240
|
+
return values.reduce((acc, value) => acc * value, 1);
|
|
241
|
+
}
|
|
242
|
+
//#endregion
|
|
243
|
+
export { axesUnder, axisKey, axisNameKey, inputRowsMax, isJoinNode, joinChildren, joinShapes, structuralFindings };
|
|
244
|
+
|
|
245
|
+
//# sourceMappingURL=rules.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rules.js","names":[],"sources":["../src/rules.ts"],"sourcesContent":["import type {\n JoinEntry,\n SpecQueryLinkerJoin,\n SpecQueryOuterJoin,\n SpecQuerySymmetricJoin,\n} from \"@milaboratories/pl-model-common\";\nimport type { DataSummary } from \"./data_summary\";\n\n/**\n * Structural faults in a join, read from the recorded shape of its definition.\n *\n * Two faults make a join's output size unbounded and both are visible before any\n * data is touched: siblings that share no axis at all, which is a cartesian\n * product, and siblings whose axes agree on name and type but disagree on\n * domain, where the join key silently fails to match.\n *\n * These run in the analyzer rather than at record time. Nothing is computed on\n * the hot path, the thresholds and the rules can be revised against logs that\n * already exist, and one implementation covers every definition shape: the join\n * nodes are recognised by their discriminator and their children by position, so\n * both the original tree API and the V2 query API are read by the same walk.\n */\n\nexport type FindingSeverity = \"critical\" | \"high\" | \"medium\" | \"low\";\n\nexport type StructuralFinding = {\n rule: \"cross-join\" | \"axis-domain-mismatch\" | \"partial-key-fan-out\";\n severity: FindingSeverity;\n /** Position in the definition, e.g. `root/innerJoin[1]`. */\n path: string;\n join: string;\n detail: string;\n rowsUpperBound?: number;\n domains?: { domain: string; children: number[] }[];\n missing?: { index: number; missing: string[] }[];\n};\n\nexport type AxisDescriptor = {\n name: string;\n type: string;\n domain?: Record<string, string>;\n};\n\nexport type JoinShape = {\n join: string;\n path: string;\n childCount: number;\n axisUnion: string[];\n sharedAxes: string[];\n disjointPairs: [number, number][];\n inputRowsMax?: number;\n /** Loose but true: no join of these inputs can exceed the product of their rows. */\n rowsUpperBound?: number;\n};\n\n// The discriminators are pinned to the model's own literal types, so renaming a\n// join kind in pl-model-common fails this build instead of silently disabling a\n// rule. The model exports the names only as types, never as runtime constants.\ntype TreeJoinType = Extract<\n JoinEntry<unknown>,\n { entries: unknown } | { primary: unknown }\n>[\"type\"];\ntype QueryJoinType = (SpecQuerySymmetricJoin | SpecQueryOuterJoin | SpecQueryLinkerJoin)[\"type\"];\n\n/** Joins that keep only keys present in every entry; an entry missing part of the key fans out. */\nconst INTERSECT_JOINS: ReadonlySet<string> = new Set([\"inner\", \"innerJoin\"] satisfies (\n | TreeJoinType\n | QueryJoinType\n)[]);\n/** Joins that keep keys present in any entry, filling the rest with nulls. */\nconst UNION_JOINS: ReadonlySet<string> = new Set([\"full\", \"fullJoin\"] satisfies (\n | TreeJoinType\n | QueryJoinType\n)[]);\n/** Joins driven by one side: the primary or linker decides which keys exist. */\nconst DRIVEN_JOINS: ReadonlySet<string> = new Set([\"outer\", \"outerJoin\", \"linkerJoin\"] satisfies (\n | TreeJoinType\n | QueryJoinType\n)[]);\n\n/** Structural findings for a recorded definition, most specific first. */\nexport function structuralFindings(def: unknown): StructuralFinding[] {\n const findings: StructuralFinding[] = [];\n visit(def, \"root\", (node, path) => collect(node, path, findings));\n return findings;\n}\n\n/** Shape of every join node in a definition, outermost first. */\nexport function joinShapes(def: unknown): JoinShape[] {\n const shapes: JoinShape[] = [];\n visit(def, \"root\", (node, path) => shapes.push(shapeOf(node, path)));\n return shapes;\n}\n\n/**\n * Largest input row count the definition declares, used to judge how much a\n * join amplified. Unknown when no workflow recorded chunk statistics.\n */\nexport function inputRowsMax(def: unknown): number | undefined {\n const rows = childRowCounts(def);\n return rows.length > 0 ? Math.max(...rows) : estimateRows(def);\n}\n\n/** Canonical axis identity, matching how join keys are formed. */\nexport function axisKey(axis: AxisDescriptor): string {\n return `${axis.type}|${axis.name}|${canonicalDomain(axis.domain)}`;\n}\n\n/** Axis identity ignoring domain, used to spot near-miss axes that fail to join. */\nexport function axisNameKey(axis: AxisDescriptor): string {\n return `${axis.type}|${axis.name}`;\n}\n\n/** True when a node is a join, by its discriminator. */\nexport function isJoinNode(node: unknown): boolean {\n const type = discriminator(node);\n return (\n type !== undefined &&\n (INTERSECT_JOINS.has(type) || UNION_JOINS.has(type) || DRIVEN_JOINS.has(type))\n );\n}\n\n/**\n * A join's children, by position. The V2 API wraps each child in `{ entry }`;\n * that wrapper is left in place because every read here descends through it.\n */\nexport function joinChildren(node: unknown): unknown[] {\n const record = asRecord(node);\n if (!record) return [];\n if (Array.isArray(record.entries)) return record.entries;\n const driven = [record.primary ?? record.linker, ...toArray(record.secondary)];\n return driven.filter((child) => child !== undefined && child !== null);\n}\n\n/** Axis descriptors anywhere beneath a node, deduplicated by identity. */\nexport function axesUnder(node: unknown): AxisDescriptor[] {\n const out = new Map<string, AxisDescriptor>();\n gatherAxes(node, out, 0);\n return [...out.values()];\n}\n\n// Internals\n\nconst MAX_WALK_DEPTH = 40;\n\nfunction visit(\n node: unknown,\n path: string,\n onJoin: (node: unknown, path: string) => void,\n depth = 0,\n): void {\n if (depth > MAX_WALK_DEPTH || !isTraversable(node)) return;\n if (isJoinNode(node)) {\n onJoin(node, path);\n const join = discriminator(node) ?? \"join\";\n for (const [index, child] of joinChildren(node).entries()) {\n visit(child, `${path}/${join}[${index}]`, onJoin, depth + 1);\n }\n return;\n }\n for (const child of childValues(node)) visit(child, path, onJoin, depth + 1);\n}\n\nfunction shapeOf(node: unknown, path: string): JoinShape {\n const join = discriminator(node) ?? \"join\";\n const children = joinChildren(node);\n const keySets = children.map((child) => new Set(axesUnder(child).map(axisKey)));\n\n const union = new Set<string>();\n for (const set of keySets) for (const key of set) union.add(key);\n\n const disjointPairs: [number, number][] = [];\n for (let i = 0; i < keySets.length; i++) {\n for (let j = i + 1; j < keySets.length; j++) {\n if (keySets[i].size === 0 || keySets[j].size === 0) continue;\n if ([...keySets[i]].some((key) => keySets[j].has(key))) continue;\n disjointPairs.push([i, j]);\n }\n }\n\n const rows = children.map(estimateRows);\n const known = rows.filter((value): value is number => typeof value === \"number\");\n return {\n join,\n path,\n childCount: children.length,\n axisUnion: [...union],\n sharedAxes: [...union].filter((key) => keySets.every((set) => set.has(key))),\n disjointPairs,\n inputRowsMax: known.length > 0 ? Math.max(...known) : undefined,\n rowsUpperBound: known.length === rows.length && known.length > 0 ? product(known) : undefined,\n };\n}\n\nfunction collect(node: unknown, path: string, findings: StructuralFinding[]): void {\n const shape = shapeOf(node, path);\n const children = joinChildren(node);\n\n if (shape.disjointPairs.length > 0) {\n findings.push({\n rule: \"cross-join\",\n severity: \"critical\",\n path,\n join: shape.join,\n detail: `join siblings share no axis: pairs ${JSON.stringify(shape.disjointPairs)}`,\n rowsUpperBound: shape.rowsUpperBound,\n });\n }\n\n for (const nearMiss of nearMissAxes(children)) {\n findings.push({\n rule: \"axis-domain-mismatch\",\n severity: \"high\",\n path,\n join: shape.join,\n detail: `axis ${nearMiss.axis} appears with ${nearMiss.domains.length} different domains`,\n domains: nearMiss.domains,\n });\n }\n\n // Fan-out is worth reporting only where the node still has a working join key\n // and its entries are peers; on a cartesian node it restates the cross-join,\n // and on a driven join a narrower secondary is the intended behaviour.\n if (!INTERSECT_JOINS.has(shape.join) || shape.disjointPairs.length > 0) return;\n const missing = children\n .map((child, index) => {\n const own = new Set(axesUnder(child).map(axisKey));\n return { index, missing: shape.axisUnion.filter((key) => !own.has(key)) };\n })\n .filter((entry) => entry.missing.length > 0);\n if (missing.length === 0) return;\n findings.push({\n rule: \"partial-key-fan-out\",\n severity: \"medium\",\n path,\n join: shape.join,\n detail: `${missing.length} sibling(s) lack part of the node's axis union and get replicated`,\n missing,\n });\n}\n\n// Axes agreeing on name and type but disagreeing on domain never match as a\n// join key, which turns an intended join into a product or an empty result.\nfunction nearMissAxes(\n children: unknown[],\n): { axis: string; domains: { domain: string; children: number[] }[] }[] {\n const byName = new Map<string, Map<string, Set<number>>>();\n for (const [index, child] of children.entries()) {\n for (const axis of axesUnder(child)) {\n const nameKey = axisNameKey(axis);\n let perDomain = byName.get(nameKey);\n if (!perDomain) byName.set(nameKey, (perDomain = new Map()));\n const domainKey = canonicalDomain(axis.domain);\n let indices = perDomain.get(domainKey);\n if (!indices) perDomain.set(domainKey, (indices = new Set()));\n indices.add(index);\n }\n }\n const out: { axis: string; domains: { domain: string; children: number[] }[] }[] = [];\n for (const [axis, perDomain] of byName) {\n if (perDomain.size < 2) continue;\n out.push({\n axis,\n domains: [...perDomain.entries()].map(([domain, indices]) => ({\n domain: domain || \"(none)\",\n children: [...indices],\n })),\n });\n }\n return out;\n}\n\nfunction estimateRows(node: unknown, depth = 0): number | undefined {\n if (depth > MAX_WALK_DEPTH || !isTraversable(node)) return undefined;\n if (isJoinNode(node)) {\n const join = discriminator(node) ?? \"\";\n const rows = joinChildren(node)\n .map((child) => estimateRows(child, depth + 1))\n .filter((value): value is number => typeof value === \"number\");\n if (rows.length === 0) return undefined;\n // An intersection cannot exceed its largest input; a union adds up.\n return INTERSECT_JOINS.has(join) ? Math.max(...rows) : sum(rows);\n }\n const own = ownRows(node);\n if (own !== undefined) return own;\n const rows = childValues(node)\n .map((child) => estimateRows(child, depth + 1))\n .filter((value): value is number => typeof value === \"number\");\n return rows.length > 0 ? Math.max(...rows) : undefined;\n}\n\nfunction childRowCounts(def: unknown): number[] {\n const shapes = joinShapes(def);\n return shapes\n .map((shape) => shape.inputRowsMax)\n .filter((value): value is number => typeof value === \"number\");\n}\n\nfunction ownRows(node: unknown): number | undefined {\n const record = asRecord(node);\n if (!record) return undefined;\n for (const key of [\"data\", \"dataInfo\"]) {\n const summary = record[key] as DataSummary | undefined;\n if (!summary || typeof summary !== \"object\") continue;\n if (typeof summary.rows === \"number\") return summary.rows;\n if (typeof summary.entries === \"number\") return summary.entries;\n }\n return undefined;\n}\n\nfunction gatherAxes(node: unknown, out: Map<string, AxisDescriptor>, depth: number): void {\n if (depth > MAX_WALK_DEPTH || !isTraversable(node)) return;\n const record = asRecord(node);\n if (record) {\n for (const key of [\"axesSpec\", \"axes\"]) {\n const value = record[key];\n if (!Array.isArray(value)) continue;\n for (const item of value) {\n const axis = asAxis(item);\n if (axis) out.set(axisKey(axis), axis);\n }\n }\n }\n for (const child of childValues(node)) gatherAxes(child, out, depth + 1);\n}\n\nfunction asAxis(value: unknown): AxisDescriptor | undefined {\n const record = asRecord(value);\n if (!record) return undefined;\n const { name, type, domain } = record;\n if (typeof name !== \"string\" || typeof type !== \"string\") return undefined;\n return { name, type, domain: plainStringMap(domain) };\n}\n\nfunction plainStringMap(value: unknown): Record<string, string> | undefined {\n const record = asRecord(value);\n if (!record) return undefined;\n const out: Record<string, string> = {};\n for (const [key, item] of Object.entries(record)) {\n if (typeof item === \"string\") out[key] = item;\n }\n return Object.keys(out).length > 0 ? out : undefined;\n}\n\nfunction canonicalDomain(domain: Record<string, string> | undefined): string {\n return Object.entries(domain ?? {})\n .sort(([lhs], [rhs]) => (lhs < rhs ? -1 : 1))\n .map(([key, value]) => `${key}=${value}`)\n .join(\",\");\n}\n\nfunction discriminator(node: unknown): string | undefined {\n const type = asRecord(node)?.type;\n return typeof type === \"string\" ? type : undefined;\n}\n\nfunction childValues(node: unknown): unknown[] {\n if (Array.isArray(node)) return node;\n const record = asRecord(node);\n return record ? Object.values(record) : [];\n}\n\nfunction asRecord(value: unknown): Record<string, unknown> | undefined {\n return typeof value === \"object\" && value !== null && !Array.isArray(value)\n ? (value as Record<string, unknown>)\n : undefined;\n}\n\nfunction isTraversable(value: unknown): boolean {\n return typeof value === \"object\" && value !== null;\n}\n\nfunction toArray(value: unknown): unknown[] {\n return Array.isArray(value) ? value : value === undefined || value === null ? [] : [value];\n}\n\nfunction sum(values: number[]): number {\n return values.reduce((acc, value) => acc + value, 0);\n}\n\nfunction product(values: number[]): number {\n return values.reduce((acc, value) => acc * value, 1);\n}\n"],"mappings":";;AAiEA,MAAM,kCAAuC,IAAI,IAAI,CAAC,SAAS,WAAW,CAGvE;;AAEH,MAAM,8BAAmC,IAAI,IAAI,CAAC,QAAQ,UAAU,CAGjE;;AAEH,MAAM,+BAAoC,IAAI,IAAI;CAAC;CAAS;CAAa;AAAY,CAGlF;;AAGH,SAAgB,mBAAmB,KAAmC;CACpE,MAAM,WAAgC,CAAC;CACvC,MAAM,KAAK,SAAS,MAAM,SAAS,QAAQ,MAAM,MAAM,QAAQ,CAAC;CAChE,OAAO;AACT;;AAGA,SAAgB,WAAW,KAA2B;CACpD,MAAM,SAAsB,CAAC;CAC7B,MAAM,KAAK,SAAS,MAAM,SAAS,OAAO,KAAK,QAAQ,MAAM,IAAI,CAAC,CAAC;CACnE,OAAO;AACT;;;;;AAMA,SAAgB,aAAa,KAAkC;CAC7D,MAAM,OAAO,eAAe,GAAG;CAC/B,OAAO,KAAK,SAAS,IAAI,KAAK,IAAI,GAAG,IAAI,IAAI,aAAa,GAAG;AAC/D;;AAGA,SAAgB,QAAQ,MAA8B;CACpD,OAAO,GAAG,KAAK,KAAK,GAAG,KAAK,KAAK,GAAG,gBAAgB,KAAK,MAAM;AACjE;;AAGA,SAAgB,YAAY,MAA8B;CACxD,OAAO,GAAG,KAAK,KAAK,GAAG,KAAK;AAC9B;;AAGA,SAAgB,WAAW,MAAwB;CACjD,MAAM,OAAO,cAAc,IAAI;CAC/B,OACE,SAAS,KAAA,MACR,gBAAgB,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,aAAa,IAAI,IAAI;AAEhF;;;;;AAMA,SAAgB,aAAa,MAA0B;CACrD,MAAM,SAAS,SAAS,IAAI;CAC5B,IAAI,CAAC,QAAQ,OAAO,CAAC;CACrB,IAAI,MAAM,QAAQ,OAAO,OAAO,GAAG,OAAO,OAAO;CAEjD,OAAO,CADS,OAAO,WAAW,OAAO,QAAQ,GAAG,QAAQ,OAAO,SAAS,CAChE,CAAC,CAAC,QAAQ,UAAU,UAAU,KAAA,KAAa,UAAU,IAAI;AACvE;;AAGA,SAAgB,UAAU,MAAiC;CACzD,MAAM,sBAAM,IAAI,IAA4B;CAC5C,WAAW,MAAM,KAAK,CAAC;CACvB,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC;AACzB;AAIA,MAAM,iBAAiB;AAEvB,SAAS,MACP,MACA,MACA,QACA,QAAQ,GACF;CACN,IAAI,QAAQ,kBAAkB,CAAC,cAAc,IAAI,GAAG;CACpD,IAAI,WAAW,IAAI,GAAG;EACpB,OAAO,MAAM,IAAI;EACjB,MAAM,OAAO,cAAc,IAAI,KAAK;EACpC,KAAK,MAAM,CAAC,OAAO,UAAU,aAAa,IAAI,CAAC,CAAC,QAAQ,GACtD,MAAM,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,IAAI,QAAQ,QAAQ,CAAC;EAE7D;CACF;CACA,KAAK,MAAM,SAAS,YAAY,IAAI,GAAG,MAAM,OAAO,MAAM,QAAQ,QAAQ,CAAC;AAC7E;AAEA,SAAS,QAAQ,MAAe,MAAyB;CACvD,MAAM,OAAO,cAAc,IAAI,KAAK;CACpC,MAAM,WAAW,aAAa,IAAI;CAClC,MAAM,UAAU,SAAS,KAAK,UAAU,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC;CAE9E,MAAM,wBAAQ,IAAI,IAAY;CAC9B,KAAK,MAAM,OAAO,SAAS,KAAK,MAAM,OAAO,KAAK,MAAM,IAAI,GAAG;CAE/D,MAAM,gBAAoC,CAAC;CAC3C,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAClC,KAAK,IAAI,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EAC3C,IAAI,QAAQ,EAAE,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC,SAAS,GAAG;EACpD,IAAI,CAAC,GAAG,QAAQ,EAAE,CAAC,CAAC,MAAM,QAAQ,QAAQ,EAAE,CAAC,IAAI,GAAG,CAAC,GAAG;EACxD,cAAc,KAAK,CAAC,GAAG,CAAC,CAAC;CAC3B;CAGF,MAAM,OAAO,SAAS,IAAI,YAAY;CACtC,MAAM,QAAQ,KAAK,QAAQ,UAA2B,OAAO,UAAU,QAAQ;CAC/E,OAAO;EACL;EACA;EACA,YAAY,SAAS;EACrB,WAAW,CAAC,GAAG,KAAK;EACpB,YAAY,CAAC,GAAG,KAAK,CAAC,CAAC,QAAQ,QAAQ,QAAQ,OAAO,QAAQ,IAAI,IAAI,GAAG,CAAC,CAAC;EAC3E;EACA,cAAc,MAAM,SAAS,IAAI,KAAK,IAAI,GAAG,KAAK,IAAI,KAAA;EACtD,gBAAgB,MAAM,WAAW,KAAK,UAAU,MAAM,SAAS,IAAI,QAAQ,KAAK,IAAI,KAAA;CACtF;AACF;AAEA,SAAS,QAAQ,MAAe,MAAc,UAAqC;CACjF,MAAM,QAAQ,QAAQ,MAAM,IAAI;CAChC,MAAM,WAAW,aAAa,IAAI;CAElC,IAAI,MAAM,cAAc,SAAS,GAC/B,SAAS,KAAK;EACZ,MAAM;EACN,UAAU;EACV;EACA,MAAM,MAAM;EACZ,QAAQ,sCAAsC,KAAK,UAAU,MAAM,aAAa;EAChF,gBAAgB,MAAM;CACxB,CAAC;CAGH,KAAK,MAAM,YAAY,aAAa,QAAQ,GAC1C,SAAS,KAAK;EACZ,MAAM;EACN,UAAU;EACV;EACA,MAAM,MAAM;EACZ,QAAQ,QAAQ,SAAS,KAAK,gBAAgB,SAAS,QAAQ,OAAO;EACtE,SAAS,SAAS;CACpB,CAAC;CAMH,IAAI,CAAC,gBAAgB,IAAI,MAAM,IAAI,KAAK,MAAM,cAAc,SAAS,GAAG;CACxE,MAAM,UAAU,SACb,KAAK,OAAO,UAAU;EACrB,MAAM,MAAM,IAAI,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,OAAO,CAAC;EACjD,OAAO;GAAE;GAAO,SAAS,MAAM,UAAU,QAAQ,QAAQ,CAAC,IAAI,IAAI,GAAG,CAAC;EAAE;CAC1E,CAAC,CAAC,CACD,QAAQ,UAAU,MAAM,QAAQ,SAAS,CAAC;CAC7C,IAAI,QAAQ,WAAW,GAAG;CAC1B,SAAS,KAAK;EACZ,MAAM;EACN,UAAU;EACV;EACA,MAAM,MAAM;EACZ,QAAQ,GAAG,QAAQ,OAAO;EAC1B;CACF,CAAC;AACH;AAIA,SAAS,aACP,UACuE;CACvE,MAAM,yBAAS,IAAI,IAAsC;CACzD,KAAK,MAAM,CAAC,OAAO,UAAU,SAAS,QAAQ,GAC5C,KAAK,MAAM,QAAQ,UAAU,KAAK,GAAG;EACnC,MAAM,UAAU,YAAY,IAAI;EAChC,IAAI,YAAY,OAAO,IAAI,OAAO;EAClC,IAAI,CAAC,WAAW,OAAO,IAAI,SAAU,4BAAY,IAAI,IAAI,CAAE;EAC3D,MAAM,YAAY,gBAAgB,KAAK,MAAM;EAC7C,IAAI,UAAU,UAAU,IAAI,SAAS;EACrC,IAAI,CAAC,SAAS,UAAU,IAAI,WAAY,0BAAU,IAAI,IAAI,CAAE;EAC5D,QAAQ,IAAI,KAAK;CACnB;CAEF,MAAM,MAA6E,CAAC;CACpF,KAAK,MAAM,CAAC,MAAM,cAAc,QAAQ;EACtC,IAAI,UAAU,OAAO,GAAG;EACxB,IAAI,KAAK;GACP;GACA,SAAS,CAAC,GAAG,UAAU,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,cAAc;IAC5D,QAAQ,UAAU;IAClB,UAAU,CAAC,GAAG,OAAO;GACvB,EAAE;EACJ,CAAC;CACH;CACA,OAAO;AACT;AAEA,SAAS,aAAa,MAAe,QAAQ,GAAuB;CAClE,IAAI,QAAQ,kBAAkB,CAAC,cAAc,IAAI,GAAG,OAAO,KAAA;CAC3D,IAAI,WAAW,IAAI,GAAG;EACpB,MAAM,OAAO,cAAc,IAAI,KAAK;EACpC,MAAM,OAAO,aAAa,IAAI,CAAC,CAC5B,KAAK,UAAU,aAAa,OAAO,QAAQ,CAAC,CAAC,CAAC,CAC9C,QAAQ,UAA2B,OAAO,UAAU,QAAQ;EAC/D,IAAI,KAAK,WAAW,GAAG,OAAO,KAAA;EAE9B,OAAO,gBAAgB,IAAI,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,IAAI,IAAI,IAAI;CACjE;CACA,MAAM,MAAM,QAAQ,IAAI;CACxB,IAAI,QAAQ,KAAA,GAAW,OAAO;CAC9B,MAAM,OAAO,YAAY,IAAI,CAAC,CAC3B,KAAK,UAAU,aAAa,OAAO,QAAQ,CAAC,CAAC,CAAC,CAC9C,QAAQ,UAA2B,OAAO,UAAU,QAAQ;CAC/D,OAAO,KAAK,SAAS,IAAI,KAAK,IAAI,GAAG,IAAI,IAAI,KAAA;AAC/C;AAEA,SAAS,eAAe,KAAwB;CAE9C,OADe,WAAW,GACd,CAAC,CACV,KAAK,UAAU,MAAM,YAAY,CAAC,CAClC,QAAQ,UAA2B,OAAO,UAAU,QAAQ;AACjE;AAEA,SAAS,QAAQ,MAAmC;CAClD,MAAM,SAAS,SAAS,IAAI;CAC5B,IAAI,CAAC,QAAQ,OAAO,KAAA;CACpB,KAAK,MAAM,OAAO,CAAC,QAAQ,UAAU,GAAG;EACtC,MAAM,UAAU,OAAO;EACvB,IAAI,CAAC,WAAW,OAAO,YAAY,UAAU;EAC7C,IAAI,OAAO,QAAQ,SAAS,UAAU,OAAO,QAAQ;EACrD,IAAI,OAAO,QAAQ,YAAY,UAAU,OAAO,QAAQ;CAC1D;AAEF;AAEA,SAAS,WAAW,MAAe,KAAkC,OAAqB;CACxF,IAAI,QAAQ,kBAAkB,CAAC,cAAc,IAAI,GAAG;CACpD,MAAM,SAAS,SAAS,IAAI;CAC5B,IAAI,QACF,KAAK,MAAM,OAAO,CAAC,YAAY,MAAM,GAAG;EACtC,MAAM,QAAQ,OAAO;EACrB,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG;EAC3B,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,OAAO,OAAO,IAAI;GACxB,IAAI,MAAM,IAAI,IAAI,QAAQ,IAAI,GAAG,IAAI;EACvC;CACF;CAEF,KAAK,MAAM,SAAS,YAAY,IAAI,GAAG,WAAW,OAAO,KAAK,QAAQ,CAAC;AACzE;AAEA,SAAS,OAAO,OAA4C;CAC1D,MAAM,SAAS,SAAS,KAAK;CAC7B,IAAI,CAAC,QAAQ,OAAO,KAAA;CACpB,MAAM,EAAE,MAAM,MAAM,WAAW;CAC/B,IAAI,OAAO,SAAS,YAAY,OAAO,SAAS,UAAU,OAAO,KAAA;CACjE,OAAO;EAAE;EAAM;EAAM,QAAQ,eAAe,MAAM;CAAE;AACtD;AAEA,SAAS,eAAe,OAAoD;CAC1E,MAAM,SAAS,SAAS,KAAK;CAC7B,IAAI,CAAC,QAAQ,OAAO,KAAA;CACpB,MAAM,MAA8B,CAAC;CACrC,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,MAAM,GAC7C,IAAI,OAAO,SAAS,UAAU,IAAI,OAAO;CAE3C,OAAO,OAAO,KAAK,GAAG,CAAC,CAAC,SAAS,IAAI,MAAM,KAAA;AAC7C;AAEA,SAAS,gBAAgB,QAAoD;CAC3E,OAAO,OAAO,QAAQ,UAAU,CAAC,CAAC,CAAC,CAChC,MAAM,CAAC,MAAM,CAAC,SAAU,MAAM,MAAM,KAAK,CAAE,CAAC,CAC5C,KAAK,CAAC,KAAK,WAAW,GAAG,IAAI,GAAG,OAAO,CAAC,CACxC,KAAK,GAAG;AACb;AAEA,SAAS,cAAc,MAAmC;CACxD,MAAM,OAAO,SAAS,IAAI,CAAC,EAAE;CAC7B,OAAO,OAAO,SAAS,WAAW,OAAO,KAAA;AAC3C;AAEA,SAAS,YAAY,MAA0B;CAC7C,IAAI,MAAM,QAAQ,IAAI,GAAG,OAAO;CAChC,MAAM,SAAS,SAAS,IAAI;CAC5B,OAAO,SAAS,OAAO,OAAO,MAAM,IAAI,CAAC;AAC3C;AAEA,SAAS,SAAS,OAAqD;CACrE,OAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK,IACrE,QACD,KAAA;AACN;AAEA,SAAS,cAAc,OAAyB;CAC9C,OAAO,OAAO,UAAU,YAAY,UAAU;AAChD;AAEA,SAAS,QAAQ,OAA2B;CAC1C,OAAO,MAAM,QAAQ,KAAK,IAAI,QAAQ,UAAU,KAAA,KAAa,UAAU,OAAO,CAAC,IAAI,CAAC,KAAK;AAC3F;AAEA,SAAS,IAAI,QAA0B;CACrC,OAAO,OAAO,QAAQ,KAAK,UAAU,MAAM,OAAO,CAAC;AACrD;AAEA,SAAS,QAAQ,QAA0B;CACzC,OAAO,OAAO,QAAQ,KAAK,UAAU,MAAM,OAAO,CAAC;AACrD"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
//#region src/sampler.d.ts
|
|
2
|
+
export type MemorySamplerOptions = {
|
|
3
|
+
dir: string;
|
|
4
|
+
sessionId: string;
|
|
5
|
+
/** Sampling period; 250 ms is roughly four short appends per second. */
|
|
6
|
+
intervalMs?: number;
|
|
7
|
+
};
|
|
8
|
+
export type MemorySampler = {
|
|
9
|
+
/** Sibling log the sampler appends to. */
|
|
10
|
+
readonly file: string;
|
|
11
|
+
stop(): void;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Starts the out-of-band memory sampler for a session.
|
|
15
|
+
*
|
|
16
|
+
* The sampler runs on a thread of its own with a small heap of its own, so it
|
|
17
|
+
* keeps producing readings when the observed thread is blocked and when the
|
|
18
|
+
* observed thread's heap is the thing that is full.
|
|
19
|
+
*/
|
|
20
|
+
export declare function startMemorySampler(options: MemorySamplerOptions): MemorySampler;
|
|
21
|
+
//#endregion
|
|
22
|
+
//# sourceMappingURL=sampler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sampler.d.ts","names":[],"sources":["../src/sampler.ts"],"mappings":";YAIY;EACV;EACA;;EAEA;;YAGU;;WAED;EACT;;;;;;;;;wBAUc,mBAAmB,SAAS,uBAAuB"}
|
package/dist/sampler.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import "./events.js";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { Worker } from "node:worker_threads";
|
|
4
|
+
//#region src/sampler.ts
|
|
5
|
+
/**
|
|
6
|
+
* Starts the out-of-band memory sampler for a session.
|
|
7
|
+
*
|
|
8
|
+
* The sampler runs on a thread of its own with a small heap of its own, so it
|
|
9
|
+
* keeps producing readings when the observed thread is blocked and when the
|
|
10
|
+
* observed thread's heap is the thing that is full.
|
|
11
|
+
*/
|
|
12
|
+
function startMemorySampler(options) {
|
|
13
|
+
const { dir, sessionId, intervalMs = 250 } = options;
|
|
14
|
+
const file = path.join(dir, `mem-${sessionId}.ndjson`);
|
|
15
|
+
const worker = new Worker(new URL("./sampler_thread.js", import.meta.url), {
|
|
16
|
+
workerData: {
|
|
17
|
+
file,
|
|
18
|
+
intervalMs
|
|
19
|
+
},
|
|
20
|
+
resourceLimits: { maxOldGenerationSizeMb: 32 }
|
|
21
|
+
});
|
|
22
|
+
worker.unref();
|
|
23
|
+
return {
|
|
24
|
+
file,
|
|
25
|
+
stop: () => {
|
|
26
|
+
worker.terminate();
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
//#endregion
|
|
31
|
+
export { startMemorySampler };
|
|
32
|
+
|
|
33
|
+
//# sourceMappingURL=sampler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sampler.js","names":[],"sources":["../src/sampler.ts"],"sourcesContent":["import path from \"node:path\";\nimport { Worker } from \"node:worker_threads\";\nimport { SAMPLER_FILE_PREFIX } from \"./events\";\n\nexport type MemorySamplerOptions = {\n dir: string;\n sessionId: string;\n /** Sampling period; 250 ms is roughly four short appends per second. */\n intervalMs?: number;\n};\n\nexport type MemorySampler = {\n /** Sibling log the sampler appends to. */\n readonly file: string;\n stop(): void;\n};\n\n/**\n * Starts the out-of-band memory sampler for a session.\n *\n * The sampler runs on a thread of its own with a small heap of its own, so it\n * keeps producing readings when the observed thread is blocked and when the\n * observed thread's heap is the thing that is full.\n */\nexport function startMemorySampler(options: MemorySamplerOptions): MemorySampler {\n const { dir, sessionId, intervalMs = 250 } = options;\n const file = path.join(dir, `${SAMPLER_FILE_PREFIX}-${sessionId}.ndjson`);\n const worker = new Worker(new URL(\"./sampler_thread.js\", import.meta.url), {\n workerData: { file, intervalMs },\n resourceLimits: { maxOldGenerationSizeMb: 32 },\n });\n // Unreferenced so a sampler that is never stopped cannot hold the process open.\n worker.unref();\n return {\n file,\n stop: () => {\n void worker.terminate();\n },\n };\n}\n"],"mappings":";;;;;;;;;;;AAwBA,SAAgB,mBAAmB,SAA8C;CAC/E,MAAM,EAAE,KAAK,WAAW,aAAa,QAAQ;CAC7C,MAAM,OAAO,KAAK,KAAK,KAAK,OAA0B,UAAU,QAAQ;CACxE,MAAM,SAAS,IAAI,OAAO,IAAI,IAAI,uBAAuB,OAAO,KAAK,GAAG,GAAG;EACzE,YAAY;GAAE;GAAM;EAAW;EAC/B,gBAAgB,EAAE,wBAAwB,GAAG;CAC/C,CAAC;CAED,OAAO,MAAM;CACb,OAAO;EACL;EACA,YAAY;GACV,OAAY,UAAU;EACxB;CACF;AACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import { workerData } from "node:worker_threads";
|
|
4
|
+
//#region src/sampler_thread.ts
|
|
5
|
+
/**
|
|
6
|
+
* Memory sampler, run on its own worker thread.
|
|
7
|
+
*
|
|
8
|
+
* It exists because the thread worth watching is the one that blocks. While the
|
|
9
|
+
* middle layer sits inside a synchronous pframes call its own timers do not
|
|
10
|
+
* fire, so its memory series goes dark exactly while memory is growing fastest.
|
|
11
|
+
* This thread stays responsive and keeps the resident-size curve intact right up
|
|
12
|
+
* to the moment the process dies.
|
|
13
|
+
*
|
|
14
|
+
* `rss` and `freeMemory` are process- and machine-wide and so are meaningful
|
|
15
|
+
* from here. Heap figures are per-isolate and would describe only this thread,
|
|
16
|
+
* so they are deliberately not recorded; the observed thread reports its own.
|
|
17
|
+
*/
|
|
18
|
+
const { file, intervalMs } = workerData;
|
|
19
|
+
const fd = fs.openSync(file, "a");
|
|
20
|
+
let seq = 0;
|
|
21
|
+
let peakRss = 0;
|
|
22
|
+
setInterval(() => {
|
|
23
|
+
const rss = process.memoryUsage.rss();
|
|
24
|
+
if (rss > peakRss) peakRss = rss;
|
|
25
|
+
const record = {
|
|
26
|
+
seq: ++seq,
|
|
27
|
+
t: Math.round(performance.now() * 1e3) / 1e3,
|
|
28
|
+
wall: Date.now(),
|
|
29
|
+
type: "mem-sampler",
|
|
30
|
+
rss,
|
|
31
|
+
peakRss,
|
|
32
|
+
freeMemory: os.freemem(),
|
|
33
|
+
totalMemory: os.totalmem()
|
|
34
|
+
};
|
|
35
|
+
try {
|
|
36
|
+
fs.writeSync(fd, `${JSON.stringify(record)}\n`);
|
|
37
|
+
} catch {}
|
|
38
|
+
}, intervalMs);
|
|
39
|
+
//#endregion
|
|
40
|
+
|
|
41
|
+
//# sourceMappingURL=sampler_thread.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sampler_thread.js","names":[],"sources":["../src/sampler_thread.ts"],"sourcesContent":["/**\n * Memory sampler, run on its own worker thread.\n *\n * It exists because the thread worth watching is the one that blocks. While the\n * middle layer sits inside a synchronous pframes call its own timers do not\n * fire, so its memory series goes dark exactly while memory is growing fastest.\n * This thread stays responsive and keeps the resident-size curve intact right up\n * to the moment the process dies.\n *\n * `rss` and `freeMemory` are process- and machine-wide and so are meaningful\n * from here. Heap figures are per-isolate and would describe only this thread,\n * so they are deliberately not recorded; the observed thread reports its own.\n */\n\nimport fs from \"node:fs\";\nimport os from \"node:os\";\nimport { workerData } from \"node:worker_threads\";\nimport type { SamplerRecord } from \"./events\";\n\ntype SamplerWorkerData = { file: string; intervalMs: number };\n\nconst { file, intervalMs } = workerData as SamplerWorkerData;\nconst fd = fs.openSync(file, \"a\");\nlet seq = 0;\nlet peakRss = 0;\n\nsetInterval(() => {\n const rss = process.memoryUsage.rss();\n if (rss > peakRss) peakRss = rss;\n const record: SamplerRecord = {\n seq: ++seq,\n t: Math.round(performance.now() * 1000) / 1000,\n wall: Date.now(),\n type: \"mem-sampler\",\n rss,\n peakRss,\n freeMemory: os.freemem(),\n totalMemory: os.totalmem(),\n };\n try {\n fs.writeSync(fd, `${JSON.stringify(record)}\\n`);\n } catch {\n // Sampling must never take the application down.\n }\n}, intervalMs);\n"],"mappings":";;;;;;;;;;;;;;;;;AAqBA,MAAM,EAAE,MAAM,eAAe;AAC7B,MAAM,KAAK,GAAG,SAAS,MAAM,GAAG;AAChC,IAAI,MAAM;AACV,IAAI,UAAU;AAEd,kBAAkB;CAChB,MAAM,MAAM,QAAQ,YAAY,IAAI;CACpC,IAAI,MAAM,SAAS,UAAU;CAC7B,MAAM,SAAwB;EAC5B,KAAK,EAAE;EACP,GAAG,KAAK,MAAM,YAAY,IAAI,IAAI,GAAI,IAAI;EAC1C,MAAM,KAAK,IAAI;EACf,MAAM;EACN;EACA;EACA,YAAY,GAAG,QAAQ;EACvB,aAAa,GAAG,SAAS;CAC3B;CACA,IAAI;EACF,GAAG,UAAU,IAAI,GAAG,KAAK,UAAU,MAAM,EAAE,GAAG;CAChD,QAAQ,CAER;AACF,GAAG,UAAU"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { Recorder } from "./recorder.js";
|
|
2
|
+
import { MemorySampler } from "./sampler.js";
|
|
3
|
+
import { HandleRegistry } from "./instrument.js";
|
|
4
|
+
//#region src/session.d.ts
|
|
5
|
+
/** Environment variable naming the directory flight logs are written to. */
|
|
6
|
+
export declare const FLIGHT_DIR_ENV = "MI_FLIGHT_RECORDER_DIR";
|
|
7
|
+
/**
|
|
8
|
+
* Environment variable carrying the session id a supervising parent assigned.
|
|
9
|
+
* Set it alongside {@link FLIGHT_DIR_ENV} when spawning the worker and pass the
|
|
10
|
+
* same id to `superviseWorker`.
|
|
11
|
+
*/
|
|
12
|
+
export declare const FLIGHT_SESSION_ENV = "MI_FLIGHT_RECORDER_SESSION";
|
|
13
|
+
export type FlightSessionOptions = {
|
|
14
|
+
/** Overrides the directory from the environment. */
|
|
15
|
+
dir?: string;
|
|
16
|
+
/** Overrides the session id from the environment. */
|
|
17
|
+
sessionId?: string;
|
|
18
|
+
role?: string;
|
|
19
|
+
meta?: Record<string, unknown>;
|
|
20
|
+
samplerIntervalMs?: number;
|
|
21
|
+
selfSamplerIntervalMs?: number;
|
|
22
|
+
};
|
|
23
|
+
export type FlightSession = {
|
|
24
|
+
readonly recorder: Recorder;
|
|
25
|
+
readonly sampler: MemorySampler;
|
|
26
|
+
/** Shared so create calls and later data calls agree on handle identity. */
|
|
27
|
+
readonly registry: HandleRegistry;
|
|
28
|
+
close(reason?: string): void;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Opens a flight session, or returns undefined when recording is not enabled.
|
|
32
|
+
*
|
|
33
|
+
* Recording is opt-in for now: it appends synchronously on every recorded
|
|
34
|
+
* operation, and that cost has not been measured against a real project, so it
|
|
35
|
+
* is switched on by pointing {@link FLIGHT_DIR_ENV} at a directory rather than
|
|
36
|
+
* being on by default.
|
|
37
|
+
*/
|
|
38
|
+
export declare function openFlightSession(options?: FlightSessionOptions): FlightSession | undefined;
|
|
39
|
+
//#endregion
|
|
40
|
+
//# sourceMappingURL=session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session.d.ts","names":[],"sources":["../src/session.ts"],"mappings":";;;;;qBAKa;;;;;;qBAOA;YAED;;EAEV;;EAEA;EACA;EACA,OAAO;EACP;EACA;;YAGU;WACD,UAAU;WACV,SAAS;;WAET,UAAU;EACnB,MAAM;;;;;;;;;;wBAWQ,kBAAkB,UAAS,uBAA4B"}
|
package/dist/session.js
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { openRecorder, startSelfSampler } from "./recorder.js";
|
|
2
|
+
import { startMemorySampler } from "./sampler.js";
|
|
3
|
+
import { createHandleRegistry } from "./instrument.js";
|
|
4
|
+
//#region src/session.ts
|
|
5
|
+
/** Environment variable naming the directory flight logs are written to. */
|
|
6
|
+
const FLIGHT_DIR_ENV = "MI_FLIGHT_RECORDER_DIR";
|
|
7
|
+
/**
|
|
8
|
+
* Environment variable carrying the session id a supervising parent assigned.
|
|
9
|
+
* Set it alongside {@link FLIGHT_DIR_ENV} when spawning the worker and pass the
|
|
10
|
+
* same id to `superviseWorker`.
|
|
11
|
+
*/
|
|
12
|
+
const FLIGHT_SESSION_ENV = "MI_FLIGHT_RECORDER_SESSION";
|
|
13
|
+
/**
|
|
14
|
+
* Opens a flight session, or returns undefined when recording is not enabled.
|
|
15
|
+
*
|
|
16
|
+
* Recording is opt-in for now: it appends synchronously on every recorded
|
|
17
|
+
* operation, and that cost has not been measured against a real project, so it
|
|
18
|
+
* is switched on by pointing {@link FLIGHT_DIR_ENV} at a directory rather than
|
|
19
|
+
* being on by default.
|
|
20
|
+
*/
|
|
21
|
+
function openFlightSession(options = {}) {
|
|
22
|
+
const dir = options.dir ?? process.env["MI_FLIGHT_RECORDER_DIR"];
|
|
23
|
+
if (!dir) return void 0;
|
|
24
|
+
const recorder = openRecorder({
|
|
25
|
+
dir,
|
|
26
|
+
role: options.role,
|
|
27
|
+
meta: options.meta,
|
|
28
|
+
sessionId: options.sessionId ?? process.env["MI_FLIGHT_RECORDER_SESSION"] ?? void 0
|
|
29
|
+
});
|
|
30
|
+
const sampler = startMemorySampler({
|
|
31
|
+
dir,
|
|
32
|
+
sessionId: recorder.sessionId,
|
|
33
|
+
intervalMs: options.samplerIntervalMs
|
|
34
|
+
});
|
|
35
|
+
const stopSelfSampler = startSelfSampler(recorder, options.selfSamplerIntervalMs);
|
|
36
|
+
return {
|
|
37
|
+
recorder,
|
|
38
|
+
sampler,
|
|
39
|
+
registry: createHandleRegistry(),
|
|
40
|
+
close(reason) {
|
|
41
|
+
stopSelfSampler();
|
|
42
|
+
sampler.stop();
|
|
43
|
+
recorder.close(reason);
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
//#endregion
|
|
48
|
+
export { FLIGHT_DIR_ENV, FLIGHT_SESSION_ENV, openFlightSession };
|
|
49
|
+
|
|
50
|
+
//# sourceMappingURL=session.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session.js","names":[],"sources":["../src/session.ts"],"sourcesContent":["import { openRecorder, startSelfSampler, type Recorder } from \"./recorder\";\nimport { startMemorySampler, type MemorySampler } from \"./sampler\";\nimport { createHandleRegistry, type HandleRegistry } from \"./instrument\";\n\n/** Environment variable naming the directory flight logs are written to. */\nexport const FLIGHT_DIR_ENV = \"MI_FLIGHT_RECORDER_DIR\";\n\n/**\n * Environment variable carrying the session id a supervising parent assigned.\n * Set it alongside {@link FLIGHT_DIR_ENV} when spawning the worker and pass the\n * same id to `superviseWorker`.\n */\nexport const FLIGHT_SESSION_ENV = \"MI_FLIGHT_RECORDER_SESSION\";\n\nexport type FlightSessionOptions = {\n /** Overrides the directory from the environment. */\n dir?: string;\n /** Overrides the session id from the environment. */\n sessionId?: string;\n role?: string;\n meta?: Record<string, unknown>;\n samplerIntervalMs?: number;\n selfSamplerIntervalMs?: number;\n};\n\nexport type FlightSession = {\n readonly recorder: Recorder;\n readonly sampler: MemorySampler;\n /** Shared so create calls and later data calls agree on handle identity. */\n readonly registry: HandleRegistry;\n close(reason?: string): void;\n};\n\n/**\n * Opens a flight session, or returns undefined when recording is not enabled.\n *\n * Recording is opt-in for now: it appends synchronously on every recorded\n * operation, and that cost has not been measured against a real project, so it\n * is switched on by pointing {@link FLIGHT_DIR_ENV} at a directory rather than\n * being on by default.\n */\nexport function openFlightSession(options: FlightSessionOptions = {}): FlightSession | undefined {\n const dir = options.dir ?? process.env[FLIGHT_DIR_ENV];\n if (!dir) return undefined;\n\n const recorder = openRecorder({\n dir,\n role: options.role,\n meta: options.meta,\n sessionId: options.sessionId ?? process.env[FLIGHT_SESSION_ENV] ?? undefined,\n });\n const sampler = startMemorySampler({\n dir,\n sessionId: recorder.sessionId,\n intervalMs: options.samplerIntervalMs,\n });\n const stopSelfSampler = startSelfSampler(recorder, options.selfSamplerIntervalMs);\n\n return {\n recorder,\n sampler,\n registry: createHandleRegistry(),\n close(reason) {\n stopSelfSampler();\n sampler.stop();\n recorder.close(reason);\n },\n };\n}\n"],"mappings":";;;;;AAKA,MAAa,iBAAiB;;;;;;AAO9B,MAAa,qBAAqB;;;;;;;;;AA6BlC,SAAgB,kBAAkB,UAAgC,CAAC,GAA8B;CAC/F,MAAM,MAAM,QAAQ,OAAO,QAAQ,IAAA;CACnC,IAAI,CAAC,KAAK,OAAO,KAAA;CAEjB,MAAM,WAAW,aAAa;EAC5B;EACA,MAAM,QAAQ;EACd,MAAM,QAAQ;EACd,WAAW,QAAQ,aAAa,QAAQ,IAAA,iCAA2B,KAAA;CACrE,CAAC;CACD,MAAM,UAAU,mBAAmB;EACjC;EACA,WAAW,SAAS;EACpB,YAAY,QAAQ;CACtB,CAAC;CACD,MAAM,kBAAkB,iBAAiB,UAAU,QAAQ,qBAAqB;CAEhF,OAAO;EACL;EACA;EACA,UAAU,qBAAqB;EAC/B,MAAM,QAAQ;GACZ,gBAAgB;GAChB,QAAQ,KAAK;GACb,SAAS,MAAM,MAAM;EACvB;CACF;AACF"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { CrashMarker, CrashReason } from "./events.js";
|
|
2
|
+
//#region src/supervisor.d.ts
|
|
3
|
+
export type CrashMarkerInput = {
|
|
4
|
+
/** Session id the parent assigned to the worker. Omitted, the marker carries no identity. */
|
|
5
|
+
sessionId?: string;
|
|
6
|
+
reason?: CrashReason;
|
|
7
|
+
error?: (Error & {
|
|
8
|
+
code?: string;
|
|
9
|
+
}) | unknown;
|
|
10
|
+
code?: number;
|
|
11
|
+
signal?: string;
|
|
12
|
+
stderrTail?: string;
|
|
13
|
+
};
|
|
14
|
+
export type SuperviseOptions = {
|
|
15
|
+
/**
|
|
16
|
+
* The session id handed to the worker at spawn (see `FLIGHT_SESSION_ENV`).
|
|
17
|
+
* With it the marker names the dying session with certainty. Without it the
|
|
18
|
+
* analyzer has to attribute the marker by timing, and will decline to
|
|
19
|
+
* attribute it at all when more than one session looks dead.
|
|
20
|
+
*/
|
|
21
|
+
sessionId?: string;
|
|
22
|
+
onCrash?: (info: {
|
|
23
|
+
kind: "error" | "exit";
|
|
24
|
+
markerFile: string;
|
|
25
|
+
error?: unknown;
|
|
26
|
+
code?: number;
|
|
27
|
+
}) => void;
|
|
28
|
+
};
|
|
29
|
+
/** Minimal view of a worker, so callers are not forced to import worker_threads. */
|
|
30
|
+
export type SupervisedWorker = {
|
|
31
|
+
on(event: "error", listener: (error: Error) => void): unknown;
|
|
32
|
+
on(event: "exit", listener: (code: number) => void): unknown;
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Records an abnormal end observed from outside the dying thread.
|
|
36
|
+
*
|
|
37
|
+
* A thread that runs out of heap cannot describe its own death: the last reading
|
|
38
|
+
* it wrote predates the blow-up, and when the blow-up is synchronous no sampler
|
|
39
|
+
* tick of its own lands either. The parent is the only place where the cause is
|
|
40
|
+
* known rather than inferred — Node reports `ERR_WORKER_OUT_OF_MEMORY` to it —
|
|
41
|
+
* so the parent writes the verdict down on the dead thread's behalf.
|
|
42
|
+
*/
|
|
43
|
+
export declare function writeCrashMarker(dir: string, input?: CrashMarkerInput): string;
|
|
44
|
+
/** Crash markers in a directory, oldest first. */
|
|
45
|
+
export declare function readCrashMarkers(dir: string): CrashMarker[];
|
|
46
|
+
/**
|
|
47
|
+
* Attaches crash recording to a middle-layer worker thread.
|
|
48
|
+
*
|
|
49
|
+
* A worker whose isolate exhausts its heap dies alone and the parent receives
|
|
50
|
+
* `ERR_WORKER_OUT_OF_MEMORY`, with or without `resourceLimits`. What
|
|
51
|
+
* `resourceLimits.maxOldGenerationSizeMb` adds is a chosen ceiling: V8's default
|
|
52
|
+
* is several gigabytes, so on a small machine the OS can run out of memory and
|
|
53
|
+
* kill the whole process before V8 ever reports the worker's heap as full — and
|
|
54
|
+
* then there is no parent left to write anything.
|
|
55
|
+
*/
|
|
56
|
+
export declare function superviseWorker(worker: SupervisedWorker, dir: string, options?: SuperviseOptions): void;
|
|
57
|
+
//#endregion
|
|
58
|
+
//# sourceMappingURL=supervisor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervisor.d.ts","names":[],"sources":["../src/supervisor.ts"],"mappings":";;YAKY;;EAEV;EACA,SAAS;EACT,SAAS;IAAU;;EACnB;EACA;EACA;;YAGU;;;;;;;EAOV;EACA,WAAW;IACT;IACA;IACA;IACA;;;;YAKQ;EACV,GAAG,gBAAgB,WAAW,OAAO;EACrC,GAAG,eAAe,WAAW;;;;;;;;;;;wBAYf,iBAAiB,aAAa,QAAO;;wBA2BrC,iBAAiB,cAAc;;;;;;;;;;;wBA+B/B,gBACd,QAAQ,kBACR,aACA,UAAS"}
|