@camstack/system 1.2.151 → 1.2.154

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/addon-runner.js +14 -15
  2. package/dist/addon-runner.mjs +14 -14
  3. package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
  4. package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
  5. package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
  6. package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
  7. package/dist/builtins/alerts/alerts.addon.js +1 -1
  8. package/dist/builtins/alerts/alerts.addon.mjs +1 -1
  9. package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
  10. package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
  11. package/dist/builtins/console-logging/index.js +1 -1
  12. package/dist/builtins/console-logging/index.mjs +1 -1
  13. package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
  14. package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
  15. package/dist/builtins/device-manager/device-manager.addon.js +2 -2
  16. package/dist/builtins/device-manager/device-manager.addon.mjs +2 -2
  17. package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
  18. package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
  19. package/dist/builtins/hub-forwarder/index.js +1 -1
  20. package/dist/builtins/hub-forwarder/index.mjs +1 -1
  21. package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
  22. package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
  23. package/dist/builtins/local-auth/local-auth.addon.js +1 -1
  24. package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
  25. package/dist/builtins/local-network/local-network.addon.js +3 -3
  26. package/dist/builtins/local-network/local-network.addon.mjs +3 -3
  27. package/dist/builtins/loki-logging/index.js +1 -1
  28. package/dist/builtins/loki-logging/index.mjs +1 -1
  29. package/dist/builtins/native-metrics/cgroup-memory.d.ts +193 -0
  30. package/dist/builtins/native-metrics/container-memory-series.d.ts +226 -0
  31. package/dist/builtins/native-metrics/native-metrics.addon.d.ts +26 -0
  32. package/dist/builtins/native-metrics/native-metrics.addon.js +1536 -661
  33. package/dist/builtins/native-metrics/native-metrics.addon.mjs +1536 -660
  34. package/dist/builtins/platform-probe/index.js +3 -3
  35. package/dist/builtins/platform-probe/index.mjs +2 -2
  36. package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
  37. package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
  38. package/dist/builtins/snapshot/index.js +1 -1
  39. package/dist/builtins/snapshot/index.mjs +1 -1
  40. package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +2 -2
  41. package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +2 -2
  42. package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
  43. package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
  44. package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
  45. package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
  46. package/dist/builtins/system-config/system-config.addon.js +1 -1
  47. package/dist/builtins/system-config/system-config.addon.mjs +1 -1
  48. package/dist/builtins/winston-logging/index.js +1 -1
  49. package/dist/builtins/winston-logging/index.mjs +1 -1
  50. package/dist/{dist-CImxMt5h.js → dist-BPlfW-CG.js} +96 -10
  51. package/dist/{dist-CbRciAbu.mjs → dist-D_NyUV6Y.mjs} +96 -10
  52. package/dist/index.d.ts +6 -2
  53. package/dist/index.js +24 -5
  54. package/dist/index.mjs +5 -5
  55. package/dist/kernel/event-plane-report.d.ts +202 -0
  56. package/dist/kernel/heap-spaces.d.ts +138 -0
  57. package/dist/kernel/heap-watch.d.ts +142 -5
  58. package/dist/kernel/index.d.ts +2 -0
  59. package/dist/kernel/transport/local-child-registry.d.ts +22 -0
  60. package/dist/{manifest-python-deps-DuQ3Uqoq.mjs → manifest-python-deps-BILJm2mI.mjs} +522 -15
  61. package/dist/{manifest-python-deps-BqWrDjSN.js → manifest-python-deps-COeSr7el.js} +636 -15
  62. package/dist/{retired-settings-keys-DhFPXIRD.mjs → retired-settings-keys-Bq1Tb84I.mjs} +1 -1
  63. package/dist/{retired-settings-keys-C7gLUS3x.js → retired-settings-keys-DBY6ebwV.js} +1 -1
  64. package/package.json +1 -1
  65. package/dist/{tls-u8QCJCFE.js → tls-BxQlomxd.js} +1 -1
  66. package/dist/{tls-CQhPGSJm.mjs → tls-DOTmtLCW.mjs} +1 -1
@@ -3,736 +3,1550 @@ Object.defineProperties(exports, {
3
3
  [Symbol.toStringTag]: { value: "Module" }
4
4
  });
5
5
  const require_chunk = require("../../chunk-Cek0wNdY.js");
6
- const require_dist = require("../../dist-CImxMt5h.js");
6
+ const require_dist = require("../../dist-BPlfW-CG.js");
7
7
  let node_fs_promises = require("node:fs/promises");
8
8
  let node_child_process = require("node:child_process");
9
9
  let node_util = require("node:util");
10
- let node_os = require("node:os");
11
- node_os = require_chunk.__toESM(node_os);
12
10
  let node_fs = require("node:fs");
13
11
  node_fs = require_chunk.__toESM(node_fs);
14
- //#region src/builtins/native-metrics/native-metrics-provider.ts
15
- var IS_DARWIN = node_os.platform() === "darwin";
16
- var IS_LINUX = node_os.platform() === "linux";
17
- var NativeMetricsProvider = class {
18
- id = "native";
19
- cachedSnapshot = null;
20
- samplingTimer = null;
21
- macMemTimer = null;
22
- prevCpu;
23
- diagnostics = null;
24
- consecutiveSampleErrors = 0;
25
- constructor() {
26
- this.prevCpu = aggregateCpuTimes();
12
+ let node_os = require("node:os");
13
+ node_os = require_chunk.__toESM(node_os);
14
+ //#region src/builtins/native-metrics/cgroup-memory.ts
15
+ /** `readFileSync`, with every failure flattened to `null`. */
16
+ var nodeCgroupFileReader = { read(path) {
17
+ try {
18
+ return (0, node_fs.readFileSync)(path, "utf8");
19
+ } catch {
20
+ return null;
21
+ }
22
+ } };
23
+ /** Where the v2 unified hierarchy is mounted, and where v1 hangs its controllers. */
24
+ var CGROUP_MOUNT_ROOT = "/sys/fs/cgroup";
25
+ /**
26
+ * The i915 GEM accounting file.
27
+ *
28
+ * **debugfs, and it is not mounted in our containers.** Verified on the live
29
+ * hub on 2026-08-29: readable on the Unraid host (`6014 shrinkable [0 free]
30
+ * objects, 1194934272 bytes`), `ENOENT` inside the `CamStack` container where
31
+ * `native-metrics` actually runs. It is probed ONCE per process and latched —
32
+ * a bind mount cannot appear under a running container — so an absent file
33
+ * costs one failed `open` for the life of the runner and one log line, never a
34
+ * failure per tick.
35
+ *
36
+ * Making it populate is a one-line change on the container, not here:
37
+ * mounting `/sys/kernel/debug` read-only into `CamStack` fills this field on
38
+ * the next sample with no code change.
39
+ */
40
+ var I915_GEM_OBJECTS_PATH = "/sys/kernel/debug/dri/0/i915_gem_objects";
41
+ /**
42
+ * v1's "no limit" sentinel — `PAGE_COUNTER_MAX << PAGE_SHIFT`, which is what a
43
+ * 64-bit kernel writes into `memory.limit_in_bytes` when nothing is set. It is
44
+ * a limit in the same sense that `Infinity` is a number, so it reads as `null`.
45
+ */
46
+ var V1_UNLIMITED_SENTINEL = 0x7ffffffffffff000;
47
+ /**
48
+ * Parse `/proc/self/cgroup`.
49
+ *
50
+ * Two formats, and the file may contain both on a hybrid host:
51
+ *
52
+ * ```
53
+ * 0::/ ← v2 unified
54
+ * 5:memory:/docker/9f3c… ← v1, memory controller
55
+ * 4:cpu,cpuacct:/docker/9f3c… ← v1, other controllers
56
+ * ```
57
+ *
58
+ * v2 wins when present: on a hybrid host the unified hierarchy is the one
59
+ * carrying `memory.current`, and every camstack deployment is v2 (verified on
60
+ * the live hub, which reports the single line `0::/`).
61
+ *
62
+ * Returns `null` when the file names no memory hierarchy at all — which is
63
+ * also what a non-Linux host produces, because there is no file to read.
64
+ */
65
+ function parseCgroupSelf(text) {
66
+ let v1 = null;
67
+ for (const rawLine of text.split("\n")) {
68
+ const line = rawLine.trim();
69
+ if (line.length === 0) continue;
70
+ const first = line.indexOf(":");
71
+ if (first < 0) continue;
72
+ const second = line.indexOf(":", first + 1);
73
+ if (second < 0) continue;
74
+ const controllers = line.slice(first + 1, second);
75
+ const path = line.slice(second + 1);
76
+ if (path.length === 0) continue;
77
+ const normalized = path.startsWith("/") ? path : `/${path}`;
78
+ if (controllers.length === 0) return {
79
+ version: "v2",
80
+ path: normalized
81
+ };
82
+ if (v1 === null && controllers.split(",").includes("memory")) v1 = {
83
+ version: "v1",
84
+ path: normalized
85
+ };
86
+ }
87
+ return v1;
88
+ }
89
+ /**
90
+ * Parse a `memory.stat` file — v1 or v2, both are `key value` per line.
91
+ *
92
+ * Unknown keys are KEPT rather than refused: a v2 `memory.stat` carries ~50
93
+ * fields and we read five of them, and a kernel that adds a sixth must not
94
+ * turn into a parse failure. A malformed line is skipped, not thrown on.
95
+ */
96
+ function parseCgroupStat(text) {
97
+ const out = /* @__PURE__ */ new Map();
98
+ for (const rawLine of text.split("\n")) {
99
+ const line = rawLine.trim();
100
+ if (line.length === 0) continue;
101
+ const space = line.indexOf(" ");
102
+ if (space <= 0) continue;
103
+ const key = line.slice(0, space);
104
+ const value = Number(line.slice(space + 1).trim());
105
+ if (!Number.isFinite(value)) continue;
106
+ out.set(key, value);
107
+ }
108
+ return out;
109
+ }
110
+ /**
111
+ * Parse a single-scalar cgroup file (`memory.current`, `memory.max`, …).
112
+ *
113
+ * `max` — v2's word for "no limit" — and v1's `V1_UNLIMITED_SENTINEL` both
114
+ * become `null`, because a ceiling nothing can reach is not a ceiling and
115
+ * charting it as 8 exabytes would flatten every real series to zero.
116
+ */
117
+ function parseCgroupScalar(text) {
118
+ const trimmed = text.trim();
119
+ if (trimmed.length === 0) return null;
120
+ if (trimmed === "max") return null;
121
+ const value = Number(trimmed);
122
+ if (!Number.isFinite(value) || value < 0) return null;
123
+ if (value >= V1_UNLIMITED_SENTINEL) return null;
124
+ return value;
125
+ }
126
+ /**
127
+ * Parse the first line of `i915_gem_objects`.
128
+ *
129
+ * ```
130
+ * 6014 shrinkable [0 free] objects, 1194934272 bytes
131
+ * ```
132
+ *
133
+ * Only the byte total is taken. The format has changed across kernel releases
134
+ * more than once, so a line that does not match reads `null` — an unrecognised
135
+ * format is UNKNOWN, never zero.
136
+ */
137
+ function parseI915GemObjects(text) {
138
+ const match = text.match(/^\s*\d+\s+shrinkable\b[^,\n]*,\s*(\d+)\s+bytes/m);
139
+ if (match === null) return null;
140
+ const bytes = Number(match[1]);
141
+ return Number.isFinite(bytes) ? bytes : null;
142
+ }
143
+ /**
144
+ * Find the cgroup from INSIDE this process — never a hard-coded path.
145
+ *
146
+ * The path the hub happens to have (`/sys/fs/cgroup/docker/<id>/memory.stat`
147
+ * as seen from the Unraid host) is an artefact of where you are standing. From
148
+ * inside the container the same cgroup is at the mount ROOT, because Docker
149
+ * gives the container its own cgroup namespace and `/proc/self/cgroup` reports
150
+ * `0::/`. Hard-coding either one breaks the other.
151
+ *
152
+ * So the resolution is: ask `/proc/self/cgroup` which hierarchy and which
153
+ * path, then take the FIRST candidate directory that actually publishes the
154
+ * usage file.
155
+ *
156
+ * - **v2** — `<mount><path>` then `<mount>`. The second candidate is the
157
+ * namespaced case and the one that answers on our hub; the first is the
158
+ * un-namespaced case (a host process, or `--cgroupns=host`).
159
+ * - **v1** — `<mount>/memory<path>` then `<mount>/memory`, same reasoning.
160
+ * - **neither** — `null`. macOS has no `/proc/self/cgroup` at all, so a
161
+ * developer Mac lands here and reports nothing, which is the truth.
162
+ */
163
+ function resolveCgroupMemoryLayout(fs) {
164
+ const selfText = fs.read("/proc/self/cgroup");
165
+ if (selfText === null) return null;
166
+ const entry = parseCgroupSelf(selfText);
167
+ if (entry === null) return null;
168
+ const suffix = entry.path === "/" ? "" : entry.path;
169
+ const candidates = entry.version === "v2" ? [`${CGROUP_MOUNT_ROOT}${suffix}`, CGROUP_MOUNT_ROOT] : [`${CGROUP_MOUNT_ROOT}/memory${suffix}`, `${CGROUP_MOUNT_ROOT}/memory`];
170
+ const usageFile = entry.version === "v2" ? "memory.current" : "memory.usage_in_bytes";
171
+ const limitFile = entry.version === "v2" ? "memory.max" : "memory.limit_in_bytes";
172
+ for (const directory of candidates) {
173
+ if (fs.read(`${directory}/${usageFile}`) === null) continue;
174
+ return {
175
+ source: entry.version === "v2" ? "cgroup-v2" : "cgroup-v1",
176
+ directory,
177
+ currentPath: `${directory}/${usageFile}`,
178
+ limitPath: `${directory}/${limitFile}`,
179
+ statPath: `${directory}/memory.stat`
180
+ };
181
+ }
182
+ return null;
183
+ }
184
+ /** A stat lookup that answers `null` for a key this hierarchy does not publish. */
185
+ function statValue(stat, key) {
186
+ const value = stat.get(key);
187
+ return value === void 0 ? null : value;
188
+ }
189
+ /**
190
+ * v2 publishes `slab` directly; older v2 kernels publish only the two halves.
191
+ * Summing them is the same number, so the fallback is exact rather than an
192
+ * approximation — and when neither is present the answer is `null`.
193
+ */
194
+ function resolveSlab(stat) {
195
+ const direct = statValue(stat, "slab");
196
+ if (direct !== null) return direct;
197
+ const reclaimable = stat.get("slab_reclaimable");
198
+ const unreclaimable = stat.get("slab_unreclaimable");
199
+ if (reclaimable === void 0 || unreclaimable === void 0) return null;
200
+ return reclaimable + unreclaimable;
201
+ }
202
+ /**
203
+ * Turn one layout's three files into a snapshot.
204
+ *
205
+ * Pure over the text, so the whole mapping — including a v2 `memory.stat` full
206
+ * of fields we do not read — is testable against a real fixture.
207
+ *
208
+ * v1's `memory.stat` names the same quantities differently and publishes fewer
209
+ * of them: `rss` is the anonymous charge, `cache` is the page cache, and there
210
+ * is no slab accounting at all. Mapping them onto the same field names is what
211
+ * makes a v1 node and a v2 node comparable; reporting `slabBytes: null` on v1
212
+ * is what stops that comparison from lying.
213
+ */
214
+ function buildContainerMemorySnapshot(input) {
215
+ const currentBytes = parseCgroupScalar(input.currentText);
216
+ if (currentBytes === null) return null;
217
+ const stat = input.statText === null ? /* @__PURE__ */ new Map() : parseCgroupStat(input.statText);
218
+ const isV2 = input.source === "cgroup-v2";
219
+ return {
220
+ source: input.source,
221
+ currentBytes,
222
+ limitBytes: input.limitText === null ? null : parseCgroupScalar(input.limitText),
223
+ anonBytes: statValue(stat, isV2 ? "anon" : "rss"),
224
+ fileBytes: statValue(stat, isV2 ? "file" : "cache"),
225
+ shmemBytes: statValue(stat, "shmem"),
226
+ slabBytes: isV2 ? resolveSlab(stat) : null,
227
+ gpuShmemBytes: input.gpuShmemBytes
228
+ };
229
+ }
230
+ /**
231
+ * The sampler. Resolves once, latches, and logs TRANSITIONS only.
232
+ *
233
+ * Three states are worth a line and none of them is worth one per tick:
234
+ *
235
+ * 1. **Resolution** — once, at the first sample. Either the layout that
236
+ * answered, or "no cgroup on this node", which is a fact about the node and
237
+ * not an error.
238
+ * 2. **GPU probe** — once, at the first sample. Absent debugfs is the normal
239
+ * case inside a container and is stated as such, then never retried: a bind
240
+ * mount cannot appear under a running container.
241
+ * 3. **Read failing / recovering** — a cgroup that resolved and then stopped
242
+ * answering is a real change and says so, once in each direction. A tick
243
+ * that fails while already failing is silent, because the first line
244
+ * already said it.
245
+ *
246
+ * The silence is bounded by design: at the 10 s snapshot cadence a per-tick
247
+ * warning would be 8 640 lines a day on a node that simply is not Linux.
248
+ */
249
+ var ContainerMemoryProbe = class {
250
+ logger;
251
+ fs;
252
+ wantGpu;
253
+ resolved = false;
254
+ layout = null;
255
+ gpuProbed = false;
256
+ gpuReadable = false;
257
+ /** `null` until the first sample — so the FIRST outcome is a transition too. */
258
+ lastReadOk = null;
259
+ constructor(deps) {
260
+ this.logger = deps.logger;
261
+ this.fs = deps.fs ?? nodeCgroupFileReader;
262
+ this.wantGpu = deps.readGpu ?? true;
263
+ }
264
+ /** Did this node resolve a cgroup at all? Absent is a reportable fact. */
265
+ get available() {
266
+ return this.layout !== null;
27
267
  }
28
268
  /**
29
- * Optional diagnostics sink for sampling errors. The first failure and
30
- * every Nth failure thereafter are reported so a permanently-broken
31
- * collector doesn't spam logs but is still observable.
269
+ * One reading, or `null` when this node has no cgroup or the cgroup stopped
270
+ * answering. `null` is ABSENT the caller contributes no point, and no
271
+ * point is what a chart must show, never a zero.
32
272
  */
33
- setDiagnostics(diag) {
34
- this.diagnostics = diag;
35
- }
36
- startSampling(intervalMs) {
37
- if (IS_DARWIN) {
38
- sampleMacMemory(this);
39
- this.macMemTimer = setInterval(() => sampleMacMemory(this), 5e3);
40
- if (this.macMemTimer.unref) this.macMemTimer.unref();
273
+ sample() {
274
+ const layout = this.resolveOnce();
275
+ if (layout === null) return null;
276
+ const currentText = this.fs.read(layout.currentPath);
277
+ if (currentText === null) {
278
+ this.noteRead(false, `unreadable: ${layout.currentPath}`);
279
+ return null;
41
280
  }
42
- this.collectSnapshot().then((s) => {
43
- this.cachedSnapshot = s;
44
- this.consecutiveSampleErrors = 0;
45
- }).catch((err) => this.reportSampleError(err));
46
- this.samplingTimer = setInterval(() => {
47
- this.collectSnapshot().then((s) => {
48
- this.cachedSnapshot = s;
49
- this.consecutiveSampleErrors = 0;
50
- }).catch((err) => this.reportSampleError(err));
51
- }, intervalMs);
52
- if (this.samplingTimer.unref) this.samplingTimer.unref();
53
- }
54
- reportSampleError(err) {
55
- this.consecutiveSampleErrors++;
56
- if (!this.diagnostics) return;
57
- if (!(this.consecutiveSampleErrors === 1 || this.consecutiveSampleErrors % 60 === 0)) return;
58
- this.diagnostics.warn("metrics-provider sample failed", {
59
- consecutiveErrors: this.consecutiveSampleErrors,
60
- error: err instanceof Error ? err.message : String(err)
281
+ const snapshot = buildContainerMemorySnapshot({
282
+ source: layout.source,
283
+ currentText,
284
+ limitText: this.fs.read(layout.limitPath),
285
+ statText: this.fs.read(layout.statPath),
286
+ gpuShmemBytes: this.readGpuShmem()
61
287
  });
288
+ if (snapshot === null) {
289
+ this.noteRead(false, `unparseable: ${layout.currentPath}`);
290
+ return null;
291
+ }
292
+ this.noteRead(true, null);
293
+ return snapshot;
62
294
  }
63
- stopSampling() {
64
- if (this.samplingTimer) {
65
- clearInterval(this.samplingTimer);
66
- this.samplingTimer = null;
295
+ /** Resolve on first use and never again — a cgroup does not move under a pid. */
296
+ resolveOnce() {
297
+ if (this.resolved) return this.layout;
298
+ this.resolved = true;
299
+ try {
300
+ this.layout = resolveCgroupMemoryLayout(this.fs);
301
+ } catch (err) {
302
+ this.layout = null;
303
+ this.logger.warn("cgroup memory resolution failed — container memory will be ABSENT", { meta: { error: require_dist.errMsg(err) } });
304
+ return null;
67
305
  }
68
- if (this.macMemTimer) {
69
- clearInterval(this.macMemTimer);
70
- this.macMemTimer = null;
306
+ if (this.layout === null) {
307
+ this.logger.info("no cgroup on this node — container memory is ABSENT, not zero, for every sample", { meta: {
308
+ platform: process.platform,
309
+ probed: "/proc/self/cgroup"
310
+ } });
311
+ return null;
71
312
  }
313
+ this.logger.info("container memory resolved from the cgroup", { meta: {
314
+ source: this.layout.source,
315
+ directory: this.layout.directory
316
+ } });
317
+ return this.layout;
72
318
  }
73
- async collectSnapshot() {
74
- const cpu = this.sampleCpu();
75
- const memory = this.getMemoryInfo();
76
- const processInfo = getProcessResources();
77
- const [gpu, network, disk, cpuTemp] = await Promise.all([
78
- getGpuInfoInternal().catch(() => null),
79
- takeNetworkIoSnapshot(),
80
- takeDiskIoSnapshot(),
81
- getCpuTemperatureInternal().catch(() => null)
82
- ]);
83
- const snapshot = {
84
- cpu,
85
- memory,
86
- gpu,
87
- network,
88
- disk,
89
- pressure: {
90
- cpu: getPressure("cpu"),
91
- memory: getPressure("memory"),
92
- io: getPressure("io")
93
- },
94
- process: processInfo,
95
- cpuTemperature: cpuTemp,
96
- timestampMs: Date.now()
97
- };
98
- this.cachedSnapshot = snapshot;
99
- return snapshot;
319
+ /**
320
+ * The GPU number, probed once.
321
+ *
322
+ * It is host-wide and lives in debugfs, so it is absent far more often than
323
+ * it is present inside every camstack container today. One `open` decides
324
+ * it for the life of the process; after that an unreadable file costs
325
+ * nothing at all, not even a syscall.
326
+ */
327
+ readGpuShmem() {
328
+ if (!this.wantGpu) return null;
329
+ if (!this.gpuProbed) {
330
+ this.gpuProbed = true;
331
+ const text = this.fs.read(I915_GEM_OBJECTS_PATH);
332
+ this.gpuReadable = text !== null && parseI915GemObjects(text) !== null;
333
+ this.logger.info(this.gpuReadable ? "i915 GEM accounting is readable — GPU-backed shmem will be reported" : "i915 GEM accounting is NOT readable — gpuShmemBytes stays UNKNOWN (this is normal: debugfs is not mounted in the container, and a node with no Intel GPU has no such file)", { meta: { path: I915_GEM_OBJECTS_PATH } });
334
+ if (!this.gpuReadable) return null;
335
+ return text === null ? null : parseI915GemObjects(text);
336
+ }
337
+ if (!this.gpuReadable) return null;
338
+ const text = this.fs.read(I915_GEM_OBJECTS_PATH);
339
+ return text === null ? null : parseI915GemObjects(text);
340
+ }
341
+ /** Log a read outcome only when it CHANGES. Never once per tick. */
342
+ noteRead(ok, reason) {
343
+ if (this.lastReadOk === ok) return;
344
+ const first = this.lastReadOk === null;
345
+ this.lastReadOk = ok;
346
+ if (!ok) {
347
+ this.logger.warn("cgroup memory stopped answering — container memory is ABSENT until it returns", { meta: { reason: reason ?? "unknown" } });
348
+ return;
349
+ }
350
+ if (!first) this.logger.info("cgroup memory is answering again");
100
351
  }
101
- async getCached() {
102
- return this.cachedSnapshot;
352
+ };
353
+ /**
354
+ * One node's fixed-capacity circular buffer of samples.
355
+ *
356
+ * Mirrors {@link LogRingBuffer}: write at `head`, wrap, and once full every
357
+ * accepted sample evicts the oldest. `rows` is carried alongside so the fleet
358
+ * budget never has to walk the samples to know what a partition costs.
359
+ */
360
+ var LoadPartition = class {
361
+ capacity;
362
+ slots;
363
+ head = 0;
364
+ count = 0;
365
+ rows = 0;
366
+ /** Newest retained sample's timestamp — the monotonic gate for `push`. */
367
+ newestAtMs = null;
368
+ constructor(capacity) {
369
+ this.capacity = capacity;
370
+ this.slots = Array.from({ length: capacity });
103
371
  }
104
- async getCurrent() {
105
- const s = this.cachedSnapshot ?? await this.collectSnapshot();
106
- const mb = (bytes) => Math.round(bytes / 1024 / 1024);
372
+ /**
373
+ * Accept a sample.
374
+ *
375
+ * A sample at or before the newest one already held is REFUSED. The bus drops
376
+ * a node's own broadcast echo, but a cross-node redelivery or a replayed
377
+ * subscription must not be able to double a point — and idempotence here is
378
+ * what lets every reader above treat the series as a set.
379
+ *
380
+ * `accepted` is reported separately from `rowDelta` on purpose: a full ring
381
+ * that evicts a sample of the same size has a delta of zero and is not a
382
+ * refusal, and conflating the two would silently stop advancing the write
383
+ * ordinal on a steady-state cluster.
384
+ */
385
+ push(sample) {
386
+ if (this.newestAtMs !== null && sample.atMs <= this.newestAtMs) return {
387
+ accepted: false,
388
+ rowDelta: 0
389
+ };
390
+ const evicted = this.count === this.capacity ? this.slots[this.head]?.processes.length ?? 0 : 0;
391
+ this.slots[this.head] = sample;
392
+ this.head = (this.head + 1) % this.capacity;
393
+ if (this.count < this.capacity) this.count++;
394
+ this.newestAtMs = sample.atMs;
395
+ const rowDelta = sample.processes.length - evicted;
396
+ this.rows += rowDelta;
107
397
  return {
108
- cpuPercent: s.cpu.total,
109
- memoryPercent: s.memory.percent,
110
- memoryUsedMB: mb(s.memory.usedBytes),
111
- memoryTotalMB: mb(s.memory.totalBytes),
112
- temperature: s.cpuTemperature ?? void 0,
113
- gpuPercent: s.gpu?.utilization,
114
- gpuMemoryPercent: s.gpu && s.gpu.memoryTotalBytes > 0 ? Math.round(s.gpu.memoryUsedBytes / s.gpu.memoryTotalBytes * 1e3) / 10 : void 0
398
+ accepted: true,
399
+ rowDelta
115
400
  };
116
401
  }
117
- async getDiskSpace(input) {
118
- return getDiskSpaceInternal(input.dirPath);
402
+ /** Drop the oldest sample. Returns the rows reclaimed (0 when empty). */
403
+ dropOldest() {
404
+ if (this.count === 0) return 0;
405
+ const index = (this.head - this.count + this.capacity) % this.capacity;
406
+ const victim = this.slots[index];
407
+ this.slots[index] = void 0;
408
+ this.count--;
409
+ const reclaimed = victim?.processes.length ?? 0;
410
+ this.rows -= reclaimed;
411
+ if (this.count === 0) this.newestAtMs = null;
412
+ return reclaimed;
119
413
  }
120
- async getGpuInfo() {
121
- return getGpuInfoInternal();
414
+ /** Oldest-first, optionally only what is strictly newer than `sinceMs`. */
415
+ list(sinceMs) {
416
+ const out = [];
417
+ for (let i = 0; i < this.count; i++) {
418
+ const index = (this.head - this.count + i + this.capacity) % this.capacity;
419
+ const sample = this.slots[index];
420
+ if (sample === void 0) continue;
421
+ if (sinceMs !== void 0 && sample.atMs <= sinceMs) continue;
422
+ out.push(sample);
423
+ }
424
+ return out;
122
425
  }
123
- async getCpuTemperature() {
124
- return getCpuTemperatureInternal();
426
+ size() {
427
+ return this.count;
125
428
  }
126
- async getProcessStats(input) {
127
- const { getPidStats } = await Promise.resolve().then(() => require("../../resource-monitor-CdnzxBLP.js")).then((n) => n.resource_monitor_exports);
128
- const raw = await getPidStats(input.pids);
129
- const result = [];
130
- for (const [pid, s] of raw) result.push({
131
- pid,
132
- cpu: s.cpu,
133
- memory: s.memory,
134
- privateBytes: s.privateBytes,
135
- sharedBytes: s.sharedBytes
136
- });
137
- return result;
429
+ rowCount() {
430
+ return this.rows;
138
431
  }
139
- sampleCpu() {
140
- const curr = aggregateCpuTimes();
141
- const prev = this.prevCpu;
142
- this.prevCpu = curr;
143
- const dUser = curr.user - prev.user;
144
- const dNice = curr.nice - prev.nice;
145
- const dSys = curr.sys - prev.sys;
146
- const dIrq = curr.irq - prev.irq;
147
- const dIdle = curr.idle - prev.idle;
148
- const dTotal = dUser + dNice + dSys + dIrq + dIdle;
149
- const pct = (v) => dTotal === 0 ? 0 : Math.round(v / dTotal * 1e3) / 10;
150
- return {
151
- total: pct(dUser + dNice + dSys + dIrq),
152
- user: pct(dUser),
153
- system: pct(dSys),
154
- irq: pct(dIrq),
155
- nice: pct(dNice),
156
- loadAvg: node_os.loadavg(),
157
- cores: node_os.cpus().length
158
- };
432
+ oldestAtMs() {
433
+ if (this.count === 0) return null;
434
+ const index = (this.head - this.count + this.capacity) % this.capacity;
435
+ return this.slots[index]?.atMs ?? null;
159
436
  }
160
- /** Cached macOS memory (set by sampleMacMemory timer). */
161
- _cachedMacMem = null;
162
- getMemoryInfo() {
163
- const total = node_os.totalmem();
164
- if (total === 0) return {
165
- percent: 0,
166
- totalBytes: 0,
167
- usedBytes: 0,
168
- availableBytes: 0,
169
- swapUsedBytes: 0,
170
- swapTotalBytes: 0
171
- };
172
- if (IS_DARWIN && this._cachedMacMem !== null) return this._cachedMacMem;
173
- const free = node_os.freemem();
174
- const used = total - free;
175
- let swapUsedBytes = 0;
176
- let swapTotalBytes = 0;
177
- if (IS_LINUX) try {
178
- const meminfo = node_fs.readFileSync("/proc/meminfo", "utf-8");
179
- const swapTotal = parseInt(meminfo.match(/SwapTotal:\s+(\d+)/)?.[1] ?? "0", 10) * 1024;
180
- const swapFree = parseInt(meminfo.match(/SwapFree:\s+(\d+)/)?.[1] ?? "0", 10) * 1024;
181
- swapTotalBytes = swapTotal;
182
- swapUsedBytes = swapTotal - swapFree;
183
- } catch {}
184
- return {
185
- percent: Math.round((total - free) / total * 1e3) / 10,
186
- totalBytes: total,
187
- usedBytes: used,
188
- availableBytes: free,
189
- swapUsedBytes,
190
- swapTotalBytes
191
- };
437
+ lastWriteAtMs() {
438
+ return this.newestAtMs;
192
439
  }
193
440
  };
194
- function aggregateCpuTimes() {
195
- const cpus = node_os.cpus();
196
- let user = 0, nice = 0, sys = 0, irq = 0, idle = 0;
197
- for (const cpu of cpus) {
198
- user += cpu.times.user;
199
- nice += cpu.times.nice;
200
- sys += cpu.times.sys;
201
- irq += cpu.times.irq;
202
- idle += cpu.times.idle;
441
+ /**
442
+ * The partitioned ring: one {@link LoadPartition} per node id.
443
+ *
444
+ * Three bounds, doing three different jobs — the same division as the log
445
+ * buffer, for the same reason:
446
+ *
447
+ * - **`samplesPerNode`** is HARD and per partition. It is what the operator
448
+ * asked for (an hour) and it can never be exceeded.
449
+ * - **`maxTotalProcessRows`** is the FLEET budget, and it is denominated in the
450
+ * thing memory is proportional to. When it bites, the oldest sample of the
451
+ * HEAVIEST partition is dropped — the node creating the pressure is the node
452
+ * that pays, and a quiet node keeps its full hour. The node just written to is
453
+ * NOT protected here: if it is the heaviest, it is the culprit.
454
+ * - **`maxNodes`** is the backstop against an unbounded key space, and it drops
455
+ * whole partitions least-recently-written first, never the one being written.
456
+ *
457
+ * Plus {@link DEFAULT_NODE_IDLE_EVICTION_MS}, which is what stops a node that
458
+ * simply stopped emitting from living in memory forever.
459
+ */
460
+ var NodeLoadRing = class {
461
+ partitions = /* @__PURE__ */ new Map();
462
+ /**
463
+ * Node id → write ordinal of its last accepted sample. A monotonic counter,
464
+ * not a clock: partition eviction must follow the order writes actually
465
+ * happened, and node clocks disagree.
466
+ */
467
+ lastWriteSeq = /* @__PURE__ */ new Map();
468
+ writeSeq = 0;
469
+ totalRows = 0;
470
+ samplesPerNode;
471
+ maxTotalProcessRows;
472
+ maxNodes;
473
+ idleEvictionMs;
474
+ now;
475
+ constructor(options = {}) {
476
+ this.samplesPerNode = options.samplesPerNode ?? 180;
477
+ this.maxTotalProcessRows = options.maxTotalProcessRows ?? 32e3;
478
+ this.maxNodes = options.maxNodes ?? 16;
479
+ this.idleEvictionMs = options.idleEvictionMs ?? 36e5;
480
+ this.now = options.now ?? Date.now;
203
481
  }
204
- return {
205
- user,
206
- nice,
207
- sys,
208
- irq,
209
- idle
210
- };
211
- }
212
- function sampleMacMemory(provider) {
213
- try {
214
- const output = (0, node_child_process.execFileSync)("vm_stat", {
215
- timeout: 2e3,
216
- encoding: "utf-8"
482
+ /**
483
+ * Retain one snapshot. `processes` is stored by reference — the payload is
484
+ * already an immutable arrival off the bus, and copying it would double the
485
+ * measured cost for nothing.
486
+ *
487
+ * Returns whether the sample was ACCEPTED — that is, whether it was new
488
+ * rather than a replay of a timestamp this node has already delivered.
489
+ *
490
+ * The return value is not diagnostics. It is the idempotence gate the
491
+ * DURABLE tier rides on (`load-series-store.ts`): only an accepted sample is
492
+ * appended to the table, which is what lets that table carry an `INTEGER`
493
+ * rowid key instead of a composite unique index over two million rows. The
494
+ * gate is one monotonic comparison in memory; the index it replaces was
495
+ * measured at ~40 bytes per row.
496
+ */
497
+ record(nodeId, atMs, processes) {
498
+ if (nodeId.length === 0) return false;
499
+ this.sweepIdle();
500
+ let partition = this.partitions.get(nodeId);
501
+ if (partition === void 0) {
502
+ partition = new LoadPartition(this.samplesPerNode);
503
+ this.partitions.set(nodeId, partition);
504
+ }
505
+ const outcome = partition.push({
506
+ atMs,
507
+ processes
217
508
  });
218
- const pageSize = 16384;
219
- const parse = (label) => {
220
- const match = output.match(new RegExp(`${label}:\\s+(\\d+)`));
221
- return match ? parseInt(match[1], 10) * pageSize : 0;
509
+ if (!outcome.accepted) {
510
+ if (partition.size() === 0) this.partitions.delete(nodeId);
511
+ return false;
512
+ }
513
+ this.totalRows += outcome.rowDelta;
514
+ this.lastWriteSeq.set(nodeId, ++this.writeSeq);
515
+ if (this.partitions.size > this.maxNodes) this.evictPartitions(this.partitions.size - this.maxNodes, nodeId);
516
+ this.enforceRowBudget();
517
+ return true;
518
+ }
519
+ /**
520
+ * Read one node's retained series, oldest-first.
521
+ *
522
+ * `sinceMs` is EXCLUSIVE: a caller passes the newest timestamp it already
523
+ * holds and gets back only what it is missing. That is the whole contract
524
+ * that lets the admin UI seed from here and then continue live without
525
+ * doubling a point it already drew.
526
+ *
527
+ * A node nobody has recorded answers with an empty series, not an error —
528
+ * unknown is the truth about a node that has not reported.
529
+ */
530
+ read(nodeId, sinceMs) {
531
+ this.sweepIdle();
532
+ const partition = this.partitions.get(nodeId);
533
+ if (partition === void 0) return {
534
+ nodeId,
535
+ samples: [],
536
+ retainedSamples: 0,
537
+ oldestAtMs: null,
538
+ capacity: this.samplesPerNode
222
539
  };
223
- const free = parse("Pages free");
224
- const inactive = parse("Pages inactive");
225
- const purgeable = parse("Pages purgeable");
226
- const speculative = parse("Pages speculative");
227
- const available = free + inactive + purgeable + speculative;
228
- const total = node_os.totalmem();
229
- const used = total - available;
230
- let swapUsedBytes = 0;
231
- let swapTotalBytes = 0;
232
- try {
233
- const swapOut = (0, node_child_process.execFileSync)("sysctl", ["-n", "vm.swapusage"], {
234
- timeout: 2e3,
235
- encoding: "utf-8"
236
- });
237
- const totalMatch = swapOut.match(/total\s*=\s*([\d.]+)M/);
238
- const usedMatch = swapOut.match(/used\s*=\s*([\d.]+)M/);
239
- if (totalMatch) swapTotalBytes = parseFloat(totalMatch[1]) * 1024 * 1024;
240
- if (usedMatch) swapUsedBytes = parseFloat(usedMatch[1]) * 1024 * 1024;
241
- } catch {}
242
- provider._cachedMacMem = {
243
- percent: Math.round(used / total * 1e3) / 10,
244
- totalBytes: total,
245
- usedBytes: used,
246
- availableBytes: available,
247
- swapUsedBytes,
248
- swapTotalBytes
540
+ return {
541
+ nodeId,
542
+ samples: partition.list(sinceMs),
543
+ retainedSamples: partition.size(),
544
+ oldestAtMs: partition.oldestAtMs(),
545
+ capacity: this.samplesPerNode
249
546
  };
250
- } catch {
251
- provider._cachedMacMem = null;
252
547
  }
253
- }
254
- function getDiskSpaceInternal(dirPath) {
255
- return new Promise((resolve, reject) => {
256
- node_fs.statfs(dirPath, (err, stats) => {
257
- if (err) return reject(err);
258
- const totalBytes = stats.blocks * stats.bsize;
259
- const availableBytes = stats.bavail * stats.bsize;
260
- const usedBytes = totalBytes - stats.bfree * stats.bsize;
261
- resolve({
262
- path: dirPath,
263
- totalBytes,
264
- usedBytes,
265
- availableBytes,
266
- percent: totalBytes > 0 ? Math.round(usedBytes / totalBytes * 1e3) / 10 : 0
267
- });
268
- });
269
- });
270
- }
271
- async function takeDiskIoSnapshot() {
272
- const now = Date.now();
273
- if (IS_LINUX) try {
274
- const data = node_fs.readFileSync("/proc/diskstats", "utf-8");
275
- let readOps = 0, readBytes = 0, writeOps = 0, writeBytes = 0;
276
- for (const line of data.split("\n")) {
277
- const parts = line.trim().split(/\s+/);
278
- if (parts.length < 14) continue;
279
- const devName = parts[2];
280
- if (/\d+$/.test(devName) && !/^nvme\d+n\d+$/.test(devName)) continue;
281
- readOps += parseInt(parts[3], 10) || 0;
282
- readBytes += (parseInt(parts[5], 10) || 0) * 512;
283
- writeOps += parseInt(parts[7], 10) || 0;
284
- writeBytes += (parseInt(parts[9], 10) || 0) * 512;
548
+ /** Node ids with a live partition. Observability for the fleet bound. */
549
+ nodeIds() {
550
+ return [...this.partitions.keys()];
551
+ }
552
+ /** Process rows retained across every partition — the number that IS memory. */
553
+ rowCount() {
554
+ return this.totalRows;
555
+ }
556
+ /** Samples retained across every partition. */
557
+ sampleCount() {
558
+ let total = 0;
559
+ for (const partition of this.partitions.values()) total += partition.size();
560
+ return total;
561
+ }
562
+ /**
563
+ * Drop partitions whose newest sample is older than the retention window.
564
+ *
565
+ * Lazy, on write and on read — never a timer. A timer would be a new
566
+ * periodic cost in a subsystem whose entire premise is that it adds none,
567
+ * and a ring that nobody writes to and nobody reads is not growing either.
568
+ */
569
+ sweepIdle() {
570
+ const cutoff = this.now() - this.idleEvictionMs;
571
+ for (const [nodeId, partition] of [...this.partitions.entries()]) {
572
+ const lastWrite = partition.lastWriteAtMs();
573
+ if (lastWrite !== null && lastWrite > cutoff) continue;
574
+ this.dropPartition(nodeId);
285
575
  }
286
- return {
287
- readBytes,
288
- writeBytes,
289
- readOps,
290
- writeOps,
291
- timestampMs: now
292
- };
293
- } catch {}
294
- return {
295
- readBytes: 0,
296
- writeBytes: 0,
297
- readOps: 0,
298
- writeOps: 0,
299
- timestampMs: now
300
- };
576
+ }
577
+ /**
578
+ * Bring the fleet back under the row budget by dropping the OLDEST sample of
579
+ * the HEAVIEST partition, repeatedly.
580
+ *
581
+ * Terminates: every iteration removes one sample from a non-empty partition,
582
+ * and the population of samples is finite and strictly decreasing.
583
+ */
584
+ enforceRowBudget() {
585
+ while (this.totalRows > this.maxTotalProcessRows) {
586
+ const victim = this.heaviestPartition();
587
+ if (victim === null) return;
588
+ const [nodeId, partition] = victim;
589
+ this.totalRows -= partition.dropOldest();
590
+ if (partition.size() === 0) this.dropPartition(nodeId);
591
+ }
592
+ }
593
+ heaviestPartition() {
594
+ let best = null;
595
+ for (const entry of this.partitions.entries()) {
596
+ if (entry[1].size() === 0) continue;
597
+ if (best === null || entry[1].rowCount() > best[1].rowCount()) best = entry;
598
+ }
599
+ return best;
600
+ }
601
+ /** Drop `count` whole partitions, least-recently-written first. */
602
+ evictPartitions(count, protectedNodeId) {
603
+ const order = [...this.partitions.keys()].filter((nodeId) => nodeId !== protectedNodeId).toSorted((a, b) => (this.lastWriteSeq.get(a) ?? 0) - (this.lastWriteSeq.get(b) ?? 0));
604
+ let remaining = count;
605
+ for (const nodeId of order) {
606
+ if (remaining <= 0) return;
607
+ this.dropPartition(nodeId);
608
+ remaining -= 1;
609
+ }
610
+ }
611
+ /** Remove a partition and everything that indexes it. */
612
+ dropPartition(nodeId) {
613
+ const partition = this.partitions.get(nodeId);
614
+ if (partition === void 0) return;
615
+ this.totalRows -= partition.rowCount();
616
+ this.partitions.delete(nodeId);
617
+ this.lastWriteSeq.delete(nodeId);
618
+ }
619
+ };
620
+ //#endregion
621
+ //#region src/builtins/native-metrics/container-memory-series.ts
622
+ /**
623
+ * @durable class=audit owner=metrics-native
624
+ * write="one row per NODE per sample, appended by the HUB's `native-metrics`
625
+ * off its subscription to `metrics.node-processes-snapshot` — the same
626
+ * subscription, the same accepted-sample gate and the same timestamps as
627
+ * `metrics:node-load-samples`, so a memory row and the process rows of the
628
+ * same instant share an `atMs`. A node with no cgroup writes NOTHING; it
629
+ * never writes a zeroed row."
630
+ * retention="The operator's `loadSeriesRetentionHours` by age and
631
+ * CONTAINER_MEMORY_MAX_ROWS by count, swept in the same pass as the process
632
+ * table and through the same bounded page. Losing the table loses the cold
633
+ * window only: the ring keeps serving the last hour."
634
+ */
635
+ var CONTAINER_MEMORY_COLLECTION = "metrics:node-memory-samples";
636
+ /**
637
+ * Fleet ceiling on the hot ring, in READINGS.
638
+ *
639
+ * One reading per node per sample, so this is simply the per-node ceiling times
640
+ * the partition ceiling the process ring already declares (16). 2 880 readings
641
+ * of ten numbers each is well under a megabyte — the bound exists so a runaway
642
+ * node id space costs a fixed amount, not because the number is large.
643
+ */
644
+ var CONTAINER_MEMORY_MAX_TOTAL_ROWS = 2880;
645
+ /**
646
+ * Hard ceiling on the durable table, in ROWS. The guarantee behind the
647
+ * retention intention, exactly as `DEFAULT_MAX_ROWS` is for the process table.
648
+ *
649
+ * | cadence | retention | rows (3 nodes) |
650
+ * | --- | --- | --- |
651
+ * | 10 s | 6 h (default) | 6 480 |
652
+ * | 5 s | 6 h | 12 960 |
653
+ * | 5 s | 72 h, 16 nodes | 829 440 — capped here |
654
+ *
655
+ * 50 000 rows is 7.7x the shipped default. This row is strictly narrower than
656
+ * a `metrics:node-load-samples` row — fewer columns, one index, every value an
657
+ * integer varint — and that one was MEASURED at 87 B/row, so 50 000 rows is
658
+ * **under 4.2 MiB** with the real figure lower still. It is not a knob: a
659
+ * guarantee that moves with the hand that raised the intention is not one.
660
+ */
661
+ var CONTAINER_MEMORY_MAX_ROWS = 5e4;
662
+ var CONTAINER_MEMORY_COLUMNS = [
663
+ {
664
+ name: "id",
665
+ type: "INTEGER",
666
+ primaryKey: true,
667
+ notNull: true
668
+ },
669
+ {
670
+ name: "nodeId",
671
+ type: "TEXT",
672
+ notNull: true
673
+ },
674
+ (
675
+ /** Shared with the process rows of the same tick — that is the whole point. */
676
+ {
677
+ name: "atMs",
678
+ type: "INTEGER",
679
+ notNull: true
680
+ }),
681
+ {
682
+ name: "source",
683
+ type: "TEXT",
684
+ notNull: true
685
+ },
686
+ (
687
+ /**
688
+ * Whole MiB, like `rssMib` on the process row and for the same reason: a
689
+ * byte-exact residency on a value in the tens of gigabytes is precision no
690
+ * chart can draw, and it costs bytes on every row forever.
691
+ */
692
+ {
693
+ name: "currentMib",
694
+ type: "INTEGER",
695
+ notNull: true
696
+ }),
697
+ (
698
+ /** `NULL` = NO LIMIT. Never a sentinel. */
699
+ {
700
+ name: "limitMib",
701
+ type: "INTEGER"
702
+ }),
703
+ (
704
+ /** `NULL` = UNKNOWN on all four. Never zero. */
705
+ {
706
+ name: "anonMib",
707
+ type: "INTEGER"
708
+ }),
709
+ {
710
+ name: "fileMib",
711
+ type: "INTEGER"
712
+ },
713
+ {
714
+ name: "shmemMib",
715
+ type: "INTEGER"
716
+ },
717
+ {
718
+ name: "slabMib",
719
+ type: "INTEGER"
720
+ },
721
+ (
722
+ /** Host-wide i915 GEM bytes. `NULL` wherever debugfs is not mounted. */
723
+ {
724
+ name: "gpuShmemMib",
725
+ type: "INTEGER"
726
+ })
727
+ ];
728
+ var CONTAINER_MEMORY_INDEXES = [(
729
+ /** The one index, and it serves both the read and the prune — see the
730
+ * process table's `idx_load_samples_node_at` for the full reasoning. */
731
+ {
732
+ name: "idx_node_memory_node_at",
733
+ columns: ["nodeId", "atMs"]
734
+ })];
735
+ var BYTES_PER_MIB$1 = 1048576;
736
+ /** Bytes → whole MiB. `null` stays `null`: unknown is not zero. */
737
+ function toMib(value) {
738
+ return value === null ? null : Math.round(value / BYTES_PER_MIB$1);
739
+ }
740
+ /** Whole MiB → bytes. `null` stays `null`. */
741
+ function fromMib(value) {
742
+ return value === null ? null : value * BYTES_PER_MIB$1;
301
743
  }
302
- async function takeNetworkIoSnapshot() {
303
- const now = Date.now();
304
- if (IS_LINUX) try {
305
- const data = node_fs.readFileSync("/proc/net/dev", "utf-8");
306
- let rxBytes = 0, txBytes = 0, rxPackets = 0, txPackets = 0, rxErrors = 0, txErrors = 0;
307
- for (const line of data.split("\n")) {
308
- const match = line.match(/^\s*(\w+):\s+(.+)$/);
309
- if (!match) continue;
310
- if (match[1] === "lo") continue;
311
- const cols = match[2].trim().split(/\s+/);
312
- rxBytes += parseInt(cols[0], 10) || 0;
313
- rxPackets += parseInt(cols[1], 10) || 0;
314
- rxErrors += parseInt(cols[2], 10) || 0;
315
- txBytes += parseInt(cols[8], 10) || 0;
316
- txPackets += parseInt(cols[9], 10) || 0;
317
- txErrors += parseInt(cols[10], 10) || 0;
318
- }
319
- return {
320
- rxBytes,
321
- txBytes,
322
- rxPackets,
323
- txPackets,
324
- rxErrors,
325
- txErrors,
326
- timestampMs: now
327
- };
328
- } catch {}
329
- if (IS_DARWIN) try {
330
- const stdout = await execAsync("netstat", ["-ib"]);
331
- let rxBytes = 0, txBytes = 0, rxPackets = 0, txPackets = 0, rxErrors = 0, txErrors = 0;
332
- for (const line of stdout.split("\n")) {
333
- const parts = line.trim().split(/\s+/);
334
- if (parts.length < 11) continue;
335
- if (parts[0] === "lo0" || parts[0] === "Name") continue;
336
- if (!parts[2]?.startsWith("Link#")) continue;
337
- rxPackets += parseInt(parts[4], 10) || 0;
338
- rxErrors += parseInt(parts[5], 10) || 0;
339
- rxBytes += parseInt(parts[6], 10) || 0;
340
- txPackets += parseInt(parts[7], 10) || 0;
341
- txErrors += parseInt(parts[8], 10) || 0;
342
- txBytes += parseInt(parts[9], 10) || 0;
343
- }
344
- return {
345
- rxBytes,
346
- txBytes,
347
- rxPackets,
348
- txPackets,
349
- rxErrors,
350
- txErrors,
351
- timestampMs: now
352
- };
353
- } catch {}
744
+ /** The ONE encode. */
745
+ function snapshotToRow(nodeId, atMs, snapshot) {
354
746
  return {
355
- rxBytes: 0,
356
- txBytes: 0,
357
- rxPackets: 0,
358
- txPackets: 0,
359
- rxErrors: 0,
360
- txErrors: 0,
361
- timestampMs: now
747
+ nodeId,
748
+ atMs,
749
+ source: snapshot.source,
750
+ currentMib: Math.round(snapshot.currentBytes / BYTES_PER_MIB$1),
751
+ limitMib: toMib(snapshot.limitBytes),
752
+ anonMib: toMib(snapshot.anonBytes),
753
+ fileMib: toMib(snapshot.fileBytes),
754
+ shmemMib: toMib(snapshot.shmemBytes),
755
+ slabMib: toMib(snapshot.slabBytes),
756
+ gpuShmemMib: toMib(snapshot.gpuShmemBytes)
362
757
  };
363
758
  }
364
- async function getCpuTemperatureInternal() {
365
- if (IS_LINUX) for (const p of ["/sys/class/thermal/thermal_zone0/temp", "/sys/class/hwmon/hwmon0/temp1_input"]) try {
366
- const raw = node_fs.readFileSync(p, "utf-8").trim();
367
- const millideg = parseInt(raw, 10);
368
- if (!isNaN(millideg)) return Math.round(millideg / 100) / 10;
369
- } catch {
370
- continue;
371
- }
372
- if (IS_DARWIN) try {
373
- const match = (await execAsync("sudo", [
374
- "-n",
375
- "powermetrics",
376
- "--samplers",
377
- "smc",
378
- "-i",
379
- "1",
380
- "-n",
381
- "1"
382
- ], 3e3)).match(/CPU die temperature:\s+([\d.]+)\s*C/);
383
- if (match) return Math.round(parseFloat(match[1]) * 10) / 10;
384
- } catch {}
385
- return null;
386
- }
387
- async function getGpuInfoInternal() {
388
- if (IS_DARWIN) try {
389
- const utilMatch = (await execAsync("ioreg", [
390
- "-r",
391
- "-d",
392
- "1",
393
- "-c",
394
- "IOAccelerator"
395
- ])).match(/"GPU Core Utilization"\s*=\s*(\d+)/);
396
- const util = utilMatch ? parseInt(utilMatch[1], 10) : 0;
397
- const utilPct = util > 1e3 ? Math.round(util / 1e7) : util;
398
- return {
399
- utilization: Math.min(100, utilPct),
400
- model: "Apple Silicon GPU",
401
- memoryUsedBytes: 0,
402
- memoryTotalBytes: 0,
403
- temperature: null
404
- };
405
- } catch {}
406
- if (IS_LINUX) try {
407
- const parts = (await execAsync("nvidia-smi", ["--query-gpu=utilization.gpu,name,memory.used,memory.total,temperature.gpu", "--format=csv,noheader,nounits"])).trim().split(",").map((s) => s.trim());
408
- if (parts.length >= 5) return {
409
- utilization: parseInt(parts[0], 10) || 0,
410
- model: parts[1] ?? "Unknown GPU",
411
- memoryUsedBytes: (parseInt(parts[2], 10) || 0) * 1024 * 1024,
412
- memoryTotalBytes: (parseInt(parts[3], 10) || 0) * 1024 * 1024,
413
- temperature: parseInt(parts[4], 10) || null
414
- };
415
- } catch {}
416
- return null;
759
+ /**
760
+ * The ONE decode. A row whose `source` is not a hierarchy this build knows is
761
+ * skipped rather than coerced — a stored string that no longer means anything
762
+ * must not be charted as though it did.
763
+ */
764
+ function rowToPoint(row) {
765
+ if (row.source !== "cgroup-v2" && row.source !== "cgroup-v1") return null;
766
+ return {
767
+ atMs: row.atMs,
768
+ source: row.source,
769
+ currentBytes: row.currentMib * BYTES_PER_MIB$1,
770
+ limitBytes: fromMib(row.limitMib),
771
+ anonBytes: fromMib(row.anonMib),
772
+ fileBytes: fromMib(row.fileMib),
773
+ shmemBytes: fromMib(row.shmemMib),
774
+ slabBytes: fromMib(row.slabMib),
775
+ gpuShmemBytes: fromMib(row.gpuShmemMib)
776
+ };
417
777
  }
418
- function getProcessResources() {
419
- let openFds = 0;
420
- let threadCount = 0;
421
- if (IS_LINUX) {
422
- try {
423
- openFds = node_fs.readdirSync(`/proc/${process.pid}/fd`).length;
424
- } catch {}
425
- try {
426
- const match = node_fs.readFileSync(`/proc/${process.pid}/status`, "utf-8").match(/Threads:\s+(\d+)/);
427
- if (match) threadCount = parseInt(match[1], 10);
428
- } catch {}
429
- }
430
- const nodeProcess = process;
778
+ /** Structural validation on read. A malformed row is skipped, never charted. */
779
+ function recordToMemoryRow(data) {
780
+ const nodeId = data["nodeId"];
781
+ const source = data["source"];
782
+ if (typeof nodeId !== "string" || typeof source !== "string") return null;
783
+ const atMs = Number(data["atMs"]);
784
+ const currentMib = Number(data["currentMib"]);
785
+ if (!Number.isFinite(atMs) || !Number.isFinite(currentMib)) return null;
786
+ const optional = (raw) => {
787
+ if (raw === null || raw === void 0) return null;
788
+ const n = Number(raw);
789
+ return Number.isFinite(n) ? n : null;
790
+ };
431
791
  return {
432
- openFds,
433
- threadCount,
434
- activeHandles: (nodeProcess._getActiveHandles?.() ?? []).length,
435
- activeRequests: (nodeProcess._getActiveRequests?.() ?? []).length
792
+ nodeId,
793
+ atMs,
794
+ source,
795
+ currentMib,
796
+ limitMib: optional(data["limitMib"]),
797
+ anonMib: optional(data["anonMib"]),
798
+ fileMib: optional(data["fileMib"]),
799
+ shmemMib: optional(data["shmemMib"]),
800
+ slabMib: optional(data["slabMib"]),
801
+ gpuShmemMib: optional(data["gpuShmemMib"])
436
802
  };
437
803
  }
438
- function getPressure(resource) {
439
- if (!IS_LINUX) return null;
440
- try {
441
- const data = node_fs.readFileSync(`/proc/pressure/${resource}`, "utf-8");
442
- const parseLine = (prefix) => {
443
- const match = data.match(new RegExp(`${prefix} avg10=(\\d+\\.\\d+) avg60=(\\d+\\.\\d+) avg300=(\\d+\\.\\d+)`));
444
- if (!match) return null;
445
- return {
446
- avg10: parseFloat(match[1]),
447
- avg60: parseFloat(match[2]),
448
- avg300: parseFloat(match[3])
449
- };
450
- };
451
- const some = parseLine("some");
452
- if (!some) return null;
453
- return {
454
- some,
455
- full: parseLine("full")
456
- };
457
- } catch {
458
- return null;
804
+ /**
805
+ * The HOT tier — a naming adapter over {@link NodeLoadRing}, not a second ring.
806
+ *
807
+ * One reading per sample, so the ring's "rows" and its samples are the same
808
+ * count and its fleet budget is a sample budget. Every bound, the idle sweep
809
+ * and the monotonic replay gate come from there unchanged.
810
+ */
811
+ var ContainerMemoryRing = class {
812
+ ring;
813
+ constructor(options = {}) {
814
+ this.ring = new NodeLoadRing({
815
+ samplesPerNode: options.samplesPerNode ?? 180,
816
+ maxTotalProcessRows: CONTAINER_MEMORY_MAX_TOTAL_ROWS,
817
+ ...options.now !== void 0 ? { now: options.now } : {}
818
+ });
459
819
  }
460
- }
461
- function execAsync(cmd, args, timeoutMs = 5e3) {
462
- return new Promise((resolve, reject) => {
463
- (0, node_child_process.execFile)(cmd, args, { timeout: timeoutMs }, (err, stdout) => {
464
- if (err) return reject(err);
465
- resolve(stdout);
820
+ /** Retain one reading. `false` = a timestamp this node already delivered. */
821
+ record(nodeId, atMs, snapshot) {
822
+ return this.ring.record(nodeId, atMs, [snapshot]);
823
+ }
824
+ /**
825
+ * Read one node's retained readings, oldest-first. `sinceMs` is EXCLUSIVE,
826
+ * matching every other reader in this subsystem.
827
+ *
828
+ * A node nobody has recorded answers EMPTY — which is what "this node has no
829
+ * cgroup" looks like, and it must stay distinguishable from a zero.
830
+ */
831
+ read(nodeId, sinceMs) {
832
+ const series = this.ring.read(nodeId, sinceMs);
833
+ const out = [];
834
+ for (const sample of series.samples) {
835
+ const snapshot = sample.processes[0];
836
+ if (snapshot === void 0) continue;
837
+ out.push({
838
+ ...snapshot,
839
+ atMs: sample.atMs
840
+ });
841
+ }
842
+ return out;
843
+ }
844
+ };
845
+ /**
846
+ * Merge the two tiers and reduce, oldest-first.
847
+ *
848
+ * Dedupe is on `atMs` with the HOT copy winning, exactly as the process reader
849
+ * does: the ring holds the reading as it arrived, the table holds it after the
850
+ * MiB encode.
851
+ *
852
+ * Reduction keeps, for each bucket, the reading with the largest
853
+ * `currentBytes` — WHOLE, every field from that one sample. A per-field
854
+ * maximum would synthesise a row whose parts never coexisted (an `anon` peak
855
+ * and a `file` peak minutes apart, summed into a total that never happened),
856
+ * and a mean would smear away the spike this series exists to find. `atMs`
857
+ * becomes the bucket START so the points lie on the same x-axis as the folded
858
+ * process series, which is the only reason both travel in one answer.
859
+ *
860
+ * An empty bucket stays ABSENT — never zero, never carried forward.
861
+ */
862
+ function reduceContainerMemory(points, bucketMs, origin) {
863
+ if (bucketMs <= 0 || points.length === 0) return points;
864
+ const buckets = /* @__PURE__ */ new Map();
865
+ for (const point of points) {
866
+ const start = origin + Math.floor((point.atMs - origin) / bucketMs) * bucketMs;
867
+ const held = buckets.get(start);
868
+ if (held === void 0 || point.currentBytes > held.currentBytes) buckets.set(start, {
869
+ ...point,
870
+ atMs: start
466
871
  });
467
- });
872
+ }
873
+ return [...buckets.values()].toSorted((a, b) => a.atMs - b.atMs);
874
+ }
875
+ /** Merge cold + hot on `atMs`, hot wins. Exported because dedup is the
876
+ * property worth testing on its own. */
877
+ function mergeContainerMemory(cold, hot) {
878
+ const byAt = /* @__PURE__ */ new Map();
879
+ for (const point of cold) byAt.set(point.atMs, point);
880
+ for (const point of hot) byAt.set(point.atMs, point);
881
+ return [...byAt.values()].toSorted((a, b) => a.atMs - b.atMs);
468
882
  }
469
883
  /**
470
- * One node's fixed-capacity circular buffer of samples.
884
+ * The COLD tier. Append-only, bounded, and never the authority on anything the
885
+ * hot ring can answer.
471
886
  *
472
- * Mirrors {@link LogRingBuffer}: write at `head`, wrap, and once full every
473
- * accepted sample evicts the oldest. `rows` is carried alongside so the fleet
474
- * budget never has to walk the samples to know what a partition costs.
887
+ * Every method is best-effort and says so in the log rather than throwing at
888
+ * the emitter: a storage hiccup must cost a gap in a chart, never a dropped
889
+ * snapshot on the bus.
475
890
  */
476
- var LoadPartition = class {
477
- capacity;
478
- slots;
479
- head = 0;
480
- count = 0;
481
- rows = 0;
482
- /** Newest retained sample's timestamp — the monotonic gate for `push`. */
483
- newestAtMs = null;
484
- constructor(capacity) {
485
- this.capacity = capacity;
486
- this.slots = Array.from({ length: capacity });
891
+ var ContainerMemoryStore = class {
892
+ declared = false;
893
+ store;
894
+ logger;
895
+ nowFn;
896
+ prunePageRows;
897
+ constructor(deps) {
898
+ this.store = deps.store;
899
+ this.logger = deps.logger;
900
+ this.nowFn = deps.now ?? (() => Date.now());
901
+ this.prunePageRows = deps.prunePageRows ?? 2e4;
902
+ }
903
+ /** Idempotently declare the collection. `false` when the store refused. */
904
+ async declare() {
905
+ if (this.declared) return true;
906
+ try {
907
+ await this.store.declareCollection.mutate({
908
+ collection: CONTAINER_MEMORY_COLLECTION,
909
+ columns: [...CONTAINER_MEMORY_COLUMNS],
910
+ indexes: CONTAINER_MEMORY_INDEXES.map((i) => ({
911
+ name: i.name,
912
+ columns: [...i.columns]
913
+ }))
914
+ });
915
+ this.declared = true;
916
+ return true;
917
+ } catch (err) {
918
+ this.logger.warn("container memory declareCollection failed — nothing will be retained on disk", { meta: {
919
+ collection: CONTAINER_MEMORY_COLLECTION,
920
+ error: require_dist.errMsg(err)
921
+ } });
922
+ return false;
923
+ }
924
+ }
925
+ /** Append ONE reading. */
926
+ async append(nodeId, atMs, snapshot) {
927
+ if (!await this.declare()) return 0;
928
+ const record = { ...snapshotToRow(nodeId, atMs, snapshot) };
929
+ try {
930
+ const { inserted } = await this.store.insertMany.mutate({
931
+ collection: CONTAINER_MEMORY_COLLECTION,
932
+ records: [{ data: record }]
933
+ });
934
+ return inserted;
935
+ } catch (err) {
936
+ this.logger.warn("container memory reading not retained — this interval will be missing", { meta: {
937
+ nodeId,
938
+ atMs,
939
+ error: require_dist.errMsg(err)
940
+ } });
941
+ return 0;
942
+ }
943
+ }
944
+ /** Read one node's cold readings, oldest-first. `sinceMs` is EXCLUSIVE. */
945
+ async read(nodeId, sinceMs, limitRows) {
946
+ if (!await this.declare()) return [];
947
+ try {
948
+ const records = await this.store.query.query({
949
+ collection: CONTAINER_MEMORY_COLLECTION,
950
+ filter: {
951
+ where: { nodeId },
952
+ whereBetween: { atMs: [sinceMs + 1, Number.MAX_SAFE_INTEGER] },
953
+ orderBy: {
954
+ field: "atMs",
955
+ direction: "asc"
956
+ },
957
+ limit: limitRows
958
+ }
959
+ });
960
+ const points = [];
961
+ for (const record of records) {
962
+ const row = recordToMemoryRow(record.data);
963
+ if (row === null) continue;
964
+ const point = rowToPoint(row);
965
+ if (point !== null) points.push(point);
966
+ }
967
+ return points;
968
+ } catch (err) {
969
+ this.logger.warn("container memory cold read failed — answering from the hot window only", { meta: {
970
+ nodeId,
971
+ sinceMs,
972
+ error: require_dist.errMsg(err)
973
+ } });
974
+ return [];
975
+ }
487
976
  }
488
977
  /**
489
- * Accept a sample.
490
- *
491
- * A sample at or before the newest one already held is REFUSED. The bus drops
492
- * a node's own broadcast echo, but a cross-node redelivery or a replayed
493
- * subscription must not be able to double a point — and idempotence here is
494
- * what lets every reader above treat the series as a set.
978
+ * Enforce BOTH bounds, oldest-first, through a bounded page each.
495
979
  *
496
- * `accepted` is reported separately from `rowDelta` on purpose: a full ring
497
- * that evicts a sample of the same size has a delta of zero and is not a
498
- * refusal, and conflating the two would silently stop advancing the write
499
- * ordinal on a steady-state cluster.
980
+ * Not rate-limited here: it is driven by the process store's own prune
981
+ * cadence, so it runs at most once a minute by construction and a second
982
+ * timer would be a second thing to get wrong.
500
983
  */
501
- push(sample) {
502
- if (this.newestAtMs !== null && sample.atMs <= this.newestAtMs) return {
503
- accepted: false,
504
- rowDelta: 0
505
- };
506
- const evicted = this.count === this.capacity ? this.slots[this.head]?.processes.length ?? 0 : 0;
507
- this.slots[this.head] = sample;
508
- this.head = (this.head + 1) % this.capacity;
509
- if (this.count < this.capacity) this.count++;
510
- this.newestAtMs = sample.atMs;
511
- const rowDelta = sample.processes.length - evicted;
512
- this.rows += rowDelta;
984
+ async prune(nodeIds, retention) {
985
+ if (!await this.declare()) return null;
986
+ const ageCutoff = this.nowFn() - retention.retentionHours * 36e5;
987
+ let deletedByAge = 0;
988
+ let rowsExamined = 0;
989
+ for (const nodeId of nodeIds) {
990
+ const outcome = await this.deleteOldestPage(nodeId, ageCutoff, this.prunePageRows);
991
+ deletedByAge += outcome.deleted;
992
+ rowsExamined += outcome.examined;
993
+ }
994
+ const total = await this.count();
995
+ const excess = total === null ? 0 : total - CONTAINER_MEMORY_MAX_ROWS;
996
+ let deletedByCap = 0;
997
+ if (excess > 0) {
998
+ let remaining = Math.min(excess, this.prunePageRows);
999
+ for (const nodeId of nodeIds) {
1000
+ if (remaining <= 0) break;
1001
+ const outcome = await this.deleteOldestPage(nodeId, null, remaining);
1002
+ deletedByCap += outcome.deleted;
1003
+ remaining -= outcome.deleted;
1004
+ rowsExamined += outcome.examined;
1005
+ }
1006
+ this.logger.warn("container memory ROW CAP bit — evicting the oldest readings", { meta: {
1007
+ collection: CONTAINER_MEMORY_COLLECTION,
1008
+ rows: total,
1009
+ cap: CONTAINER_MEMORY_MAX_ROWS,
1010
+ over: excess,
1011
+ deleted: deletedByCap,
1012
+ hint: "lower the retention or the sampling cadence — the cap is the guarantee, not the intention"
1013
+ } });
1014
+ }
513
1015
  return {
514
- accepted: true,
515
- rowDelta
1016
+ deletedByAge,
1017
+ deletedByCap,
1018
+ rowsExamined,
1019
+ capBit: excess > 0
516
1020
  };
517
1021
  }
518
- /** Drop the oldest sample. Returns the rows reclaimed (0 when empty). */
519
- dropOldest() {
520
- if (this.count === 0) return 0;
521
- const index = (this.head - this.count + this.capacity) % this.capacity;
522
- const victim = this.slots[index];
523
- this.slots[index] = void 0;
524
- this.count--;
525
- const reclaimed = victim?.processes.length ?? 0;
526
- this.rows -= reclaimed;
527
- if (this.count === 0) this.newestAtMs = null;
528
- return reclaimed;
529
- }
530
- /** Oldest-first, optionally only what is strictly newer than `sinceMs`. */
531
- list(sinceMs) {
532
- const out = [];
533
- for (let i = 0; i < this.count; i++) {
534
- const index = (this.head - this.count + i + this.capacity) % this.capacity;
535
- const sample = this.slots[index];
536
- if (sample === void 0) continue;
537
- if (sinceMs !== void 0 && sample.atMs <= sinceMs) continue;
538
- out.push(sample);
1022
+ /** Total rows, or `null` when the store could not answer. */
1023
+ async count() {
1024
+ if (!await this.declare()) return null;
1025
+ try {
1026
+ return await this.store.count.query({ collection: CONTAINER_MEMORY_COLLECTION });
1027
+ } catch (err) {
1028
+ this.logger.warn("container memory count failed — the row cap is not enforced this pass", { meta: { error: require_dist.errMsg(err) } });
1029
+ return null;
539
1030
  }
540
- return out;
541
- }
542
- size() {
543
- return this.count;
544
- }
545
- rowCount() {
546
- return this.rows;
547
- }
548
- oldestAtMs() {
549
- if (this.count === 0) return null;
550
- const index = (this.head - this.count + this.capacity) % this.capacity;
551
- return this.slots[index]?.atMs ?? null;
552
- }
553
- lastWriteAtMs() {
554
- return this.newestAtMs;
555
1031
  }
556
- };
557
- /**
558
- * The partitioned ring: one {@link LoadPartition} per node id.
559
- *
560
- * Three bounds, doing three different jobs — the same division as the log
561
- * buffer, for the same reason:
562
- *
563
- * - **`samplesPerNode`** is HARD and per partition. It is what the operator
564
- * asked for (an hour) and it can never be exceeded.
565
- * - **`maxTotalProcessRows`** is the FLEET budget, and it is denominated in the
566
- * thing memory is proportional to. When it bites, the oldest sample of the
567
- * HEAVIEST partition is dropped — the node creating the pressure is the node
568
- * that pays, and a quiet node keeps its full hour. The node just written to is
569
- * NOT protected here: if it is the heaviest, it is the culprit.
570
- * - **`maxNodes`** is the backstop against an unbounded key space, and it drops
571
- * whole partitions least-recently-written first, never the one being written.
572
- *
573
- * Plus {@link DEFAULT_NODE_IDLE_EVICTION_MS}, which is what stops a node that
574
- * simply stopped emitting from living in memory forever.
575
- */
576
- var NodeLoadRing = class {
577
- partitions = /* @__PURE__ */ new Map();
578
1032
  /**
579
- * Node id write ordinal of its last accepted sample. A monotonic counter,
580
- * not a clock: partition eviction must follow the order writes actually
581
- * happened, and node clocks disagree.
1033
+ * Delete this node's oldest rows, at most one page's worth.
1034
+ *
1035
+ * `cutoff === null` means "the oldest rows, whatever their age" — the
1036
+ * row-cap path, which is bounded by `limitRows` instead. Either way the page
1037
+ * is READ FIRST, ordered by the one index, and its last `atMs` becomes the
1038
+ * effective cutoff for a single bounded delete. An unbounded `deleteWhere`
1039
+ * would be one statement and one stalling transaction the first time a
1040
+ * retention is lowered.
582
1041
  */
583
- lastWriteSeq = /* @__PURE__ */ new Map();
584
- writeSeq = 0;
585
- totalRows = 0;
586
- samplesPerNode;
587
- maxTotalProcessRows;
588
- maxNodes;
589
- idleEvictionMs;
590
- now;
591
- constructor(options = {}) {
592
- this.samplesPerNode = options.samplesPerNode ?? 180;
593
- this.maxTotalProcessRows = options.maxTotalProcessRows ?? 32e3;
594
- this.maxNodes = options.maxNodes ?? 16;
595
- this.idleEvictionMs = options.idleEvictionMs ?? 36e5;
596
- this.now = options.now ?? Date.now;
1042
+ async deleteOldestPage(nodeId, cutoff, limitRows) {
1043
+ if (limitRows <= 0) return {
1044
+ deleted: 0,
1045
+ examined: 0
1046
+ };
1047
+ try {
1048
+ const page = await this.store.query.query({
1049
+ collection: CONTAINER_MEMORY_COLLECTION,
1050
+ filter: {
1051
+ where: { nodeId },
1052
+ ...cutoff === null ? {} : { whereBetween: { atMs: [0, cutoff] } },
1053
+ orderBy: {
1054
+ field: "atMs",
1055
+ direction: "asc"
1056
+ },
1057
+ limit: Math.min(limitRows, this.prunePageRows)
1058
+ },
1059
+ columns: ["atMs"]
1060
+ });
1061
+ if (page.length === 0) return {
1062
+ deleted: 0,
1063
+ examined: 0
1064
+ };
1065
+ const effectiveCutoff = Number(page.at(-1)?.data["atMs"]);
1066
+ if (!Number.isFinite(effectiveCutoff)) return {
1067
+ deleted: 0,
1068
+ examined: page.length
1069
+ };
1070
+ const { deleted } = await this.store.deleteWhere.mutate({
1071
+ collection: CONTAINER_MEMORY_COLLECTION,
1072
+ filter: {
1073
+ where: { nodeId },
1074
+ whereBetween: { atMs: [0, effectiveCutoff] }
1075
+ }
1076
+ });
1077
+ return {
1078
+ deleted,
1079
+ examined: page.length
1080
+ };
1081
+ } catch (err) {
1082
+ this.logger.warn("container memory prune failed — the table keeps growing this pass", { meta: {
1083
+ nodeId,
1084
+ cutoff,
1085
+ error: require_dist.errMsg(err)
1086
+ } });
1087
+ return {
1088
+ deleted: 0,
1089
+ examined: 0
1090
+ };
1091
+ }
1092
+ }
1093
+ };
1094
+ //#endregion
1095
+ //#region src/builtins/native-metrics/native-metrics-provider.ts
1096
+ var IS_DARWIN = node_os.platform() === "darwin";
1097
+ var IS_LINUX = node_os.platform() === "linux";
1098
+ var NativeMetricsProvider = class {
1099
+ id = "native";
1100
+ cachedSnapshot = null;
1101
+ samplingTimer = null;
1102
+ macMemTimer = null;
1103
+ prevCpu;
1104
+ diagnostics = null;
1105
+ consecutiveSampleErrors = 0;
1106
+ constructor() {
1107
+ this.prevCpu = aggregateCpuTimes();
597
1108
  }
598
1109
  /**
599
- * Retain one snapshot. `processes` is stored by reference the payload is
600
- * already an immutable arrival off the bus, and copying it would double the
601
- * measured cost for nothing.
602
- *
603
- * Returns whether the sample was ACCEPTED — that is, whether it was new
604
- * rather than a replay of a timestamp this node has already delivered.
605
- *
606
- * The return value is not diagnostics. It is the idempotence gate the
607
- * DURABLE tier rides on (`load-series-store.ts`): only an accepted sample is
608
- * appended to the table, which is what lets that table carry an `INTEGER`
609
- * rowid key instead of a composite unique index over two million rows. The
610
- * gate is one monotonic comparison in memory; the index it replaces was
611
- * measured at ~40 bytes per row.
1110
+ * Optional diagnostics sink for sampling errors. The first failure and
1111
+ * every Nth failure thereafter are reported so a permanently-broken
1112
+ * collector doesn't spam logs but is still observable.
612
1113
  */
613
- record(nodeId, atMs, processes) {
614
- if (nodeId.length === 0) return false;
615
- this.sweepIdle();
616
- let partition = this.partitions.get(nodeId);
617
- if (partition === void 0) {
618
- partition = new LoadPartition(this.samplesPerNode);
619
- this.partitions.set(nodeId, partition);
1114
+ setDiagnostics(diag) {
1115
+ this.diagnostics = diag;
1116
+ }
1117
+ startSampling(intervalMs) {
1118
+ if (IS_DARWIN) {
1119
+ sampleMacMemory(this);
1120
+ this.macMemTimer = setInterval(() => sampleMacMemory(this), 5e3);
1121
+ if (this.macMemTimer.unref) this.macMemTimer.unref();
620
1122
  }
621
- const outcome = partition.push({
622
- atMs,
623
- processes
1123
+ this.collectSnapshot().then((s) => {
1124
+ this.cachedSnapshot = s;
1125
+ this.consecutiveSampleErrors = 0;
1126
+ }).catch((err) => this.reportSampleError(err));
1127
+ this.samplingTimer = setInterval(() => {
1128
+ this.collectSnapshot().then((s) => {
1129
+ this.cachedSnapshot = s;
1130
+ this.consecutiveSampleErrors = 0;
1131
+ }).catch((err) => this.reportSampleError(err));
1132
+ }, intervalMs);
1133
+ if (this.samplingTimer.unref) this.samplingTimer.unref();
1134
+ }
1135
+ reportSampleError(err) {
1136
+ this.consecutiveSampleErrors++;
1137
+ if (!this.diagnostics) return;
1138
+ if (!(this.consecutiveSampleErrors === 1 || this.consecutiveSampleErrors % 60 === 0)) return;
1139
+ this.diagnostics.warn("metrics-provider sample failed", {
1140
+ consecutiveErrors: this.consecutiveSampleErrors,
1141
+ error: err instanceof Error ? err.message : String(err)
624
1142
  });
625
- if (!outcome.accepted) {
626
- if (partition.size() === 0) this.partitions.delete(nodeId);
627
- return false;
1143
+ }
1144
+ stopSampling() {
1145
+ if (this.samplingTimer) {
1146
+ clearInterval(this.samplingTimer);
1147
+ this.samplingTimer = null;
1148
+ }
1149
+ if (this.macMemTimer) {
1150
+ clearInterval(this.macMemTimer);
1151
+ this.macMemTimer = null;
628
1152
  }
629
- this.totalRows += outcome.rowDelta;
630
- this.lastWriteSeq.set(nodeId, ++this.writeSeq);
631
- if (this.partitions.size > this.maxNodes) this.evictPartitions(this.partitions.size - this.maxNodes, nodeId);
632
- this.enforceRowBudget();
633
- return true;
634
1153
  }
635
- /**
636
- * Read one node's retained series, oldest-first.
637
- *
638
- * `sinceMs` is EXCLUSIVE: a caller passes the newest timestamp it already
639
- * holds and gets back only what it is missing. That is the whole contract
640
- * that lets the admin UI seed from here and then continue live without
641
- * doubling a point it already drew.
642
- *
643
- * A node nobody has recorded answers with an empty series, not an error —
644
- * unknown is the truth about a node that has not reported.
645
- */
646
- read(nodeId, sinceMs) {
647
- this.sweepIdle();
648
- const partition = this.partitions.get(nodeId);
649
- if (partition === void 0) return {
650
- nodeId,
651
- samples: [],
652
- retainedSamples: 0,
653
- oldestAtMs: null,
654
- capacity: this.samplesPerNode
1154
+ async collectSnapshot() {
1155
+ const cpu = this.sampleCpu();
1156
+ const memory = this.getMemoryInfo();
1157
+ const processInfo = getProcessResources();
1158
+ const [gpu, network, disk, cpuTemp] = await Promise.all([
1159
+ getGpuInfoInternal().catch(() => null),
1160
+ takeNetworkIoSnapshot(),
1161
+ takeDiskIoSnapshot(),
1162
+ getCpuTemperatureInternal().catch(() => null)
1163
+ ]);
1164
+ const snapshot = {
1165
+ cpu,
1166
+ memory,
1167
+ gpu,
1168
+ network,
1169
+ disk,
1170
+ pressure: {
1171
+ cpu: getPressure("cpu"),
1172
+ memory: getPressure("memory"),
1173
+ io: getPressure("io")
1174
+ },
1175
+ process: processInfo,
1176
+ cpuTemperature: cpuTemp,
1177
+ timestampMs: Date.now()
655
1178
  };
1179
+ this.cachedSnapshot = snapshot;
1180
+ return snapshot;
1181
+ }
1182
+ async getCached() {
1183
+ return this.cachedSnapshot;
1184
+ }
1185
+ async getCurrent() {
1186
+ const s = this.cachedSnapshot ?? await this.collectSnapshot();
1187
+ const mb = (bytes) => Math.round(bytes / 1024 / 1024);
656
1188
  return {
657
- nodeId,
658
- samples: partition.list(sinceMs),
659
- retainedSamples: partition.size(),
660
- oldestAtMs: partition.oldestAtMs(),
661
- capacity: this.samplesPerNode
1189
+ cpuPercent: s.cpu.total,
1190
+ memoryPercent: s.memory.percent,
1191
+ memoryUsedMB: mb(s.memory.usedBytes),
1192
+ memoryTotalMB: mb(s.memory.totalBytes),
1193
+ temperature: s.cpuTemperature ?? void 0,
1194
+ gpuPercent: s.gpu?.utilization,
1195
+ gpuMemoryPercent: s.gpu && s.gpu.memoryTotalBytes > 0 ? Math.round(s.gpu.memoryUsedBytes / s.gpu.memoryTotalBytes * 1e3) / 10 : void 0
662
1196
  };
663
1197
  }
664
- /** Node ids with a live partition. Observability for the fleet bound. */
665
- nodeIds() {
666
- return [...this.partitions.keys()];
1198
+ async getDiskSpace(input) {
1199
+ return getDiskSpaceInternal(input.dirPath);
667
1200
  }
668
- /** Process rows retained across every partition — the number that IS memory. */
669
- rowCount() {
670
- return this.totalRows;
1201
+ async getGpuInfo() {
1202
+ return getGpuInfoInternal();
671
1203
  }
672
- /** Samples retained across every partition. */
673
- sampleCount() {
674
- let total = 0;
675
- for (const partition of this.partitions.values()) total += partition.size();
676
- return total;
1204
+ async getCpuTemperature() {
1205
+ return getCpuTemperatureInternal();
677
1206
  }
678
- /**
679
- * Drop partitions whose newest sample is older than the retention window.
680
- *
681
- * Lazy, on write and on read — never a timer. A timer would be a new
682
- * periodic cost in a subsystem whose entire premise is that it adds none,
683
- * and a ring that nobody writes to and nobody reads is not growing either.
684
- */
685
- sweepIdle() {
686
- const cutoff = this.now() - this.idleEvictionMs;
687
- for (const [nodeId, partition] of [...this.partitions.entries()]) {
688
- const lastWrite = partition.lastWriteAtMs();
689
- if (lastWrite !== null && lastWrite > cutoff) continue;
690
- this.dropPartition(nodeId);
1207
+ async getProcessStats(input) {
1208
+ const { getPidStats } = await Promise.resolve().then(() => require("../../resource-monitor-CdnzxBLP.js")).then((n) => n.resource_monitor_exports);
1209
+ const raw = await getPidStats(input.pids);
1210
+ const result = [];
1211
+ for (const [pid, s] of raw) result.push({
1212
+ pid,
1213
+ cpu: s.cpu,
1214
+ memory: s.memory,
1215
+ privateBytes: s.privateBytes,
1216
+ sharedBytes: s.sharedBytes
1217
+ });
1218
+ return result;
1219
+ }
1220
+ sampleCpu() {
1221
+ const curr = aggregateCpuTimes();
1222
+ const prev = this.prevCpu;
1223
+ this.prevCpu = curr;
1224
+ const dUser = curr.user - prev.user;
1225
+ const dNice = curr.nice - prev.nice;
1226
+ const dSys = curr.sys - prev.sys;
1227
+ const dIrq = curr.irq - prev.irq;
1228
+ const dIdle = curr.idle - prev.idle;
1229
+ const dTotal = dUser + dNice + dSys + dIrq + dIdle;
1230
+ const pct = (v) => dTotal === 0 ? 0 : Math.round(v / dTotal * 1e3) / 10;
1231
+ return {
1232
+ total: pct(dUser + dNice + dSys + dIrq),
1233
+ user: pct(dUser),
1234
+ system: pct(dSys),
1235
+ irq: pct(dIrq),
1236
+ nice: pct(dNice),
1237
+ loadAvg: node_os.loadavg(),
1238
+ cores: node_os.cpus().length
1239
+ };
1240
+ }
1241
+ /** Cached macOS memory (set by sampleMacMemory timer). */
1242
+ _cachedMacMem = null;
1243
+ getMemoryInfo() {
1244
+ const total = node_os.totalmem();
1245
+ if (total === 0) return {
1246
+ percent: 0,
1247
+ totalBytes: 0,
1248
+ usedBytes: 0,
1249
+ availableBytes: 0,
1250
+ swapUsedBytes: 0,
1251
+ swapTotalBytes: 0
1252
+ };
1253
+ if (IS_DARWIN && this._cachedMacMem !== null) return this._cachedMacMem;
1254
+ const free = node_os.freemem();
1255
+ const used = total - free;
1256
+ let swapUsedBytes = 0;
1257
+ let swapTotalBytes = 0;
1258
+ if (IS_LINUX) try {
1259
+ const meminfo = node_fs.readFileSync("/proc/meminfo", "utf-8");
1260
+ const swapTotal = parseInt(meminfo.match(/SwapTotal:\s+(\d+)/)?.[1] ?? "0", 10) * 1024;
1261
+ const swapFree = parseInt(meminfo.match(/SwapFree:\s+(\d+)/)?.[1] ?? "0", 10) * 1024;
1262
+ swapTotalBytes = swapTotal;
1263
+ swapUsedBytes = swapTotal - swapFree;
1264
+ } catch {}
1265
+ return {
1266
+ percent: Math.round((total - free) / total * 1e3) / 10,
1267
+ totalBytes: total,
1268
+ usedBytes: used,
1269
+ availableBytes: free,
1270
+ swapUsedBytes,
1271
+ swapTotalBytes
1272
+ };
1273
+ }
1274
+ };
1275
+ function aggregateCpuTimes() {
1276
+ const cpus = node_os.cpus();
1277
+ let user = 0, nice = 0, sys = 0, irq = 0, idle = 0;
1278
+ for (const cpu of cpus) {
1279
+ user += cpu.times.user;
1280
+ nice += cpu.times.nice;
1281
+ sys += cpu.times.sys;
1282
+ irq += cpu.times.irq;
1283
+ idle += cpu.times.idle;
1284
+ }
1285
+ return {
1286
+ user,
1287
+ nice,
1288
+ sys,
1289
+ irq,
1290
+ idle
1291
+ };
1292
+ }
1293
+ function sampleMacMemory(provider) {
1294
+ try {
1295
+ const output = (0, node_child_process.execFileSync)("vm_stat", {
1296
+ timeout: 2e3,
1297
+ encoding: "utf-8"
1298
+ });
1299
+ const pageSize = 16384;
1300
+ const parse = (label) => {
1301
+ const match = output.match(new RegExp(`${label}:\\s+(\\d+)`));
1302
+ return match ? parseInt(match[1], 10) * pageSize : 0;
1303
+ };
1304
+ const free = parse("Pages free");
1305
+ const inactive = parse("Pages inactive");
1306
+ const purgeable = parse("Pages purgeable");
1307
+ const speculative = parse("Pages speculative");
1308
+ const available = free + inactive + purgeable + speculative;
1309
+ const total = node_os.totalmem();
1310
+ const used = total - available;
1311
+ let swapUsedBytes = 0;
1312
+ let swapTotalBytes = 0;
1313
+ try {
1314
+ const swapOut = (0, node_child_process.execFileSync)("sysctl", ["-n", "vm.swapusage"], {
1315
+ timeout: 2e3,
1316
+ encoding: "utf-8"
1317
+ });
1318
+ const totalMatch = swapOut.match(/total\s*=\s*([\d.]+)M/);
1319
+ const usedMatch = swapOut.match(/used\s*=\s*([\d.]+)M/);
1320
+ if (totalMatch) swapTotalBytes = parseFloat(totalMatch[1]) * 1024 * 1024;
1321
+ if (usedMatch) swapUsedBytes = parseFloat(usedMatch[1]) * 1024 * 1024;
1322
+ } catch {}
1323
+ provider._cachedMacMem = {
1324
+ percent: Math.round(used / total * 1e3) / 10,
1325
+ totalBytes: total,
1326
+ usedBytes: used,
1327
+ availableBytes: available,
1328
+ swapUsedBytes,
1329
+ swapTotalBytes
1330
+ };
1331
+ } catch {
1332
+ provider._cachedMacMem = null;
1333
+ }
1334
+ }
1335
+ function getDiskSpaceInternal(dirPath) {
1336
+ return new Promise((resolve, reject) => {
1337
+ node_fs.statfs(dirPath, (err, stats) => {
1338
+ if (err) return reject(err);
1339
+ const totalBytes = stats.blocks * stats.bsize;
1340
+ const availableBytes = stats.bavail * stats.bsize;
1341
+ const usedBytes = totalBytes - stats.bfree * stats.bsize;
1342
+ resolve({
1343
+ path: dirPath,
1344
+ totalBytes,
1345
+ usedBytes,
1346
+ availableBytes,
1347
+ percent: totalBytes > 0 ? Math.round(usedBytes / totalBytes * 1e3) / 10 : 0
1348
+ });
1349
+ });
1350
+ });
1351
+ }
1352
+ async function takeDiskIoSnapshot() {
1353
+ const now = Date.now();
1354
+ if (IS_LINUX) try {
1355
+ const data = node_fs.readFileSync("/proc/diskstats", "utf-8");
1356
+ let readOps = 0, readBytes = 0, writeOps = 0, writeBytes = 0;
1357
+ for (const line of data.split("\n")) {
1358
+ const parts = line.trim().split(/\s+/);
1359
+ if (parts.length < 14) continue;
1360
+ const devName = parts[2];
1361
+ if (/\d+$/.test(devName) && !/^nvme\d+n\d+$/.test(devName)) continue;
1362
+ readOps += parseInt(parts[3], 10) || 0;
1363
+ readBytes += (parseInt(parts[5], 10) || 0) * 512;
1364
+ writeOps += parseInt(parts[7], 10) || 0;
1365
+ writeBytes += (parseInt(parts[9], 10) || 0) * 512;
691
1366
  }
692
- }
693
- /**
694
- * Bring the fleet back under the row budget by dropping the OLDEST sample of
695
- * the HEAVIEST partition, repeatedly.
696
- *
697
- * Terminates: every iteration removes one sample from a non-empty partition,
698
- * and the population of samples is finite and strictly decreasing.
699
- */
700
- enforceRowBudget() {
701
- while (this.totalRows > this.maxTotalProcessRows) {
702
- const victim = this.heaviestPartition();
703
- if (victim === null) return;
704
- const [nodeId, partition] = victim;
705
- this.totalRows -= partition.dropOldest();
706
- if (partition.size() === 0) this.dropPartition(nodeId);
1367
+ return {
1368
+ readBytes,
1369
+ writeBytes,
1370
+ readOps,
1371
+ writeOps,
1372
+ timestampMs: now
1373
+ };
1374
+ } catch {}
1375
+ return {
1376
+ readBytes: 0,
1377
+ writeBytes: 0,
1378
+ readOps: 0,
1379
+ writeOps: 0,
1380
+ timestampMs: now
1381
+ };
1382
+ }
1383
+ async function takeNetworkIoSnapshot() {
1384
+ const now = Date.now();
1385
+ if (IS_LINUX) try {
1386
+ const data = node_fs.readFileSync("/proc/net/dev", "utf-8");
1387
+ let rxBytes = 0, txBytes = 0, rxPackets = 0, txPackets = 0, rxErrors = 0, txErrors = 0;
1388
+ for (const line of data.split("\n")) {
1389
+ const match = line.match(/^\s*(\w+):\s+(.+)$/);
1390
+ if (!match) continue;
1391
+ if (match[1] === "lo") continue;
1392
+ const cols = match[2].trim().split(/\s+/);
1393
+ rxBytes += parseInt(cols[0], 10) || 0;
1394
+ rxPackets += parseInt(cols[1], 10) || 0;
1395
+ rxErrors += parseInt(cols[2], 10) || 0;
1396
+ txBytes += parseInt(cols[8], 10) || 0;
1397
+ txPackets += parseInt(cols[9], 10) || 0;
1398
+ txErrors += parseInt(cols[10], 10) || 0;
707
1399
  }
708
- }
709
- heaviestPartition() {
710
- let best = null;
711
- for (const entry of this.partitions.entries()) {
712
- if (entry[1].size() === 0) continue;
713
- if (best === null || entry[1].rowCount() > best[1].rowCount()) best = entry;
1400
+ return {
1401
+ rxBytes,
1402
+ txBytes,
1403
+ rxPackets,
1404
+ txPackets,
1405
+ rxErrors,
1406
+ txErrors,
1407
+ timestampMs: now
1408
+ };
1409
+ } catch {}
1410
+ if (IS_DARWIN) try {
1411
+ const stdout = await execAsync("netstat", ["-ib"]);
1412
+ let rxBytes = 0, txBytes = 0, rxPackets = 0, txPackets = 0, rxErrors = 0, txErrors = 0;
1413
+ for (const line of stdout.split("\n")) {
1414
+ const parts = line.trim().split(/\s+/);
1415
+ if (parts.length < 11) continue;
1416
+ if (parts[0] === "lo0" || parts[0] === "Name") continue;
1417
+ if (!parts[2]?.startsWith("Link#")) continue;
1418
+ rxPackets += parseInt(parts[4], 10) || 0;
1419
+ rxErrors += parseInt(parts[5], 10) || 0;
1420
+ rxBytes += parseInt(parts[6], 10) || 0;
1421
+ txPackets += parseInt(parts[7], 10) || 0;
1422
+ txErrors += parseInt(parts[8], 10) || 0;
1423
+ txBytes += parseInt(parts[9], 10) || 0;
714
1424
  }
715
- return best;
1425
+ return {
1426
+ rxBytes,
1427
+ txBytes,
1428
+ rxPackets,
1429
+ txPackets,
1430
+ rxErrors,
1431
+ txErrors,
1432
+ timestampMs: now
1433
+ };
1434
+ } catch {}
1435
+ return {
1436
+ rxBytes: 0,
1437
+ txBytes: 0,
1438
+ rxPackets: 0,
1439
+ txPackets: 0,
1440
+ rxErrors: 0,
1441
+ txErrors: 0,
1442
+ timestampMs: now
1443
+ };
1444
+ }
1445
+ async function getCpuTemperatureInternal() {
1446
+ if (IS_LINUX) for (const p of ["/sys/class/thermal/thermal_zone0/temp", "/sys/class/hwmon/hwmon0/temp1_input"]) try {
1447
+ const raw = node_fs.readFileSync(p, "utf-8").trim();
1448
+ const millideg = parseInt(raw, 10);
1449
+ if (!isNaN(millideg)) return Math.round(millideg / 100) / 10;
1450
+ } catch {
1451
+ continue;
716
1452
  }
717
- /** Drop `count` whole partitions, least-recently-written first. */
718
- evictPartitions(count, protectedNodeId) {
719
- const order = [...this.partitions.keys()].filter((nodeId) => nodeId !== protectedNodeId).toSorted((a, b) => (this.lastWriteSeq.get(a) ?? 0) - (this.lastWriteSeq.get(b) ?? 0));
720
- let remaining = count;
721
- for (const nodeId of order) {
722
- if (remaining <= 0) return;
723
- this.dropPartition(nodeId);
724
- remaining -= 1;
725
- }
1453
+ if (IS_DARWIN) try {
1454
+ const match = (await execAsync("sudo", [
1455
+ "-n",
1456
+ "powermetrics",
1457
+ "--samplers",
1458
+ "smc",
1459
+ "-i",
1460
+ "1",
1461
+ "-n",
1462
+ "1"
1463
+ ], 3e3)).match(/CPU die temperature:\s+([\d.]+)\s*C/);
1464
+ if (match) return Math.round(parseFloat(match[1]) * 10) / 10;
1465
+ } catch {}
1466
+ return null;
1467
+ }
1468
+ async function getGpuInfoInternal() {
1469
+ if (IS_DARWIN) try {
1470
+ const utilMatch = (await execAsync("ioreg", [
1471
+ "-r",
1472
+ "-d",
1473
+ "1",
1474
+ "-c",
1475
+ "IOAccelerator"
1476
+ ])).match(/"GPU Core Utilization"\s*=\s*(\d+)/);
1477
+ const util = utilMatch ? parseInt(utilMatch[1], 10) : 0;
1478
+ const utilPct = util > 1e3 ? Math.round(util / 1e7) : util;
1479
+ return {
1480
+ utilization: Math.min(100, utilPct),
1481
+ model: "Apple Silicon GPU",
1482
+ memoryUsedBytes: 0,
1483
+ memoryTotalBytes: 0,
1484
+ temperature: null
1485
+ };
1486
+ } catch {}
1487
+ if (IS_LINUX) try {
1488
+ const parts = (await execAsync("nvidia-smi", ["--query-gpu=utilization.gpu,name,memory.used,memory.total,temperature.gpu", "--format=csv,noheader,nounits"])).trim().split(",").map((s) => s.trim());
1489
+ if (parts.length >= 5) return {
1490
+ utilization: parseInt(parts[0], 10) || 0,
1491
+ model: parts[1] ?? "Unknown GPU",
1492
+ memoryUsedBytes: (parseInt(parts[2], 10) || 0) * 1024 * 1024,
1493
+ memoryTotalBytes: (parseInt(parts[3], 10) || 0) * 1024 * 1024,
1494
+ temperature: parseInt(parts[4], 10) || null
1495
+ };
1496
+ } catch {}
1497
+ return null;
1498
+ }
1499
+ function getProcessResources() {
1500
+ let openFds = 0;
1501
+ let threadCount = 0;
1502
+ if (IS_LINUX) {
1503
+ try {
1504
+ openFds = node_fs.readdirSync(`/proc/${process.pid}/fd`).length;
1505
+ } catch {}
1506
+ try {
1507
+ const match = node_fs.readFileSync(`/proc/${process.pid}/status`, "utf-8").match(/Threads:\s+(\d+)/);
1508
+ if (match) threadCount = parseInt(match[1], 10);
1509
+ } catch {}
726
1510
  }
727
- /** Remove a partition and everything that indexes it. */
728
- dropPartition(nodeId) {
729
- const partition = this.partitions.get(nodeId);
730
- if (partition === void 0) return;
731
- this.totalRows -= partition.rowCount();
732
- this.partitions.delete(nodeId);
733
- this.lastWriteSeq.delete(nodeId);
1511
+ const nodeProcess = process;
1512
+ return {
1513
+ openFds,
1514
+ threadCount,
1515
+ activeHandles: (nodeProcess._getActiveHandles?.() ?? []).length,
1516
+ activeRequests: (nodeProcess._getActiveRequests?.() ?? []).length
1517
+ };
1518
+ }
1519
+ function getPressure(resource) {
1520
+ if (!IS_LINUX) return null;
1521
+ try {
1522
+ const data = node_fs.readFileSync(`/proc/pressure/${resource}`, "utf-8");
1523
+ const parseLine = (prefix) => {
1524
+ const match = data.match(new RegExp(`${prefix} avg10=(\\d+\\.\\d+) avg60=(\\d+\\.\\d+) avg300=(\\d+\\.\\d+)`));
1525
+ if (!match) return null;
1526
+ return {
1527
+ avg10: parseFloat(match[1]),
1528
+ avg60: parseFloat(match[2]),
1529
+ avg300: parseFloat(match[3])
1530
+ };
1531
+ };
1532
+ const some = parseLine("some");
1533
+ if (!some) return null;
1534
+ return {
1535
+ some,
1536
+ full: parseLine("full")
1537
+ };
1538
+ } catch {
1539
+ return null;
734
1540
  }
735
- };
1541
+ }
1542
+ function execAsync(cmd, args, timeoutMs = 5e3) {
1543
+ return new Promise((resolve, reject) => {
1544
+ (0, node_child_process.execFile)(cmd, args, { timeout: timeoutMs }, (err, stdout) => {
1545
+ if (err) return reject(err);
1546
+ resolve(stdout);
1547
+ });
1548
+ });
1549
+ }
736
1550
  //#endregion
737
1551
  //#region src/builtins/native-metrics/load-series-store.ts
738
1552
  /**
@@ -1740,6 +2554,20 @@ var NativeMetricsAddon = class NativeMetricsAddon extends require_dist.BaseAddon
1740
2554
  * per sample, to reach that very same table.
1741
2555
  */
1742
2556
  loadStore = null;
2557
+ /**
2558
+ * The container's OWN memory, sampled on the process-snapshot tick.
2559
+ *
2560
+ * It rides that tick rather than a timer of its own so a memory reading and
2561
+ * the process rows it must be compared against carry the SAME timestamp —
2562
+ * the whole question is a subtraction, and two clocks make it an estimate.
2563
+ * Constructed in `onInitialize`, where the logger exists; it resolves the
2564
+ * cgroup once and latches.
2565
+ */
2566
+ memoryProbe = null;
2567
+ /** Hot tier for the container reading. Bounds in `container-memory-series.ts`. */
2568
+ memoryRing = new ContainerMemoryRing();
2569
+ /** Cold tier, HUB only — same reasoning as `loadStore`. */
2570
+ memoryStore = null;
1743
2571
  /** The resolved knobs. Re-resolved on every settings write. */
1744
2572
  loadConfig = {
1745
2573
  cadenceSec: 10,
@@ -1812,13 +2640,20 @@ var NativeMetricsAddon = class NativeMetricsAddon extends require_dist.BaseAddon
1812
2640
  dumpHeapSnapshot: (params) => this.dumpHeapSnapshot(params)
1813
2641
  };
1814
2642
  this.applyLoadSeriesConfig();
1815
- if (this.isHub) this.loadStore = new LoadSeriesStore({
1816
- store: this.ctx.api.settingsStore,
1817
- logger: this.ctx.logger.child("LoadSeries")
1818
- });
2643
+ if (this.isHub) {
2644
+ this.loadStore = new LoadSeriesStore({
2645
+ store: this.ctx.api.settingsStore,
2646
+ logger: this.ctx.logger.child("LoadSeries")
2647
+ });
2648
+ this.memoryStore = new ContainerMemoryStore({
2649
+ store: this.ctx.api.settingsStore,
2650
+ logger: this.ctx.logger.child("ContainerMemory")
2651
+ });
2652
+ }
2653
+ this.memoryProbe = new ContainerMemoryProbe({ logger: this.ctx.logger.child("ContainerMemory") });
1819
2654
  this.snapshotTimer = setInterval(() => this.emitResourcesSnapshot(), METRICS_SNAPSHOT_INTERVAL_MS);
1820
2655
  this.startProcessSnapshotTimer();
1821
- this.ctx.addDisposer(this.ctx.eventBus.subscribe({ category: require_dist.EventCategory.MetricsNodeProcessesSnapshot }, (event) => this.retainSnapshot(event.data.nodeId, event.data.timestamp, event.data.processes)));
2656
+ this.ctx.addDisposer(this.ctx.eventBus.subscribe({ category: require_dist.EventCategory.MetricsNodeProcessesSnapshot }, (event) => this.retainSnapshot(event.data.nodeId, event.data.timestamp, event.data.processes, event.data.containerMemory)));
1822
2657
  return [{
1823
2658
  capability: require_dist.metricsProviderCapability,
1824
2659
  provider: composed
@@ -1836,19 +2671,31 @@ var NativeMetricsAddon = class NativeMetricsAddon extends require_dist.BaseAddon
1836
2671
  * The durable append is fire-and-forget: a storage stall must cost a gap in
1837
2672
  * the cold window, never a blocked event-bus handler. Every failure logs.
1838
2673
  */
1839
- retainSnapshot(nodeId, atMs, processes) {
2674
+ retainSnapshot(nodeId, atMs, processes, containerMemory) {
1840
2675
  const retained = processes.map((p) => NativeMetricsAddon.toRetained(p));
1841
2676
  if (!this.loadRing.record(nodeId, atMs, retained)) return;
1842
2677
  this.observedRowsByNode.set(nodeId, retained.length);
2678
+ if (containerMemory !== null) this.memoryRing.record(nodeId, atMs, containerMemory);
1843
2679
  const store = this.loadStore;
1844
2680
  if (store === null) return;
1845
- store.append(nodeId, atMs, retained).then(() => store.prune([...this.observedRowsByNode.keys()], this.loadConfig)).catch((err) => {
2681
+ store.append(nodeId, atMs, retained).then(() => store.prune([...this.observedRowsByNode.keys()], this.loadConfig)).then(async (outcome) => {
2682
+ if (outcome === null) return;
2683
+ await this.memoryStore?.prune([...this.observedRowsByNode.keys()], this.loadConfig);
2684
+ }).catch((err) => {
1846
2685
  this.ctx.logger.warn("durable load series write failed", { meta: {
1847
2686
  nodeId,
1848
2687
  atMs,
1849
2688
  error: err instanceof Error ? err.message : String(err)
1850
2689
  } });
1851
2690
  });
2691
+ if (containerMemory === null) return;
2692
+ this.memoryStore?.append(nodeId, atMs, containerMemory).catch((err) => {
2693
+ this.ctx.logger.warn("durable container memory write failed", { meta: {
2694
+ nodeId,
2695
+ atMs,
2696
+ error: err instanceof Error ? err.message : String(err)
2697
+ } });
2698
+ });
1852
2699
  }
1853
2700
  /**
1854
2701
  * The ONE reader, over both tiers.
@@ -1877,6 +2724,14 @@ var NativeMetricsAddon = class NativeMetricsAddon extends require_dist.BaseAddon
1877
2724
  return {
1878
2725
  nodeId: params.forNodeId,
1879
2726
  series: merged.series,
2727
+ containerMemory: await this.readContainerMemory({
2728
+ forNodeId: params.forNodeId,
2729
+ sinceMs,
2730
+ ...params.sinceMs !== void 0 ? { exclusiveSinceMs: params.sinceMs } : {},
2731
+ bucketMs: merged.bucketMs,
2732
+ cadenceMs,
2733
+ origin: merged.oldestAtMs
2734
+ }),
1880
2735
  bucketMs: merged.bucketMs,
1881
2736
  retainedSamples: merged.retainedSamples,
1882
2737
  oldestAtMs: merged.oldestAtMs,
@@ -1885,6 +2740,24 @@ var NativeMetricsAddon = class NativeMetricsAddon extends require_dist.BaseAddon
1885
2740
  };
1886
2741
  }
1887
2742
  /**
2743
+ * The container's memory over the same window, on the same x-axis.
2744
+ *
2745
+ * Both tiers, merged and deduped on `atMs` like the process series, then
2746
+ * reduced against the SAME `bucketMs` and the SAME origin — otherwise the
2747
+ * two series in one answer would sit on two different axes and the
2748
+ * subtraction they exist for would be an eyeball.
2749
+ *
2750
+ * An EMPTY answer means the node has no cgroup, or nothing has been retained
2751
+ * for it yet. It never means zero.
2752
+ */
2753
+ async readContainerMemory(params) {
2754
+ const hot = this.memoryRing.read(params.forNodeId, params.exclusiveSinceMs);
2755
+ const store = this.memoryStore;
2756
+ const merged = mergeContainerMemory(store === null ? [] : await store.read(params.forNodeId, params.sinceMs, CONTAINER_MEMORY_MAX_ROWS), hot);
2757
+ if (params.bucketMs <= params.cadenceMs) return merged;
2758
+ return reduceContainerMemory(merged, params.bucketMs, params.origin ?? 0);
2759
+ }
2760
+ /**
1888
2761
  * Re-resolve the knobs and restate the fixed cadence.
1889
2762
  *
1890
2763
  * A REFUSED value (out of 5-60 s, or out of 1-72 h) leaves the previous
@@ -2000,6 +2873,7 @@ var NativeMetricsAddon = class NativeMetricsAddon extends require_dist.BaseAddon
2000
2873
  const timestamp = Date.now();
2001
2874
  try {
2002
2875
  const processes = await this.listNodeProcesses();
2876
+ const containerMemory = this.memoryProbe?.sample() ?? null;
2003
2877
  eventBus.emit(require_dist.createEvent(require_dist.EventCategory.MetricsNodeProcessesSnapshot, {
2004
2878
  type: "node",
2005
2879
  id: nodeId,
@@ -2007,7 +2881,8 @@ var NativeMetricsAddon = class NativeMetricsAddon extends require_dist.BaseAddon
2007
2881
  }, {
2008
2882
  nodeId,
2009
2883
  processes,
2010
- timestamp
2884
+ timestamp,
2885
+ containerMemory
2011
2886
  }));
2012
2887
  } catch (err) {
2013
2888
  this.ctx.logger.warn("process snapshot skipped — this interval will be missing", { meta: {