@camstack/system 1.2.151 → 1.2.154

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/dist/addon-runner.js +14 -15
  2. package/dist/addon-runner.mjs +14 -14
  3. package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
  4. package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
  5. package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
  6. package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
  7. package/dist/builtins/alerts/alerts.addon.js +1 -1
  8. package/dist/builtins/alerts/alerts.addon.mjs +1 -1
  9. package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
  10. package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
  11. package/dist/builtins/console-logging/index.js +1 -1
  12. package/dist/builtins/console-logging/index.mjs +1 -1
  13. package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
  14. package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
  15. package/dist/builtins/device-manager/device-manager.addon.js +2 -2
  16. package/dist/builtins/device-manager/device-manager.addon.mjs +2 -2
  17. package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
  18. package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
  19. package/dist/builtins/hub-forwarder/index.js +1 -1
  20. package/dist/builtins/hub-forwarder/index.mjs +1 -1
  21. package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
  22. package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
  23. package/dist/builtins/local-auth/local-auth.addon.js +1 -1
  24. package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
  25. package/dist/builtins/local-network/local-network.addon.js +3 -3
  26. package/dist/builtins/local-network/local-network.addon.mjs +3 -3
  27. package/dist/builtins/loki-logging/index.js +1 -1
  28. package/dist/builtins/loki-logging/index.mjs +1 -1
  29. package/dist/builtins/native-metrics/cgroup-memory.d.ts +193 -0
  30. package/dist/builtins/native-metrics/container-memory-series.d.ts +226 -0
  31. package/dist/builtins/native-metrics/native-metrics.addon.d.ts +26 -0
  32. package/dist/builtins/native-metrics/native-metrics.addon.js +1536 -661
  33. package/dist/builtins/native-metrics/native-metrics.addon.mjs +1536 -660
  34. package/dist/builtins/platform-probe/index.js +3 -3
  35. package/dist/builtins/platform-probe/index.mjs +2 -2
  36. package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
  37. package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
  38. package/dist/builtins/snapshot/index.js +1 -1
  39. package/dist/builtins/snapshot/index.mjs +1 -1
  40. package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +2 -2
  41. package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +2 -2
  42. package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
  43. package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
  44. package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
  45. package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
  46. package/dist/builtins/system-config/system-config.addon.js +1 -1
  47. package/dist/builtins/system-config/system-config.addon.mjs +1 -1
  48. package/dist/builtins/winston-logging/index.js +1 -1
  49. package/dist/builtins/winston-logging/index.mjs +1 -1
  50. package/dist/{dist-CImxMt5h.js → dist-BPlfW-CG.js} +96 -10
  51. package/dist/{dist-CbRciAbu.mjs → dist-D_NyUV6Y.mjs} +96 -10
  52. package/dist/index.d.ts +6 -2
  53. package/dist/index.js +24 -5
  54. package/dist/index.mjs +5 -5
  55. package/dist/kernel/event-plane-report.d.ts +202 -0
  56. package/dist/kernel/heap-spaces.d.ts +138 -0
  57. package/dist/kernel/heap-watch.d.ts +142 -5
  58. package/dist/kernel/index.d.ts +2 -0
  59. package/dist/kernel/transport/local-child-registry.d.ts +22 -0
  60. package/dist/{manifest-python-deps-DuQ3Uqoq.mjs → manifest-python-deps-BILJm2mI.mjs} +522 -15
  61. package/dist/{manifest-python-deps-BqWrDjSN.js → manifest-python-deps-COeSr7el.js} +636 -15
  62. package/dist/{retired-settings-keys-DhFPXIRD.mjs → retired-settings-keys-Bq1Tb84I.mjs} +1 -1
  63. package/dist/{retired-settings-keys-C7gLUS3x.js → retired-settings-keys-DBY6ebwV.js} +1 -1
  64. package/package.json +1 -1
  65. package/dist/{tls-u8QCJCFE.js → tls-BxQlomxd.js} +1 -1
  66. package/dist/{tls-CQhPGSJm.mjs → tls-DOTmtLCW.mjs} +1 -1
@@ -1,731 +1,1546 @@
1
- import { Ht as createEvent, Ot as errMsg, ft as reducePoints, kt as BaseAddon, ot as metricsProviderCapability, pt as resolveBucketMs, q as foldSnapshotByFunction, tn as EventCategory } from "../../dist-CbRciAbu.mjs";
1
+ import { Ht as createEvent, Ot as errMsg, ft as reducePoints, kt as BaseAddon, ot as metricsProviderCapability, pt as resolveBucketMs, q as foldSnapshotByFunction, tn as EventCategory } from "../../dist-D_NyUV6Y.mjs";
2
2
  import { readFile, readdir } from "node:fs/promises";
3
3
  import { execFile, execFileSync } from "node:child_process";
4
4
  import { promisify } from "node:util";
5
- import * as os from "node:os";
6
5
  import * as fs from "node:fs";
7
- //#region src/builtins/native-metrics/native-metrics-provider.ts
8
- var IS_DARWIN = os.platform() === "darwin";
9
- var IS_LINUX = os.platform() === "linux";
10
- var NativeMetricsProvider = class {
11
- id = "native";
12
- cachedSnapshot = null;
13
- samplingTimer = null;
14
- macMemTimer = null;
15
- prevCpu;
16
- diagnostics = null;
17
- consecutiveSampleErrors = 0;
18
- constructor() {
19
- this.prevCpu = aggregateCpuTimes();
6
+ import { readFileSync } from "node:fs";
7
+ import * as os from "node:os";
8
+ //#region src/builtins/native-metrics/cgroup-memory.ts
9
+ /** `readFileSync`, with every failure flattened to `null`. */
10
+ var nodeCgroupFileReader = { read(path) {
11
+ try {
12
+ return readFileSync(path, "utf8");
13
+ } catch {
14
+ return null;
15
+ }
16
+ } };
17
+ /** Where the v2 unified hierarchy is mounted, and where v1 hangs its controllers. */
18
+ var CGROUP_MOUNT_ROOT = "/sys/fs/cgroup";
19
+ /**
20
+ * The i915 GEM accounting file.
21
+ *
22
+ * **debugfs, and it is not mounted in our containers.** Verified on the live
23
+ * hub on 2026-08-29: readable on the Unraid host (`6014 shrinkable [0 free]
24
+ * objects, 1194934272 bytes`), `ENOENT` inside the `CamStack` container where
25
+ * `native-metrics` actually runs. It is probed ONCE per process and latched —
26
+ * a bind mount cannot appear under a running container — so an absent file
27
+ * costs one failed `open` for the life of the runner and one log line, never a
28
+ * failure per tick.
29
+ *
30
+ * Making it populate is a one-line change on the container, not here:
31
+ * mounting `/sys/kernel/debug` read-only into `CamStack` fills this field on
32
+ * the next sample with no code change.
33
+ */
34
+ var I915_GEM_OBJECTS_PATH = "/sys/kernel/debug/dri/0/i915_gem_objects";
35
+ /**
36
+ * v1's "no limit" sentinel — `PAGE_COUNTER_MAX << PAGE_SHIFT`, which is what a
37
+ * 64-bit kernel writes into `memory.limit_in_bytes` when nothing is set. It is
38
+ * a limit in the same sense that `Infinity` is a number, so it reads as `null`.
39
+ */
40
+ var V1_UNLIMITED_SENTINEL = 0x7ffffffffffff000;
41
+ /**
42
+ * Parse `/proc/self/cgroup`.
43
+ *
44
+ * Two formats, and the file may contain both on a hybrid host:
45
+ *
46
+ * ```
47
+ * 0::/ ← v2 unified
48
+ * 5:memory:/docker/9f3c… ← v1, memory controller
49
+ * 4:cpu,cpuacct:/docker/9f3c… ← v1, other controllers
50
+ * ```
51
+ *
52
+ * v2 wins when present: on a hybrid host the unified hierarchy is the one
53
+ * carrying `memory.current`, and every camstack deployment is v2 (verified on
54
+ * the live hub, which reports the single line `0::/`).
55
+ *
56
+ * Returns `null` when the file names no memory hierarchy at all — which is
57
+ * also what a non-Linux host produces, because there is no file to read.
58
+ */
59
+ function parseCgroupSelf(text) {
60
+ let v1 = null;
61
+ for (const rawLine of text.split("\n")) {
62
+ const line = rawLine.trim();
63
+ if (line.length === 0) continue;
64
+ const first = line.indexOf(":");
65
+ if (first < 0) continue;
66
+ const second = line.indexOf(":", first + 1);
67
+ if (second < 0) continue;
68
+ const controllers = line.slice(first + 1, second);
69
+ const path = line.slice(second + 1);
70
+ if (path.length === 0) continue;
71
+ const normalized = path.startsWith("/") ? path : `/${path}`;
72
+ if (controllers.length === 0) return {
73
+ version: "v2",
74
+ path: normalized
75
+ };
76
+ if (v1 === null && controllers.split(",").includes("memory")) v1 = {
77
+ version: "v1",
78
+ path: normalized
79
+ };
80
+ }
81
+ return v1;
82
+ }
83
+ /**
84
+ * Parse a `memory.stat` file — v1 or v2, both are `key value` per line.
85
+ *
86
+ * Unknown keys are KEPT rather than refused: a v2 `memory.stat` carries ~50
87
+ * fields and we read five of them, and a kernel that adds a sixth must not
88
+ * turn into a parse failure. A malformed line is skipped, not thrown on.
89
+ */
90
+ function parseCgroupStat(text) {
91
+ const out = /* @__PURE__ */ new Map();
92
+ for (const rawLine of text.split("\n")) {
93
+ const line = rawLine.trim();
94
+ if (line.length === 0) continue;
95
+ const space = line.indexOf(" ");
96
+ if (space <= 0) continue;
97
+ const key = line.slice(0, space);
98
+ const value = Number(line.slice(space + 1).trim());
99
+ if (!Number.isFinite(value)) continue;
100
+ out.set(key, value);
101
+ }
102
+ return out;
103
+ }
104
+ /**
105
+ * Parse a single-scalar cgroup file (`memory.current`, `memory.max`, …).
106
+ *
107
+ * `max` — v2's word for "no limit" — and v1's `V1_UNLIMITED_SENTINEL` both
108
+ * become `null`, because a ceiling nothing can reach is not a ceiling and
109
+ * charting it as 8 exabytes would flatten every real series to zero.
110
+ */
111
+ function parseCgroupScalar(text) {
112
+ const trimmed = text.trim();
113
+ if (trimmed.length === 0) return null;
114
+ if (trimmed === "max") return null;
115
+ const value = Number(trimmed);
116
+ if (!Number.isFinite(value) || value < 0) return null;
117
+ if (value >= V1_UNLIMITED_SENTINEL) return null;
118
+ return value;
119
+ }
120
+ /**
121
+ * Parse the first line of `i915_gem_objects`.
122
+ *
123
+ * ```
124
+ * 6014 shrinkable [0 free] objects, 1194934272 bytes
125
+ * ```
126
+ *
127
+ * Only the byte total is taken. The format has changed across kernel releases
128
+ * more than once, so a line that does not match reads `null` — an unrecognised
129
+ * format is UNKNOWN, never zero.
130
+ */
131
+ function parseI915GemObjects(text) {
132
+ const match = text.match(/^\s*\d+\s+shrinkable\b[^,\n]*,\s*(\d+)\s+bytes/m);
133
+ if (match === null) return null;
134
+ const bytes = Number(match[1]);
135
+ return Number.isFinite(bytes) ? bytes : null;
136
+ }
137
+ /**
138
+ * Find the cgroup from INSIDE this process — never a hard-coded path.
139
+ *
140
+ * The path the hub happens to have (`/sys/fs/cgroup/docker/<id>/memory.stat`
141
+ * as seen from the Unraid host) is an artefact of where you are standing. From
142
+ * inside the container the same cgroup is at the mount ROOT, because Docker
143
+ * gives the container its own cgroup namespace and `/proc/self/cgroup` reports
144
+ * `0::/`. Hard-coding either one breaks the other.
145
+ *
146
+ * So the resolution is: ask `/proc/self/cgroup` which hierarchy and which
147
+ * path, then take the FIRST candidate directory that actually publishes the
148
+ * usage file.
149
+ *
150
+ * - **v2** — `<mount><path>` then `<mount>`. The second candidate is the
151
+ * namespaced case and the one that answers on our hub; the first is the
152
+ * un-namespaced case (a host process, or `--cgroupns=host`).
153
+ * - **v1** — `<mount>/memory<path>` then `<mount>/memory`, same reasoning.
154
+ * - **neither** — `null`. macOS has no `/proc/self/cgroup` at all, so a
155
+ * developer Mac lands here and reports nothing, which is the truth.
156
+ */
157
+ function resolveCgroupMemoryLayout(fs) {
158
+ const selfText = fs.read("/proc/self/cgroup");
159
+ if (selfText === null) return null;
160
+ const entry = parseCgroupSelf(selfText);
161
+ if (entry === null) return null;
162
+ const suffix = entry.path === "/" ? "" : entry.path;
163
+ const candidates = entry.version === "v2" ? [`${CGROUP_MOUNT_ROOT}${suffix}`, CGROUP_MOUNT_ROOT] : [`${CGROUP_MOUNT_ROOT}/memory${suffix}`, `${CGROUP_MOUNT_ROOT}/memory`];
164
+ const usageFile = entry.version === "v2" ? "memory.current" : "memory.usage_in_bytes";
165
+ const limitFile = entry.version === "v2" ? "memory.max" : "memory.limit_in_bytes";
166
+ for (const directory of candidates) {
167
+ if (fs.read(`${directory}/${usageFile}`) === null) continue;
168
+ return {
169
+ source: entry.version === "v2" ? "cgroup-v2" : "cgroup-v1",
170
+ directory,
171
+ currentPath: `${directory}/${usageFile}`,
172
+ limitPath: `${directory}/${limitFile}`,
173
+ statPath: `${directory}/memory.stat`
174
+ };
175
+ }
176
+ return null;
177
+ }
178
+ /** A stat lookup that answers `null` for a key this hierarchy does not publish. */
179
+ function statValue(stat, key) {
180
+ const value = stat.get(key);
181
+ return value === void 0 ? null : value;
182
+ }
183
+ /**
184
+ * v2 publishes `slab` directly; older v2 kernels publish only the two halves.
185
+ * Summing them is the same number, so the fallback is exact rather than an
186
+ * approximation — and when neither is present the answer is `null`.
187
+ */
188
+ function resolveSlab(stat) {
189
+ const direct = statValue(stat, "slab");
190
+ if (direct !== null) return direct;
191
+ const reclaimable = stat.get("slab_reclaimable");
192
+ const unreclaimable = stat.get("slab_unreclaimable");
193
+ if (reclaimable === void 0 || unreclaimable === void 0) return null;
194
+ return reclaimable + unreclaimable;
195
+ }
196
+ /**
197
+ * Turn one layout's three files into a snapshot.
198
+ *
199
+ * Pure over the text, so the whole mapping — including a v2 `memory.stat` full
200
+ * of fields we do not read — is testable against a real fixture.
201
+ *
202
+ * v1's `memory.stat` names the same quantities differently and publishes fewer
203
+ * of them: `rss` is the anonymous charge, `cache` is the page cache, and there
204
+ * is no slab accounting at all. Mapping them onto the same field names is what
205
+ * makes a v1 node and a v2 node comparable; reporting `slabBytes: null` on v1
206
+ * is what stops that comparison from lying.
207
+ */
208
+ function buildContainerMemorySnapshot(input) {
209
+ const currentBytes = parseCgroupScalar(input.currentText);
210
+ if (currentBytes === null) return null;
211
+ const stat = input.statText === null ? /* @__PURE__ */ new Map() : parseCgroupStat(input.statText);
212
+ const isV2 = input.source === "cgroup-v2";
213
+ return {
214
+ source: input.source,
215
+ currentBytes,
216
+ limitBytes: input.limitText === null ? null : parseCgroupScalar(input.limitText),
217
+ anonBytes: statValue(stat, isV2 ? "anon" : "rss"),
218
+ fileBytes: statValue(stat, isV2 ? "file" : "cache"),
219
+ shmemBytes: statValue(stat, "shmem"),
220
+ slabBytes: isV2 ? resolveSlab(stat) : null,
221
+ gpuShmemBytes: input.gpuShmemBytes
222
+ };
223
+ }
224
+ /**
225
+ * The sampler. Resolves once, latches, and logs TRANSITIONS only.
226
+ *
227
+ * Three states are worth a line and none of them is worth one per tick:
228
+ *
229
+ * 1. **Resolution** — once, at the first sample. Either the layout that
230
+ * answered, or "no cgroup on this node", which is a fact about the node and
231
+ * not an error.
232
+ * 2. **GPU probe** — once, at the first sample. Absent debugfs is the normal
233
+ * case inside a container and is stated as such, then never retried: a bind
234
+ * mount cannot appear under a running container.
235
+ * 3. **Read failing / recovering** — a cgroup that resolved and then stopped
236
+ * answering is a real change and says so, once in each direction. A tick
237
+ * that fails while already failing is silent, because the first line
238
+ * already said it.
239
+ *
240
+ * The silence is bounded by design: at the 10 s snapshot cadence a per-tick
241
+ * warning would be 8 640 lines a day on a node that simply is not Linux.
242
+ */
243
+ var ContainerMemoryProbe = class {
244
+ logger;
245
+ fs;
246
+ wantGpu;
247
+ resolved = false;
248
+ layout = null;
249
+ gpuProbed = false;
250
+ gpuReadable = false;
251
+ /** `null` until the first sample — so the FIRST outcome is a transition too. */
252
+ lastReadOk = null;
253
+ constructor(deps) {
254
+ this.logger = deps.logger;
255
+ this.fs = deps.fs ?? nodeCgroupFileReader;
256
+ this.wantGpu = deps.readGpu ?? true;
257
+ }
258
+ /** Did this node resolve a cgroup at all? Absent is a reportable fact. */
259
+ get available() {
260
+ return this.layout !== null;
20
261
  }
21
262
  /**
22
- * Optional diagnostics sink for sampling errors. The first failure and
23
- * every Nth failure thereafter are reported so a permanently-broken
24
- * collector doesn't spam logs but is still observable.
263
+ * One reading, or `null` when this node has no cgroup or the cgroup stopped
264
+ * answering. `null` is ABSENT the caller contributes no point, and no
265
+ * point is what a chart must show, never a zero.
25
266
  */
26
- setDiagnostics(diag) {
27
- this.diagnostics = diag;
28
- }
29
- startSampling(intervalMs) {
30
- if (IS_DARWIN) {
31
- sampleMacMemory(this);
32
- this.macMemTimer = setInterval(() => sampleMacMemory(this), 5e3);
33
- if (this.macMemTimer.unref) this.macMemTimer.unref();
267
+ sample() {
268
+ const layout = this.resolveOnce();
269
+ if (layout === null) return null;
270
+ const currentText = this.fs.read(layout.currentPath);
271
+ if (currentText === null) {
272
+ this.noteRead(false, `unreadable: ${layout.currentPath}`);
273
+ return null;
34
274
  }
35
- this.collectSnapshot().then((s) => {
36
- this.cachedSnapshot = s;
37
- this.consecutiveSampleErrors = 0;
38
- }).catch((err) => this.reportSampleError(err));
39
- this.samplingTimer = setInterval(() => {
40
- this.collectSnapshot().then((s) => {
41
- this.cachedSnapshot = s;
42
- this.consecutiveSampleErrors = 0;
43
- }).catch((err) => this.reportSampleError(err));
44
- }, intervalMs);
45
- if (this.samplingTimer.unref) this.samplingTimer.unref();
46
- }
47
- reportSampleError(err) {
48
- this.consecutiveSampleErrors++;
49
- if (!this.diagnostics) return;
50
- if (!(this.consecutiveSampleErrors === 1 || this.consecutiveSampleErrors % 60 === 0)) return;
51
- this.diagnostics.warn("metrics-provider sample failed", {
52
- consecutiveErrors: this.consecutiveSampleErrors,
53
- error: err instanceof Error ? err.message : String(err)
275
+ const snapshot = buildContainerMemorySnapshot({
276
+ source: layout.source,
277
+ currentText,
278
+ limitText: this.fs.read(layout.limitPath),
279
+ statText: this.fs.read(layout.statPath),
280
+ gpuShmemBytes: this.readGpuShmem()
54
281
  });
282
+ if (snapshot === null) {
283
+ this.noteRead(false, `unparseable: ${layout.currentPath}`);
284
+ return null;
285
+ }
286
+ this.noteRead(true, null);
287
+ return snapshot;
55
288
  }
56
- stopSampling() {
57
- if (this.samplingTimer) {
58
- clearInterval(this.samplingTimer);
59
- this.samplingTimer = null;
289
+ /** Resolve on first use and never again — a cgroup does not move under a pid. */
290
+ resolveOnce() {
291
+ if (this.resolved) return this.layout;
292
+ this.resolved = true;
293
+ try {
294
+ this.layout = resolveCgroupMemoryLayout(this.fs);
295
+ } catch (err) {
296
+ this.layout = null;
297
+ this.logger.warn("cgroup memory resolution failed — container memory will be ABSENT", { meta: { error: errMsg(err) } });
298
+ return null;
60
299
  }
61
- if (this.macMemTimer) {
62
- clearInterval(this.macMemTimer);
63
- this.macMemTimer = null;
300
+ if (this.layout === null) {
301
+ this.logger.info("no cgroup on this node — container memory is ABSENT, not zero, for every sample", { meta: {
302
+ platform: process.platform,
303
+ probed: "/proc/self/cgroup"
304
+ } });
305
+ return null;
64
306
  }
307
+ this.logger.info("container memory resolved from the cgroup", { meta: {
308
+ source: this.layout.source,
309
+ directory: this.layout.directory
310
+ } });
311
+ return this.layout;
65
312
  }
66
- async collectSnapshot() {
67
- const cpu = this.sampleCpu();
68
- const memory = this.getMemoryInfo();
69
- const processInfo = getProcessResources();
70
- const [gpu, network, disk, cpuTemp] = await Promise.all([
71
- getGpuInfoInternal().catch(() => null),
72
- takeNetworkIoSnapshot(),
73
- takeDiskIoSnapshot(),
74
- getCpuTemperatureInternal().catch(() => null)
75
- ]);
76
- const snapshot = {
77
- cpu,
78
- memory,
79
- gpu,
80
- network,
81
- disk,
82
- pressure: {
83
- cpu: getPressure("cpu"),
84
- memory: getPressure("memory"),
85
- io: getPressure("io")
86
- },
87
- process: processInfo,
88
- cpuTemperature: cpuTemp,
89
- timestampMs: Date.now()
90
- };
91
- this.cachedSnapshot = snapshot;
92
- return snapshot;
313
+ /**
314
+ * The GPU number, probed once.
315
+ *
316
+ * It is host-wide and lives in debugfs, so it is absent far more often than
317
+ * it is present inside every camstack container today. One `open` decides
318
+ * it for the life of the process; after that an unreadable file costs
319
+ * nothing at all, not even a syscall.
320
+ */
321
+ readGpuShmem() {
322
+ if (!this.wantGpu) return null;
323
+ if (!this.gpuProbed) {
324
+ this.gpuProbed = true;
325
+ const text = this.fs.read(I915_GEM_OBJECTS_PATH);
326
+ this.gpuReadable = text !== null && parseI915GemObjects(text) !== null;
327
+ this.logger.info(this.gpuReadable ? "i915 GEM accounting is readable — GPU-backed shmem will be reported" : "i915 GEM accounting is NOT readable — gpuShmemBytes stays UNKNOWN (this is normal: debugfs is not mounted in the container, and a node with no Intel GPU has no such file)", { meta: { path: I915_GEM_OBJECTS_PATH } });
328
+ if (!this.gpuReadable) return null;
329
+ return text === null ? null : parseI915GemObjects(text);
330
+ }
331
+ if (!this.gpuReadable) return null;
332
+ const text = this.fs.read(I915_GEM_OBJECTS_PATH);
333
+ return text === null ? null : parseI915GemObjects(text);
334
+ }
335
+ /** Log a read outcome only when it CHANGES. Never once per tick. */
336
+ noteRead(ok, reason) {
337
+ if (this.lastReadOk === ok) return;
338
+ const first = this.lastReadOk === null;
339
+ this.lastReadOk = ok;
340
+ if (!ok) {
341
+ this.logger.warn("cgroup memory stopped answering — container memory is ABSENT until it returns", { meta: { reason: reason ?? "unknown" } });
342
+ return;
343
+ }
344
+ if (!first) this.logger.info("cgroup memory is answering again");
93
345
  }
94
- async getCached() {
95
- return this.cachedSnapshot;
346
+ };
347
+ /**
348
+ * One node's fixed-capacity circular buffer of samples.
349
+ *
350
+ * Mirrors {@link LogRingBuffer}: write at `head`, wrap, and once full every
351
+ * accepted sample evicts the oldest. `rows` is carried alongside so the fleet
352
+ * budget never has to walk the samples to know what a partition costs.
353
+ */
354
+ var LoadPartition = class {
355
+ capacity;
356
+ slots;
357
+ head = 0;
358
+ count = 0;
359
+ rows = 0;
360
+ /** Newest retained sample's timestamp — the monotonic gate for `push`. */
361
+ newestAtMs = null;
362
+ constructor(capacity) {
363
+ this.capacity = capacity;
364
+ this.slots = Array.from({ length: capacity });
96
365
  }
97
- async getCurrent() {
98
- const s = this.cachedSnapshot ?? await this.collectSnapshot();
99
- const mb = (bytes) => Math.round(bytes / 1024 / 1024);
366
+ /**
367
+ * Accept a sample.
368
+ *
369
+ * A sample at or before the newest one already held is REFUSED. The bus drops
370
+ * a node's own broadcast echo, but a cross-node redelivery or a replayed
371
+ * subscription must not be able to double a point — and idempotence here is
372
+ * what lets every reader above treat the series as a set.
373
+ *
374
+ * `accepted` is reported separately from `rowDelta` on purpose: a full ring
375
+ * that evicts a sample of the same size has a delta of zero and is not a
376
+ * refusal, and conflating the two would silently stop advancing the write
377
+ * ordinal on a steady-state cluster.
378
+ */
379
+ push(sample) {
380
+ if (this.newestAtMs !== null && sample.atMs <= this.newestAtMs) return {
381
+ accepted: false,
382
+ rowDelta: 0
383
+ };
384
+ const evicted = this.count === this.capacity ? this.slots[this.head]?.processes.length ?? 0 : 0;
385
+ this.slots[this.head] = sample;
386
+ this.head = (this.head + 1) % this.capacity;
387
+ if (this.count < this.capacity) this.count++;
388
+ this.newestAtMs = sample.atMs;
389
+ const rowDelta = sample.processes.length - evicted;
390
+ this.rows += rowDelta;
100
391
  return {
101
- cpuPercent: s.cpu.total,
102
- memoryPercent: s.memory.percent,
103
- memoryUsedMB: mb(s.memory.usedBytes),
104
- memoryTotalMB: mb(s.memory.totalBytes),
105
- temperature: s.cpuTemperature ?? void 0,
106
- gpuPercent: s.gpu?.utilization,
107
- gpuMemoryPercent: s.gpu && s.gpu.memoryTotalBytes > 0 ? Math.round(s.gpu.memoryUsedBytes / s.gpu.memoryTotalBytes * 1e3) / 10 : void 0
392
+ accepted: true,
393
+ rowDelta
108
394
  };
109
395
  }
110
- async getDiskSpace(input) {
111
- return getDiskSpaceInternal(input.dirPath);
396
+ /** Drop the oldest sample. Returns the rows reclaimed (0 when empty). */
397
+ dropOldest() {
398
+ if (this.count === 0) return 0;
399
+ const index = (this.head - this.count + this.capacity) % this.capacity;
400
+ const victim = this.slots[index];
401
+ this.slots[index] = void 0;
402
+ this.count--;
403
+ const reclaimed = victim?.processes.length ?? 0;
404
+ this.rows -= reclaimed;
405
+ if (this.count === 0) this.newestAtMs = null;
406
+ return reclaimed;
112
407
  }
113
- async getGpuInfo() {
114
- return getGpuInfoInternal();
408
+ /** Oldest-first, optionally only what is strictly newer than `sinceMs`. */
409
+ list(sinceMs) {
410
+ const out = [];
411
+ for (let i = 0; i < this.count; i++) {
412
+ const index = (this.head - this.count + i + this.capacity) % this.capacity;
413
+ const sample = this.slots[index];
414
+ if (sample === void 0) continue;
415
+ if (sinceMs !== void 0 && sample.atMs <= sinceMs) continue;
416
+ out.push(sample);
417
+ }
418
+ return out;
115
419
  }
116
- async getCpuTemperature() {
117
- return getCpuTemperatureInternal();
420
+ size() {
421
+ return this.count;
118
422
  }
119
- async getProcessStats(input) {
120
- const { getPidStats } = await import("../../resource-monitor-BWmQ5i-o.mjs").then((n) => n.r);
121
- const raw = await getPidStats(input.pids);
122
- const result = [];
123
- for (const [pid, s] of raw) result.push({
124
- pid,
125
- cpu: s.cpu,
126
- memory: s.memory,
127
- privateBytes: s.privateBytes,
128
- sharedBytes: s.sharedBytes
129
- });
130
- return result;
423
+ rowCount() {
424
+ return this.rows;
131
425
  }
132
- sampleCpu() {
133
- const curr = aggregateCpuTimes();
134
- const prev = this.prevCpu;
135
- this.prevCpu = curr;
136
- const dUser = curr.user - prev.user;
137
- const dNice = curr.nice - prev.nice;
138
- const dSys = curr.sys - prev.sys;
139
- const dIrq = curr.irq - prev.irq;
140
- const dIdle = curr.idle - prev.idle;
141
- const dTotal = dUser + dNice + dSys + dIrq + dIdle;
142
- const pct = (v) => dTotal === 0 ? 0 : Math.round(v / dTotal * 1e3) / 10;
143
- return {
144
- total: pct(dUser + dNice + dSys + dIrq),
145
- user: pct(dUser),
146
- system: pct(dSys),
147
- irq: pct(dIrq),
148
- nice: pct(dNice),
149
- loadAvg: os.loadavg(),
150
- cores: os.cpus().length
151
- };
426
+ oldestAtMs() {
427
+ if (this.count === 0) return null;
428
+ const index = (this.head - this.count + this.capacity) % this.capacity;
429
+ return this.slots[index]?.atMs ?? null;
152
430
  }
153
- /** Cached macOS memory (set by sampleMacMemory timer). */
154
- _cachedMacMem = null;
155
- getMemoryInfo() {
156
- const total = os.totalmem();
157
- if (total === 0) return {
158
- percent: 0,
159
- totalBytes: 0,
160
- usedBytes: 0,
161
- availableBytes: 0,
162
- swapUsedBytes: 0,
163
- swapTotalBytes: 0
164
- };
165
- if (IS_DARWIN && this._cachedMacMem !== null) return this._cachedMacMem;
166
- const free = os.freemem();
167
- const used = total - free;
168
- let swapUsedBytes = 0;
169
- let swapTotalBytes = 0;
170
- if (IS_LINUX) try {
171
- const meminfo = fs.readFileSync("/proc/meminfo", "utf-8");
172
- const swapTotal = parseInt(meminfo.match(/SwapTotal:\s+(\d+)/)?.[1] ?? "0", 10) * 1024;
173
- const swapFree = parseInt(meminfo.match(/SwapFree:\s+(\d+)/)?.[1] ?? "0", 10) * 1024;
174
- swapTotalBytes = swapTotal;
175
- swapUsedBytes = swapTotal - swapFree;
176
- } catch {}
177
- return {
178
- percent: Math.round((total - free) / total * 1e3) / 10,
179
- totalBytes: total,
180
- usedBytes: used,
181
- availableBytes: free,
182
- swapUsedBytes,
183
- swapTotalBytes
184
- };
431
+ lastWriteAtMs() {
432
+ return this.newestAtMs;
185
433
  }
186
434
  };
187
- function aggregateCpuTimes() {
188
- const cpus = os.cpus();
189
- let user = 0, nice = 0, sys = 0, irq = 0, idle = 0;
190
- for (const cpu of cpus) {
191
- user += cpu.times.user;
192
- nice += cpu.times.nice;
193
- sys += cpu.times.sys;
194
- irq += cpu.times.irq;
195
- idle += cpu.times.idle;
435
+ /**
436
+ * The partitioned ring: one {@link LoadPartition} per node id.
437
+ *
438
+ * Three bounds, doing three different jobs — the same division as the log
439
+ * buffer, for the same reason:
440
+ *
441
+ * - **`samplesPerNode`** is HARD and per partition. It is what the operator
442
+ * asked for (an hour) and it can never be exceeded.
443
+ * - **`maxTotalProcessRows`** is the FLEET budget, and it is denominated in the
444
+ * thing memory is proportional to. When it bites, the oldest sample of the
445
+ * HEAVIEST partition is dropped — the node creating the pressure is the node
446
+ * that pays, and a quiet node keeps its full hour. The node just written to is
447
+ * NOT protected here: if it is the heaviest, it is the culprit.
448
+ * - **`maxNodes`** is the backstop against an unbounded key space, and it drops
449
+ * whole partitions least-recently-written first, never the one being written.
450
+ *
451
+ * Plus {@link DEFAULT_NODE_IDLE_EVICTION_MS}, which is what stops a node that
452
+ * simply stopped emitting from living in memory forever.
453
+ */
454
+ var NodeLoadRing = class {
455
+ partitions = /* @__PURE__ */ new Map();
456
+ /**
457
+ * Node id → write ordinal of its last accepted sample. A monotonic counter,
458
+ * not a clock: partition eviction must follow the order writes actually
459
+ * happened, and node clocks disagree.
460
+ */
461
+ lastWriteSeq = /* @__PURE__ */ new Map();
462
+ writeSeq = 0;
463
+ totalRows = 0;
464
+ samplesPerNode;
465
+ maxTotalProcessRows;
466
+ maxNodes;
467
+ idleEvictionMs;
468
+ now;
469
+ constructor(options = {}) {
470
+ this.samplesPerNode = options.samplesPerNode ?? 180;
471
+ this.maxTotalProcessRows = options.maxTotalProcessRows ?? 32e3;
472
+ this.maxNodes = options.maxNodes ?? 16;
473
+ this.idleEvictionMs = options.idleEvictionMs ?? 36e5;
474
+ this.now = options.now ?? Date.now;
196
475
  }
197
- return {
198
- user,
199
- nice,
200
- sys,
201
- irq,
202
- idle
203
- };
204
- }
205
- function sampleMacMemory(provider) {
206
- try {
207
- const output = execFileSync("vm_stat", {
208
- timeout: 2e3,
209
- encoding: "utf-8"
476
+ /**
477
+ * Retain one snapshot. `processes` is stored by reference — the payload is
478
+ * already an immutable arrival off the bus, and copying it would double the
479
+ * measured cost for nothing.
480
+ *
481
+ * Returns whether the sample was ACCEPTED — that is, whether it was new
482
+ * rather than a replay of a timestamp this node has already delivered.
483
+ *
484
+ * The return value is not diagnostics. It is the idempotence gate the
485
+ * DURABLE tier rides on (`load-series-store.ts`): only an accepted sample is
486
+ * appended to the table, which is what lets that table carry an `INTEGER`
487
+ * rowid key instead of a composite unique index over two million rows. The
488
+ * gate is one monotonic comparison in memory; the index it replaces was
489
+ * measured at ~40 bytes per row.
490
+ */
491
+ record(nodeId, atMs, processes) {
492
+ if (nodeId.length === 0) return false;
493
+ this.sweepIdle();
494
+ let partition = this.partitions.get(nodeId);
495
+ if (partition === void 0) {
496
+ partition = new LoadPartition(this.samplesPerNode);
497
+ this.partitions.set(nodeId, partition);
498
+ }
499
+ const outcome = partition.push({
500
+ atMs,
501
+ processes
210
502
  });
211
- const pageSize = 16384;
212
- const parse = (label) => {
213
- const match = output.match(new RegExp(`${label}:\\s+(\\d+)`));
214
- return match ? parseInt(match[1], 10) * pageSize : 0;
503
+ if (!outcome.accepted) {
504
+ if (partition.size() === 0) this.partitions.delete(nodeId);
505
+ return false;
506
+ }
507
+ this.totalRows += outcome.rowDelta;
508
+ this.lastWriteSeq.set(nodeId, ++this.writeSeq);
509
+ if (this.partitions.size > this.maxNodes) this.evictPartitions(this.partitions.size - this.maxNodes, nodeId);
510
+ this.enforceRowBudget();
511
+ return true;
512
+ }
513
+ /**
514
+ * Read one node's retained series, oldest-first.
515
+ *
516
+ * `sinceMs` is EXCLUSIVE: a caller passes the newest timestamp it already
517
+ * holds and gets back only what it is missing. That is the whole contract
518
+ * that lets the admin UI seed from here and then continue live without
519
+ * doubling a point it already drew.
520
+ *
521
+ * A node nobody has recorded answers with an empty series, not an error —
522
+ * unknown is the truth about a node that has not reported.
523
+ */
524
+ read(nodeId, sinceMs) {
525
+ this.sweepIdle();
526
+ const partition = this.partitions.get(nodeId);
527
+ if (partition === void 0) return {
528
+ nodeId,
529
+ samples: [],
530
+ retainedSamples: 0,
531
+ oldestAtMs: null,
532
+ capacity: this.samplesPerNode
215
533
  };
216
- const free = parse("Pages free");
217
- const inactive = parse("Pages inactive");
218
- const purgeable = parse("Pages purgeable");
219
- const speculative = parse("Pages speculative");
220
- const available = free + inactive + purgeable + speculative;
221
- const total = os.totalmem();
222
- const used = total - available;
223
- let swapUsedBytes = 0;
224
- let swapTotalBytes = 0;
225
- try {
226
- const swapOut = execFileSync("sysctl", ["-n", "vm.swapusage"], {
227
- timeout: 2e3,
228
- encoding: "utf-8"
229
- });
230
- const totalMatch = swapOut.match(/total\s*=\s*([\d.]+)M/);
231
- const usedMatch = swapOut.match(/used\s*=\s*([\d.]+)M/);
232
- if (totalMatch) swapTotalBytes = parseFloat(totalMatch[1]) * 1024 * 1024;
233
- if (usedMatch) swapUsedBytes = parseFloat(usedMatch[1]) * 1024 * 1024;
234
- } catch {}
235
- provider._cachedMacMem = {
236
- percent: Math.round(used / total * 1e3) / 10,
237
- totalBytes: total,
238
- usedBytes: used,
239
- availableBytes: available,
240
- swapUsedBytes,
241
- swapTotalBytes
534
+ return {
535
+ nodeId,
536
+ samples: partition.list(sinceMs),
537
+ retainedSamples: partition.size(),
538
+ oldestAtMs: partition.oldestAtMs(),
539
+ capacity: this.samplesPerNode
242
540
  };
243
- } catch {
244
- provider._cachedMacMem = null;
245
541
  }
246
- }
247
- function getDiskSpaceInternal(dirPath) {
248
- return new Promise((resolve, reject) => {
249
- fs.statfs(dirPath, (err, stats) => {
250
- if (err) return reject(err);
251
- const totalBytes = stats.blocks * stats.bsize;
252
- const availableBytes = stats.bavail * stats.bsize;
253
- const usedBytes = totalBytes - stats.bfree * stats.bsize;
254
- resolve({
255
- path: dirPath,
256
- totalBytes,
257
- usedBytes,
258
- availableBytes,
259
- percent: totalBytes > 0 ? Math.round(usedBytes / totalBytes * 1e3) / 10 : 0
260
- });
261
- });
262
- });
263
- }
264
- async function takeDiskIoSnapshot() {
265
- const now = Date.now();
266
- if (IS_LINUX) try {
267
- const data = fs.readFileSync("/proc/diskstats", "utf-8");
268
- let readOps = 0, readBytes = 0, writeOps = 0, writeBytes = 0;
269
- for (const line of data.split("\n")) {
270
- const parts = line.trim().split(/\s+/);
271
- if (parts.length < 14) continue;
272
- const devName = parts[2];
273
- if (/\d+$/.test(devName) && !/^nvme\d+n\d+$/.test(devName)) continue;
274
- readOps += parseInt(parts[3], 10) || 0;
275
- readBytes += (parseInt(parts[5], 10) || 0) * 512;
276
- writeOps += parseInt(parts[7], 10) || 0;
277
- writeBytes += (parseInt(parts[9], 10) || 0) * 512;
542
+ /** Node ids with a live partition. Observability for the fleet bound. */
543
+ nodeIds() {
544
+ return [...this.partitions.keys()];
545
+ }
546
+ /** Process rows retained across every partition — the number that IS memory. */
547
+ rowCount() {
548
+ return this.totalRows;
549
+ }
550
+ /** Samples retained across every partition. */
551
+ sampleCount() {
552
+ let total = 0;
553
+ for (const partition of this.partitions.values()) total += partition.size();
554
+ return total;
555
+ }
556
+ /**
557
+ * Drop partitions whose newest sample is older than the retention window.
558
+ *
559
+ * Lazy, on write and on read — never a timer. A timer would be a new
560
+ * periodic cost in a subsystem whose entire premise is that it adds none,
561
+ * and a ring that nobody writes to and nobody reads is not growing either.
562
+ */
563
+ sweepIdle() {
564
+ const cutoff = this.now() - this.idleEvictionMs;
565
+ for (const [nodeId, partition] of [...this.partitions.entries()]) {
566
+ const lastWrite = partition.lastWriteAtMs();
567
+ if (lastWrite !== null && lastWrite > cutoff) continue;
568
+ this.dropPartition(nodeId);
278
569
  }
279
- return {
280
- readBytes,
281
- writeBytes,
282
- readOps,
283
- writeOps,
284
- timestampMs: now
285
- };
286
- } catch {}
287
- return {
288
- readBytes: 0,
289
- writeBytes: 0,
290
- readOps: 0,
291
- writeOps: 0,
292
- timestampMs: now
293
- };
570
+ }
571
+ /**
572
+ * Bring the fleet back under the row budget by dropping the OLDEST sample of
573
+ * the HEAVIEST partition, repeatedly.
574
+ *
575
+ * Terminates: every iteration removes one sample from a non-empty partition,
576
+ * and the population of samples is finite and strictly decreasing.
577
+ */
578
+ enforceRowBudget() {
579
+ while (this.totalRows > this.maxTotalProcessRows) {
580
+ const victim = this.heaviestPartition();
581
+ if (victim === null) return;
582
+ const [nodeId, partition] = victim;
583
+ this.totalRows -= partition.dropOldest();
584
+ if (partition.size() === 0) this.dropPartition(nodeId);
585
+ }
586
+ }
587
+ heaviestPartition() {
588
+ let best = null;
589
+ for (const entry of this.partitions.entries()) {
590
+ if (entry[1].size() === 0) continue;
591
+ if (best === null || entry[1].rowCount() > best[1].rowCount()) best = entry;
592
+ }
593
+ return best;
594
+ }
595
+ /** Drop `count` whole partitions, least-recently-written first. */
596
+ evictPartitions(count, protectedNodeId) {
597
+ const order = [...this.partitions.keys()].filter((nodeId) => nodeId !== protectedNodeId).toSorted((a, b) => (this.lastWriteSeq.get(a) ?? 0) - (this.lastWriteSeq.get(b) ?? 0));
598
+ let remaining = count;
599
+ for (const nodeId of order) {
600
+ if (remaining <= 0) return;
601
+ this.dropPartition(nodeId);
602
+ remaining -= 1;
603
+ }
604
+ }
605
+ /** Remove a partition and everything that indexes it. */
606
+ dropPartition(nodeId) {
607
+ const partition = this.partitions.get(nodeId);
608
+ if (partition === void 0) return;
609
+ this.totalRows -= partition.rowCount();
610
+ this.partitions.delete(nodeId);
611
+ this.lastWriteSeq.delete(nodeId);
612
+ }
613
+ };
614
+ //#endregion
615
+ //#region src/builtins/native-metrics/container-memory-series.ts
616
+ /**
617
+ * @durable class=audit owner=metrics-native
618
+ * write="one row per NODE per sample, appended by the HUB's `native-metrics`
619
+ * off its subscription to `metrics.node-processes-snapshot` — the same
620
+ * subscription, the same accepted-sample gate and the same timestamps as
621
+ * `metrics:node-load-samples`, so a memory row and the process rows of the
622
+ * same instant share an `atMs`. A node with no cgroup writes NOTHING; it
623
+ * never writes a zeroed row."
624
+ * retention="The operator's `loadSeriesRetentionHours` by age and
625
+ * CONTAINER_MEMORY_MAX_ROWS by count, swept in the same pass as the process
626
+ * table and through the same bounded page. Losing the table loses the cold
627
+ * window only: the ring keeps serving the last hour."
628
+ */
629
+ var CONTAINER_MEMORY_COLLECTION = "metrics:node-memory-samples";
630
+ /**
631
+ * Fleet ceiling on the hot ring, in READINGS.
632
+ *
633
+ * One reading per node per sample, so this is simply the per-node ceiling times
634
+ * the partition ceiling the process ring already declares (16). 2 880 readings
635
+ * of ten numbers each is well under a megabyte — the bound exists so a runaway
636
+ * node id space costs a fixed amount, not because the number is large.
637
+ */
638
+ var CONTAINER_MEMORY_MAX_TOTAL_ROWS = 2880;
639
+ /**
640
+ * Hard ceiling on the durable table, in ROWS. The guarantee behind the
641
+ * retention intention, exactly as `DEFAULT_MAX_ROWS` is for the process table.
642
+ *
643
+ * | cadence | retention | rows (3 nodes) |
644
+ * | --- | --- | --- |
645
+ * | 10 s | 6 h (default) | 6 480 |
646
+ * | 5 s | 6 h | 12 960 |
647
+ * | 5 s | 72 h, 16 nodes | 829 440 — capped here |
648
+ *
649
+ * 50 000 rows is 7.7x the shipped default. This row is strictly narrower than
650
+ * a `metrics:node-load-samples` row — fewer columns, one index, every value an
651
+ * integer varint — and that one was MEASURED at 87 B/row, so 50 000 rows is
652
+ * **under 4.2 MiB** with the real figure lower still. It is not a knob: a
653
+ * guarantee that moves with the hand that raised the intention is not one.
654
+ */
655
+ var CONTAINER_MEMORY_MAX_ROWS = 5e4;
656
+ var CONTAINER_MEMORY_COLUMNS = [
657
+ {
658
+ name: "id",
659
+ type: "INTEGER",
660
+ primaryKey: true,
661
+ notNull: true
662
+ },
663
+ {
664
+ name: "nodeId",
665
+ type: "TEXT",
666
+ notNull: true
667
+ },
668
+ (
669
+ /** Shared with the process rows of the same tick — that is the whole point. */
670
+ {
671
+ name: "atMs",
672
+ type: "INTEGER",
673
+ notNull: true
674
+ }),
675
+ {
676
+ name: "source",
677
+ type: "TEXT",
678
+ notNull: true
679
+ },
680
+ (
681
+ /**
682
+ * Whole MiB, like `rssMib` on the process row and for the same reason: a
683
+ * byte-exact residency on a value in the tens of gigabytes is precision no
684
+ * chart can draw, and it costs bytes on every row forever.
685
+ */
686
+ {
687
+ name: "currentMib",
688
+ type: "INTEGER",
689
+ notNull: true
690
+ }),
691
+ (
692
+ /** `NULL` = NO LIMIT. Never a sentinel. */
693
+ {
694
+ name: "limitMib",
695
+ type: "INTEGER"
696
+ }),
697
+ (
698
+ /** `NULL` = UNKNOWN on all four. Never zero. */
699
+ {
700
+ name: "anonMib",
701
+ type: "INTEGER"
702
+ }),
703
+ {
704
+ name: "fileMib",
705
+ type: "INTEGER"
706
+ },
707
+ {
708
+ name: "shmemMib",
709
+ type: "INTEGER"
710
+ },
711
+ {
712
+ name: "slabMib",
713
+ type: "INTEGER"
714
+ },
715
+ (
716
+ /** Host-wide i915 GEM bytes. `NULL` wherever debugfs is not mounted. */
717
+ {
718
+ name: "gpuShmemMib",
719
+ type: "INTEGER"
720
+ })
721
+ ];
722
+ var CONTAINER_MEMORY_INDEXES = [(
723
+ /** The one index, and it serves both the read and the prune — see the
724
+ * process table's `idx_load_samples_node_at` for the full reasoning. */
725
+ {
726
+ name: "idx_node_memory_node_at",
727
+ columns: ["nodeId", "atMs"]
728
+ })];
729
+ var BYTES_PER_MIB$1 = 1048576;
730
+ /** Bytes → whole MiB. `null` stays `null`: unknown is not zero. */
731
+ function toMib(value) {
732
+ return value === null ? null : Math.round(value / BYTES_PER_MIB$1);
733
+ }
734
+ /** Whole MiB → bytes. `null` stays `null`. */
735
+ function fromMib(value) {
736
+ return value === null ? null : value * BYTES_PER_MIB$1;
294
737
  }
295
- async function takeNetworkIoSnapshot() {
296
- const now = Date.now();
297
- if (IS_LINUX) try {
298
- const data = fs.readFileSync("/proc/net/dev", "utf-8");
299
- let rxBytes = 0, txBytes = 0, rxPackets = 0, txPackets = 0, rxErrors = 0, txErrors = 0;
300
- for (const line of data.split("\n")) {
301
- const match = line.match(/^\s*(\w+):\s+(.+)$/);
302
- if (!match) continue;
303
- if (match[1] === "lo") continue;
304
- const cols = match[2].trim().split(/\s+/);
305
- rxBytes += parseInt(cols[0], 10) || 0;
306
- rxPackets += parseInt(cols[1], 10) || 0;
307
- rxErrors += parseInt(cols[2], 10) || 0;
308
- txBytes += parseInt(cols[8], 10) || 0;
309
- txPackets += parseInt(cols[9], 10) || 0;
310
- txErrors += parseInt(cols[10], 10) || 0;
311
- }
312
- return {
313
- rxBytes,
314
- txBytes,
315
- rxPackets,
316
- txPackets,
317
- rxErrors,
318
- txErrors,
319
- timestampMs: now
320
- };
321
- } catch {}
322
- if (IS_DARWIN) try {
323
- const stdout = await execAsync("netstat", ["-ib"]);
324
- let rxBytes = 0, txBytes = 0, rxPackets = 0, txPackets = 0, rxErrors = 0, txErrors = 0;
325
- for (const line of stdout.split("\n")) {
326
- const parts = line.trim().split(/\s+/);
327
- if (parts.length < 11) continue;
328
- if (parts[0] === "lo0" || parts[0] === "Name") continue;
329
- if (!parts[2]?.startsWith("Link#")) continue;
330
- rxPackets += parseInt(parts[4], 10) || 0;
331
- rxErrors += parseInt(parts[5], 10) || 0;
332
- rxBytes += parseInt(parts[6], 10) || 0;
333
- txPackets += parseInt(parts[7], 10) || 0;
334
- txErrors += parseInt(parts[8], 10) || 0;
335
- txBytes += parseInt(parts[9], 10) || 0;
336
- }
337
- return {
338
- rxBytes,
339
- txBytes,
340
- rxPackets,
341
- txPackets,
342
- rxErrors,
343
- txErrors,
344
- timestampMs: now
345
- };
346
- } catch {}
738
+ /** The ONE encode. */
739
+ function snapshotToRow(nodeId, atMs, snapshot) {
347
740
  return {
348
- rxBytes: 0,
349
- txBytes: 0,
350
- rxPackets: 0,
351
- txPackets: 0,
352
- rxErrors: 0,
353
- txErrors: 0,
354
- timestampMs: now
741
+ nodeId,
742
+ atMs,
743
+ source: snapshot.source,
744
+ currentMib: Math.round(snapshot.currentBytes / BYTES_PER_MIB$1),
745
+ limitMib: toMib(snapshot.limitBytes),
746
+ anonMib: toMib(snapshot.anonBytes),
747
+ fileMib: toMib(snapshot.fileBytes),
748
+ shmemMib: toMib(snapshot.shmemBytes),
749
+ slabMib: toMib(snapshot.slabBytes),
750
+ gpuShmemMib: toMib(snapshot.gpuShmemBytes)
355
751
  };
356
752
  }
357
- async function getCpuTemperatureInternal() {
358
- if (IS_LINUX) for (const p of ["/sys/class/thermal/thermal_zone0/temp", "/sys/class/hwmon/hwmon0/temp1_input"]) try {
359
- const raw = fs.readFileSync(p, "utf-8").trim();
360
- const millideg = parseInt(raw, 10);
361
- if (!isNaN(millideg)) return Math.round(millideg / 100) / 10;
362
- } catch {
363
- continue;
364
- }
365
- if (IS_DARWIN) try {
366
- const match = (await execAsync("sudo", [
367
- "-n",
368
- "powermetrics",
369
- "--samplers",
370
- "smc",
371
- "-i",
372
- "1",
373
- "-n",
374
- "1"
375
- ], 3e3)).match(/CPU die temperature:\s+([\d.]+)\s*C/);
376
- if (match) return Math.round(parseFloat(match[1]) * 10) / 10;
377
- } catch {}
378
- return null;
379
- }
380
- async function getGpuInfoInternal() {
381
- if (IS_DARWIN) try {
382
- const utilMatch = (await execAsync("ioreg", [
383
- "-r",
384
- "-d",
385
- "1",
386
- "-c",
387
- "IOAccelerator"
388
- ])).match(/"GPU Core Utilization"\s*=\s*(\d+)/);
389
- const util = utilMatch ? parseInt(utilMatch[1], 10) : 0;
390
- const utilPct = util > 1e3 ? Math.round(util / 1e7) : util;
391
- return {
392
- utilization: Math.min(100, utilPct),
393
- model: "Apple Silicon GPU",
394
- memoryUsedBytes: 0,
395
- memoryTotalBytes: 0,
396
- temperature: null
397
- };
398
- } catch {}
399
- if (IS_LINUX) try {
400
- const parts = (await execAsync("nvidia-smi", ["--query-gpu=utilization.gpu,name,memory.used,memory.total,temperature.gpu", "--format=csv,noheader,nounits"])).trim().split(",").map((s) => s.trim());
401
- if (parts.length >= 5) return {
402
- utilization: parseInt(parts[0], 10) || 0,
403
- model: parts[1] ?? "Unknown GPU",
404
- memoryUsedBytes: (parseInt(parts[2], 10) || 0) * 1024 * 1024,
405
- memoryTotalBytes: (parseInt(parts[3], 10) || 0) * 1024 * 1024,
406
- temperature: parseInt(parts[4], 10) || null
407
- };
408
- } catch {}
409
- return null;
753
+ /**
754
+ * The ONE decode. A row whose `source` is not a hierarchy this build knows is
755
+ * skipped rather than coerced — a stored string that no longer means anything
756
+ * must not be charted as though it did.
757
+ */
758
+ function rowToPoint(row) {
759
+ if (row.source !== "cgroup-v2" && row.source !== "cgroup-v1") return null;
760
+ return {
761
+ atMs: row.atMs,
762
+ source: row.source,
763
+ currentBytes: row.currentMib * BYTES_PER_MIB$1,
764
+ limitBytes: fromMib(row.limitMib),
765
+ anonBytes: fromMib(row.anonMib),
766
+ fileBytes: fromMib(row.fileMib),
767
+ shmemBytes: fromMib(row.shmemMib),
768
+ slabBytes: fromMib(row.slabMib),
769
+ gpuShmemBytes: fromMib(row.gpuShmemMib)
770
+ };
410
771
  }
411
- function getProcessResources() {
412
- let openFds = 0;
413
- let threadCount = 0;
414
- if (IS_LINUX) {
415
- try {
416
- openFds = fs.readdirSync(`/proc/${process.pid}/fd`).length;
417
- } catch {}
418
- try {
419
- const match = fs.readFileSync(`/proc/${process.pid}/status`, "utf-8").match(/Threads:\s+(\d+)/);
420
- if (match) threadCount = parseInt(match[1], 10);
421
- } catch {}
422
- }
423
- const nodeProcess = process;
772
+ /** Structural validation on read. A malformed row is skipped, never charted. */
773
+ function recordToMemoryRow(data) {
774
+ const nodeId = data["nodeId"];
775
+ const source = data["source"];
776
+ if (typeof nodeId !== "string" || typeof source !== "string") return null;
777
+ const atMs = Number(data["atMs"]);
778
+ const currentMib = Number(data["currentMib"]);
779
+ if (!Number.isFinite(atMs) || !Number.isFinite(currentMib)) return null;
780
+ const optional = (raw) => {
781
+ if (raw === null || raw === void 0) return null;
782
+ const n = Number(raw);
783
+ return Number.isFinite(n) ? n : null;
784
+ };
424
785
  return {
425
- openFds,
426
- threadCount,
427
- activeHandles: (nodeProcess._getActiveHandles?.() ?? []).length,
428
- activeRequests: (nodeProcess._getActiveRequests?.() ?? []).length
786
+ nodeId,
787
+ atMs,
788
+ source,
789
+ currentMib,
790
+ limitMib: optional(data["limitMib"]),
791
+ anonMib: optional(data["anonMib"]),
792
+ fileMib: optional(data["fileMib"]),
793
+ shmemMib: optional(data["shmemMib"]),
794
+ slabMib: optional(data["slabMib"]),
795
+ gpuShmemMib: optional(data["gpuShmemMib"])
429
796
  };
430
797
  }
431
- function getPressure(resource) {
432
- if (!IS_LINUX) return null;
433
- try {
434
- const data = fs.readFileSync(`/proc/pressure/${resource}`, "utf-8");
435
- const parseLine = (prefix) => {
436
- const match = data.match(new RegExp(`${prefix} avg10=(\\d+\\.\\d+) avg60=(\\d+\\.\\d+) avg300=(\\d+\\.\\d+)`));
437
- if (!match) return null;
438
- return {
439
- avg10: parseFloat(match[1]),
440
- avg60: parseFloat(match[2]),
441
- avg300: parseFloat(match[3])
442
- };
443
- };
444
- const some = parseLine("some");
445
- if (!some) return null;
446
- return {
447
- some,
448
- full: parseLine("full")
449
- };
450
- } catch {
451
- return null;
798
+ /**
799
+ * The HOT tier — a naming adapter over {@link NodeLoadRing}, not a second ring.
800
+ *
801
+ * One reading per sample, so the ring's "rows" and its samples are the same
802
+ * count and its fleet budget is a sample budget. Every bound, the idle sweep
803
+ * and the monotonic replay gate come from there unchanged.
804
+ */
805
+ var ContainerMemoryRing = class {
806
+ ring;
807
+ constructor(options = {}) {
808
+ this.ring = new NodeLoadRing({
809
+ samplesPerNode: options.samplesPerNode ?? 180,
810
+ maxTotalProcessRows: CONTAINER_MEMORY_MAX_TOTAL_ROWS,
811
+ ...options.now !== void 0 ? { now: options.now } : {}
812
+ });
452
813
  }
453
- }
454
- function execAsync(cmd, args, timeoutMs = 5e3) {
455
- return new Promise((resolve, reject) => {
456
- execFile(cmd, args, { timeout: timeoutMs }, (err, stdout) => {
457
- if (err) return reject(err);
458
- resolve(stdout);
814
+ /** Retain one reading. `false` = a timestamp this node already delivered. */
815
+ record(nodeId, atMs, snapshot) {
816
+ return this.ring.record(nodeId, atMs, [snapshot]);
817
+ }
818
+ /**
819
+ * Read one node's retained readings, oldest-first. `sinceMs` is EXCLUSIVE,
820
+ * matching every other reader in this subsystem.
821
+ *
822
+ * A node nobody has recorded answers EMPTY — which is what "this node has no
823
+ * cgroup" looks like, and it must stay distinguishable from a zero.
824
+ */
825
+ read(nodeId, sinceMs) {
826
+ const series = this.ring.read(nodeId, sinceMs);
827
+ const out = [];
828
+ for (const sample of series.samples) {
829
+ const snapshot = sample.processes[0];
830
+ if (snapshot === void 0) continue;
831
+ out.push({
832
+ ...snapshot,
833
+ atMs: sample.atMs
834
+ });
835
+ }
836
+ return out;
837
+ }
838
+ };
839
+ /**
840
+ * Merge the two tiers and reduce, oldest-first.
841
+ *
842
+ * Dedupe is on `atMs` with the HOT copy winning, exactly as the process reader
843
+ * does: the ring holds the reading as it arrived, the table holds it after the
844
+ * MiB encode.
845
+ *
846
+ * Reduction keeps, for each bucket, the reading with the largest
847
+ * `currentBytes` — WHOLE, every field from that one sample. A per-field
848
+ * maximum would synthesise a row whose parts never coexisted (an `anon` peak
849
+ * and a `file` peak minutes apart, summed into a total that never happened),
850
+ * and a mean would smear away the spike this series exists to find. `atMs`
851
+ * becomes the bucket START so the points lie on the same x-axis as the folded
852
+ * process series, which is the only reason both travel in one answer.
853
+ *
854
+ * An empty bucket stays ABSENT — never zero, never carried forward.
855
+ */
856
+ function reduceContainerMemory(points, bucketMs, origin) {
857
+ if (bucketMs <= 0 || points.length === 0) return points;
858
+ const buckets = /* @__PURE__ */ new Map();
859
+ for (const point of points) {
860
+ const start = origin + Math.floor((point.atMs - origin) / bucketMs) * bucketMs;
861
+ const held = buckets.get(start);
862
+ if (held === void 0 || point.currentBytes > held.currentBytes) buckets.set(start, {
863
+ ...point,
864
+ atMs: start
459
865
  });
460
- });
866
+ }
867
+ return [...buckets.values()].toSorted((a, b) => a.atMs - b.atMs);
868
+ }
869
+ /** Merge cold + hot on `atMs`, hot wins. Exported because dedup is the
870
+ * property worth testing on its own. */
871
+ function mergeContainerMemory(cold, hot) {
872
+ const byAt = /* @__PURE__ */ new Map();
873
+ for (const point of cold) byAt.set(point.atMs, point);
874
+ for (const point of hot) byAt.set(point.atMs, point);
875
+ return [...byAt.values()].toSorted((a, b) => a.atMs - b.atMs);
461
876
  }
462
877
  /**
463
- * One node's fixed-capacity circular buffer of samples.
878
+ * The COLD tier. Append-only, bounded, and never the authority on anything the
879
+ * hot ring can answer.
464
880
  *
465
- * Mirrors {@link LogRingBuffer}: write at `head`, wrap, and once full every
466
- * accepted sample evicts the oldest. `rows` is carried alongside so the fleet
467
- * budget never has to walk the samples to know what a partition costs.
881
+ * Every method is best-effort and says so in the log rather than throwing at
882
+ * the emitter: a storage hiccup must cost a gap in a chart, never a dropped
883
+ * snapshot on the bus.
468
884
  */
469
- var LoadPartition = class {
470
- capacity;
471
- slots;
472
- head = 0;
473
- count = 0;
474
- rows = 0;
475
- /** Newest retained sample's timestamp — the monotonic gate for `push`. */
476
- newestAtMs = null;
477
- constructor(capacity) {
478
- this.capacity = capacity;
479
- this.slots = Array.from({ length: capacity });
885
+ var ContainerMemoryStore = class {
886
+ declared = false;
887
+ store;
888
+ logger;
889
+ nowFn;
890
+ prunePageRows;
891
+ constructor(deps) {
892
+ this.store = deps.store;
893
+ this.logger = deps.logger;
894
+ this.nowFn = deps.now ?? (() => Date.now());
895
+ this.prunePageRows = deps.prunePageRows ?? 2e4;
896
+ }
897
+ /** Idempotently declare the collection. `false` when the store refused. */
898
+ async declare() {
899
+ if (this.declared) return true;
900
+ try {
901
+ await this.store.declareCollection.mutate({
902
+ collection: CONTAINER_MEMORY_COLLECTION,
903
+ columns: [...CONTAINER_MEMORY_COLUMNS],
904
+ indexes: CONTAINER_MEMORY_INDEXES.map((i) => ({
905
+ name: i.name,
906
+ columns: [...i.columns]
907
+ }))
908
+ });
909
+ this.declared = true;
910
+ return true;
911
+ } catch (err) {
912
+ this.logger.warn("container memory declareCollection failed — nothing will be retained on disk", { meta: {
913
+ collection: CONTAINER_MEMORY_COLLECTION,
914
+ error: errMsg(err)
915
+ } });
916
+ return false;
917
+ }
918
+ }
919
+ /** Append ONE reading. */
920
+ async append(nodeId, atMs, snapshot) {
921
+ if (!await this.declare()) return 0;
922
+ const record = { ...snapshotToRow(nodeId, atMs, snapshot) };
923
+ try {
924
+ const { inserted } = await this.store.insertMany.mutate({
925
+ collection: CONTAINER_MEMORY_COLLECTION,
926
+ records: [{ data: record }]
927
+ });
928
+ return inserted;
929
+ } catch (err) {
930
+ this.logger.warn("container memory reading not retained — this interval will be missing", { meta: {
931
+ nodeId,
932
+ atMs,
933
+ error: errMsg(err)
934
+ } });
935
+ return 0;
936
+ }
937
+ }
938
+ /** Read one node's cold readings, oldest-first. `sinceMs` is EXCLUSIVE. */
939
+ async read(nodeId, sinceMs, limitRows) {
940
+ if (!await this.declare()) return [];
941
+ try {
942
+ const records = await this.store.query.query({
943
+ collection: CONTAINER_MEMORY_COLLECTION,
944
+ filter: {
945
+ where: { nodeId },
946
+ whereBetween: { atMs: [sinceMs + 1, Number.MAX_SAFE_INTEGER] },
947
+ orderBy: {
948
+ field: "atMs",
949
+ direction: "asc"
950
+ },
951
+ limit: limitRows
952
+ }
953
+ });
954
+ const points = [];
955
+ for (const record of records) {
956
+ const row = recordToMemoryRow(record.data);
957
+ if (row === null) continue;
958
+ const point = rowToPoint(row);
959
+ if (point !== null) points.push(point);
960
+ }
961
+ return points;
962
+ } catch (err) {
963
+ this.logger.warn("container memory cold read failed — answering from the hot window only", { meta: {
964
+ nodeId,
965
+ sinceMs,
966
+ error: errMsg(err)
967
+ } });
968
+ return [];
969
+ }
480
970
  }
481
971
  /**
482
- * Accept a sample.
483
- *
484
- * A sample at or before the newest one already held is REFUSED. The bus drops
485
- * a node's own broadcast echo, but a cross-node redelivery or a replayed
486
- * subscription must not be able to double a point — and idempotence here is
487
- * what lets every reader above treat the series as a set.
972
+ * Enforce BOTH bounds, oldest-first, through a bounded page each.
488
973
  *
489
- * `accepted` is reported separately from `rowDelta` on purpose: a full ring
490
- * that evicts a sample of the same size has a delta of zero and is not a
491
- * refusal, and conflating the two would silently stop advancing the write
492
- * ordinal on a steady-state cluster.
974
+ * Not rate-limited here: it is driven by the process store's own prune
975
+ * cadence, so it runs at most once a minute by construction and a second
976
+ * timer would be a second thing to get wrong.
493
977
  */
494
- push(sample) {
495
- if (this.newestAtMs !== null && sample.atMs <= this.newestAtMs) return {
496
- accepted: false,
497
- rowDelta: 0
498
- };
499
- const evicted = this.count === this.capacity ? this.slots[this.head]?.processes.length ?? 0 : 0;
500
- this.slots[this.head] = sample;
501
- this.head = (this.head + 1) % this.capacity;
502
- if (this.count < this.capacity) this.count++;
503
- this.newestAtMs = sample.atMs;
504
- const rowDelta = sample.processes.length - evicted;
505
- this.rows += rowDelta;
978
+ async prune(nodeIds, retention) {
979
+ if (!await this.declare()) return null;
980
+ const ageCutoff = this.nowFn() - retention.retentionHours * 36e5;
981
+ let deletedByAge = 0;
982
+ let rowsExamined = 0;
983
+ for (const nodeId of nodeIds) {
984
+ const outcome = await this.deleteOldestPage(nodeId, ageCutoff, this.prunePageRows);
985
+ deletedByAge += outcome.deleted;
986
+ rowsExamined += outcome.examined;
987
+ }
988
+ const total = await this.count();
989
+ const excess = total === null ? 0 : total - CONTAINER_MEMORY_MAX_ROWS;
990
+ let deletedByCap = 0;
991
+ if (excess > 0) {
992
+ let remaining = Math.min(excess, this.prunePageRows);
993
+ for (const nodeId of nodeIds) {
994
+ if (remaining <= 0) break;
995
+ const outcome = await this.deleteOldestPage(nodeId, null, remaining);
996
+ deletedByCap += outcome.deleted;
997
+ remaining -= outcome.deleted;
998
+ rowsExamined += outcome.examined;
999
+ }
1000
+ this.logger.warn("container memory ROW CAP bit — evicting the oldest readings", { meta: {
1001
+ collection: CONTAINER_MEMORY_COLLECTION,
1002
+ rows: total,
1003
+ cap: CONTAINER_MEMORY_MAX_ROWS,
1004
+ over: excess,
1005
+ deleted: deletedByCap,
1006
+ hint: "lower the retention or the sampling cadence — the cap is the guarantee, not the intention"
1007
+ } });
1008
+ }
506
1009
  return {
507
- accepted: true,
508
- rowDelta
1010
+ deletedByAge,
1011
+ deletedByCap,
1012
+ rowsExamined,
1013
+ capBit: excess > 0
509
1014
  };
510
1015
  }
511
- /** Drop the oldest sample. Returns the rows reclaimed (0 when empty). */
512
- dropOldest() {
513
- if (this.count === 0) return 0;
514
- const index = (this.head - this.count + this.capacity) % this.capacity;
515
- const victim = this.slots[index];
516
- this.slots[index] = void 0;
517
- this.count--;
518
- const reclaimed = victim?.processes.length ?? 0;
519
- this.rows -= reclaimed;
520
- if (this.count === 0) this.newestAtMs = null;
521
- return reclaimed;
522
- }
523
- /** Oldest-first, optionally only what is strictly newer than `sinceMs`. */
524
- list(sinceMs) {
525
- const out = [];
526
- for (let i = 0; i < this.count; i++) {
527
- const index = (this.head - this.count + i + this.capacity) % this.capacity;
528
- const sample = this.slots[index];
529
- if (sample === void 0) continue;
530
- if (sinceMs !== void 0 && sample.atMs <= sinceMs) continue;
531
- out.push(sample);
1016
+ /** Total rows, or `null` when the store could not answer. */
1017
+ async count() {
1018
+ if (!await this.declare()) return null;
1019
+ try {
1020
+ return await this.store.count.query({ collection: CONTAINER_MEMORY_COLLECTION });
1021
+ } catch (err) {
1022
+ this.logger.warn("container memory count failed — the row cap is not enforced this pass", { meta: { error: errMsg(err) } });
1023
+ return null;
532
1024
  }
533
- return out;
534
- }
535
- size() {
536
- return this.count;
537
- }
538
- rowCount() {
539
- return this.rows;
540
- }
541
- oldestAtMs() {
542
- if (this.count === 0) return null;
543
- const index = (this.head - this.count + this.capacity) % this.capacity;
544
- return this.slots[index]?.atMs ?? null;
545
- }
546
- lastWriteAtMs() {
547
- return this.newestAtMs;
548
1025
  }
549
- };
550
- /**
551
- * The partitioned ring: one {@link LoadPartition} per node id.
552
- *
553
- * Three bounds, doing three different jobs — the same division as the log
554
- * buffer, for the same reason:
555
- *
556
- * - **`samplesPerNode`** is HARD and per partition. It is what the operator
557
- * asked for (an hour) and it can never be exceeded.
558
- * - **`maxTotalProcessRows`** is the FLEET budget, and it is denominated in the
559
- * thing memory is proportional to. When it bites, the oldest sample of the
560
- * HEAVIEST partition is dropped — the node creating the pressure is the node
561
- * that pays, and a quiet node keeps its full hour. The node just written to is
562
- * NOT protected here: if it is the heaviest, it is the culprit.
563
- * - **`maxNodes`** is the backstop against an unbounded key space, and it drops
564
- * whole partitions least-recently-written first, never the one being written.
565
- *
566
- * Plus {@link DEFAULT_NODE_IDLE_EVICTION_MS}, which is what stops a node that
567
- * simply stopped emitting from living in memory forever.
568
- */
569
- var NodeLoadRing = class {
570
- partitions = /* @__PURE__ */ new Map();
571
1026
  /**
572
- * Node id write ordinal of its last accepted sample. A monotonic counter,
573
- * not a clock: partition eviction must follow the order writes actually
574
- * happened, and node clocks disagree.
1027
+ * Delete this node's oldest rows, at most one page's worth.
1028
+ *
1029
+ * `cutoff === null` means "the oldest rows, whatever their age" — the
1030
+ * row-cap path, which is bounded by `limitRows` instead. Either way the page
1031
+ * is READ FIRST, ordered by the one index, and its last `atMs` becomes the
1032
+ * effective cutoff for a single bounded delete. An unbounded `deleteWhere`
1033
+ * would be one statement and one stalling transaction the first time a
1034
+ * retention is lowered.
575
1035
  */
576
- lastWriteSeq = /* @__PURE__ */ new Map();
577
- writeSeq = 0;
578
- totalRows = 0;
579
- samplesPerNode;
580
- maxTotalProcessRows;
581
- maxNodes;
582
- idleEvictionMs;
583
- now;
584
- constructor(options = {}) {
585
- this.samplesPerNode = options.samplesPerNode ?? 180;
586
- this.maxTotalProcessRows = options.maxTotalProcessRows ?? 32e3;
587
- this.maxNodes = options.maxNodes ?? 16;
588
- this.idleEvictionMs = options.idleEvictionMs ?? 36e5;
589
- this.now = options.now ?? Date.now;
1036
+ async deleteOldestPage(nodeId, cutoff, limitRows) {
1037
+ if (limitRows <= 0) return {
1038
+ deleted: 0,
1039
+ examined: 0
1040
+ };
1041
+ try {
1042
+ const page = await this.store.query.query({
1043
+ collection: CONTAINER_MEMORY_COLLECTION,
1044
+ filter: {
1045
+ where: { nodeId },
1046
+ ...cutoff === null ? {} : { whereBetween: { atMs: [0, cutoff] } },
1047
+ orderBy: {
1048
+ field: "atMs",
1049
+ direction: "asc"
1050
+ },
1051
+ limit: Math.min(limitRows, this.prunePageRows)
1052
+ },
1053
+ columns: ["atMs"]
1054
+ });
1055
+ if (page.length === 0) return {
1056
+ deleted: 0,
1057
+ examined: 0
1058
+ };
1059
+ const effectiveCutoff = Number(page.at(-1)?.data["atMs"]);
1060
+ if (!Number.isFinite(effectiveCutoff)) return {
1061
+ deleted: 0,
1062
+ examined: page.length
1063
+ };
1064
+ const { deleted } = await this.store.deleteWhere.mutate({
1065
+ collection: CONTAINER_MEMORY_COLLECTION,
1066
+ filter: {
1067
+ where: { nodeId },
1068
+ whereBetween: { atMs: [0, effectiveCutoff] }
1069
+ }
1070
+ });
1071
+ return {
1072
+ deleted,
1073
+ examined: page.length
1074
+ };
1075
+ } catch (err) {
1076
+ this.logger.warn("container memory prune failed — the table keeps growing this pass", { meta: {
1077
+ nodeId,
1078
+ cutoff,
1079
+ error: errMsg(err)
1080
+ } });
1081
+ return {
1082
+ deleted: 0,
1083
+ examined: 0
1084
+ };
1085
+ }
1086
+ }
1087
+ };
1088
+ //#endregion
1089
+ //#region src/builtins/native-metrics/native-metrics-provider.ts
1090
+ var IS_DARWIN = os.platform() === "darwin";
1091
+ var IS_LINUX = os.platform() === "linux";
1092
+ var NativeMetricsProvider = class {
1093
+ id = "native";
1094
+ cachedSnapshot = null;
1095
+ samplingTimer = null;
1096
+ macMemTimer = null;
1097
+ prevCpu;
1098
+ diagnostics = null;
1099
+ consecutiveSampleErrors = 0;
1100
+ constructor() {
1101
+ this.prevCpu = aggregateCpuTimes();
590
1102
  }
591
1103
  /**
592
- * Retain one snapshot. `processes` is stored by reference the payload is
593
- * already an immutable arrival off the bus, and copying it would double the
594
- * measured cost for nothing.
595
- *
596
- * Returns whether the sample was ACCEPTED — that is, whether it was new
597
- * rather than a replay of a timestamp this node has already delivered.
598
- *
599
- * The return value is not diagnostics. It is the idempotence gate the
600
- * DURABLE tier rides on (`load-series-store.ts`): only an accepted sample is
601
- * appended to the table, which is what lets that table carry an `INTEGER`
602
- * rowid key instead of a composite unique index over two million rows. The
603
- * gate is one monotonic comparison in memory; the index it replaces was
604
- * measured at ~40 bytes per row.
1104
+ * Optional diagnostics sink for sampling errors. The first failure and
1105
+ * every Nth failure thereafter are reported so a permanently-broken
1106
+ * collector doesn't spam logs but is still observable.
605
1107
  */
606
- record(nodeId, atMs, processes) {
607
- if (nodeId.length === 0) return false;
608
- this.sweepIdle();
609
- let partition = this.partitions.get(nodeId);
610
- if (partition === void 0) {
611
- partition = new LoadPartition(this.samplesPerNode);
612
- this.partitions.set(nodeId, partition);
1108
+ setDiagnostics(diag) {
1109
+ this.diagnostics = diag;
1110
+ }
1111
+ startSampling(intervalMs) {
1112
+ if (IS_DARWIN) {
1113
+ sampleMacMemory(this);
1114
+ this.macMemTimer = setInterval(() => sampleMacMemory(this), 5e3);
1115
+ if (this.macMemTimer.unref) this.macMemTimer.unref();
613
1116
  }
614
- const outcome = partition.push({
615
- atMs,
616
- processes
1117
+ this.collectSnapshot().then((s) => {
1118
+ this.cachedSnapshot = s;
1119
+ this.consecutiveSampleErrors = 0;
1120
+ }).catch((err) => this.reportSampleError(err));
1121
+ this.samplingTimer = setInterval(() => {
1122
+ this.collectSnapshot().then((s) => {
1123
+ this.cachedSnapshot = s;
1124
+ this.consecutiveSampleErrors = 0;
1125
+ }).catch((err) => this.reportSampleError(err));
1126
+ }, intervalMs);
1127
+ if (this.samplingTimer.unref) this.samplingTimer.unref();
1128
+ }
1129
+ reportSampleError(err) {
1130
+ this.consecutiveSampleErrors++;
1131
+ if (!this.diagnostics) return;
1132
+ if (!(this.consecutiveSampleErrors === 1 || this.consecutiveSampleErrors % 60 === 0)) return;
1133
+ this.diagnostics.warn("metrics-provider sample failed", {
1134
+ consecutiveErrors: this.consecutiveSampleErrors,
1135
+ error: err instanceof Error ? err.message : String(err)
617
1136
  });
618
- if (!outcome.accepted) {
619
- if (partition.size() === 0) this.partitions.delete(nodeId);
620
- return false;
1137
+ }
1138
+ stopSampling() {
1139
+ if (this.samplingTimer) {
1140
+ clearInterval(this.samplingTimer);
1141
+ this.samplingTimer = null;
1142
+ }
1143
+ if (this.macMemTimer) {
1144
+ clearInterval(this.macMemTimer);
1145
+ this.macMemTimer = null;
621
1146
  }
622
- this.totalRows += outcome.rowDelta;
623
- this.lastWriteSeq.set(nodeId, ++this.writeSeq);
624
- if (this.partitions.size > this.maxNodes) this.evictPartitions(this.partitions.size - this.maxNodes, nodeId);
625
- this.enforceRowBudget();
626
- return true;
627
1147
  }
628
- /**
629
- * Read one node's retained series, oldest-first.
630
- *
631
- * `sinceMs` is EXCLUSIVE: a caller passes the newest timestamp it already
632
- * holds and gets back only what it is missing. That is the whole contract
633
- * that lets the admin UI seed from here and then continue live without
634
- * doubling a point it already drew.
635
- *
636
- * A node nobody has recorded answers with an empty series, not an error —
637
- * unknown is the truth about a node that has not reported.
638
- */
639
- read(nodeId, sinceMs) {
640
- this.sweepIdle();
641
- const partition = this.partitions.get(nodeId);
642
- if (partition === void 0) return {
643
- nodeId,
644
- samples: [],
645
- retainedSamples: 0,
646
- oldestAtMs: null,
647
- capacity: this.samplesPerNode
1148
+ async collectSnapshot() {
1149
+ const cpu = this.sampleCpu();
1150
+ const memory = this.getMemoryInfo();
1151
+ const processInfo = getProcessResources();
1152
+ const [gpu, network, disk, cpuTemp] = await Promise.all([
1153
+ getGpuInfoInternal().catch(() => null),
1154
+ takeNetworkIoSnapshot(),
1155
+ takeDiskIoSnapshot(),
1156
+ getCpuTemperatureInternal().catch(() => null)
1157
+ ]);
1158
+ const snapshot = {
1159
+ cpu,
1160
+ memory,
1161
+ gpu,
1162
+ network,
1163
+ disk,
1164
+ pressure: {
1165
+ cpu: getPressure("cpu"),
1166
+ memory: getPressure("memory"),
1167
+ io: getPressure("io")
1168
+ },
1169
+ process: processInfo,
1170
+ cpuTemperature: cpuTemp,
1171
+ timestampMs: Date.now()
648
1172
  };
1173
+ this.cachedSnapshot = snapshot;
1174
+ return snapshot;
1175
+ }
1176
+ async getCached() {
1177
+ return this.cachedSnapshot;
1178
+ }
1179
+ async getCurrent() {
1180
+ const s = this.cachedSnapshot ?? await this.collectSnapshot();
1181
+ const mb = (bytes) => Math.round(bytes / 1024 / 1024);
649
1182
  return {
650
- nodeId,
651
- samples: partition.list(sinceMs),
652
- retainedSamples: partition.size(),
653
- oldestAtMs: partition.oldestAtMs(),
654
- capacity: this.samplesPerNode
1183
+ cpuPercent: s.cpu.total,
1184
+ memoryPercent: s.memory.percent,
1185
+ memoryUsedMB: mb(s.memory.usedBytes),
1186
+ memoryTotalMB: mb(s.memory.totalBytes),
1187
+ temperature: s.cpuTemperature ?? void 0,
1188
+ gpuPercent: s.gpu?.utilization,
1189
+ gpuMemoryPercent: s.gpu && s.gpu.memoryTotalBytes > 0 ? Math.round(s.gpu.memoryUsedBytes / s.gpu.memoryTotalBytes * 1e3) / 10 : void 0
655
1190
  };
656
1191
  }
657
- /** Node ids with a live partition. Observability for the fleet bound. */
658
- nodeIds() {
659
- return [...this.partitions.keys()];
1192
+ async getDiskSpace(input) {
1193
+ return getDiskSpaceInternal(input.dirPath);
660
1194
  }
661
- /** Process rows retained across every partition — the number that IS memory. */
662
- rowCount() {
663
- return this.totalRows;
1195
+ async getGpuInfo() {
1196
+ return getGpuInfoInternal();
664
1197
  }
665
- /** Samples retained across every partition. */
666
- sampleCount() {
667
- let total = 0;
668
- for (const partition of this.partitions.values()) total += partition.size();
669
- return total;
1198
+ async getCpuTemperature() {
1199
+ return getCpuTemperatureInternal();
670
1200
  }
671
- /**
672
- * Drop partitions whose newest sample is older than the retention window.
673
- *
674
- * Lazy, on write and on read — never a timer. A timer would be a new
675
- * periodic cost in a subsystem whose entire premise is that it adds none,
676
- * and a ring that nobody writes to and nobody reads is not growing either.
677
- */
678
- sweepIdle() {
679
- const cutoff = this.now() - this.idleEvictionMs;
680
- for (const [nodeId, partition] of [...this.partitions.entries()]) {
681
- const lastWrite = partition.lastWriteAtMs();
682
- if (lastWrite !== null && lastWrite > cutoff) continue;
683
- this.dropPartition(nodeId);
1201
+ async getProcessStats(input) {
1202
+ const { getPidStats } = await import("../../resource-monitor-BWmQ5i-o.mjs").then((n) => n.r);
1203
+ const raw = await getPidStats(input.pids);
1204
+ const result = [];
1205
+ for (const [pid, s] of raw) result.push({
1206
+ pid,
1207
+ cpu: s.cpu,
1208
+ memory: s.memory,
1209
+ privateBytes: s.privateBytes,
1210
+ sharedBytes: s.sharedBytes
1211
+ });
1212
+ return result;
1213
+ }
1214
+ sampleCpu() {
1215
+ const curr = aggregateCpuTimes();
1216
+ const prev = this.prevCpu;
1217
+ this.prevCpu = curr;
1218
+ const dUser = curr.user - prev.user;
1219
+ const dNice = curr.nice - prev.nice;
1220
+ const dSys = curr.sys - prev.sys;
1221
+ const dIrq = curr.irq - prev.irq;
1222
+ const dIdle = curr.idle - prev.idle;
1223
+ const dTotal = dUser + dNice + dSys + dIrq + dIdle;
1224
+ const pct = (v) => dTotal === 0 ? 0 : Math.round(v / dTotal * 1e3) / 10;
1225
+ return {
1226
+ total: pct(dUser + dNice + dSys + dIrq),
1227
+ user: pct(dUser),
1228
+ system: pct(dSys),
1229
+ irq: pct(dIrq),
1230
+ nice: pct(dNice),
1231
+ loadAvg: os.loadavg(),
1232
+ cores: os.cpus().length
1233
+ };
1234
+ }
1235
+ /** Cached macOS memory (set by sampleMacMemory timer). */
1236
+ _cachedMacMem = null;
1237
+ getMemoryInfo() {
1238
+ const total = os.totalmem();
1239
+ if (total === 0) return {
1240
+ percent: 0,
1241
+ totalBytes: 0,
1242
+ usedBytes: 0,
1243
+ availableBytes: 0,
1244
+ swapUsedBytes: 0,
1245
+ swapTotalBytes: 0
1246
+ };
1247
+ if (IS_DARWIN && this._cachedMacMem !== null) return this._cachedMacMem;
1248
+ const free = os.freemem();
1249
+ const used = total - free;
1250
+ let swapUsedBytes = 0;
1251
+ let swapTotalBytes = 0;
1252
+ if (IS_LINUX) try {
1253
+ const meminfo = fs.readFileSync("/proc/meminfo", "utf-8");
1254
+ const swapTotal = parseInt(meminfo.match(/SwapTotal:\s+(\d+)/)?.[1] ?? "0", 10) * 1024;
1255
+ const swapFree = parseInt(meminfo.match(/SwapFree:\s+(\d+)/)?.[1] ?? "0", 10) * 1024;
1256
+ swapTotalBytes = swapTotal;
1257
+ swapUsedBytes = swapTotal - swapFree;
1258
+ } catch {}
1259
+ return {
1260
+ percent: Math.round((total - free) / total * 1e3) / 10,
1261
+ totalBytes: total,
1262
+ usedBytes: used,
1263
+ availableBytes: free,
1264
+ swapUsedBytes,
1265
+ swapTotalBytes
1266
+ };
1267
+ }
1268
+ };
1269
+ function aggregateCpuTimes() {
1270
+ const cpus = os.cpus();
1271
+ let user = 0, nice = 0, sys = 0, irq = 0, idle = 0;
1272
+ for (const cpu of cpus) {
1273
+ user += cpu.times.user;
1274
+ nice += cpu.times.nice;
1275
+ sys += cpu.times.sys;
1276
+ irq += cpu.times.irq;
1277
+ idle += cpu.times.idle;
1278
+ }
1279
+ return {
1280
+ user,
1281
+ nice,
1282
+ sys,
1283
+ irq,
1284
+ idle
1285
+ };
1286
+ }
1287
+ function sampleMacMemory(provider) {
1288
+ try {
1289
+ const output = execFileSync("vm_stat", {
1290
+ timeout: 2e3,
1291
+ encoding: "utf-8"
1292
+ });
1293
+ const pageSize = 16384;
1294
+ const parse = (label) => {
1295
+ const match = output.match(new RegExp(`${label}:\\s+(\\d+)`));
1296
+ return match ? parseInt(match[1], 10) * pageSize : 0;
1297
+ };
1298
+ const free = parse("Pages free");
1299
+ const inactive = parse("Pages inactive");
1300
+ const purgeable = parse("Pages purgeable");
1301
+ const speculative = parse("Pages speculative");
1302
+ const available = free + inactive + purgeable + speculative;
1303
+ const total = os.totalmem();
1304
+ const used = total - available;
1305
+ let swapUsedBytes = 0;
1306
+ let swapTotalBytes = 0;
1307
+ try {
1308
+ const swapOut = execFileSync("sysctl", ["-n", "vm.swapusage"], {
1309
+ timeout: 2e3,
1310
+ encoding: "utf-8"
1311
+ });
1312
+ const totalMatch = swapOut.match(/total\s*=\s*([\d.]+)M/);
1313
+ const usedMatch = swapOut.match(/used\s*=\s*([\d.]+)M/);
1314
+ if (totalMatch) swapTotalBytes = parseFloat(totalMatch[1]) * 1024 * 1024;
1315
+ if (usedMatch) swapUsedBytes = parseFloat(usedMatch[1]) * 1024 * 1024;
1316
+ } catch {}
1317
+ provider._cachedMacMem = {
1318
+ percent: Math.round(used / total * 1e3) / 10,
1319
+ totalBytes: total,
1320
+ usedBytes: used,
1321
+ availableBytes: available,
1322
+ swapUsedBytes,
1323
+ swapTotalBytes
1324
+ };
1325
+ } catch {
1326
+ provider._cachedMacMem = null;
1327
+ }
1328
+ }
1329
+ function getDiskSpaceInternal(dirPath) {
1330
+ return new Promise((resolve, reject) => {
1331
+ fs.statfs(dirPath, (err, stats) => {
1332
+ if (err) return reject(err);
1333
+ const totalBytes = stats.blocks * stats.bsize;
1334
+ const availableBytes = stats.bavail * stats.bsize;
1335
+ const usedBytes = totalBytes - stats.bfree * stats.bsize;
1336
+ resolve({
1337
+ path: dirPath,
1338
+ totalBytes,
1339
+ usedBytes,
1340
+ availableBytes,
1341
+ percent: totalBytes > 0 ? Math.round(usedBytes / totalBytes * 1e3) / 10 : 0
1342
+ });
1343
+ });
1344
+ });
1345
+ }
1346
+ async function takeDiskIoSnapshot() {
1347
+ const now = Date.now();
1348
+ if (IS_LINUX) try {
1349
+ const data = fs.readFileSync("/proc/diskstats", "utf-8");
1350
+ let readOps = 0, readBytes = 0, writeOps = 0, writeBytes = 0;
1351
+ for (const line of data.split("\n")) {
1352
+ const parts = line.trim().split(/\s+/);
1353
+ if (parts.length < 14) continue;
1354
+ const devName = parts[2];
1355
+ if (/\d+$/.test(devName) && !/^nvme\d+n\d+$/.test(devName)) continue;
1356
+ readOps += parseInt(parts[3], 10) || 0;
1357
+ readBytes += (parseInt(parts[5], 10) || 0) * 512;
1358
+ writeOps += parseInt(parts[7], 10) || 0;
1359
+ writeBytes += (parseInt(parts[9], 10) || 0) * 512;
684
1360
  }
685
- }
686
- /**
687
- * Bring the fleet back under the row budget by dropping the OLDEST sample of
688
- * the HEAVIEST partition, repeatedly.
689
- *
690
- * Terminates: every iteration removes one sample from a non-empty partition,
691
- * and the population of samples is finite and strictly decreasing.
692
- */
693
- enforceRowBudget() {
694
- while (this.totalRows > this.maxTotalProcessRows) {
695
- const victim = this.heaviestPartition();
696
- if (victim === null) return;
697
- const [nodeId, partition] = victim;
698
- this.totalRows -= partition.dropOldest();
699
- if (partition.size() === 0) this.dropPartition(nodeId);
1361
+ return {
1362
+ readBytes,
1363
+ writeBytes,
1364
+ readOps,
1365
+ writeOps,
1366
+ timestampMs: now
1367
+ };
1368
+ } catch {}
1369
+ return {
1370
+ readBytes: 0,
1371
+ writeBytes: 0,
1372
+ readOps: 0,
1373
+ writeOps: 0,
1374
+ timestampMs: now
1375
+ };
1376
+ }
1377
+ async function takeNetworkIoSnapshot() {
1378
+ const now = Date.now();
1379
+ if (IS_LINUX) try {
1380
+ const data = fs.readFileSync("/proc/net/dev", "utf-8");
1381
+ let rxBytes = 0, txBytes = 0, rxPackets = 0, txPackets = 0, rxErrors = 0, txErrors = 0;
1382
+ for (const line of data.split("\n")) {
1383
+ const match = line.match(/^\s*(\w+):\s+(.+)$/);
1384
+ if (!match) continue;
1385
+ if (match[1] === "lo") continue;
1386
+ const cols = match[2].trim().split(/\s+/);
1387
+ rxBytes += parseInt(cols[0], 10) || 0;
1388
+ rxPackets += parseInt(cols[1], 10) || 0;
1389
+ rxErrors += parseInt(cols[2], 10) || 0;
1390
+ txBytes += parseInt(cols[8], 10) || 0;
1391
+ txPackets += parseInt(cols[9], 10) || 0;
1392
+ txErrors += parseInt(cols[10], 10) || 0;
700
1393
  }
701
- }
702
- heaviestPartition() {
703
- let best = null;
704
- for (const entry of this.partitions.entries()) {
705
- if (entry[1].size() === 0) continue;
706
- if (best === null || entry[1].rowCount() > best[1].rowCount()) best = entry;
1394
+ return {
1395
+ rxBytes,
1396
+ txBytes,
1397
+ rxPackets,
1398
+ txPackets,
1399
+ rxErrors,
1400
+ txErrors,
1401
+ timestampMs: now
1402
+ };
1403
+ } catch {}
1404
+ if (IS_DARWIN) try {
1405
+ const stdout = await execAsync("netstat", ["-ib"]);
1406
+ let rxBytes = 0, txBytes = 0, rxPackets = 0, txPackets = 0, rxErrors = 0, txErrors = 0;
1407
+ for (const line of stdout.split("\n")) {
1408
+ const parts = line.trim().split(/\s+/);
1409
+ if (parts.length < 11) continue;
1410
+ if (parts[0] === "lo0" || parts[0] === "Name") continue;
1411
+ if (!parts[2]?.startsWith("Link#")) continue;
1412
+ rxPackets += parseInt(parts[4], 10) || 0;
1413
+ rxErrors += parseInt(parts[5], 10) || 0;
1414
+ rxBytes += parseInt(parts[6], 10) || 0;
1415
+ txPackets += parseInt(parts[7], 10) || 0;
1416
+ txErrors += parseInt(parts[8], 10) || 0;
1417
+ txBytes += parseInt(parts[9], 10) || 0;
707
1418
  }
708
- return best;
1419
+ return {
1420
+ rxBytes,
1421
+ txBytes,
1422
+ rxPackets,
1423
+ txPackets,
1424
+ rxErrors,
1425
+ txErrors,
1426
+ timestampMs: now
1427
+ };
1428
+ } catch {}
1429
+ return {
1430
+ rxBytes: 0,
1431
+ txBytes: 0,
1432
+ rxPackets: 0,
1433
+ txPackets: 0,
1434
+ rxErrors: 0,
1435
+ txErrors: 0,
1436
+ timestampMs: now
1437
+ };
1438
+ }
1439
+ async function getCpuTemperatureInternal() {
1440
+ if (IS_LINUX) for (const p of ["/sys/class/thermal/thermal_zone0/temp", "/sys/class/hwmon/hwmon0/temp1_input"]) try {
1441
+ const raw = fs.readFileSync(p, "utf-8").trim();
1442
+ const millideg = parseInt(raw, 10);
1443
+ if (!isNaN(millideg)) return Math.round(millideg / 100) / 10;
1444
+ } catch {
1445
+ continue;
709
1446
  }
710
- /** Drop `count` whole partitions, least-recently-written first. */
711
- evictPartitions(count, protectedNodeId) {
712
- const order = [...this.partitions.keys()].filter((nodeId) => nodeId !== protectedNodeId).toSorted((a, b) => (this.lastWriteSeq.get(a) ?? 0) - (this.lastWriteSeq.get(b) ?? 0));
713
- let remaining = count;
714
- for (const nodeId of order) {
715
- if (remaining <= 0) return;
716
- this.dropPartition(nodeId);
717
- remaining -= 1;
718
- }
1447
+ if (IS_DARWIN) try {
1448
+ const match = (await execAsync("sudo", [
1449
+ "-n",
1450
+ "powermetrics",
1451
+ "--samplers",
1452
+ "smc",
1453
+ "-i",
1454
+ "1",
1455
+ "-n",
1456
+ "1"
1457
+ ], 3e3)).match(/CPU die temperature:\s+([\d.]+)\s*C/);
1458
+ if (match) return Math.round(parseFloat(match[1]) * 10) / 10;
1459
+ } catch {}
1460
+ return null;
1461
+ }
1462
+ async function getGpuInfoInternal() {
1463
+ if (IS_DARWIN) try {
1464
+ const utilMatch = (await execAsync("ioreg", [
1465
+ "-r",
1466
+ "-d",
1467
+ "1",
1468
+ "-c",
1469
+ "IOAccelerator"
1470
+ ])).match(/"GPU Core Utilization"\s*=\s*(\d+)/);
1471
+ const util = utilMatch ? parseInt(utilMatch[1], 10) : 0;
1472
+ const utilPct = util > 1e3 ? Math.round(util / 1e7) : util;
1473
+ return {
1474
+ utilization: Math.min(100, utilPct),
1475
+ model: "Apple Silicon GPU",
1476
+ memoryUsedBytes: 0,
1477
+ memoryTotalBytes: 0,
1478
+ temperature: null
1479
+ };
1480
+ } catch {}
1481
+ if (IS_LINUX) try {
1482
+ const parts = (await execAsync("nvidia-smi", ["--query-gpu=utilization.gpu,name,memory.used,memory.total,temperature.gpu", "--format=csv,noheader,nounits"])).trim().split(",").map((s) => s.trim());
1483
+ if (parts.length >= 5) return {
1484
+ utilization: parseInt(parts[0], 10) || 0,
1485
+ model: parts[1] ?? "Unknown GPU",
1486
+ memoryUsedBytes: (parseInt(parts[2], 10) || 0) * 1024 * 1024,
1487
+ memoryTotalBytes: (parseInt(parts[3], 10) || 0) * 1024 * 1024,
1488
+ temperature: parseInt(parts[4], 10) || null
1489
+ };
1490
+ } catch {}
1491
+ return null;
1492
+ }
1493
+ function getProcessResources() {
1494
+ let openFds = 0;
1495
+ let threadCount = 0;
1496
+ if (IS_LINUX) {
1497
+ try {
1498
+ openFds = fs.readdirSync(`/proc/${process.pid}/fd`).length;
1499
+ } catch {}
1500
+ try {
1501
+ const match = fs.readFileSync(`/proc/${process.pid}/status`, "utf-8").match(/Threads:\s+(\d+)/);
1502
+ if (match) threadCount = parseInt(match[1], 10);
1503
+ } catch {}
719
1504
  }
720
- /** Remove a partition and everything that indexes it. */
721
- dropPartition(nodeId) {
722
- const partition = this.partitions.get(nodeId);
723
- if (partition === void 0) return;
724
- this.totalRows -= partition.rowCount();
725
- this.partitions.delete(nodeId);
726
- this.lastWriteSeq.delete(nodeId);
1505
+ const nodeProcess = process;
1506
+ return {
1507
+ openFds,
1508
+ threadCount,
1509
+ activeHandles: (nodeProcess._getActiveHandles?.() ?? []).length,
1510
+ activeRequests: (nodeProcess._getActiveRequests?.() ?? []).length
1511
+ };
1512
+ }
1513
+ function getPressure(resource) {
1514
+ if (!IS_LINUX) return null;
1515
+ try {
1516
+ const data = fs.readFileSync(`/proc/pressure/${resource}`, "utf-8");
1517
+ const parseLine = (prefix) => {
1518
+ const match = data.match(new RegExp(`${prefix} avg10=(\\d+\\.\\d+) avg60=(\\d+\\.\\d+) avg300=(\\d+\\.\\d+)`));
1519
+ if (!match) return null;
1520
+ return {
1521
+ avg10: parseFloat(match[1]),
1522
+ avg60: parseFloat(match[2]),
1523
+ avg300: parseFloat(match[3])
1524
+ };
1525
+ };
1526
+ const some = parseLine("some");
1527
+ if (!some) return null;
1528
+ return {
1529
+ some,
1530
+ full: parseLine("full")
1531
+ };
1532
+ } catch {
1533
+ return null;
727
1534
  }
728
- };
1535
+ }
1536
+ function execAsync(cmd, args, timeoutMs = 5e3) {
1537
+ return new Promise((resolve, reject) => {
1538
+ execFile(cmd, args, { timeout: timeoutMs }, (err, stdout) => {
1539
+ if (err) return reject(err);
1540
+ resolve(stdout);
1541
+ });
1542
+ });
1543
+ }
729
1544
  //#endregion
730
1545
  //#region src/builtins/native-metrics/load-series-store.ts
731
1546
  /**
@@ -1733,6 +2548,20 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
1733
2548
  * per sample, to reach that very same table.
1734
2549
  */
1735
2550
  loadStore = null;
2551
+ /**
2552
+ * The container's OWN memory, sampled on the process-snapshot tick.
2553
+ *
2554
+ * It rides that tick rather than a timer of its own so a memory reading and
2555
+ * the process rows it must be compared against carry the SAME timestamp —
2556
+ * the whole question is a subtraction, and two clocks make it an estimate.
2557
+ * Constructed in `onInitialize`, where the logger exists; it resolves the
2558
+ * cgroup once and latches.
2559
+ */
2560
+ memoryProbe = null;
2561
+ /** Hot tier for the container reading. Bounds in `container-memory-series.ts`. */
2562
+ memoryRing = new ContainerMemoryRing();
2563
+ /** Cold tier, HUB only — same reasoning as `loadStore`. */
2564
+ memoryStore = null;
1736
2565
  /** The resolved knobs. Re-resolved on every settings write. */
1737
2566
  loadConfig = {
1738
2567
  cadenceSec: 10,
@@ -1805,13 +2634,20 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
1805
2634
  dumpHeapSnapshot: (params) => this.dumpHeapSnapshot(params)
1806
2635
  };
1807
2636
  this.applyLoadSeriesConfig();
1808
- if (this.isHub) this.loadStore = new LoadSeriesStore({
1809
- store: this.ctx.api.settingsStore,
1810
- logger: this.ctx.logger.child("LoadSeries")
1811
- });
2637
+ if (this.isHub) {
2638
+ this.loadStore = new LoadSeriesStore({
2639
+ store: this.ctx.api.settingsStore,
2640
+ logger: this.ctx.logger.child("LoadSeries")
2641
+ });
2642
+ this.memoryStore = new ContainerMemoryStore({
2643
+ store: this.ctx.api.settingsStore,
2644
+ logger: this.ctx.logger.child("ContainerMemory")
2645
+ });
2646
+ }
2647
+ this.memoryProbe = new ContainerMemoryProbe({ logger: this.ctx.logger.child("ContainerMemory") });
1812
2648
  this.snapshotTimer = setInterval(() => this.emitResourcesSnapshot(), METRICS_SNAPSHOT_INTERVAL_MS);
1813
2649
  this.startProcessSnapshotTimer();
1814
- this.ctx.addDisposer(this.ctx.eventBus.subscribe({ category: EventCategory.MetricsNodeProcessesSnapshot }, (event) => this.retainSnapshot(event.data.nodeId, event.data.timestamp, event.data.processes)));
2650
+ this.ctx.addDisposer(this.ctx.eventBus.subscribe({ category: EventCategory.MetricsNodeProcessesSnapshot }, (event) => this.retainSnapshot(event.data.nodeId, event.data.timestamp, event.data.processes, event.data.containerMemory)));
1815
2651
  return [{
1816
2652
  capability: metricsProviderCapability,
1817
2653
  provider: composed
@@ -1829,19 +2665,31 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
1829
2665
  * The durable append is fire-and-forget: a storage stall must cost a gap in
1830
2666
  * the cold window, never a blocked event-bus handler. Every failure logs.
1831
2667
  */
1832
- retainSnapshot(nodeId, atMs, processes) {
2668
+ retainSnapshot(nodeId, atMs, processes, containerMemory) {
1833
2669
  const retained = processes.map((p) => NativeMetricsAddon.toRetained(p));
1834
2670
  if (!this.loadRing.record(nodeId, atMs, retained)) return;
1835
2671
  this.observedRowsByNode.set(nodeId, retained.length);
2672
+ if (containerMemory !== null) this.memoryRing.record(nodeId, atMs, containerMemory);
1836
2673
  const store = this.loadStore;
1837
2674
  if (store === null) return;
1838
- store.append(nodeId, atMs, retained).then(() => store.prune([...this.observedRowsByNode.keys()], this.loadConfig)).catch((err) => {
2675
+ store.append(nodeId, atMs, retained).then(() => store.prune([...this.observedRowsByNode.keys()], this.loadConfig)).then(async (outcome) => {
2676
+ if (outcome === null) return;
2677
+ await this.memoryStore?.prune([...this.observedRowsByNode.keys()], this.loadConfig);
2678
+ }).catch((err) => {
1839
2679
  this.ctx.logger.warn("durable load series write failed", { meta: {
1840
2680
  nodeId,
1841
2681
  atMs,
1842
2682
  error: err instanceof Error ? err.message : String(err)
1843
2683
  } });
1844
2684
  });
2685
+ if (containerMemory === null) return;
2686
+ this.memoryStore?.append(nodeId, atMs, containerMemory).catch((err) => {
2687
+ this.ctx.logger.warn("durable container memory write failed", { meta: {
2688
+ nodeId,
2689
+ atMs,
2690
+ error: err instanceof Error ? err.message : String(err)
2691
+ } });
2692
+ });
1845
2693
  }
1846
2694
  /**
1847
2695
  * The ONE reader, over both tiers.
@@ -1870,6 +2718,14 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
1870
2718
  return {
1871
2719
  nodeId: params.forNodeId,
1872
2720
  series: merged.series,
2721
+ containerMemory: await this.readContainerMemory({
2722
+ forNodeId: params.forNodeId,
2723
+ sinceMs,
2724
+ ...params.sinceMs !== void 0 ? { exclusiveSinceMs: params.sinceMs } : {},
2725
+ bucketMs: merged.bucketMs,
2726
+ cadenceMs,
2727
+ origin: merged.oldestAtMs
2728
+ }),
1873
2729
  bucketMs: merged.bucketMs,
1874
2730
  retainedSamples: merged.retainedSamples,
1875
2731
  oldestAtMs: merged.oldestAtMs,
@@ -1878,6 +2734,24 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
1878
2734
  };
1879
2735
  }
1880
2736
  /**
2737
+ * The container's memory over the same window, on the same x-axis.
2738
+ *
2739
+ * Both tiers, merged and deduped on `atMs` like the process series, then
2740
+ * reduced against the SAME `bucketMs` and the SAME origin — otherwise the
2741
+ * two series in one answer would sit on two different axes and the
2742
+ * subtraction they exist for would be an eyeball.
2743
+ *
2744
+ * An EMPTY answer means the node has no cgroup, or nothing has been retained
2745
+ * for it yet. It never means zero.
2746
+ */
2747
+ async readContainerMemory(params) {
2748
+ const hot = this.memoryRing.read(params.forNodeId, params.exclusiveSinceMs);
2749
+ const store = this.memoryStore;
2750
+ const merged = mergeContainerMemory(store === null ? [] : await store.read(params.forNodeId, params.sinceMs, CONTAINER_MEMORY_MAX_ROWS), hot);
2751
+ if (params.bucketMs <= params.cadenceMs) return merged;
2752
+ return reduceContainerMemory(merged, params.bucketMs, params.origin ?? 0);
2753
+ }
2754
+ /**
1881
2755
  * Re-resolve the knobs and restate the fixed cadence.
1882
2756
  *
1883
2757
  * A REFUSED value (out of 5-60 s, or out of 1-72 h) leaves the previous
@@ -1993,6 +2867,7 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
1993
2867
  const timestamp = Date.now();
1994
2868
  try {
1995
2869
  const processes = await this.listNodeProcesses();
2870
+ const containerMemory = this.memoryProbe?.sample() ?? null;
1996
2871
  eventBus.emit(createEvent(EventCategory.MetricsNodeProcessesSnapshot, {
1997
2872
  type: "node",
1998
2873
  id: nodeId,
@@ -2000,7 +2875,8 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
2000
2875
  }, {
2001
2876
  nodeId,
2002
2877
  processes,
2003
- timestamp
2878
+ timestamp,
2879
+ containerMemory
2004
2880
  }));
2005
2881
  } catch (err) {
2006
2882
  this.ctx.logger.warn("process snapshot skipped — this interval will be missing", { meta: {