@camstack/system 1.2.446 → 1.2.447

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -688,11 +688,9 @@ var NodeLoadRing = class {
688
688
  * rather than a replay of a timestamp this node has already delivered.
689
689
  *
690
690
  * The return value is not diagnostics. It is the idempotence gate the
691
- * DURABLE tier rides on (`load-series-store.ts`): only an accepted sample is
692
- * appended to the table, which is what lets that table carry an `INTEGER`
693
- * rowid key instead of a composite unique index over two million rows. The
694
- * gate is one monotonic comparison in memory; the index it replaces was
695
- * measured at ~40 bytes per row.
691
+ * DURABLE tier rides on this return value (`node-metrics-history-recorder.ts`):
692
+ * only an accepted sample is sent to device-history. The gate is one monotonic
693
+ * comparison in memory.
696
694
  */
697
695
  record(nodeId, atMs, processes) {
698
696
  if (nodeId.length === 0) return false;
@@ -817,197 +815,7 @@ var NodeLoadRing = class {
817
815
  this.lastWriteSeq.delete(nodeId);
818
816
  }
819
817
  };
820
- //#endregion
821
- //#region src/builtins/native-metrics/container-memory-series.ts
822
- /**
823
- * @durable class=audit owner=metrics-native
824
- * write="one row per NODE per sample, appended by the HUB's `native-metrics`
825
- * off its subscription to `metrics.node-processes-snapshot` — the same
826
- * subscription, the same accepted-sample gate and the same timestamps as
827
- * `metrics:node-load-samples`, so a memory row and the process rows of the
828
- * same instant share an `atMs`. A node with no cgroup writes NOTHING; it
829
- * never writes a zeroed row."
830
- * retention="The operator's `loadSeriesRetentionHours` by age and
831
- * CONTAINER_MEMORY_MAX_ROWS by count, swept in the same pass as the process
832
- * table and through the same bounded page. Losing the table loses the cold
833
- * window only: the ring keeps serving the last hour."
834
- */
835
- var CONTAINER_MEMORY_COLLECTION = "metrics:node-memory-samples";
836
- /**
837
- * Fleet ceiling on the hot ring, in READINGS.
838
- *
839
- * One reading per node per sample, so this is simply the per-node ceiling times
840
- * the partition ceiling the process ring already declares (16). 2 880 readings
841
- * of ten numbers each is well under a megabyte — the bound exists so a runaway
842
- * node id space costs a fixed amount, not because the number is large.
843
- */
844
818
  var CONTAINER_MEMORY_MAX_TOTAL_ROWS = 2880;
845
- /**
846
- * Hard ceiling on the durable table, in ROWS. The guarantee behind the
847
- * retention intention, exactly as `DEFAULT_MAX_ROWS` is for the process table.
848
- *
849
- * | cadence | retention | rows (3 nodes) |
850
- * | --- | --- | --- |
851
- * | 10 s | 6 h (default) | 6 480 |
852
- * | 5 s | 6 h | 12 960 |
853
- * | 5 s | 72 h, 16 nodes | 829 440 — capped here |
854
- *
855
- * 50 000 rows is 7.7x the shipped default. This row is strictly narrower than
856
- * a `metrics:node-load-samples` row — fewer columns, one index, every value an
857
- * integer varint — and that one was MEASURED at 87 B/row, so 50 000 rows is
858
- * **under 4.2 MiB** with the real figure lower still. It is not a knob: a
859
- * guarantee that moves with the hand that raised the intention is not one.
860
- */
861
- var CONTAINER_MEMORY_MAX_ROWS = 5e4;
862
- var CONTAINER_MEMORY_COLUMNS = [
863
- {
864
- name: "id",
865
- type: "INTEGER",
866
- primaryKey: true,
867
- notNull: true
868
- },
869
- {
870
- name: "nodeId",
871
- type: "TEXT",
872
- notNull: true
873
- },
874
- (
875
- /** Shared with the process rows of the same tick — that is the whole point. */
876
- {
877
- name: "atMs",
878
- type: "INTEGER",
879
- notNull: true
880
- }),
881
- {
882
- name: "source",
883
- type: "TEXT",
884
- notNull: true
885
- },
886
- (
887
- /**
888
- * Whole MiB, like `rssMib` on the process row and for the same reason: a
889
- * byte-exact residency on a value in the tens of gigabytes is precision no
890
- * chart can draw, and it costs bytes on every row forever.
891
- */
892
- {
893
- name: "currentMib",
894
- type: "INTEGER",
895
- notNull: true
896
- }),
897
- (
898
- /** `NULL` = NO LIMIT. Never a sentinel. */
899
- {
900
- name: "limitMib",
901
- type: "INTEGER"
902
- }),
903
- (
904
- /** `NULL` = UNKNOWN on all four. Never zero. */
905
- {
906
- name: "anonMib",
907
- type: "INTEGER"
908
- }),
909
- {
910
- name: "fileMib",
911
- type: "INTEGER"
912
- },
913
- {
914
- name: "shmemMib",
915
- type: "INTEGER"
916
- },
917
- {
918
- name: "slabMib",
919
- type: "INTEGER"
920
- },
921
- (
922
- /** Host-wide i915 GEM bytes. `NULL` wherever debugfs is not mounted. */
923
- {
924
- name: "gpuShmemMib",
925
- type: "INTEGER"
926
- })
927
- ];
928
- var CONTAINER_MEMORY_INDEXES = [(
929
- /** The one index, and it serves both the read and the prune — see the
930
- * process table's `idx_load_samples_node_at` for the full reasoning. */
931
- {
932
- name: "idx_node_memory_node_at",
933
- columns: ["nodeId", "atMs"]
934
- })];
935
- var BYTES_PER_MIB$1 = 1048576;
936
- /** Bytes → whole MiB. `null` stays `null`: unknown is not zero. */
937
- function toMib(value) {
938
- return value === null ? null : Math.round(value / BYTES_PER_MIB$1);
939
- }
940
- /** Whole MiB → bytes. `null` stays `null`. */
941
- function fromMib(value) {
942
- return value === null ? null : value * BYTES_PER_MIB$1;
943
- }
944
- /** The ONE encode. */
945
- function snapshotToRow(nodeId, atMs, snapshot) {
946
- return {
947
- nodeId,
948
- atMs,
949
- source: snapshot.source,
950
- currentMib: Math.round(snapshot.currentBytes / BYTES_PER_MIB$1),
951
- limitMib: toMib(snapshot.limitBytes),
952
- anonMib: toMib(snapshot.anonBytes),
953
- fileMib: toMib(snapshot.fileBytes),
954
- shmemMib: toMib(snapshot.shmemBytes),
955
- slabMib: toMib(snapshot.slabBytes),
956
- gpuShmemMib: toMib(snapshot.gpuShmemBytes)
957
- };
958
- }
959
- /**
960
- * The ONE decode. A row whose `source` is not a hierarchy this build knows is
961
- * skipped rather than coerced — a stored string that no longer means anything
962
- * must not be charted as though it did.
963
- */
964
- function rowToPoint(row) {
965
- if (row.source !== "cgroup-v2" && row.source !== "cgroup-v1") return null;
966
- return {
967
- atMs: row.atMs,
968
- source: row.source,
969
- currentBytes: row.currentMib * BYTES_PER_MIB$1,
970
- limitBytes: fromMib(row.limitMib),
971
- anonBytes: fromMib(row.anonMib),
972
- fileBytes: fromMib(row.fileMib),
973
- shmemBytes: fromMib(row.shmemMib),
974
- slabBytes: fromMib(row.slabMib),
975
- gpuShmemBytes: fromMib(row.gpuShmemMib)
976
- };
977
- }
978
- /** Structural validation on read. A malformed row is skipped, never charted. */
979
- function recordToMemoryRow(data) {
980
- const nodeId = data["nodeId"];
981
- const source = data["source"];
982
- if (typeof nodeId !== "string" || typeof source !== "string") return null;
983
- const atMs = Number(data["atMs"]);
984
- const currentMib = Number(data["currentMib"]);
985
- if (!Number.isFinite(atMs) || !Number.isFinite(currentMib)) return null;
986
- const optional = (raw) => {
987
- if (raw === null || raw === void 0) return null;
988
- const n = Number(raw);
989
- return Number.isFinite(n) ? n : null;
990
- };
991
- return {
992
- nodeId,
993
- atMs,
994
- source,
995
- currentMib,
996
- limitMib: optional(data["limitMib"]),
997
- anonMib: optional(data["anonMib"]),
998
- fileMib: optional(data["fileMib"]),
999
- shmemMib: optional(data["shmemMib"]),
1000
- slabMib: optional(data["slabMib"]),
1001
- gpuShmemMib: optional(data["gpuShmemMib"])
1002
- };
1003
- }
1004
- /**
1005
- * The HOT tier — a naming adapter over {@link NodeLoadRing}, not a second ring.
1006
- *
1007
- * One reading per sample, so the ring's "rows" and its samples are the same
1008
- * count and its fleet budget is a sample budget. Every bound, the idle sweep
1009
- * and the monotonic replay gate come from there unchanged.
1010
- */
1011
819
  var ContainerMemoryRing = class {
1012
820
  ring;
1013
821
  constructor(options = {}) {
@@ -1017,17 +825,9 @@ var ContainerMemoryRing = class {
1017
825
  ...options.now !== void 0 ? { now: options.now } : {}
1018
826
  });
1019
827
  }
1020
- /** Retain one reading. `false` = a timestamp this node already delivered. */
1021
828
  record(nodeId, atMs, snapshot) {
1022
829
  return this.ring.record(nodeId, atMs, [snapshot]);
1023
830
  }
1024
- /**
1025
- * Read one node's retained readings, oldest-first. `sinceMs` is EXCLUSIVE,
1026
- * matching every other reader in this subsystem.
1027
- *
1028
- * A node nobody has recorded answers EMPTY — which is what "this node has no
1029
- * cgroup" looks like, and it must stay distinguishable from a zero.
1030
- */
1031
831
  read(nodeId, sinceMs) {
1032
832
  const series = this.ring.read(nodeId, sinceMs);
1033
833
  const out = [];
@@ -1042,23 +842,6 @@ var ContainerMemoryRing = class {
1042
842
  return out;
1043
843
  }
1044
844
  };
1045
- /**
1046
- * Merge the two tiers and reduce, oldest-first.
1047
- *
1048
- * Dedupe is on `atMs` with the HOT copy winning, exactly as the process reader
1049
- * does: the ring holds the reading as it arrived, the table holds it after the
1050
- * MiB encode.
1051
- *
1052
- * Reduction keeps, for each bucket, the reading with the largest
1053
- * `currentBytes` — WHOLE, every field from that one sample. A per-field
1054
- * maximum would synthesise a row whose parts never coexisted (an `anon` peak
1055
- * and a `file` peak minutes apart, summed into a total that never happened),
1056
- * and a mean would smear away the spike this series exists to find. `atMs`
1057
- * becomes the bucket START so the points lie on the same x-axis as the folded
1058
- * process series, which is the only reason both travel in one answer.
1059
- *
1060
- * An empty bucket stays ABSENT — never zero, never carried forward.
1061
- */
1062
845
  function reduceContainerMemory(points, bucketMs, origin) {
1063
846
  if (bucketMs <= 0 || points.length === 0) return points;
1064
847
  const buckets = /* @__PURE__ */ new Map();
@@ -1072,727 +855,13 @@ function reduceContainerMemory(points, bucketMs, origin) {
1072
855
  }
1073
856
  return [...buckets.values()].toSorted((a, b) => a.atMs - b.atMs);
1074
857
  }
1075
- /** Merge cold + hot on `atMs`, hot wins. Exported because dedup is the
1076
- * property worth testing on its own. */
1077
858
  function mergeContainerMemory(cold, hot) {
1078
859
  const byAt = /* @__PURE__ */ new Map();
1079
860
  for (const point of cold) byAt.set(point.atMs, point);
1080
861
  for (const point of hot) byAt.set(point.atMs, point);
1081
862
  return [...byAt.values()].toSorted((a, b) => a.atMs - b.atMs);
1082
863
  }
1083
- /**
1084
- * The COLD tier. Append-only, bounded, and never the authority on anything the
1085
- * hot ring can answer.
1086
- *
1087
- * Every method is best-effort and says so in the log rather than throwing at
1088
- * the emitter: a storage hiccup must cost a gap in a chart, never a dropped
1089
- * snapshot on the bus.
1090
- */
1091
- var ContainerMemoryStore = class {
1092
- declared = false;
1093
- store;
1094
- logger;
1095
- nowFn;
1096
- prunePageRows;
1097
- constructor(deps) {
1098
- this.store = deps.store;
1099
- this.logger = deps.logger;
1100
- this.nowFn = deps.now ?? (() => Date.now());
1101
- this.prunePageRows = deps.prunePageRows ?? 2e4;
1102
- }
1103
- /** Idempotently declare the collection. `false` when the store refused. */
1104
- async declare() {
1105
- if (this.declared) return true;
1106
- try {
1107
- await this.store.declareCollection.mutate({
1108
- collection: CONTAINER_MEMORY_COLLECTION,
1109
- columns: [...CONTAINER_MEMORY_COLUMNS],
1110
- indexes: CONTAINER_MEMORY_INDEXES.map((i) => ({
1111
- name: i.name,
1112
- columns: [...i.columns]
1113
- }))
1114
- });
1115
- this.declared = true;
1116
- return true;
1117
- } catch (err) {
1118
- this.logger.warn("container memory declareCollection failed — nothing will be retained on disk", { meta: {
1119
- collection: CONTAINER_MEMORY_COLLECTION,
1120
- error: errMsg(err)
1121
- } });
1122
- return false;
1123
- }
1124
- }
1125
- /** Append ONE reading. */
1126
- async append(nodeId, atMs, snapshot) {
1127
- if (!await this.declare()) return 0;
1128
- const record = { ...snapshotToRow(nodeId, atMs, snapshot) };
1129
- try {
1130
- const { inserted } = await this.store.insertMany.mutate({
1131
- collection: CONTAINER_MEMORY_COLLECTION,
1132
- records: [{ data: record }]
1133
- });
1134
- return inserted;
1135
- } catch (err) {
1136
- this.logger.warn("container memory reading not retained — this interval will be missing", { meta: {
1137
- nodeId,
1138
- atMs,
1139
- error: errMsg(err)
1140
- } });
1141
- return 0;
1142
- }
1143
- }
1144
- /** Read one node's cold readings, oldest-first. `sinceMs` is EXCLUSIVE. */
1145
- async read(nodeId, sinceMs, limitRows) {
1146
- if (!await this.declare()) return [];
1147
- try {
1148
- const records = await this.store.query.query({
1149
- collection: CONTAINER_MEMORY_COLLECTION,
1150
- filter: {
1151
- where: { nodeId },
1152
- whereBetween: { atMs: [sinceMs + 1, Number.MAX_SAFE_INTEGER] },
1153
- orderBy: {
1154
- field: "atMs",
1155
- direction: "asc"
1156
- },
1157
- limit: limitRows
1158
- }
1159
- });
1160
- const points = [];
1161
- for (const record of records) {
1162
- const row = recordToMemoryRow(record.data);
1163
- if (row === null) continue;
1164
- const point = rowToPoint(row);
1165
- if (point !== null) points.push(point);
1166
- }
1167
- return points;
1168
- } catch (err) {
1169
- this.logger.warn("container memory cold read failed — answering from the hot window only", { meta: {
1170
- nodeId,
1171
- sinceMs,
1172
- error: errMsg(err)
1173
- } });
1174
- return [];
1175
- }
1176
- }
1177
- /**
1178
- * Enforce BOTH bounds, oldest-first, through a bounded page each.
1179
- *
1180
- * Not rate-limited here: it is driven by the process store's own prune
1181
- * cadence, so it runs at most once a minute by construction and a second
1182
- * timer would be a second thing to get wrong.
1183
- */
1184
- async prune(nodeIds, retention) {
1185
- if (!await this.declare()) return null;
1186
- const ageCutoff = this.nowFn() - retention.retentionHours * 36e5;
1187
- let deletedByAge = 0;
1188
- let rowsExamined = 0;
1189
- for (const nodeId of nodeIds) {
1190
- const outcome = await this.deleteOldestPage(nodeId, ageCutoff, this.prunePageRows);
1191
- deletedByAge += outcome.deleted;
1192
- rowsExamined += outcome.examined;
1193
- }
1194
- const total = await this.count();
1195
- const excess = total === null ? 0 : total - CONTAINER_MEMORY_MAX_ROWS;
1196
- let deletedByCap = 0;
1197
- if (excess > 0) {
1198
- let remaining = Math.min(excess, this.prunePageRows);
1199
- for (const nodeId of nodeIds) {
1200
- if (remaining <= 0) break;
1201
- const outcome = await this.deleteOldestPage(nodeId, null, remaining);
1202
- deletedByCap += outcome.deleted;
1203
- remaining -= outcome.deleted;
1204
- rowsExamined += outcome.examined;
1205
- }
1206
- this.logger.warn("container memory ROW CAP bit — evicting the oldest readings", { meta: {
1207
- collection: CONTAINER_MEMORY_COLLECTION,
1208
- rows: total,
1209
- cap: CONTAINER_MEMORY_MAX_ROWS,
1210
- over: excess,
1211
- deleted: deletedByCap,
1212
- hint: "lower the retention or the sampling cadence — the cap is the guarantee, not the intention"
1213
- } });
1214
- }
1215
- return {
1216
- deletedByAge,
1217
- deletedByCap,
1218
- rowsExamined,
1219
- capBit: excess > 0
1220
- };
1221
- }
1222
- /** Total rows, or `null` when the store could not answer. */
1223
- async count() {
1224
- if (!await this.declare()) return null;
1225
- try {
1226
- return await this.store.count.query({ collection: CONTAINER_MEMORY_COLLECTION });
1227
- } catch (err) {
1228
- this.logger.warn("container memory count failed — the row cap is not enforced this pass", { meta: { error: errMsg(err) } });
1229
- return null;
1230
- }
1231
- }
1232
- /**
1233
- * Delete this node's oldest rows, at most one page's worth.
1234
- *
1235
- * `cutoff === null` means "the oldest rows, whatever their age" — the
1236
- * row-cap path, which is bounded by `limitRows` instead. Either way the page
1237
- * is READ FIRST, ordered by the one index, and its last `atMs` becomes the
1238
- * effective cutoff for a single bounded delete. An unbounded `deleteWhere`
1239
- * would be one statement and one stalling transaction the first time a
1240
- * retention is lowered.
1241
- */
1242
- async deleteOldestPage(nodeId, cutoff, limitRows) {
1243
- if (limitRows <= 0) return {
1244
- deleted: 0,
1245
- examined: 0
1246
- };
1247
- try {
1248
- const page = await this.store.query.query({
1249
- collection: CONTAINER_MEMORY_COLLECTION,
1250
- filter: {
1251
- where: { nodeId },
1252
- ...cutoff === null ? {} : { whereBetween: { atMs: [0, cutoff] } },
1253
- orderBy: {
1254
- field: "atMs",
1255
- direction: "asc"
1256
- },
1257
- limit: Math.min(limitRows, this.prunePageRows)
1258
- },
1259
- columns: ["atMs"]
1260
- });
1261
- if (page.length === 0) return {
1262
- deleted: 0,
1263
- examined: 0
1264
- };
1265
- const effectiveCutoff = Number(page.at(-1)?.data["atMs"]);
1266
- if (!Number.isFinite(effectiveCutoff)) return {
1267
- deleted: 0,
1268
- examined: page.length
1269
- };
1270
- const { deleted } = await this.store.deleteWhere.mutate({
1271
- collection: CONTAINER_MEMORY_COLLECTION,
1272
- filter: {
1273
- where: { nodeId },
1274
- whereBetween: { atMs: [0, effectiveCutoff] }
1275
- }
1276
- });
1277
- return {
1278
- deleted,
1279
- examined: page.length
1280
- };
1281
- } catch (err) {
1282
- this.logger.warn("container memory prune failed — the table keeps growing this pass", { meta: {
1283
- nodeId,
1284
- cutoff,
1285
- error: errMsg(err)
1286
- } });
1287
- return {
1288
- deleted: 0,
1289
- examined: 0
1290
- };
1291
- }
1292
- }
1293
- };
1294
- //#endregion
1295
- //#region src/builtins/native-metrics/load-series-store.ts
1296
- /**
1297
- * @durable class=audit owner=metrics-native
1298
- * write="one row per process per sample, appended in ONE transaction per
1299
- * sample (`settingsStore.insertMany`) by the HUB's `native-metrics`, off its
1300
- * subscription to `metrics.node-processes-snapshot`. Emission is
1301
- * fixed-cadence, so the write rate is deterministic: nodes x processes /
1302
- * cadence, 7.6 rows/s on the live fleet at the 10 s default. Only a sample
1303
- * the in-memory ring ACCEPTED is written, which is what makes the append
1304
- * idempotent without a unique key."
1305
- * retention="TWO bounds, swept together at most once a minute. By AGE, the
1306
- * operator's `loadSeriesRetentionHours` (default 6, the intention); by ROWS,
1307
- * DEFAULT_MAX_ROWS (the guarantee, which bites regardless of the age knob).
1308
- * Both prune oldest-first through a bounded page — never a full scan — and
1309
- * the row cap logs when it bites. Losing the table loses the cold window
1310
- * only: the ring keeps serving the last hour and the charts keep drawing."
1311
- */
1312
- var LOAD_SERIES_COLLECTION = "metrics:node-load-samples";
1313
- /**
1314
- * Hard ceiling on the table, in ROWS.
1315
- *
1316
- * The guarantee behind the retention knob. Sized against the shipped default
1317
- * and against the worst configuration the knob can express:
1318
- *
1319
- * | cadence | retention | rows (3 nodes, 76 processes) |
1320
- * | --- | --- | --- |
1321
- * | 10 s | 6 h (default) | 164 160 |
1322
- * | 5 s | 6 h | 328 320 |
1323
- * | 5 s | 72 h | 3 939 840 — capped here |
1324
- *
1325
- * 500 000 rows x the measured 87 B = **41.5 MiB**, roughly 3x what the shipped
1326
- * default needs and far below the 2 M rows a three-day retention would reach.
1327
- * It is deliberately NOT a knob: the point of a guarantee is that it is not
1328
- * subject to the same hand that raised the intention. This is the discipline
1329
- * the in-memory ring already adopted (`DEFAULT_MAX_TOTAL_PROCESS_ROWS`) and
1330
- * the one the RTP pre-roll rings still lack — a maximum per instance and no
1331
- * budget at all for the whole, which reached 898 MB across 47 rings
1332
- * (`docs/architecture/load-ledger.md` #5).
1333
- */
1334
864
  var DEFAULT_MAX_ROWS = 5e5;
1335
- var LOAD_SERIES_COLUMNS = [
1336
- (
1337
- /** `INTEGER PRIMARY KEY` = SQLite rowid alias: the key IS the row's address,
1338
- * so it costs no separate index and no stored string. */
1339
- {
1340
- name: "id",
1341
- type: "INTEGER",
1342
- primaryKey: true,
1343
- notNull: true
1344
- }),
1345
- {
1346
- name: "nodeId",
1347
- type: "TEXT",
1348
- notNull: true
1349
- },
1350
- (
1351
- /** The EMITTING node's timestamp for the sample this row belongs to. Every
1352
- * row of one sample shares it — that is what makes a sample reassemblable. */
1353
- {
1354
- name: "atMs",
1355
- type: "INTEGER",
1356
- notNull: true
1357
- }),
1358
- {
1359
- name: "pid",
1360
- type: "INTEGER",
1361
- notNull: true
1362
- },
1363
- (
1364
- /** `NULL` for a process no addon owns — `root` and `system` both. */
1365
- {
1366
- name: "addonId",
1367
- type: "TEXT"
1368
- }),
1369
- {
1370
- name: "classification",
1371
- type: "TEXT",
1372
- notNull: true
1373
- },
1374
- (
1375
- /** `ps pcpu` x 10. A LIFETIME average, not a rate — see `NodeProcess`. */
1376
- {
1377
- name: "cpuDeci",
1378
- type: "INTEGER",
1379
- notNull: true
1380
- }),
1381
- {
1382
- name: "rssMib",
1383
- type: "INTEGER",
1384
- notNull: true
1385
- },
1386
- (
1387
- /** Instantaneous main-thread CPU% x 10. `NULL` = UNKNOWN, never zero. */
1388
- {
1389
- name: "cpuMainDeci",
1390
- type: "INTEGER"
1391
- }),
1392
- (
1393
- /** Instantaneous V8-helper-pool CPU% x 10. `NULL` = UNKNOWN, never zero. */
1394
- {
1395
- name: "cpuGcDeci",
1396
- type: "INTEGER"
1397
- })
1398
- ];
1399
- var LOAD_SERIES_INDEXES = [(
1400
- /**
1401
- * The ONE index, and it serves both jobs.
1402
- *
1403
- * Reads are always "this node, newer than T" — `nodeId` leads so the
1404
- * equality is a range scan and `atMs` supplies the order without a sort.
1405
- * Prunes are always "this node, oldest first" — the same index, walked from
1406
- * the other end. A second index on `atMs` alone was measured and rejected:
1407
- * it cost 23 B/row (196.0 → 172.8 with the two column drops) to serve a
1408
- * cross-node prune that a per-node loop over three nodes already serves.
1409
- */
1410
- {
1411
- name: "idx_load_samples_node_at",
1412
- columns: ["nodeId", "atMs"]
1413
- })];
1414
- var BYTES_PER_MIB = 1048576;
1415
- /** Tenths → percent. `null` stays `null`: unknown is not zero. */
1416
- function fromDeci(value) {
1417
- return value === null ? null : Math.round(value) / 10;
1418
- }
1419
- /** Percent → tenths, lossless because both producers already round to tenths. */
1420
- function toDeci(value) {
1421
- return value === null ? null : Math.round(value * 10);
1422
- }
1423
- /** Storage row → the shape a reader gets. The ONE decode. */
1424
- function rowToProcess(row) {
1425
- return {
1426
- pid: row.pid,
1427
- addonId: row.addonId,
1428
- classification: row.classification,
1429
- cpuPercent: fromDeci(row.cpuDeci) ?? 0,
1430
- memoryRssBytes: row.rssMib * BYTES_PER_MIB,
1431
- cpuMainPercent: fromDeci(row.cpuMainDeci),
1432
- cpuGcPercent: fromDeci(row.cpuGcDeci)
1433
- };
1434
- }
1435
- /** The shape a reader gets → the storage row. The ONE encode. */
1436
- function processToRow(nodeId, atMs, process) {
1437
- return {
1438
- nodeId,
1439
- atMs,
1440
- pid: process.pid,
1441
- addonId: process.addonId,
1442
- classification: process.classification,
1443
- cpuDeci: toDeci(process.cpuPercent) ?? 0,
1444
- rssMib: Math.round(process.memoryRssBytes / BYTES_PER_MIB),
1445
- cpuMainDeci: toDeci(process.cpuMainPercent),
1446
- cpuGcDeci: toDeci(process.cpuGcPercent)
1447
- };
1448
- }
1449
- /** Group flat rows into samples, oldest-first. A sample is one `atMs`. */
1450
- function rowsToSamples(rows) {
1451
- const byAt = /* @__PURE__ */ new Map();
1452
- for (const row of rows) {
1453
- const held = byAt.get(row.atMs);
1454
- if (held === void 0) byAt.set(row.atMs, [rowToProcess(row)]);
1455
- else held.push(rowToProcess(row));
1456
- }
1457
- return [...byAt.entries()].toSorted((a, b) => a[0] - b[0]).map(([atMs, processes]) => ({
1458
- atMs,
1459
- processes
1460
- }));
1461
- }
1462
- /**
1463
- * The cold tier. Append-only, bounded, and never the authority on anything the
1464
- * hot ring can answer.
1465
- *
1466
- * Every method is best-effort and says so in the log rather than throwing at
1467
- * the emitter: a storage hiccup must cost a gap in a chart, never a dropped
1468
- * snapshot on the bus.
1469
- */
1470
- var LoadSeriesStore = class {
1471
- declared = false;
1472
- lastPruneAtMs = 0;
1473
- store;
1474
- logger;
1475
- nowFn;
1476
- prunePageRows;
1477
- pruneIntervalMs;
1478
- constructor(deps) {
1479
- this.store = deps.store;
1480
- this.logger = deps.logger;
1481
- this.nowFn = deps.now ?? (() => Date.now());
1482
- this.prunePageRows = deps.prunePageRows ?? 2e4;
1483
- this.pruneIntervalMs = deps.pruneIntervalMs ?? 6e4;
1484
- }
1485
- /** Idempotently declare the collection. `false` when the store refused. */
1486
- async declare() {
1487
- if (this.declared) return true;
1488
- try {
1489
- await this.store.declareCollection.mutate({
1490
- collection: LOAD_SERIES_COLLECTION,
1491
- columns: [...LOAD_SERIES_COLUMNS],
1492
- indexes: LOAD_SERIES_INDEXES.map((i) => ({
1493
- name: i.name,
1494
- columns: [...i.columns]
1495
- }))
1496
- });
1497
- this.declared = true;
1498
- return true;
1499
- } catch (err) {
1500
- this.logger.warn("load series declareCollection failed — nothing will be retained on disk", { meta: {
1501
- collection: LOAD_SERIES_COLLECTION,
1502
- error: errMsg(err)
1503
- } });
1504
- return false;
1505
- }
1506
- }
1507
- /**
1508
- * Append ONE sample — every process row of it — in ONE transaction.
1509
- *
1510
- * Never a write per row. `insertMany` exists for exactly this: at the 10 s
1511
- * default the fleet produces 7.6 rows/s, and 7.6 separate commits per second
1512
- * on the connection that also serves every cluster-wide configuration read
1513
- * is a constant load nobody asked for.
1514
- */
1515
- async append(nodeId, atMs, processes) {
1516
- if (processes.length === 0) return 0;
1517
- if (!await this.declare()) return 0;
1518
- const records = processes.map((p) => ({ data: { ...processToRow(nodeId, atMs, p) } }));
1519
- try {
1520
- const { inserted } = await this.store.insertMany.mutate({
1521
- collection: LOAD_SERIES_COLLECTION,
1522
- records
1523
- });
1524
- return inserted;
1525
- } catch (err) {
1526
- this.logger.warn("load series sample not retained — this interval will be missing", { meta: {
1527
- nodeId,
1528
- atMs,
1529
- rows: processes.length,
1530
- error: errMsg(err)
1531
- } });
1532
- return 0;
1533
- }
1534
- }
1535
- /**
1536
- * Read one node's cold samples, oldest-first.
1537
- *
1538
- * `sinceMs` is EXCLUSIVE, matching the ring, so a caller passing the newest
1539
- * timestamp it holds gets only what it is missing. `limitRows` bounds the
1540
- * read in ROWS (not samples) because rows are what the query costs.
1541
- */
1542
- async read(nodeId, sinceMs, limitRows) {
1543
- if (!await this.declare()) return [];
1544
- try {
1545
- const records = await this.store.query.query({
1546
- collection: LOAD_SERIES_COLLECTION,
1547
- filter: {
1548
- where: { nodeId },
1549
- whereBetween: { atMs: [sinceMs + 1, Number.MAX_SAFE_INTEGER] },
1550
- orderBy: {
1551
- field: "atMs",
1552
- direction: "asc"
1553
- },
1554
- limit: limitRows
1555
- }
1556
- });
1557
- const rows = [];
1558
- for (const record of records) {
1559
- const row = recordToRow(record.data);
1560
- if (row !== null) rows.push(row);
1561
- }
1562
- return rowsToSamples(rows);
1563
- } catch (err) {
1564
- this.logger.warn("load series cold read failed — answering from the hot window only", { meta: {
1565
- nodeId,
1566
- sinceMs,
1567
- error: errMsg(err)
1568
- } });
1569
- return [];
1570
- }
1571
- }
1572
- /**
1573
- * Enforce BOTH bounds, oldest-first, through a bounded page each.
1574
- *
1575
- * Rate-limited to {@link pruneIntervalMs}: a bound is not a deadline, and the
1576
- * append path must not pay a sweep on every sample.
1577
- */
1578
- async prune(nodeIds, retention, force = false) {
1579
- const now = this.nowFn();
1580
- if (!force && now - this.lastPruneAtMs < this.pruneIntervalMs) return null;
1581
- this.lastPruneAtMs = now;
1582
- if (!await this.declare()) return null;
1583
- let deletedByAge = 0;
1584
- let deletedByCap = 0;
1585
- let rowsExamined = 0;
1586
- const ageCutoff = now - retention.retentionHours * 36e5;
1587
- for (const nodeId of nodeIds) {
1588
- const outcome = await this.pruneNodeToCutoff(nodeId, ageCutoff);
1589
- deletedByAge += outcome.deleted;
1590
- rowsExamined += outcome.examined;
1591
- }
1592
- const total = await this.count();
1593
- const excess = total === null ? 0 : total - retention.maxRows;
1594
- if (excess > 0) {
1595
- const outcome = await this.pruneOldestRows(nodeIds, excess);
1596
- deletedByCap = outcome.deleted;
1597
- rowsExamined += outcome.examined;
1598
- this.logger.warn("load series ROW CAP bit — evicting the oldest samples", { meta: {
1599
- collection: LOAD_SERIES_COLLECTION,
1600
- rows: total,
1601
- cap: retention.maxRows,
1602
- over: excess,
1603
- deleted: outcome.deleted,
1604
- retentionHours: retention.retentionHours,
1605
- hint: "lower the retention or the sampling cadence — the cap is the guarantee, not the intention"
1606
- } });
1607
- }
1608
- return {
1609
- deletedByAge,
1610
- deletedByCap,
1611
- rowsExamined,
1612
- capBit: excess > 0
1613
- };
1614
- }
1615
- /** Total rows, or `null` when the store could not answer. */
1616
- async count() {
1617
- if (!await this.declare()) return null;
1618
- try {
1619
- return await this.store.count.query({ collection: LOAD_SERIES_COLLECTION });
1620
- } catch (err) {
1621
- this.logger.warn("load series count failed — the row cap is not enforced this pass", { meta: { error: errMsg(err) } });
1622
- return null;
1623
- }
1624
- }
1625
- /**
1626
- * Delete this node's rows older than `cutoff`, at most one page's worth.
1627
- *
1628
- * The page is the whole point. `deleteWhere({ atMs: [0, cutoff] })` on its
1629
- * own is one statement but an UNBOUNDED one — a first pass after a retention
1630
- * change would delete millions of rows inside a single stalling transaction.
1631
- * So the page is read first (keys only, ordered by the index), its last
1632
- * `atMs` becomes the EFFECTIVE cutoff, and the delete is bounded by it.
1633
- */
1634
- async pruneNodeToCutoff(nodeId, cutoff) {
1635
- try {
1636
- const page = await this.store.query.query({
1637
- collection: LOAD_SERIES_COLLECTION,
1638
- filter: {
1639
- where: { nodeId },
1640
- whereBetween: { atMs: [0, cutoff] },
1641
- orderBy: {
1642
- field: "atMs",
1643
- direction: "asc"
1644
- },
1645
- limit: this.prunePageRows
1646
- },
1647
- columns: ["atMs"]
1648
- });
1649
- if (page.length === 0) return {
1650
- deleted: 0,
1651
- examined: 0
1652
- };
1653
- const effectiveCutoff = Number(page.at(-1)?.data["atMs"]);
1654
- if (!Number.isFinite(effectiveCutoff)) return {
1655
- deleted: 0,
1656
- examined: page.length
1657
- };
1658
- const { deleted } = await this.store.deleteWhere.mutate({
1659
- collection: LOAD_SERIES_COLLECTION,
1660
- filter: {
1661
- where: { nodeId },
1662
- whereBetween: { atMs: [0, effectiveCutoff] }
1663
- }
1664
- });
1665
- return {
1666
- deleted,
1667
- examined: page.length
1668
- };
1669
- } catch (err) {
1670
- this.logger.warn("load series age prune failed — the table keeps growing this pass", { meta: {
1671
- nodeId,
1672
- cutoff,
1673
- error: errMsg(err)
1674
- } });
1675
- return {
1676
- deleted: 0,
1677
- examined: 0
1678
- };
1679
- }
1680
- }
1681
- /**
1682
- * Drop the oldest rows across the known nodes until `excess` is covered.
1683
- *
1684
- * Same bounded-page technique, walked per node so the one index serves it.
1685
- * The node holding the oldest rows pays first, which is also the node
1686
- * producing the pressure when a runaway process count is the cause.
1687
- */
1688
- async pruneOldestRows(nodeIds, excess) {
1689
- let remaining = Math.min(excess, this.prunePageRows);
1690
- let deleted = 0;
1691
- let examined = 0;
1692
- for (const nodeId of nodeIds) {
1693
- if (remaining <= 0) break;
1694
- try {
1695
- const page = await this.store.query.query({
1696
- collection: LOAD_SERIES_COLLECTION,
1697
- filter: {
1698
- where: { nodeId },
1699
- orderBy: {
1700
- field: "atMs",
1701
- direction: "asc"
1702
- },
1703
- limit: remaining
1704
- },
1705
- columns: ["atMs"]
1706
- });
1707
- examined += page.length;
1708
- if (page.length === 0) continue;
1709
- const cutoff = Number(page.at(-1)?.data["atMs"]);
1710
- if (!Number.isFinite(cutoff)) continue;
1711
- const result = await this.store.deleteWhere.mutate({
1712
- collection: LOAD_SERIES_COLLECTION,
1713
- filter: {
1714
- where: { nodeId },
1715
- whereBetween: { atMs: [0, cutoff] }
1716
- }
1717
- });
1718
- deleted += result.deleted;
1719
- remaining -= result.deleted;
1720
- } catch (err) {
1721
- this.logger.warn("load series cap prune failed — the cap is not enforced this pass", { meta: {
1722
- nodeId,
1723
- error: errMsg(err)
1724
- } });
1725
- }
1726
- }
1727
- return {
1728
- deleted,
1729
- examined
1730
- };
1731
- }
1732
- };
1733
- /** Structural validation on read. A malformed row is skipped, never charted. */
1734
- function recordToRow(data) {
1735
- const nodeId = data["nodeId"];
1736
- const classification = data["classification"];
1737
- if (typeof nodeId !== "string" || typeof classification !== "string") return null;
1738
- const atMs = Number(data["atMs"]);
1739
- const pid = Number(data["pid"]);
1740
- const cpuDeci = Number(data["cpuDeci"]);
1741
- const rssMib = Number(data["rssMib"]);
1742
- if (![
1743
- atMs,
1744
- pid,
1745
- cpuDeci,
1746
- rssMib
1747
- ].every((n) => Number.isFinite(n))) return null;
1748
- const rawAddon = data["addonId"];
1749
- const optional = (raw) => {
1750
- if (raw === null || raw === void 0) return null;
1751
- const n = Number(raw);
1752
- return Number.isFinite(n) ? n : null;
1753
- };
1754
- return {
1755
- nodeId,
1756
- atMs,
1757
- pid,
1758
- addonId: typeof rawAddon === "string" ? rawAddon : null,
1759
- classification,
1760
- cpuDeci,
1761
- rssMib,
1762
- cpuMainDeci: optional(data["cpuMainDeci"]),
1763
- cpuGcDeci: optional(data["cpuGcDeci"])
1764
- };
1765
- }
1766
- //#endregion
1767
- //#region src/builtins/native-metrics/load-series-config.ts
1768
- /**
1769
- * The two knobs of the durable load series, and the arithmetic that makes them
1770
- * honest.
1771
- *
1772
- * ## Where they live, and why there
1773
- *
1774
- * In `native-metrics`'s OWN global settings document — the one that already
1775
- * exists and already carries `samplingIntervalMs`. Fields on a document, never
1776
- * a method per knob: `system.getLoggingSettings` / `setLoggingSettings` set
1777
- * that precedent and it holds here.
1778
- *
1779
- * They are NOT in the logging document, and that is a decision rather than an
1780
- * omission. That document is about levels and diagnostic windows; a metrics
1781
- * cadence and a table's retention are neither. Two documents that both claim a
1782
- * knob is how this repo has already shipped a switch nobody read.
1783
- *
1784
- * They are cluster-wide, not per-node. Every node emits on the same cadence or
1785
- * the fleet's series cannot be laid over each other, and the table is single
1786
- * and hub-resident, so a per-node retention would be a promise nothing could
1787
- * keep.
1788
- *
1789
- * ## Refused, not clamped
1790
- *
1791
- * A cadence outside 5-60 s is REFUSED. Clamping would silently store 10 when
1792
- * the operator typed 2 and then report 10 back as though it had been accepted
1793
- * — the failure mode where a knob and its readback agree on a value nobody
1794
- * chose. The schema advertises the bounds; this refuses anything else.
1795
- */
1796
865
  /** Why a value was refused — the message an operator is shown verbatim. */
1797
866
  var LoadSeriesConfigError = class extends Error {
1798
867
  constructor(message) {
@@ -1835,7 +904,7 @@ function resolveLoadSeriesConfig(raw) {
1835
904
  * This is what makes the knob honest: an operator raising retention to three
1836
905
  * days sees the row count and the megabytes BEFORE applying it, rather than
1837
906
  * discovering them afterwards. `MEASURED_BYTES_PER_ROW` is measured, not
1838
- * guessed — see `load-series-store.ts`.
907
+ * guessed — see `node-metrics-history-recorder.ts`.
1839
908
  */
1840
909
  function projectLoadSeriesCost(input) {
1841
910
  const samplesPerNodeWindow = input.config.retentionHours * 3600 / input.config.cadenceSec;
@@ -1861,8 +930,8 @@ function describeLoadSeriesCost(projection) {
1861
930
  /**
1862
931
  * The ONE reader over both retention tiers.
1863
932
  *
1864
- * The hot window is the in-memory ring; the cold one is
1865
- * `metrics:node-load-samples`. A caller never learns which tier a point came
933
+ * The hot window is the in-memory ring; the cold one is device-history raw
934
+ * points. A caller never learns which tier a point came
1866
935
  * from, and that is a hard requirement rather than a nicety: a second reader
1867
936
  * is how two charts start disagreeing about the same node, and it is how "the
1868
937
  * ring says X, the table says Y" becomes a question nobody can answer.
@@ -1950,6 +1019,203 @@ function mergeLoadSeries(input) {
1950
1019
  };
1951
1020
  }
1952
1021
  //#endregion
1022
+ //#region src/builtins/native-metrics/node-metrics-history-recorder.ts
1023
+ var ADDON_ID = "metrics-native";
1024
+ var NODE_PROCESS_CPU_KEY = `addon:${ADDON_ID}/process-cpu-percent`;
1025
+ var NODE_PROCESS_RSS_KEY = `addon:${ADDON_ID}/process-rss-mib`;
1026
+ var NODE_CONTAINER_CURRENT_KEY = `addon:${ADDON_ID}/container-current-mib`;
1027
+ function nodeMetricsHistoryClient(api) {
1028
+ return {
1029
+ declareSeries: (input) => api.declareSeries.mutate(input),
1030
+ record: (input) => api.record.mutate({ points: [...input.points] }),
1031
+ queryNode: async (input) => {
1032
+ return { series: (await api.queryNode.query(input)).series.map((s) => ({
1033
+ scope: s.scope,
1034
+ points: s.points.map((p) => ({
1035
+ t: p.t,
1036
+ samples: p.samples,
1037
+ value: p.last ?? p.mean
1038
+ }))
1039
+ })) };
1040
+ }
1041
+ };
1042
+ }
1043
+ var NodeMetricsHistoryRecorder = class {
1044
+ deps;
1045
+ declared = false;
1046
+ constructor(deps) {
1047
+ this.deps = deps;
1048
+ }
1049
+ async declare() {
1050
+ if (this.declared) return;
1051
+ const { deviceHistory } = this.deps;
1052
+ await deviceHistory.declareSeries({
1053
+ name: "process-cpu-percent",
1054
+ subjectKind: "node",
1055
+ kind: "gauge",
1056
+ displayName: "Process CPU %",
1057
+ ingest: "points",
1058
+ unit: "%"
1059
+ });
1060
+ await deviceHistory.declareSeries({
1061
+ name: "process-rss-mib",
1062
+ subjectKind: "node",
1063
+ kind: "gauge",
1064
+ displayName: "Process RSS (MiB)",
1065
+ ingest: "points",
1066
+ unit: "MiB"
1067
+ });
1068
+ await deviceHistory.declareSeries({
1069
+ name: "container-current-mib",
1070
+ subjectKind: "node",
1071
+ kind: "gauge",
1072
+ displayName: "Container memory current (MiB)",
1073
+ ingest: "points",
1074
+ unit: "MiB"
1075
+ });
1076
+ this.declared = true;
1077
+ }
1078
+ async recordSample(nodeId, atMs, processes, containerMemory) {
1079
+ if (!await this.ensureDeclared()) return;
1080
+ const points = [];
1081
+ for (const p of processes) {
1082
+ const scope = String(p.pid);
1083
+ points.push({
1084
+ key: NODE_PROCESS_CPU_KEY,
1085
+ subject: {
1086
+ kind: "node",
1087
+ nodeId
1088
+ },
1089
+ scope,
1090
+ ts: atMs,
1091
+ value: p.cpuPercent
1092
+ });
1093
+ points.push({
1094
+ key: NODE_PROCESS_RSS_KEY,
1095
+ subject: {
1096
+ kind: "node",
1097
+ nodeId
1098
+ },
1099
+ scope,
1100
+ ts: atMs,
1101
+ value: Math.round(p.memoryRssBytes / 1048576)
1102
+ });
1103
+ }
1104
+ if (containerMemory !== null) points.push({
1105
+ key: NODE_CONTAINER_CURRENT_KEY,
1106
+ subject: {
1107
+ kind: "node",
1108
+ nodeId
1109
+ },
1110
+ scope: "",
1111
+ ts: atMs,
1112
+ value: Math.round(containerMemory.currentBytes / 1048576)
1113
+ });
1114
+ if (points.length === 0) return;
1115
+ try {
1116
+ await this.deps.deviceHistory.record({ points });
1117
+ } catch (err) {
1118
+ this.deps.logger.warn("node metrics history sample not stored", { meta: {
1119
+ nodeId,
1120
+ atMs,
1121
+ error: err instanceof Error ? err.message : String(err)
1122
+ } });
1123
+ }
1124
+ }
1125
+ async readContainerPoints(nodeId, sinceMs, maxPoints) {
1126
+ if (!await this.ensureDeclared()) return [];
1127
+ const result = await this.deps.deviceHistory.queryNode({
1128
+ nodeId,
1129
+ key: NODE_CONTAINER_CURRENT_KEY,
1130
+ from: sinceMs + 1,
1131
+ to: Date.now(),
1132
+ resolution: "raw",
1133
+ maxPoints
1134
+ });
1135
+ const out = [];
1136
+ for (const s of result.series) for (const p of s.points) {
1137
+ if (p.value === null || !Number.isFinite(p.value)) continue;
1138
+ out.push({
1139
+ atMs: p.t,
1140
+ source: "cgroup-v2",
1141
+ currentBytes: p.value * 1048576,
1142
+ limitBytes: null,
1143
+ anonBytes: null,
1144
+ fileBytes: null,
1145
+ shmemBytes: null,
1146
+ slabBytes: null,
1147
+ gpuShmemBytes: null
1148
+ });
1149
+ }
1150
+ return out.toSorted((a, b) => a.atMs - b.atMs);
1151
+ }
1152
+ async readProcessSamples(nodeId, sinceMs, maxPoints) {
1153
+ if (!await this.ensureDeclared()) return [];
1154
+ const to = Date.now();
1155
+ const [cpuResult, rssResult] = await Promise.all([this.deps.deviceHistory.queryNode({
1156
+ nodeId,
1157
+ key: NODE_PROCESS_CPU_KEY,
1158
+ from: sinceMs + 1,
1159
+ to,
1160
+ resolution: "raw",
1161
+ maxPoints
1162
+ }), this.deps.deviceHistory.queryNode({
1163
+ nodeId,
1164
+ key: NODE_PROCESS_RSS_KEY,
1165
+ from: sinceMs + 1,
1166
+ to,
1167
+ resolution: "raw",
1168
+ maxPoints
1169
+ })]);
1170
+ const byAt = /* @__PURE__ */ new Map();
1171
+ const ingest = (series, field) => {
1172
+ for (const s of series) {
1173
+ const pid = Number(s.scope);
1174
+ if (!Number.isFinite(pid)) continue;
1175
+ for (const p of s.points) {
1176
+ if (p.value === null || !Number.isFinite(p.value)) continue;
1177
+ let perPid = byAt.get(p.t);
1178
+ if (perPid === void 0) {
1179
+ perPid = /* @__PURE__ */ new Map();
1180
+ byAt.set(p.t, perPid);
1181
+ }
1182
+ const base = perPid.get(pid) ?? {
1183
+ pid,
1184
+ addonId: null,
1185
+ classification: "managed",
1186
+ cpuPercent: 0,
1187
+ memoryRssBytes: 0,
1188
+ cpuMainPercent: null,
1189
+ cpuGcPercent: null
1190
+ };
1191
+ perPid.set(pid, field === "cpuPercent" ? {
1192
+ ...base,
1193
+ cpuPercent: p.value
1194
+ } : {
1195
+ ...base,
1196
+ memoryRssBytes: p.value * 1048576
1197
+ });
1198
+ }
1199
+ }
1200
+ };
1201
+ ingest(cpuResult.series, "cpuPercent");
1202
+ ingest(rssResult.series, "memoryRssBytes");
1203
+ return [...byAt.entries()].toSorted((a, b) => a[0] - b[0]).map(([atMs, perPid]) => ({
1204
+ atMs,
1205
+ processes: [...perPid.values()]
1206
+ }));
1207
+ }
1208
+ async ensureDeclared() {
1209
+ try {
1210
+ await this.declare();
1211
+ return true;
1212
+ } catch (err) {
1213
+ this.deps.logger.warn("node metrics history declare failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
1214
+ return false;
1215
+ }
1216
+ }
1217
+ };
1218
+ //#endregion
1953
1219
  //#region src/builtins/native-metrics/gpu-probe.ts
1954
1220
  /**
1955
1221
  * The Linux GPU probe, forking only when there is something to fork.
@@ -2998,7 +2264,7 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
2998
2264
  * the `settings-store` singleton would ship its rows over Moleculer, once
2999
2265
  * per sample, to reach that very same table.
3000
2266
  */
3001
- loadStore = null;
2267
+ metricsHistory = null;
3002
2268
  /**
3003
2269
  * The container's OWN memory, sampled on the process-snapshot tick.
3004
2270
  *
@@ -3010,10 +2276,8 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
3010
2276
  */
3011
2277
  memoryProbe = null;
3012
2278
  pidsProbe = null;
3013
- /** Hot tier for the container reading. Bounds in `container-memory-series.ts`. */
2279
+ /** Hot tier for the container reading. Bounds in `container-memory-ring.ts`. */
3014
2280
  memoryRing = new ContainerMemoryRing();
3015
- /** Cold tier, HUB only — same reasoning as `loadStore`. */
3016
- memoryStore = null;
3017
2281
  /** The resolved knobs. Re-resolved on every settings write. */
3018
2282
  loadConfig = {
3019
2283
  cadenceSec: 10,
@@ -3086,13 +2350,12 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
3086
2350
  };
3087
2351
  this.applyLoadSeriesConfig();
3088
2352
  if (this.isHub) {
3089
- this.loadStore = new LoadSeriesStore({
3090
- store: this.ctx.api.settingsStore,
3091
- logger: this.ctx.logger.child("LoadSeries")
2353
+ this.metricsHistory = new NodeMetricsHistoryRecorder({
2354
+ deviceHistory: nodeMetricsHistoryClient(this.ctx.api.deviceHistory),
2355
+ logger: this.ctx.logger.child("NodeMetricsHistory")
3092
2356
  });
3093
- this.memoryStore = new ContainerMemoryStore({
3094
- store: this.ctx.api.settingsStore,
3095
- logger: this.ctx.logger.child("ContainerMemory")
2357
+ this.metricsHistory.declare().catch((err) => {
2358
+ this.ctx.logger.warn("node metrics history declare failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
3096
2359
  });
3097
2360
  }
3098
2361
  this.memoryProbe = new ContainerMemoryProbe({ logger: this.ctx.logger.child("ContainerMemory") });
@@ -3113,7 +2376,7 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
3113
2376
  * this node has already delivered, and only an accepted sample reaches the
3114
2377
  * table. That is what makes the append idempotent without a unique index
3115
2378
  * over two million rows — the composite key it replaces was measured at ~40
3116
- * bytes per row (`load-series-store.ts`).
2379
+ * bytes per row (measured on the old SQLite tier; see D850).
3117
2380
  *
3118
2381
  * The durable append is fire-and-forget: a storage stall must cost a gap in
3119
2382
  * the cold window, never a blocked event-bus handler. Every failure logs.
@@ -3123,26 +2386,9 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
3123
2386
  if (!this.loadRing.record(nodeId, atMs, retained)) return;
3124
2387
  this.observedRowsByNode.set(nodeId, retained.length);
3125
2388
  if (containerMemory !== null) this.memoryRing.record(nodeId, atMs, containerMemory);
3126
- const store = this.loadStore;
3127
- if (store === null) return;
3128
- store.append(nodeId, atMs, retained).then(() => store.prune([...this.observedRowsByNode.keys()], this.loadConfig)).then(async (outcome) => {
3129
- if (outcome === null) return;
3130
- await this.memoryStore?.prune([...this.observedRowsByNode.keys()], this.loadConfig);
3131
- }).catch((err) => {
3132
- this.ctx.logger.warn("durable load series write failed", { meta: {
3133
- nodeId,
3134
- atMs,
3135
- error: err instanceof Error ? err.message : String(err)
3136
- } });
3137
- });
3138
- if (containerMemory === null) return;
3139
- this.memoryStore?.append(nodeId, atMs, containerMemory).catch((err) => {
3140
- this.ctx.logger.warn("durable container memory write failed", { meta: {
3141
- nodeId,
3142
- atMs,
3143
- error: err instanceof Error ? err.message : String(err)
3144
- } });
3145
- });
2389
+ const history = this.metricsHistory;
2390
+ if (history === null) return;
2391
+ history.recordSample(nodeId, atMs, retained, containerMemory);
3146
2392
  }
3147
2393
  /**
3148
2394
  * The ONE reader, over both tiers.
@@ -3161,9 +2407,9 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
3161
2407
  atMs: sample.atMs,
3162
2408
  processes: sample.processes
3163
2409
  }));
3164
- const store = this.loadStore;
2410
+ const history = this.metricsHistory;
3165
2411
  const merged = mergeLoadSeries({
3166
- cold: store === null ? [] : await store.read(params.forNodeId, sinceMs, this.loadConfig.maxRows),
2412
+ cold: history === null ? [] : await history.readProcessSamples(params.forNodeId, sinceMs, this.loadConfig.maxRows),
3167
2413
  hot: hotSamples,
3168
2414
  cadenceMs,
3169
2415
  ...params.maxPoints !== void 0 ? { maxPoints: params.maxPoints } : {}
@@ -3183,7 +2429,7 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
3183
2429
  retainedSamples: merged.retainedSamples,
3184
2430
  oldestAtMs: merged.oldestAtMs,
3185
2431
  cadenceMs,
3186
- durable: store !== null
2432
+ durable: history !== null
3187
2433
  };
3188
2434
  }
3189
2435
  /**
@@ -3199,8 +2445,8 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
3199
2445
  */
3200
2446
  async readContainerMemory(params) {
3201
2447
  const hot = this.memoryRing.read(params.forNodeId, params.exclusiveSinceMs);
3202
- const store = this.memoryStore;
3203
- const merged = mergeContainerMemory(store === null ? [] : await store.read(params.forNodeId, params.sinceMs, CONTAINER_MEMORY_MAX_ROWS), hot);
2448
+ const history = this.metricsHistory;
2449
+ const merged = mergeContainerMemory(history === null ? [] : await history.readContainerPoints(params.forNodeId, params.sinceMs, 5e3), hot);
3204
2450
  if (params.bucketMs <= params.cadenceMs) return merged;
3205
2451
  return reduceContainerMemory(merged, params.bucketMs, params.origin ?? 0);
3206
2452
  }