@camstack/system 1.2.446 → 1.2.447
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtins/native-metrics/container-memory-ring.d.ts +18 -0
- package/dist/builtins/native-metrics/load-series-config.d.ts +38 -1
- package/dist/builtins/native-metrics/load-series-reader.d.ts +3 -3
- package/dist/builtins/native-metrics/load-series-types.d.ts +14 -0
- package/dist/builtins/native-metrics/native-metrics.addon.d.ts +5 -7
- package/dist/builtins/native-metrics/native-metrics.addon.js +219 -973
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +219 -973
- package/dist/builtins/native-metrics/node-load-ring.d.ts +3 -5
- package/dist/builtins/native-metrics/node-metrics-history-recorder.d.ts +70 -0
- package/package.json +1 -1
- package/dist/builtins/native-metrics/container-memory-series.d.ts +0 -260
- package/dist/builtins/native-metrics/load-series-store.d.ts +0 -293
|
@@ -688,11 +688,9 @@ var NodeLoadRing = class {
|
|
|
688
688
|
* rather than a replay of a timestamp this node has already delivered.
|
|
689
689
|
*
|
|
690
690
|
* The return value is not diagnostics. It is the idempotence gate the
|
|
691
|
-
* DURABLE tier rides on (`
|
|
692
|
-
*
|
|
693
|
-
*
|
|
694
|
-
* gate is one monotonic comparison in memory; the index it replaces was
|
|
695
|
-
* measured at ~40 bytes per row.
|
|
691
|
+
* DURABLE tier rides on this return value (`node-metrics-history-recorder.ts`):
|
|
692
|
+
* only an accepted sample is sent to device-history. The gate is one monotonic
|
|
693
|
+
* comparison in memory.
|
|
696
694
|
*/
|
|
697
695
|
record(nodeId, atMs, processes) {
|
|
698
696
|
if (nodeId.length === 0) return false;
|
|
@@ -817,197 +815,7 @@ var NodeLoadRing = class {
|
|
|
817
815
|
this.lastWriteSeq.delete(nodeId);
|
|
818
816
|
}
|
|
819
817
|
};
|
|
820
|
-
//#endregion
|
|
821
|
-
//#region src/builtins/native-metrics/container-memory-series.ts
|
|
822
|
-
/**
|
|
823
|
-
* @durable class=audit owner=metrics-native
|
|
824
|
-
* write="one row per NODE per sample, appended by the HUB's `native-metrics`
|
|
825
|
-
* off its subscription to `metrics.node-processes-snapshot` — the same
|
|
826
|
-
* subscription, the same accepted-sample gate and the same timestamps as
|
|
827
|
-
* `metrics:node-load-samples`, so a memory row and the process rows of the
|
|
828
|
-
* same instant share an `atMs`. A node with no cgroup writes NOTHING; it
|
|
829
|
-
* never writes a zeroed row."
|
|
830
|
-
* retention="The operator's `loadSeriesRetentionHours` by age and
|
|
831
|
-
* CONTAINER_MEMORY_MAX_ROWS by count, swept in the same pass as the process
|
|
832
|
-
* table and through the same bounded page. Losing the table loses the cold
|
|
833
|
-
* window only: the ring keeps serving the last hour."
|
|
834
|
-
*/
|
|
835
|
-
var CONTAINER_MEMORY_COLLECTION = "metrics:node-memory-samples";
|
|
836
|
-
/**
|
|
837
|
-
* Fleet ceiling on the hot ring, in READINGS.
|
|
838
|
-
*
|
|
839
|
-
* One reading per node per sample, so this is simply the per-node ceiling times
|
|
840
|
-
* the partition ceiling the process ring already declares (16). 2 880 readings
|
|
841
|
-
* of ten numbers each is well under a megabyte — the bound exists so a runaway
|
|
842
|
-
* node id space costs a fixed amount, not because the number is large.
|
|
843
|
-
*/
|
|
844
818
|
var CONTAINER_MEMORY_MAX_TOTAL_ROWS = 2880;
|
|
845
|
-
/**
|
|
846
|
-
* Hard ceiling on the durable table, in ROWS. The guarantee behind the
|
|
847
|
-
* retention intention, exactly as `DEFAULT_MAX_ROWS` is for the process table.
|
|
848
|
-
*
|
|
849
|
-
* | cadence | retention | rows (3 nodes) |
|
|
850
|
-
* | --- | --- | --- |
|
|
851
|
-
* | 10 s | 6 h (default) | 6 480 |
|
|
852
|
-
* | 5 s | 6 h | 12 960 |
|
|
853
|
-
* | 5 s | 72 h, 16 nodes | 829 440 — capped here |
|
|
854
|
-
*
|
|
855
|
-
* 50 000 rows is 7.7x the shipped default. This row is strictly narrower than
|
|
856
|
-
* a `metrics:node-load-samples` row — fewer columns, one index, every value an
|
|
857
|
-
* integer varint — and that one was MEASURED at 87 B/row, so 50 000 rows is
|
|
858
|
-
* **under 4.2 MiB** with the real figure lower still. It is not a knob: a
|
|
859
|
-
* guarantee that moves with the hand that raised the intention is not one.
|
|
860
|
-
*/
|
|
861
|
-
var CONTAINER_MEMORY_MAX_ROWS = 5e4;
|
|
862
|
-
var CONTAINER_MEMORY_COLUMNS = [
|
|
863
|
-
{
|
|
864
|
-
name: "id",
|
|
865
|
-
type: "INTEGER",
|
|
866
|
-
primaryKey: true,
|
|
867
|
-
notNull: true
|
|
868
|
-
},
|
|
869
|
-
{
|
|
870
|
-
name: "nodeId",
|
|
871
|
-
type: "TEXT",
|
|
872
|
-
notNull: true
|
|
873
|
-
},
|
|
874
|
-
(
|
|
875
|
-
/** Shared with the process rows of the same tick — that is the whole point. */
|
|
876
|
-
{
|
|
877
|
-
name: "atMs",
|
|
878
|
-
type: "INTEGER",
|
|
879
|
-
notNull: true
|
|
880
|
-
}),
|
|
881
|
-
{
|
|
882
|
-
name: "source",
|
|
883
|
-
type: "TEXT",
|
|
884
|
-
notNull: true
|
|
885
|
-
},
|
|
886
|
-
(
|
|
887
|
-
/**
|
|
888
|
-
* Whole MiB, like `rssMib` on the process row and for the same reason: a
|
|
889
|
-
* byte-exact residency on a value in the tens of gigabytes is precision no
|
|
890
|
-
* chart can draw, and it costs bytes on every row forever.
|
|
891
|
-
*/
|
|
892
|
-
{
|
|
893
|
-
name: "currentMib",
|
|
894
|
-
type: "INTEGER",
|
|
895
|
-
notNull: true
|
|
896
|
-
}),
|
|
897
|
-
(
|
|
898
|
-
/** `NULL` = NO LIMIT. Never a sentinel. */
|
|
899
|
-
{
|
|
900
|
-
name: "limitMib",
|
|
901
|
-
type: "INTEGER"
|
|
902
|
-
}),
|
|
903
|
-
(
|
|
904
|
-
/** `NULL` = UNKNOWN on all four. Never zero. */
|
|
905
|
-
{
|
|
906
|
-
name: "anonMib",
|
|
907
|
-
type: "INTEGER"
|
|
908
|
-
}),
|
|
909
|
-
{
|
|
910
|
-
name: "fileMib",
|
|
911
|
-
type: "INTEGER"
|
|
912
|
-
},
|
|
913
|
-
{
|
|
914
|
-
name: "shmemMib",
|
|
915
|
-
type: "INTEGER"
|
|
916
|
-
},
|
|
917
|
-
{
|
|
918
|
-
name: "slabMib",
|
|
919
|
-
type: "INTEGER"
|
|
920
|
-
},
|
|
921
|
-
(
|
|
922
|
-
/** Host-wide i915 GEM bytes. `NULL` wherever debugfs is not mounted. */
|
|
923
|
-
{
|
|
924
|
-
name: "gpuShmemMib",
|
|
925
|
-
type: "INTEGER"
|
|
926
|
-
})
|
|
927
|
-
];
|
|
928
|
-
var CONTAINER_MEMORY_INDEXES = [(
|
|
929
|
-
/** The one index, and it serves both the read and the prune — see the
|
|
930
|
-
* process table's `idx_load_samples_node_at` for the full reasoning. */
|
|
931
|
-
{
|
|
932
|
-
name: "idx_node_memory_node_at",
|
|
933
|
-
columns: ["nodeId", "atMs"]
|
|
934
|
-
})];
|
|
935
|
-
var BYTES_PER_MIB$1 = 1048576;
|
|
936
|
-
/** Bytes → whole MiB. `null` stays `null`: unknown is not zero. */
|
|
937
|
-
function toMib(value) {
|
|
938
|
-
return value === null ? null : Math.round(value / BYTES_PER_MIB$1);
|
|
939
|
-
}
|
|
940
|
-
/** Whole MiB → bytes. `null` stays `null`. */
|
|
941
|
-
function fromMib(value) {
|
|
942
|
-
return value === null ? null : value * BYTES_PER_MIB$1;
|
|
943
|
-
}
|
|
944
|
-
/** The ONE encode. */
|
|
945
|
-
function snapshotToRow(nodeId, atMs, snapshot) {
|
|
946
|
-
return {
|
|
947
|
-
nodeId,
|
|
948
|
-
atMs,
|
|
949
|
-
source: snapshot.source,
|
|
950
|
-
currentMib: Math.round(snapshot.currentBytes / BYTES_PER_MIB$1),
|
|
951
|
-
limitMib: toMib(snapshot.limitBytes),
|
|
952
|
-
anonMib: toMib(snapshot.anonBytes),
|
|
953
|
-
fileMib: toMib(snapshot.fileBytes),
|
|
954
|
-
shmemMib: toMib(snapshot.shmemBytes),
|
|
955
|
-
slabMib: toMib(snapshot.slabBytes),
|
|
956
|
-
gpuShmemMib: toMib(snapshot.gpuShmemBytes)
|
|
957
|
-
};
|
|
958
|
-
}
|
|
959
|
-
/**
|
|
960
|
-
* The ONE decode. A row whose `source` is not a hierarchy this build knows is
|
|
961
|
-
* skipped rather than coerced — a stored string that no longer means anything
|
|
962
|
-
* must not be charted as though it did.
|
|
963
|
-
*/
|
|
964
|
-
function rowToPoint(row) {
|
|
965
|
-
if (row.source !== "cgroup-v2" && row.source !== "cgroup-v1") return null;
|
|
966
|
-
return {
|
|
967
|
-
atMs: row.atMs,
|
|
968
|
-
source: row.source,
|
|
969
|
-
currentBytes: row.currentMib * BYTES_PER_MIB$1,
|
|
970
|
-
limitBytes: fromMib(row.limitMib),
|
|
971
|
-
anonBytes: fromMib(row.anonMib),
|
|
972
|
-
fileBytes: fromMib(row.fileMib),
|
|
973
|
-
shmemBytes: fromMib(row.shmemMib),
|
|
974
|
-
slabBytes: fromMib(row.slabMib),
|
|
975
|
-
gpuShmemBytes: fromMib(row.gpuShmemMib)
|
|
976
|
-
};
|
|
977
|
-
}
|
|
978
|
-
/** Structural validation on read. A malformed row is skipped, never charted. */
|
|
979
|
-
function recordToMemoryRow(data) {
|
|
980
|
-
const nodeId = data["nodeId"];
|
|
981
|
-
const source = data["source"];
|
|
982
|
-
if (typeof nodeId !== "string" || typeof source !== "string") return null;
|
|
983
|
-
const atMs = Number(data["atMs"]);
|
|
984
|
-
const currentMib = Number(data["currentMib"]);
|
|
985
|
-
if (!Number.isFinite(atMs) || !Number.isFinite(currentMib)) return null;
|
|
986
|
-
const optional = (raw) => {
|
|
987
|
-
if (raw === null || raw === void 0) return null;
|
|
988
|
-
const n = Number(raw);
|
|
989
|
-
return Number.isFinite(n) ? n : null;
|
|
990
|
-
};
|
|
991
|
-
return {
|
|
992
|
-
nodeId,
|
|
993
|
-
atMs,
|
|
994
|
-
source,
|
|
995
|
-
currentMib,
|
|
996
|
-
limitMib: optional(data["limitMib"]),
|
|
997
|
-
anonMib: optional(data["anonMib"]),
|
|
998
|
-
fileMib: optional(data["fileMib"]),
|
|
999
|
-
shmemMib: optional(data["shmemMib"]),
|
|
1000
|
-
slabMib: optional(data["slabMib"]),
|
|
1001
|
-
gpuShmemMib: optional(data["gpuShmemMib"])
|
|
1002
|
-
};
|
|
1003
|
-
}
|
|
1004
|
-
/**
|
|
1005
|
-
* The HOT tier — a naming adapter over {@link NodeLoadRing}, not a second ring.
|
|
1006
|
-
*
|
|
1007
|
-
* One reading per sample, so the ring's "rows" and its samples are the same
|
|
1008
|
-
* count and its fleet budget is a sample budget. Every bound, the idle sweep
|
|
1009
|
-
* and the monotonic replay gate come from there unchanged.
|
|
1010
|
-
*/
|
|
1011
819
|
var ContainerMemoryRing = class {
|
|
1012
820
|
ring;
|
|
1013
821
|
constructor(options = {}) {
|
|
@@ -1017,17 +825,9 @@ var ContainerMemoryRing = class {
|
|
|
1017
825
|
...options.now !== void 0 ? { now: options.now } : {}
|
|
1018
826
|
});
|
|
1019
827
|
}
|
|
1020
|
-
/** Retain one reading. `false` = a timestamp this node already delivered. */
|
|
1021
828
|
record(nodeId, atMs, snapshot) {
|
|
1022
829
|
return this.ring.record(nodeId, atMs, [snapshot]);
|
|
1023
830
|
}
|
|
1024
|
-
/**
|
|
1025
|
-
* Read one node's retained readings, oldest-first. `sinceMs` is EXCLUSIVE,
|
|
1026
|
-
* matching every other reader in this subsystem.
|
|
1027
|
-
*
|
|
1028
|
-
* A node nobody has recorded answers EMPTY — which is what "this node has no
|
|
1029
|
-
* cgroup" looks like, and it must stay distinguishable from a zero.
|
|
1030
|
-
*/
|
|
1031
831
|
read(nodeId, sinceMs) {
|
|
1032
832
|
const series = this.ring.read(nodeId, sinceMs);
|
|
1033
833
|
const out = [];
|
|
@@ -1042,23 +842,6 @@ var ContainerMemoryRing = class {
|
|
|
1042
842
|
return out;
|
|
1043
843
|
}
|
|
1044
844
|
};
|
|
1045
|
-
/**
|
|
1046
|
-
* Merge the two tiers and reduce, oldest-first.
|
|
1047
|
-
*
|
|
1048
|
-
* Dedupe is on `atMs` with the HOT copy winning, exactly as the process reader
|
|
1049
|
-
* does: the ring holds the reading as it arrived, the table holds it after the
|
|
1050
|
-
* MiB encode.
|
|
1051
|
-
*
|
|
1052
|
-
* Reduction keeps, for each bucket, the reading with the largest
|
|
1053
|
-
* `currentBytes` — WHOLE, every field from that one sample. A per-field
|
|
1054
|
-
* maximum would synthesise a row whose parts never coexisted (an `anon` peak
|
|
1055
|
-
* and a `file` peak minutes apart, summed into a total that never happened),
|
|
1056
|
-
* and a mean would smear away the spike this series exists to find. `atMs`
|
|
1057
|
-
* becomes the bucket START so the points lie on the same x-axis as the folded
|
|
1058
|
-
* process series, which is the only reason both travel in one answer.
|
|
1059
|
-
*
|
|
1060
|
-
* An empty bucket stays ABSENT — never zero, never carried forward.
|
|
1061
|
-
*/
|
|
1062
845
|
function reduceContainerMemory(points, bucketMs, origin) {
|
|
1063
846
|
if (bucketMs <= 0 || points.length === 0) return points;
|
|
1064
847
|
const buckets = /* @__PURE__ */ new Map();
|
|
@@ -1072,727 +855,13 @@ function reduceContainerMemory(points, bucketMs, origin) {
|
|
|
1072
855
|
}
|
|
1073
856
|
return [...buckets.values()].toSorted((a, b) => a.atMs - b.atMs);
|
|
1074
857
|
}
|
|
1075
|
-
/** Merge cold + hot on `atMs`, hot wins. Exported because dedup is the
|
|
1076
|
-
* property worth testing on its own. */
|
|
1077
858
|
function mergeContainerMemory(cold, hot) {
|
|
1078
859
|
const byAt = /* @__PURE__ */ new Map();
|
|
1079
860
|
for (const point of cold) byAt.set(point.atMs, point);
|
|
1080
861
|
for (const point of hot) byAt.set(point.atMs, point);
|
|
1081
862
|
return [...byAt.values()].toSorted((a, b) => a.atMs - b.atMs);
|
|
1082
863
|
}
|
|
1083
|
-
/**
|
|
1084
|
-
* The COLD tier. Append-only, bounded, and never the authority on anything the
|
|
1085
|
-
* hot ring can answer.
|
|
1086
|
-
*
|
|
1087
|
-
* Every method is best-effort and says so in the log rather than throwing at
|
|
1088
|
-
* the emitter: a storage hiccup must cost a gap in a chart, never a dropped
|
|
1089
|
-
* snapshot on the bus.
|
|
1090
|
-
*/
|
|
1091
|
-
var ContainerMemoryStore = class {
|
|
1092
|
-
declared = false;
|
|
1093
|
-
store;
|
|
1094
|
-
logger;
|
|
1095
|
-
nowFn;
|
|
1096
|
-
prunePageRows;
|
|
1097
|
-
constructor(deps) {
|
|
1098
|
-
this.store = deps.store;
|
|
1099
|
-
this.logger = deps.logger;
|
|
1100
|
-
this.nowFn = deps.now ?? (() => Date.now());
|
|
1101
|
-
this.prunePageRows = deps.prunePageRows ?? 2e4;
|
|
1102
|
-
}
|
|
1103
|
-
/** Idempotently declare the collection. `false` when the store refused. */
|
|
1104
|
-
async declare() {
|
|
1105
|
-
if (this.declared) return true;
|
|
1106
|
-
try {
|
|
1107
|
-
await this.store.declareCollection.mutate({
|
|
1108
|
-
collection: CONTAINER_MEMORY_COLLECTION,
|
|
1109
|
-
columns: [...CONTAINER_MEMORY_COLUMNS],
|
|
1110
|
-
indexes: CONTAINER_MEMORY_INDEXES.map((i) => ({
|
|
1111
|
-
name: i.name,
|
|
1112
|
-
columns: [...i.columns]
|
|
1113
|
-
}))
|
|
1114
|
-
});
|
|
1115
|
-
this.declared = true;
|
|
1116
|
-
return true;
|
|
1117
|
-
} catch (err) {
|
|
1118
|
-
this.logger.warn("container memory declareCollection failed — nothing will be retained on disk", { meta: {
|
|
1119
|
-
collection: CONTAINER_MEMORY_COLLECTION,
|
|
1120
|
-
error: errMsg(err)
|
|
1121
|
-
} });
|
|
1122
|
-
return false;
|
|
1123
|
-
}
|
|
1124
|
-
}
|
|
1125
|
-
/** Append ONE reading. */
|
|
1126
|
-
async append(nodeId, atMs, snapshot) {
|
|
1127
|
-
if (!await this.declare()) return 0;
|
|
1128
|
-
const record = { ...snapshotToRow(nodeId, atMs, snapshot) };
|
|
1129
|
-
try {
|
|
1130
|
-
const { inserted } = await this.store.insertMany.mutate({
|
|
1131
|
-
collection: CONTAINER_MEMORY_COLLECTION,
|
|
1132
|
-
records: [{ data: record }]
|
|
1133
|
-
});
|
|
1134
|
-
return inserted;
|
|
1135
|
-
} catch (err) {
|
|
1136
|
-
this.logger.warn("container memory reading not retained — this interval will be missing", { meta: {
|
|
1137
|
-
nodeId,
|
|
1138
|
-
atMs,
|
|
1139
|
-
error: errMsg(err)
|
|
1140
|
-
} });
|
|
1141
|
-
return 0;
|
|
1142
|
-
}
|
|
1143
|
-
}
|
|
1144
|
-
/** Read one node's cold readings, oldest-first. `sinceMs` is EXCLUSIVE. */
|
|
1145
|
-
async read(nodeId, sinceMs, limitRows) {
|
|
1146
|
-
if (!await this.declare()) return [];
|
|
1147
|
-
try {
|
|
1148
|
-
const records = await this.store.query.query({
|
|
1149
|
-
collection: CONTAINER_MEMORY_COLLECTION,
|
|
1150
|
-
filter: {
|
|
1151
|
-
where: { nodeId },
|
|
1152
|
-
whereBetween: { atMs: [sinceMs + 1, Number.MAX_SAFE_INTEGER] },
|
|
1153
|
-
orderBy: {
|
|
1154
|
-
field: "atMs",
|
|
1155
|
-
direction: "asc"
|
|
1156
|
-
},
|
|
1157
|
-
limit: limitRows
|
|
1158
|
-
}
|
|
1159
|
-
});
|
|
1160
|
-
const points = [];
|
|
1161
|
-
for (const record of records) {
|
|
1162
|
-
const row = recordToMemoryRow(record.data);
|
|
1163
|
-
if (row === null) continue;
|
|
1164
|
-
const point = rowToPoint(row);
|
|
1165
|
-
if (point !== null) points.push(point);
|
|
1166
|
-
}
|
|
1167
|
-
return points;
|
|
1168
|
-
} catch (err) {
|
|
1169
|
-
this.logger.warn("container memory cold read failed — answering from the hot window only", { meta: {
|
|
1170
|
-
nodeId,
|
|
1171
|
-
sinceMs,
|
|
1172
|
-
error: errMsg(err)
|
|
1173
|
-
} });
|
|
1174
|
-
return [];
|
|
1175
|
-
}
|
|
1176
|
-
}
|
|
1177
|
-
/**
|
|
1178
|
-
* Enforce BOTH bounds, oldest-first, through a bounded page each.
|
|
1179
|
-
*
|
|
1180
|
-
* Not rate-limited here: it is driven by the process store's own prune
|
|
1181
|
-
* cadence, so it runs at most once a minute by construction and a second
|
|
1182
|
-
* timer would be a second thing to get wrong.
|
|
1183
|
-
*/
|
|
1184
|
-
async prune(nodeIds, retention) {
|
|
1185
|
-
if (!await this.declare()) return null;
|
|
1186
|
-
const ageCutoff = this.nowFn() - retention.retentionHours * 36e5;
|
|
1187
|
-
let deletedByAge = 0;
|
|
1188
|
-
let rowsExamined = 0;
|
|
1189
|
-
for (const nodeId of nodeIds) {
|
|
1190
|
-
const outcome = await this.deleteOldestPage(nodeId, ageCutoff, this.prunePageRows);
|
|
1191
|
-
deletedByAge += outcome.deleted;
|
|
1192
|
-
rowsExamined += outcome.examined;
|
|
1193
|
-
}
|
|
1194
|
-
const total = await this.count();
|
|
1195
|
-
const excess = total === null ? 0 : total - CONTAINER_MEMORY_MAX_ROWS;
|
|
1196
|
-
let deletedByCap = 0;
|
|
1197
|
-
if (excess > 0) {
|
|
1198
|
-
let remaining = Math.min(excess, this.prunePageRows);
|
|
1199
|
-
for (const nodeId of nodeIds) {
|
|
1200
|
-
if (remaining <= 0) break;
|
|
1201
|
-
const outcome = await this.deleteOldestPage(nodeId, null, remaining);
|
|
1202
|
-
deletedByCap += outcome.deleted;
|
|
1203
|
-
remaining -= outcome.deleted;
|
|
1204
|
-
rowsExamined += outcome.examined;
|
|
1205
|
-
}
|
|
1206
|
-
this.logger.warn("container memory ROW CAP bit — evicting the oldest readings", { meta: {
|
|
1207
|
-
collection: CONTAINER_MEMORY_COLLECTION,
|
|
1208
|
-
rows: total,
|
|
1209
|
-
cap: CONTAINER_MEMORY_MAX_ROWS,
|
|
1210
|
-
over: excess,
|
|
1211
|
-
deleted: deletedByCap,
|
|
1212
|
-
hint: "lower the retention or the sampling cadence — the cap is the guarantee, not the intention"
|
|
1213
|
-
} });
|
|
1214
|
-
}
|
|
1215
|
-
return {
|
|
1216
|
-
deletedByAge,
|
|
1217
|
-
deletedByCap,
|
|
1218
|
-
rowsExamined,
|
|
1219
|
-
capBit: excess > 0
|
|
1220
|
-
};
|
|
1221
|
-
}
|
|
1222
|
-
/** Total rows, or `null` when the store could not answer. */
|
|
1223
|
-
async count() {
|
|
1224
|
-
if (!await this.declare()) return null;
|
|
1225
|
-
try {
|
|
1226
|
-
return await this.store.count.query({ collection: CONTAINER_MEMORY_COLLECTION });
|
|
1227
|
-
} catch (err) {
|
|
1228
|
-
this.logger.warn("container memory count failed — the row cap is not enforced this pass", { meta: { error: errMsg(err) } });
|
|
1229
|
-
return null;
|
|
1230
|
-
}
|
|
1231
|
-
}
|
|
1232
|
-
/**
|
|
1233
|
-
* Delete this node's oldest rows, at most one page's worth.
|
|
1234
|
-
*
|
|
1235
|
-
* `cutoff === null` means "the oldest rows, whatever their age" — the
|
|
1236
|
-
* row-cap path, which is bounded by `limitRows` instead. Either way the page
|
|
1237
|
-
* is READ FIRST, ordered by the one index, and its last `atMs` becomes the
|
|
1238
|
-
* effective cutoff for a single bounded delete. An unbounded `deleteWhere`
|
|
1239
|
-
* would be one statement and one stalling transaction the first time a
|
|
1240
|
-
* retention is lowered.
|
|
1241
|
-
*/
|
|
1242
|
-
async deleteOldestPage(nodeId, cutoff, limitRows) {
|
|
1243
|
-
if (limitRows <= 0) return {
|
|
1244
|
-
deleted: 0,
|
|
1245
|
-
examined: 0
|
|
1246
|
-
};
|
|
1247
|
-
try {
|
|
1248
|
-
const page = await this.store.query.query({
|
|
1249
|
-
collection: CONTAINER_MEMORY_COLLECTION,
|
|
1250
|
-
filter: {
|
|
1251
|
-
where: { nodeId },
|
|
1252
|
-
...cutoff === null ? {} : { whereBetween: { atMs: [0, cutoff] } },
|
|
1253
|
-
orderBy: {
|
|
1254
|
-
field: "atMs",
|
|
1255
|
-
direction: "asc"
|
|
1256
|
-
},
|
|
1257
|
-
limit: Math.min(limitRows, this.prunePageRows)
|
|
1258
|
-
},
|
|
1259
|
-
columns: ["atMs"]
|
|
1260
|
-
});
|
|
1261
|
-
if (page.length === 0) return {
|
|
1262
|
-
deleted: 0,
|
|
1263
|
-
examined: 0
|
|
1264
|
-
};
|
|
1265
|
-
const effectiveCutoff = Number(page.at(-1)?.data["atMs"]);
|
|
1266
|
-
if (!Number.isFinite(effectiveCutoff)) return {
|
|
1267
|
-
deleted: 0,
|
|
1268
|
-
examined: page.length
|
|
1269
|
-
};
|
|
1270
|
-
const { deleted } = await this.store.deleteWhere.mutate({
|
|
1271
|
-
collection: CONTAINER_MEMORY_COLLECTION,
|
|
1272
|
-
filter: {
|
|
1273
|
-
where: { nodeId },
|
|
1274
|
-
whereBetween: { atMs: [0, effectiveCutoff] }
|
|
1275
|
-
}
|
|
1276
|
-
});
|
|
1277
|
-
return {
|
|
1278
|
-
deleted,
|
|
1279
|
-
examined: page.length
|
|
1280
|
-
};
|
|
1281
|
-
} catch (err) {
|
|
1282
|
-
this.logger.warn("container memory prune failed — the table keeps growing this pass", { meta: {
|
|
1283
|
-
nodeId,
|
|
1284
|
-
cutoff,
|
|
1285
|
-
error: errMsg(err)
|
|
1286
|
-
} });
|
|
1287
|
-
return {
|
|
1288
|
-
deleted: 0,
|
|
1289
|
-
examined: 0
|
|
1290
|
-
};
|
|
1291
|
-
}
|
|
1292
|
-
}
|
|
1293
|
-
};
|
|
1294
|
-
//#endregion
|
|
1295
|
-
//#region src/builtins/native-metrics/load-series-store.ts
|
|
1296
|
-
/**
|
|
1297
|
-
* @durable class=audit owner=metrics-native
|
|
1298
|
-
* write="one row per process per sample, appended in ONE transaction per
|
|
1299
|
-
* sample (`settingsStore.insertMany`) by the HUB's `native-metrics`, off its
|
|
1300
|
-
* subscription to `metrics.node-processes-snapshot`. Emission is
|
|
1301
|
-
* fixed-cadence, so the write rate is deterministic: nodes x processes /
|
|
1302
|
-
* cadence, 7.6 rows/s on the live fleet at the 10 s default. Only a sample
|
|
1303
|
-
* the in-memory ring ACCEPTED is written, which is what makes the append
|
|
1304
|
-
* idempotent without a unique key."
|
|
1305
|
-
* retention="TWO bounds, swept together at most once a minute. By AGE, the
|
|
1306
|
-
* operator's `loadSeriesRetentionHours` (default 6, the intention); by ROWS,
|
|
1307
|
-
* DEFAULT_MAX_ROWS (the guarantee, which bites regardless of the age knob).
|
|
1308
|
-
* Both prune oldest-first through a bounded page — never a full scan — and
|
|
1309
|
-
* the row cap logs when it bites. Losing the table loses the cold window
|
|
1310
|
-
* only: the ring keeps serving the last hour and the charts keep drawing."
|
|
1311
|
-
*/
|
|
1312
|
-
var LOAD_SERIES_COLLECTION = "metrics:node-load-samples";
|
|
1313
|
-
/**
|
|
1314
|
-
* Hard ceiling on the table, in ROWS.
|
|
1315
|
-
*
|
|
1316
|
-
* The guarantee behind the retention knob. Sized against the shipped default
|
|
1317
|
-
* and against the worst configuration the knob can express:
|
|
1318
|
-
*
|
|
1319
|
-
* | cadence | retention | rows (3 nodes, 76 processes) |
|
|
1320
|
-
* | --- | --- | --- |
|
|
1321
|
-
* | 10 s | 6 h (default) | 164 160 |
|
|
1322
|
-
* | 5 s | 6 h | 328 320 |
|
|
1323
|
-
* | 5 s | 72 h | 3 939 840 — capped here |
|
|
1324
|
-
*
|
|
1325
|
-
* 500 000 rows x the measured 87 B = **41.5 MiB**, roughly 3x what the shipped
|
|
1326
|
-
* default needs and far below the 2 M rows a three-day retention would reach.
|
|
1327
|
-
* It is deliberately NOT a knob: the point of a guarantee is that it is not
|
|
1328
|
-
* subject to the same hand that raised the intention. This is the discipline
|
|
1329
|
-
* the in-memory ring already adopted (`DEFAULT_MAX_TOTAL_PROCESS_ROWS`) and
|
|
1330
|
-
* the one the RTP pre-roll rings still lack — a maximum per instance and no
|
|
1331
|
-
* budget at all for the whole, which reached 898 MB across 47 rings
|
|
1332
|
-
* (`docs/architecture/load-ledger.md` #5).
|
|
1333
|
-
*/
|
|
1334
864
|
var DEFAULT_MAX_ROWS = 5e5;
|
|
1335
|
-
var LOAD_SERIES_COLUMNS = [
|
|
1336
|
-
(
|
|
1337
|
-
/** `INTEGER PRIMARY KEY` = SQLite rowid alias: the key IS the row's address,
|
|
1338
|
-
* so it costs no separate index and no stored string. */
|
|
1339
|
-
{
|
|
1340
|
-
name: "id",
|
|
1341
|
-
type: "INTEGER",
|
|
1342
|
-
primaryKey: true,
|
|
1343
|
-
notNull: true
|
|
1344
|
-
}),
|
|
1345
|
-
{
|
|
1346
|
-
name: "nodeId",
|
|
1347
|
-
type: "TEXT",
|
|
1348
|
-
notNull: true
|
|
1349
|
-
},
|
|
1350
|
-
(
|
|
1351
|
-
/** The EMITTING node's timestamp for the sample this row belongs to. Every
|
|
1352
|
-
* row of one sample shares it — that is what makes a sample reassemblable. */
|
|
1353
|
-
{
|
|
1354
|
-
name: "atMs",
|
|
1355
|
-
type: "INTEGER",
|
|
1356
|
-
notNull: true
|
|
1357
|
-
}),
|
|
1358
|
-
{
|
|
1359
|
-
name: "pid",
|
|
1360
|
-
type: "INTEGER",
|
|
1361
|
-
notNull: true
|
|
1362
|
-
},
|
|
1363
|
-
(
|
|
1364
|
-
/** `NULL` for a process no addon owns — `root` and `system` both. */
|
|
1365
|
-
{
|
|
1366
|
-
name: "addonId",
|
|
1367
|
-
type: "TEXT"
|
|
1368
|
-
}),
|
|
1369
|
-
{
|
|
1370
|
-
name: "classification",
|
|
1371
|
-
type: "TEXT",
|
|
1372
|
-
notNull: true
|
|
1373
|
-
},
|
|
1374
|
-
(
|
|
1375
|
-
/** `ps pcpu` x 10. A LIFETIME average, not a rate — see `NodeProcess`. */
|
|
1376
|
-
{
|
|
1377
|
-
name: "cpuDeci",
|
|
1378
|
-
type: "INTEGER",
|
|
1379
|
-
notNull: true
|
|
1380
|
-
}),
|
|
1381
|
-
{
|
|
1382
|
-
name: "rssMib",
|
|
1383
|
-
type: "INTEGER",
|
|
1384
|
-
notNull: true
|
|
1385
|
-
},
|
|
1386
|
-
(
|
|
1387
|
-
/** Instantaneous main-thread CPU% x 10. `NULL` = UNKNOWN, never zero. */
|
|
1388
|
-
{
|
|
1389
|
-
name: "cpuMainDeci",
|
|
1390
|
-
type: "INTEGER"
|
|
1391
|
-
}),
|
|
1392
|
-
(
|
|
1393
|
-
/** Instantaneous V8-helper-pool CPU% x 10. `NULL` = UNKNOWN, never zero. */
|
|
1394
|
-
{
|
|
1395
|
-
name: "cpuGcDeci",
|
|
1396
|
-
type: "INTEGER"
|
|
1397
|
-
})
|
|
1398
|
-
];
|
|
1399
|
-
var LOAD_SERIES_INDEXES = [(
|
|
1400
|
-
/**
|
|
1401
|
-
* The ONE index, and it serves both jobs.
|
|
1402
|
-
*
|
|
1403
|
-
* Reads are always "this node, newer than T" — `nodeId` leads so the
|
|
1404
|
-
* equality is a range scan and `atMs` supplies the order without a sort.
|
|
1405
|
-
* Prunes are always "this node, oldest first" — the same index, walked from
|
|
1406
|
-
* the other end. A second index on `atMs` alone was measured and rejected:
|
|
1407
|
-
* it cost 23 B/row (196.0 → 172.8 with the two column drops) to serve a
|
|
1408
|
-
* cross-node prune that a per-node loop over three nodes already serves.
|
|
1409
|
-
*/
|
|
1410
|
-
{
|
|
1411
|
-
name: "idx_load_samples_node_at",
|
|
1412
|
-
columns: ["nodeId", "atMs"]
|
|
1413
|
-
})];
|
|
1414
|
-
var BYTES_PER_MIB = 1048576;
|
|
1415
|
-
/** Tenths → percent. `null` stays `null`: unknown is not zero. */
|
|
1416
|
-
function fromDeci(value) {
|
|
1417
|
-
return value === null ? null : Math.round(value) / 10;
|
|
1418
|
-
}
|
|
1419
|
-
/** Percent → tenths, lossless because both producers already round to tenths. */
|
|
1420
|
-
function toDeci(value) {
|
|
1421
|
-
return value === null ? null : Math.round(value * 10);
|
|
1422
|
-
}
|
|
1423
|
-
/** Storage row → the shape a reader gets. The ONE decode. */
|
|
1424
|
-
function rowToProcess(row) {
|
|
1425
|
-
return {
|
|
1426
|
-
pid: row.pid,
|
|
1427
|
-
addonId: row.addonId,
|
|
1428
|
-
classification: row.classification,
|
|
1429
|
-
cpuPercent: fromDeci(row.cpuDeci) ?? 0,
|
|
1430
|
-
memoryRssBytes: row.rssMib * BYTES_PER_MIB,
|
|
1431
|
-
cpuMainPercent: fromDeci(row.cpuMainDeci),
|
|
1432
|
-
cpuGcPercent: fromDeci(row.cpuGcDeci)
|
|
1433
|
-
};
|
|
1434
|
-
}
|
|
1435
|
-
/** The shape a reader gets → the storage row. The ONE encode. */
|
|
1436
|
-
function processToRow(nodeId, atMs, process) {
|
|
1437
|
-
return {
|
|
1438
|
-
nodeId,
|
|
1439
|
-
atMs,
|
|
1440
|
-
pid: process.pid,
|
|
1441
|
-
addonId: process.addonId,
|
|
1442
|
-
classification: process.classification,
|
|
1443
|
-
cpuDeci: toDeci(process.cpuPercent) ?? 0,
|
|
1444
|
-
rssMib: Math.round(process.memoryRssBytes / BYTES_PER_MIB),
|
|
1445
|
-
cpuMainDeci: toDeci(process.cpuMainPercent),
|
|
1446
|
-
cpuGcDeci: toDeci(process.cpuGcPercent)
|
|
1447
|
-
};
|
|
1448
|
-
}
|
|
1449
|
-
/** Group flat rows into samples, oldest-first. A sample is one `atMs`. */
|
|
1450
|
-
function rowsToSamples(rows) {
|
|
1451
|
-
const byAt = /* @__PURE__ */ new Map();
|
|
1452
|
-
for (const row of rows) {
|
|
1453
|
-
const held = byAt.get(row.atMs);
|
|
1454
|
-
if (held === void 0) byAt.set(row.atMs, [rowToProcess(row)]);
|
|
1455
|
-
else held.push(rowToProcess(row));
|
|
1456
|
-
}
|
|
1457
|
-
return [...byAt.entries()].toSorted((a, b) => a[0] - b[0]).map(([atMs, processes]) => ({
|
|
1458
|
-
atMs,
|
|
1459
|
-
processes
|
|
1460
|
-
}));
|
|
1461
|
-
}
|
|
1462
|
-
/**
|
|
1463
|
-
* The cold tier. Append-only, bounded, and never the authority on anything the
|
|
1464
|
-
* hot ring can answer.
|
|
1465
|
-
*
|
|
1466
|
-
* Every method is best-effort and says so in the log rather than throwing at
|
|
1467
|
-
* the emitter: a storage hiccup must cost a gap in a chart, never a dropped
|
|
1468
|
-
* snapshot on the bus.
|
|
1469
|
-
*/
|
|
1470
|
-
var LoadSeriesStore = class {
|
|
1471
|
-
declared = false;
|
|
1472
|
-
lastPruneAtMs = 0;
|
|
1473
|
-
store;
|
|
1474
|
-
logger;
|
|
1475
|
-
nowFn;
|
|
1476
|
-
prunePageRows;
|
|
1477
|
-
pruneIntervalMs;
|
|
1478
|
-
constructor(deps) {
|
|
1479
|
-
this.store = deps.store;
|
|
1480
|
-
this.logger = deps.logger;
|
|
1481
|
-
this.nowFn = deps.now ?? (() => Date.now());
|
|
1482
|
-
this.prunePageRows = deps.prunePageRows ?? 2e4;
|
|
1483
|
-
this.pruneIntervalMs = deps.pruneIntervalMs ?? 6e4;
|
|
1484
|
-
}
|
|
1485
|
-
/** Idempotently declare the collection. `false` when the store refused. */
|
|
1486
|
-
async declare() {
|
|
1487
|
-
if (this.declared) return true;
|
|
1488
|
-
try {
|
|
1489
|
-
await this.store.declareCollection.mutate({
|
|
1490
|
-
collection: LOAD_SERIES_COLLECTION,
|
|
1491
|
-
columns: [...LOAD_SERIES_COLUMNS],
|
|
1492
|
-
indexes: LOAD_SERIES_INDEXES.map((i) => ({
|
|
1493
|
-
name: i.name,
|
|
1494
|
-
columns: [...i.columns]
|
|
1495
|
-
}))
|
|
1496
|
-
});
|
|
1497
|
-
this.declared = true;
|
|
1498
|
-
return true;
|
|
1499
|
-
} catch (err) {
|
|
1500
|
-
this.logger.warn("load series declareCollection failed — nothing will be retained on disk", { meta: {
|
|
1501
|
-
collection: LOAD_SERIES_COLLECTION,
|
|
1502
|
-
error: errMsg(err)
|
|
1503
|
-
} });
|
|
1504
|
-
return false;
|
|
1505
|
-
}
|
|
1506
|
-
}
|
|
1507
|
-
/**
|
|
1508
|
-
* Append ONE sample — every process row of it — in ONE transaction.
|
|
1509
|
-
*
|
|
1510
|
-
* Never a write per row. `insertMany` exists for exactly this: at the 10 s
|
|
1511
|
-
* default the fleet produces 7.6 rows/s, and 7.6 separate commits per second
|
|
1512
|
-
* on the connection that also serves every cluster-wide configuration read
|
|
1513
|
-
* is a constant load nobody asked for.
|
|
1514
|
-
*/
|
|
1515
|
-
async append(nodeId, atMs, processes) {
|
|
1516
|
-
if (processes.length === 0) return 0;
|
|
1517
|
-
if (!await this.declare()) return 0;
|
|
1518
|
-
const records = processes.map((p) => ({ data: { ...processToRow(nodeId, atMs, p) } }));
|
|
1519
|
-
try {
|
|
1520
|
-
const { inserted } = await this.store.insertMany.mutate({
|
|
1521
|
-
collection: LOAD_SERIES_COLLECTION,
|
|
1522
|
-
records
|
|
1523
|
-
});
|
|
1524
|
-
return inserted;
|
|
1525
|
-
} catch (err) {
|
|
1526
|
-
this.logger.warn("load series sample not retained — this interval will be missing", { meta: {
|
|
1527
|
-
nodeId,
|
|
1528
|
-
atMs,
|
|
1529
|
-
rows: processes.length,
|
|
1530
|
-
error: errMsg(err)
|
|
1531
|
-
} });
|
|
1532
|
-
return 0;
|
|
1533
|
-
}
|
|
1534
|
-
}
|
|
1535
|
-
/**
|
|
1536
|
-
* Read one node's cold samples, oldest-first.
|
|
1537
|
-
*
|
|
1538
|
-
* `sinceMs` is EXCLUSIVE, matching the ring, so a caller passing the newest
|
|
1539
|
-
* timestamp it holds gets only what it is missing. `limitRows` bounds the
|
|
1540
|
-
* read in ROWS (not samples) because rows are what the query costs.
|
|
1541
|
-
*/
|
|
1542
|
-
async read(nodeId, sinceMs, limitRows) {
|
|
1543
|
-
if (!await this.declare()) return [];
|
|
1544
|
-
try {
|
|
1545
|
-
const records = await this.store.query.query({
|
|
1546
|
-
collection: LOAD_SERIES_COLLECTION,
|
|
1547
|
-
filter: {
|
|
1548
|
-
where: { nodeId },
|
|
1549
|
-
whereBetween: { atMs: [sinceMs + 1, Number.MAX_SAFE_INTEGER] },
|
|
1550
|
-
orderBy: {
|
|
1551
|
-
field: "atMs",
|
|
1552
|
-
direction: "asc"
|
|
1553
|
-
},
|
|
1554
|
-
limit: limitRows
|
|
1555
|
-
}
|
|
1556
|
-
});
|
|
1557
|
-
const rows = [];
|
|
1558
|
-
for (const record of records) {
|
|
1559
|
-
const row = recordToRow(record.data);
|
|
1560
|
-
if (row !== null) rows.push(row);
|
|
1561
|
-
}
|
|
1562
|
-
return rowsToSamples(rows);
|
|
1563
|
-
} catch (err) {
|
|
1564
|
-
this.logger.warn("load series cold read failed — answering from the hot window only", { meta: {
|
|
1565
|
-
nodeId,
|
|
1566
|
-
sinceMs,
|
|
1567
|
-
error: errMsg(err)
|
|
1568
|
-
} });
|
|
1569
|
-
return [];
|
|
1570
|
-
}
|
|
1571
|
-
}
|
|
1572
|
-
/**
|
|
1573
|
-
* Enforce BOTH bounds, oldest-first, through a bounded page each.
|
|
1574
|
-
*
|
|
1575
|
-
* Rate-limited to {@link pruneIntervalMs}: a bound is not a deadline, and the
|
|
1576
|
-
* append path must not pay a sweep on every sample.
|
|
1577
|
-
*/
|
|
1578
|
-
async prune(nodeIds, retention, force = false) {
|
|
1579
|
-
const now = this.nowFn();
|
|
1580
|
-
if (!force && now - this.lastPruneAtMs < this.pruneIntervalMs) return null;
|
|
1581
|
-
this.lastPruneAtMs = now;
|
|
1582
|
-
if (!await this.declare()) return null;
|
|
1583
|
-
let deletedByAge = 0;
|
|
1584
|
-
let deletedByCap = 0;
|
|
1585
|
-
let rowsExamined = 0;
|
|
1586
|
-
const ageCutoff = now - retention.retentionHours * 36e5;
|
|
1587
|
-
for (const nodeId of nodeIds) {
|
|
1588
|
-
const outcome = await this.pruneNodeToCutoff(nodeId, ageCutoff);
|
|
1589
|
-
deletedByAge += outcome.deleted;
|
|
1590
|
-
rowsExamined += outcome.examined;
|
|
1591
|
-
}
|
|
1592
|
-
const total = await this.count();
|
|
1593
|
-
const excess = total === null ? 0 : total - retention.maxRows;
|
|
1594
|
-
if (excess > 0) {
|
|
1595
|
-
const outcome = await this.pruneOldestRows(nodeIds, excess);
|
|
1596
|
-
deletedByCap = outcome.deleted;
|
|
1597
|
-
rowsExamined += outcome.examined;
|
|
1598
|
-
this.logger.warn("load series ROW CAP bit — evicting the oldest samples", { meta: {
|
|
1599
|
-
collection: LOAD_SERIES_COLLECTION,
|
|
1600
|
-
rows: total,
|
|
1601
|
-
cap: retention.maxRows,
|
|
1602
|
-
over: excess,
|
|
1603
|
-
deleted: outcome.deleted,
|
|
1604
|
-
retentionHours: retention.retentionHours,
|
|
1605
|
-
hint: "lower the retention or the sampling cadence — the cap is the guarantee, not the intention"
|
|
1606
|
-
} });
|
|
1607
|
-
}
|
|
1608
|
-
return {
|
|
1609
|
-
deletedByAge,
|
|
1610
|
-
deletedByCap,
|
|
1611
|
-
rowsExamined,
|
|
1612
|
-
capBit: excess > 0
|
|
1613
|
-
};
|
|
1614
|
-
}
|
|
1615
|
-
/** Total rows, or `null` when the store could not answer. */
|
|
1616
|
-
async count() {
|
|
1617
|
-
if (!await this.declare()) return null;
|
|
1618
|
-
try {
|
|
1619
|
-
return await this.store.count.query({ collection: LOAD_SERIES_COLLECTION });
|
|
1620
|
-
} catch (err) {
|
|
1621
|
-
this.logger.warn("load series count failed — the row cap is not enforced this pass", { meta: { error: errMsg(err) } });
|
|
1622
|
-
return null;
|
|
1623
|
-
}
|
|
1624
|
-
}
|
|
1625
|
-
/**
|
|
1626
|
-
* Delete this node's rows older than `cutoff`, at most one page's worth.
|
|
1627
|
-
*
|
|
1628
|
-
* The page is the whole point. `deleteWhere({ atMs: [0, cutoff] })` on its
|
|
1629
|
-
* own is one statement but an UNBOUNDED one — a first pass after a retention
|
|
1630
|
-
* change would delete millions of rows inside a single stalling transaction.
|
|
1631
|
-
* So the page is read first (keys only, ordered by the index), its last
|
|
1632
|
-
* `atMs` becomes the EFFECTIVE cutoff, and the delete is bounded by it.
|
|
1633
|
-
*/
|
|
1634
|
-
async pruneNodeToCutoff(nodeId, cutoff) {
|
|
1635
|
-
try {
|
|
1636
|
-
const page = await this.store.query.query({
|
|
1637
|
-
collection: LOAD_SERIES_COLLECTION,
|
|
1638
|
-
filter: {
|
|
1639
|
-
where: { nodeId },
|
|
1640
|
-
whereBetween: { atMs: [0, cutoff] },
|
|
1641
|
-
orderBy: {
|
|
1642
|
-
field: "atMs",
|
|
1643
|
-
direction: "asc"
|
|
1644
|
-
},
|
|
1645
|
-
limit: this.prunePageRows
|
|
1646
|
-
},
|
|
1647
|
-
columns: ["atMs"]
|
|
1648
|
-
});
|
|
1649
|
-
if (page.length === 0) return {
|
|
1650
|
-
deleted: 0,
|
|
1651
|
-
examined: 0
|
|
1652
|
-
};
|
|
1653
|
-
const effectiveCutoff = Number(page.at(-1)?.data["atMs"]);
|
|
1654
|
-
if (!Number.isFinite(effectiveCutoff)) return {
|
|
1655
|
-
deleted: 0,
|
|
1656
|
-
examined: page.length
|
|
1657
|
-
};
|
|
1658
|
-
const { deleted } = await this.store.deleteWhere.mutate({
|
|
1659
|
-
collection: LOAD_SERIES_COLLECTION,
|
|
1660
|
-
filter: {
|
|
1661
|
-
where: { nodeId },
|
|
1662
|
-
whereBetween: { atMs: [0, effectiveCutoff] }
|
|
1663
|
-
}
|
|
1664
|
-
});
|
|
1665
|
-
return {
|
|
1666
|
-
deleted,
|
|
1667
|
-
examined: page.length
|
|
1668
|
-
};
|
|
1669
|
-
} catch (err) {
|
|
1670
|
-
this.logger.warn("load series age prune failed — the table keeps growing this pass", { meta: {
|
|
1671
|
-
nodeId,
|
|
1672
|
-
cutoff,
|
|
1673
|
-
error: errMsg(err)
|
|
1674
|
-
} });
|
|
1675
|
-
return {
|
|
1676
|
-
deleted: 0,
|
|
1677
|
-
examined: 0
|
|
1678
|
-
};
|
|
1679
|
-
}
|
|
1680
|
-
}
|
|
1681
|
-
/**
|
|
1682
|
-
* Drop the oldest rows across the known nodes until `excess` is covered.
|
|
1683
|
-
*
|
|
1684
|
-
* Same bounded-page technique, walked per node so the one index serves it.
|
|
1685
|
-
* The node holding the oldest rows pays first, which is also the node
|
|
1686
|
-
* producing the pressure when a runaway process count is the cause.
|
|
1687
|
-
*/
|
|
1688
|
-
async pruneOldestRows(nodeIds, excess) {
|
|
1689
|
-
let remaining = Math.min(excess, this.prunePageRows);
|
|
1690
|
-
let deleted = 0;
|
|
1691
|
-
let examined = 0;
|
|
1692
|
-
for (const nodeId of nodeIds) {
|
|
1693
|
-
if (remaining <= 0) break;
|
|
1694
|
-
try {
|
|
1695
|
-
const page = await this.store.query.query({
|
|
1696
|
-
collection: LOAD_SERIES_COLLECTION,
|
|
1697
|
-
filter: {
|
|
1698
|
-
where: { nodeId },
|
|
1699
|
-
orderBy: {
|
|
1700
|
-
field: "atMs",
|
|
1701
|
-
direction: "asc"
|
|
1702
|
-
},
|
|
1703
|
-
limit: remaining
|
|
1704
|
-
},
|
|
1705
|
-
columns: ["atMs"]
|
|
1706
|
-
});
|
|
1707
|
-
examined += page.length;
|
|
1708
|
-
if (page.length === 0) continue;
|
|
1709
|
-
const cutoff = Number(page.at(-1)?.data["atMs"]);
|
|
1710
|
-
if (!Number.isFinite(cutoff)) continue;
|
|
1711
|
-
const result = await this.store.deleteWhere.mutate({
|
|
1712
|
-
collection: LOAD_SERIES_COLLECTION,
|
|
1713
|
-
filter: {
|
|
1714
|
-
where: { nodeId },
|
|
1715
|
-
whereBetween: { atMs: [0, cutoff] }
|
|
1716
|
-
}
|
|
1717
|
-
});
|
|
1718
|
-
deleted += result.deleted;
|
|
1719
|
-
remaining -= result.deleted;
|
|
1720
|
-
} catch (err) {
|
|
1721
|
-
this.logger.warn("load series cap prune failed — the cap is not enforced this pass", { meta: {
|
|
1722
|
-
nodeId,
|
|
1723
|
-
error: errMsg(err)
|
|
1724
|
-
} });
|
|
1725
|
-
}
|
|
1726
|
-
}
|
|
1727
|
-
return {
|
|
1728
|
-
deleted,
|
|
1729
|
-
examined
|
|
1730
|
-
};
|
|
1731
|
-
}
|
|
1732
|
-
};
|
|
1733
|
-
/** Structural validation on read. A malformed row is skipped, never charted. */
|
|
1734
|
-
function recordToRow(data) {
|
|
1735
|
-
const nodeId = data["nodeId"];
|
|
1736
|
-
const classification = data["classification"];
|
|
1737
|
-
if (typeof nodeId !== "string" || typeof classification !== "string") return null;
|
|
1738
|
-
const atMs = Number(data["atMs"]);
|
|
1739
|
-
const pid = Number(data["pid"]);
|
|
1740
|
-
const cpuDeci = Number(data["cpuDeci"]);
|
|
1741
|
-
const rssMib = Number(data["rssMib"]);
|
|
1742
|
-
if (![
|
|
1743
|
-
atMs,
|
|
1744
|
-
pid,
|
|
1745
|
-
cpuDeci,
|
|
1746
|
-
rssMib
|
|
1747
|
-
].every((n) => Number.isFinite(n))) return null;
|
|
1748
|
-
const rawAddon = data["addonId"];
|
|
1749
|
-
const optional = (raw) => {
|
|
1750
|
-
if (raw === null || raw === void 0) return null;
|
|
1751
|
-
const n = Number(raw);
|
|
1752
|
-
return Number.isFinite(n) ? n : null;
|
|
1753
|
-
};
|
|
1754
|
-
return {
|
|
1755
|
-
nodeId,
|
|
1756
|
-
atMs,
|
|
1757
|
-
pid,
|
|
1758
|
-
addonId: typeof rawAddon === "string" ? rawAddon : null,
|
|
1759
|
-
classification,
|
|
1760
|
-
cpuDeci,
|
|
1761
|
-
rssMib,
|
|
1762
|
-
cpuMainDeci: optional(data["cpuMainDeci"]),
|
|
1763
|
-
cpuGcDeci: optional(data["cpuGcDeci"])
|
|
1764
|
-
};
|
|
1765
|
-
}
|
|
1766
|
-
//#endregion
|
|
1767
|
-
//#region src/builtins/native-metrics/load-series-config.ts
|
|
1768
|
-
/**
|
|
1769
|
-
* The two knobs of the durable load series, and the arithmetic that makes them
|
|
1770
|
-
* honest.
|
|
1771
|
-
*
|
|
1772
|
-
* ## Where they live, and why there
|
|
1773
|
-
*
|
|
1774
|
-
* In `native-metrics`'s OWN global settings document — the one that already
|
|
1775
|
-
* exists and already carries `samplingIntervalMs`. Fields on a document, never
|
|
1776
|
-
* a method per knob: `system.getLoggingSettings` / `setLoggingSettings` set
|
|
1777
|
-
* that precedent and it holds here.
|
|
1778
|
-
*
|
|
1779
|
-
* They are NOT in the logging document, and that is a decision rather than an
|
|
1780
|
-
* omission. That document is about levels and diagnostic windows; a metrics
|
|
1781
|
-
* cadence and a table's retention are neither. Two documents that both claim a
|
|
1782
|
-
* knob is how this repo has already shipped a switch nobody read.
|
|
1783
|
-
*
|
|
1784
|
-
* They are cluster-wide, not per-node. Every node emits on the same cadence or
|
|
1785
|
-
* the fleet's series cannot be laid over each other, and the table is single
|
|
1786
|
-
* and hub-resident, so a per-node retention would be a promise nothing could
|
|
1787
|
-
* keep.
|
|
1788
|
-
*
|
|
1789
|
-
* ## Refused, not clamped
|
|
1790
|
-
*
|
|
1791
|
-
* A cadence outside 5-60 s is REFUSED. Clamping would silently store 10 when
|
|
1792
|
-
* the operator typed 2 and then report 10 back as though it had been accepted
|
|
1793
|
-
* — the failure mode where a knob and its readback agree on a value nobody
|
|
1794
|
-
* chose. The schema advertises the bounds; this refuses anything else.
|
|
1795
|
-
*/
|
|
1796
865
|
/** Why a value was refused — the message an operator is shown verbatim. */
|
|
1797
866
|
var LoadSeriesConfigError = class extends Error {
|
|
1798
867
|
constructor(message) {
|
|
@@ -1835,7 +904,7 @@ function resolveLoadSeriesConfig(raw) {
|
|
|
1835
904
|
* This is what makes the knob honest: an operator raising retention to three
|
|
1836
905
|
* days sees the row count and the megabytes BEFORE applying it, rather than
|
|
1837
906
|
* discovering them afterwards. `MEASURED_BYTES_PER_ROW` is measured, not
|
|
1838
|
-
* guessed — see `
|
|
907
|
+
* guessed — see `node-metrics-history-recorder.ts`.
|
|
1839
908
|
*/
|
|
1840
909
|
function projectLoadSeriesCost(input) {
|
|
1841
910
|
const samplesPerNodeWindow = input.config.retentionHours * 3600 / input.config.cadenceSec;
|
|
@@ -1861,8 +930,8 @@ function describeLoadSeriesCost(projection) {
|
|
|
1861
930
|
/**
|
|
1862
931
|
* The ONE reader over both retention tiers.
|
|
1863
932
|
*
|
|
1864
|
-
* The hot window is the in-memory ring; the cold one is
|
|
1865
|
-
*
|
|
933
|
+
* The hot window is the in-memory ring; the cold one is device-history raw
|
|
934
|
+
* points. A caller never learns which tier a point came
|
|
1866
935
|
* from, and that is a hard requirement rather than a nicety: a second reader
|
|
1867
936
|
* is how two charts start disagreeing about the same node, and it is how "the
|
|
1868
937
|
* ring says X, the table says Y" becomes a question nobody can answer.
|
|
@@ -1950,6 +1019,203 @@ function mergeLoadSeries(input) {
|
|
|
1950
1019
|
};
|
|
1951
1020
|
}
|
|
1952
1021
|
//#endregion
|
|
1022
|
+
//#region src/builtins/native-metrics/node-metrics-history-recorder.ts
|
|
1023
|
+
var ADDON_ID = "metrics-native";
|
|
1024
|
+
var NODE_PROCESS_CPU_KEY = `addon:${ADDON_ID}/process-cpu-percent`;
|
|
1025
|
+
var NODE_PROCESS_RSS_KEY = `addon:${ADDON_ID}/process-rss-mib`;
|
|
1026
|
+
var NODE_CONTAINER_CURRENT_KEY = `addon:${ADDON_ID}/container-current-mib`;
|
|
1027
|
+
function nodeMetricsHistoryClient(api) {
|
|
1028
|
+
return {
|
|
1029
|
+
declareSeries: (input) => api.declareSeries.mutate(input),
|
|
1030
|
+
record: (input) => api.record.mutate({ points: [...input.points] }),
|
|
1031
|
+
queryNode: async (input) => {
|
|
1032
|
+
return { series: (await api.queryNode.query(input)).series.map((s) => ({
|
|
1033
|
+
scope: s.scope,
|
|
1034
|
+
points: s.points.map((p) => ({
|
|
1035
|
+
t: p.t,
|
|
1036
|
+
samples: p.samples,
|
|
1037
|
+
value: p.last ?? p.mean
|
|
1038
|
+
}))
|
|
1039
|
+
})) };
|
|
1040
|
+
}
|
|
1041
|
+
};
|
|
1042
|
+
}
|
|
1043
|
+
var NodeMetricsHistoryRecorder = class {
|
|
1044
|
+
deps;
|
|
1045
|
+
declared = false;
|
|
1046
|
+
constructor(deps) {
|
|
1047
|
+
this.deps = deps;
|
|
1048
|
+
}
|
|
1049
|
+
async declare() {
|
|
1050
|
+
if (this.declared) return;
|
|
1051
|
+
const { deviceHistory } = this.deps;
|
|
1052
|
+
await deviceHistory.declareSeries({
|
|
1053
|
+
name: "process-cpu-percent",
|
|
1054
|
+
subjectKind: "node",
|
|
1055
|
+
kind: "gauge",
|
|
1056
|
+
displayName: "Process CPU %",
|
|
1057
|
+
ingest: "points",
|
|
1058
|
+
unit: "%"
|
|
1059
|
+
});
|
|
1060
|
+
await deviceHistory.declareSeries({
|
|
1061
|
+
name: "process-rss-mib",
|
|
1062
|
+
subjectKind: "node",
|
|
1063
|
+
kind: "gauge",
|
|
1064
|
+
displayName: "Process RSS (MiB)",
|
|
1065
|
+
ingest: "points",
|
|
1066
|
+
unit: "MiB"
|
|
1067
|
+
});
|
|
1068
|
+
await deviceHistory.declareSeries({
|
|
1069
|
+
name: "container-current-mib",
|
|
1070
|
+
subjectKind: "node",
|
|
1071
|
+
kind: "gauge",
|
|
1072
|
+
displayName: "Container memory current (MiB)",
|
|
1073
|
+
ingest: "points",
|
|
1074
|
+
unit: "MiB"
|
|
1075
|
+
});
|
|
1076
|
+
this.declared = true;
|
|
1077
|
+
}
|
|
1078
|
+
async recordSample(nodeId, atMs, processes, containerMemory) {
|
|
1079
|
+
if (!await this.ensureDeclared()) return;
|
|
1080
|
+
const points = [];
|
|
1081
|
+
for (const p of processes) {
|
|
1082
|
+
const scope = String(p.pid);
|
|
1083
|
+
points.push({
|
|
1084
|
+
key: NODE_PROCESS_CPU_KEY,
|
|
1085
|
+
subject: {
|
|
1086
|
+
kind: "node",
|
|
1087
|
+
nodeId
|
|
1088
|
+
},
|
|
1089
|
+
scope,
|
|
1090
|
+
ts: atMs,
|
|
1091
|
+
value: p.cpuPercent
|
|
1092
|
+
});
|
|
1093
|
+
points.push({
|
|
1094
|
+
key: NODE_PROCESS_RSS_KEY,
|
|
1095
|
+
subject: {
|
|
1096
|
+
kind: "node",
|
|
1097
|
+
nodeId
|
|
1098
|
+
},
|
|
1099
|
+
scope,
|
|
1100
|
+
ts: atMs,
|
|
1101
|
+
value: Math.round(p.memoryRssBytes / 1048576)
|
|
1102
|
+
});
|
|
1103
|
+
}
|
|
1104
|
+
if (containerMemory !== null) points.push({
|
|
1105
|
+
key: NODE_CONTAINER_CURRENT_KEY,
|
|
1106
|
+
subject: {
|
|
1107
|
+
kind: "node",
|
|
1108
|
+
nodeId
|
|
1109
|
+
},
|
|
1110
|
+
scope: "",
|
|
1111
|
+
ts: atMs,
|
|
1112
|
+
value: Math.round(containerMemory.currentBytes / 1048576)
|
|
1113
|
+
});
|
|
1114
|
+
if (points.length === 0) return;
|
|
1115
|
+
try {
|
|
1116
|
+
await this.deps.deviceHistory.record({ points });
|
|
1117
|
+
} catch (err) {
|
|
1118
|
+
this.deps.logger.warn("node metrics history sample not stored", { meta: {
|
|
1119
|
+
nodeId,
|
|
1120
|
+
atMs,
|
|
1121
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1122
|
+
} });
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
async readContainerPoints(nodeId, sinceMs, maxPoints) {
|
|
1126
|
+
if (!await this.ensureDeclared()) return [];
|
|
1127
|
+
const result = await this.deps.deviceHistory.queryNode({
|
|
1128
|
+
nodeId,
|
|
1129
|
+
key: NODE_CONTAINER_CURRENT_KEY,
|
|
1130
|
+
from: sinceMs + 1,
|
|
1131
|
+
to: Date.now(),
|
|
1132
|
+
resolution: "raw",
|
|
1133
|
+
maxPoints
|
|
1134
|
+
});
|
|
1135
|
+
const out = [];
|
|
1136
|
+
for (const s of result.series) for (const p of s.points) {
|
|
1137
|
+
if (p.value === null || !Number.isFinite(p.value)) continue;
|
|
1138
|
+
out.push({
|
|
1139
|
+
atMs: p.t,
|
|
1140
|
+
source: "cgroup-v2",
|
|
1141
|
+
currentBytes: p.value * 1048576,
|
|
1142
|
+
limitBytes: null,
|
|
1143
|
+
anonBytes: null,
|
|
1144
|
+
fileBytes: null,
|
|
1145
|
+
shmemBytes: null,
|
|
1146
|
+
slabBytes: null,
|
|
1147
|
+
gpuShmemBytes: null
|
|
1148
|
+
});
|
|
1149
|
+
}
|
|
1150
|
+
return out.toSorted((a, b) => a.atMs - b.atMs);
|
|
1151
|
+
}
|
|
1152
|
+
async readProcessSamples(nodeId, sinceMs, maxPoints) {
|
|
1153
|
+
if (!await this.ensureDeclared()) return [];
|
|
1154
|
+
const to = Date.now();
|
|
1155
|
+
const [cpuResult, rssResult] = await Promise.all([this.deps.deviceHistory.queryNode({
|
|
1156
|
+
nodeId,
|
|
1157
|
+
key: NODE_PROCESS_CPU_KEY,
|
|
1158
|
+
from: sinceMs + 1,
|
|
1159
|
+
to,
|
|
1160
|
+
resolution: "raw",
|
|
1161
|
+
maxPoints
|
|
1162
|
+
}), this.deps.deviceHistory.queryNode({
|
|
1163
|
+
nodeId,
|
|
1164
|
+
key: NODE_PROCESS_RSS_KEY,
|
|
1165
|
+
from: sinceMs + 1,
|
|
1166
|
+
to,
|
|
1167
|
+
resolution: "raw",
|
|
1168
|
+
maxPoints
|
|
1169
|
+
})]);
|
|
1170
|
+
const byAt = /* @__PURE__ */ new Map();
|
|
1171
|
+
const ingest = (series, field) => {
|
|
1172
|
+
for (const s of series) {
|
|
1173
|
+
const pid = Number(s.scope);
|
|
1174
|
+
if (!Number.isFinite(pid)) continue;
|
|
1175
|
+
for (const p of s.points) {
|
|
1176
|
+
if (p.value === null || !Number.isFinite(p.value)) continue;
|
|
1177
|
+
let perPid = byAt.get(p.t);
|
|
1178
|
+
if (perPid === void 0) {
|
|
1179
|
+
perPid = /* @__PURE__ */ new Map();
|
|
1180
|
+
byAt.set(p.t, perPid);
|
|
1181
|
+
}
|
|
1182
|
+
const base = perPid.get(pid) ?? {
|
|
1183
|
+
pid,
|
|
1184
|
+
addonId: null,
|
|
1185
|
+
classification: "managed",
|
|
1186
|
+
cpuPercent: 0,
|
|
1187
|
+
memoryRssBytes: 0,
|
|
1188
|
+
cpuMainPercent: null,
|
|
1189
|
+
cpuGcPercent: null
|
|
1190
|
+
};
|
|
1191
|
+
perPid.set(pid, field === "cpuPercent" ? {
|
|
1192
|
+
...base,
|
|
1193
|
+
cpuPercent: p.value
|
|
1194
|
+
} : {
|
|
1195
|
+
...base,
|
|
1196
|
+
memoryRssBytes: p.value * 1048576
|
|
1197
|
+
});
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
};
|
|
1201
|
+
ingest(cpuResult.series, "cpuPercent");
|
|
1202
|
+
ingest(rssResult.series, "memoryRssBytes");
|
|
1203
|
+
return [...byAt.entries()].toSorted((a, b) => a[0] - b[0]).map(([atMs, perPid]) => ({
|
|
1204
|
+
atMs,
|
|
1205
|
+
processes: [...perPid.values()]
|
|
1206
|
+
}));
|
|
1207
|
+
}
|
|
1208
|
+
async ensureDeclared() {
|
|
1209
|
+
try {
|
|
1210
|
+
await this.declare();
|
|
1211
|
+
return true;
|
|
1212
|
+
} catch (err) {
|
|
1213
|
+
this.deps.logger.warn("node metrics history declare failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
1214
|
+
return false;
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
};
|
|
1218
|
+
//#endregion
|
|
1953
1219
|
//#region src/builtins/native-metrics/gpu-probe.ts
|
|
1954
1220
|
/**
|
|
1955
1221
|
* The Linux GPU probe, forking only when there is something to fork.
|
|
@@ -2998,7 +2264,7 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
|
|
|
2998
2264
|
* the `settings-store` singleton would ship its rows over Moleculer, once
|
|
2999
2265
|
* per sample, to reach that very same table.
|
|
3000
2266
|
*/
|
|
3001
|
-
|
|
2267
|
+
metricsHistory = null;
|
|
3002
2268
|
/**
|
|
3003
2269
|
* The container's OWN memory, sampled on the process-snapshot tick.
|
|
3004
2270
|
*
|
|
@@ -3010,10 +2276,8 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
|
|
|
3010
2276
|
*/
|
|
3011
2277
|
memoryProbe = null;
|
|
3012
2278
|
pidsProbe = null;
|
|
3013
|
-
/** Hot tier for the container reading. Bounds in `container-memory-
|
|
2279
|
+
/** Hot tier for the container reading. Bounds in `container-memory-ring.ts`. */
|
|
3014
2280
|
memoryRing = new ContainerMemoryRing();
|
|
3015
|
-
/** Cold tier, HUB only — same reasoning as `loadStore`. */
|
|
3016
|
-
memoryStore = null;
|
|
3017
2281
|
/** The resolved knobs. Re-resolved on every settings write. */
|
|
3018
2282
|
loadConfig = {
|
|
3019
2283
|
cadenceSec: 10,
|
|
@@ -3086,13 +2350,12 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
|
|
|
3086
2350
|
};
|
|
3087
2351
|
this.applyLoadSeriesConfig();
|
|
3088
2352
|
if (this.isHub) {
|
|
3089
|
-
this.
|
|
3090
|
-
|
|
3091
|
-
logger: this.ctx.logger.child("
|
|
2353
|
+
this.metricsHistory = new NodeMetricsHistoryRecorder({
|
|
2354
|
+
deviceHistory: nodeMetricsHistoryClient(this.ctx.api.deviceHistory),
|
|
2355
|
+
logger: this.ctx.logger.child("NodeMetricsHistory")
|
|
3092
2356
|
});
|
|
3093
|
-
this.
|
|
3094
|
-
|
|
3095
|
-
logger: this.ctx.logger.child("ContainerMemory")
|
|
2357
|
+
this.metricsHistory.declare().catch((err) => {
|
|
2358
|
+
this.ctx.logger.warn("node metrics history declare failed", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
3096
2359
|
});
|
|
3097
2360
|
}
|
|
3098
2361
|
this.memoryProbe = new ContainerMemoryProbe({ logger: this.ctx.logger.child("ContainerMemory") });
|
|
@@ -3113,7 +2376,7 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
|
|
|
3113
2376
|
* this node has already delivered, and only an accepted sample reaches the
|
|
3114
2377
|
* table. That is what makes the append idempotent without a unique index
|
|
3115
2378
|
* over two million rows — the composite key it replaces was measured at ~40
|
|
3116
|
-
* bytes per row (
|
|
2379
|
+
* bytes per row (measured on the old SQLite tier; see D850).
|
|
3117
2380
|
*
|
|
3118
2381
|
* The durable append is fire-and-forget: a storage stall must cost a gap in
|
|
3119
2382
|
* the cold window, never a blocked event-bus handler. Every failure logs.
|
|
@@ -3123,26 +2386,9 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
|
|
|
3123
2386
|
if (!this.loadRing.record(nodeId, atMs, retained)) return;
|
|
3124
2387
|
this.observedRowsByNode.set(nodeId, retained.length);
|
|
3125
2388
|
if (containerMemory !== null) this.memoryRing.record(nodeId, atMs, containerMemory);
|
|
3126
|
-
const
|
|
3127
|
-
if (
|
|
3128
|
-
|
|
3129
|
-
if (outcome === null) return;
|
|
3130
|
-
await this.memoryStore?.prune([...this.observedRowsByNode.keys()], this.loadConfig);
|
|
3131
|
-
}).catch((err) => {
|
|
3132
|
-
this.ctx.logger.warn("durable load series write failed", { meta: {
|
|
3133
|
-
nodeId,
|
|
3134
|
-
atMs,
|
|
3135
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3136
|
-
} });
|
|
3137
|
-
});
|
|
3138
|
-
if (containerMemory === null) return;
|
|
3139
|
-
this.memoryStore?.append(nodeId, atMs, containerMemory).catch((err) => {
|
|
3140
|
-
this.ctx.logger.warn("durable container memory write failed", { meta: {
|
|
3141
|
-
nodeId,
|
|
3142
|
-
atMs,
|
|
3143
|
-
error: err instanceof Error ? err.message : String(err)
|
|
3144
|
-
} });
|
|
3145
|
-
});
|
|
2389
|
+
const history = this.metricsHistory;
|
|
2390
|
+
if (history === null) return;
|
|
2391
|
+
history.recordSample(nodeId, atMs, retained, containerMemory);
|
|
3146
2392
|
}
|
|
3147
2393
|
/**
|
|
3148
2394
|
* The ONE reader, over both tiers.
|
|
@@ -3161,9 +2407,9 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
|
|
|
3161
2407
|
atMs: sample.atMs,
|
|
3162
2408
|
processes: sample.processes
|
|
3163
2409
|
}));
|
|
3164
|
-
const
|
|
2410
|
+
const history = this.metricsHistory;
|
|
3165
2411
|
const merged = mergeLoadSeries({
|
|
3166
|
-
cold:
|
|
2412
|
+
cold: history === null ? [] : await history.readProcessSamples(params.forNodeId, sinceMs, this.loadConfig.maxRows),
|
|
3167
2413
|
hot: hotSamples,
|
|
3168
2414
|
cadenceMs,
|
|
3169
2415
|
...params.maxPoints !== void 0 ? { maxPoints: params.maxPoints } : {}
|
|
@@ -3183,7 +2429,7 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
|
|
|
3183
2429
|
retainedSamples: merged.retainedSamples,
|
|
3184
2430
|
oldestAtMs: merged.oldestAtMs,
|
|
3185
2431
|
cadenceMs,
|
|
3186
|
-
durable:
|
|
2432
|
+
durable: history !== null
|
|
3187
2433
|
};
|
|
3188
2434
|
}
|
|
3189
2435
|
/**
|
|
@@ -3199,8 +2445,8 @@ var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
|
|
|
3199
2445
|
*/
|
|
3200
2446
|
async readContainerMemory(params) {
|
|
3201
2447
|
const hot = this.memoryRing.read(params.forNodeId, params.exclusiveSinceMs);
|
|
3202
|
-
const
|
|
3203
|
-
const merged = mergeContainerMemory(
|
|
2448
|
+
const history = this.metricsHistory;
|
|
2449
|
+
const merged = mergeContainerMemory(history === null ? [] : await history.readContainerPoints(params.forNodeId, params.sinceMs, 5e3), hot);
|
|
3204
2450
|
if (params.bucketMs <= params.cadenceMs) return merged;
|
|
3205
2451
|
return reduceContainerMemory(merged, params.bucketMs, params.origin ?? 0);
|
|
3206
2452
|
}
|