@camstack/system 1.2.146 → 1.2.147
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +1 -1
- package/dist/addon-runner.mjs +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +2 -2
- package/dist/builtins/device-manager/device-manager.addon.mjs +2 -2
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/load-series-config.d.ts +74 -0
- package/dist/builtins/native-metrics/load-series-reader.d.ts +37 -0
- package/dist/builtins/native-metrics/load-series-store.d.ts +232 -0
- package/dist/builtins/native-metrics/native-metrics.addon.d.ts +118 -35
- package/dist/builtins/native-metrics/native-metrics.addon.js +1463 -199
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1463 -199
- package/dist/builtins/native-metrics/node-load-ring.d.ts +214 -0
- package/dist/builtins/native-metrics/process-classification.d.ts +51 -0
- package/dist/builtins/native-metrics/thread-cpu-sampler.d.ts +116 -0
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings-backend.d.ts +25 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +2 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +2 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-BVU5JADq.js → dist-Dl6MFXPr.js} +496 -35
- package/dist/{dist-CG7kwIZp.mjs → dist-DrBc-7Y3.mjs} +479 -36
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/dist/{manifest-python-deps-CVeckyXx.mjs → manifest-python-deps-DhT0oFOZ.mjs} +1 -1
- package/dist/{manifest-python-deps-CktMcXzS.js → manifest-python-deps-FYZBHc3v.js} +1 -1
- package/dist/{retired-settings-keys-_PLI9w0k.js → retired-settings-keys-6w_JOqBg.js} +1 -1
- package/dist/{retired-settings-keys-Dm1z4mO4.mjs → retired-settings-keys-CF5r4YKq.mjs} +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Ht as createEvent, Ot as errMsg, ft as reducePoints, kt as BaseAddon, ot as metricsProviderCapability, pt as resolveBucketMs, q as foldSnapshotByFunction, tn as EventCategory } from "../../dist-DrBc-7Y3.mjs";
|
|
2
|
+
import { readFile, readdir } from "node:fs/promises";
|
|
2
3
|
import { execFile, execFileSync } from "node:child_process";
|
|
3
4
|
import { promisify } from "node:util";
|
|
4
5
|
import * as os from "node:os";
|
|
@@ -458,9 +459,933 @@ function execAsync(cmd, args, timeoutMs = 5e3) {
|
|
|
458
459
|
});
|
|
459
460
|
});
|
|
460
461
|
}
|
|
462
|
+
/**
|
|
463
|
+
* One node's fixed-capacity circular buffer of samples.
|
|
464
|
+
*
|
|
465
|
+
* Mirrors {@link LogRingBuffer}: write at `head`, wrap, and once full every
|
|
466
|
+
* accepted sample evicts the oldest. `rows` is carried alongside so the fleet
|
|
467
|
+
* budget never has to walk the samples to know what a partition costs.
|
|
468
|
+
*/
|
|
469
|
+
var LoadPartition = class {
|
|
470
|
+
capacity;
|
|
471
|
+
slots;
|
|
472
|
+
head = 0;
|
|
473
|
+
count = 0;
|
|
474
|
+
rows = 0;
|
|
475
|
+
/** Newest retained sample's timestamp — the monotonic gate for `push`. */
|
|
476
|
+
newestAtMs = null;
|
|
477
|
+
constructor(capacity) {
|
|
478
|
+
this.capacity = capacity;
|
|
479
|
+
this.slots = Array.from({ length: capacity });
|
|
480
|
+
}
|
|
481
|
+
/**
|
|
482
|
+
* Accept a sample.
|
|
483
|
+
*
|
|
484
|
+
* A sample at or before the newest one already held is REFUSED. The bus drops
|
|
485
|
+
* a node's own broadcast echo, but a cross-node redelivery or a replayed
|
|
486
|
+
* subscription must not be able to double a point — and idempotence here is
|
|
487
|
+
* what lets every reader above treat the series as a set.
|
|
488
|
+
*
|
|
489
|
+
* `accepted` is reported separately from `rowDelta` on purpose: a full ring
|
|
490
|
+
* that evicts a sample of the same size has a delta of zero and is not a
|
|
491
|
+
* refusal, and conflating the two would silently stop advancing the write
|
|
492
|
+
* ordinal on a steady-state cluster.
|
|
493
|
+
*/
|
|
494
|
+
push(sample) {
|
|
495
|
+
if (this.newestAtMs !== null && sample.atMs <= this.newestAtMs) return {
|
|
496
|
+
accepted: false,
|
|
497
|
+
rowDelta: 0
|
|
498
|
+
};
|
|
499
|
+
const evicted = this.count === this.capacity ? this.slots[this.head]?.processes.length ?? 0 : 0;
|
|
500
|
+
this.slots[this.head] = sample;
|
|
501
|
+
this.head = (this.head + 1) % this.capacity;
|
|
502
|
+
if (this.count < this.capacity) this.count++;
|
|
503
|
+
this.newestAtMs = sample.atMs;
|
|
504
|
+
const rowDelta = sample.processes.length - evicted;
|
|
505
|
+
this.rows += rowDelta;
|
|
506
|
+
return {
|
|
507
|
+
accepted: true,
|
|
508
|
+
rowDelta
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
/** Drop the oldest sample. Returns the rows reclaimed (0 when empty). */
|
|
512
|
+
dropOldest() {
|
|
513
|
+
if (this.count === 0) return 0;
|
|
514
|
+
const index = (this.head - this.count + this.capacity) % this.capacity;
|
|
515
|
+
const victim = this.slots[index];
|
|
516
|
+
this.slots[index] = void 0;
|
|
517
|
+
this.count--;
|
|
518
|
+
const reclaimed = victim?.processes.length ?? 0;
|
|
519
|
+
this.rows -= reclaimed;
|
|
520
|
+
if (this.count === 0) this.newestAtMs = null;
|
|
521
|
+
return reclaimed;
|
|
522
|
+
}
|
|
523
|
+
/** Oldest-first, optionally only what is strictly newer than `sinceMs`. */
|
|
524
|
+
list(sinceMs) {
|
|
525
|
+
const out = [];
|
|
526
|
+
for (let i = 0; i < this.count; i++) {
|
|
527
|
+
const index = (this.head - this.count + i + this.capacity) % this.capacity;
|
|
528
|
+
const sample = this.slots[index];
|
|
529
|
+
if (sample === void 0) continue;
|
|
530
|
+
if (sinceMs !== void 0 && sample.atMs <= sinceMs) continue;
|
|
531
|
+
out.push(sample);
|
|
532
|
+
}
|
|
533
|
+
return out;
|
|
534
|
+
}
|
|
535
|
+
size() {
|
|
536
|
+
return this.count;
|
|
537
|
+
}
|
|
538
|
+
rowCount() {
|
|
539
|
+
return this.rows;
|
|
540
|
+
}
|
|
541
|
+
oldestAtMs() {
|
|
542
|
+
if (this.count === 0) return null;
|
|
543
|
+
const index = (this.head - this.count + this.capacity) % this.capacity;
|
|
544
|
+
return this.slots[index]?.atMs ?? null;
|
|
545
|
+
}
|
|
546
|
+
lastWriteAtMs() {
|
|
547
|
+
return this.newestAtMs;
|
|
548
|
+
}
|
|
549
|
+
};
|
|
550
|
+
/**
|
|
551
|
+
* The partitioned ring: one {@link LoadPartition} per node id.
|
|
552
|
+
*
|
|
553
|
+
* Three bounds, doing three different jobs — the same division as the log
|
|
554
|
+
* buffer, for the same reason:
|
|
555
|
+
*
|
|
556
|
+
* - **`samplesPerNode`** is HARD and per partition. It is what the operator
|
|
557
|
+
* asked for (an hour) and it can never be exceeded.
|
|
558
|
+
* - **`maxTotalProcessRows`** is the FLEET budget, and it is denominated in the
|
|
559
|
+
* thing memory is proportional to. When it bites, the oldest sample of the
|
|
560
|
+
* HEAVIEST partition is dropped — the node creating the pressure is the node
|
|
561
|
+
* that pays, and a quiet node keeps its full hour. The node just written to is
|
|
562
|
+
* NOT protected here: if it is the heaviest, it is the culprit.
|
|
563
|
+
* - **`maxNodes`** is the backstop against an unbounded key space, and it drops
|
|
564
|
+
* whole partitions least-recently-written first, never the one being written.
|
|
565
|
+
*
|
|
566
|
+
* Plus {@link DEFAULT_NODE_IDLE_EVICTION_MS}, which is what stops a node that
|
|
567
|
+
* simply stopped emitting from living in memory forever.
|
|
568
|
+
*/
|
|
569
|
+
var NodeLoadRing = class {
|
|
570
|
+
partitions = /* @__PURE__ */ new Map();
|
|
571
|
+
/**
|
|
572
|
+
* Node id → write ordinal of its last accepted sample. A monotonic counter,
|
|
573
|
+
* not a clock: partition eviction must follow the order writes actually
|
|
574
|
+
* happened, and node clocks disagree.
|
|
575
|
+
*/
|
|
576
|
+
lastWriteSeq = /* @__PURE__ */ new Map();
|
|
577
|
+
writeSeq = 0;
|
|
578
|
+
totalRows = 0;
|
|
579
|
+
samplesPerNode;
|
|
580
|
+
maxTotalProcessRows;
|
|
581
|
+
maxNodes;
|
|
582
|
+
idleEvictionMs;
|
|
583
|
+
now;
|
|
584
|
+
constructor(options = {}) {
|
|
585
|
+
this.samplesPerNode = options.samplesPerNode ?? 180;
|
|
586
|
+
this.maxTotalProcessRows = options.maxTotalProcessRows ?? 32e3;
|
|
587
|
+
this.maxNodes = options.maxNodes ?? 16;
|
|
588
|
+
this.idleEvictionMs = options.idleEvictionMs ?? 36e5;
|
|
589
|
+
this.now = options.now ?? Date.now;
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Retain one snapshot. `processes` is stored by reference — the payload is
|
|
593
|
+
* already an immutable arrival off the bus, and copying it would double the
|
|
594
|
+
* measured cost for nothing.
|
|
595
|
+
*
|
|
596
|
+
* Returns whether the sample was ACCEPTED — that is, whether it was new
|
|
597
|
+
* rather than a replay of a timestamp this node has already delivered.
|
|
598
|
+
*
|
|
599
|
+
* The return value is not diagnostics. It is the idempotence gate the
|
|
600
|
+
* DURABLE tier rides on (`load-series-store.ts`): only an accepted sample is
|
|
601
|
+
* appended to the table, which is what lets that table carry an `INTEGER`
|
|
602
|
+
* rowid key instead of a composite unique index over two million rows. The
|
|
603
|
+
* gate is one monotonic comparison in memory; the index it replaces was
|
|
604
|
+
* measured at ~40 bytes per row.
|
|
605
|
+
*/
|
|
606
|
+
record(nodeId, atMs, processes) {
|
|
607
|
+
if (nodeId.length === 0) return false;
|
|
608
|
+
this.sweepIdle();
|
|
609
|
+
let partition = this.partitions.get(nodeId);
|
|
610
|
+
if (partition === void 0) {
|
|
611
|
+
partition = new LoadPartition(this.samplesPerNode);
|
|
612
|
+
this.partitions.set(nodeId, partition);
|
|
613
|
+
}
|
|
614
|
+
const outcome = partition.push({
|
|
615
|
+
atMs,
|
|
616
|
+
processes
|
|
617
|
+
});
|
|
618
|
+
if (!outcome.accepted) {
|
|
619
|
+
if (partition.size() === 0) this.partitions.delete(nodeId);
|
|
620
|
+
return false;
|
|
621
|
+
}
|
|
622
|
+
this.totalRows += outcome.rowDelta;
|
|
623
|
+
this.lastWriteSeq.set(nodeId, ++this.writeSeq);
|
|
624
|
+
if (this.partitions.size > this.maxNodes) this.evictPartitions(this.partitions.size - this.maxNodes, nodeId);
|
|
625
|
+
this.enforceRowBudget();
|
|
626
|
+
return true;
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* Read one node's retained series, oldest-first.
|
|
630
|
+
*
|
|
631
|
+
* `sinceMs` is EXCLUSIVE: a caller passes the newest timestamp it already
|
|
632
|
+
* holds and gets back only what it is missing. That is the whole contract
|
|
633
|
+
* that lets the admin UI seed from here and then continue live without
|
|
634
|
+
* doubling a point it already drew.
|
|
635
|
+
*
|
|
636
|
+
* A node nobody has recorded answers with an empty series, not an error —
|
|
637
|
+
* unknown is the truth about a node that has not reported.
|
|
638
|
+
*/
|
|
639
|
+
read(nodeId, sinceMs) {
|
|
640
|
+
this.sweepIdle();
|
|
641
|
+
const partition = this.partitions.get(nodeId);
|
|
642
|
+
if (partition === void 0) return {
|
|
643
|
+
nodeId,
|
|
644
|
+
samples: [],
|
|
645
|
+
retainedSamples: 0,
|
|
646
|
+
oldestAtMs: null,
|
|
647
|
+
capacity: this.samplesPerNode
|
|
648
|
+
};
|
|
649
|
+
return {
|
|
650
|
+
nodeId,
|
|
651
|
+
samples: partition.list(sinceMs),
|
|
652
|
+
retainedSamples: partition.size(),
|
|
653
|
+
oldestAtMs: partition.oldestAtMs(),
|
|
654
|
+
capacity: this.samplesPerNode
|
|
655
|
+
};
|
|
656
|
+
}
|
|
657
|
+
/** Node ids with a live partition. Observability for the fleet bound. */
|
|
658
|
+
nodeIds() {
|
|
659
|
+
return [...this.partitions.keys()];
|
|
660
|
+
}
|
|
661
|
+
/** Process rows retained across every partition — the number that IS memory. */
|
|
662
|
+
rowCount() {
|
|
663
|
+
return this.totalRows;
|
|
664
|
+
}
|
|
665
|
+
/** Samples retained across every partition. */
|
|
666
|
+
sampleCount() {
|
|
667
|
+
let total = 0;
|
|
668
|
+
for (const partition of this.partitions.values()) total += partition.size();
|
|
669
|
+
return total;
|
|
670
|
+
}
|
|
671
|
+
/**
|
|
672
|
+
* Drop partitions whose newest sample is older than the retention window.
|
|
673
|
+
*
|
|
674
|
+
* Lazy, on write and on read — never a timer. A timer would be a new
|
|
675
|
+
* periodic cost in a subsystem whose entire premise is that it adds none,
|
|
676
|
+
* and a ring that nobody writes to and nobody reads is not growing either.
|
|
677
|
+
*/
|
|
678
|
+
sweepIdle() {
|
|
679
|
+
const cutoff = this.now() - this.idleEvictionMs;
|
|
680
|
+
for (const [nodeId, partition] of [...this.partitions.entries()]) {
|
|
681
|
+
const lastWrite = partition.lastWriteAtMs();
|
|
682
|
+
if (lastWrite !== null && lastWrite > cutoff) continue;
|
|
683
|
+
this.dropPartition(nodeId);
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Bring the fleet back under the row budget by dropping the OLDEST sample of
|
|
688
|
+
* the HEAVIEST partition, repeatedly.
|
|
689
|
+
*
|
|
690
|
+
* Terminates: every iteration removes one sample from a non-empty partition,
|
|
691
|
+
* and the population of samples is finite and strictly decreasing.
|
|
692
|
+
*/
|
|
693
|
+
enforceRowBudget() {
|
|
694
|
+
while (this.totalRows > this.maxTotalProcessRows) {
|
|
695
|
+
const victim = this.heaviestPartition();
|
|
696
|
+
if (victim === null) return;
|
|
697
|
+
const [nodeId, partition] = victim;
|
|
698
|
+
this.totalRows -= partition.dropOldest();
|
|
699
|
+
if (partition.size() === 0) this.dropPartition(nodeId);
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
heaviestPartition() {
|
|
703
|
+
let best = null;
|
|
704
|
+
for (const entry of this.partitions.entries()) {
|
|
705
|
+
if (entry[1].size() === 0) continue;
|
|
706
|
+
if (best === null || entry[1].rowCount() > best[1].rowCount()) best = entry;
|
|
707
|
+
}
|
|
708
|
+
return best;
|
|
709
|
+
}
|
|
710
|
+
/** Drop `count` whole partitions, least-recently-written first. */
|
|
711
|
+
evictPartitions(count, protectedNodeId) {
|
|
712
|
+
const order = [...this.partitions.keys()].filter((nodeId) => nodeId !== protectedNodeId).toSorted((a, b) => (this.lastWriteSeq.get(a) ?? 0) - (this.lastWriteSeq.get(b) ?? 0));
|
|
713
|
+
let remaining = count;
|
|
714
|
+
for (const nodeId of order) {
|
|
715
|
+
if (remaining <= 0) return;
|
|
716
|
+
this.dropPartition(nodeId);
|
|
717
|
+
remaining -= 1;
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
/** Remove a partition and everything that indexes it. */
|
|
721
|
+
dropPartition(nodeId) {
|
|
722
|
+
const partition = this.partitions.get(nodeId);
|
|
723
|
+
if (partition === void 0) return;
|
|
724
|
+
this.totalRows -= partition.rowCount();
|
|
725
|
+
this.partitions.delete(nodeId);
|
|
726
|
+
this.lastWriteSeq.delete(nodeId);
|
|
727
|
+
}
|
|
728
|
+
};
|
|
461
729
|
//#endregion
|
|
462
|
-
//#region src/builtins/native-metrics/
|
|
463
|
-
|
|
730
|
+
//#region src/builtins/native-metrics/load-series-store.ts
|
|
731
|
+
/**
|
|
732
|
+
* @durable class=audit owner=metrics-native
|
|
733
|
+
* write="one row per process per sample, appended in ONE transaction per
|
|
734
|
+
* sample (`settingsStore.insertMany`) by the HUB's `native-metrics`, off its
|
|
735
|
+
* subscription to `metrics.node-processes-snapshot`. Emission is
|
|
736
|
+
* fixed-cadence, so the write rate is deterministic: nodes x processes /
|
|
737
|
+
* cadence, 7.6 rows/s on the live fleet at the 10 s default. Only a sample
|
|
738
|
+
* the in-memory ring ACCEPTED is written, which is what makes the append
|
|
739
|
+
* idempotent without a unique key."
|
|
740
|
+
* retention="TWO bounds, swept together at most once a minute. By AGE, the
|
|
741
|
+
* operator's `loadSeriesRetentionHours` (default 6, the intention); by ROWS,
|
|
742
|
+
* DEFAULT_MAX_ROWS (the guarantee, which bites regardless of the age knob).
|
|
743
|
+
* Both prune oldest-first through a bounded page — never a full scan — and
|
|
744
|
+
* the row cap logs when it bites. Losing the table loses the cold window
|
|
745
|
+
* only: the ring keeps serving the last hour and the charts keep drawing."
|
|
746
|
+
*/
|
|
747
|
+
var LOAD_SERIES_COLLECTION = "metrics:node-load-samples";
|
|
748
|
+
/**
|
|
749
|
+
* Hard ceiling on the table, in ROWS.
|
|
750
|
+
*
|
|
751
|
+
* The guarantee behind the retention knob. Sized against the shipped default
|
|
752
|
+
* and against the worst configuration the knob can express:
|
|
753
|
+
*
|
|
754
|
+
* | cadence | retention | rows (3 nodes, 76 processes) |
|
|
755
|
+
* | --- | --- | --- |
|
|
756
|
+
* | 10 s | 6 h (default) | 164 160 |
|
|
757
|
+
* | 5 s | 6 h | 328 320 |
|
|
758
|
+
* | 5 s | 72 h | 3 939 840 — capped here |
|
|
759
|
+
*
|
|
760
|
+
* 500 000 rows x the measured 87 B = **41.5 MiB**, roughly 3x what the shipped
|
|
761
|
+
* default needs and far below the 2 M rows a three-day retention would reach.
|
|
762
|
+
* It is deliberately NOT a knob: the point of a guarantee is that it is not
|
|
763
|
+
* subject to the same hand that raised the intention. This is the discipline
|
|
764
|
+
* the in-memory ring already adopted (`DEFAULT_MAX_TOTAL_PROCESS_ROWS`) and
|
|
765
|
+
* the one the RTP pre-roll rings still lack — a maximum per instance and no
|
|
766
|
+
* budget at all for the whole, which reached 898 MB across 47 rings
|
|
767
|
+
* (`docs/architecture/load-ledger.md` #5).
|
|
768
|
+
*/
|
|
769
|
+
var DEFAULT_MAX_ROWS = 5e5;
|
|
770
|
+
var LOAD_SERIES_COLUMNS = [
|
|
771
|
+
(
|
|
772
|
+
/** `INTEGER PRIMARY KEY` = SQLite rowid alias: the key IS the row's address,
|
|
773
|
+
* so it costs no separate index and no stored string. */
|
|
774
|
+
{
|
|
775
|
+
name: "id",
|
|
776
|
+
type: "INTEGER",
|
|
777
|
+
primaryKey: true,
|
|
778
|
+
notNull: true
|
|
779
|
+
}),
|
|
780
|
+
{
|
|
781
|
+
name: "nodeId",
|
|
782
|
+
type: "TEXT",
|
|
783
|
+
notNull: true
|
|
784
|
+
},
|
|
785
|
+
(
|
|
786
|
+
/** The EMITTING node's timestamp for the sample this row belongs to. Every
|
|
787
|
+
* row of one sample shares it — that is what makes a sample reassemblable. */
|
|
788
|
+
{
|
|
789
|
+
name: "atMs",
|
|
790
|
+
type: "INTEGER",
|
|
791
|
+
notNull: true
|
|
792
|
+
}),
|
|
793
|
+
{
|
|
794
|
+
name: "pid",
|
|
795
|
+
type: "INTEGER",
|
|
796
|
+
notNull: true
|
|
797
|
+
},
|
|
798
|
+
(
|
|
799
|
+
/** `NULL` for a process no addon owns — `root` and `system` both. */
|
|
800
|
+
{
|
|
801
|
+
name: "addonId",
|
|
802
|
+
type: "TEXT"
|
|
803
|
+
}),
|
|
804
|
+
{
|
|
805
|
+
name: "classification",
|
|
806
|
+
type: "TEXT",
|
|
807
|
+
notNull: true
|
|
808
|
+
},
|
|
809
|
+
(
|
|
810
|
+
/** `ps pcpu` x 10. A LIFETIME average, not a rate — see `NodeProcess`. */
|
|
811
|
+
{
|
|
812
|
+
name: "cpuDeci",
|
|
813
|
+
type: "INTEGER",
|
|
814
|
+
notNull: true
|
|
815
|
+
}),
|
|
816
|
+
{
|
|
817
|
+
name: "rssMib",
|
|
818
|
+
type: "INTEGER",
|
|
819
|
+
notNull: true
|
|
820
|
+
},
|
|
821
|
+
(
|
|
822
|
+
/** Instantaneous main-thread CPU% x 10. `NULL` = UNKNOWN, never zero. */
|
|
823
|
+
{
|
|
824
|
+
name: "cpuMainDeci",
|
|
825
|
+
type: "INTEGER"
|
|
826
|
+
}),
|
|
827
|
+
(
|
|
828
|
+
/** Instantaneous V8-helper-pool CPU% x 10. `NULL` = UNKNOWN, never zero. */
|
|
829
|
+
{
|
|
830
|
+
name: "cpuGcDeci",
|
|
831
|
+
type: "INTEGER"
|
|
832
|
+
})
|
|
833
|
+
];
|
|
834
|
+
var LOAD_SERIES_INDEXES = [(
|
|
835
|
+
/**
|
|
836
|
+
* The ONE index, and it serves both jobs.
|
|
837
|
+
*
|
|
838
|
+
* Reads are always "this node, newer than T" — `nodeId` leads so the
|
|
839
|
+
* equality is a range scan and `atMs` supplies the order without a sort.
|
|
840
|
+
* Prunes are always "this node, oldest first" — the same index, walked from
|
|
841
|
+
* the other end. A second index on `atMs` alone was measured and rejected:
|
|
842
|
+
* it cost 23 B/row (196.0 → 172.8 with the two column drops) to serve a
|
|
843
|
+
* cross-node prune that a per-node loop over three nodes already serves.
|
|
844
|
+
*/
|
|
845
|
+
{
|
|
846
|
+
name: "idx_load_samples_node_at",
|
|
847
|
+
columns: ["nodeId", "atMs"]
|
|
848
|
+
})];
|
|
849
|
+
var BYTES_PER_MIB = 1048576;
|
|
850
|
+
/** Tenths → percent. `null` stays `null`: unknown is not zero. */
|
|
851
|
+
function fromDeci(value) {
|
|
852
|
+
return value === null ? null : Math.round(value) / 10;
|
|
853
|
+
}
|
|
854
|
+
/** Percent → tenths, lossless because both producers already round to tenths. */
|
|
855
|
+
function toDeci(value) {
|
|
856
|
+
return value === null ? null : Math.round(value * 10);
|
|
857
|
+
}
|
|
858
|
+
/** Storage row → the shape a reader gets. The ONE decode. */
|
|
859
|
+
function rowToProcess(row) {
|
|
860
|
+
return {
|
|
861
|
+
pid: row.pid,
|
|
862
|
+
addonId: row.addonId,
|
|
863
|
+
classification: row.classification,
|
|
864
|
+
cpuPercent: fromDeci(row.cpuDeci) ?? 0,
|
|
865
|
+
memoryRssBytes: row.rssMib * BYTES_PER_MIB,
|
|
866
|
+
cpuMainPercent: fromDeci(row.cpuMainDeci),
|
|
867
|
+
cpuGcPercent: fromDeci(row.cpuGcDeci)
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
/** The shape a reader gets → the storage row. The ONE encode. */
|
|
871
|
+
function processToRow(nodeId, atMs, process) {
|
|
872
|
+
return {
|
|
873
|
+
nodeId,
|
|
874
|
+
atMs,
|
|
875
|
+
pid: process.pid,
|
|
876
|
+
addonId: process.addonId,
|
|
877
|
+
classification: process.classification,
|
|
878
|
+
cpuDeci: toDeci(process.cpuPercent) ?? 0,
|
|
879
|
+
rssMib: Math.round(process.memoryRssBytes / BYTES_PER_MIB),
|
|
880
|
+
cpuMainDeci: toDeci(process.cpuMainPercent),
|
|
881
|
+
cpuGcDeci: toDeci(process.cpuGcPercent)
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
/** Group flat rows into samples, oldest-first. A sample is one `atMs`. */
|
|
885
|
+
function rowsToSamples(rows) {
|
|
886
|
+
const byAt = /* @__PURE__ */ new Map();
|
|
887
|
+
for (const row of rows) {
|
|
888
|
+
const held = byAt.get(row.atMs);
|
|
889
|
+
if (held === void 0) byAt.set(row.atMs, [rowToProcess(row)]);
|
|
890
|
+
else held.push(rowToProcess(row));
|
|
891
|
+
}
|
|
892
|
+
return [...byAt.entries()].toSorted((a, b) => a[0] - b[0]).map(([atMs, processes]) => ({
|
|
893
|
+
atMs,
|
|
894
|
+
processes
|
|
895
|
+
}));
|
|
896
|
+
}
|
|
897
|
+
/**
|
|
898
|
+
* The cold tier. Append-only, bounded, and never the authority on anything the
|
|
899
|
+
* hot ring can answer.
|
|
900
|
+
*
|
|
901
|
+
* Every method is best-effort and says so in the log rather than throwing at
|
|
902
|
+
* the emitter: a storage hiccup must cost a gap in a chart, never a dropped
|
|
903
|
+
* snapshot on the bus.
|
|
904
|
+
*/
|
|
905
|
+
var LoadSeriesStore = class {
|
|
906
|
+
declared = false;
|
|
907
|
+
lastPruneAtMs = 0;
|
|
908
|
+
store;
|
|
909
|
+
logger;
|
|
910
|
+
nowFn;
|
|
911
|
+
prunePageRows;
|
|
912
|
+
pruneIntervalMs;
|
|
913
|
+
constructor(deps) {
|
|
914
|
+
this.store = deps.store;
|
|
915
|
+
this.logger = deps.logger;
|
|
916
|
+
this.nowFn = deps.now ?? (() => Date.now());
|
|
917
|
+
this.prunePageRows = deps.prunePageRows ?? 2e4;
|
|
918
|
+
this.pruneIntervalMs = deps.pruneIntervalMs ?? 6e4;
|
|
919
|
+
}
|
|
920
|
+
/** Idempotently declare the collection. `false` when the store refused. */
|
|
921
|
+
async declare() {
|
|
922
|
+
if (this.declared) return true;
|
|
923
|
+
try {
|
|
924
|
+
await this.store.declareCollection.mutate({
|
|
925
|
+
collection: LOAD_SERIES_COLLECTION,
|
|
926
|
+
columns: [...LOAD_SERIES_COLUMNS],
|
|
927
|
+
indexes: LOAD_SERIES_INDEXES.map((i) => ({
|
|
928
|
+
name: i.name,
|
|
929
|
+
columns: [...i.columns]
|
|
930
|
+
}))
|
|
931
|
+
});
|
|
932
|
+
this.declared = true;
|
|
933
|
+
return true;
|
|
934
|
+
} catch (err) {
|
|
935
|
+
this.logger.warn("load series declareCollection failed — nothing will be retained on disk", { meta: {
|
|
936
|
+
collection: LOAD_SERIES_COLLECTION,
|
|
937
|
+
error: errMsg(err)
|
|
938
|
+
} });
|
|
939
|
+
return false;
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
/**
|
|
943
|
+
* Append ONE sample — every process row of it — in ONE transaction.
|
|
944
|
+
*
|
|
945
|
+
* Never a write per row. `insertMany` exists for exactly this: at the 10 s
|
|
946
|
+
* default the fleet produces 7.6 rows/s, and 7.6 separate commits per second
|
|
947
|
+
* on the connection that also serves every cluster-wide configuration read
|
|
948
|
+
* is a constant load nobody asked for.
|
|
949
|
+
*/
|
|
950
|
+
async append(nodeId, atMs, processes) {
|
|
951
|
+
if (processes.length === 0) return 0;
|
|
952
|
+
if (!await this.declare()) return 0;
|
|
953
|
+
const records = processes.map((p) => ({ data: { ...processToRow(nodeId, atMs, p) } }));
|
|
954
|
+
try {
|
|
955
|
+
const { inserted } = await this.store.insertMany.mutate({
|
|
956
|
+
collection: LOAD_SERIES_COLLECTION,
|
|
957
|
+
records
|
|
958
|
+
});
|
|
959
|
+
return inserted;
|
|
960
|
+
} catch (err) {
|
|
961
|
+
this.logger.warn("load series sample not retained — this interval will be missing", { meta: {
|
|
962
|
+
nodeId,
|
|
963
|
+
atMs,
|
|
964
|
+
rows: processes.length,
|
|
965
|
+
error: errMsg(err)
|
|
966
|
+
} });
|
|
967
|
+
return 0;
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
/**
|
|
971
|
+
* Read one node's cold samples, oldest-first.
|
|
972
|
+
*
|
|
973
|
+
* `sinceMs` is EXCLUSIVE, matching the ring, so a caller passing the newest
|
|
974
|
+
* timestamp it holds gets only what it is missing. `limitRows` bounds the
|
|
975
|
+
* read in ROWS (not samples) because rows are what the query costs.
|
|
976
|
+
*/
|
|
977
|
+
async read(nodeId, sinceMs, limitRows) {
|
|
978
|
+
if (!await this.declare()) return [];
|
|
979
|
+
try {
|
|
980
|
+
const records = await this.store.query.query({
|
|
981
|
+
collection: LOAD_SERIES_COLLECTION,
|
|
982
|
+
filter: {
|
|
983
|
+
where: { nodeId },
|
|
984
|
+
whereBetween: { atMs: [sinceMs + 1, Number.MAX_SAFE_INTEGER] },
|
|
985
|
+
orderBy: {
|
|
986
|
+
field: "atMs",
|
|
987
|
+
direction: "asc"
|
|
988
|
+
},
|
|
989
|
+
limit: limitRows
|
|
990
|
+
}
|
|
991
|
+
});
|
|
992
|
+
const rows = [];
|
|
993
|
+
for (const record of records) {
|
|
994
|
+
const row = recordToRow(record.data);
|
|
995
|
+
if (row !== null) rows.push(row);
|
|
996
|
+
}
|
|
997
|
+
return rowsToSamples(rows);
|
|
998
|
+
} catch (err) {
|
|
999
|
+
this.logger.warn("load series cold read failed — answering from the hot window only", { meta: {
|
|
1000
|
+
nodeId,
|
|
1001
|
+
sinceMs,
|
|
1002
|
+
error: errMsg(err)
|
|
1003
|
+
} });
|
|
1004
|
+
return [];
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Enforce BOTH bounds, oldest-first, through a bounded page each.
|
|
1009
|
+
*
|
|
1010
|
+
* Rate-limited to {@link pruneIntervalMs}: a bound is not a deadline, and the
|
|
1011
|
+
* append path must not pay a sweep on every sample.
|
|
1012
|
+
*/
|
|
1013
|
+
async prune(nodeIds, retention, force = false) {
|
|
1014
|
+
const now = this.nowFn();
|
|
1015
|
+
if (!force && now - this.lastPruneAtMs < this.pruneIntervalMs) return null;
|
|
1016
|
+
this.lastPruneAtMs = now;
|
|
1017
|
+
if (!await this.declare()) return null;
|
|
1018
|
+
let deletedByAge = 0;
|
|
1019
|
+
let deletedByCap = 0;
|
|
1020
|
+
let rowsExamined = 0;
|
|
1021
|
+
const ageCutoff = now - retention.retentionHours * 36e5;
|
|
1022
|
+
for (const nodeId of nodeIds) {
|
|
1023
|
+
const outcome = await this.pruneNodeToCutoff(nodeId, ageCutoff);
|
|
1024
|
+
deletedByAge += outcome.deleted;
|
|
1025
|
+
rowsExamined += outcome.examined;
|
|
1026
|
+
}
|
|
1027
|
+
const total = await this.count();
|
|
1028
|
+
const excess = total === null ? 0 : total - retention.maxRows;
|
|
1029
|
+
if (excess > 0) {
|
|
1030
|
+
const outcome = await this.pruneOldestRows(nodeIds, excess);
|
|
1031
|
+
deletedByCap = outcome.deleted;
|
|
1032
|
+
rowsExamined += outcome.examined;
|
|
1033
|
+
this.logger.warn("load series ROW CAP bit — evicting the oldest samples", { meta: {
|
|
1034
|
+
collection: LOAD_SERIES_COLLECTION,
|
|
1035
|
+
rows: total,
|
|
1036
|
+
cap: retention.maxRows,
|
|
1037
|
+
over: excess,
|
|
1038
|
+
deleted: outcome.deleted,
|
|
1039
|
+
retentionHours: retention.retentionHours,
|
|
1040
|
+
hint: "lower the retention or the sampling cadence — the cap is the guarantee, not the intention"
|
|
1041
|
+
} });
|
|
1042
|
+
}
|
|
1043
|
+
return {
|
|
1044
|
+
deletedByAge,
|
|
1045
|
+
deletedByCap,
|
|
1046
|
+
rowsExamined,
|
|
1047
|
+
capBit: excess > 0
|
|
1048
|
+
};
|
|
1049
|
+
}
|
|
1050
|
+
/** Total rows, or `null` when the store could not answer. */
|
|
1051
|
+
async count() {
|
|
1052
|
+
if (!await this.declare()) return null;
|
|
1053
|
+
try {
|
|
1054
|
+
return await this.store.count.query({ collection: LOAD_SERIES_COLLECTION });
|
|
1055
|
+
} catch (err) {
|
|
1056
|
+
this.logger.warn("load series count failed — the row cap is not enforced this pass", { meta: { error: errMsg(err) } });
|
|
1057
|
+
return null;
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
/**
|
|
1061
|
+
* Delete this node's rows older than `cutoff`, at most one page's worth.
|
|
1062
|
+
*
|
|
1063
|
+
* The page is the whole point. `deleteWhere({ atMs: [0, cutoff] })` on its
|
|
1064
|
+
* own is one statement but an UNBOUNDED one — a first pass after a retention
|
|
1065
|
+
* change would delete millions of rows inside a single stalling transaction.
|
|
1066
|
+
* So the page is read first (keys only, ordered by the index), its last
|
|
1067
|
+
* `atMs` becomes the EFFECTIVE cutoff, and the delete is bounded by it.
|
|
1068
|
+
*/
|
|
1069
|
+
async pruneNodeToCutoff(nodeId, cutoff) {
|
|
1070
|
+
try {
|
|
1071
|
+
const page = await this.store.query.query({
|
|
1072
|
+
collection: LOAD_SERIES_COLLECTION,
|
|
1073
|
+
filter: {
|
|
1074
|
+
where: { nodeId },
|
|
1075
|
+
whereBetween: { atMs: [0, cutoff] },
|
|
1076
|
+
orderBy: {
|
|
1077
|
+
field: "atMs",
|
|
1078
|
+
direction: "asc"
|
|
1079
|
+
},
|
|
1080
|
+
limit: this.prunePageRows
|
|
1081
|
+
},
|
|
1082
|
+
columns: ["atMs"]
|
|
1083
|
+
});
|
|
1084
|
+
if (page.length === 0) return {
|
|
1085
|
+
deleted: 0,
|
|
1086
|
+
examined: 0
|
|
1087
|
+
};
|
|
1088
|
+
const effectiveCutoff = Number(page.at(-1)?.data["atMs"]);
|
|
1089
|
+
if (!Number.isFinite(effectiveCutoff)) return {
|
|
1090
|
+
deleted: 0,
|
|
1091
|
+
examined: page.length
|
|
1092
|
+
};
|
|
1093
|
+
const { deleted } = await this.store.deleteWhere.mutate({
|
|
1094
|
+
collection: LOAD_SERIES_COLLECTION,
|
|
1095
|
+
filter: {
|
|
1096
|
+
where: { nodeId },
|
|
1097
|
+
whereBetween: { atMs: [0, effectiveCutoff] }
|
|
1098
|
+
}
|
|
1099
|
+
});
|
|
1100
|
+
return {
|
|
1101
|
+
deleted,
|
|
1102
|
+
examined: page.length
|
|
1103
|
+
};
|
|
1104
|
+
} catch (err) {
|
|
1105
|
+
this.logger.warn("load series age prune failed — the table keeps growing this pass", { meta: {
|
|
1106
|
+
nodeId,
|
|
1107
|
+
cutoff,
|
|
1108
|
+
error: errMsg(err)
|
|
1109
|
+
} });
|
|
1110
|
+
return {
|
|
1111
|
+
deleted: 0,
|
|
1112
|
+
examined: 0
|
|
1113
|
+
};
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
/**
|
|
1117
|
+
* Drop the oldest rows across the known nodes until `excess` is covered.
|
|
1118
|
+
*
|
|
1119
|
+
* Same bounded-page technique, walked per node so the one index serves it.
|
|
1120
|
+
* The node holding the oldest rows pays first, which is also the node
|
|
1121
|
+
* producing the pressure when a runaway process count is the cause.
|
|
1122
|
+
*/
|
|
1123
|
+
async pruneOldestRows(nodeIds, excess) {
|
|
1124
|
+
let remaining = Math.min(excess, this.prunePageRows);
|
|
1125
|
+
let deleted = 0;
|
|
1126
|
+
let examined = 0;
|
|
1127
|
+
for (const nodeId of nodeIds) {
|
|
1128
|
+
if (remaining <= 0) break;
|
|
1129
|
+
try {
|
|
1130
|
+
const page = await this.store.query.query({
|
|
1131
|
+
collection: LOAD_SERIES_COLLECTION,
|
|
1132
|
+
filter: {
|
|
1133
|
+
where: { nodeId },
|
|
1134
|
+
orderBy: {
|
|
1135
|
+
field: "atMs",
|
|
1136
|
+
direction: "asc"
|
|
1137
|
+
},
|
|
1138
|
+
limit: remaining
|
|
1139
|
+
},
|
|
1140
|
+
columns: ["atMs"]
|
|
1141
|
+
});
|
|
1142
|
+
examined += page.length;
|
|
1143
|
+
if (page.length === 0) continue;
|
|
1144
|
+
const cutoff = Number(page.at(-1)?.data["atMs"]);
|
|
1145
|
+
if (!Number.isFinite(cutoff)) continue;
|
|
1146
|
+
const result = await this.store.deleteWhere.mutate({
|
|
1147
|
+
collection: LOAD_SERIES_COLLECTION,
|
|
1148
|
+
filter: {
|
|
1149
|
+
where: { nodeId },
|
|
1150
|
+
whereBetween: { atMs: [0, cutoff] }
|
|
1151
|
+
}
|
|
1152
|
+
});
|
|
1153
|
+
deleted += result.deleted;
|
|
1154
|
+
remaining -= result.deleted;
|
|
1155
|
+
} catch (err) {
|
|
1156
|
+
this.logger.warn("load series cap prune failed — the cap is not enforced this pass", { meta: {
|
|
1157
|
+
nodeId,
|
|
1158
|
+
error: errMsg(err)
|
|
1159
|
+
} });
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
return {
|
|
1163
|
+
deleted,
|
|
1164
|
+
examined
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
};
|
|
1168
|
+
/** Structural validation on read. A malformed row is skipped, never charted. */
|
|
1169
|
+
function recordToRow(data) {
|
|
1170
|
+
const nodeId = data["nodeId"];
|
|
1171
|
+
const classification = data["classification"];
|
|
1172
|
+
if (typeof nodeId !== "string" || typeof classification !== "string") return null;
|
|
1173
|
+
const atMs = Number(data["atMs"]);
|
|
1174
|
+
const pid = Number(data["pid"]);
|
|
1175
|
+
const cpuDeci = Number(data["cpuDeci"]);
|
|
1176
|
+
const rssMib = Number(data["rssMib"]);
|
|
1177
|
+
if (![
|
|
1178
|
+
atMs,
|
|
1179
|
+
pid,
|
|
1180
|
+
cpuDeci,
|
|
1181
|
+
rssMib
|
|
1182
|
+
].every((n) => Number.isFinite(n))) return null;
|
|
1183
|
+
const rawAddon = data["addonId"];
|
|
1184
|
+
const optional = (raw) => {
|
|
1185
|
+
if (raw === null || raw === void 0) return null;
|
|
1186
|
+
const n = Number(raw);
|
|
1187
|
+
return Number.isFinite(n) ? n : null;
|
|
1188
|
+
};
|
|
1189
|
+
return {
|
|
1190
|
+
nodeId,
|
|
1191
|
+
atMs,
|
|
1192
|
+
pid,
|
|
1193
|
+
addonId: typeof rawAddon === "string" ? rawAddon : null,
|
|
1194
|
+
classification,
|
|
1195
|
+
cpuDeci,
|
|
1196
|
+
rssMib,
|
|
1197
|
+
cpuMainDeci: optional(data["cpuMainDeci"]),
|
|
1198
|
+
cpuGcDeci: optional(data["cpuGcDeci"])
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
1201
|
+
//#endregion
|
|
1202
|
+
//#region src/builtins/native-metrics/load-series-config.ts
|
|
1203
|
+
/**
|
|
1204
|
+
* The two knobs of the durable load series, and the arithmetic that makes them
|
|
1205
|
+
* honest.
|
|
1206
|
+
*
|
|
1207
|
+
* ## Where they live, and why there
|
|
1208
|
+
*
|
|
1209
|
+
* In `native-metrics`'s OWN global settings document — the one that already
|
|
1210
|
+
* exists and already carries `samplingIntervalMs`. Fields on a document, never
|
|
1211
|
+
* a method per knob: `system.getLoggingSettings` / `setLoggingSettings` set
|
|
1212
|
+
* that precedent and it holds here.
|
|
1213
|
+
*
|
|
1214
|
+
* They are NOT in the logging document, and that is a decision rather than an
|
|
1215
|
+
* omission. That document is about levels and diagnostic windows; a metrics
|
|
1216
|
+
* cadence and a table's retention are neither. Two documents that both claim a
|
|
1217
|
+
* knob is how this repo has already shipped a switch nobody read.
|
|
1218
|
+
*
|
|
1219
|
+
* They are cluster-wide, not per-node. Every node emits on the same cadence or
|
|
1220
|
+
* the fleet's series cannot be laid over each other, and the table is single
|
|
1221
|
+
* and hub-resident, so a per-node retention would be a promise nothing could
|
|
1222
|
+
* keep.
|
|
1223
|
+
*
|
|
1224
|
+
* ## Refused, not clamped
|
|
1225
|
+
*
|
|
1226
|
+
* A cadence outside 5-60 s is REFUSED. Clamping would silently store 10 when
|
|
1227
|
+
* the operator typed 2 and then report 10 back as though it had been accepted
|
|
1228
|
+
* — the failure mode where a knob and its readback agree on a value nobody
|
|
1229
|
+
* chose. The schema advertises the bounds; this refuses anything else.
|
|
1230
|
+
*/
|
|
1231
|
+
/** Why a value was refused — the message an operator is shown verbatim. */
|
|
1232
|
+
var LoadSeriesConfigError = class extends Error {
|
|
1233
|
+
constructor(message) {
|
|
1234
|
+
super(message);
|
|
1235
|
+
this.name = "LoadSeriesConfigError";
|
|
1236
|
+
}
|
|
1237
|
+
};
|
|
1238
|
+
function requireInteger(value, field) {
|
|
1239
|
+
const n = typeof value === "number" ? value : Number(value);
|
|
1240
|
+
if (!Number.isFinite(n)) throw new LoadSeriesConfigError(`${field} must be a number, received ${String(value)}`);
|
|
1241
|
+
return Math.round(n);
|
|
1242
|
+
}
|
|
1243
|
+
/**
|
|
1244
|
+
* Resolve the configuration, REFUSING anything out of range.
|
|
1245
|
+
*
|
|
1246
|
+
* An ABSENT field takes its default — that is not a refusal, it is a document
|
|
1247
|
+
* that has never been written. A PRESENT field outside its bounds throws.
|
|
1248
|
+
*/
|
|
1249
|
+
function resolveLoadSeriesConfig(raw) {
|
|
1250
|
+
let cadenceSec = 10;
|
|
1251
|
+
if (raw.loadSeriesCadenceSec !== void 0 && raw.loadSeriesCadenceSec !== null) {
|
|
1252
|
+
cadenceSec = requireInteger(raw.loadSeriesCadenceSec, "loadSeriesCadenceSec");
|
|
1253
|
+
if (cadenceSec < 5 || cadenceSec > 60) throw new LoadSeriesConfigError(`load series cadence must be between 5 and 60 seconds — refused ${cadenceSec}`);
|
|
1254
|
+
}
|
|
1255
|
+
let retentionHours = 6;
|
|
1256
|
+
if (raw.loadSeriesRetentionHours !== void 0 && raw.loadSeriesRetentionHours !== null) {
|
|
1257
|
+
retentionHours = requireInteger(raw.loadSeriesRetentionHours, "loadSeriesRetentionHours");
|
|
1258
|
+
if (retentionHours < 1 || retentionHours > 72) throw new LoadSeriesConfigError(`load series retention must be between 1 and 72 hours — refused ${retentionHours}`);
|
|
1259
|
+
}
|
|
1260
|
+
return {
|
|
1261
|
+
cadenceSec,
|
|
1262
|
+
retentionHours,
|
|
1263
|
+
maxRows: DEFAULT_MAX_ROWS
|
|
1264
|
+
};
|
|
1265
|
+
}
|
|
1266
|
+
/**
|
|
1267
|
+
* Project the cost of a configuration from the numbers the cluster is actually
|
|
1268
|
+
* reporting — nodes and processes OBSERVED, never a constant.
|
|
1269
|
+
*
|
|
1270
|
+
* This is what makes the knob honest: an operator raising retention to three
|
|
1271
|
+
* days sees the row count and the megabytes BEFORE applying it, rather than
|
|
1272
|
+
* discovering them afterwards. `MEASURED_BYTES_PER_ROW` is measured, not
|
|
1273
|
+
* guessed — see `load-series-store.ts`.
|
|
1274
|
+
*/
|
|
1275
|
+
function projectLoadSeriesCost(input) {
|
|
1276
|
+
const samplesPerNodeWindow = input.config.retentionHours * 3600 / input.config.cadenceSec;
|
|
1277
|
+
const intendedRows = Math.round(samplesPerNodeWindow * Math.max(input.observedProcessRows, 0));
|
|
1278
|
+
const boundedRows = Math.min(intendedRows, input.config.maxRows);
|
|
1279
|
+
return {
|
|
1280
|
+
intendedRows,
|
|
1281
|
+
boundedRows,
|
|
1282
|
+
estimatedBytes: boundedRows * 87,
|
|
1283
|
+
capBites: intendedRows > input.config.maxRows
|
|
1284
|
+
};
|
|
1285
|
+
}
|
|
1286
|
+
/** Human-readable one-liner for the settings form. */
|
|
1287
|
+
function describeLoadSeriesCost(projection) {
|
|
1288
|
+
const mib = (projection.estimatedBytes / 1048576).toFixed(1);
|
|
1289
|
+
const rows = projection.intendedRows.toLocaleString("en-US");
|
|
1290
|
+
const capped = projection.boundedRows.toLocaleString("en-US");
|
|
1291
|
+
if (!projection.capBites) return `This configuration retains ~${rows} rows ≈ ${mib} MiB on the hub database (NVMe cache), measured at 87 bytes per row.`;
|
|
1292
|
+
return `This configuration WANTS ~${rows} rows, above the ${projection.boundedRows.toLocaleString("en-US")}-row hard cap. The table will hold ~${capped} rows ≈ ${mib} MiB and the OLDEST samples will be evicted, so the effective window is shorter than the retention you asked for.`;
|
|
1293
|
+
}
|
|
1294
|
+
//#endregion
|
|
1295
|
+
//#region src/builtins/native-metrics/load-series-reader.ts
|
|
1296
|
+
/**
|
|
1297
|
+
* The ONE reader over both retention tiers.
|
|
1298
|
+
*
|
|
1299
|
+
* The hot window is the in-memory ring; the cold one is
|
|
1300
|
+
* `metrics:node-load-samples`. A caller never learns which tier a point came
|
|
1301
|
+
* from, and that is a hard requirement rather than a nicety: a second reader
|
|
1302
|
+
* is how two charts start disagreeing about the same node, and it is how "the
|
|
1303
|
+
* ring says X, the table says Y" becomes a question nobody can answer.
|
|
1304
|
+
*
|
|
1305
|
+
* Three things happen here, in this order, and each one is load-bearing:
|
|
1306
|
+
*
|
|
1307
|
+
* 1. **Merge and dedupe on `atMs`.** The two tiers OVERLAP by construction —
|
|
1308
|
+
* the ring holds the last 180 snapshots and the table holds every snapshot
|
|
1309
|
+
* the ring accepted, including those same 180. A snapshot present in both
|
|
1310
|
+
* must contribute exactly once. Identity is `atMs`, because one node emits
|
|
1311
|
+
* one snapshot per tick and a timestamp already held IS that snapshot. The
|
|
1312
|
+
* HOT copy wins on a tie: it is the payload as it arrived, while the cold
|
|
1313
|
+
* one has been through the storage encode.
|
|
1314
|
+
* 2. **Fold per function.** The chart draws one series per addon, so the fold
|
|
1315
|
+
* happens here rather than in the browser.
|
|
1316
|
+
* 3. **Reduce, preserving the extremes.** See `load-series-fold.ts` for why
|
|
1317
|
+
* the representative of a bucket is its MAX and never its mean.
|
|
1318
|
+
*
|
|
1319
|
+
* A GAP stays a gap through all three. Emission is fixed-cadence, so a missing
|
|
1320
|
+
* interval means one thing — nobody reported — and nothing here fills it in.
|
|
1321
|
+
*/
|
|
1322
|
+
/**
|
|
1323
|
+
* Merge the two tiers into one deduped, oldest-first list of samples.
|
|
1324
|
+
*
|
|
1325
|
+
* Exported because the dedup is the property worth testing on its own: a
|
|
1326
|
+
* doubled point is invisible in a total and obvious only as a chart that reads
|
|
1327
|
+
* twice as loud as the machine.
|
|
1328
|
+
*/
|
|
1329
|
+
function mergeSamples(cold, hot) {
|
|
1330
|
+
const byAt = /* @__PURE__ */ new Map();
|
|
1331
|
+
for (const sample of cold) byAt.set(sample.atMs, sample);
|
|
1332
|
+
for (const sample of hot) byAt.set(sample.atMs, sample);
|
|
1333
|
+
return [...byAt.values()].toSorted((a, b) => a.atMs - b.atMs);
|
|
1334
|
+
}
|
|
1335
|
+
/** `RetainedProcess` is already the fold's row shape — named, not re-derived. */
|
|
1336
|
+
function toFoldRows(processes) {
|
|
1337
|
+
return processes;
|
|
1338
|
+
}
|
|
1339
|
+
/**
|
|
1340
|
+
* Merge, fold and reduce. The whole read path in one pure function, so every
|
|
1341
|
+
* property that matters — dedup, peak survival, an empty bucket staying empty
|
|
1342
|
+
* — is testable without a database or a ring.
|
|
1343
|
+
*/
|
|
1344
|
+
function mergeLoadSeries(input) {
|
|
1345
|
+
const samples = mergeSamples(input.cold, input.hot);
|
|
1346
|
+
if (samples.length === 0) return {
|
|
1347
|
+
series: [],
|
|
1348
|
+
bucketMs: input.cadenceMs,
|
|
1349
|
+
retainedSamples: 0,
|
|
1350
|
+
oldestAtMs: null
|
|
1351
|
+
};
|
|
1352
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
1353
|
+
for (const sample of samples) for (const bucket of foldSnapshotByFunction(toFoldRows(sample.processes), sample.atMs)) {
|
|
1354
|
+
const held = byKey.get(bucket.key);
|
|
1355
|
+
if (held === void 0) byKey.set(bucket.key, {
|
|
1356
|
+
kind: bucket.kind,
|
|
1357
|
+
points: [bucket.point]
|
|
1358
|
+
});
|
|
1359
|
+
else held.points.push(bucket.point);
|
|
1360
|
+
}
|
|
1361
|
+
const oldestAtMs = samples[0]?.atMs ?? null;
|
|
1362
|
+
const newestAtMs = samples.at(-1)?.atMs ?? oldestAtMs;
|
|
1363
|
+
const spanMs = oldestAtMs === null || newestAtMs === null ? 0 : newestAtMs - oldestAtMs + input.cadenceMs;
|
|
1364
|
+
const bucketMs = input.maxPoints === void 0 ? input.cadenceMs : resolveBucketMs(spanMs, input.cadenceMs, input.maxPoints);
|
|
1365
|
+
const origin = oldestAtMs ?? 0;
|
|
1366
|
+
const series = [...byKey.entries()].map(([key, held]) => ({
|
|
1367
|
+
key,
|
|
1368
|
+
kind: held.kind,
|
|
1369
|
+
points: bucketMs > input.cadenceMs ? reducePoints(held.points, bucketMs, origin) : held.points
|
|
1370
|
+
}));
|
|
1371
|
+
const weight = (s) => {
|
|
1372
|
+
const last = s.points.at(-1);
|
|
1373
|
+
if (last === void 0) return -1;
|
|
1374
|
+
if (last.cpuMainPercent !== null && last.cpuGcPercent !== null) return last.cpuMainPercent + last.cpuGcPercent;
|
|
1375
|
+
return last.cpuLifetimePercent;
|
|
1376
|
+
};
|
|
1377
|
+
return {
|
|
1378
|
+
series: series.toSorted((a, b) => {
|
|
1379
|
+
const d = weight(b) - weight(a);
|
|
1380
|
+
return d !== 0 ? d : a.key.localeCompare(b.key);
|
|
1381
|
+
}),
|
|
1382
|
+
bucketMs,
|
|
1383
|
+
retainedSamples: samples.length,
|
|
1384
|
+
oldestAtMs
|
|
1385
|
+
};
|
|
1386
|
+
}
|
|
1387
|
+
//#endregion
|
|
1388
|
+
//#region src/builtins/native-metrics/process-classification.ts
|
|
464
1389
|
/**
|
|
465
1390
|
* Regex matching camstack-related command lines. We surface anything with
|
|
466
1391
|
* an obvious camstack fingerprint so operators can spot stray inference
|
|
@@ -472,23 +1397,243 @@ var execFileAsync = promisify(execFile);
|
|
|
472
1397
|
*/
|
|
473
1398
|
var CAMSTACK_CMD_RE = /(camstack|tsx\s+watch\s.*launcher\.ts|packages\/agent\/dist\/cli\.js|inference_pool\.py|bench-(inference-pool|nodeav)|node .*\/packages\/)/;
|
|
474
1399
|
/**
|
|
475
|
-
*
|
|
476
|
-
*
|
|
477
|
-
*
|
|
478
|
-
*
|
|
479
|
-
*
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
1400
|
+
* Classify one pid.
|
|
1401
|
+
*
|
|
1402
|
+
* Identity-driven, not ancestry-driven: a process is `root` because it IS
|
|
1403
|
+
* this node, `managed` because the kernel named it, and `system` otherwise.
|
|
1404
|
+
* There is no third source of truth to consult and no walk to get wrong.
|
|
1405
|
+
*/
|
|
1406
|
+
function classifyProcess(pid, selfPid, managed) {
|
|
1407
|
+
if (pid === selfPid) return "root";
|
|
1408
|
+
if (managed.has(pid)) return "managed";
|
|
1409
|
+
return "system";
|
|
1410
|
+
}
|
|
1411
|
+
/** Turn a `ps` snapshot into the node's process table. */
|
|
1412
|
+
function buildNodeProcesses(input) {
|
|
1413
|
+
const out = [];
|
|
1414
|
+
for (const p of input.psRows) {
|
|
1415
|
+
if (!CAMSTACK_CMD_RE.test(p.command)) continue;
|
|
1416
|
+
const managed = input.managed.get(p.pid);
|
|
1417
|
+
const split = input.cpuSplits.get(p.pid);
|
|
1418
|
+
const classification = classifyProcess(p.pid, input.selfPid, input.managed);
|
|
1419
|
+
out.push({
|
|
1420
|
+
pid: p.pid,
|
|
1421
|
+
ppid: p.ppid,
|
|
1422
|
+
pgid: p.pgid,
|
|
1423
|
+
classification,
|
|
1424
|
+
addonId: managed?.addonId ?? null,
|
|
1425
|
+
nodeId: managed?.nodeId ?? (p.pid === input.selfPid ? input.selfNodeId : null),
|
|
1426
|
+
command: p.command,
|
|
1427
|
+
cpuPercent: p.cpuPercent,
|
|
1428
|
+
memoryRssBytes: p.memoryRssBytes,
|
|
1429
|
+
cpuMainPercent: split?.mainPercent ?? null,
|
|
1430
|
+
cpuGcPercent: split?.gcPercent ?? null,
|
|
1431
|
+
threadCount: split?.threadCount ?? null,
|
|
1432
|
+
uptimeSec: p.uptimeSec
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
return out;
|
|
1436
|
+
}
|
|
1437
|
+
//#endregion
|
|
1438
|
+
//#region src/builtins/native-metrics/thread-cpu-sampler.ts
|
|
1439
|
+
/**
|
|
1440
|
+
* Per-process CPU split: the process's OWN threads vs V8's helper-thread pool.
|
|
1441
|
+
*
|
|
1442
|
+
* ## Why this exists
|
|
1443
|
+
*
|
|
1444
|
+
* `ps pcpu` — the number every CPU column in this app renders today — is the
|
|
1445
|
+
* process's CPU averaged over its **entire lifetime**. On a runner that has
|
|
1446
|
+
* been up for three days it barely moves, which makes it useless as a series:
|
|
1447
|
+
* a chart drawn from it is a flat line that cannot show a spike.
|
|
1448
|
+
*
|
|
1449
|
+
* Worse, a single total hides the finding that rewrote the 2026-08-27
|
|
1450
|
+
* diagnosis. Separating the V8 helper threads from the rest showed that ~2
|
|
1451
|
+
* whole cores of the container **were not computing** — hub-main was at 73%
|
|
1452
|
+
* and `stream-broker` at 61% of their CPU in that pool
|
|
1453
|
+
* (`docs/architecture/load-ledger.md`). A CPU chart that does not split the
|
|
1454
|
+
* two renders "busy" where the truth is "allocating too much".
|
|
1455
|
+
*
|
|
1456
|
+
* This module produces both at once, from the source the ledger prescribes:
|
|
1457
|
+
* a delta over `/proc/<pid>/task/<tid>/stat`, classified by thread name.
|
|
1458
|
+
*
|
|
1459
|
+
* ## What `gcPercent` actually contains — read this before quoting it
|
|
1460
|
+
*
|
|
1461
|
+
* It is the CPU of the threads V8 names `V8Worker`: its platform worker pool.
|
|
1462
|
+
* Concurrent marking and sweeping run there, and so does background
|
|
1463
|
+
* compilation. So `gcPercent` is **"the V8 helper pool"**, of which GC is the
|
|
1464
|
+
* dominant but not the only tenant. It is exactly the quantity the load ledger
|
|
1465
|
+
* measured — the 73% / 61% figures are this number — and it is reported under
|
|
1466
|
+
* that name for continuity, not because every jiffy in it is a collection.
|
|
1467
|
+
* The UI labels it `GC / V8 helpers` for the same reason.
|
|
1468
|
+
*
|
|
1469
|
+
* ## Cost
|
|
1470
|
+
*
|
|
1471
|
+
* One `readdir` plus one small `read` per thread, for the camstack-shaped pids
|
|
1472
|
+
* ONLY — on a busy node that is ~10 processes × ~10-40 threads. It rides the
|
|
1473
|
+
* existing 20 s process-snapshot tick and never adds a timer of its own, so it
|
|
1474
|
+
* is strictly cheaper than the full-process-table `ps -eo` scan that tick
|
|
1475
|
+
* already pays. There is no high-frequency sampler here; that is the defect
|
|
1476
|
+
* the load ledger documents.
|
|
1477
|
+
*
|
|
1478
|
+
* ## Linux only
|
|
1479
|
+
*
|
|
1480
|
+
* `/proc/<pid>/task` does not exist on macOS or Windows. Every read returns
|
|
1481
|
+
* `null` there, and `null` must reach the UI as "unknown" — never as zero.
|
|
1482
|
+
*/
|
|
1483
|
+
/**
|
|
1484
|
+
* Kernel jiffies per second (`USER_HZ`). `sysconf(_SC_CLK_TCK)` is not exposed
|
|
1485
|
+
* to Node, and this has been 100 on every Linux/x86-64 and arm64 kernel
|
|
1486
|
+
* configuration we ship to. It scales both numerator and denominator of the
|
|
1487
|
+
* ratio identically, so a wrong value would scale all CPU numbers by a
|
|
1488
|
+
* constant — visible immediately, not a silent skew.
|
|
1489
|
+
*/
|
|
1490
|
+
var CLOCK_TICKS_PER_SEC = 100;
|
|
1491
|
+
/**
|
|
1492
|
+
* Thread names V8 gives its platform worker pool. `comm` is capped at 15
|
|
1493
|
+
* characters by the kernel, which is why this matches a PREFIX.
|
|
1494
|
+
*/
|
|
1495
|
+
var V8_HELPER_THREAD_RE = /^V8Worker/;
|
|
1496
|
+
function parseThreadStat(line) {
|
|
1497
|
+
const close = line.lastIndexOf(")");
|
|
1498
|
+
const open = line.indexOf("(");
|
|
1499
|
+
if (close < 0 || open < 0 || close < open) return null;
|
|
1500
|
+
const comm = line.slice(open + 1, close);
|
|
1501
|
+
const rest = line.slice(close + 1).trim().split(/\s+/);
|
|
1502
|
+
const utime = Number(rest[11]);
|
|
1503
|
+
const stime = Number(rest[12]);
|
|
1504
|
+
const startTicks = Number(rest[19]);
|
|
1505
|
+
if (!Number.isFinite(utime) || !Number.isFinite(stime)) return null;
|
|
1506
|
+
if (!Number.isFinite(startTicks)) return null;
|
|
1507
|
+
return {
|
|
1508
|
+
comm,
|
|
1509
|
+
ticks: utime + stime,
|
|
1510
|
+
startTicks
|
|
1511
|
+
};
|
|
1512
|
+
}
|
|
1513
|
+
var nodeProcFsReader = {
|
|
1514
|
+
listTaskIds: (pid) => readdir(`/proc/${pid}/task`),
|
|
1515
|
+
readTaskStat: (pid, tid) => readFile(`/proc/${pid}/task/${tid}/stat`, "utf8")
|
|
1516
|
+
};
|
|
1517
|
+
/**
|
|
1518
|
+
* Read one instant's tick counts for `pid`, or `null` when the process has no
|
|
1519
|
+
* readable `/proc/<pid>/task` — it exited, we lack permission, or this is not
|
|
1520
|
+
* Linux. `null` is "unknown"; it must never be turned into a zero downstream.
|
|
1521
|
+
*/
|
|
1522
|
+
async function readThreadTicks(pid, reader = nodeProcFsReader) {
|
|
1523
|
+
let tids;
|
|
1524
|
+
try {
|
|
1525
|
+
tids = await reader.listTaskIds(pid);
|
|
1526
|
+
} catch {
|
|
1527
|
+
return null;
|
|
1528
|
+
}
|
|
1529
|
+
let mainTicks = 0;
|
|
1530
|
+
let gcTicks = 0;
|
|
1531
|
+
let threadCount = 0;
|
|
1532
|
+
let startTicks = null;
|
|
1533
|
+
const mainThreadTid = String(pid);
|
|
1534
|
+
for (const tid of tids) {
|
|
1535
|
+
let line;
|
|
1536
|
+
try {
|
|
1537
|
+
line = await reader.readTaskStat(pid, tid);
|
|
1538
|
+
} catch {
|
|
1539
|
+
continue;
|
|
1540
|
+
}
|
|
1541
|
+
const parsed = parseThreadStat(line);
|
|
1542
|
+
if (parsed === null) continue;
|
|
1543
|
+
threadCount += 1;
|
|
1544
|
+
if (tid === mainThreadTid) startTicks = parsed.startTicks;
|
|
1545
|
+
if (V8_HELPER_THREAD_RE.test(parsed.comm)) gcTicks += parsed.ticks;
|
|
1546
|
+
else mainTicks += parsed.ticks;
|
|
1547
|
+
}
|
|
1548
|
+
if (threadCount === 0) return null;
|
|
1549
|
+
return {
|
|
1550
|
+
mainTicks,
|
|
1551
|
+
gcTicks,
|
|
1552
|
+
threadCount,
|
|
1553
|
+
atMs: Date.now(),
|
|
1554
|
+
startTicks
|
|
1555
|
+
};
|
|
1556
|
+
}
|
|
1557
|
+
/**
|
|
1558
|
+
* The CPU split between two samples of the SAME process.
|
|
1559
|
+
*
|
|
1560
|
+
* Returns `null` — never a zero — when the window cannot produce a rate:
|
|
1561
|
+
* - the two samples are not separated in time (no window to divide by);
|
|
1562
|
+
* - the pid's `starttime` changed, or is unknown on either side. A changed
|
|
1563
|
+
* starttime means the pid was RECYCLED: this is a different process that
|
|
1564
|
+
* happens to wear the same number. See below for why the tick check alone
|
|
1565
|
+
* does not catch it;
|
|
1566
|
+
* - either tick count went BACKWARDS, which is the same fault seen from the
|
|
1567
|
+
* other side. A negative delta divided by a window would render as a
|
|
1568
|
+
* plausible small number for the wrong process, and a plausible wrong
|
|
1569
|
+
* number is the failure this whole surface exists to avoid.
|
|
1570
|
+
*
|
|
1571
|
+
* ## Why `starttime` and not just the backwards-tick check
|
|
1572
|
+
*
|
|
1573
|
+
* The backwards check only fires when the NEW process has burned FEWER ticks
|
|
1574
|
+
* than the dead one had. A recycled pid landing on a busier process — an
|
|
1575
|
+
* ffmpeg that has been transcoding for a minute inheriting the pid of an
|
|
1576
|
+
* ffmpeg that idled — produces a large POSITIVE delta, which divides cleanly
|
|
1577
|
+
* and renders as a spike. That is not a hypothetical rounding concern now that
|
|
1578
|
+
* the process snapshot attributes each pid to a camera: it would charge one
|
|
1579
|
+
* camera for another camera's whole lifetime of CPU, in a chart built
|
|
1580
|
+
* specifically to answer "why is 617 worse than 615". `starttime` is the pid's
|
|
1581
|
+
* incarnation number and it catches both directions.
|
|
1582
|
+
*/
|
|
1583
|
+
function cpuSplitBetween(prev, next) {
|
|
1584
|
+
const windowMs = next.atMs - prev.atMs;
|
|
1585
|
+
if (windowMs <= 0) return null;
|
|
1586
|
+
if (prev.startTicks === null || next.startTicks === null) return null;
|
|
1587
|
+
if (prev.startTicks !== next.startTicks) return null;
|
|
1588
|
+
const mainDelta = next.mainTicks - prev.mainTicks;
|
|
1589
|
+
const gcDelta = next.gcTicks - prev.gcTicks;
|
|
1590
|
+
if (mainDelta < 0 || gcDelta < 0) return null;
|
|
1591
|
+
const windowTicks = windowMs / 1e3 * CLOCK_TICKS_PER_SEC;
|
|
1592
|
+
const pct = (delta) => Math.round(delta / windowTicks * 1e3) / 10;
|
|
1593
|
+
return {
|
|
1594
|
+
mainPercent: pct(mainDelta),
|
|
1595
|
+
gcPercent: pct(gcDelta),
|
|
1596
|
+
threadCount: next.threadCount
|
|
1597
|
+
};
|
|
1598
|
+
}
|
|
1599
|
+
/**
|
|
1600
|
+
* Keeps the previous tick sample per pid so the next process-snapshot tick can
|
|
1601
|
+
* turn it into a rate.
|
|
1602
|
+
*
|
|
1603
|
+
* Bounded by construction: {@link sample} replaces the whole map with only the
|
|
1604
|
+
* pids it was just asked about, so a pid that stops appearing in the scan is
|
|
1605
|
+
* dropped on the following tick rather than accumulating forever.
|
|
490
1606
|
*/
|
|
491
|
-
var
|
|
1607
|
+
var ThreadCpuTracker = class {
|
|
1608
|
+
reader;
|
|
1609
|
+
previous = /* @__PURE__ */ new Map();
|
|
1610
|
+
constructor(reader = nodeProcFsReader) {
|
|
1611
|
+
this.reader = reader;
|
|
1612
|
+
}
|
|
1613
|
+
/**
|
|
1614
|
+
* Sample `pids` and return the split for each one that HAS a usable previous
|
|
1615
|
+
* sample. A pid absent from the result has no answer yet — the caller must
|
|
1616
|
+
* report `null`, not `0`.
|
|
1617
|
+
*/
|
|
1618
|
+
async sample(pids) {
|
|
1619
|
+
const out = /* @__PURE__ */ new Map();
|
|
1620
|
+
const nextPrevious = /* @__PURE__ */ new Map();
|
|
1621
|
+
for (const pid of pids) {
|
|
1622
|
+
const next = await readThreadTicks(pid, this.reader);
|
|
1623
|
+
if (next === null) continue;
|
|
1624
|
+
nextPrevious.set(pid, next);
|
|
1625
|
+
const prev = this.previous.get(pid);
|
|
1626
|
+
if (prev === void 0) continue;
|
|
1627
|
+
const split = cpuSplitBetween(prev, next);
|
|
1628
|
+
if (split !== null) out.set(pid, split);
|
|
1629
|
+
}
|
|
1630
|
+
this.previous = nextPrevious;
|
|
1631
|
+
return out;
|
|
1632
|
+
}
|
|
1633
|
+
};
|
|
1634
|
+
//#endregion
|
|
1635
|
+
//#region src/builtins/native-metrics/native-metrics.addon.ts
|
|
1636
|
+
var execFileAsync = promisify(execFile);
|
|
492
1637
|
/**
|
|
493
1638
|
* Cadence for the per-node metric snapshot bus events. ~5 s matches
|
|
494
1639
|
* the existing UI polling cadence (ProcessesTab, ProcessResources)
|
|
@@ -497,18 +1642,20 @@ var SUPERVISOR_BOUNDARY_RE = /(tsx\s+watch\s.*launcher\.ts|packages\/agent\/dist
|
|
|
497
1642
|
*/
|
|
498
1643
|
var METRICS_SNAPSHOT_INTERVAL_MS = 5e3;
|
|
499
1644
|
/**
|
|
500
|
-
*
|
|
501
|
-
*
|
|
502
|
-
*
|
|
503
|
-
*
|
|
504
|
-
*
|
|
505
|
-
*
|
|
506
|
-
*
|
|
507
|
-
*
|
|
508
|
-
*
|
|
509
|
-
*
|
|
1645
|
+
* The process-tree snapshot is FIXED-CADENCE, and its cadence is the
|
|
1646
|
+
* operator's `loadSeriesCadenceSec` (5-60 s, 10 by default).
|
|
1647
|
+
*
|
|
1648
|
+
* It used to emit only ON A CHANGE, with a 60 s fallback heartbeat, and the
|
|
1649
|
+
* dedup is gone rather than tuned. A conditional emit makes a hole in the
|
|
1650
|
+
* series mean two different things — "nothing changed" and "nobody reported"
|
|
1651
|
+
* — and telling those apart is the entire reason an operator opens this chart
|
|
1652
|
+
* after a spike. Now a gap means exactly one thing.
|
|
1653
|
+
*
|
|
1654
|
+
* It is not free: the emit rate is deterministic, so the durable table's row
|
|
1655
|
+
* count is too (nodes x processes / cadence, no discount for a quiet fleet).
|
|
1656
|
+
* The measured cost of that is in `docs/architecture/load-ledger.md`, and it
|
|
1657
|
+
* is why the retention default is six hours rather than three days.
|
|
510
1658
|
*/
|
|
511
|
-
var PROCESS_SNAPSHOT_INTERVAL_MS = 2e4;
|
|
512
1659
|
/**
|
|
513
1660
|
* Force a metrics-snapshot emit at least every 60s even when the
|
|
514
1661
|
* coarsened payload looks unchanged. Without this an idle node
|
|
@@ -538,25 +1685,6 @@ function coarsenResourcesSnapshot(snapshot) {
|
|
|
538
1685
|
};
|
|
539
1686
|
return JSON.stringify(round(snapshot));
|
|
540
1687
|
}
|
|
541
|
-
/**
|
|
542
|
-
* Coarsen a process list before deduping — pid, addonId, state +
|
|
543
|
-
* bucketed cpu%. Drops cmdline / cwd / start time / uptime which
|
|
544
|
-
* change every tick on a process that's just running. The result
|
|
545
|
-
* is "process X has the same role and roughly the same load" =
|
|
546
|
-
* skip emit.
|
|
547
|
-
*/
|
|
548
|
-
function coarsenProcessList(processes) {
|
|
549
|
-
const summary = processes.filter((p) => !!p && typeof p === "object").map((p) => {
|
|
550
|
-
return [
|
|
551
|
-
p["pid"],
|
|
552
|
-
p["addonId"],
|
|
553
|
-
p["state"],
|
|
554
|
-
typeof p["cpuPercent"] === "number" ? Math.round(p["cpuPercent"] / 5) * 5 : null,
|
|
555
|
-
typeof p["memoryRss"] === "number" ? Math.round(p["memoryRss"] / (50 * 1024 * 1024)) : null
|
|
556
|
-
];
|
|
557
|
-
});
|
|
558
|
-
return JSON.stringify(summary);
|
|
559
|
-
}
|
|
560
1688
|
function narrowWorkerState(state) {
|
|
561
1689
|
switch (state) {
|
|
562
1690
|
case "starting":
|
|
@@ -571,7 +1699,7 @@ function narrowWorkerState(state) {
|
|
|
571
1699
|
* Native metrics — CPU, memory, disk usage sampling.
|
|
572
1700
|
* Settings appear under Cluster → NodeDetail → Settings.
|
|
573
1701
|
*/
|
|
574
|
-
var NativeMetricsAddon = class extends BaseAddon {
|
|
1702
|
+
var NativeMetricsAddon = class NativeMetricsAddon extends BaseAddon {
|
|
575
1703
|
provider = null;
|
|
576
1704
|
startedAtMs = Date.now();
|
|
577
1705
|
snapshotTimer = null;
|
|
@@ -583,9 +1711,77 @@ var NativeMetricsAddon = class extends BaseAddon {
|
|
|
583
1711
|
* elapsed) is skipped.
|
|
584
1712
|
*/
|
|
585
1713
|
lastResourcesEmit = null;
|
|
586
|
-
|
|
1714
|
+
/**
|
|
1715
|
+
* Holds the previous `/proc/<pid>/task/*` tick counts so each process scan
|
|
1716
|
+
* can turn them into an instantaneous CPU rate split between the process's
|
|
1717
|
+
* own threads and V8's helper pool. Self-bounding — see `ThreadCpuTracker`.
|
|
1718
|
+
*/
|
|
1719
|
+
threadCpu = new ThreadCpuTracker();
|
|
1720
|
+
/**
|
|
1721
|
+
* Retention for the snapshots this addon emits. Fed by SUBSCRIBING to
|
|
1722
|
+
* `metrics.node-processes-snapshot`, never by a second sampler — on the hub
|
|
1723
|
+
* that bus carries every node, so the hub's ring is the cluster's. Bounds and
|
|
1724
|
+
* measured cost live in `node-load-ring.ts`.
|
|
1725
|
+
*/
|
|
1726
|
+
loadRing = new NodeLoadRing();
|
|
1727
|
+
/**
|
|
1728
|
+
* The COLD tier, and it exists only on the HUB.
|
|
1729
|
+
*
|
|
1730
|
+
* The table is hub-resident and the hub already hears every node's snapshot
|
|
1731
|
+
* on its bus, so the hub's writer is the cluster's. An agent writing through
|
|
1732
|
+
* the `settings-store` singleton would ship its rows over Moleculer, once
|
|
1733
|
+
* per sample, to reach that very same table.
|
|
1734
|
+
*/
|
|
1735
|
+
loadStore = null;
|
|
1736
|
+
/** The resolved knobs. Re-resolved on every settings write. */
|
|
1737
|
+
loadConfig = {
|
|
1738
|
+
cadenceSec: 10,
|
|
1739
|
+
retentionHours: 6,
|
|
1740
|
+
maxRows: DEFAULT_MAX_ROWS
|
|
1741
|
+
};
|
|
1742
|
+
/**
|
|
1743
|
+
* Process rows seen in each node's most recent snapshot — the OBSERVED
|
|
1744
|
+
* numbers the settings form projects its cost from. A constant here would be
|
|
1745
|
+
* a projection that stops being true the first time the fleet changes.
|
|
1746
|
+
*/
|
|
1747
|
+
observedRowsByNode = /* @__PURE__ */ new Map();
|
|
587
1748
|
constructor() {
|
|
588
|
-
super({
|
|
1749
|
+
super({
|
|
1750
|
+
samplingIntervalMs: 5e3,
|
|
1751
|
+
loadSeriesCadenceSec: 10,
|
|
1752
|
+
loadSeriesRetentionHours: 6
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1755
|
+
/** Is this the hub? The same test every other addon uses (`addon-ai`). */
|
|
1756
|
+
get isHub() {
|
|
1757
|
+
return (this.ctx.kernel.cluster?.broker?.nodeID ?? "hub") === "hub";
|
|
1758
|
+
}
|
|
1759
|
+
/** Process rows observed across the whole fleet, for the cost projection. */
|
|
1760
|
+
observedFleetRows() {
|
|
1761
|
+
let total = 0;
|
|
1762
|
+
for (const rows of this.observedRowsByNode.values()) total += rows;
|
|
1763
|
+
return total;
|
|
1764
|
+
}
|
|
1765
|
+
/**
|
|
1766
|
+
* Project a `NodeProcess` onto the series' own row.
|
|
1767
|
+
*
|
|
1768
|
+
* `command`, `ppid`, `pgid`, `nodeId`, `threadCount` and `uptimeSec` are
|
|
1769
|
+
* dropped here, at the single point both tiers pass through, so the hot ring
|
|
1770
|
+
* and the cold table carry the SAME shape and the merged read cannot tell
|
|
1771
|
+
* them apart. `command` in particular is the fattest field in a snapshot and
|
|
1772
|
+
* the same identical string on every runner — the runner id travels in the
|
|
1773
|
+
* environment, not in argv — and nothing that draws this series reads it.
|
|
1774
|
+
*/
|
|
1775
|
+
static toRetained(process) {
|
|
1776
|
+
return {
|
|
1777
|
+
pid: process.pid,
|
|
1778
|
+
addonId: process.addonId,
|
|
1779
|
+
classification: process.classification,
|
|
1780
|
+
cpuPercent: process.cpuPercent,
|
|
1781
|
+
memoryRssBytes: process.memoryRssBytes,
|
|
1782
|
+
cpuMainPercent: process.cpuMainPercent,
|
|
1783
|
+
cpuGcPercent: process.cpuGcPercent
|
|
1784
|
+
};
|
|
589
1785
|
}
|
|
590
1786
|
async onInitialize() {
|
|
591
1787
|
const provider = new NativeMetricsProvider();
|
|
@@ -605,16 +1801,127 @@ var NativeMetricsAddon = class extends BaseAddon {
|
|
|
605
1801
|
listAddonInstances: () => this.listAddonInstances(),
|
|
606
1802
|
getAddonStats: (params) => this.getAddonStats(params.addonId),
|
|
607
1803
|
listNodeProcesses: () => this.listNodeProcesses(),
|
|
608
|
-
|
|
1804
|
+
getLoadSeries: (params) => this.readLoadSeries(params),
|
|
609
1805
|
dumpHeapSnapshot: (params) => this.dumpHeapSnapshot(params)
|
|
610
1806
|
};
|
|
1807
|
+
this.applyLoadSeriesConfig();
|
|
1808
|
+
if (this.isHub) this.loadStore = new LoadSeriesStore({
|
|
1809
|
+
store: this.ctx.api.settingsStore,
|
|
1810
|
+
logger: this.ctx.logger.child("LoadSeries")
|
|
1811
|
+
});
|
|
611
1812
|
this.snapshotTimer = setInterval(() => this.emitResourcesSnapshot(), METRICS_SNAPSHOT_INTERVAL_MS);
|
|
612
|
-
this.
|
|
1813
|
+
this.startProcessSnapshotTimer();
|
|
1814
|
+
this.ctx.addDisposer(this.ctx.eventBus.subscribe({ category: EventCategory.MetricsNodeProcessesSnapshot }, (event) => this.retainSnapshot(event.data.nodeId, event.data.timestamp, event.data.processes)));
|
|
613
1815
|
return [{
|
|
614
1816
|
capability: metricsProviderCapability,
|
|
615
1817
|
provider: composed
|
|
616
1818
|
}];
|
|
617
1819
|
}
|
|
1820
|
+
/**
|
|
1821
|
+
* Retain one arriving snapshot in BOTH tiers.
|
|
1822
|
+
*
|
|
1823
|
+
* The ring is written FIRST and it is the gate: `record` refuses a timestamp
|
|
1824
|
+
* this node has already delivered, and only an accepted sample reaches the
|
|
1825
|
+
* table. That is what makes the append idempotent without a unique index
|
|
1826
|
+
* over two million rows — the composite key it replaces was measured at ~40
|
|
1827
|
+
* bytes per row (`load-series-store.ts`).
|
|
1828
|
+
*
|
|
1829
|
+
* The durable append is fire-and-forget: a storage stall must cost a gap in
|
|
1830
|
+
* the cold window, never a blocked event-bus handler. Every failure logs.
|
|
1831
|
+
*/
|
|
1832
|
+
retainSnapshot(nodeId, atMs, processes) {
|
|
1833
|
+
const retained = processes.map((p) => NativeMetricsAddon.toRetained(p));
|
|
1834
|
+
if (!this.loadRing.record(nodeId, atMs, retained)) return;
|
|
1835
|
+
this.observedRowsByNode.set(nodeId, retained.length);
|
|
1836
|
+
const store = this.loadStore;
|
|
1837
|
+
if (store === null) return;
|
|
1838
|
+
store.append(nodeId, atMs, retained).then(() => store.prune([...this.observedRowsByNode.keys()], this.loadConfig)).catch((err) => {
|
|
1839
|
+
this.ctx.logger.warn("durable load series write failed", { meta: {
|
|
1840
|
+
nodeId,
|
|
1841
|
+
atMs,
|
|
1842
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1843
|
+
} });
|
|
1844
|
+
});
|
|
1845
|
+
}
|
|
1846
|
+
/**
|
|
1847
|
+
* The ONE reader, over both tiers.
|
|
1848
|
+
*
|
|
1849
|
+
* Cold first, hot second, merged and deduped on `atMs` — see
|
|
1850
|
+
* `load-series-reader.ts`. The cold read is bounded in ROWS, because rows
|
|
1851
|
+
* are what a query costs, and the fold + reduction happen HERE rather than
|
|
1852
|
+
* in the browser: six hours at the 10 s cadence is 2 160 snapshots, and
|
|
1853
|
+
* shipping them to a page that will discard most of them is precisely the
|
|
1854
|
+
* cost this subsystem exists to avoid.
|
|
1855
|
+
*/
|
|
1856
|
+
async readLoadSeries(params) {
|
|
1857
|
+
const cadenceMs = this.loadConfig.cadenceSec * 1e3;
|
|
1858
|
+
const sinceMs = params.sinceMs ?? 0;
|
|
1859
|
+
const hotSamples = this.loadRing.read(params.forNodeId, params.sinceMs).samples.map((sample) => ({
|
|
1860
|
+
atMs: sample.atMs,
|
|
1861
|
+
processes: sample.processes
|
|
1862
|
+
}));
|
|
1863
|
+
const store = this.loadStore;
|
|
1864
|
+
const merged = mergeLoadSeries({
|
|
1865
|
+
cold: store === null ? [] : await store.read(params.forNodeId, sinceMs, this.loadConfig.maxRows),
|
|
1866
|
+
hot: hotSamples,
|
|
1867
|
+
cadenceMs,
|
|
1868
|
+
...params.maxPoints !== void 0 ? { maxPoints: params.maxPoints } : {}
|
|
1869
|
+
});
|
|
1870
|
+
return {
|
|
1871
|
+
nodeId: params.forNodeId,
|
|
1872
|
+
series: merged.series,
|
|
1873
|
+
bucketMs: merged.bucketMs,
|
|
1874
|
+
retainedSamples: merged.retainedSamples,
|
|
1875
|
+
oldestAtMs: merged.oldestAtMs,
|
|
1876
|
+
cadenceMs,
|
|
1877
|
+
durable: store !== null
|
|
1878
|
+
};
|
|
1879
|
+
}
|
|
1880
|
+
/**
|
|
1881
|
+
* Re-resolve the knobs and restate the fixed cadence.
|
|
1882
|
+
*
|
|
1883
|
+
* A REFUSED value (out of 5-60 s, or out of 1-72 h) leaves the previous
|
|
1884
|
+
* configuration in force and says so. Refused, never clamped: storing 10
|
|
1885
|
+
* when the operator typed 2 and reading 10 back is a knob and a readback
|
|
1886
|
+
* agreeing on a value nobody chose.
|
|
1887
|
+
*/
|
|
1888
|
+
applyLoadSeriesConfig() {
|
|
1889
|
+
try {
|
|
1890
|
+
const next = resolveLoadSeriesConfig({
|
|
1891
|
+
loadSeriesCadenceSec: this.config.loadSeriesCadenceSec,
|
|
1892
|
+
loadSeriesRetentionHours: this.config.loadSeriesRetentionHours
|
|
1893
|
+
});
|
|
1894
|
+
const changed = next.cadenceSec !== this.loadConfig.cadenceSec || next.retentionHours !== this.loadConfig.retentionHours;
|
|
1895
|
+
this.loadConfig = next;
|
|
1896
|
+
if (changed) {
|
|
1897
|
+
const projection = projectLoadSeriesCost({
|
|
1898
|
+
config: next,
|
|
1899
|
+
observedProcessRows: this.observedFleetRows()
|
|
1900
|
+
});
|
|
1901
|
+
this.ctx.logger.info("load series configuration applied", { meta: {
|
|
1902
|
+
nodeId: this.ctx.kernel.localNodeId ?? this.ctx.id,
|
|
1903
|
+
cadenceSec: next.cadenceSec,
|
|
1904
|
+
retentionHours: next.retentionHours,
|
|
1905
|
+
maxRows: next.maxRows,
|
|
1906
|
+
projectedRows: projection.intendedRows,
|
|
1907
|
+
projectedMib: Math.round(projection.estimatedBytes / 1048576 * 10) / 10,
|
|
1908
|
+
capBites: projection.capBites
|
|
1909
|
+
} });
|
|
1910
|
+
}
|
|
1911
|
+
} catch (err) {
|
|
1912
|
+
this.ctx.logger.warn("load series configuration REFUSED — keeping the previous values", { meta: {
|
|
1913
|
+
nodeId: this.ctx.kernel.localNodeId ?? this.ctx.id,
|
|
1914
|
+
cadenceSec: this.loadConfig.cadenceSec,
|
|
1915
|
+
retentionHours: this.loadConfig.retentionHours,
|
|
1916
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1917
|
+
} });
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
/** (Re)arm the fixed-cadence process-tree timer at the configured interval. */
|
|
1921
|
+
startProcessSnapshotTimer() {
|
|
1922
|
+
if (this.processSnapshotTimer) clearInterval(this.processSnapshotTimer);
|
|
1923
|
+
this.processSnapshotTimer = setInterval(() => this.emitProcessesSnapshot(), this.loadConfig.cadenceSec * 1e3);
|
|
1924
|
+
}
|
|
618
1925
|
async onShutdown() {
|
|
619
1926
|
if (this.snapshotTimer) {
|
|
620
1927
|
clearInterval(this.snapshotTimer);
|
|
@@ -670,10 +1977,14 @@ var NativeMetricsAddon = class extends BaseAddon {
|
|
|
670
1977
|
} catch {}
|
|
671
1978
|
}
|
|
672
1979
|
/**
|
|
673
|
-
* Emit one `metrics.node-processes-snapshot` for this node.
|
|
674
|
-
*
|
|
675
|
-
*
|
|
676
|
-
*
|
|
1980
|
+
* Emit one `metrics.node-processes-snapshot` for this node.
|
|
1981
|
+
*
|
|
1982
|
+
* Heavy — a full OS `ps -eo` scan plus a `$process.list` broker call — and
|
|
1983
|
+
* UNCONDITIONAL. The change-detection that used to guard it is gone: with a
|
|
1984
|
+
* fixed cadence a missing interval means exactly one thing, and that is the
|
|
1985
|
+
* property an operator investigating a spike is actually looking for.
|
|
1986
|
+
*
|
|
1987
|
+
* A failed scan emits nothing, which is the same signal: nobody reported.
|
|
677
1988
|
*/
|
|
678
1989
|
async emitProcessesSnapshot() {
|
|
679
1990
|
const eventBus = this.ctx.eventBus;
|
|
@@ -682,29 +1993,27 @@ var NativeMetricsAddon = class extends BaseAddon {
|
|
|
682
1993
|
const timestamp = Date.now();
|
|
683
1994
|
try {
|
|
684
1995
|
const processes = await this.listNodeProcesses();
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
timestamp
|
|
701
|
-
}));
|
|
702
|
-
}
|
|
703
|
-
} catch {}
|
|
1996
|
+
eventBus.emit(createEvent(EventCategory.MetricsNodeProcessesSnapshot, {
|
|
1997
|
+
type: "node",
|
|
1998
|
+
id: nodeId,
|
|
1999
|
+
nodeId
|
|
2000
|
+
}, {
|
|
2001
|
+
nodeId,
|
|
2002
|
+
processes,
|
|
2003
|
+
timestamp
|
|
2004
|
+
}));
|
|
2005
|
+
} catch (err) {
|
|
2006
|
+
this.ctx.logger.warn("process snapshot skipped — this interval will be missing", { meta: {
|
|
2007
|
+
nodeId,
|
|
2008
|
+
error: err instanceof Error ? err.message : String(err)
|
|
2009
|
+
} });
|
|
2010
|
+
}
|
|
704
2011
|
}
|
|
705
2012
|
async onConfigChanged() {
|
|
706
2013
|
this.provider?.stopSampling();
|
|
707
2014
|
this.provider?.startSampling(this.config.samplingIntervalMs);
|
|
2015
|
+
this.applyLoadSeriesConfig();
|
|
2016
|
+
this.startProcessSnapshotTimer();
|
|
708
2017
|
}
|
|
709
2018
|
async listWorkerInstances() {
|
|
710
2019
|
const broker = this.ctx.kernel.cluster?.broker;
|
|
@@ -748,137 +2057,29 @@ var NativeMetricsAddon = class extends BaseAddon {
|
|
|
748
2057
|
/**
|
|
749
2058
|
* Walk the OS process table and classify each camstack-shaped process.
|
|
750
2059
|
*
|
|
751
|
-
* Classification
|
|
752
|
-
*
|
|
753
|
-
*
|
|
754
|
-
*
|
|
755
|
-
* - system — ancestry walk crosses a SUPERVISOR_BOUNDARY_RE match
|
|
756
|
-
* (tsx-watch launcher, agent CLI, concurrently, vite,
|
|
757
|
-
* npm exec wrapper). The process belongs to the dev
|
|
758
|
-
* tree even if not in `$process.list`. NEVER killable.
|
|
759
|
-
* - ghost — ancestry walk reaches `ppid=1` without crossing any
|
|
760
|
-
* supervisor boundary AND the parent isn't visible in
|
|
761
|
-
* `ps`. A truly orphaned camstack-shaped process. The
|
|
762
|
-
* ONLY classification that's eligible for kill.
|
|
763
|
-
*
|
|
764
|
-
* Old pattern-only ghost detection produced false positives: every
|
|
765
|
-
* monorepo-path process matched CAMSTACK_CMD_RE, ancestry walk
|
|
766
|
-
* stopping at ppid=hub returned false-positive ghosts whenever a
|
|
767
|
-
* concurrently sibling sat above hub. Ancestry-driven classification
|
|
768
|
-
* fixes that.
|
|
2060
|
+
* Classification is IDENTITY-driven: `root` is this pid, `managed` is a pid
|
|
2061
|
+
* the kernel's `$process.list` names, and every other camstack-shaped
|
|
2062
|
+
* process is `system`. The rules — and why there is no longer an ancestry
|
|
2063
|
+
* walk behind them — live in `process-classification.ts`.
|
|
769
2064
|
*/
|
|
770
2065
|
async listNodeProcesses() {
|
|
771
2066
|
const ps = await this.runPs();
|
|
772
2067
|
if (ps.length === 0) return [];
|
|
773
|
-
const
|
|
2068
|
+
const managed = /* @__PURE__ */ new Map();
|
|
774
2069
|
const workers = await this.listWorkerInstances();
|
|
775
|
-
for (const w of workers)
|
|
2070
|
+
for (const w of workers) managed.set(w.pid, {
|
|
776
2071
|
addonId: w.addonId,
|
|
777
2072
|
nodeId: w.nodeId
|
|
778
2073
|
});
|
|
779
|
-
const
|
|
780
|
-
const
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
2074
|
+
const camstackPids = ps.filter((p) => CAMSTACK_CMD_RE.test(p.command)).map((p) => p.pid);
|
|
2075
|
+
const cpuSplits = await this.threadCpu.sample(camstackPids);
|
|
2076
|
+
return buildNodeProcesses({
|
|
2077
|
+
psRows: ps,
|
|
2078
|
+
selfPid: process.pid,
|
|
2079
|
+
selfNodeId: this.ctx.kernel.cluster?.broker?.nodeID ?? "hub",
|
|
2080
|
+
managed,
|
|
2081
|
+
cpuSplits
|
|
785
2082
|
});
|
|
786
|
-
/**
|
|
787
|
-
* Walk ancestry from `pid` up to ppid=1 (or until we leave the ps
|
|
788
|
-
* snapshot). Returns:
|
|
789
|
-
* - 'system' — crossed a SUPERVISOR_BOUNDARY_RE ancestor
|
|
790
|
-
* - 'ghost' — reached ppid=1 OR the parent isn't visible in ps
|
|
791
|
-
* without crossing a boundary
|
|
792
|
-
*
|
|
793
|
-
* `selfPid` itself is treated as a boundary (this node's own root).
|
|
794
|
-
*/
|
|
795
|
-
const classifyByAncestry = (startPid) => {
|
|
796
|
-
let cur = startPid;
|
|
797
|
-
for (let depth = 0; depth < 32; depth++) {
|
|
798
|
-
const node = psIndex.get(cur);
|
|
799
|
-
if (!node) return "ghost";
|
|
800
|
-
if (cur === selfPid) return "system";
|
|
801
|
-
if (SUPERVISOR_BOUNDARY_RE.test(node.command)) return "system";
|
|
802
|
-
if (node.ppid === 1) return "ghost";
|
|
803
|
-
if (node.ppid === selfPid) return "system";
|
|
804
|
-
cur = node.ppid;
|
|
805
|
-
}
|
|
806
|
-
return "system";
|
|
807
|
-
};
|
|
808
|
-
const out = [];
|
|
809
|
-
for (const p of ps) {
|
|
810
|
-
if (!CAMSTACK_CMD_RE.test(p.command)) continue;
|
|
811
|
-
const managed = managedPids.get(p.pid);
|
|
812
|
-
let classification;
|
|
813
|
-
if (p.pid === selfPid) classification = "root";
|
|
814
|
-
else if (managed) classification = "managed";
|
|
815
|
-
else classification = classifyByAncestry(p.pid);
|
|
816
|
-
const orphaned = classification === "ghost";
|
|
817
|
-
out.push({
|
|
818
|
-
pid: p.pid,
|
|
819
|
-
ppid: p.ppid,
|
|
820
|
-
pgid: p.pgid,
|
|
821
|
-
classification,
|
|
822
|
-
addonId: managed?.addonId ?? null,
|
|
823
|
-
nodeId: managed?.nodeId ?? (p.pid === selfPid ? hubNodeId : null),
|
|
824
|
-
command: p.command,
|
|
825
|
-
cpuPercent: p.cpuPercent,
|
|
826
|
-
memoryRssBytes: p.memoryRssBytes,
|
|
827
|
-
uptimeSec: p.uptimeSec,
|
|
828
|
-
orphaned
|
|
829
|
-
});
|
|
830
|
-
}
|
|
831
|
-
return out;
|
|
832
|
-
}
|
|
833
|
-
/**
|
|
834
|
-
* Send SIGTERM / SIGKILL to a pid. Refuses pids that don't appear in
|
|
835
|
-
* `listNodeProcesses()` to prevent arbitrary system kills — a dedicated
|
|
836
|
-
* admin-path for resurrected zombies, not a generic shell replacement.
|
|
837
|
-
*
|
|
838
|
-
* `root`-classified pids (the running launcher / agent CLI / hub itself)
|
|
839
|
-
* are also refused: killing them tears down the whole node and the
|
|
840
|
-
* operator's intent is almost always to nuke a leaked child, not the
|
|
841
|
-
* supervisor that keeps the rest alive. Process restart goes through
|
|
842
|
-
* the dedicated `$process.restart` action, not this kill API.
|
|
843
|
-
*/
|
|
844
|
-
async killProcess(input) {
|
|
845
|
-
const match = (await this.listNodeProcesses()).find((p) => p.pid === input.pid);
|
|
846
|
-
if (!match) return {
|
|
847
|
-
success: false,
|
|
848
|
-
reason: "pid not in node process table"
|
|
849
|
-
};
|
|
850
|
-
if (match.classification === "root" || match.classification === "system") {
|
|
851
|
-
this.ctx.logger.warn("Refused to kill protected process", { meta: {
|
|
852
|
-
pid: input.pid,
|
|
853
|
-
classification: match.classification,
|
|
854
|
-
addonId: match.addonId,
|
|
855
|
-
command: match.command
|
|
856
|
-
} });
|
|
857
|
-
return {
|
|
858
|
-
success: false,
|
|
859
|
-
reason: match.classification === "root" ? "cannot kill root (current node supervisor)" : "cannot kill system (intentional dev-tree ancestor — vite, concurrently, npm, etc.)"
|
|
860
|
-
};
|
|
861
|
-
}
|
|
862
|
-
const signal = input.force ? "SIGKILL" : "SIGTERM";
|
|
863
|
-
try {
|
|
864
|
-
process.kill(input.pid, signal);
|
|
865
|
-
this.ctx.logger.info("Killed node process", { meta: {
|
|
866
|
-
pid: input.pid,
|
|
867
|
-
signal,
|
|
868
|
-
classification: match.classification,
|
|
869
|
-
addonId: match.addonId
|
|
870
|
-
} });
|
|
871
|
-
return {
|
|
872
|
-
success: true,
|
|
873
|
-
signal
|
|
874
|
-
};
|
|
875
|
-
} catch (err) {
|
|
876
|
-
return {
|
|
877
|
-
success: false,
|
|
878
|
-
reason: err instanceof Error ? err.message : String(err),
|
|
879
|
-
signal
|
|
880
|
-
};
|
|
881
|
-
}
|
|
882
2083
|
}
|
|
883
2084
|
/**
|
|
884
2085
|
* Ask the addon's forked runner to write a V8 heap snapshot (SIGUSR2 → the
|
|
@@ -955,7 +2156,29 @@ var NativeMetricsAddon = class extends BaseAddon {
|
|
|
955
2156
|
return [];
|
|
956
2157
|
}
|
|
957
2158
|
}
|
|
2159
|
+
/**
|
|
2160
|
+
* The knobs live HERE, on the document this addon already owns.
|
|
2161
|
+
*
|
|
2162
|
+
* Fields on an existing document, never a method per knob —
|
|
2163
|
+
* `system.getLoggingSettings` / `setLoggingSettings` set that precedent.
|
|
2164
|
+
* They are deliberately NOT in the logging document: that one is about
|
|
2165
|
+
* levels and diagnostic windows, and two documents both claiming a knob is
|
|
2166
|
+
* how this repo has already shipped a switch nobody read.
|
|
2167
|
+
*
|
|
2168
|
+
* Cluster-wide, not per-node: every node must emit on the same cadence or
|
|
2169
|
+
* the fleet's series cannot be laid over each other, and the table is single
|
|
2170
|
+
* and hub-resident, so a per-node retention would be a promise nothing could
|
|
2171
|
+
* keep.
|
|
2172
|
+
*
|
|
2173
|
+
* The cost line is computed from OBSERVED numbers — the nodes and process
|
|
2174
|
+
* counts the cluster is actually reporting — so an operator raising the
|
|
2175
|
+
* retention sees what it costs BEFORE applying it, not afterwards.
|
|
2176
|
+
*/
|
|
958
2177
|
globalSettingsSchema() {
|
|
2178
|
+
const projection = projectLoadSeriesCost({
|
|
2179
|
+
config: this.loadConfig,
|
|
2180
|
+
observedProcessRows: this.observedFleetRows()
|
|
2181
|
+
});
|
|
959
2182
|
return this.schema({ sections: [{
|
|
960
2183
|
id: "native-metrics-settings",
|
|
961
2184
|
title: "System Metrics",
|
|
@@ -970,6 +2193,47 @@ var NativeMetricsAddon = class extends BaseAddon {
|
|
|
970
2193
|
default: 5e3,
|
|
971
2194
|
unit: "ms"
|
|
972
2195
|
})]
|
|
2196
|
+
}, {
|
|
2197
|
+
id: "native-metrics-load-series",
|
|
2198
|
+
title: "Load history",
|
|
2199
|
+
fields: [
|
|
2200
|
+
this.field({
|
|
2201
|
+
type: "number",
|
|
2202
|
+
key: "loadSeriesCadenceSec",
|
|
2203
|
+
label: "Sampling cadence",
|
|
2204
|
+
description: "How often every node reports its process tree. FIXED — a sample is emitted on every interval whether or not anything changed, so a gap in the chart means one thing only: nobody reported. Outside 5-60 s the value is refused.",
|
|
2205
|
+
min: 5,
|
|
2206
|
+
max: 60,
|
|
2207
|
+
step: 1,
|
|
2208
|
+
default: 10,
|
|
2209
|
+
unit: "s"
|
|
2210
|
+
}),
|
|
2211
|
+
this.field({
|
|
2212
|
+
type: "number",
|
|
2213
|
+
key: "loadSeriesRetentionHours",
|
|
2214
|
+
label: "Retention",
|
|
2215
|
+
description: "How long the load history is kept on disk (the hub database, on the NVMe cache — never the recordings disk). This is the INTENTION; the row cap below is the guarantee.",
|
|
2216
|
+
min: 1,
|
|
2217
|
+
max: 72,
|
|
2218
|
+
step: 1,
|
|
2219
|
+
default: 6,
|
|
2220
|
+
unit: "h"
|
|
2221
|
+
}),
|
|
2222
|
+
{
|
|
2223
|
+
type: "info",
|
|
2224
|
+
key: "load-series-cost",
|
|
2225
|
+
label: "What this configuration costs",
|
|
2226
|
+
content: describeLoadSeriesCost(projection),
|
|
2227
|
+
variant: projection.capBites ? "warning" : "info"
|
|
2228
|
+
},
|
|
2229
|
+
{
|
|
2230
|
+
type: "info",
|
|
2231
|
+
key: "load-series-cap",
|
|
2232
|
+
label: "Hard row cap",
|
|
2233
|
+
content: `The table never exceeds ${this.loadConfig.maxRows.toLocaleString("en-US")} rows. When it would, the OLDEST samples are evicted and a warning is logged — so raising the retention can never turn into an incident, only into a shorter effective window.`,
|
|
2234
|
+
variant: "info"
|
|
2235
|
+
}
|
|
2236
|
+
]
|
|
973
2237
|
}] });
|
|
974
2238
|
}
|
|
975
2239
|
};
|