@camstack/system 1.2.218 → 1.2.220
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +2 -2
- package/dist/addon-runner.mjs +2 -2
- package/dist/addon-utils.js +2 -2
- package/dist/addon-utils.mjs +2 -2
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +2 -2
- package/dist/builtins/device-manager/device-manager.addon.mjs +2 -2
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +2 -2
- package/dist/builtins/local-network/local-network.addon.mjs +2 -2
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/gpu-probe.d.ts +18 -0
- package/dist/builtins/native-metrics/native-metrics-provider.d.ts +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.d.ts +26 -3
- package/dist/builtins/native-metrics/native-metrics.addon.js +1320 -1093
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1320 -1096
- package/dist/builtins/native-metrics/proc-process-table.d.ts +25 -0
- package/dist/builtins/platform-probe/index.js +3 -3
- package/dist/builtins/platform-probe/index.mjs +3 -3
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-pragmas.d.ts +23 -0
- package/dist/builtins/sqlite-storage/sqlite-settings-backend.d.ts +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.d.ts +1 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
- package/dist/builtins/sqlite-storage/wal-checkpoint-policy.d.ts +115 -0
- package/dist/builtins/sqlite-storage/wal-checkpoint-worker.d.ts +1 -0
- package/dist/builtins/sqlite-storage/wal-checkpointer.d.ts +70 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{custom-action-registry-BDwM5qAK.js → custom-action-registry-B42MC_Pv.js} +1 -1
- package/dist/{custom-action-registry-CsONUlhC.mjs → custom-action-registry-CpR58DXT.mjs} +1 -1
- package/dist/{dist-DigmGsQS.mjs → dist-Ca_jaxXs.mjs} +232 -4
- package/dist/{dist-BOlMCOeT.js → dist-CllKpcTd.js} +232 -4
- package/dist/index.d.ts +6 -1
- package/dist/index.js +99 -65
- package/dist/index.mjs +84 -67
- package/dist/kernel/gc-census.d.ts +144 -0
- package/dist/kernel/heap-watch.d.ts +32 -1
- package/dist/kernel/moleculer/mesh-queue-report.d.ts +51 -0
- package/dist/kernel/outbound-queue-watch.d.ts +94 -0
- package/dist/kernel/socket-plane-report.d.ts +112 -4
- package/dist/kernel/transport/cap-action-census.d.ts +218 -0
- package/dist/kernel/transport/cap-child-index.d.ts +49 -0
- package/dist/kernel/transport/cap-routing-hints.d.ts +21 -0
- package/dist/kernel/transport/child-cap-protocol.d.ts +28 -2
- package/dist/kernel/transport/frame-codec.d.ts +49 -0
- package/dist/kernel/transport/index.d.ts +2 -0
- package/dist/kernel/transport/local-child-client.d.ts +7 -0
- package/dist/kernel/transport/local-child-registry.d.ts +99 -5
- package/dist/kernel/transport/local-transport.d.ts +54 -2
- package/dist/kernel/transport/socket-channel.d.ts +40 -1
- package/dist/{manifest-system-deps-Bo92lg5u.js → manifest-system-deps-DfpwaXkh.js} +1725 -614
- package/dist/{manifest-system-deps-D--wyMD9.mjs → manifest-system-deps-iadzxSBr.mjs} +1850 -841
- package/dist/process/proc-stat.d.ts +75 -0
- package/dist/process/resource-monitor.d.ts +9 -1
- package/dist/{resource-monitor-CdnzxBLP.js → resource-monitor-LVE1BoLs.js} +42 -23
- package/dist/{resource-monitor-BWmQ5i-o.mjs → resource-monitor-Vp0bPV3U.mjs} +42 -22
- package/dist/{retired-settings-keys-xEAlYnv7.js → retired-settings-keys-7iEsSzmD.js} +1 -1
- package/dist/{retired-settings-keys-vK_U-CG2.mjs → retired-settings-keys-DQnfr__p.mjs} +1 -1
- package/dist/wal-checkpoint-policy-CAcg63o-.mjs +92 -0
- package/dist/wal-checkpoint-policy-CCJZngds.js +127 -0
- package/dist/wal-checkpoint-worker.js +138 -0
- package/dist/wal-checkpoint-worker.mjs +137 -0
- package/package.json +1 -1
- package/dist/{event-loop-stall-monitor-DM9OAzy8.js → event-loop-stall-monitor-CAsHdMwP.js} +1 -1
- package/dist/{event-loop-stall-monitor-B3CFd9tM.mjs → event-loop-stall-monitor-GPrVvx14.mjs} +1 -1
- package/dist/{tls-DOTmtLCW.mjs → tls-2xbgg78V.mjs} +2 -2
- package/dist/{tls-BxQlomxd.js → tls-BFs4PzoW.js} +2 -2
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* gc-census — HOW OFTEN V8 collects, on the ordinary heartbeat.
|
|
3
|
+
*
|
|
4
|
+
* ## The question this exists to answer, and why four records could not
|
|
5
|
+
*
|
|
6
|
+
* A garbage collector's cost is a product of two independent things: the LIVE
|
|
7
|
+
* SET (how much there is to mark) and the RATE at which a cycle is triggered.
|
|
8
|
+
* The `[mem]` line has carried the live set since 2026-08-29 — `heapUsed`,
|
|
9
|
+
* `oldSpace`, `largeObjectSpace`, `external`, `arrayBuffers` — and has never
|
|
10
|
+
* carried the rate. So D443, D444, D446 and D456 each read a level, saw it flat
|
|
11
|
+
* or falling, and could not say whether the GC share they were looking at came
|
|
12
|
+
* from the size of the heap or from the frequency of the collection. D444 cut
|
|
13
|
+
* hub-main's live set from 2 491 MB to 714 MB — a factor of 3.5 — and the GC
|
|
14
|
+
* share did not move. That result was unreadable until the rate was measured.
|
|
15
|
+
*
|
|
16
|
+
* Measured on hub-main 2026-09-11 with the observer below, the answer was on
|
|
17
|
+
* sight: **1.39 major mark-compacts per second against a 602 MB live set whose
|
|
18
|
+
* old space grew 22 MB/s.** A major GC every 0.7 s cannot be explained by 22
|
|
19
|
+
* MB/s of heap growth under a 3 264 MB ceiling — V8 was being driven by
|
|
20
|
+
* EXTERNAL memory pressure (537 MB/s of ArrayBuffer allocation, D458), and each
|
|
21
|
+
* of those cycles then had to mark the 602 MB live set on four platform
|
|
22
|
+
* workers. That is 70 % of the process.
|
|
23
|
+
*
|
|
24
|
+
* ## Why counts and not a rate of bytes
|
|
25
|
+
*
|
|
26
|
+
* An allocation rate sampled from `process.memoryUsage()` on this heartbeat
|
|
27
|
+
* would be a lie with a number on it. The probe runs every 2 s; external memory
|
|
28
|
+
* on hub-main cycles many times inside 2 s, so a sum of positive deltas
|
|
29
|
+
* under-reports it by more than an order of magnitude (a 100 ms sampler already
|
|
30
|
+
* reported 234 MB/s where a 5 ms one would have said 537). The GC COUNT is
|
|
31
|
+
* exact at any cadence — V8 reports every cycle — and it is the half of the
|
|
32
|
+
* product the line was missing. Read against the `oldSpace=` field already on
|
|
33
|
+
* the line, the two together say which economy is paying:
|
|
34
|
+
*
|
|
35
|
+
* - many majors, little old-space growth between them ⇒ EXTERNAL pressure.
|
|
36
|
+
* The fix is an allocation site, and shrinking the live set buys nothing.
|
|
37
|
+
* - majors that track old-space growth ⇒ the heap is genuinely filling.
|
|
38
|
+
* The fix is retention.
|
|
39
|
+
*
|
|
40
|
+
* ## Cost
|
|
41
|
+
*
|
|
42
|
+
* One prologue/epilogue callback per GC, into a counter. On hub-main at its
|
|
43
|
+
* worst that is ~4 entries/s. The observer was run against the live hub three
|
|
44
|
+
* times for 90 s while this was written, with no measurable effect on the
|
|
45
|
+
* process; `Profiler.start`, by comparison, stalls it 867 ms (D456).
|
|
46
|
+
*
|
|
47
|
+
* Absence is never zero (D393): a runtime that will not give an observer
|
|
48
|
+
* reports NOTHING on the line, not a census of zero collections.
|
|
49
|
+
*/
|
|
50
|
+
/** One window of collections, as V8 counted them. */
|
|
51
|
+
export interface GcCensusWindow {
|
|
52
|
+
/** Full mark-compacts. The expensive kind: every one marks the live set. */
|
|
53
|
+
readonly majors: number;
|
|
54
|
+
/** Scavenges of the young generation. */
|
|
55
|
+
readonly minors: number;
|
|
56
|
+
/** Incremental marking steps on the main thread. */
|
|
57
|
+
readonly incrementals: number;
|
|
58
|
+
/** Total MAIN-THREAD pause across the window. Never the concurrent work. */
|
|
59
|
+
readonly pauseMs: number;
|
|
60
|
+
/** How long the window was, so a rate can be computed from the line. */
|
|
61
|
+
readonly windowMs: number;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* A census that accumulates until it is read, then starts a new window.
|
|
65
|
+
*
|
|
66
|
+
* Window semantics, not cumulative, for the reason D456 gives: a line printing
|
|
67
|
+
* a lifetime total says nothing about the minute it describes, and the reader
|
|
68
|
+
* has to diff two lines by hand to learn anything.
|
|
69
|
+
*/
|
|
70
|
+
export interface GcCensus {
|
|
71
|
+
/** The window since the previous read. `null` until the first full window. */
|
|
72
|
+
readonly read: (at: number) => GcCensusWindow | null;
|
|
73
|
+
readonly stop: () => void;
|
|
74
|
+
}
|
|
75
|
+
/** What the census counts. Node's `PerformanceEntry.detail.kind` values. */
|
|
76
|
+
export interface GcObservation {
|
|
77
|
+
readonly kind: number;
|
|
78
|
+
readonly durationMs: number;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* How the census is fed. Injectable so a spec can pose a sequence of
|
|
82
|
+
* collections without provoking real ones, which is not something a test can
|
|
83
|
+
* do deterministically.
|
|
84
|
+
*/
|
|
85
|
+
export type GcObserverInstall = (observe: (entry: GcObservation) => void) => (() => void) | undefined;
|
|
86
|
+
/**
|
|
87
|
+
* V8's kinds, as Node reports them on `detail.kind`.
|
|
88
|
+
*
|
|
89
|
+
* Read from `perf_hooks.constants` rather than written as literals: the values
|
|
90
|
+
* are a Node ABI, and a literal `4` in this file would keep compiling and stop
|
|
91
|
+
* meaning "major" the release it changed.
|
|
92
|
+
*/
|
|
93
|
+
export interface GcKinds {
|
|
94
|
+
readonly major: number;
|
|
95
|
+
readonly minor: number;
|
|
96
|
+
readonly incremental: number;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* The shortest window a rate may be quoted over.
|
|
100
|
+
*
|
|
101
|
+
* One second, and the number comes from the line this field lives on: the boot
|
|
102
|
+
* line is printed microseconds after the watch starts, and `maj0@0s` there
|
|
103
|
+
* would be a rate computed over no time at all — indistinguishable on sight
|
|
104
|
+
* from a process that is genuinely not collecting. Below the floor the census
|
|
105
|
+
* says NOTHING, which is the same discipline as an unobservable runtime
|
|
106
|
+
* (D393): a window too short to answer is not an answer of zero.
|
|
107
|
+
*/
|
|
108
|
+
export declare const GC_CENSUS_MIN_WINDOW_MS = 1000;
|
|
109
|
+
/**
|
|
110
|
+
* Build a census over an injected observer.
|
|
111
|
+
*
|
|
112
|
+
* Returns `undefined` when the observer cannot be installed — the caller then
|
|
113
|
+
* prints nothing, which is the honest reading. It must never degrade into a
|
|
114
|
+
* census that reports zero collections: a process whose GC is not being
|
|
115
|
+
* observed and a process that is not collecting look identical on the line, and
|
|
116
|
+
* they are the two answers furthest apart.
|
|
117
|
+
*/
|
|
118
|
+
export declare function createGcCensus(install: GcObserverInstall, kinds: GcKinds, startedAt: number): GcCensus | undefined;
|
|
119
|
+
/** The real Node observer and the real kind constants, wrapped so they can be replaced in a spec. */
|
|
120
|
+
export interface NodeGcObserver {
|
|
121
|
+
readonly install: GcObserverInstall;
|
|
122
|
+
readonly kinds: GcKinds;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* The observer this process actually gets.
|
|
126
|
+
*
|
|
127
|
+
* Everything that can fail is inside the try: a runtime without
|
|
128
|
+
* `perf_hooks`, without the `gc` entry type, or without the kind constants
|
|
129
|
+
* yields `undefined` from `install`, and {@link createGcCensus} then declines
|
|
130
|
+
* to exist rather than reporting zeros.
|
|
131
|
+
*/
|
|
132
|
+
export declare function nodeGcObserver(): NodeGcObserver;
|
|
133
|
+
/**
|
|
134
|
+
* The two fields, APPENDED LAST on the `[mem]` line.
|
|
135
|
+
*
|
|
136
|
+
* D446's rule: no existing field moves, so every Loki prefix written since
|
|
137
|
+
* 2026-08-29 still matches. `gc=` prints even when nothing was collected — a
|
|
138
|
+
* window with no collections is a reading, and it is the reading that says a
|
|
139
|
+
* process is idle rather than unobserved. `gcPauseMs=` is the MAIN-THREAD
|
|
140
|
+
* pause only; the concurrent marking that cost hub-main 70 % of its CPU is on
|
|
141
|
+
* the platform workers and no in-process counter sees it, which is why the
|
|
142
|
+
* field is named for the thread it describes (D456, part 4).
|
|
143
|
+
*/
|
|
144
|
+
export declare function formatGcCensus(window: GcCensusWindow | null | undefined): string;
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { EventPlaneReader } from './event-plane-report.js';
|
|
2
|
-
import { SocketPlaneReader } from './socket-plane-report.js';
|
|
2
|
+
import { SocketPlaneCounters, SocketPlaneReader } from './socket-plane-report.js';
|
|
3
|
+
import { MeshPeerQueue, MeshQueueReader } from './moleculer/mesh-queue-report.js';
|
|
4
|
+
import { OutboundQueueGateOptions, OutboundQueueReading } from './outbound-queue-watch.js';
|
|
3
5
|
import { HeapSpaceReader } from './heap-spaces.js';
|
|
6
|
+
import { NodeGcObserver } from './gc-census.js';
|
|
4
7
|
import { ArrayBufferCensus, ArrayBufferCensusGateOptions } from './array-buffer-census.js';
|
|
5
8
|
/**
|
|
6
9
|
* What hub-main calls itself in a `[mem]` line.
|
|
@@ -607,6 +610,20 @@ export interface HeapWatchProbes {
|
|
|
607
610
|
* from the parent's side. Only hub-main and an agent's main pass one.
|
|
608
611
|
*/
|
|
609
612
|
readonly socketPlane?: SocketPlaneReader;
|
|
613
|
+
/**
|
|
614
|
+
* The write queue of every Moleculer TCP socket to a peer node. No default:
|
|
615
|
+
* only a process with a broker on a real transporter has one. Those sockets
|
|
616
|
+
* are UNREF'D by Moleculer, so a sum over `_getActiveHandles()` never listed
|
|
617
|
+
* them — the hole in the 2026-09-10 exclusion of "every socket queue"
|
|
618
|
+
* (D446). Read on every probe, judged with the UDS queues below.
|
|
619
|
+
*/
|
|
620
|
+
readonly meshQueue?: MeshQueueReader;
|
|
621
|
+
/**
|
|
622
|
+
* Thresholds for the outbound-queue gate. The gate itself is created
|
|
623
|
+
* whenever {@link socketPlane} or {@link meshQueue} is given; this only
|
|
624
|
+
* tunes it (tests, or a process with a different notion of "held").
|
|
625
|
+
*/
|
|
626
|
+
readonly outboundQueue?: OutboundQueueGateOptions;
|
|
610
627
|
/**
|
|
611
628
|
* The ArrayBuffer census that ARMS ITSELF. No default: it costs a forced
|
|
612
629
|
* full GC when it fires, and only hub-main has an episode on record (two
|
|
@@ -617,11 +634,25 @@ export interface HeapWatchProbes {
|
|
|
617
634
|
* `array-buffer-census.ts` for the gate, the cost and what it is NOT.
|
|
618
635
|
*/
|
|
619
636
|
readonly arrayBufferCensus?: ArrayBufferCensusProbe;
|
|
637
|
+
/**
|
|
638
|
+
* How the GC census observes collections. Defaults to the real Node
|
|
639
|
+
* observer, for EVERY process — the rate is half of what a GC costs and the
|
|
640
|
+
* line has never carried it (see `gc-census.ts`). A spec passes its own so a
|
|
641
|
+
* sequence of collections can be posed; nothing in production passes one.
|
|
642
|
+
*/
|
|
643
|
+
readonly gcObserver?: NodeGcObserver;
|
|
620
644
|
}
|
|
621
645
|
/** How the census is wired: the census itself plus the gate's thresholds. */
|
|
622
646
|
export interface ArrayBufferCensusProbe extends ArrayBufferCensusGateOptions {
|
|
623
647
|
readonly census: ArrayBufferCensus;
|
|
624
648
|
}
|
|
649
|
+
/**
|
|
650
|
+
* Every outbound queue the heartbeat can reach, as the gate wants them: the
|
|
651
|
+
* UDS children (only those whose channel keeps a reading — an unmeasured
|
|
652
|
+
* peer is not a zero) and the mesh writer sockets, each prefixed with its
|
|
653
|
+
* plane.
|
|
654
|
+
*/
|
|
655
|
+
export declare function outboundQueueReadings(uds: SocketPlaneCounters | undefined, mesh: readonly MeshPeerQueue[] | undefined): readonly OutboundQueueReading[];
|
|
625
656
|
/**
|
|
626
657
|
* Start the heartbeat. Returns a stop function.
|
|
627
658
|
*
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* mesh-queue-report — the write queue of every Moleculer TCP socket this
|
|
3
|
+
* process holds to a peer node, on the `[mem]` line (D446).
|
|
4
|
+
*
|
|
5
|
+
* ## Why a separate reader
|
|
6
|
+
*
|
|
7
|
+
* The UDS plane is measured on the channel (`SocketChannel.queuedBytes`). The
|
|
8
|
+
* mesh plane is Moleculer's: `TcpTransporter.writer` is a `TcpWriter` whose
|
|
9
|
+
* `sockets` map holds one `net.Socket` per peer node, opened lazily on the
|
|
10
|
+
* first send and — this is the point — `unref()`'d at connect
|
|
11
|
+
* (`transporters/tcp/tcp-writer.js`). An unref'd handle is not in
|
|
12
|
+
* `process._getActiveHandles()`, so the 2026-09-10 census that summed
|
|
13
|
+
* `writableLength` over that list and excluded "every socket queue" could not
|
|
14
|
+
* have seen a backlog to an agent, however large.
|
|
15
|
+
*
|
|
16
|
+
* ## What is read, and how carefully
|
|
17
|
+
*
|
|
18
|
+
* `broker.transit.tx.writer.sockets` — an internal path, walked with type
|
|
19
|
+
* guards and never a cast. Every step that is not the expected shape yields
|
|
20
|
+
* `undefined` for the WHOLE reading: a test broker (`Fake` transporter), a
|
|
21
|
+
* broker that does not exist yet, a Moleculer whose internals moved. `undefined`
|
|
22
|
+
* is "this process has no mesh writer to report", never "the mesh is quiet".
|
|
23
|
+
* A socket entry without numeric `writableLength`/`bytesWritten` is skipped
|
|
24
|
+
* on its own.
|
|
25
|
+
*/
|
|
26
|
+
/** One peer node's outbound socket, as the writer holds it. */
|
|
27
|
+
export interface MeshPeerQueue {
|
|
28
|
+
readonly nodeId: string;
|
|
29
|
+
/** Bytes handed to the socket that the kernel has not taken yet. */
|
|
30
|
+
readonly queuedBytes: number;
|
|
31
|
+
/** Lifetime bytes written on this socket — so a stalled queue can be read
|
|
32
|
+
* against the peer's normal rate. */
|
|
33
|
+
readonly bytesWritten: number;
|
|
34
|
+
}
|
|
35
|
+
/** `undefined` = no mesh writer to report (no broker, `Fake` transporter). */
|
|
36
|
+
export type MeshQueueReader = () => readonly MeshPeerQueue[] | undefined;
|
|
37
|
+
/**
|
|
38
|
+
* Build a reader over a lazily-resolved broker. Wrapped: a broker mid-teardown
|
|
39
|
+
* may throw, and a diagnostic that can crash the process it watches is worse
|
|
40
|
+
* than none.
|
|
41
|
+
*/
|
|
42
|
+
export declare function createMeshQueueReader(getBroker: () => unknown): MeshQueueReader;
|
|
43
|
+
/** At most this many nodes named. A cluster has a handful. */
|
|
44
|
+
export declare const MESH_QUEUE_TOP_N = 3;
|
|
45
|
+
/**
|
|
46
|
+
* The suffix appended to a `[mem]` line — leading space included, empty when
|
|
47
|
+
* this process has no mesh writer. `meshQueued` is printed always, at zero
|
|
48
|
+
* included: a quiet mesh is a reading. `meshQueuedTop` names only nodes with
|
|
49
|
+
* a queue, largest first — a ranked list of zeros is noise.
|
|
50
|
+
*/
|
|
51
|
+
export declare function formatMeshQueue(peers: readonly MeshPeerQueue[] | undefined): string;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* outbound-queue-watch — an outbound socket queue that HOLDS is named, and its
|
|
3
|
+
* release is timestamped (D446).
|
|
4
|
+
*
|
|
5
|
+
* ## The blind spot this closes
|
|
6
|
+
*
|
|
7
|
+
* On 2026-09-10 hub-main's `arrayBuffers` climbed twice at a constant rate
|
|
8
|
+
* (20 MB/min to 1 264 MB; 63 MB/min for 77 min to 4 968 MB), LIVE across a
|
|
9
|
+
* forced compaction, with ~4 % of JS heap alongside, and each time released in
|
|
10
|
+
* one instant with no line from any owner. The investigation excluded "every
|
|
11
|
+
* socket write queue" by summing `writableLength` over
|
|
12
|
+
* `process._getActiveHandles()` — 84 MB, all of it the phone's WebSocket.
|
|
13
|
+
*
|
|
14
|
+
* `_getActiveHandles()` lists only handles that keep the loop alive. An
|
|
15
|
+
* UNREF'D socket is not one of them, and Moleculer's `TcpWriter.connect`
|
|
16
|
+
* unrefs every socket it opens to a peer node (`transporters/tcp/tcp-writer.js`,
|
|
17
|
+
* `socket.unref()`). So the one outbound plane that writes to another MACHINE
|
|
18
|
+
* — where a reader that stops draining is the ordinary failure — was never in
|
|
19
|
+
* the sum. The UDS children were in it, but only as a total, at one instant,
|
|
20
|
+
* by hand.
|
|
21
|
+
*
|
|
22
|
+
* ## What this is
|
|
23
|
+
*
|
|
24
|
+
* A pure gate over per-peer queue depths the heartbeat reads on every probe:
|
|
25
|
+
* the UDS children's `socket.writableLength` (`SocketChannel.queuedBytes`) and
|
|
26
|
+
* the mesh writer sockets' (`mesh-queue-report.ts`). It says three things and
|
|
27
|
+
* nothing else:
|
|
28
|
+
*
|
|
29
|
+
* - a peer whose queue has been at or above {@link OUTBOUND_QUEUE_THRESHOLD_BYTES}
|
|
30
|
+
* for {@link OUTBOUND_QUEUE_SUSTAIN_MS} is NAMED, with its current and peak
|
|
31
|
+
* depth and how long it has held — at most once per
|
|
32
|
+
* {@link OUTBOUND_QUEUE_COOLDOWN_MS};
|
|
33
|
+
* - a named peer whose queue then falls under half the threshold, or that
|
|
34
|
+
* vanishes from the readings (its socket died — which is how a queue is
|
|
35
|
+
* dropped in one instant), is reported as RELEASED with its peak and the
|
|
36
|
+
* hold, so the release the episodes never logged has a line and a peer;
|
|
37
|
+
* - a burst that drains inside the sustain window says nothing. Backpressure
|
|
38
|
+
* measures LIVENESS, not identity (D444): a peer momentarily behind is
|
|
39
|
+
* coming back.
|
|
40
|
+
*
|
|
41
|
+
* ## What it is NOT
|
|
42
|
+
*
|
|
43
|
+
* Not a bound. It terminates nothing: a UDS child whose channel is destroyed
|
|
44
|
+
* loses every in-flight call, and a mesh socket Moleculer reconnects on its
|
|
45
|
+
* own terms. The WS guard (`ws-slow-consumer-guard.ts`) bounds the one plane
|
|
46
|
+
* whose client is expendable. Here the line is the deliverable; what to do
|
|
47
|
+
* about a peer that holds is decided with the peer named, not before.
|
|
48
|
+
*/
|
|
49
|
+
/** 64 MiB. Below this a queue is a burst on a busy peer (the orchestrator's
|
|
50
|
+
* channel moves ~1.5 MB/s of audio responses); above it, and held, it is
|
|
51
|
+
* retention with a name. */
|
|
52
|
+
export declare const OUTBOUND_QUEUE_THRESHOLD_BYTES: number;
|
|
53
|
+
/** Matches the ArrayBuffer census sustain: a minute above the bound. */
|
|
54
|
+
export declare const OUTBOUND_QUEUE_SUSTAIN_MS = 60000;
|
|
55
|
+
/** A held queue is re-named every ten minutes, not every probe. */
|
|
56
|
+
export declare const OUTBOUND_QUEUE_COOLDOWN_MS: number;
|
|
57
|
+
/** One outbound channel's queue depth at one probe. `peerId` carries its
|
|
58
|
+
* plane as a prefix (`uds:<childId>`, `mesh:<nodeId>`). */
|
|
59
|
+
export interface OutboundQueueReading {
|
|
60
|
+
readonly peerId: string;
|
|
61
|
+
readonly bytes: number;
|
|
62
|
+
}
|
|
63
|
+
export type OutboundQueueVerdict = {
|
|
64
|
+
readonly kind: 'sustained';
|
|
65
|
+
readonly peerId: string;
|
|
66
|
+
readonly bytes: number;
|
|
67
|
+
readonly peakBytes: number;
|
|
68
|
+
readonly heldForMs: number;
|
|
69
|
+
} | {
|
|
70
|
+
readonly kind: 'released';
|
|
71
|
+
readonly peerId: string;
|
|
72
|
+
readonly peakBytes: number;
|
|
73
|
+
readonly toBytes: number;
|
|
74
|
+
readonly heldForMs: number;
|
|
75
|
+
/** The peer left the readings altogether — its socket is gone. */
|
|
76
|
+
readonly gone: boolean;
|
|
77
|
+
};
|
|
78
|
+
export interface OutboundQueueGateOptions {
|
|
79
|
+
readonly thresholdBytes?: number;
|
|
80
|
+
readonly sustainMs?: number;
|
|
81
|
+
readonly cooldownMs?: number;
|
|
82
|
+
}
|
|
83
|
+
export interface OutboundQueueGate {
|
|
84
|
+
/** Judge one probe's readings. Every peer is judged on its own. */
|
|
85
|
+
readonly observe: (readings: readonly OutboundQueueReading[], at: number) => readonly OutboundQueueVerdict[];
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* The pure decision — threshold, sustain, cooldown, release, gone — testable
|
|
89
|
+
* without a socket.
|
|
90
|
+
*/
|
|
91
|
+
export declare function createOutboundQueueGate(options?: OutboundQueueGateOptions): OutboundQueueGate;
|
|
92
|
+
/** The line for one verdict. Both carry the peer, the bytes and the hold;
|
|
93
|
+
* the sustained one also says why the older census could not see it. */
|
|
94
|
+
export declare function formatOutboundQueueVerdict(label: string, verdict: OutboundQueueVerdict): string;
|
|
@@ -1,24 +1,64 @@
|
|
|
1
|
+
import { CapActionReading, CapActionWindow } from './transport/cap-action-census.js';
|
|
1
2
|
import { SocketDirectionSample, SocketTrafficSample } from './transport/socket-traffic.js';
|
|
2
|
-
/** One peer's cumulative traffic, as the channel keeps it
|
|
3
|
+
/** One peer's cumulative traffic, as the channel keeps it, plus its outbound
|
|
4
|
+
* queue depth at the instant of the read. */
|
|
3
5
|
export interface SocketPeerSample {
|
|
4
6
|
readonly peerId: string;
|
|
5
7
|
readonly tx: SocketDirectionSample;
|
|
6
8
|
readonly rx: SocketDirectionSample;
|
|
9
|
+
/**
|
|
10
|
+
* Bytes written to this peer that it has not yet taken — `writableLength`
|
|
11
|
+
* on the channel, read live (D446). `null` when the channel keeps no such
|
|
12
|
+
* reading: a measurement that failed is null, never 0 (D393), and the
|
|
13
|
+
* window counts it as unmeasured rather than as empty.
|
|
14
|
+
*/
|
|
15
|
+
readonly queuedBytes: number | null;
|
|
7
16
|
}
|
|
8
17
|
/** What `LocalChildRegistry` exposes, structurally — the kernel's heartbeat
|
|
9
18
|
* must not depend on the transport layer's class. */
|
|
10
19
|
export interface SocketPlaneRegistryChild {
|
|
11
20
|
readonly childId: string;
|
|
12
21
|
}
|
|
22
|
+
/**
|
|
23
|
+
* How many child→sibling cap calls the parent forwarded without reading
|
|
24
|
+
* their args (`raw`) against how many it decoded — see
|
|
25
|
+
* `transport/local-child-registry.ts` `ForwardCensus`. Cumulative on the
|
|
26
|
+
* registry; a delta on the window.
|
|
27
|
+
*/
|
|
28
|
+
export interface SocketForwardCensus {
|
|
29
|
+
readonly raw: number;
|
|
30
|
+
readonly decoded: number;
|
|
31
|
+
}
|
|
13
32
|
export interface SocketPlaneRegistry {
|
|
14
33
|
readonly listChildren: () => readonly SocketPlaneRegistryChild[];
|
|
15
34
|
readonly getChildSocketTraffic: (childId: string) => SocketTrafficSample | null;
|
|
35
|
+
/** Live outbound queue depth to one child, or `null` when unmeasured. */
|
|
36
|
+
readonly getChildQueuedBytes: (childId: string) => number | null;
|
|
37
|
+
/**
|
|
38
|
+
* The raw-forward census, when the registry keeps one. OPTIONAL: a registry
|
|
39
|
+
* that predates raw-forward has no such number, and the line must then omit
|
|
40
|
+
* the field rather than print a zero that reads as "nothing was forwarded".
|
|
41
|
+
*/
|
|
42
|
+
readonly readForwardCensus?: () => SocketForwardCensus;
|
|
43
|
+
/**
|
|
44
|
+
* The per-ACTION byte census, when the registry keeps one (D456).
|
|
45
|
+
*
|
|
46
|
+
* OPTIONAL on the same terms as {@link readForwardCensus}: a registry whose
|
|
47
|
+
* channels carry no byte counters has nothing to attribute, and the line
|
|
48
|
+
* must omit the fields rather than print a `0/0` that reads as "the plane
|
|
49
|
+
* moved nothing" instead of "nobody measured".
|
|
50
|
+
*/
|
|
51
|
+
readonly readActionCensus?: () => CapActionReading;
|
|
16
52
|
}
|
|
17
53
|
/** One cumulative reading of every measured peer. */
|
|
18
54
|
export interface SocketPlaneCounters {
|
|
19
55
|
readonly peers: readonly SocketPeerSample[];
|
|
20
56
|
/** Connected children, INCLUDING any whose channel keeps no counters. */
|
|
21
57
|
readonly peersConnected: number;
|
|
58
|
+
/** Cumulative forward census, absent when the registry keeps none. */
|
|
59
|
+
readonly forward?: SocketForwardCensus;
|
|
60
|
+
/** Cumulative per-action census, absent when the registry keeps none. */
|
|
61
|
+
readonly actions?: CapActionReading;
|
|
22
62
|
}
|
|
23
63
|
/** `undefined` = this process has no local socket plane to report (every forked
|
|
24
64
|
* runner, and hub-main before the registry exists). */
|
|
@@ -57,6 +97,23 @@ export interface SocketPeerDelta {
|
|
|
57
97
|
readonly tx: SocketDirectionSample;
|
|
58
98
|
readonly rx: SocketDirectionSample;
|
|
59
99
|
}
|
|
100
|
+
/** One peer holding an outbound queue at the instant of the read. */
|
|
101
|
+
export interface SocketPeerQueue {
|
|
102
|
+
readonly peerId: string;
|
|
103
|
+
readonly bytes: number;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* The outbound queues at the instant of the read — NOT a delta: a queue is a
|
|
107
|
+
* level, and the level is the reading (D446).
|
|
108
|
+
*/
|
|
109
|
+
export interface SocketQueueReading {
|
|
110
|
+
/** Summed over every peer with a reading. */
|
|
111
|
+
readonly totalBytes: number;
|
|
112
|
+
/** Peers whose channel keeps no queue reading. Never folded into the total. */
|
|
113
|
+
readonly unmeasured: number;
|
|
114
|
+
/** Peers holding a queue, largest first, at most {@link SOCKET_QUEUE_TOP_N}. */
|
|
115
|
+
readonly top: readonly SocketPeerQueue[];
|
|
116
|
+
}
|
|
60
117
|
/** The whole plane over one heartbeat window. */
|
|
61
118
|
export interface SocketPlaneWindow {
|
|
62
119
|
readonly peersConnected: number;
|
|
@@ -67,10 +124,56 @@ export interface SocketPlaneWindow {
|
|
|
67
124
|
readonly rx: SocketDirectionSample;
|
|
68
125
|
/** Busiest first (tx+rx messages), at most {@link SOCKET_PLANE_TOP_N}. */
|
|
69
126
|
readonly top: readonly SocketPeerDelta[];
|
|
127
|
+
/**
|
|
128
|
+
* Heaviest first (tx+rx BYTES), at most {@link SOCKET_PLANE_TOP_N}.
|
|
129
|
+
*
|
|
130
|
+
* A separate ranking rather than a replacement, because the two questions
|
|
131
|
+
* have different answers and both are live: `top` names the peer paying for
|
|
132
|
+
* round trips (`audio-codec` at 8 379 calls/min of small frames), `topBytes`
|
|
133
|
+
* names the peer paying for copies and GC. A peer that moves FEW, HUGE
|
|
134
|
+
* frames is absent from `top` by construction — measured 2026-09-11, the
|
|
135
|
+
* five peers named by messages left 116 MB/min of hub-main's outbound bytes
|
|
136
|
+
* with no name on them (D454).
|
|
137
|
+
*/
|
|
138
|
+
readonly topBytes: readonly SocketPeerDelta[];
|
|
139
|
+
readonly queued: SocketQueueReading;
|
|
140
|
+
/** Forwards in this window, absent when the registry keeps no census. */
|
|
141
|
+
readonly forward?: SocketForwardCensus;
|
|
142
|
+
/**
|
|
143
|
+
* The per-action census over this window, absent when none is kept.
|
|
144
|
+
*
|
|
145
|
+
* This is the field that survives raw-forward. Once 85.7 % of sibling calls
|
|
146
|
+
* cross hub-main without being read (D448, measured on 1.2.270), the bytes
|
|
147
|
+
* a peer moves stop being the proxy for what hub-main PAYS — a forwarded
|
|
148
|
+
* body costs a memcpy, a decoded one costs msgpack plus the garbage. So the
|
|
149
|
+
* entry carries the raw/decoded split BESIDE the bytes, per method, and the
|
|
150
|
+
* two questions are read off one row.
|
|
151
|
+
*/
|
|
152
|
+
readonly actions?: CapActionWindow;
|
|
70
153
|
}
|
|
154
|
+
/** How many queue-holding peers get named. Three: a queue is an exception,
|
|
155
|
+
* and the line already carries five names for traffic. */
|
|
156
|
+
export declare const SOCKET_QUEUE_TOP_N = 3;
|
|
157
|
+
/**
|
|
158
|
+
* How many (peer, action) pairs get named.
|
|
159
|
+
*
|
|
160
|
+
* Five, matching {@link SOCKET_PLANE_TOP_N}, and for the same measured reason:
|
|
161
|
+
* the distribution is not flat. Queried on this hub 2026-09-11, the heaviest
|
|
162
|
+
* peer's top THREE caps were 2 077 of its 2 141 calls/min — five names the
|
|
163
|
+
* dominant action, its challenger, and enough of the tail to see the shape
|
|
164
|
+
* change. The map behind it keeps every action (see `cap-action-census.ts` on
|
|
165
|
+
* why the bound that matters is the line's width, not the map's size); this
|
|
166
|
+
* constant governs only what is printed.
|
|
167
|
+
*/
|
|
168
|
+
export declare const SOCKET_ACTION_TOP_N = 5;
|
|
169
|
+
/** The queue reading over the current counters — a level, so no baseline. */
|
|
170
|
+
export declare function readSocketQueues(current: SocketPlaneCounters): SocketQueueReading;
|
|
71
171
|
/** Previous cumulative reading, kept between lines. */
|
|
72
172
|
export interface SocketPlaneBaseline {
|
|
73
173
|
readonly peers: ReadonlyMap<string, SocketPeerSample>;
|
|
174
|
+
readonly forward?: SocketForwardCensus;
|
|
175
|
+
/** Previous per-action reading; absent = nothing has been read yet. */
|
|
176
|
+
readonly actions?: CapActionReading;
|
|
74
177
|
}
|
|
75
178
|
/** A process that has never read the plane. */
|
|
76
179
|
export declare const EMPTY_SOCKET_PLANE_BASELINE: SocketPlaneBaseline;
|
|
@@ -92,8 +195,13 @@ export declare function createSocketPlaneMeter(reader: SocketPlaneReader, topN?:
|
|
|
92
195
|
* is the only way to see an unmeasured channel, and the per-kind splits because
|
|
93
196
|
* "the plane went quiet for a minute" is a reading.
|
|
94
197
|
*
|
|
95
|
-
* `socketTop` is
|
|
96
|
-
*
|
|
97
|
-
*
|
|
198
|
+
* `socketTop` is a conditional field — a ranked list of nothing is noise — and
|
|
199
|
+
* it is a single comma-separated value so the field COUNT does not change from
|
|
200
|
+
* line to line.
|
|
201
|
+
*
|
|
202
|
+
* `socketTopBytes` is the other one, and its PRESENCE is the reading: it is
|
|
203
|
+
* printed only when ranking by bytes names a peer that ranking by messages
|
|
204
|
+
* does not. A line without it says the two rankings agree; a line with it says
|
|
205
|
+
* a peer is moving few, heavy frames and names it (D454).
|
|
98
206
|
*/
|
|
99
207
|
export declare function formatSocketPlane(window: SocketPlaneWindow | undefined): string;
|