@causal-order/monitor 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.build/src/config.d.ts +1 -0
- package/.build/src/config.js +1 -0
- package/.build/src/health/HealthTracker.d.ts +12 -0
- package/.build/src/health/HealthTracker.js +133 -0
- package/.build/src/health.d.ts +1 -0
- package/.build/src/health.js +1 -0
- package/.build/src/index.d.ts +16 -0
- package/.build/src/index.js +15 -0
- package/.build/src/inspect/inspectMonitorSnapshot.d.ts +2 -0
- package/.build/src/inspect/inspectMonitorSnapshot.js +79 -0
- package/.build/src/inspect.d.ts +1 -0
- package/.build/src/inspect.js +1 -0
- package/.build/src/replay/ReplayCoordinator.d.ts +24 -0
- package/.build/src/replay/ReplayCoordinator.js +205 -0
- package/.build/src/replay.d.ts +1 -0
- package/.build/src/replay.js +1 -0
- package/.build/src/routing/DeliveryRouter.d.ts +8 -0
- package/.build/src/routing/DeliveryRouter.js +48 -0
- package/.build/src/routing.d.ts +1 -0
- package/.build/src/routing.js +1 -0
- package/.build/src/runtime/MonitorRuntime.d.ts +35 -0
- package/.build/src/runtime/MonitorRuntime.js +253 -0
- package/.build/src/runtime/createMonitorRuntime.d.ts +3 -0
- package/.build/src/runtime/createMonitorRuntime.js +4 -0
- package/.build/src/runtime.d.ts +2 -0
- package/.build/src/runtime.js +2 -0
- package/.build/src/storage/SQLiteReservoir.d.ts +44 -0
- package/.build/src/storage/SQLiteReservoir.js +369 -0
- package/.build/src/storage/schema.d.ts +3 -0
- package/.build/src/storage/schema.js +80 -0
- package/.build/src/storage.d.ts +1 -0
- package/.build/src/storage.js +1 -0
- package/.build/src/testing/monitorHarness.d.ts +27 -0
- package/.build/src/testing/monitorHarness.js +98 -0
- package/.build/src/throttle/ThrottleController.d.ts +14 -0
- package/.build/src/throttle/ThrottleController.js +51 -0
- package/.build/src/throttle.d.ts +1 -0
- package/.build/src/throttle.js +1 -0
- package/.build/src/transport/TransportMonitorAdapter.d.ts +42 -0
- package/.build/src/transport/TransportMonitorAdapter.js +135 -0
- package/.build/src/transport.d.ts +1 -0
- package/.build/src/transport.js +1 -0
- package/.build/src/types/config.d.ts +46 -0
- package/.build/src/types/config.js +54 -0
- package/.build/src/types/events.d.ts +51 -0
- package/.build/src/types/events.js +1 -0
- package/.build/src/types/snapshots.d.ts +63 -0
- package/.build/src/types/snapshots.js +1 -0
- package/.build/src/types.d.ts +3 -0
- package/.build/src/types.js +3 -0
- package/CHANGELOG.md +85 -0
- package/LICENSE +21 -0
- package/README.md +247 -0
- package/package.json +128 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { createDefaultMonitorConfig, type MonitorComponentHealthConfig, type MonitorConfig, type MonitorHealthConfig, type MonitorReplayConfig, type MonitorReservoirConfig, type MonitorThrottleConfig, type MonitorThrottleTierConfig, type MonitorTransportConfig, } from "./types/config.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { createDefaultMonitorConfig, } from "./types/config.js";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { MonitorHealthConfig } from "../types/config.js";
|
|
2
|
+
import type { MonitorComponent, MonitorHealthUpdate } from "../types/events.js";
|
|
3
|
+
import type { MonitorComponentHealthSnapshot } from "../types/snapshots.js";
|
|
4
|
+
export declare class HealthTracker {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(config: MonitorHealthConfig, now: () => bigint);
|
|
7
|
+
getSnapshot(): Record<MonitorComponent, MonitorComponentHealthSnapshot>;
|
|
8
|
+
getComponentSnapshot(component: MonitorComponent): MonitorComponentHealthSnapshot;
|
|
9
|
+
updateComponentHealth(component: MonitorComponent, update: MonitorHealthUpdate): MonitorComponentHealthSnapshot;
|
|
10
|
+
observeHeartbeat(component: MonitorComponent, observedAt?: bigint, details?: Record<string, unknown>): MonitorComponentHealthSnapshot;
|
|
11
|
+
refreshStates(at?: bigint): Record<MonitorComponent, MonitorComponentHealthSnapshot>;
|
|
12
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
function createComponentSnapshot(component, observedAt) {
|
|
2
|
+
return {
|
|
3
|
+
component,
|
|
4
|
+
state: "online",
|
|
5
|
+
observedAt,
|
|
6
|
+
lastChangedAt: observedAt,
|
|
7
|
+
reasonCode: null,
|
|
8
|
+
details: {},
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
export class HealthTracker {
|
|
12
|
+
#config;
|
|
13
|
+
#now;
|
|
14
|
+
#components;
|
|
15
|
+
constructor(config, now) {
|
|
16
|
+
this.#config = config;
|
|
17
|
+
this.#now = now;
|
|
18
|
+
const observedAt = now();
|
|
19
|
+
this.#components = {
|
|
20
|
+
transport: {
|
|
21
|
+
snapshot: createComponentSnapshot("transport", observedAt),
|
|
22
|
+
lastHeartbeatAt: observedAt,
|
|
23
|
+
},
|
|
24
|
+
dedupe: {
|
|
25
|
+
snapshot: createComponentSnapshot("dedupe", observedAt),
|
|
26
|
+
lastHeartbeatAt: observedAt,
|
|
27
|
+
},
|
|
28
|
+
"causal-order": {
|
|
29
|
+
snapshot: createComponentSnapshot("causal-order", observedAt),
|
|
30
|
+
lastHeartbeatAt: observedAt,
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
getSnapshot() {
|
|
35
|
+
return {
|
|
36
|
+
transport: {
|
|
37
|
+
...this.#components.transport.snapshot,
|
|
38
|
+
details: { ...this.#components.transport.snapshot.details },
|
|
39
|
+
},
|
|
40
|
+
dedupe: {
|
|
41
|
+
...this.#components.dedupe.snapshot,
|
|
42
|
+
details: { ...this.#components.dedupe.snapshot.details },
|
|
43
|
+
},
|
|
44
|
+
"causal-order": {
|
|
45
|
+
...this.#components["causal-order"].snapshot,
|
|
46
|
+
details: { ...this.#components["causal-order"].snapshot.details },
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
getComponentSnapshot(component) {
|
|
51
|
+
const snapshot = this.#components[component].snapshot;
|
|
52
|
+
return {
|
|
53
|
+
...snapshot,
|
|
54
|
+
details: { ...snapshot.details },
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
updateComponentHealth(component, update) {
|
|
58
|
+
const componentState = this.#components[component];
|
|
59
|
+
const observedAt = update.observedAt ?? this.#now();
|
|
60
|
+
const previous = componentState.snapshot;
|
|
61
|
+
const nextState = update.state;
|
|
62
|
+
componentState.snapshot = {
|
|
63
|
+
component,
|
|
64
|
+
state: nextState,
|
|
65
|
+
observedAt,
|
|
66
|
+
lastChangedAt: previous.state === nextState ? previous.lastChangedAt : observedAt,
|
|
67
|
+
reasonCode: update.reasonCode ?? null,
|
|
68
|
+
details: update.details ?? {},
|
|
69
|
+
};
|
|
70
|
+
if (nextState !== "offline") {
|
|
71
|
+
componentState.lastHeartbeatAt = observedAt;
|
|
72
|
+
}
|
|
73
|
+
return this.getComponentSnapshot(component);
|
|
74
|
+
}
|
|
75
|
+
observeHeartbeat(component, observedAt = this.#now(), details = {}) {
|
|
76
|
+
const componentState = this.#components[component];
|
|
77
|
+
componentState.lastHeartbeatAt = observedAt;
|
|
78
|
+
return this.updateComponentHealth(component, {
|
|
79
|
+
state: "online",
|
|
80
|
+
observedAt,
|
|
81
|
+
reasonCode: "heartbeat",
|
|
82
|
+
details,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
refreshStates(at = this.#now()) {
|
|
86
|
+
const componentNames = [
|
|
87
|
+
"transport",
|
|
88
|
+
"dedupe",
|
|
89
|
+
"causal-order",
|
|
90
|
+
];
|
|
91
|
+
for (const component of componentNames) {
|
|
92
|
+
const componentState = this.#components[component];
|
|
93
|
+
const elapsed = at - componentState.lastHeartbeatAt;
|
|
94
|
+
const thresholds = this.#resolveThresholds(component);
|
|
95
|
+
const nextState = this.#resolveStateFromElapsed(elapsed, thresholds);
|
|
96
|
+
if (nextState !== componentState.snapshot.state) {
|
|
97
|
+
this.updateComponentHealth(component, {
|
|
98
|
+
state: nextState,
|
|
99
|
+
observedAt: at,
|
|
100
|
+
reasonCode: "heartbeat_timeout",
|
|
101
|
+
details: {
|
|
102
|
+
elapsedSinceHeartbeatMs: elapsed.toString(),
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
componentState.snapshot = {
|
|
108
|
+
...componentState.snapshot,
|
|
109
|
+
observedAt: at,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return this.getSnapshot();
|
|
114
|
+
}
|
|
115
|
+
#resolveThresholds(component) {
|
|
116
|
+
if (component === "transport") {
|
|
117
|
+
return this.#config.transport;
|
|
118
|
+
}
|
|
119
|
+
if (component === "dedupe") {
|
|
120
|
+
return this.#config.dedupe;
|
|
121
|
+
}
|
|
122
|
+
return this.#config.causalOrder;
|
|
123
|
+
}
|
|
124
|
+
#resolveStateFromElapsed(elapsed, thresholds) {
|
|
125
|
+
if (elapsed >= thresholds.offlineAfterMs) {
|
|
126
|
+
return "offline";
|
|
127
|
+
}
|
|
128
|
+
if (elapsed >= thresholds.degradedAfterMs) {
|
|
129
|
+
return "degraded";
|
|
130
|
+
}
|
|
131
|
+
return "online";
|
|
132
|
+
}
|
|
133
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { HealthTracker } from "./health/HealthTracker.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { HealthTracker } from "./health/HealthTracker.js";
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export declare const monitorPackageVersion = "0.0.9";
|
|
2
|
+
export type MonitorImplementationStatus = "replay_operational";
|
|
3
|
+
export declare const monitorImplementationStatus: MonitorImplementationStatus;
|
|
4
|
+
export { createDefaultMonitorConfig, type MonitorComponentHealthConfig, type MonitorConfig, type MonitorHealthConfig, type MonitorReplayConfig, type MonitorReservoirConfig, type MonitorThrottleConfig, type MonitorThrottleTierConfig, type MonitorTransportConfig, } from "./types/config.js";
|
|
5
|
+
export { type MonitorClock, type MonitorComponent, type MonitorDeliveryMode, type MonitorHealthState, type MonitorHealthUpdate, type MonitorIngressAction, type MonitorIngressDecision, type MonitorIngressEvent, type MonitorPeerState, type MonitorPeerStateEvent, type MonitorReplaySessionState, type MonitorReplayTargetPath, type MonitorRoutingMode, type MonitorSourcePath, type MonitorThrottleTier, } from "./types/events.js";
|
|
6
|
+
export { type MonitorOperationalState, type InspectedMonitorSnapshot, type MonitorComponentHealthSnapshot, type MonitorSnapshot, type ReplaySessionSnapshot, type ReservoirStats, } from "./types/snapshots.js";
|
|
7
|
+
export { inspectMonitorSnapshot } from "./inspect/inspectMonitorSnapshot.js";
|
|
8
|
+
export { HealthTracker } from "./health/HealthTracker.js";
|
|
9
|
+
export { ReplayCoordinator, type ReplayBatch } from "./replay/ReplayCoordinator.js";
|
|
10
|
+
export { DeliveryRouter } from "./routing/DeliveryRouter.js";
|
|
11
|
+
export { createMonitorRuntime } from "./runtime/createMonitorRuntime.js";
|
|
12
|
+
export { MonitorRuntime } from "./runtime/MonitorRuntime.js";
|
|
13
|
+
export { SQLiteReservoir, type ReservoirReplayEntry, } from "./storage/SQLiteReservoir.js";
|
|
14
|
+
export { TransportMonitorAdapter, type MonitorAdapterForwardContext, type MonitorAdapterHandlers, type MonitorIngestResult, type ReplayPumpResult, } from "./transport/TransportMonitorAdapter.js";
|
|
15
|
+
export { monitorHarnessArtifacts, monitorHarnessScenarios, type MonitorHarnessAdapterContract, type MonitorHarnessArtifactSpec, type MonitorHarnessExpectation, type MonitorHarnessRuntimeKind, type MonitorHarnessScenarioDefinition, type MonitorHarnessScenarioId, } from "./testing/monitorHarness.js";
|
|
16
|
+
export { ThrottleController } from "./throttle/ThrottleController.js";
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export const monitorPackageVersion = "0.0.9";
|
|
2
|
+
export const monitorImplementationStatus = "replay_operational";
|
|
3
|
+
export { createDefaultMonitorConfig, } from "./types/config.js";
|
|
4
|
+
export {} from "./types/events.js";
|
|
5
|
+
export {} from "./types/snapshots.js";
|
|
6
|
+
export { inspectMonitorSnapshot } from "./inspect/inspectMonitorSnapshot.js";
|
|
7
|
+
export { HealthTracker } from "./health/HealthTracker.js";
|
|
8
|
+
export { ReplayCoordinator } from "./replay/ReplayCoordinator.js";
|
|
9
|
+
export { DeliveryRouter } from "./routing/DeliveryRouter.js";
|
|
10
|
+
export { createMonitorRuntime } from "./runtime/createMonitorRuntime.js";
|
|
11
|
+
export { MonitorRuntime } from "./runtime/MonitorRuntime.js";
|
|
12
|
+
export { SQLiteReservoir, } from "./storage/SQLiteReservoir.js";
|
|
13
|
+
export { TransportMonitorAdapter, } from "./transport/TransportMonitorAdapter.js";
|
|
14
|
+
export { monitorHarnessArtifacts, monitorHarnessScenarios, } from "./testing/monitorHarness.js";
|
|
15
|
+
export { ThrottleController } from "./throttle/ThrottleController.js";
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
function deriveOperationalState(snapshot, replayRetryBackoffActive) {
|
|
2
|
+
if (replayRetryBackoffActive) {
|
|
3
|
+
return "replay_retry_waiting";
|
|
4
|
+
}
|
|
5
|
+
if (snapshot.replay.state === "queued" || snapshot.replay.state === "running") {
|
|
6
|
+
return "replay_draining";
|
|
7
|
+
}
|
|
8
|
+
if (snapshot.replay.state === "completed") {
|
|
9
|
+
return "recovery_confirming";
|
|
10
|
+
}
|
|
11
|
+
if (snapshot.routingMode === "order_buffer_only" ||
|
|
12
|
+
snapshot.routingMode === "full_outage_buffer") {
|
|
13
|
+
return "buffering_only";
|
|
14
|
+
}
|
|
15
|
+
if (snapshot.routingMode === "dedupe_bypass_throttled") {
|
|
16
|
+
return "degraded_live";
|
|
17
|
+
}
|
|
18
|
+
return "healthy_live";
|
|
19
|
+
}
|
|
20
|
+
function deriveLiveFlowGateReason(snapshot, replayRetryBackoffActive) {
|
|
21
|
+
if (replayRetryBackoffActive) {
|
|
22
|
+
return "replay retry backoff in progress";
|
|
23
|
+
}
|
|
24
|
+
if (snapshot.replay.state === "queued" || snapshot.replay.state === "running") {
|
|
25
|
+
return "replay backlog is draining";
|
|
26
|
+
}
|
|
27
|
+
if (snapshot.replay.state === "completed") {
|
|
28
|
+
return "post-replay health confirmation is still in progress";
|
|
29
|
+
}
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
export function inspectMonitorSnapshot(snapshot) {
|
|
33
|
+
const unhealthyComponents = Object.values(snapshot.components)
|
|
34
|
+
.filter((component) => component.state !== "online")
|
|
35
|
+
.map((component) => component.component);
|
|
36
|
+
const replayRetryBackoffActive = snapshot.replay.nextRetryAt !== null &&
|
|
37
|
+
snapshot.replay.nextRetryAt > snapshot.generatedAt;
|
|
38
|
+
const operationalState = deriveOperationalState(snapshot, replayRetryBackoffActive);
|
|
39
|
+
const liveFlowGateReason = deriveLiveFlowGateReason(snapshot, replayRetryBackoffActive);
|
|
40
|
+
const liveFlowGateClosed = liveFlowGateReason !== null;
|
|
41
|
+
const replayReadyRows = Math.max(0, snapshot.reservoir.totalPendingRows - snapshot.reservoir.retryWaitingRows);
|
|
42
|
+
const replayProgressBase = snapshot.replay.deliveredEventCount + snapshot.reservoir.totalPendingRows;
|
|
43
|
+
const replayProgressPercent = replayProgressBase === 0
|
|
44
|
+
? null
|
|
45
|
+
: Math.min(100, Math.max(0, Math.round((snapshot.replay.deliveredEventCount / replayProgressBase) * 100)));
|
|
46
|
+
const replayRetryDelayMs = snapshot.replay.nextRetryAt !== null &&
|
|
47
|
+
snapshot.replay.nextRetryAt > snapshot.generatedAt
|
|
48
|
+
? snapshot.replay.nextRetryAt - snapshot.generatedAt
|
|
49
|
+
: null;
|
|
50
|
+
const requiresOperatorAttention = unhealthyComponents.length > 0 ||
|
|
51
|
+
snapshot.replay.state === "failed" ||
|
|
52
|
+
snapshot.replay.state === "aborted" ||
|
|
53
|
+
snapshot.reservoir.retryWaitingRows > 0 ||
|
|
54
|
+
snapshot.reservoir.totalPendingRows > 0;
|
|
55
|
+
return {
|
|
56
|
+
generatedAt: snapshot.generatedAt,
|
|
57
|
+
operationalState,
|
|
58
|
+
routingMode: snapshot.routingMode,
|
|
59
|
+
throttleTier: snapshot.throttleTier,
|
|
60
|
+
unhealthyComponents,
|
|
61
|
+
requiresOperatorAttention,
|
|
62
|
+
liveFlowGateClosed,
|
|
63
|
+
liveFlowGateReason,
|
|
64
|
+
totalPendingRows: snapshot.reservoir.totalPendingRows,
|
|
65
|
+
oldestPendingAgeMs: snapshot.reservoir.oldestPendingAgeMs,
|
|
66
|
+
replayReadyRows,
|
|
67
|
+
retryWaitingRows: snapshot.reservoir.retryWaitingRows,
|
|
68
|
+
earliestRetryAt: snapshot.reservoir.earliestRetryAt,
|
|
69
|
+
replayState: snapshot.replay.state,
|
|
70
|
+
replayBacklogRemainingRows: snapshot.reservoir.totalPendingRows,
|
|
71
|
+
replayProgressPercent,
|
|
72
|
+
replayQueuedEventCount: snapshot.replay.queuedEventCount,
|
|
73
|
+
replayDeliveredEventCount: snapshot.replay.deliveredEventCount,
|
|
74
|
+
replayNextRetryAt: snapshot.replay.nextRetryAt,
|
|
75
|
+
replayRetryDelayMs,
|
|
76
|
+
replayConsecutiveFailureCount: snapshot.replay.consecutiveFailureCount,
|
|
77
|
+
replayRetryBackoffActive,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { inspectMonitorSnapshot } from "./inspect/inspectMonitorSnapshot.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { inspectMonitorSnapshot } from "./inspect/inspectMonitorSnapshot.js";
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { MonitorReplayConfig } from "../types/config.js";
|
|
2
|
+
import type { MonitorComponent, MonitorHealthState } from "../types/events.js";
|
|
3
|
+
import type { ReplaySessionSnapshot } from "../types/snapshots.js";
|
|
4
|
+
import { SQLiteReservoir, type ReservoirReplayEntry } from "../storage/SQLiteReservoir.js";
|
|
5
|
+
export interface ReplayBatch {
|
|
6
|
+
session: ReplaySessionSnapshot;
|
|
7
|
+
entries: ReservoirReplayEntry[];
|
|
8
|
+
isDrainComplete: boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare class ReplayCoordinator {
|
|
11
|
+
#private;
|
|
12
|
+
constructor(config: MonitorReplayConfig, reservoir: SQLiteReservoir, now: () => bigint);
|
|
13
|
+
getSnapshot(): ReplaySessionSnapshot;
|
|
14
|
+
isGateClosed(): boolean;
|
|
15
|
+
queueIfNeeded(): ReplaySessionSnapshot;
|
|
16
|
+
start(): ReplaySessionSnapshot;
|
|
17
|
+
claimNextBatch(limit: number): ReplayBatch;
|
|
18
|
+
acknowledgeBatch(rowIds: ReadonlyArray<number>): ReplaySessionSnapshot;
|
|
19
|
+
fail(error: string, rowIds?: ReadonlyArray<number>): ReplaySessionSnapshot;
|
|
20
|
+
abort(reason: string, rowIds?: ReadonlyArray<number>): ReplaySessionSnapshot;
|
|
21
|
+
complete(): ReplaySessionSnapshot;
|
|
22
|
+
observeRecoveryHeartbeat(component: MonitorComponent, state: MonitorHealthState, allHealthy: boolean): ReplaySessionSnapshot;
|
|
23
|
+
resetToIdle(): ReplaySessionSnapshot;
|
|
24
|
+
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { SQLiteReservoir, } from "../storage/SQLiteReservoir.js";
|
|
2
|
+
function createSnapshot(requiredRecoveryHeartbeats) {
|
|
3
|
+
return {
|
|
4
|
+
state: "idle",
|
|
5
|
+
targetPath: "dedupe_then_order",
|
|
6
|
+
queuedEventCount: 0,
|
|
7
|
+
deliveredEventCount: 0,
|
|
8
|
+
startedAt: null,
|
|
9
|
+
endedAt: null,
|
|
10
|
+
lastError: null,
|
|
11
|
+
nextRetryAt: null,
|
|
12
|
+
consecutiveFailureCount: 0,
|
|
13
|
+
recoveryHeartbeatCount: 0,
|
|
14
|
+
requiredRecoveryHeartbeats,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
export class ReplayCoordinator {
|
|
18
|
+
#config;
|
|
19
|
+
#reservoir;
|
|
20
|
+
#now;
|
|
21
|
+
#snapshot;
|
|
22
|
+
constructor(config, reservoir, now) {
|
|
23
|
+
this.#config = config;
|
|
24
|
+
this.#reservoir = reservoir;
|
|
25
|
+
this.#now = now;
|
|
26
|
+
this.#snapshot = createSnapshot(config.healthConfirmationHeartbeats);
|
|
27
|
+
}
|
|
28
|
+
getSnapshot() {
|
|
29
|
+
return { ...this.#snapshot };
|
|
30
|
+
}
|
|
31
|
+
isGateClosed() {
|
|
32
|
+
return (this.#snapshot.state === "queued" ||
|
|
33
|
+
this.#snapshot.state === "running" ||
|
|
34
|
+
this.#snapshot.state === "failed" ||
|
|
35
|
+
(this.#snapshot.state === "completed" &&
|
|
36
|
+
this.#snapshot.recoveryHeartbeatCount <
|
|
37
|
+
this.#snapshot.requiredRecoveryHeartbeats));
|
|
38
|
+
}
|
|
39
|
+
queueIfNeeded() {
|
|
40
|
+
const backlog = this.#reservoir.getStats().totalPendingRows;
|
|
41
|
+
if (backlog === 0) {
|
|
42
|
+
return this.getSnapshot();
|
|
43
|
+
}
|
|
44
|
+
if (this.#snapshot.state === "running" || this.#snapshot.state === "queued") {
|
|
45
|
+
return this.#commit({
|
|
46
|
+
...this.#snapshot,
|
|
47
|
+
queuedEventCount: Math.max(this.#snapshot.queuedEventCount, backlog),
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
if (this.#snapshot.state === "failed" &&
|
|
51
|
+
this.#snapshot.nextRetryAt !== null &&
|
|
52
|
+
this.#now() < this.#snapshot.nextRetryAt) {
|
|
53
|
+
return this.#commit({
|
|
54
|
+
...this.#snapshot,
|
|
55
|
+
queuedEventCount: Math.max(this.#snapshot.queuedEventCount, backlog),
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return this.#commit({
|
|
59
|
+
state: "queued",
|
|
60
|
+
targetPath: "dedupe_then_order",
|
|
61
|
+
queuedEventCount: backlog,
|
|
62
|
+
deliveredEventCount: 0,
|
|
63
|
+
startedAt: null,
|
|
64
|
+
endedAt: null,
|
|
65
|
+
lastError: null,
|
|
66
|
+
nextRetryAt: null,
|
|
67
|
+
consecutiveFailureCount: this.#snapshot.state === "failed"
|
|
68
|
+
? this.#snapshot.consecutiveFailureCount
|
|
69
|
+
: 0,
|
|
70
|
+
recoveryHeartbeatCount: 0,
|
|
71
|
+
requiredRecoveryHeartbeats: this.#config.healthConfirmationHeartbeats,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
start() {
|
|
75
|
+
const backlog = this.#reservoir.getStats().totalPendingRows;
|
|
76
|
+
if (backlog === 0) {
|
|
77
|
+
return this.resetToIdle();
|
|
78
|
+
}
|
|
79
|
+
if (this.#snapshot.state === "running") {
|
|
80
|
+
return this.getSnapshot();
|
|
81
|
+
}
|
|
82
|
+
if (this.#snapshot.state !== "queued" && this.#snapshot.state !== "failed") {
|
|
83
|
+
throw new Error(`Cannot start replay from state '${this.#snapshot.state}'. Queue replay first.`);
|
|
84
|
+
}
|
|
85
|
+
return this.#commit({
|
|
86
|
+
state: "running",
|
|
87
|
+
targetPath: "dedupe_then_order",
|
|
88
|
+
queuedEventCount: backlog,
|
|
89
|
+
deliveredEventCount: 0,
|
|
90
|
+
startedAt: this.#now(),
|
|
91
|
+
endedAt: null,
|
|
92
|
+
lastError: null,
|
|
93
|
+
nextRetryAt: null,
|
|
94
|
+
recoveryHeartbeatCount: 0,
|
|
95
|
+
requiredRecoveryHeartbeats: this.#config.healthConfirmationHeartbeats,
|
|
96
|
+
consecutiveFailureCount: this.#snapshot.state === "failed"
|
|
97
|
+
? this.#snapshot.consecutiveFailureCount
|
|
98
|
+
: 0,
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
claimNextBatch(limit) {
|
|
102
|
+
if (this.#snapshot.state === "queued") {
|
|
103
|
+
this.start();
|
|
104
|
+
}
|
|
105
|
+
if (this.#snapshot.state !== "running") {
|
|
106
|
+
throw new Error(`Cannot claim replay batch while replay is '${this.#snapshot.state}'.`);
|
|
107
|
+
}
|
|
108
|
+
const entries = this.#reservoir.claimReplayBatch(limit);
|
|
109
|
+
if (entries.length === 0) {
|
|
110
|
+
const remainingRows = this.#reservoir.getStats().totalPendingRows;
|
|
111
|
+
if (remainingRows === 0) {
|
|
112
|
+
return {
|
|
113
|
+
session: this.complete(),
|
|
114
|
+
entries: [],
|
|
115
|
+
isDrainComplete: true,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
session: this.getSnapshot(),
|
|
121
|
+
entries,
|
|
122
|
+
isDrainComplete: false,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
acknowledgeBatch(rowIds) {
|
|
126
|
+
const deliveredNow = this.#reservoir.markReplayBatchDelivered(rowIds);
|
|
127
|
+
const remainingRows = this.#reservoir.getStats().totalPendingRows;
|
|
128
|
+
const deliveredEventCount = this.#snapshot.deliveredEventCount + deliveredNow;
|
|
129
|
+
const nextSnapshot = this.#commit({
|
|
130
|
+
...this.#snapshot,
|
|
131
|
+
deliveredEventCount,
|
|
132
|
+
queuedEventCount: deliveredEventCount + remainingRows,
|
|
133
|
+
});
|
|
134
|
+
if (remainingRows === 0 && nextSnapshot.state === "running") {
|
|
135
|
+
return this.complete();
|
|
136
|
+
}
|
|
137
|
+
return nextSnapshot;
|
|
138
|
+
}
|
|
139
|
+
fail(error, rowIds = []) {
|
|
140
|
+
const now = this.#now();
|
|
141
|
+
const nextRetryAt = now + this.#config.retryBackoffMs;
|
|
142
|
+
this.#reservoir.resetReplayBatchToPending(rowIds, nextRetryAt);
|
|
143
|
+
return this.#commit({
|
|
144
|
+
...this.#snapshot,
|
|
145
|
+
state: "failed",
|
|
146
|
+
endedAt: now,
|
|
147
|
+
lastError: error,
|
|
148
|
+
nextRetryAt,
|
|
149
|
+
consecutiveFailureCount: this.#snapshot.consecutiveFailureCount + 1,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
abort(reason, rowIds = []) {
|
|
153
|
+
this.#reservoir.resetReplayBatchToPending(rowIds);
|
|
154
|
+
return this.#commit({
|
|
155
|
+
...this.#snapshot,
|
|
156
|
+
state: "aborted",
|
|
157
|
+
endedAt: this.#now(),
|
|
158
|
+
lastError: reason,
|
|
159
|
+
nextRetryAt: null,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
complete() {
|
|
163
|
+
return this.#commit({
|
|
164
|
+
...this.#snapshot,
|
|
165
|
+
state: "completed",
|
|
166
|
+
queuedEventCount: this.#snapshot.deliveredEventCount,
|
|
167
|
+
endedAt: this.#now(),
|
|
168
|
+
lastError: null,
|
|
169
|
+
nextRetryAt: null,
|
|
170
|
+
recoveryHeartbeatCount: 0,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
observeRecoveryHeartbeat(component, state, allHealthy) {
|
|
174
|
+
if (this.#snapshot.state !== "completed") {
|
|
175
|
+
return this.getSnapshot();
|
|
176
|
+
}
|
|
177
|
+
if (component === "transport") {
|
|
178
|
+
return this.getSnapshot();
|
|
179
|
+
}
|
|
180
|
+
if (state !== "online" || !allHealthy) {
|
|
181
|
+
return this.#commit({
|
|
182
|
+
...this.#snapshot,
|
|
183
|
+
recoveryHeartbeatCount: 0,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
const recoveryHeartbeatCount = this.#snapshot.recoveryHeartbeatCount + 1;
|
|
187
|
+
const nextSnapshot = this.#commit({
|
|
188
|
+
...this.#snapshot,
|
|
189
|
+
recoveryHeartbeatCount,
|
|
190
|
+
});
|
|
191
|
+
if (recoveryHeartbeatCount >= this.#config.healthConfirmationHeartbeats &&
|
|
192
|
+
this.#reservoir.getStats().totalPendingRows === 0) {
|
|
193
|
+
return this.resetToIdle();
|
|
194
|
+
}
|
|
195
|
+
return nextSnapshot;
|
|
196
|
+
}
|
|
197
|
+
resetToIdle() {
|
|
198
|
+
return this.#commit(createSnapshot(this.#config.healthConfirmationHeartbeats));
|
|
199
|
+
}
|
|
200
|
+
#commit(snapshot) {
|
|
201
|
+
this.#snapshot = { ...snapshot };
|
|
202
|
+
this.#reservoir.recordReplaySnapshot(this.#snapshot);
|
|
203
|
+
return this.getSnapshot();
|
|
204
|
+
}
|
|
205
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { ReplayCoordinator, type ReplayBatch } from "./replay/ReplayCoordinator.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { ReplayCoordinator } from "./replay/ReplayCoordinator.js";
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { ThrottleController } from "../throttle/ThrottleController.js";
|
|
2
|
+
import type { MonitorIngressDecision, MonitorRoutingMode } from "../types/events.js";
|
|
3
|
+
import type { ReservoirStats } from "../types/snapshots.js";
|
|
4
|
+
export declare class DeliveryRouter {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(throttleController: ThrottleController);
|
|
7
|
+
decideIngress(routingMode: MonitorRoutingMode, reservoir: ReservoirStats): MonitorIngressDecision;
|
|
8
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { ThrottleController } from "../throttle/ThrottleController.js";
|
|
2
|
+
export class DeliveryRouter {
|
|
3
|
+
#throttleController;
|
|
4
|
+
constructor(throttleController) {
|
|
5
|
+
this.#throttleController = throttleController;
|
|
6
|
+
}
|
|
7
|
+
decideIngress(routingMode, reservoir) {
|
|
8
|
+
const throttle = this.#throttleController.decide(routingMode, reservoir);
|
|
9
|
+
if (throttle.tier === "paused") {
|
|
10
|
+
return {
|
|
11
|
+
action: routingMode === "full_outage_buffer" || routingMode === "order_buffer_only"
|
|
12
|
+
? "buffer_only"
|
|
13
|
+
: "pause",
|
|
14
|
+
routingMode,
|
|
15
|
+
deliveryMode: this.#resolveDeliveryMode(routingMode),
|
|
16
|
+
throttleTier: throttle.tier,
|
|
17
|
+
targetEventsPerSecond: throttle.targetEventsPerSecond,
|
|
18
|
+
targetBatchSize: throttle.targetBatchSize,
|
|
19
|
+
reason: throttle.reason,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
return {
|
|
23
|
+
action: routingMode === "full_outage_buffer" || routingMode === "order_buffer_only"
|
|
24
|
+
? "buffer_only"
|
|
25
|
+
: "accept",
|
|
26
|
+
routingMode,
|
|
27
|
+
deliveryMode: this.#resolveDeliveryMode(routingMode),
|
|
28
|
+
throttleTier: throttle.tier,
|
|
29
|
+
targetEventsPerSecond: throttle.targetEventsPerSecond,
|
|
30
|
+
targetBatchSize: throttle.targetBatchSize,
|
|
31
|
+
reason: throttle.reason,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
#resolveDeliveryMode(routingMode) {
|
|
35
|
+
switch (routingMode) {
|
|
36
|
+
case "dedupe_bypass_throttled":
|
|
37
|
+
return "dedupe_bypass";
|
|
38
|
+
case "order_buffer_only":
|
|
39
|
+
return "order_buffer_only";
|
|
40
|
+
case "full_outage_buffer":
|
|
41
|
+
return "full_outage_buffer";
|
|
42
|
+
case "replay_through_dedupe":
|
|
43
|
+
return "replay_through_dedupe";
|
|
44
|
+
default:
|
|
45
|
+
return "normal";
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { DeliveryRouter } from "./routing/DeliveryRouter.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { DeliveryRouter } from "./routing/DeliveryRouter.js";
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { type MonitorConfig } from "../types/config.js";
|
|
2
|
+
import type { MonitorComponent, MonitorIngressDecision, MonitorHealthUpdate, MonitorIngressEvent, MonitorThrottleTier } from "../types/events.js";
|
|
3
|
+
import type { InspectedMonitorSnapshot, MonitorSnapshot, ReplaySessionSnapshot, ReservoirStats } from "../types/snapshots.js";
|
|
4
|
+
export declare class MonitorRuntime {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(config?: Partial<MonitorConfig>);
|
|
7
|
+
getConfig(): Readonly<MonitorConfig>;
|
|
8
|
+
getSnapshot(): MonitorSnapshot;
|
|
9
|
+
getInspectedSnapshot(): InspectedMonitorSnapshot;
|
|
10
|
+
getHealthSnapshot(): Record<MonitorComponent, import("../types/snapshots.js").MonitorComponentHealthSnapshot>;
|
|
11
|
+
getReservoirStats(): ReservoirStats;
|
|
12
|
+
getReplaySnapshot(): ReplaySessionSnapshot;
|
|
13
|
+
getIngressDecision(): MonitorIngressDecision;
|
|
14
|
+
updateComponentHealth(component: MonitorComponent, update: MonitorHealthUpdate): import("../types/snapshots.js").MonitorComponentHealthSnapshot;
|
|
15
|
+
observeHeartbeat(component: MonitorComponent, observedAt?: bigint, details?: Record<string, unknown>): import("../types/snapshots.js").MonitorComponentHealthSnapshot;
|
|
16
|
+
setThrottleTier(tier: MonitorThrottleTier): MonitorThrottleTier;
|
|
17
|
+
queueReplay(): ReplaySessionSnapshot;
|
|
18
|
+
startReplay(): ReplaySessionSnapshot;
|
|
19
|
+
claimReplayBatch(limit?: number): import("../replay/ReplayCoordinator.js").ReplayBatch;
|
|
20
|
+
acknowledgeReplayBatch(rowIds: ReadonlyArray<number>): ReplaySessionSnapshot;
|
|
21
|
+
acknowledgeIngressDelivery(rowIds: ReadonlyArray<number>): number;
|
|
22
|
+
failReplay(error: string, rowIds?: ReadonlyArray<number>): ReplaySessionSnapshot;
|
|
23
|
+
abortReplay(reason: string, rowIds?: ReadonlyArray<number>): ReplaySessionSnapshot;
|
|
24
|
+
ingestTransportEvent(event: MonitorIngressEvent, sourceStreamId?: string | null): {
|
|
25
|
+
rowId: number;
|
|
26
|
+
decision: MonitorIngressDecision;
|
|
27
|
+
};
|
|
28
|
+
observeDedupeEvent(event: MonitorIngressEvent, sourceStreamId?: string | null): number;
|
|
29
|
+
pruneReservoir(): {
|
|
30
|
+
markedDeadLetter: number;
|
|
31
|
+
deletedRows: number;
|
|
32
|
+
};
|
|
33
|
+
refreshHealthStates(at?: bigint): Record<MonitorComponent, import("../types/snapshots.js").MonitorComponentHealthSnapshot>;
|
|
34
|
+
close(): void;
|
|
35
|
+
}
|