@causal-order/monitor 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.build/src/config.d.ts +1 -0
- package/.build/src/config.js +1 -0
- package/.build/src/health/HealthTracker.d.ts +12 -0
- package/.build/src/health/HealthTracker.js +133 -0
- package/.build/src/health.d.ts +1 -0
- package/.build/src/health.js +1 -0
- package/.build/src/index.d.ts +16 -0
- package/.build/src/index.js +15 -0
- package/.build/src/inspect/inspectMonitorSnapshot.d.ts +2 -0
- package/.build/src/inspect/inspectMonitorSnapshot.js +79 -0
- package/.build/src/inspect.d.ts +1 -0
- package/.build/src/inspect.js +1 -0
- package/.build/src/replay/ReplayCoordinator.d.ts +24 -0
- package/.build/src/replay/ReplayCoordinator.js +205 -0
- package/.build/src/replay.d.ts +1 -0
- package/.build/src/replay.js +1 -0
- package/.build/src/routing/DeliveryRouter.d.ts +8 -0
- package/.build/src/routing/DeliveryRouter.js +48 -0
- package/.build/src/routing.d.ts +1 -0
- package/.build/src/routing.js +1 -0
- package/.build/src/runtime/MonitorRuntime.d.ts +35 -0
- package/.build/src/runtime/MonitorRuntime.js +253 -0
- package/.build/src/runtime/createMonitorRuntime.d.ts +3 -0
- package/.build/src/runtime/createMonitorRuntime.js +4 -0
- package/.build/src/runtime.d.ts +2 -0
- package/.build/src/runtime.js +2 -0
- package/.build/src/storage/SQLiteReservoir.d.ts +44 -0
- package/.build/src/storage/SQLiteReservoir.js +369 -0
- package/.build/src/storage/schema.d.ts +3 -0
- package/.build/src/storage/schema.js +80 -0
- package/.build/src/storage.d.ts +1 -0
- package/.build/src/storage.js +1 -0
- package/.build/src/testing/monitorHarness.d.ts +27 -0
- package/.build/src/testing/monitorHarness.js +98 -0
- package/.build/src/throttle/ThrottleController.d.ts +14 -0
- package/.build/src/throttle/ThrottleController.js +51 -0
- package/.build/src/throttle.d.ts +1 -0
- package/.build/src/throttle.js +1 -0
- package/.build/src/transport/TransportMonitorAdapter.d.ts +42 -0
- package/.build/src/transport/TransportMonitorAdapter.js +135 -0
- package/.build/src/transport.d.ts +1 -0
- package/.build/src/transport.js +1 -0
- package/.build/src/types/config.d.ts +46 -0
- package/.build/src/types/config.js +54 -0
- package/.build/src/types/events.d.ts +51 -0
- package/.build/src/types/events.js +1 -0
- package/.build/src/types/snapshots.d.ts +63 -0
- package/.build/src/types/snapshots.js +1 -0
- package/.build/src/types.d.ts +3 -0
- package/.build/src/types.js +3 -0
- package/CHANGELOG.md +85 -0
- package/LICENSE +21 -0
- package/README.md +247 -0
- package/package.json +128 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
export const MONITOR_SQLITE_SCHEMA = `
|
|
3
|
+
CREATE TABLE IF NOT EXISTS ingress_events (
|
|
4
|
+
monitor_ingest_seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
5
|
+
id TEXT NOT NULL,
|
|
6
|
+
monitor_ingest_at_ms INTEGER NOT NULL,
|
|
7
|
+
source_node_id TEXT NOT NULL,
|
|
8
|
+
source_stream_id TEXT,
|
|
9
|
+
source_path TEXT NOT NULL,
|
|
10
|
+
event_id TEXT NOT NULL,
|
|
11
|
+
trace_id TEXT,
|
|
12
|
+
sequence TEXT,
|
|
13
|
+
logical_time_ms INTEGER NOT NULL,
|
|
14
|
+
payload_json TEXT NOT NULL,
|
|
15
|
+
payload_encoding TEXT NOT NULL DEFAULT 'json',
|
|
16
|
+
delivery_mode TEXT NOT NULL,
|
|
17
|
+
replay_state TEXT NOT NULL,
|
|
18
|
+
replay_attempts INTEGER NOT NULL DEFAULT 0,
|
|
19
|
+
retry_not_before_ms INTEGER,
|
|
20
|
+
expires_at_ms INTEGER NOT NULL
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
CREATE INDEX IF NOT EXISTS idx_ingress_events_expires_at
|
|
24
|
+
ON ingress_events(expires_at_ms);
|
|
25
|
+
|
|
26
|
+
CREATE INDEX IF NOT EXISTS idx_ingress_events_replay_state
|
|
27
|
+
ON ingress_events(replay_state);
|
|
28
|
+
|
|
29
|
+
CREATE INDEX IF NOT EXISTS idx_ingress_events_replay_retry
|
|
30
|
+
ON ingress_events(replay_state, retry_not_before_ms);
|
|
31
|
+
|
|
32
|
+
CREATE INDEX IF NOT EXISTS idx_ingress_events_source_path
|
|
33
|
+
ON ingress_events(source_path);
|
|
34
|
+
|
|
35
|
+
CREATE TABLE IF NOT EXISTS component_health_log (
|
|
36
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
37
|
+
recorded_at_ms INTEGER NOT NULL,
|
|
38
|
+
component TEXT NOT NULL,
|
|
39
|
+
health_state TEXT NOT NULL,
|
|
40
|
+
reason_code TEXT,
|
|
41
|
+
details_json TEXT NOT NULL
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
CREATE INDEX IF NOT EXISTS idx_component_health_log_component_time
|
|
45
|
+
ON component_health_log(component, recorded_at_ms);
|
|
46
|
+
|
|
47
|
+
CREATE TABLE IF NOT EXISTS outage_windows (
|
|
48
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
49
|
+
started_at_ms INTEGER NOT NULL,
|
|
50
|
+
ended_at_ms INTEGER,
|
|
51
|
+
dedupe_state TEXT NOT NULL,
|
|
52
|
+
causal_order_state TEXT NOT NULL,
|
|
53
|
+
buffer_mode TEXT NOT NULL,
|
|
54
|
+
notes_json TEXT NOT NULL
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
CREATE TABLE IF NOT EXISTS replay_sessions (
|
|
58
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
59
|
+
started_at_ms INTEGER,
|
|
60
|
+
ended_at_ms INTEGER,
|
|
61
|
+
target_path TEXT NOT NULL,
|
|
62
|
+
session_state TEXT NOT NULL,
|
|
63
|
+
event_count_attempted INTEGER NOT NULL DEFAULT 0,
|
|
64
|
+
event_count_delivered INTEGER NOT NULL DEFAULT 0,
|
|
65
|
+
error_count INTEGER NOT NULL DEFAULT 0,
|
|
66
|
+
details_json TEXT NOT NULL
|
|
67
|
+
);
|
|
68
|
+
`;
|
|
69
|
+
export function applyMonitorSchema(db) {
|
|
70
|
+
db.exec(MONITOR_SQLITE_SCHEMA);
|
|
71
|
+
const ingressColumns = db
|
|
72
|
+
.prepare(`PRAGMA table_info(ingress_events)`)
|
|
73
|
+
.all();
|
|
74
|
+
const columnNames = new Set(ingressColumns
|
|
75
|
+
.map((column) => column.name)
|
|
76
|
+
.filter((name) => typeof name === "string"));
|
|
77
|
+
if (!columnNames.has("retry_not_before_ms")) {
|
|
78
|
+
db.exec(`ALTER TABLE ingress_events ADD COLUMN retry_not_before_ms INTEGER`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { SQLiteReservoir, type ReservoirReplayEntry, } from "./storage/SQLiteReservoir.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { SQLiteReservoir, } from "./storage/SQLiteReservoir.js";
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export type MonitorHarnessScenarioId = "monitor-healthy-rolling-4h" | "monitor-transport-outage-burst" | "monitor-dedupe-outage" | "monitor-order-outage" | "monitor-dual-outage" | "monitor-recovery-through-dedupe";
|
|
2
|
+
export type MonitorHarnessRuntimeKind = "deployment-runtime" | "adapter-runtime";
|
|
3
|
+
export interface MonitorHarnessArtifactSpec {
|
|
4
|
+
fileName: string;
|
|
5
|
+
purpose: string;
|
|
6
|
+
}
|
|
7
|
+
export interface MonitorHarnessExpectation {
|
|
8
|
+
description: string;
|
|
9
|
+
}
|
|
10
|
+
export interface MonitorHarnessScenarioDefinition {
|
|
11
|
+
id: MonitorHarnessScenarioId;
|
|
12
|
+
runtimeKinds: ReadonlyArray<MonitorHarnessRuntimeKind>;
|
|
13
|
+
title: string;
|
|
14
|
+
description: string;
|
|
15
|
+
expectedArtifacts: ReadonlyArray<MonitorHarnessArtifactSpec>;
|
|
16
|
+
expectedRoutingModes: ReadonlyArray<string>;
|
|
17
|
+
expectations: ReadonlyArray<MonitorHarnessExpectation>;
|
|
18
|
+
}
|
|
19
|
+
export declare const monitorHarnessArtifacts: ReadonlyArray<MonitorHarnessArtifactSpec>;
|
|
20
|
+
export declare const monitorHarnessScenarios: ReadonlyArray<MonitorHarnessScenarioDefinition>;
|
|
21
|
+
export interface MonitorHarnessAdapterContract {
|
|
22
|
+
ingest(event: Record<string, unknown>): Promise<void> | void;
|
|
23
|
+
updateComponentHealth(component: "transport" | "dedupe" | "causal-order", state: string): void;
|
|
24
|
+
getSnapshot(): Record<string, unknown>;
|
|
25
|
+
reconcileRecovery?(): Promise<Record<string, unknown> | null> | Record<string, unknown> | null;
|
|
26
|
+
destroy(): void;
|
|
27
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
export const monitorHarnessArtifacts = [
|
|
2
|
+
{
|
|
3
|
+
fileName: "monitor-heartbeats.ndjson",
|
|
4
|
+
purpose: "Component heartbeat and blackout timeline for transport, dedupe, and causal-order.",
|
|
5
|
+
},
|
|
6
|
+
{
|
|
7
|
+
fileName: "monitor-health.ndjson",
|
|
8
|
+
purpose: "Health-state transitions, routing changes, and throttle posture.",
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
fileName: "monitor-replay.ndjson",
|
|
12
|
+
purpose: "Replay session lifecycle, batch drain progress, and replay failures.",
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
fileName: "monitor-summary.json",
|
|
16
|
+
purpose: "Compact run summary with backlog, routing, replay, and retention signals.",
|
|
17
|
+
},
|
|
18
|
+
];
|
|
19
|
+
export const monitorHarnessScenarios = [
|
|
20
|
+
{
|
|
21
|
+
id: "monitor-healthy-rolling-4h",
|
|
22
|
+
runtimeKinds: ["deployment-runtime", "adapter-runtime"],
|
|
23
|
+
title: "Healthy rolling reservoir",
|
|
24
|
+
description: "Validates steady-state buffering, pruning, and health visibility with all downstream components online.",
|
|
25
|
+
expectedArtifacts: monitorHarnessArtifacts,
|
|
26
|
+
expectedRoutingModes: ["normal"],
|
|
27
|
+
expectations: [
|
|
28
|
+
{ description: "rolling retention stays bounded at 4 hours" },
|
|
29
|
+
{ description: "no replay session starts during healthy flow" },
|
|
30
|
+
{ description: "throttle tier remains open for normal throughput" },
|
|
31
|
+
],
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
id: "monitor-transport-outage-burst",
|
|
35
|
+
runtimeKinds: ["adapter-runtime"],
|
|
36
|
+
title: "Transport outage and reconnect burst",
|
|
37
|
+
description: "Exercises transport blackout visibility and reconnect burst control without inventing separate monitor logic in the harness.",
|
|
38
|
+
expectedArtifacts: monitorHarnessArtifacts,
|
|
39
|
+
expectedRoutingModes: ["normal", "order_buffer_only"],
|
|
40
|
+
expectations: [
|
|
41
|
+
{ description: "transport blackout is visible in heartbeat artifacts" },
|
|
42
|
+
{ description: "reconnect burst markers appear in health or summary output" },
|
|
43
|
+
{ description: "recovery does not bypass monitor routing decisions" },
|
|
44
|
+
],
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
id: "monitor-dedupe-outage",
|
|
48
|
+
runtimeKinds: ["deployment-runtime", "adapter-runtime"],
|
|
49
|
+
title: "Dedupe outage with throttled bypass",
|
|
50
|
+
description: "Verifies that monitor can continue feeding causal-order directly while slowing ingress when dedupe is offline.",
|
|
51
|
+
expectedArtifacts: monitorHarnessArtifacts,
|
|
52
|
+
expectedRoutingModes: ["dedupe_bypass_throttled"],
|
|
53
|
+
expectations: [
|
|
54
|
+
{ description: "routing switches into dedupe bypass mode" },
|
|
55
|
+
{ description: "throttle tier drops below open during bypass" },
|
|
56
|
+
{ description: "buffer growth remains inspectable in summary output" },
|
|
57
|
+
],
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
id: "monitor-order-outage",
|
|
61
|
+
runtimeKinds: ["deployment-runtime", "adapter-runtime"],
|
|
62
|
+
title: "Order outage with buffered recovery",
|
|
63
|
+
description: "Confirms backlog accumulation while causal-order is offline and replay once the normal path is restored.",
|
|
64
|
+
expectedArtifacts: monitorHarnessArtifacts,
|
|
65
|
+
expectedRoutingModes: ["order_buffer_only", "replay_through_dedupe"],
|
|
66
|
+
expectations: [
|
|
67
|
+
{ description: "live ingress buffers instead of forwarding to causal-order" },
|
|
68
|
+
{ description: "replay is recorded as a distinct recovery session" },
|
|
69
|
+
{ description: "live flow reopens only after replay completion" },
|
|
70
|
+
],
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
id: "monitor-dual-outage",
|
|
74
|
+
runtimeKinds: ["deployment-runtime", "adapter-runtime"],
|
|
75
|
+
title: "Dual outage with hard retention ceiling",
|
|
76
|
+
description: "Checks that monitor remains bounded when both dedupe and causal-order are unavailable.",
|
|
77
|
+
expectedArtifacts: monitorHarnessArtifacts,
|
|
78
|
+
expectedRoutingModes: ["full_outage_buffer", "replay_through_dedupe"],
|
|
79
|
+
expectations: [
|
|
80
|
+
{ description: "buffer retention never exceeds the hard 6 hour ceiling" },
|
|
81
|
+
{ description: "expired undeliverable rows are visible as dead-letter outcomes" },
|
|
82
|
+
{ description: "recovery still returns through dedupe before normal reopen" },
|
|
83
|
+
],
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
id: "monitor-recovery-through-dedupe",
|
|
87
|
+
runtimeKinds: ["deployment-runtime", "adapter-runtime"],
|
|
88
|
+
title: "Recovery through dedupe",
|
|
89
|
+
description: "Validates the key current-line rule that backlog replay always routes back through restored dedupe.",
|
|
90
|
+
expectedArtifacts: monitorHarnessArtifacts,
|
|
91
|
+
expectedRoutingModes: ["replay_through_dedupe", "normal"],
|
|
92
|
+
expectations: [
|
|
93
|
+
{ description: "replay target stays on the dedupe then order path" },
|
|
94
|
+
{ description: "replay drain reaches zero pending rows" },
|
|
95
|
+
{ description: "health confirmation gate completes before live reopen" },
|
|
96
|
+
],
|
|
97
|
+
},
|
|
98
|
+
];
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { MonitorThrottleConfig } from "../types/config.js";
|
|
2
|
+
import type { MonitorRoutingMode, MonitorThrottleTier } from "../types/events.js";
|
|
3
|
+
import type { ReservoirStats } from "../types/snapshots.js";
|
|
4
|
+
export interface ThrottleDecision {
|
|
5
|
+
tier: MonitorThrottleTier;
|
|
6
|
+
targetEventsPerSecond: number;
|
|
7
|
+
targetBatchSize: number;
|
|
8
|
+
reason: string;
|
|
9
|
+
}
|
|
10
|
+
export declare class ThrottleController {
|
|
11
|
+
#private;
|
|
12
|
+
constructor(config: MonitorThrottleConfig);
|
|
13
|
+
decide(routingMode: MonitorRoutingMode, reservoir: ReservoirStats): ThrottleDecision;
|
|
14
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
export class ThrottleController {
|
|
2
|
+
#config;
|
|
3
|
+
constructor(config) {
|
|
4
|
+
this.#config = config;
|
|
5
|
+
}
|
|
6
|
+
decide(routingMode, reservoir) {
|
|
7
|
+
if (routingMode === "full_outage_buffer" || routingMode === "order_buffer_only") {
|
|
8
|
+
return this.#toDecision("paused", "downstream ordering path unavailable");
|
|
9
|
+
}
|
|
10
|
+
if (routingMode === "replay_through_dedupe") {
|
|
11
|
+
return this.#toDecision("very_slow", "replay session active");
|
|
12
|
+
}
|
|
13
|
+
if (routingMode === "dedupe_bypass_throttled") {
|
|
14
|
+
if (reservoir.totalPendingRows >= 10_000) {
|
|
15
|
+
return this.#toDecision("paused", "dedupe bypass backlog exceeds hard safety threshold");
|
|
16
|
+
}
|
|
17
|
+
if (reservoir.totalPendingRows >= 2_000) {
|
|
18
|
+
return this.#toDecision("very_slow", "dedupe bypass backlog is elevated");
|
|
19
|
+
}
|
|
20
|
+
return this.#toDecision("slow", "dedupe offline, protect causal-order");
|
|
21
|
+
}
|
|
22
|
+
if (reservoir.totalPendingRows >= 20_000) {
|
|
23
|
+
return this.#toDecision("very_slow", "backlog remains very high");
|
|
24
|
+
}
|
|
25
|
+
if (reservoir.totalPendingRows >= 5_000) {
|
|
26
|
+
return this.#toDecision("slow", "backlog is elevated");
|
|
27
|
+
}
|
|
28
|
+
return this.#toDecision(this.#config.defaultTier, "normal operating posture");
|
|
29
|
+
}
|
|
30
|
+
#toDecision(tier, reason) {
|
|
31
|
+
const target = this.#resolveTierConfig(tier);
|
|
32
|
+
return {
|
|
33
|
+
tier,
|
|
34
|
+
targetEventsPerSecond: target.maxEventsPerSecond,
|
|
35
|
+
targetBatchSize: target.batchSize,
|
|
36
|
+
reason,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
#resolveTierConfig(tier) {
|
|
40
|
+
switch (tier) {
|
|
41
|
+
case "slow":
|
|
42
|
+
return this.#config.slow;
|
|
43
|
+
case "very_slow":
|
|
44
|
+
return this.#config.verySlow;
|
|
45
|
+
case "paused":
|
|
46
|
+
return this.#config.paused;
|
|
47
|
+
default:
|
|
48
|
+
return this.#config.open;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { ThrottleController, type ThrottleDecision, } from "./throttle/ThrottleController.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { ThrottleController, } from "./throttle/ThrottleController.js";
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { MonitorRuntime } from "../runtime/MonitorRuntime.js";
|
|
2
|
+
import type { MonitorConfig } from "../types/config.js";
|
|
3
|
+
import type { MonitorComponent, MonitorDeliveryMode, MonitorIngressDecision, MonitorIngressEvent } from "../types/events.js";
|
|
4
|
+
import type { InspectedMonitorSnapshot, MonitorSnapshot, ReplaySessionSnapshot } from "../types/snapshots.js";
|
|
5
|
+
export interface MonitorAdapterForwardContext {
|
|
6
|
+
rowId: number;
|
|
7
|
+
sourceStreamId: string | null;
|
|
8
|
+
deliveryMode: MonitorDeliveryMode;
|
|
9
|
+
replay: boolean;
|
|
10
|
+
}
|
|
11
|
+
export interface MonitorAdapterHandlers {
|
|
12
|
+
deliverToDedupe: (event: MonitorIngressEvent, context: MonitorAdapterForwardContext) => Promise<void> | void;
|
|
13
|
+
deliverToOrder: (event: MonitorIngressEvent, context: MonitorAdapterForwardContext) => Promise<void> | void;
|
|
14
|
+
onBuffered?: (event: MonitorIngressEvent, decision: MonitorIngressDecision, rowId: number) => Promise<void> | void;
|
|
15
|
+
onDecision?: (event: MonitorIngressEvent, decision: MonitorIngressDecision, rowId: number) => Promise<void> | void;
|
|
16
|
+
onReplayStateChange?: (snapshot: ReplaySessionSnapshot) => Promise<void> | void;
|
|
17
|
+
}
|
|
18
|
+
export interface MonitorIngestResult {
|
|
19
|
+
rowId: number;
|
|
20
|
+
decision: MonitorIngressDecision;
|
|
21
|
+
forwardedTo: "dedupe" | "causal-order" | "buffer";
|
|
22
|
+
}
|
|
23
|
+
export interface ReplayPumpResult {
|
|
24
|
+
snapshot: ReplaySessionSnapshot;
|
|
25
|
+
claimedCount: number;
|
|
26
|
+
deliveredCount: number;
|
|
27
|
+
completed: boolean;
|
|
28
|
+
}
|
|
29
|
+
export declare class TransportMonitorAdapter {
|
|
30
|
+
#private;
|
|
31
|
+
constructor(handlers: MonitorAdapterHandlers, config?: Partial<MonitorConfig>);
|
|
32
|
+
getRuntime(): MonitorRuntime;
|
|
33
|
+
getSnapshot(): MonitorSnapshot;
|
|
34
|
+
getInspectedSnapshot(): InspectedMonitorSnapshot;
|
|
35
|
+
getReplaySnapshot(): ReplaySessionSnapshot;
|
|
36
|
+
updateComponentHealth(component: MonitorComponent, update: Parameters<MonitorRuntime["updateComponentHealth"]>[1]): import("../types/snapshots.js").MonitorComponentHealthSnapshot;
|
|
37
|
+
observeHeartbeat(component: MonitorComponent, observedAt?: bigint, details?: Record<string, unknown>): import("../types/snapshots.js").MonitorComponentHealthSnapshot;
|
|
38
|
+
ingest(event: MonitorIngressEvent, sourceStreamId?: string | null): Promise<MonitorIngestResult>;
|
|
39
|
+
pumpReplayBatch(limit?: number): Promise<ReplayPumpResult>;
|
|
40
|
+
reconcileRecovery(limit?: number): Promise<ReplayPumpResult | null>;
|
|
41
|
+
close(): void;
|
|
42
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { MonitorRuntime } from "../runtime/MonitorRuntime.js";
|
|
2
|
+
export class TransportMonitorAdapter {
|
|
3
|
+
#runtime;
|
|
4
|
+
#handlers;
|
|
5
|
+
constructor(handlers, config = {}) {
|
|
6
|
+
this.#runtime = new MonitorRuntime(config);
|
|
7
|
+
this.#handlers = handlers;
|
|
8
|
+
}
|
|
9
|
+
getRuntime() {
|
|
10
|
+
return this.#runtime;
|
|
11
|
+
}
|
|
12
|
+
getSnapshot() {
|
|
13
|
+
return this.#runtime.getSnapshot();
|
|
14
|
+
}
|
|
15
|
+
getInspectedSnapshot() {
|
|
16
|
+
return this.#runtime.getInspectedSnapshot();
|
|
17
|
+
}
|
|
18
|
+
getReplaySnapshot() {
|
|
19
|
+
return this.#runtime.getReplaySnapshot();
|
|
20
|
+
}
|
|
21
|
+
updateComponentHealth(component, update) {
|
|
22
|
+
const snapshot = this.#runtime.updateComponentHealth(component, update);
|
|
23
|
+
void this.#notifyReplayChange();
|
|
24
|
+
return snapshot;
|
|
25
|
+
}
|
|
26
|
+
observeHeartbeat(component, observedAt, details) {
|
|
27
|
+
const snapshot = this.#runtime.observeHeartbeat(component, observedAt, details);
|
|
28
|
+
void this.#notifyReplayChange();
|
|
29
|
+
return snapshot;
|
|
30
|
+
}
|
|
31
|
+
async ingest(event, sourceStreamId) {
|
|
32
|
+
const { rowId, decision } = this.#runtime.ingestTransportEvent(event, sourceStreamId);
|
|
33
|
+
await this.#handlers.onDecision?.(event, decision, rowId);
|
|
34
|
+
if (decision.action === "buffer_only" || decision.action === "pause") {
|
|
35
|
+
await this.#handlers.onBuffered?.(event, decision, rowId);
|
|
36
|
+
return {
|
|
37
|
+
rowId,
|
|
38
|
+
decision,
|
|
39
|
+
forwardedTo: "buffer",
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
if (decision.deliveryMode === "dedupe_bypass") {
|
|
43
|
+
await this.#handlers.deliverToOrder(event, {
|
|
44
|
+
rowId,
|
|
45
|
+
sourceStreamId: sourceStreamId ?? null,
|
|
46
|
+
deliveryMode: decision.deliveryMode,
|
|
47
|
+
replay: false,
|
|
48
|
+
});
|
|
49
|
+
this.#runtime.acknowledgeIngressDelivery([rowId]);
|
|
50
|
+
return {
|
|
51
|
+
rowId,
|
|
52
|
+
decision,
|
|
53
|
+
forwardedTo: "causal-order",
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
await this.#handlers.deliverToDedupe(event, {
|
|
57
|
+
rowId,
|
|
58
|
+
sourceStreamId: sourceStreamId ?? null,
|
|
59
|
+
deliveryMode: decision.deliveryMode,
|
|
60
|
+
replay: false,
|
|
61
|
+
});
|
|
62
|
+
this.#runtime.acknowledgeIngressDelivery([rowId]);
|
|
63
|
+
return {
|
|
64
|
+
rowId,
|
|
65
|
+
decision,
|
|
66
|
+
forwardedTo: "dedupe",
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
async pumpReplayBatch(limit) {
|
|
70
|
+
const batch = this.#runtime.claimReplayBatch(limit);
|
|
71
|
+
await this.#notifyReplayChange();
|
|
72
|
+
if (batch.entries.length === 0) {
|
|
73
|
+
await this.#notifyReplayChange();
|
|
74
|
+
return {
|
|
75
|
+
snapshot: this.#runtime.getReplaySnapshot(),
|
|
76
|
+
claimedCount: 0,
|
|
77
|
+
deliveredCount: 0,
|
|
78
|
+
completed: batch.isDrainComplete,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
const claimedRowIds = [];
|
|
82
|
+
try {
|
|
83
|
+
for (const entry of batch.entries) {
|
|
84
|
+
claimedRowIds.push(entry.rowId);
|
|
85
|
+
await this.#handlers.deliverToDedupe(entry.event, {
|
|
86
|
+
rowId: entry.rowId,
|
|
87
|
+
sourceStreamId: entry.sourceStreamId,
|
|
88
|
+
deliveryMode: "replay_through_dedupe",
|
|
89
|
+
replay: true,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
const snapshot = this.#runtime.acknowledgeReplayBatch(claimedRowIds);
|
|
93
|
+
await this.#notifyReplayChange();
|
|
94
|
+
return {
|
|
95
|
+
snapshot,
|
|
96
|
+
claimedCount: claimedRowIds.length,
|
|
97
|
+
deliveredCount: claimedRowIds.length,
|
|
98
|
+
completed: snapshot.state === "completed" || snapshot.state === "idle",
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
const message = error instanceof Error ? error.message : "unknown replay failure";
|
|
103
|
+
const snapshot = this.#runtime.failReplay(message, claimedRowIds);
|
|
104
|
+
await this.#notifyReplayChange();
|
|
105
|
+
return {
|
|
106
|
+
snapshot,
|
|
107
|
+
claimedCount: claimedRowIds.length,
|
|
108
|
+
deliveredCount: 0,
|
|
109
|
+
completed: false,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
async reconcileRecovery(limit) {
|
|
114
|
+
const snapshot = this.#runtime.getReplaySnapshot();
|
|
115
|
+
const backlog = this.#runtime.getReservoirStats().totalPendingRows;
|
|
116
|
+
if (backlog === 0) {
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
if (snapshot.state === "idle" || snapshot.state === "failed") {
|
|
120
|
+
this.#runtime.queueReplay();
|
|
121
|
+
await this.#notifyReplayChange();
|
|
122
|
+
}
|
|
123
|
+
const nextSnapshot = this.#runtime.getReplaySnapshot();
|
|
124
|
+
if (nextSnapshot.state !== "queued" && nextSnapshot.state !== "running") {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
return this.pumpReplayBatch(limit);
|
|
128
|
+
}
|
|
129
|
+
close() {
|
|
130
|
+
this.#runtime.close();
|
|
131
|
+
}
|
|
132
|
+
async #notifyReplayChange() {
|
|
133
|
+
await this.#handlers.onReplayStateChange?.(this.#runtime.getReplaySnapshot());
|
|
134
|
+
}
|
|
135
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { TransportMonitorAdapter, type MonitorAdapterForwardContext, type MonitorAdapterHandlers, type MonitorIngestResult, type ReplayPumpResult, } from "./transport/TransportMonitorAdapter.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { TransportMonitorAdapter, } from "./transport/TransportMonitorAdapter.js";
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { MonitorThrottleTier } from "./events.js";
|
|
2
|
+
export interface MonitorReservoirConfig {
|
|
3
|
+
databasePath: string;
|
|
4
|
+
rollingBufferWindowMs: bigint;
|
|
5
|
+
fullOutageMaxWindowMs: bigint;
|
|
6
|
+
pruneIntervalMs: bigint;
|
|
7
|
+
}
|
|
8
|
+
export interface MonitorTransportConfig {
|
|
9
|
+
heartbeatGraceMs: bigint;
|
|
10
|
+
reconnectBurstWindowMs: bigint;
|
|
11
|
+
sourceLabel?: string;
|
|
12
|
+
}
|
|
13
|
+
export interface MonitorComponentHealthConfig {
|
|
14
|
+
degradedAfterMs: bigint;
|
|
15
|
+
offlineAfterMs: bigint;
|
|
16
|
+
}
|
|
17
|
+
export interface MonitorHealthConfig {
|
|
18
|
+
transport: MonitorComponentHealthConfig;
|
|
19
|
+
dedupe: MonitorComponentHealthConfig;
|
|
20
|
+
causalOrder: MonitorComponentHealthConfig;
|
|
21
|
+
}
|
|
22
|
+
export interface MonitorThrottleTierConfig {
|
|
23
|
+
maxEventsPerSecond: number;
|
|
24
|
+
batchSize: number;
|
|
25
|
+
}
|
|
26
|
+
export interface MonitorThrottleConfig {
|
|
27
|
+
open: MonitorThrottleTierConfig;
|
|
28
|
+
slow: MonitorThrottleTierConfig;
|
|
29
|
+
verySlow: MonitorThrottleTierConfig;
|
|
30
|
+
paused: MonitorThrottleTierConfig;
|
|
31
|
+
defaultTier: MonitorThrottleTier;
|
|
32
|
+
}
|
|
33
|
+
export interface MonitorReplayConfig {
|
|
34
|
+
healthConfirmationHeartbeats: number;
|
|
35
|
+
pauseLiveFlowDuringReplay: boolean;
|
|
36
|
+
retryBackoffMs: bigint;
|
|
37
|
+
}
|
|
38
|
+
export interface MonitorConfig {
|
|
39
|
+
reservoir: MonitorReservoirConfig;
|
|
40
|
+
transport: MonitorTransportConfig;
|
|
41
|
+
health: MonitorHealthConfig;
|
|
42
|
+
throttle: MonitorThrottleConfig;
|
|
43
|
+
replay: MonitorReplayConfig;
|
|
44
|
+
now?: () => bigint;
|
|
45
|
+
}
|
|
46
|
+
export declare function createDefaultMonitorConfig(): MonitorConfig;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
export function createDefaultMonitorConfig() {
|
|
2
|
+
return {
|
|
3
|
+
reservoir: {
|
|
4
|
+
databasePath: ":memory:",
|
|
5
|
+
rollingBufferWindowMs: 4n * 60n * 60n * 1000n,
|
|
6
|
+
fullOutageMaxWindowMs: 6n * 60n * 60n * 1000n,
|
|
7
|
+
pruneIntervalMs: 60000n,
|
|
8
|
+
},
|
|
9
|
+
transport: {
|
|
10
|
+
heartbeatGraceMs: 15000n,
|
|
11
|
+
reconnectBurstWindowMs: 30000n,
|
|
12
|
+
sourceLabel: "@causal-order/transport",
|
|
13
|
+
},
|
|
14
|
+
health: {
|
|
15
|
+
transport: {
|
|
16
|
+
degradedAfterMs: 10000n,
|
|
17
|
+
offlineAfterMs: 30000n,
|
|
18
|
+
},
|
|
19
|
+
dedupe: {
|
|
20
|
+
degradedAfterMs: 10000n,
|
|
21
|
+
offlineAfterMs: 30000n,
|
|
22
|
+
},
|
|
23
|
+
causalOrder: {
|
|
24
|
+
degradedAfterMs: 10000n,
|
|
25
|
+
offlineAfterMs: 30000n,
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
throttle: {
|
|
29
|
+
open: {
|
|
30
|
+
maxEventsPerSecond: 5_000,
|
|
31
|
+
batchSize: 500,
|
|
32
|
+
},
|
|
33
|
+
slow: {
|
|
34
|
+
maxEventsPerSecond: 1_000,
|
|
35
|
+
batchSize: 200,
|
|
36
|
+
},
|
|
37
|
+
verySlow: {
|
|
38
|
+
maxEventsPerSecond: 250,
|
|
39
|
+
batchSize: 50,
|
|
40
|
+
},
|
|
41
|
+
paused: {
|
|
42
|
+
maxEventsPerSecond: 0,
|
|
43
|
+
batchSize: 0,
|
|
44
|
+
},
|
|
45
|
+
defaultTier: "open",
|
|
46
|
+
},
|
|
47
|
+
replay: {
|
|
48
|
+
healthConfirmationHeartbeats: 2,
|
|
49
|
+
pauseLiveFlowDuringReplay: true,
|
|
50
|
+
retryBackoffMs: 5000n,
|
|
51
|
+
},
|
|
52
|
+
now: () => BigInt(Date.now()),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
export type MonitorComponent = "transport" | "dedupe" | "causal-order";
|
|
2
|
+
export type MonitorHealthState = "online" | "degraded" | "offline";
|
|
3
|
+
export type MonitorRoutingMode = "normal" | "dedupe_bypass_throttled" | "order_buffer_only" | "full_outage_buffer" | "replay_through_dedupe";
|
|
4
|
+
export type MonitorThrottleTier = "open" | "slow" | "very_slow" | "paused";
|
|
5
|
+
export type MonitorSourcePath = "transport_normalized_stream" | "deduped_observation";
|
|
6
|
+
export type MonitorDeliveryMode = "normal" | "dedupe_bypass" | "order_buffer_only" | "full_outage_buffer" | "replay_through_dedupe";
|
|
7
|
+
export type MonitorReplaySessionState = "idle" | "queued" | "running" | "completed" | "failed" | "aborted";
|
|
8
|
+
export type MonitorReplayTargetPath = "dedupe_then_order";
|
|
9
|
+
export type MonitorPeerState = "connected" | "disconnected" | "degraded";
|
|
10
|
+
export type MonitorIngressAction = "accept" | "buffer_only" | "pause";
|
|
11
|
+
export interface MonitorClock extends Record<string, unknown> {
|
|
12
|
+
physicalTimeMs: bigint;
|
|
13
|
+
}
|
|
14
|
+
export interface MonitorEventPayload {
|
|
15
|
+
traceId?: string | null;
|
|
16
|
+
entityId?: string | null;
|
|
17
|
+
service?: string;
|
|
18
|
+
operation?: string;
|
|
19
|
+
[key: string]: unknown;
|
|
20
|
+
}
|
|
21
|
+
export interface MonitorIngressEvent {
|
|
22
|
+
id: string;
|
|
23
|
+
nodeId: string;
|
|
24
|
+
clock: MonitorClock;
|
|
25
|
+
payload: MonitorEventPayload;
|
|
26
|
+
sequence?: bigint;
|
|
27
|
+
traceId?: string | null;
|
|
28
|
+
ingestedAt?: bigint;
|
|
29
|
+
[key: string]: unknown;
|
|
30
|
+
}
|
|
31
|
+
export interface MonitorPeerStateEvent {
|
|
32
|
+
peerId: string;
|
|
33
|
+
state: MonitorPeerState;
|
|
34
|
+
observedAt: bigint;
|
|
35
|
+
details?: Record<string, unknown>;
|
|
36
|
+
}
|
|
37
|
+
export interface MonitorHealthUpdate {
|
|
38
|
+
state: MonitorHealthState;
|
|
39
|
+
observedAt?: bigint;
|
|
40
|
+
reasonCode?: string | null;
|
|
41
|
+
details?: Record<string, unknown>;
|
|
42
|
+
}
|
|
43
|
+
export interface MonitorIngressDecision {
|
|
44
|
+
action: MonitorIngressAction;
|
|
45
|
+
routingMode: MonitorRoutingMode;
|
|
46
|
+
deliveryMode: MonitorDeliveryMode;
|
|
47
|
+
throttleTier: MonitorThrottleTier;
|
|
48
|
+
targetEventsPerSecond: number;
|
|
49
|
+
targetBatchSize: number;
|
|
50
|
+
reason: string;
|
|
51
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|