bunsane 0.2.8 → 0.2.10

package/core/remote/RemoteManager.ts

@@ -0,0 +1,400 @@
+/**
+ * Remote Communication: RemoteManager
+ *
+ * Phase 1: events + RPC over Redis Streams.
+ * - emit(target, event, data): fire-and-forget XADD to `remote:<target>`
+ * - call(target, method, data, options): RPC with correlationId + deadline
+ * - Dedicated Redis connections:
+ *     publisher   — XADD only (non-blocking, retries enabled)
+ *     consumer    — XREADGROUP BLOCK (retries=null, required for blocking)
+ *     rpcListener — XREAD BLOCK on per-instance response stream
+ * - Graceful shutdown drains pending RPC calls within `shutdownDrainMs`
+ */
+
+import Redis, { type RedisOptions } from "ioredis";
+import { logger } from "../Logger";
+import { StreamConsumer } from "./StreamConsumer";
+import { RpcCaller } from "./RpcCaller";
+import { OutboxWorker } from "./OutboxWorker";
+import { ensureOutboxSchema } from "./outboxSchema";
+import { CircuitBreaker } from "./CircuitBreaker";
+import { RemoteMetrics, type RemoteMetricsSnapshot } from "./metrics";
+import { collectRemoteHealth, type RemoteHealthCheck } from "./health";
+import db from "../../database";
+import type {
+    CallOptions,
+    EmitOptions,
+    RemoteHandler,
+    RemoteManagerConfig,
+    RpcHandler,
+} from "./types";
+
+const loggerInstance = logger.child({ scope: "RemoteManager" });
+
+function buildRedisOptions(blocking: boolean): RedisOptions {
+    return {
+        host: process.env.REDIS_HOST || "localhost",
+        port: parseInt(process.env.REDIS_PORT || "6379", 10),
+        password: process.env.REDIS_PASSWORD,
+        db: parseInt(process.env.REDIS_DB || "0", 10),
+        maxRetriesPerRequest: blocking ? null : 3,
+        enableReadyCheck: false,
+        retryStrategy: (times: number) => Math.min(times * 50, 2000),
+    };
+}
+
+export class RemoteManager {
+    private publisher: Redis | null = null;
+    private consumerRedis: Redis | null = null;
+    private rpcListenerRedis: Redis | null = null;
+    private consumer: StreamConsumer | null = null;
+    private caller: RpcCaller | null = null;
+    private outboxWorker: OutboxWorker | null = null;
+    private breaker: CircuitBreaker;
+    private metrics = new RemoteMetrics();
+    private config: RemoteManagerConfig;
+    private _instanceId: string;
+    private started = false;
+
+    constructor(config: RemoteManagerConfig) {
+        this.config = config;
+        this._instanceId = crypto.randomUUID();
+        this.breaker = new CircuitBreaker({
+            threshold: config.circuitBreakerThreshold,
+            resetTimeoutMs: config.circuitBreakerResetMs,
+        });
+        this.breaker.onTrip = () => this.metrics.cbTripped();
+        this.breaker.onReject = () => this.metrics.cbRejected();
+    }
+
+    getMetrics(): RemoteMetricsSnapshot & {
+        circuitBreaker: RemoteMetricsSnapshot["circuitBreaker"] & {
+            state: string;
+        };
+    } {
+        const snap = this.metrics.getSnapshot();
+        return {
+            ...snap,
+            circuitBreaker: {
+                ...snap.circuitBreaker,
+                state: this.breaker.getState(),
+            },
+        };
+    }
+
+    getCircuitBreaker(): CircuitBreaker {
+        return this.breaker;
+    }
+
+    async health(): Promise<RemoteHealthCheck> {
+        return collectRemoteHealth({
+            publisher: this.publisher,
+            consumerRedis: this.consumerRedis,
+            streamKey: `${this.streamPrefix}${this.config.appName}`,
+            consumerGroup:
+                this.config.consumerGroup ?? this.config.appName,
+            dlqStream: `${this.streamPrefix}${this.config.appName}:dlq`,
+            outboxEnabled: this.config.enableOutbox ?? false,
+            db,
+            breaker: this.breaker,
+        });
+    }
+
+    get appName(): string {
+        return this.config.appName;
+    }
+
+    get instanceId(): string {
+        return this._instanceId;
+    }
+
+    get streamPrefix(): string {
+        return this.config.streamPrefix ?? "remote:";
+    }
+
+    get responseStream(): string {
+        return `rpc:responses:${this._instanceId}`;
+    }
+
+    /**
+     * Emit an event.
+     *
+     * Without `{ trx }`: direct XADD to Redis (fire-and-forget, no DB write).
+     * With `{ trx }`: insert a row into `remote_outbox` within the caller's
+     * transaction. The OutboxWorker publishes the row to Redis after commit.
+     *
+     * Returns the Redis message id (direct path) or the outbox row id
+     * (transactional path).
+     */
+    async emit(
+        target: string,
+        event: string,
+        data: unknown,
+        options: EmitOptions = {}
+    ): Promise<string | null> {
+        if (!this.publisher) {
+            throw new Error(
+                "RemoteManager not started — call start() before emit()"
+            );
+        }
+
+        if (options.trx) {
+            const trx = options.trx as any;
+            const rows = await trx`
+                INSERT INTO remote_outbox (target, event, data)
+                VALUES (${target}, ${event}, ${data})
+                RETURNING id
+            `;
+            const id = rows?.[0]?.id ?? null;
+            this.metrics.emitOutbox();
+            if (this.config.enableLogging) {
+                loggerInstance.debug(
+                    `emit outbox → target=${target} event=${event} id=${id}`
+                );
+            }
+            return id;
+        }
+
+        const stream = `${this.streamPrefix}${target}`;
+        const envelope = JSON.stringify({
+            kind: "event",
+            sourceApp: this.config.appName,
+            event,
+            data,
+            emittedAt: Date.now(),
+        });
+        try {
+            const publisher = this.publisher;
+            const id = await this.breaker.exec(() =>
+                publisher.xadd(stream, "*", "data", envelope)
+            );
+            this.metrics.emitDirect();
+            if (this.config.enableLogging) {
+                loggerInstance.debug(
+                    `emit → ${stream} event=${event} id=${id}`
+                );
+            }
+            return id;
+        } catch (error) {
+            this.metrics.emitFailed();
+            throw error;
+        }
+    }
+
+    /**
+     * RPC call — awaits a response or rejects on timeout/error.
+     * Throws `RemoteError { code: "INVALID_TARGET" }` for broadcast target "*".
+     */
+    async call<T = unknown>(
+        target: string,
+        method: string,
+        data: unknown,
+        options: CallOptions = {}
+    ): Promise<T> {
+        if (!this.caller) {
+            throw new Error(
+                "RemoteManager not started — call start() before call()"
+            );
+        }
+        return this.caller.call<T>(
+            target,
+            method,
+            data,
+            this.streamPrefix,
+            this.config.appName,
+            options
+        );
+    }
+
+    on(event: string, fn: RemoteHandler, handlerId: string): void {
+        if (!this.consumer) {
+            throw new Error(
+                "RemoteManager consumer not initialized — call start() first"
+            );
+        }
+        this.consumer.addHandler(event, fn, handlerId);
+    }
+
+    onRpc(event: string, fn: RpcHandler, handlerId: string): void {
+        if (!this.consumer) {
+            throw new Error(
+                "RemoteManager consumer not initialized — call start() first"
+            );
+        }
+        this.consumer.addRpcHandler(event, fn, handlerId);
+    }
+
+    async start(): Promise<void> {
+        if (this.started) return;
+
+        const factory =
+            this.config.redisFactory ??
+            ((blocking: boolean) => new Redis(buildRedisOptions(blocking)));
+
+        this.publisher = factory(false) as Redis;
+        this.consumerRedis = factory(true) as Redis;
+        this.rpcListenerRedis = factory(true) as Redis;
+
+        for (const [name, client] of [
+            ["publisher", this.publisher],
+            ["consumer", this.consumerRedis],
+            ["rpcListener", this.rpcListenerRedis],
+        ] as const) {
+            client.on("error", (err) => {
+                loggerInstance.warn(
+                    { err, name, msg: `${name} Redis error` }
+                );
+            });
+        }
+
+        this.consumer = new StreamConsumer(
+            this.consumerRedis,
+            this.publisher,
+            this.config,
+            this.metrics
+        );
+        await this.consumer.start();
+
+        this.caller = new RpcCaller(
+            this.rpcListenerRedis,
+            this.publisher,
+            {
+                instanceId: this._instanceId,
+                responseStream: this.responseStream,
+                defaultTimeout: this.config.defaultCallTimeout ?? 5000,
+                responseStreamMaxLen:
+                    this.config.responseStreamMaxLen ?? 1000,
+                enableLogging: this.config.enableLogging ?? false,
+            },
+            this.breaker,
+            this.metrics
+        );
+        await this.caller.start();
+
+        if (this.config.enableOutbox) {
+            try {
+                await ensureOutboxSchema(db);
+                this.outboxWorker = new OutboxWorker(
+                    db,
+                    this.publisher,
+                    {
+                        sourceApp: this.config.appName,
+                        streamPrefix: this.streamPrefix,
+                        pollIntervalMs:
+                            this.config.outboxPollIntervalMs ?? 1000,
+                        batchSize: this.config.outboxBatchSize ?? 100,
+                        enableLogging: this.config.enableLogging ?? false,
+                    },
+                    this.metrics
+                );
+                await this.outboxWorker.start();
+            } catch (error) {
+                loggerInstance.error(
+                    { err: error, msg: "Failed to start OutboxWorker" }
+                );
+                this.outboxWorker = null;
+            }
+        }
+
+        this.started = true;
+        loggerInstance.info(
+            `RemoteManager started app="${this.config.appName}" instance=${this._instanceId} outbox=${this.config.enableOutbox ?? false}`
+        );
+    }
+
+    async shutdown(): Promise<void> {
+        if (!this.started) return;
+        this.started = false;
+
+        // 1. Stop outbox worker first — best-effort flush so committed rows
+        //    emitted right before shutdown still reach Redis.
+        if (this.outboxWorker) {
+            try {
+                await this.outboxWorker.flush();
+            } catch (error) {
+                loggerInstance.warn(
+                    { err: error, msg: "OutboxWorker flush error" }
+                );
+            }
+            try {
+                await this.outboxWorker.stop();
+            } catch (error) {
+                loggerInstance.warn(
+                    { err: error, msg: "OutboxWorker stop error" }
+                );
+            }
+            this.outboxWorker = null;
+        }
+
+        // 2. Drain pending RPC calls first (caller rejects new)
+        const drainMs = this.config.shutdownDrainMs ?? 2000;
+        if (this.caller) {
+            try {
+                await this.caller.stop(drainMs);
+            } catch (error) {
+                loggerInstance.warn(
+                    { err: error, msg: "RpcCaller stop error" }
+                );
+            }
+            this.caller = null;
+        }
+
+        // 3. Stop consumer — waits for in-flight handler
+        if (this.consumer) {
+            try {
+                await this.consumer.stop();
+            } catch (error) {
+                loggerInstance.warn(
+                    { err: error, msg: "Consumer stop error" }
+                );
+            }
+            this.consumer = null;
+        }
+
+        // 4. Disconnect Redis conns
+        if (this.rpcListenerRedis) {
+            try {
+                this.rpcListenerRedis.disconnect();
+            } catch (error) {
+                loggerInstance.warn(
+                    { err: error, msg: "RPC listener disconnect error" }
+                );
+            }
+            this.rpcListenerRedis = null;
+        }
+
+        if (this.consumerRedis) {
+            try {
+                this.consumerRedis.disconnect();
+            } catch (error) {
+                loggerInstance.warn(
+                    { err: error, msg: "Consumer Redis disconnect error" }
+                );
+            }
+            this.consumerRedis = null;
+        }
+
+        if (this.publisher) {
+            try {
+                await this.publisher.quit();
+            } catch (error) {
+                loggerInstance.warn(
+                    { err: error, msg: "Publisher quit error" }
+                );
+            }
+            this.publisher = null;
+        }
+
+        loggerInstance.info("RemoteManager shutdown completed");
+    }
+}
+
+let remoteManagerInstance: RemoteManager | null = null;
+
+export function getRemoteManager(): RemoteManager | null {
+    return remoteManagerInstance;
+}
+
+export function setRemoteManager(instance: RemoteManager | null): void {
+    remoteManagerInstance = instance;
+}

package/core/remote/RpcCaller.ts

@@ -0,0 +1,310 @@
+/**
+ * Remote Communication: RpcCaller
+ *
+ * Phase 1: Request/response over Redis Streams.
+ * - Per-instance response stream `rpc:responses:<instanceId>` — no consumer group
+ * - Dedicated blocking Redis connection for XREAD $
+ * - Pending map keyed by correlationId, timer per call
+ * - Response writes capped via MAXLEN ~ to prevent unbounded growth
+ */
+
+import Redis from "ioredis";
+import { logger } from "../Logger";
+import { RemoteError } from "./types";
+import type {
+    CallOptions,
+    RpcResponse,
+} from "./types";
+import type { CircuitBreaker } from "./CircuitBreaker";
+import type { RemoteMetrics } from "./metrics";
+
+const loggerInstance = logger.child({ scope: "RpcCaller" });
+
+interface PendingEntry {
+    resolve: (value: unknown) => void;
+    reject: (err: unknown) => void;
+    timer: ReturnType<typeof setTimeout>;
+    method: string;
+    target: string;
+}
+
+export interface RpcCallerConfig {
+    instanceId: string;
+    responseStream: string;
+    defaultTimeout: number;
+    responseStreamMaxLen: number;
+    enableLogging: boolean;
+}
+
+export class RpcCaller {
+    private listenerRedis: Redis;
+    private publisher: Redis;
+    private config: RpcCallerConfig;
+    private pending = new Map<string, PendingEntry>();
+    private running = false;
+    private draining = false;
+    private loopPromise: Promise<void> | null = null;
+    private lastId = "$";
+    private breaker?: CircuitBreaker;
+    private metrics?: RemoteMetrics;
+
+    constructor(
+        listenerRedis: Redis,
+        publisher: Redis,
+        config: RpcCallerConfig,
+        breaker?: CircuitBreaker,
+        metrics?: RemoteMetrics
+    ) {
+        this.listenerRedis = listenerRedis;
+        this.publisher = publisher;
+        this.config = config;
+        this.breaker = breaker;
+        this.metrics = metrics;
+    }
+
+    get responseStream(): string {
+        return this.config.responseStream;
+    }
+
+    get instanceId(): string {
+        return this.config.instanceId;
+    }
+
+    async start(): Promise<void> {
+        if (this.running) return;
+        this.running = true;
+        this.loopPromise = this.listenLoop();
+        loggerInstance.info(
+            `RpcCaller started: responseStream=${this.config.responseStream}`
+        );
+    }
+
+    /**
+     * Stop accepting new calls and drain pending within `drainMs`.
+     * Pending calls remaining after drain are rejected with code="SHUTDOWN".
+     */
+    async stop(drainMs: number): Promise<void> {
+        if (!this.running) return;
+        this.draining = true;
+
+        // Wait for pending to settle — bounded by drainMs
+        const deadline = Date.now() + drainMs;
+        while (this.pending.size > 0 && Date.now() < deadline) {
+            await this.sleep(50);
+        }
+
+        // Force-reject remaining
+        for (const [id, entry] of this.pending) {
+            clearTimeout(entry.timer);
+            entry.reject(
+                new RemoteError(
+                    `RPC ${entry.method} cancelled by shutdown`,
+                    { code: "SHUTDOWN", sourceApp: entry.target }
+                )
+            );
+            this.pending.delete(id);
+        }
+
+        this.running = false;
+        if (this.loopPromise) {
+            await this.loopPromise.catch(() => {});
+            this.loopPromise = null;
+        }
+        loggerInstance.info("RpcCaller stopped");
+    }
+
+    async call<T = unknown>(
+        target: string,
+        method: string,
+        data: unknown,
+        requestStreamPrefix: string,
+        sourceApp: string,
+        options: CallOptions = {}
+    ): Promise<T> {
+        if (target === "*") {
+            throw new RemoteError(
+                "call() does not support broadcast target '*' — use emit() for fan-out",
+                { code: "INVALID_TARGET" }
+            );
+        }
+        if (this.draining || !this.running) {
+            throw new RemoteError(
+                `RPC ${method} rejected — RemoteManager draining/stopped`,
+                { code: "SHUTDOWN" }
+            );
+        }
+
+        this.metrics?.rpcCalled();
+
+        const timeout = options.timeout ?? this.config.defaultTimeout;
+        const correlationId = crypto.randomUUID();
+        const deadline = Date.now() + timeout;
+        const requestStream = `${requestStreamPrefix}${target}`;
+
+        return new Promise<T>((resolve, reject) => {
+            const timer = setTimeout(() => {
+                if (this.pending.delete(correlationId)) {
+                    this.metrics?.rpcTimedOut();
+                    reject(
+                        new RemoteError(
+                            `RPC ${method} to "${target}" timed out after ${timeout}ms`,
+                            { code: "TIMEOUT", sourceApp: target }
+                        )
+                    );
+                }
+            }, timeout);
+
+            this.pending.set(correlationId, {
+                resolve: (v) => {
+                    this.metrics?.rpcSucceeded();
+                    (resolve as (x: unknown) => void)(v);
+                },
+                reject: (err) => {
+                    this.metrics?.rpcFailed();
+                    reject(err);
+                },
+                timer,
+                method,
+                target,
+            });
+
+            const envelope = JSON.stringify({
+                kind: "rpc_request",
+                sourceApp,
+                event: method,
+                data,
+                emittedAt: Date.now(),
+                correlationId,
+                replyTo: this.config.responseStream,
+                deadline,
+            });
+
+            const publish = this.breaker
+                ? this.breaker.exec(() =>
+                      this.publisher.xadd(requestStream, "*", "data", envelope)
+                  )
+                : this.publisher.xadd(
+                      requestStream,
+                      "*",
+                      "data",
+                      envelope
+                  );
+
+            Promise.resolve(publish)
+                .then((id) => {
+                    if (this.config.enableLogging) {
+                        loggerInstance.debug(
+                            `call → ${requestStream} method=${method} id=${id} cid=${correlationId}`
+                        );
+                    }
+                })
+                .catch((err) => {
+                    if (this.pending.delete(correlationId)) {
+                        clearTimeout(timer);
+                        const code =
+                            (err && (err as any).code) === "CIRCUIT_OPEN"
+                                ? "CIRCUIT_OPEN"
+                                : "PUBLISH_FAILED";
+                        reject(
+                            new RemoteError(
+                                `Failed to publish RPC request: ${err?.message ?? err}`,
+                                { code, sourceApp: target }
+                            )
+                        );
+                    }
+                });
+        });
+    }
+
+    private async listenLoop(): Promise<void> {
+        while (this.running) {
+            try {
+                const result: any = await (this.listenerRedis as any).xread(
+                    "COUNT",
+                    50,
+                    "BLOCK",
+                    2000,
+                    "STREAMS",
+                    this.config.responseStream,
+                    this.lastId
+                );
+
+                if (!result || !this.running) continue;
+
+                for (const [, entries] of result) {
+                    for (const [msgId, fields] of entries) {
+                        this.lastId = msgId;
+                        const response = this.parseResponse(fields);
+                        if (response) {
+                            this.dispatchResponse(response);
+                        }
+                    }
+                }
+            } catch (error: any) {
+                if (!this.running) break;
+                if (String(error?.message).includes("Connection is closed")) {
+                    break;
+                }
+                loggerInstance.error(
+                    { err: error, msg: "Response listen error" }
+                );
+                await this.sleep(500);
+            }
+        }
+    }
+
+    private dispatchResponse(response: RpcResponse): void {
+        const entry = this.pending.get(response.correlationId);
+        if (!entry) {
+            // Orphan response — caller already timed out or never existed
+            if (this.config.enableLogging) {
+                loggerInstance.debug(
+                    `Orphan response cid=${response.correlationId}`
+                );
+            }
+            return;
+        }
+        clearTimeout(entry.timer);
+        this.pending.delete(response.correlationId);
+
+        if (response.success) {
+            entry.resolve(response.result);
+        } else {
+            entry.reject(
+                new RemoteError(response.error.message, {
+                    code: response.error.code,
+                    sourceApp: response.sourceApp,
+                    extensions: response.error.extensions,
+                })
+            );
+        }
+    }
+
+    private parseResponse(fields: string[]): RpcResponse | null {
+        let payload: string | undefined;
+        for (let i = 0; i < fields.length - 1; i += 2) {
+            if (fields[i] === "data") {
+                payload = fields[i + 1];
+                break;
+            }
+        }
+        if (!payload) return null;
+        try {
+            const parsed = JSON.parse(payload);
+            if (
+                !parsed ||
+                typeof parsed.correlationId !== "string" ||
+                typeof parsed.success !== "boolean"
+            ) {
+                return null;
+            }
+            return parsed as RpcResponse;
+        } catch {
+            return null;
+        }
+    }
+
+    private sleep(ms: number): Promise<void> {
+        return new Promise((r) => setTimeout(r, ms));
+    }
+}