bunsane 0.2.9 → 0.2.10

package/core/App.ts

@@ -26,6 +26,12 @@ import studioEndpoint from "../endpoints";
 import { type Middleware, composeMiddleware } from "./Middleware";
 import { deepHealthCheck, readinessCheck } from "./health";
 import { validateEnv } from "./validateEnv";
+import {
+    RemoteManager,
+    registerRemoteHandlers,
+    setRemoteManager,
+} from "./remote";
+import type { RemoteManagerConfig } from "./remote";
 
 export type CorsConfig = {
     origin?: string | string[] | ((origin: string) => boolean);
@@ -70,6 +76,8 @@ export default class App {
     private composedHandler: ((req: Request) => Promise<Response>) | null = null;
 
     private studioEnabled: boolean = false;
+    private remote: RemoteManager | null = null;
+    private remoteConfig: Partial<RemoteManagerConfig> | null = null;
     private server: ReturnType<typeof Bun.serve> | null = null;
     private isShuttingDown = false;
     private isReady = false;
@@ -254,6 +262,40 @@ export default class App {
                             `Registered scheduled tasks for ${services.length} services`
                         );
 
+                        // Initialize RemoteManager (opt-in via enableRemote())
+                        if (this.remoteConfig) {
+                            try {
+                                const rmConfig: RemoteManagerConfig = {
+                                    appName:
+                                        this.remoteConfig.appName ||
+                                        this.name,
+                                    ...this.remoteConfig,
+                                };
+                                this.remote = new RemoteManager(rmConfig);
+                                setRemoteManager(this.remote);
+                                await this.remote.start();
+
+                                for (const service of services) {
+                                    try {
+                                        registerRemoteHandlers(service);
+                                    } catch (error) {
+                                        logger.warn(
+                                            `Failed to register remote handlers for service ${service.constructor.name}`
+                                        );
+                                        logger.warn(error);
+                                    }
+                                }
+                                logger.info(
+                                    `RemoteManager initialized for app "${rmConfig.appName}"`
+                                );
+                            } catch (error) {
+                                logger.error(
+                                    "Failed to start RemoteManager:"
+                                );
+                                logger.error(error);
+                            }
+                        }
+
                         // Collect REST endpoints from all services
                         for (const service of services) {
                             const endpoints = (service.constructor as any)
@@ -561,6 +603,31 @@ export default class App {
                 ), req);
             }
 
+            // Remote health check
+            if (url.pathname === "/health/remote") {
+                clearTimeout(timeoutId);
+                if (!this.remote) {
+                    return this.addCorsHeaders(new Response(
+                        JSON.stringify({
+                            healthy: false,
+                            error: "Remote subsystem not enabled",
+                        }),
+                        {
+                            status: 503,
+                            headers: { "Content-Type": "application/json" },
+                        }
+                    ), req);
+                }
+                const health = await this.remote.health();
+                return this.addCorsHeaders(new Response(
+                    JSON.stringify(health),
+                    {
+                        status: health.healthy ? 200 : 503,
+                        headers: { "Content-Type": "application/json" },
+                    }
+                ), req);
+            }
+
             // Readiness probe
             if (url.pathname === "/health/ready") {
                 clearTimeout(timeoutId);
@@ -911,10 +978,27 @@ export default class App {
         this.name = name;
     }
 
+    public getName(): string {
+        return this.name;
+    }
+
     public setVersion(version: string) {
         this.version = version;
     }
 
+    /**
+     * Enable remote cross-app communication over Redis Streams.
+     * Must be called before `init()` (initialization happens in SYSTEM_READY).
+     * `appName` defaults to the app name.
+     */
+    public enableRemote(config: Partial<RemoteManagerConfig> = {}) {
+        this.remoteConfig = config;
+    }
+
+    public getRemote(): RemoteManager | null {
+        return this.remote;
+    }
+
     public subscribeAppReady(callback: () => void) {
         this.appReadyCallbacks.push(callback);
     }
@@ -1023,6 +1107,7 @@ export default class App {
             cache: cacheStats,
             scheduler: SchedulerManager.getInstance().getMetrics(),
             preparedStatements: preparedStatementCache.getStats(),
+            remote: this.remote ? this.remote.getMetrics() : null,
         };
     }
 
@@ -1127,6 +1212,18 @@ export default class App {
             logger.warn({ scope: 'app', component: 'App', msg: 'Scheduler stop error', error });
         }
 
+        // Shutdown RemoteManager (after scheduler, before cache — DB still available)
+        if (this.remote) {
+            try {
+                await this.remote.shutdown();
+                setRemoteManager(null);
+                this.remote = null;
+                logger.info({ scope: 'app', component: 'App', msg: 'RemoteManager shutdown' });
+            } catch (error) {
+                logger.warn({ scope: 'app', component: 'App', msg: 'RemoteManager shutdown error', error });
+            }
+        }
+
         // Shutdown cache
         try {
             const { CacheManager } = await import('./cache/CacheManager');

package/core/remote/CircuitBreaker.ts

@@ -0,0 +1,115 @@
+/**
+ * Remote Communication: CircuitBreaker
+ *
+ * Three-state breaker: closed -> open -> half-open -> closed.
+ *
+ * closed: pass through; increment failure count on error; trip to open at N.
+ * open:   reject immediately (fail-fast) until reset timeout elapses.
+ * half:   one trial call allowed; success -> closed, failure -> open again.
+ *
+ * Wraps Redis publish operations so a sustained Redis outage does not stall
+ * callers waiting for command timeouts on every request.
+ */
+
+export type CircuitState = "closed" | "open" | "half-open";
+
+export interface CircuitBreakerConfig {
+    /** Consecutive failures before opening (default 5) */
+    threshold?: number;
+    /** ms after opening before a half-open trial is allowed (default 30000) */
+    resetTimeoutMs?: number;
+}
+
+export class CircuitOpenError extends Error {
+    public readonly code = "CIRCUIT_OPEN";
+    constructor(message = "Circuit breaker is open") {
+        super(message);
+        this.name = "CircuitOpenError";
+    }
+}
+
+export class CircuitBreaker {
+    private state: CircuitState = "closed";
+    private failures = 0;
+    private openedAt = 0;
+    private threshold: number;
+    private resetTimeoutMs: number;
+
+    /** Hooks for metrics. */
+    public onTrip?: () => void;
+    public onReject?: () => void;
+
+    constructor(config: CircuitBreakerConfig = {}) {
+        this.threshold = config.threshold ?? 5;
+        this.resetTimeoutMs = config.resetTimeoutMs ?? 30_000;
+    }
+
+    getState(): CircuitState {
+        // Lazy transition from open -> half-open when reset window elapses.
+        if (
+            this.state === "open" &&
+            Date.now() - this.openedAt >= this.resetTimeoutMs
+        ) {
+            this.state = "half-open";
+        }
+        return this.state;
+    }
+
+    async exec<T>(fn: () => Promise<T>): Promise<T> {
+        const state = this.getState();
+        if (state === "open") {
+            this.onReject?.();
+            throw new CircuitOpenError();
+        }
+
+        try {
+            const result = await fn();
+            this.recordSuccess();
+            return result;
+        } catch (err) {
+            this.recordFailure();
+            throw err;
+        }
+    }
+
+    recordSuccess(): void {
+        const current = this.getState();
+        if (current === "half-open") {
+            this.state = "closed";
+        }
+        this.failures = 0;
+    }
+
+    recordFailure(): void {
+        // Force lazy open->half-open transition before deciding what to do.
+        const current = this.getState();
+        this.failures++;
+        if (current === "half-open") {
+            // Trial failed — back to open.
+            this.state = "open";
+            this.openedAt = Date.now();
+            this.onTrip?.();
+            return;
+        }
+        if (current === "closed" && this.failures >= this.threshold) {
+            this.state = "open";
+            this.openedAt = Date.now();
+            this.onTrip?.();
+        }
+    }
+
+    /** Force reset (useful for tests or manual recovery). */
+    reset(): void {
+        this.state = "closed";
+        this.failures = 0;
+        this.openedAt = 0;
+    }
+
+    getStats() {
+        return {
+            state: this.getState(),
+            failures: this.failures,
+            openedAt: this.openedAt,
+        };
+    }
+}

package/core/remote/OutboxWorker.ts

@@ -0,0 +1,176 @@
+/**
+ * Remote Communication: OutboxWorker
+ *
+ * Polls `remote_outbox` for unpublished rows, publishes each to Redis, and
+ * marks the row published. Uses `FOR UPDATE SKIP LOCKED` so multiple
+ * instances can run workers concurrently without double-publishing:
+ * each row is claimed by exactly one worker per batch.
+ *
+ * At-least-once semantics: if the worker crashes after XADD but before the
+ * UPDATE commits, the row stays pending and will be republished. Consumers
+ * must be idempotent — enforce this at the handler level (e.g., dedup on
+ * `ctx.messageId` or domain-level idempotency keys).
+ */
+
+import type Redis from "ioredis";
+import type { SQL } from "bun";
+import { logger } from "../Logger";
+import type { RemoteMetrics } from "./metrics";
+
+const loggerInstance = logger.child({ scope: "OutboxWorker" });
+
+export interface OutboxWorkerConfig {
+    sourceApp: string;
+    streamPrefix: string;
+    pollIntervalMs: number;
+    batchSize: number;
+    enableLogging: boolean;
+}
+
+interface OutboxRow {
+    id: string;
+    target: string;
+    event: string;
+    data: unknown;
+    created_at: Date;
+}
+
+export class OutboxWorker {
+    private db: SQL;
+    private publisher: Redis;
+    private config: OutboxWorkerConfig;
+    private running = false;
+    private timer: ReturnType<typeof setTimeout> | null = null;
+    private currentTick: Promise<void> | null = null;
+    private metrics?: RemoteMetrics;
+
+    constructor(
+        db: SQL,
+        publisher: Redis,
+        config: OutboxWorkerConfig,
+        metrics?: RemoteMetrics
+    ) {
+        this.db = db;
+        this.publisher = publisher;
+        this.config = config;
+        this.metrics = metrics;
+    }
+
+    async start(): Promise<void> {
+        if (this.running) return;
+        this.running = true;
+        this.scheduleNext(0);
+        loggerInstance.info(
+            `OutboxWorker started pollMs=${this.config.pollIntervalMs} batch=${this.config.batchSize}`
+        );
+    }
+
+    async stop(): Promise<void> {
+        if (!this.running) return;
+        this.running = false;
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        if (this.currentTick) {
+            await this.currentTick.catch(() => {});
+        }
+        loggerInstance.info("OutboxWorker stopped");
+    }
+
+    /**
+     * Force an immediate tick. Used during shutdown to flush any
+     * committed-but-unpublished rows before the process exits.
+     */
+    async flush(): Promise<void> {
+        await this.tick();
+    }
+
+    private scheduleNext(delayMs: number): void {
+        if (!this.running) return;
+        this.timer = setTimeout(() => {
+            this.currentTick = this.tick().finally(() => {
+                this.currentTick = null;
+                this.scheduleNext(this.config.pollIntervalMs);
+            });
+        }, delayMs);
+    }
+
+    private async tick(): Promise<void> {
+        if (!this.running) return;
+        try {
+            await this.processBatch();
+        } catch (error: any) {
+            loggerInstance.error(
+                { err: error, msg: "OutboxWorker tick error" }
+            );
+        }
+    }
+
+    private async processBatch(): Promise<void> {
+        const db = this.db as any;
+        await db.begin(async (trx: any) => {
+            const rows: OutboxRow[] = await trx`
+                SELECT id, target, event, data, created_at
+                FROM remote_outbox
+                WHERE published_at IS NULL
+                ORDER BY created_at
+                LIMIT ${this.config.batchSize}
+                FOR UPDATE SKIP LOCKED
+            `;
+
+            if (rows.length === 0) return;
+
+            this.metrics?.outboxClaimed(rows.length);
+            if (this.config.enableLogging) {
+                loggerInstance.debug(`Claimed ${rows.length} outbox rows`);
+            }
+
+            const successIds: string[] = [];
+
+            for (const row of rows) {
+                const stream = `${this.config.streamPrefix}${row.target}`;
+                const envelope = JSON.stringify({
+                    kind: "event",
+                    sourceApp: this.config.sourceApp,
+                    event: row.event,
+                    data: row.data,
+                    emittedAt: row.created_at.getTime(),
+                });
+                try {
+                    await this.publisher.xadd(
+                        stream,
+                        "*",
+                        "data",
+                        envelope
+                    );
+                    successIds.push(row.id);
+                } catch (err: any) {
+                    this.metrics?.outboxPublishFailed();
+                    loggerInstance.error(
+                        {
+                            err,
+                            outboxId: row.id,
+                            target: row.target,
+                            event: row.event,
+                            msg: "Outbox XADD failed — row will retry next tick",
+                        }
+                    );
+                    // Leave row unpublished; SKIP LOCKED releases on tx end
+                    // so next tick (or another instance) picks it up.
+                }
+            }
+
+            if (successIds.length > 0) {
+                for (const id of successIds) {
+                    await trx`
+                        UPDATE remote_outbox
+                        SET published_at = NOW()
+                        WHERE id = ${id}::uuid
+                    `;
+                }
+                this.metrics?.outboxPublished(successIds.length);
+            }
+        });
+    }
+}