@rudderhq/server 0.7.2 → 0.7.3

package/dist/services/plugin-worker-manager.js

@@ -1,836 +0,0 @@
-/**
- * PluginWorkerManager — spawns and manages out-of-process plugin worker child
- * processes, routes JSON-RPC 2.0 calls over stdio, and handles lifecycle
- * management including crash recovery with exponential backoff.
- *
- * Each installed plugin gets one dedicated worker process. The host sends
- * JSON-RPC requests over the child's stdin and reads responses from stdout.
- * Worker stderr is captured and forwarded to the host logger.
- *
- * Process model. See doc/engineering/PLUGIN_RUNTIME_CONTRACT.md.
- * - One worker process per installed plugin
- * - Failure isolation: plugin crashes do not affect the host
- * - Graceful shutdown: 10-second drain, then SIGTERM, then SIGKILL
- * - Automatic restart with exponential backoff on unexpected exits
- *
- * @see doc/engineering/PLUGIN_RUNTIME_CONTRACT.md — Process Model
- * @see doc/engineering/PLUGIN_RUNTIME_CONTRACT.md — Graceful Shutdown Policy
- * @see doc/engineering/PLUGIN_RUNTIME_CONTRACT.md — Host-Worker Protocol
- */
-import { createErrorResponse, createRequest, isJsonRpcNotification, isJsonRpcRequest, isJsonRpcResponse, isJsonRpcSuccessResponse, JSONRPC_ERROR_CODES, JSONRPC_VERSION, JsonRpcCallError, JsonRpcParseError, parseMessage, PLUGIN_RPC_ERROR_CODES, serializeMessage, } from "@rudderhq/plugin-sdk";
-import { fork } from "node:child_process";
-import { EventEmitter } from "node:events";
-import { createInterface } from "node:readline";
-import { logger } from "../middleware/logger.js";
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-/** Default timeout for RPC calls in milliseconds. */
-const DEFAULT_RPC_TIMEOUT_MS = 30_000;
-/** Hard upper bound for any RPC timeout (5 minutes). Prevents unbounded waits. */
-const MAX_RPC_TIMEOUT_MS = 5 * 60 * 1_000;
-/** Timeout for the initialize RPC call. */
-const INITIALIZE_TIMEOUT_MS = 15_000;
-/** Timeout for the shutdown RPC call before escalating to SIGTERM. */
-const SHUTDOWN_DRAIN_MS = 10_000;
-/** Time to wait after SIGTERM before sending SIGKILL. */
-const SIGTERM_GRACE_MS = 5_000;
-/** Minimum backoff delay for crash recovery (1 second). */
-const MIN_BACKOFF_MS = 1_000;
-/** Maximum backoff delay for crash recovery (5 minutes). */
-const MAX_BACKOFF_MS = 5 * 60 * 1_000;
-/** Backoff multiplier on each consecutive crash. */
-const BACKOFF_MULTIPLIER = 2;
-/** Maximum number of consecutive crashes before giving up on auto-restart. */
-const MAX_CONSECUTIVE_CRASHES = 10;
-/** Time window in which crashes are considered consecutive (10 minutes). */
-const CRASH_WINDOW_MS = 10 * 60 * 1_000;
-/** Maximum number of stderr characters retained for worker failure context. */
-const MAX_STDERR_EXCERPT_CHARS = 8_000;
-export function appendStderrExcerpt(current, chunk) {
-    const next = current ? `${current}\n${chunk}` : chunk;
-    return next.length <= MAX_STDERR_EXCERPT_CHARS
-        ? next
-        : next.slice(-MAX_STDERR_EXCERPT_CHARS);
-}
-export function formatWorkerFailureMessage(message, stderrExcerpt) {
-    const excerpt = stderrExcerpt.trim();
-    if (!excerpt)
-        return message;
-    if (message.includes(excerpt))
-        return message;
-    return `${message}\n\nWorker stderr:\n${excerpt}`;
-}
-// ---------------------------------------------------------------------------
-// Implementation: createPluginWorkerHandle
-// ---------------------------------------------------------------------------
-/**
- * Create a handle for a single plugin worker process.
- *
- * @internal Exported for testing; consumers should use `createPluginWorkerManager`.
- */
-export function createPluginWorkerHandle(pluginId, options) {
-    const log = logger.child({ service: "plugin-worker", pluginId });
-    const emitter = new EventEmitter();
-    /**
-     * Higher than default (10) to accommodate multiple subscribers to
-     * crash/ready/exit events during integration tests and runtime monitoring.
-     */
-    emitter.setMaxListeners(50);
-    // Worker process state
-    let childProcess = null;
-    let readline = null;
-    let stderrReadline = null;
-    let status = "stopped";
-    let startedAt = null;
-    let stderrExcerpt = "";
-    // Pending RPC requests awaiting a response
-    const pendingRequests = new Map();
-    let nextRequestId = 1;
-    // Optional methods reported by the worker during initialization
-    let supportedMethods = [];
-    // Crash tracking for exponential backoff
-    let consecutiveCrashes = 0;
-    let totalCrashes = 0;
-    let lastCrashAt = null;
-    let backoffTimer = null;
-    let nextRestartAt = null;
-    // Track open stream channels so we can emit synthetic close on crash.
-    // Maps channel → orgId.
-    const openStreamChannels = new Map();
-    // Shutdown coordination
-    let intentionalStop = false;
-    const rpcTimeoutMs = options.rpcTimeoutMs ?? DEFAULT_RPC_TIMEOUT_MS;
-    const autoRestart = options.autoRestart ?? true;
-    // -----------------------------------------------------------------------
-    // Status management
-    // -----------------------------------------------------------------------
-    function setStatus(newStatus) {
-        const prev = status;
-        if (prev === newStatus)
-            return;
-        status = newStatus;
-        log.debug({ from: prev, to: newStatus }, "worker status change");
-        emitter.emit("status", { pluginId, status: newStatus, previousStatus: prev });
-    }
-    // -----------------------------------------------------------------------
-    // JSON-RPC message sending
-    // -----------------------------------------------------------------------
-    function sendMessage(message) {
-        if (!childProcess?.stdin?.writable) {
-            throw new Error(`Worker process for plugin "${pluginId}" is not writable`);
-        }
-        const serialized = serializeMessage(message);
-        childProcess.stdin.write(serialized);
-    }
-    // -----------------------------------------------------------------------
-    // Incoming message handling
-    // -----------------------------------------------------------------------
-    function handleLine(line) {
-        if (!line.trim())
-            return;
-        let message;
-        try {
-            message = parseMessage(line);
-        }
-        catch (err) {
-            if (err instanceof JsonRpcParseError) {
-                log.warn({ rawLine: line.slice(0, 200) }, "unparseable message from worker");
-            }
-            else {
-                log.warn({ err }, "error parsing worker message");
-            }
-            return;
-        }
-        if (isJsonRpcResponse(message)) {
-            handleResponse(message);
-        }
-        else if (isJsonRpcRequest(message)) {
-            handleWorkerRequest(message);
-        }
-        else if (isJsonRpcNotification(message)) {
-            handleWorkerNotification(message);
-        }
-        else {
-            log.warn("unknown message type from worker");
-        }
-    }
-    /**
-     * Handle a JSON-RPC response from the worker (matching a pending request).
-     */
-    function handleResponse(response) {
-        const id = response.id;
-        if (id === null || id === undefined) {
-            log.warn("received response with null/undefined id");
-            return;
-        }
-        const pending = pendingRequests.get(id);
-        if (!pending) {
-            log.warn({ id }, "received response for unknown request id");
-            return;
-        }
-        clearTimeout(pending.timer);
-        pendingRequests.delete(id);
-        pending.resolve(response);
-    }
-    /**
-     * Handle a JSON-RPC request from the worker (worker→host call).
-     */
-    async function handleWorkerRequest(request) {
-        const method = request.method;
-        const handler = options.hostHandlers[method];
-        if (!handler) {
-            log.warn({ method }, "worker called unregistered host method");
-            try {
-                sendMessage(createErrorResponse(request.id, JSONRPC_ERROR_CODES.METHOD_NOT_FOUND, `Host does not handle method "${method}"`));
-            }
-            catch {
-                // Worker may have exited, ignore send error
-            }
-            return;
-        }
-        try {
-            const result = await handler(request.params);
-            sendMessage({
-                jsonrpc: JSONRPC_VERSION,
-                id: request.id,
-                result: result ?? null,
-            });
-        }
-        catch (err) {
-            const errorMessage = err instanceof Error ? err.message : String(err);
-            log.error({ method, err: errorMessage }, "host handler error");
-            try {
-                sendMessage(createErrorResponse(request.id, JSONRPC_ERROR_CODES.INTERNAL_ERROR, errorMessage));
-            }
-            catch {
-                // Worker may have exited, ignore send error
-            }
-        }
-    }
-    /**
-     * Handle a JSON-RPC notification from the worker (fire-and-forget).
-     *
-     * The `log` notification is the primary case — worker `ctx.logger` calls
-     * arrive here. We append structured plugin context (pluginId, timestamp,
-     * level) so every log entry is queryable by the current plugin runtime
-     * contract.
-     */
-    function handleWorkerNotification(notification) {
-        if (notification.method === "log") {
-            const params = notification.params;
-            const level = params?.level ?? "info";
-            const msg = params?.message ?? "";
-            const meta = params?.meta;
-            // Build a structured log object that includes the plugin context fields:
-            // pluginId, timestamp, level, message, and metadata.
-            // The child logger already carries `pluginId` in its bindings, but we
-            // add explicit `pluginLogLevel` and `pluginTimestamp` so downstream
-            // consumers (log storage, UI queries) can filter without parsing.
-            const logFields = {
-                ...meta,
-                pluginLogLevel: level,
-                pluginTimestamp: new Date().toISOString(),
-            };
-            if (level === "error") {
-                log.error(logFields, `[plugin] ${msg}`);
-            }
-            else if (level === "warn") {
-                log.warn(logFields, `[plugin] ${msg}`);
-            }
-            else if (level === "debug") {
-                log.debug(logFields, `[plugin] ${msg}`);
-            }
-            else {
-                log.info(logFields, `[plugin] ${msg}`);
-            }
-            return;
-        }
-        // Stream notifications: forward to the stream bus via callback
-        if (notification.method === "streams.open" ||
-            notification.method === "streams.emit" ||
-            notification.method === "streams.close") {
-            const params = (notification.params ?? {});
-            // Track open channels so we can emit synthetic close on crash
-            if (notification.method === "streams.open") {
-                const ch = String(params.channel ?? "");
-                const co = String(params.orgId ?? "");
-                if (ch)
-                    openStreamChannels.set(ch, co);
-            }
-            else if (notification.method === "streams.close") {
-                openStreamChannels.delete(String(params.channel ?? ""));
-            }
-            if (options.onStreamNotification) {
-                try {
-                    options.onStreamNotification(notification.method, params);
-                }
-                catch (err) {
-                    log.error({
-                        method: notification.method,
-                        err: err instanceof Error ? err.message : String(err),
-                    }, "stream notification handler failed");
-                }
-            }
-            return;
-        }
-        log.debug({ method: notification.method }, "received notification from worker");
-    }
-    // -----------------------------------------------------------------------
-    // Process lifecycle
-    // -----------------------------------------------------------------------
-    function spawnProcess() {
-        // Security: Do NOT spread process.env into the worker. Plugins should only
-        // receive a minimal, controlled environment to prevent leaking host
-        // secrets (like DATABASE_URL, internal API keys, etc.).
-        const workerEnv = {
-            ...options.env,
-            PATH: process.env.PATH ?? "",
-            NODE_PATH: process.env.NODE_PATH ?? "",
-            RUDDER_PLUGIN_ID: pluginId,
-            NODE_ENV: process.env.NODE_ENV ?? "production",
-            TZ: process.env.TZ ?? "UTC",
-        };
-        const child = fork(options.entrypointPath, [], {
-            stdio: ["pipe", "pipe", "pipe", "ipc"],
-            execArgv: options.execArgv ?? [],
-            env: workerEnv,
-            // Don't let the child keep the parent alive
-            detached: false,
-        });
-        return child;
-    }
-    function attachStdioHandlers(child) {
-        // Read NDJSON from stdout
-        if (child.stdout) {
-            readline = createInterface({ input: child.stdout });
-            readline.on("line", handleLine);
-        }
-        // Capture stderr for logging
-        if (child.stderr) {
-            stderrReadline = createInterface({ input: child.stderr });
-            stderrReadline.on("line", (line) => {
-                stderrExcerpt = appendStderrExcerpt(stderrExcerpt, line);
-                log.warn({ stream: "stderr" }, `[plugin stderr] ${line}`);
-            });
-        }
-        // Handle process exit
-        child.on("exit", (code, signal) => {
-            handleProcessExit(code, signal);
-        });
-        // Handle process errors (e.g. spawn failure)
-        child.on("error", (err) => {
-            log.error({ err: err.message }, "worker process error");
-            emitter.emit("error", { pluginId, error: err });
-            if (status === "starting") {
-                setStatus("crashed");
-                rejectAllPending(new Error(formatWorkerFailureMessage(`Worker process failed to start: ${err.message}`, stderrExcerpt)));
-            }
-        });
-    }
-    function handleProcessExit(code, signal) {
-        const wasIntentional = intentionalStop;
-        // Clean up readline interfaces
-        if (readline) {
-            readline.close();
-            readline = null;
-        }
-        if (stderrReadline) {
-            stderrReadline.close();
-            stderrReadline = null;
-        }
-        childProcess = null;
-        startedAt = null;
-        // Reject all pending requests
-        rejectAllPending(new Error(formatWorkerFailureMessage(`Worker process exited (code=${code}, signal=${signal})`, stderrExcerpt)));
-        // Emit synthetic close for any orphaned stream channels so SSE clients
-        // are notified instead of hanging indefinitely.
-        if (openStreamChannels.size > 0 && options.onStreamNotification) {
-            for (const [channel, orgId] of openStreamChannels) {
-                try {
-                    options.onStreamNotification("streams.close", { channel, orgId });
-                }
-                catch {
-                    // Best-effort cleanup — don't let it interfere with exit handling
-                }
-            }
-            openStreamChannels.clear();
-        }
-        emitter.emit("exit", { pluginId, code, signal });
-        if (wasIntentional) {
-            // Graceful stop — status is already "stopping" or will be set to "stopped"
-            setStatus("stopped");
-            log.info({ code, signal }, "worker process stopped");
-            return;
-        }
-        // Unexpected exit — crash recovery
-        totalCrashes++;
-        const now = Date.now();
-        // Reset consecutive crash counter if enough time passed
-        if (lastCrashAt !== null && now - lastCrashAt > CRASH_WINDOW_MS) {
-            consecutiveCrashes = 0;
-        }
-        consecutiveCrashes++;
-        lastCrashAt = now;
-        log.error({ code, signal, consecutiveCrashes, totalCrashes }, "worker process crashed");
-        const willRestart = autoRestart && consecutiveCrashes <= MAX_CONSECUTIVE_CRASHES;
-        setStatus("crashed");
-        emitter.emit("crash", { pluginId, code, signal, willRestart });
-        if (willRestart) {
-            scheduleRestart();
-        }
-        else {
-            log.error({ consecutiveCrashes, maxCrashes: MAX_CONSECUTIVE_CRASHES }, "max consecutive crashes reached, not restarting");
-        }
-    }
-    function rejectAllPending(error) {
-        for (const [id, pending] of pendingRequests) {
-            clearTimeout(pending.timer);
-            pending.resolve(createErrorResponse(pending.id, PLUGIN_RPC_ERROR_CODES.WORKER_UNAVAILABLE, error.message));
-        }
-        pendingRequests.clear();
-    }
-    // -----------------------------------------------------------------------
-    // Crash recovery with exponential backoff
-    // -----------------------------------------------------------------------
-    function computeBackoffMs() {
-        // Exponential backoff: MIN_BACKOFF * MULTIPLIER^(consecutiveCrashes - 1)
-        const delay = MIN_BACKOFF_MS * Math.pow(BACKOFF_MULTIPLIER, consecutiveCrashes - 1);
-        // Add jitter: ±25%
-        const jitter = delay * 0.25 * (Math.random() * 2 - 1);
-        return Math.min(Math.round(delay + jitter), MAX_BACKOFF_MS);
-    }
-    function scheduleRestart() {
-        const delay = computeBackoffMs();
-        nextRestartAt = Date.now() + delay;
-        setStatus("backoff");
-        log.info({ delayMs: delay, consecutiveCrashes }, "scheduling restart with backoff");
-        backoffTimer = setTimeout(async () => {
-            backoffTimer = null;
-            nextRestartAt = null;
-            try {
-                await startInternal();
-            }
-            catch (err) {
-                log.error({ err: err instanceof Error ? err.message : String(err) }, "restart after backoff failed");
-            }
-        }, delay);
-    }
-    function cancelPendingRestart() {
-        if (backoffTimer !== null) {
-            clearTimeout(backoffTimer);
-            backoffTimer = null;
-            nextRestartAt = null;
-        }
-    }
-    // -----------------------------------------------------------------------
-    // Start / Stop
-    // -----------------------------------------------------------------------
-    async function startInternal() {
-        if (status === "running" || status === "starting") {
-            throw new Error(`Worker for plugin "${pluginId}" is already ${status}`);
-        }
-        intentionalStop = false;
-        setStatus("starting");
-        stderrExcerpt = "";
-        const child = spawnProcess();
-        childProcess = child;
-        attachStdioHandlers(child);
-        startedAt = Date.now();
-        // Send the initialize RPC call
-        const initParams = {
-            manifest: options.manifest,
-            config: options.config,
-            instanceInfo: options.instanceInfo,
-            apiVersion: options.apiVersion,
-        };
-        try {
-            const result = await callInternal("initialize", initParams, INITIALIZE_TIMEOUT_MS);
-            if (!result || !result.ok) {
-                throw new Error("Worker initialize returned ok=false");
-            }
-            supportedMethods = result.supportedMethods ?? [];
-        }
-        catch (err) {
-            // Initialize failed — kill the process and propagate
-            const msg = err instanceof Error ? err.message : String(err);
-            log.error({ err: msg }, "worker initialize failed");
-            await killProcess();
-            setStatus("crashed");
-            throw new Error(`Worker initialize failed for "${pluginId}": ${msg}`);
-        }
-        // Reset crash counter on successful start
-        consecutiveCrashes = 0;
-        setStatus("running");
-        emitter.emit("ready", { pluginId });
-        log.info({ pid: child.pid }, "worker process started and initialized");
-    }
-    async function stopInternal() {
-        cancelPendingRestart();
-        if (status === "stopped" || status === "stopping") {
-            return;
-        }
-        intentionalStop = true;
-        setStatus("stopping");
-        if (!childProcess) {
-            setStatus("stopped");
-            return;
-        }
-        // Step 1: Send shutdown RPC and wait for the worker to exit gracefully.
-        // We race the shutdown call against a timeout. The worker should process
-        // the shutdown and exit on its own within the drain period.
-        try {
-            await Promise.race([
-                callInternal("shutdown", {}, SHUTDOWN_DRAIN_MS),
-                waitForExit(SHUTDOWN_DRAIN_MS),
-            ]);
-        }
-        catch {
-            // Shutdown call failed or timed out — proceed to kill
-            log.warn("shutdown RPC failed or timed out, escalating to SIGTERM");
-        }
-        // Give the process a brief moment to exit after the shutdown response
-        if (childProcess) {
-            await waitForExit(500);
-        }
-        // Check if process already exited
-        if (!childProcess) {
-            setStatus("stopped");
-            return;
-        }
-        // Step 2: Send SIGTERM and wait
-        log.info("worker did not exit after shutdown RPC, sending SIGTERM");
-        await killWithSignal("SIGTERM", SIGTERM_GRACE_MS);
-        if (!childProcess) {
-            setStatus("stopped");
-            return;
-        }
-        // Step 3: Forcefully kill with SIGKILL
-        log.warn("worker did not exit after SIGTERM, sending SIGKILL");
-        await killWithSignal("SIGKILL", 2_000);
-        if (childProcess) {
-            log.error("worker process still alive after SIGKILL — this should not happen");
-        }
-        setStatus("stopped");
-    }
-    /**
-     * Wait for the child process to exit, up to `timeoutMs`.
-     * Resolves immediately if the process is already gone.
-     */
-    function waitForExit(timeoutMs) {
-        return new Promise((resolve) => {
-            if (!childProcess) {
-                resolve();
-                return;
-            }
-            let settled = false;
-            const timer = setTimeout(() => {
-                if (settled)
-                    return;
-                settled = true;
-                resolve();
-            }, timeoutMs);
-            childProcess.once("exit", () => {
-                if (settled)
-                    return;
-                settled = true;
-                clearTimeout(timer);
-                resolve();
-            });
-        });
-    }
-    function killWithSignal(signal, waitMs) {
-        return new Promise((resolve) => {
-            if (!childProcess) {
-                resolve();
-                return;
-            }
-            const timer = setTimeout(() => {
-                resolve();
-            }, waitMs);
-            childProcess.once("exit", () => {
-                clearTimeout(timer);
-                resolve();
-            });
-            try {
-                childProcess.kill(signal);
-            }
-            catch {
-                clearTimeout(timer);
-                resolve();
-            }
-        });
-    }
-    async function killProcess() {
-        if (!childProcess)
-            return;
-        intentionalStop = true;
-        try {
-            childProcess.kill("SIGKILL");
-        }
-        catch {
-            // Process may already be dead
-        }
-        // Wait briefly for exit event
-        await new Promise((resolve) => {
-            if (!childProcess) {
-                resolve();
-                return;
-            }
-            const timer = setTimeout(() => {
-                resolve();
-            }, 1_000);
-            childProcess.once("exit", () => {
-                clearTimeout(timer);
-                resolve();
-            });
-        });
-    }
-    // -----------------------------------------------------------------------
-    // RPC call implementation
-    // -----------------------------------------------------------------------
-    function callInternal(method, params, timeoutMs) {
-        return new Promise((resolve, reject) => {
-            if (!childProcess?.stdin?.writable) {
-                reject(new Error(`Cannot call "${method}" — worker for "${pluginId}" is not running`));
-                return;
-            }
-            const id = nextRequestId++;
-            const timeout = Math.min(timeoutMs ?? rpcTimeoutMs, MAX_RPC_TIMEOUT_MS);
-            // Guard against double-settlement. When a process exits all pending
-            // requests are rejected via rejectAllPending(), but the timeout timer
-            // may still be running. Without this guard the timer's reject fires on
-            // an already-settled promise, producing an unhandled rejection.
-            let settled = false;
-            const settle = (fn, value) => {
-                if (settled)
-                    return;
-                settled = true;
-                clearTimeout(timer);
-                pendingRequests.delete(id);
-                fn(value);
-            };
-            const timer = setTimeout(() => {
-                settle(reject, new JsonRpcCallError({
-                    code: PLUGIN_RPC_ERROR_CODES.TIMEOUT,
-                    message: `RPC call "${method}" timed out after ${timeout}ms`,
-                }));
-            }, timeout);
-            const pending = {
-                id,
-                method,
-                resolve: (response) => {
-                    if (isJsonRpcSuccessResponse(response)) {
-                        settle(resolve, response.result);
-                    }
-                    else if ("error" in response && response.error) {
-                        settle(reject, new JsonRpcCallError(response.error));
-                    }
-                    else {
-                        settle(reject, new Error(`Unexpected response format for "${method}"`));
-                    }
-                },
-                timer,
-                sentAt: Date.now(),
-            };
-            pendingRequests.set(id, pending);
-            try {
-                const request = createRequest(method, params, id);
-                sendMessage(request);
-            }
-            catch (err) {
-                clearTimeout(timer);
-                pendingRequests.delete(id);
-                reject(new Error(`Failed to send "${method}" to worker: ${err instanceof Error ? err.message : String(err)}`));
-            }
-        });
-    }
-    // -----------------------------------------------------------------------
-    // Public API
-    // -----------------------------------------------------------------------
-    const handle = {
-        get pluginId() {
-            return pluginId;
-        },
-        get status() {
-            return status;
-        },
-        get supportedMethods() {
-            return supportedMethods;
-        },
-        async start() {
-            await startInternal();
-        },
-        async stop() {
-            await stopInternal();
-        },
-        async restart() {
-            await stopInternal();
-            await startInternal();
-        },
-        call(method, params, timeoutMs) {
-            if (status !== "running" && status !== "starting") {
-                return Promise.reject(new Error(`Cannot call "${method}" — worker for "${pluginId}" is ${status}`));
-            }
-            return callInternal(method, params, timeoutMs);
-        },
-        notify(method, params) {
-            if (status !== "running")
-                return;
-            try {
-                sendMessage({
-                    jsonrpc: JSONRPC_VERSION,
-                    method,
-                    params,
-                });
-            }
-            catch {
-                log.warn({ method }, "failed to send notification to worker");
-            }
-        },
-        on(event, listener) {
-            emitter.on(event, listener);
-        },
-        off(event, listener) {
-            emitter.off(event, listener);
-        },
-        diagnostics() {
-            return {
-                pluginId,
-                status,
-                pid: childProcess?.pid ?? null,
-                uptime: startedAt !== null && status === "running"
-                    ? Date.now() - startedAt
-                    : null,
-                consecutiveCrashes,
-                totalCrashes,
-                pendingRequests: pendingRequests.size,
-                lastCrashAt,
-                nextRestartAt,
-            };
-        },
-    };
-    return handle;
-}
-/**
- * Create a new PluginWorkerManager.
- *
- * The manager holds all plugin worker handles and provides a unified API for
- * starting, stopping, and communicating with plugin workers.
- *
- * @example
- * ```ts
- * const manager = createPluginWorkerManager();
- *
- * const handle = await manager.startWorker("acme.linear", {
- *   entrypointPath: "/path/to/worker.cjs",
- *   manifest,
- *   config: resolvedConfig,
- *   instanceInfo: { instanceId: "inst-1", hostVersion: "1.0.0" },
- *   apiVersion: 1,
- *   hostHandlers: { "config.get": async () => resolvedConfig, ... },
- * });
- *
- * // Send RPC call to the worker
- * const health = await manager.call("acme.linear", "health", {});
- *
- * // Shutdown all workers on server exit
- * await manager.stopAll();
- * ```
- */
-export function createPluginWorkerManager(managerOptions) {
-    const log = logger.child({ service: "plugin-worker-manager" });
-    const workers = new Map();
-    /** Per-plugin startup locks to prevent concurrent spawn races. */
-    const startupLocks = new Map();
-    return {
-        async startWorker(pluginId, options) {
-            // Mutex: if a start is already in-flight for this plugin, wait for it
-            const inFlight = startupLocks.get(pluginId);
-            if (inFlight) {
-                log.warn({ pluginId }, "concurrent startWorker call — waiting for in-flight start");
-                return inFlight;
-            }
-            const existing = workers.get(pluginId);
-            if (existing && existing.status !== "stopped") {
-                throw new Error(`Worker already registered for plugin "${pluginId}" (status: ${existing.status})`);
-            }
-            const handle = createPluginWorkerHandle(pluginId, options);
-            workers.set(pluginId, handle);
-            // Subscribe to crash/ready events for live event forwarding
-            if (managerOptions?.onWorkerEvent) {
-                const notify = managerOptions.onWorkerEvent;
-                handle.on("crash", (payload) => {
-                    notify({
-                        type: "plugin.worker.crashed",
-                        pluginId: payload.pluginId,
-                        code: payload.code,
-                        signal: payload.signal,
-                        willRestart: payload.willRestart,
-                    });
-                });
-                handle.on("ready", (payload) => {
-                    // Only emit restarted if this was a crash recovery (totalCrashes > 0)
-                    const diag = handle.diagnostics();
-                    if (diag.totalCrashes > 0) {
-                        notify({
-                            type: "plugin.worker.restarted",
-                            pluginId: payload.pluginId,
-                        });
-                    }
-                });
-            }
-            log.info({ pluginId }, "starting plugin worker");
-            // Set the lock before awaiting start() to prevent concurrent spawns
-            const startPromise = handle.start().then(() => handle).finally(() => {
-                startupLocks.delete(pluginId);
-            });
-            startupLocks.set(pluginId, startPromise);
-            return startPromise;
-        },
-        async stopWorker(pluginId) {
-            const handle = workers.get(pluginId);
-            if (!handle) {
-                log.warn({ pluginId }, "no worker registered for plugin, nothing to stop");
-                return;
-            }
-            log.info({ pluginId }, "stopping plugin worker");
-            await handle.stop();
-            workers.delete(pluginId);
-        },
-        getWorker(pluginId) {
-            return workers.get(pluginId);
-        },
-        isRunning(pluginId) {
-            const handle = workers.get(pluginId);
-            return handle?.status === "running";
-        },
-        async stopAll() {
-            log.info({ count: workers.size }, "stopping all plugin workers");
-            const promises = Array.from(workers.values()).map(async (handle) => {
-                try {
-                    await handle.stop();
-                }
-                catch (err) {
-                    log.error({
-                        pluginId: handle.pluginId,
-                        err: err instanceof Error ? err.message : String(err),
-                    }, "error stopping worker during shutdown");
-                }
-            });
-            await Promise.all(promises);
-            workers.clear();
-        },
-        diagnostics() {
-            return Array.from(workers.values()).map((h) => h.diagnostics());
-        },
-        call(pluginId, method, params, timeoutMs) {
-            const handle = workers.get(pluginId);
-            if (!handle) {
-                return Promise.reject(new Error(`No worker registered for plugin "${pluginId}"`));
-            }
-            return handle.call(method, params, timeoutMs);
-        },
-    };
-}
-//# sourceMappingURL=plugin-worker-manager.js.map