nebula-notebook 0.1.1 → 0.2.1

package/node-server/dist/kernel/kernel-service.js

@@ -39,7 +39,8 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.kernelService = exports.KernelService = void 0;
+exports.kernelService = exports.KernelService = exports.IOPUB_QUEUE_TIMEOUT = exports.IOPUB_QUEUE_CLOSED = void 0;
+exports.rewriteShellLines = rewriteShellLines;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const os = __importStar(require("os"));
@@ -53,7 +54,130 @@ const types_1 = require("./types");
 const display_data_1 = require("../output/display-data");
 const output_limits_1 = require("../config/output-limits");
 const session_store_1 = require("./session-store");
+const private_tmp_1 = require("../private-tmp");
 const kernelspec_1 = require("./kernelspec");
+const discovery_service_1 = require("../discovery/discovery-service");
+const types_2 = require("../discovery/types");
+/** Resolved by BoundedAsyncQueue.next() when the queue has been closed. */
+exports.IOPUB_QUEUE_CLOSED = Symbol('iopub-queue-closed');
+/** Resolved by BoundedAsyncQueue.next(timeoutMs) when the wait timed out. */
+exports.IOPUB_QUEUE_TIMEOUT = Symbol('iopub-queue-timeout');
+// Back-pressure bounds for the unified iopub reader. Parent-keyed queues carry
+// full execution output streams, so they get a generous bound; drop-oldest
+// keeps the terminal `status: idle` reachable so executions can't hang.
+const MAX_PARENT_QUEUE_MESSAGES = 5000;
+const MAX_CATCHALL_QUEUE_MESSAGES = 500;
+// Light v1 comm-state store bound (late-joiner replay of open widget comms).
+const MAX_TRACKED_COMMS_PER_SESSION = 512;
+/**
+ * Bounded producer/consumer queue used to fan iopub messages out from the
+ * single reader loop. Never blocks the producer: when full, the oldest item
+ * is dropped (newest wins, so terminal status messages still arrive).
+ */
+class BoundedAsyncQueue {
+    maxSize;
+    items = [];
+    waiters = [];
+    closed = false;
+    dropped = 0;
+    constructor(maxSize) {
+        this.maxSize = maxSize;
+    }
+    push(item) {
+        if (this.closed)
+            return;
+        const waiter = this.waiters.shift();
+        if (waiter) {
+            waiter(item);
+            return;
+        }
+        if (this.items.length >= this.maxSize) {
+            this.items.shift();
+            this.dropped++;
+            if (this.dropped === 1 || this.dropped % 1000 === 0) {
+                console.warn(`[Kernel] iopub queue overflow: dropped ${this.dropped} message(s)`);
+            }
+        }
+        this.items.push(item);
+    }
+    /**
+     * Take the next item. Resolves IOPUB_QUEUE_CLOSED once the queue is closed
+     * and drained. With timeoutMs, resolves IOPUB_QUEUE_TIMEOUT if nothing
+     * arrives in time — the waiter is deregistered on timeout so no message can
+     * be lost to an abandoned waiter.
+     */
+    next(timeoutMs) {
+        if (this.items.length > 0) {
+            return Promise.resolve(this.items.shift());
+        }
+        if (this.closed) {
+            return Promise.resolve(exports.IOPUB_QUEUE_CLOSED);
+        }
+        return new Promise((resolve) => {
+            let timer = null;
+            const waiter = (value) => {
+                if (timer)
+                    clearTimeout(timer);
+                resolve(value);
+            };
+            this.waiters.push(waiter);
+            if (timeoutMs !== undefined) {
+                timer = setTimeout(() => {
+                    const idx = this.waiters.indexOf(waiter);
+                    if (idx !== -1) {
+                        this.waiters.splice(idx, 1);
+                        resolve(exports.IOPUB_QUEUE_TIMEOUT);
+                    }
+                }, timeoutMs);
+                timer.unref?.();
+            }
+        });
+    }
+    close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        this.items = [];
+        for (const waiter of this.waiters.splice(0)) {
+            waiter(exports.IOPUB_QUEUE_CLOSED);
+        }
+    }
+}
+/**
+ * Polyfill IPython-style `!command` lines for kernels whose language lacks
+ * them, by rewriting each such line into the language's own shell-call idiom
+ * before the code reaches the kernel. Because the rewritten code still runs
+ * in the kernel process, cwd, environment (conda env, loaded modules), and
+ * machine (allocation node) are all the kernel's — the same guarantees
+ * IPython's native `!` gives Python users.
+ *
+ * Python kernels are deliberately untouched: IPython's `!` is richer than a
+ * plain shell call (output capture `x = !ls`, `{var}` interpolation,
+ * streaming output) and a rewrite would break it.
+ *
+ * Only whole lines that start with `!` are rewritten. Known limits (v1):
+ * a line inside a multi-line string that happens to start with `!` would be
+ * rewritten too, and R's intern=TRUE buffers output until the command exits.
+ */
+function rewriteShellLines(code, language) {
+    const lang = (language || '').toLowerCase();
+    if (lang !== 'r')
+        return code; // python: native support; others: no safe idiom mapped yet
+    if (!/^[ \t]*!/m.test(code))
+        return code;
+    return code
+        .split('\n')
+        .map((line) => {
+        const m = line.match(/^([ \t]*)!(.+)$/);
+        if (!m)
+            return line;
+        // Merge stderr at the shell level — intern=TRUE captures only stdout.
+        const cmd = `${m[2].trim()} 2>&1`;
+        const rString = cmd.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+        return `${m[1]}cat(system("${rString}", intern = TRUE), sep = "\\n")`;
+    })
+        .join('\n');
+}
 class KernelService {
     sessions = new Map();
     fileToSession = new Map();
@@ -77,11 +201,98 @@ class KernelService {
     // Per-cell truncation tracking: sessionId -> cellId -> tracking state
     cellOutputTracking = new Map();
     executingCellIds = new Map();
+    // ---- Unified iopub reader state ----
+    // One long-lived reader loop per session owns ALL iopub .receive() calls.
+    // Consumers (execute loop, busy monitor) subscribe to bounded queues.
+    iopubReaders = new Map();
+    // sessionId -> parent msg_id -> queue (execute-parented message routing)
+    iopubParentSubscribers = new Map();
+    // sessionId -> catch-all queues (busy monitor and other any-parent consumers)
+    iopubCatchAllSubscribers = new Map();
+    // Comm-state store for late joiners: sessionId -> comm_id -> open info
+    commStates = new Map();
+    // ---- Unified shell reply reader state ----
+    // One long-lived reader loop per session owns ALL shell .receive() calls.
+    // Replies are dispatched by parent msg_id to one-shot waiters; a reply with
+    // no registered waiter (e.g. execute_reply — executions are iopub-driven —
+    // or a reply whose waiter already timed out) is dropped. This means a
+    // timed-out request can never hold the socket or eat a later reply.
+    shellReaders = new Map();
+    // sessionId -> request msg_id -> one-shot resolver
+    shellReplyWaiters = new Map();
     constructor(config, sessionStore) {
         this.config = { ...types_1.DEFAULT_CONFIG, ...config };
         this.sessionStore = sessionStore || new session_store_1.SessionStore();
         this.serverId = process.env.NEBULA_SERVER_ID || 'local';
         this.serverInstanceId = process.env.NEBULA_SERVER_INSTANCE_ID || (0, uuid_1.v4)();
+        this.startLivenessSweep();
+    }
+    // ---- Kernel death notification --------------------------------------
+    // Process exit/error handlers (and the liveness sweep below) notify
+    // listeners so the route layer can broadcast status to WebSocket clients.
+    // Without this, a crashed kernel leaves every client showing 'busy'.
+    deadListeners = [];
+    onSessionDead(cb) {
+        this.deadListeners.push(cb);
+    }
+    notifySessionDead(sessionId) {
+        for (const cb of this.deadListeners) {
+            try {
+                cb(sessionId);
+            }
+            catch (err) {
+                console.error('[Kernel] onSessionDead listener failed:', err);
+            }
+        }
+    }
+    // ---- Comm message notification (ipywidgets et al.) -------------------
+    // The unified iopub reader demuxes comm_open/comm_msg/comm_close messages
+    // (regardless of parent — including idle-time traffic from widget
+    // observers) to these listeners so the route layer can broadcast them.
+    commListeners = [];
+    onComm(cb) {
+        this.commListeners.push(cb);
+    }
+    notifyComm(sessionId, comm) {
+        for (const cb of this.commListeners) {
+            try {
+                cb(sessionId, comm);
+            }
+            catch (err) {
+                console.error('[Kernel] onComm listener failed:', err);
+            }
+        }
+    }
+    // ---- Liveness sweep ---------------------------------------------------
+    // ChildProcess handles get 'exit' events, but REATTACHED sessions only
+    // have a PID — nothing fires when that process dies. Sweep PIDs so a dead
+    // reattached kernel is noticed within ~15s instead of at the next execute.
+    livenessTimer = null;
+    startLivenessSweep() {
+        this.livenessTimer = setInterval(() => this.sweepLiveness(), 15000);
+        // Never keep the process alive just for the sweep
+        this.livenessTimer.unref?.();
+    }
+    /** One liveness pass (exposed for tests; normally driven by the timer). */
+    sweepLiveness() {
+        for (const [sessionId, session] of this.sessions.entries()) {
+            if (session.status === 'dead')
+                continue;
+            // Sessions with a live ChildProcess handle are covered by 'exit'
+            if (this.kernelProcesses.has(sessionId))
+                continue;
+            if (!session.pid)
+                continue;
+            try {
+                process.kill(session.pid, 0);
+            }
+            catch {
+                console.log(`[Kernel] Liveness sweep: reattached kernel ${sessionId} (pid ${session.pid}) is gone`);
+                session.status = 'dead';
+                this.cleanupKernelResources(sessionId);
+                this.notifySessionDead(sessionId);
+            }
+        }
     }
     setServerIdentity(serverId, serverInstanceId) {
         this.serverId = serverId;
@@ -108,6 +319,7 @@ class KernelService {
             }
             console.error(`Kernel ${sessionId} error:`, err);
             session.status = 'dead';
+            this.notifySessionDead(sessionId);
         };
         const onExit = (code) => {
             if (this.kernelProcesses.get(sessionId) !== proc) {
@@ -116,6 +328,7 @@ class KernelService {
             console.log(`Kernel ${sessionId} exited with code ${code}`);
             session.status = 'dead';
             this.cleanupKernelResources(sessionId);
+            this.notifySessionDead(sessionId);
         };
         this.processLifecycleHandlers.set(proc, { onError, onExit });
         proc.on('error', onError);
@@ -288,6 +501,9 @@ class KernelService {
                     console.log(`[Kernel] Failed to reattach session ${sessionId}: ${err instanceof Error ? err.message : err}`);
                 }
             }
+            // Rows just marked terminated (process gone) are pure bookkeeping —
+            // remove them now so they never surface as "orphaned" in the UI.
+            await this.autoCleanupDeadSessions().catch((err) => console.warn('[Kernel] Auto-cleanup after reattach failed:', err));
             return {
                 attempted: orphanedSessions.length,
                 reattached,
@@ -396,8 +612,9 @@ class KernelService {
             hbPort,
         };
         // Write connection file
-        const connDir = path.join(os.tmpdir(), 'nebula-kernels');
-        fs.mkdirSync(connDir, { recursive: true });
+        // Per-user private dir (0700): connection files carry the HMAC key —
+        // world-readable means arbitrary code execution as this user.
+        const connDir = (0, private_tmp_1.privateTmpDir)('kernels');
         const connFile = path.join(connDir, `kernel-${sessionId}.json`);
         const connData = {
             ip: config.ip,
@@ -410,7 +627,7 @@ class KernelService {
             iopub_port: config.iopubPort,
             hb_port: config.hbPort,
         };
-        fs.writeFileSync(connFile, JSON.stringify(connData, null, 2));
+        fs.writeFileSync(connFile, JSON.stringify(connData, null, 2), { mode: 0o600 });
         return { config, filePath: connFile };
     }
     /**
@@ -418,7 +635,27 @@ class KernelService {
      */
     async startKernel(options = {}) {
         const kernelName = options.kernelName || 'python3';
-        const spec = (0, kernelspec_1.getKernelSpec)(kernelName);
+        // env:<pythonPath> kernels raw-launch a Python environment directly
+        // (VSCode-style) — no kernelspec registration involved. Preflight here so
+        // callers get a crisp, coded error instead of a spawn failure + ready
+        // timeout: python_not_found for a dead path, needs_ipykernel (with an
+        // ecosystem-appropriate install hint) when the env can't host a kernel yet.
+        const envPython = (0, kernelspec_1.envKernelPythonPath)(kernelName);
+        let spec;
+        if (envPython) {
+            if (!fs.existsSync(envPython)) {
+                throw new types_2.KernelProvisionError(`Python not found: ${envPython}`, 'python_not_found');
+            }
+            const probe = await discovery_service_1.pythonDiscovery.probeForKernel(envPython);
+            if (!probe.hasIpykernel) {
+                throw new types_2.KernelProvisionError(`ipykernel is not installed in ${envPython}`, 'needs_ipykernel', probe.installHint || undefined);
+            }
+            const cached = discovery_service_1.pythonDiscovery.getFromCache()?.[envPython];
+            spec = (0, kernelspec_1.makeEnvKernelSpec)(envPython, cached?.displayName);
+        }
+        else {
+            spec = (0, kernelspec_1.getKernelSpec)(kernelName);
+        }
         if (!spec) {
             throw new Error(`Kernel '${kernelName}' not found`);
         }
@@ -523,6 +760,12 @@ class KernelService {
         iopub.connect(iopubEndpoint);
         iopub.subscribe('');
         this.zmqSockets.set(sessionId, { shell, iopub });
+        // Start the unified readers: from here on, these loops own every
+        // iopub.receive()/shell.receive() for the session. The shell reader must
+        // be live before the kernel_info probing below, which awaits its reply
+        // through the waiter mechanism.
+        this.startIopubReader(sessionId, iopub);
+        this.startShellReader(sessionId, shell);
         // Send kernel_info_request to verify connectivity
         const startTime = Date.now();
         const actualTimeout = timeoutSeconds ?? this.config.startupTimeoutSeconds;
@@ -545,6 +788,374 @@ class KernelService {
         }
         throw new Error(`Kernel did not start within ${actualTimeout} seconds`);
     }
+    // ---- Unified iopub reader ---------------------------------------------
+    // Exactly one loop per session calls iopub.receive(). It demuxes:
+    //   - comm_open/comm_msg/comm_close (any parent) -> comm store + onComm
+    //   - everything else -> the parent-keyed queue (if a consumer registered
+    //     for that parent msg_id) AND every catch-all queue (busy monitor).
+    // The loop never spins: receive() blocks until a message arrives, and a
+    // socket close (stop/restart/cleanup) rejects the pending receive, which
+    // exits the loop and closes all subscriber queues.
+    startIopubReader(sessionId, iopub) {
+        // Never allow two loops for one session (e.g. restart replaces sockets).
+        this.stopIopubReader(sessionId);
+        const reader = { stopped: false };
+        this.iopubReaders.set(sessionId, reader);
+        void (async () => {
+            try {
+                while (!reader.stopped) {
+                    let frames;
+                    try {
+                        frames = await iopub.receive();
+                    }
+                    catch {
+                        // Socket closed or errored (kernel stop/restart/cleanup) — exit.
+                        break;
+                    }
+                    if (reader.stopped)
+                        break;
+                    try {
+                        const parsed = this.parseJupyterMessage(frames);
+                        this.dispatchIopubMessage(sessionId, parsed);
+                    }
+                    catch (err) {
+                        console.warn(`[Kernel] Dropping unparseable iopub message for ${sessionId}:`, err);
+                    }
+                }
+            }
+            finally {
+                // Only tear down if we're still the current reader — on restart a new
+                // reader (with fresh subscriber queues) may already have taken over.
+                if (this.iopubReaders.get(sessionId) === reader) {
+                    this.iopubReaders.delete(sessionId);
+                    this.closeIopubSubscribers(sessionId);
+                }
+            }
+        })();
+    }
+    stopIopubReader(sessionId) {
+        const reader = this.iopubReaders.get(sessionId);
+        if (reader) {
+            reader.stopped = true;
+            this.iopubReaders.delete(sessionId);
+        }
+        this.closeIopubSubscribers(sessionId);
+    }
+    closeIopubSubscribers(sessionId) {
+        const parents = this.iopubParentSubscribers.get(sessionId);
+        if (parents) {
+            for (const queue of parents.values())
+                queue.close();
+            this.iopubParentSubscribers.delete(sessionId);
+        }
+        const catchAll = this.iopubCatchAllSubscribers.get(sessionId);
+        if (catchAll) {
+            for (const queue of catchAll)
+                queue.close();
+            this.iopubCatchAllSubscribers.delete(sessionId);
+        }
+    }
+    dispatchIopubMessage(sessionId, msg) {
+        if (msg.msgType === 'comm_open' || msg.msgType === 'comm_msg' || msg.msgType === 'comm_close') {
+            this.handleKernelCommMessage(sessionId, msg);
+            return;
+        }
+        if (msg.parentMsgId) {
+            this.iopubParentSubscribers.get(sessionId)?.get(msg.parentMsgId)?.push(msg);
+        }
+        const catchAll = this.iopubCatchAllSubscribers.get(sessionId);
+        if (catchAll) {
+            for (const queue of catchAll)
+                queue.push(msg);
+        }
+    }
+    /**
+     * Subscribe to iopub messages parented by a specific msg_id. Must be called
+     * BEFORE sending the request so no reply can slip past the demux. Callers
+     * must unsubscribe in a finally block.
+     */
+    subscribeIopubParent(sessionId, parentMsgId) {
+        let parents = this.iopubParentSubscribers.get(sessionId);
+        if (!parents) {
+            parents = new Map();
+            this.iopubParentSubscribers.set(sessionId, parents);
+        }
+        const queue = new BoundedAsyncQueue(MAX_PARENT_QUEUE_MESSAGES);
+        parents.set(parentMsgId, queue);
+        // If no reader loop is alive (it crashed or was never started), close the
+        // queue immediately so consumers fail fast instead of hanging forever.
+        if (!this.iopubReaders.has(sessionId)) {
+            queue.close();
+        }
+        return queue;
+    }
+    unsubscribeIopubParent(sessionId, parentMsgId) {
+        const parents = this.iopubParentSubscribers.get(sessionId);
+        const queue = parents?.get(parentMsgId);
+        if (!parents || !queue)
+            return;
+        queue.close();
+        parents.delete(parentMsgId);
+        if (parents.size === 0) {
+            this.iopubParentSubscribers.delete(sessionId);
+        }
+    }
+    /** Subscribe to ALL iopub messages for a session (any parent). */
+    subscribeIopubCatchAll(sessionId) {
+        let queues = this.iopubCatchAllSubscribers.get(sessionId);
+        if (!queues) {
+            queues = new Set();
+            this.iopubCatchAllSubscribers.set(sessionId, queues);
+        }
+        const queue = new BoundedAsyncQueue(MAX_CATCHALL_QUEUE_MESSAGES);
+        queues.add(queue);
+        if (!this.iopubReaders.has(sessionId)) {
+            queue.close();
+        }
+        return queue;
+    }
+    unsubscribeIopubCatchAll(sessionId, queue) {
+        queue.close();
+        const queues = this.iopubCatchAllSubscribers.get(sessionId);
+        if (!queues)
+            return;
+        queues.delete(queue);
+        if (queues.size === 0) {
+            this.iopubCatchAllSubscribers.delete(sessionId);
+        }
+    }
+    // ---- Unified shell reply reader -----------------------------------------
+    // Mirrors the iopub reader: exactly one loop per session calls
+    // shell.receive(). Consumers register a one-shot waiter keyed by their
+    // request msg_id BEFORE sending, then await the reply with a timeout. A
+    // timeout only deregisters the waiter — the reader keeps owning the socket,
+    // so a slow reply (e.g. complete_request against a busy kernel) can never
+    // block or swallow a later request's reply.
+    startShellReader(sessionId, shell) {
+        // Never allow two loops for one session (e.g. restart replaces sockets).
+        this.stopShellReader(sessionId);
+        const reader = { stopped: false };
+        this.shellReaders.set(sessionId, reader);
+        void (async () => {
+            try {
+                while (!reader.stopped) {
+                    let frames;
+                    try {
+                        frames = await shell.receive();
+                    }
+                    catch {
+                        // Socket closed or errored (kernel stop/restart/cleanup) — exit.
+                        break;
+                    }
+                    if (reader.stopped)
+                        break;
+                    try {
+                        const parsed = this.parseJupyterMessage(frames);
+                        const waiters = this.shellReplyWaiters.get(sessionId);
+                        const waiter = parsed.parentMsgId ? waiters?.get(parsed.parentMsgId) : undefined;
+                        if (waiters && waiter) {
+                            waiters.delete(parsed.parentMsgId);
+                            if (waiters.size === 0)
+                                this.shellReplyWaiters.delete(sessionId);
+                            waiter(parsed);
+                        }
+                        else {
+                            // No waiter: execute_reply (executions are iopub-driven), or the
+                            // waiter timed out and deregistered. Drop it.
+                            console.debug(`[Kernel] Dropping unclaimed shell ${parsed.msgType} for ${sessionId}`);
+                        }
+                    }
+                    catch (err) {
+                        console.warn(`[Kernel] Dropping unparseable shell message for ${sessionId}:`, err);
+                    }
+                }
+            }
+            finally {
+                // Only tear down if we're still the current reader — on restart a new
+                // reader (with fresh waiters) may already have taken over.
+                if (this.shellReaders.get(sessionId) === reader) {
+                    this.shellReaders.delete(sessionId);
+                    this.closeShellReplyWaiters(sessionId);
+                }
+            }
+        })();
+    }
+    stopShellReader(sessionId) {
+        const reader = this.shellReaders.get(sessionId);
+        if (reader) {
+            reader.stopped = true;
+            this.shellReaders.delete(sessionId);
+        }
+        this.closeShellReplyWaiters(sessionId);
+    }
+    closeShellReplyWaiters(sessionId) {
+        const waiters = this.shellReplyWaiters.get(sessionId);
+        if (!waiters)
+            return;
+        this.shellReplyWaiters.delete(sessionId);
+        for (const resolve of waiters.values()) {
+            resolve('closed');
+        }
+    }
+    /**
+     * Register a one-shot waiter for the shell reply parented by msgId. Must be
+     * called BEFORE sending the request so the reply cannot slip past the
+     * dispatch. Resolves 'closed' immediately if no reader loop is alive, or
+     * later when the reader stops (kernel stop/restart/cleanup).
+     */
+    registerShellReplyWaiter(sessionId, msgId) {
+        if (!this.shellReaders.has(sessionId)) {
+            return Promise.resolve('closed');
+        }
+        let waiters = this.shellReplyWaiters.get(sessionId);
+        if (!waiters) {
+            waiters = new Map();
+            this.shellReplyWaiters.set(sessionId, waiters);
+        }
+        const map = waiters;
+        return new Promise((resolve) => {
+            map.set(msgId, resolve);
+        });
+    }
+    removeShellReplyWaiter(sessionId, msgId) {
+        const waiters = this.shellReplyWaiters.get(sessionId);
+        if (!waiters)
+            return;
+        waiters.delete(msgId);
+        if (waiters.size === 0) {
+            this.shellReplyWaiters.delete(sessionId);
+        }
+    }
+    /**
+     * Await a previously registered shell reply with a timeout. The waiter is
+     * always deregistered on the way out (timeout, reply, or error), so a late
+     * reply is dropped by the reader instead of leaking a waiter.
+     */
+    async waitForShellReply(sessionId, msgId, replyPromise, timeoutMs) {
+        let timer = null;
+        const timeout = new Promise((resolve) => {
+            timer = setTimeout(() => resolve('timeout'), timeoutMs);
+            timer.unref?.();
+        });
+        try {
+            return await Promise.race([replyPromise, timeout]);
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+            this.removeShellReplyWaiter(sessionId, msgId);
+        }
+    }
+    // ---- Comm handling ------------------------------------------------------
+    handleKernelCommMessage(sessionId, msg) {
+        const commId = typeof msg.content.comm_id === 'string' ? msg.content.comm_id : null;
+        if (!commId)
+            return;
+        const data = (msg.content.data && typeof msg.content.data === 'object')
+            ? msg.content.data
+            : {};
+        const targetName = typeof msg.content.target_name === 'string' ? msg.content.target_name : undefined;
+        // Maintain the comm-state store so late-joining clients can discover
+        // open comms (comm_info) after the fact.
+        if (msg.msgType === 'comm_open') {
+            this.rememberOpenComm(sessionId, commId, targetName ?? '', data);
+        }
+        else if (msg.msgType === 'comm_close') {
+            this.commStates.get(sessionId)?.delete(commId);
+        }
+        const comm = {
+            msgType: msg.msgType,
+            commId,
+            data,
+        };
+        if (targetName !== undefined) {
+            comm.targetName = targetName;
+        }
+        if (msg.buffers.length > 0) {
+            comm.buffers = msg.buffers.map((b) => b.toString('base64'));
+        }
+        this.notifyComm(sessionId, comm);
+    }
+    rememberOpenComm(sessionId, commId, targetName, openData) {
+        let comms = this.commStates.get(sessionId);
+        if (!comms) {
+            comms = new Map();
+            this.commStates.set(sessionId, comms);
+        }
+        // Bounded store: evict the oldest tracked comm when full (light v1).
+        if (!comms.has(commId) && comms.size >= MAX_TRACKED_COMMS_PER_SESSION) {
+            const oldest = comms.keys().next().value;
+            if (oldest !== undefined) {
+                comms.delete(oldest);
+            }
+        }
+        comms.set(commId, { targetName, openData });
+    }
+    /**
+     * Open comms known for a session (for late-joining clients).
+     * Returns comm_id -> { targetName, openData } where openData is the last
+     * state-carrying comm_open payload observed for that comm.
+     */
+    getOpenComms(sessionId) {
+        const comms = this.commStates.get(sessionId);
+        const result = {};
+        if (!comms)
+            return result;
+        for (const [commId, info] of comms) {
+            result[commId] = { targetName: info.targetName, openData: info.openData };
+        }
+        return result;
+    }
+    /**
+     * Send a comm message (comm_open / comm_msg / comm_close) to the kernel on
+     * the shell channel. Comm messages produce NO shell reply, so the queued
+     * slot releases as soon as the send completes — we never block waiting for
+     * a reply that will not come.
+     *
+     * @param buffers Optional binary buffer frames, base64-encoded.
+     */
+    async sendCommMessage(sessionId, msgType, content, buffers) {
+        const session = this.sessions.get(sessionId);
+        if (!session || !session.connectionConfig) {
+            throw new Error(`Session ${sessionId} not found`);
+        }
+        if (!this.zmq || !this.zmqSockets.has(sessionId)) {
+            throw new Error('ZeroMQ not available for kernel communication');
+        }
+        const binaryBuffers = (buffers ?? []).map((b64) => Buffer.from(b64, 'base64'));
+        const header = {
+            msg_id: (0, uuid_1.v4)(),
+            session: sessionId,
+            username: 'nebula',
+            msg_type: msgType,
+            version: '5.3',
+            date: new Date().toISOString(),
+        };
+        const message = this.createJupyterMessage(header, {}, content, session.connectionConfig.key, binaryBuffers);
+        // Track client-opened comms too, so comm_info reflects both directions.
+        const commId = typeof content.comm_id === 'string' ? content.comm_id : null;
+        if (commId) {
+            if (msgType === 'comm_open') {
+                const targetName = typeof content.target_name === 'string' ? content.target_name : '';
+                const data = (content.data && typeof content.data === 'object')
+                    ? content.data
+                    : {};
+                this.rememberOpenComm(sessionId, commId, targetName, data);
+            }
+            else if (msgType === 'comm_close') {
+                this.commStates.get(sessionId)?.delete(commId);
+            }
+        }
+        // Route through the shell queue so we never interleave with an in-flight
+        // request/reply pair on the Dealer socket. Send-only: no receive() here.
+        await this.enqueueShellRequest(sessionId, async () => {
+            const sockets = this.zmqSockets.get(sessionId);
+            if (!sockets) {
+                throw new Error(`Session ${sessionId} sockets are gone (kernel stopped?)`);
+            }
+            await sockets.shell.send(message);
+        });
+    }
     /**
      * Monitor a busy reattached kernel on iopub. When its current execution
      * finishes (status: idle on iopub), verify shell connectivity and update
@@ -552,19 +1163,17 @@ class KernelService {
      */
     monitorBusyKernel(sessionId) {
         const session = this.sessions.get(sessionId);
-        const sockets = this.zmqSockets.get(sessionId);
-        if (!session || !sockets)
+        if (!session || !this.zmqSockets.has(sessionId))
             return;
         console.log(`[Kernel] Monitoring busy kernel ${sessionId} for idle transition...`);
+        // Consume from the unified reader's catch-all channel — the monitor cares
+        // about status messages from executions we did not initiate (any parent).
+        const channel = this.subscribeIopubCatchAll(sessionId);
         const poll = async () => {
             try {
                 while (session.status === 'busy') {
-                    const receivePromise = sockets.iopub.receive();
-                    const timeoutPromise = new Promise((resolve) => {
-                        setTimeout(() => resolve(null), 5000);
-                    });
-                    const frames = await Promise.race([receivePromise, timeoutPromise]);
-                    if (!frames) {
+                    const msg = await channel.next(5000);
+                    if (msg === exports.IOPUB_QUEUE_TIMEOUT) {
                         // Periodic liveness check
                         if (session.pid && !this.isPidAlive(session.pid)) {
                             console.log(`[Kernel] Busy kernel ${sessionId} PID ${session.pid} died`);
@@ -573,8 +1182,11 @@ class KernelService {
                         }
                         continue;
                     }
-                    const { msgType, content } = this.parseJupyterMessage(frames);
-                    if (msgType === 'status' && content.execution_state === 'idle') {
+                    if (msg === exports.IOPUB_QUEUE_CLOSED) {
+                        // Kernel stopped/restarted while monitoring — nothing more to do.
+                        return;
+                    }
+                    if (msg.msgType === 'status' && msg.content.execution_state === 'idle') {
                         break;
                     }
                 }
@@ -593,6 +1205,9 @@ class KernelService {
                 console.error(`[Kernel] Error monitoring busy kernel ${sessionId}:`, err);
                 session.status = 'idle'; // Assume idle — next operation will detect real issues
             }
+            finally {
+                this.unsubscribeIopubCatchAll(sessionId, channel);
+            }
         };
         void poll();
     }
@@ -616,35 +1231,42 @@ class KernelService {
             date: new Date().toISOString(),
         };
         const message = this.createJupyterMessage(header, {}, {}, session.connectionConfig.key);
-        await sockets.shell.send(message);
-        // Wait for kernel_info_reply with timeout
-        const receivePromise = sockets.shell.receive();
-        const timeoutPromise = new Promise((_, reject) => {
-            setTimeout(() => reject(new Error('Kernel not responding')), 1000);
-        });
-        const frames = await Promise.race([receivePromise, timeoutPromise]);
-        const { msgType } = this.parseJupyterMessage(frames);
-        if (msgType !== 'kernel_info_reply') {
-            throw new Error(`Unexpected message type: ${msgType}`);
+        // Register the waiter before sending so the reply cannot race the
+        // dispatch, then wait for kernel_info_reply with timeout.
+        const replyPromise = this.registerShellReplyWaiter(sessionId, msgId);
+        try {
+            await this.enqueueShellRequest(sessionId, () => sockets.shell.send(message));
+        }
+        catch (err) {
+            this.removeShellReplyWaiter(sessionId, msgId);
+            throw err;
+        }
+        const reply = await this.waitForShellReply(sessionId, msgId, replyPromise, 1000);
+        if (reply === 'timeout' || reply === 'closed') {
+            throw new Error('Kernel not responding');
+        }
+        if (reply.msgType !== 'kernel_info_reply') {
+            throw new Error(`Unexpected message type: ${reply.msgType}`);
         }
         // Kernel is ready!
     }
     /**
      * Create a Jupyter protocol message
      */
-    createJupyterMessage(header, parentHeader, content, key) {
+    createJupyterMessage(header, parentHeader, content, key, buffers = []) {
         const headerJson = JSON.stringify(header);
         const parentHeaderJson = JSON.stringify(parentHeader);
         const metadataJson = JSON.stringify({});
         const contentJson = JSON.stringify(content);
-        // Create HMAC signature
+        // Create HMAC signature (buffers are NOT part of the signature per the
+        // Jupyter wire protocol — only header/parent_header/metadata/content).
         const hmac = crypto.createHmac('sha256', key);
         hmac.update(headerJson);
         hmac.update(parentHeaderJson);
         hmac.update(metadataJson);
         hmac.update(contentJson);
         const signature = hmac.digest('hex');
-        // Build multipart message
+        // Build multipart message; binary buffer frames follow content
         return [
             Buffer.from('<IDS|MSG>'),
             Buffer.from(signature),
@@ -652,13 +1274,31 @@ class KernelService {
             Buffer.from(parentHeaderJson),
             Buffer.from(metadataJson),
             Buffer.from(contentJson),
+            ...buffers,
         ];
     }
     /**
      * Get or create kernel for a file (one notebook = one kernel).
      * Returns whether a new session was created.
      */
+    /** In-flight create/attach per normalized file path (single-flight).
+     *  Without this, two near-simultaneous requests (UI re-render + agent op,
+     *  double-click) both miss the fileToSession check and spawn TWO kernel
+     *  processes — the second overwrites the mapping and the first leaks. */
+    inflightKernelCreates = new Map();
     async getOrCreateKernel(filePath, kernelName = 'python3') {
+        const key = this.normalizePath(filePath);
+        const inflight = this.inflightKernelCreates.get(key);
+        if (inflight) {
+            return inflight;
+        }
+        const run = this.getOrCreateKernelInternal(key, kernelName).finally(() => {
+            this.inflightKernelCreates.delete(key);
+        });
+        this.inflightKernelCreates.set(key, run);
+        return run;
+    }
+    async getOrCreateKernelInternal(filePath, kernelName) {
         const normalizedPath = this.normalizePath(filePath);
         // Check for existing session in fileToSession map
         let existingSessionId = this.fileToSession.get(normalizedPath);
@@ -769,6 +1409,14 @@ class KernelService {
      * Execute code in a kernel session
      */
     async executeCode(sessionId, code, onOutput, onQueueInfo, cellId) {
+        // Polyfill `!command` lines for kernels whose language lacks them (see
+        // rewriteShellLines) — rewritten code still runs IN the kernel process,
+        // so cwd/env/node are the kernel's, exactly like IPython's native `!`.
+        {
+            const kernelName = this.sessions.get(sessionId)?.kernelName;
+            const spec = kernelName ? (0, kernelspec_1.getKernelSpec)(kernelName) : null;
+            code = rewriteShellLines(code, spec?.language);
+        }
         const queueInfo = this.reserveExecutionSlot(sessionId);
         if (onQueueInfo) {
             onQueueInfo(queueInfo);
@@ -815,7 +1463,11 @@ class KernelService {
         return run;
     }
     /**
-     * Queue shell socket requests to prevent concurrent receive operations
+     * Serialize shell socket SENDS. Receives are owned by the unified shell
+     * reader, so tasks queued here must be send-only and release the slot as
+     * soon as the send completes (register any reply waiter BEFORE enqueueing,
+     * await the reply AFTER the slot is released). Holding the slot across a
+     * reply wait would let one slow request delay every later send.
      */
     enqueueShellRequest(sessionId, task) {
         const previous = this.shellRequestQueues.get(sessionId) || Promise.resolve();
@@ -868,9 +1520,12 @@ class KernelService {
         session.status = 'busy';
         const sockets = this.zmqSockets.get(sessionId);
         let execCount = null;
+        // Subscribe to iopub messages for this execution BEFORE sending the
+        // request so the unified reader can't demux replies into the void.
+        const msgId = (0, uuid_1.v4)();
+        const channel = this.subscribeIopubParent(sessionId, msgId);
         try {
             // Create execute_request message
-            const msgId = (0, uuid_1.v4)();
             const header = {
                 msg_id: msgId,
                 session: sessionId,
@@ -888,15 +1543,20 @@ class KernelService {
                 stop_on_error: true,
             };
             const message = this.createJupyterMessage(header, {}, content, session.connectionConfig.key);
-            await sockets.shell.send(message);
-            // Process iopub messages
+            // Serialize the send with other shell sends; no shell reply waiter —
+            // execution progress and completion are driven entirely by iopub
+            // (execute_reply is dropped by the shell reader as unclaimed).
+            await this.enqueueShellRequest(sessionId, () => sockets.shell.send(message));
+            // Process iopub messages for our execution (already parent-filtered by
+            // the unified reader; comm messages are demuxed to onComm separately)
             while (true) {
-                const frames = await sockets.iopub.receive();
-                const { msgType, content: msgContent, parentMsgId } = this.parseJupyterMessage(frames);
-                // Only process messages for our execution
-                if (parentMsgId !== msgId) {
-                    continue;
+                const msg = await channel.next();
+                if (typeof msg === 'symbol') {
+                    // Queue closed: kernel stopped/restarted mid-execution. Surface the
+                    // same way a socket-close used to — via the catch block below.
+                    throw new Error('Kernel iopub stream closed during execution');
                 }
+                const { msgType, content: msgContent } = msg;
                 // Handle different message types
                 if (msgType === 'execute_input') {
                     execCount = msgContent.execution_count;
@@ -931,11 +1591,24 @@ class KernelService {
             return { status: 'ok', executionCount: execCount };
         }
         catch (err) {
+            // A restart closes the old sockets while we may be mid-receive: surface
+            // that as a clean cancellation, and never clobber 'starting'/'dead'
+            // status set concurrently by the restart/exit paths (re-read through
+            // the map — TS narrowing can't see cross-async mutation).
+            const currentStatus = this.sessions.get(sessionId)?.status;
+            if (currentStatus === 'starting' || currentStatus === 'dead') {
+                const msg = currentStatus === 'dead' ? 'Kernel died during execution' : 'Kernel was restarted — execution cancelled';
+                await onOutput({ type: 'error', content: msg });
+                return { status: 'error', executionCount: null, error: msg };
+            }
             session.status = 'idle';
             const errorMsg = this.formatExecutionError(err);
             await onOutput({ type: 'error', content: errorMsg });
             return { status: 'error', executionCount: null, error: errorMsg };
         }
+        finally {
+            this.unsubscribeIopubParent(sessionId, msgId);
+        }
     }
     /**
      * Request code completion from the kernel
@@ -949,11 +1622,13 @@ class KernelService {
         if (!this.zmq || !this.zmqSockets.has(sessionId)) {
             return { status: 'error', matches: [], cursor_start: cursorPos, cursor_end: cursorPos };
         }
-        // Queue completion requests to avoid "socket busy" errors
-        return this.enqueueShellRequest(sessionId, () => this.completeInternal(sessionId, code, cursorPos));
+        return this.completeInternal(sessionId, code, cursorPos);
     }
     /**
-     * Internal completion implementation (runs within shell queue)
+     * Internal completion implementation. The send is serialized through the
+     * shell queue; the reply is awaited via the unified shell reader, so a
+     * timeout here simply drops the waiter and can never block or swallow a
+     * later request's reply.
      */
     async completeInternal(sessionId, code, cursorPos) {
         const session = this.sessions.get(sessionId);
@@ -977,30 +1652,30 @@ class KernelService {
                 cursor_pos: cursorPos,
             };
             const message = this.createJupyterMessage(header, {}, content, session.connectionConfig.key);
-            await sockets.shell.send(message);
-            // Wait for complete_reply with timeout
-            const timeoutPromise = new Promise((resolve) => {
-                setTimeout(() => resolve(null), 3000);
-            });
-            const receivePromise = (async () => {
-                while (true) {
-                    const frames = await sockets.shell.receive();
-                    const { msgType, content: msgContent, parentMsgId } = this.parseJupyterMessage(frames);
-                    if (parentMsgId === msgId && msgType === 'complete_reply') {
-                        return {
-                            status: msgContent.status || 'ok',
-                            matches: msgContent.matches || [],
-                            cursor_start: msgContent.cursor_start ?? cursorPos,
-                            cursor_end: msgContent.cursor_end ?? cursorPos,
-                        };
-                    }
-                }
-            })();
-            const result = await Promise.race([receivePromise, timeoutPromise]);
-            if (result === null) {
+            // Register the waiter before sending so the reply cannot race the
+            // dispatch, then wait for complete_reply with timeout.
+            const replyPromise = this.registerShellReplyWaiter(sessionId, msgId);
+            try {
+                await this.enqueueShellRequest(sessionId, () => sockets.shell.send(message));
+            }
+            catch (err) {
+                this.removeShellReplyWaiter(sessionId, msgId);
+                throw err;
+            }
+            const reply = await this.waitForShellReply(sessionId, msgId, replyPromise, 3000);
+            if (reply === 'timeout') {
                 return { status: 'timeout', matches: [], cursor_start: cursorPos, cursor_end: cursorPos };
             }
-            return result;
+            if (reply === 'closed' || reply.msgType !== 'complete_reply') {
+                return { status: 'error', matches: [], cursor_start: cursorPos, cursor_end: cursorPos };
+            }
+            const msgContent = reply.content;
+            return {
+                status: msgContent.status || 'ok',
+                matches: msgContent.matches || [],
+                cursor_start: msgContent.cursor_start ?? cursorPos,
+                cursor_end: msgContent.cursor_end ?? cursorPos,
+            };
         }
         catch (err) {
             console.error('[KernelService] Completion error:', err);
@@ -1028,10 +1703,13 @@ class KernelService {
         const parentHeader = JSON.parse(frames[delimIdx + 3].toString());
         const metadata = JSON.parse(frames[delimIdx + 4].toString());
         const content = JSON.parse(frames[delimIdx + 5].toString());
+        // Any frames after content are binary buffers (e.g. ipywidgets comm data)
+        const buffers = frames.slice(delimIdx + 6);
         return {
             msgType: header.msg_type,
             content,
             parentMsgId: parentHeader.msg_id || '',
+            buffers,
         };
     }
     /**
@@ -1281,6 +1959,10 @@ class KernelService {
         // stopKernel() already handles these, but unexpected process exits and failed startups
         // may call cleanup directly. Ensure we don't leak sockets or stale ChildProcess handles.
         this.detachProcessLifecycle(this.kernelProcesses.get(sessionId));
+        // Stop the unified readers and release any waiting consumers
+        this.stopIopubReader(sessionId);
+        this.stopShellReader(sessionId);
+        this.commStates.delete(sessionId);
         const sockets = this.zmqSockets.get(sessionId);
         if (sockets) {
             try {
@@ -1329,6 +2011,9 @@ class KernelService {
             this.fileToSession.delete(session.filePath);
         }
         this.detachProcessLifecycle(this.kernelProcesses.get(sessionId));
+        this.stopIopubReader(sessionId);
+        this.stopShellReader(sessionId);
+        this.commStates.delete(sessionId);
         const sockets = this.zmqSockets.get(sessionId);
         if (sockets) {
             try {
@@ -1393,6 +2078,22 @@ class KernelService {
         const { kernelName, filePath, connectionFile: oldConnFile } = session;
         // Set status to restarting
         session.status = 'starting';
+        // Restart hygiene: drop buffered outputs from the old process (otherwise
+        // stale outputs resurface on the next sync), and reset execution queues
+        // so nothing chains onto promises from the old kernel. In-flight executes
+        // fail when the old sockets close below; executeCode surfaces that as a
+        // clean 'Kernel was restarted' error because status is 'starting'.
+        this.cellOutputBuffers.delete(sessionId);
+        this.cellOutputTracking.delete(sessionId);
+        this.executionQueues.delete(sessionId);
+        this.shellRequestQueues.delete(sessionId);
+        // Stop the old kernel's readers and drop its comm state — widget
+        // comms do not survive a restart. In-flight executes see their queues
+        // close and fail with a clean 'Kernel was restarted' error; pending
+        // shell reply waiters resolve 'closed'.
+        this.stopIopubReader(sessionId);
+        this.stopShellReader(sessionId);
+        this.commStates.delete(sessionId);
         // Close ZeroMQ sockets
         const sockets = this.zmqSockets.get(sessionId);
         if (sockets) {
@@ -1542,6 +2243,23 @@ class KernelService {
             createdAt: session.createdAt,
         };
     }
+    /**
+     * Activity snapshot for the idle auto-release monitor (client mode):
+     * whether any kernel is busy/starting, and the most recent kernel activity
+     * across sessions in ms since epoch (sessions track it in seconds).
+     */
+    getIdleSnapshot() {
+        let anyBusy = false;
+        let lastActivityMs = null;
+        for (const session of this.sessions.values()) {
+            if (session.status === 'busy' || session.status === 'starting')
+                anyBusy = true;
+            const ms = session.lastActivity * 1000;
+            if (lastActivityMs === null || ms > lastActivityMs)
+                lastActivityMs = ms;
+        }
+        return { anyBusy, lastActivityMs };
+    }
     /**
      * Get all sessions
      */
@@ -1574,7 +2292,16 @@ class KernelService {
      */
     getDeadSessions() {
         const deadSessions = this.sessionStore.getDeadSessions(this.serverId);
-        return deadSessions.map(s => ({
+        return deadSessions
+            // A session that is live in memory is NOT dead — e.g. it was
+            // successfully reattached (or is mid-reattach) after a server restart
+            // and the DB 'orphaned' status is stale or racing the reattach loop.
+            // Only sessions with no live entry (or a dead one) are true orphans.
+            .filter(s => {
+            const live = this.sessions.get(s.sessionId);
+            return !live || live.status === 'dead';
+        })
+            .map(s => ({
             sessionId: s.sessionId,
             kernelName: s.kernelName,
             filePath: s.filePath,
@@ -1582,11 +2309,50 @@ class KernelService {
             lastHeartbeat: s.lastHeartbeat,
         }));
     }
+    /**
+     * Auto-delete dead session rows whose kernel process is CONFIRMED gone
+     * (no PID, PID not running, or PID reused by another process). These are
+     * pure bookkeeping — nothing to kill, nothing to lose — so they need no
+     * user confirmation. Rows whose PID is still alive are kept for the
+     * explicit "Clean Up" flow (killing a process should stay a user action,
+     * and legacy rows without a start-time fingerprint can't be verified).
+     */
+    async autoCleanupDeadSessions() {
+        const dead = this.sessionStore.getDeadSessions(this.serverId).filter(s => {
+            const live = this.sessions.get(s.sessionId);
+            return !live || live.status === 'dead';
+        });
+        const deletable = [];
+        for (const s of dead) {
+            if (!s.kernelPid || !this.isPidAlive(s.kernelPid)) {
+                deletable.push(s.sessionId);
+                continue;
+            }
+            if (s.kernelStartTime) {
+                const currentStartTime = await this.getProcessStartTime(s.kernelPid);
+                if (!currentStartTime || currentStartTime !== s.kernelStartTime) {
+                    // PID reused by an unrelated process — the kernel itself is gone
+                    deletable.push(s.sessionId);
+                }
+            }
+        }
+        if (deletable.length > 0) {
+            this.sessionStore.deleteSessions(deletable);
+            console.log(`[Kernel] Auto-cleaned ${deletable.length} dead session record(s) (process gone)`);
+        }
+        return deletable.length;
+    }
     /**
      * Cleanup dead sessions by deleting them from the database
      */
     async cleanupDeadSessions(sessionIds) {
-        const deadSessions = this.sessionStore.getDeadSessions(this.serverId);
+        // Same live-session guard as getDeadSessions: never clean up a session
+        // that is currently live in memory (e.g. reattached after a restart while
+        // its DB row still says 'orphaned').
+        const deadSessions = this.sessionStore.getDeadSessions(this.serverId).filter(s => {
+            const live = this.sessions.get(s.sessionId);
+            return !live || live.status === 'dead';
+        });
         const targets = sessionIds && sessionIds.length > 0
             ? deadSessions.filter(s => sessionIds.includes(s.sessionId))
             : deadSessions;
@@ -1635,6 +2401,10 @@ class KernelService {
      * If preserveKernels is true, keep kernel processes running and close the session store.
      */
     async shutdown(options) {
+        if (this.livenessTimer) {
+            clearInterval(this.livenessTimer);
+            this.livenessTimer = null;
+        }
         if (options?.preserveKernels) {
             try {
                 this.sessionStore.markAllOrphaned(this.serverId, this.serverInstanceId);