@tlockney/cdp-kernel 0.4.0 → 0.5.0

package/dist/wire.js

@@ -41,7 +41,7 @@ export function makeHeader(session, msgType) {
 }
 export function encodeMessage(session, key, msgType, content, options = {}) {
     const parts = [
-        makeHeader(session, msgType),
+        options.header ?? makeHeader(session, msgType),
         options.parentHeader ?? {},
         options.metadata ?? {},
         content,
@@ -70,6 +70,8 @@ function parseObject(frame) {
     }
     return value;
 }
+// Only msg_id and msg_type are required of a received header; nothing
+// reads the rest, so a client that leaves them out still works.
 function isHeader(value) {
     return typeof value.msg_id === "string" && typeof value.msg_type === "string";
 }

package/dist/workerd-target.d.ts

@@ -1,4 +1,4 @@
-import type { CdpParams, EvaluateResult, RemoteObject } from "./cdp.ts";
+import { type CdpParams, type EvaluateResult, type RemoteObject } from "./cdp.ts";
 import { transformCellForEval } from "./transform.ts";
 import { type KernelTarget, type TargetEvents } from "./target.ts";
 import { type WsChannel } from "./ws-client.ts";
@@ -53,6 +53,12 @@ export interface WorkerHost {
     stop(now: boolean): void;
     onExit(listener: (code: number | null) => void): void;
 }
+/**
+ * Removes workerd directories under `root` whose kernel has died without
+ * cleaning up (SIGKILL runs no exit hook), killing their workerd if it's
+ * still running (#220).
+ */
+export declare function sweepStaleWorkerdDirs(root?: string): void;
 /** The driver on a workerd of its own, behind a Unix socket. */
 export declare class RawWorkerd implements WorkerHost {
     #private;

package/dist/workerd-target.js

@@ -18,18 +18,20 @@ import { Buffer } from "node:buffer";
 import process from "node:process";
 import { execFileSync, spawn } from "node:child_process";
 import { randomBytes } from "node:crypto";
-import { copyFileSync, existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { copyFileSync, existsSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, rmSync, writeFileSync } from "node:fs";
 import { builtinModules } from "node:module";
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import { pathToFileURL } from "node:url";
 import { build } from "esbuild";
+import { asEvaluateResult } from "./cdp.js";
 import { transformCellForEval } from "./transform.js";
 import { runtimeDir } from "./package.js";
+import { killOnExit } from "./children.js";
 import { Interrupted, isTargetMessage } from "./target.js";
 import { connectUnixWebSocket } from "./ws-client.js";
 import { findWranglerConfig, startMiniflare } from "./workers-project.js";
-import { ensureInstalled, findLockFile, jsrToNpm, parsePackageSpecifier } from "../runtime/node-hooks.mjs";
+import { ensureInstalledAsync, findLockFile, jsrToNpm, parsePackageSpecifier } from "../runtime/node-hooks.mjs";
 /** The driver Worker's modules; the first is its main module. */
 export const WORKER_MODULES = ["workerd-worker.mjs", "display.mjs", "comm.mjs", "widget.mjs", "evaluator.mjs", "introspect.mjs"];
 /** See BunTarget: how long an interrupted cell has to show it isn't busy. */
@@ -121,7 +123,8 @@ export async function bundleModule(specifier, cwd) {
                 ? jsrToNpm(specifier.slice(4))
                 : null;
         if (pkg) {
-            const { dir, alias } = ensureInstalled(pkg, "npm", { lockFile: findLockFile(cwd) });
+            // Async: a sync install would stall the heartbeat and interrupts (#149).
+            const { dir, alias } = await ensureInstalledAsync(pkg, "npm", { lockFile: findLockFile(cwd) });
             entry = alias + pkg.subpath;
             resolveDir = dir;
         }
@@ -168,6 +171,65 @@ export function cellMeta(cwd) {
     const replUrl = new URL("$cdpk$repl.mjs", pathToFileURL(cwd.endsWith("/") ? cwd : cwd + "/"));
     return { url: replUrl.href, dirname: cwd, filename: join(cwd, "$cdpk$repl.mjs") };
 }
+const WORKERD_DIR_PREFIX = "cdpk-workerd-";
+/** In each workerd directory: `{ host, workerd }` pids. */
+const OWNER_FILE = "owner.json";
+function pidAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err.code === "EPERM";
+    }
+}
+/** Whether `pid` is still the workerd serving `dir`, not a reused pid. */
+function isWorkerdIn(pid, dir) {
+    try {
+        if (process.platform === "linux")
+            return readlinkSync(`/proc/${pid}/cwd`) === dir;
+        const args = execFileSync("ps", ["-o", "args=", "-p", String(pid)], { encoding: "utf8" });
+        return /workerd/.test(args) && args.includes("config.capnp");
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Removes workerd directories under `root` whose kernel has died without
+ * cleaning up (SIGKILL runs no exit hook), killing their workerd if it's
+ * still running (#220).
+ */
+export function sweepStaleWorkerdDirs(root = tmpdir()) {
+    let names;
+    try {
+        names = readdirSync(root).filter((name) => name.startsWith(WORKERD_DIR_PREFIX));
+    }
+    catch {
+        return;
+    }
+    for (const name of names) {
+        const dir = join(root, name);
+        let owner;
+        try {
+            owner = JSON.parse(readFileSync(join(dir, OWNER_FILE), "utf8"));
+        }
+        catch {
+            continue; // Another user's, from before owner files, or being set up.
+        }
+        if (typeof owner.host !== "number" || pidAlive(owner.host))
+            continue;
+        if (typeof owner.workerd === "number" && isWorkerdIn(owner.workerd, dir)) {
+            try {
+                process.kill(owner.workerd, "SIGKILL");
+            }
+            catch {
+                // Gone already.
+            }
+        }
+        rmSync(dir, { recursive: true, force: true });
+    }
+}
 /** The driver on a workerd of its own, behind a Unix socket. */
 export class RawWorkerd {
     runtime;
@@ -184,7 +246,9 @@ export class RawWorkerd {
         const source = await runtimeDir(new URL("../runtime/", import.meta.url).href);
         // mkdtemp makes the directory 0700: only this user reaches the socket,
         // and the token stops anything else running as them by accident.
-        const dir = mkdtempSync(join(tmpdir(), "cdpk-workerd-"));
+        sweepStaleWorkerdDirs();
+        const dir = mkdtempSync(join(tmpdir(), WORKERD_DIR_PREFIX));
+        writeFileSync(join(dir, OWNER_FILE), JSON.stringify({ host: process.pid }));
         const socket = join(dir, "sock");
         for (const name of WORKER_MODULES)
             copyFileSync(join(source, name), join(dir, name));
@@ -194,11 +258,21 @@ export class RawWorkerd {
         // --experimental: the UnsafeEval binding.
         const child = spawn(workerd, ["serve", "config.capnp", "--experimental"], { cwd: dir, stdio: ["ignore", "pipe", "pipe"], env });
         child.once("exit", () => rmSync(dir, { recursive: true, force: true }));
+        if (child.pid !== undefined)
+            writeFileSync(join(dir, OWNER_FILE), JSON.stringify({ host: process.pid, workerd: child.pid }));
+        killOnExit(child, () => {
+            child.kill("SIGKILL");
+            rmSync(dir, { recursive: true, force: true });
+        });
         // Cell output is captured inside the Worker; what workerd itself prints
         // (its own warnings, errors from background work) goes to the server log.
+        // `early` keeps that output for a startup error only, and stops growing
+        // once the session is open.
         let early = "";
+        let starting = true;
         const log = (chunk) => {
-            early += chunk.toString();
+            if (starting)
+                early += chunk.toString();
             process.stderr.write(`[cdp-kernel workerd] ${chunk}`);
         };
         child.stdout?.on("data", log);
@@ -221,7 +295,10 @@ export class RawWorkerd {
             }
         })();
         try {
-            return new RawWorkerd(child, await Promise.race([ready, exited]), compatibilityDate);
+            const host = new RawWorkerd(child, await Promise.race([ready, exited]), compatibilityDate);
+            starting = false;
+            early = "";
+            return host;
         }
         catch (err) {
             child.kill();
@@ -229,7 +306,16 @@ export class RawWorkerd {
         }
     }
     stop(now) {
-        this.#child.kill(now ? "SIGKILL" : "SIGTERM");
+        if (now) {
+            this.#child.kill("SIGKILL");
+            return;
+        }
+        this.#child.kill("SIGTERM");
+        // A busy isolate keeps workerd's event loop from ever seeing SIGTERM (#128).
+        setTimeout(() => {
+            if (this.#child.exitCode === null && this.#child.signalCode === null)
+                this.#child.kill("SIGKILL");
+        }, 3000).unref();
     }
     onExit(listener) {
         this.#child.once("exit", listener);
@@ -342,6 +428,9 @@ export class WorkerdTarget {
         this.#pending.clear();
     }
     #request(op, params = {}, interruptible = false) {
+        // The channel drops writes after it closes, so nothing would answer.
+        if (this.#exitedOnce)
+            return Promise.reject(new Abandoned("the workerd session is closed"));
         const id = ++this.#nextId;
         return new Promise((resolve, reject) => {
             this.#pending.set(id, { resolve, reject });
@@ -377,7 +466,12 @@ export class WorkerdTarget {
             if (!modules[specifier].startsWith("Promise.reject("))
                 this.#sent.add(specifier);
         }
-        return (await this.#interruptibly("eval", { code: script, modules }, "Execution"));
+        const { failedModules, ...reply } = await this.#interruptibly("eval", { code: script, modules }, "Execution");
+        // A module that threw while loading is rebuilt for its next import (#147).
+        if (Array.isArray(failedModules))
+            for (const specifier of failedModules)
+                this.#sent.delete(specifier);
+        return asEvaluateResult(reply);
     }
     /**
      * A cell that's waiting on something leaves the driver free: it answers a

package/dist/ws-client.d.ts

@@ -1,6 +1,7 @@
 import { Buffer } from "node:buffer";
 export declare const OP_CONTINUATION = 0;
 export declare const OP_TEXT = 1;
+export declare const OP_BINARY = 2;
 export declare const OP_CLOSE = 8;
 export declare const OP_PING = 9;
 export declare const OP_PONG = 10;
@@ -11,7 +12,10 @@ export interface Frame {
 }
 /** One frame; clients must mask what they send. */
 export declare function encodeFrame(opcode: number, payload: Buffer, mask?: Buffer | null): Buffer;
-/** Splits a byte stream into frames, holding back a partial one. */
+/**
+ * Splits a byte stream into frames, holding back a partial one. Chunks of a
+ * large frame are only joined once the whole frame has arrived.
+ */
 export declare class FrameDecoder {
     #private;
     push(chunk: Buffer): Frame[];

package/dist/ws-client.js

@@ -2,13 +2,14 @@
 // target's session with its driver Worker (#26). The WebSocket that Node
 // and Deno provide can't connect to a Unix socket, and a TCP listener would
 // be reachable by every local process. Text messages only: that's all the
-// session carries.
+// session carries, so binary messages are dropped.
 import { Buffer } from "node:buffer";
 import { createHash, randomBytes } from "node:crypto";
 import { connect } from "node:net";
 const GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
 export const OP_CONTINUATION = 0x0;
 export const OP_TEXT = 0x1;
+export const OP_BINARY = 0x2;
 export const OP_CLOSE = 0x8;
 export const OP_PING = 0x9;
 export const OP_PONG = 0xa;
@@ -40,14 +41,24 @@ export function encodeFrame(opcode, payload, mask = randomBytes(4)) {
     }
     return out;
 }
-/** Splits a byte stream into frames, holding back a partial one. */
+/**
+ * Splits a byte stream into frames, holding back a partial one. Chunks of a
+ * large frame are only joined once the whole frame has arrived.
+ */
 export class FrameDecoder {
-    #buffer = Buffer.alloc(0);
+    #chunks = [];
+    #length = 0;
+    /** Bytes the frame at the front needs, once its header is known. */
+    #needed = 0;
     push(chunk) {
-        this.#buffer = Buffer.concat([this.#buffer, chunk]);
+        this.#chunks.push(chunk);
+        this.#length += chunk.length;
+        if (this.#length < this.#needed)
+            return [];
+        let b = this.#chunks.length === 1 ? this.#chunks[0] : Buffer.concat(this.#chunks, this.#length);
+        this.#needed = 0;
         const frames = [];
         for (;;) {
-            const b = this.#buffer;
             if (b.length < 2)
                 break;
             const masked = (b[1] & 0x80) !== 0;
@@ -68,15 +79,19 @@ export class FrameDecoder {
             const maskAt = offset;
             if (masked)
                 offset += 4;
-            if (b.length < offset + len)
+            if (b.length < offset + len) {
+                this.#needed = offset + len;
                 break;
+            }
             const payload = Buffer.from(b.subarray(offset, offset + len));
             if (masked)
                 for (let i = 0; i < len; i++)
                     payload[i] ^= b[maskAt + (i & 3)];
             frames.push({ fin: (b[0] & 0x80) !== 0, opcode: b[0] & 0x0f, payload });
-            this.#buffer = b.subarray(offset + len);
+            b = b.subarray(offset + len);
         }
+        this.#chunks = b.length ? [b] : [];
+        this.#length = b.length;
         return frames;
     }
 }
@@ -95,6 +110,9 @@ export function connectUnixWebSocket(socketPath, { path = "/", headers = {} } =
         let head = Buffer.alloc(0);
         const decoder = new FrameDecoder();
         let parts = [];
+        // The opcode of the message whose fragments are arriving.
+        let messageOpcode = null;
+        let closeSent = false;
         const finish = () => {
             if (closed)
                 return;
@@ -108,30 +126,45 @@ export function connectUnixWebSocket(socketPath, { path = "/", headers = {} } =
             if (!closed)
                 socket.write(encodeFrame(opcode, payload));
         };
+        const sendClose = (payload) => {
+            if (closeSent)
+                return;
+            closeSent = true;
+            write(OP_CLOSE, payload);
+        };
         const channel = {
             send: (text) => write(OP_TEXT, Buffer.from(text)),
             onMessage: (listener) => void messageListeners.push(listener),
             onClose: (listener) => void closeListeners.push(listener),
             close() {
-                write(OP_CLOSE, Buffer.alloc(0));
+                sendClose(Buffer.alloc(0));
                 socket.end();
             },
         };
         const onFrames = (frames) => {
             for (const frame of frames) {
-                if (frame.opcode === OP_TEXT || frame.opcode === OP_CONTINUATION) {
-                    parts.push(frame.payload);
+                if (frame.opcode === OP_TEXT || frame.opcode === OP_BINARY || frame.opcode === OP_CONTINUATION) {
+                    if (frame.opcode !== OP_CONTINUATION) {
+                        messageOpcode = frame.opcode;
+                        parts = [];
+                    }
+                    if (messageOpcode === OP_TEXT)
+                        parts.push(frame.payload);
                     if (frame.fin) {
-                        const text = Buffer.concat(parts).toString("utf8");
+                        const text = messageOpcode === OP_TEXT ? Buffer.concat(parts).toString("utf8") : null;
                         parts = [];
-                        for (const listener of messageListeners)
-                            listener(text);
+                        messageOpcode = null;
+                        if (text !== null)
+                            for (const listener of messageListeners)
+                                listener(text);
                     }
                 }
                 else if (frame.opcode === OP_PING) {
                     write(OP_PONG, frame.payload);
                 }
                 else if (frame.opcode === OP_CLOSE) {
+                    // Echo the status code, as RFC 6455 section 5.5.1 asks.
+                    sendClose(frame.payload.subarray(0, 2));
                     socket.end();
                 }
             }

package/dist/zmtp.d.ts

@@ -20,6 +20,8 @@ export declare function recvMultipart(reader: SocketReader): Promise<Uint8Array[
 /** Async FIFO with blocking pop. */
 export declare class Queue<T> {
     #private;
+    /** Items waiting to be popped. */
+    get size(): number;
     push(item: T): void;
     pop(): Promise<T>;
 }
@@ -29,7 +31,11 @@ export interface RoutedMessage {
     peer: string;
     frames: Uint8Array[];
 }
-/** ROUTER socket: each TCP connection is one peer with a generated identity. */
+/**
+ * ROUTER socket: each TCP connection is one peer with a generated identity.
+ * Writes ignore socket.write's backpressure signal: node:net buffers the
+ * rest, which is fine for loopback peers that read promptly.
+ */
 export declare class RouterSocket {
     #private;
     readonly incoming: Queue<RoutedMessage>;

package/dist/zmtp.js

@@ -36,6 +36,10 @@ export class SocketReader {
         this.#waiter = null;
         waiter?.();
     }
+    // Each call joins every buffered chunk, so a reader that keeps a large
+    // backlog while small chunks trickle in recopies it per call (quadratic).
+    // Jupyter traffic over loopback arrives in few, large chunks; a chunk
+    // list with an offset would avoid the copies if that ever changes.
     async readExact(n) {
         while (this.#buffered < n) {
             if (this.#error)
@@ -136,11 +140,28 @@ export async function recvMultipart(reader) {
             return frames;
     }
 }
+/** How long a port from the connection file may stay taken before the kernel gives up. */
+const LISTEN_RETRY_MS = 5000;
+const LISTEN_RETRY_INTERVAL_MS = 100;
 function listen(port, host, onConn) {
     const server = createServer((socket) => {
         socket.setNoDelay(true);
         onConn(socket);
     });
+    // The frontend picks these ports by binding and releasing them, so another
+    // socket can take one before the kernel binds it. Wait for it to be freed
+    // rather than die before kernel_info.
+    const deadline = Date.now() + LISTEN_RETRY_MS;
+    const onError = (err) => {
+        if (err.code !== "EADDRINUSE" || Date.now() >= deadline) {
+            server.off("error", onError);
+            server.emit("error", err);
+            return;
+        }
+        setTimeout(() => server.listen(port, host), LISTEN_RETRY_INTERVAL_MS);
+    };
+    server.on("error", onError);
+    server.once("listening", () => server.off("error", onError));
     server.listen(port, host);
     return server;
 }
@@ -148,6 +169,10 @@ function listen(port, host, onConn) {
 export class Queue {
     #items = [];
     #waiters = [];
+    /** Items waiting to be popped. */
+    get size() {
+        return this.#items.length;
+    }
     push(item) {
         const waiter = this.#waiters.shift();
         if (waiter)
@@ -173,7 +198,11 @@ export function startHeartbeat(port, host) {
         })().catch(() => socket.destroy());
     });
 }
-/** ROUTER socket: each TCP connection is one peer with a generated identity. */
+/**
+ * ROUTER socket: each TCP connection is one peer with a generated identity.
+ * Writes ignore socket.write's backpressure signal: node:net buffers the
+ * rest, which is fine for loopback peers that read promptly.
+ */
 export class RouterSocket {
     incoming = new Queue();
     peers = new Map();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tlockney/cdp-kernel",
-  "version": "0.4.0",
-  "description": "Deno's Jupyter kernel, extracted to run notebooks on any V8 runtime over the Chrome DevTools Protocol",
+  "version": "0.5.0",
+  "description": "Deno's Jupyter kernel, extracted to run notebooks on Node, Deno, Bun, workerd and celld",
   "license": "MIT",
   "author": "Thomas Lockney <thomas@lockney.net>",
   "repository": {
@@ -42,7 +42,8 @@
   "files": [
     "dist/",
     "runtime/*.mjs",
-    "runtime/*.d.mts"
+    "runtime/*.d.mts",
+    "THIRD-PARTY-NOTICES"
   ],
   "publishConfig": {
     "access": "public"
@@ -50,6 +51,10 @@
   "scripts": {
     "build": "rm -rf dist && tsc -p tsconfig.build.json",
     "check": "tsc -p tsconfig.json",
+    "docs:dev": "vitepress dev docs",
+    "docs:build": "vitepress build docs",
+    "docs:notebooks": "node docs/.vitepress/run-notebooks.ts",
+    "docs:widgets": "uv run --with jupyterlab --with anywidget --with playwright --with nbformat python test/widget_screenshots.py",
     "install:node": "node src/main.ts install --runtime node",
     "install:deno": "node src/main.ts install --runtime deno",
     "install:node-deno": "node src/main.ts install --runtime node --deno-compat",
@@ -69,6 +74,7 @@
   "devDependencies": {
     "@types/node": "^24.19.2",
     "typescript": "^5.9.3",
+    "vitepress": "^1.6.4",
     "workerd": "1.20261015.1",
     "wrangler": "4.151.0"
   }