@tlockney/cdp-kernel 0.5.0 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tlockney/cdp-kernel",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Deno's Jupyter kernel, extracted to run notebooks on Node, Deno, Bun, workerd and celld",
   "license": "MIT",
   "author": "Thomas Lockney <thomas@lockney.net>",
@@ -20,7 +20,7 @@
   ],
   "type": "module",
   "engines": {
-    "node": ">=24"
+    "node": ">=22.18"
   },
   "bin": {
     "cdp-kernel": "dist/main.js"
@@ -64,18 +64,23 @@
     "prepack": "npm run build",
     "test": "node --test --test-timeout=300000 test/*.test.ts",
     "e2e": "uv run --with nbclient --with nbformat --with jupyter_client python test/e2e.py",
+    "bench": "uv run --with jupyter_client python test/bench.py",
+    "soak": "uv run --with jupyter_client python test/soak.py",
     "test:lab": "uv run --with jupyterlab --with anywidget --with playwright --with nbformat python test/lab_widgets.py"
   },
   "dependencies": {
     "acorn": "^8.15.0",
     "acorn-walk": "^8.3.4",
-    "esbuild": "^0.25.10"
+    "esbuild": "^0.25.10",
+    "typescript": "^6.0.3"
   },
   "devDependencies": {
     "@types/node": "^24.19.2",
-    "typescript": "^5.9.3",
     "vitepress": "^1.6.4",
     "workerd": "1.20261015.1",
     "wrangler": "4.151.0"
+  },
+  "optionalDependencies": {
+    "sucrase": "^3.35.1"
   }
 }

package/runtime/bootstrap.mjs

@@ -2,8 +2,8 @@
 //   node --inspect=127.0.0.1:0 runtime/bootstrap.mjs
 //   deno run -A --inspect=127.0.0.1:0 runtime/bootstrap.mjs
 // then drives it over CDP. Everything the host needs from inside the
-// runtime lives on globalThis.__cdpk; output to the host goes through the
-// `__cdpk_send` CDP binding (Runtime.addBinding) plus plain stdout/stderr.
+// runtime lives on globalThis.__cdpk; output to the host goes over plain
+// stdout/stderr, with messages as records on stdout (see send below).
 // Bun is launched as `bun runtime/bootstrap.mjs` and driven over a pipe
 // instead; see bun-channel.mjs.
 //
@@ -27,7 +27,7 @@ const cwdUrl = pathToFileURL(cwd.endsWith("/") ? cwd : cwd + "/");
 // Stand-in referrer for cell code, like Deno's ./$deno$jupyter_repl.mts.
 const replUrl = new URL("$cdpk$repl.mjs", cwdUrl);
 
-const { inspect, inspectPlain, readFile, writeOut, writeErr, readStdinLine } = await platform();
+const { inspect, inspectPlain, readFile, writeOut, writeErr, readStdin } = await platform();
 
 function base64(bytes) {
   let binary = "";
@@ -38,11 +38,12 @@ function base64(bytes) {
 }
 
 // --- Output ordering ----------------------------------------------------
-// stdout/stderr reach the host over pipes; display messages over the
-// WebSocket. To keep them in order, every binding message is preceded by a
-// numbered marker written to both pipes. The host holds the message until
-// both pipes have delivered that marker, and a gated marker ("m") holds
-// later pipe output until the message has gone out.
+// stdout/stderr reach the host over pipes, and so do messages: as records
+// on stdout on Node and Deno, over Bun's channel on Bun. Every message has
+// a number; stderr gets a gated marker ("m") with it, and stdout the record
+// (Bun: a gated marker on both pipes). The host holds the message until
+// both pipes have reached that number, and holds later pipe output until
+// the message has gone out.
 let barrierSeq = 0;
 function barrier(gated = false) {
   const n = ++barrierSeq;
@@ -52,15 +53,32 @@ function barrier(gated = false) {
   return n;
 }
 
+// Node and Deno: messages were a CDP binding's events, but an event just
+// before an evaluate's reply delays the reply ~40 ms (#271). A record is
+// the message's JSON, which holds no raw ESC, between "\x1b[cdpk:<" and
+// "\x1b]". Cells can call this too; the host drops what isn't a message
+// (#122).
+let attached = false;
+function sendRecord(text) {
+  text = String(text);
+  if (attached && !text.includes("\x1b")) writeOut(`\x1b[cdpk:<${text}\x1b]`);
+}
+if (!isBun) globalThis.__cdpk_send = sendRecord;
+
 function send(payload) {
-  const binding = globalThis.__cdpk_send;
-  if (typeof binding !== "function") return; // host not attached yet
+  if (isBun ? typeof globalThis.__cdpk_send !== "function" : !attached) return; // host not attached yet
   // Serialize before writing the marker: if the payload can't be
   // serialized (a BigInt, a cycle), the error reaches the cell without a
   // gated marker the host would wait on forever (#129). `payload` always
   // has a `kind`, so `body` is never "{}".
   const body = JSON.stringify(payload);
-  binding(`{"barrier":${barrier(true)},${body.slice(1)}`);
+  if (isBun) {
+    globalThis.__cdpk_send(`{"barrier":${barrier(true)},${body.slice(1)}`);
+    return;
+  }
+  const n = ++barrierSeq;
+  writeErr(`\x1b[cdpk:${n}m]\x1b`);
+  sendRecord(`{"barrier":${n},${body.slice(1)}`);
 }
 
 const jupyter = createJupyter({
@@ -74,7 +92,7 @@ const jupyter = createJupyter({
   // stdin for its JSON-line reply.
   input(prompt, password) {
     send({ kind: "input_request", prompt, password });
-    const line = readStdinLine();
+    const line = readInputReply();
     if (line === null) return null;
     const reply = JSON.parse(line);
     if (reply.interrupted === true) {
@@ -157,13 +175,23 @@ function hold(value) {
   return promise;
 }
 
+// Yields one macrotask. Code the inspector dispatches calls this, and Node
+// and Deno dispatch a message that arrives while JavaScript runs from a V8
+// interrupt, which can land inside their timer code. A setTimeout there can
+// create a timer list while the timer heap is mid-update, and the heap then
+// loses it: every later setTimeout of that duration never fires (#237).
+// setImmediate only appends to a linked list.
+const yieldTask = () => new Promise((resolve) => setImmediate(resolve));
+
 // Adds the cause to an error from a Deno member the --deno-compat shim
 // lacks (#172); the identity everywhere else.
 let explainError = (text) => text;
 
 globalThis.__cdpk = {
+  // The host calls this once it reads message records (Node and Deno).
+  attach: () => void (attached = true),
   // Cell code awaits this first; see PRELUDE in src/transform.ts.
-  tick: () => new Promise((resolve) => setTimeout(resolve, 0)),
+  tick: yieldTask,
   // Node: the resolve hook re-parents these imports. Deno would resolve
   // relative paths against this file, so denoImport does it here.
   import: (specifier, options) =>
@@ -196,10 +224,19 @@ globalThis.__cdpk = {
   // Frontend comm messages (#9), from the host's shell channel. Handlers
   // run after one macrotask, like cells, so interrupt can reach them.
   async commDispatch(msgType, content, metadata, buffers64) {
-    await new Promise((resolve) => setTimeout(resolve, 0));
+    await yieldTask();
     const buffers = buffers64.map((b) => Uint8Array.from(atob(b), (c) => c.charCodeAt(0)));
     return await jupyter.commDispatch(msgType, content, metadata, buffers);
   },
+  // The same, for a message the host wrote to stdin (Node and Deno). A
+  // value passed as a CDP call argument is compiled as source, and V8's
+  // compilation cache keeps that source until the heap nears its limit, so
+  // each message's data would stay in memory (#287). Read before the first
+  // await, so messages are taken in the order the host wrote them.
+  commDispatchFromStdin() {
+    const { msgType, content, metadata, buffers } = readCommRecord();
+    return globalThis.__cdpk.commDispatch(msgType, content, metadata, buffers);
+  },
   describeError(value) {
     if (value instanceof Error) {
       const message = explainError(value.message);
@@ -308,21 +345,24 @@ if (isBun) {
   });
 }
 
+function writeAllSync(stream, bytes) {
+  while (bytes.length > 0) bytes = bytes.subarray(stream.writeSync(bytes));
+}
+
 async function platform() {
   if (isDeno) {
-    const stdinBuf = new Uint8Array(1);
     return {
       inspect: (v) => Deno.inspect(v, { colors: true, depth: 4 }),
       inspectPlain: (v) => Deno.inspect(v, { colors: false, depth: 2 }),
       readFile: (p) => Deno.readFileSync(p),
-      writeOut: (s) => Deno.stdout.writeSync(ENC.encode(s)),
-      writeErr: (s) => Deno.stderr.writeSync(ENC.encode(s)),
-      readStdinLine: () => readLine(() => (Deno.stdin.readSync(stdinBuf) ? stdinBuf[0] : null)),
+      // writeSync can write part of the buffer; a record must go out whole.
+      writeOut: (s) => writeAllSync(Deno.stdout, ENC.encode(s)),
+      writeErr: (s) => writeAllSync(Deno.stderr, ENC.encode(s)),
+      readStdin: (bytes) => Deno.stdin.readSync(bytes) ?? 0,
     };
   }
   const util = await import("node:util");
   const fs = await import("node:fs");
-  const buf = Buffer.alloc(1);
   // Node writes to a pipe without blocking and queues what doesn't fit for
   // the event loop to flush. input() blocks the main thread in readSync, so
   // queued output, and the input_request's barrier marker after it, would
@@ -335,19 +375,20 @@ async function platform() {
     inspect: (v) => util.inspect(v, { colors: true, depth: 4 }),
     inspectPlain: (v) => util.inspect(v, { colors: false, depth: 2 }),
     readFile: (p) => fs.readFileSync(p),
-    writeOut: (s) => process.stdout.write(s),
-    writeErr: (s) => process.stderr.write(s),
-    readStdinLine: () =>
-      readLine(() => {
-        for (;;) {
-          try {
-            return fs.readSync(0, buf, 0, 1, null) === 1 ? buf[0] : null;
-          } catch (err) {
-            if (err.code !== "EAGAIN") throw err;
-            Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
-          }
+    // Bound now: a cell or library that patches process.stdout.write to
+    // capture output must not swallow message records (#271).
+    writeOut: process.stdout.write.bind(process.stdout),
+    writeErr: process.stderr.write.bind(process.stderr),
+    readStdin: (bytes) => {
+      for (;;) {
+        try {
+          return fs.readSync(0, bytes, 0, bytes.length, null);
+        } catch (err) {
+          if (err.code !== "EAGAIN") throw err;
+          Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
         }
-      }),
+      }
+    },
   };
 }
 
@@ -376,12 +417,51 @@ async function syncBunOutput() {
   }
 }
 
-function readLine(nextByte) {
+// The host writes two kinds of record to stdin, both read synchronously: a
+// prompt's reply, as one JSON line, and a frontend comm message (#287), as
+// a `comm <length>` line followed by that many bytes of JSON.
+const COMM_HEADER = /^comm (\d+)$/;
+
+/** `n` bytes from stdin, or null if it ends first. */
+function readExactly(n) {
+  const bytes = new Uint8Array(n);
+  for (let at = 0; at < n;) {
+    const got = readStdin(bytes.subarray(at));
+    if (!got) return null;
+    at += got;
+  }
+  return bytes;
+}
+
+function readLine() {
   const bytes = [];
   for (;;) {
-    const byte = nextByte();
+    const byte = readExactly(1)?.[0] ?? null;
     if (byte === null) return bytes.length ? new TextDecoder().decode(new Uint8Array(bytes)) : null;
     if (byte === 0x0a) return new TextDecoder().decode(new Uint8Array(bytes));
     bytes.push(byte);
   }
 }
+
+/** The next prompt reply's line, past any comm record left unread. */
+function readInputReply() {
+  for (;;) {
+    const line = readLine();
+    const comm = line === null ? null : COMM_HEADER.exec(line);
+    if (comm === null) return line;
+    readExactly(Number(comm[1]));
+  }
+}
+
+/** The next comm message, past any prompt reply left unread. */
+function readCommRecord() {
+  for (;;) {
+    const line = readLine();
+    if (line === null) throw new Error("stdin closed before a comm message arrived");
+    const comm = COMM_HEADER.exec(line);
+    if (comm === null) continue;
+    const body = readExactly(Number(comm[1]));
+    if (body === null) throw new Error("stdin closed in the middle of a comm message");
+    return JSON.parse(new TextDecoder().decode(body));
+  }
+}

package/runtime/introspect.mjs

@@ -77,7 +77,7 @@ function safeSource(fn) {
 function propertyNames(value) {
   if ((typeof value !== "object" && typeof value !== "function") || value === null) return [];
   try {
-    // A function's own internals aren't worth listing; older V8 (Node 24)
+    // A function's own internals aren't worth listing; older V8 (Node 22 and 24)
     // also gives sloppy-mode functions own `arguments` and `caller`.
     const internal = ["length", "name", "prototype", "arguments", "caller"];
     return Object.getOwnPropertyNames(value).filter((n) => !(typeof value === "function" && internal.includes(n)));

package/runtime/node-hooks.d.mts

@@ -47,6 +47,12 @@ export function ensureInstalledAsync(
     run?: (...args: Parameters<InstallRunner>) => ReturnType<InstallRunner> | Promise<ReturnType<InstallRunner>>;
   },
 ): Promise<{ dir: string; alias: string }>;
+/** Where a package is installed in the cache, or null; never installs. */
+export function installedPackageDir(
+  pkg: { name: string; range: string },
+  installer?: Installer,
+  options?: { lockFile?: string | null },
+): { dir: string; alias: string } | null;
 
 export const LOCK_FILE: string;
 export interface Lock {

package/runtime/node-hooks.mjs

@@ -320,6 +320,17 @@ export function ensureInstalledAsync({ name, range }, installer = "npm", { lockF
   });
 }
 
+/**
+ * Where `name@range` is installed in the installer's cache, or null if it
+ * isn't: nothing is installed. For type information (#273), which must not
+ * start downloads.
+ */
+export function installedPackageDir({ name, range }, installer = "npm", { lockFile = null } = {}) {
+  const dir = cacheDirPath(installer);
+  const { alias } = lockedRequest(name, range, lockFile);
+  return installedVersion(dir, alias) === null ? null : { dir, alias };
+}
+
 // --- Imported TypeScript (#73) --------------------------------------------------
 // Node strips types in place, so line and column numbers stay those of the
 // file, but it rejects TypeScript that needs code generation. Such a file