@tlockney/cdp-kernel 0.4.0 → 0.6.0

package/runtime/node-hooks.mjs

@@ -9,15 +9,19 @@
 //     one package can coexist (Deno keeps a global cache the same way).
 //   - jsr:@scope/name[@range][/subpath] maps onto JSR's npm compatibility
 //     registry (npm.jsr.io) as @jsr/scope__name.
+//   - TypeScript files a cell imports load with Node's own type stripping
+//     where it can; syntax that needs code generation (enum, namespaces with
+//     values, parameter properties) is compiled by esbuild instead (#73).
+//     Node can't load JSX at all, so .tsx/.jsx files are always compiled (#91).
 // Uses module.registerHooks (synchronous, in-thread; Node >= 23.5), so the
 // install can run with spawnSync inside resolve.
 
 // Not a named import: Bun, which loads this file for its npm:/jsr: helpers,
 // has no registerHooks, and a missing named export fails at link time.
 import module from "node:module";
-import { spawnSync } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
@@ -44,12 +48,19 @@ export function aliasFor(name, range) {
   return `cdpk-${digest}`;
 }
 
-// Each installer keeps its own cache, so npm's and Bun's lockfiles never
-// meet; both read the @jsr registry from .npmrc.
+/**
+ * Each installer keeps its own cache, so npm's and Bun's lockfiles never
+ * meet. Both sit beside the runtime cache (cacheRoot in src/package.ts);
+ * CDPK_CACHE_DIR overrides npm's.
+ */
+export function cacheDirPath(installer, env = process.env, home = homedir()) {
+  if (installer === "npm" && env.CDPK_CACHE_DIR) return env.CDPK_CACHE_DIR;
+  return join(env.XDG_CACHE_HOME || join(home, ".cache"), "cdp-kernel", installer);
+}
+
+// Both caches read the @jsr registry from .npmrc.
 function cacheDir(installer) {
-  const dir = installer === "npm"
-    ? process.env.CDPK_CACHE_DIR ?? join(homedir(), ".cache", "cdp-kernel", "npm")
-    : join(homedir(), ".cache", "cdp-kernel", installer);
+  const dir = cacheDirPath(installer);
   if (!existsSync(join(dir, "package.json"))) {
     mkdirSync(dir, { recursive: true });
     writeFileSync(join(dir, "package.json"), '{ "private": true }\n');
@@ -94,32 +105,55 @@ export function findLockFile(dir) {
 
 /** `{ version: 1, packages: { "name@range": "exact version" } }`; `{}` counts as empty. */
 export function readLock(path) {
-  const parsed = JSON.parse(readFileSync(path, "utf8") || "{}");
+  let parsed;
+  try {
+    parsed = JSON.parse(readFileSync(path, "utf8") || "{}");
+  } catch (err) {
+    throw new Error(`cannot read ${path}: ${err instanceof Error ? err.message : err}`);
+  }
   const packages = parsed && typeof parsed.packages === "object" && parsed.packages !== null ? parsed.packages : {};
   return { version: 1, packages };
 }
 
-/** Sorted, so a lockfile's diff shows only what changed. */
+/**
+ * Sorted, so a lockfile's diff shows only what changed. Written to a
+ * temporary file and renamed into place, so a crash mid-write can't leave
+ * a truncated lockfile (#144).
+ */
 export function writeLock(path, lock) {
   const packages = Object.fromEntries(Object.entries(lock.packages).sort(([a], [b]) => a.localeCompare(b)));
-  writeFileSync(path, JSON.stringify({ version: 1, packages }, null, 2) + "\n");
+  const tmp = `${path}.${process.pid}.tmp`;
+  writeFileSync(tmp, JSON.stringify({ version: 1, packages }, null, 2) + "\n");
+  renameSync(tmp, path);
 }
 
 // --- Serialized installs (#14) -------------------------------------------------
 // Kernels share one cache per installer, and npm/bun both rewrite its
 // package.json and node_modules, so installs take a lock: a directory, which
-// mkdir creates atomically. A lock left by a process that died is taken over.
+// mkdir creates atomically. A lock left by a process that died is taken over;
+// one whose holder is alive is waited on.
 
 const LOCK_DIR = ".install-lock";
-const STALE_MS = 10 * 60 * 1000;
+/** How long a waiter waits for a live holder before giving up. */
+const WAIT_MS = 10 * 60 * 1000;
+/** A lock this old is taken over even if its pid is alive: the pid was reused. */
+const ABANDONED_MS = 60 * 60 * 1000;
 
 function sleep(ms) {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
 
+function holderPid(lockDir) {
+  try {
+    return Number(readFileSync(join(lockDir, "pid"), "utf8"));
+  } catch {
+    return NaN;
+  }
+}
+
 function holderAlive(lockDir) {
   try {
-    const pid = Number(readFileSync(join(lockDir, "pid"), "utf8"));
+    const pid = holderPid(lockDir);
     if (!Number.isInteger(pid) || pid <= 0) return false;
     process.kill(pid, 0);
     return true;
@@ -129,15 +163,13 @@ function holderAlive(lockDir) {
   }
 }
 
-/** Runs `fn` holding the cache's install lock. Synchronous, like the resolve hook. */
-export function withInstallLock(dir, fn, { timeoutMs = STALE_MS } = {}) {
-  const lockDir = join(dir, LOCK_DIR);
-  const deadline = Date.now() + timeoutMs;
+/** Takes the install lock at `lockDir` if it's free: false means wait and try again. */
+function tryLock(lockDir, deadline) {
   for (;;) {
     try {
       mkdirSync(lockDir);
       writeFileSync(join(lockDir, "pid"), String(process.pid));
-      break;
+      return true;
     } catch (err) {
       if (err?.code !== "EEXIST") throw err;
       let age = 0;
@@ -146,19 +178,46 @@ export function withInstallLock(dir, fn, { timeoutMs = STALE_MS } = {}) {
       } catch {
         continue; // released meanwhile
       }
-      // A holder that died, or one stuck for longer than any install takes.
-      if ((age > 1000 && !holderAlive(lockDir)) || age > STALE_MS) {
+      // A holder that died. The grace period covers the gap between mkdir
+      // and writing the pid. A live holder is waited on rather than stolen
+      // from, which would run a second install alongside its own (#144),
+      // unless the lock is so old its pid must have been reused.
+      if ((age > 1000 && !holderAlive(lockDir)) || age > ABANDONED_MS) {
         rmSync(lockDir, { recursive: true, force: true });
         continue;
       }
       if (Date.now() > deadline) throw new Error(`timed out waiting for the install lock ${lockDir}`);
-      sleep(100);
+      return false;
     }
   }
+}
+
+/** Removes the lock only if still ours: a waiter that took it over owns it now. */
+function releaseLock(lockDir) {
+  if (holderPid(lockDir) === process.pid) rmSync(lockDir, { recursive: true, force: true });
+}
+
+/** Runs `fn` holding the cache's install lock. Synchronous, like the resolve hook. */
+export function withInstallLock(dir, fn, { timeoutMs = WAIT_MS } = {}) {
+  const lockDir = join(dir, LOCK_DIR);
+  const deadline = Date.now() + timeoutMs;
+  while (!tryLock(lockDir, deadline)) sleep(100);
   try {
     return fn();
   } finally {
-    rmSync(lockDir, { recursive: true, force: true });
+    releaseLock(lockDir);
+  }
+}
+
+/** withInstallLock for a caller that mustn't block its event loop while it waits. */
+export async function withInstallLockAsync(dir, fn, { timeoutMs = WAIT_MS } = {}) {
+  const lockDir = join(dir, LOCK_DIR);
+  const deadline = Date.now() + timeoutMs;
+  while (!tryLock(lockDir, deadline)) await new Promise((resolve) => setTimeout(resolve, 100));
+  try {
+    return await fn();
+  } finally {
+    releaseLock(lockDir);
   }
 }
 
@@ -170,8 +229,62 @@ function installedVersion(dir, alias) {
   }
 }
 
+/**
+ * How to spawn an install command on `platform`. On Windows npm is npm.cmd,
+ * which Node runs only through a shell (#51). cmd.exe reads ^ < > | & as
+ * operators, and semver ranges use them, so each argument is double-quoted,
+ * where cmd.exe leaves them alone. A " or % would still be read, so they're
+ * refused; no package name or valid range contains either.
+ */
+export function spawnCommand(cmd, args, platform = process.platform) {
+  if (platform !== "win32" || cmd !== "npm") return { cmd, args, shell: false };
+  const bad = args.find((arg) => /["%\r\n]/.test(arg));
+  if (bad !== undefined) throw new TypeError(`can't pass ${JSON.stringify(bad)} to npm through cmd.exe`);
+  return { cmd, args: args.map((arg) => `"${arg}"`), shell: true };
+}
+
 function spawnInstall(cmd, args, cwd, env = {}) {
-  return spawnSync(cmd, args, { cwd, encoding: "utf8", env: { ...process.env, ...env } });
+  const spawned = spawnCommand(cmd, args);
+  return spawnSync(spawned.cmd, spawned.args, { cwd, encoding: "utf8", env: { ...process.env, ...env }, shell: spawned.shell });
+}
+
+/** spawnInstall without blocking: the kernel host keeps answering while it runs (#149). */
+function spawnInstallAsync(cmd, args, cwd, env = {}) {
+  const spawned = spawnCommand(cmd, args);
+  return new Promise((resolve) => {
+    let stderr = "";
+    const child = spawn(spawned.cmd, spawned.args, {
+      cwd,
+      env: { ...process.env, ...env },
+      shell: spawned.shell,
+      stdio: ["ignore", "ignore", "pipe"],
+    });
+    child.stderr.setEncoding("utf8").on("data", (chunk) => (stderr += chunk));
+    child.on("error", (error) => resolve({ status: null, stderr, error }));
+    child.on("close", (status) => resolve({ status, stderr }));
+  });
+}
+
+/** The version to install for `name@range`: the locked one, if the lockfile has it. */
+function lockedRequest(name, range, lockFile) {
+  const lock = lockFile ? readLock(lockFile) : null;
+  const locked = lock?.packages[`${name}@${range}`];
+  const version = typeof locked === "string" ? locked : range;
+  return { lock, locked: typeof locked === "string", version, alias: aliasFor(name, version) };
+}
+
+/** Records an unlocked request's installed version in the lockfile. */
+function recordLock(dir, name, range, lockFile, { lock, locked, alias }) {
+  if (!lock || locked) return;
+  const resolved = installedVersion(dir, alias);
+  if (resolved !== null) {
+    lock.packages[`${name}@${range}`] = resolved;
+    writeLock(lockFile, lock);
+  }
+}
+
+function checkInstall(result, installer, name, version) {
+  if (result.status !== 0) throw new Error(`${installer} install ${name}@${version} failed:\n${result.stderr || result.error}`);
 }
 
 /**
@@ -182,32 +295,135 @@ function spawnInstall(cmd, args, cwd, env = {}) {
  */
 export function ensureInstalled({ name, range }, installer = "npm", { lockFile = null, run = spawnInstall } = {}) {
   const dir = cacheDir(installer);
-  const key = `${name}@${range}`;
   return withInstallLock(dir, () => {
-    const lock = lockFile ? readLock(lockFile) : null;
-    const locked = lock?.packages[key];
-    const version = typeof locked === "string" ? locked : range;
-    const alias = aliasFor(name, version);
-    if (installedVersion(dir, alias) === null) {
-      const { cmd, args, env } = installCommand(installer, alias, name, version);
-      const result = run(cmd, args, dir, env);
-      if (result.status !== 0) {
-        throw new Error(`${installer} install ${name}@${version} failed:\n${result.stderr || result.error}`);
-      }
+    const request = lockedRequest(name, range, lockFile);
+    if (installedVersion(dir, request.alias) === null) {
+      const { cmd, args, env } = installCommand(installer, request.alias, name, request.version);
+      checkInstall(run(cmd, args, dir, env), installer, name, request.version);
     }
-    if (lock && typeof locked !== "string") {
-      const resolved = installedVersion(dir, alias);
-      if (resolved !== null) {
-        lock.packages[key] = resolved;
-        writeLock(lockFile, lock);
-      }
+    recordLock(dir, name, range, lockFile, request);
+    return { dir, alias: request.alias };
+  });
+}
+
+/** ensureInstalled for the kernel host (workerd's bundler), which mustn't block (#149). */
+export function ensureInstalledAsync({ name, range }, installer = "npm", { lockFile = null, run = spawnInstallAsync } = {}) {
+  const dir = cacheDir(installer);
+  return withInstallLockAsync(dir, async () => {
+    const request = lockedRequest(name, range, lockFile);
+    if (installedVersion(dir, request.alias) === null) {
+      const { cmd, args, env } = installCommand(installer, request.alias, name, request.version);
+      checkInstall(await run(cmd, args, dir, env), installer, name, request.version);
     }
-    return { dir, alias };
+    recordLock(dir, name, range, lockFile, request);
+    return { dir, alias: request.alias };
   });
 }
 
+/**
+ * Where `name@range` is installed in the installer's cache, or null if it
+ * isn't: nothing is installed. For type information (#273), which must not
+ * start downloads.
+ */
+export function installedPackageDir({ name, range }, installer = "npm", { lockFile = null } = {}) {
+  const dir = cacheDirPath(installer);
+  const { alias } = lockedRequest(name, range, lockFile);
+  return installedVersion(dir, alias) === null ? null : { dir, alias };
+}
+
+// --- Imported TypeScript (#73) --------------------------------------------------
+// Node strips types in place, so line and column numbers stay those of the
+// file, but it rejects TypeScript that needs code generation. Such a file
+// is compiled with esbuild, as cells are, with an inline source map. The
+// host passes its esbuild binary in CDPK_ESBUILD_BINARY: the runtime files
+// may have been copied away from any node_modules (src/package.ts), and the
+// esbuild binary, unlike its JavaScript API, works from anywhere.
+
+// Both match a file: URL's path, so a query or fragment (`./x.ts?v=2`, the
+// cache-busting idiom for a module being edited) doesn't hide the extension (#145).
+const TS_FILE = /\.[cm]?ts$/;
+// Node has no format for these extensions, so its loader is never asked.
+const JSX_FILE = /\.([jt]sx)$/;
+const TS_FORMATS = { "module-typescript": "module", "commonjs-typescript": "commonjs" };
+
+function isUnsupportedSyntax(err) {
+  return err?.code === "ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX";
+}
+
+/** Node's strip-only transform, without its once-per-process ExperimentalWarning. */
+function stripTypes(source) {
+  const emitWarning = process.emitWarning;
+  process.emitWarning = function (warning, ...rest) {
+    if (String(warning?.message ?? warning).includes("stripTypeScriptTypes")) return;
+    return emitWarning.call(this, warning, ...rest);
+  };
+  try {
+    return module.stripTypeScriptTypes(source, { mode: "strip" });
+  } finally {
+    process.emitWarning = emitWarning;
+  }
+}
+
+/**
+ * Compiles one TypeScript (or, with loader "tsx"/"jsx", JSX) file with the
+ * esbuild binary at `binary`, as cells are compiled. JSX takes esbuild's
+ * default, classic React.createElement, which is what cells use on Node
+ * (no project config there); a file's own @jsx/@jsxImportSource pragma wins.
+ */
+export function compileTypeScript(source, filename, binary, run = spawnSync, loader = "ts") {
+  const args = [
+    `--loader=${loader}`,
+    "--target=esnext",
+    "--sourcemap=inline",
+    `--sourcefile=${filename}`,
+    '--tsconfig-raw={"compilerOptions":{"verbatimModuleSyntax":true}}',
+    "--log-level=error",
+    "--color=false",
+  ];
+  const result = run(binary, args, { input: source, encoding: "utf8", maxBuffer: 1 << 30 });
+  if (result.status !== 0) {
+    throw new SyntaxError(`${filename}: ${(result.stderr || String(result.error)).trim()}`);
+  }
+  return result.stdout;
+}
+
+/** The load hook: Node's result unless it would reject the file's TypeScript. */
+export function loadTypeScript(url, context, nextLoad, binary) {
+  const path = url.startsWith("file:") ? new URL(url).pathname : "";
+  const jsx = JSX_FILE.exec(path);
+  if (binary && jsx) {
+    // Notebooks import ES modules, so it's compiled as one.
+    const filename = fileURLToPath(url);
+    const source = compileTypeScript(readFileSync(filename, "utf8"), filename, binary, spawnSync, jsx[1]);
+    return { format: "module", source, shortCircuit: true };
+  }
+  if (!binary || !TS_FILE.test(path)) return nextLoad(url, context);
+  let result;
+  try {
+    result = nextLoad(url, context);
+  } catch (err) {
+    // A .ts file outside a package.json "type" is stripped while Node
+    // detects its format, so the rejection surfaces here. Notebooks import
+    // ES modules, so it's compiled as one.
+    if (!isUnsupportedSyntax(err)) throw err;
+    const filename = fileURLToPath(url);
+    return { format: "module", source: compileTypeScript(readFileSync(filename, "utf8"), filename, binary), shortCircuit: true };
+  }
+  const format = TS_FORMATS[result.format];
+  if (format === undefined || typeof module.stripTypeScriptTypes !== "function") return result;
+  const source = typeof result.source === "string" ? result.source : new TextDecoder().decode(result.source);
+  try {
+    return { ...result, format, source: stripTypes(source) };
+  } catch (err) {
+    if (!isUnsupportedSyntax(err)) throw err;
+    return { ...result, format, source: compileTypeScript(source, fileURLToPath(url), binary) };
+  }
+}
+
 export function installNodeResolveHooks({ shimUrl, replUrl }) {
+  const esbuild = process.env.CDPK_ESBUILD_BINARY;
   module.registerHooks({
+    load: (url, context, nextLoad) => loadTypeScript(url, context, nextLoad, esbuild),
     resolve(specifier, context, nextResolve) {
       let pkg = null;
       if (specifier.startsWith("npm:")) pkg = parsePackageSpecifier(specifier.slice(4));

package/runtime/workerd-worker.mjs

@@ -51,6 +51,44 @@ Object.assign(console, {
   error: stream("stderr"),
 });
 
+// An error thrown later by a cell's timer or promise goes to stderr, as on
+// the other targets (runtime/bootstrap.mjs). Workers has no global "error"
+// event, so timer callbacks report their own (#148), and so do the
+// promises they return: celld doesn't dispatch "unhandledrejection".
+function reportUncaught(kind, err) {
+  emit({ kind: "stream", name: "stderr", text: `${kind}: ${err instanceof Error ? (err.stack ?? err.message) : inspect(err)}\n` });
+}
+// A rejection both runtimes' events report is shown once.
+const reported = new WeakSet();
+function reportRejection(reason) {
+  if (typeof reason === "object" && reason !== null) {
+    if (reported.has(reason)) return;
+    reported.add(reason);
+  }
+  reportUncaught("Uncaught (in promise)", reason);
+}
+globalThis.addEventListener("unhandledrejection", (event) => {
+  event.preventDefault();
+  reportRejection(event.reason);
+});
+globalThis.process?.on?.("unhandledRejection", reportRejection);
+const reporting = (callback) =>
+  typeof callback !== "function" ? callback : function (...args) {
+    let result;
+    try {
+      result = callback.apply(this, args);
+    } catch (err) {
+      reportUncaught("Uncaught", err);
+      return;
+    }
+    if (typeof result?.then === "function") result.then(undefined, reportRejection);
+    return result;
+  };
+for (const name of ["setTimeout", "setInterval", "queueMicrotask"]) {
+  const original = globalThis[name];
+  globalThis[name] = { [name]: (callback, ...rest) => original(reporting(callback), ...rest) }[name];
+}
+
 const jupyter = createJupyter({
   inspect: (value) => inspect(value, { colors: true, depth: 4 }),
   base64,
@@ -131,13 +169,23 @@ function install(env) {
 
 let evaluate = null;
 
+// Modules whose load rejected. Importing one again gives its error again,
+// and the next eval reply names it, so the host bundles it afresh for the
+// next import (#147), which replaces it here.
+const rejected = new Set();
+const unreported = new Set();
+
 /** Evaluates the bundles a cell brings, once each. */
 async function loadModules(bundles = {}) {
   for (const [specifier, script] of Object.entries(bundles)) {
-    if (modules.has(specifier)) continue;
+    if (modules.has(specifier) && !rejected.has(specifier)) continue;
+    rejected.delete(specifier);
     const loading = Promise.resolve().then(() => evaluate(script));
-    // A module that failed to load is retried by the next import of it.
-    loading.catch(() => modules.delete(specifier));
+    loading.catch(() => {
+      if (modules.get(specifier) !== loading) return;
+      rejected.add(specifier);
+      unreported.add(specifier);
+    });
     modules.set(specifier, loading);
   }
 }
@@ -151,8 +199,31 @@ async function answer(req) {
       worker: globalThis.worker !== undefined,
     };
   }
-  if (req.op === "eval") await loadModules(req.modules);
-  return await handle(req);
+  if (req.op !== "eval") return await handle(req);
+  await loadModules(req.modules);
+  const reply = await handle(req);
+  // After the cell, so a load that failed in it is counted.
+  const failedModules = [...unreported];
+  unreported.clear();
+  return { ...reply, failedModules };
+}
+
+const encoder = new TextEncoder();
+
+/**
+ * Compares in constant time: under celld and Miniflare the driver listens on
+ * loopback TCP, where any local user can try tokens. The length (fixed by
+ * the host) is not secret.
+ */
+function tokenMatches(token, expected) {
+  if (typeof token !== "string" || typeof expected !== "string") return false;
+  const a = encoder.encode(token);
+  const b = encoder.encode(expected);
+  if (a.byteLength !== b.byteLength) return false;
+  if (typeof crypto.subtle.timingSafeEqual === "function") return crypto.subtle.timingSafeEqual(a, b);
+  let diff = 0;
+  for (let i = 0; i < a.byteLength; i++) diff |= a[i] ^ b[i];
+  return diff === 0;
 }
 
 export default {
@@ -160,7 +231,7 @@ export default {
     // A header, or the query string for a WebSocket client that can't set
     // headers (celld's session uses the host's built-in WebSocket).
     const token = request.headers.get("x-cdpk-token") ?? new URL(request.url).searchParams.get("token");
-    if (token !== env.TOKEN) return new Response("forbidden", { status: 403 });
+    if (!tokenMatches(token, env.TOKEN)) return new Response("forbidden", { status: 403 });
     if (request.headers.get("upgrade") !== "websocket") return new Response("expected a WebSocket", { status: 426 });
     evaluate ??= install(env);