@tlockney/cdp-kernel 0.1.1 → 0.3.0

package/runtime/display.mjs

@@ -1,3 +1,4 @@
+// @ts-self-types="./display.d.mts"
 // The Jupyter display API that Deno exposes as `Deno.jupyter`, made
 // runtime-neutral. Runs inside the target runtime (Node or Deno).
 //
@@ -10,7 +11,11 @@
 //   Deno.readFileSync               -> host.readFile
 //   op_jupyter_broadcast            -> host.send
 // WebGPU texture/buffer display depends on Deno's wgpu bindings and is
-// omitted.
+// omitted. Comms and widgets (jupyter.comm, jupyter.widget) are additions
+// Deno doesn't have; see comm.mjs and widget.mjs.
+
+import { createComms } from "./comm.mjs";
+import { createWidgetFactory } from "./widget.mjs";
 
 export const $display = Symbol.for("Jupyter.display");
 
@@ -219,6 +224,8 @@ export function createJupyter(host) {
     return answer === "" ? defaultValue : answer;
   }
 
+  const comms = createComms((msgType, content, metadata, buffers) => host.send(msgType, content, metadata, buffers));
+
   return {
     namespace: {
       broadcast,
@@ -229,9 +236,12 @@ export function createJupyter(host) {
       svg: taggedTemplate("image/svg+xml"),
       image,
       $display,
+      comm: comms.api,
+      widget: createWidgetFactory(comms.api, $display),
     },
     broadcastResult,
     confirm,
     prompt,
+    commDispatch: comms.dispatch,
   };
 }

package/runtime/evaluator.d.mts

@@ -0,0 +1,18 @@
+/** The parts of globalThis.__cdpk the evaluator calls. */
+export interface EvaluatorShim {
+  broadcastResult(executionCount: number, value: unknown): void | Promise<void>;
+  describeError(value: unknown): string;
+  commDispatch(msgType: string, content: object, metadata: object, buffers64: string[]): Promise<unknown>;
+  barrier(): number;
+}
+
+/** A request from the host: an `op` plus its parameters. */
+export interface EvaluatorRequest {
+  op: string;
+  [key: string]: unknown;
+}
+
+export type Handler = (request: EvaluatorRequest) => Promise<{ [key: string]: unknown }>;
+
+export function propertyNames(value: unknown): string[];
+export function createHandler(shim: EvaluatorShim, evaluate?: (code: string) => unknown): Handler;

package/runtime/evaluator.mjs

@@ -0,0 +1,83 @@
+// The in-process evaluator behind the targets that don't use CDP: Bun (#7),
+// over a pipe (bun-channel.mjs), and workerd (#8), inside a request
+// (workerd-worker.mjs). Cells arrive as global code from
+// src/transform.ts's transformCellForEval, whose value is a promise of the
+// boxed result. Requests and replies mirror the CDP calls the kernel makes
+// of the other targets.
+//
+// No imports: this file must load in every target.
+
+/** Own and inherited property names, as Runtime.getProperties lists them. */
+export function propertyNames(value) {
+  if (value === null || value === undefined) return [];
+  const names = new Set();
+  for (let o = Object(value); o !== null; o = Object.getPrototypeOf(o)) {
+    for (const name of Object.getOwnPropertyNames(o)) names.add(name);
+  }
+  return [...names];
+}
+
+/**
+ * Answers the host's requests. `shim` is globalThis.__cdpk; `evaluate` runs
+ * global code. Values the host refers to later (a result to display, an
+ * error to describe) stay here under a handle, the stand-in for a CDP
+ * objectId.
+ */
+export function createHandler(shim, evaluate = (code) => (0, eval)(code)) {
+  const handles = new Map();
+  let nextHandle = 0;
+  const remote = (value) => {
+    const objectId = `h${++nextHandle}`;
+    handles.set(objectId, value);
+    return { type: value === null ? "object" : typeof value, objectId };
+  };
+  const take = (objectId) => {
+    const value = handles.get(objectId);
+    handles.delete(objectId);
+    return value;
+  };
+
+  return async function handle(request) {
+    switch (request.op) {
+      case "ping":
+        return {};
+      case "eval": {
+        // The host uses a result's handle before its next cell.
+        handles.clear();
+        try {
+          const box = await evaluate(request.code);
+          return { result: box === undefined ? { type: "undefined" } : remote(box[0]) };
+        } catch (err) {
+          return {
+            exceptionDetails: { text: "Uncaught", lineNumber: 0, columnNumber: 0, exception: remote(err) },
+          };
+        }
+      }
+      case "broadcast":
+        await shim.broadcastResult(request.count, take(request.handle));
+        return {};
+      case "describe":
+        return { value: shim.describeError(take(request.handle)) };
+      case "comm":
+        try {
+          const handled = await shim.commDispatch(request.msgType, request.content, request.metadata, request.buffers);
+          return { handled: handled === true };
+        } catch (err) {
+          return { error: JSON.parse(shim.describeError(err)) };
+        }
+      case "props": {
+        let value;
+        try {
+          value = evaluate(`(${request.expr})`);
+        } catch {
+          return { names: [] };
+        }
+        return { names: propertyNames(value) };
+      }
+      case "barrier":
+        return { value: shim.barrier() };
+      default:
+        throw new Error(`unknown request ${JSON.stringify(request.op)}`);
+    }
+  };
+}

package/runtime/node-hooks.d.mts

@@ -7,4 +7,15 @@ export interface PackageSpecifier {
 export function parsePackageSpecifier(spec: string): PackageSpecifier;
 export function jsrToNpm(spec: string): PackageSpecifier;
 export function aliasFor(name: string, range: string): string;
+export type Installer = "npm" | "bun";
+export function installCommand(
+  installer: Installer,
+  alias: string,
+  name: string,
+  range: string,
+): { cmd: string; args: string[] };
+export function ensureInstalled(
+  pkg: { name: string; range: string },
+  installer?: Installer,
+): { dir: string; alias: string };
 export function installNodeResolveHooks(urls: { shimUrl: string; replUrl: string }): void;

package/runtime/node-hooks.mjs

@@ -11,7 +11,9 @@
 // Uses module.registerHooks (synchronous, in-thread; Node >= 23.5), so the
 // install can run with spawnSync inside resolve.
 
-import { registerHooks } from "node:module";
+// Not a named import: Bun, which loads this file for its npm:/jsr: helpers,
+// has no registerHooks, and a missing named export fails at link time.
+import module from "node:module";
 import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
@@ -41,8 +43,12 @@ export function aliasFor(name, range) {
   return `cdpk-${digest}`;
 }
 
-function cacheDir() {
-  const dir = process.env.CDPK_CACHE_DIR ?? join(homedir(), ".cache", "cdp-kernel", "npm");
+// Each installer keeps its own cache, so npm's and Bun's lockfiles never
+// meet; both read the @jsr registry from .npmrc.
+function cacheDir(installer) {
+  const dir = installer === "npm"
+    ? process.env.CDPK_CACHE_DIR ?? join(homedir(), ".cache", "cdp-kernel", "npm")
+    : join(homedir(), ".cache", "cdp-kernel", installer);
   if (!existsSync(join(dir, "package.json"))) {
     mkdirSync(dir, { recursive: true });
     writeFileSync(join(dir, "package.json"), '{ "private": true }\n');
@@ -51,24 +57,30 @@ function cacheDir() {
   return dir;
 }
 
-function ensureInstalled({ name, range }) {
-  const dir = cacheDir();
+/** The command that installs name@range under `alias` with `installer`. */
+export function installCommand(installer, alias, name, range) {
+  const spec = `${alias}@npm:${name}@${range}`;
+  // The Bun target runs this inside Bun, so it uses its own binary.
+  if (installer === "bun") return { cmd: process.execPath, args: ["add", "--silent", spec] };
+  return { cmd: "npm", args: ["install", "--no-audit", "--no-fund", "--loglevel=error", spec] };
+}
+
+/** Installs a package into the installer's cache unless it's there already. */
+export function ensureInstalled({ name, range }, installer = "npm") {
+  const dir = cacheDir(installer);
   const alias = aliasFor(name, range);
   if (!existsSync(join(dir, "node_modules", alias, "package.json"))) {
-    const result = spawnSync(
-      "npm",
-      ["install", "--no-audit", "--no-fund", "--loglevel=error", `${alias}@npm:${name}@${range}`],
-      { cwd: dir, encoding: "utf8" },
-    );
+    const { cmd, args } = installCommand(installer, alias, name, range);
+    const result = spawnSync(cmd, args, { cwd: dir, encoding: "utf8" });
     if (result.status !== 0) {
-      throw new Error(`npm install ${name}@${range} failed:\n${result.stderr || result.error}`);
+      throw new Error(`${installer} install ${name}@${range} failed:\n${result.stderr || result.error}`);
     }
   }
   return { dir, alias };
 }
 
 export function installNodeResolveHooks({ shimUrl, replUrl }) {
-  registerHooks({
+  module.registerHooks({
     resolve(specifier, context, nextResolve) {
       let pkg = null;
       if (specifier.startsWith("npm:")) pkg = parsePackageSpecifier(specifier.slice(4));

package/runtime/node-modules-resolve.d.mts

@@ -0,0 +1,8 @@
+/** True for "pkg", "@scope/pkg", "pkg/sub"; false for paths, URLs and aliases like "@/x". */
+export function isBareSpecifier(specifier: string): boolean;
+
+/**
+ * An absolute file URL for a bare package import, resolved from the
+ * node_modules at or above `fromDir`, or null when none has the package.
+ */
+export function resolveFromNodeModules(specifier: string, fromDir: string): string | null;

package/runtime/node-modules-resolve.mjs

@@ -0,0 +1,106 @@
+// @ts-self-types="./node-modules-resolve.d.mts"
+// Resolves a bare package import against a notebook directory's
+// node_modules, for Deno targets in a Deno project (#13).
+//
+// Deno resolves package.json dependencies relative to the *importing*
+// module. Cell imports are issued from runtime/bootstrap.mjs, outside the
+// project, and code evaluated over the inspector resolves against the
+// notebook directory treated as a file (one level too high), so neither
+// finds the project's node_modules. Resolving the package to an absolute
+// file URL here sidesteps both; the package's own imports then resolve
+// normally from inside node_modules.
+//
+// Implements the parts of Node's package resolution notebooks need:
+// "exports" (string, subpath map, "*" patterns, nested conditions taken in
+// the package's key order), then "module"/"main"/index.js. Returns null when
+// no node_modules has the package, so the caller can fall back to Deno's
+// own resolution (import map, npm:, jsr:, builtins).
+
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+// The conditions Deno matches for npm packages, plus the common "module".
+const CONDITIONS = new Set(["deno", "node", "import", "module", "default"]);
+
+/** True for "pkg", "@scope/pkg", "pkg/sub"; false for paths, URLs, aliases. */
+export function isBareSpecifier(specifier) {
+  if (/^[./]/.test(specifier) || specifier.startsWith("#")) return false;
+  if (/^[a-z][a-z0-9+.-]*:/i.test(specifier)) return false; // npm:, node:, file:, https:
+  if (specifier.startsWith("@")) return /^@[^/]+\/[^/]+/.test(specifier) && !specifier.startsWith("@/");
+  return true;
+}
+
+function splitPackage(specifier) {
+  const parts = specifier.split("/");
+  const nameLength = specifier.startsWith("@") ? 2 : 1;
+  return { name: parts.slice(0, nameLength).join("/"), subpath: parts.slice(nameLength).join("/") };
+}
+
+function findPackageDir(name, fromDir) {
+  for (let dir = fromDir; ; dir = dirname(dir)) {
+    const candidate = join(dir, "node_modules", name);
+    if (existsSync(join(candidate, "package.json"))) return candidate;
+    if (dirname(dir) === dir) return null;
+  }
+}
+
+/** Picks a target from an exports value: a string, or nested conditions. */
+function conditionalTarget(value) {
+  if (typeof value === "string") return value;
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      const target = conditionalTarget(item);
+      if (target !== null) return target;
+    }
+    return null;
+  }
+  if (value !== null && typeof value === "object") {
+    for (const [key, nested] of Object.entries(value)) {
+      if (!CONDITIONS.has(key)) continue;
+      const target = conditionalTarget(nested);
+      if (target !== null) return target;
+    }
+  }
+  return null;
+}
+
+function exportsTarget(exports, subpath) {
+  const key = subpath === "" ? "." : `./${subpath}`;
+  const isSubpathMap = exports !== null && typeof exports === "object" && !Array.isArray(exports) &&
+    Object.keys(exports).some((k) => k.startsWith("."));
+  if (!isSubpathMap) return key === "." ? conditionalTarget(exports) : null;
+  if (Object.hasOwn(exports, key)) return conditionalTarget(exports[key]);
+  for (const [pattern, value] of Object.entries(exports)) {
+    const star = pattern.indexOf("*");
+    if (star === -1) continue;
+    const prefix = pattern.slice(0, star);
+    const suffix = pattern.slice(star + 1);
+    if (key.startsWith(prefix) && key.endsWith(suffix) && key.length >= prefix.length + suffix.length) {
+      const match = key.slice(prefix.length, key.length - suffix.length);
+      const target = conditionalTarget(value);
+      return target === null ? null : target.replaceAll("*", match);
+    }
+  }
+  return null;
+}
+
+/** An absolute file URL for `specifier`, or null if no node_modules has it. */
+export function resolveFromNodeModules(specifier, fromDir) {
+  if (!isBareSpecifier(specifier)) return null;
+  const { name, subpath } = splitPackage(specifier);
+  const pkgDir = findPackageDir(name, fromDir);
+  if (pkgDir === null) return null;
+  const pkg = JSON.parse(readFileSync(join(pkgDir, "package.json"), "utf8"));
+
+  let target;
+  if (pkg.exports !== undefined) {
+    target = exportsTarget(pkg.exports, subpath);
+    if (target === null) return null; // not exported: not importable
+  } else if (subpath !== "") {
+    target = subpath;
+  } else {
+    target = pkg.module ?? pkg.main ?? "index.js";
+  }
+  return pathToFileURL(join(pkgDir, target)).href;
+}

package/runtime/widget.d.mts

@@ -0,0 +1,40 @@
+import type { CommApi } from "./comm.d.mts";
+import type { $display, MediaBundle } from "./display.d.mts";
+
+export const WIDGET_PROTOCOL_VERSION: "2.1.0";
+
+export type WidgetState = { [key: string]: unknown };
+
+export interface WidgetOptions {
+  /** The anywidget ES module: `export default { render({ model, el }) { ... } }`. */
+  esm: string;
+  /** Optional CSS for the view. */
+  css?: string;
+  /** Initial synced state. Uint8Array/ArrayBuffer values travel as binary buffers. */
+  state?: WidgetState;
+}
+
+/** An anywidget model, synced with its frontend views. Display it to show a view. */
+export interface Widget {
+  /** The model id (the comm id). */
+  readonly id: string;
+  readonly closed: boolean;
+  get(key: string): unknown;
+  /** Sets one key or several, and syncs the change to the frontend. */
+  set(key: string, value: unknown): void;
+  set(changes: WidgetState): void;
+  on(event: `change:${string}`, listener: (value: unknown, previous: unknown) => void | Promise<void>): void;
+  on(event: "change", listener: (changes: WidgetState) => void | Promise<void>): void;
+  on(event: "msg:custom", listener: (content: unknown, buffers: Uint8Array[]) => void | Promise<void>): void;
+  off(event: string, listener: (...args: never[]) => unknown): void;
+  /** A custom message to the view's `model.on("msg:custom", ...)`. */
+  send(content: unknown, buffers?: Uint8Array[]): void;
+  close(): void;
+  fullState(): WidgetState;
+  /** Displaying a widget shows a view of it (application/vnd.jupyter.widget-view+json). */
+  [$display](): MediaBundle;
+}
+
+export function removeBuffers(state: unknown): { state: unknown; bufferPaths: (string | number)[][]; buffers: Uint8Array[] };
+export function putBuffers<T>(state: T, bufferPaths?: (string | number)[][], buffers?: Uint8Array[]): T;
+export function createWidgetFactory(commApi: CommApi, $display: symbol): (options: WidgetOptions) => Widget;

package/runtime/widget.mjs

@@ -0,0 +1,208 @@
+// @ts-self-types="./widget.d.mts"
+// anywidget-compatible widgets over Jupyter comms (#9): the kernel half of
+// the ipywidgets model protocol (version 2.1.0) for anywidget's AnyModel,
+// whose frontend is the `anywidget` JupyterLab/Notebook/VS Code extension.
+//
+//   const counter = jupyter.widget({ esm, state: { count: 0 } });
+//   counter            // display it
+//   counter.get("count"); counter.set("count", 1);
+//   counter.on("change:count", (value, old) => ...);
+//
+// State is synced as in ipywidgets: `update` messages carry changed keys,
+// with binary values lifted out into message buffers at `buffer_paths`
+// (ipywidgets' _remove_buffers/_put_buffers).
+
+export const WIDGET_PROTOCOL_VERSION = "2.1.0";
+const WIDGET_TARGET = "jupyter.widget";
+const CONTROL_TARGET = "jupyter.widget.control";
+const VIEW_MIME = "application/vnd.jupyter.widget-view+json";
+
+// The anywidget model the frontend instantiates. The module version is a
+// semver range for the frontend's registry; anywidget sends "~major.minor.*"
+// of its own version, and "*" accepts whichever release is installed.
+const ANYWIDGET_MODEL = {
+  _model_name: "AnyModel",
+  _model_module: "anywidget",
+  _model_module_version: "*",
+  _view_name: "AnyView",
+  _view_module: "anywidget",
+  _view_module_version: "*",
+  _view_count: null,
+};
+
+function isBinary(value) {
+  return value instanceof Uint8Array || value instanceof ArrayBuffer || ArrayBuffer.isView(value);
+}
+
+function toBytes(value) {
+  if (value instanceof Uint8Array) return value;
+  if (value instanceof ArrayBuffer) return new Uint8Array(value);
+  return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
+}
+
+/** Copies `state` with binary values replaced by null, collecting them. */
+export function removeBuffers(state) {
+  const bufferPaths = [];
+  const buffers = [];
+  function walk(value, path) {
+    if (isBinary(value)) {
+      bufferPaths.push(path);
+      buffers.push(toBytes(value));
+      return null;
+    }
+    if (Array.isArray(value)) return value.map((item, i) => walk(item, [...path, i]));
+    if (value !== null && typeof value === "object") {
+      return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, walk(v, [...path, k])]));
+    }
+    return value;
+  }
+  return { state: walk(state, []), bufferPaths, buffers };
+}
+
+/** Puts received buffers back into `state` at `bufferPaths` (in place). */
+export function putBuffers(state, bufferPaths = [], buffers = []) {
+  bufferPaths.forEach((path, i) => {
+    let obj = state;
+    for (const key of path.slice(0, -1)) obj = obj[key];
+    obj[path[path.length - 1]] = buffers[i];
+  });
+  return state;
+}
+
+class Widget {
+  #comm;
+  #state;
+  #listeners = new Map(); // event -> Set<listener>
+
+  constructor(comm, state, $display) {
+    this.#comm = comm;
+    this.#state = state;
+    this[$display] = () => ({
+      [VIEW_MIME]: { version_major: 2, version_minor: 1, model_id: this.id },
+      "text/plain": `Widget(${this.id})`,
+    });
+  }
+
+  get id() {
+    return this.#comm.id;
+  }
+
+  get closed() {
+    return this.#comm.closed;
+  }
+
+  get(key) {
+    return this.#state[key];
+  }
+
+  /** set("key", value) or set({ key: value, ... }); syncs to the frontend. */
+  set(keyOrChanges, value) {
+    const changes = typeof keyOrChanges === "string" ? { [keyOrChanges]: value } : keyOrChanges;
+    this.#apply(changes);
+    if (this.closed) return;
+    const { state, bufferPaths, buffers } = removeBuffers(changes);
+    this.#comm.send({ method: "update", state, buffer_paths: bufferPaths }, { buffers });
+  }
+
+  on(event, listener) {
+    if (!this.#listeners.has(event)) this.#listeners.set(event, new Set());
+    this.#listeners.get(event).add(listener);
+  }
+
+  off(event, listener) {
+    this.#listeners.get(event)?.delete(listener);
+  }
+
+  /** A custom message, delivered to the view's model.on("msg:custom"). */
+  send(content, buffers = []) {
+    this.#comm.send({ method: "custom", content }, { buffers });
+  }
+
+  close() {
+    this.#comm.close();
+  }
+
+  /** The full state, as the frontend sees it (for request_state). */
+  fullState() {
+    return { ...ANYWIDGET_MODEL, ...this.#state };
+  }
+
+  async #emit(event, ...args) {
+    for (const listener of this.#listeners.get(event) ?? []) await listener(...args);
+  }
+
+  async #apply(changes) {
+    const previous = {};
+    for (const [key, value] of Object.entries(changes)) {
+      previous[key] = this.#state[key];
+      this.#state[key] = value;
+    }
+    for (const key of Object.keys(changes)) await this.#emit(`change:${key}`, changes[key], previous[key]);
+    await this.#emit("change", changes);
+  }
+
+  async _receive(msg) {
+    const { method } = msg.data;
+    if (method === "update") {
+      const changes = putBuffers(msg.data.state ?? {}, msg.data.buffer_paths, msg.buffers);
+      await this.#apply(changes);
+    } else if (method === "request_state") {
+      const { state, bufferPaths, buffers } = removeBuffers(this.fullState());
+      this.#comm.send({ method: "update", state, buffer_paths: bufferPaths }, { buffers });
+    } else if (method === "custom") {
+      await this.#emit("msg:custom", msg.data.content, msg.buffers);
+    }
+  }
+}
+
+/**
+ * @param {import("./comm.d.mts").CommApi} commApi
+ * @param {symbol} $display
+ */
+export function createWidgetFactory(commApi, $display) {
+  const live = new Map(); // model id -> Widget
+
+  // The frontend opens this comm to restore every widget's state, e.g.
+  // after the page reloads.
+  commApi.registerTarget(CONTROL_TARGET, (control) => {
+    control.onMsg((msg) => {
+      if (msg.data.method !== "request_states") return;
+      const states = {};
+      for (const widget of live.values()) {
+        const full = widget.fullState();
+        states[widget.id] = {
+          model_name: full._model_name,
+          model_module: full._model_module,
+          model_module_version: full._model_module_version,
+          state: full,
+        };
+      }
+      const { state, bufferPaths, buffers } = removeBuffers(states);
+      control.send({ method: "update_states", states: state, buffer_paths: bufferPaths }, { buffers });
+    });
+  });
+
+  return function widget({ esm, css, state = {} } = {}) {
+    if (typeof esm !== "string") throw new TypeError("jupyter.widget needs an `esm` module string");
+    const initial = { ...state };
+    const full = {
+      ...ANYWIDGET_MODEL,
+      _esm: esm,
+      ...(css === undefined ? {} : { _css: css }),
+      _anywidget_id: "cdp-kernel.widget",
+      ...initial,
+    };
+    const { state: wire, bufferPaths, buffers } = removeBuffers(full);
+    const comm = commApi.open(WIDGET_TARGET, { state: wire, buffer_paths: bufferPaths }, {
+      metadata: { version: WIDGET_PROTOCOL_VERSION },
+      buffers,
+    });
+    const instance = new Widget(comm, { ...full }, $display);
+    live.set(comm.id, instance);
+    comm.onMsg((msg) => instance._receive(msg));
+    comm.onClose(() => {
+      live.delete(comm.id);
+    });
+    return instance;
+  };
+}