@tlockney/cdp-kernel 0.11.0 → 0.12.1

package/dist/browser/mod.d.ts

@@ -4,6 +4,33 @@ import { type Channel, type JupyterMessage, type MessageListener } from "./chann
 import { type BrowserTargetOptions } from "./target.ts";
 export type { Channel, JupyterMessage, MessageListener } from "./channels.ts";
 export { BrowserTarget, type BrowserTargetOptions, type HostHandler, type HostRequest, type WorkerLike } from "./target.ts";
+/**
+ * The packages `jupyter.chart` imports, as it writes them: map them in
+ * `modules` to copies the page serves, and charts draw offline. The page
+ * serves ES modules that export what the packages do. Same as
+ * runtime/chart.mjs's VEGA, VEGA_LITE and LINKEDOM.
+ */
+export declare const CHART_MODULES: {
+    readonly vega: "npm:vega@6";
+    readonly vegaLite: "npm:vega-lite@6";
+    readonly linkedom: "npm:linkedom@0.18";
+};
+/**
+ * What `jupyter.sql` loads, as it writes them: map them in `modules` to
+ * copies the page serves, and SQL runs offline. `duckdb` is an ES module
+ * that exports what @duckdb/duckdb-wasm does; the others are that
+ * package's own dist files, served as they are. A page may map only the
+ * `eh` pair (every current browser runs it); a variant left unmapped
+ * loads from jsDelivr. Same as runtime/sql.mjs's DUCKDB_WASM and
+ * DUCKDB_FILES.
+ */
+export declare const SQL_MODULES: {
+    readonly duckdb: "npm:@duckdb/duckdb-wasm@1.29.0";
+    readonly ehWasm: "npm:@duckdb/duckdb-wasm@1.29.0/dist/duckdb-eh.wasm";
+    readonly ehWorker: "npm:@duckdb/duckdb-wasm@1.29.0/dist/duckdb-browser-eh.worker.js";
+    readonly mvpWasm: "npm:@duckdb/duckdb-wasm@1.29.0/dist/duckdb-mvp.wasm";
+    readonly mvpWorker: "npm:@duckdb/duckdb-wasm@1.29.0/dist/duckdb-browser-mvp.worker.js";
+};
 export interface BrowserKernelOptions extends BrowserTargetOptions {
     /** What strips cells' types; sucrase by default, since esbuild's native binary can't run in a page. */
     typeStrip?: TypeStripperName | TypeStripper;

package/dist/browser/mod.js

@@ -15,6 +15,33 @@ import { VERSION } from "../package.js";
 import { PageChannels } from "./channels.js";
 import { BrowserTarget } from "./target.js";
 export { BrowserTarget } from "./target.js";
+/**
+ * The packages `jupyter.chart` imports, as it writes them: map them in
+ * `modules` to copies the page serves, and charts draw offline. The page
+ * serves ES modules that export what the packages do. Same as
+ * runtime/chart.mjs's VEGA, VEGA_LITE and LINKEDOM.
+ */
+export const CHART_MODULES = {
+    vega: "npm:vega@6",
+    vegaLite: "npm:vega-lite@6",
+    linkedom: "npm:linkedom@0.18",
+};
+/**
+ * What `jupyter.sql` loads, as it writes them: map them in `modules` to
+ * copies the page serves, and SQL runs offline. `duckdb` is an ES module
+ * that exports what @duckdb/duckdb-wasm does; the others are that
+ * package's own dist files, served as they are. A page may map only the
+ * `eh` pair (every current browser runs it); a variant left unmapped
+ * loads from jsDelivr. Same as runtime/sql.mjs's DUCKDB_WASM and
+ * DUCKDB_FILES.
+ */
+export const SQL_MODULES = {
+    duckdb: "npm:@duckdb/duckdb-wasm@1.29.0",
+    ehWasm: "npm:@duckdb/duckdb-wasm@1.29.0/dist/duckdb-eh.wasm",
+    ehWorker: "npm:@duckdb/duckdb-wasm@1.29.0/dist/duckdb-browser-eh.worker.js",
+    mvpWasm: "npm:@duckdb/duckdb-wasm@1.29.0/dist/duckdb-mvp.wasm",
+    mvpWorker: "npm:@duckdb/duckdb-wasm@1.29.0/dist/duckdb-browser-mvp.worker.js",
+};
 export function startBrowserKernel(options) {
     const channels = new PageChannels();
     const log = options.log ?? ((text) => console.debug(`[cdp-kernel] ${text}`));

package/dist/browser/target.d.ts

@@ -22,6 +22,16 @@ export interface BrowserTargetOptions {
     base?: string;
     /** Where npm:, jsr: and bare imports load from; esm.sh by default. */
     cdn?: string;
+    /**
+     * Specifiers, as written, mapped to URLs the page serves (resolved
+     * against `base`), which win over `cdn`: a cell's imports and the
+     * libraries `jupyter.chart` and `jupyter.sql` import load from there.
+     * Mapping `CHART_MODULES` makes charts draw offline, and mapping
+     * `SQL_MODULES` does the same for `jupyter.sql`.
+     */
+    modules?: {
+        readonly [specifier: string]: string;
+    };
     /**
      * Modules the Worker imports before the first cell, each time it starts,
      * resolved against `base`. A module's default export, if a function, is

package/dist/browser/target.js

@@ -49,7 +49,7 @@ export class BrowserTarget {
         const base = options.base ?? globalThis.location?.href ?? "file:///";
         try {
             const params = options.params === undefined ? undefined : JSON.stringify(options.params);
-            const { userAgent } = await target.#request("info", { base, cdn: options.cdn, preload: options.preload ?? [], params });
+            const { userAgent } = await target.#request("info", { base, cdn: options.cdn, modules: options.modules, preload: options.preload ?? [], params });
             if (typeof userAgent === "string")
                 target.versions.runtime = browserName(userAgent);
             target.versions.engine = engineName(typeof userAgent === "string" ? userAgent : "");

package/dist/browser/worker.js

@@ -715,6 +715,17 @@ function textTable(fields, numbered) {
   for (const row of cells) row.forEach((c, i) => widths[i] = Math.max(widths[i], c.length));
   return cells.map((row) => `  ${row.map((c, i) => c.padEnd(widths[i])).join(" | ").trimEnd()}`);
 }
+var PAGEABLE = /* @__PURE__ */ Symbol("cdpk.table.pageable");
+function encodeRow(source, fields, row) {
+  return fields.map((f, i) => encodeValue(row[i], f.type, source.columns[i].declared));
+}
+function pageTable(bundle, pager, key2) {
+  const resource = bundle?.[TABLE_MIME];
+  const pending = resource?.[PAGEABLE];
+  if (!pending || !pager || resource.ganymede?.page !== void 0) return bundle;
+  resource.ganymede.page = pager.register(pending.source, pending.fields, key2);
+  return bundle;
+}
 function tableBundle(source, options = {}) {
   const requested = options.limit ?? DEFAULT_LIMIT;
   if (!Number.isInteger(requested) || requested < 0) throw new TypeError("jupyter.table: limit must be a whole number");
@@ -723,7 +734,7 @@ function tableBundle(source, options = {}) {
   const total = source.total;
   const raw = source.read(0, Math.min(limit, total));
   const fields = fieldsOf(source, raw);
-  const encode = (r) => fields.map((f, i) => encodeValue(r[i], f.type, source.columns[i].declared));
+  const encode = (r) => encodeRow(source, fields, r);
   const utf8 = new TextEncoder();
   const encoded = [];
   const data = [];
@@ -772,6 +783,10 @@ function tableBundle(source, options = {}) {
   if (text.length > TEXT_MAX) text = `${text.slice(0, TEXT_MAX - 1)}\u2026`;
   const resource = { schema: { fields }, data, ganymede: { rows: total, shown } };
   if (title) resource.ganymede.title = title;
+  if (shown < total) {
+    Object.defineProperty(resource, PAGEABLE, { value: { source, fields } });
+    if (options.pager) pageTable({ [TABLE_MIME]: resource }, options.pager);
+  }
   return {
     [TABLE_MIME]: resource,
     "text/html": htmlTable(fields, htmlHead, htmlTail, total, title),
@@ -782,6 +797,20 @@ function tableBundle(source, options = {}) {
 // runtime/sql.mjs
 var NODE_API = "npm:@duckdb/node-api@1.5.6-r.2";
 var DUCKDB_WASM = "npm:@duckdb/duckdb-wasm@1.29.0";
+var DUCKDB_FILES = {
+  mvp: { mainModule: `${DUCKDB_WASM}/dist/duckdb-mvp.wasm`, mainWorker: `${DUCKDB_WASM}/dist/duckdb-browser-mvp.worker.js` },
+  eh: { mainModule: `${DUCKDB_WASM}/dist/duckdb-eh.wasm`, mainWorker: `${DUCKDB_WASM}/dist/duckdb-browser-eh.worker.js` }
+};
+function duckdbBundles(jsDelivr, mapped) {
+  const bundles = { ...jsDelivr };
+  if (!mapped) return bundles;
+  for (const [variant, files] of Object.entries(DUCKDB_FILES)) {
+    const mainModule = mapped(files.mainModule);
+    const mainWorker = mapped(files.mainWorker);
+    if (mainModule && mainWorker) bundles[variant] = { mainModule, mainWorker, pthreadWorker: null };
+  }
+  return bundles;
+}
 var $display = /* @__PURE__ */ Symbol.for("Jupyter.display");
 var RAW = /* @__PURE__ */ Symbol("cdpk.sql.raw");
 function jsValue(value, declared) {
@@ -909,9 +938,9 @@ var DUCK_TYPES = {
   object: "JSON",
   duration: "VARCHAR"
 };
-function createSql({ importModule, signal, openEngine }) {
+function createSql({ importModule, mapped, signal, openEngine }) {
   let engine = null;
-  const open = () => engine ??= (openEngine ?? (() => defaultEngine(importModule)))().catch((err) => {
+  const open = () => engine ??= (openEngine ?? (() => defaultEngine(importModule, mapped)))().catch((err) => {
     engine = null;
     throw err;
   });
@@ -1009,9 +1038,9 @@ function inWorker() {
 function hasNativeRuntime() {
   return typeof globalThis.Deno?.version?.deno === "string" || typeof globalThis.Bun?.version === "string" || typeof globalThis.process?.versions?.node === "string" && !inWorker();
 }
-async function defaultEngine(importModule) {
+async function defaultEngine(importModule, mapped) {
   if (hasNativeRuntime()) return nativeEngine(importModule);
-  if (inWorker() && typeof Worker === "function") return wasmEngine(importModule);
+  if (inWorker() && typeof Worker === "function") return wasmEngine(importModule, mapped);
   throw new Error("jupyter.sql needs Node, Deno, Bun or the browser target; workerd and celld have no DuckDB");
 }
 async function nativeEngine(importModule) {
@@ -1039,9 +1068,9 @@ async function nativeEngine(importModule) {
     }
   };
 }
-async function wasmEngine(importModule) {
+async function wasmEngine(importModule, mapped) {
   const duckdb = await importModule(DUCKDB_WASM);
-  const bundle = await duckdb.selectBundle(duckdb.getJsDelivrBundles());
+  const bundle = await duckdb.selectBundle(duckdbBundles(duckdb.getJsDelivrBundles(), mapped));
   const loader = URL.createObjectURL(new Blob([`importScripts(${JSON.stringify(bundle.mainWorker)});`], { type: "text/javascript" }));
   const worker = new Worker(loader);
   const db = new duckdb.AsyncDuckDB(new duckdb.VoidLogger(), worker);
@@ -1357,6 +1386,133 @@ function createChart({ importModule, summarizeSpec, warn = (m) => console.warn(m
   return chart;
 }
 
+// runtime/table-paging.mjs
+var PAGE_TARGET = "ganymede.table.v1";
+var KEEP = 16;
+var MAX_PAGE = 1e3;
+var MAX_SORT = 1e6;
+var CHUNK = 1e4;
+var SORTS = 2;
+var NUMERIC = /* @__PURE__ */ new Set(["integer", "number"]);
+var INTEGER_TEXT = /^-?\d+$/;
+function sortNumber(value) {
+  if (typeof value === "number") return value;
+  if (typeof value === "string" && value.trim() !== "") return Number(value);
+  return Number.NaN;
+}
+function textOf(value) {
+  if (value === null || value === void 0) return "null";
+  if (typeof value === "string") return value;
+  if (typeof value === "number" || typeof value === "boolean") return String(value);
+  return JSON.stringify(value);
+}
+var collator = new Intl.Collator(void 0, { numeric: true });
+function compareEncoded(a, b, type, descending = false) {
+  const numeric = NUMERIC.has(type) || typeof a === "number" && typeof b === "number";
+  const last = (v) => v === null || v === void 0 || numeric && Number.isNaN(sortNumber(v));
+  const aLast = last(a);
+  const bLast = last(b);
+  if (aLast || bLast) return aLast === bLast ? 0 : aLast ? 1 : -1;
+  const sign = descending ? -1 : 1;
+  if (numeric) {
+    if (typeof a === "string" && typeof b === "string" && INTEGER_TEXT.test(a) && INTEGER_TEXT.test(b)) {
+      const x = BigInt(a);
+      const y = BigInt(b);
+      return sign * (x < y ? -1 : x > y ? 1 : 0);
+    }
+    return sign * (sortNumber(a) - sortNumber(b));
+  }
+  if (typeof a === "boolean" && typeof b === "boolean") return sign * (Number(a) - Number(b));
+  return sign * collator.compare(textOf(a), textOf(b));
+}
+function createTablePager(commApi, { keep = KEEP, newId = () => crypto.randomUUID() } = {}) {
+  const tables = /* @__PURE__ */ new Map();
+  const keyed = /* @__PURE__ */ new Map();
+  function register(source, fields, key2) {
+    let id = key2 === void 0 ? void 0 : keyed.get(key2);
+    if (id === void 0 || !tables.has(id)) id = newId();
+    if (key2 !== void 0) keyed.set(key2, id);
+    tables.delete(id);
+    tables.set(id, { source, fields, key: key2, orders: /* @__PURE__ */ new Map() });
+    while (tables.size > keep) {
+      const [oldest, entry] = tables.entries().next().value;
+      tables.delete(oldest);
+      if (entry.key !== void 0 && keyed.get(entry.key) === oldest) keyed.delete(entry.key);
+    }
+    return id;
+  }
+  function encodeRows(entry, raw) {
+    return raw.map((r) => encodeRow(entry.source, entry.fields, r));
+  }
+  function orderFor(entry, column, descending) {
+    const key2 = `${column}\0${descending}`;
+    const kept = entry.orders.get(key2);
+    if (kept) return kept;
+    const { source, fields } = entry;
+    const c = fields.findIndex((f) => f.name === column);
+    if (c < 0) throw new Error(`no column ${JSON.stringify(column)}`);
+    if (source.total > MAX_SORT) throw new Error(`sorting is offered up to ${MAX_SORT.toLocaleString("en-US")} rows`);
+    const type = fields[c].type;
+    const declared = source.columns[c].declared;
+    const values = new Array(source.total);
+    for (let start = 0; start < source.total; start += CHUNK) {
+      const raw = source.read(start, Math.min(start + CHUNK, source.total));
+      raw.forEach((r, i) => values[start + i] = encodeValue(r[c], type, declared));
+    }
+    const index = Array.from({ length: source.total }, (_, i) => i);
+    index.sort((i, j) => compareEncoded(values[i], values[j], type, descending));
+    entry.orders.set(key2, index);
+    while (entry.orders.size > SORTS) entry.orders.delete(entry.orders.keys().next().value);
+    return index;
+  }
+  function page(entry, { offset, limit, sort }) {
+    const { source } = entry;
+    const start = Math.max(0, Math.min(offset, source.total));
+    const end = Math.min(source.total, start + Math.min(limit, MAX_PAGE));
+    if (!sort) return encodeRows(entry, source.read(start, end));
+    const index = orderFor(entry, String(sort.column), sort.descending === true).slice(start, end);
+    const rows = [];
+    for (let i = 0; i < index.length; ) {
+      let j = i + 1;
+      while (j < index.length && index[j] === index[j - 1] + 1) j++;
+      rows.push(...source.read(index[i], index[j - 1] + 1));
+      i = j;
+    }
+    return encodeRows(entry, rows);
+  }
+  const gone = "The kernel no longer has this table: run the cell again to page through it.";
+  commApi.registerTarget(PAGE_TARGET, (comm, msg) => {
+    const id = msg.data?.id;
+    if (typeof id !== "string" || !tables.has(id)) {
+      comm.send({ error: gone });
+      comm.close();
+      return;
+    }
+    comm.onMsg(({ data }) => {
+      const request = data?.request;
+      const entry = tables.get(id);
+      if (!entry) {
+        comm.send({ request, error: gone });
+        comm.close();
+        return;
+      }
+      const offset = data?.offset;
+      const limit = data?.limit ?? MAX_PAGE;
+      if (!Number.isInteger(offset) || offset < 0 || !Number.isInteger(limit) || limit < 1) {
+        comm.send({ request, error: "offset and limit must be whole numbers" });
+        return;
+      }
+      try {
+        const rows = page(entry, { offset, limit, sort: data?.sort ?? void 0 });
+        comm.send({ request, offset, total: entry.source.total, rows });
+      } catch (err) {
+        comm.send({ request, error: err instanceof Error ? err.message : String(err) });
+      }
+    });
+  });
+  return { register, size: () => tables.size };
+}
+
 // runtime/display.mjs
 var $display3 = /* @__PURE__ */ Symbol.for("Jupyter.display");
 var rawToEntity3 = /* @__PURE__ */ new Map([
@@ -1631,10 +1787,11 @@ function createJupyter(host2) {
   function broadcast(msgType, content, { metadata = {}, buffers = [] } = {}) {
     host2.send(msgType, content, metadata, buffers);
   }
+  let pager;
   async function display(obj, options = { raw: false, update: false }) {
     const bundle = options.raw && isMediaBundle(obj) ? obj : await format2(obj);
     broadcast(options.update ? "update_display_data" : "display_data", {
-      data: bundle,
+      data: pageTable(bundle, pager, options.display_id),
       metadata: {},
       transient: options.display_id ? { display_id: options.display_id } : {}
     });
@@ -1644,7 +1801,7 @@ function createJupyter(host2) {
     try {
       broadcast("execute_result", {
         execution_count: executionCount,
-        data: await format2(result2),
+        data: pageTable(await format2(result2), pager),
         metadata: {},
         // nbclient crashes on a null `transient`; Deno learned this too.
         transient: {}
@@ -1703,9 +1860,14 @@ function createJupyter(host2) {
     if (typeof load !== "function") return Promise.reject(new Error(`${what}: no module loader in this runtime`));
     return load(specifier);
   };
-  const sql = createSql({ importModule: cellImport("jupyter.sql"), signal: () => current.signal });
+  const sql = createSql({
+    importModule: cellImport("jupyter.sql"),
+    mapped: (specifier) => globalThis.__cdpk?.mapped?.(specifier) ?? null,
+    signal: () => current.signal
+  });
   const chart = createChart({ importModule: cellImport("jupyter.chart"), summarizeSpec: summarizeChart });
   const comms = createComms((msgType, content, metadata, buffers) => host2.send(msgType, content, metadata, buffers));
+  pager = createTablePager(comms.api);
   let cell = new AbortController();
   let current = cell;
   const handlers = /* @__PURE__ */ new Map();
@@ -2052,7 +2214,8 @@ ${resolved.name} has a getter, which isn't called for inspection: it could have
 
 // runtime/cdn.mjs
 var DEFAULT_CDN = "https://esm.sh/";
-function resolveSpecifier(specifier, base, cdn2 = DEFAULT_CDN) {
+function resolveSpecifier(specifier, base, cdn2 = DEFAULT_CDN, modules2 = void 0) {
+  if (modules2 && Object.hasOwn(modules2, specifier)) return new URL(modules2[specifier], base).href;
   if (/^(?:https?|data|blob):/.test(specifier)) return specifier;
   if (/^\.{0,2}\//.test(specifier)) return new URL(specifier, base).href;
   if (specifier.startsWith("npm:")) return cdn2 + specifier.slice(4);
@@ -2107,6 +2270,7 @@ var jupyter = createJupyter({
 });
 var paramsText;
 var cdn = DEFAULT_CDN;
+var modules;
 var hostCalls = /* @__PURE__ */ new Map();
 var hostListeners = /* @__PURE__ */ new Set();
 var nextHostId = 0;
@@ -2173,13 +2337,17 @@ async function preload(urls, base) {
   }
 }
 var handle = null;
-function install({ base, cdn: cdnBase, params }) {
+function install({ base, cdn: cdnBase, modules: moduleMap, params }) {
   if (typeof params === "string") paramsText = params;
   if (typeof cdnBase === "string") cdn = cdnBase.endsWith("/") ? cdnBase : cdnBase + "/";
+  if (moduleMap !== null && typeof moduleMap === "object") modules = { ...moduleMap };
   const url = new URL("$cdpk$repl.mjs", base).href;
   globalThis.__cdpk = {
-    import: (specifier, options) => import(resolveSpecifier(specifier, url, cdn)),
-    meta: { url, resolve: (specifier) => resolveSpecifier(specifier, url, cdn) },
+    import: (specifier, options) => import(resolveSpecifier(specifier, url, cdn, modules)),
+    meta: { url, resolve: (specifier) => resolveSpecifier(specifier, url, cdn, modules) },
+    // The URL `modules` maps a specifier to, or null: jupyter.sql's DuckDB
+    // files load from the page this way.
+    mapped: (specifier) => modules && Object.hasOwn(modules, specifier) ? new URL(modules[specifier], url).href : null,
     // inspect_request (#10): by descriptor, so nothing runs.
     describeForInspect: (value, detail) => describeValue(value, { inspect: (v) => inspect(v, { depth: 2 }), detail }),
     inspectPath: (path, detail) => inspectPath(path, { inspect: (v) => inspect(v, { depth: 2 }), detail }),