@tlockney/cdp-kernel 0.5.0 → 0.7.0

package/dist/language-service-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/language-service-worker.js

@@ -0,0 +1,88 @@
+// The worker thread that runs TypeScript's language service for the kernel
+// (#273). It owns a NotebookTypes (src/notebook-types.ts) and answers the
+// host's requests (src/language-service.ts) in order. Loading TypeScript
+// and the first program take most of a second, which here doesn't hold up
+// heartbeats, shell or control messages.
+import { parentPort, workerData } from "node:worker_threads";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import ts from "typescript";
+import { ensureInstalledAsync } from "../runtime/node-hooks.mjs";
+import { kernelGlobals, NotebookTypes } from "./notebook-types.js";
+import { DenoResolver, denoTypes } from "./deno-types.js";
+const options = workerData;
+const types = new NotebookTypes(ts, {
+    notebookDir: options.notebookDir,
+    npmCache: options.npmCache,
+    deno: options.deno === null
+        ? undefined
+        : new DenoResolver({ config: options.deno.config, root: join(options.notebookDir, "__cdpk_remote__") }),
+    types: [],
+    typeFiles: options.typeFiles,
+    globals: kernelGlobals(options.display),
+});
+if (options.denoTypes) {
+    const declarations = denoTypes();
+    if (declarations !== null)
+        types.addVirtualTypeFile("__cdpk_deno__.d.ts", declarations);
+    else
+        process.stderr.write("[cdp-kernel] no Deno types for completion: `deno types` failed\n");
+}
+/** A package's declaration entry point: package.json "types"/"typings", else index.d.ts. */
+function typesEntry(dir) {
+    try {
+        const manifest = JSON.parse(readFileSync(join(dir, "package.json"), "utf8"));
+        const entry = manifest.types ?? manifest.typings;
+        if (typeof entry === "string")
+            return join(dir, entry);
+    }
+    catch {
+        // No readable package.json: fall back to the convention.
+    }
+    return join(dir, "index.d.ts");
+}
+/**
+ * A type package for the target's globals (@types/node on Node, @types/bun,
+ * @cloudflare/workers-types): the
+ * notebook project's own when it has one, else installed once into the
+ * kernel's npm cache, in the background. Completion goes on without it
+ * meanwhile, and without it at all if the install fails (offline).
+ */
+async function addTypePackage({ name, range }) {
+    const short = name.replace(/^@types\//, "");
+    if (types.resolvesTypes(short)) {
+        types.addTypeNames([short]);
+        return;
+    }
+    try {
+        const { dir, alias } = await ensureInstalledAsync({ name, range });
+        types.addTypeFiles([typesEntry(join(dir, "node_modules", alias))]);
+    }
+    catch (err) {
+        process.stderr.write(`[cdp-kernel] no ${name} for completion: ${err instanceof Error ? err.message : err}\n`);
+    }
+}
+for (const pkg of options.typePackages)
+    void addTypePackage(pkg);
+parentPort?.on("message", (request) => {
+    let response;
+    switch (request.type) {
+        case "cell":
+            types.addCell(request.code);
+            return;
+        case "reset":
+            types.reset();
+            return;
+        case "complete":
+        case "quickInfo":
+            try {
+                response = request.type === "complete"
+                    ? { id: request.id, result: types.complete(request.code, request.cursor) }
+                    : { id: request.id, result: types.quickInfo(request.code, request.cursor) };
+            }
+            catch (err) {
+                response = { id: request.id, error: err instanceof Error ? err.stack ?? err.message : String(err) };
+            }
+            parentPort?.postMessage(response);
+    }
+});

package/dist/language-service.d.ts

@@ -0,0 +1,64 @@
+import type { CompletionResult } from "./notebook-types.ts";
+export type { CompletionEntry, CompletionResult } from "./notebook-types.ts";
+/** What the worker is started with. */
+export interface WorkerOptions {
+    notebookDir: string;
+    /** See NotebookTypesOptions.npmCache. */
+    npmCache: "npm" | "bun" | null;
+    /** A Deno target: resolve imports with `deno info`, under this project config if any. */
+    deno: {
+        config: string | null;
+    } | null;
+    /** Add `deno types` (the Deno namespace) to the globals. */
+    denoTypes: boolean;
+    /** Declaration files for the target's globals, e.g. a Workers project's worker-configuration.d.ts. */
+    typeFiles: string[];
+    /** runtime/display.mjs, which types `jupyter`; null if not a local file. */
+    display: string | null;
+    /** Type packages for the target's globals, e.g. @types/node. */
+    typePackages: {
+        name: string;
+        range: string;
+    }[];
+}
+export type WorkerRequest = {
+    type: "cell";
+    code: string;
+} | {
+    type: "reset";
+} | {
+    type: "complete" | "quickInfo";
+    id: number;
+    code: string;
+    cursor: number;
+};
+export type WorkerResponse = {
+    id: number;
+    result: CompletionResult | string | null;
+} | {
+    id: number;
+    error: string;
+};
+/**
+ * What to start the worker from. Deno's node:worker_threads takes only
+ * file: and data: URLs, so a kernel run from JSR (https:) starts a data:
+ * module that imports the worker.
+ */
+export declare function workerEntry(url?: URL): URL;
+export declare class LanguageService {
+    #private;
+    constructor(options: Omit<WorkerOptions, "display">, { budgetMs, log }?: {
+        budgetMs?: number;
+        log?: (text: string) => void;
+    });
+    /** A cell that ran (even if it threw): its declarations exist now. */
+    addCell(code: string): void;
+    /** The target restarted, so what the cells declared is gone. */
+    reset(): void;
+    /** Typed completions, or null if the service isn't ready within the budget. */
+    complete(code: string, cursor: number): Promise<CompletionResult | null>;
+    /** Type, signature and JSDoc for the name at the cursor, or null. */
+    quickInfo(code: string, cursor: number): Promise<string | null>;
+    /** Ends the worker; the next request starts another. */
+    close(): void;
+}

package/dist/language-service.js

@@ -0,0 +1,128 @@
+// The kernel's side of typed completion and inspection (#273, #274): starts the language-service
+// worker (src/language-service-worker.ts) on the first request, keeps it fed
+// with executed cells, and gives each request a time budget, after which
+// the kernel answers from the runtime alone. The worker is ended after a
+// while without requests, which gives its memory back (about 150 MB).
+import { Worker } from "node:worker_threads";
+import { fileURLToPath } from "node:url";
+/**
+ * How long a request waits for the worker. JupyterLab gives a kernel's
+ * completions a second, and the runtime's half runs alongside.
+ */
+const BUDGET_MS = 500;
+const IDLE_MS = 10 * 60 * 1000;
+// Compiled to dist/*.js for the npm package, run as .ts from a checkout.
+const WORKER_URL = new URL(import.meta.url.endsWith(".ts") ? "./language-service-worker.ts" : "./language-service-worker.js", import.meta.url);
+const DISPLAY_URL = new URL("../runtime/display.mjs", import.meta.url);
+/**
+ * What to start the worker from. Deno's node:worker_threads takes only
+ * file: and data: URLs, so a kernel run from JSR (https:) starts a data:
+ * module that imports the worker.
+ */
+export function workerEntry(url = WORKER_URL) {
+    if (url.protocol === "file:")
+        return url;
+    return new URL(`data:text/javascript,${encodeURIComponent(`import ${JSON.stringify(url.href)};`)}`);
+}
+export class LanguageService {
+    #options;
+    #budgetMs;
+    #log;
+    /** Executed cells, replayed to a new worker. */
+    #cells = [];
+    #pending = new Map();
+    #worker = null;
+    #nextId = 1;
+    #idle = null;
+    /** The worker couldn't start: don't keep trying. */
+    #broken = false;
+    constructor(options, { budgetMs = BUDGET_MS, log = () => { } } = {}) {
+        this.#options = { ...options, display: DISPLAY_URL.protocol === "file:" ? fileURLToPath(DISPLAY_URL) : null };
+        this.#budgetMs = budgetMs;
+        this.#log = log;
+    }
+    /** A cell that ran (even if it threw): its declarations exist now. */
+    addCell(code) {
+        this.#cells.push(code);
+        this.#worker?.postMessage({ type: "cell", code });
+    }
+    /** The target restarted, so what the cells declared is gone. */
+    reset() {
+        this.#cells.length = 0;
+        this.#worker?.postMessage({ type: "reset" });
+    }
+    /** Typed completions, or null if the service isn't ready within the budget. */
+    async complete(code, cursor) {
+        return (await this.#request("complete", code, cursor));
+    }
+    /** Type, signature and JSDoc for the name at the cursor, or null. */
+    async quickInfo(code, cursor) {
+        return (await this.#request("quickInfo", code, cursor));
+    }
+    #request(type, code, cursor) {
+        const worker = this.#ensureWorker();
+        if (worker === null)
+            return Promise.resolve(null);
+        const id = this.#nextId++;
+        return new Promise((resolve) => {
+            const timer = setTimeout(() => settle(null), this.#budgetMs);
+            const settle = (response) => {
+                clearTimeout(timer);
+                this.#pending.delete(id);
+                if (response !== null && "error" in response)
+                    this.#log(`${type} failed: ${response.error}`);
+                resolve(response !== null && "result" in response ? response.result : null);
+            };
+            this.#pending.set(id, settle);
+            worker.postMessage({ type, id, code, cursor });
+        });
+    }
+    /** Ends the worker; the next request starts another. */
+    close() {
+        if (this.#idle !== null)
+            clearTimeout(this.#idle);
+        this.#idle = null;
+        const worker = this.#worker;
+        this.#worker = null;
+        for (const settle of this.#pending.values())
+            settle(null);
+        void worker?.terminate();
+    }
+    #ensureWorker() {
+        if (this.#broken)
+            return null;
+        if (this.#idle !== null)
+            clearTimeout(this.#idle);
+        this.#idle = setTimeout(() => this.close(), IDLE_MS);
+        this.#idle.unref();
+        if (this.#worker !== null)
+            return this.#worker;
+        let worker;
+        try {
+            worker = new Worker(workerEntry(), { workerData: this.#options });
+        }
+        catch (err) {
+            this.#broken = true;
+            this.#log(`typed completion unavailable: ${err instanceof Error ? err.message : err}`);
+            return null;
+        }
+        // An idle worker mustn't keep the kernel alive.
+        worker.unref();
+        worker.on("message", (response) => this.#pending.get(response.id)?.(response));
+        worker.on("error", (err) => {
+            // E.g. typescript failed to load. Completion goes on from the runtime.
+            this.#broken = true;
+            this.#log(`typed completion unavailable: ${err instanceof Error ? err.message : err}`);
+            if (this.#worker === worker)
+                this.close();
+        });
+        worker.on("exit", () => {
+            if (this.#worker === worker)
+                this.close();
+        });
+        for (const code of this.#cells)
+            worker.postMessage({ type: "cell", code });
+        this.#worker = worker;
+        return worker;
+    }
+}

package/dist/main.js

@@ -1,17 +1,23 @@
 #!/usr/bin/env node
 // cdp-kernel CLI.
-//   cdp-kernel install [--runtime node|deno|bun|workerd|celld] [--deno-compat] [--name NAME] [--display NAME]
-//   cdp-kernel kernel --runtime node|deno|bun|workerd|celld [--deno-compat] --conn CONNECTION_FILE
+//   cdp-kernel install [--runtime node|deno|bun|workerd|celld] [--deno-compat] [--type-strip esbuild|sucrase] [--no-language-service] [--name NAME] [--display NAME]
+//   cdp-kernel kernel --runtime node|deno|bun|workerd|celld [--deno-compat] [--type-strip esbuild|sucrase] [--no-language-service] --conn CONNECTION_FILE
+//   cdp-kernel list
+//   cdp-kernel uninstall [NAME...] [--name NAME] [--all]
 //   cdp-kernel version
 //
 // --deno-compat (Node only) adds a `Deno` namespace for notebooks written for
-// `deno jupyter`; see runtime/deno-compat.mjs.
+// `deno jupyter`; see runtime/deno-compat.mjs. --type-strip picks what
+// removes cells' types (src/transform.ts). --no-language-service turns off
+// typed completion (src/language-service.ts), which costs about 150 MB
+// while in use.
 import process from "node:process";
-import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { homedir, platform } from "node:os";
 import { join } from "node:path";
 import { parseArgs } from "node:util";
 import { Kernel, parseConnectionInfo } from "./kernel.js";
+import { isTypeStripperName } from "./transform.js";
 import { hostRuntime, kernelArgv, moduleOrigin, VERSION } from "./package.js";
 function isRuntime(value) {
     return value === "node" || value === "deno" || value === "bun" || value === "workerd" || value === "celld";
@@ -25,7 +31,7 @@ function kernelsDir() {
         return join(process.env.APPDATA ?? homedir(), "jupyter", "kernels");
     return join(process.env.XDG_DATA_HOME ?? join(homedir(), ".local", "share"), "jupyter", "kernels");
 }
-function install(runtime, denoCompat, name, display) {
+function install(runtime, denoCompat, typeStrip, languageService, name, display) {
     const dir = join(kernelsDir(), name);
     mkdirSync(dir, { recursive: true });
     const spec = {
@@ -34,6 +40,8 @@ function install(runtime, denoCompat, name, display) {
             "--runtime",
             runtime,
             ...(denoCompat ? ["--deno-compat"] : []),
+            ...(typeStrip !== "esbuild" ? ["--type-strip", typeStrip] : []),
+            ...(languageService ? [] : ["--no-language-service"]),
             "--conn",
             "{connection_file}",
         ]),
@@ -49,6 +57,77 @@ function install(runtime, denoCompat, name, display) {
     writeFileSync(join(dir, "kernel.json"), JSON.stringify(spec, null, 2) + "\n");
     console.log(`installed kernelspec ${name} -> ${dir}`);
 }
+// A kernelspec this package wrote: install() always records the runtime in
+// metadata and passes the connection file to the kernel subcommand. Other
+// kernels' specs in the same directory are left alone.
+function readOwnSpec(name) {
+    const dir = join(kernelsDir(), name);
+    let spec;
+    try {
+        spec = JSON.parse(readFileSync(join(dir, "kernel.json"), "utf8"));
+    }
+    catch {
+        return undefined;
+    }
+    const argv = Array.isArray(spec.argv) ? spec.argv : [];
+    const runtime = spec.metadata?.runtime;
+    if (!isRuntime(runtime) || !argv.includes("kernel") || !argv.includes("{connection_file}"))
+        return undefined;
+    const version = typeof spec.metadata?.version === "string" ? spec.metadata.version : undefined;
+    return { name, dir, runtime, version };
+}
+function installedSpecs() {
+    const root = kernelsDir();
+    if (!existsSync(root))
+        return [];
+    return readdirSync(root, { withFileTypes: true })
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => readOwnSpec(entry.name))
+        .filter((spec) => spec !== undefined)
+        .sort((a, b) => a.name.localeCompare(b.name));
+}
+function list() {
+    const specs = installedSpecs();
+    if (specs.length === 0) {
+        console.log(`no cdp-kernel kernelspecs in ${kernelsDir()}`);
+        return;
+    }
+    const width = Math.max(...specs.map((s) => s.name.length));
+    for (const s of specs) {
+        console.log(`${s.name.padEnd(width)}  ${s.runtime.padEnd(7)}  ${(s.version ?? "-").padEnd(7)}  ${s.dir}`);
+    }
+}
+// Jupyter's kernel names; anything else could point outside the kernels
+// directory.
+const KERNEL_NAME = /^[A-Za-z0-9._-]+$/;
+function uninstall(names, all) {
+    if (all)
+        names = installedSpecs().map((s) => s.name);
+    else if (names.length === 0) {
+        console.error("uninstall: name the kernelspecs to remove, or pass --all. Installed:");
+        list();
+        process.exit(2);
+    }
+    let failed = false;
+    for (const name of names) {
+        const valid = KERNEL_NAME.test(name) && name !== "." && name !== "..";
+        const spec = valid ? readOwnSpec(name) : undefined;
+        if (!spec) {
+            const exists = valid && existsSync(join(kernelsDir(), name));
+            console.error(exists
+                ? `uninstall: ${name} wasn't installed by cdp-kernel; leaving it (use jupyter kernelspec remove)`
+                : `uninstall: no kernelspec ${name} in ${kernelsDir()}`);
+            failed = true;
+            continue;
+        }
+        rmSync(spec.dir, { recursive: true, force: true });
+        console.log(`removed kernelspec ${name} (${spec.dir})`);
+    }
+    if (all && names.length === 0)
+        console.log(`no cdp-kernel kernelspecs in ${kernelsDir()}`);
+    if (failed)
+        process.exit(1);
+}
 const { positionals, values } = parseArgs({
     allowPositionals: true,
     options: {
@@ -56,7 +135,10 @@ const { positionals, values } = parseArgs({
         conn: { type: "string" },
         name: { type: "string" },
         display: { type: "string" },
+        all: { type: "boolean", default: false },
         "deno-compat": { type: "boolean", default: false },
+        "type-strip": { type: "string", default: "esbuild" },
+        "no-language-service": { type: "boolean", default: false },
     },
 });
 const runtime = values.runtime;
@@ -64,12 +146,18 @@ if (!isRuntime(runtime)) {
     console.error(`unknown runtime ${runtime}; expected node, deno, bun, workerd or celld`);
     process.exit(2);
 }
+const typeStrip = values["type-strip"];
+if (!isTypeStripperName(typeStrip)) {
+    console.error(`unknown type stripper ${typeStrip}; expected esbuild or sucrase`);
+    process.exit(2);
+}
+const languageService = values["no-language-service"] !== true;
 // Deno targets have the real namespace; the shim is for Node.
 const denoCompat = values["deno-compat"] === true && runtime === "node";
 switch (positionals[0]) {
     case "install": {
         const suffix = denoCompat ? "-deno" : "";
-        install(runtime, denoCompat, values.name ?? `cdp-${runtime}${suffix}`, values.display ?? `TypeScript (cdp-kernel, ${runtime}${denoCompat ? ", Deno compat" : ""})`);
+        install(runtime, denoCompat, typeStrip, languageService, values.name ?? `cdp-${runtime}${suffix}`, values.display ?? `TypeScript (cdp-kernel, ${runtime}${denoCompat ? ", Deno compat" : ""})`);
         break;
     }
     case "kernel": {
@@ -77,14 +165,20 @@ switch (positionals[0]) {
             console.error("kernel: --conn CONNECTION_FILE is required");
             process.exit(2);
         }
-        const kernel = new Kernel(parseConnectionInfo(readFileSync(values.conn, "utf8")), runtime, { denoCompat });
+        const kernel = new Kernel(parseConnectionInfo(readFileSync(values.conn, "utf8")), runtime, { denoCompat, typeStrip, languageService });
         await kernel.run();
         break;
     }
+    case "list":
+        list();
+        break;
+    case "uninstall":
+        uninstall([...positionals.slice(1), ...(values.name ? [values.name] : [])], values.all === true);
+        break;
     case "version":
         console.log(VERSION);
         break;
     default:
-        console.error("usage: cdp-kernel install [--runtime node|deno|bun|workerd|celld] [--deno-compat] | kernel --runtime node|deno|bun|workerd|celld --conn FILE | version");
+        console.error("usage: cdp-kernel install [--runtime node|deno|bun|workerd|celld] [--deno-compat] [--type-strip esbuild|sucrase] [--no-language-service] | list | uninstall [NAME...] [--all] | kernel --runtime node|deno|bun|workerd|celld --conn FILE | version");
         process.exit(2);
 }

package/dist/message.d.ts

@@ -0,0 +1,34 @@
+export declare const PROTOCOL_VERSION = "5.3";
+export type JsonObject = {
+    [key: string]: unknown;
+};
+export interface Header extends JsonObject {
+    msg_id: string;
+    session: string;
+    date: string;
+    username: string;
+    msg_type: string;
+    version: string;
+}
+/** A header as received: the fields the kernel relies on, plus whatever else came. */
+export interface ReceivedHeader extends JsonObject {
+    msg_id: string;
+    msg_type: string;
+}
+/** A message from a frontend, decoded (and on ZeroMQ, verified). */
+export interface KernelMessage {
+    header: ReceivedHeader;
+    parentHeader: JsonObject;
+    metadata: JsonObject;
+    content: JsonObject;
+    buffers: Uint8Array[];
+}
+/** A message the kernel sends. */
+export interface OutgoingMessage {
+    header: Header;
+    parentHeader: JsonObject;
+    metadata: JsonObject;
+    content: JsonObject;
+    buffers: Uint8Array[];
+}
+export declare function makeHeader(session: string, msgType: string): Header;

package/dist/message.js

@@ -0,0 +1,14 @@
+// Jupyter message shapes and headers, without the wire format: what the
+// kernel's message handling (src/kernel-core.ts) builds and reads. The
+// framing and HMAC signing that ZeroMQ peers need are in src/wire.ts.
+export const PROTOCOL_VERSION = "5.3";
+export function makeHeader(session, msgType) {
+    return {
+        msg_id: crypto.randomUUID(),
+        session,
+        date: new Date().toISOString(),
+        username: "kernel",
+        msg_type: msgType,
+        version: PROTOCOL_VERSION,
+    };
+}

package/dist/mod.d.ts

@@ -22,4 +22,5 @@
  */
 export { type ConnectionInfo, Kernel, parseConnectionInfo } from "./kernel.ts";
 export type { RuntimeName } from "./target.ts";
+export { CellSyntaxError, type CellTransformOptions, type JsxOptions, type TypeStripper } from "./transform.ts";
 export { VERSION } from "./package.ts";

package/dist/mod.js

@@ -21,4 +21,5 @@
  * @module
  */
 export { Kernel, parseConnectionInfo } from "./kernel.js";
+export { CellSyntaxError } from "./transform.js";
 export { VERSION } from "./package.js";

package/dist/notebook-types.d.ts

@@ -0,0 +1,62 @@
+import type TS from "typescript";
+import type { DenoResolver } from "./deno-types.ts";
+export interface NotebookTypesOptions {
+    notebookDir: string;
+    /**
+     * The installer cache npm: and jsr: imports resolve from: where the
+     * target installs them (Bun's own for Bun, npm's for the rest). Null for
+     * Deno targets, which resolve them natively (see `deno`).
+     */
+    npmCache: "npm" | "bun" | null;
+    /**
+     * For Deno targets: where Deno put what the executed cells imported
+     * (npm:, jsr:, URLs, the project's import map).
+     */
+    deno?: DenoResolver;
+    /** compilerOptions.types: type packages found from the notebook's directory. */
+    types: string[];
+    /** Declaration files every cell sees, e.g. runtime types. */
+    typeFiles: string[];
+    /** Declarations for the kernel's own globals (jupyter, prompt, confirm). */
+    globals: string;
+}
+export interface CompletionEntry {
+    name: string;
+    /** TypeScript's ScriptElementKind: "method", "property", "const", ... */
+    kind: string;
+}
+export interface CompletionResult {
+    /** Where the completed name starts in the cell, as a UTF-16 index. */
+    start: number;
+    entries: CompletionEntry[];
+}
+export declare class NotebookTypes {
+    #private;
+    constructor(ts: typeof TS, options: NotebookTypesOptions);
+    /** An executed cell: its declarations are in scope for later ones. */
+    addCell(code: string): void;
+    /** The runtime restarted and lost its state. */
+    reset(): void;
+    /** Adds declaration files, e.g. runtime types installed after startup. */
+    addTypeFiles(files: string[]): void;
+    /** Adds a declaration file held in memory, e.g. the output of `deno types`. */
+    addVirtualTypeFile(name: string, text: string): void;
+    /** Adds type packages (compilerOptions.types) that resolve from the notebook. */
+    addTypeNames(names: string[]): void;
+    /** Whether a type package (compilerOptions.types) resolves from the notebook. */
+    resolvesTypes(name: string): boolean;
+    /** Completions at `cursor` (a UTF-16 index) in `code`, the cell being edited. */
+    complete(code: string, cursor: number): CompletionResult;
+    /**
+     * The type, signature and JSDoc of the name at `cursor`: the one the
+     * cursor is in or just after, or, right after "name(", that callee.
+     */
+    quickInfo(code: string, cursor: number): string | null;
+}
+/**
+ * Declarations for the names the kernel adds to every target, as in
+ * src/globals.ts. `display` is runtime/display.mjs's path; its sibling
+ * display.d.mts types `jupyter`. Without it (a kernel run from JSR, whose
+ * runtime files aren't local), `jupyter` is untyped.
+ */
+export declare function kernelGlobals(display: string | null): string;