@tlockney/cdp-kernel 0.5.0 → 0.7.0

package/dist/target.js

@@ -7,11 +7,20 @@ import { spawn } from "node:child_process";
 import { createRequire } from "node:module";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { asEvaluateResult, CdpAbandoned, CdpSession } from "./cdp.js";
+import { asEvaluateResult, CdpAbandoned, CdpError, CdpSession } from "./cdp.js";
 import { SHIM, transformCell } from "./transform.js";
 import { hostRuntime, isInsideNodeModules, runtimeDir } from "./package.js";
 import { killOnExit } from "./children.js";
-const BINDING = "__cdpk_send";
+import { Interrupted, isTargetMessage, parseJson } from "./target-api.js";
+export { Interrupted, isTargetMessage, parseJson };
+/**
+ * The inspector keeps every object it returns a remote reference to alive
+ * until it is released, so a cell's result would never be collected (#286).
+ * Cells and lookups put theirs in these groups, which are released once the
+ * kernel is done with them.
+ */
+const CELL_GROUP = "cdpk-cell";
+const LOOKUP_GROUP = "cdpk-lookup";
 /** How long a target may take from spawn to a usable session. */
 const START_TIMEOUT_MS = 60_000;
 const MARKER_RE = /\x1b\[cdpk:(\d+)(m?)\]\x1b/;
@@ -27,7 +36,9 @@ const INSPECTOR_CHATTER = [
     "Visit chrome://inspect",
     "Waiting for the debugger to disconnect",
 ];
-const DENO_CHATTER = [...INSPECTOR_CHATTER, "Download http://", "Download https://"];
+// Deno prints "Debugger session started." for each session the host opens
+// (#281).
+const DENO_CHATTER = [...INSPECTOR_CHATTER, "Debugger session started.", "Download http://", "Download https://"];
 const ANSI_SGR = /\x1b\[[0-9;]*m/g;
 /** A colour code cut off at the end of a chunk. */
 const PARTIAL_SGR = /\x1b(?:\[[0-9;]*)?$/;
@@ -75,23 +86,11 @@ export function esbuildBinary(resolve = import.meta.resolve, platform = process.
         return null;
     }
 }
-/** The parsed value, or undefined for text that isn't JSON. */
-export function parseJson(text) {
-    try {
-        return JSON.parse(text);
-    }
-    catch {
-        return undefined;
-    }
-}
-export function isTargetMessage(value) {
-    if (typeof value !== "object" || value === null)
-        return false;
-    const kind = value.kind;
-    return (kind === "iopub" || kind === "input_request") &&
-        typeof value.barrier === "number";
-}
 const MARKER_PREFIX = "\x1b[cdpk:";
+// A message record: the message's JSON, which never holds a raw ESC, between
+// "\x1b[cdpk:<" and "\x1b]" (#271).
+const RECORD_PREFIX = "\x1b[cdpk:<";
+const RECORD_RE = /\x1b\[cdpk:<([^\x1b]*)\x1b\]/;
 function isPartialMarker(tail) {
     if (tail.length >= MAX_MARKER_LEN)
         return false;
@@ -99,10 +98,13 @@ function isPartialMarker(tail) {
 }
 /**
  * Splits one output pipe into text and barrier markers, and releases them
- * in order. A gated marker (one that announces a binding message) pauses
+ * in order. A gated marker (one that announces a message) pauses
  * the pipe until `gate(n)` resolves, i.e. until that message has been
  * delivered, so later output can't overtake it. A possible partial marker
  * at the end of a chunk is held back until the next chunk decides it.
+ *
+ * With `onMessage`, the pipe also carries message records. A record stands
+ * where its gated marker would, and pauses the pipe the same way.
  */
 export class PipeDemux {
     #buffer = "";
@@ -111,28 +113,49 @@ export class PipeDemux {
     #closed = false;
     #reached = 0;
     #waiters = [];
+    /** The gated marker the pipe is paused on, if any. */
+    #gatedOn = null;
     onText;
     gate;
-    constructor(onText, gate = () => Promise.resolve()) {
+    onMessage;
+    constructor(onText, gate = () => Promise.resolve(), onMessage = null) {
         this.onText = onText;
         this.gate = gate;
+        this.onMessage = onMessage;
     }
     push(chunk) {
         this.#buffer += chunk;
         for (;;) {
             const m = MARKER_RE.exec(this.#buffer);
-            if (!m)
+            const r = this.onMessage ? RECORD_RE.exec(this.#buffer) : null;
+            const first = m && r ? (m.index < r.index ? m : r) : m ?? r;
+            if (!first)
                 break;
-            if (m.index > 0)
-                this.#items.push({ text: this.#buffer.slice(0, m.index) });
-            this.#items.push({ barrier: Number(m[1]), gated: m[2] === "m" });
-            this.#buffer = this.#buffer.slice(m.index + m[0].length);
+            if (first.index > 0)
+                this.#items.push({ text: this.#buffer.slice(0, first.index) });
+            if (first === m) {
+                this.#items.push({ barrier: Number(m[1]), gated: m[2] === "m" });
+            }
+            else {
+                // Anything else a cell prints in this shape is dropped (#122).
+                const msg = parseJson(first[1]);
+                if (isTargetMessage(msg))
+                    this.#items.push({ message: msg });
+            }
+            this.#buffer = this.#buffer.slice(first.index + first[0].length);
         }
         // Hold back a tail that could still grow into a marker, including one
-        // cut inside the "\x1b[cdpk:" prefix itself.
+        // cut inside the "\x1b[cdpk:" prefix itself, or into a record: one
+        // whose only ESC after its prefix is a final one.
         const esc = this.#buffer.lastIndexOf("\x1b");
         const tail = esc === -1 ? "" : this.#buffer.slice(esc);
-        const holdFrom = esc !== -1 && isPartialMarker(tail) ? esc : this.#buffer.length;
+        let holdFrom = esc !== -1 && isPartialMarker(tail) ? esc : this.#buffer.length;
+        const record = this.onMessage ? this.#buffer.lastIndexOf(RECORD_PREFIX) : -1;
+        if (record !== -1) {
+            const next = this.#buffer.indexOf("\x1b", record + RECORD_PREFIX.length);
+            if (next === -1 || next === this.#buffer.length - 1)
+                holdFrom = Math.min(holdFrom, record);
+        }
         if (holdFrom > 0) {
             this.#items.push({ text: this.#buffer.slice(0, holdFrom) });
             this.#buffer = this.#buffer.slice(holdFrom);
@@ -149,9 +172,22 @@ export class PipeDemux {
                     this.onText(item.text);
                     continue;
                 }
+                if ("message" in item) {
+                    this.#reach(item.message.barrier);
+                    this.onMessage?.(item.message);
+                    if (!this.#closed) {
+                        this.#gatedOn = item.message.barrier;
+                        await this.gate(item.message.barrier);
+                        this.#gatedOn = null;
+                    }
+                    continue;
+                }
                 this.#reach(item.barrier);
-                if (item.gated && !this.#closed)
+                if (item.gated && !this.#closed) {
+                    this.#gatedOn = item.barrier;
                     await this.gate(item.barrier);
+                    this.#gatedOn = null;
+                }
             }
         }
         finally {
@@ -172,6 +208,16 @@ export class PipeDemux {
             return Promise.resolve();
         return new Promise((resolve) => this.#waiters.push({ n, resolve }));
     }
+    /** Where the pipe stands, for a state dump (#237). */
+    state() {
+        return {
+            reached: this.#reached,
+            waitingFor: this.#waiters.map((w) => w.n),
+            gatedOn: this.#gatedOn,
+            queued: this.#items.length,
+            held: this.#buffer.length,
+        };
+    }
     /** Release everything (the process died). */
     close() {
         this.#closed = true;
@@ -267,9 +313,6 @@ export class Gates {
         this.#map.clear();
     }
 }
-export class Interrupted extends Error {
-    name = "Interrupted";
-}
 /** A catch handler: a call interrupt() abandoned becomes Interrupted. */
 function interrupted(what) {
     return (err) => {
@@ -287,8 +330,13 @@ export class Target {
     #gates;
     #contextId;
     #messages = Promise.resolve();
+    /** Messages received but not yet delivered. */
+    #undelivered = 0;
     #interrupted = false;
     #paused = false;
+    /** What the debugger asked to pause on (#307), and how many kernel calls hold it off. */
+    #pauseOnExceptions = "none";
+    #unpausable = 0;
     /** Set once the main context is gone: the runtime is exiting (#130). */
     #exiting = false;
     #exited;
@@ -332,6 +380,9 @@ export class Target {
         delete env.NO_COLOR;
         const child = spawn(cmd, args, { cwd: options.cwd, stdio: ["pipe", "pipe", "pipe"], env });
         killOnExit(child);
+        // Prompt replies and comm messages go to stdin; one written as the
+        // runtime exits fails with EPIPE, and the exit is handled on its own.
+        child.stdin.on("error", () => { });
         // A start that fails or stalls must not leave the runtime running: its
         // pipes and inspector sockets would keep this process alive (#52).
         const sessions = [];
@@ -357,7 +408,12 @@ export class Target {
         child.stdout.setEncoding("utf8");
         child.stderr.setEncoding("utf8");
         const gates = new Gates();
-        const stdout = new PipeDemux((text) => events.stream("stdout", text), (n) => gates.wait(n));
+        // Messages come only after attach(), so the target exists by then.
+        let target;
+        const stdout = new PipeDemux((text) => events.stream("stdout", text), (n) => gates.wait(n), (msg) => {
+            if (target)
+                target.#deliver(msg, events);
+        });
         const stderr = new PipeDemux((text) => events.stream("stderr", text), (n) => gates.wait(n));
         const filter = new StderrFilter(runtime, (text) => stderr.push(text));
         const pushStderr = (chunk) => filter.push(chunk);
@@ -398,25 +454,22 @@ export class Target {
             });
         });
         await shell.send("Runtime.enable");
-        await shell.send("Runtime.addBinding", { name: BINDING });
         const contextId = await contextReady;
-        const target = new Target(runtime, child, shell, control, stdout, stderr, gates, contextId, { runtime: "", engine: "" });
+        // The inspector socket leaves Nagle's algorithm on, so an event written
+        // just before an evaluate's reply holds that reply until the event is
+        // acknowledged, ~40 ms on Linux (#271). So the shell session, which runs
+        // the cells, takes no Runtime events (console calls among them), and
+        // messages come over stdout. The control session watches for exit.
+        await shell.send("Runtime.disable");
+        await control.send("Runtime.enable");
+        target = new Target(runtime, child, shell, control, stdout, stderr, gates, contextId, { runtime: "", engine: "" });
         // process.exit() or Deno.exit() destroys the context, then the runtime
         // waits for its inspector clients to disconnect before it exits. Let go,
         // so it exits and the kernel starts a fresh one (#130).
-        shell.on("Runtime.executionContextDestroyed", (params) => {
-            if (params.executionContextId === contextId)
+        control.on("Runtime.executionContextDestroyed", (params) => {
+            if (params.executionContextId === contextId && target)
                 target.#onExit();
         });
-        shell.on("Runtime.bindingCalled", (params) => {
-            if (params.name !== BINDING || typeof params.payload !== "string")
-                return;
-            // __cdpk_send is an ordinary global, so a cell can call it with
-            // anything; that mustn't take the host down (#122).
-            const msg = parseJson(params.payload);
-            if (isTargetMessage(msg))
-                target.#deliver(msg, events);
-        });
         // Wait for the bootstrap's top-level await to finish installing the
         // shim. There is no cap here: launch's start timeout bounds the wait and
         // kills the child, so a slow bootstrap never yields a target without
@@ -427,6 +480,7 @@ export class Target {
                 break;
             await new Promise((r) => setTimeout(r, 25));
         }
+        await target.#evaluateSync(`${SHIM}.attach()`);
         const versions = await target.#evaluateSync(`JSON.stringify(typeof Deno !== "undefined" ? { runtime: Deno.version.deno, engine: "V8 " + Deno.version.v8 } : { runtime: process.versions.node, engine: "V8 " + process.versions.v8 })`);
         Object.assign(target.versions, JSON.parse(String(versions.result.value)));
         // The runtime's own enumerable globals, before any cell: the debugger's
@@ -443,21 +497,102 @@ export class Target {
     /** The Debugger domain on this target's session, for src/debugger.ts. */
     get debug() {
         return {
-            send: (method, params = {}) => this.#shell.send(method, params),
+            send: (method, params = {}) => {
+                if (method === "Debugger.setPauseOnExceptions") {
+                    this.#pauseOnExceptions = String(params.state ?? "none");
+                    // Applied when the kernel's own call in flight finishes.
+                    if (this.#unpausable > 0)
+                        return Promise.resolve({});
+                }
+                if (method === "Debugger.disable")
+                    this.#pauseOnExceptions = "none";
+                return this.#shell.send(method, params);
+            },
             on: (method, listener) => this.#shell.on(method, listener),
             contextId: this.#contextId,
             baselineGlobals: this.#baselineGlobals,
+            withoutPausing: (call) => this.#withoutPausing(call),
         };
     }
-    // Binding messages wait until both pipes have caught up to their marker,
+    /**
+     * Runs the kernel's own inspector call with pausing on exceptions off. A
+     * pause inside a call the inspector dispatches holds that call (Node), or
+     * resumes by itself and leaves Deno's event loop stopped for good, so no
+     * later cell runs (#307). A cell that throws meanwhile doesn't pause.
+     */
+    async #withoutPausing(call) {
+        if (this.#pauseOnExceptions === "none" || this.#paused)
+            return call();
+        if (this.#unpausable++ === 0)
+            await this.#shell.send("Debugger.setPauseOnExceptions", { state: "none" });
+        try {
+            return await call();
+        }
+        finally {
+            if (--this.#unpausable === 0 && !this.#exiting) {
+                await this.#shell.send("Debugger.setPauseOnExceptions", { state: this.#pauseOnExceptions }).catch(() => { });
+            }
+        }
+    }
+    // Messages wait until both pipes have caught up to their marker,
     // and are delivered strictly in order.
     #deliver(msg, events) {
+        this.#undelivered++;
         this.#messages = this.#messages.then(async () => {
             await Promise.all([this.#stdout.waitFor(msg.barrier), this.#stderr.waitFor(msg.barrier)]);
+            this.#undelivered--;
             events.message(msg);
             this.#gates.open(msg.barrier);
         });
     }
+    /**
+     * What the host is waiting on, whether the runtime still evaluates, runs
+     * immediates and runs timers, and where its JavaScript is if it is
+     * running any, for a kernel that has stopped answering (#237). The probes
+     * go over the control session, which no cell holds.
+     */
+    async diagnose() {
+        const within = (promise, fallback) => Promise.race([promise, new Promise((resolve) => setTimeout(() => resolve(fallback), 2000).unref())]);
+        const probe = (expression, awaitPromise) => within(this.#control.send("Runtime.evaluate", { expression, contextId: this.#contextId, awaitPromise, returnByValue: true })
+            .then((res) => res.exceptionDetails ? `threw: ${JSON.stringify(res.exceptionDetails)}` : String(res.result?.value), (err) => `failed: ${err}`), "no answer in 2 s");
+        const [evaluates, immediates, timers] = await Promise.all([
+            probe("'yes'", false),
+            probe("new Promise((resolve) => setImmediate(() => resolve('yes')))", true),
+            probe("new Promise((resolve) => setTimeout(() => resolve('yes'), 0))", true),
+        ]);
+        // A runtime whose event loop is stuck may be stuck in JavaScript: say
+        // where. (An idle one would take the pause in whatever runs next.)
+        let running = "event loop runs";
+        if (immediates !== "yes") {
+            try {
+                const paused = new Promise((resolve) => this.#control.on("Debugger.paused", resolve));
+                await this.#control.send("Debugger.enable");
+                await this.#control.send("Debugger.pause");
+                const event = await within(paused, null);
+                running = event === null ? "no" : event.callFrames
+                    .slice(0, 10)
+                    .map((f) => `${f.functionName || "(anonymous)"} ${f.url || "(no url)"}:${f.location.lineNumber + 1}`);
+                await this.#control.send("Debugger.resume").catch(() => { });
+                await this.#control.send("Debugger.disable");
+            }
+            catch (err) {
+                running = `failed: ${err}`;
+            }
+        }
+        return {
+            pid: this.#child.pid ?? null,
+            calls: { shell: this.#shell.pendingCalls(), control: this.#control.pendingCalls() },
+            stdout: this.#stdout.state(),
+            stderr: this.#stderr.state(),
+            undelivered: this.#undelivered,
+            paused: this.#paused,
+            exiting: this.#exiting,
+            evaluates,
+            immediates,
+            timers,
+            running,
+        };
+    }
     /** Resolves once all output and messages sent so far have been delivered. */
     async flush() {
         const n = await this.#evaluateSync(`${SHIM}.barrier()`).catch(interrupted("Execution"));
@@ -479,7 +614,13 @@ export class Target {
     }
     #evaluateRaw(expression, session = this.#shell) {
         return session
-            .send("Runtime.evaluate", { expression, contextId: this.#contextId, replMode: true, awaitPromise: true })
+            .send("Runtime.evaluate", {
+            expression,
+            contextId: this.#contextId,
+            replMode: true,
+            awaitPromise: true,
+            objectGroup: CELL_GROUP,
+        })
             .then(asEvaluateResult);
     }
     /** Runs transformed cell code. Rejects with Interrupted after interrupt(). */
@@ -491,6 +632,10 @@ export class Target {
         catch (err) {
             if (err instanceof CdpAbandoned || this.#interrupted)
                 throw new Interrupted("Execution interrupted");
+            // The control session's executionContextDestroyed event can arrive
+            // after the shell session's error reply (#271 moved it there).
+            if (err instanceof CdpError && err.message.includes("Execution context was destroyed"))
+                this.#onExit();
             if (this.#exiting) {
                 // The child's exit, which drops this target in the kernel, comes
                 // first, so the next cell starts a fresh one.
@@ -500,6 +645,11 @@ export class Target {
             throw err;
         }
     }
+    async releaseResults() {
+        if (this.#exiting)
+            return;
+        await this.#shell.send("Runtime.releaseObjectGroup", { objectGroup: CELL_GROUP }).catch(() => { });
+    }
     #onExit() {
         if (this.#exiting)
             return;
@@ -532,12 +682,12 @@ export class Target {
      * even a plain return value in a promise it holds only weakly (#55).
      */
     callFunctionOn(functionDeclaration, args) {
-        return this.#shell.send("Runtime.callFunctionOn", {
+        return this.#withoutPausing(() => this.#shell.send("Runtime.callFunctionOn", {
             functionDeclaration,
             arguments: args,
             executionContextId: this.#contextId,
             returnByValue: true,
-        });
+        }));
     }
     /**
      * For functions that return a promise. The target holds it until after
@@ -573,14 +723,19 @@ export class Target {
      */
     async dispatchComm(msgType, content, metadata, buffers) {
         this.#interrupted = false;
+        // Over stdin, not as call arguments: V8 caches an argument's source,
+        // so the data would stay in the target's memory (#287).
+        const record = Buffer.from(JSON.stringify({
+            msgType,
+            content,
+            metadata,
+            buffers: buffers.map((b) => Buffer.from(b).toString("base64")),
+        }));
+        this.#child.stdin.write(`comm ${record.length}\n`);
+        this.#child.stdin.write(record);
         let res;
         try {
-            res = await this.#callAsyncFunctionOn(`function (t, c, m, b) { return globalThis.${SHIM}.commDispatch(t, c, m, b); }`, [
-                { value: msgType },
-                { value: content },
-                { value: metadata },
-                { value: buffers.map((b) => Buffer.from(b).toString("base64")) },
-            ]);
+            res = await this.#callAsyncFunctionOn(`function () { return globalThis.${SHIM}.commDispatchFromStdin(); }`, []);
         }
         catch (err) {
             if (err instanceof CdpAbandoned || this.#interrupted)
@@ -611,11 +766,22 @@ export class Target {
      */
     #evaluatePure(expression) {
         return this.#shell
-            .send("Runtime.evaluate", { expression, contextId: this.#contextId, throwOnSideEffect: true })
+            .send("Runtime.evaluate", { expression, contextId: this.#contextId, throwOnSideEffect: true, objectGroup: LOOKUP_GROUP })
             .then(asEvaluateResult)
             .catch(() => null);
     }
+    #releaseLookup() {
+        return this.#shell.send("Runtime.releaseObjectGroup", { objectGroup: LOOKUP_GROUP }).then(() => { }, () => { });
+    }
     async propertyNames(expression) {
+        try {
+            return await this.#propertyNames(expression);
+        }
+        finally {
+            await this.#releaseLookup();
+        }
+    }
+    async #propertyNames(expression) {
         const evaluated = await this.#evaluatePure(`(${expression})`);
         const objectId = evaluated?.result.objectId;
         if (!objectId || evaluated?.exceptionDetails)
@@ -625,6 +791,14 @@ export class Target {
         return (list ?? []).map((p) => p.name);
     }
     async inspect(path, detail) {
+        try {
+            return await this.#inspect(path, detail);
+        }
+        finally {
+            await this.#releaseLookup();
+        }
+    }
+    async #inspect(path, detail) {
         const found = await this.#evaluatePure(path);
         if (!found || found.exceptionDetails)
             return null;

package/dist/traceback.d.ts

@@ -0,0 +1,75 @@
+import type { LineMap } from "./sourcemap.ts";
+export declare const TRACEBACK_MIME = "application/vnd.nteract.traceback+json";
+/** The //# sourceURL a cell's script runs under while not debugging. */
+export declare function cellName(code: string): string;
+/** A cell that ran: its code, and where the notebook knows it from. */
+export interface CellRecord {
+    code: string;
+    executionCount: number | null;
+    cellId: string | null;
+}
+export interface SourceLine extends Record<string, string | number | boolean> {
+    lineno: number;
+    source: string;
+}
+export interface TracebackFrame extends Record<string, unknown> {
+    filename: string;
+    lineno: number;
+    colno?: number;
+    name: string;
+    library: boolean;
+    lines: SourceLine[];
+    execution_count?: number;
+    cell_id?: string;
+}
+/** A frame of CDP's exceptionDetails.stackTrace (0-based positions). */
+export interface CallFrame {
+    url: string;
+    lineNumber: number;
+    columnNumber: number;
+}
+/** A frame as parsed from one line of a stack. */
+export interface StackFrame {
+    name: string;
+    file: string;
+    line: number | null;
+    column: number | null;
+}
+/** One line of a stack as a frame, or null for a line that isn't one (the message). */
+export declare function parseFrame(text: string): StackFrame | null;
+/**
+ * Frames from packages, runtimes and the kernel itself, which a frontend
+ * can dim or fold: a hint, not a boundary.
+ */
+export declare function isLibraryFile(file: string, kernelFiles?: readonly string[]): boolean;
+export interface TracebackSources {
+    /** The cell a script name belongs to, if it is one. */
+    cell(file: string): (CellRecord & {
+        lines: LineMap | null;
+    }) | undefined;
+    /** Another file's source, when it can be read. */
+    readFile?(file: string): string | undefined;
+    /** Path or URL prefixes of the kernel's own runtime files. */
+    kernelFiles?: readonly string[];
+}
+export interface Traceback {
+    /** The classic traceback lines, with cell frames pointing at the cell. */
+    lines: string[];
+    /** The structured payload. */
+    payload: Record<string, unknown>;
+}
+/**
+ * `stack` is the error's stack as the target describes it (src/target.ts's
+ * describeError), message lines included. `callFrames`, from CDP, name the
+ * scripts of frames the stack shows as "<anonymous>": Deno formats stacks
+ * itself and doesn't use a script's //# sourceURL.
+ */
+export declare function buildTraceback(error: {
+    ename: string;
+    evalue: string;
+    stack: string;
+    callFrames?: readonly CallFrame[];
+}, sources: TracebackSources, execution: {
+    executionCount: number | null;
+    cellId: string | null;
+}): Traceback;