@hpcc-js/observablehq-compiler 3.3.9 → 3.5.0

package/src/kit/compiler.ts

@@ -0,0 +1,41 @@
+
+import { type Notebook, transpile } from "@observablehq/notebook-kit";
+import { type Definition } from "@observablehq/notebook-kit/runtime";
+import { constructFunction } from "./util.ts";
+
+export interface CompileKitOptions {
+    inline?: boolean;
+}
+
+export function compile(notebook: Notebook, options: CompileKitOptions = { inline: true }): Definition[] {
+    const retVal: Definition[] = [];
+    let id = 1;
+    for (const cell of notebook.cells) {
+        try {
+            const compiled = transpile(cell);
+            retVal.push({
+                id: id++,
+                ...compiled,
+                body: options.inline ? constructFunction(compiled.body, `cell_${id}`) : compiled.body,
+            });
+            if (cell.pinned) {
+                const compiled = transpile({
+                    ...cell,
+                    mode: "md",
+                    value: `\
+\`\`\`${cell.mode}
+${cell.value}
+\`\`\``
+                });
+                retVal.push({
+                    id: id++,
+                    ...compiled,
+                    body: options.inline ? constructFunction(compiled.body, `cell_${id}`) : compiled.body,
+                });
+            }
+        } catch (error) {
+            console.error(`Error compiling cell ${id}:`, error);
+        }
+    }
+    return retVal;
+}

package/src/kit/index.ts

@@ -0,0 +1,3 @@
+export * from "./util.ts";
+export * from "./compiler.ts";
+export * from "./util.ts";

package/src/kit/runtime.ts

@@ -0,0 +1,57 @@
+import { type DefineState, NotebookRuntime as NotebookRuntimeBase } from "@observablehq/notebook-kit/runtime";
+import { type Definition } from "./index.ts";
+
+import "@observablehq/notebook-kit/index.css";
+import "@observablehq/notebook-kit/theme-air.css";
+
+export class NotebookRuntime extends NotebookRuntimeBase {
+
+    stateById = new Map<number, DefineState>();
+
+    constructor() {
+        super();
+    }
+
+    has(cellId: number): boolean {
+        return this.stateById.has(cellId);
+    }
+
+    async add(cellId: number, definition: Definition, placeholderDiv: HTMLDivElement): Promise<void> {
+        let state: DefineState | undefined = this.stateById.get(cellId);
+        if (state) {
+            this.remove(cellId);
+        }
+        state = { root: placeholderDiv, expanded: [], variables: [] };
+        this.stateById.set(cellId, state);
+        this.define(state, definition);
+        await this.runtime._computeNow();
+    }
+
+    async remove(cellId: number): Promise<void> {
+        const state = this.stateById.get(cellId);
+        if (!state) return;
+        [...state.variables].forEach(v => v.delete());
+        this.stateById.delete(cellId);
+        state.root?.remove();
+        await this.runtime._computeNow();
+    }
+
+    async removeAll(): Promise<void> {
+        const keys = [...this.stateById.keys()];
+        for (const key of keys) {
+            this.remove(key);
+        }
+        await this.runtime._computeNow();
+    }
+
+    render(definitions: Definition[], target: HTMLElement) {
+        definitions.forEach(definition => {
+            const placeholderDiv = document.createElement("div");
+            placeholderDiv.id = `cell-${definition.id}`;
+            placeholderDiv.className = "observablehq observablehq--cell";
+            this.stateById.set(definition.id, { root: placeholderDiv, expanded: [], variables: [] });
+            this.define(this.stateById.get(definition.id)!, definition);
+            target.appendChild(placeholderDiv);
+        });
+    }
+}

package/src/kit/util.ts

@@ -0,0 +1,157 @@
+import { type Notebook, type Cell, parseJavaScript, serialize, deserialize, toNotebook, toCell } from "@observablehq/notebook-kit";
+import { type Definition } from "@observablehq/notebook-kit/runtime";
+
+export type { Notebook, Cell, Definition };
+export { toNotebook, toCell };
+
+// Shared function constructor utilities to avoid duplication between util modules.
+
+export type RegularFunction = (...args: any[]) => any;
+interface RegularFunctionConstructor {
+    new(...args: string[]): RegularFunction;
+    (...args: string[]): RegularFunction;
+    readonly prototype: RegularFunction;
+}
+
+export type AsyncFunction = (...args: any[]) => Promise<any>;
+interface AsyncFunctionConstructor {
+    new(...args: string[]): AsyncFunction;
+    (...args: string[]): AsyncFunction;
+    readonly prototype: AsyncFunction;
+}
+
+export type GeneratorFunction = (...args: any[]) => Generator<any, any, any>;
+interface GeneratorFunctionConstructor {
+    new(...args: string[]): GeneratorFunction;
+    (...args: string[]): GeneratorFunction;
+    readonly prototype: GeneratorFunction;
+}
+
+export type AsyncGeneratorFunction = (...args: any[]) => AsyncGenerator<any, any, any>;
+interface AsyncGeneratorFunctionConstructor {
+    new(...args: string[]): AsyncGeneratorFunction;
+    (...args: string[]): AsyncGeneratorFunction;
+    readonly prototype: AsyncGeneratorFunction;
+}
+
+export type AnyFunction = RegularFunction | AsyncFunction | GeneratorFunction | AsyncGeneratorFunction;
+
+export const FunctionConstructors: {
+    regular: RegularFunctionConstructor;
+    async: AsyncFunctionConstructor;
+    generator: GeneratorFunctionConstructor;
+    asyncGenerator: AsyncGeneratorFunctionConstructor;
+} = {
+    regular: Object.getPrototypeOf(function () { }).constructor as RegularFunctionConstructor,
+    async: Object.getPrototypeOf(async function () { }).constructor as AsyncFunctionConstructor,
+    generator: Object.getPrototypeOf(function* () { }).constructor as GeneratorFunctionConstructor,
+    asyncGenerator: Object.getPrototypeOf(async function* () { }).constructor as AsyncGeneratorFunctionConstructor,
+};
+
+function funcType(async: boolean = false, generator: boolean = false) {
+    if (!async && !generator) return FunctionConstructors.regular;
+    if (async && !generator) return FunctionConstructors.async;
+    if (!async && generator) return FunctionConstructors.generator;
+    return FunctionConstructors.asyncGenerator;
+}
+
+interface Ref {
+    start: number,
+    end: number,
+    newText: string
+}
+
+export interface Refs {
+    inputs: string[];
+    args: string[];
+    patches: Ref[];
+}
+
+export function createFunction(refs: Refs, async = false, generator = false, blockStatement = false, body?: string) {
+    if (body === undefined) {
+        return undefined;
+    }
+
+    refs.patches.sort((l, r) => r.start - l.start);
+    refs.patches.forEach(r => {
+        body = body!.substring(0, r.start) + r.newText + body!.substring(r.end);
+    });
+    return new (funcType(async, generator))(...refs.args, blockStatement ?
+        body.substring(1, body.length - 1).trim() :
+        `return (\n${body}\n);`);
+}
+
+function join(baseURL: string, relativeURL: string) {
+    return relativeURL
+        ? baseURL.replace(/\/+$/, "") + "/" + relativeURL.replace(/^\/+/, "")
+        : baseURL;
+}
+
+export const isRelativePath = (path: string) => path[0] === ".";
+export const fixRelativeUrl = (path: string, basePath: string) => {
+    if (isRelativePath(path)) {
+        return join(basePath, path);
+    }
+    return path;
+};
+
+//  Hide "import" from bundlers as they have a habit of replacing "import" with "require"
+const obfuscatedImportFunction = new FunctionConstructors.async("url", "return import(url)");
+export async function obfuscatedImport(url: string) {
+    return obfuscatedImportFunction(url);
+}
+
+function _constructFunction(body: any, bodyStr: string, name?: string) {
+    if (body.type !== "FunctionExpression" && body.type !== "FunctionDeclaration" && body.type !== "ArrowFunctionExpression") {
+        throw new Error(`Unsupported function type: ${body.type}`);
+    }
+    const func = body.async && body.generator ?
+        FunctionConstructors.asyncGenerator :
+        body.async ?
+            FunctionConstructors.async :
+            body.generator ?
+                FunctionConstructors.generator :
+                FunctionConstructors.regular;
+
+    const params = body.params?.map((param) => bodyStr.slice(param.start, param.end)).join(", ") ?? "";
+    const isBlock = body.body.type === "BlockStatement";
+    const { start, end } = body.body;
+    const inner = isBlock
+        ? bodyStr.slice(start + 1, end - 1)
+        : `return ${bodyStr.slice(start, end)}`;
+    // If a name is provided, build a true named function expression for reliable naming.
+    if (name) {
+        // Sanitize to a valid identifier; fallback to underscored name when necessary.
+        const sanitize = (n: string): string => {
+            let s = n.replace(/[^A-Za-z0-9_$]/g, "_");
+            if (!/^[A-Za-z_$]/.test(s)) s = "_" + s;
+            return s;
+        };
+        const id = sanitize(name);
+        const asyncKw = body.async ? "async " : "";
+        const genMark = body.generator ? "*" : "";
+        const src = `return ${asyncKw}function${genMark} ${id}(${params}) {\n${inner}\n}`;
+        // Use a plain Function wrapper to evaluate the named function expression.
+        const builtNamed = (new Function(src)()) as AnyFunction;
+        // Expose the intended display name even if sanitization changed the identifier.
+        try { (builtNamed as any).displayName = name; } catch { /* noop */ }
+        return builtNamed;
+    }
+
+    const built = func(params, inner) as AnyFunction;
+    return built;
+}
+
+export function constructFunction(bodyStr: string, name?: string) {
+    const { body } = parseJavaScript(bodyStr);
+    if (body.type === "Program") {
+        if (body.body.length !== 1) {
+            throw new Error(`Expected a single function, but found ${body.body.length} statements`);
+        }
+        return _constructFunction(body.body[0], bodyStr, name);
+    }
+    return _constructFunction(body, bodyStr, name);
+}
+
+export const html2notebook = (html: string): Notebook => deserialize(html);
+export const notebook2html = (notebook: Notebook): string => serialize(notebook);

package/src/util.ts

@@ -1,65 +1,8 @@
+import { type Notebook, type Cell, toNotebook, toCell } from "./kit/index.ts";
+
 import type { ohq } from "./observable-shim.ts";
 import { parseCell, splitModule } from "./observable-shim.ts";
 
-const FuncTypes = {
-    functionType: Object.getPrototypeOf(function () { }).constructor,
-    asyncFunctionType: Object.getPrototypeOf(async function () { }).constructor,
-    generatorFunctionType: Object.getPrototypeOf(function* () { }).constructor,
-    asyncGeneratorFunctionType: Object.getPrototypeOf(async function* () { }).constructor
-};
-
-function funcType(async: boolean = false, generator: boolean = false) {
-    if (!async && !generator) return FuncTypes.functionType;
-    if (async && !generator) return FuncTypes.asyncFunctionType;
-    if (!async && generator) return FuncTypes.generatorFunctionType;
-    return FuncTypes.asyncGeneratorFunctionType;
-}
-
-interface Ref {
-    start: number,
-    end: number,
-    newText: string
-}
-
-export interface Refs {
-    inputs: string[];
-    args: string[];
-    patches: Ref[];
-}
-
-export function createFunction(refs: Refs, async = false, generator = false, blockStatement = false, body?: string) {
-    if (body === undefined) {
-        return undefined;
-    }
-
-    refs.patches.sort((l, r) => r.start - l.start);
-    refs.patches.forEach(r => {
-        body = body!.substring(0, r.start) + r.newText + body!.substring(r.end);
-    });
-    return new (funcType(async, generator))(...refs.args, blockStatement ?
-        body.substring(1, body.length - 1).trim() :
-        `return (\n${body}\n);`);
-}
-
-function join(baseURL: string, relativeURL: string) {
-    return relativeURL
-        ? baseURL.replace(/\/+$/, "") + "/" + relativeURL.replace(/^\/+/, "")
-        : baseURL;
-}
-
-export const isRelativePath = (path: string) => path[0] === ".";
-export const fixRelativeUrl = (path: string, basePath: string) => {
-    if (isRelativePath(path)) {
-        return join(basePath, path);
-    }
-    return path;
-};
-
-//  Hide "import" from bundlers as they have a habit of replacing "import" with "require"
-export async function obfuscatedImport(url: string) {
-    return new FuncTypes.asyncFunctionType("url", "return import(url)")(url);
-}
-
 interface ParsedOJS {
     ojs: string;
     offset: number;
@@ -141,6 +84,17 @@ export function ojs2notebook(ojs: string): ohq.Notebook {
     } as ohq.Notebook;
 }
 
+export function ojs2notebookKit(ojs: string): Notebook {
+    const cells: Cell[] = splitModule(ojs).map((cell, idx) => {
+        return toCell({
+            id: idx,
+            mode: "ojs",
+            value: cell.text
+        });
+    });
+    return toNotebook({ cells });
+}
+
 export function omd2notebook(omd: string): ohq.Notebook {
     const cells = splitOmd(omd);
     return {
@@ -157,6 +111,28 @@ export function omd2notebook(omd: string): ohq.Notebook {
     } as ohq.Notebook;
 }
 
+export function omd2notebookKit(omd: string): Notebook {
+    const cells: Cell[] = [];
+    splitOmd(omd).forEach((cell) => {
+        if (!cell.inlineMD) {
+            splitModule(cell.ojs).forEach((subCell) => {
+                cells.push(toCell({
+                    id: cells.length + 1,
+                    mode: "ojs",
+                    value: subCell.text
+                }));
+            });
+        } else {
+            cells.push(toCell({
+                id: cells.length + 1,
+                mode: "md",
+                value: cell.ojs
+            }));
+        }
+    });
+    return toNotebook({ cells });
+}
+
 export function fetchEx(url: string, proxyPrefix = "https://api.codetabs.com/v1/proxy/?quest=", proxyPostfix = "") {
     const matches = url.match(/^(?:https?:\/\/)?(?:[^@\n]+@)?(?:www\.)?([^:\/\n?]+)/img);
     if (!matches || matches.length === 0) {
@@ -176,4 +152,4 @@ export function download(impUrl: string, proxyPrefix?: string, proxyPostfix?: st
     return fetchEx(impUrl.replace(`https://observablehq.com/${isShared ? "d/" : ""}`, "https://api.observablehq.com/document/"), proxyPrefix, proxyPostfix)
         .then(r => r.json())
         ;
-}
+}

package/types/compiler.d.ts

@@ -1,3 +1,4 @@
+import { type Notebook, type Definition } from "./kit/index.ts";
 import { ohq } from "./observable-shim.ts";
 import { Writer } from "./writer.ts";
 export type InspectorFactoryEx = (name: string | undefined, id: string | number) => Inspector;
@@ -72,40 +73,11 @@ export declare function notebook(_files?: ohq.File[], _cells?: CellFunc[], { bas
     write(w: Writer): void;
     toString(w?: Writer): string;
 };
-export declare function compile(notebookOrOjs: ohq.Notebook | string, { baseUrl, importMode }?: CompileOptions): Promise<{
-    (runtime: ohq.Runtime, inspector?: InspectorFactoryEx): ohq.Module;
-    fileAttachments: Map<string, any>;
-    cells: Map<string | number, {
-        (runtime: ohq.Runtime, main: ohq.Module, inspector?: InspectorFactoryEx): void;
-        id: string | number;
-        modules: {
-            (runtime: ohq.Runtime, main: ohq.Module, inspector?: InspectorFactoryEx): ohq.Module;
-            importVariables: {
-                (main: ohq.Module, otherModule: ohq.Module): void;
-                delete(): void;
-            }[];
-            variables: {
-                (module: ohq.Module, inspector?: InspectorFactoryEx): ohq.Variable;
-                delete(): void;
-                write(w: Writer): void;
-            }[];
-            delete(): void;
-            write(w: Writer): void;
-        }[];
-        variables: {
-            (module: ohq.Module, inspector?: InspectorFactoryEx): ohq.Variable;
-            delete(): void;
-            write(w: Writer): void;
-        }[];
-        delete(): void;
-        write(w: Writer): void;
-    }>;
-    set(n: ohq.Node): Promise<CellFunc>;
-    get(id: string | number): CellFunc | undefined;
-    delete(id: string | number): boolean;
-    clear(): void;
-    write(w: Writer): void;
-    toString(w?: Writer): string;
-}>;
-export type compileFunc = Awaited<ReturnType<typeof compile>>;
+type NotebookFunc = ReturnType<typeof notebook>;
+export declare function isNotebookKit(value: any): value is Notebook;
+export declare function isOhqNotebook(value: any): value is ohq.Notebook;
+export declare function compile(notebookOrOjs: Notebook): Promise<Definition[]>;
+export declare function compile(notebookOrOjs: ohq.Notebook, options?: CompileOptions): Promise<NotebookFunc>;
+export declare function compile(notebookOrOjs: string, options?: CompileOptions): Promise<NotebookFunc>;
+export type CompileFunc = Awaited<ReturnType<typeof compile>>;
 export {};

package/types/index.d.ts

@@ -1,5 +1,5 @@
 export type { ohq } from "./observable-shim.ts";
 export * from "./compiler.ts";
-export { ojs2notebook, omd2notebook, download } from "./util.ts";
+export { ojs2notebook, omd2notebook, omd2notebookKit, ojs2notebookKit, download } from "./util.ts";
+export { compile as compileKit, html2notebook, notebook2html } from "./kit/index.ts";
 export * from "./writer.ts";
-import "../src/index.css";

package/types/kit/compiler.d.ts

@@ -0,0 +1,6 @@
+import { type Notebook } from "@observablehq/notebook-kit";
+import { type Definition } from "@observablehq/notebook-kit/runtime";
+export interface CompileKitOptions {
+    inline?: boolean;
+}
+export declare function compile(notebook: Notebook, options?: CompileKitOptions): Definition[];

package/types/kit/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./util.ts";
+export * from "./compiler.ts";
+export * from "./util.ts";

package/types/kit/runtime.d.ts

@@ -0,0 +1,13 @@
+import { type DefineState, NotebookRuntime as NotebookRuntimeBase } from "@observablehq/notebook-kit/runtime";
+import { type Definition } from "./index.ts";
+import "@observablehq/notebook-kit/index.css";
+import "@observablehq/notebook-kit/theme-air.css";
+export declare class NotebookRuntime extends NotebookRuntimeBase {
+    stateById: Map<number, DefineState>;
+    constructor();
+    has(cellId: number): boolean;
+    add(cellId: number, definition: Definition, placeholderDiv: HTMLDivElement): Promise<void>;
+    remove(cellId: number): Promise<void>;
+    removeAll(): Promise<void>;
+    render(definitions: Definition[], target: HTMLElement): void;
+}

package/types/kit/util.d.ts

@@ -0,0 +1,52 @@
+import { type Notebook, type Cell, toNotebook, toCell } from "@observablehq/notebook-kit";
+import { type Definition } from "@observablehq/notebook-kit/runtime";
+export type { Notebook, Cell, Definition };
+export { toNotebook, toCell };
+export type RegularFunction = (...args: any[]) => any;
+interface RegularFunctionConstructor {
+    new (...args: string[]): RegularFunction;
+    (...args: string[]): RegularFunction;
+    readonly prototype: RegularFunction;
+}
+export type AsyncFunction = (...args: any[]) => Promise<any>;
+interface AsyncFunctionConstructor {
+    new (...args: string[]): AsyncFunction;
+    (...args: string[]): AsyncFunction;
+    readonly prototype: AsyncFunction;
+}
+export type GeneratorFunction = (...args: any[]) => Generator<any, any, any>;
+interface GeneratorFunctionConstructor {
+    new (...args: string[]): GeneratorFunction;
+    (...args: string[]): GeneratorFunction;
+    readonly prototype: GeneratorFunction;
+}
+export type AsyncGeneratorFunction = (...args: any[]) => AsyncGenerator<any, any, any>;
+interface AsyncGeneratorFunctionConstructor {
+    new (...args: string[]): AsyncGeneratorFunction;
+    (...args: string[]): AsyncGeneratorFunction;
+    readonly prototype: AsyncGeneratorFunction;
+}
+export type AnyFunction = RegularFunction | AsyncFunction | GeneratorFunction | AsyncGeneratorFunction;
+export declare const FunctionConstructors: {
+    regular: RegularFunctionConstructor;
+    async: AsyncFunctionConstructor;
+    generator: GeneratorFunctionConstructor;
+    asyncGenerator: AsyncGeneratorFunctionConstructor;
+};
+interface Ref {
+    start: number;
+    end: number;
+    newText: string;
+}
+export interface Refs {
+    inputs: string[];
+    args: string[];
+    patches: Ref[];
+}
+export declare function createFunction(refs: Refs, async?: boolean, generator?: boolean, blockStatement?: boolean, body?: string): RegularFunction | undefined;
+export declare const isRelativePath: (path: string) => boolean;
+export declare const fixRelativeUrl: (path: string, basePath: string) => string;
+export declare function obfuscatedImport(url: string): Promise<any>;
+export declare function constructFunction(bodyStr: string, name?: string): AnyFunction;
+export declare const html2notebook: (html: string) => Notebook;
+export declare const notebook2html: (notebook: Notebook) => string;

package/types/util.d.ts

@@ -1,18 +1,5 @@
+import { type Notebook } from "./kit/index.ts";
 import type { ohq } from "./observable-shim.ts";
-interface Ref {
-    start: number;
-    end: number;
-    newText: string;
-}
-export interface Refs {
-    inputs: string[];
-    args: string[];
-    patches: Ref[];
-}
-export declare function createFunction(refs: Refs, async?: boolean, generator?: boolean, blockStatement?: boolean, body?: string): any;
-export declare const isRelativePath: (path: string) => boolean;
-export declare const fixRelativeUrl: (path: string, basePath: string) => string;
-export declare function obfuscatedImport(url: string): Promise<any>;
 interface ParsedOJS {
     ojs: string;
     offset: number;
@@ -23,7 +10,9 @@ interface ParsedOJS {
 export declare function encodeBacktick(str: string): string;
 export declare function notebook2ojs(_: string): ParsedOJS[];
 export declare function ojs2notebook(ojs: string): ohq.Notebook;
+export declare function ojs2notebookKit(ojs: string): Notebook;
 export declare function omd2notebook(omd: string): ohq.Notebook;
+export declare function omd2notebookKit(omd: string): Notebook;
 export declare function fetchEx(url: string, proxyPrefix?: string, proxyPostfix?: string): Promise<Response>;
 export declare function download(impUrl: string, proxyPrefix?: string, proxyPostfix?: string): Promise<ohq.Notebook>;
 export {};