@hpcc-js/observablehq-compiler 1.1.0

package/src/__tests__/index.ts

@@ -0,0 +1,13 @@
+// @ts-ignore
+import fetch, { Blob, blobFrom, blobFromSync, File, fileFrom, fileFromSync, FormData, Headers, Request, Response } from "node-fetch";
+if (!globalThis.fetch) {
+    // @ts-ignore
+    globalThis.fetch = fetch;
+    // @ts-ignore
+    globalThis.Headers = Headers;
+    // @ts-ignore
+    globalThis.Request = Request;
+    // @ts-ignore
+    globalThis.Response = Response;
+}
+export * from "./node";

package/src/__tests__/node.ts

@@ -0,0 +1,177 @@
+import { Library, Runtime, Inspector } from "@observablehq/runtime";
+import { expect } from "chai";
+import { compile, download, ohq, ojs2notebook } from "../index";
+import { Writer } from "../writer";
+import { fa } from "./File Attachments";
+import { imports } from "./Introduction to Imports";
+import { timechart } from "./Observable TimeChart";
+
+const placeholder = globalThis?.document?.getElementById("placeholder");
+
+const ojs = `
+42;
+a = 1
+b = 2
+c = a + b
+d = {
+  yield 1;
+  yield 2;
+  yield 3;
+}
+
+viewof e = {
+  let output = {};
+  let listeners = [];
+  output.value = 10;
+  output.addEventListener = (listener) => listeners.push(listener);;
+  output.removeEventListener = (listener) => {
+    listeners = listeners.filter(l => l !== listener);
+  };
+  return output;
+}
+
+tenTimes = {
+  for (let i = 0; i < 10; i++) {
+    yield Promises.delay(100);
+  }
+}
+
+f = {
+  tenTimes;
+  return (this || 0) + 1;
+}
+`;
+
+describe("ojs", function () {
+  it("quickstart", async function () {
+    this.timeout(10000);
+
+    const notebook = await download("https://observablehq.com/@observablehq/summary-table");
+    const compiledNB = await compile(notebook);
+
+    const library = new Library();
+    const runtime = new Runtime(library);
+
+    compiledNB(runtime, name => {
+      const div = globalThis?.document.createElement("div");
+      placeholder.appendChild(div);
+      return new Inspector(div);
+    });
+  });
+
+  it.only("simple", async function () {
+    this.timeout(10000);
+
+    const notebook = ojs2notebook(ojs);
+    const define = await compile(notebook);
+
+    const library = new Library();
+    const runtime = new Runtime(library);
+    const main: ohq.Module = define(runtime, name => {
+      if (placeholder) {
+        const div = globalThis?.document.createElement("div");
+        placeholder.appendChild(div);
+        return new Inspector(div);
+      }
+      return {
+        pending() { },
+        fulfilled(value) { console.info("fulfilled", name, value); },
+        rejected(error) { console.error("rejected", name, error); },
+      };
+    });
+
+    expect(await main.value("d")).to.equal(1);
+    expect(await main.value("d")).to.equal(2);
+    expect(await main.value("d")).to.equal(3);
+    expect(await main.value("d")).to.equal(3);
+    expect(await main.value("d")).to.equal(3);
+
+    expect(await main.value("c")).to.equal(3);
+    expect(await main.value("a")).to.equal(1);
+    expect(await main.value("b")).to.equal(2);
+
+    expect(await main.value("e")).to.equal(10);
+    const viewOfE = await main.value("viewof e");
+    expect(viewOfE).to.have.property("value");
+    expect(viewOfE).to.have.property("addEventListener");
+    expect(viewOfE).to.have.property("removeEventListener");
+
+    await main.value("tenTimes");
+
+    for (const cell of define.cells) {
+      cell.dispose();
+      break;
+    }
+  });
+
+  it("compiler", async function () {
+    const writer = new Writer();
+    const define = await compile(imports as any);
+    define.write(writer);
+    console.info(writer.toString());
+  });
+
+  it("File Attachements", async function () {
+
+    const define = await compile(fa as any);
+
+    const library = new Library();
+    const runtime = new Runtime(library);
+    /*const main: ohq.Module =*/ define(runtime, name => {
+      if (placeholder) {
+        const div = globalThis?.document.createElement("div");
+        placeholder.appendChild(div);
+        return new Inspector(div);
+      }
+      return {
+        pending() { console.info("pending", name); },
+        fulfilled(value) { console.info("fulfilled", name, value); },
+        rejected(error) { console.error("rejected", name, error); },
+      };
+    });
+
+  });
+
+  it("Introduction to Imports", async function () {
+
+    const define = await compile(imports as any);
+
+    const library = new Library();
+    const runtime = new Runtime(library);
+    /*const main: ohq.Module =*/ define(runtime, name => {
+      if (placeholder) {
+        const div = globalThis?.document.createElement("div");
+        placeholder.appendChild(div);
+        return new Inspector(div);
+      }
+      return {
+        pending() { console.info("pending", name); },
+        fulfilled(value) { console.info("fulfilled", name, value); },
+        rejected(error) { console.error("rejected", name, error); },
+      };
+    });
+
+  });
+
+  it("Observable TimeChart", async function () {
+
+    const define2 = await compile(timechart as any);
+
+    const library = new Library();
+    const runtime = new Runtime(library);
+    /*const main: ohq.Module =*/ define2(runtime, name => {
+      if (placeholder) {
+        const div = globalThis?.document.createElement("div");
+        placeholder.appendChild(div);
+        return new Inspector(div);
+      }
+      return {
+        pending() { console.info("pending", name); },
+        fulfilled(value) { console.info("fulfilled", name, value); },
+        rejected(error) { console.error("rejected", name, error); },
+      };
+    });
+
+  });
+});
+

package/src/__tests__/tsconfig.json

@@ -0,0 +1,21 @@
+{
+    "extends": "../tsconfig.settings.json",
+    "compilerOptions": {
+        "rootDir": "./src",
+        "outDir": "./lib-umd",
+        "declarationDir": "./types",
+        "target": "ES2018",
+        "noImplicitThis": true,
+        "resolveJsonModule": true,
+        "lib": [
+            "dom",
+            "ES2018"
+        ],
+        "types": [
+            "mocha"
+        ]
+    },
+    "include": [
+        "./src/**/*"
+    ]
+}

package/src/compiler.md

@@ -0,0 +1,234 @@
+# Compiler
+
+Observable HQ Notebook Compiler and Interpreter
+
+##  Minimal JSON Notebook
+
+While the compiler supports and persists the entire Observable HQ Notebook (v3), it only needs a minimal subset to actually function:
+
+```ts
+interface Notebook {
+    files: File[];
+    nodes: Node[];
+}
+
+interface Node {
+    id: number | string;
+    mode: "js" | "md";
+    value: string;
+}
+
+interface File {
+    name: string;
+    url: string;
+    mime_type?: string;
+}
+```  
+
+---
+
+<a name="compile" href="#compile">#</a> **compile**(_notebook_) => Promise\<function\> · [<>](https://github.com/hpcc-systems/Visualization/blob/trunk/packages/observablehq/compiler/src/compiler.ts "Source")
+
+* _notebook_:  JSON Notebook, see [utilities](./util) for examples on how to fetch or create a notebook.
+* _returns_: Returns a Promise to a "define" function that is compatible with the ["@observablehq/runtime"](https://github.com/observablehq/runtime) and ["@observablehq/inspector"](https://github.com/observablehq/inspector)
+
+## Hello World Example
+
+First take a simple hello world notebook 
+
+```js
+const notebook = {
+    files: [],
+    nodes: [
+        {
+            id: 1,
+            value: "md`# ${h} ${w}`",
+            mode: "js"
+        },
+        {
+            id: 1,
+            value: "h = \"Hello\"",
+            mode: "js"
+        },
+        {
+            id: 2,
+            value: "w = \"World\"",
+            mode: "js"
+        }
+    ]
+}
+```
+
+Then import the compiler and invoke it:
+
+```js
+import { compile } from "@hpcc-js/observablehq-compiler";
+
+const compiledNotebook = await compile(notebook);
+```
+
+To render it to a web page, simply follow the same steps as if you had downloaded a compiled version from ObservableHQ site:
+
+```js
+
+import { Library, Runtime, Inspector } from "https://cdn.skypack.dev/@observablehq/runtime";
+
+const placeholder = document.getElementById("placeholder");
+
+const library = new Library();
+const runtime = new Runtime(library);
+
+compiledNotebook(runtime, name => {
+    const div = document.createElement("div");
+    placeholder.appendChild(div);
+    return new Inspector(div);
+});
+
+```
+
+Putting it all together:
+
+<ClientOnly>
+  <hpcc-vitepress preview_height_ratio=0.4 style="width:100%;height:400px">
+    <div id="placeholder" style="height:100%;overflow-y:scroll">
+    </div>
+    <script type="module">
+      import { Library, Runtime, Inspector } from "https://cdn.skypack.dev/@observablehq/runtime";
+      import { compile } from "@hpcc-js/observablehq-compiler";
+
+      const placeholder = document.getElementById("placeholder");
+
+      const notebook = {
+          files: [],
+          nodes: [
+              {
+                  id: 1,
+                  value: "md`# ${h} ${w}`",
+                  mode: "js"
+              },
+              {
+                  id: 1,
+                  value: "h = \"Hello\"",
+                  mode: "js"
+              },
+              {
+                  id: 2,
+                  value: "w = \"World\"",
+                  mode: "js"
+              }
+          ]
+      }
+
+      const compiledNotebook = await compile(notebook);
+
+      const library = new Library();
+      const runtime = new Runtime(library);
+      compiledNotebook(runtime, name => {
+          const div = document.createElement("div");
+          placeholder.appendChild(div);
+          return new Inspector(div);
+      });
+    </script>
+  </hpcc-vitepress>
+</ClientOnly>
+
+---
+
+To output the generated code simply call `toString` on the compiled function:
+
+<ClientOnly>
+  <hpcc-vitepress style="width:100%;height:600px">
+    <hpcc-codemirror id="placeholder" mode="javascript" theme="dark" style="width:100%;height:100%">
+    </hpcc-codemirror>
+    <script type="module">
+      import "@hpcc-js/wc-editor";
+      import { compile } from "@hpcc-js/observablehq-compiler";
+
+      const notebook = {
+          files: [],
+          nodes: [
+              {
+                  id: 1,
+                  value: "md`# ${h} ${w}`",
+                  mode: "js"
+              },
+              {
+                  id: 1,
+                  value: "h = \"Hello\"",
+                  mode: "js"
+              },
+              {
+                  id: 2,
+                  value: "w = \"World\"",
+                  mode: "js"
+              }
+          ]
+      }
+
+      const compiledNotebook = await compile(notebook);
+      const placeholder = document.getElementById("placeholder");
+      placeholder.text = compiledNotebook.toString();
+    </script>
+  </hpcc-vitepress>
+</ClientOnly>
+
+---
+
+## MoreExamples
+
+* [@observablehq/plot](https://observablehq.com/@observablehq/plot)
+
+<ClientOnly>
+  <hpcc-vitepress style="width:100%;height:600px">
+    <div id="placeholder" style="height:400px;overflow-y:scroll">
+    </div>
+    <script type="module">
+      import { Library, Runtime, Inspector } from "https://cdn.skypack.dev/@observablehq/runtime";
+      import { download, compile } from "@hpcc-js/observablehq-compiler";
+
+      const notebookUrl = "https://observablehq.com/@observablehq/plot";
+      const placeholder = document.getElementById("placeholder");
+
+      const notebook = await download(notebookUrl);
+      const compiledNB = await compile(notebook);
+
+      const library = new Library();
+      const runtime = new Runtime(library);
+      compiledNB(runtime, name => {
+          const div = document.createElement("div");
+          placeholder.appendChild(div);
+          return new Inspector(div);
+      });
+    </script>
+  </hpcc-vitepress>
+</ClientOnly>
+
+---
+
+* [@mbostock/fullscreen-canvas](https://observablehq.com/@mbostock/fullscreen-canvas)
+
+<ClientOnly>
+  <hpcc-vitepress style="width:100%;height:600px">
+    <div id="placeholder" style="height:400px;overflow-y:scroll">
+    </div>
+    <script type="module">
+      import { Library, Runtime, Inspector } from "https://cdn.skypack.dev/@observablehq/runtime";
+      import { download, compile } from "@hpcc-js/observablehq-compiler";
+
+      const notebookUrl = "https://observablehq.com/@mbostock/fullscreen-canvas";
+      const placeholder = document.getElementById("placeholder");
+
+      const notebook = await download(notebookUrl);
+      const compiledNB = await compile(notebook);
+
+      const library = new Library();
+      const runtime = new Runtime(library);
+      compiledNB(runtime, name => {
+          const div = document.createElement("div");
+          placeholder.appendChild(div);
+          return new Inspector(div);
+      });
+    </script>
+  </hpcc-vitepress>
+</ClientOnly>
+

package/src/compiler.ts

@@ -0,0 +1,264 @@
+import type { ohq } from "@hpcc-js/observable-shim";
+
+import { endsWith, join } from "@hpcc-js/util";
+import { parseCell, ParsedImportCell } from "./cst";
+import { Writer } from "./writer";
+import { encodeBacktick, fetchEx, obfuscatedImport, ojs2notebook, omd2notebook } from "./util";
+
+interface ImportDefine {
+    (runtime: ohq.Runtime, inspector?: ohq.InspectorFactory): ohq.Module;
+    dispose: () => void;
+    write: (w: Writer) => void;
+}
+
+async function importFile(relativePath: string, baseUrl: string) {
+    const path = join(baseUrl, relativePath);
+    const content = await fetchEx(path).then(r => r.text());
+    let notebook: ohq.Notebook;
+    if (endsWith(relativePath, ".ojsnb")) {
+        notebook = JSON.parse(content);
+    } else if (endsWith(relativePath, ".ojs")) {
+        notebook = ojs2notebook(content);
+    } else if (endsWith(relativePath, ".omd")) {
+        notebook = omd2notebook(content);
+    }
+    const retVal: ImportDefine = compile(notebook, baseUrl) as any;
+    retVal.dispose = () => { };
+    retVal.write = (w: Writer) => {
+        w.import(path);
+    };
+    return retVal;
+}
+
+// @ts-ignore - use precompiled notebook from observable
+async function importCompiledNotebook(partial: string) {
+    const url = `https://api.observablehq.com/${partial[0] === "@" ? partial : `d/${partial}`}.js?v=3`;
+    let impMod = {
+        default: function (runtime: ohq.Runtime, inspector?: ohq.InspectorFactory): ohq.Module {
+            return undefined;
+        } as any
+    };
+    try {
+        impMod = await obfuscatedImport(url);
+    } catch (e) {
+    }
+    const retVal: ImportDefine = impMod.default;
+    retVal.dispose = () => { };
+    retVal.write = (w: Writer) => {
+        w.import(url);
+    };
+    return retVal;
+}
+
+// @ts-ignore - recursive notebook parsing and compiling
+async function importNotebook(partial: string) {
+    const url = `https://api.observablehq.com/document/${partial}`;
+    const notebook = fetchEx(url)
+        .then(r => {
+            return r.json();
+        }).catch(e => {
+            console.error(url);
+            console.error(e);
+        });
+    const retVal: ImportDefine = compile(await notebook) as any;
+    retVal.dispose = () => { };
+    retVal.write = (w: Writer) => {
+        w.import(url);
+    };
+    return retVal;
+}
+
+function createVariable(inspect: boolean, name?: string, inputs?: string[], definition?: any, inline = false) {
+
+    let i: ohq.Inspector;
+    let v: ohq.Variable;
+
+    const retVal = (module: ohq.Module, inspector?: ohq.InspectorFactory) => {
+        i = inspect ? inspector(name) : undefined;
+        v = module.variable(i);
+        if (arguments.length > 1) {
+            try {
+                v.define(name, inputs, definition);
+            } catch (e: any) {
+                console.error(e?.message);
+            }
+        }
+        return v;
+    };
+    retVal.dispose = () => {
+        try {
+            i?._node?.remove();
+        } catch (e) {
+        }
+        i = undefined;
+        try {
+            v?.delete();
+        } catch (e) {
+        }
+        v = undefined;
+    };
+    retVal.write = (w: Writer) => {
+        if (inline) {
+            w.define({ id: name, inputs, func: definition }, inspect, true);
+        } else {
+            const id = w.function({ id: name, func: definition });
+            w.define({ id: name, inputs, func: definition }, inspect, false, id);
+        }
+    };
+    return retVal;
+}
+type VariableFunc = ReturnType<typeof createVariable>;
+
+function createImportVariable(name?: string, alias?: string) {
+
+    let v: ohq.Variable;
+
+    const retVal = (main: ohq.Module, otherModule: ohq.Module) => {
+        v = main.variable();
+        v.import(name, alias, otherModule);
+    };
+
+    retVal.dispose = () => {
+        v?.delete();
+    };
+    return retVal;
+}
+type ImportVariableFunc = ReturnType<typeof createImportVariable>;
+
+async function createModule(parsed: ParsedImportCell, text: string, baseUrl: string) {
+    const otherModule = [".", "/"].indexOf(parsed.src[0]) === 0 ?
+        await importFile(parsed.src, baseUrl) :
+        await importCompiledNotebook(parsed.src);
+
+    const importVariables: ImportVariableFunc[] = [];
+    const variables: VariableFunc[] = [];
+    parsed.specifiers.forEach(spec => {
+        const viewof = spec.view ? "viewof " : "";
+        importVariables.push(createImportVariable(viewof + spec.name, viewof + spec.alias));
+        if (spec.view) {
+            importVariables.push(createImportVariable(spec.name, spec.alias));
+        }
+    });
+    variables.push(createVariable(true, undefined, ["md"], md => {
+        return md`\`\`\`JavaScript
+${text}
+\`\`\``;
+    }));
+
+    const retVal = (runtime: ohq.Runtime, main: ohq.Module, inspector?: ohq.InspectorFactory) => {
+
+        let mod = runtime.module(otherModule);
+        if (parsed.injections.length) {
+            mod = mod.derive(parsed.injections, main);
+        }
+        variables.forEach(v => v(main, inspector));
+        importVariables.forEach(v => v(main, mod));
+        return mod;
+    };
+    retVal.importVariables = importVariables;
+    retVal.variables = variables;
+    retVal.dispose = () => {
+        importVariables.forEach(v => v.dispose());
+        variables.forEach(v => v.dispose());
+        otherModule.dispose();
+    };
+    retVal.write = (w: Writer) => {
+        otherModule.write(w);
+        w.importDefine(parsed);
+    };
+    return retVal;
+}
+type ModuleFunc = Awaited<ReturnType<typeof createModule>>;
+
+async function createCell(node: ohq.Node, baseUrl: string) {
+    const modules: ModuleFunc[] = [];
+    const variables: VariableFunc[] = [];
+    try {
+        const text = node.mode && node.mode !== "js" ? `${node.mode}\`${encodeBacktick(node.value)}\`` : node.value;
+        const parsed = parseCell(text);
+        switch (parsed.type) {
+            case "import":
+                modules.push(await createModule(parsed, text, baseUrl));
+                break;
+            case "viewof":
+                variables.push(createVariable(true, parsed.variable.id, parsed.variable.inputs, parsed.variable.func));
+                variables.push(createVariable(false, parsed.variableValue.id, parsed.variableValue.inputs, parsed.variableValue.func, true));
+                break;
+            case "mutable":
+                variables.push(createVariable(false, parsed.initial.id, parsed.initial.inputs, parsed.initial.func));
+                variables.push(createVariable(false, parsed.variable.id, parsed.variable.inputs, parsed.variable.func));
+                variables.push(createVariable(true, parsed.variableValue.id, parsed.variableValue.inputs, parsed.variableValue.func, true));
+                break;
+            case "variable":
+                variables.push(createVariable(true, parsed.id, parsed.inputs, parsed.func));
+                break;
+        }
+    } catch (e) {
+    }
+
+    const retVal = (runtime: ohq.Runtime, main: ohq.Module, inspector?: ohq.InspectorFactory) => {
+        modules.forEach(imp => imp(runtime, main, inspector));
+        variables.forEach(v => v(main, inspector));
+    };
+    retVal.modules = modules;
+    retVal.variables = variables;
+    retVal.dispose = () => {
+        variables.forEach(v => v.dispose());
+        modules.forEach(mod => mod.dispose());
+    };
+    retVal.write = (w: Writer) => {
+        modules.forEach(imp => imp.write(w));
+        variables.forEach(v => v.write(w));
+    };
+    return retVal;
+}
+export type CellFunc = Awaited<ReturnType<typeof createCell>>;
+
+function createFile(file: ohq.File): [string, any] {
+    function toString() { return globalThis.url; }
+    return [file.name, { url: new URL(file.url), mimeType: file.mime_type, toString }];
+}
+export type FileFunc = ReturnType<typeof createFile>;
+
+export async function compile(notebook: ohq.Notebook, baseUrl: string = ".") {
+
+    const files = notebook.files.map(f => createFile(f));
+    const fileAttachments = new Map<string, any>(files);
+    let cells: CellFunc[] = await Promise.all(notebook.nodes.map(n => createCell(n, baseUrl)));
+
+    const retVal = (runtime: ohq.Runtime, inspector?: ohq.InspectorFactory): ohq.Module => {
+        const main = runtime.module();
+        main.builtin("FileAttachment", runtime.fileAttachments(name => {
+            return fileAttachments.get(name) ?? { url: new URL(name), mimeType: null };
+        }));
+        cells.forEach(cell => {
+            cell(runtime, main, inspector);
+        });
+        return main;
+    };
+    retVal.fileAttachments = fileAttachments;
+    retVal.cells = cells;
+    retVal.appendCell = async (n: ohq.Node, baseUrl: string = ".") => {
+        const cell = await createCell(n, baseUrl);
+        cells.push(cell);
+        return cell;
+    };
+    retVal.disposeCell = async (cell: CellFunc) => {
+        cells = cells.filter(c => c !== cell);
+        cell.dispose();
+    };
+    retVal.dispose = () => {
+        cells.forEach(cell => cell.dispose());
+        cells = [];
+    };
+    retVal.write = (w: Writer) => {
+        w.files(notebook.files);
+        cells.forEach(cell => cell.write(w));
+    };
+    retVal.toString = (w = new Writer()) => {
+        retVal.write(w);
+        return w.toString().trim();
+    };
+    return retVal;
+}
+export type compileFunc = Awaited<ReturnType<typeof compile>>;