@hpcc-js/observablehq-compiler 1.1.0

package/src/util.md

@@ -0,0 +1,113 @@
+# Utilities
+
+Various utilities and helpers.
+
+---
+
+<a name="download" href="#download">#</a> **download**(_impUrl_) => Promise\<ohq.Module\> · [<>](https://github.com/hpcc-systems/Visualization/blob/trunk/packages/observablehq/compiler/src/util.ts "Source")
+
+* _impUrl_:  Full url to Obaservable HQ notebook as a string.
+* _returns_: Returns a Promise of a notebook (a JSON object).
+
+Downloads a notebook directly from [Observable HQ](https://observablehq.com/) as a JSON object.  The following example downloads the [@observablehq/plot](https://observablehq.com/@observablehq/plot) notebook as JSON:
+
+<ClientOnly>
+  <hpcc-vitepress style="width:100%;height:600px">
+    <hpcc-codemirror id="placeholder" mode="json" theme="dark" style="width:100%;height:100%">
+    </hpcc-codemirror>
+    <script type="module">
+      import "@hpcc-js/wc-editor";
+      import { download } from "@hpcc-js/observablehq-compiler";
+
+      const notebook = await download("https://observablehq.com/@observablehq/plot");
+      const placeholder = document.getElementById("placeholder");
+      placeholder.text = JSON.stringify(notebook, undefined, 4);
+    </script>
+  </hpcc-vitepress>
+</ClientOnly>
+
+---
+
+<a name="ojs2notebook" href="#ojs2notebook">#</a> **ojs2notebook**(_ojs_) => ohq.Module · [<>](https://github.com/hpcc-systems/Visualization/blob/trunk/packages/observablehq/compiler/src/util.ts "Source")
+
+* _ojs_:  String containing Observable JavaScript.
+* _returns_: Returns the notebook as a JSON object.
+
+Transforms Observable JavaScript to a JSON notebook.
+
+<ClientOnly>
+  <hpcc-vitepress style="width:100%;height:600px">
+    <hpcc-codemirror id="placeholder" mode="json" theme="dark" style="width:100%;height:100%">
+    </hpcc-codemirror>
+    <script type="module">
+      import "@hpcc-js/wc-editor";
+      import { ojs2notebook } from "@hpcc-js/observablehq-compiler";
+
+      const notebook = ojs2notebook(`
+        md\`# Simple Notebook\`
+        a = 1
+        b = 2
+        c = a + b
+        d = {
+          yield 1;
+          yield 2;
+          yield 3;
+        }
+
+        viewof e = {
+          let output = {};
+          let listeners = [];
+          output.value = 10;
+          output.addEventListener = (listener) => listeners.push(listener);;
+          output.removeEventListener = (listener) => {
+            listeners = listeners.filter(l => l !== listener);
+          };
+          return output;
+        }
+      `);
+      const placeholder = document.getElementById("placeholder");
+      placeholder.text = JSON.stringify(notebook, undefined, 4);
+    </script>
+  </hpcc-vitepress>
+</ClientOnly>
+
+<a name="omd2notebook" href="#omd2notebook">#</a> **omd2notebook**(_omd_) => ohq.Module · [<>](https://github.com/hpcc-systems/Visualization/blob/trunk/packages/observablehq/compiler/src/util.ts "Source")
+
+* _omd_:  String containing Observable Markdown.
+* _returns_: Returns the notebook as a JSON object.
+
+Transforms Observable Markdown to a JSON notebook.
+
+<ClientOnly>
+  <hpcc-vitepress style="width:100%;height:600px">
+    <hpcc-codemirror id="placeholder" mode="json" theme="dark" style="width:100%;height:100%">
+    </hpcc-codemirror>
+    <script type="module">
+      import "@hpcc-js/wc-editor";
+      import { omd2notebook } from "@hpcc-js/observablehq-compiler";
+
+      const notebook = omd2notebook(`\
+        # Simple Notebook
+
+        * Set A
+        \`\`\`
+        a = 1
+        \`\`\`
+
+        * Set B
+        \`\`\`
+        b = 2
+        \`\`\`
+
+        * Calculate c
+
+        \`\`\`
+        c = a + b
+        \`\`\`
+      `);
+      const placeholder = document.getElementById("placeholder");
+      placeholder.text = JSON.stringify(notebook, undefined, 4);
+    </script>
+  </hpcc-vitepress>
+</ClientOnly>
+

package/src/util.ts

@@ -0,0 +1,154 @@
+import type { ohq } from "@hpcc-js/observable-shim";
+import { parseModule } from "@hpcc-js/observable-shim";
+
+const FuncTypes = {
+    functionType: Object.getPrototypeOf(function () { }).constructor,
+    asyncFunctionType: Object.getPrototypeOf(async function () { }).constructor,
+    generatorFunctionType: Object.getPrototypeOf(function* () { }).constructor,
+    asyncGeneratorFunctionType: Object.getPrototypeOf(async function* () { }).constructor
+};
+
+function funcType(async: boolean = false, generator: boolean = false) {
+    if (!async && !generator) return FuncTypes.functionType;
+    if (async && !generator) return FuncTypes.asyncFunctionType;
+    if (!async && generator) return FuncTypes.generatorFunctionType;
+    return FuncTypes.asyncGeneratorFunctionType;
+}
+
+interface Ref {
+    start: number,
+    end: number,
+    newText: string
+}
+
+export interface Refs {
+    inputs: string[];
+    args: string[];
+    patches: Ref[];
+}
+
+export function createFunction(refs: Refs, async = false, generator = false, blockStatement = false, body?: string) {
+    if (body === undefined) {
+        return undefined;
+    }
+
+    refs.patches.sort((l, r) => r.start - l.start);
+    refs.patches.forEach(r => {
+        body = body!.substring(0, r.start) + r.newText + body!.substring(r.end);
+    });
+    return new (funcType(async, generator))(...refs.args, blockStatement ?
+        body.substring(1, body.length - 1).trim() :
+        `return (\n${body}\n);`);
+}
+
+//  Hide "import" from bundlers as they have a habit of replacing "import" with "require"
+export async function obfuscatedImport(url: string) {
+    return new FuncTypes.asyncFunctionType("url", "return import(url)")(url);
+}
+
+interface ParsedOJS {
+    ojs: string;
+    offset: number;
+    inlineMD: boolean;
+}
+
+export function encodeBacktick(str: string) {
+    return str
+        .split("`").join("\\`")
+        ;
+}
+
+function createParsedOJS(ojs: string, offset: number, inlineMD: boolean): ParsedOJS {
+    return {
+        ojs,
+        offset,
+        inlineMD
+    };
+}
+
+function parseOmd(_: string): ParsedOJS[] {
+    const retVal: ParsedOJS[] = [];
+    //  Load Markdown  ---
+    const re = /(```(?:\s|\S)[\s\S]*?```)/g;
+    let prevOffset = 0;
+    let match = re.exec(_);
+    while (match !== null) {
+        if (match.index > prevOffset) {
+            retVal.push(createParsedOJS(_.substring(prevOffset, match.index), prevOffset, true));
+        }
+
+        const outer = match[0];
+        if (outer.indexOf("``` ") === 0 || outer.indexOf("```\n") === 0 || outer.indexOf("```\r\n") === 0) {
+            const prefixLen = 3;
+            const inner = outer.substring(prefixLen, outer.length - prefixLen);
+            retVal.push(createParsedOJS(inner, match.index + prefixLen, false));
+        } else {
+            retVal.push(createParsedOJS(outer, match.index, true));
+        }
+
+        prevOffset = match.index + match[0].length;
+        match = re.exec(_);
+    }
+    if (_.length > prevOffset) {
+        retVal.push(createParsedOJS(_.substring(prevOffset, _.length), prevOffset, true));
+    }
+    return retVal;
+}
+
+export function notebook2ojs(_: string): ParsedOJS[] {
+    const parsed: ohq.Notebook = JSON.parse(_);
+    return parsed.nodes.map(node => createParsedOJS(node.value, 0, node.mode === "md"));
+}
+
+export function ojs2notebook(ojs: string): ohq.Notebook {
+    const cells = parseModule(ojs);
+    return {
+        files: [],
+        nodes: cells.map((cell, idx) => {
+            return {
+                id: idx,
+                mode: "js",
+                value: cell
+            };
+        })
+    } as ohq.Notebook;
+}
+
+export function omd2notebook(omd: string): ohq.Notebook {
+    const cells = parseOmd(omd);
+    return {
+        files: [],
+        nodes: cells.map((cell, idx) => {
+            return {
+                id: idx,
+                mode: cell.inlineMD ? "md" : "js",
+                value: cell.ojs
+            };
+        })
+    } as ohq.Notebook;
+}
+
+export function fetchEx(url: string) {
+    return fetch(url)
+        .then(response => {
+            if (response.ok) return response;
+            throw new Error("CORS?");
+        }).catch(e => {
+            const matches = url.match(/^(?:https?:\/\/)?(?:[^@\n]+@)?(?:www\.)?([^:\/\n?]+)/img);
+            url = "https://observable-cors.glitch.me/" + url;
+            return fetch(url, {
+                headers: {
+                    origin: matches[0],
+                    referer: url
+                }
+            });
+        });
+}
+
+export function download(impUrl: string): Promise<ohq.Notebook> {
+    const isShared = impUrl.indexOf("https://observablehq.com/d") === 0;
+    return fetchEx(impUrl.replace(`https://observablehq.com/${isShared ? "d/" : ""}`, "https://api.observablehq.com/document/"))
+        .then(r => r.json())
+        ;
+
+}

package/src/writer.ts

@@ -0,0 +1,80 @@
+import { ohq } from "@hpcc-js/observable-shim";
+import { ParsedImportCell, ParsedVariable } from "./cst";
+
+export class Writer {
+
+    protected _files: ohq.File[] = [];
+    protected _imports: string[] = [];
+    protected _functions: string[] = [];
+    protected _defines: string[] = [];
+    protected _defineUid = 0;
+    protected _functionUid = 0;
+
+    constructor() {
+    }
+
+    toString() {
+        return `\
+${this._imports.join("\n")}
+
+${this._functions.join("\n").split("\n) {").join("){")}
+
+export default function define(runtime, observer) {
+  const main = runtime.module();
+
+  function toString() { return this.url; }
+  const fileAttachments = new Map([
+    ${this._files.map(f => `["${f.name}", { url: new URL("${f.url}"), mimeType: ${JSON.stringify(f.mime_type)}, toString }]`).join(",\n    ")}
+  ]);
+  main.builtin("FileAttachment", runtime.fileAttachments(name => fileAttachments.get(name)));
+
+  ${this._defines.join("\n  ")}
+
+  return main;
+}\n`;
+    }
+
+    files(files: ohq.File[]) {
+        this._files = [...this._files, ...files];
+    }
+
+    import(url: string) {
+        this._imports.push(`import define${++this._defineUid} from "${url}"; `);
+    }
+
+    importDefine(imp: Partial<ParsedImportCell>) {
+        const injections = imp.injections.map(inj => {
+            return inj.name === inj.alias ?
+                `"${inj.name}"` :
+                `{name: "${inj.name}", alias: "${inj.alias}"}`;
+        });
+        const derive = imp.injections.length ? `.derive([${injections.join(", ")}], main)` : "";
+        this._defines.push(`const child${this._defineUid} = runtime.module(define${this._defineUid})${derive};`);
+        imp.specifiers.forEach(s => {
+            this._defines.push(`main.import("${s.name}"${s.alias && s.alias !== s.name ? `, "${s.alias}"` : ""}, child${this._defineUid}); `);
+        });
+    }
+
+    function(variable: Partial<ParsedVariable>) {
+        let id = variable.id ?? `${++this._functionUid}`;
+        const idParts = id.split(" ");
+        id = `_${idParts[idParts.length - 1]}`;
+        this._functions.push(`${variable.func?.toString()?.replace("anonymous", `${id}`)}`);
+        return id;
+    }
+
+    define(variable: Partial<ParsedVariable>, observable = true, inlineFunc = false, funcId?: string) {
+        funcId = funcId ?? variable.id;
+        const observe = observable ? `.variable(observer(${variable.id ? JSON.stringify(variable.id) : ""}))` : "";
+        const id = variable.id ? `${JSON.stringify(variable.id)}, ` : "";
+        const inputs = variable.inputs.length ? `[${variable.inputs.map(i => JSON.stringify(i)).join(", ")}], ` : "";
+        const func = inlineFunc ?
+            variable.func?.toString() :
+            funcId;
+        this._defines.push(`main${observe}.define(${id}${inputs}${func});`);
+    }
+
+    error(msg: string) {
+    }
+}
+

package/types/__package__.d.ts

@@ -0,0 +1,4 @@
+export declare const PKG_NAME = "@hpcc-js/observablehq-compiler";
+export declare const PKG_VERSION = "1.1.0";
+export declare const BUILD_VERSION = "2.104.5";
+//# sourceMappingURL=__package__.d.ts.map

package/types/__package__.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"__package__.d.ts","sourceRoot":"","sources":["../src/__package__.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,QAAQ,mCAAmC,CAAC;AACzD,eAAO,MAAM,WAAW,UAAU,CAAC;AACnC,eAAO,MAAM,aAAa,YAAY,CAAC"}

package/types/__tests__/File Attachments.d.ts

@@ -0,0 +1,110 @@
+export declare const fa: {
+    id: string;
+    slug: string;
+    trashed: boolean;
+    description: string;
+    likes: number;
+    publish_level: string;
+    forks: number;
+    fork_of: any;
+    update_time: string;
+    publish_time: string;
+    publish_version: number;
+    latest_version: number;
+    thumbnail: string;
+    default_thumbnail: string;
+    roles: any[];
+    sharing: any;
+    owner: {
+        id: string;
+        avatar_url: string;
+        login: string;
+        name: string;
+        bio: string;
+        home_url: string;
+        type: string;
+        tier: string;
+    };
+    creator: {
+        id: string;
+        github_login: string;
+        avatar_url: string;
+        login: string;
+        name: string;
+        bio: string;
+        home_url: string;
+        tier: string;
+    };
+    authors: any[];
+    collections: {
+        id: string;
+        type: string;
+        slug: string;
+        title: string;
+        description: string;
+        update_time: string;
+        pinned: boolean;
+        ordered: boolean;
+        custom_thumbnail: string;
+        default_thumbnail: string;
+        thumbnail: string;
+        listing_count: number;
+        parent_collection_count: number;
+        owner: {
+            id: string;
+            avatar_url: string;
+            login: string;
+            name: string;
+            bio: string;
+            home_url: string;
+            type: string;
+            tier: string;
+        };
+    }[];
+    files: {
+        id: string;
+        url: string;
+        download_url: string;
+        name: string;
+        create_time: string;
+        status: string;
+        size: number;
+        mime_type: string;
+        content_encoding: string;
+    }[];
+    comments: {
+        id: string;
+        content: string;
+        node_id: number;
+        create_time: string;
+        update_time: any;
+        resolved: boolean;
+        user: {
+            id: string;
+            github_login: string;
+            avatar_url: string;
+            login: string;
+            name: string;
+            bio: string;
+            home_url: string;
+            tier: string;
+        };
+    }[];
+    commenting_lock: any;
+    suggestion_from: any;
+    suggestions_to: any[];
+    version: number;
+    title: string;
+    license: any;
+    copyright: string;
+    nodes: {
+        id: number;
+        value: string;
+        pinned: boolean;
+        mode: string;
+        data: any;
+        name: string;
+    }[];
+    resolutions: any[];
+};
+//# sourceMappingURL=File%20Attachments.d.ts.map

package/types/__tests__/File Attachments.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"File Attachments.d.ts","sourceRoot":"","sources":["../../src/__tests__/File Attachments.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA83Bd,CAAC"}

package/types/__tests__/Introduction to Imports.d.ts

@@ -0,0 +1,120 @@
+export declare const imports: {
+    id: string;
+    slug: string;
+    trashed: boolean;
+    description: string;
+    likes: number;
+    publish_level: string;
+    forks: number;
+    fork_of: any;
+    update_time: string;
+    publish_time: string;
+    publish_version: number;
+    latest_version: number;
+    thumbnail: string;
+    default_thumbnail: string;
+    roles: any[];
+    sharing: any;
+    owner: {
+        id: string;
+        avatar_url: string;
+        login: string;
+        name: string;
+        bio: string;
+        home_url: string;
+        type: string;
+        tier: string;
+    };
+    creator: {
+        id: string;
+        github_login: string;
+        avatar_url: string;
+        login: string;
+        name: string;
+        bio: string;
+        home_url: string;
+        tier: string;
+    };
+    authors: any[];
+    collections: {
+        id: string;
+        type: string;
+        slug: string;
+        title: string;
+        description: string;
+        update_time: string;
+        pinned: boolean;
+        ordered: boolean;
+        custom_thumbnail: string;
+        default_thumbnail: string;
+        thumbnail: string;
+        listing_count: number;
+        parent_collection_count: number;
+        owner: {
+            id: string;
+            avatar_url: string;
+            login: string;
+            name: string;
+            bio: string;
+            home_url: string;
+            type: string;
+            tier: string;
+        };
+    }[];
+    files: ({
+        id: string;
+        url: string;
+        download_url: string;
+        name: string;
+        create_time: string;
+        status: string;
+        size: number;
+        mime_type: any;
+        content_encoding: string;
+    } | {
+        id: string;
+        url: string;
+        download_url: string;
+        name: string;
+        create_time: string;
+        status: string;
+        size: number;
+        mime_type: string;
+        content_encoding: any;
+    })[];
+    comments: {
+        id: string;
+        content: string;
+        node_id: number;
+        create_time: string;
+        update_time: any;
+        resolved: boolean;
+        user: {
+            id: string;
+            github_login: string;
+            avatar_url: string;
+            login: string;
+            name: string;
+            bio: string;
+            home_url: string;
+            tier: string;
+        };
+    }[];
+    commenting_lock: any;
+    suggestion_from: any;
+    suggestions_to: any[];
+    version: number;
+    title: string;
+    license: any;
+    copyright: string;
+    nodes: {
+        id: number;
+        value: string;
+        pinned: boolean;
+        mode: string;
+        data: any;
+        name: string;
+    }[];
+    resolutions: any[];
+};
+//# sourceMappingURL=Introduction%20to%20Imports.d.ts.map

package/types/__tests__/Introduction to Imports.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Introduction to Imports.d.ts","sourceRoot":"","sources":["../../src/__tests__/Introduction to Imports.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4uBnB,CAAC"}

package/types/__tests__/Observable TimeChart.d.ts

@@ -0,0 +1,111 @@
+export declare const timechart: {
+    id: string;
+    slug: string;
+    trashed: boolean;
+    description: string;
+    likes: number;
+    publish_level: string;
+    forks: number;
+    fork_of: any;
+    update_time: string;
+    publish_time: string;
+    publish_version: number;
+    latest_version: number;
+    thumbnail: string;
+    default_thumbnail: string;
+    roles: any[];
+    sharing: any;
+    owner: {
+        id: string;
+        avatar_url: string;
+        login: string;
+        name: string;
+        bio: string;
+        home_url: string;
+        type: string;
+        tier: string;
+    };
+    creator: {
+        id: string;
+        github_login: string;
+        avatar_url: string;
+        login: string;
+        name: string;
+        bio: string;
+        home_url: string;
+        tier: string;
+    };
+    authors: {
+        id: string;
+        avatar_url: string;
+        name: string;
+        login: string;
+        bio: string;
+        home_url: string;
+        github_login: string;
+        tier: string;
+        approved: boolean;
+        description: string;
+    }[];
+    collections: {
+        id: string;
+        type: string;
+        slug: string;
+        title: string;
+        description: string;
+        update_time: string;
+        pinned: boolean;
+        ordered: boolean;
+        custom_thumbnail: string;
+        default_thumbnail: string;
+        thumbnail: string;
+        listing_count: number;
+        parent_collection_count: number;
+        owner: {
+            id: string;
+            avatar_url: string;
+            login: string;
+            name: string;
+            bio: string;
+            home_url: string;
+            type: string;
+            tier: string;
+        };
+    }[];
+    files: any[];
+    comments: {
+        id: string;
+        content: string;
+        node_id: number;
+        create_time: string;
+        update_time: any;
+        resolved: boolean;
+        user: {
+            id: string;
+            github_login: string;
+            avatar_url: string;
+            login: string;
+            name: string;
+            bio: string;
+            home_url: string;
+            tier: string;
+        };
+    }[];
+    commenting_lock: any;
+    suggestion_from: any;
+    suggestions_to: any[];
+    version: number;
+    title: string;
+    license: string;
+    copyright: string;
+    nodes: {
+        id: number;
+        value: string;
+        pinned: boolean;
+        mode: string;
+        data: any;
+        name: any;
+    }[];
+    resolutions: any[];
+};
+//# sourceMappingURL=Observable%20TimeChart.d.ts.map