@hpcc-js/observablehq-compiler 1.4.0 → 3.2.0

package/package.json

@@ -1,74 +1,64 @@
 {
     "name": "@hpcc-js/observablehq-compiler",
-    "version": "1.4.0",
+    "version": "3.2.0",
     "description": "hpcc-js - ObservableHQ Compiler (unoffical)",
-    "keywords": [
-        "observablehq",
-        "markdown",
-        "observable",
-        "compiler",
-        "interpreter",
-        "renderer"
-    ],
-    "main": "dist/index.js",
-    "module": "dist/index.esm.js",
-    "browser": "dist/index.js",
-    "unpkg": "dist/index.min.js",
-    "jsdelivr": "dist/index.min.js",
-    "types": "types/index.d.ts",
-    "typesVersions": {
-        "<3.8": {
-            "*": [
-                "types-3.4/index.d.ts"
-            ]
-        }
-    },
-    "bin": {
-        "ojscc": "bin/ojscc.mjs"
+    "type": "module",
+    "exports": {
+        ".": {
+            "types": "./types/index.d.ts",
+            "default": "./dist/index.js"
+        },
+        "./dist/*": "./dist/*"
     },
+    "main": "./dist/index.js",
+    "types": "./types/index.d.ts",
     "files": [
         "dist/*",
-        "types/*",
-        "types-3.4/*",
-        "src/*"
+        "src/*",
+        "types/*"
     ],
+    "bin": {
+        "ojscc": "bin/ojscc.mjs"
+    },
     "scripts": {
-        "clean": "rimraf --glob lib* dist* types *.tsbuildinfo",
-        "compile-es6": "tsc --module es2020 --outDir ./lib-es6",
-        "compile-es6-watch": "npm run compile-es6 -- -w",
-        "compile-umd": "tsc --module umd --outDir ./lib-umd",
-        "compile-umd-watch": "npm run compile-umd -- -w",
-        "bundle": "rollup -c",
-        "bundle-watch": "npm run bundle -- -w",
-        "minimize-index": "terser dist/index.js -c -m --source-map \"content='dist/index.js.map',url='index.min.js.map'\" -o dist/index.min.js",
-        "minimize-index-es6": "terser dist/index.esm.js -c -m --source-map \"content='dist/index.esm.js.map',url='index.esm.min.js.map'\" -o dist/index.esm.min.js",
-        "minimize": "run-p minimize-index minimize-index-es6",
-        "gen-legacy-types": "downlevel-dts ./types ./types-3.4",
-        "build": "npm run compile-es6 && npm run bundle",
-        "watch": "npm-run-all compile-es6 -p compile-es6-watch bundle-watch",
-        "stamp": "node ../../node_modules/@hpcc-js/bundle/src/stamp.js",
+        "clean": "rimraf --glob lib* types dist dist-test *.tsbuildinfo .turbo",
+        "bundle": "node esbuild.js",
+        "bundle-watch": "npm run bundle -- --development --watch",
+        "gen-types": "tsc --project tsconfig.json",
+        "gen-types-watch": "npm run gen-types -- --watch",
+        "build": "run-p gen-types bundle",
         "lint": "eslint ./src",
+        "lint-fix": "eslint --fix src/**/*.ts",
         "docs": "typedoc --options tdoptions.json .",
-        "dev-start": "ws",
-        "test-cli": "node ./bin/ojscc.mjs --version",
-        "test-node": "mocha ./dist-test/index.mjs --reporter spec",
-        "test": "run-s test-cli test-node",
-        "update": "npx --yes npm-check-updates -u -t minor"
+        "test-browser": "vitest run --project browser",
+        "test-node": "vitest run --project node",
+        "test-both": "vitest run",
+        "test-bin": "node ./bin/ojscc.mjs --version",
+        "test": "run-p test-both test-bin",
+        "coverage": "vitest run --coverage",
+        "update": "npx -y npm-check-updates -u -t minor",
+        "update-major": "npx -y npm-check-updates -u"
     },
     "dependencies": {
-        "@hpcc-js/observable-shim": "^1.0.0",
-        "node-fetch": "3.3.2",
         "yargs": "17.7.2"
     },
     "devDependencies": {
-        "@hpcc-js/bundle": "^2.12.0",
-        "@observablehq/runtime": "5.9.9",
-        "tslib": "2.7.0"
+        "@hpcc-js/esbuild-plugins": "^1.3.0",
+        "@observablehq/parser": "6.1.0",
+        "@observablehq/runtime": "5.9.9"
     },
     "repository": {
         "type": "git",
         "url": "git+https://github.com/hpcc-systems/Visualization.git"
     },
+    "keywords": [
+        "observablehq",
+        "markdown",
+        "observable",
+        "compiler",
+        "interpreter",
+        "renderer"
+    ],
     "author": "Gordon Smith <gordonjsmith@gmail.com>",
     "contributors": [],
     "license": "Apache-2.0",
@@ -76,5 +66,5 @@
         "url": "https://github.com/hpcc-systems/Visualization/issues"
     },
     "homepage": "https://github.com/hpcc-systems/Visualization/tree/trunk/packages/observablehq-compiler",
-    "gitHead": "fbbef050700b0e9d76ef99714856383d95155149"
+    "gitHead": "658c50fd965a7744ba8db675ba6878607c44d5e2"
 }

package/src/__package__.ts

@@ -1,3 +1,3 @@
 export const PKG_NAME = "@hpcc-js/observablehq-compiler";
-export const PKG_VERSION = "1.4.0";
-export const BUILD_VERSION = "2.107.0";
+export const PKG_VERSION = "3.2.0";
+export const BUILD_VERSION = "3.2.0";

package/src/compiler.ts

@@ -1,16 +1,16 @@
-import { ohq, splitModule } from "@hpcc-js/observable-shim";
-import { parseCell, ParsedImportCell } from "./cst";
-import { Writer } from "./writer";
-import { fixRelativeUrl, isRelativePath, encodeBacktick, fetchEx, obfuscatedImport, ojs2notebook, omd2notebook } from "./util";
+import { ohq, splitModule } from "./observable-shim.ts";
+import { parseCell, ParsedImportCell } from "./cst.ts";
+import { Writer } from "./writer.ts";
+import { fixRelativeUrl, isRelativePath, encodeBacktick, fetchEx, obfuscatedImport, ojs2notebook, omd2notebook } from "./util.ts";
 
 //  Inspector Factory  ---
 export type InspectorFactoryEx = (name: string | undefined, id: string | number) => Inspector;
 
 export interface Inspector {
     _node?: HTMLDivElement;
-    pending();
-    fulfilled(value);
-    rejected(error);
+    pending(): void;
+    fulfilled(value: any): void;
+    rejected(error: Error): void;
 }
 
 //  Module  ---
@@ -31,6 +31,9 @@ async function importFile(relativePath: string, baseUrl: string) {
         notebook = ojs2notebook(content);
     } else if (relativePath.endsWith(".omd")) {
         notebook = omd2notebook(content);
+    } else {
+        console.warn(`Unknown file type: ${relativePath}, assuming .ojsnb`);
+        notebook = JSON.parse(content);
     }
     const retVal: ImportDefine = compile(notebook, { baseUrl }) as any;
     retVal.delete = () => { };
@@ -44,7 +47,7 @@ async function importFile(relativePath: string, baseUrl: string) {
 async function importCompiledNotebook(partial: string) {
     const url = `https://api.observablehq.com/${partial[0] === "@" ? partial : `d/${partial}`}.js?v=3`;
     let impMod = {
-        default: function (runtime: ohq.Runtime, inspector?: InspectorFactoryEx): ohq.Module {
+        default: function (runtime: ohq.Runtime, inspector?: InspectorFactoryEx): ohq.Module | undefined {
             return undefined;
         } as any
     };
@@ -80,7 +83,7 @@ async function importNotebook(partial: string) {
 
 async function createModule(node: ohq.Node, parsed: ParsedImportCell, text: string, { baseUrl, importMode }: CompileOptions) {
     const otherModule = isRelativePath(parsed.src) ?
-        await importFile(parsed.src, baseUrl) :
+        await importFile(parsed.src, baseUrl ?? "") :
         importMode === "recursive" ?
             await importNotebook(parsed.src) :
             await importCompiledNotebook(parsed.src);
@@ -123,11 +126,13 @@ type ModuleFunc = Awaited<ReturnType<typeof createModule>>;
 //  Variable  ---
 function createVariable(node: ohq.Node, inspect: boolean, name?: string, inputs?: string[], definition?: any, inline = false) {
 
-    let i: ohq.Inspector;
-    let v: ohq.Variable;
+    let i: ohq.Inspector | undefined;
+    let v: ohq.Variable | undefined;
 
     const retVal = (module: ohq.Module, inspector?: InspectorFactoryEx) => {
-        i = inspect ? inspector(name, node.id) : undefined;
+        if (inspect && inspector) {
+            i = inspector(name, node.id);
+        }
         v = module.variable(i);
         if (arguments.length > 1) {
             try {
@@ -137,9 +142,9 @@ function createVariable(node: ohq.Node, inspect: boolean, name?: string, inputs?
             }
         }
         if (node.pinned) {
-            v = module.variable(inspector(name, node.id));
+            v = inspector ? module.variable(inspector(name, node.id)) : module.variable();
             try {
-                v.define(undefined, ["md"], md => {
+                v.define(undefined, ["md"], (md: any) => {
                     return md`\`\`\`js
 ${node.value}
 \`\`\``;
@@ -174,13 +179,17 @@ ${node.value}
 }
 type VariableFunc = ReturnType<typeof createVariable>;
 
-function createImportVariable(name?: string, alias?: string) {
+function createImportVariable(name: string, alias?: string) {
 
     let v: ohq.Variable;
 
     const retVal = (main: ohq.Module, otherModule: ohq.Module) => {
         v = main.variable();
-        v.import(name, alias, otherModule);
+        if (alias === undefined) {
+            v.import(name, otherModule);
+        } else {
+            v.import(name, alias, otherModule);
+        }
     };
 
     retVal.delete = () => {
@@ -198,7 +207,7 @@ async function createCell(node: ohq.Node, options: CompileOptions) {
         const text = node.mode && node.mode !== "js" ? `${node.mode}\`${encodeBacktick(node.value)}\`` : node.value;
         const parsedModule = splitModule(text);
         for (const cell of parsedModule) {
-            const parsed = parseCell(cell.text, options.baseUrl);
+            const parsed = parseCell(cell.text, options.baseUrl ?? "");
             switch (parsed.type) {
                 case "import":
                     modules.push(await createModule(node, parsed, cell.text, options));
@@ -217,8 +226,8 @@ async function createCell(node: ohq.Node, options: CompileOptions) {
                     break;
             }
         }
-    } catch (e) {
-        variables.push(createVariable(node, true, undefined, [], e.message));
+    } catch (e: any) {
+        variables.push(createVariable(node, true, undefined, [], e.message ?? "Unkown error"));
     }
 
     const retVal = (runtime: ohq.Runtime, main: ohq.Module, inspector?: InspectorFactoryEx) => {
@@ -242,8 +251,11 @@ export type CellFunc = Awaited<ReturnType<typeof createCell>>;
 
 //  File  ---
 function createFile(file: ohq.File, options: CompileOptions): [string, any] {
-    function toString() { return globalThis.url; }
-    return [file.name, { url: new URL(fixRelativeUrl(file.url, options.baseUrl)), mimeType: file.mime_type, toString }];
+    function toString() {
+        // TODO Double check url should not be URL?
+        return (globalThis as any).url ?? "";
+    }
+    return [file.name, { url: new URL(fixRelativeUrl(file.url, options.baseUrl ?? "")), mimeType: file.mime_type, toString }];
 }
 type FileFunc = ReturnType<typeof createFile>;
 
@@ -277,7 +289,7 @@ export function notebook(_files: ohq.File[] = [], _cells: CellFunc[] = [], { bas
         cells.set(cell.id, cell);
         return cell;
     };
-    retVal.get = (id: string | number): CellFunc => {
+    retVal.get = (id: string | number): CellFunc | undefined => {
         return cells.get(id);
     };
     retVal.delete = (id: string | number): boolean => {

package/src/cst.ts

@@ -1,20 +1,18 @@
-import { parseCell as ohqParseCell } from "@hpcc-js/observable-shim";
-import { acorn, walk } from "@observablehq/parser";
-import { ancestor } from "acorn-walk";
+import { ancestor, parseCell as ohqParseCell, Cell, Node, walk, AncestorVisitors } from "./observable-shim.ts";
 
-import { fixRelativeUrl, createFunction, Refs } from "./util";
+import { fixRelativeUrl, createFunction, Refs } from "./util.ts";
 
-function calcRefs(cellAst, cellStr): Refs {
+function calcRefs(cellAst: Cell, cellStr: string): Refs {
     if (cellAst.references === undefined) return { inputs: [], args: [], patches: [] };
 
-    const dedup = {};
-    cellAst.references.forEach(r => dedup[cellStr.substring(r.start, r.end)] = true);
+    const dedup: { [id: string]: boolean } = {};
+    cellAst.references.forEach((r: any) => dedup[cellStr.substring(r.start, r.end)] = true);
     const retVal: Refs = {
         inputs: Object.keys(dedup),
         args: Object.keys(dedup).map(r => r.split(" ").join("_")),
         patches: []
     };
-    const pushPatch = (node, newText) => retVal.patches.push({ start: node.start - cellAst.body.start, end: node.end - cellAst.body.start, newText });
+    const pushPatch = (node: Node, newText: string) => retVal.patches.push({ start: node.start - (cellAst.body?.start ?? 0), end: node.end - (cellAst.body?.start ?? 0), newText });
     if (cellAst.body) {
         ancestor(cellAst.body, {
             Identifier(node) {
@@ -22,23 +20,23 @@ function calcRefs(cellAst, cellStr): Refs {
                 if (dedup[value]) {
                 }
             },
-            MutableExpression(node) {
+            MutableExpression(node: Node) {
                 const value = cellStr.substring(node.start, node.end);
                 const newText = value.split(" ").join("_") + ".value";
                 pushPatch(node, newText);
             },
-            ViewExpression(node) {
+            ViewExpression(node: Node) {
                 const value = cellStr.substring(node.start, node.end);
                 const newText = value.split(" ").join("_");
                 pushPatch(node, newText);
             },
-            ThisExpression(node, ancestors: acorn.Node[]) {
+            ThisExpression(node: Node, ancestors: Node[]) {
                 const value = cellStr.substring(node.start, node.end);
                 if (value === "this" && !ancestors.find(n => n.type === "FunctionExpression")) {
                     pushPatch(node, "((this === globalThis || this === globalThis.window)? undefined : this?.valueOf())");
                 }
             }
-        } as acorn.AncestorVisitors, walk);
+        } as AncestorVisitors<any>, walk);
     }
     return retVal;
 }
@@ -54,18 +52,18 @@ export interface ParsedImportCell extends ParsedCell {
     injections: { name: string, alias: string }[];
 }
 
-function parseImportDeclaration(cellAst): ParsedImportCell {
+function parseImportDeclaration(cellAst: any): ParsedImportCell {
     return {
         type: "import",
         src: cellAst.body.source.value,
-        specifiers: cellAst.body.specifiers?.map(spec => {
+        specifiers: cellAst.body.specifiers?.map((spec: any) => {
             return {
                 view: spec.view,
                 name: spec.imported.name,
                 alias: (spec.local?.name && spec.imported.name !== spec.local.name) ? spec.local.name : spec.imported.name
             };
         }) ?? [],
-        injections: cellAst.body.injections?.map(inj => {
+        injections: cellAst.body.injections?.map((inj: any) => {
             return {
                 name: inj.imported.name,
                 alias: inj.local?.name ?? inj.imported.name
@@ -87,7 +85,7 @@ export interface ParsedViewCell extends ParsedCell {
     variableValue: ParsedVariable;
 }
 
-function parseViewExpression(cellStr: string, cellAst, refs: Refs, bodyStr?: string): ParsedViewCell {
+function parseViewExpression(cellStr: string, cellAst: any, refs: Refs, bodyStr?: string): ParsedViewCell {
     const id = cellAst.id && cellStr.substring(cellAst.id.start, cellAst.id.end);
     return {
         type: "viewof",
@@ -101,7 +99,7 @@ function parseViewExpression(cellStr: string, cellAst, refs: Refs, bodyStr?: str
             type: "variable",
             id: cellAst?.id?.id?.name,
             inputs: ["Generators", id],
-            func: (G, _) => G.input(_)
+            func: (G: any, _: any) => G.input(_)
         }
     };
 }
@@ -113,7 +111,7 @@ interface ParsedMutableCell extends ParsedCell {
     variableValue: ParsedVariable;
 }
 
-function parseMutableExpression(cellStr: string, cellAst, refs: Refs, bodyStr?: string): ParsedMutableCell {
+function parseMutableExpression(cellStr: string, cellAst: any, refs: Refs, bodyStr?: string): ParsedMutableCell {
     const id = cellAst.id && cellStr.substring(cellAst.id.start, cellAst.id.end);
     const initialValueId = cellAst?.id?.id?.name;
     const initialId = `initial ${initialValueId}`;
@@ -129,13 +127,13 @@ function parseMutableExpression(cellStr: string, cellAst, refs: Refs, bodyStr?:
             type: "variable",
             id,
             inputs: ["Mutable", initialId],
-            func: (M, _) => new M(_)
+            func: (M: any, _: any) => new M(_)
         },
         variableValue: {
             type: "variable",
             id: initialValueId,
             inputs: [id],
-            func: _ => _.generator
+            func: (_: any) => _.generator
         }
     };
 }
@@ -147,7 +145,7 @@ interface ParsedVariableCell extends ParsedCell {
     func: any,
 }
 
-function parseVariableExpression(cellStr: string, cellAst, refs: Refs, bodyStr?: string): ParsedVariableCell {
+function parseVariableExpression(cellStr: string, cellAst: any, refs: Refs, bodyStr?: string): ParsedVariableCell {
     return {
         type: "variable",
         id: cellAst.id && cellStr.substring(cellAst.id?.start, cellAst.id?.end),
@@ -163,10 +161,15 @@ export function parseCell(cellStr: string, baseUrl: string): ParsedImportCell |
         case "ImportDeclaration":
             return parseImportDeclaration(cellAst);
         case "ImportExpression":
-            bodyStr = `import("${fixRelativeUrl(cellAst.body.source.value, baseUrl)}")`;
+            switch (cellAst.body.source.type) {
+                case "Literal":
+                    bodyStr = `import("${fixRelativeUrl("" + cellAst.body.source.value, baseUrl)}")`;
+                    break;
+                default:
+                    console.error("Unexpected import value");
+            }
     }
     const refs = calcRefs(cellAst, cellStr);
-
     switch (cellAst.id?.type) {
         case "ViewExpression":
             return parseViewExpression(cellStr, cellAst, refs, bodyStr);
@@ -176,3 +179,4 @@ export function parseCell(cellStr: string, baseUrl: string): ParsedImportCell |
             return parseVariableExpression(cellStr, cellAst, refs, bodyStr);
     }
 }
+

package/src/index.ts

@@ -1,6 +1,7 @@
-export type { ohq } from "@hpcc-js/observable-shim";
+export type { ohq } from "./observable-shim.ts";
 
-export * from "./compiler";
-export { ojs2notebook, omd2notebook, download } from "./util";
+export * from "./compiler.ts";
+export { ojs2notebook, omd2notebook, download } from "./util.ts";
+export * from "./writer.ts";
 
 import "../src/index.css";

package/src/observable-shim.ts

@@ -0,0 +1,2 @@
+export type { ohq } from "./types.ts";
+export * from "./parse.ts";

package/src/parse.ts

@@ -0,0 +1,136 @@
+// Compare with ../../../node_modules/@observablehq/parser/src/parse.js
+import { getLineInfo, tokTypes, Statement, ModuleDeclaration, Expression as ExpressionBase, Node } from "acorn";
+import { ancestor, RecursiveVisitors, AncestorVisitors } from "acorn-walk";
+import { CellParser, parseCell as ohqParseCell, walk as ohqWalk } from "@observablehq/parser";
+import defaultGlobals from "../../../node_modules/@observablehq/parser/src/globals.js";
+import findReferences from "../../../node_modules/@observablehq/parser/src/references.js";
+import findFeatures from "../../../node_modules/@observablehq/parser/src/features.js";
+
+export interface MutableExpression extends Node {
+    type: "MutableExpression"
+}
+
+export interface ViewExpression extends Node {
+    type: "ViewExpression"
+}
+
+export type Expression = ExpressionBase | MutableExpression | ViewExpression;
+
+// Find references.
+// Check for illegal references to arguments.
+// Check for illegal assignments to global references.
+function parseReferences(cell, input, globals = defaultGlobals) {
+    if (!cell.body) {
+        cell.references = [];
+    } else if (cell.body.type === "ImportDeclaration") {
+        //  This is correct?!?
+        cell.references = cell.body.specifiers
+            ? cell.body.specifiers.map(i => i.imported)
+            : [];
+    } else {
+        try {
+            cell.references = findReferences(cell, globals);
+        } catch (error: any) {
+            if (error.node) {
+                const loc = getLineInfo(input, error.node.start);
+                error.message += ` (${loc.line}:${loc.column})`;
+                error.pos = error.node.start;
+                error.loc = loc;
+                delete error.node;
+            }
+            throw error;
+        }
+    }
+    return cell;
+}
+
+// Find features: file attachments, secrets, database clients.
+// Check for illegal references to arguments.
+// Check for illegal assignments to global references.
+function parseFeatures(cell, input) {
+    if (cell.body && cell.body.type !== "ImportDeclaration") {
+        try {
+            cell.fileAttachments = findFeatures(cell, "FileAttachment");
+            cell.databaseClients = findFeatures(cell, "DatabaseClient");
+            cell.secrets = findFeatures(cell, "Secret");
+        } catch (error: any) {
+            if (error.node) {
+                const loc = getLineInfo(input, error.node.start);
+                error.message += ` (${loc.line}:${loc.column})`;
+                error.pos = error.node.start;
+                error.loc = loc;
+                delete error.node;
+            }
+            throw error;
+        }
+    } else {
+        cell.fileAttachments = new Map();
+        cell.databaseClients = new Map();
+        cell.secrets = new Map();
+    }
+    return cell;
+}
+
+class ModuleParser extends CellParser {
+
+    parseTopLevel(node: { cells?: Cell[] }) {
+        if (!node.cells) node.cells = [];
+        // @ts-ignore
+        while (this.type !== tokTypes.eof) {
+            // @ts-ignore
+            const cell: Cell = this.parseCell(this.startNode());
+            // @ts-ignore
+            cell.input = this.input;
+            node.cells.push(cell);
+        }
+        // @ts-ignore
+        this.next();
+        // @ts-ignore
+        return this.finishNode(node, "Program");
+    }
+}
+
+// @ts-ignore
+export function parseModule(input, { globals }: { globals: any } = {}) {
+    // @ts-ignore
+    const program = ModuleParser.parse(input, { ecmaVersion: 2020 });
+    for (const cell of program.cells) {
+        parseReferences(cell, input, globals);
+        parseFeatures(cell, input);
+    }
+    return program;
+}
+
+export interface Cell extends Node {
+    type: "Cell";
+    id: Expression;
+    text: string;
+    body?: Statement | ModuleDeclaration | Expression;
+    references: unknown[];
+    async: boolean;
+    generator: boolean;
+    strict: boolean;
+}
+
+export function splitModule(input: string): Cell[] {
+    return (ModuleParser as any)
+        .parse(input, { ecmaVersion: "latest" })
+        .cells.map((cell: any) => ({
+            type: "Cell",
+            text: input.substring(cell.start, cell.end),
+            start: cell.start,
+            end: cell.end
+        }));
+}
+
+export {
+    Node,
+    ancestor,
+    AncestorVisitors
+};
+
+export function parseCell(input: string): Cell {
+    return ohqParseCell(input);
+}
+
+export const walk: RecursiveVisitors<any> = ohqWalk;