@hpcc-js/observablehq-compiler 3.1.0 → 3.2.0

package/package.json

@@ -1,13 +1,14 @@
 {
     "name": "@hpcc-js/observablehq-compiler",
-    "version": "3.1.0",
+    "version": "3.2.0",
     "description": "hpcc-js - ObservableHQ Compiler (unoffical)",
     "type": "module",
     "exports": {
         ".": {
             "types": "./types/index.d.ts",
             "default": "./dist/index.js"
-        }
+        },
+        "./dist/*": "./dist/*"
     },
     "main": "./dist/index.js",
     "types": "./types/index.d.ts",
@@ -26,8 +27,8 @@
         "gen-types": "tsc --project tsconfig.json",
         "gen-types-watch": "npm run gen-types -- --watch",
         "build": "run-p gen-types bundle",
-        "stamp": "node ../../node_modules/@hpcc-js/bundle/src/stamp.js",
         "lint": "eslint ./src",
+        "lint-fix": "eslint --fix src/**/*.ts",
         "docs": "typedoc --options tdoptions.json .",
         "test-browser": "vitest run --project browser",
         "test-node": "vitest run --project node",
@@ -42,8 +43,8 @@
         "yargs": "17.7.2"
     },
     "devDependencies": {
-        "@hpcc-js/esbuild-plugins": "^1.2.0",
-        "@hpcc-js/observable-shim": "^3.1.0",
+        "@hpcc-js/esbuild-plugins": "^1.3.0",
+        "@observablehq/parser": "6.1.0",
         "@observablehq/runtime": "5.9.9"
     },
     "repository": {
@@ -65,5 +66,5 @@
         "url": "https://github.com/hpcc-systems/Visualization/issues"
     },
     "homepage": "https://github.com/hpcc-systems/Visualization/tree/trunk/packages/observablehq-compiler",
-    "gitHead": "1fc4d0ad31fe8e30ae5348fdee67284df4c928ed"
+    "gitHead": "658c50fd965a7744ba8db675ba6878607c44d5e2"
 }

package/src/__package__.ts

@@ -1,3 +1,3 @@
 export const PKG_NAME = "@hpcc-js/observablehq-compiler";
-export const PKG_VERSION = "3.1.0";
-export const BUILD_VERSION = "2.107.0";
+export const PKG_VERSION = "3.2.0";
+export const BUILD_VERSION = "3.2.0";

package/src/compiler.ts

@@ -1,4 +1,4 @@
-import { ohq, splitModule } from "@hpcc-js/observable-shim";
+import { ohq, splitModule } from "./observable-shim.ts";
 import { parseCell, ParsedImportCell } from "./cst.ts";
 import { Writer } from "./writer.ts";
 import { fixRelativeUrl, isRelativePath, encodeBacktick, fetchEx, obfuscatedImport, ojs2notebook, omd2notebook } from "./util.ts";

package/src/cst.ts

@@ -1,4 +1,4 @@
-import { ancestor, parseCell as ohqParseCell, Cell, Node, walk, AncestorVisitors } from "@hpcc-js/observable-shim";
+import { ancestor, parseCell as ohqParseCell, Cell, Node, walk, AncestorVisitors } from "./observable-shim.ts";
 
 import { fixRelativeUrl, createFunction, Refs } from "./util.ts";
 

package/src/index.ts

@@ -1,4 +1,4 @@
-export type { ohq } from "@hpcc-js/observable-shim";
+export type { ohq } from "./observable-shim.ts";
 
 export * from "./compiler.ts";
 export { ojs2notebook, omd2notebook, download } from "./util.ts";

package/src/observable-shim.ts

@@ -0,0 +1,2 @@
+export type { ohq } from "./types.ts";
+export * from "./parse.ts";

package/src/parse.ts

@@ -0,0 +1,136 @@
+// Compare with ../../../node_modules/@observablehq/parser/src/parse.js
+import { getLineInfo, tokTypes, Statement, ModuleDeclaration, Expression as ExpressionBase, Node } from "acorn";
+import { ancestor, RecursiveVisitors, AncestorVisitors } from "acorn-walk";
+import { CellParser, parseCell as ohqParseCell, walk as ohqWalk } from "@observablehq/parser";
+import defaultGlobals from "../../../node_modules/@observablehq/parser/src/globals.js";
+import findReferences from "../../../node_modules/@observablehq/parser/src/references.js";
+import findFeatures from "../../../node_modules/@observablehq/parser/src/features.js";
+
+export interface MutableExpression extends Node {
+    type: "MutableExpression"
+}
+
+export interface ViewExpression extends Node {
+    type: "ViewExpression"
+}
+
+export type Expression = ExpressionBase | MutableExpression | ViewExpression;
+
+// Find references.
+// Check for illegal references to arguments.
+// Check for illegal assignments to global references.
+function parseReferences(cell, input, globals = defaultGlobals) {
+    if (!cell.body) {
+        cell.references = [];
+    } else if (cell.body.type === "ImportDeclaration") {
+        //  This is correct?!?
+        cell.references = cell.body.specifiers
+            ? cell.body.specifiers.map(i => i.imported)
+            : [];
+    } else {
+        try {
+            cell.references = findReferences(cell, globals);
+        } catch (error: any) {
+            if (error.node) {
+                const loc = getLineInfo(input, error.node.start);
+                error.message += ` (${loc.line}:${loc.column})`;
+                error.pos = error.node.start;
+                error.loc = loc;
+                delete error.node;
+            }
+            throw error;
+        }
+    }
+    return cell;
+}
+
+// Find features: file attachments, secrets, database clients.
+// Check for illegal references to arguments.
+// Check for illegal assignments to global references.
+function parseFeatures(cell, input) {
+    if (cell.body && cell.body.type !== "ImportDeclaration") {
+        try {
+            cell.fileAttachments = findFeatures(cell, "FileAttachment");
+            cell.databaseClients = findFeatures(cell, "DatabaseClient");
+            cell.secrets = findFeatures(cell, "Secret");
+        } catch (error: any) {
+            if (error.node) {
+                const loc = getLineInfo(input, error.node.start);
+                error.message += ` (${loc.line}:${loc.column})`;
+                error.pos = error.node.start;
+                error.loc = loc;
+                delete error.node;
+            }
+            throw error;
+        }
+    } else {
+        cell.fileAttachments = new Map();
+        cell.databaseClients = new Map();
+        cell.secrets = new Map();
+    }
+    return cell;
+}
+
+class ModuleParser extends CellParser {
+
+    parseTopLevel(node: { cells?: Cell[] }) {
+        if (!node.cells) node.cells = [];
+        // @ts-ignore
+        while (this.type !== tokTypes.eof) {
+            // @ts-ignore
+            const cell: Cell = this.parseCell(this.startNode());
+            // @ts-ignore
+            cell.input = this.input;
+            node.cells.push(cell);
+        }
+        // @ts-ignore
+        this.next();
+        // @ts-ignore
+        return this.finishNode(node, "Program");
+    }
+}
+
+// @ts-ignore
+export function parseModule(input, { globals }: { globals: any } = {}) {
+    // @ts-ignore
+    const program = ModuleParser.parse(input, { ecmaVersion: 2020 });
+    for (const cell of program.cells) {
+        parseReferences(cell, input, globals);
+        parseFeatures(cell, input);
+    }
+    return program;
+}
+
+export interface Cell extends Node {
+    type: "Cell";
+    id: Expression;
+    text: string;
+    body?: Statement | ModuleDeclaration | Expression;
+    references: unknown[];
+    async: boolean;
+    generator: boolean;
+    strict: boolean;
+}
+
+export function splitModule(input: string): Cell[] {
+    return (ModuleParser as any)
+        .parse(input, { ecmaVersion: "latest" })
+        .cells.map((cell: any) => ({
+            type: "Cell",
+            text: input.substring(cell.start, cell.end),
+            start: cell.start,
+            end: cell.end
+        }));
+}
+
+export {
+    Node,
+    ancestor,
+    AncestorVisitors
+};
+
+export function parseCell(input: string): Cell {
+    return ohqParseCell(input);
+}
+
+export const walk: RecursiveVisitors<any> = ohqWalk;

package/src/types.ts

@@ -0,0 +1,180 @@
+export namespace ohq {
+
+    //  ObservableHQ Notebook Format  ---
+    export interface Owner {
+        id?: string;
+        github_login?: string;
+        avatar_url?: string;
+        login?: string;
+        name?: string;
+        bio?: string;
+        home_url?: string;
+        type?: string;
+        tier?: string;
+    }
+
+    export interface Creator {
+        id?: string;
+        github_login?: string;
+        avatar_url?: string;
+        login?: string;
+        name?: string;
+        bio?: string;
+        home_url?: string;
+        tier?: string;
+    }
+
+    export interface Author {
+        id?: string;
+        avatar_url?: string;
+        name?: string;
+        login?: string;
+        bio?: string;
+        home_url?: string;
+        github_login?: string;
+        tier?: string;
+        approved?: boolean;
+        description?: string;
+    }
+
+    export interface Owner2 {
+        id?: string;
+        github_login?: string;
+        avatar_url?: string;
+        login?: string;
+        name?: string;
+        bio?: string;
+        home_url?: string;
+        type?: string;
+        tier?: string;
+    }
+
+    export interface Collection {
+        id?: string;
+        type?: string;
+        slug?: string;
+        title?: string;
+        description?: string;
+        update_time?: Date;
+        pinned?: boolean;
+        ordered?: boolean;
+        custom_thumbnail?: any;
+        default_thumbnail?: string;
+        thumbnail?: string;
+        listing_count?: number;
+        parent_collection_count?: number;
+        owner?: Owner2;
+    }
+
+    export interface File {
+        id?: string;
+        url: string;
+        download_url?: string;
+        name: string;
+        create_time?: Date;
+        status?: string;
+        size?: number;
+        mime_type?: string;
+        content_encoding?: string;
+    }
+
+    export interface User {
+        id?: string;
+        github_login?: string;
+        avatar_url?: string;
+        login?: string;
+        name?: string;
+        bio?: string;
+        home_url?: string;
+        tier?: string;
+    }
+
+    export interface Comment {
+        id?: string;
+        content?: string;
+        node_id?: number;
+        create_time?: Date;
+        update_time?: any;
+        resolved?: boolean;
+        user?: User;
+    }
+
+    export interface Node {
+        id: string | number;
+        mode: string;
+        value: string;
+        pinned?: boolean;   // Show source code?
+        data?: any;
+        name?: string;
+        //  NodeEx  ---
+        start?: number;
+        end?: number;
+        private?: boolean;
+        outputs?: string[];
+    }
+
+    export interface Notebook {
+        id?: string;
+        slug?: any;
+        trashed?: boolean;
+        description?: string;
+        likes?: number;
+        publish_level?: string;
+        forks?: number;
+        fork_of?: any;
+        update_time?: Date;
+        publish_time?: Date;
+        publish_version?: number;
+        latest_version?: number;
+        thumbnail?: string;
+        default_thumbnail?: string;
+        roles?: any[];
+        sharing?: any;
+        owner?: Owner;
+        creator?: Creator;
+        authors?: Author[];
+        collections?: Collection[];
+        files: File[];
+        comments?: Comment[];
+        commenting_lock?: any;
+        suggestion_from?: any;
+        suggestions_to?: any[];
+        version?: number;
+        title?: string;
+        license?: string;
+        copyright?: string;
+        nodes: Node[];
+        resolutions?: any[];
+    }
+
+    //  @observablehq/runtime API  ---
+    export type InspectorFactory = (name?: string) => Inspector;
+
+    export interface Inspector {
+        _node?: HTMLDivElement;
+        pending();
+        fulfilled(value);
+        rejected(error);
+    }
+
+    export interface Runtime {
+        fileAttachments(func: (name: string) => ohq.File): any;
+        module(define?, inspector?: InspectorFactory): Module;
+        dispose(): void;
+    }
+
+    export interface Module {
+        derive(specifiers: string[] | { name: string, alias: string }[], source: any);
+        import(name: string, alias: string | undefined, mod: Module): Variable;
+        builtin(name: string, _: any);
+        variable(inspector?: Inspector): Variable;
+        value(name: string): Promise<any>;
+    }
+
+    export interface Variable {
+        delete();
+        define(name?: string, inputs?: string[], definition?: any);
+        import(name: string, otherModule: ohq.Module);
+        import(name: string, alias: string, otherModule: ohq.Module);
+    }
+}

package/src/util.ts

@@ -1,5 +1,5 @@
-import type { ohq } from "@hpcc-js/observable-shim";
-import { parseCell, splitModule } from "@hpcc-js/observable-shim";
+import type { ohq } from "./observable-shim.ts";
+import { parseCell, splitModule } from "./observable-shim.ts";
 
 const FuncTypes = {
     functionType: Object.getPrototypeOf(function () { }).constructor,

package/src/writer.ts

@@ -1,4 +1,4 @@
-import { ohq } from "@hpcc-js/observable-shim";
+import { ohq } from "./observable-shim.ts";
 import { ParsedImportCell, ParsedVariable } from "./cst.ts";
 
 export class Writer {

package/types/compiler.d.ts

@@ -1,4 +1,4 @@
-import { ohq } from "@hpcc-js/observable-shim";
+import { ohq } from "./observable-shim.ts";
 import { Writer } from "./writer.ts";
 export type InspectorFactoryEx = (name: string | undefined, id: string | number) => Inspector;
 export interface Inspector {

package/types/index.d.ts

@@ -1,4 +1,4 @@
-export type { ohq } from "@hpcc-js/observable-shim";
+export type { ohq } from "./observable-shim.ts";
 export * from "./compiler.ts";
 export { ojs2notebook, omd2notebook, download } from "./util.ts";
 export * from "./writer.ts";

package/types/observable-shim.d.ts

@@ -0,0 +1,2 @@
+export type { ohq } from "./types.ts";
+export * from "./parse.ts";

package/types/parse.d.ts

@@ -0,0 +1,26 @@
+import { Statement, ModuleDeclaration, Expression as ExpressionBase, Node } from "acorn";
+import { ancestor, RecursiveVisitors, AncestorVisitors } from "acorn-walk";
+export interface MutableExpression extends Node {
+    type: "MutableExpression";
+}
+export interface ViewExpression extends Node {
+    type: "ViewExpression";
+}
+export type Expression = ExpressionBase | MutableExpression | ViewExpression;
+export declare function parseModule(input: any, { globals }?: {
+    globals: any;
+}): any;
+export interface Cell extends Node {
+    type: "Cell";
+    id: Expression;
+    text: string;
+    body?: Statement | ModuleDeclaration | Expression;
+    references: unknown[];
+    async: boolean;
+    generator: boolean;
+    strict: boolean;
+}
+export declare function splitModule(input: string): Cell[];
+export { Node, ancestor, AncestorVisitors };
+export declare function parseCell(input: string): Cell;
+export declare const walk: RecursiveVisitors<any>;

package/types/types.d.ts

@@ -0,0 +1,165 @@
+export declare namespace ohq {
+    interface Owner {
+        id?: string;
+        github_login?: string;
+        avatar_url?: string;
+        login?: string;
+        name?: string;
+        bio?: string;
+        home_url?: string;
+        type?: string;
+        tier?: string;
+    }
+    interface Creator {
+        id?: string;
+        github_login?: string;
+        avatar_url?: string;
+        login?: string;
+        name?: string;
+        bio?: string;
+        home_url?: string;
+        tier?: string;
+    }
+    interface Author {
+        id?: string;
+        avatar_url?: string;
+        name?: string;
+        login?: string;
+        bio?: string;
+        home_url?: string;
+        github_login?: string;
+        tier?: string;
+        approved?: boolean;
+        description?: string;
+    }
+    interface Owner2 {
+        id?: string;
+        github_login?: string;
+        avatar_url?: string;
+        login?: string;
+        name?: string;
+        bio?: string;
+        home_url?: string;
+        type?: string;
+        tier?: string;
+    }
+    interface Collection {
+        id?: string;
+        type?: string;
+        slug?: string;
+        title?: string;
+        description?: string;
+        update_time?: Date;
+        pinned?: boolean;
+        ordered?: boolean;
+        custom_thumbnail?: any;
+        default_thumbnail?: string;
+        thumbnail?: string;
+        listing_count?: number;
+        parent_collection_count?: number;
+        owner?: Owner2;
+    }
+    interface File {
+        id?: string;
+        url: string;
+        download_url?: string;
+        name: string;
+        create_time?: Date;
+        status?: string;
+        size?: number;
+        mime_type?: string;
+        content_encoding?: string;
+    }
+    interface User {
+        id?: string;
+        github_login?: string;
+        avatar_url?: string;
+        login?: string;
+        name?: string;
+        bio?: string;
+        home_url?: string;
+        tier?: string;
+    }
+    interface Comment {
+        id?: string;
+        content?: string;
+        node_id?: number;
+        create_time?: Date;
+        update_time?: any;
+        resolved?: boolean;
+        user?: User;
+    }
+    interface Node {
+        id: string | number;
+        mode: string;
+        value: string;
+        pinned?: boolean;
+        data?: any;
+        name?: string;
+        start?: number;
+        end?: number;
+        private?: boolean;
+        outputs?: string[];
+    }
+    interface Notebook {
+        id?: string;
+        slug?: any;
+        trashed?: boolean;
+        description?: string;
+        likes?: number;
+        publish_level?: string;
+        forks?: number;
+        fork_of?: any;
+        update_time?: Date;
+        publish_time?: Date;
+        publish_version?: number;
+        latest_version?: number;
+        thumbnail?: string;
+        default_thumbnail?: string;
+        roles?: any[];
+        sharing?: any;
+        owner?: Owner;
+        creator?: Creator;
+        authors?: Author[];
+        collections?: Collection[];
+        files: File[];
+        comments?: Comment[];
+        commenting_lock?: any;
+        suggestion_from?: any;
+        suggestions_to?: any[];
+        version?: number;
+        title?: string;
+        license?: string;
+        copyright?: string;
+        nodes: Node[];
+        resolutions?: any[];
+    }
+    type InspectorFactory = (name?: string) => Inspector;
+    interface Inspector {
+        _node?: HTMLDivElement;
+        pending(): any;
+        fulfilled(value: any): any;
+        rejected(error: any): any;
+    }
+    interface Runtime {
+        fileAttachments(func: (name: string) => ohq.File): any;
+        module(define?: any, inspector?: InspectorFactory): Module;
+        dispose(): void;
+    }
+    interface Module {
+        derive(specifiers: string[] | {
+            name: string;
+            alias: string;
+        }[], source: any): any;
+        import(name: string, alias: string | undefined, mod: Module): Variable;
+        builtin(name: string, _: any): any;
+        variable(inspector?: Inspector): Variable;
+        value(name: string): Promise<any>;
+    }
+    interface Variable {
+        delete(): any;
+        define(name?: string, inputs?: string[], definition?: any): any;
+        import(name: string, otherModule: ohq.Module): any;
+        import(name: string, alias: string, otherModule: ohq.Module): any;
+    }
+}

package/types/util.d.ts

@@ -1,4 +1,4 @@
-import type { ohq } from "@hpcc-js/observable-shim";
+import type { ohq } from "./observable-shim.ts";
 interface Ref {
     start: number;
     end: number;

package/types/writer.d.ts

@@ -1,4 +1,4 @@
-import { ohq } from "@hpcc-js/observable-shim";
+import { ohq } from "./observable-shim.ts";
 import { ParsedImportCell, ParsedVariable } from "./cst.ts";
 export declare class Writer {
     protected _files: ohq.File[];