@observablehq/notebook-kit 1.0.1 → 1.1.0-rc.2

package/dist/src/runtime/stdlib/fileAttachment.js

@@ -1,8 +1,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 const files = new Map();
-export function FileAttachment(name, base = document.baseURI) {
-    if (new.target !== undefined)
-        throw new TypeError("FileAttachment is not a constructor");
+// TODO Enforce that files have been registered; throw error if not found.
+export const FileAttachment = (name, base = document.baseURI) => {
     const href = new URL(name, base).href;
     let file = files.get(href);
     if (!file) {
@@ -10,8 +9,20 @@ export function FileAttachment(name, base = document.baseURI) {
         files.set(href, file);
     }
     return file;
+};
+export function registerFile(name, info, base = location.href) {
+    const href = new URL(name, base).href;
+    if (info == null) {
+        files.delete(href);
+    }
+    else {
+        const { path, mimeType, lastModified, size } = info;
+        const file = new FileAttachmentImpl(new URL(path, base).href, name.split("/").pop(), mimeType, lastModified, size);
+        files.set(href, file);
+        return file;
+    }
 }
-async function remote_fetch(file) {
+async function fetchFile(file) {
     const response = await fetch(file.href);
     if (!response.ok)
         throw new Error(`Unable to load file: ${file.name}`);
@@ -54,21 +65,21 @@ export class AbstractFile {
         return this.href;
     }
     async blob() {
-        return (await remote_fetch(this)).blob();
+        return (await fetchFile(this)).blob();
     }
     async arrayBuffer() {
-        return (await remote_fetch(this)).arrayBuffer();
+        return (await fetchFile(this)).arrayBuffer();
     }
     async text(encoding) {
         return encoding === undefined
-            ? (await remote_fetch(this)).text()
+            ? (await fetchFile(this)).text()
             : new TextDecoder(encoding).decode(await this.arrayBuffer());
     }
     async json() {
-        return (await remote_fetch(this)).json();
+        return (await fetchFile(this)).json();
     }
     async stream() {
-        return (await remote_fetch(this)).body;
+        return (await fetchFile(this)).body;
     }
     async dsv({ delimiter = ",", array = false, typed = false } = {}) {
         const [text, d3] = await Promise.all([this.text(), import("npm:d3-dsv")]);
@@ -95,7 +106,7 @@ export class AbstractFile {
         });
     }
     async arrow() {
-        const [Arrow, response] = await Promise.all([import("npm:apache-arrow"), remote_fetch(this)]);
+        const [Arrow, response] = await Promise.all([import("npm:apache-arrow"), fetchFile(this)]);
         return Arrow.tableFromIPC(response);
     }
     async arquero(options) {

package/dist/src/runtime/stdlib/generators/input.d.ts

@@ -1 +1 @@
-export declare function input(element: Element): AsyncGenerator<string | number | boolean | string[] | Date | File | FileList | null, void, unknown>;
+export declare function input(element: Element): AsyncGenerator<string | number | boolean | string[] | Date | FileList | File | null, void, unknown>;

package/dist/src/runtime/stdlib/index.d.ts

@@ -1,10 +1,10 @@
-import { Mutable } from "./mutable.js";
-import * as Generators from "./generators/index.js";
-import { FileAttachment } from "./fileAttachment.js";
+import { DatabaseClient } from "./databaseClient.js";
 import * as DOM from "./dom/index.js";
+import { FileAttachment } from "./fileAttachment.js";
+import * as Generators from "./generators/index.js";
+import { Mutable } from "./mutable.js";
 import { Observer } from "./observer.js";
 import { require } from "./require.js";
-import { __sql } from "./sql.js";
 export declare const root: HTMLElement;
 export declare const library: {
     aapl: () => Promise<any>;
@@ -46,11 +46,48 @@ export declare const library: {
     vl: () => Promise<any>;
     now: () => AsyncGenerator<number, void, unknown>;
     width: () => AsyncGenerator<number, void, unknown>;
-    FileAttachment: () => typeof FileAttachment;
+    DatabaseClient: () => {
+        (name: string, options?: import("./databaseClient.js").QueryOptionsSpec): DatabaseClient;
+        hash: (strings: string[], ...params: unknown[]) => Promise<string>;
+        revive: ({ rows, schema, date, ...meta }: import("../../databases/index.js").SerializableQueryResult) => import("./databaseClient.js").QueryResult;
+        prototype: {
+            readonly name: string;
+            readonly options: import("./databaseClient.js").QueryOptions;
+            sql(strings: string[], ...params: import("./databaseClient.js").QueryParam[]): Promise<import("./databaseClient.js").QueryResult>;
+        };
+    };
+    FileAttachment: () => {
+        (name: string, base?: string): FileAttachment;
+        prototype: {
+            href: string;
+            name: string;
+            mimeType: string;
+            lastModified: number | undefined;
+            size: number | undefined;
+            url(): Promise<string>;
+            blob(): Promise<Blob>;
+            arrayBuffer(): Promise<ArrayBuffer>;
+            text(encoding?: string): Promise<string>;
+            json(): Promise<any>;
+            stream(): Promise<ReadableStream<Uint8Array<ArrayBufferLike>>>;
+            dsv({ delimiter, array, typed }?: {
+                delimiter?: string | undefined;
+                array?: boolean | undefined;
+                typed?: boolean | undefined;
+            }): Promise<import("./fileAttachment.js").DsvResult>;
+            csv(options?: Omit<import("./fileAttachment.js").DsvOptions, "delimiter">): Promise<import("./fileAttachment.js").DsvResult>;
+            tsv(options?: Omit<import("./fileAttachment.js").DsvOptions, "delimiter">): Promise<import("./fileAttachment.js").DsvResult>;
+            image(props?: Partial<HTMLImageElement>): Promise<HTMLImageElement>;
+            arrow(): Promise<any>;
+            arquero(options?: any): Promise<any>;
+            parquet(): Promise<any>;
+            xml(mimeType?: DOMParserSupportedType): Promise<Document>;
+            html(): Promise<Document>;
+        };
+    };
     Generators: () => typeof Generators;
     Mutable: () => typeof Mutable;
     DOM: () => typeof DOM;
     require: () => typeof require;
-    __sql: () => typeof __sql;
     __ojs_observer: () => () => Observer;
 };

package/dist/src/runtime/stdlib/index.js

@@ -1,22 +1,22 @@
-import { Mutable } from "./mutable.js";
-import * as Generators from "./generators/index.js";
-import { FileAttachment } from "./fileAttachment.js";
+import { DatabaseClient } from "./databaseClient.js";
 import * as DOM from "./dom/index.js";
+import { FileAttachment } from "./fileAttachment.js";
+import * as Generators from "./generators/index.js";
+import { Mutable } from "./mutable.js";
 import { Observer } from "./observer.js";
 import * as recommendedLibraries from "./recommendedLibraries.js";
 import { require } from "./require.js";
 import * as sampleDatasets from "./sampleDatasets.js";
-import { __sql } from "./sql.js";
 export const root = document.querySelector("main") ?? document.body;
 export const library = {
     now: () => Generators.now(),
     width: () => Generators.width(root),
+    DatabaseClient: () => DatabaseClient,
     FileAttachment: () => FileAttachment,
     Generators: () => Generators,
     Mutable: () => Mutable,
     DOM: () => DOM, // deprecated!
     require: () => require, // deprecated!
-    __sql: () => __sql,
     __ojs_observer: () => () => new Observer(),
     ...recommendedLibraries,
     ...sampleDatasets

package/dist/src/styles/global.css

@@ -17,7 +17,7 @@ html {
   -moz-osx-font-smoothing: grayscale;
   background: var(--theme-background);
   color: var(--theme-foreground);
-  margin-top: 16px;
+  margin: 16px 0;
   padding: 16px 24px;
 }
 

package/dist/src/templates/default.html

@@ -5,24 +5,24 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <style type="text/css">
       @import url("observable:styles/index.css");
-      @import "@fontsource/spline-sans-mono/400.css";
-      @import "@fontsource/spline-sans-mono/600.css";
-      @import "@fontsource/spline-sans-mono/700.css";
-      @import "@fontsource/spline-sans-mono/400-italic.css";
-      @import "@fontsource/spline-sans-mono/600-italic.css";
-      @import "@fontsource/spline-sans-mono/700-italic.css";
-      @import "@fontsource/source-serif-4/400.css";
-      @import "@fontsource/source-serif-4/700.css";
-      @import "@fontsource/source-serif-4/400-italic.css";
-      @import "@fontsource/source-serif-4/700-italic.css";
-      @import "@fontsource/inter/400.css";
-      @import "@fontsource/inter/500.css";
-      @import "@fontsource/inter/600.css";
-      @import "@fontsource/inter/700.css";
-      @import "@fontsource/inter/400-italic.css";
-      @import "@fontsource/inter/500-italic.css";
-      @import "@fontsource/inter/600-italic.css";
-      @import "@fontsource/inter/700-italic.css";
+      @import url("@fontsource/spline-sans-mono/400.css");
+      @import url("@fontsource/spline-sans-mono/600.css");
+      @import url("@fontsource/spline-sans-mono/700.css");
+      @import url("@fontsource/spline-sans-mono/400-italic.css");
+      @import url("@fontsource/spline-sans-mono/600-italic.css");
+      @import url("@fontsource/spline-sans-mono/700-italic.css");
+      @import url("@fontsource/source-serif-4/400.css");
+      @import url("@fontsource/source-serif-4/700.css");
+      @import url("@fontsource/source-serif-4/400-italic.css");
+      @import url("@fontsource/source-serif-4/700-italic.css");
+      @import url("@fontsource/inter/400.css");
+      @import url("@fontsource/inter/500.css");
+      @import url("@fontsource/inter/600.css");
+      @import url("@fontsource/inter/700.css");
+      @import url("@fontsource/inter/400-italic.css");
+      @import url("@fontsource/inter/500-italic.css");
+      @import url("@fontsource/inter/600-italic.css");
+      @import url("@fontsource/inter/700-italic.css");
     </style>
   </head>
   <body>

package/dist/src/vite/config.js

@@ -1,10 +1,12 @@
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { resolveNpmImport } from "../javascript/imports/npm.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 export function config() {
     return {
         base: "./",
+        appType: "mpa", // return 404 for missing pages
         esbuild: {
             supported: {
                 "top-level-await": true
@@ -13,8 +15,9 @@ export function config() {
         resolve: {
             alias: [
                 {
-                    find: /^npm:(.*)$/,
-                    replacement: "https://cdn.jsdelivr.net/npm/$1/+esm"
+                    find: /^(npm:.*)$/,
+                    replacement: "$1",
+                    customResolver: resolveNpmImport
                 },
                 {
                     find: /^jsr:(.*)$/,

package/dist/src/vite/observable.d.ts

@@ -1,12 +1,31 @@
-import type { PluginOption } from "vite";
+import type { PluginOption, IndexHtmlTransformContext } from "vite";
+import type { Notebook } from "../lib/notebook.js";
+/**
+ * A function which performs a per-page transformation of the template HTML.
+ *
+ * @param source The source of the template (typically HTML).
+ * @param context The Vite plugin context.
+ * @returns The transformed template source HTML.
+ */
+export type TemplateTransform = (source: string, context: IndexHtmlTransformContext) => string | Promise<string>;
+/**
+ * A function which transforms the parsed notebook.
+ *
+ * @param notebook The current (parsed) notebook.
+ * @param context The Vite plugin context.
+ * @returns The transformed notebook.
+ */
+export type NotebookTransform = (notebook: Notebook, context: IndexHtmlTransformContext) => Notebook | Promise<Notebook>;
 export interface ObservableOptions {
-    /** The global window, for the default parser and serializer implementations. */
-    window?: Pick<typeof globalThis, "DOMParser" | "XMLSerializer">;
+    /** The global window, for the default parser implementations. */
+    window?: Pick<typeof globalThis, "DOMParser">;
     /** The parser implementation; defaults to `new window.DOMParser()`. */
     parser?: DOMParser;
-    /** The serializer implementation; defaults to `new window.XMLSerializer()`. */
-    serializer?: XMLSerializer;
     /** The path to the page template; defaults to the default template. */
     template?: string;
+    /** An optional function which transforms the template HTML for the current page. */
+    transformTemplate?: TemplateTransform;
+    /** An optional function which transforms the notebook for the current page. */
+    transformNotebook?: NotebookTransform;
 }
-export declare function observable({ window, parser, serializer, template }?: ObservableOptions): PluginOption;
+export declare function observable({ window, parser, template, transformTemplate, transformNotebook }?: ObservableOptions): PluginOption;

package/dist/src/vite/observable.js

@@ -1,16 +1,20 @@
-import { existsSync } from "node:fs";
-import { readFile } from "node:fs/promises";
+import { createReadStream, existsSync } from "node:fs";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { json } from "node:stream/consumers";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { JSDOM } from "jsdom";
+import { getDatabase, isDefaultDatabase } from "../databases/index.js";
+import { isEnoent } from "../lib/error.js";
 import { deserialize } from "../lib/serialize.js";
 import { Sourcemap } from "../javascript/sourcemap.js";
 import { transpile } from "../javascript/transpile.js";
 import { parseTemplate } from "../javascript/template.js";
 import { collectAssets } from "../runtime/stdlib/assets.js";
+import { DatabaseClient } from "../runtime/stdlib/databaseClient.js";
 import { highlight } from "../runtime/stdlib/highlight.js";
 import { MarkdownRenderer } from "../runtime/stdlib/md.js";
-export function observable({ window = new JSDOM().window, parser = new window.DOMParser(), serializer = new window.XMLSerializer(), template = fileURLToPath(import.meta.resolve("../templates/default.html")) } = {}) {
+export function observable({ window = new JSDOM().window, parser = new window.DOMParser(), template = fileURLToPath(import.meta.resolve("../templates/default.html")), transformTemplate = (template) => template, transformNotebook = (notebook) => notebook } = {}) {
     return {
         name: "observable",
         buildStart() {
@@ -24,9 +28,9 @@ export function observable({ window = new JSDOM().window, parser = new window.DO
         transformIndexHtml: {
             order: "pre",
             async handler(input, context) {
-                const notebook = deserialize(input, { parser });
-                const tsource = await readFile(template, "utf-8");
-                const document = parser.parseFromString(tsource, "text/html");
+                const notebook = await transformNotebook(deserialize(input, { parser }), context);
+                const templateHtml = await transformTemplate(await readFile(template, "utf-8"), context);
+                const document = parser.parseFromString(templateHtml, "text/html");
                 const statics = new Set();
                 const assets = new Set();
                 const md = MarkdownRenderer({ document });
@@ -41,26 +45,64 @@ export function observable({ window = new JSDOM().window, parser = new window.DO
                 let cells = document.querySelector("main");
                 cells ?? (cells = document.body.appendChild(document.createElement("main")));
                 for (const cell of notebook.cells) {
-                    const { id, mode, pinned, value } = cell;
+                    const { id, mode, pinned, hidden, value } = cell;
                     const contents = document.createDocumentFragment();
                     const div = contents.appendChild(document.createElement("div"));
                     div.id = `cell-${id}`;
                     div.className = "observablehq observablehq--cell";
-                    if (mode === "md") {
+                    if (mode === "md" && !hidden) {
                         const template = parseTemplate(value);
-                        if (!template.expressions.length)
+                        if (!template.expressions.length && !cell.output)
                             statics.add(cell);
                         const content = md([stripExpressions(template, value)]);
                         const codes = content.querySelectorAll("code[class^=language-]");
                         await Promise.all(Array.from(codes, highlight));
                         div.appendChild(content);
                     }
-                    else if (mode === "html") {
+                    else if (mode === "html" && !hidden) {
                         const template = parseTemplate(value);
-                        if (!template.expressions.length)
+                        if (!template.expressions.length && !cell.output)
                             statics.add(cell);
                         div.innerHTML = stripExpressions(template, value);
                     }
+                    else if (mode === "sql" && cell.database && !cell.database.startsWith("var:")) {
+                        const template = parseTemplate(value);
+                        if (!template.expressions.length) {
+                            const dir = dirname(context.filename);
+                            const cacheDir = join(dir, ".observable", "cache");
+                            const hash = await DatabaseClient.hash.call(null, [value]);
+                            const cacheName = `${cell.database}-${hash}.json`;
+                            const cachePath = join(cacheDir, cacheName);
+                            if (!existsSync(cachePath)) {
+                                let config;
+                                try {
+                                    const configPath = join(dir, ".observable", "databases.json");
+                                    const configStream = createReadStream(configPath, "utf-8");
+                                    const configs = (await json(configStream));
+                                    config = configs[cell.database];
+                                }
+                                catch (error) {
+                                    if (!isEnoent(error))
+                                        throw error;
+                                }
+                                if (isDefaultDatabase(cell.database))
+                                    config ?? (config = { type: cell.database });
+                                if (!config)
+                                    throw new Error(`database not found: ${cell.database}`);
+                                try {
+                                    const database = await getDatabase(config, { cwd: dir });
+                                    const results = await database.call(null, [value]);
+                                    await mkdir(cacheDir, { recursive: true });
+                                    await writeFile(cachePath, JSON.stringify(results));
+                                }
+                                catch (error) {
+                                    console.error(error);
+                                }
+                            }
+                            cell.mode = "js";
+                            cell.value = `FileAttachment(${JSON.stringify(`.observable/cache/${cacheName}`)}).json().then(DatabaseClient.revive)${hidden ? "" : `.then(Inputs.table)${cell.output ? ".then(view)" : ""}`}`;
+                        }
+                    }
                     collectAssets(assets, div);
                     if (pinned) {
                         const pre = contents.appendChild(document.createElement("pre"));
@@ -73,15 +115,16 @@ export function observable({ window = new JSDOM().window, parser = new window.DO
                 }
                 // Don’t error if assets are missing (matching Vite’s behavior).
                 filterMissingAssets(assets, dirname(context.filename));
-                const output = serializer.serializeToString(document);
+                const output = document.documentElement.outerHTML;
                 const i = output.indexOf("</body>");
                 if (!(i >= 0))
                     throw new Error("body not found");
-                return (output.slice(0, i) +
+                return (`<!doctype html>` +
+                    output.slice(0, i) +
                     `<style type="text/css">
 @import url("observable:styles/theme-${notebook.theme}.css");
 </style><script type="module">
-import {define} from "observable:runtime/define";${Array.from(assets)
+import {define} from "observable:runtime";${Array.from(assets)
                         .map((asset, i) => `
 import asset${i + 1} from ${JSON.stringify(`${asset}?url`)};`)
                         .join("")}${assets.size > 0
@@ -96,7 +139,7 @@ ${Array.from(assets)
 ${notebook.cells
                         .filter((cell) => !statics.has(cell))
                         .map((cell) => {
-                        const transpiled = transpile(cell.value, cell.mode, { resolveFiles: true });
+                        const transpiled = transpile(cell, { resolveFiles: true });
                         return `
 define(
   {

package/package.json

@@ -5,7 +5,7 @@
     "type": "git",
     "url": "git+https://github.com/observablehq/notebook-kit.git"
   },
-  "version": "1.0.1",
+  "version": "1.1.0-rc.2",
   "type": "module",
   "scripts": {
     "test": "vitest",
@@ -27,6 +27,10 @@
       "types": "./dist/src/index.d.ts",
       "import": "./dist/src/index.js"
     },
+    "./databases": {
+      "types": "./dist/src/databases/index.d.ts",
+      "import": "./dist/src/databases/index.js"
+    },
     "./runtime": {
       "types": "./dist/src/runtime/index.d.ts",
       "import": "./dist/src/runtime/index.js"
@@ -59,15 +63,34 @@
     "vite": "^7.0.0"
   },
   "devDependencies": {
+    "@duckdb/node-api": "^1.3.2-alpha.26",
     "@eslint/js": "^9.29.0",
     "@types/jsdom": "^21.1.7",
     "@types/markdown-it": "^14.1.2",
     "eslint": "^9.29.0",
     "globals": "^16.2.0",
     "htl": "^0.3.1",
+    "postgres": "^3.4.7",
+    "snowflake-sdk": "^2.1.3",
     "tsx": "^4.20.3",
     "typescript": "^5.8.3",
     "typescript-eslint": "^8.35.0",
     "vitest": "^3.2.4"
+  },
+  "peerDependencies": {
+    "@duckdb/node-api": "^1.3.2-alpha.26",
+    "postgres": "^3.4.7",
+    "snowflake-sdk": "^2.1.3"
+  },
+  "peerDependenciesMeta": {
+    "@duckdb/node-api": {
+      "optional": true
+    },
+    "postgres": {
+      "optional": true
+    },
+    "snowflake-sdk": {
+      "optional": true
+    }
   }
 }

package/dist/src/runtime/stdlib/sql.d.ts

@@ -1,5 +0,0 @@
-type Template = (template: TemplateStringsArray, ...values: unknown[]) => Promise<unknown>;
-export declare function __sql(db: {
-    sql: Template;
-}, render: (data: unknown) => unknown): Template;
-export {};

package/dist/src/runtime/stdlib/sql.js

@@ -1,5 +0,0 @@
-export function __sql(db, render) {
-    return (template, ...values) => {
-        return db.sql.call(db, template, ...values).then(render);
-    };
-}