@observablehq/notebook-kit 1.0.1 → 1.1.0-rc.10

package/dist/bin/build.js

@@ -23,6 +23,10 @@ export default async function run(args) {
                 type: "string",
                 default: "./"
             },
+            minify: {
+                type: "boolean",
+                default: true
+            },
             template: {
                 type: "string"
             },
@@ -42,6 +46,7 @@ export default async function run(args) {
   --template <path>        path to the HTML template
   -o, --out <dir>          path to the output directory (relative to root)
   --base <path>            serving base path; defaults to ./
+  --no-minify              disable JS/CSS minification
   --empty                  whether to empty the output directory before building
   -h, --help               show this message
 `);
@@ -53,6 +58,8 @@ export default async function run(args) {
         root: values.root,
         base: values.base,
         build: {
+            minify: values.minify ? "esbuild" : false,
+            cssMinify: values.minify ? "esbuild" : false,
             outDir: values.out,
             emptyOutDir: values.empty,
             rollupOptions: {

package/dist/bin/download.js

@@ -3,6 +3,8 @@ import { parseArgs } from "node:util";
 import { JSDOM } from "jsdom";
 import { toNotebook } from "../src/lib/notebook.js";
 import { serialize } from "../src/lib/serialize.js";
+const UI_ORIGIN = "https://observablehq.com";
+const API_ORIGIN = "https://api.observablehq.com";
 if (process.argv[1] === import.meta.filename)
     run();
 export default async function run(args) {
@@ -25,25 +27,19 @@ export default async function run(args) {
 `);
         return;
     }
-    const globalDocument = globalThis.document;
-    globalThis.document = new JSDOM().window.document;
-    try {
-        for (const positional of positionals) {
-            let url = new URL(positional, "https://observablehq.com");
-            if (url.origin === "https://observablehq.com") {
-                url = new URL(`/document${url.pathname.replace(/^\/d\//, "/")}`, "https://api.observablehq.com");
-            }
-            const response = await fetch(url);
-            if (!response.ok)
-                throw new Error(`unable to fetch: ${url}`);
-            const { title, nodes } = await response.json();
-            for (const node of nodes)
-                if (node.mode === "js")
-                    node.mode = "ojs";
-            process.stdout.write(serialize(toNotebook({ title, cells: nodes })));
+    const { window } = new JSDOM();
+    for (const positional of positionals) {
+        let url = new URL(positional, UI_ORIGIN);
+        if (url.origin === UI_ORIGIN) {
+            url = new URL(`/document${url.pathname.replace(/^\/d\//, "/")}`, API_ORIGIN);
         }
-    }
-    finally {
-        globalThis.document = globalDocument;
+        const response = await fetch(url);
+        if (!response.ok)
+            throw new Error(`unable to fetch: ${url}`);
+        const { title, nodes } = await response.json();
+        for (const node of nodes)
+            if (node.mode === "js")
+                node.mode = "ojs";
+        process.stdout.write(serialize(toNotebook({ title, cells: nodes }), { document: window.document }));
     }
 }

package/dist/package.json

@@ -5,7 +5,7 @@
         "type": "git",
         "url": "git+https://github.com/observablehq/notebook-kit.git"
     },
-    "version": "1.0.1",
+    "version": "1.1.0-rc.10",
     "type": "module",
     "scripts": {
         "test": "vitest",
@@ -13,7 +13,7 @@
         "lint": "tsc --noEmit && eslint bin src types",
         "notebooks": "tsx bin/notebooks.ts",
         "download": "tsx bin/notebooks.ts download",
-        "docs:preview": "tsx --watch bin/notebooks.ts preview --root docs --template docs/observable.tmpl",
+        "docs:preview": "tsx --watch bin/notebooks.ts preview --base /notebook-kit/ --root docs --template docs/observable.tmpl",
         "docs:build": "tsx bin/notebooks.ts build --root docs --template docs/observable.tmpl -- $(find docs -path 'docs/.observable' -prune -o -name '*.html' -print)"
     },
     "bin": {
@@ -27,6 +27,10 @@
             "types": "./dist/src/index.d.ts",
             "import": "./dist/src/index.js"
         },
+        "./databases": {
+            "types": "./dist/src/databases/index.d.ts",
+            "import": "./dist/src/databases/index.js"
+        },
         "./runtime": {
             "types": "./dist/src/runtime/index.d.ts",
             "import": "./dist/src/runtime/index.js"
@@ -59,15 +63,40 @@
         "vite": "^7.0.0"
     },
     "devDependencies": {
+        "@duckdb/node-api": "^1.3.2-alpha.26",
         "@eslint/js": "^9.29.0",
+        "@types/better-sqlite3": "^7.6.13",
         "@types/jsdom": "^21.1.7",
         "@types/markdown-it": "^14.1.2",
+        "better-sqlite3": "^12.2.0",
         "eslint": "^9.29.0",
         "globals": "^16.2.0",
         "htl": "^0.3.1",
+        "postgres": "^3.4.7",
+        "snowflake-sdk": "^2.1.3",
         "tsx": "^4.20.3",
         "typescript": "^5.8.3",
         "typescript-eslint": "^8.35.0",
         "vitest": "^3.2.4"
+    },
+    "peerDependencies": {
+        "@duckdb/node-api": "^1.3.2-alpha.26",
+        "better-sqlite3": "^12.2.0",
+        "postgres": "^3.4.7",
+        "snowflake-sdk": "^2.1.3"
+    },
+    "peerDependenciesMeta": {
+        "@duckdb/node-api": {
+            "optional": true
+        },
+        "better-sqlite3": {
+            "optional": true
+        },
+        "postgres": {
+            "optional": true
+        },
+        "snowflake-sdk": {
+            "optional": true
+        }
     }
 }

package/dist/src/databases/duckdb.d.ts

@@ -0,0 +1,2 @@
+import type { DatabaseContext, DuckDBConfig, QueryTemplateFunction } from "./index.js";
+export default function duckdb({ path, options }: DuckDBConfig, context: DatabaseContext): QueryTemplateFunction;

package/dist/src/databases/duckdb.js

@@ -0,0 +1,72 @@
+import { DuckDBConnection, DuckDBInstance } from "@duckdb/node-api";
+import { BIGINT, BIT, BLOB, BOOLEAN, DATE, DOUBLE, FLOAT, HUGEINT, INTEGER, INTERVAL, SMALLINT, TIME, TIMESTAMP, TIMESTAMP_MS, TIMESTAMP_NS, TIMESTAMP_S, TIMESTAMPTZ, TINYINT, UBIGINT, UHUGEINT, UINTEGER, USMALLINT, UTINYINT, UUID, VARCHAR, VARINT } from "@duckdb/node-api"; // prettier-ignore
+import { join } from "node:path";
+export default function duckdb({ path, options }, context) {
+    if (path !== undefined)
+        path = join(context.cwd, path);
+    return async (strings, ...params) => {
+        const instance = await DuckDBInstance.create(path, options);
+        const connection = await DuckDBConnection.create(instance);
+        await connection.run(`SET file_search_path=$0`, [context.cwd]);
+        const date = new Date();
+        let result;
+        let rows;
+        try {
+            result = await connection.run(strings.reduce((p, c, i) => `${p}$${i - 1}${c}`), params);
+            rows = await result.getRowObjectsJson();
+        }
+        finally {
+            connection.disconnectSync();
+            instance.closeSync();
+        }
+        return {
+            rows,
+            schema: getResultSchema(result),
+            duration: Date.now() - +date,
+            date
+        };
+    };
+}
+function getResultSchema(result) {
+    return result.columnNames().map((name, i) => ({ name, type: getColumnType(result.columnType(i)) }));
+}
+function getColumnType(type) {
+    switch (type) {
+        case BOOLEAN:
+            return "boolean";
+        case BIT:
+        case TINYINT:
+        case SMALLINT:
+        case INTEGER:
+        case UTINYINT:
+        case USMALLINT:
+        case UINTEGER:
+        case VARINT:
+            return "integer";
+        case BIGINT:
+        case UBIGINT:
+        case HUGEINT:
+        case UHUGEINT:
+            return "bigint";
+        case FLOAT:
+        case DOUBLE:
+            return "number";
+        case TIMESTAMP:
+        case TIMESTAMP_S:
+        case TIMESTAMP_MS:
+        case TIMESTAMP_NS:
+        case TIMESTAMPTZ:
+        case DATE:
+            return "date";
+        case TIME:
+        case VARCHAR:
+        case UUID:
+            return "string";
+        case BLOB:
+            return "buffer";
+        case INTERVAL:
+            return "array";
+        default:
+            return "other";
+    }
+}

package/dist/src/databases/index.d.ts

@@ -0,0 +1,46 @@
+import type { ColumnSchema, QueryParam } from "../runtime/index.js";
+export { hash as getQueryHash, nameHash as getNameHash } from "../lib/hash.js";
+export type DatabaseConfig = DuckDBConfig | SQLiteConfig | SnowflakeConfig | PostgresConfig;
+export type DuckDBConfig = {
+    type: "duckdb";
+    path?: string;
+    options?: {
+        [key: string]: string;
+    };
+};
+export type SQLiteConfig = {
+    type: "sqlite";
+    path?: string;
+};
+export type SnowflakeConfig = {
+    type: "snowflake";
+    account: string;
+    database?: string;
+    role?: string;
+    schema?: string;
+    username?: string;
+    warehouse?: string;
+    password?: string;
+};
+export type PostgresConfig = {
+    type: "postgres";
+    host?: string;
+    port?: string | number;
+    username?: string;
+    password?: string;
+    database?: string;
+    ssl?: boolean;
+};
+export type DatabaseContext = {
+    cwd: string;
+};
+export type QueryTemplateFunction = (strings: readonly string[], ...params: QueryParam[]) => Promise<SerializableQueryResult>;
+export type SerializableQueryResult = {
+    rows: Record<string, unknown>[];
+    schema: ColumnSchema[];
+    duration: number;
+    date: Date;
+};
+export declare function getDatabaseConfig(sourcePath: string, databaseName: string): Promise<DatabaseConfig>;
+export declare function getDatabase(config: DatabaseConfig, context: DatabaseContext): Promise<QueryTemplateFunction>;
+export declare function getQueryCachePath(sourcePath: string, databaseName: string, strings: readonly string[], ...params: unknown[]): Promise<string>;

package/dist/src/databases/index.js

@@ -0,0 +1,54 @@
+import { createReadStream } from "node:fs";
+import { dirname, join } from "node:path";
+import { json } from "node:stream/consumers";
+import { isEnoent } from "../lib/error.js";
+import { hash as getQueryHash, nameHash as getNameHash } from "../lib/hash.js";
+export { hash as getQueryHash, nameHash as getNameHash } from "../lib/hash.js";
+export async function getDatabaseConfig(sourcePath, databaseName) {
+    const sourceDir = dirname(sourcePath);
+    const configPath = join(sourceDir, ".observable", "databases.json");
+    let config;
+    try {
+        const configStream = createReadStream(configPath, "utf-8");
+        const configs = (await json(configStream));
+        config = configs[databaseName];
+    }
+    catch (error) {
+        if (!isEnoent(error))
+            throw error;
+    }
+    if (config === undefined) {
+        if (databaseName === "postgres")
+            config = { type: "postgres" };
+        else if (databaseName === "duckdb")
+            config = { type: "duckdb" };
+        else if (databaseName === "sqlite")
+            config = { type: "sqlite" };
+        else if (/\.duckdb$/i.test(databaseName))
+            config = { type: "duckdb", path: databaseName };
+        else if (/\.db$/i.test(databaseName))
+            config = { type: "sqlite", path: databaseName }; // TODO disambiguate
+        else
+            throw new Error(`database not found: ${databaseName}`);
+    }
+    return config;
+}
+export async function getDatabase(config, context) {
+    switch (config.type) {
+        case "duckdb":
+            return (await import("./duckdb.js")).default(config, context);
+        case "sqlite":
+            return (await import("./sqlite.js")).default(config, context);
+        case "snowflake":
+            return (await import("./snowflake.js")).default(config);
+        case "postgres":
+            return (await import("./postgres.js")).default(config);
+        default:
+            throw new Error(`unsupported database type: ${config["type"]}`);
+    }
+}
+export async function getQueryCachePath(sourcePath, databaseName, strings, ...params) {
+    const sourceDir = dirname(sourcePath);
+    const cacheName = `${await getNameHash(databaseName)}-${await getQueryHash(strings, ...params)}.json`;
+    return join(sourceDir, ".observable", "cache", cacheName);
+}

package/dist/src/databases/options.d.ts

@@ -0,0 +1,3 @@
+export declare const optionalString: (value: unknown) => string | undefined;
+export declare const optionalNumber: (value: unknown) => number | undefined;
+export declare const optionalBoolean: (value: unknown) => boolean | undefined;

package/dist/src/databases/options.js

@@ -0,0 +1,6 @@
+function optional(type) {
+    return (value) => (value == null ? undefined : type(value));
+}
+export const optionalString = optional(String);
+export const optionalNumber = optional(Number);
+export const optionalBoolean = optional(Boolean);

package/dist/src/databases/postgres.d.ts

@@ -0,0 +1,2 @@
+import type { PostgresConfig, QueryTemplateFunction } from "./index.js";
+export default function postgres(options: PostgresConfig): QueryTemplateFunction;

package/dist/src/databases/postgres.js

@@ -0,0 +1,95 @@
+import Postgres from "postgres";
+import { optionalBoolean, optionalNumber, optionalString } from "./options.js";
+export default function postgres(options) {
+    return async (strings, ...params) => {
+        const sql = Postgres({
+            host: optionalString(options.host),
+            port: optionalNumber(options.port),
+            username: optionalString(options.username),
+            password: optionalString(options.password),
+            database: optionalString(options.database),
+            ssl: optionalBoolean(options.ssl) ? { rejectUnauthorized: false } : false
+        });
+        const date = new Date();
+        let rows;
+        try {
+            rows = await sql.unsafe(strings.reduce((p, c, i) => `${p}$${i}${c}`), params);
+        }
+        finally {
+            await sql.end();
+        }
+        return {
+            rows,
+            schema: rows.columns.map(getColumnSchema),
+            duration: Date.now() - +date,
+            date
+        };
+    };
+}
+function getColumnSchema(column) {
+    return { name: column.name, type: getColumnType(column.type) };
+}
+// https://github.com/brianc/node-pg-types/blob/master/lib/textParsers.js#L166
+function getColumnType(oid) {
+    switch (oid) {
+        case 20: // int8
+            return "bigint";
+        case 21: // int2
+        case 23: // int4
+        case 26: // oid
+            return "integer";
+        case 700: // float4/real
+        case 701: // float8/double
+            return "number";
+        case 16: // bool
+            return "boolean";
+        case 1082: // date
+        case 1114: // timestamp without timezone
+        case 1184: // timestamp
+            return "date";
+        case 651: // cidr[]
+        case 1000: // bool[]
+        case 1001: // byte[]
+        case 1002: // string[]
+        case 1005: // int2[]
+        case 1007: // int4[]
+        case 1028: // oid[]
+        case 1016: // int8[]
+        case 1017: // point[]
+        case 1021: // float4[]
+        case 1022: // float8[]
+        case 1231: // numeric[]
+        case 1014: // char[]
+        case 1015: // varchar[]
+        case 1008: // string[]
+        case 1009: // string[]
+        case 1040: // macaddr[]
+        case 1041: // inet[]
+        case 1115: // timestamp without time zone[]
+        case 1182: // date[]
+        case 1185: // timestamp with time zone[]
+        case 1187: // interval[]
+        case 199: // json[]
+        case 3807: // jsonb[]
+        case 3907: // numrange[]
+        case 2951: // uuid[]
+        case 791: // money[]
+        case 1183: // time[]
+        case 1270: // timetz[]
+            return "array";
+        case 1186: // interval
+        case 114: // json
+        case 3802: // jsonb
+        case 600: // point
+        case 718: // circle
+            return "object";
+        case 17: // bytea
+            return "buffer";
+        case 18: // char
+        case 1700: // numeric
+        case 25: // text
+        case 24: // regproc
+        default:
+            return "string";
+    }
+}

package/dist/src/databases/snowflake.d.ts

@@ -0,0 +1,2 @@
+import { QueryTemplateFunction, SnowflakeConfig } from "./index.js";
+export default function snowflake(options: SnowflakeConfig): QueryTemplateFunction;

package/dist/src/databases/snowflake.js

@@ -0,0 +1,102 @@
+import Snowflake from "snowflake-sdk";
+import { optionalString } from "./options.js";
+Snowflake.configure({ logLevel: "OFF" });
+export default function snowflake(options) {
+    return async (strings, ...params) => {
+        const connection = await connect({
+            account: String(options.account),
+            username: optionalString(options.username),
+            password: optionalString(options.password),
+            database: optionalString(options.database),
+            schema: optionalString(options.schema),
+            warehouse: optionalString(options.warehouse),
+            role: optionalString(options.role)
+        });
+        let result;
+        try {
+            result = await execute(connection, strings.join("?"), params);
+        }
+        finally {
+            await destroy(connection);
+        }
+        return result;
+    };
+}
+async function connect(options) {
+    const connection = Snowflake.createConnection(options);
+    await new Promise((resolve, reject) => {
+        connection.connect((error) => {
+            if (error)
+                return reject(error);
+            resolve();
+        });
+    });
+    return connection;
+}
+async function destroy(connection) {
+    await new Promise((resolve, reject) => {
+        connection.destroy((error) => {
+            if (error)
+                return reject(error);
+            resolve();
+        });
+    });
+}
+async function execute(connection, sql, params) {
+    return new Promise((resolve, reject) => {
+        const date = new Date();
+        connection.execute({
+            sqlText: sql,
+            binds: params,
+            complete(error, statement, rows) {
+                if (error)
+                    return reject(error);
+                resolve({
+                    rows: rows,
+                    schema: getStatementSchema(statement),
+                    duration: Date.now() - +date,
+                    date
+                });
+            }
+        });
+    });
+}
+function getStatementSchema(statement) {
+    return statement.getColumns().map(getColumnSchema);
+}
+function getColumnSchema(column) {
+    return { name: column.getName(), type: getColumnType(column), nullable: column.isNullable() };
+}
+function getColumnType(column) {
+    const type = column.getType();
+    switch (type.toLowerCase()) {
+        case "date":
+        case "datetime":
+        case "timestamp":
+        case "timestamp_ltz":
+        case "timestamp_ntz":
+        case "timestamp_tz":
+            return "date";
+        case "time":
+        case "text":
+            return "string";
+        case "fixed":
+            return column.getScale() === 0 ? "integer" : "number";
+        case "float":
+        case "number":
+        case "real":
+            return "number";
+        case "binary":
+            return "buffer";
+        case "array":
+            return "array";
+        case "boolean":
+            return "boolean";
+        case "object":
+        case "variant":
+            return "object";
+        default:
+            console.warn(`unknown type: ${type}`);
+            return "other";
+    }
+}

package/dist/src/databases/sqlite.d.ts

@@ -0,0 +1,2 @@
+import type { DatabaseContext, SQLiteConfig, QueryTemplateFunction } from "./index.js";
+export default function sqlite({ path }: SQLiteConfig, context: DatabaseContext): QueryTemplateFunction;

package/dist/src/databases/sqlite.js

@@ -0,0 +1,64 @@
+import { join } from "node:path";
+import Database from "better-sqlite3";
+export default function sqlite({ path = ":memory:" }, context) {
+    if (path !== undefined)
+        path = join(context.cwd, path);
+    return async (strings, ...params) => {
+        const date = new Date();
+        const database = new Database(path);
+        try {
+            const statement = database.prepare(strings.join("?"));
+            const rows = statement.all(...params);
+            return {
+                rows,
+                schema: getStatementSchema(statement),
+                duration: Date.now() - +date,
+                date
+            };
+        }
+        finally {
+            database.close();
+        }
+    };
+}
+function getStatementSchema(statement) {
+    return statement
+        .columns()
+        .map((column) => ({ name: column.name, type: getColumnType(column.type) }));
+}
+function getColumnType(type) {
+    switch (type) {
+        case "INT":
+        case "INTEGER":
+        case "TINYINT":
+        case "SMALLINT":
+        case "MEDIUMINT":
+        case "BIGINT":
+        case "UNSIGNED BIG INT":
+        case "INT2":
+        case "INT8":
+            return "integer";
+        case "TEXT":
+        case "CLOB":
+            return "string";
+        case "REAL":
+        case "DOUBLE":
+        case "DOUBLE PRECISION":
+        case "FLOAT":
+        case "NUMERIC":
+            return "number";
+        case "BLOB":
+            return "buffer";
+        case "DATE":
+        case "DATETIME":
+            return "string"; // TODO convert strings to Date instances in sql.js
+        case null:
+            return "other";
+        default:
+            return /^(?:(?:(?:VARYING|NATIVE) )?CHARACTER|(?:N|VAR|NVAR)CHAR)\(/.test(type)
+                ? "string"
+                : /^(?:DECIMAL|NUMERIC)\(/.test(type)
+                    ? "number"
+                    : "other";
+    }
+}

package/dist/src/javascript/imports/npm.js

@@ -25,6 +25,8 @@ function getDefaultRange(name) {
     switch (name) {
         case "@duckdb/duckdb-wasm":
             return "@1.29.0"; // https://github.com/duckdb/duckdb-wasm/issues/1994
+        case "apache-arrow":
+            return "@17.0.0"; // to match @duckdb/duckdb-wasm 1.29.0
         default:
             return "";
     }