@cereusdb/standard 0.2.0 → 0.4.0

package/dist/index.js

@@ -1,4 +1,80 @@
 import init, { CereusDB as WasmCereusDB } from './wasm/cereusdb.js';
+import { OPFSStorageBackend } from './storage.js';
+export { MemoryStorageBackend, OPFSStorageBackend, } from './storage.js';
+/** MIME type of GeoParquet exports. */
+export const PARQUET_MIME_TYPE = 'application/vnd.apache.parquet';
+/**
+ * Save a file with the browser's download mechanism. Needs a DOM `document`
+ * (the main thread); it does not work in Web Workers or Node.js.
+ */
+export function downloadFile(filename, data, mimeType = 'application/octet-stream') {
+    if (typeof document === 'undefined') {
+        throw new Error('downloadFile() needs a browser document; pass CereusDB.create({ onExport }) to handle exports elsewhere');
+    }
+    const url = URL.createObjectURL(new Blob([data], { type: mimeType }));
+    const link = document.createElement('a');
+    link.href = url;
+    link.download = filename;
+    link.style.display = 'none';
+    document.body.appendChild(link);
+    try {
+        link.click();
+    }
+    finally {
+        link.remove();
+        // Revoking right away can cancel the download in some browsers.
+        setTimeout(() => URL.revokeObjectURL(url), 60000);
+    }
+}
+const QUERY_PATTERN = /^\s*(select|with|values|\()/i;
+function resolveExportHandler(option) {
+    if (option === false) {
+        return undefined;
+    }
+    if (option !== undefined) {
+        return option;
+    }
+    return typeof document !== 'undefined' ? downloadFile : undefined;
+}
+function defaultExportFilename(queryOrTable) {
+    if (QUERY_PATTERN.test(queryOrTable)) {
+        return 'export.parquet';
+    }
+    const name = queryOrTable.trim().split('.').pop()?.replace(/^"|"$/g, '');
+    return `${name || 'export'}.parquet`;
+}
+function resolveStorageBackends(options) {
+    const backends = {};
+    for (const [scheme, backend] of Object.entries(options ?? {})) {
+        if (backend) {
+            backends[scheme.toLowerCase()] = backend;
+        }
+    }
+    if (options?.opfs === undefined && OPFSStorageBackend.isSupported()) {
+        backends.opfs = new OPFSStorageBackend();
+    }
+    return backends;
+}
+function alterTableClause(operation) {
+    if ('renameTo' in operation) {
+        return `RENAME TO ${operation.renameTo}`;
+    }
+    if ('addColumn' in operation) {
+        const column = operation.addColumn;
+        const notNull = column.notNull ? ' NOT NULL' : '';
+        const defaultValue = column.default !== undefined ? ` DEFAULT ${column.default}` : '';
+        const ifNotExists = operation.ifNotExists ? 'IF NOT EXISTS ' : '';
+        return `ADD COLUMN ${ifNotExists}${column.name} ${column.type}${notNull}${defaultValue}`;
+    }
+    if ('dropColumn' in operation) {
+        const columns = Array.isArray(operation.dropColumn)
+            ? operation.dropColumn
+            : [operation.dropColumn];
+        const ifExists = operation.ifExists ? 'IF EXISTS ' : '';
+        return columns.map((column) => `DROP COLUMN ${ifExists}${column}`).join(', ');
+    }
+    return `RENAME COLUMN ${operation.renameColumn.from} TO ${operation.renameColumn.to}`;
+}
 function toUint8Array(data) {
     if (ArrayBuffer.isView(data)) {
         return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
@@ -14,6 +90,7 @@ function normalizeRasterFormat(format) {
 }
 export class CereusDB {
     constructor(inner) {
+        this.storageBackends = {};
         this.inner = inner;
     }
     /**
@@ -33,6 +110,17 @@ export class CereusDB {
         if (options?.objectStores !== undefined) {
             db.registerObjectStores(options.objectStores);
         }
+        db.exportHandler = resolveExportHandler(options?.onExport);
+        if (db.exportHandler !== undefined) {
+            inner.register_export_handler(db.exportHandler);
+        }
+        db.storageBackends = resolveStorageBackends(options?.storage);
+        for (const [scheme, backend] of Object.entries(db.storageBackends)) {
+            inner.register_storage_backend(scheme, backend);
+        }
+        for (const location of options?.attach ?? []) {
+            await db.createDatabase(location, { ifNotExists: true });
+        }
         return db;
     }
     /**
@@ -109,13 +197,140 @@ export class CereusDB {
     registerGeoTIFF(name, data) {
         this.registerRaster(name, data, 'geotiff');
     }
-    /** Drop a table. */
+    /**
+     * Drop a table. The table is removed immediately; for tables of persistent
+     * databases the returned promise resolves once the change is written.
+     */
     dropTable(name) {
         this.inner.drop_table(name);
+        return this.inner.flush();
+    }
+    /**
+     * Create a persistent database, for example `opfs://mydb`, and attach it
+     * as a catalog. Equivalent to `CREATE DATABASE 'opfs://mydb'`.
+     * Resolves to the database name.
+     */
+    async createDatabase(location, options = {}) {
+        return await this.inner.attach_database(location, options.name, true, options.ifNotExists ?? false);
+    }
+    /**
+     * Open an existing persistent database. Equivalent to
+     * `ATTACH 'opfs://mydb' [AS name]`. Resolves to the database name.
+     */
+    async attachDatabase(location, options = {}) {
+        return await this.inner.attach_database(location, options.name, false, options.ifNotExists ?? false);
+    }
+    /** Write pending changes and close a persistent database. Equivalent to `DETACH name`. */
+    async detachDatabase(name, options = {}) {
+        await this.inner.detach_database(name, options.ifExists ?? false);
+    }
+    /**
+     * Drop a database by name or location. Persistent databases are deleted
+     * from storage. Equivalent to `DROP DATABASE`.
+     */
+    async dropDatabase(nameOrLocation, options = {}) {
+        await this.inner.drop_database(nameOrLocation, options.ifExists ?? false);
+    }
+    /** Set the default database (and schema). Equivalent to `USE database[.schema]`. */
+    async useDatabase(database, schema) {
+        await this.inner.sql_json(schema === undefined ? `USE ${database}` : `USE ${database}.${schema}`);
+    }
+    /** Attached databases plus stored databases that are not attached. */
+    async listDatabases() {
+        const rows = (await this.sqlJSON('SELECT database_name, storage, location FROM cereusdb_databases()'));
+        const databases = rows.map((row) => ({
+            name: row.database_name,
+            storage: row.storage,
+            location: row.location ?? null,
+            attached: true,
+        }));
+        for (const scheme of Object.keys(this.storageBackends)) {
+            const locations = (await this.inner.list_database_locations(scheme));
+            for (const location of locations) {
+                if (!databases.some((database) => database.location === location)) {
+                    databases.push({ name: null, storage: scheme, location, attached: false });
+                }
+            }
+        }
+        return databases;
+    }
+    /** Describe all databases, schemas, tables and views. */
+    async catalog() {
+        return JSON.parse(await this.inner.catalog_json());
     }
-    /** List registered tables. */
-    tables() {
-        return this.inner.tables();
+    /**
+     * Create a table from column definitions or a query. `name` may be
+     * qualified (`mydb.public.cities`).
+     */
+    async createTable(name, definition, options = {}) {
+        const orReplace = options.orReplace ? 'OR REPLACE ' : '';
+        const ifNotExists = options.ifNotExists ? 'IF NOT EXISTS ' : '';
+        const body = 'as' in definition
+            ? `AS ${definition.as}`
+            : `(${Object.entries(definition.columns)
+                .map(([column, type]) => `${column} ${type}`)
+                .join(', ')})`;
+        await this.inner.sql_json(`CREATE ${orReplace}TABLE ${ifNotExists}${name} ${body}`);
+    }
+    /** Rename a table or add, drop, or rename a column. */
+    async alterTable(name, operation) {
+        await this.inner.sql_json(`ALTER TABLE ${name} ${alterTableClause(operation)}`);
+    }
+    /**
+     * Insert Arrow IPC data (stream or file format, for example from
+     * `tableToIPC()` of apache-arrow) into a table. Columns are matched by
+     * name; missing columns are filled with NULL. Resolves to the number of
+     * inserted rows.
+     */
+    async insertArrow(table, data) {
+        return await this.inner.insert_arrow(table, toUint8Array(data));
+    }
+    /**
+     * Export a table or query result as a GeoParquet (1.1) file. Geometry and
+     * geography columns are described in the file's `geo` metadata.
+     *
+     * @param queryOrTable A query (`SELECT ...`, `WITH ...`) or a table name.
+     */
+    async exportGeoParquet(queryOrTable, options = {}) {
+        const query = QUERY_PATTERN.test(queryOrTable)
+            ? queryOrTable
+            : `SELECT * FROM ${queryOrTable}`;
+        return await this.inner.export_geoparquet(query, options);
+    }
+    /**
+     * Export a table or query result as GeoParquet and pass it to the export
+     * handler: a browser download by default (see `onExport` in
+     * {@link CereusDBOptions}). Resolves to the file name.
+     *
+     * @param queryOrTable A query (`SELECT ...`, `WITH ...`) or a table name.
+     */
+    async downloadGeoParquet(queryOrTable, options = {}) {
+        const { filename, ...exportOptions } = options;
+        const handler = this.exportHandler;
+        if (handler === undefined) {
+            throw new Error('No export handler: downloads need a browser document; pass CereusDB.create({ onExport }) elsewhere');
+        }
+        const data = await this.exportGeoParquet(queryOrTable, exportOptions);
+        const name = filename ?? defaultExportFilename(queryOrTable);
+        await handler(name, data, PARQUET_MIME_TYPE);
+        return name;
+    }
+    /** Rewrite all tables of a persistent database into one segment each. */
+    async compactDatabase(name) {
+        await this.inner.compact_database(name);
+    }
+    /** Write pending changes of persistent databases (for example after registerGeoJSON()). */
+    async flush() {
+        await this.inner.flush();
+    }
+    /**
+     * List the tables and views of all databases. By default only the table
+     * names are returned, which is ambiguous when databases contain tables with
+     * the same name; pass `{ qualified: true }` for `database.schema.table`
+     * names, or use {@link CereusDB.catalog} for full details.
+     */
+    tables(options = {}) {
+        return options.qualified ? this.inner.qualified_tables() : this.inner.tables();
     }
     /** Version string. */
     version() {

package/dist/storage.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Storage backends for persistent databases (`CREATE DATABASE 'opfs://mydb'`).
+ *
+ * A backend stores the files of persistent databases. Paths are relative to
+ * the backend root and use `/` as separator, for example
+ * `mydb/manifest.json` or `mydb/segments/1.arrow`.
+ */
+export interface StorageBackend {
+    /** Read a file. Resolves to `null` if it does not exist. */
+    readFile(path: string): Promise<Uint8Array | null>;
+    /** Create or replace a file. Should replace the file atomically. */
+    writeFile(path: string, data: Uint8Array): Promise<void>;
+    /** Remove a file or directory (recursively). Missing entries are ignored. */
+    remove(path: string): Promise<void>;
+    /** List entry names of a directory; directory names end with `/`. Missing directories list as empty. */
+    list(path: string): Promise<string[]>;
+    /** Acquire an exclusive lock (for example across browser tabs). Rejects if it is held elsewhere. */
+    lock?(name: string): Promise<void>;
+    /** Release a lock acquired with `lock()`. */
+    unlock?(name: string): Promise<void>;
+}
+/**
+ * Keeps files in memory. Useful for tests, Node.js, and environments without
+ * OPFS. Share one instance between CereusDB instances to simulate a reload.
+ */
+export declare class MemoryStorageBackend implements StorageBackend {
+    private readonly files;
+    private readonly locks;
+    readFile(path: string): Promise<Uint8Array | null>;
+    writeFile(path: string, data: Uint8Array): Promise<void>;
+    remove(path: string): Promise<void>;
+    list(path: string): Promise<string[]>;
+    lock(name: string): Promise<void>;
+    unlock(name: string): Promise<void>;
+    /** All stored file paths (for inspection and tests). */
+    paths(): string[];
+}
+export interface OPFSStorageBackendOptions {
+    /** Directory inside the origin private file system. Defaults to `cereusdb`. */
+    directory?: string;
+}
+/**
+ * Stores databases in the Origin Private File System (OPFS). Used for
+ * `opfs://` locations by default when the browser supports OPFS.
+ *
+ * Opening a database takes a Web Lock, so a database can be open in only one
+ * tab or worker at a time.
+ */
+export declare class OPFSStorageBackend implements StorageBackend {
+    private readonly directory;
+    private rootPromise?;
+    private readonly releases;
+    constructor(options?: OPFSStorageBackendOptions);
+    /** Whether OPFS is available in the current environment. */
+    static isSupported(): boolean;
+    private root;
+    private directoryHandle;
+    readFile(path: string): Promise<Uint8Array | null>;
+    writeFile(path: string, data: Uint8Array): Promise<void>;
+    remove(path: string): Promise<void>;
+    list(path: string): Promise<string[]>;
+    lock(name: string): Promise<void>;
+    unlock(name: string): Promise<void>;
+}

package/dist/storage.js

@@ -0,0 +1,211 @@
+function splitPath(path) {
+    return path.split('/').filter((segment) => segment.length > 0);
+}
+function isNotFound(error) {
+    const name = error?.name;
+    return name === 'NotFoundError' || name === 'TypeMismatchError';
+}
+/**
+ * Keeps files in memory. Useful for tests, Node.js, and environments without
+ * OPFS. Share one instance between CereusDB instances to simulate a reload.
+ */
+export class MemoryStorageBackend {
+    constructor() {
+        this.files = new Map();
+        this.locks = new Set();
+    }
+    async readFile(path) {
+        const data = this.files.get(splitPath(path).join('/'));
+        return data ? data.slice() : null;
+    }
+    async writeFile(path, data) {
+        this.files.set(splitPath(path).join('/'), data.slice());
+    }
+    async remove(path) {
+        const key = splitPath(path).join('/');
+        for (const existing of [...this.files.keys()]) {
+            if (key === '' || existing === key || existing.startsWith(`${key}/`)) {
+                this.files.delete(existing);
+            }
+        }
+    }
+    async list(path) {
+        const prefix = splitPath(path).join('/');
+        const entries = new Set();
+        for (const key of this.files.keys()) {
+            if (prefix !== '' && !key.startsWith(`${prefix}/`)) {
+                continue;
+            }
+            const rest = prefix === '' ? key : key.slice(prefix.length + 1);
+            const [head, ...tail] = rest.split('/');
+            entries.add(tail.length > 0 ? `${head}/` : head);
+        }
+        return [...entries].sort();
+    }
+    async lock(name) {
+        if (this.locks.has(name)) {
+            throw new Error('it is already open in another CereusDB instance');
+        }
+        this.locks.add(name);
+    }
+    async unlock(name) {
+        this.locks.delete(name);
+    }
+    /** All stored file paths (for inspection and tests). */
+    paths() {
+        return [...this.files.keys()].sort();
+    }
+}
+/**
+ * Stores databases in the Origin Private File System (OPFS). Used for
+ * `opfs://` locations by default when the browser supports OPFS.
+ *
+ * Opening a database takes a Web Lock, so a database can be open in only one
+ * tab or worker at a time.
+ */
+export class OPFSStorageBackend {
+    constructor(options = {}) {
+        this.releases = new Map();
+        this.directory = options.directory ?? 'cereusdb';
+    }
+    /** Whether OPFS is available in the current environment. */
+    static isSupported() {
+        return (typeof navigator !== 'undefined' &&
+            typeof navigator.storage?.getDirectory === 'function');
+    }
+    root() {
+        this.rootPromise ?? (this.rootPromise = navigator.storage
+            .getDirectory()
+            .then((root) => root.getDirectoryHandle(this.directory, { create: true })));
+        return this.rootPromise;
+    }
+    async directoryHandle(segments, create) {
+        let directory = await this.root();
+        for (const segment of segments) {
+            try {
+                directory = await directory.getDirectoryHandle(segment, { create });
+            }
+            catch (error) {
+                if (isNotFound(error)) {
+                    return null;
+                }
+                throw error;
+            }
+        }
+        return directory;
+    }
+    async readFile(path) {
+        const segments = splitPath(path);
+        const name = segments.pop();
+        if (name === undefined) {
+            return null;
+        }
+        const directory = await this.directoryHandle(segments, false);
+        if (directory === null) {
+            return null;
+        }
+        try {
+            const file = await (await directory.getFileHandle(name)).getFile();
+            return new Uint8Array(await file.arrayBuffer());
+        }
+        catch (error) {
+            if (isNotFound(error)) {
+                return null;
+            }
+            throw error;
+        }
+    }
+    async writeFile(path, data) {
+        const segments = splitPath(path);
+        const name = segments.pop();
+        if (name === undefined) {
+            throw new Error('writeFile requires a file path');
+        }
+        const directory = await this.directoryHandle(segments, true);
+        if (directory === null) {
+            throw new Error(`cannot create directory for ${path}`);
+        }
+        const handle = await directory.getFileHandle(name, { create: true });
+        if (typeof handle.createWritable === 'function') {
+            // Writes go to a swap file that replaces the original on close().
+            const writable = await handle.createWritable();
+            try {
+                await writable.write(data);
+                await writable.close();
+            }
+            catch (error) {
+                await writable.abort().catch(() => undefined);
+                throw error;
+            }
+            return;
+        }
+        // Older Safari only supports synchronous access handles (in workers).
+        const createSyncAccessHandle = handle.createSyncAccessHandle;
+        if (typeof createSyncAccessHandle !== 'function') {
+            throw new Error('this browser cannot write OPFS files outside a worker');
+        }
+        const access = await createSyncAccessHandle.call(handle);
+        try {
+            access.truncate(0);
+            access.write(data, { at: 0 });
+            access.flush();
+        }
+        finally {
+            access.close();
+        }
+    }
+    async remove(path) {
+        const segments = splitPath(path);
+        const name = segments.pop();
+        if (name === undefined) {
+            throw new Error('remove requires a path');
+        }
+        const directory = await this.directoryHandle(segments, false);
+        if (directory === null) {
+            return;
+        }
+        try {
+            await directory.removeEntry(name, { recursive: true });
+        }
+        catch (error) {
+            if (!isNotFound(error)) {
+                throw error;
+            }
+        }
+    }
+    async list(path) {
+        const directory = await this.directoryHandle(splitPath(path), false);
+        if (directory === null) {
+            return [];
+        }
+        const entries = [];
+        const iterable = directory;
+        for await (const [name, handle] of iterable.entries()) {
+            entries.push(handle.kind === 'directory' ? `${name}/` : name);
+        }
+        return entries.sort();
+    }
+    async lock(name) {
+        if (typeof navigator === 'undefined' || navigator.locks === undefined) {
+            return;
+        }
+        await new Promise((resolve, reject) => {
+            navigator.locks
+                .request(name, { ifAvailable: true }, (lock) => {
+                if (lock === null) {
+                    reject(new Error('it is open in another tab or worker'));
+                    return undefined;
+                }
+                resolve();
+                // Hold the lock until unlock() resolves this promise.
+                return new Promise((release) => this.releases.set(name, release));
+            })
+                .catch(reject);
+        });
+    }
+    async unlock(name) {
+        const release = this.releases.get(name);
+        this.releases.delete(name);
+        release?.();
+    }
+}