@cereusdb/standard 0.2.0 → 0.4.0

package/README.md

@@ -10,11 +10,15 @@ This package includes browser object stores for ranged remote Parquet reads and
 npm install @cereusdb/standard
 ```
 
+## Parquet support
+
+Parquet files compressed with Snappy, Gzip, Brotli, LZ4, or ZSTD can be registered.
+
 ## SQL function availability
 
 Current runtime surface:
 
-- `131` runtime `ST_*` names
+- `151` runtime `ST_*` names
 - `0` runtime `RS_*` names
 
 Included function families:
@@ -37,6 +41,44 @@ Not included in this package:
 
 Browser object stores are included in `@cereusdb/standard`. Use `registerObjectStores()` to configure `http`, `s3`, `gcs`, or `azure` providers, then `registerParquetTable()` to register an exact Parquet object or provider-backed prefix.
 
+## Persistent databases (OPFS)
+
+Tables normally live in memory. Persistent databases are stored in the browser's Origin Private File System and survive page reloads:
+
+```ts
+const db = await CereusDB.create({ attach: ['opfs://mydb'] }); // opens or creates mydb
+
+await db.sqlJSON(`CREATE TABLE mydb.public.cities AS SELECT 1 AS id, 'Berlin' AS name, ST_Point(13.4, 52.5) AS geom`);
+await db.sqlJSON(`INSERT INTO mydb.public.cities VALUES (2, 'Paris', ST_Point(2.35, 48.86))`);
+await db.sqlJSON(`USE mydb`); // `cities` now resolves to mydb.public.cities
+```
+
+SQL: `CREATE DATABASE 'opfs://mydb'`, `ATTACH 'opfs://mydb' [AS name]`, `DETACH name`, `DROP DATABASE name`, `USE name`, `ALTER TABLE ... RENAME TO | ADD COLUMN | DROP COLUMN | RENAME COLUMN`, and `SELECT * FROM cereusdb_databases()`. Databases appear as catalogs in `information_schema` and `db.catalog()`. A table's data is loaded the first time it is used. See the [persistent databases guide](https://github.com/tobilg/cereusdb/blob/main/packages/documentation/guides/persistent-databases.md).
+
+## GeoParquet export
+
+```ts
+const bytes = await db.exportGeoParquet('mydb.public.cities'); // Uint8Array
+await db.downloadGeoParquet('mydb.public.cities');            // downloads cities.parquet
+await db.sqlJSON(`COPY (SELECT * FROM cities WHERE id > 1) TO 'some.parquet'`);
+```
+
+See the [GeoParquet export guide](https://github.com/tobilg/cereusdb/blob/main/packages/documentation/guides/geoparquet-export.md).
+
+## Loading the WASM module
+
+The wasm binary ships as a separate file, `dist/wasm/cereusdb_bg.wasm`. The default entry finds it automatically, and bundlers emit it as an asset.
+
+If you host the wasm yourself, or your bundler inlines assets into large `data:application/wasm;base64,...` strings, use the `external` entry. It has no built-in wasm reference and requires `wasmUrl` or `wasmSource`:
+
+```ts
+import { CereusDB } from '@cereusdb/standard/external';
+
+const db = await CereusDB.create({ wasmUrl: '/wasm/cereusdb_bg.wasm' });
+```
+
+Copy `node_modules/@cereusdb/standard/dist/wasm/cereusdb_bg.wasm` to your static assets, or import its URL with Vite: `import wasmUrl from '@cereusdb/standard/wasm?url'`. See the [WASM loading guide](https://github.com/tobilg/cereusdb/blob/main/packages/documentation/guides/wasm-loading.md) for details.
+
 ## JS / TS API
 
 Exports:
@@ -49,10 +91,49 @@ Exports:
 - `RegisterParquetTableOptions`
 - `RasterFormat`
 - `QueryResult`
+- `MemoryStorageBackend`, `OPFSStorageBackend`, `StorageBackend`, `StorageOptions`
+- `CreateDatabaseOptions`, `AttachDatabaseOptions`, `DatabaseListing`
+- `CatalogDatabase`, `CatalogSchema`, `CatalogTable`, `CatalogColumn`
+- `CreateTableDefinition`, `CreateTableOptions`, `AlterTableOperation`
+- `GeoParquetExportOptions`, `DownloadGeoParquetOptions`, `ExportHandler`
+- `downloadFile`, `PARQUET_MIME_TYPE`
 
 Main types:
 
 ```ts
+interface StorageOptions {
+  opfs?: StorageBackend | false; // default: OPFSStorageBackend when the browser has OPFS
+  [scheme: string]: StorageBackend | false | undefined;
+}
+
+interface StorageBackend {
+  readFile(path: string): Promise<Uint8Array | null>;
+  writeFile(path: string, data: Uint8Array): Promise<void>;
+  remove(path: string): Promise<void>;
+  list(path: string): Promise<string[]>;
+  lock?(name: string): Promise<void>;
+  unlock?(name: string): Promise<void>;
+}
+
+type ExportHandler = (filename: string, data: Uint8Array, mimeType: string) => void | Promise<void>;
+
+type CreateTableDefinition = { columns: Record<string, string> } | { as: string };
+
+type AlterTableOperation =
+  | { renameTo: string }
+  | { addColumn: { name: string; type: string; notNull?: boolean; default?: string }; ifNotExists?: boolean }
+  | { dropColumn: string | string[]; ifExists?: boolean }
+  | { renameColumn: { from: string; to: string } };
+
+interface GeoParquetExportOptions {
+  compression?: 'zstd' | 'snappy' | 'lz4' | 'gzip' | 'brotli' | 'uncompressed';
+  rowGroupSize?: number;
+}
+
+interface DownloadGeoParquetOptions extends GeoParquetExportOptions {
+  filename?: string;
+}
+
 type RasterFormat = 'geotiff' | 'tiff';
 type ObjectStoreProvider = 'http' | 's3' | 'gcs' | 'azure';
 
@@ -66,6 +147,9 @@ interface CereusDBOptions {
     | WebAssembly.Module
     | Promise<Response>;
   objectStores?: ObjectStoreRegistryConfig;
+  storage?: StorageOptions;
+  attach?: string[];
+  onExport?: ExportHandler | false;
 }
 
 interface ObjectStoreRegistryConfig {
@@ -104,8 +188,22 @@ class CereusDB {
   registerGeoJSON(name: string, geojson: string | object): void;
   registerRaster(name: string, data: BufferSource, format: RasterFormat): void;
   registerGeoTIFF(name: string, data: BufferSource): void;
-  dropTable(name: string): void;
-  tables(): string[];
+  dropTable(name: string): Promise<void>;
+  catalog(): Promise<CatalogDatabase[]>;
+  createDatabase(location: string, options?: CreateDatabaseOptions): Promise<string>;
+  attachDatabase(location: string, options?: AttachDatabaseOptions): Promise<string>;
+  detachDatabase(name: string, options?: { ifExists?: boolean }): Promise<void>;
+  dropDatabase(nameOrLocation: string, options?: { ifExists?: boolean }): Promise<void>;
+  useDatabase(database: string, schema?: string): Promise<void>;
+  listDatabases(): Promise<DatabaseListing[]>;
+  createTable(name: string, definition: CreateTableDefinition, options?: CreateTableOptions): Promise<void>;
+  alterTable(name: string, operation: AlterTableOperation): Promise<void>;
+  insertArrow(table: string, data: BufferSource): Promise<number>;
+  compactDatabase(name: string): Promise<void>;
+  flush(): Promise<void>;
+  exportGeoParquet(queryOrTable: string, options?: GeoParquetExportOptions): Promise<Uint8Array>;
+  downloadGeoParquet(queryOrTable: string, options?: DownloadGeoParquetOptions): Promise<string>;
+  tables(options?: { qualified?: boolean }): string[];
   version(): string;
 }
 ```
@@ -119,6 +217,8 @@ API notes:
 - S3 temporary credentials use `access_key_id`, `secret_access_key`, and `token`, where `token` is the STS `SessionToken`.
 - S3-compatible endpoints can be configured with `endpoint`; use `allow_http: true` for local HTTP endpoints such as MinIO or LocalStack.
 - `registerRaster()` and `registerGeoTIFF()` are part of the shared wrapper, but raster registration requires `@cereusdb/full`.
+- Persistent databases (`createDatabase()` and related methods), `ALTER TABLE`, `insertArrow()`, `catalog()` and GeoParquet export are available in every package.
+- `downloadGeoParquet()` and `COPY ... TO` pass the file to the `onExport` handler: a browser download by default on the main thread; pass your own handler in Web Workers or Node.js.
 
 ## Example
 

package/dist/external.d.ts

@@ -0,0 +1,293 @@
+import { type StorageBackend } from './storage.js';
+export { MemoryStorageBackend, OPFSStorageBackend, type OPFSStorageBackendOptions, type StorageBackend, } from './storage.js';
+export interface QueryResult {
+    /** Raw JSON data parsed from query */
+    data: Record<string, unknown>[];
+    /** Number of rows */
+    numRows: number;
+    /** Raw Arrow IPC bytes */
+    toIPC(): Uint8Array;
+    /** Convert to array of plain JS objects */
+    toJSON(): Record<string, unknown>[];
+}
+export interface CereusDBOptions {
+    /** Custom WASM module URL (for CDN hosting) */
+    wasmUrl?: string;
+    /** Preloaded WASM bytes/module for Node or custom loaders. */
+    wasmSource?: RequestInfo | URL | Response | BufferSource | WebAssembly.Module | Promise<Response>;
+    /** Browser-backed object stores to register at startup. */
+    objectStores?: ObjectStoreRegistryConfig;
+    /**
+     * Storage backends for persistent databases, by URL scheme. `opfs` defaults
+     * to an {@link OPFSStorageBackend} when the browser supports OPFS; pass
+     * `false` to disable it.
+     */
+    storage?: StorageOptions;
+    /**
+     * Persistent databases to open at startup, for example `['opfs://mydb']`.
+     * Each is created if it does not exist yet.
+     */
+    attach?: string[];
+    /**
+     * Receives files written by `COPY ... TO '<file>'` and
+     * {@link CereusDB.downloadGeoParquet}. Defaults to {@link downloadFile}
+     * (a browser download) when a DOM `document` is available; pass `false` to
+     * disable exports or a function to handle them yourself (for example in a
+     * Web Worker or in Node.js).
+     */
+    onExport?: ExportHandler | false;
+}
+/** Receives an exported file. */
+export type ExportHandler = (filename: string, data: Uint8Array, mimeType: string) => void | Promise<void>;
+/** MIME type of GeoParquet exports. */
+export declare const PARQUET_MIME_TYPE = "application/vnd.apache.parquet";
+/**
+ * Save a file with the browser's download mechanism. Needs a DOM `document`
+ * (the main thread); it does not work in Web Workers or Node.js.
+ */
+export declare function downloadFile(filename: string, data: Uint8Array, mimeType?: string): void;
+export interface StorageOptions {
+    opfs?: StorageBackend | false;
+    [scheme: string]: StorageBackend | false | undefined;
+}
+export interface CreateDatabaseOptions {
+    /** Database (catalog) name. Defaults to the lowercased name in the location URL. */
+    name?: string;
+    /** Open the database if it already exists instead of failing. */
+    ifNotExists?: boolean;
+}
+export interface AttachDatabaseOptions {
+    /** Database (catalog) name. Defaults to the lowercased name in the location URL. */
+    name?: string;
+    /** Succeed if the database is already attached under this name. */
+    ifNotExists?: boolean;
+}
+export interface DatabaseListing {
+    /** Catalog name, or `null` for stored databases that are not attached. */
+    name: string | null;
+    /** `memory` or the storage scheme, for example `opfs`. */
+    storage: string;
+    /** Location URL for persistent databases. */
+    location: string | null;
+    attached: boolean;
+}
+export interface CatalogColumn {
+    name: string;
+    /** Arrow data type. */
+    type: string;
+    nullable: boolean;
+    /** Arrow extension type, for example `geoarrow.wkb` for geometry columns. */
+    extension?: string;
+}
+export interface CatalogTable {
+    name: string;
+    type: 'BASE TABLE' | 'VIEW' | 'LOCAL TEMPORARY';
+    columns: CatalogColumn[];
+}
+export interface CatalogSchema {
+    name: string;
+    tables: CatalogTable[];
+}
+export interface CatalogDatabase {
+    name: string;
+    storage: string;
+    location: string | null;
+    schemas: CatalogSchema[];
+}
+/** Columns as `{ name: 'SQL type' }`, or a query for CREATE TABLE ... AS. */
+export type CreateTableDefinition = {
+    columns: Record<string, string>;
+} | {
+    as: string;
+};
+export interface GeoParquetExportOptions {
+    /** Parquet compression codec. Defaults to `zstd`. */
+    compression?: 'zstd' | 'snappy' | 'lz4' | 'gzip' | 'brotli' | 'uncompressed';
+    /** Maximum number of rows per row group. */
+    rowGroupSize?: number;
+}
+export interface DownloadGeoParquetOptions extends GeoParquetExportOptions {
+    /**
+     * File name. Defaults to `<table>.parquet` for a table and
+     * `export.parquet` for a query.
+     */
+    filename?: string;
+}
+export interface CreateTableOptions {
+    ifNotExists?: boolean;
+    orReplace?: boolean;
+}
+export type AlterTableOperation = {
+    renameTo: string;
+} | {
+    addColumn: {
+        name: string;
+        /** SQL type, for example `VARCHAR` or `DOUBLE`. */
+        type: string;
+        notNull?: boolean;
+        /** SQL expression used for existing rows and as column default. */
+        default?: string;
+    };
+    ifNotExists?: boolean;
+} | {
+    dropColumn: string | string[];
+    ifExists?: boolean;
+} | {
+    renameColumn: {
+        from: string;
+        to: string;
+    };
+};
+export type RasterFormat = 'geotiff' | 'tiff';
+export type ObjectStoreProvider = 'http' | 's3' | 'gcs' | 'azure';
+export interface ObjectStoreRegistryConfig {
+    /** Maximum concurrent browser fetches used by object_store. */
+    maxConcurrency?: number;
+    /** Stores registered by URL prefix. */
+    stores: ObjectStoreConfig[];
+}
+export interface ObjectStoreConfig {
+    /** Optional diagnostic name. */
+    name?: string;
+    /** Backing provider. */
+    provider: ObjectStoreProvider;
+    /** URL prefix, for example https://host, s3://bucket, gs://bucket. */
+    url: string;
+    /** Upstream object_store option keys and primitive values. */
+    options?: Record<string, string | number | boolean>;
+}
+export interface RegisterParquetTableOptions {
+    /** File extension used during listing discovery. Defaults to .parquet. */
+    fileExtension?: string;
+    /** Optional DataFusion target partition count. */
+    targetPartitions?: number;
+}
+export declare class CereusDB {
+    private inner;
+    private storageBackends;
+    private exportHandler?;
+    private constructor();
+    /**
+     * Create and initialize a new CereusDB instance.
+     * This loads the WASM module and initializes the query engine.
+     */
+    static create(options?: CereusDBOptions): Promise<CereusDB>;
+    /**
+     * Execute a SQL query and return results as Arrow IPC bytes.
+     */
+    sql(query: string): Promise<Uint8Array>;
+    /**
+     * Execute a SQL query and return results as JSON.
+     */
+    sqlJSON(query: string): Promise<Record<string, unknown>[]>;
+    /**
+     * Register a remote Parquet file as a table.
+     * The server must support CORS.
+     */
+    registerRemoteParquet(name: string, url: string): Promise<void>;
+    /**
+     * Register browser-backed object stores for ranged and listing reads.
+     */
+    registerObjectStores(config: ObjectStoreRegistryConfig): void;
+    /**
+     * Register a remote Parquet object or prefix through DataFusion's listing table path.
+     */
+    registerParquetTable(name: string, url: string, options?: RegisterParquetTableOptions): Promise<void>;
+    /**
+     * Register a local file (from File API / drag-and-drop) as a table.
+     * Currently supports Parquet, GeoJSON, and GeoTIFF rasters.
+     */
+    registerFile(name: string, file: File): Promise<void>;
+    /**
+     * Register a GeoJSON object or string as a table.
+     */
+    registerGeoJSON(name: string, geojson: string | object): void;
+    /**
+     * Register a raster buffer as a single-column raster table.
+     * Requires the full GDAL-enabled package build.
+     */
+    registerRaster(name: string, data: BufferSource, format: RasterFormat): void;
+    /**
+     * Register a GeoTIFF buffer as a single-column raster table.
+     * Requires the full GDAL-enabled package build.
+     */
+    registerGeoTIFF(name: string, data: BufferSource): void;
+    /**
+     * Drop a table. The table is removed immediately; for tables of persistent
+     * databases the returned promise resolves once the change is written.
+     */
+    dropTable(name: string): Promise<void>;
+    /**
+     * Create a persistent database, for example `opfs://mydb`, and attach it
+     * as a catalog. Equivalent to `CREATE DATABASE 'opfs://mydb'`.
+     * Resolves to the database name.
+     */
+    createDatabase(location: string, options?: CreateDatabaseOptions): Promise<string>;
+    /**
+     * Open an existing persistent database. Equivalent to
+     * `ATTACH 'opfs://mydb' [AS name]`. Resolves to the database name.
+     */
+    attachDatabase(location: string, options?: AttachDatabaseOptions): Promise<string>;
+    /** Write pending changes and close a persistent database. Equivalent to `DETACH name`. */
+    detachDatabase(name: string, options?: {
+        ifExists?: boolean;
+    }): Promise<void>;
+    /**
+     * Drop a database by name or location. Persistent databases are deleted
+     * from storage. Equivalent to `DROP DATABASE`.
+     */
+    dropDatabase(nameOrLocation: string, options?: {
+        ifExists?: boolean;
+    }): Promise<void>;
+    /** Set the default database (and schema). Equivalent to `USE database[.schema]`. */
+    useDatabase(database: string, schema?: string): Promise<void>;
+    /** Attached databases plus stored databases that are not attached. */
+    listDatabases(): Promise<DatabaseListing[]>;
+    /** Describe all databases, schemas, tables and views. */
+    catalog(): Promise<CatalogDatabase[]>;
+    /**
+     * Create a table from column definitions or a query. `name` may be
+     * qualified (`mydb.public.cities`).
+     */
+    createTable(name: string, definition: CreateTableDefinition, options?: CreateTableOptions): Promise<void>;
+    /** Rename a table or add, drop, or rename a column. */
+    alterTable(name: string, operation: AlterTableOperation): Promise<void>;
+    /**
+     * Insert Arrow IPC data (stream or file format, for example from
+     * `tableToIPC()` of apache-arrow) into a table. Columns are matched by
+     * name; missing columns are filled with NULL. Resolves to the number of
+     * inserted rows.
+     */
+    insertArrow(table: string, data: BufferSource): Promise<number>;
+    /**
+     * Export a table or query result as a GeoParquet (1.1) file. Geometry and
+     * geography columns are described in the file's `geo` metadata.
+     *
+     * @param queryOrTable A query (`SELECT ...`, `WITH ...`) or a table name.
+     */
+    exportGeoParquet(queryOrTable: string, options?: GeoParquetExportOptions): Promise<Uint8Array>;
+    /**
+     * Export a table or query result as GeoParquet and pass it to the export
+     * handler: a browser download by default (see `onExport` in
+     * {@link CereusDBOptions}). Resolves to the file name.
+     *
+     * @param queryOrTable A query (`SELECT ...`, `WITH ...`) or a table name.
+     */
+    downloadGeoParquet(queryOrTable: string, options?: DownloadGeoParquetOptions): Promise<string>;
+    /** Rewrite all tables of a persistent database into one segment each. */
+    compactDatabase(name: string): Promise<void>;
+    /** Write pending changes of persistent databases (for example after registerGeoJSON()). */
+    flush(): Promise<void>;
+    /**
+     * List the tables and views of all databases. By default only the table
+     * names are returned, which is ambiguous when databases contain tables with
+     * the same name; pass `{ qualified: true }` for `database.schema.table`
+     * names, or use {@link CereusDB.catalog} for full details.
+     */
+    tables(options?: {
+        qualified?: boolean;
+    }): string[];
+    /** Version string. */
+    version(): string;
+    private objectStoreApi;
+}