@gscdump/engine-duckdb-wasm 1.3.2 → 1.4.1

package/dist/opfs-registry.mjs

@@ -0,0 +1,89 @@
+function createOpfsHandleRegistry(backend) {
+	const entries = /* @__PURE__ */ new Map();
+	const pending = /* @__PURE__ */ new Map();
+	function reportCleanupFailure(operation, error) {
+		console.warn(`[gscdump/engine-duckdb-wasm] ${operation} failed`, error);
+	}
+	async function acquire(name, openHandle) {
+		const existing = entries.get(name);
+		if (existing) {
+			existing.refs++;
+			return;
+		}
+		const inflight = pending.get(name);
+		if (inflight) {
+			await inflight.then(() => void 0, () => void 0);
+			const settled = entries.get(name);
+			if (settled) {
+				settled.refs++;
+				return;
+			}
+			return acquire(name, openHandle);
+		}
+		const registration = (async () => {
+			const handle = await openHandle();
+			await backend.register(name, handle);
+			entries.set(name, {
+				handle,
+				refs: 1
+			});
+		})();
+		pending.set(name, registration);
+		try {
+			await registration;
+		} finally {
+			pending.delete(name);
+		}
+	}
+	async function release(names) {
+		for (const name of names) {
+			const entry = entries.get(name);
+			if (!entry) continue;
+			entry.refs--;
+			if (entry.refs > 0) continue;
+			entries.delete(name);
+			await backend.drop(name).catch((error) => {
+				reportCleanupFailure(`dropping OPFS handle ${name}`, error);
+			});
+		}
+	}
+	const viewRefs = /* @__PURE__ */ new Map();
+	function acquireView(key, signature) {
+		const existing = viewRefs.get(key);
+		if (existing) {
+			if (existing.signature !== signature) throw new Error(`view ${key} is already attached with a different signature`);
+			existing.refs++;
+			return;
+		}
+		viewRefs.set(key, {
+			refs: 1,
+			signature
+		});
+	}
+	async function releaseView(key, drop) {
+		const existing = viewRefs.get(key);
+		const next = (existing?.refs ?? 0) - 1;
+		if (next > 0) {
+			viewRefs.set(key, {
+				refs: next,
+				signature: existing?.signature
+			});
+			return;
+		}
+		viewRefs.delete(key);
+		await drop().catch((error) => {
+			reportCleanupFailure(`dropping view ${key}`, error);
+		});
+	}
+	return {
+		acquire,
+		release,
+		size: () => entries.size,
+		refs: (name) => entries.get(name)?.refs ?? 0,
+		acquireView,
+		releaseView,
+		viewRefs: (key) => viewRefs.get(key)?.refs ?? 0,
+		viewSignature: (key) => viewRefs.get(key)?.signature
+	};
+}
+export { createOpfsHandleRegistry };

package/dist/opfs.d.mts

@@ -0,0 +1,195 @@
+import { AsyncDuckDB, AsyncDuckDBConnection } from "@duckdb/duckdb-wasm";
+/** A parquet data file to materialise into OPFS. */
+interface OpfsParquetFile {
+  /** Same-origin URL carrying a signed size hint + short-lived access token. */
+  url: string;
+  /** Expected byte size — drives progress + a cheap pre-verify shortcut. */
+  bytes: number;
+  /**
+   * Opaque content-stable identifier for this file (e.g. the Iceberg data-
+   * file object key). Encoded into the OPFS filename so the same hash is the
+   * same cache entry. NOT required to be a SHA-256. When omitted, the cache
+   * key falls back to `(table, index)` and only the byte size is verified
+   * (degraded — stale entries can survive a content change).
+   */
+  contentHash?: string;
+  /** Row count — diagnostics only. */
+  rowCount?: number;
+}
+/** One logical table and the parquet files that compose it. */
+interface OpfsParquetTable {
+  /** Iceberg table name — becomes the DuckDB view name. */
+  table: string;
+  files: OpfsParquetFile[];
+  /**
+   * Recent-window overlay parquet (the non-stable tail the lake excludes). When
+   * present it is materialised into OPFS alongside `files` and the view unions
+   * it with an anti-join dedup: the lake (`files`) serves every day it has, the
+   * overlay serves ONLY days the lake lacks. So a stale overlay whose days have
+   * since landed in the lake can't double-count, and a day that stabilised but
+   * isn't yet in the lake still serves from the overlay. Its `contentHash` must
+   * change when the overlay bytes change (it is overwritten in place) so the
+   * cache re-downloads.
+   */
+  overlay?: OpfsParquetFile;
+}
+interface AttachOpfsTablesOptions {
+  db: AsyncDuckDB;
+  conn: AsyncDuckDBConnection;
+  tables: OpfsParquetTable[];
+  /** DuckDB schema the views are created in. Default `main`. */
+  schema?: string;
+  /** `fetch` override (tests). Default `globalThis.fetch`. */
+  fetch?: typeof fetch;
+  /** Request init for the parquet downloads (auth headers, credentials mode). */
+  fetchInit?: RequestInit;
+  /** Caps simultaneous downloads. Default 2. */
+  fetchConcurrency?: number;
+  /** Abort signal threaded through downloads + registration. */
+  signal?: AbortSignal;
+  /** Snapshot version associated with this file set — echoed on the handle. */
+  version?: string;
+  /** Ticks once per file as it lands in OPFS + registers. UI progress. */
+  onFileProgress?: (info: OpfsFileProgress) => void;
+  /**
+   * Low-volume diagnostics for the OPFS attach waterfall. This is intentionally
+   * phase-level rather than a logger so hosts can join it with their own route
+   * timings without parsing console text.
+   */
+  onTiming?: (info: OpfsAttachTiming) => void;
+  /**
+   * Serializer for the operations that mutate the DuckDB-WASM virtual
+   * filesystem / catalog (`registerFileHandle`, `dropFile`, `CREATE`/`DROP
+   * VIEW`) — those are not concurrency-safe across consumers sharing one
+   * `AsyncDuckDB`. Callers with a shared DB pass their global attach mutex
+   * here so ONLY these mutations serialize; the parquet downloads (network +
+   * OPFS writes, no DB interaction) run outside the lock and overlap across
+   * concurrent attaches. Wrapping the whole `attachOpfsParquetTables` call in
+   * the mutex instead serializes every table's downloads end-to-end — the
+   * dominant cold-load cost.
+   *
+   * The returned handle's `detach()` is NOT routed through this — callers
+   * that pass a lock typically already hold it around teardown, and the lock
+   * is not reentrant.
+   *
+   * Default: run inline (caller owns the DB exclusively).
+   */
+  withDb?: <T>(fn: () => Promise<T>) => Promise<T>;
+  /**
+   * Recover a table degraded by an OPFS sync-access-handle / write conflict
+   * (the multi-tab case — another tab holds the file's exclusive handle) by
+   * reading the already-cached bytes via the lock-free File API (or HTTP on a
+   * genuine cache miss) and registering them as in-memory buffers, so the table
+   * attaches from the shared cache instead of being pushed back to the caller.
+   * Quota / incomplete degradations are NEVER recovered this way — buffering a
+   * quota-exceeding set risks OOM, and the caller routes those to its tail.
+   * Default true.
+   */
+  recoverContention?: boolean;
+}
+interface OpfsFileProgress {
+  table: string;
+  /** OPFS file name. */
+  file: string;
+  /** Index within the flat file list. */
+  index: number;
+  /** Total files across all tables. */
+  total: number;
+  /** Bytes of this file (cumulative caller-side). */
+  bytes: number;
+  /** `'cache-hit'` — already in OPFS, verified; `'downloaded'` — fetched. */
+  outcome: 'cache-hit' | 'downloaded';
+  /** Time spent probing/downloading/writing this file before DuckDB registration. */
+  materialiseMs?: number;
+  /** Time spent inside DuckDB's file-registration mutation lock for this file. */
+  registerMs?: number;
+  /** End-to-end file phase time, materialise + register + callback overhead. */
+  totalMs?: number;
+}
+type OpfsAttachTimingStage = 'persist' | 'root' | 'plan' | 'duckdb-import' | 'sweep' | 'materialise' | 'register' | 'downloads' | 'view' | 'recovery' | 'total';
+interface OpfsAttachTiming {
+  stage: OpfsAttachTimingStage;
+  durationMs: number;
+  table?: string;
+  file?: string;
+  files?: number;
+  tables?: number;
+  bytes?: number;
+  outcome?: 'cache-hit' | 'downloaded';
+}
+/** Handle returned from {@link attachOpfsParquetTables}. */
+interface OpfsAttachedHandle {
+  version: string | undefined;
+  /** Tables that successfully attached (a per-table failure drops only that table). */
+  tables: string[];
+  schema: string;
+  /** Total bytes materialised into OPFS for this attach. */
+  bytesAttached: number;
+  /**
+   * Tables that could NOT be attached because OPFS ran out of quota. The
+   * caller routes these to the server tail. Empty on a clean attach.
+   */
+  degradedTables: string[];
+  /**
+   * Why each degraded table degraded. 'quota' is actionable (the caller can
+   * reclaim space via `clearOpfsSnapshotCache` and retry once); 'contention'
+   * and 'incomplete' are transient multi-tab races the buffer fallback
+   * already absorbs.
+   */
+  degradeReasons: Partial<Record<string, 'quota' | 'contention' | 'incomplete'>>;
+  /** Detach the created views + release the registered OPFS file handles. */
+  detach: () => Promise<void>;
+}
+/**
+ * Raised when OPFS cannot hold the file set. Carries the partial state so the
+ * caller can degrade — attach what fit, route the rest server-side.
+ */
+declare class OpfsQuotaExceededError extends Error {
+  name: string;
+  /** Tables that did not fit. */
+  readonly degradedTables: string[];
+  constructor(message: string, degradedTables: string[]);
+}
+/**
+ * Request persistent storage so the browser is less likely to evict the OPFS
+ * cache under pressure. Returns the granted state — `false` is normal for an
+ * un-engaged origin and is NOT an error; it just means eviction is possible.
+ */
+declare function requestPersistentStorage(): Promise<boolean>;
+/** Best-effort `{ usageBytes, quotaBytes }` from the Storage API. */
+declare function estimateOpfsStorage(): Promise<{
+  usageBytes?: number;
+  quotaBytes?: number;
+}>;
+/**
+ * Read a content-addressed OPFS snapshot file via the async File API and return
+ * its bytes, or null when absent / size-mismatched (partial / stale write).
+ *
+ * `getFile()` takes NO lock — unlike DuckDB's `BROWSER_FSACCESS` sync access
+ * handle — so it reads cleanly while ANOTHER tab holds the same file open. That
+ * is the basis for cross-tab cache sharing: a tab that loses the exclusive-handle
+ * race reads the shared cached bytes here instead of re-downloading. The
+ * filename derivation is the SAME `opfsFileName` + `contentHashSlug` the attach
+ * path writes, so consumers reuse the engine's naming with no replicated slug
+ * logic to drift out of lock-step. `index` is only the disambiguator for the
+ * degraded no-`contentHash` fallback (mirrors `attachOpfsParquetTables`).
+ */
+declare function readOpfsSnapshotFile(table: string, contentHash: string | undefined, index: number, expectedBytes: number): Promise<Uint8Array | null>;
+/**
+ * Download every parquet file in `tables` into OPFS, content-hash verify, and
+ * attach them as DuckDB-WASM views. Attach-once: call this once per
+ * `(site, table)` span; re-query without re-attaching for filter / range
+ * changes inside the span.
+ *
+ * Quota handling: a `QuotaExceededError` while writing a table's files does
+ * NOT throw — that table is recorded in `degradedTables` and skipped; the
+ * caller routes it to the server tail. The remaining tables still attach.
+ */
+declare function attachOpfsParquetTables(options: AttachOpfsTablesOptions): Promise<OpfsAttachedHandle>;
+/**
+ * Delete every OPFS entry this module created. Used to reclaim space after a
+ * quota error, or to force a clean re-download. Best-effort — missing entries
+ * are ignored.
+ */
+declare function clearOpfsSnapshotCache(): Promise<void>;
+export { AttachOpfsTablesOptions, OpfsAttachTiming, OpfsAttachTimingStage, OpfsAttachedHandle, OpfsFileProgress, OpfsParquetFile, OpfsParquetTable, OpfsQuotaExceededError, attachOpfsParquetTables, clearOpfsSnapshotCache, estimateOpfsStorage, readOpfsSnapshotFile, requestPersistentStorage };