@gscdump/engine-duckdb-wasm 0.31.10 → 0.32.1

package/dist/index.d.mts

@@ -154,6 +154,17 @@ interface AttachOpfsTablesOptions {
    * Default: run inline (caller owns the DB exclusively).
    */
   withDb?: <T>(fn: () => Promise<T>) => Promise<T>;
+  /**
+   * Recover a table degraded by an OPFS sync-access-handle / write conflict
+   * (the multi-tab case — another tab holds the file's exclusive handle) by
+   * reading the already-cached bytes via the lock-free File API (or HTTP on a
+   * genuine cache miss) and registering them as in-memory buffers, so the table
+   * attaches from the shared cache instead of being pushed back to the caller.
+   * Quota / incomplete degradations are NEVER recovered this way — buffering a
+   * quota-exceeding set risks OOM, and the caller routes those to its tail.
+   * Default true.
+   */
+  recoverContention?: boolean;
 }
 interface OpfsFileProgress {
   table: string;
@@ -205,6 +216,20 @@ declare function estimateOpfsStorage(): Promise<{
   usageBytes?: number;
   quotaBytes?: number;
 }>;
+/**
+ * Read a content-addressed OPFS snapshot file via the async File API and return
+ * its bytes, or null when absent / size-mismatched (partial / stale write).
+ *
+ * `getFile()` takes NO lock — unlike DuckDB's `BROWSER_FSACCESS` sync access
+ * handle — so it reads cleanly while ANOTHER tab holds the same file open. That
+ * is the basis for cross-tab cache sharing: a tab that loses the exclusive-handle
+ * race reads the shared cached bytes here instead of re-downloading. The
+ * filename derivation is the SAME `opfsFileName` + `contentHashSlug` the attach
+ * path writes, so consumers reuse the engine's naming with no replicated slug
+ * logic to drift out of lock-step. `index` is only the disambiguator for the
+ * degraded no-`contentHash` fallback (mirrors `attachOpfsParquetTables`).
+ */
+declare function readOpfsSnapshotFile(table: string, contentHash: string | undefined, index: number, expectedBytes: number): Promise<Uint8Array | null>;
 /**
  * Download every parquet file in `tables` into OPFS, content-hash verify, and
  * attach them as DuckDB-WASM views. Attach-once: call this once per
@@ -459,4 +484,4 @@ declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, option
   version?: number | string;
   attachedTables?: readonly string[];
 }): BrowserAnalysisRuntime;
-export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
+export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, readOpfsSnapshotFile, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };

package/dist/index.mjs

@@ -566,6 +566,7 @@ var OpfsQuotaExceededError = class extends Error {
 };
 const DEFAULT_CONCURRENCY = 2;
 const OPFS_PREFIX = "gscdump-snapshot__";
+const RECOVER_BUFFER_PREFIX = "gscdump-recover__";
 const dbRegistries = /* @__PURE__ */ new WeakMap();
 function getOpfsRegistry(db, protocol) {
 	let registry = dbRegistries.get(db);
@@ -646,6 +647,17 @@ async function getOpfsRoot() {
 	if (!dir) throw new Error("[engine-duckdb-wasm/opfs] OPFS unavailable: navigator.storage.getDirectory missing");
 	return dir.call(globalThis.navigator.storage);
 }
+async function readOpfsSnapshotFile(table, contentHash, index, expectedBytes) {
+	try {
+		const root = await getOpfsRoot();
+		const name = opfsFileName(table, contentHash ? await contentHashSlug(contentHash) : void 0, index);
+		const file = await (await root.getFileHandle(name)).getFile();
+		if (file.size !== expectedBytes) return null;
+		return new Uint8Array(await file.arrayBuffer());
+	} catch {
+		return null;
+	}
+}
 async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
 	signal?.throwIfAborted();
 	let handle;
@@ -712,7 +724,7 @@ async function runWithConcurrency$1(items, concurrency, fn) {
 	await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
 }
 async function attachOpfsParquetTables(options) {
-	const { db, conn, tables, schema = "main", fetch: fetchImpl = globalThis.fetch.bind(globalThis), fetchInit, fetchConcurrency = DEFAULT_CONCURRENCY, signal, version, onFileProgress, withDb = (fn) => fn() } = options;
+	const { db, conn, tables, schema = "main", fetch: fetchImpl = globalThis.fetch.bind(globalThis), fetchInit, fetchConcurrency = DEFAULT_CONCURRENCY, signal, version, onFileProgress, withDb = (fn) => fn(), recoverContention = true } = options;
 	await requestPersistentStorage();
 	const root = await getOpfsRoot();
 	const flat = [];
@@ -751,7 +763,10 @@ async function attachOpfsParquetTables(options) {
 	await sweepStaleEntries(root, registry, expectedByTable).catch(() => {});
 	const tableFiles = /* @__PURE__ */ new Map();
 	const degraded = /* @__PURE__ */ new Set();
+	const degradeReason = /* @__PURE__ */ new Map();
 	const acquiredNames = /* @__PURE__ */ new Set();
+	const bufferFiles = [];
+	const bufferViews = [];
 	let bytesAttached = 0;
 	const releaseTable = async (table) => {
 		const names = (tableFiles.get(table) ?? []).map((f) => f.name);
@@ -769,10 +784,12 @@ async function attachOpfsParquetTables(options) {
 			if (isAbortError$1(err)) throw err;
 			if (isQuotaError(err)) {
 				degraded.add(item.table);
+				degradeReason.set(item.table, "quota");
 				return;
 			}
 			if (isOpfsWriteConflict(err)) {
 				degraded.add(item.table);
+				degradeReason.set(item.table, "contention");
 				return;
 			}
 			throw err;
@@ -784,6 +801,7 @@ async function attachOpfsParquetTables(options) {
 			if (isAbortError$1(err)) throw err;
 			if (isOpfsAccessHandleConflict(err)) {
 				degraded.add(item.table);
+				degradeReason.set(item.table, "contention");
 				return;
 			}
 			throw err;
@@ -821,6 +839,7 @@ async function attachOpfsParquetTables(options) {
 		const files = tableFiles.get(t.table) ?? [];
 		if (files.length !== (expectedCount.get(t.table) ?? t.files.length)) {
 			degraded.add(t.table);
+			degradeReason.set(t.table, degradeReason.get(t.table) ?? "incomplete");
 			await releaseTable(t.table);
 			continue;
 		}
@@ -837,6 +856,7 @@ async function attachOpfsParquetTables(options) {
 			if (isOpfsAccessHandleConflict(err)) {
 				if (registry.viewRefs(`${schema}.${t.table}`) === 0) await withDb(() => conn.query(`DROP VIEW IF EXISTS ${schema}.${t.table}`)).catch(() => {});
 				degraded.add(t.table);
+				degradeReason.set(t.table, "contention");
 				await releaseTable(t.table);
 				continue;
 			}
@@ -847,10 +867,58 @@ async function attachOpfsParquetTables(options) {
 		registry.acquireView(`${schema}.${t.table}`);
 		for (const f of files) registeredNames.push(f.name);
 	}
+	const recoverTableToBuffers = async (t) => {
+		const readOne = async (file, index) => {
+			const cached = await readOpfsSnapshotFile(t.table, file.contentHash, index, file.bytes);
+			if (cached) return cached;
+			signal?.throwIfAborted();
+			const resp = await fetchImpl(file.url, {
+				...fetchInit,
+				signal
+			});
+			if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] recover ${file.url} failed: ${resp.status}`);
+			return new Uint8Array(await resp.arrayBuffer());
+		};
+		const lakeNames = [];
+		for (let i = 0; i < t.files.length; i++) {
+			const buf = await readOne(t.files[i], i);
+			const name = `${RECOVER_BUFFER_PREFIX}${t.table}_${i}.parquet`;
+			await withDb(() => db.registerFileBuffer(name, buf));
+			bufferFiles.push(name);
+			lakeNames.push(name);
+			bytesAttached += t.files[i].bytes;
+		}
+		let overlayName;
+		if (t.overlay) {
+			const buf = await readOne(t.overlay, t.files.length);
+			overlayName = `${RECOVER_BUFFER_PREFIX}${t.table}_overlay.parquet`;
+			await withDb(() => db.registerFileBuffer(overlayName, buf));
+			bufferFiles.push(overlayName);
+			bytesAttached += t.overlay.bytes;
+		}
+		const body = overlayViewBody({
+			lakeSelect: lakeNames.length ? lakeSelect(lakeNames) : null,
+			overlaySelect: overlayName ? `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayName.replace(/'/g, "''")}'], union_by_name = true)` : null,
+			materializeLake: false
+		});
+		if (!body) return false;
+		await withDb(() => conn.query(`CREATE OR REPLACE VIEW ${schema}.${t.table} AS ${body}`));
+		bufferViews.push(t.table);
+		return true;
+	};
+	if (recoverContention && !signal?.aborted) for (const t of tables) {
+		if (!degraded.has(t.table) || degradeReason.get(t.table) !== "contention") continue;
+		const before = bufferFiles.length;
+		if (await recoverTableToBuffers(t).catch(() => false)) degraded.delete(t.table);
+		else {
+			for (const n of bufferFiles.slice(before)) await withDb(() => db.dropFile(n)).catch(() => {});
+			bufferFiles.length = before;
+		}
+	}
 	let detached = false;
 	return {
 		version,
-		tables: attached,
+		tables: [...attached, ...bufferViews],
 		schema,
 		bytesAttached,
 		degradedTables: [...degraded],
@@ -858,6 +926,8 @@ async function attachOpfsParquetTables(options) {
 			if (detached) return;
 			detached = true;
 			await detachOpfs(registry, conn, schema, attached, registeredNames);
+			for (const v of bufferViews) await conn.query(`DROP VIEW IF EXISTS ${schema}.${v}`).catch(() => {});
+			for (const n of bufferFiles) await db.dropFile(n).catch(() => {});
 		}
 	};
 }
@@ -1307,4 +1377,4 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
 		}
 	};
 }
-export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, OpfsQuotaExceededError, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
+export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, OpfsQuotaExceededError, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, readOpfsSnapshotFile, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-duckdb-wasm",
   "type": "module",
-  "version": "0.31.10",
+  "version": "0.32.1",
   "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
   "author": {
     "name": "Harlan Wilton",
@@ -45,15 +45,15 @@
   },
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
-    "@gscdump/contracts": "0.31.10",
-    "@gscdump/engine": "0.31.10",
-    "gscdump": "0.31.10"
+    "@gscdump/engine": "0.32.1",
+    "@gscdump/contracts": "0.32.1",
+    "gscdump": "0.32.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
     "@vitest/browser": "^4.1.9",
     "@vitest/browser-playwright": "^4.1.9",
-    "playwright": "^1.61.0",
+    "playwright": "^1.61.1",
     "vitest": "^4.1.9"
   },
   "scripts": {