@gscdump/engine-duckdb-wasm 0.32.9 → 0.32.11

package/dist/index.d.mts

@@ -136,6 +136,12 @@ interface AttachOpfsTablesOptions {
   version?: string;
   /** Ticks once per file as it lands in OPFS + registers. UI progress. */
   onFileProgress?: (info: OpfsFileProgress) => void;
+  /**
+   * Low-volume diagnostics for the OPFS attach waterfall. This is intentionally
+   * phase-level rather than a logger so hosts can join it with their own route
+   * timings without parsing console text.
+   */
+  onTiming?: (info: OpfsAttachTiming) => void;
   /**
    * Serializer for the operations that mutate the DuckDB-WASM virtual
    * filesystem / catalog (`registerFileHandle`, `dropFile`, `CREATE`/`DROP
@@ -178,6 +184,23 @@ interface OpfsFileProgress {
   bytes: number;
   /** `'cache-hit'` — already in OPFS, verified; `'downloaded'` — fetched. */
   outcome: 'cache-hit' | 'downloaded';
+  /** Time spent probing/downloading/writing this file before DuckDB registration. */
+  materialiseMs?: number;
+  /** Time spent inside DuckDB's file-registration mutation lock for this file. */
+  registerMs?: number;
+  /** End-to-end file phase time, materialise + register + callback overhead. */
+  totalMs?: number;
+}
+type OpfsAttachTimingStage = 'persist' | 'root' | 'plan' | 'duckdb-import' | 'sweep' | 'materialise' | 'register' | 'downloads' | 'view' | 'recovery' | 'total';
+interface OpfsAttachTiming {
+  stage: OpfsAttachTimingStage;
+  durationMs: number;
+  table?: string;
+  file?: string;
+  files?: number;
+  tables?: number;
+  bytes?: number;
+  outcome?: 'cache-hit' | 'downloaded';
 }
 /** Handle returned from {@link attachOpfsParquetTables}. */
 interface OpfsAttachedHandle {
@@ -484,4 +507,4 @@ declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, option
   version?: number | string;
   attachedTables?: readonly string[];
 }): BrowserAnalysisRuntime;
-export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, readOpfsSnapshotFile, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
+export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachTiming, type OpfsAttachTimingStage, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, readOpfsSnapshotFile, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };

package/dist/index.mjs

@@ -567,6 +567,28 @@ var OpfsQuotaExceededError = class extends Error {
 const DEFAULT_CONCURRENCY = 2;
 const OPFS_PREFIX = "gscdump-snapshot__";
 const RECOVER_BUFFER_PREFIX = "gscdump-recover__";
+const MAX_CONTENT_HASH_SLUG_CACHE = 4096;
+const contentHashSlugCache = /* @__PURE__ */ new Map();
+function nowMs() {
+	return globalThis.performance?.now?.() ?? Date.now();
+}
+function emitTiming(onTiming, info) {
+	try {
+		onTiming?.(info);
+	} catch {}
+}
+async function timed(onTiming, stage, meta, fn) {
+	const started = nowMs();
+	try {
+		return await fn();
+	} finally {
+		emitTiming(onTiming, {
+			...meta,
+			stage,
+			durationMs: nowMs() - started
+		});
+	}
+}
 const dbRegistries = /* @__PURE__ */ new WeakMap();
 function getOpfsRegistry(db, protocol) {
 	let registry = dbRegistries.get(db);
@@ -636,6 +658,20 @@ async function estimateOpfsStorage() {
 	} : {};
 }
 async function contentHashSlug(contentHash) {
+	const cached = contentHashSlugCache.get(contentHash);
+	if (cached) return cached;
+	if (contentHashSlugCache.size >= MAX_CONTENT_HASH_SLUG_CACHE) {
+		const oldest = contentHashSlugCache.keys().next().value;
+		if (oldest !== void 0) contentHashSlugCache.delete(oldest);
+	}
+	const promise = deriveContentHashSlug(contentHash).catch((err) => {
+		contentHashSlugCache.delete(contentHash);
+		throw err;
+	});
+	contentHashSlugCache.set(contentHash, promise);
+	return promise;
+}
+async function deriveContentHashSlug(contentHash) {
 	const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(contentHash));
 	const bytes = new Uint8Array(digest);
 	let hex = "";
@@ -724,43 +760,49 @@ async function runWithConcurrency$1(items, concurrency, fn) {
 	await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
 }
 async function attachOpfsParquetTables(options) {
-	const { db, conn, tables, schema = "main", fetch: fetchImpl = globalThis.fetch.bind(globalThis), fetchInit, fetchConcurrency = DEFAULT_CONCURRENCY, signal, version, onFileProgress, withDb = (fn) => fn(), recoverContention = true } = options;
-	await requestPersistentStorage();
-	const root = await getOpfsRoot();
+	const { db, conn, tables, schema = "main", fetch: fetchImpl = globalThis.fetch.bind(globalThis), fetchInit, fetchConcurrency = DEFAULT_CONCURRENCY, signal, version, onFileProgress, onTiming, withDb = (fn) => fn(), recoverContention = true } = options;
+	const totalStarted = nowMs();
+	await timed(onTiming, "persist", {}, requestPersistentStorage);
+	const root = await timed(onTiming, "root", {}, getOpfsRoot);
 	const flat = [];
 	const expectedByTable = /* @__PURE__ */ new Map();
-	for (const t of tables) {
-		const expected = /* @__PURE__ */ new Set();
-		for (let i = 0; i < t.files.length; i++) {
-			const file = t.files[i];
-			const slug = file.contentHash ? await contentHashSlug(file.contentHash) : void 0;
-			const name = opfsFileName(t.table, slug, i);
-			flat.push({
-				table: t.table,
-				file,
-				name
-			});
-			expected.add(name);
-		}
-		if (t.overlay) {
-			const slug = t.overlay.contentHash ? await contentHashSlug(t.overlay.contentHash) : void 0;
-			const name = opfsFileName(t.table, slug, t.files.length);
-			flat.push({
-				table: t.table,
-				file: t.overlay,
-				name,
-				overlay: true
-			});
-			expected.add(name);
+	await timed(onTiming, "plan", { tables: tables.length }, async () => {
+		for (const t of tables) {
+			const expected = /* @__PURE__ */ new Set();
+			for (let i = 0; i < t.files.length; i++) {
+				const file = t.files[i];
+				const slug = file.contentHash ? await contentHashSlug(file.contentHash) : void 0;
+				const name = opfsFileName(t.table, slug, i);
+				flat.push({
+					table: t.table,
+					file,
+					name
+				});
+				expected.add(name);
+			}
+			if (t.overlay) {
+				const slug = t.overlay.contentHash ? await contentHashSlug(t.overlay.contentHash) : void 0;
+				const name = opfsFileName(t.table, slug, t.files.length);
+				flat.push({
+					table: t.table,
+					file: t.overlay,
+					name,
+					overlay: true
+				});
+				expected.add(name);
+			}
+			expectedByTable.set(t.table, expected);
 		}
-		expectedByTable.set(t.table, expected);
-	}
+	});
 	const total = flat.length;
 	const overlayNames = /* @__PURE__ */ new Map();
 	const expectedCount = new Map(tables.map((t) => [t.table, t.files.length + (t.overlay ? 1 : 0)]));
-	const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
+	const { DuckDBDataProtocol } = await timed(onTiming, "duckdb-import", {}, () => import("@duckdb/duckdb-wasm"));
 	const registry = getOpfsRegistry(db, DuckDBDataProtocol.BROWSER_FSACCESS);
-	await sweepStaleEntries(root, registry, expectedByTable).catch(() => {});
+	await timed(onTiming, "sweep", {
+		files: total,
+		tables: tables.length
+	}, () => sweepStaleEntries(root, registry, expectedByTable)).catch(() => {});
 	const tableFiles = /* @__PURE__ */ new Map();
 	const degraded = /* @__PURE__ */ new Set();
 	const degradeReason = /* @__PURE__ */ new Map();
@@ -776,10 +818,23 @@ async function attachOpfsParquetTables(options) {
 	const runDownloads = () => runWithConcurrency$1(flat, Math.max(1, fetchConcurrency), async (item, index) => {
 		if (degraded.has(item.table)) return;
 		signal?.throwIfAborted();
+		const fileStarted = nowMs();
 		const name = item.name;
 		let result;
+		let materialiseMs = 0;
+		let registerMs = 0;
 		try {
+			const materialiseStarted = nowMs();
 			result = await materialiseFile(root, name, item.file, fetchImpl, fetchInit, signal);
+			materialiseMs = nowMs() - materialiseStarted;
+			emitTiming(onTiming, {
+				stage: "materialise",
+				durationMs: materialiseMs,
+				table: item.table,
+				file: name,
+				bytes: item.file.bytes,
+				outcome: result.outcome
+			});
 		} catch (err) {
 			if (isAbortError$1(err)) throw err;
 			if (isQuotaError(err)) {
@@ -795,7 +850,17 @@ async function attachOpfsParquetTables(options) {
 			throw err;
 		}
 		try {
+			const registerStarted = nowMs();
 			await withDb(() => registry.acquire(name, () => result.handle));
+			registerMs = nowMs() - registerStarted;
+			emitTiming(onTiming, {
+				stage: "register",
+				durationMs: registerMs,
+				table: item.table,
+				file: name,
+				bytes: item.file.bytes,
+				outcome: result.outcome
+			});
 			acquiredNames.add(name);
 		} catch (err) {
 			if (isAbortError$1(err)) throw err;
@@ -820,11 +885,17 @@ async function attachOpfsParquetTables(options) {
 			index,
 			total,
 			bytes: item.file.bytes,
-			outcome: result.outcome
+			outcome: result.outcome,
+			materialiseMs,
+			registerMs,
+			totalMs: nowMs() - fileStarted
 		});
 	});
 	try {
-		await runDownloads();
+		await timed(onTiming, "downloads", {
+			files: total,
+			tables: tables.length
+		}, runDownloads);
 	} catch (err) {
 		await withDb(() => registry.release([...acquiredNames])).catch(() => {});
 		throw err;
@@ -847,7 +918,10 @@ async function attachOpfsParquetTables(options) {
 		const lakeNames = overlayName ? files.map((f) => f.name).filter((n) => n !== overlayName) : files.map((f) => f.name);
 		try {
 			signal?.throwIfAborted();
-			await withDb(() => conn.query(overlayName ? readParquetViewWithOverlaySql(schema, t.table, lakeNames, overlayName) : readParquetViewSql$1(schema, t.table, lakeNames)));
+			await timed(onTiming, "view", {
+				table: t.table,
+				files: files.length
+			}, () => withDb(() => conn.query(overlayName ? readParquetViewWithOverlaySql(schema, t.table, lakeNames, overlayName) : readParquetViewSql$1(schema, t.table, lakeNames))));
 		} catch (err) {
 			if (isAbortError$1(err)) {
 				await withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames])).catch(() => {});
@@ -868,43 +942,48 @@ async function attachOpfsParquetTables(options) {
 		for (const f of files) registeredNames.push(f.name);
 	}
 	const recoverTableToBuffers = async (t) => {
-		const readOne = async (file, index) => {
-			const cached = await readOpfsSnapshotFile(t.table, file.contentHash, index, file.bytes);
-			if (cached) return cached;
-			signal?.throwIfAborted();
-			const resp = await fetchImpl(file.url, {
-				...fetchInit,
-				signal
+		return timed(onTiming, "recovery", {
+			table: t.table,
+			files: t.files.length + (t.overlay ? 1 : 0)
+		}, async () => {
+			const readOne = async (file, index) => {
+				const cached = await readOpfsSnapshotFile(t.table, file.contentHash, index, file.bytes);
+				if (cached) return cached;
+				signal?.throwIfAborted();
+				const resp = await fetchImpl(file.url, {
+					...fetchInit,
+					signal
+				});
+				if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] recover ${file.url} failed: ${resp.status}`);
+				return new Uint8Array(await resp.arrayBuffer());
+			};
+			const lakeNames = [];
+			for (let i = 0; i < t.files.length; i++) {
+				const buf = await readOne(t.files[i], i);
+				const name = `${RECOVER_BUFFER_PREFIX}${t.table}_${i}.parquet`;
+				await withDb(() => db.registerFileBuffer(name, buf));
+				bufferFiles.push(name);
+				lakeNames.push(name);
+				bytesAttached += t.files[i].bytes;
+			}
+			let overlayName;
+			if (t.overlay) {
+				const buf = await readOne(t.overlay, t.files.length);
+				overlayName = `${RECOVER_BUFFER_PREFIX}${t.table}_overlay.parquet`;
+				await withDb(() => db.registerFileBuffer(overlayName, buf));
+				bufferFiles.push(overlayName);
+				bytesAttached += t.overlay.bytes;
+			}
+			const body = overlayViewBody({
+				lakeSelect: lakeNames.length ? lakeSelect(lakeNames) : null,
+				overlaySelect: overlayName ? `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayName.replace(/'/g, "''")}'], union_by_name = true)` : null,
+				materializeLake: false
 			});
-			if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] recover ${file.url} failed: ${resp.status}`);
-			return new Uint8Array(await resp.arrayBuffer());
-		};
-		const lakeNames = [];
-		for (let i = 0; i < t.files.length; i++) {
-			const buf = await readOne(t.files[i], i);
-			const name = `${RECOVER_BUFFER_PREFIX}${t.table}_${i}.parquet`;
-			await withDb(() => db.registerFileBuffer(name, buf));
-			bufferFiles.push(name);
-			lakeNames.push(name);
-			bytesAttached += t.files[i].bytes;
-		}
-		let overlayName;
-		if (t.overlay) {
-			const buf = await readOne(t.overlay, t.files.length);
-			overlayName = `${RECOVER_BUFFER_PREFIX}${t.table}_overlay.parquet`;
-			await withDb(() => db.registerFileBuffer(overlayName, buf));
-			bufferFiles.push(overlayName);
-			bytesAttached += t.overlay.bytes;
-		}
-		const body = overlayViewBody({
-			lakeSelect: lakeNames.length ? lakeSelect(lakeNames) : null,
-			overlaySelect: overlayName ? `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayName.replace(/'/g, "''")}'], union_by_name = true)` : null,
-			materializeLake: false
+			if (!body) return false;
+			await withDb(() => conn.query(`CREATE OR REPLACE VIEW ${schema}.${t.table} AS ${body}`));
+			bufferViews.push(t.table);
+			return true;
 		});
-		if (!body) return false;
-		await withDb(() => conn.query(`CREATE OR REPLACE VIEW ${schema}.${t.table} AS ${body}`));
-		bufferViews.push(t.table);
-		return true;
 	};
 	if (recoverContention && !signal?.aborted) for (const t of tables) {
 		if (!degraded.has(t.table) || degradeReason.get(t.table) !== "contention") continue;
@@ -915,6 +994,13 @@ async function attachOpfsParquetTables(options) {
 			bufferFiles.length = before;
 		}
 	}
+	emitTiming(onTiming, {
+		stage: "total",
+		durationMs: nowMs() - totalStarted,
+		files: total,
+		tables: tables.length,
+		bytes: bytesAttached
+	});
 	let detached = false;
 	return {
 		version,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-duckdb-wasm",
   "type": "module",
-  "version": "0.32.9",
+  "version": "0.32.11",
   "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
   "author": {
     "name": "Harlan Wilton",
@@ -45,9 +45,9 @@
   },
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
-    "@gscdump/engine": "0.32.9",
-    "@gscdump/contracts": "0.32.9",
-    "gscdump": "0.32.9"
+    "@gscdump/contracts": "0.32.11",
+    "@gscdump/engine": "0.32.11",
+    "gscdump": "0.32.11"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",