@gscdump/engine-duckdb-wasm 0.21.3 → 0.22.4

package/dist/index.d.mts

@@ -58,7 +58,7 @@ declare function createInsightRunner(opts: InsightRunnerOptions): Promise<Insigh
  * so consumers can add the predicate without an interface change when
  * multi-site snapshots land.
  */
-declare const scopeFor: (table: "pages" | "queries" | "countries" | "page_queries" | "dates" | "search_appearance" | "hourly_pages", opts: ScopedRunnerOptions) => TableScope, mergeScope: typeof import("@gscdump/engine/scope").mergeScope;
+declare const scopeFor: (table: "pages" | "queries" | "countries" | "page_queries" | "dates" | "search_appearance" | "search_appearance_pages" | "search_appearance_queries" | "search_appearance_page_queries" | "hourly_pages", opts: ScopedRunnerOptions) => TableScope, mergeScope: typeof import("@gscdump/engine/scope").mergeScope;
 interface StrikingMomentumOptions {
   /** Anchor date (YYYY-MM-DD). Defaults to today. */
   anchor?: string;
@@ -88,9 +88,11 @@ interface OpfsParquetFile {
   /** Expected byte size — drives progress + a cheap pre-verify shortcut. */
   bytes: number;
   /**
-   * Lowercase hex SHA-256 of the file's bytes (the Iceberg data-file digest).
-   * The OPFS-cached copy is verified against this before it is trusted.
-   * When omitted, only the byte size is checked (degraded trust).
+   * Opaque content-stable identifier for this file (e.g. the Iceberg data-
+   * file object key). Encoded into the OPFS filename so the same hash is the
+   * same cache entry. NOT required to be a SHA-256. When omitted, the cache
+   * key falls back to `(table, index)` and only the byte size is verified
+   * (degraded — stale entries can survive a content change).
    */
   contentHash?: string;
   /** Row count — diagnostics only. */
@@ -258,6 +260,26 @@ interface AttachParquetUrlTablesOptions {
   maxBytes?: number;
   /** Abort signal passed through to URL preflights and registration. */
   signal?: AbortSignal;
+  /**
+   * How DuckDB reads the parquet bytes:
+   *  - `'http'` (default): register the URL as an HTTP file; DuckDB issues
+   *    its own range reads during query execution. Right for large parquet
+   *    where only some column chunks are touched per query.
+   *  - `'buffer'`: fetch the full file once into an `ArrayBuffer` and register
+   *    it via `registerFileBuffer`. DuckDB makes zero HTTP calls after
+   *    registration — every query reads from the in-memory copy. Right for
+   *    tiny files (e.g. `dates` daily-totals parquet, <50 KB per file) where
+   *    the per-file round-trip overhead dominates the actual bytes moved.
+   */
+  attachMode?: 'http' | 'buffer';
+  /**
+   * When `true` (default), skip per-file HEAD/Range preflight if the URL
+   * carries a signed `?s=<bytes>.<sig>` size hint — the size is already
+   * known and DuckDB will learn `Accept-Ranges` from its first read.
+   * Eliminates one round-trip per file on hosts that mint size hints. Set
+   * `false` to force preflight against URLs whose size you don't trust.
+   */
+  trustSizeHint?: boolean;
   /**
    * Manifest version the caller associates with this set of URLs. Returned
    * on the resulting handle so callers can compare against a fresh manifest

package/dist/index.mjs

@@ -399,6 +399,18 @@ var OpfsQuotaExceededError = class extends Error {
 };
 const DEFAULT_CONCURRENCY = 2;
 const OPFS_PREFIX = "gscdump-snapshot__";
+const dbFileRegistrations = /* @__PURE__ */ new WeakMap();
+function isAlreadyRegistered(db, name) {
+	return dbFileRegistrations.get(db)?.has(name) === true;
+}
+function markRegistered(db, name) {
+	let set = dbFileRegistrations.get(db);
+	if (!set) {
+		set = /* @__PURE__ */ new Set();
+		dbFileRegistrations.set(db, set);
+	}
+	set.add(name);
+}
 function isQuotaError(err) {
 	if (typeof err !== "object" || err === null) return false;
 	return err.name === "QuotaExceededError" || err.code === 22;
@@ -406,8 +418,12 @@ function isQuotaError(err) {
 function isAbortError$1(err) {
 	return typeof err === "object" && err !== null && err.name === "AbortError";
 }
-function opfsFileName(table, index) {
-	return `${OPFS_PREFIX}${table}_${index}.parquet`;
+function opfsFileName(table, index, hashSlug) {
+	const base = `${OPFS_PREFIX}${table}_${index}`;
+	return hashSlug ? `${base}_${hashSlug}.parquet` : `${base}.parquet`;
+}
+function opfsFileNamePrefix(table, index) {
+	return `${OPFS_PREFIX}${table}_${index}`;
 }
 async function requestPersistentStorage() {
 	const storage = globalThis.navigator?.storage;
@@ -424,32 +440,34 @@ async function estimateOpfsStorage() {
 		quotaBytes: est.quota
 	} : {};
 }
-async function sha256Hex(bytes) {
-	const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
-	return Array.from(new Uint8Array(digest)).map((b) => b.toString(16).padStart(2, "0")).join("");
+async function contentHashSlug(contentHash) {
+	const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(contentHash));
+	const bytes = new Uint8Array(digest);
+	let hex = "";
+	for (let i = 0; i < 8; i++) hex += bytes[i].toString(16).padStart(2, "0");
+	return hex;
 }
 async function getOpfsRoot() {
 	const dir = globalThis.navigator?.storage?.getDirectory;
 	if (!dir) throw new Error("[engine-duckdb-wasm/opfs] OPFS unavailable: navigator.storage.getDirectory missing");
 	return dir.call(globalThis.navigator.storage);
 }
-async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
+async function materialiseFile(root, name, staleSweepPrefix, file, fetchImpl, fetchInit, signal) {
 	signal?.throwIfAborted();
 	let handle;
 	try {
 		handle = await root.getFileHandle(name);
-		const cached = await handle.getFile();
-		if (cached.size === file.bytes) {
-			if (!file.contentHash) return {
-				handle,
-				outcome: "cache-hit"
-			};
-			if (await sha256Hex(await cached.arrayBuffer()) === file.contentHash.toLowerCase()) return {
-				handle,
-				outcome: "cache-hit"
-			};
-		}
+		if ((await handle.getFile()).size === file.bytes) return {
+			handle,
+			outcome: "cache-hit"
+		};
 	} catch {}
+	const dir = root;
+	if (dir.keys) for await (const existing of dir.keys()) {
+		if (existing === name || !existing.startsWith(staleSweepPrefix) || !existing.endsWith(".parquet")) continue;
+		const next = existing.charAt(staleSweepPrefix.length);
+		if (next === "." || next === "_") await root.removeEntry(existing).catch(() => {});
+	}
 	signal?.throwIfAborted();
 	const resp = await fetchImpl(file.url, {
 		...fetchInit,
@@ -457,10 +475,6 @@ async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
 	});
 	if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} failed: ${resp.status}`);
 	const buf = await resp.arrayBuffer();
-	if (file.contentHash) {
-		const hash = await sha256Hex(buf);
-		if (hash !== file.contentHash.toLowerCase()) throw new Error(`[engine-duckdb-wasm/opfs] content-hash mismatch for ${file.url}: expected ${file.contentHash}, got ${hash}`);
-	}
 	handle = await root.getFileHandle(name, { create: true });
 	const writable = await handle.createWritable();
 	try {
@@ -513,10 +527,12 @@ async function attachOpfsParquetTables(options) {
 	await runWithConcurrency$1(flat, Math.max(1, fetchConcurrency), async (item, index) => {
 		if (degraded.has(item.table)) return;
 		signal?.throwIfAborted();
-		const name = opfsFileName(item.table, item.fileIndex);
+		const hashSlug = item.file.contentHash ? await contentHashSlug(item.file.contentHash) : void 0;
+		const name = opfsFileName(item.table, item.fileIndex, hashSlug);
+		const sweepPrefix = opfsFileNamePrefix(item.table, item.fileIndex);
 		let result;
 		try {
-			result = await materialiseFile(root, name, item.file, fetchImpl, fetchInit, signal);
+			result = await materialiseFile(root, name, sweepPrefix, item.file, fetchImpl, fetchInit, signal);
 		} catch (err) {
 			if (isAbortError$1(err)) throw err;
 			if (isQuotaError(err)) {
@@ -525,7 +541,10 @@ async function attachOpfsParquetTables(options) {
 			}
 			throw err;
 		}
-		await db.registerFileHandle(name, result.handle, DuckDBDataProtocol.BROWSER_FSACCESS, true);
+		if (!isAlreadyRegistered(db, name)) {
+			await db.registerFileHandle(name, result.handle, DuckDBDataProtocol.BROWSER_FSACCESS, true);
+			markRegistered(db, name);
+		}
 		const list = tableFiles.get(item.table) ?? [];
 		list.push({
 			name,
@@ -575,9 +594,8 @@ async function attachOpfsParquetTables(options) {
 		}
 	};
 }
-async function detachOpfs(db, conn, schema, tables, files) {
+async function detachOpfs(db, conn, schema, tables, _files) {
 	for (const table of tables) await conn.query(`DROP VIEW IF EXISTS ${schema}.${table}`).catch(() => {});
-	if (files.length > 0) await db.dropFiles([...files]).catch(() => {});
 }
 async function clearOpfsSnapshotCache() {
 	const root = await getOpfsRoot().catch(() => null);
@@ -725,9 +743,9 @@ function rangeOnlyConfig(config) {
 	return {
 		...config ?? {},
 		filesystem: {
-			...config?.filesystem ?? {},
 			reliableHeadRequests: true,
 			allowFullHTTPReads: false,
+			...config?.filesystem ?? {},
 			forceFullHTTPReads: false
 		}
 	};
@@ -764,7 +782,7 @@ async function attachParquetTables(options) {
 	}
 }
 async function attachParquetUrlTables(options) {
-	const { db, conn, tables, fetch: fetchImpl = globalThis.fetch.bind(globalThis), schema = "main", fetchInit, fetchConcurrency, maxFiles, maxBytes, signal, version, onFileAttached } = options;
+	const { db, conn, tables, fetch: fetchImpl = globalThis.fetch.bind(globalThis), schema = "main", fetchInit, fetchConcurrency, maxFiles, maxBytes, signal, version, onFileAttached, attachMode = "http", trustSizeHint = true } = options;
 	const concurrency = positiveInteger(fetchConcurrency, DEFAULT_ATTACH_FETCH_CONCURRENCY, "fetchConcurrency");
 	const fileBudget = positiveInteger(maxFiles, DEFAULT_ATTACH_MAX_FILES, "maxFiles");
 	const byteBudget = positiveInteger(maxBytes, DEFAULT_ATTACH_MAX_BYTES, "maxBytes");
@@ -791,11 +809,21 @@ async function attachParquetUrlTables(options) {
 	const effectiveSignal = mergeAbortSignals(signal, budgetController.signal);
 	let plannedBytes = 0;
 	const total = flat.length;
-	const preflighted = [];
+	const prepared = [];
 	await runWithConcurrency(flat, concurrency, async ({ table, url, index }) => {
 		if (tableFailures.has(table)) return;
 		effectiveSignal?.throwIfAborted();
-		await preflightHttpUrl(url, fetchImpl, fetchInit, effectiveSignal).then((bytes) => {
+		try {
+			let bytes = null;
+			let body = null;
+			if (attachMode === "buffer") {
+				const res = await fetchImpl(url, fetchInitFor(fetchInit, "GET", effectiveSignal));
+				if (!res.ok) throw new Error(`GET ${url} failed: ${res.status}`);
+				const buf = new Uint8Array(await res.arrayBuffer());
+				body = buf;
+				bytes = buf.byteLength;
+			} else if (trustSizeHint && sizeHintFromUrl(url) !== null) bytes = sizeHintFromUrl(url);
+			else bytes = await preflightHttpUrl(url, fetchImpl, fetchInit, effectiveSignal);
 			plannedBytes += bytes;
 			if (plannedBytes > byteBudget) {
 				const err = new BrowserAttachBudgetExceededError(`browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${byteBudget}`);
@@ -803,25 +831,28 @@ async function attachParquetUrlTables(options) {
 				throw err;
 			}
 			effectiveSignal?.throwIfAborted();
-			preflighted.push({
+			prepared.push({
 				table,
 				url,
 				index,
-				name: attachFileName(attachId, table, index)
+				name: attachFileName(attachId, table, index),
+				bytes,
+				body
 			});
-		}).catch((err) => {
+		} catch (err) {
 			if (effectiveSignal?.aborted || err instanceof BrowserAttachBudgetExceededError || isAbortError(err)) throw err;
 			tableFailures.set(table, err instanceof Error ? err : new Error(String(err)));
-		});
+		}
 	});
 	const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
 	const attached = [];
 	const registeredFiles = [];
 	try {
-		for (const file of preflighted) {
+		for (const file of prepared) {
 			if (tableFailures.has(file.table)) continue;
 			effectiveSignal?.throwIfAborted();
-			await db.registerFileURL(file.name, file.url, DuckDBDataProtocol.HTTP, false);
+			if (file.body !== null) await db.registerFileBuffer(file.name, file.body);
+			else await db.registerFileURL(file.name, file.url, DuckDBDataProtocol.HTTP, false);
 			registeredFiles.push(file.name);
 			onFileAttached?.({
 				table: file.table,
@@ -831,7 +862,7 @@ async function attachParquetUrlTables(options) {
 		}
 		for (const table of Object.keys(counts)) {
 			if (tableFailures.has(table)) continue;
-			const files = preflighted.filter((file) => file.table === table).sort((a, b) => a.index - b.index);
+			const files = prepared.filter((file) => file.table === table).sort((a, b) => a.index - b.index);
 			if (files.length !== counts[table]) continue;
 			effectiveSignal?.throwIfAborted();
 			await conn.query(readParquetViewSql(schema, table, files.map((file) => file.name)));

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-duckdb-wasm",
   "type": "module",
-  "version": "0.21.3",
+  "version": "0.22.4",
   "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
   "author": {
     "name": "Harlan Wilton",
@@ -45,13 +45,13 @@
   },
   "dependencies": {
     "drizzle-orm": "^0.45.2",
-    "@gscdump/engine": "0.21.3",
-    "@gscdump/sdk": "0.21.3",
-    "gscdump": "0.21.3"
+    "@gscdump/engine": "0.22.4",
+    "@gscdump/sdk": "0.22.4",
+    "gscdump": "0.22.4"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
-    "vitest": "^4.1.6"
+    "vitest": "^4.1.7"
   },
   "scripts": {
     "build": "obuild",