@gscdump/engine-duckdb-wasm 0.25.13 → 0.25.14

package/dist/index.mjs

@@ -166,7 +166,8 @@ function compileArchetypeSql(query) {
 			}
 			const col = DIM_COLUMN[query.dimension];
 			const facet = facetPredicate(query);
-			let sql = `SELECT ${col} AS ${query.dimension}, ${metricSelectList(query.metrics)} FROM ${table} WHERE ${where.sql}${facet.sql} GROUP BY ${col} ORDER BY ${query.orderBy.metric} ${dir} LIMIT ?`;
+			const totalCol = query.includeTotal ? ", COUNT(*) OVER() AS __total" : "";
+			let sql = `SELECT ${col} AS ${query.dimension}, ${metricSelectList(query.metrics)}${totalCol} FROM ${table} WHERE ${where.sql}${facet.sql} GROUP BY ${col} ORDER BY ${query.orderBy.metric} ${dir} LIMIT ?`;
 			const params = [
 				...where.params,
 				...facet.params,
@@ -489,12 +490,27 @@ function isOpfsAccessHandleConflict(err) {
 	const msg = err instanceof Error ? err.message : String(err);
 	return /createSyncAccessHandle|Access Handle/i.test(msg);
 }
-function opfsFileName(table, index, hashSlug) {
-	const base = `${OPFS_PREFIX}${table}_${index}`;
-	return hashSlug ? `${base}_${hashSlug}.parquet` : `${base}.parquet`;
+function opfsFileName(table, hashSlug, index) {
+	return hashSlug ? `${OPFS_PREFIX}${table}_${hashSlug}.parquet` : `${OPFS_PREFIX}${table}_${index}.parquet`;
 }
-function opfsFileNamePrefix(table, index) {
-	return `${OPFS_PREFIX}${table}_${index}`;
+function escapeRegExp(s) {
+	return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function tableEntryMatcher(table) {
+	return new RegExp(`^${escapeRegExp(`${OPFS_PREFIX}${table}_`)}(?:[0-9a-f]{16}|\\d+(?:_[0-9a-f]{16})?)\\.parquet$`);
+}
+async function sweepStaleEntries(root, registry, expectedByTable) {
+	const dir = root;
+	if (!dir.keys) return;
+	const matchers = [...expectedByTable].map(([table, expected]) => ({
+		expected,
+		re: tableEntryMatcher(table)
+	}));
+	for await (const name of dir.keys()) {
+		const m = matchers.find((m) => m.re.test(name));
+		if (!m || m.expected.has(name) || registry.refs(name) > 0) continue;
+		await root.removeEntry(name).catch(() => {});
+	}
 }
 async function requestPersistentStorage() {
 	const storage = globalThis.navigator?.storage;
@@ -523,7 +539,7 @@ async function getOpfsRoot() {
 	if (!dir) throw new Error("[engine-duckdb-wasm/opfs] OPFS unavailable: navigator.storage.getDirectory missing");
 	return dir.call(globalThis.navigator.storage);
 }
-async function materialiseFile(root, name, staleSweepPrefix, file, fetchImpl, fetchInit, signal) {
+async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
 	signal?.throwIfAborted();
 	let handle;
 	try {
@@ -533,12 +549,6 @@ async function materialiseFile(root, name, staleSweepPrefix, file, fetchImpl, fe
 			outcome: "cache-hit"
 		};
 	} catch {}
-	const dir = root;
-	if (dir.keys) for await (const existing of dir.keys()) {
-		if (existing === name || !existing.startsWith(staleSweepPrefix) || !existing.endsWith(".parquet")) continue;
-		const next = existing.charAt(staleSweepPrefix.length);
-		if (next === "." || next === "_") await root.removeEntry(existing).catch(() => {});
-	}
 	signal?.throwIfAborted();
 	const resp = await fetchImpl(file.url, {
 		...fetchInit,
@@ -596,24 +606,39 @@ async function attachOpfsParquetTables(options) {
 	await requestPersistentStorage();
 	const root = await getOpfsRoot();
 	const flat = [];
+	const expectedByTable = /* @__PURE__ */ new Map();
 	for (const t of tables) {
-		for (let i = 0; i < t.files.length; i++) flat.push({
-			table: t.table,
-			file: t.files[i],
-			fileIndex: i
-		});
-		if (t.overlay) flat.push({
-			table: t.table,
-			file: t.overlay,
-			fileIndex: t.files.length,
-			overlay: true
-		});
+		const expected = /* @__PURE__ */ new Set();
+		for (let i = 0; i < t.files.length; i++) {
+			const file = t.files[i];
+			const slug = file.contentHash ? await contentHashSlug(file.contentHash) : void 0;
+			const name = opfsFileName(t.table, slug, i);
+			flat.push({
+				table: t.table,
+				file,
+				name
+			});
+			expected.add(name);
+		}
+		if (t.overlay) {
+			const slug = t.overlay.contentHash ? await contentHashSlug(t.overlay.contentHash) : void 0;
+			const name = opfsFileName(t.table, slug, t.files.length);
+			flat.push({
+				table: t.table,
+				file: t.overlay,
+				name,
+				overlay: true
+			});
+			expected.add(name);
+		}
+		expectedByTable.set(t.table, expected);
 	}
 	const total = flat.length;
 	const overlayNames = /* @__PURE__ */ new Map();
 	const expectedCount = new Map(tables.map((t) => [t.table, t.files.length + (t.overlay ? 1 : 0)]));
 	const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
 	const registry = getOpfsRegistry(db, DuckDBDataProtocol.BROWSER_FSACCESS);
+	await sweepStaleEntries(root, registry, expectedByTable).catch(() => {});
 	const tableFiles = /* @__PURE__ */ new Map();
 	const degraded = /* @__PURE__ */ new Set();
 	const acquiredNames = /* @__PURE__ */ new Set();
@@ -626,12 +651,10 @@ async function attachOpfsParquetTables(options) {
 	const runDownloads = () => runWithConcurrency$1(flat, Math.max(1, fetchConcurrency), async (item, index) => {
 		if (degraded.has(item.table)) return;
 		signal?.throwIfAborted();
-		const hashSlug = item.file.contentHash ? await contentHashSlug(item.file.contentHash) : void 0;
-		const name = opfsFileName(item.table, item.fileIndex, hashSlug);
-		const sweepPrefix = opfsFileNamePrefix(item.table, item.fileIndex);
+		const name = item.name;
 		let result;
 		try {
-			result = await materialiseFile(root, name, sweepPrefix, item.file, fetchImpl, fetchInit, signal);
+			result = await materialiseFile(root, name, item.file, fetchImpl, fetchInit, signal);
 		} catch (err) {
 			if (isAbortError$1(err)) throw err;
 			if (isQuotaError(err)) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-duckdb-wasm",
   "type": "module",
-  "version": "0.25.13",
+  "version": "0.25.14",
   "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
   "author": {
     "name": "Harlan Wilton",
@@ -45,16 +45,16 @@
   },
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
-    "@gscdump/engine": "0.25.13",
-    "gscdump": "0.25.13",
-    "@gscdump/sdk": "0.25.13"
+    "@gscdump/engine": "0.25.14",
+    "@gscdump/sdk": "0.25.14",
+    "gscdump": "0.25.14"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
-    "@vitest/browser": "^4.1.7",
-    "@vitest/browser-playwright": "^4.1.7",
+    "@vitest/browser": "^4.1.8",
+    "@vitest/browser-playwright": "^4.1.8",
     "playwright": "^1.60.0",
-    "vitest": "^4.1.7"
+    "vitest": "^4.1.8"
   },
   "scripts": {
     "build": "obuild",