@gscdump/engine-duckdb-wasm 0.32.11 → 0.33.0

package/dist/index.mjs

@@ -20,11 +20,16 @@ const METRIC_SQL = {
 const DIM_COLUMN = {
 	page: "url",
 	query: "query",
-	queryCanonical: "query_canonical",
 	country: "country",
 	date: "date",
 	searchAppearance: "search_appearance"
 };
+function queryCanonicalExpr(queryRef = "query") {
+	return `COALESCE((SELECT qd.query_canonical FROM query_dim qd WHERE qd.query = ${queryRef} LIMIT 1), ${queryRef})`;
+}
+function dimExpr(dim) {
+	return dim === "queryCanonical" ? queryCanonicalExpr() : DIM_COLUMN[dim] ?? dim;
+}
 function tableForDimensions(dims) {
 	const set = new Set(dims.filter((d) => d !== "date"));
 	if (set.has("page") && (set.has("query") || set.has("queryCanonical"))) return "page_queries";
@@ -128,7 +133,7 @@ function facetPredicate(query) {
 	const parts = [];
 	const params = [];
 	for (const f of facets) {
-		const col = DIM_COLUMN[f.column] ?? f.column;
+		const col = dimExpr(f.column);
 		switch (f.op) {
 			case "eq":
 				parts.push(`${col} = ?`);
@@ -163,7 +168,7 @@ function compileArchetypeSql(query) {
 		}
 		case "entity-daily-timeseries": {
 			const table = tableForDimensions([query.entity.dimension]);
-			const col = DIM_COLUMN[query.entity.dimension];
+			const col = dimExpr(query.entity.dimension);
 			const where = rangePredicate(query);
 			return {
 				table,
@@ -173,7 +178,7 @@ function compileArchetypeSql(query) {
 		}
 		case "entity-daily-sparkline": {
 			const table = tableForDimensions([query.dimension]);
-			const col = DIM_COLUMN[query.dimension];
+			const col = dimExpr(query.dimension);
 			const where = rangePredicate(query);
 			if (query.entities.length === 0) throw new Error("[archetype-sql] entity-daily-sparkline requires resolved entities");
 			const placeholders = query.entities.map(() => "?").join(", ");
@@ -231,7 +236,7 @@ function compileArchetypeSql(query) {
 					params
 				};
 			}
-			const col = DIM_COLUMN[query.dimension];
+			const col = dimExpr(query.dimension);
 			const facet = facetPredicate(query);
 			const totalCol = query.includeTotal ? ", COUNT(*) OVER() AS __total" : "";
 			if (cmp) {
@@ -281,7 +286,7 @@ function compileArchetypeSql(query) {
 			const matchParts = [];
 			const matchParams = [];
 			for (const [dim, value] of Object.entries(query.match)) {
-				const col = DIM_COLUMN[dim];
+				const col = dimExpr(dim);
 				if (!col) throw new Error(`[archetype-sql] single-row-lookup: unknown dimension ${dim}`);
 				matchParts.push(`${col} = ?`);
 				matchParams.push(value);
@@ -305,7 +310,7 @@ function compileArchetypeSql(query) {
 					...where.params
 				]
 			};
-			const col = DIM_COLUMN[query.seriesDimension];
+			const col = dimExpr(query.seriesDimension);
 			return {
 				table,
 				sql: `SELECT date, ${col} AS ${query.seriesDimension}, ${metricExpr(query.metric)} AS ${query.metric} FROM ${table} WHERE ${where.sql} GROUP BY date, ${col} ORDER BY date, ${col}`,
@@ -525,13 +530,26 @@ function createOpfsHandleRegistry(backend) {
 		}
 	}
 	const viewRefs = /* @__PURE__ */ new Map();
-	function acquireView(key) {
-		viewRefs.set(key, (viewRefs.get(key) ?? 0) + 1);
+	function acquireView(key, signature) {
+		const existing = viewRefs.get(key);
+		if (existing) {
+			if (existing.signature !== signature) throw new Error(`view ${key} is already attached with a different signature`);
+			existing.refs++;
+			return;
+		}
+		viewRefs.set(key, {
+			refs: 1,
+			signature
+		});
 	}
 	async function releaseView(key, drop) {
-		const next = (viewRefs.get(key) ?? 0) - 1;
+		const existing = viewRefs.get(key);
+		const next = (existing?.refs ?? 0) - 1;
 		if (next > 0) {
-			viewRefs.set(key, next);
+			viewRefs.set(key, {
+				refs: next,
+				signature: existing?.signature
+			});
 			return;
 		}
 		viewRefs.delete(key);
@@ -544,7 +562,8 @@ function createOpfsHandleRegistry(backend) {
 		refs: (name) => entries.get(name)?.refs ?? 0,
 		acquireView,
 		releaseView,
-		viewRefs: (key) => viewRefs.get(key) ?? 0
+		viewRefs: (key) => viewRefs.get(key)?.refs ?? 0,
+		viewSignature: (key) => viewRefs.get(key)?.signature
 	};
 }
 function overlayViewBody(args) {
@@ -569,6 +588,7 @@ const OPFS_PREFIX = "gscdump-snapshot__";
 const RECOVER_BUFFER_PREFIX = "gscdump-recover__";
 const MAX_CONTENT_HASH_SLUG_CACHE = 4096;
 const contentHashSlugCache = /* @__PURE__ */ new Map();
+const IDENT_RE = /^[A-Z_][\w$]*$/i;
 function nowMs() {
 	return globalThis.performance?.now?.() ?? Date.now();
 }
@@ -603,6 +623,23 @@ function getOpfsRegistry(db, protocol) {
 	}
 	return registry;
 }
+const dbBufferRegistries = /* @__PURE__ */ new WeakMap();
+function getBufferRegistry(db) {
+	let registry = dbBufferRegistries.get(db);
+	if (!registry) {
+		registry = createOpfsHandleRegistry({
+			register: (name, handle) => db.registerFileBuffer(name, handle),
+			drop: async (name) => {
+				await db.dropFile(name);
+			}
+		});
+		dbBufferRegistries.set(db, registry);
+	}
+	return registry;
+}
+function assertSqlIdentifier(kind, value) {
+	if (!IDENT_RE.test(value)) throw new TypeError(`[engine-duckdb-wasm/opfs] invalid ${kind} identifier ${JSON.stringify(value)}`);
+}
 function isQuotaError(err) {
 	if (typeof err !== "object" || err === null) return false;
 	return err.name === "QuotaExceededError" || err.code === 22;
@@ -743,6 +780,28 @@ function readParquetViewWithOverlaySql(schema, table, lakeFiles, overlayFile) {
 		materializeLake: true
 	})}`;
 }
+function viewKey(schema, table) {
+	return `${schema}.${table}`;
+}
+function viewSignature(lakeFiles, overlayFile) {
+	return JSON.stringify({
+		lake: [...lakeFiles].sort(),
+		overlay: overlayFile ?? null
+	});
+}
+async function ensureView(registry, conn, schema, table, signature, sql) {
+	const key = viewKey(schema, table);
+	if (registry.viewSignature(key) === signature) {
+		registry.acquireView(key, signature);
+		return;
+	}
+	if (registry.viewRefs(key) > 0) throw new Error(`[engine-duckdb-wasm/opfs] view ${key} is already attached with a different fileset`);
+	await conn.query(sql);
+	registry.acquireView(key, signature);
+}
+async function recoveredBufferName(table, file, index) {
+	return `${RECOVER_BUFFER_PREFIX}${table}_${file.contentHash ? await contentHashSlug(file.contentHash) : String(index)}.parquet`;
+}
 async function runWithConcurrency$1(items, concurrency, fn) {
 	let next = 0;
 	let failed;
@@ -757,10 +816,13 @@ async function runWithConcurrency$1(items, concurrency, fn) {
 			}
 		}
 	}
-	await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
+	const rejected = (await Promise.allSettled(Array.from({ length: Math.min(concurrency, items.length) }, worker))).find((r) => r.status === "rejected");
+	if (rejected) throw rejected.reason;
 }
 async function attachOpfsParquetTables(options) {
 	const { db, conn, tables, schema = "main", fetch: fetchImpl = globalThis.fetch.bind(globalThis), fetchInit, fetchConcurrency = DEFAULT_CONCURRENCY, signal, version, onFileProgress, onTiming, withDb = (fn) => fn(), recoverContention = true } = options;
+	assertSqlIdentifier("schema", schema);
+	for (const table of tables) assertSqlIdentifier("table", table.table);
 	const totalStarted = nowMs();
 	await timed(onTiming, "persist", {}, requestPersistentStorage);
 	const root = await timed(onTiming, "root", {}, getOpfsRoot);
@@ -799,6 +861,7 @@ async function attachOpfsParquetTables(options) {
 	const expectedCount = new Map(tables.map((t) => [t.table, t.files.length + (t.overlay ? 1 : 0)]));
 	const { DuckDBDataProtocol } = await timed(onTiming, "duckdb-import", {}, () => import("@duckdb/duckdb-wasm"));
 	const registry = getOpfsRegistry(db, DuckDBDataProtocol.BROWSER_FSACCESS);
+	const bufferRegistry = getBufferRegistry(db);
 	await timed(onTiming, "sweep", {
 		files: total,
 		tables: tables.length
@@ -915,20 +978,21 @@ async function attachOpfsParquetTables(options) {
 			continue;
 		}
 		const overlayName = overlayNames.get(t.table);
-		const lakeNames = overlayName ? files.map((f) => f.name).filter((n) => n !== overlayName) : files.map((f) => f.name);
+		const lakeNames = (overlayName ? files.map((f) => f.name).filter((n) => n !== overlayName) : files.map((f) => f.name)).sort();
+		const signature = viewSignature(lakeNames, overlayName);
 		try {
 			signal?.throwIfAborted();
 			await timed(onTiming, "view", {
 				table: t.table,
 				files: files.length
-			}, () => withDb(() => conn.query(overlayName ? readParquetViewWithOverlaySql(schema, t.table, lakeNames, overlayName) : readParquetViewSql$1(schema, t.table, lakeNames))));
+			}, () => withDb(() => ensureView(registry, conn, schema, t.table, signature, overlayName ? readParquetViewWithOverlaySql(schema, t.table, lakeNames, overlayName) : readParquetViewSql$1(schema, t.table, lakeNames))));
 		} catch (err) {
 			if (isAbortError$1(err)) {
 				await withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames])).catch(() => {});
 				throw err;
 			}
 			if (isOpfsAccessHandleConflict(err)) {
-				if (registry.viewRefs(`${schema}.${t.table}`) === 0) await withDb(() => conn.query(`DROP VIEW IF EXISTS ${schema}.${t.table}`)).catch(() => {});
+				if (registry.viewRefs(viewKey(schema, t.table)) === 0) await withDb(() => conn.query(`DROP VIEW IF EXISTS ${schema}.${t.table}`)).catch(() => {});
 				degraded.add(t.table);
 				degradeReason.set(t.table, "contention");
 				await releaseTable(t.table);
@@ -938,7 +1002,6 @@ async function attachOpfsParquetTables(options) {
 			throw err;
 		}
 		attached.push(t.table);
-		registry.acquireView(`${schema}.${t.table}`);
 		for (const f of files) registeredNames.push(f.name);
 	}
 	const recoverTableToBuffers = async (t) => {
@@ -959,28 +1022,30 @@ async function attachOpfsParquetTables(options) {
 			};
 			const lakeNames = [];
 			for (let i = 0; i < t.files.length; i++) {
-				const buf = await readOne(t.files[i], i);
-				const name = `${RECOVER_BUFFER_PREFIX}${t.table}_${i}.parquet`;
-				await withDb(() => db.registerFileBuffer(name, buf));
+				const file = t.files[i];
+				const name = await recoveredBufferName(t.table, file, i);
+				const buf = await readOne(file, i);
+				await withDb(() => bufferRegistry.acquire(name, () => buf));
 				bufferFiles.push(name);
 				lakeNames.push(name);
-				bytesAttached += t.files[i].bytes;
+				bytesAttached += file.bytes;
 			}
 			let overlayName;
 			if (t.overlay) {
+				overlayName = await recoveredBufferName(t.table, t.overlay, t.files.length);
 				const buf = await readOne(t.overlay, t.files.length);
-				overlayName = `${RECOVER_BUFFER_PREFIX}${t.table}_overlay.parquet`;
-				await withDb(() => db.registerFileBuffer(overlayName, buf));
+				await withDb(() => bufferRegistry.acquire(overlayName, () => buf));
 				bufferFiles.push(overlayName);
 				bytesAttached += t.overlay.bytes;
 			}
+			lakeNames.sort();
 			const body = overlayViewBody({
 				lakeSelect: lakeNames.length ? lakeSelect(lakeNames) : null,
 				overlaySelect: overlayName ? `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayName.replace(/'/g, "''")}'], union_by_name = true)` : null,
 				materializeLake: false
 			});
 			if (!body) return false;
-			await withDb(() => conn.query(`CREATE OR REPLACE VIEW ${schema}.${t.table} AS ${body}`));
+			await withDb(() => ensureView(registry, conn, schema, t.table, viewSignature(lakeNames, overlayName), `CREATE OR REPLACE VIEW ${schema}.${t.table} AS ${body}`));
 			bufferViews.push(t.table);
 			return true;
 		});
@@ -990,7 +1055,7 @@ async function attachOpfsParquetTables(options) {
 		const before = bufferFiles.length;
 		if (await recoverTableToBuffers(t).catch(() => false)) degraded.delete(t.table);
 		else {
-			for (const n of bufferFiles.slice(before)) await withDb(() => db.dropFile(n)).catch(() => {});
+			await withDb(() => bufferRegistry.release(bufferFiles.slice(before))).catch(() => {});
 			bufferFiles.length = before;
 		}
 	}
@@ -1012,8 +1077,8 @@ async function attachOpfsParquetTables(options) {
 			if (detached) return;
 			detached = true;
 			await detachOpfs(registry, conn, schema, attached, registeredNames);
-			for (const v of bufferViews) await conn.query(`DROP VIEW IF EXISTS ${schema}.${v}`).catch(() => {});
-			for (const n of bufferFiles) await db.dropFile(n).catch(() => {});
+			for (const v of bufferViews) await registry.releaseView(viewKey(schema, v), () => conn.query(`DROP VIEW IF EXISTS ${schema}.${v}`).then(() => {}));
+			await bufferRegistry.release(bufferFiles);
 		}
 	};
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-duckdb-wasm",
   "type": "module",
-  "version": "0.32.11",
+  "version": "0.33.0",
   "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
   "author": {
     "name": "Harlan Wilton",
@@ -45,9 +45,9 @@
   },
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
-    "@gscdump/contracts": "0.32.11",
-    "@gscdump/engine": "0.32.11",
-    "gscdump": "0.32.11"
+    "gscdump": "0.33.0",
+    "@gscdump/engine": "0.33.0",
+    "@gscdump/contracts": "0.33.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",