@gscdump/engine-duckdb-wasm 0.25.4 → 0.25.6

package/dist/index.d.mts

@@ -105,6 +105,17 @@ interface OpfsParquetTable {
   /** Iceberg table name — becomes the DuckDB view name. */
   table: string;
   files: OpfsParquetFile[];
+  /**
+   * Recent-window overlay parquet (the non-stable tail the lake excludes). When
+   * present it is materialised into OPFS alongside `files` and the view unions
+   * it with an anti-join dedup: the lake (`files`) serves every day it has, the
+   * overlay serves ONLY days the lake lacks. So a stale overlay whose days have
+   * since landed in the lake can't double-count, and a day that stabilised but
+   * isn't yet in the lake still serves from the overlay. Its `contentHash` must
+   * change when the overlay bytes change (it is overwritten in place) so the
+   * cache re-downloads.
+   */
+  overlay?: OpfsParquetFile;
 }
 interface AttachOpfsTablesOptions {
   db: AsyncDuckDB;

package/dist/index.mjs

@@ -540,8 +540,19 @@ async function materialiseFile(root, name, staleSweepPrefix, file, fetchImpl, fe
 		outcome: "downloaded"
 	};
 }
+function quoteList(files) {
+	return files.map((f) => `'${f.replace(/'/g, "''")}'`).join(", ");
+}
+function lakeSelect(files) {
+	return `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet([${quoteList(files)}], union_by_name = true)`;
+}
 function readParquetViewSql$1(schema, table, files) {
-	return `CREATE OR REPLACE VIEW ${schema}.${table} AS SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet([${files.map((f) => `'${f.replace(/'/g, "''")}'`).join(", ")}], union_by_name = true)`;
+	return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${lakeSelect(files)}`;
+}
+function readParquetViewWithOverlaySql(schema, table, lakeFiles, overlayFile) {
+	const overlay = `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayFile.replace(/'/g, "''")}'], union_by_name = true)`;
+	if (lakeFiles.length === 0) return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${overlay}`;
+	return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${lakeSelect(lakeFiles)} UNION ALL BY NAME ${overlay} WHERE CAST(date AS DATE) NOT IN (SELECT DISTINCT CAST(date AS DATE) FROM read_parquet([${quoteList(lakeFiles)}], union_by_name = true))`;
 }
 async function runWithConcurrency$1(items, concurrency, fn) {
 	let next = 0;
@@ -564,12 +575,22 @@ async function attachOpfsParquetTables(options) {
 	await requestPersistentStorage();
 	const root = await getOpfsRoot();
 	const flat = [];
-	for (const t of tables) for (let i = 0; i < t.files.length; i++) flat.push({
-		table: t.table,
-		file: t.files[i],
-		fileIndex: i
-	});
+	for (const t of tables) {
+		for (let i = 0; i < t.files.length; i++) flat.push({
+			table: t.table,
+			file: t.files[i],
+			fileIndex: i
+		});
+		if (t.overlay) flat.push({
+			table: t.table,
+			file: t.overlay,
+			fileIndex: t.files.length,
+			overlay: true
+		});
+	}
 	const total = flat.length;
+	const overlayNames = /* @__PURE__ */ new Map();
+	const expectedCount = new Map(tables.map((t) => [t.table, t.files.length + (t.overlay ? 1 : 0)]));
 	const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
 	const registry = getOpfsRegistry(db, DuckDBDataProtocol.BROWSER_FSACCESS);
 	const tableFiles = /* @__PURE__ */ new Map();
@@ -615,6 +636,7 @@ async function attachOpfsParquetTables(options) {
 			handle: result.handle
 		});
 		tableFiles.set(item.table, list);
+		if (item.overlay) overlayNames.set(item.table, name);
 		bytesAttached += item.file.bytes;
 		onFileProgress?.({
 			table: item.table,
@@ -639,14 +661,16 @@ async function attachOpfsParquetTables(options) {
 			continue;
 		}
 		const files = tableFiles.get(t.table) ?? [];
-		if (files.length !== t.files.length) {
+		if (files.length !== (expectedCount.get(t.table) ?? t.files.length)) {
 			degraded.add(t.table);
 			await releaseTable(t.table);
 			continue;
 		}
+		const overlayName = overlayNames.get(t.table);
+		const lakeNames = overlayName ? files.map((f) => f.name).filter((n) => n !== overlayName) : files.map((f) => f.name);
 		try {
 			signal?.throwIfAborted();
-			await conn.query(readParquetViewSql$1(schema, t.table, files.map((f) => f.name)));
+			await conn.query(overlayName ? readParquetViewWithOverlaySql(schema, t.table, lakeNames, overlayName) : readParquetViewSql$1(schema, t.table, lakeNames));
 		} catch (err) {
 			if (isAbortError$1(err)) {
 				await detachOpfs(registry, conn, schema, attached, [...acquiredNames]).catch(() => {});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-duckdb-wasm",
   "type": "module",
-  "version": "0.25.4",
+  "version": "0.25.6",
   "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
   "author": {
     "name": "Harlan Wilton",
@@ -45,9 +45,9 @@
   },
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
-    "@gscdump/engine": "0.25.4",
-    "@gscdump/sdk": "0.25.4",
-    "gscdump": "0.25.4"
+    "@gscdump/engine": "0.25.6",
+    "@gscdump/sdk": "0.25.6",
+    "gscdump": "0.25.6"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",