@gscdump/engine-duckdb-wasm 1.3.2 → 1.4.1

package/dist/opfs.mjs

@@ -0,0 +1,584 @@
+import { createOpfsHandleRegistry } from "./opfs-registry.mjs";
+import { overlayViewBody } from "./overlay-view.mjs";
+var OpfsQuotaExceededError = class extends Error {
+	name = "OpfsQuotaExceededError";
+	degradedTables;
+	constructor(message, degradedTables) {
+		super(message);
+		this.degradedTables = degradedTables;
+	}
+};
+const DEFAULT_CONCURRENCY = 2;
+const OPFS_PREFIX = "gscdump-snapshot__";
+const RECOVER_BUFFER_PREFIX = "gscdump-recover__";
+const MAX_CONTENT_HASH_SLUG_CACHE = 4096;
+const contentHashSlugCache = /* @__PURE__ */ new Map();
+const IDENT_RE = /^[A-Z_][\w$]*$/i;
+function nowMs() {
+	return globalThis.performance?.now?.() ?? Date.now();
+}
+function isNotFoundError(error) {
+	return typeof error === "object" && error !== null && error.name === "NotFoundError";
+}
+function reportBestEffortFailure(operation, error) {
+	console.warn(`[gscdump/engine-duckdb-wasm] ${operation} failed`, error);
+}
+async function attemptCleanup(operation, cleanup) {
+	try {
+		await cleanup();
+	} catch (error) {
+		reportBestEffortFailure(operation, error);
+	}
+}
+function emitTiming(onTiming, info) {
+	try {
+		onTiming?.(info);
+	} catch (error) {
+		reportBestEffortFailure("timing callback", error);
+	}
+}
+async function timed(onTiming, stage, meta, fn) {
+	const started = nowMs();
+	try {
+		return await fn();
+	} finally {
+		emitTiming(onTiming, {
+			...meta,
+			stage,
+			durationMs: nowMs() - started
+		});
+	}
+}
+const dbRegistries = /* @__PURE__ */ new WeakMap();
+function getOpfsRegistry(db, protocol) {
+	let registry = dbRegistries.get(db);
+	if (!registry) {
+		registry = createOpfsHandleRegistry({
+			register: (name, handle) => db.registerFileHandle(name, handle, protocol, true),
+			drop: async (name) => {
+				await db.dropFile(name);
+			}
+		});
+		dbRegistries.set(db, registry);
+	}
+	return registry;
+}
+const dbBufferRegistries = /* @__PURE__ */ new WeakMap();
+function getBufferRegistry(db) {
+	let registry = dbBufferRegistries.get(db);
+	if (!registry) {
+		registry = createOpfsHandleRegistry({
+			register: (name, handle) => db.registerFileBuffer(name, handle),
+			drop: async (name) => {
+				await db.dropFile(name);
+			}
+		});
+		dbBufferRegistries.set(db, registry);
+	}
+	return registry;
+}
+function assertSqlIdentifier(kind, value) {
+	if (!IDENT_RE.test(value)) throw new TypeError(`[engine-duckdb-wasm/opfs] invalid ${kind} identifier ${JSON.stringify(value)}`);
+}
+const DOWNLOAD_DEADLINE_MS = 12e4;
+function isQuotaError(err) {
+	if (typeof err !== "object" || err === null) return false;
+	return err.name === "QuotaExceededError" || err.code === 22;
+}
+function isAbortError(err) {
+	return typeof err === "object" && err !== null && err.name === "AbortError";
+}
+function isOpfsAccessHandleConflict(err) {
+	const msg = err instanceof Error ? err.message : String(err);
+	return /createSyncAccessHandle|Access Handle/i.test(msg);
+}
+function isOpfsWriteConflict(err) {
+	if (typeof err !== "object" || err === null) return false;
+	const name = err.name;
+	const msg = err instanceof Error ? err.message : String(err);
+	return name === "NoModificationAllowedError" || err.code === 7 || /createWritable|modifications are not allowed/i.test(msg);
+}
+function opfsFileName(table, hashSlug, index) {
+	return hashSlug ? `${OPFS_PREFIX}${table}_${hashSlug}.parquet` : `${OPFS_PREFIX}${table}_${index}.parquet`;
+}
+function escapeRegExp(s) {
+	return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function tableEntryMatcher(table) {
+	return new RegExp(`^${escapeRegExp(`${OPFS_PREFIX}${table}_`)}(?:[0-9a-f]{16}|\\d+(?:_[0-9a-f]{16})?)\\.parquet$`);
+}
+async function sweepStaleEntries(root, registry, expectedByTable) {
+	const dir = root;
+	if (!dir.keys) return;
+	const matchers = [...expectedByTable].map(([table, expected]) => ({
+		expected,
+		re: tableEntryMatcher(table)
+	}));
+	for await (const name of dir.keys()) {
+		const m = matchers.find((m) => m.re.test(name));
+		if (!m || m.expected.has(name) || registry.refs(name) > 0) continue;
+		try {
+			await root.removeEntry(name);
+		} catch (error) {
+			if (!isNotFoundError(error)) reportBestEffortFailure(`removing stale OPFS entry ${name}`, error);
+		}
+	}
+}
+async function requestPersistentStorage() {
+	const storage = globalThis.navigator?.storage;
+	if (!storage?.persist) return false;
+	if (await storage.persisted?.().catch(() => false)) return true;
+	return storage.persist().catch(() => false);
+}
+async function estimateOpfsStorage() {
+	const estimate = globalThis.navigator?.storage?.estimate;
+	if (!estimate) return {};
+	const est = await estimate.call(globalThis.navigator.storage).catch(() => null);
+	return est ? {
+		usageBytes: est.usage,
+		quotaBytes: est.quota
+	} : {};
+}
+async function contentHashSlug(contentHash) {
+	const cached = contentHashSlugCache.get(contentHash);
+	if (cached) return cached;
+	if (contentHashSlugCache.size >= MAX_CONTENT_HASH_SLUG_CACHE) {
+		const oldest = contentHashSlugCache.keys().next().value;
+		if (oldest !== void 0) contentHashSlugCache.delete(oldest);
+	}
+	const promise = deriveContentHashSlug(contentHash).catch((err) => {
+		contentHashSlugCache.delete(contentHash);
+		throw err;
+	});
+	contentHashSlugCache.set(contentHash, promise);
+	return promise;
+}
+async function deriveContentHashSlug(contentHash) {
+	const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(contentHash));
+	const bytes = new Uint8Array(digest);
+	let hex = "";
+	for (let i = 0; i < 8; i++) hex += bytes[i].toString(16).padStart(2, "0");
+	return hex;
+}
+async function getOpfsRoot() {
+	const dir = globalThis.navigator?.storage?.getDirectory;
+	if (!dir) throw new Error("[engine-duckdb-wasm/opfs] OPFS unavailable: navigator.storage.getDirectory missing");
+	return dir.call(globalThis.navigator.storage);
+}
+async function readOpfsSnapshotFile(table, contentHash, index, expectedBytes) {
+	try {
+		const root = await getOpfsRoot();
+		const name = opfsFileName(table, contentHash ? await contentHashSlug(contentHash) : void 0, index);
+		const file = await (await root.getFileHandle(name)).getFile();
+		if (file.size !== expectedBytes) return null;
+		return new Uint8Array(await file.arrayBuffer());
+	} catch (error) {
+		if (!isNotFoundError(error)) reportBestEffortFailure(`reading OPFS snapshot ${table}`, error);
+		return null;
+	}
+}
+async function materialiseFile(root, name, file, fetchImpl, fetchInit, signal) {
+	signal?.throwIfAborted();
+	let handle;
+	try {
+		handle = await root.getFileHandle(name);
+		if ((await handle.getFile()).size === file.bytes) return {
+			handle,
+			outcome: "cache-hit"
+		};
+	} catch (error) {
+		if (!isNotFoundError(error)) throw error;
+	}
+	signal?.throwIfAborted();
+	const deadline = AbortSignal.timeout(DOWNLOAD_DEADLINE_MS);
+	const fetchSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
+	const resp = await fetchImpl(file.url, {
+		...fetchInit,
+		signal: fetchSignal
+	});
+	if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} failed: ${resp.status}`);
+	handle = await root.getFileHandle(name, { create: true });
+	let writable;
+	try {
+		writable = await handle.createWritable();
+	} catch (err) {
+		await resp.body?.cancel().catch(() => void 0);
+		throw err;
+	}
+	let reader;
+	let bytesWritten = 0;
+	try {
+		if (resp.body) {
+			reader = resp.body.getReader();
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+				bytesWritten += value.byteLength;
+				if (bytesWritten > file.bytes) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} byte length mismatch: expected ${file.bytes}, got more than ${file.bytes}`);
+				await writable.write(value);
+			}
+		} else {
+			const buf = await resp.arrayBuffer();
+			bytesWritten = buf.byteLength;
+			await writable.write(buf);
+		}
+		if (bytesWritten !== file.bytes) throw new Error(`[engine-duckdb-wasm/opfs] download ${file.url} byte length mismatch: expected ${file.bytes}, got ${bytesWritten}`);
+		await writable.close();
+	} catch (err) {
+		await reader?.cancel().catch(() => void 0);
+		if (writable.abort) await attemptCleanup(`aborting partial OPFS write ${name}`, () => writable.abort());
+		await attemptCleanup(`removing partial OPFS file ${name}`, () => root.removeEntry(name));
+		throw err;
+	} finally {
+		reader?.releaseLock();
+	}
+	return {
+		handle,
+		outcome: "downloaded"
+	};
+}
+function quoteList(files) {
+	return files.map((f) => `'${f.replace(/'/g, "''")}'`).join(", ");
+}
+function lakeSelect(files) {
+	return `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet([${quoteList(files)}], union_by_name = true)`;
+}
+function readParquetViewSql(schema, table, files) {
+	return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${lakeSelect(files)}`;
+}
+function readParquetViewWithOverlaySql(schema, table, lakeFiles, overlayFile) {
+	const overlay = `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayFile.replace(/'/g, "''")}'], union_by_name = true)`;
+	return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${overlayViewBody({
+		lakeSelect: lakeFiles.length === 0 ? null : lakeSelect(lakeFiles),
+		overlaySelect: overlay,
+		materializeLake: false
+	})}`;
+}
+function viewKey(schema, table) {
+	return `${schema}.${table}`;
+}
+function viewSignature(lakeFiles, overlayFile) {
+	return JSON.stringify({
+		lake: [...lakeFiles].sort(),
+		overlay: overlayFile ?? null
+	});
+}
+async function ensureView(registry, conn, schema, table, signature, sql) {
+	const key = viewKey(schema, table);
+	if (registry.viewSignature(key) === signature) {
+		registry.acquireView(key, signature);
+		return;
+	}
+	if (registry.viewRefs(key) > 0) throw new Error(`[engine-duckdb-wasm/opfs] view ${key} is already attached with a different fileset`);
+	await conn.query(sql);
+	registry.acquireView(key, signature);
+}
+async function recoveredBufferName(table, file, index) {
+	const slug = file.contentHash ? await contentHashSlug(file.contentHash) : String(index);
+	return `${RECOVER_BUFFER_PREFIX}${table}_${slug}.parquet`;
+}
+async function runWithConcurrency(items, concurrency, fn) {
+	let next = 0;
+	let failed;
+	async function worker() {
+		while (failed === void 0 && next < items.length) {
+			const index = next++;
+			try {
+				await fn(items[index], index);
+			} catch (err) {
+				failed = err;
+				throw err;
+			}
+		}
+	}
+	const rejected = (await Promise.allSettled(Array.from({ length: Math.min(concurrency, items.length) }, worker))).find((r) => r.status === "rejected");
+	if (rejected) throw rejected.reason;
+}
+async function attachOpfsParquetTables(options) {
+	const { db, conn, tables, schema = "main", fetch: fetchImpl = globalThis.fetch.bind(globalThis), fetchInit, fetchConcurrency = DEFAULT_CONCURRENCY, signal, version, onFileProgress, onTiming, withDb = (fn) => fn(), recoverContention = true } = options;
+	assertSqlIdentifier("schema", schema);
+	for (const table of tables) assertSqlIdentifier("table", table.table);
+	const totalStarted = nowMs();
+	await timed(onTiming, "persist", {}, requestPersistentStorage);
+	const root = await timed(onTiming, "root", {}, getOpfsRoot);
+	const flat = [];
+	const expectedByTable = /* @__PURE__ */ new Map();
+	await timed(onTiming, "plan", { tables: tables.length }, async () => {
+		for (const t of tables) {
+			const expected = /* @__PURE__ */ new Set();
+			for (let i = 0; i < t.files.length; i++) {
+				const file = t.files[i];
+				const slug = file.contentHash ? await contentHashSlug(file.contentHash) : void 0;
+				const name = opfsFileName(t.table, slug, i);
+				flat.push({
+					table: t.table,
+					file,
+					name
+				});
+				expected.add(name);
+			}
+			if (t.overlay) {
+				const slug = t.overlay.contentHash ? await contentHashSlug(t.overlay.contentHash) : void 0;
+				const name = opfsFileName(t.table, slug, t.files.length);
+				flat.push({
+					table: t.table,
+					file: t.overlay,
+					name,
+					overlay: true
+				});
+				expected.add(name);
+			}
+			expectedByTable.set(t.table, expected);
+		}
+	});
+	const total = flat.length;
+	const overlayNames = /* @__PURE__ */ new Map();
+	const expectedCount = new Map(tables.map((t) => [t.table, t.files.length + (t.overlay ? 1 : 0)]));
+	const { DuckDBDataProtocol } = await timed(onTiming, "duckdb-import", {}, () => import("@duckdb/duckdb-wasm"));
+	const registry = getOpfsRegistry(db, DuckDBDataProtocol.BROWSER_FSACCESS);
+	const bufferRegistry = getBufferRegistry(db);
+	await timed(onTiming, "sweep", {
+		files: total,
+		tables: tables.length
+	}, () => sweepStaleEntries(root, registry, expectedByTable)).catch((error) => {
+		reportBestEffortFailure("stale OPFS cache sweep", error);
+	});
+	const tableFiles = /* @__PURE__ */ new Map();
+	const degraded = /* @__PURE__ */ new Set();
+	const degradeReason = /* @__PURE__ */ new Map();
+	const acquiredNames = /* @__PURE__ */ new Set();
+	const bufferFiles = [];
+	const bufferViews = [];
+	let bytesAttached = 0;
+	const releaseTable = async (table) => {
+		const names = (tableFiles.get(table) ?? []).map((f) => f.name);
+		for (const n of names) acquiredNames.delete(n);
+		await withDb(() => registry.release(names));
+	};
+	const runDownloads = () => runWithConcurrency(flat, Math.max(1, fetchConcurrency), async (item, index) => {
+		if (degraded.has(item.table)) return;
+		signal?.throwIfAborted();
+		const fileStarted = nowMs();
+		const name = item.name;
+		let result;
+		let materialiseMs = 0;
+		let registerMs = 0;
+		try {
+			const materialiseStarted = nowMs();
+			result = await materialiseFile(root, name, item.file, fetchImpl, fetchInit, signal);
+			materialiseMs = nowMs() - materialiseStarted;
+			emitTiming(onTiming, {
+				stage: "materialise",
+				durationMs: materialiseMs,
+				table: item.table,
+				file: name,
+				bytes: item.file.bytes,
+				outcome: result.outcome
+			});
+		} catch (err) {
+			if (isAbortError(err)) throw err;
+			if (isQuotaError(err)) {
+				degraded.add(item.table);
+				degradeReason.set(item.table, "quota");
+				return;
+			}
+			if (isOpfsWriteConflict(err)) {
+				degraded.add(item.table);
+				degradeReason.set(item.table, "contention");
+				return;
+			}
+			throw err;
+		}
+		try {
+			const registerStarted = nowMs();
+			await withDb(() => registry.acquire(name, () => result.handle));
+			registerMs = nowMs() - registerStarted;
+			emitTiming(onTiming, {
+				stage: "register",
+				durationMs: registerMs,
+				table: item.table,
+				file: name,
+				bytes: item.file.bytes,
+				outcome: result.outcome
+			});
+			acquiredNames.add(name);
+		} catch (err) {
+			if (isAbortError(err)) throw err;
+			if (isOpfsAccessHandleConflict(err)) {
+				degraded.add(item.table);
+				degradeReason.set(item.table, "contention");
+				return;
+			}
+			throw err;
+		}
+		const list = tableFiles.get(item.table) ?? [];
+		list.push({
+			name,
+			handle: result.handle
+		});
+		tableFiles.set(item.table, list);
+		if (item.overlay) overlayNames.set(item.table, name);
+		bytesAttached += item.file.bytes;
+		onFileProgress?.({
+			table: item.table,
+			file: name,
+			index,
+			total,
+			bytes: item.file.bytes,
+			outcome: result.outcome,
+			materialiseMs,
+			registerMs,
+			totalMs: nowMs() - fileStarted
+		});
+	});
+	try {
+		await timed(onTiming, "downloads", {
+			files: total,
+			tables: tables.length
+		}, runDownloads);
+	} catch (err) {
+		await attemptCleanup("releasing handles after download failure", () => withDb(() => registry.release([...acquiredNames])));
+		throw err;
+	}
+	const attached = [];
+	const registeredNames = [];
+	for (const t of tables) {
+		if (degraded.has(t.table)) {
+			await releaseTable(t.table);
+			continue;
+		}
+		const files = tableFiles.get(t.table) ?? [];
+		if (files.length !== (expectedCount.get(t.table) ?? t.files.length)) {
+			degraded.add(t.table);
+			degradeReason.set(t.table, degradeReason.get(t.table) ?? "incomplete");
+			await releaseTable(t.table);
+			continue;
+		}
+		const overlayName = overlayNames.get(t.table);
+		const lakeNames = (overlayName ? files.map((f) => f.name).filter((n) => n !== overlayName) : files.map((f) => f.name)).sort();
+		const signature = viewSignature(lakeNames, overlayName);
+		try {
+			signal?.throwIfAborted();
+			await timed(onTiming, "view", {
+				table: t.table,
+				files: files.length
+			}, () => withDb(() => ensureView(registry, conn, schema, t.table, signature, overlayName ? readParquetViewWithOverlaySql(schema, t.table, lakeNames, overlayName) : readParquetViewSql(schema, t.table, lakeNames))));
+		} catch (err) {
+			if (isAbortError(err)) {
+				await attemptCleanup("detaching OPFS resources after abort", () => withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames])));
+				throw err;
+			}
+			if (isOpfsAccessHandleConflict(err)) {
+				if (registry.viewRefs(viewKey(schema, t.table)) === 0) await attemptCleanup(`dropping incomplete view ${schema}.${t.table}`, () => withDb(() => conn.query(`DROP VIEW IF EXISTS ${schema}.${t.table}`)));
+				degraded.add(t.table);
+				degradeReason.set(t.table, "contention");
+				await releaseTable(t.table);
+				continue;
+			}
+			await attemptCleanup("detaching OPFS resources after view failure", () => withDb(() => detachOpfs(registry, conn, schema, attached, [...acquiredNames])));
+			throw err;
+		}
+		attached.push(t.table);
+		for (const f of files) registeredNames.push(f.name);
+	}
+	const recoverTableToBuffers = async (t) => {
+		return timed(onTiming, "recovery", {
+			table: t.table,
+			files: t.files.length + (t.overlay ? 1 : 0)
+		}, async () => {
+			const readOne = async (file, index) => {
+				const cached = await readOpfsSnapshotFile(t.table, file.contentHash, index, file.bytes);
+				if (cached) return cached;
+				signal?.throwIfAborted();
+				const resp = await fetchImpl(file.url, {
+					...fetchInit,
+					signal
+				});
+				if (!resp.ok) throw new Error(`[engine-duckdb-wasm/opfs] recover ${file.url} failed: ${resp.status}`);
+				return new Uint8Array(await resp.arrayBuffer());
+			};
+			const lakeNames = [];
+			for (let i = 0; i < t.files.length; i++) {
+				const file = t.files[i];
+				const name = await recoveredBufferName(t.table, file, i);
+				const buf = await readOne(file, i);
+				await withDb(() => bufferRegistry.acquire(name, () => buf));
+				bufferFiles.push(name);
+				lakeNames.push(name);
+				bytesAttached += file.bytes;
+			}
+			let overlayName;
+			if (t.overlay) {
+				overlayName = await recoveredBufferName(t.table, t.overlay, t.files.length);
+				const buf = await readOne(t.overlay, t.files.length);
+				await withDb(() => bufferRegistry.acquire(overlayName, () => buf));
+				bufferFiles.push(overlayName);
+				bytesAttached += t.overlay.bytes;
+			}
+			lakeNames.sort();
+			const body = overlayViewBody({
+				lakeSelect: lakeNames.length ? lakeSelect(lakeNames) : null,
+				overlaySelect: overlayName ? `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayName.replace(/'/g, "''")}'], union_by_name = true)` : null,
+				materializeLake: false
+			});
+			if (!body) return false;
+			await withDb(() => ensureView(registry, conn, schema, t.table, viewSignature(lakeNames, overlayName), `CREATE OR REPLACE VIEW ${schema}.${t.table} AS ${body}`));
+			bufferViews.push(t.table);
+			return true;
+		});
+	};
+	if (recoverContention && !signal?.aborted) for (const t of tables) {
+		if (!degraded.has(t.table) || degradeReason.get(t.table) !== "contention") continue;
+		const before = bufferFiles.length;
+		if (await recoverTableToBuffers(t).catch(() => false)) degraded.delete(t.table);
+		else {
+			await attemptCleanup(`releasing recovery buffers for ${t.table}`, () => withDb(() => bufferRegistry.release(bufferFiles.slice(before))));
+			bufferFiles.length = before;
+		}
+	}
+	emitTiming(onTiming, {
+		stage: "total",
+		durationMs: nowMs() - totalStarted,
+		files: total,
+		tables: tables.length,
+		bytes: bytesAttached
+	});
+	let detached = false;
+	return {
+		version,
+		tables: [...attached, ...bufferViews],
+		schema,
+		bytesAttached,
+		degradedTables: [...degraded],
+		degradeReasons: Object.fromEntries(degradeReason),
+		async detach() {
+			if (detached) return;
+			detached = true;
+			await detachOpfs(registry, conn, schema, attached, registeredNames);
+			for (const v of bufferViews) await registry.releaseView(viewKey(schema, v), () => conn.query(`DROP VIEW IF EXISTS ${schema}.${v}`).then(() => {}));
+			await bufferRegistry.release(bufferFiles);
+		}
+	};
+}
+async function detachOpfs(registry, conn, schema, tables, files) {
+	for (const table of tables) await registry.releaseView(`${schema}.${table}`, () => conn.query(`DROP VIEW IF EXISTS ${schema}.${table}`).then(() => {}));
+	await registry.release(files);
+}
+async function clearOpfsSnapshotCache() {
+	const root = await getOpfsRoot().catch((error) => {
+		reportBestEffortFailure("opening OPFS cache for clearing", error);
+		return null;
+	});
+	if (!root) return;
+	const removable = [];
+	const dir = root;
+	if (dir.keys) {
+		for await (const name of dir.keys()) if (name.startsWith(OPFS_PREFIX)) removable.push(name);
+	}
+	for (const name of removable) try {
+		await root.removeEntry(name);
+	} catch (error) {
+		if (!isNotFoundError(error)) reportBestEffortFailure(`clearing OPFS cache entry ${name}`, error);
+	}
+}
+export { OpfsQuotaExceededError, attachOpfsParquetTables, clearOpfsSnapshotCache, estimateOpfsStorage, readOpfsSnapshotFile, requestPersistentStorage };

package/dist/overlay-view.d.mts

@@ -0,0 +1,33 @@
+/**
+ * Browser-side recent-overlay merge-on-read SQL — shared by the OPFS attach path
+ * (`opfs.ts`) and the in-memory buffer attach path (consumers'
+ * `registerParquetView`).
+ *
+ * The lake serves every day it has; the overlay serves ONLY days the lake lacks
+ * (`date NOT IN (SELECT DISTINCT date FROM lake)`), unioned with
+ * `UNION ALL BY NAME`. This is the browser sibling of the server's
+ * `overlayMergeEnvelope` (in gscdump.com); kept SEPARATE because the browser path
+ * can read the lake once via a `MATERIALIZED` CTE (CPU-bound DuckDB-WASM, bounded
+ * per-site buffers) where the server's many-s3-file reads prefer projection
+ * pushdown over a re-scan.
+ *
+ * Both `lakeSelect` / `overlaySelect` must already expose a `date` column cast to
+ * DATE (the overlay stores ISO strings; `SELECT * REPLACE (CAST(date AS DATE) AS
+ * date)`), so `UNION ALL BY NAME` merges them type-uniformly.
+ */
+/**
+ * The view BODY (no `CREATE VIEW` wrapper) merging a base lake relation and a
+ * recent overlay. Returns null when neither side exists; the lake verbatim with
+ * no overlay; the overlay verbatim with no lake (every recent row is wanted).
+ *
+ * `materializeLake` (default true): wrap the lake in a `MATERIALIZED` CTE so it is
+ * scanned ONCE and reused for both the served rows and the anti-join date set — a
+ * plain CTE inlines and re-scans the parquet, the dominant cost in DuckDB-WASM.
+ * Set false to keep the streaming re-scan shape (lower peak memory, two scans).
+ */
+declare function overlayViewBody(args: {
+  lakeSelect: string | null;
+  overlaySelect: string | null;
+  materializeLake?: boolean;
+}): string | null;
+export { overlayViewBody };

package/dist/overlay-view.mjs

@@ -0,0 +1,10 @@
+function overlayViewBody(args) {
+	const { lakeSelect, overlaySelect } = args;
+	const materialize = args.materializeLake ?? true;
+	if (!lakeSelect && !overlaySelect) return null;
+	if (!overlaySelect) return lakeSelect;
+	if (!lakeSelect) return overlaySelect;
+	if (materialize) return `WITH lake AS MATERIALIZED (${lakeSelect}), lake_dates AS (SELECT DISTINCT date FROM lake) SELECT * FROM lake UNION ALL BY NAME SELECT * FROM (${overlaySelect}) AS overlay WHERE overlay.date NOT IN (SELECT date FROM lake_dates)`;
+	return `${lakeSelect} UNION ALL BY NAME SELECT * FROM (${overlaySelect}) AS overlay WHERE overlay.date NOT IN (SELECT DISTINCT date FROM (${lakeSelect}))`;
+}
+export { overlayViewBody };

package/dist/runner.d.mts

@@ -0,0 +1,28 @@
+import { DuckDBWasmClient } from "./drizzle-adapter/client.mjs";
+import { DuckDBWasmDrizzleDatabase } from "./drizzle-adapter/driver.mjs";
+import "./drizzle-adapter/index.mjs";
+import { Schema } from "./schema.mjs";
+import { ScopedRunnerOptions, TableScope } from "@gscdump/engine/scope";
+import { AsyncDuckDB, AsyncDuckDBConnection } from "@duckdb/duckdb-wasm";
+interface InsightRunnerOptions {
+  db: AsyncDuckDB;
+  conn: AsyncDuckDBConnection;
+  logger?: boolean;
+}
+interface InsightRunner {
+  db: DuckDBWasmDrizzleDatabase<Schema>;
+  client: Promise<DuckDBWasmClient>;
+  close: () => Promise<void>;
+}
+declare function createInsightRunner(opts: InsightRunnerOptions): Promise<InsightRunner>;
+/**
+ * Build a per-table predicate set from {siteId, window}. The returned
+ * `wherePredicates` composes with user-level filters via `mergeScope`.
+ *
+ * Note: the current SCHEMAS don't include `site_id` on any table (snapshots
+ * are already per-site), so `siteId` is a no-op for now — kept in the API
+ * so consumers can add the predicate without an interface change when
+ * multi-site snapshots land.
+ */
+declare const scopeFor: (table: "countries" | "dates" | "hourly_pages" | "page_queries" | "pages" | "queries" | "search_appearance" | "search_appearance_page_queries" | "search_appearance_pages" | "search_appearance_queries", opts: ScopedRunnerOptions) => TableScope, mergeScope: typeof import("@gscdump/engine/scope").mergeScope;
+export { InsightRunner, InsightRunnerOptions, type ScopedRunnerOptions, type TableScope, createInsightRunner, mergeScope, scopeFor };

package/dist/runner.mjs

@@ -0,0 +1,19 @@
+import { createClient } from "./drizzle-adapter/client.mjs";
+import { drizzle } from "./drizzle-adapter/driver.mjs";
+import "./drizzle-adapter/index.mjs";
+import { schema } from "./schema.mjs";
+import { createScopedHelpers } from "@gscdump/engine/scope";
+async function createInsightRunner(opts) {
+	const client = await createClient(opts.db, opts.conn);
+	const clientPromise = Promise.resolve(client);
+	return {
+		db: drizzle(clientPromise, {
+			schema,
+			logger: opts.logger
+		}),
+		client: clientPromise,
+		close: () => client.close()
+	};
+}
+const { scopeFor, mergeScope } = createScopedHelpers(schema);
+export { createInsightRunner, mergeScope, scopeFor };