@gscdump/engine-duckdb-wasm 0.31.3 → 0.31.5

package/dist/index.d.mts

@@ -222,6 +222,38 @@ declare function attachOpfsParquetTables(options: AttachOpfsTablesOptions): Prom
  * are ignored.
  */
 declare function clearOpfsSnapshotCache(): Promise<void>;
+/**
+ * Browser-side recent-overlay merge-on-read SQL — shared by the OPFS attach path
+ * (`opfs.ts`) and the in-memory buffer attach path (consumers'
+ * `registerParquetView`).
+ *
+ * The lake serves every day it has; the overlay serves ONLY days the lake lacks
+ * (`date NOT IN (SELECT DISTINCT date FROM lake)`), unioned with
+ * `UNION ALL BY NAME`. This is the browser sibling of the server's
+ * `overlayMergeEnvelope` (in gscdump.com); kept SEPARATE because the browser path
+ * can read the lake once via a `MATERIALIZED` CTE (CPU-bound DuckDB-WASM, bounded
+ * per-site buffers) where the server's many-s3-file reads prefer projection
+ * pushdown over a re-scan.
+ *
+ * Both `lakeSelect` / `overlaySelect` must already expose a `date` column cast to
+ * DATE (the overlay stores ISO strings; `SELECT * REPLACE (CAST(date AS DATE) AS
+ * date)`), so `UNION ALL BY NAME` merges them type-uniformly.
+ */
+/**
+ * The view BODY (no `CREATE VIEW` wrapper) merging a base lake relation and a
+ * recent overlay. Returns null when neither side exists; the lake verbatim with
+ * no overlay; the overlay verbatim with no lake (every recent row is wanted).
+ *
+ * `materializeLake` (default true): wrap the lake in a `MATERIALIZED` CTE so it is
+ * scanned ONCE and reused for both the served rows and the anti-join date set — a
+ * plain CTE inlines and re-scans the parquet, the dominant cost in DuckDB-WASM.
+ * Set false to keep the streaming re-scan shape (lower peak memory, two scans).
+ */
+declare function overlayViewBody(args: {
+  lakeSelect: string | null;
+  overlaySelect: string | null;
+  materializeLake?: boolean;
+}): string | null;
 interface QueryResult {
   rows: Record<string, unknown>[];
   queryMs: number;
@@ -427,4 +459,4 @@ declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, option
   version?: number | string;
   attachedTables?: readonly string[];
 }): BrowserAnalysisRuntime;
-export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
+export { type AnalyzeResult, type AttachOpfsTablesOptions, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserAttachError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type CompiledArchetypeSql, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type OpfsAttachedHandle, type OpfsFileProgress, type OpfsParquetFile, type OpfsParquetTable, OpfsQuotaExceededError, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };

package/dist/index.mjs

@@ -547,6 +547,15 @@ function createOpfsHandleRegistry(backend) {
 		viewRefs: (key) => viewRefs.get(key) ?? 0
 	};
 }
+function overlayViewBody(args) {
+	const { lakeSelect, overlaySelect } = args;
+	const materialize = args.materializeLake ?? true;
+	if (!lakeSelect && !overlaySelect) return null;
+	if (!overlaySelect) return lakeSelect;
+	if (!lakeSelect) return overlaySelect;
+	if (materialize) return `WITH lake AS MATERIALIZED (${lakeSelect}), lake_dates AS (SELECT DISTINCT date FROM lake) SELECT * FROM lake UNION ALL BY NAME SELECT * FROM (${overlaySelect}) AS overlay WHERE overlay.date NOT IN (SELECT date FROM lake_dates)`;
+	return `${lakeSelect} UNION ALL BY NAME SELECT * FROM (${overlaySelect}) AS overlay WHERE overlay.date NOT IN (SELECT DISTINCT date FROM (${lakeSelect}))`;
+}
 var OpfsQuotaExceededError = class extends Error {
 	name = "OpfsQuotaExceededError";
 	degradedTables;
@@ -680,8 +689,11 @@ function readParquetViewSql$1(schema, table, files) {
 }
 function readParquetViewWithOverlaySql(schema, table, lakeFiles, overlayFile) {
 	const overlay = `SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet(['${overlayFile.replace(/'/g, "''")}'], union_by_name = true)`;
-	if (lakeFiles.length === 0) return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${overlay}`;
-	return `CREATE OR REPLACE VIEW ${schema}.${table} AS WITH lake AS MATERIALIZED (${lakeSelect(lakeFiles)}), lake_dates AS (SELECT DISTINCT date FROM lake) SELECT * FROM lake UNION ALL BY NAME SELECT * FROM (${overlay}) AS overlay WHERE overlay.date NOT IN (SELECT date FROM lake_dates)`;
+	return `CREATE OR REPLACE VIEW ${schema}.${table} AS ${overlayViewBody({
+		lakeSelect: lakeFiles.length === 0 ? null : lakeSelect(lakeFiles),
+		overlaySelect: overlay,
+		materializeLake: true
+	})}`;
 }
 async function runWithConcurrency$1(items, concurrency, fn) {
 	let next = 0;
@@ -1295,4 +1307,4 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
 		}
 	};
 }
-export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, OpfsQuotaExceededError, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };
+export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, OpfsQuotaExceededError, attachOpfsParquetTables, attachParquetTables, attachParquetUrlTables, attachParquetUrlTablesResult, bootDuckDBWasm, browserAttachErrors, clearOpfsSnapshotCache, compileArchetypeSql, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, dates, drizzle, estimateOpfsStorage, hourly_pages, isBrowserAttachError, mergeScope, overlayViewBody, page_queries, pages, queries, requestPersistentStorage, resolveWindow, schema, scopeFor, strikingMomentum, tableForArchetype };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-duckdb-wasm",
   "type": "module",
-  "version": "0.31.3",
+  "version": "0.31.5",
   "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
   "author": {
     "name": "Harlan Wilton",
@@ -45,9 +45,9 @@
   },
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
-    "@gscdump/engine": "0.31.3",
-    "@gscdump/contracts": "0.31.3",
-    "gscdump": "0.31.3"
+    "@gscdump/contracts": "0.31.5",
+    "@gscdump/engine": "0.31.5",
+    "gscdump": "0.31.5"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",