@gscdump/engine-duckdb-wasm 0.19.6 → 0.20.0

package/README.md

@@ -38,6 +38,26 @@ const scope = scopeFor('keywords', { siteId, window })
 const rows = await strikingMomentum(runner, { ...scope, limit: 50 })
 ```
 
+## Browser Parquet Attachment Strategy
+
+`attachParquetUrlTables()` uses URL registration with DuckDB-WASM's HTTP file
+reader rather than downloading parquet objects into JS memory. Each exact-object
+URL is preflighted with `HEAD`; if the endpoint does not support `HEAD`, the
+runtime performs a one-byte `Range: bytes=0-0` probe. Attachment fails closed
+unless the response proves `Content-Length` / `Content-Range` and byte-range
+support.
+
+The runtime keeps local guards independent of the server response:
+`maxFiles`, `maxBytes`, `fetchConcurrency`, and `AbortSignal` are enforced
+before registering files. `bootDuckDBWasm()` opens DuckDB with full HTTP reads
+disabled, so a server that cannot satisfy range reads routes to server-side
+fallback instead of causing broad browser object downloads.
+
+OPFS is intentionally not used as a write-through parquet cache yet. If it is
+added later, the cache key must include manifest version and object key, and
+the same file/byte budgets must gate admission before any object is materialized
+locally.
+
 ### Engine source for analyzer dispatch
 
 ```ts

package/dist/index.d.mts

@@ -4,7 +4,7 @@ import { DrizzleSchema as Schema, countries, devices, drizzleSchema as schema, h
 import { ScopedRunnerOptions, TableScope } from "@gscdump/engine/scope";
 import { AnalyzerRegistry } from "@gscdump/engine/analyzer";
 import { ComparisonMode, ResolveWindowOptions, ResolvedWindow, WindowPreset, resolveWindow } from "@gscdump/engine/period";
-import { AsyncDuckDB, AsyncDuckDBConnection, DuckDBBundles } from "@duckdb/duckdb-wasm";
+import { AsyncDuckDB, AsyncDuckDBConnection, DuckDBBundles, DuckDBConfig } from "@duckdb/duckdb-wasm";
 import { AnalysisParams } from "@gscdump/engine/analysis-types";
 interface DuckDBWasmClient {
   db: AsyncDuckDB;
@@ -88,6 +88,12 @@ interface BootDuckDBWasmOptions {
    * worker assets themselves (e.g. Cloudflare Workers' 25 MB per-asset cap).
    */
   bundles?: DuckDBBundles;
+  /**
+   * Extra DuckDB open config. The browser runtime always forces HTTP files
+   * into range-only mode (reliable HEAD probes, no full HTTP fallback) so a
+   * server that cannot answer bounded reads fails closed.
+   */
+  config?: DuckDBConfig;
 }
 interface BrowserParquetFile {
   bytes: Uint8Array;
@@ -113,7 +119,24 @@ interface AttachParquetUrlTablesOptions {
   tables: BrowserParquetUrlTable[];
   fetch?: typeof fetch;
   schema?: string;
+  /**
+   * Request init used only for runtime-owned HEAD / one-byte Range preflights.
+   * DuckDB-WASM's internal HTTP reader cannot receive custom fetch headers;
+   * URL reads must therefore be authorized by the URL itself.
+   */
   fetchInit?: RequestInit;
+  /**
+   * Caps simultaneous URL preflights. Browser source endpoints should already
+   * return small, coverage-planned URL sets; this is the runtime's local
+   * guard against accidental unbounded attachment.
+   */
+  fetchConcurrency?: number;
+  /** Reject before preflight when the URL set exceeds this many parquet files. */
+  maxFiles?: number;
+  /** Reject before registration when hinted or authoritative bytes exceed budget. */
+  maxBytes?: number;
+  /** Abort signal passed through to URL preflights and registration. */
+  signal?: AbortSignal;
   /**
    * Manifest version the caller associates with this set of URLs. Returned
    * on the resulting handle so callers can compare against a fresh manifest
@@ -123,9 +146,11 @@ interface AttachParquetUrlTablesOptions {
   version?: number | string;
   /**
    * Called once per parquet file after it's been fetched and registered with
-   * DuckDB. Fires in non-deterministic order (Promise.all under the hood).
-   * Used by UI progress indicators to tick a per-site counter; a no-op
-   * default keeps the hot path free.
+   * DuckDB. For URL-backed HTTP files this means "preflighted and registered"
+   * rather than fully downloaded; DuckDB then performs range reads during the
+   * query. Fires in bounded-concurrency completion order, which is still not
+   * guaranteed to match manifest URL order. Used by UI progress indicators to
+   * tick a per-site counter; a no-op default keeps the hot path free.
    */
   onFileAttached?: (info: {
     table: string;
@@ -170,6 +195,9 @@ interface BrowserAnalysisRuntime {
   setAttachedTables: (tables: readonly string[]) => void;
   close: () => Promise<void>;
 }
+declare class BrowserAttachBudgetExceededError extends Error {
+  name: string;
+}
 declare function bootDuckDBWasm(options?: BootDuckDBWasmOptions): Promise<DuckDBWasmBootResult>;
 declare function attachParquetTables(options: AttachParquetTablesOptions): Promise<void>;
 declare function attachParquetUrlTables(options: AttachParquetUrlTablesOptions): Promise<AttachedTablesHandle>;
@@ -178,4 +206,4 @@ declare function createBrowserAnalysisRuntime(boot: DuckDBWasmBootResult, option
   version?: number | string;
   attachedTables?: readonly string[];
 }): BrowserAnalysisRuntime;
-export { type AnalyzeResult, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachParquetTables, attachParquetUrlTables, bootDuckDBWasm, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, devices, drizzle, hourly_pages, keywords, mergeScope, page_keywords, pages, resolveWindow, schema, scopeFor, strikingMomentum };
+export { type AnalyzeResult, type AttachParquetTablesOptions, type AttachParquetUrlTablesOptions, type AttachedTablesHandle, type BootDuckDBWasmOptions, type BrowserAnalysisRuntime, BrowserAttachBudgetExceededError, type BrowserParquetFile, type BrowserParquetTable, type BrowserParquetUrlTable, type ComparisonMode, type DuckDBWasmBootResult, type DuckDBWasmClient, DuckDBWasmDatabase, type DuckDBWasmDrizzleDatabase, type InsightRunner, type InsightRunnerOptions, type QueryResult, type ResolveWindowOptions, type ResolvedWindow, type Schema, type ScopedRunnerOptions, type StrikingMomentumOptions, type StrikingMomentumRow, type TableScope, type WindowPreset, attachParquetTables, attachParquetUrlTables, bootDuckDBWasm, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, devices, drizzle, hourly_pages, keywords, mergeScope, page_keywords, pages, resolveWindow, schema, scopeFor, strikingMomentum };

package/dist/index.mjs

@@ -167,12 +167,140 @@ async function createInsightRunner(opts) {
 	};
 }
 const { scopeFor, mergeScope } = createScopedHelpers(schema);
+const DEFAULT_ATTACH_FETCH_CONCURRENCY = 2;
+const DEFAULT_ATTACH_MAX_FILES = 32;
+const DEFAULT_ATTACH_MAX_BYTES = 16 * 1024 * 1024;
+let nextAttachId = 0;
+var BrowserAttachBudgetExceededError = class extends Error {
+	name = "BrowserAttachBudgetExceededError";
+};
 function fileName(table, index, provided) {
 	return provided ?? `${table}_${index}.parquet`;
 }
+function attachFileName(attachId, table, index) {
+	return `__gscdump_attach_${attachId}_${fileName(table, index)}`;
+}
 function readParquetViewSql(schema, table, files) {
 	return `CREATE OR REPLACE VIEW ${schema}.${table} AS SELECT * REPLACE (CAST(date AS DATE) AS date) FROM read_parquet([${files.map((name) => `'${sqlEscape(name)}'`).join(", ")}], union_by_name = true)`;
 }
+function positiveInteger(value, fallback, label) {
+	const raw = value ?? fallback;
+	if (!Number.isFinite(raw) || raw < 1) throw new Error(`${label} must be a positive integer`);
+	return Math.floor(raw);
+}
+async function runWithConcurrency(items, concurrency, fn) {
+	let next = 0;
+	let failed = false;
+	async function worker() {
+		while (!failed && next < items.length) {
+			const index = next++;
+			try {
+				await fn(items[index], index);
+			} catch (err) {
+				failed = true;
+				throw err;
+			}
+		}
+	}
+	await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
+}
+function sizeHintFromUrl(url) {
+	try {
+		const base = typeof globalThis.location?.href === "string" ? globalThis.location.href : "http://localhost";
+		const raw = new URL(url, base).searchParams.get("s")?.split(".")[0];
+		if (!raw) return null;
+		const size = Number(raw);
+		return Number.isFinite(size) && size >= 0 ? size : null;
+	} catch {
+		return null;
+	}
+}
+function mergeAbortSignals(primary, secondary) {
+	if (!primary) return secondary;
+	if (!secondary) return primary;
+	if (primary.aborted) return primary;
+	if (secondary.aborted) return secondary;
+	const controller = new AbortController();
+	const abort = (signal) => {
+		controller.abort(signal.reason);
+	};
+	primary.addEventListener("abort", () => abort(primary), { once: true });
+	secondary.addEventListener("abort", () => abort(secondary), { once: true });
+	return controller.signal;
+}
+function fetchInitFor(fetchInit, method, signal, extraHeaders) {
+	const { body: _body, method: _method, signal: initSignal, headers: initHeaders, ...rest } = fetchInit ?? {};
+	const headers = new Headers(initHeaders);
+	for (const [key, value] of Object.entries(extraHeaders ?? {})) headers.set(key, value);
+	return {
+		...rest,
+		method,
+		headers,
+		signal: mergeAbortSignals(signal, initSignal ?? void 0)
+	};
+}
+function isAbortError(err) {
+	return typeof err === "object" && err !== null && err.name === "AbortError";
+}
+function parseContentLength(headers) {
+	const raw = headers.get("content-length");
+	if (!raw) return null;
+	const size = Number(raw);
+	return Number.isFinite(size) && size >= 0 ? size : null;
+}
+function parseContentRangeSize(headers) {
+	const raw = headers.get("content-range");
+	if (!raw) return null;
+	const match = /^bytes\s+\d+-\d+\/(\d+)$/i.exec(raw);
+	if (!match) return null;
+	const size = Number(match[1]);
+	return Number.isFinite(size) && size >= 0 ? size : null;
+}
+function supportsRangeReads(headers) {
+	return headers.get("accept-ranges")?.toLowerCase().split(",").map((v) => v.trim()).includes("bytes") === true;
+}
+async function cancelBody(response) {
+	await response.body?.cancel().catch(() => void 0);
+}
+async function preflightHttpUrl(url, fetchImpl, fetchInit, signal) {
+	const head = await fetchImpl(url, fetchInitFor(fetchInit, "HEAD", signal));
+	if (head.ok) {
+		const size = parseContentLength(head.headers);
+		if (size === null) throw new Error(`HEAD ${url} missing Content-Length`);
+		if (!supportsRangeReads(head.headers)) throw new Error(`HEAD ${url} missing Accept-Ranges: bytes`);
+		return size;
+	}
+	await cancelBody(head);
+	if (![
+		403,
+		405,
+		501
+	].includes(head.status)) throw new Error(`HEAD ${url} failed: ${head.status}`);
+	const probe = await fetchImpl(url, fetchInitFor(fetchInit, "GET", signal, { Range: "bytes=0-0" }));
+	try {
+		if (probe.status !== 206) throw new Error(`range probe ${url} failed: ${probe.status}`);
+		const size = parseContentRangeSize(probe.headers);
+		if (size === null) throw new Error(`range probe ${url} missing Content-Range size`);
+		return size;
+	} finally {
+		await cancelBody(probe);
+	}
+}
+function rangeOnlyConfig(config) {
+	return {
+		...config ?? {},
+		filesystem: {
+			...config?.filesystem ?? {},
+			reliableHeadRequests: true,
+			allowFullHTTPReads: false,
+			forceFullHTTPReads: false
+		}
+	};
+}
+async function dropAttachedResources(db, conn, schema, tables, files) {
+	for (const table of tables) await conn.query(`DROP VIEW IF EXISTS ${schema}.${table}`);
+	if (files.length > 0) await db.dropFiles([...files]);
+}
 async function bootDuckDBWasm(options = {}) {
 	const { getJsDelivrBundles, selectBundle, AsyncDuckDB, ConsoleLogger } = await import("@duckdb/duckdb-wasm");
 	const bundle = await selectBundle(options.bundles ?? getJsDelivrBundles());
@@ -180,6 +308,7 @@ async function bootDuckDBWasm(options = {}) {
 	const worker = new Worker(workerUrl);
 	const db = new AsyncDuckDB(options.logger ?? new ConsoleLogger(), worker);
 	await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+	await db.open(rangeOnlyConfig(options.config));
 	URL.revokeObjectURL(workerUrl);
 	return {
 		db,
@@ -200,7 +329,11 @@ async function attachParquetTables(options) {
 	}
 }
 async function attachParquetUrlTables(options) {
-	const { db, conn, tables, fetch: fetchImpl = globalThis.fetch.bind(globalThis), schema = "main", fetchInit, version, onFileAttached } = options;
+	const { db, conn, tables, fetch: fetchImpl = globalThis.fetch.bind(globalThis), schema = "main", fetchInit, fetchConcurrency, maxFiles, maxBytes, signal, version, onFileAttached } = options;
+	const concurrency = positiveInteger(fetchConcurrency, DEFAULT_ATTACH_FETCH_CONCURRENCY, "fetchConcurrency");
+	const fileBudget = positiveInteger(maxFiles, DEFAULT_ATTACH_MAX_FILES, "maxFiles");
+	const byteBudget = positiveInteger(maxBytes, DEFAULT_ATTACH_MAX_BYTES, "maxBytes");
+	const attachId = nextAttachId++;
 	const flat = [];
 	const counts = {};
 	for (const [table, urls] of tables.map((t) => [t.table, t.urls])) {
@@ -212,38 +345,79 @@ async function attachParquetUrlTables(options) {
 			index: i
 		});
 	}
+	if (flat.length > fileBudget) throw new BrowserAttachBudgetExceededError(`browser parquet attach requires ${flat.length} files, above maxFiles=${fileBudget}`);
+	const hintedBytes = flat.reduce((acc, item) => {
+		const hint = sizeHintFromUrl(item.url);
+		return hint === null ? acc : acc + hint;
+	}, 0);
+	if (hintedBytes > byteBudget) throw new BrowserAttachBudgetExceededError(`browser parquet attach requires ${hintedBytes} hinted bytes, above maxBytes=${byteBudget}`);
 	const tableFailures = /* @__PURE__ */ new Map();
+	const budgetController = new AbortController();
+	const effectiveSignal = mergeAbortSignals(signal, budgetController.signal);
+	let plannedBytes = 0;
 	const total = flat.length;
-	await Promise.all(flat.map(async ({ table, url, index }) => {
+	const preflighted = [];
+	await runWithConcurrency(flat, concurrency, async ({ table, url, index }) => {
 		if (tableFailures.has(table)) return;
-		await fetchImpl(url, fetchInit).then(async (response) => {
-			if (!response.ok) throw new Error(`fetch ${url} failed: ${response.status}`);
-			const bytes = new Uint8Array(await response.arrayBuffer());
-			await db.registerFileBuffer(fileName(table, index), bytes);
-			onFileAttached?.({
+		effectiveSignal?.throwIfAborted();
+		await preflightHttpUrl(url, fetchImpl, fetchInit, effectiveSignal).then((bytes) => {
+			plannedBytes += bytes;
+			if (plannedBytes > byteBudget) {
+				const err = new BrowserAttachBudgetExceededError(`browser parquet attach planned ${plannedBytes} bytes, above maxBytes=${byteBudget}`);
+				budgetController.abort(err);
+				throw err;
+			}
+			effectiveSignal?.throwIfAborted();
+			preflighted.push({
 				table,
+				url,
 				index,
-				total
+				name: attachFileName(attachId, table, index)
 			});
 		}).catch((err) => {
+			if (effectiveSignal?.aborted || err instanceof BrowserAttachBudgetExceededError || isAbortError(err)) throw err;
 			tableFailures.set(table, err instanceof Error ? err : new Error(String(err)));
 		});
-	}));
+	});
+	const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
 	const attached = [];
-	for (const table of Object.keys(counts)) {
-		if (tableFailures.has(table)) continue;
-		const names = [];
-		for (let i = 0; i < counts[table]; i++) names.push(fileName(table, i));
-		await conn.query(readParquetViewSql(schema, table, names));
-		attached.push(table);
+	const registeredFiles = [];
+	try {
+		for (const file of preflighted) {
+			if (tableFailures.has(file.table)) continue;
+			effectiveSignal?.throwIfAborted();
+			await db.registerFileURL(file.name, file.url, DuckDBDataProtocol.HTTP, false);
+			registeredFiles.push(file.name);
+			onFileAttached?.({
+				table: file.table,
+				index: file.index,
+				total
+			});
+		}
+		for (const table of Object.keys(counts)) {
+			if (tableFailures.has(table)) continue;
+			const files = preflighted.filter((file) => file.table === table).sort((a, b) => a.index - b.index);
+			if (files.length !== counts[table]) continue;
+			effectiveSignal?.throwIfAborted();
+			await conn.query(readParquetViewSql(schema, table, files.map((file) => file.name)));
+			attached.push(table);
+		}
+	} catch (err) {
+		await dropAttachedResources(db, conn, schema, attached, registeredFiles).catch((cleanupErr) => {
+			console.warn("[gscdump/engine-duckdb-wasm] cleanup after failed attach failed", cleanupErr);
+		});
+		throw err;
 	}
 	if (tableFailures.size > 0) for (const [table, err] of tableFailures) console.warn(`[gscdump/engine-duckdb-wasm] dropped table "${table}" — ${err.message}`);
+	let detached = false;
 	return {
 		version,
 		tables: attached,
 		schema,
 		async detach() {
-			for (const table of attached) await conn.query(`DROP VIEW IF EXISTS ${schema}.${table}`);
+			if (detached) return;
+			detached = true;
+			await dropAttachedResources(db, conn, schema, attached, registeredFiles);
 		}
 	};
 }
@@ -253,12 +427,32 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
 	let version = options.version;
 	let attachedTables = options.attachedTables;
 	let chain = Promise.resolve();
+	function abortError(signal) {
+		return signal.reason ?? new DOMException("aborted", "AbortError");
+	}
+	function raceSignal(promise, signal) {
+		if (!signal) return promise;
+		if (signal.aborted) return Promise.reject(abortError(signal));
+		return new Promise((resolve, reject) => {
+			const onAbort = () => reject(abortError(signal));
+			signal.addEventListener("abort", onAbort, { once: true });
+			promise.then((value) => {
+				signal.removeEventListener("abort", onAbort);
+				resolve(value);
+			}, (err) => {
+				signal.removeEventListener("abort", onAbort);
+				reject(err);
+			});
+		});
+	}
+	function runExclusive(signal, work) {
+		const next = chain.then(work, work);
+		chain = next.catch(() => {});
+		return raceSignal(next, signal);
+	}
 	async function cancelOnAbort(signal, work) {
 		if (!signal) return work;
-		if (signal.aborted) {
-			conn.cancelSent().catch(() => {});
-			throw signal.reason ?? new DOMException("aborted", "AbortError");
-		}
+		if (signal.aborted) throw abortError(signal);
 		const onAbort = () => {
 			conn.cancelSent().catch(() => {});
 		};
@@ -269,7 +463,7 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
 			signal.removeEventListener("abort", onAbort);
 		}
 	}
-	async function runParameterized(sql, params, signal) {
+	async function runParameterizedDirect(sql, params, signal) {
 		signal?.throwIfAborted();
 		return cancelOnAbort(signal, (async () => {
 			if (!params || params.length === 0) return conn.query(sql);
@@ -287,7 +481,7 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
 		async query(sql, params, signal) {
 			const t0 = performance.now();
 			return {
-				rows: arrowToRows(await runParameterized(sql, params, signal)),
+				rows: arrowToRows(await runExclusive(signal, () => runParameterizedDirect(sql, params, signal))),
 				queryMs: performance.now() - t0
 			};
 		},
@@ -297,7 +491,7 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
 				signal?.throwIfAborted();
 				const t0 = performance.now();
 				const result = await runAnalyzerFromSource(createAttachedTableSource({ query: async (sql, bindParams, innerSignal) => {
-					return arrowToRows(await runParameterized(sql, bindParams, innerSignal ?? signal));
+					return arrowToRows(await runParameterizedDirect(sql, bindParams, innerSignal ?? signal));
 				} }, {
 					schema,
 					signal,
@@ -310,9 +504,7 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
 					queryMs: performance.now() - t0
 				};
 			};
-			const next = chain.then(run, run);
-			chain = next.catch(() => {});
-			return next;
+			return runExclusive(signal, run);
 		},
 		isStale(expected) {
 			return expected !== version;
@@ -329,4 +521,4 @@ function createBrowserAnalysisRuntime(boot, options = {}) {
 		}
 	};
 }
-export { DuckDBWasmDatabase, attachParquetTables, attachParquetUrlTables, bootDuckDBWasm, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, devices, drizzle, hourly_pages, keywords, mergeScope, page_keywords, pages, resolveWindow, schema, scopeFor, strikingMomentum };
+export { BrowserAttachBudgetExceededError, DuckDBWasmDatabase, attachParquetTables, attachParquetUrlTables, bootDuckDBWasm, countries, createBrowserAnalysisRuntime, createClient, createInsightRunner, devices, drizzle, hourly_pages, keywords, mergeScope, page_keywords, pages, resolveWindow, schema, scopeFor, strikingMomentum };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-duckdb-wasm",
   "type": "module",
-  "version": "0.19.6",
+  "version": "0.20.0",
   "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
   "author": {
     "name": "Harlan Wilton",
@@ -45,8 +45,8 @@
   },
   "dependencies": {
     "drizzle-orm": "^0.45.2",
-    "@gscdump/engine": "0.19.6",
-    "gscdump": "0.19.6"
+    "@gscdump/engine": "0.20.0",
+    "gscdump": "0.20.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",