@gscdump/cloudflare 0.38.1 → 0.39.0

package/dist/index.d.mts

@@ -21,6 +21,7 @@ interface AnalyticsEnv {
       tables?: Record<string, {
         ipc: Uint8Array;
       }>;
+      deadlineAt?: number;
     }) => Promise<{
       rows: unknown[];
       sql: string;
@@ -61,6 +62,8 @@ declare function useAnalyticsEnv(event: H3Event): AnalyticsEnv;
 interface AnalyticsEngineHooks {
   /** Called once per R2 PUT, with the byte size of the payload. */
   onR2Write?: (byteLength: number) => void;
+  /** Override the interactive 22-second DuckDB RPC budget for background work. */
+  duckdbRpcTimeoutMs?: number;
 }
 interface AnalyticsEngineRuntime {
   getEngine: (env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks) => ReturnType<typeof createStorageEngine> | null;
@@ -94,6 +97,8 @@ interface DucklingsExecutorOptions {
   ipcChunkBytes?: number;
   ipcDirectCallBytes?: number;
   ipcTotalBytes?: number;
+  /** Per-RPC wall/queue budget. Interactive default is 22 seconds. */
+  rpcTimeoutMs?: number;
   rowCache?: DucklingsRowCache;
 }
 declare function createDucklingsExecutor(env: AnalyticsEnv, opts?: DucklingsExecutorOptions): QueryExecutor;

package/dist/index.mjs

@@ -74,15 +74,32 @@ var DuckDBServiceTimeoutError = class extends Error {
 function withDuckDBDeadline(op, timeoutMs, signal) {
 	if (signal?.aborted) return Promise.reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
 	return new Promise((resolve, reject) => {
-		const timer = setTimeout(() => reject(new DuckDBServiceTimeoutError(timeoutMs)), timeoutMs);
-		const onAbort = () => reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
+		let settled = false;
+		let timer;
+		let onAbort;
+		const cleanup = () => {
+			if (timer !== void 0) clearTimeout(timer);
+			if (onAbort) signal?.removeEventListener("abort", onAbort);
+		};
+		const settle = (complete, value) => {
+			if (settled) return;
+			settled = true;
+			cleanup();
+			complete(value);
+		};
+		onAbort = () => settle(reject, signal.reason ?? new DOMException("Aborted", "AbortError"));
+		timer = setTimeout(() => settle(reject, new DuckDBServiceTimeoutError(timeoutMs)), timeoutMs);
 		signal?.addEventListener("abort", onAbort, { once: true });
-		op.then(resolve, reject).finally(() => {
-			clearTimeout(timer);
-			signal?.removeEventListener("abort", onAbort);
-		});
+		op.then((value) => settle(resolve, value), (error) => settle(reject, error));
 	});
 }
+function runSQLWithDeadline(svc, args, timeoutMs, signal) {
+	const deadlineAt = Date.now() + timeoutMs;
+	return withDuckDBDeadline(svc.runSQL({
+		...args,
+		deadlineAt
+	}), timeoutMs, signal);
+}
 function createDucklingsCodec(_env) {
 	return createHyparquetCodec();
 }
@@ -95,13 +112,21 @@ async function mapLimit(items, concurrency, fn) {
 	const limit = Math.max(1, Math.floor(concurrency));
 	const out = Array.from({ length: items.length });
 	let next = 0;
+	let failed = false;
+	let failure;
 	async function worker() {
-		while (next < items.length) {
+		while (!failed && next < items.length) {
 			const index = next++;
-			out[index] = await fn(items[index], index);
+			try {
+				out[index] = await fn(items[index], index);
+			} catch (error) {
+				failed = true;
+				failure = error;
+			}
 		}
 	}
 	await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+	if (failed) throw failure;
 	return out;
 }
 function assertWorkerReadBudget(opts) {
@@ -247,19 +272,42 @@ function createDucklingsRowCache(maxBytes = ROW_CACHE_MAX_BYTES) {
 }
 function createDucklingsExecutor(env, opts = {}) {
 	const rowCache = opts.rowCache ?? createDucklingsRowCache();
+	const rpcTimeoutMs = opts.rpcTimeoutMs ?? DUCKDB_RPC_TIMEOUT_MS;
+	if (!Number.isFinite(rpcTimeoutMs) || rpcTimeoutMs <= 0) throw new TypeError("createDucklingsExecutor: rpcTimeoutMs must be a positive finite number");
 	return { async execute({ sql, params, fileKeys, placeholderTables, pushdownFilters, dataSource, signal, table }) {
 		signal?.throwIfAborted();
 		const svc = resolveSvc(env);
 		assertWorkerReadBudget({ fileKeys });
-		if (dataSource.head) {
-			const uniqueKeys = [...new Set(Object.values(fileKeys).flat())];
+		const cachedUnfiltered = /* @__PURE__ */ new Map();
+		const scheduledUnfiltered = /* @__PURE__ */ new Set();
+		const plannedReadKeys = [];
+		for (const [placeholder, keys] of Object.entries(fileKeys)) {
+			const filter = pushdownFilters?.[placeholder];
+			for (const key of keys) {
+				if (filter) {
+					plannedReadKeys.push(key);
+					continue;
+				}
+				const cached = cachedUnfiltered.get(key) ?? rowCache.get(key);
+				if (cached !== void 0) {
+					cachedUnfiltered.set(key, cached);
+					continue;
+				}
+				if (!scheduledUnfiltered.has(key)) {
+					scheduledUnfiltered.add(key);
+					plannedReadKeys.push(key);
+				}
+			}
+		}
+		if (dataSource.head && plannedReadKeys.length > 0) {
+			const uniqueKeys = [...new Set(plannedReadKeys)];
 			const sizes = {};
 			await mapLimit(uniqueKeys, WORKER_R2_HEAD_CONCURRENCY, async (key) => {
 				signal?.throwIfAborted();
 				sizes[key] = (await dataSource.head(key))?.bytes;
 			});
 			assertWorkerReadBudget({
-				fileKeys,
+				fileKeys: { READS: plannedReadKeys },
 				sizes
 			});
 		}
@@ -269,20 +317,32 @@ function createDucklingsExecutor(env, opts = {}) {
 		const maxChunkBytes = opts.ipcChunkBytes ?? IPC_CHUNK_BUDGET;
 		const maxDirectCallBytes = opts.ipcDirectCallBytes ?? IPC_DIRECT_CALL_BUDGET;
 		const maxTotalBytes = opts.ipcTotalBytes ?? IPC_STAGED_TOTAL_BUDGET;
+		const unfilteredLoads = /* @__PURE__ */ new Map();
+		for (const [key, rows] of cachedUnfiltered) unfilteredLoads.set(key, Promise.resolve(rows));
+		const loadUnfiltered = (key) => {
+			let loading = unfilteredLoads.get(key);
+			if (!loading) {
+				loading = (async () => {
+					signal?.throwIfAborted();
+					const bytes = await dataSource.read(key, void 0, signal);
+					signal?.throwIfAborted();
+					const rows = await decodeParquetToRows(bytes);
+					rowCache.put(key, rows);
+					return rows;
+				})();
+				unfilteredLoads.set(key, loading);
+			}
+			return loading;
+		};
 		for (const [placeholder, keys] of Object.entries(fileKeys)) {
 			signal?.throwIfAborted();
 			const filter = pushdownFilters?.[placeholder];
 			const perFile = await mapLimit(keys, WORKER_R2_DECODE_CONCURRENCY, async (key) => {
-				if (!filter) {
-					const cached = rowCache.get(key);
-					if (cached) return cached;
-				}
+				if (!filter) return loadUnfiltered(key);
 				signal?.throwIfAborted();
 				const bytes = await dataSource.read(key, void 0, signal);
 				signal?.throwIfAborted();
-				const rows = await decodeParquetToRows(bytes, filter ? { filter } : {});
-				if (!filter) rowCache.put(key, rows);
-				return rows;
+				return await decodeParquetToRows(bytes, filter ? { filter } : {});
 			});
 			const merged = [];
 			for (const rows of perFile) merged.push(...rows);
@@ -308,10 +368,10 @@ function createDucklingsExecutor(env, opts = {}) {
 		if (canInlineTables) {
 			const tables = {};
 			for (const [name, chunks] of Object.entries(tableChunks)) tables[name] = { ipc: chunks[0].ipc };
-			result = await withDuckDBDeadline(svc.runSQL({
+			result = await runSQLWithDeadline(svc, {
 				sql: finalSql,
 				tables
-			}), DUCKDB_RPC_TIMEOUT_MS, signal);
+			}, rpcTimeoutMs, signal);
 		} else {
 			if (!svc.stageArrowTable || !svc.dropTables) throw new Error("createDucklingsExecutor: DUCKDB_SVC does not support chunked Arrow IPC staging. Deploy the gscdump-duckdb worker with stageArrowTable/dropTables support.");
 			const staged = /* @__PURE__ */ new Set();
@@ -324,14 +384,14 @@ function createDucklingsExecutor(env, opts = {}) {
 					await withDuckDBDeadline(svc.stageArrowTable({
 						table: name,
 						ipc: chunk.ipc
-					}), DUCKDB_RPC_TIMEOUT_MS, signal);
+					}), rpcTimeoutMs, signal);
 				}
-				result = await withDuckDBDeadline(svc.runSQL({ sql: finalSql }), DUCKDB_RPC_TIMEOUT_MS, signal);
+				result = await runSQLWithDeadline(svc, { sql: finalSql }, rpcTimeoutMs, signal);
 			} catch (error) {
 				primaryError = error;
 			} finally {
 				if (staged.size > 0) try {
-					await withDuckDBDeadline(svc.dropTables({ tables: [...staged] }), DUCKDB_RPC_TIMEOUT_MS);
+					await withDuckDBDeadline(svc.dropTables({ tables: [...staged] }), rpcTimeoutMs);
 				} catch (error) {
 					cleanupError = error;
 				}
@@ -364,7 +424,10 @@ function createAnalyticsEngineRuntime() {
 			} : baseDataSource,
 			manifestStore: createD1ManifestStore(db),
 			codec: createDucklingsCodec(env),
-			executor: createDucklingsExecutor(env, { rowCache })
+			executor: createDucklingsExecutor(env, {
+				rowCache,
+				...hooks.duckdbRpcTimeoutMs !== void 0 ? { rpcTimeoutMs: hooks.duckdbRpcTimeoutMs } : {}
+			})
 		});
 	} };
 }

package/dist/server-tail/index.d.mts

@@ -47,6 +47,7 @@ type DuckDbIcebergRow = Record<string, string | number | null>;
 interface DuckDbSvc {
   runSQL: (args: {
     sql: string;
+    deadlineAt?: number;
   }) => Promise<{
     rows: unknown[];
     sql: string;

package/dist/server-tail/index.mjs

@@ -460,7 +460,11 @@ function createDuckDbIcebergExecutor(config) {
 	const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS$1;
 	async function sendResult(sql) {
 		const started = Date.now();
-		const raced = await withDeadline(config.svc.runSQL({ sql }), timeoutMs).then((value) => ok(value)).catch((error) => error instanceof DuckDbIcebergTimeoutError ? err(error) : err(new DuckDbIcebergError(`DUCKDB_SVC.runSQL failed: ${error.message}`)));
+		const deadlineAt = Date.now() + timeoutMs;
+		const raced = await withDeadline(config.svc.runSQL({
+			sql,
+			deadlineAt
+		}), timeoutMs).then((value) => ok(value)).catch((error) => error instanceof DuckDbIcebergTimeoutError ? err(error) : err(new DuckDbIcebergError(`DUCKDB_SVC.runSQL failed: ${error.message}`)));
 		if (!raced.ok) return raced;
 		const result = raced.value;
 		return ok({

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/cloudflare",
   "type": "module",
-  "version": "0.38.1",
+  "version": "0.39.0",
   "description": "Cloudflare-Workers-flavored helpers for the gscdump analytics stack: AnalyticsEnv binding contract, R2 SigV4 presigner, size-hint HMAC, DuckDB Workers shims, engine factory.",
   "author": {
     "name": "Harlan Wilton",
@@ -46,10 +46,10 @@
   "dependencies": {
     "@uwdata/flechette": "^2.5.0",
     "aws4fetch": "^1.0.20",
-    "@gscdump/contracts": "0.38.1",
-    "@gscdump/engine": "0.38.1",
-    "@gscdump/engine-sqlite": "0.38.1",
-    "gscdump": "0.38.1"
+    "@gscdump/engine": "0.39.0",
+    "@gscdump/contracts": "0.39.0",
+    "gscdump": "0.39.0",
+    "@gscdump/engine-sqlite": "0.39.0"
   },
   "devDependencies": {
     "@cloudflare/vitest-pool-workers": "^0.18.4",