@gscdump/cloudflare 0.37.2 → 0.37.4

package/dist/index.d.mts

@@ -1,4 +1,4 @@
-import { ParquetCodec, QueryExecutor, createStorageEngine } from "@gscdump/engine";
+import { ParquetCodec, QueryExecutor, Row, createStorageEngine } from "@gscdump/engine";
 import { H3Event } from "h3";
 interface AnalyticsEnv {
   /** R2 bucket holding parquet + rollup + entity data. Required in origin mode. */
@@ -62,6 +62,10 @@ interface AnalyticsEngineHooks {
   /** Called once per R2 PUT, with the byte size of the payload. */
   onR2Write?: (byteLength: number) => void;
 }
+interface AnalyticsEngineRuntime {
+  getEngine: (env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks) => ReturnType<typeof createStorageEngine> | null;
+}
+declare function createAnalyticsEngineRuntime(): AnalyticsEngineRuntime;
 declare function getAnalyticsEngine(env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks): ReturnType<typeof createStorageEngine> | null;
 interface InflightDedupe<T> {
   dedupe: (key: string, run: () => Promise<T>) => Promise<T>;
@@ -80,10 +84,17 @@ declare function getHostedR2QueryKey(input: HostedR2QueryKeyInput): string;
 declare function signSizeHint(env: AnalyticsEnv, key: string, bytes: number): Promise<string>;
 declare function verifySizeHint(env: AnalyticsEnv, key: string, bytes: number, providedHex: string): Promise<boolean>;
 declare function createDucklingsCodec(_env: AnalyticsEnv): ParquetCodec;
+interface DucklingsRowCache {
+  clear: () => void;
+  get: (key: string) => Row[] | undefined;
+  put: (key: string, rows: Row[]) => void;
+}
+declare function createDucklingsRowCache(maxBytes?: number): DucklingsRowCache;
 interface DucklingsExecutorOptions {
   ipcChunkBytes?: number;
   ipcDirectCallBytes?: number;
   ipcTotalBytes?: number;
+  rowCache?: DucklingsRowCache;
 }
 declare function createDucklingsExecutor(env: AnalyticsEnv, opts?: DucklingsExecutorOptions): QueryExecutor;
-export { type AnalyticsEngineHooks, type AnalyticsEnv, type HostedR2QueryKeyInput, type InflightDedupe, createDucklingsCodec, createDucklingsExecutor, createInflightDedupe, getAnalyticsEngine, getHostedR2QueryKey, signSizeHint, useAnalyticsEnv, verifySizeHint };
+export { type AnalyticsEngineHooks, type AnalyticsEngineRuntime, type AnalyticsEnv, type DucklingsRowCache, type HostedR2QueryKeyInput, type InflightDedupe, createAnalyticsEngineRuntime, createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache, createInflightDedupe, getAnalyticsEngine, getHostedR2QueryKey, signSizeHint, useAnalyticsEnv, verifySizeHint };

package/dist/index.mjs

@@ -119,55 +119,54 @@ function assertWorkerReadBudget(opts) {
 	if (totalBytes > maxBytes) throw new Error(`createDucklingsExecutor: planned read spans ${totalBytes} bytes, exceeding the ${maxBytes} byte Worker budget. Narrow the date range or route through a background/windowed query.`);
 }
 const ROW_CACHE_MAX_BYTES = 16 * 1024 * 1024;
-let rowCacheBytes = 0;
-const rowCache = /* @__PURE__ */ new Map();
 function estimateRowsBytes(rows) {
 	if (rows.length === 0) return 0;
 	const cols = Object.keys(rows[0]).length;
 	return rows.length * cols * 64;
 }
-function inferExtraColumnType(values) {
-	let hasValue = false;
-	let hasString = false;
-	let hasFloat = false;
-	let hasBigInt = false;
-	for (const value of values) {
-		if (value === null || value === void 0) continue;
-		hasValue = true;
-		if (typeof value === "string") {
-			hasString = true;
-			break;
-		}
-		if (typeof value === "bigint") {
-			hasBigInt = true;
-			continue;
-		}
-		if (typeof value === "number") {
-			if (!Number.isInteger(value)) hasFloat = true;
-			if (value > 2147483647 || value < -2147483648) hasBigInt = true;
-		}
+function addInferredValue(inference, value) {
+	if (value === null || value === void 0) return;
+	inference.hasValue = true;
+	if (typeof value === "string") {
+		inference.hasString = true;
+		return;
 	}
-	if (!hasValue || hasString) return "VARCHAR";
-	if (hasFloat) return "DOUBLE";
-	return hasBigInt ? "BIGINT" : "INTEGER";
+	if (typeof value === "bigint") {
+		inference.hasBigInt = true;
+		return;
+	}
+	if (typeof value === "number") {
+		if (!Number.isInteger(value)) inference.hasFloat = true;
+		if (value > 2147483647 || value < -2147483648) inference.hasBigInt = true;
+	}
+}
+function inferredColumnType(inference) {
+	if (!inference.hasValue || inference.hasString) return "VARCHAR";
+	if (inference.hasFloat) return "DOUBLE";
+	return inference.hasBigInt ? "BIGINT" : "INTEGER";
 }
 function chunkSchemaColumns(rows, schemaColumns) {
 	const columns = schemaColumns ? [...schemaColumns] : [];
 	const seen = new Set(columns.map((c) => c.name));
-	const extraValues = /* @__PURE__ */ new Map();
+	const extraColumns = /* @__PURE__ */ new Map();
 	for (const row of rows) for (const key in row) {
 		if (seen.has(key)) continue;
-		let values = extraValues.get(key);
-		if (!values) {
-			values = [];
-			extraValues.set(key, values);
+		let inference = extraColumns.get(key);
+		if (!inference) {
+			inference = {
+				hasValue: false,
+				hasString: false,
+				hasFloat: false,
+				hasBigInt: false
+			};
+			extraColumns.set(key, inference);
 		}
-		values.push(row[key]);
+		addInferredValue(inference, row[key]);
 	}
-	if (extraValues.size === 0) return schemaColumns ? columns : void 0;
-	for (const [name, values] of extraValues) columns.push({
+	if (extraColumns.size === 0) return schemaColumns ? columns : void 0;
+	for (const [name, inference] of extraColumns) columns.push({
 		name,
-		type: inferExtraColumnType(values),
+		type: inferredColumnType(inference),
 		nullable: true
 	});
 	return columns;
@@ -208,30 +207,46 @@ function rowsToArrowIPCChunks(rows, schemaColumns, opts = {}) {
 	}
 	return chunks;
 }
-function rowCacheGet(key) {
-	const hit = rowCache.get(key);
-	if (!hit) return void 0;
-	rowCache.delete(key);
-	rowCache.set(key, hit);
-	return hit.rows;
-}
-function rowCachePut(key, rows) {
-	const bytes = estimateRowsBytes(rows);
-	if (bytes > ROW_CACHE_MAX_BYTES) return;
-	while (rowCacheBytes + bytes > ROW_CACHE_MAX_BYTES) {
-		const oldest = rowCache.keys().next().value;
-		if (oldest === void 0) break;
-		const evicted = rowCache.get(oldest);
-		rowCache.delete(oldest);
-		rowCacheBytes -= evicted.bytes;
-	}
-	rowCache.set(key, {
-		rows,
-		bytes
-	});
-	rowCacheBytes += bytes;
+function createDucklingsRowCache(maxBytes = ROW_CACHE_MAX_BYTES) {
+	let totalBytes = 0;
+	const entries = /* @__PURE__ */ new Map();
+	return {
+		clear() {
+			entries.clear();
+			totalBytes = 0;
+		},
+		get(key) {
+			const hit = entries.get(key);
+			if (!hit) return void 0;
+			entries.delete(key);
+			entries.set(key, hit);
+			return hit.rows;
+		},
+		put(key, rows) {
+			const bytes = estimateRowsBytes(rows);
+			if (bytes > maxBytes) return;
+			const existing = entries.get(key);
+			if (existing) {
+				entries.delete(key);
+				totalBytes -= existing.bytes;
+			}
+			while (totalBytes + bytes > maxBytes) {
+				const oldest = entries.keys().next().value;
+				if (oldest === void 0) break;
+				const evicted = entries.get(oldest);
+				entries.delete(oldest);
+				totalBytes -= evicted.bytes;
+			}
+			entries.set(key, {
+				rows,
+				bytes
+			});
+			totalBytes += bytes;
+		}
+	};
 }
 function createDucklingsExecutor(env, opts = {}) {
+	const rowCache = opts.rowCache ?? createDucklingsRowCache();
 	return { async execute({ sql, params, fileKeys, placeholderTables, pushdownFilters, dataSource, signal, table }) {
 		signal?.throwIfAborted();
 		const svc = resolveSvc(env);
@@ -259,14 +274,14 @@ function createDucklingsExecutor(env, opts = {}) {
 			const filter = pushdownFilters?.[placeholder];
 			const perFile = await mapLimit(keys, WORKER_R2_DECODE_CONCURRENCY, async (key) => {
 				if (!filter) {
-					const cached = rowCacheGet(key);
+					const cached = rowCache.get(key);
 					if (cached) return cached;
 				}
 				signal?.throwIfAborted();
 				const bytes = await dataSource.read(key, void 0, signal);
 				signal?.throwIfAborted();
 				const rows = await decodeParquetToRows(bytes, filter ? { filter } : {});
-				if (!filter) rowCachePut(key, rows);
+				if (!filter) rowCache.put(key, rows);
 				return rows;
 			});
 			const merged = [];
@@ -331,24 +346,32 @@ function createDucklingsExecutor(env, opts = {}) {
 		};
 	} };
 }
+function createAnalyticsEngineRuntime() {
+	const rowCache = createDucklingsRowCache();
+	return { getEngine(env, db, hooks = {}) {
+		if (!env.R2_DATA) return null;
+		const baseDataSource = createR2DataSource({
+			bucket: env.R2_DATA,
+			bucketName: env.R2_BUCKET_NAME
+		});
+		return createStorageEngine({
+			dataSource: hooks.onR2Write ? {
+				...baseDataSource,
+				async write(key, bytes) {
+					hooks.onR2Write(bytes.byteLength);
+					return baseDataSource.write(key, bytes);
+				}
+			} : baseDataSource,
+			manifestStore: createD1ManifestStore(db),
+			codec: createDucklingsCodec(env),
+			executor: createDucklingsExecutor(env, { rowCache })
+		});
+	} };
+}
+let sharedRuntime;
 function getAnalyticsEngine(env, db, hooks = {}) {
-	if (!env.R2_DATA) return null;
-	const baseDataSource = createR2DataSource({
-		bucket: env.R2_DATA,
-		bucketName: env.R2_BUCKET_NAME
-	});
-	return createStorageEngine({
-		dataSource: hooks.onR2Write ? {
-			...baseDataSource,
-			async write(key, bytes) {
-				hooks.onR2Write(bytes.byteLength);
-				return baseDataSource.write(key, bytes);
-			}
-		} : baseDataSource,
-		manifestStore: createD1ManifestStore(db),
-		codec: createDucklingsCodec(env),
-		executor: createDucklingsExecutor(env)
-	});
+	sharedRuntime ??= createAnalyticsEngineRuntime();
+	return sharedRuntime.getEngine(env, db, hooks);
 }
 function useAnalyticsEnv(event) {
 	const fromCtx = event.context.analyticsEnv;
@@ -436,4 +459,4 @@ async function verifySizeHint(env, key, bytes, providedHex) {
 	for (let i = 0; i < SIG_HEX_LEN; i++) diff |= expected.charCodeAt(i) ^ providedHex.charCodeAt(i);
 	return diff === 0;
 }
-export { createDucklingsCodec, createDucklingsExecutor, createInflightDedupe, getAnalyticsEngine, getHostedR2QueryKey, signSizeHint, useAnalyticsEnv, verifySizeHint };
+export { createAnalyticsEngineRuntime, createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache, createInflightDedupe, getAnalyticsEngine, getHostedR2QueryKey, signSizeHint, useAnalyticsEnv, verifySizeHint };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/cloudflare",
   "type": "module",
-  "version": "0.37.2",
+  "version": "0.37.4",
   "description": "Cloudflare-Workers-flavored helpers for the gscdump analytics stack: AnalyticsEnv binding contract, R2 SigV4 presigner, size-hint HMAC, DuckDB Workers shims, engine factory.",
   "author": {
     "name": "Harlan Wilton",
@@ -46,10 +46,10 @@
   "dependencies": {
     "@uwdata/flechette": "^2.5.0",
     "aws4fetch": "^1.0.20",
-    "@gscdump/contracts": "0.37.2",
-    "gscdump": "0.37.2",
-    "@gscdump/engine-sqlite": "0.37.2",
-    "@gscdump/engine": "0.37.2"
+    "@gscdump/contracts": "0.37.4",
+    "@gscdump/engine-sqlite": "0.37.4",
+    "@gscdump/engine": "0.37.4",
+    "gscdump": "0.37.4"
   },
   "devDependencies": {
     "@cloudflare/vitest-pool-workers": "^0.18.0",