@gscdump/cloudflare 1.4.0 → 1.4.1

package/dist/arrow.mjs

@@ -0,0 +1,50 @@
+import { float64, int32, int64, tableFromArrays, tableToIPC, utf8 } from "@uwdata/flechette";
+function arrowTypeForColumn(type) {
+	switch (type) {
+		case "VARCHAR":
+		case "DATE": return utf8();
+		case "BIGINT": return int64();
+		case "INTEGER": return int32();
+		case "DOUBLE": return float64();
+	}
+}
+function rowsToArrowIPC(rows, schemaColumns) {
+	const colNames = [];
+	const seen = /* @__PURE__ */ new Set();
+	const add = (name) => {
+		if (!seen.has(name)) {
+			seen.add(name);
+			colNames.push(name);
+		}
+	};
+	if (schemaColumns) for (const c of schemaColumns) add(c.name);
+	for (const row of rows) for (const key in row) add(key);
+	const schemaType = /* @__PURE__ */ new Map();
+	if (schemaColumns) for (const c of schemaColumns) schemaType.set(c.name, c.type);
+	const data = {};
+	const types = {};
+	for (const name of colNames) {
+		const values = rows.map((r) => r[name] ?? null);
+		data[name] = values;
+		const known = schemaType.get(name);
+		if (known) {
+			types[name] = arrowTypeForColumn(known);
+			continue;
+		}
+		let hasString = false;
+		let hasValue = false;
+		for (const v of values) {
+			if (v === null || v === void 0) continue;
+			hasValue = true;
+			if (typeof v === "string") {
+				hasString = true;
+				break;
+			}
+		}
+		if (hasString || !hasValue) types[name] = utf8();
+	}
+	const ipc = tableToIPC(tableFromArrays(data, { types }), { format: "stream" });
+	if (!ipc) throw new Error("rowsToArrowIPC: tableToIPC returned null");
+	return ipc;
+}
+export { rowsToArrowIPC };

package/dist/engine.d.mts

@@ -0,0 +1,17 @@
+import { AnalyticsEnv } from "./env.mjs";
+import { createStorageEngine } from "@gscdump/engine";
+/**
+ * Optional per-request hooks for telemetry / tracing. Hosts wire these in
+ * to bridge engine activity into their own metrics pipeline.
+ */
+interface AnalyticsEngineHooks {
+  /** Called once per R2 PUT, with the byte size of the payload. */
+  onR2Write?: (byteLength: number) => void;
+  /** Override the interactive 22-second DuckDB RPC budget for background work. */
+  duckdbRpcTimeoutMs?: number;
+}
+interface AnalyticsEngineRuntime {
+  getEngine: (env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks) => ReturnType<typeof createStorageEngine> | null;
+}
+declare function getAnalyticsEngine(env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks): ReturnType<typeof createStorageEngine> | null;
+export { AnalyticsEngineHooks, AnalyticsEngineRuntime, getAnalyticsEngine };

package/dist/engine.mjs

@@ -0,0 +1,35 @@
+import { createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache } from "./workers-duckdb.mjs";
+import { createStorageEngine } from "@gscdump/engine";
+import { createD1ManifestStore } from "@gscdump/engine-sqlite";
+import { createR2DataSource } from "@gscdump/engine/r2";
+function createAnalyticsEngineRuntime() {
+	const rowCache = createDucklingsRowCache();
+	return { getEngine(env, db, hooks = {}) {
+		if (!env.R2_DATA) return null;
+		const baseDataSource = createR2DataSource({
+			bucket: env.R2_DATA,
+			bucketName: env.R2_BUCKET_NAME
+		});
+		return createStorageEngine({
+			dataSource: hooks.onR2Write ? {
+				...baseDataSource,
+				async write(key, bytes) {
+					hooks.onR2Write(bytes.byteLength);
+					return baseDataSource.write(key, bytes);
+				}
+			} : baseDataSource,
+			manifestStore: createD1ManifestStore(db),
+			codec: createDucklingsCodec(env),
+			executor: createDucklingsExecutor(env, {
+				rowCache,
+				...hooks.duckdbRpcTimeoutMs !== void 0 ? { rpcTimeoutMs: hooks.duckdbRpcTimeoutMs } : {}
+			})
+		});
+	} };
+}
+let sharedRuntime;
+function getAnalyticsEngine(env, db, hooks = {}) {
+	sharedRuntime ??= createAnalyticsEngineRuntime();
+	return sharedRuntime.getEngine(env, db, hooks);
+}
+export { getAnalyticsEngine };

package/dist/env.d.mts

@@ -0,0 +1,45 @@
+interface AnalyticsEnv {
+  /** R2 bucket holding parquet + rollup + entity data. Required in origin mode. */
+  R2_DATA?: R2Bucket;
+  /** Human name of the R2 bucket (used by the presigner when building public URLs). */
+  R2_BUCKET_NAME?: string;
+  /** Optional: D1 database holding the r2_manifest mirror + sync state. */
+  DB?: D1Database;
+  /**
+   * Optional: DuckDB service binding (Workers RPC) for server-side execution.
+   * Structural shape — any binding with `runSQL` + `ping` satisfies it, so
+   * hosts can declare their own binding interface without coupling to this one.
+   * Request tables use Arrow IPC chunks. Small queries pass tables inline to
+   * `runSQL`; larger queries stage chunks via `stageArrowTable`, then call
+   * `runSQL` with only SQL, and finally `dropTables`.
+   */
+  DUCKDB_SVC?: {
+    runSQL: (args: {
+      sql: string;
+      tables?: Record<string, {
+        ipc: Uint8Array;
+      }>;
+      deadlineAt?: number;
+    }) => Promise<{
+      rows: unknown[];
+      sql: string;
+    }>;
+    stageArrowTable?: (args: {
+      table: string;
+      ipc: Uint8Array;
+    }) => Promise<void>;
+    dropTables?: (args: {
+      tables: string[];
+    }) => Promise<void>;
+    ping: () => Promise<string>;
+  };
+  /** Route override: force D1 as the manifest source even if R2 is bound. */
+  ANALYTICS_FORCE_D1?: string;
+  /** Route override: disable R2 reads entirely (consumer / legacy mode). */
+  R2_READS_ENABLED?: string;
+  /** R2 S3-API credentials for presigning (origin mode when the worker can't proxy). */
+  R2_ACCESS_KEY_ID?: string;
+  R2_SECRET_ACCESS_KEY?: string;
+  CLOUDFLARE_ACCOUNT_ID?: string;
+}
+export { AnalyticsEnv };

package/dist/index.d.mts

@@ -1,90 +1,5 @@
-import { ParquetCodec, QueryExecutor, Row, createStorageEngine } from "@gscdump/engine";
-interface AnalyticsEnv {
-  /** R2 bucket holding parquet + rollup + entity data. Required in origin mode. */
-  R2_DATA?: R2Bucket;
-  /** Human name of the R2 bucket (used by the presigner when building public URLs). */
-  R2_BUCKET_NAME?: string;
-  /** Optional: D1 database holding the r2_manifest mirror + sync state. */
-  DB?: D1Database;
-  /**
-   * Optional: DuckDB service binding (Workers RPC) for server-side execution.
-   * Structural shape — any binding with `runSQL` + `ping` satisfies it, so
-   * hosts can declare their own binding interface without coupling to this one.
-   * Request tables use Arrow IPC chunks. Small queries pass tables inline to
-   * `runSQL`; larger queries stage chunks via `stageArrowTable`, then call
-   * `runSQL` with only SQL, and finally `dropTables`.
-   */
-  DUCKDB_SVC?: {
-    runSQL: (args: {
-      sql: string;
-      tables?: Record<string, {
-        ipc: Uint8Array;
-      }>;
-      deadlineAt?: number;
-    }) => Promise<{
-      rows: unknown[];
-      sql: string;
-    }>;
-    stageArrowTable?: (args: {
-      table: string;
-      ipc: Uint8Array;
-    }) => Promise<void>;
-    dropTables?: (args: {
-      tables: string[];
-    }) => Promise<void>;
-    ping: () => Promise<string>;
-  };
-  /** Route override: force D1 as the manifest source even if R2 is bound. */
-  ANALYTICS_FORCE_D1?: string;
-  /** Route override: disable R2 reads entirely (consumer / legacy mode). */
-  R2_READS_ENABLED?: string;
-  /** R2 S3-API credentials for presigning (origin mode when the worker can't proxy). */
-  R2_ACCESS_KEY_ID?: string;
-  R2_SECRET_ACCESS_KEY?: string;
-  CLOUDFLARE_ACCOUNT_ID?: string;
-}
-/**
- * Optional per-request hooks for telemetry / tracing. Hosts wire these in
- * to bridge engine activity into their own metrics pipeline.
- */
-interface AnalyticsEngineHooks {
-  /** Called once per R2 PUT, with the byte size of the payload. */
-  onR2Write?: (byteLength: number) => void;
-  /** Override the interactive 22-second DuckDB RPC budget for background work. */
-  duckdbRpcTimeoutMs?: number;
-}
-interface AnalyticsEngineRuntime {
-  getEngine: (env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks) => ReturnType<typeof createStorageEngine> | null;
-}
-declare function getAnalyticsEngine(env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks): ReturnType<typeof createStorageEngine> | null;
-interface InflightDedupe<T> {
-  dedupe: (key: string, run: () => Promise<T>) => Promise<T>;
-  has: (key: string) => boolean;
-  clear: () => void;
-}
-declare function createInflightDedupe<T>(): InflightDedupe<T>;
-interface HostedR2QueryKeyInput {
-  userId: string | number;
-  siteId: string;
-  state: unknown;
-  comparison?: unknown;
-  comparisonFilter?: string;
-}
-declare function getHostedR2QueryKey(input: HostedR2QueryKeyInput): string;
-declare function createDucklingsCodec(_env: AnalyticsEnv): ParquetCodec;
-interface DucklingsRowCache {
-  clear: () => void;
-  get: (key: string) => Row[] | undefined;
-  put: (key: string, rows: Row[]) => void;
-}
-declare function createDucklingsRowCache(maxBytes?: number): DucklingsRowCache;
-interface DucklingsExecutorOptions {
-  ipcChunkBytes?: number;
-  ipcDirectCallBytes?: number;
-  ipcTotalBytes?: number;
-  /** Per-RPC wall/queue budget. Interactive default is 22 seconds. */
-  rpcTimeoutMs?: number;
-  rowCache?: DucklingsRowCache;
-}
-declare function createDucklingsExecutor(env: AnalyticsEnv, opts?: DucklingsExecutorOptions): QueryExecutor;
+import { AnalyticsEnv } from "./env.mjs";
+import { AnalyticsEngineHooks, AnalyticsEngineRuntime, getAnalyticsEngine } from "./engine.mjs";
+import { HostedR2QueryKeyInput, InflightDedupe, createInflightDedupe, getHostedR2QueryKey } from "./inflight-dedupe.mjs";
+import { DucklingsRowCache, createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache } from "./workers-duckdb.mjs";
 export { type AnalyticsEngineHooks, type AnalyticsEngineRuntime, type AnalyticsEnv, type DucklingsRowCache, type HostedR2QueryKeyInput, type InflightDedupe, createDucklingsCodec, createDucklingsExecutor, createDucklingsRowCache, createInflightDedupe, getAnalyticsEngine, getHostedR2QueryKey };