@gscdump/cloudflare 0.7.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Harlan Wilton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.d.mts

@@ -0,0 +1,73 @@
+import { DuckDBFactory, ParquetCodec, QueryExecutor, Row, createStorageEngine } from "@gscdump/engine";
+import { H3Event } from "h3";
+declare function getWasmDuckDBFactory(): DuckDBFactory;
+declare function resetWasmDuckDB(): void;
+interface AnalyticsEnv {
+  /** R2 bucket holding parquet + rollup + entity data. Required in origin mode. */
+  R2_DATA?: R2Bucket;
+  /** Human name of the R2 bucket (used by the presigner when building public URLs). */
+  R2_BUCKET_NAME?: string;
+  /** Optional: D1 database holding the r2_manifest mirror + sync state. */
+  DB?: D1Database;
+  /**
+   * Optional: DuckDB service binding (Workers RPC) for server-side execution.
+   * Structural shape — any binding with `runSQL` + `ping` satisfies it, so
+   * hosts can declare their own binding interface without coupling to this one.
+   */
+  DUCKDB_SVC?: {
+    runSQL: (args: {
+      sql: string;
+      tables?: Record<string, {
+        rows: unknown[];
+        ddl?: string;
+      }>;
+    }) => Promise<{
+      rows: unknown[];
+      sql: string;
+    }>;
+    ping: () => Promise<string>;
+  };
+  /** Route override: force D1 as the manifest source even if R2 is bound. */
+  ANALYTICS_FORCE_D1?: string;
+  /** Route override: disable R2 reads entirely (consumer / legacy mode). */
+  R2_READS_ENABLED?: string;
+  /** R2 S3-API credentials for presigning (origin mode when the worker can't proxy). */
+  R2_ACCESS_KEY_ID?: string;
+  R2_SECRET_ACCESS_KEY?: string;
+  CLOUDFLARE_ACCOUNT_ID?: string;
+  /** Secret used by size-hint-sig.ts to HMAC-sign size hints. Rotating it invalidates cached hints. */
+  TOKEN_ENCRYPTION_SECRET?: string;
+}
+/**
+ * Resolve the AnalyticsEnv for the current request.
+ *
+ * Looks for, in order:
+ *  1. `event.context.analyticsEnv` — host plugin sets this explicitly.
+ *  2. `event.context.cloudflare?.env` — Cloudflare adapter convention.
+ *  3. Throws. The layer has no way to fabricate an env; the host must wire it.
+ */
+declare function useAnalyticsEnv(event: H3Event): AnalyticsEnv;
+/**
+ * Optional per-request hooks for telemetry / tracing. Hosts wire these in
+ * to bridge engine activity into their own metrics pipeline.
+ */
+interface AnalyticsEngineHooks {
+  /** Called once per R2 PUT, with the byte size of the payload. */
+  onR2Write?: (byteLength: number) => void;
+}
+declare function getAnalyticsEngine(env: AnalyticsEnv, db: any, hooks?: AnalyticsEngineHooks): ReturnType<typeof createStorageEngine> | null;
+interface PresignOptions {
+  key: string;
+  bucket: string;
+  expiresIn?: number;
+}
+declare function createR2Presigner(env: AnalyticsEnv): ({
+  key,
+  bucket,
+  expiresIn
+}: PresignOptions) => Promise<string>;
+declare function signSizeHint(env: AnalyticsEnv, key: string, bytes: number): Promise<string>;
+declare function verifySizeHint(env: AnalyticsEnv, key: string, bytes: number, providedHex: string): Promise<boolean>;
+declare function createDucklingsCodec(_env: AnalyticsEnv): ParquetCodec;
+declare function createDucklingsExecutor(env: AnalyticsEnv): QueryExecutor;
+export { type AnalyticsEngineHooks, type AnalyticsEnv, type PresignOptions, type Row, createDucklingsCodec, createDucklingsExecutor, createR2Presigner, getAnalyticsEngine, getWasmDuckDBFactory, resetWasmDuckDB, signSizeHint, useAnalyticsEnv, verifySizeHint };

package/dist/index.mjs

@@ -0,0 +1,208 @@
+import { bindLiterals, canonicalEmptyParquetSchema, createStorageEngine } from "@gscdump/engine";
+import { createD1ManifestStore } from "@gscdump/engine-sqlite";
+import { createR2DataSource } from "@gscdump/engine/r2";
+import { createHyparquetCodec, decodeParquetToRows } from "@gscdump/engine/hyparquet";
+import { createError } from "h3";
+import { AwsClient } from "aws4fetch";
+let handle = null;
+async function initHandle() {
+	throw new Error("DuckDB-WASM handle not wired for Cloudflare Workers yet. Complete duckdb-wasm-handle.ts before enabling dual-write (user.migration_phase != 'd1').");
+}
+function getWasmDuckDBFactory() {
+	return { getDuckDB() {
+		if (!handle) handle = initHandle();
+		return handle;
+	} };
+}
+function resetWasmDuckDB() {
+	handle = null;
+}
+function resolveSvc(env) {
+	const svc = env.DUCKDB_SVC;
+	if (!svc) throw new Error("DUCKDB_SVC service binding is not configured");
+	return svc;
+}
+function createDucklingsCodec(_env) {
+	return createHyparquetCodec();
+}
+function coerceRow(row) {
+	let mutated = null;
+	for (const [k, v] of Object.entries(row)) if (typeof v === "bigint") {
+		if (!mutated) mutated = { ...row };
+		mutated[k] = Number(v);
+	}
+	return mutated ?? row;
+}
+const READ_PARQUET_PLACEHOLDER = /read_parquet\(\{\{(\w+)\}\}(?:\s*,[^)]*)?\)/g;
+function tmpTableName(placeholder) {
+	const uuid = crypto.randomUUID().replace(/-/g, "_");
+	return `tmp_${placeholder.toLowerCase()}_${uuid}`;
+}
+const ROW_CACHE_MAX_BYTES = 16 * 1024 * 1024;
+let rowCacheBytes = 0;
+const rowCache = /* @__PURE__ */ new Map();
+function estimateRowsBytes(rows) {
+	if (rows.length === 0) return 0;
+	const cols = Object.keys(rows[0]).length;
+	return rows.length * cols * 64;
+}
+function rowCacheGet(key) {
+	const hit = rowCache.get(key);
+	if (!hit) return void 0;
+	rowCache.delete(key);
+	rowCache.set(key, hit);
+	return hit.rows;
+}
+function rowCachePut(key, rows) {
+	const bytes = estimateRowsBytes(rows);
+	if (bytes > ROW_CACHE_MAX_BYTES) return;
+	while (rowCacheBytes + bytes > ROW_CACHE_MAX_BYTES) {
+		const oldest = rowCache.keys().next().value;
+		if (oldest === void 0) break;
+		const evicted = rowCache.get(oldest);
+		rowCache.delete(oldest);
+		rowCacheBytes -= evicted.bytes;
+	}
+	rowCache.set(key, {
+		rows,
+		bytes
+	});
+	rowCacheBytes += bytes;
+}
+function createDucklingsExecutor(env) {
+	return { async execute({ sql, params, fileKeys, dataSource, signal, table }) {
+		signal?.throwIfAborted();
+		const svc = resolveSvc(env);
+		const tempNames = {};
+		const tables = {};
+		await Promise.all(Object.entries(fileKeys).map(async ([placeholder, keys]) => {
+			const perFile = await Promise.all(keys.map(async (key) => {
+				const cached = rowCacheGet(key);
+				if (cached) return cached;
+				const rows = await decodeParquetToRows(await dataSource.read(key));
+				rowCachePut(key, rows);
+				return rows;
+			}));
+			const merged = [];
+			for (const rows of perFile) merged.push(...rows);
+			const tmp = tmpTableName(placeholder);
+			tempNames[placeholder] = tmp;
+			tables[tmp] = {
+				rows: merged,
+				ddl: `AS SELECT * FROM ${canonicalEmptyParquetSchema(table)} WHERE FALSE`
+			};
+		}));
+		signal?.throwIfAborted();
+		const finalSql = bindLiterals(sql.replace(READ_PARQUET_PLACEHOLDER, (_, placeholder) => {
+			const tmp = tempNames[placeholder];
+			if (!tmp) throw new Error(`createDucklingsExecutor: SQL references {{${placeholder}}} but no fileKeys entry provided`);
+			return tmp;
+		}), params);
+		const result = await svc.runSQL({
+			sql: finalSql,
+			tables
+		});
+		return {
+			rows: result.rows.map(coerceRow),
+			sql: result.sql
+		};
+	} };
+}
+function getAnalyticsEngine(env, db, hooks = {}) {
+	if (!env.R2_DATA) return null;
+	const baseDataSource = createR2DataSource({
+		bucket: env.R2_DATA,
+		bucketName: env.R2_BUCKET_NAME
+	});
+	return createStorageEngine({
+		dataSource: hooks.onR2Write ? {
+			...baseDataSource,
+			async write(key, bytes) {
+				hooks.onR2Write(bytes.byteLength);
+				return baseDataSource.write(key, bytes);
+			}
+		} : baseDataSource,
+		manifestStore: createD1ManifestStore(db),
+		codec: createDucklingsCodec(env),
+		executor: createDucklingsExecutor(env)
+	});
+}
+function useAnalyticsEnv(event) {
+	const fromCtx = event.context.analyticsEnv;
+	if (fromCtx) return fromCtx;
+	const fromCf = event.context.cloudflare?.env;
+	if (fromCf) return fromCf;
+	throw createError({
+		statusCode: 500,
+		statusMessage: "AnalyticsEnv not available — host must populate event.context.analyticsEnv or use the Cloudflare adapter"
+	});
+}
+function createR2Presigner(env) {
+	if (!env.R2_ACCESS_KEY_ID || !env.R2_SECRET_ACCESS_KEY) throw createError({
+		statusCode: 500,
+		message: "R2 S3 credentials missing (R2_ACCESS_KEY_ID / R2_SECRET_ACCESS_KEY)"
+	});
+	if (!env.CLOUDFLARE_ACCOUNT_ID) throw createError({
+		statusCode: 500,
+		message: "CLOUDFLARE_ACCOUNT_ID missing"
+	});
+	const aws = new AwsClient({
+		accessKeyId: env.R2_ACCESS_KEY_ID,
+		secretAccessKey: env.R2_SECRET_ACCESS_KEY,
+		service: "s3",
+		region: "auto"
+	});
+	const endpoint = `https://${env.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`;
+	return async function presignGet({ key, bucket, expiresIn = 3600 }) {
+		const url = new URL(`${endpoint}/${bucket}/${encodeKey(key)}`);
+		url.searchParams.set("X-Amz-Expires", String(expiresIn));
+		return (await aws.sign(url.toString(), {
+			method: "GET",
+			aws: { signQuery: true }
+		})).url;
+	};
+}
+function encodeKey(key) {
+	return key.split("/").map(encodeURIComponent).join("/");
+}
+const SIG_HEX_LEN = 16;
+const keyCache = /* @__PURE__ */ new WeakMap();
+const stringKeyCache = /* @__PURE__ */ new Map();
+async function getKey(env) {
+	const secret = env.TOKEN_ENCRYPTION_SECRET;
+	if (!secret) throw new Error("size-hint-sig: TOKEN_ENCRYPTION_SECRET not configured");
+	let cached = keyCache.get(env);
+	if (!cached) {
+		cached = stringKeyCache.get(secret);
+		if (!cached) {
+			cached = crypto.subtle.importKey("raw", new TextEncoder().encode(secret).slice().buffer, {
+				name: "HMAC",
+				hash: "SHA-256"
+			}, false, ["sign", "verify"]);
+			stringKeyCache.set(secret, cached);
+		}
+		keyCache.set(env, cached);
+	}
+	return cached;
+}
+function toHex(buf, chars) {
+	const bytes = new Uint8Array(buf);
+	let hex = "";
+	for (let i = 0; i < chars / 2; i++) hex += bytes[i].toString(16).padStart(2, "0");
+	return hex;
+}
+function payload(key, bytes) {
+	return new TextEncoder().encode(`${key}\0${bytes}`).slice().buffer;
+}
+async function signSizeHint(env, key, bytes) {
+	const cryptoKey = await getKey(env);
+	return toHex(await crypto.subtle.sign("HMAC", cryptoKey, payload(key, bytes)), SIG_HEX_LEN);
+}
+async function verifySizeHint(env, key, bytes, providedHex) {
+	if (providedHex.length !== SIG_HEX_LEN) return false;
+	const expected = await signSizeHint(env, key, bytes);
+	let diff = 0;
+	for (let i = 0; i < SIG_HEX_LEN; i++) diff |= expected.charCodeAt(i) ^ providedHex.charCodeAt(i);
+	return diff === 0;
+}
+export { createDucklingsCodec, createDucklingsExecutor, createR2Presigner, getAnalyticsEngine, getWasmDuckDBFactory, resetWasmDuckDB, signSizeHint, useAnalyticsEnv, verifySizeHint };

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@gscdump/cloudflare",
+  "type": "module",
+  "version": "0.7.0",
+  "description": "Cloudflare-Workers-flavored helpers for the gscdump analytics stack: AnalyticsEnv binding contract, R2 SigV4 presigner, size-hint HMAC, DuckDB Workers shims, engine factory.",
+  "author": {
+    "name": "Harlan Wilton",
+    "email": "harlan@harlanzw.com",
+    "url": "https://harlanzw.com/"
+  },
+  "license": "MIT",
+  "funding": "https://github.com/sponsors/harlan-zw",
+  "homepage": "https://github.com/harlan-zw/gscdump/tree/main/packages/cloudflare#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/harlan-zw/gscdump.git",
+    "directory": "packages/cloudflare"
+  },
+  "bugs": {
+    "url": "https://github.com/harlan-zw/gscdump/issues"
+  },
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "peerDependencies": {
+    "h3": "^1.15.0"
+  },
+  "dependencies": {
+    "aws4fetch": "^1.0.20",
+    "@gscdump/engine": "0.7.0",
+    "@gscdump/engine-sqlite": "0.7.0"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20260426.1",
+    "h3": "^1.15.11",
+    "typescript": "^6.0.3"
+  },
+  "scripts": {
+    "build": "obuild",
+    "dev": "obuild --stub",
+    "typecheck": "tsc --noEmit"
+  }
+}