@gscdump/engine-duckdb-wasm 0.23.4 → 0.24.0

package/dist/index.d.mts

@@ -1,5 +1,6 @@
 import { Assume, DrizzleConfig, entityKind } from "drizzle-orm";
-import { PgDatabase, PgDialect, PgPreparedQuery, PgQueryResultHKT, PgSession } from "drizzle-orm/pg-core";
+import { PgAsyncDatabase, PgAsyncPreparedQuery, PgAsyncSession, PgDialect, PgQueryResultHKT } from "drizzle-orm/pg-core";
+import { AnyRelations, EmptyRelations, ExtractTablesWithRelations, Schema as Schema$1 } from "drizzle-orm/relations";
 import { DrizzleSchema as Schema, countries, dates, drizzleSchema as schema, hourly_pages, page_queries, pages, queries } from "@gscdump/engine/schema";
 import { ScopedRunnerOptions, TableScope } from "@gscdump/engine/scope";
 import { AnalyzerRegistry } from "@gscdump/engine/analyzer";
@@ -31,13 +32,14 @@ type Row = Record<string, unknown>;
 interface DuckDBWasmQueryResultHKT extends PgQueryResultHKT {
   type: Assume<this['row'], Row>[];
 }
-declare class DuckDBWasmDatabase<TSchema extends Record<string, unknown> = Record<string, never>> extends PgDatabase<DuckDBWasmQueryResultHKT, TSchema> {
+type SchemaRelations<TSchema extends Schema$1> = ExtractTablesWithRelations<Record<string, never>, TSchema>;
+declare class DuckDBWasmDatabase<TRelations extends AnyRelations = EmptyRelations> extends PgAsyncDatabase<DuckDBWasmQueryResultHKT, TRelations> {
   static readonly [entityKind]: string;
 }
-interface DuckDBWasmDrizzleDatabase<TSchema extends Record<string, unknown> = Record<string, never>> extends DuckDBWasmDatabase<TSchema> {
+interface DuckDBWasmDrizzleDatabase<TSchema extends Schema$1 = Record<string, never>, TRelations extends AnyRelations = SchemaRelations<TSchema>> extends DuckDBWasmDatabase<TRelations> {
   $client: Promise<DuckDBWasmClient>;
 }
-declare function drizzle<TSchema extends Record<string, unknown> = Record<string, never>>(client: Promise<DuckDBWasmClient> | DuckDBWasmClient, config?: DrizzleConfig<TSchema>): DuckDBWasmDrizzleDatabase<TSchema>;
+declare function drizzle<TSchema extends Schema$1 = Record<string, never>, TRelations extends AnyRelations = SchemaRelations<TSchema>>(client: Promise<DuckDBWasmClient> | DuckDBWasmClient, config?: DrizzleConfig<TSchema, TRelations>): DuckDBWasmDrizzleDatabase<TSchema, TRelations>;
 interface InsightRunnerOptions {
   db: AsyncDuckDB;
   conn: AsyncDuckDBConnection;

package/dist/index.mjs

@@ -1,6 +1,7 @@
 import { arrowToRows } from "@gscdump/engine/arrow";
-import { DefaultLogger, NoopLogger, createTableRelationsHelpers, entityKind, extractTablesRelationalConfig, fillPlaceholders, sql } from "drizzle-orm";
-import { PgDatabase, PgDialect, PgPreparedQuery, PgSession } from "drizzle-orm/pg-core";
+import { DefaultLogger, NoopLogger, entityKind, sql } from "drizzle-orm";
+import { PgAsyncDatabase, PgAsyncPreparedQuery, PgAsyncSession, PgDialect } from "drizzle-orm/pg-core";
+import { buildRelations } from "drizzle-orm/relations";
 import { toIsoDate } from "gscdump";
 import { countries, dates, drizzleSchema as schema, hourly_pages, page_queries, pages, queries } from "@gscdump/engine/schema";
 import { createScopedHelpers } from "@gscdump/engine/scope";
@@ -263,69 +264,44 @@ async function createClient(db, conn) {
 		}
 	};
 }
-var DuckDBWasmPreparedQuery = class extends PgPreparedQuery {
-	client;
-	logger;
-	static [entityKind] = "DuckDBWasmPreparedQuery";
-	constructor(client, query, logger, queryMetadata, cache) {
-		super(query, cache, queryMetadata);
-		this.client = client;
-		this.logger = logger;
-	}
-	async execute(placeholderValues = {}) {
-		const params = fillPlaceholders(this.query.params, placeholderValues);
-		this.logger.logQuery(this.query.sql, params);
-		return await (await this.client).query(this.query.sql, params);
-	}
-	async all(placeholderValues = {}) {
-		const params = fillPlaceholders(this.query.params, placeholderValues);
-		this.logger.logQuery(this.query.sql, params);
-		return await (await this.client).query(this.query.sql, params);
-	}
-};
-var DuckDBWasmSession = class extends PgSession {
+var DuckDBWasmSession = class extends PgAsyncSession {
 	client;
 	options;
 	static [entityKind] = "DuckDBWasmSession";
 	logger;
 	cache;
-	constructor(client, dialect, _schema, options = {}) {
+	constructor(client, dialect, options = {}) {
 		super(dialect);
 		this.client = client;
 		this.options = options;
 		this.logger = options.logger ?? new NoopLogger();
 		this.cache = options.cache;
 	}
-	prepareQuery(query, _fields, _name, _isResponseInArrayMode, _customResultMapper, queryMetadata) {
-		return new DuckDBWasmPreparedQuery(this.client, query, this.logger, queryMetadata, this.cache);
-	}
-	async query(sql, params) {
-		this.logger.logQuery(sql, params);
-		return (await this.client).query(sql, params);
+	prepareQuery(query, mode, _name, mapper, queryMetadata, cacheConfig) {
+		const client = this.client;
+		const executor = async (params = []) => {
+			const rows = await (await client).query(query.sql, params);
+			return mode === "arrays" ? rows.map((row) => Object.values(row)) : rows;
+		};
+		return new PgAsyncPreparedQuery(executor, query, mapper, mode, this.logger, this.cache, queryMetadata, cacheConfig);
 	}
-	transaction(_transaction, _config) {
-		throw new Error("Transactions are not supported by the DuckDB-WASM drizzle adapter. The analytics workload is read-only; if transactions become necessary, implement PgSession.transaction() in @gscdump/engine-duckdb-wasm.");
+	transaction() {
+		throw new Error("Transactions are not supported by the DuckDB-WASM drizzle adapter. The analytics workload is read-only; if transactions become necessary, implement PgAsyncSession.transaction() in @gscdump/engine-duckdb-wasm.");
 	}
 };
-var DuckDBWasmDatabase = class extends PgDatabase {
+var DuckDBWasmDatabase = class extends PgAsyncDatabase {
 	static [entityKind] = "DuckDBWasmDatabase";
 };
 function drizzle(client, config = {}) {
-	const dialect = new PgDialect({ casing: config.casing });
+	const dialect = new PgDialect();
 	const clientPromise = Promise.resolve(client);
 	let logger;
 	if (config.logger === true) logger = new DefaultLogger();
 	else if (config.logger !== false) logger = config.logger;
-	let schema;
-	if (config.schema) {
-		const tablesConfig = extractTablesRelationalConfig(config.schema, createTableRelationsHelpers);
-		schema = {
-			fullSchema: config.schema,
-			schema: tablesConfig.tables,
-			tableNamesMap: tablesConfig.tableNamesMap
-		};
-	}
-	const db = new DuckDBWasmDatabase(dialect, new DuckDBWasmSession(clientPromise, dialect, schema, { logger }), schema);
+	const db = new DuckDBWasmDatabase(dialect, new DuckDBWasmSession(clientPromise, dialect, {
+		logger,
+		cache: config.cache
+	}), config.relations ?? (config.schema ? buildRelations(config.schema, {}) : {}));
 	db.$client = clientPromise;
 	return db;
 }
@@ -390,6 +366,73 @@ async function strikingMomentum(runner, opts = {}) {
   `;
 	return await runner.db.execute(windowExpr);
 }
+function createOpfsHandleRegistry(backend) {
+	const entries = /* @__PURE__ */ new Map();
+	const pending = /* @__PURE__ */ new Map();
+	async function acquire(name, openHandle) {
+		const existing = entries.get(name);
+		if (existing) {
+			existing.refs++;
+			return;
+		}
+		const inflight = pending.get(name);
+		if (inflight) {
+			await inflight.catch(() => {});
+			const settled = entries.get(name);
+			if (settled) {
+				settled.refs++;
+				return;
+			}
+			return acquire(name, openHandle);
+		}
+		const registration = (async () => {
+			const handle = await openHandle();
+			await backend.register(name, handle);
+			entries.set(name, {
+				handle,
+				refs: 1
+			});
+		})();
+		pending.set(name, registration);
+		try {
+			await registration;
+		} finally {
+			pending.delete(name);
+		}
+	}
+	async function release(names) {
+		for (const name of names) {
+			const entry = entries.get(name);
+			if (!entry) continue;
+			entry.refs--;
+			if (entry.refs > 0) continue;
+			entries.delete(name);
+			await backend.drop(name).catch(() => {});
+		}
+	}
+	const viewRefs = /* @__PURE__ */ new Map();
+	function acquireView(key) {
+		viewRefs.set(key, (viewRefs.get(key) ?? 0) + 1);
+	}
+	async function releaseView(key, drop) {
+		const next = (viewRefs.get(key) ?? 0) - 1;
+		if (next > 0) {
+			viewRefs.set(key, next);
+			return;
+		}
+		viewRefs.delete(key);
+		await drop().catch(() => {});
+	}
+	return {
+		acquire,
+		release,
+		size: () => entries.size,
+		refs: (name) => entries.get(name)?.refs ?? 0,
+		acquireView,
+		releaseView,
+		viewRefs: (key) => viewRefs.get(key) ?? 0
+	};
+}
 var OpfsQuotaExceededError = class extends Error {
 	name = "OpfsQuotaExceededError";
 	degradedTables;
@@ -400,17 +443,19 @@ var OpfsQuotaExceededError = class extends Error {
 };
 const DEFAULT_CONCURRENCY = 2;
 const OPFS_PREFIX = "gscdump-snapshot__";
-const dbFileRegistrations = /* @__PURE__ */ new WeakMap();
-function isAlreadyRegistered(db, name) {
-	return dbFileRegistrations.get(db)?.has(name) === true;
-}
-function markRegistered(db, name) {
-	let set = dbFileRegistrations.get(db);
-	if (!set) {
-		set = /* @__PURE__ */ new Set();
-		dbFileRegistrations.set(db, set);
+const dbRegistries = /* @__PURE__ */ new WeakMap();
+function getOpfsRegistry(db, protocol) {
+	let registry = dbRegistries.get(db);
+	if (!registry) {
+		registry = createOpfsHandleRegistry({
+			register: (name, handle) => db.registerFileHandle(name, handle, protocol, true),
+			drop: async (name) => {
+				await db.dropFile(name);
+			}
+		});
+		dbRegistries.set(db, registry);
 	}
-	set.add(name);
+	return registry;
 }
 function isQuotaError(err) {
 	if (typeof err !== "object" || err === null) return false;
@@ -419,6 +464,10 @@ function isQuotaError(err) {
 function isAbortError$1(err) {
 	return typeof err === "object" && err !== null && err.name === "AbortError";
 }
+function isOpfsAccessHandleConflict(err) {
+	const msg = err instanceof Error ? err.message : String(err);
+	return /createSyncAccessHandle|Access Handle/i.test(msg);
+}
 function opfsFileName(table, index, hashSlug) {
 	const base = `${OPFS_PREFIX}${table}_${index}`;
 	return hashSlug ? `${base}_${hashSlug}.parquet` : `${base}.parquet`;
@@ -522,10 +571,17 @@ async function attachOpfsParquetTables(options) {
 	});
 	const total = flat.length;
 	const { DuckDBDataProtocol } = await import("@duckdb/duckdb-wasm");
+	const registry = getOpfsRegistry(db, DuckDBDataProtocol.BROWSER_FSACCESS);
 	const tableFiles = /* @__PURE__ */ new Map();
 	const degraded = /* @__PURE__ */ new Set();
+	const acquiredNames = /* @__PURE__ */ new Set();
 	let bytesAttached = 0;
-	await runWithConcurrency$1(flat, Math.max(1, fetchConcurrency), async (item, index) => {
+	const releaseTable = async (table) => {
+		const names = (tableFiles.get(table) ?? []).map((f) => f.name);
+		for (const n of names) acquiredNames.delete(n);
+		await registry.release(names);
+	};
+	const runDownloads = () => runWithConcurrency$1(flat, Math.max(1, fetchConcurrency), async (item, index) => {
 		if (degraded.has(item.table)) return;
 		signal?.throwIfAborted();
 		const hashSlug = item.file.contentHash ? await contentHashSlug(item.file.contentHash) : void 0;
@@ -542,9 +598,16 @@ async function attachOpfsParquetTables(options) {
 			}
 			throw err;
 		}
-		if (!isAlreadyRegistered(db, name)) {
-			await db.registerFileHandle(name, result.handle, DuckDBDataProtocol.BROWSER_FSACCESS, true);
-			markRegistered(db, name);
+		try {
+			await registry.acquire(name, () => result.handle);
+			acquiredNames.add(name);
+		} catch (err) {
+			if (isAbortError$1(err)) throw err;
+			if (isOpfsAccessHandleConflict(err)) {
+				degraded.add(item.table);
+				return;
+			}
+			throw err;
 		}
 		const list = tableFiles.get(item.table) ?? [];
 		list.push({
@@ -562,24 +625,45 @@ async function attachOpfsParquetTables(options) {
 			outcome: result.outcome
 		});
 	});
+	try {
+		await runDownloads();
+	} catch (err) {
+		await registry.release([...acquiredNames]).catch(() => {});
+		throw err;
+	}
 	const attached = [];
 	const registeredNames = [];
-	try {
-		for (const t of tables) {
-			if (degraded.has(t.table)) continue;
-			const files = tableFiles.get(t.table) ?? [];
-			if (files.length !== t.files.length) {
+	for (const t of tables) {
+		if (degraded.has(t.table)) {
+			await releaseTable(t.table);
+			continue;
+		}
+		const files = tableFiles.get(t.table) ?? [];
+		if (files.length !== t.files.length) {
+			degraded.add(t.table);
+			await releaseTable(t.table);
+			continue;
+		}
+		try {
+			signal?.throwIfAborted();
+			await conn.query(readParquetViewSql$1(schema, t.table, files.map((f) => f.name)));
+		} catch (err) {
+			if (isAbortError$1(err)) {
+				await detachOpfs(registry, conn, schema, attached, [...acquiredNames]).catch(() => {});
+				throw err;
+			}
+			if (isOpfsAccessHandleConflict(err)) {
+				if (registry.viewRefs(`${schema}.${t.table}`) === 0) await conn.query(`DROP VIEW IF EXISTS ${schema}.${t.table}`).catch(() => {});
 				degraded.add(t.table);
+				await releaseTable(t.table);
 				continue;
 			}
-			signal?.throwIfAborted();
-			await conn.query(readParquetViewSql$1(schema, t.table, files.map((f) => f.name)));
-			attached.push(t.table);
-			for (const f of files) registeredNames.push(f.name);
+			await detachOpfs(registry, conn, schema, attached, [...acquiredNames]).catch(() => {});
+			throw err;
 		}
-	} catch (err) {
-		await detachOpfs(db, conn, schema, attached, registeredNames).catch(() => {});
-		throw err;
+		attached.push(t.table);
+		registry.acquireView(`${schema}.${t.table}`);
+		for (const f of files) registeredNames.push(f.name);
 	}
 	let detached = false;
 	return {
@@ -591,12 +675,13 @@ async function attachOpfsParquetTables(options) {
 		async detach() {
 			if (detached) return;
 			detached = true;
-			await detachOpfs(db, conn, schema, attached, registeredNames);
+			await detachOpfs(registry, conn, schema, attached, registeredNames);
 		}
 	};
 }
-async function detachOpfs(db, conn, schema, tables, _files) {
-	for (const table of tables) await conn.query(`DROP VIEW IF EXISTS ${schema}.${table}`).catch(() => {});
+async function detachOpfs(registry, conn, schema, tables, files) {
+	for (const table of tables) await registry.releaseView(`${schema}.${table}`, () => conn.query(`DROP VIEW IF EXISTS ${schema}.${table}`).then(() => {}));
+	await registry.release(files);
 }
 async function clearOpfsSnapshotCache() {
 	const root = await getOpfsRoot().catch(() => null);
@@ -761,13 +846,19 @@ async function bootDuckDBWasm(options = {}) {
 	const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], { type: "text/javascript" }));
 	const worker = new Worker(workerUrl);
 	const db = new AsyncDuckDB(options.logger ?? new ConsoleLogger(), worker);
-	await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
-	await db.open(rangeOnlyConfig(options.config));
-	URL.revokeObjectURL(workerUrl);
-	return {
-		db,
-		conn: await db.connect()
-	};
+	try {
+		await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+		await db.open(rangeOnlyConfig(options.config));
+		return {
+			db,
+			conn: await db.connect()
+		};
+	} catch (err) {
+		worker.terminate();
+		throw err;
+	} finally {
+		URL.revokeObjectURL(workerUrl);
+	}
 }
 async function attachParquetTables(options) {
 	const { db, conn, tables, schema = "main" } = options;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine-duckdb-wasm",
   "type": "module",
-  "version": "0.23.4",
+  "version": "0.24.0",
   "description": "DuckDB-WASM engine adapter for @gscdump/analysis — typed browser analytics against parquet via R2.",
   "author": {
     "name": "Harlan Wilton",
@@ -44,19 +44,23 @@
     }
   },
   "dependencies": {
-    "drizzle-orm": "^0.45.2",
-    "@gscdump/engine": "0.23.4",
-    "@gscdump/sdk": "0.23.4",
-    "gscdump": "0.23.4"
+    "drizzle-orm": "1.0.0-rc.3",
+    "@gscdump/sdk": "0.24.0",
+    "gscdump": "0.24.0",
+    "@gscdump/engine": "0.24.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
+    "@vitest/browser": "^4.1.7",
+    "@vitest/browser-playwright": "^4.1.7",
+    "playwright": "^1.60.0",
     "vitest": "^4.1.7"
   },
   "scripts": {
     "build": "obuild",
     "dev": "obuild --stub",
     "typecheck": "tsc --noEmit",
-    "test": "vitest"
+    "test": "vitest",
+    "test:browser": "GSCDUMP_E2E=1 vitest --run --config vitest.browser.config.ts"
   }
 }