@peerbit/indexer-sqlite3 3.0.0-e6ea5c0 → 3.0.1

package/src/sqlite3.browser.ts

@@ -9,109 +9,264 @@ import {
 	type StatementGetResult,
 } from "./types.js";
 
+type RequestType =
+	| messages.DatabaseMessages["type"]
+	| messages.StatementMessages["type"];
+
+export type SQLiteBrowserOptions = {
+	protocol?: messages.SqliteWorkerProtocol;
+	pragmas?: messages.SQLitePragmaOptions;
+	profile?: boolean;
+	onProfile?: (sample: SQLiteProfileSample) => void;
+};
+
+export type SQLiteProfileSample = {
+	requestType: RequestType;
+	protocol: messages.SqliteWorkerProtocol;
+	databaseId: string;
+	databaseDirectory?: string;
+	sql?: string;
+	clientEncodeMs: number;
+	clientRoundTripMs: number;
+	valueCount: number;
+	blobValueCount: number;
+	blobBytes: number;
+	worker?: messages.WorkerTiming;
+};
+
+type SendMetrics = messages.ClientEncodeMetrics & {
+	requestType: RequestType;
+	sql?: string;
+};
+
+type ResponseMessage = Extract<messages.ResponseMessages, { type: "response" }>;
+type ErrorMessage = Extract<messages.ResponseMessages, { type: "error" }>;
+
+const DEFAULT_PROTOCOL: messages.SqliteWorkerProtocol = "clone";
+const EMPTY_ENCODE_METRICS: messages.ClientEncodeMetrics = {
+	encodeMs: 0,
+	valueCount: 0,
+	blobValueCount: 0,
+	blobBytes: 0,
+};
+
+const getProtocol = (
+	options?: SQLiteBrowserOptions,
+): messages.SqliteWorkerProtocol => options?.protocol ?? DEFAULT_PROTOCOL;
+
 class ProxyStatement implements IStatement {
 	id: string;
-	resolvers: {
-		[hash in string]: {
-			resolve: (...args: any) => void;
-			reject: (...args: any) => void;
-		};
-	} = {};
+	private needsReset = false;
 
 	constructor(
 		readonly send: <T>(
 			message: messages.DatabaseMessages | messages.StatementMessages,
+			metrics?: SendMetrics,
 		) => Promise<T>,
 		readonly databaseId: string,
 		readonly statementId: string,
+		readonly sql: string,
+		readonly options?: SQLiteBrowserOptions,
 	) {
 		this.id = statementId;
 	}
 
 	async bind(values: any[]) {
-		await this.send({
-			type: "bind",
-			values: values.map(messages.encodeValue),
-			id: uuid(),
-			databaseId: this.databaseId,
-			statementId: this.statementId,
-		});
+		const encoded = messages.encodeValues(values, getProtocol(this.options));
+		await this.send(
+			{
+				type: "bind",
+				values: encoded.values ?? [],
+				id: uuid(),
+				databaseId: this.databaseId,
+				statementId: this.statementId,
+			},
+				{
+					requestType: "bind",
+					sql: this.sql,
+					...encoded.metrics,
+				},
+		);
+		this.needsReset = true;
 		return this;
 	}
 
 	async finalize() {
-		await this.send({
-			type: "finalize",
-			id: uuid(),
-			databaseId: this.databaseId,
-			statementId: this.statementId,
-		});
+		await this.send(
+			{
+				type: "finalize",
+				id: uuid(),
+				databaseId: this.databaseId,
+				statementId: this.statementId,
+			},
+			{
+				requestType: "finalize",
+				sql: this.sql,
+				...EMPTY_ENCODE_METRICS,
+			},
+		);
+		this.needsReset = false;
 	}
 
-	get(values?: BindableValue[]) {
-		return this.send<StatementGetResult>({
-			type: "get",
-			values: values ? values.map(messages.encodeValue) : undefined,
-			id: uuid(),
-			databaseId: this.databaseId,
-			statementId: this.statementId,
-		});
+	async get(values?: BindableValue[]) {
+		const encoded = messages.encodeValues(values, getProtocol(this.options));
+		const result = await this.send<StatementGetResult>(
+			{
+				type: "get",
+				values: encoded.values,
+				id: uuid(),
+				databaseId: this.databaseId,
+				statementId: this.statementId,
+			},
+				{
+					requestType: "get",
+					sql: this.sql,
+					...encoded.metrics,
+				},
+		);
+		this.needsReset = false;
+		return result;
 	}
 
 	async run(values: BindableValue[]) {
-		await this.send({
-			type: "run-statement",
-			values: values.map(messages.encodeValue),
-			id: uuid(),
-			databaseId: this.databaseId,
-			statementId: this.statementId,
-		});
+		const encoded = messages.encodeValues(values, getProtocol(this.options));
+		await this.send(
+			{
+				type: "run-statement",
+				values: encoded.values ?? [],
+				id: uuid(),
+				databaseId: this.databaseId,
+				statementId: this.statementId,
+			},
+				{
+					requestType: "run-statement",
+					sql: this.sql,
+					...encoded.metrics,
+				},
+		);
+		this.needsReset = false;
 	}
 
 	async reset() {
-		await this.send({
-			type: "reset",
-			id: uuid(),
-			databaseId: this.databaseId,
-			statementId: this.statementId,
-		});
+		if (!this.needsReset) {
+			return this;
+		}
+		await this.send(
+			{
+				type: "reset",
+				id: uuid(),
+				databaseId: this.databaseId,
+				statementId: this.statementId,
+			},
+			{
+				requestType: "reset",
+				sql: this.sql,
+				...EMPTY_ENCODE_METRICS,
+			},
+		);
+		this.needsReset = false;
 		return this;
 	}
 
 	async all(values: BindableValue[]) {
-		let id = uuid();
-		const results = await this.send({
-			type: "all",
-			values: values.map(messages.encodeValue),
-			id,
-			databaseId: this.databaseId,
-			statementId: this.statementId,
-		});
-		return results;
+		const encoded = messages.encodeValues(values, getProtocol(this.options));
+		const result = await this.send(
+			{
+				type: "all",
+				values: encoded.values ?? [],
+				id: uuid(),
+				databaseId: this.databaseId,
+				statementId: this.statementId,
+			},
+				{
+					requestType: "all",
+					sql: this.sql,
+					...encoded.metrics,
+				},
+		);
+		this.needsReset = false;
+		return result;
 	}
 }
 
 class ProxyDatabase implements IDatabase {
 	statements: Map<string, ProxyStatement> = new Map();
-
-	resolvers: {
-		[hash in string]: {
-			resolve: (...args: any) => void;
-			reject: (...args: any) => void;
-		};
-	} = {};
 	databaseId!: string;
+	private directory?: string;
+
 	constructor(
-		readonly send: <T>(
+		readonly postMessage: (
 			message: messages.DatabaseMessages | messages.StatementMessages,
-		) => Promise<T>,
+		) => Promise<ResponseMessage>,
+		readonly options?: SQLiteBrowserOptions,
 	) {}
 
+	private async send<T>(
+		message: messages.DatabaseMessages | messages.StatementMessages,
+		metrics?: SendMetrics,
+	): Promise<T> {
+		const startedAt = performance.now();
+		const protocol = getProtocol(this.options);
+		const shouldProfile = Boolean(this.options?.profile || this.options?.onProfile);
+		const requestType = metrics?.requestType ?? message.type;
+
+		try {
+			const response = await this.postMessage({
+				...message,
+				protocol,
+				profile: shouldProfile,
+			});
+			this.options?.onProfile?.({
+				requestType,
+				protocol,
+				databaseId: this.databaseId,
+				databaseDirectory: this.directory,
+				sql:
+					metrics?.sql ??
+					("sql" in message && typeof message.sql === "string"
+						? message.sql
+						: undefined),
+				clientEncodeMs: metrics?.encodeMs ?? 0,
+				clientRoundTripMs: performance.now() - startedAt,
+				valueCount: metrics?.valueCount ?? 0,
+				blobValueCount: metrics?.blobValueCount ?? 0,
+				blobBytes: metrics?.blobBytes ?? 0,
+				worker: response.timing,
+			});
+			return response.result as T;
+		} catch (error: any) {
+			const responseError = error as ErrorMessage | undefined;
+			this.options?.onProfile?.({
+				requestType,
+				protocol,
+				databaseId: this.databaseId,
+				databaseDirectory: this.directory,
+				sql:
+					metrics?.sql ??
+					("sql" in message && typeof message.sql === "string"
+						? message.sql
+						: undefined),
+				clientEncodeMs: metrics?.encodeMs ?? 0,
+				clientRoundTripMs: performance.now() - startedAt,
+				valueCount: metrics?.valueCount ?? 0,
+				blobValueCount: metrics?.blobValueCount ?? 0,
+				blobBytes: metrics?.blobBytes ?? 0,
+				worker: responseError?.timing,
+			});
+			if (responseError?.type === "error") {
+				throw new Error(responseError.message);
+			}
+			throw error;
+		}
+	}
+
 	async init(directory?: string) {
 		this.databaseId = uuid();
+		this.directory = directory;
 		return this.send({
 			type: "create",
 			directory,
+			pragmas: this.options?.pragmas,
 			databaseId: this.databaseId,
 			id: uuid(),
 		});
@@ -141,9 +296,11 @@ class ProxyDatabase implements IDatabase {
 			databaseId: this.databaseId,
 		});
 		const statement = new ProxyStatement(
-			this.send,
+			this.send.bind(this),
 			this.databaseId,
 			statementId,
+			sql,
+			this.options,
 		);
 		this.statements.set(statementId, statement);
 
@@ -181,18 +338,10 @@ class ProxyDatabase implements IDatabase {
 			databaseId: this.databaseId,
 		});
 	}
-
-	/*    async get(sql: string) {
-		   return this.send({ type: 'get', sql, id: uuid() });
-	   }
-   
-	   async run(sql: string, bind: any[]) {
-		   return this.send({ type: 'run', sql, bind, id: uuid() });
-	   } */
 }
 
 interface DatabaseCreator {
-	create(directory?: string): Promise<ProxyDatabase>;
+	create(directory?: string, options?: SQLiteBrowserOptions): Promise<ProxyDatabase>;
 	close(): Promise<void> | void;
 }
 
@@ -202,21 +351,22 @@ const init = async (): Promise<DatabaseCreator> => {
 		return initialized;
 	}
 
-	let worker = new Worker(
+	const worker = new Worker(
 		new URL("/peerbit/sqlite3/sqlite3.worker.min.js", import.meta.url),
 		{ type: "module" },
 	);
-	let resolvers: {
-		[hash in string]: {
-			resolve: (...args: any) => void;
-			reject: (...args: any) => void;
-		};
-	} = {};
-
-	let send = <T>(
+	const resolvers: Record<
+		string,
+		{
+			resolve: (message: ResponseMessage) => void;
+			reject: (message: ErrorMessage) => void;
+		}
+	> = {};
+
+	const postMessage = (
 		message: messages.DatabaseMessages | messages.StatementMessages,
 	) => {
-		const promise = new Promise<T>((resolve, reject) => {
+		const promise = new Promise<ResponseMessage>((resolve, reject) => {
 			resolvers[message.id] = { resolve, reject };
 		});
 		worker.postMessage(message);
@@ -224,7 +374,7 @@ const init = async (): Promise<DatabaseCreator> => {
 		return promise.finally(() => delete resolvers[message.id]);
 	};
 
-	let isReady = pDefer();
+	const isReady = pDefer();
 
 	worker.onmessage = async (ev) => {
 		const message = ev.data as messages.ResponseMessages | messages.IsReady;
@@ -235,15 +385,21 @@ const init = async (): Promise<DatabaseCreator> => {
 		}
 
 		const resolver = resolvers[message.id];
+		if (!resolver) {
+			return;
+		}
 		if (message.type === "error") {
-			resolver.reject(new Error(message.message));
-		} else if (message.type === "response") {
-			resolver.resolve(message.result);
+			resolver.reject(message);
+		} else {
+			resolver.resolve(message);
 		}
 	};
 
-	const create = async (directory?: string) => {
-		const db = new ProxyDatabase(send);
+	const create = async (
+		directory?: string,
+		options?: SQLiteBrowserOptions,
+	) => {
+		const db = new ProxyDatabase(postMessage, options);
 		await isReady.promise;
 		await db.init(directory);
 		await db.open();
@@ -258,12 +414,14 @@ const init = async (): Promise<DatabaseCreator> => {
 	});
 };
 
-const create = (directory?: string): Promise<IDatabase> => {
+const create = (
+	directory?: string,
+	options?: SQLiteBrowserOptions,
+): Promise<IDatabase> => {
 	if (directory) {
 		// persist the database
-		return init().then((creator) => creator.create(directory));
-	} else {
-		return createDatabase();
+		return init().then((creator) => creator.create(directory, options));
 	}
+	return createDatabase(directory, options);
 };
 export { create };

package/src/sqlite3.ts

@@ -1,11 +1,28 @@
 import DB from "better-sqlite3";
 import fs from "fs";
+import type { SQLitePragmaOptions } from "./sqlite3-messages.worker.js";
 import type {
 	Database as IDatabase,
 	Statement as IStatement,
 } from "./types.js";
 
-let create = async (directory?: string) => {
+const applyPragmas = (db: DB.Database, pragmas?: SQLitePragmaOptions) => {
+	db.pragma("journal_mode = WAL");
+	db.pragma("foreign_keys = on");
+	db.pragma(`synchronous = ${(pragmas?.synchronous ?? "FULL").toUpperCase()}`);
+	if (pragmas?.lockingMode) {
+		db.pragma(`locking_mode = ${pragmas.lockingMode.toUpperCase()}`);
+	}
+	if (pragmas?.tempStore && pragmas.tempStore !== "DEFAULT") {
+		db.pragma(`temp_store = ${pragmas.tempStore.toUpperCase()}`);
+	}
+	db.defaultSafeIntegers(true);
+};
+
+let create = async (
+	directory?: string,
+	options?: { pragmas?: SQLitePragmaOptions },
+) => {
 	let db: DB.Database | undefined = undefined;
 	let statements: Map<string, IStatement> = new Map();
 	let dbFileName: string;
@@ -50,10 +67,7 @@ let create = async (directory?: string) => {
 			});
 		}
 
-		// TODO this test makes things faster, but for benchmarking it might yield wierd results where some runs are faster than others
-		db.pragma("journal_mode = WAL");
-		db.pragma("foreign_keys = on");
-		db.defaultSafeIntegers(true);
+		applyPragmas(db, options?.pragmas);
 	};
 
 	return {

package/src/sqlite3.wasm.ts

@@ -12,6 +12,7 @@ import {
 	type Statement as IStatement,
 	type StatementGetResult,
 } from "./types.js";
+import type { SQLitePragmaOptions } from "./sqlite3-messages.worker.js";
 
 export const encodeName = (name: string): string => {
 	// since "/" and perhaps other characters might not be allowed we do encode
@@ -183,7 +184,29 @@ const getSqlite3 = async (): Promise<Sqlite3Module> => {
 	return sqlite3Promise;
 };
 
-const create = async (directory?: string) => {
+const applyPragmas = (
+	db: SQLDatabase | OpfsSAHPoolDatabase,
+	pragmas?: SQLitePragmaOptions,
+) => {
+	db.exec("PRAGMA journal_mode = WAL");
+	db.exec("PRAGMA foreign_keys = on");
+	// Browser SQLite state is a rebuildable materialized view over the log, so
+	// NORMAL is a better default tradeoff than FULL for OPFS-backed write latency.
+	db.exec(
+		`PRAGMA synchronous = ${(pragmas?.synchronous ?? "NORMAL").toUpperCase()}`,
+	);
+	if (pragmas?.lockingMode) {
+		db.exec(`PRAGMA locking_mode = ${pragmas.lockingMode.toUpperCase()}`);
+	}
+	if (pragmas?.tempStore && pragmas.tempStore !== "DEFAULT") {
+		db.exec(`PRAGMA temp_store = ${pragmas.tempStore.toUpperCase()}`);
+	}
+};
+
+const create = async (
+	directory?: string,
+	options?: { pragmas?: SQLitePragmaOptions },
+) => {
 	let statements: Map<string, Statement> = new Map();
 
 	const sqlite3 = await getSqlite3();
@@ -299,8 +322,7 @@ const create = async (directory?: string) => {
 		if (!sqliteDb) {
 			throw new Error("Failed to open sqlite database");
 		}
-		sqliteDb.exec("PRAGMA journal_mode = WAL");
-		sqliteDb.exec("PRAGMA foreign_keys = on");
+		applyPragmas(sqliteDb, options?.pragmas);
 	};
 
 	return {