@peerbit/indexer-sqlite3 0.0.1-cccc078

package/src/sqlite3-messages.worker.ts

@@ -0,0 +1,123 @@
+import { fromBase64, toBase64 } from "@peerbit/crypto";
+
+interface Message {
+	id: string;
+	databaseId: string;
+}
+
+// Database messages
+interface CreateDatabase extends Message {
+	type: "create";
+	directory?: string;
+}
+
+interface Exec extends Message {
+	type: "exec";
+	sql: string;
+}
+
+interface Status extends Message {
+	type: "status";
+}
+
+interface Close extends Message {
+	type: "close";
+}
+
+interface Open extends Message {
+	type: "open";
+}
+
+interface Prepare extends Message {
+	type: "prepare";
+	sql: string;
+}
+
+type Uint8ArrayType = { type: "uint8array"; base64: string };
+
+type SimpleType = { type: "simple"; value: any };
+
+export const resolveValue = (value: Uint8ArrayType | SimpleType) =>
+	value.type === "simple" ? value.value : fromBase64(value.base64);
+export const encodeValue = (value: any): Uint8ArrayType | SimpleType => {
+	if (value instanceof Uint8Array) {
+		return { type: "uint8array", base64: toBase64(value) };
+	}
+	return { type: "simple", value };
+};
+
+interface Run extends Statement {
+	type: "run";
+	sql: string;
+	values: (Uint8ArrayType | SimpleType)[];
+}
+
+// Statement messages
+interface Statement extends Message {
+	statementId: string;
+}
+
+interface Bind extends Statement {
+	type: "bind";
+	values: (Uint8ArrayType | SimpleType)[];
+}
+
+interface Step extends Statement {
+	type: "step";
+}
+
+interface Get extends Statement {
+	type: "get";
+	values?: any[];
+}
+
+interface Reset extends Statement {
+	type: "reset";
+}
+
+interface RunStatement extends Statement {
+	type: "run-statement";
+	values: any[];
+}
+
+interface All extends Statement {
+	type: "all";
+	values: (Uint8ArrayType | SimpleType)[];
+}
+
+interface Finalize extends Statement {
+	type: "finalize";
+}
+
+// Response messages
+interface ErrorResponse {
+	type: "error";
+	id: string;
+	message: string;
+}
+
+interface Response {
+	type: "response";
+	id: string;
+	result: any;
+}
+
+export type DatabaseMessages =
+	| CreateDatabase
+	| Exec
+	| Prepare
+	| Close
+	| Open
+	| Run
+	| Status;
+export type StatementMessages =
+	| Bind
+	| Step
+	| Get
+	| Reset
+	| All
+	| Finalize
+	| RunStatement;
+export type ResponseMessages = ErrorResponse | Response;
+
+export type IsReady = { type: "ready" };

package/src/sqlite3.browser.ts

@@ -0,0 +1,245 @@
+import pDefer from "p-defer";
+import { v4 as uuid } from "uuid";
+import type { BindableValue } from "./schema.js";
+import * as messages from "./sqlite3-messages.worker.js";
+import { create as createDatabase } from "./sqlite3.wasm.js";
+import {
+	type Database as IDatabase,
+	type Statement as IStatement,
+	type StatementGetResult,
+} from "./types.js";
+
+class ProxyStatement implements IStatement {
+	resolvers: {
+		[hash in string]: {
+			resolve: (...args: any) => void;
+			reject: (...args: any) => void;
+		};
+	} = {};
+
+	constructor(
+		readonly send: <T>(
+			message: messages.DatabaseMessages | messages.StatementMessages,
+		) => Promise<T>,
+		readonly databaseId: string,
+		readonly statementId: string,
+	) {}
+
+	async bind(values: any[]) {
+		await this.send({
+			type: "bind",
+			values: values.map(messages.encodeValue),
+			id: uuid(),
+			databaseId: this.databaseId,
+			statementId: this.statementId,
+		});
+		return this;
+	}
+
+	async finalize() {
+		await this.send({
+			type: "finalize",
+			id: uuid(),
+			databaseId: this.databaseId,
+			statementId: this.statementId,
+		});
+	}
+
+	get(values?: BindableValue[]) {
+		return this.send<StatementGetResult>({
+			type: "get",
+			values: values ? values.map(messages.encodeValue) : undefined,
+			id: uuid(),
+			databaseId: this.databaseId,
+			statementId: this.statementId,
+		});
+	}
+
+	async run(values: BindableValue[]) {
+		await this.send({
+			type: "run-statement",
+			values: values.map(messages.encodeValue),
+			id: uuid(),
+			databaseId: this.databaseId,
+			statementId: this.statementId,
+		});
+	}
+
+	async reset() {
+		await this.send({
+			type: "reset",
+			id: uuid(),
+			databaseId: this.databaseId,
+			statementId: this.statementId,
+		});
+		return this;
+	}
+
+	async all(values: BindableValue[]) {
+		let id = uuid();
+		const results = await this.send({
+			type: "all",
+			values: values.map(messages.encodeValue),
+			id,
+			databaseId: this.databaseId,
+			statementId: this.statementId,
+		});
+		return results;
+	}
+}
+
+class ProxyDatabase implements IDatabase {
+	statements: Map<string, ProxyStatement> = new Map();
+	resolvers: {
+		[hash in string]: {
+			resolve: (...args: any) => void;
+			reject: (...args: any) => void;
+		};
+	} = {};
+	databaseId: string;
+	constructor(
+		readonly send: <T>(
+			message: messages.DatabaseMessages | messages.StatementMessages,
+		) => Promise<T>,
+	) {}
+
+	async init(directory?: string) {
+		this.databaseId = uuid();
+		return this.send({
+			type: "create",
+			directory,
+			databaseId: this.databaseId,
+			id: uuid(),
+		});
+	}
+
+	async exec(sql: string) {
+		return this.send({
+			type: "exec",
+			sql,
+			id: uuid(),
+			databaseId: this.databaseId,
+		});
+	}
+
+	async prepare(sql: string) {
+		const statementId = await this.send<string>({
+			type: "prepare",
+			sql,
+			id: uuid(),
+			databaseId: this.databaseId,
+		});
+		const statement = new ProxyStatement(
+			this.send,
+			this.databaseId,
+			statementId,
+		);
+		this.statements.set(statementId, statement);
+		return statement;
+	}
+
+	async open() {
+		return this.send({ type: "open", id: uuid(), databaseId: this.databaseId });
+	}
+
+	async close() {
+		return this.send({
+			type: "close",
+			id: uuid(),
+			databaseId: this.databaseId,
+		});
+	}
+
+	async status() {
+		return this.send<"open" | "closed">({
+			type: "status",
+			id: uuid(),
+			databaseId: this.databaseId,
+		});
+	}
+
+	/*    async get(sql: string) {
+		   return this.send({ type: 'get', sql, id: uuid() });
+	   }
+   
+	   async run(sql: string, bind: any[]) {
+		   return this.send({ type: 'run', sql, bind, id: uuid() });
+	   } */
+}
+
+interface DatabaseCreator {
+	create(directory?: string): Promise<ProxyDatabase>;
+	close(): Promise<void> | void;
+}
+
+let initialized: DatabaseCreator | undefined = undefined;
+const init = async (): Promise<DatabaseCreator> => {
+	if (initialized) {
+		return initialized;
+	}
+
+	let worker = new Worker(
+		new URL("/peerbit/sqlite3.worker.min.js", import.meta.url),
+		{ type: "module" },
+	);
+	let resolvers: {
+		[hash in string]: {
+			resolve: (...args: any) => void;
+			reject: (...args: any) => void;
+		};
+	} = {};
+
+	let send = <T>(
+		message: messages.DatabaseMessages | messages.StatementMessages,
+	) => {
+		const promise = new Promise<T>((resolve, reject) => {
+			resolvers[message.id] = { resolve, reject };
+		});
+		worker.postMessage(message);
+
+		return promise.finally(() => delete resolvers[message.id]);
+	};
+
+	let isReady = pDefer();
+
+	worker.onmessage = async (ev) => {
+		const message = ev.data as messages.ResponseMessages | messages.IsReady;
+
+		if (message.type === "ready") {
+			isReady.resolve();
+			return;
+		}
+
+		const resolver = resolvers[message.id];
+		if (message.type === "error") {
+			resolver.reject(message.message);
+		} else if (message.type === "response") {
+			resolver.resolve(message.result);
+		}
+	};
+
+	const create = async (directory?: string) => {
+		const db = new ProxyDatabase(send);
+		await isReady.promise;
+		await db.init(directory);
+		await db.open();
+		return db;
+	};
+	return (initialized = {
+		create,
+		close: () => {
+			initialized = undefined;
+			worker.terminate();
+		},
+	});
+};
+
+const create = (directory?: string): Promise<IDatabase> => {
+	if (directory) {
+		// persist the database
+		return init().then((creator) => creator.create(directory));
+	} else {
+		return createDatabase();
+	}
+};
+export { create };

package/src/sqlite3.ts

@@ -0,0 +1,56 @@
+import DB from "better-sqlite3";
+import fs from "fs";
+import type {
+	Database as IDatabase,
+	Statement as IStatement,
+} from "./types.js";
+
+let create = async (directory?: string) => {
+	let db: DB.Database | undefined = undefined;
+	let close = () => {
+		if (db) {
+			db.close();
+			db = undefined;
+		}
+	};
+	let open = () => {
+		if (db) {
+			return db;
+		}
+
+		let dbFileName: string;
+		if (directory) {
+			// if directory is provided, check if directory exist, if not create it
+			if (!fs.existsSync(directory)) {
+				fs.mkdirSync(directory, { recursive: true });
+			}
+			dbFileName = `${directory}/db.sqlite`;
+		} else {
+			dbFileName = ":memory:";
+		}
+
+		db = new DB(dbFileName, {
+			fileMustExist: false,
+			readonly: false /* , verbose: (message) => console.log(message)  */,
+		});
+		/* db.pragma('journal_mode = WAL'); */
+		db.pragma("foreign_keys = on");
+		db.defaultSafeIntegers(true);
+	};
+
+	return {
+		exec: (sql: string) => {
+			if (!db) throw new Error("Database not open");
+			return db.exec(sql);
+		},
+		prepare(sql: string) {
+			if (!db) throw new Error("Database not open");
+			return db.prepare(sql) as any as IStatement; // TODO types
+		},
+		close,
+		open,
+		status: () => (db ? "open" : "closed"),
+	} as IDatabase; // TODO fix this
+};
+
+export { create };

package/src/sqlite3.wasm.ts

@@ -0,0 +1,211 @@
+import { BinaryReader, BinaryWriter } from "@dao-xyz/borsh";
+import { fromBase64URL, toBase64URL } from "@peerbit/crypto";
+import {
+	type OpfsSAHPoolDatabase,
+	type SAHPoolUtil,
+	type Database as SQLDatabase,
+	type PreparedStatement as SQLStatement,
+	default as sqlite3InitModule,
+} from "@sqlite.org/sqlite-wasm";
+import type { BindableValue } from "./schema.js";
+import {
+	type Statement as IStatement,
+	type StatementGetResult,
+} from "./types.js";
+
+/* import { v4 as uuid } from 'uuid';
+ */
+export const encodeName = (name: string): string => {
+	// since "/" and perhaps other characters might not be allowed we do encode
+	const writer = new BinaryWriter();
+	writer.string(name);
+	return toBase64URL(writer.finalize());
+};
+
+export const decodeName = (name: string): string => {
+	// since "/" and perhaps other characters might not be allowed we do encode
+	const writer = new BinaryReader(fromBase64URL(name));
+	return writer.string();
+};
+
+class Statement implements IStatement {
+	constructor(private statement: SQLStatement) {}
+
+	async bind(values: any[]) {
+		await this.statement.bind(values);
+		return this;
+	}
+
+	async finalize() {
+		if ((await this.statement.finalize()) > 0) {
+			throw new Error("Error finalizing statement");
+		}
+	}
+
+	get(values?: BindableValue[]) {
+		if (values) {
+			this.statement.bind(values);
+		}
+		let step = this.statement.step();
+		if (!step) {
+			// no data available
+			this.statement.reset();
+			return undefined;
+		}
+		const results = this.statement.get({});
+		this.statement.reset();
+		return results as StatementGetResult;
+	}
+
+	run(values: BindableValue[]) {
+		this.statement.bind(values as any);
+		this.statement.stepReset();
+	}
+
+	async reset() {
+		await this.statement.reset();
+		return this;
+	}
+
+	all(values: BindableValue[]) {
+		if (values && values.length > 0) {
+			this.statement.bind(values as any);
+		}
+
+		let results = [];
+		while (this.statement.step()) {
+			results.push(this.statement.get({}));
+		}
+		this.statement.reset();
+		return results;
+	}
+
+	step() {
+		return this.statement.step();
+	}
+}
+
+/* export class Database implements IDatabase {
+
+	statements: Map<string, Statement> = new Map();
+	private db: SQLDatabase
+	constructor(private readonly _close?: () => Promise<any> | any) { }
+
+	async exec(sql: string) {
+		return this.db.exec(sql);
+	}
+
+	async prepare(sql: string) {
+		const statement = this.db.prepare(sql);
+		const wrappedStatement = new Statement(statement);
+		this.statements.set(sql, wrappedStatement)
+		return wrappedStatement
+	}
+
+	async close() {
+		await Promise.all([...this.statements.values()].map(x => x.finalize?.()))
+		await this.db.close();
+		await this._close?.()
+	}
+
+	async get(sql: string) {
+		return this.db.exec({ sql, rowMode: 'array' });
+	}
+
+	async run(sql: string, bind: any[]) {
+		return this.db.exec(sql, { bind, rowMode: 'array' });
+	}
+}
+
+ */
+
+// eslint-disable-next-line no-console
+const log = (...args: any) => console.log(...args);
+// eslint-disable-next-line no-console
+const error = (...args: any) => console.error(...args);
+
+/* let initOpfsResult: Promise<{ sqlite3: Awaited<ReturnType<typeof sqlite3InitModule>>,poolUtil: }> | undefined = undefined;
+const initOpfs = async () => {
+
+	let sqlite3: Awaited<ReturnType<typeof sqlite3InitModule>> = await sqlite3InitModule({
+		locateFile: (path, prefix) => {
+
+			return path;
+		}, print: log, printErr: error
+	});
+	let poolUtil = await sqlite3.installOpfsSAHPoolVfs({
+		directory: encodeName("helloworld")
+	});
+
+	return initOpfsResult || (initOpfsResult = { sqlite3, poolUtil })
+} */
+
+let poolUtil: SAHPoolUtil = undefined;
+let sqlite3: Awaited<ReturnType<typeof sqlite3InitModule>> | undefined =
+	undefined;
+
+const create = async (directory?: string) => {
+	let statements: Map<string, Statement> = new Map();
+
+	sqlite3 =
+		sqlite3 || (await sqlite3InitModule({ print: log, printErr: error }));
+	let sqliteDb: OpfsSAHPoolDatabase | SQLDatabase | undefined = undefined;
+	let close: (() => Promise<any> | any) | undefined = async () => {
+		await Promise.all([...statements.values()].map((x) => x.finalize?.()));
+		statements.clear();
+
+		await sqliteDb?.close();
+		sqliteDb = undefined;
+	};
+	let open = async () => {
+		if (sqliteDb) {
+			return sqliteDb;
+		}
+		if (directory) {
+			// directory has to be absolute path. Remove leading dot if any
+			// TODO show warning if directory is not absolute?
+			directory = directory.replace(/^\./, "");
+
+			let dbFileName = `${directory}/db.sqlite`;
+
+			poolUtil =
+				poolUtil ||
+				(await sqlite3.installOpfsSAHPoolVfs({
+					directory: "peerbit/sqlite", // encodeName("peerbit")
+				}));
+
+			await poolUtil.reserveMinimumCapacity(100);
+			sqliteDb = new poolUtil.OpfsSAHPoolDb(dbFileName);
+		} else {
+			sqliteDb = new sqlite3.oo1.DB(":memory:");
+		}
+
+		sqliteDb.exec("PRAGMA journal_mode = WAL");
+		sqliteDb.exec("PRAGMA foreign_keys = on");
+	};
+
+	return {
+		close,
+		exec: (sql: string) => {
+			return sqliteDb.exec(sql);
+		},
+		open,
+		prepare: (sql: string) => {
+			const statement = sqliteDb.prepare(sql);
+			const wrappedStatement = new Statement(statement);
+			statements.set(sql, wrappedStatement);
+			return wrappedStatement;
+		},
+		get(sql: string) {
+			return sqliteDb.exec({ sql, rowMode: "array" });
+		},
+
+		run(sql: string, bind: any[]) {
+			return sqliteDb.exec(sql, { bind, rowMode: "array" });
+		},
+		status: () => (sqliteDb?.isOpen() ? "open" : "closed"),
+		statements,
+	};
+};
+
+export { create };