@optimystic/db-p2p-storage-ns 0.14.1 → 0.16.3

package/src/ns-opener.ts

@@ -1,4 +1,5 @@
-import { applySchema, DEFAULT_DB_NAME, DEFAULT_DB_VERSION, type OptimysticNSDBHandle, type SqliteDb, type SqliteParam, type SqliteRow, type SqliteStatement } from './db.js';
+import { ConnectionMutex } from './connection-mutex.js';
+import { applySchema, DEFAULT_DB_NAME, DEFAULT_DB_VERSION, type OptimysticNSDBHandle, type SqliteDb, type SqliteParam, type SqliteRow, type SqliteStatement, type SqliteTransaction } from './db.js';
 
 /**
  * Minimal subset of `@nativescript-community/sqlite`'s `Db` we depend on.
@@ -7,7 +8,7 @@ import { applySchema, DEFAULT_DB_NAME, DEFAULT_DB_VERSION, type OptimysticNSDBHa
  * at typecheck time on non-NativeScript consumers).
  */
 interface NSPluginDb {
-	execSQL(sql: string, params?: ReadonlyArray<unknown>): unknown;
+	execute(sql: string, params?: ReadonlyArray<unknown>): unknown;
 	get(sql: string, params?: ReadonlyArray<unknown>): Promise<Record<string, unknown> | null | undefined> | Record<string, unknown> | null | undefined;
 	select(sql: string, params?: ReadonlyArray<unknown>): Promise<Array<Record<string, unknown>>> | Array<Record<string, unknown>>;
 	close(): void | Promise<void>;
@@ -23,9 +24,13 @@ interface NSPluginModule {
  * the migration to `version`.
  *
  * The returned `SqliteDb` handle is safe to share across `SqliteRawStorage`,
- * `SqliteKVStore`, and `loadOrCreateNSPeerKey` — SQLite serializes writes
- * inside the connection, and our reads/writes are short-lived enough that
- * single-connection contention is a non-issue for a client peer.
+ * `SqliteKVStore`, and `loadOrCreateNSPeerKey`. Because SQLite allows at most
+ * one open transaction per connection, the wrapper serializes every mutating
+ * operation — `exec`, statement `run`, and whole `transaction` bodies — through
+ * a per-connection FIFO mutex. Without it, two concurrent `transaction` bodies
+ * would each `BEGIN` on the shared connection; the second would nest and its
+ * rollback would silently discard the first's still-open writes. Reads
+ * (`get`/`all`) stay off the mutex to preserve read concurrency.
  *
  * `path` may be passed in as the full filesystem path if the caller wants
  * to control file placement; otherwise the plugin's documents-directory
@@ -47,44 +52,63 @@ export async function openOptimysticNSDb(
  * internal `SqliteDb` interface used by the storage classes. Exported only
  * for callers that already hold an open NS-plugin handle and want to skip
  * the opener (rare — typically users just call `openOptimysticNSDb`).
+ *
+ * NOTE: the serialization mutex lives on the wrapper, not the raw handle. Wrap
+ * a given raw connection exactly once — two wrappers over one handle each hold
+ * their own mutex and would not serialize against each other, reintroducing the
+ * cross-rollback this fix closes.
  */
 export function wrapNSPluginDb(raw: NSPluginDb): SqliteDb {
 	return new NSPluginDbWrapper(raw);
 }
 
 class NSPluginDbWrapper implements SqliteDb {
+	private readonly mutex = new ConnectionMutex();
+
 	constructor(private readonly raw: NSPluginDb) {}
 
 	async exec(sql: string): Promise<void> {
-		// The plugin's execSQL accepts a single statement; split semicolon-
+		// The plugin's execute accepts a single statement; split semicolon-
 		// separated DDL so callers can pass the full schema in one go.
 		const statements = sql
 			.split(';')
 			.map(s => s.trim())
 			.filter(s => s.length > 0);
-		for (const statement of statements) {
-			await this.raw.execSQL(statement);
-		}
+		await this.mutex.serialize(async () => {
+			for (const statement of statements) {
+				await this.raw.execute(statement);
+			}
+		});
 	}
 
 	prepare(sql: string): SqliteStatement {
-		return new NSPluginStatement(this.raw, sql);
+		// Outside-transaction statement: writes go through the mutex.
+		return new NSPluginStatement(this.raw, sql, this.mutex);
 	}
 
-	async transaction<T>(fn: () => Promise<T>): Promise<T> {
-		await this.raw.execSQL('BEGIN');
-		try {
-			const result = await fn();
-			await this.raw.execSQL('COMMIT');
-			return result;
-		} catch (err) {
+	async transaction<T>(fn: (tx: SqliteTransaction) => Promise<T>): Promise<T> {
+		return this.mutex.serialize(async () => {
+			// BEGIN IMMEDIATE takes the write lock up front rather than deferring
+			// it to the first write, so contention surfaces here and not mid-body.
+			await this.raw.execute('BEGIN IMMEDIATE');
 			try {
-				await this.raw.execSQL('ROLLBACK');
-			} catch {
-				// Swallow rollback failures so we surface the original error.
+				// Statements bound to the open transaction bypass the mutex — we
+				// already hold the slot; re-locking would deadlock.
+				const tx: SqliteTransaction = {
+					prepare: (sql: string) => new NSPluginStatement(this.raw, sql),
+				};
+				const result = await fn(tx);
+				await this.raw.execute('COMMIT');
+				return result;
+			} catch (err) {
+				try {
+					await this.raw.execute('ROLLBACK');
+				} catch {
+					// Swallow rollback failures so we surface the original error.
+				}
+				throw err;
 			}
-			throw err;
-		}
+		});
 	}
 
 	async close(): Promise<void> {
@@ -93,19 +117,42 @@ class NSPluginDbWrapper implements SqliteDb {
 }
 
 class NSPluginStatement implements SqliteStatement {
-	constructor(private readonly raw: NSPluginDb, private readonly sql: string) {}
+	/**
+	 * @param mutex When present, `run` is serialized on the connection mutex
+	 *   (outside-transaction writes). When absent, `run` executes directly on the
+	 *   raw connection — used for statements bound to an already-open transaction
+	 *   that already holds the mutex slot.
+	 */
+	constructor(
+		private readonly raw: NSPluginDb,
+		private readonly sql: string,
+		private readonly mutex?: ConnectionMutex,
+	) {}
 
 	async run(...params: SqliteParam[]): Promise<void> {
-		await this.raw.execSQL(this.sql, params);
+		if (this.mutex) {
+			await this.mutex.serialize(() => this.raw.execute(this.sql, params));
+		} else {
+			await this.raw.execute(this.sql, params);
+		}
 	}
 
 	async get(...params: SqliteParam[]): Promise<SqliteRow | undefined> {
+		// NOTE: reads run directly on the connection, unserialized, to preserve read
+		// concurrency. A read issued while a write transaction is open on this same
+		// connection observes that transaction's UNCOMMITTED rows (read-your-connection
+		// semantics). Fine today: the only transaction writer is same-block promote
+		// under the commit latch, and cross-block reads are independent. If a future
+		// caller reads a block on this connection while another op's transaction on the
+		// same rows is mid-flight, it may see uncommitted state — serialize reads too if
+		// that ever matters.
 		const row = await this.raw.get(this.sql, params);
 		if (row === null || row === undefined) return undefined;
 		return row as SqliteRow;
 	}
 
 	async all(...params: SqliteParam[]): Promise<SqliteRow[]> {
+		// NOTE: unserialized read — see the note on `get` above re: uncommitted reads.
 		const rows = await this.raw.select(this.sql, params);
 		return rows as SqliteRow[];
 	}

package/src/sqlite-storage.ts

@@ -1,25 +1,29 @@
-import type { ActionId, ActionRev, BlockId, IBlock, Transform } from '@optimystic/db-core';
-import type { BlockMetadata, IRawStorage } from '@optimystic/db-p2p';
+import type { ActionId, BlockId } from '@optimystic/db-core';
+import { KvRawStorage, type RawStoreDriver } from '@optimystic/db-p2p';
 import type { SqliteDb, SqliteStatement } from './db.js';
 import { createLogger } from './logger.js';
 
 const log = createLogger('storage:sqlite');
 
 /**
- * SQLite-backed `IRawStorage` implementation for NativeScript peers.
+ * SQLite {@link RawStoreDriver}: the five logical block-storage stores mapped to
+ * the five relational tables (`metadata`, `revisions`, `pending`, `transactions`,
+ * `materialized`) with their original columns and keys. This is a code refactor,
+ * not a storage-format change at the SQL level.
  *
- * Every CRUD operation goes through a prepared statement bound once in the
- * constructor. `listRevisions` and `listPendingTransactions` issue
- * `SELECT … ORDER BY` queries bounded by `block_id` (and `rev` for revisions)
- * and drain the results into an array before yielding — matching the
- * `IndexedDBRawStorage` pattern, where holding a cursor live across consumer
- * awaits would invalidate the underlying transaction.
- *
- * `promotePendingTransaction` runs as a single `BEGIN; INSERT…; DELETE…; COMMIT;`
- * via `SqliteDb.transaction(fn)`, so the move is atomic across crashes —
- * unlike the MMKV adapter, which has to maintain a separate pending-index row.
+ * `KvRawStorage` now owns all JSON serialization, so this driver only ever
+ * reads/writes `Uint8Array` values: the value columns are BLOB and SQLite binds
+ * a `Uint8Array` as a BLOB and returns it as a `Uint8Array`, so no codec lives
+ * here (a TEXT column would risk UTF-8 coercion corrupting non-ASCII JSON bytes).
+ * Everything SQLite-specific stays: each CRUD op goes through a prepared
+ * statement bound once in the constructor; range/list queries drain their rows
+ * (`.all(...)`) before yielding so no cursor straddles a consumer's `await`; and
+ * `promote` runs as a single `db.transaction(fn)` — `BEGIN IMMEDIATE; INSERT…;
+ * DELETE…; COMMIT;` — whose three statements are re-prepared against the OPEN
+ * transaction so they run on the held mutex slot without re-locking (which would
+ * deadlock — see `st-nativescript-sqlite-transaction-mutex`).
  */
-export class SqliteRawStorage implements IRawStorage {
+export class SqliteStoreDriver implements RawStoreDriver {
 	private readonly stmts: {
 		getMetadata: SqliteStatement;
 		saveMetadata: SqliteStatement;
@@ -31,6 +35,7 @@ export class SqliteRawStorage implements IRawStorage {
 		savePending: SqliteStatement;
 		deletePending: SqliteStatement;
 		listPending: SqliteStatement;
+		listBlockIds: SqliteStatement;
 		getTransaction: SqliteStatement;
 		saveTransaction: SqliteStatement;
 		getMaterialized: SqliteStatement;
@@ -52,6 +57,11 @@ export class SqliteRawStorage implements IRawStorage {
 			savePending: db.prepare('INSERT OR REPLACE INTO pending (block_id, action_id, value) VALUES (?, ?, ?)'),
 			deletePending: db.prepare('DELETE FROM pending WHERE block_id = ? AND action_id = ?'),
 			listPending: db.prepare('SELECT action_id FROM pending WHERE block_id = ? ORDER BY action_id ASC'),
+			// metadata.block_id is the PRIMARY KEY, so each row is a distinct block id —
+			// no dedup needed. NOTE: drains the whole metadata table up front; if a peer
+			// ever holds millions of blocks and this SELECT becomes a startup-latency
+			// problem, page it (LIMIT/OFFSET or keyset) — fine at current scale.
+			listBlockIds: db.prepare('SELECT block_id FROM metadata'),
 			getTransaction: db.prepare('SELECT value FROM transactions WHERE block_id = ? AND action_id = ?'),
 			saveTransaction: db.prepare('INSERT OR REPLACE INTO transactions (block_id, action_id, value) VALUES (?, ?, ?)'),
 			getMaterialized: db.prepare('SELECT value FROM materialized WHERE block_id = ? AND action_id = ?'),
@@ -62,82 +72,116 @@ export class SqliteRawStorage implements IRawStorage {
 		};
 	}
 
-	async getMetadata(blockId: BlockId): Promise<BlockMetadata | undefined> {
+	// --- metadata ---
+
+	async getMetadata(blockId: BlockId): Promise<Uint8Array | undefined> {
 		const row = await this.stmts.getMetadata.get(blockId);
-		if (!row) return undefined;
-		return JSON.parse(row.value as string) as BlockMetadata;
+		return row ? (row.value as Uint8Array) : undefined;
 	}
 
-	async saveMetadata(blockId: BlockId, metadata: BlockMetadata): Promise<void> {
-		await this.stmts.saveMetadata.run(blockId, JSON.stringify(metadata));
+	async putMetadata(blockId: BlockId, value: Uint8Array): Promise<void> {
+		await this.stmts.saveMetadata.run(blockId, value);
 	}
 
-	async getRevision(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
+	// --- revisions ---
+
+	async getRevision(blockId: BlockId, rev: number): Promise<Uint8Array | undefined> {
 		const row = await this.stmts.getRevision.get(blockId, rev);
-		return row ? (row.action_id as ActionId) : undefined;
+		return row ? (row.action_id as Uint8Array) : undefined;
 	}
 
-	async saveRevision(blockId: BlockId, rev: number, actionId: ActionId): Promise<void> {
-		await this.stmts.saveRevision.run(blockId, rev, actionId);
+	async putRevision(blockId: BlockId, rev: number, value: Uint8Array): Promise<void> {
+		await this.stmts.saveRevision.run(blockId, rev, value);
 	}
 
-	async *listRevisions(blockId: BlockId, startRev: number, endRev: number): AsyncIterable<ActionRev> {
-		const ascending = startRev <= endRev;
-		const lo = ascending ? startRev : endRev;
-		const hi = ascending ? endRev : startRev;
-		const stmt = ascending ? this.stmts.listRevisionsAsc : this.stmts.listRevisionsDesc;
+	async *rangeRevisions(blockId: BlockId, lo: number, hi: number, reverse: boolean): AsyncIterable<[number, Uint8Array]> {
+		// `.all(...)` materializes every row before we yield, so no SQLite cursor
+		// straddles the consumer's awaits (the kernel's drain-before-yield contract).
+		const stmt = reverse ? this.stmts.listRevisionsDesc : this.stmts.listRevisionsAsc;
 		const rows = await stmt.all(blockId, lo, hi);
 		for (const row of rows) {
-			yield { rev: row.rev as number, actionId: row.action_id as ActionId };
+			yield [row.rev as number, row.action_id as Uint8Array];
 		}
 	}
 
-	async getPendingTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
+	// --- pending ---
+
+	async getPending(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
 		const row = await this.stmts.getPending.get(blockId, actionId);
-		if (!row) return undefined;
-		return JSON.parse(row.value as string) as Transform;
+		return row ? (row.value as Uint8Array) : undefined;
 	}
 
-	async savePendingTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
-		await this.stmts.savePending.run(blockId, actionId, JSON.stringify(transform));
+	async putPending(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
+		await this.stmts.savePending.run(blockId, actionId, value);
 	}
 
-	async deletePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
+	async deletePending(blockId: BlockId, actionId: ActionId): Promise<void> {
 		await this.stmts.deletePending.run(blockId, actionId);
 	}
 
-	async *listPendingTransactions(blockId: BlockId): AsyncIterable<ActionId> {
+	async *listPendingActionIds(blockId: BlockId): AsyncIterable<ActionId> {
+		// Drained by `.all(...)` before yielding — same rationale as rangeRevisions.
 		const rows = await this.stmts.listPending.all(blockId);
 		for (const row of rows) {
 			yield row.action_id as ActionId;
 		}
 	}
 
-	async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
+	// --- transactions ---
+
+	async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
 		const row = await this.stmts.getTransaction.get(blockId, actionId);
-		if (!row) return undefined;
-		return JSON.parse(row.value as string) as Transform;
+		return row ? (row.value as Uint8Array) : undefined;
 	}
 
-	async saveTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
-		await this.stmts.saveTransaction.run(blockId, actionId, JSON.stringify(transform));
+	async putTransaction(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
+		await this.stmts.saveTransaction.run(blockId, actionId, value);
 	}
 
-	async getMaterializedBlock(blockId: BlockId, actionId: ActionId): Promise<IBlock | undefined> {
+	// --- materialized ---
+
+	async getMaterialized(blockId: BlockId, actionId: ActionId): Promise<Uint8Array | undefined> {
 		const row = await this.stmts.getMaterialized.get(blockId, actionId);
-		if (!row) return undefined;
-		return JSON.parse(row.value as string) as IBlock;
+		return row ? (row.value as Uint8Array) : undefined;
+	}
+
+	async putMaterialized(blockId: BlockId, actionId: ActionId, value: Uint8Array): Promise<void> {
+		await this.stmts.saveMaterialized.run(blockId, actionId, value);
+	}
+
+	async deleteMaterialized(blockId: BlockId, actionId: ActionId): Promise<void> {
+		await this.stmts.deleteMaterialized.run(blockId, actionId);
+	}
+
+	// --- promote (the only cross-key atomic op) ---
+
+	async promote(blockId: BlockId, actionId: ActionId): Promise<void> {
+		await this.db.transaction(async (tx) => {
+			// Prepare against the open transaction so these three statements run on
+			// the held mutex slot without re-locking the connection (which would
+			// deadlock). Re-preparing is cheap — the driver caches by SQL text.
+			const getPending = tx.prepare('SELECT value FROM pending WHERE block_id = ? AND action_id = ?');
+			const saveTransaction = tx.prepare('INSERT OR REPLACE INTO transactions (block_id, action_id, value) VALUES (?, ?, ?)');
+			const deletePending = tx.prepare('DELETE FROM pending WHERE block_id = ? AND action_id = ?');
+			const row = await getPending.get(blockId, actionId);
+			if (!row) {
+				throw new Error(`Pending action ${actionId} not found for block ${blockId}`);
+			}
+			await saveTransaction.run(blockId, actionId, row.value as Uint8Array);
+			await deletePending.run(blockId, actionId);
+		});
 	}
 
-	async saveMaterializedBlock(blockId: BlockId, actionId: ActionId, block?: IBlock): Promise<void> {
-		if (block) {
-			await this.stmts.saveMaterialized.run(blockId, actionId, JSON.stringify(block));
-		} else {
-			await this.stmts.deleteMaterialized.run(blockId, actionId);
+	// --- optional passthroughs ---
+
+	async *listBlockIds(): AsyncIterable<BlockId> {
+		const rows = await this.stmts.listBlockIds.all();
+		for (const row of rows) {
+			yield row.block_id as BlockId;
 		}
 	}
 
-	async getApproximateBytesUsed(): Promise<number> {
+	async approximateBytesUsed(): Promise<number> {
 		try {
 			const pageCountRow = await this.stmts.pageCount.get();
 			const pageSizeRow = await this.stmts.pageSize.get();
@@ -149,15 +193,25 @@ export class SqliteRawStorage implements IRawStorage {
 			return 0;
 		}
 	}
+}
 
-	async promotePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
-		await this.db.transaction(async () => {
-			const row = await this.stmts.getPending.get(blockId, actionId);
-			if (!row) {
-				throw new Error(`Pending action ${actionId} not found for block ${blockId}`);
-			}
-			await this.stmts.saveTransaction.run(blockId, actionId, row.value as string);
-			await this.stmts.deletePending.run(blockId, actionId);
-		});
+/**
+ * SQLite-backed {@link IRawStorage} for NativeScript peers, now a thin shell over
+ * the shared {@link KvRawStorage} kernel driven by a {@link SqliteStoreDriver}.
+ * The public name/constructor (`new SqliteRawStorage(db)`) is unchanged so
+ * existing imports keep resolving; the kernel supplies the `IRawStorage` surface
+ * and the driver supplies SQLite behavior.
+ *
+ * `listBlockIds`/`getApproximateBytesUsed` are re-declared here as always-present
+ * (the SQLite driver always implements them, so the kernel constructor always
+ * wires them) — the base declares them optional, but every NS consumer relies on
+ * them.
+ */
+export class SqliteRawStorage extends KvRawStorage {
+	declare listBlockIds: () => AsyncIterable<BlockId>;
+	declare getApproximateBytesUsed: () => Promise<number>;
+
+	constructor(db: SqliteDb) {
+		super(new SqliteStoreDriver(db));
 	}
 }