@optimystic/db-p2p-storage-ns 1.15.0 → 1.16.0

package/src/db.ts

@@ -11,6 +11,10 @@
  * consumers; the `SqliteDb` interface is an internal seam for tests.
  */
 
+import { createLogger } from './logger.js';
+
+const log = createLogger('db');
+
 /** SQL parameter value — covers everything `IRawStorage` and `IKVStore` write. */
 export type SqliteParam = string | number | Uint8Array | null;
 
@@ -49,6 +53,18 @@ export interface SqliteTransaction {
 	prepare(sql: string): SqliteStatement;
 }
 
+/** Options of {@link SqliteDb.transaction}. */
+export interface SqliteTransactionOptions {
+	/**
+	 * Commit under `PRAGMA synchronous = FULL`, which in write-ahead-log mode syncs the log file at
+	 * this commit. The log is one append-only file, so that sync covers every frame before it: every
+	 * earlier `NORMAL` commit on this database, whichever store or helper made it. The pragma is set
+	 * inside the mutex slot the transaction holds and restored to `NORMAL` after the commit, or after
+	 * the rollback, so no other write on the connection pays for the sync by accident. Default false.
+	 */
+	readonly synced?: boolean;
+}
+
 /**
  * Minimal SQLite driver surface used by this package.
  *
@@ -70,6 +86,10 @@ export interface SqliteDb {
 	 * Run `fn` inside `BEGIN IMMEDIATE ... COMMIT` (rolls back on throw), holding
 	 * the connection mutex for the whole body. `fn` receives a `SqliteTransaction`
 	 * whose statements run directly on the open transaction without re-locking.
+	 * With `options.synced` the commit also syncs the write-ahead log, which makes
+	 * this transaction and every earlier commit on the database survive a power
+	 * cut (see {@link SqliteTransactionOptions.synced}); without it the commit
+	 * survives a crash of this process only, until a later synced commit.
 	 *
 	 * NOTE: `fn` must issue its writes through the provided `tx`, never through a
 	 * class-level statement from `db.prepare(...)` (those re-acquire the mutex and
@@ -77,11 +97,70 @@ export interface SqliteDb {
 	 * `db.transaction(...)` again — nested calls deadlock for the same reason. No
 	 * current caller does either; guard here if that ever changes.
 	 */
-	transaction<T>(fn: (tx: SqliteTransaction) => Promise<T>): Promise<T>;
+	transaction<T>(fn: (tx: SqliteTransaction) => Promise<T>, options?: SqliteTransactionOptions): Promise<T>;
 	/** Release the underlying handle. */
 	close(): Promise<void>;
 }
 
+/** The raw connection surface {@link runTransaction} drives: one statement at a time, no rows back. */
+export interface TransactionConnection {
+	execute(sql: string): Promise<void> | void;
+}
+
+/**
+ * The transaction protocol both `SqliteDb` implementations run inside their mutex slot: with
+ * `synced`, `PRAGMA synchronous = FULL` first; then `BEGIN IMMEDIATE`, the body, and `COMMIT`, or
+ * `ROLLBACK` when the body or the commit throws; then, synced, `PRAGMA synchronous = NORMAL`
+ * whichever way the transaction ended, so the connection's default is restored before the slot is
+ * released. `BEGIN IMMEDIATE` takes the write lock up front rather than deferring it to the first
+ * write, so contention surfaces here and not mid-body.
+ *
+ * A failure of the restore after a failed body is logged rather than thrown, so the body's error,
+ * the one that matters, is the one the caller sees; after a committed body it is thrown, since the
+ * connection would otherwise sync every later commit unasked.
+ */
+export async function runTransaction<T>(
+	conn: TransactionConnection,
+	tx: SqliteTransaction,
+	fn: (tx: SqliteTransaction) => Promise<T>,
+	options?: SqliteTransactionOptions,
+): Promise<T> {
+	const synced = options?.synced === true;
+	if (synced) await conn.execute('PRAGMA synchronous = FULL');
+	let result: T;
+	try {
+		result = await commitOrRollBack(conn, tx, fn);
+	} catch (err) {
+		if (synced) {
+			try {
+				await conn.execute('PRAGMA synchronous = NORMAL');
+			} catch (restoreErr) {
+				log('restoring synchronous = NORMAL failed after a failed transaction: %o', restoreErr);
+			}
+		}
+		throw err;
+	}
+	if (synced) await conn.execute('PRAGMA synchronous = NORMAL');
+	return result;
+}
+
+async function commitOrRollBack<T>(conn: TransactionConnection, tx: SqliteTransaction, fn: (tx: SqliteTransaction) => Promise<T>): Promise<T> {
+	await conn.execute('BEGIN IMMEDIATE');
+	try {
+		const result = await fn(tx);
+		await conn.execute('COMMIT');
+		return result;
+	} catch (err) {
+		try {
+			await conn.execute('ROLLBACK');
+		} catch (rollbackErr) {
+			// The original error is the one to surface; the rollback's is only logged.
+			log('rollback failed after a failed transaction: %o', rollbackErr);
+		}
+		throw err;
+	}
+}
+
 export const DEFAULT_DB_NAME = 'optimystic.sqlite';
 export const DEFAULT_DB_VERSION = 1;
 
@@ -90,11 +169,11 @@ export const DEFAULT_DB_VERSION = 1;
  *
  * Mirrors the IndexedDB object stores 1:1. The six block-storage stores keep
  * their original columns and keys, but their value columns are now BLOB: the
- * shared `KvRawStorage` kernel (driven by `SqliteStoreDriver`) owns all JSON /
- * UTF-8 (de)serialization, so the driver only ever binds/reads the kernel's raw
- * `Uint8Array` values. SQLite stores a `Uint8Array` as a BLOB and returns it as
- * a `Uint8Array`, so bytes round-trip exactly (a TEXT column would risk UTF-8
- * coercion corrupting non-ASCII JSON bytes). Keys stay TEXT/INTEGER.
+ * shared `KvRawStorage` kernel (over `WriteGroupDriver` over `SqliteBatchBackend`)
+ * owns all JSON / UTF-8 (de)serialization, so the backend only ever binds/reads
+ * the kernel's raw `Uint8Array` values. SQLite stores a `Uint8Array` as a BLOB
+ * and returns it as a `Uint8Array`, so bytes round-trip exactly (a TEXT column
+ * would risk UTF-8 coercion corrupting non-ASCII JSON bytes). Keys stay TEXT/INTEGER.
  * - `metadata` → per-block `BlockMetadata` bytes keyed by `block_id`.
  * - `revisions` → revision lookup keyed by `(block_id, rev)`; `action_id` column
  *   holds the kernel's encoded `ActionId` bytes (BLOB).
@@ -107,6 +186,12 @@ export const DEFAULT_DB_VERSION = 1;
  * - `kv` → generic string keyspace shared by `IKVStore` (`s_val`) and the
  *   identity helper (`b_val`). NOT kernel-backed; unchanged. The two columns
  *   avoid a base64 round-trip for the libp2p private key.
+ * - `durability_mark` → one row, bumped by `syncSqliteDb` (`sqlite-sync.ts`) so a
+ *   sync that has no rows of its own to write still has a frame to commit.
+ *
+ * NOTE: this package's SQL is uppercase throughout, predating the lowercase rule in
+ * AGENTS.md, and new statements have kept to it rather than mix cases within one file;
+ * convert the package in one pass rather than statement by statement.
  *
  * NOTE: value columns changed TEXT→BLOB with the kernel refactor, but every
  * statement is `CREATE TABLE IF NOT EXISTS` — so a database file created by a
@@ -163,6 +248,11 @@ CREATE TABLE IF NOT EXISTS kv (
 	s_val  TEXT,
 	b_val  BLOB
 );
+
+CREATE TABLE IF NOT EXISTS durability_mark (
+	id INTEGER PRIMARY KEY CHECK (id = 1),
+	n  INTEGER NOT NULL
+);
 `;
 
 // Assignment-form pragmas return no rows, so they go through exec()/execute().

package/src/identity.ts

@@ -1,12 +1,15 @@
 import { generateKeyPair, privateKeyFromProtobuf, privateKeyToProtobuf } from '@libp2p/crypto/keys';
 import type { PrivateKey } from '@libp2p/interface';
 import type { SqliteDb } from './db.js';
+import { syncSqliteDb } from './sqlite-sync.js';
 
 export const DEFAULT_PEER_KEY_NAME = 'peer-private-key';
 
 /**
  * Loads the persisted NativeScript peer's libp2p private key, or generates a
- * fresh Ed25519 key on first call and persists it.
+ * fresh Ed25519 key on first call, persists it, and syncs the database before
+ * returning it, so a power cut right after first start cannot give the node a
+ * different peer id next time. An existing key is read as it is.
  *
  * The key is stored as raw protobuf bytes in the `kv` table's `b_val` column
  * under `keyName`. SQLite stores BLOB natively, so there is no base64
@@ -30,5 +33,6 @@ export async function loadOrCreateNSPeerKey(
 	await db
 		.prepare('INSERT INTO kv (key, s_val, b_val) VALUES (?, NULL, ?) ON CONFLICT(key) DO UPDATE SET b_val = excluded.b_val')
 		.run(keyName, bytes);
+	await syncSqliteDb(db);
 	return key;
 }

package/src/index.ts

@@ -1,6 +1,8 @@
-export type { OptimysticNSDBHandle, SqliteDb, SqliteStatement, SqliteParam, SqliteRow } from './db.js';
+export type { OptimysticNSDBHandle, SqliteDb, SqliteStatement, SqliteTransaction, SqliteTransactionOptions, SqliteParam, SqliteRow } from './db.js';
 export { DEFAULT_DB_NAME, DEFAULT_DB_VERSION } from './db.js';
 export { openOptimysticNSDb, wrapNSPluginDb } from './ns-opener.js';
-export { SqliteRawStorage, SqliteStoreDriver } from './sqlite-storage.js';
+export { SqliteRawStorage, SqliteBatchBackend } from './sqlite-storage.js';
+export type { SqliteRawStorageOptions, SqliteBatchBackendOptions } from './sqlite-storage.js';
+export { syncSqliteDb } from './sqlite-sync.js';
 export { SqliteKVStore } from './sqlite-kv-store.js';
 export { loadOrCreateNSPeerKey, DEFAULT_PEER_KEY_NAME } from './identity.js';

package/src/ns-opener.ts

@@ -1,5 +1,5 @@
 import { ConnectionMutex } from './connection-mutex.js';
-import { applySchema, DEFAULT_DB_NAME, DEFAULT_DB_VERSION, type OptimysticNSDBHandle, type SqliteDb, type SqliteParam, type SqliteRow, type SqliteStatement, type SqliteTransaction } from './db.js';
+import { applySchema, runTransaction, DEFAULT_DB_NAME, DEFAULT_DB_VERSION, type OptimysticNSDBHandle, type SqliteDb, type SqliteParam, type SqliteRow, type SqliteStatement, type SqliteTransaction, type SqliteTransactionOptions } from './db.js';
 
 /**
  * Minimal subset of `@nativescript-community/sqlite`'s `Db` we depend on.
@@ -86,29 +86,15 @@ class NSPluginDbWrapper implements SqliteDb {
 		return new NSPluginStatement(this.raw, sql, this.mutex);
 	}
 
-	async transaction<T>(fn: (tx: SqliteTransaction) => Promise<T>): Promise<T> {
-		return this.mutex.serialize(async () => {
-			// BEGIN IMMEDIATE takes the write lock up front rather than deferring
-			// it to the first write, so contention surfaces here and not mid-body.
-			await this.raw.execute('BEGIN IMMEDIATE');
-			try {
-				// Statements bound to the open transaction bypass the mutex — we
-				// already hold the slot; re-locking would deadlock.
-				const tx: SqliteTransaction = {
-					prepare: (sql: string) => new NSPluginStatement(this.raw, sql),
-				};
-				const result = await fn(tx);
-				await this.raw.execute('COMMIT');
-				return result;
-			} catch (err) {
-				try {
-					await this.raw.execute('ROLLBACK');
-				} catch {
-					// Swallow rollback failures so we surface the original error.
-				}
-				throw err;
-			}
-		});
+	async transaction<T>(fn: (tx: SqliteTransaction) => Promise<T>, options?: SqliteTransactionOptions): Promise<T> {
+		return this.mutex.serialize(() => runTransaction(
+			{ execute: async (sql) => { await this.raw.execute(sql); } },
+			// Statements bound to the open transaction bypass the mutex — we
+			// already hold the slot; re-locking would deadlock.
+			{ prepare: (sql: string) => new NSPluginStatement(this.raw, sql) },
+			fn,
+			options,
+		));
 	}
 
 	async close(): Promise<void> {
@@ -141,11 +127,16 @@ class NSPluginStatement implements SqliteStatement {
 		// NOTE: reads run directly on the connection, unserialized, to preserve read
 		// concurrency. A read issued while a write transaction is open on this same
 		// connection observes that transaction's UNCOMMITTED rows (read-your-connection
-		// semantics). Fine today: the only transaction writer is same-block promote
-		// under the block's write latch, and cross-block reads are independent. If a future
-		// caller reads a block on this connection while another op's transaction on the
-		// same rows is mid-flight, it may see uncommitted state — serialize reads too if
-		// that ever matters.
+		// semantics). Fine today: the one transaction writer of the block tables is
+		// `SqliteBatchBackend.commit`, and every row it is writing is held by the
+		// `WriteGroupDriver` overlay above it until the batch is durable. A point read of
+		// such a row is answered by the overlay and never reaches this connection; a
+		// listing does reach it, and may see the batch's uncommitted rows, but merges the
+		// overlay over them, so it answers the same whether the transaction has committed
+		// or not (and a rolled-back batch latches the store). The key-value store and the
+		// identity helper write other tables, by autocommit statements, and the sync's
+		// `durability_mark` row is read by nobody. If a reader of a batch's rows ever
+		// bypasses the overlay, serialize reads too.
 		const row = await this.raw.get(this.sql, params);
 		if (row === null || row === undefined) return undefined;
 		return row as SqliteRow;

package/src/sqlite-kv-store.ts

@@ -1,5 +1,6 @@
-import type { IKVStore } from '@optimystic/db-p2p';
+import { StoreFailedError, type IKVStore } from '@optimystic/db-p2p';
 import type { SqliteDb, SqliteStatement } from './db.js';
+import { syncSqliteDb } from './sqlite-sync.js';
 
 /**
  * SQLite-backed `IKVStore` adapter for NativeScript peers.
@@ -12,6 +13,13 @@ import type { SqliteDb, SqliteStatement } from './db.js';
  *
  * Only the `s_val` column is read or written here; the `b_val` column is
  * reserved for binary identity material.
+ *
+ * Writes are autocommit statements: each is visible, and survives a crash of this process, once it
+ * resolves, and the write-ahead log keeps them in order. `flush()` makes them survive a power cut:
+ * when anything was written since the last flush it runs `syncSqliteDb`, one synced commit that
+ * syncs the shared log and with it every earlier write on the connection; otherwise it resolves at
+ * once. A failed sync latches the store, after which every write and flush throws
+ * `StoreFailedError` (rule 5 of the flush contract on `RawStoreDriver.flush`); reads keep working.
  */
 export class SqliteKVStore implements IKVStore {
 	private readonly stmts: {
@@ -21,8 +29,11 @@ export class SqliteKVStore implements IKVStore {
 		list: SqliteStatement;
 	};
 	private readonly prefix: string;
+	/** Set once a write has resolved, cleared by the flush that syncs it. */
+	private dirty = false;
+	private fault: { cause: unknown } | undefined;
 
-	constructor(db: SqliteDb, prefix: string = 'optimystic:txn:') {
+	constructor(private readonly db: SqliteDb, prefix: string = 'optimystic:txn:') {
 		this.prefix = prefix;
 		this.stmts = {
 			get: db.prepare('SELECT s_val FROM kv WHERE key = ?'),
@@ -40,11 +51,15 @@ export class SqliteKVStore implements IKVStore {
 	}
 
 	async set(key: string, value: string): Promise<void> {
+		this.throwIfFailed();
 		await this.stmts.set.run(this.prefix + key, value);
+		this.dirty = true;
 	}
 
 	async delete(key: string): Promise<void> {
+		this.throwIfFailed();
 		await this.stmts.delete.run(this.prefix + key);
+		this.dirty = true;
 	}
 
 	async list(prefix: string): Promise<string[]> {
@@ -58,4 +73,23 @@ export class SqliteKVStore implements IKVStore {
 		const rows = await this.stmts.list.all(fullPrefix, upper);
 		return rows.map(row => (row.key as string).slice(this.prefix.length));
 	}
+
+	/** `IKVStore.flush`: one synced commit when this store wrote since its last flush, otherwise nothing. */
+	async flush(): Promise<void> {
+		this.throwIfFailed();
+		if (!this.dirty) return;
+		// Cleared before the sync, so a write resolving while the sync runs is covered by the next
+		// flush whether or not this sync's commit happened to come after it.
+		this.dirty = false;
+		try {
+			await syncSqliteDb(this.db);
+		} catch (err) {
+			this.fault ??= { cause: err };
+			throw new StoreFailedError(this.fault.cause);
+		}
+	}
+
+	private throwIfFailed(): void {
+		if (this.fault) throw new StoreFailedError(this.fault.cause);
+	}
 }