@optimystic/db-p2p 0.0.1

package/src/storage/block-storage.ts

@@ -0,0 +1,182 @@
+import type { BlockId, IBlock, Transform, ActionId, ActionRev } from "@optimystic/db-core";
+import { Latches, applyTransform } from "@optimystic/db-core";
+import type { BlockArchive, BlockMetadata, RestoreCallback, RevisionRange } from "./struct.js";
+import type { IRawStorage } from "./i-raw-storage.js";
+import { mergeRanges } from "./helpers.js";
+import type { IBlockStorage } from "./i-block-storage.js";
+
+export class BlockStorage implements IBlockStorage {
+	constructor(
+		private readonly blockId: BlockId,
+		private readonly storage: IRawStorage,
+		private readonly restoreCallback?: RestoreCallback
+	) { }
+
+	async getLatest(): Promise<ActionRev | undefined> {
+		const meta = await this.storage.getMetadata(this.blockId);
+		return meta?.latest;
+	}
+
+	async getBlock(rev?: number): Promise<{ block: IBlock, actionRev: ActionRev } | undefined> {
+		const meta = await this.storage.getMetadata(this.blockId);
+		if (!meta) {
+			return undefined;
+		}
+
+		const targetRev = rev ?? meta.latest?.rev;
+		if (targetRev === undefined) {
+			throw new Error(`No revision specified and no latest revision exists for block ${this.blockId}`);
+		}
+
+		await this.ensureRevision(meta, targetRev);
+		return await this.materializeBlock(meta, targetRev);
+	}
+
+	async getTransaction(actionId: ActionId): Promise<Transform | undefined> {
+		return await this.storage.getTransaction(this.blockId, actionId);
+	}
+
+	async getPendingTransaction(actionId: ActionId): Promise<Transform | undefined> {
+		return await this.storage.getPendingTransaction(this.blockId, actionId);
+	}
+
+	async *listPendingTransactions(): AsyncIterable<ActionId> {
+		yield* this.storage.listPendingTransactions(this.blockId);
+	}
+
+	async savePendingTransaction(actionId: ActionId, transform: Transform): Promise<void> {
+		let meta = await this.storage.getMetadata(this.blockId);
+		if (!meta) {
+			meta = { latest: undefined, ranges: [[0]] };
+			await this.storage.saveMetadata(this.blockId, meta);
+		}
+		await this.storage.savePendingTransaction(this.blockId, actionId, transform);
+	}
+
+	async deletePendingTransaction(actionId: ActionId): Promise<void> {
+		await this.storage.deletePendingTransaction(this.blockId, actionId);
+	}
+
+	async *listRevisions(startRev: number, endRev: number): AsyncIterable<ActionRev> {
+		yield* this.storage.listRevisions(this.blockId, startRev, endRev);
+	}
+
+	async saveMaterializedBlock(actionId: ActionId, block: IBlock | undefined): Promise<void> {
+		await this.storage.saveMaterializedBlock(this.blockId, actionId, block);
+	}
+
+	async saveRevision(rev: number, actionId: ActionId): Promise<void> {
+		await this.storage.saveRevision(this.blockId, rev, actionId);
+	}
+
+	async promotePendingTransaction(actionId: ActionId): Promise<void> {
+		await this.storage.promotePendingTransaction(this.blockId, actionId);
+	}
+
+	async setLatest(latest: ActionRev): Promise<void> {
+		const meta = await this.storage.getMetadata(this.blockId);
+		if (!meta) {
+			throw new Error(`Block ${this.blockId} not found`);
+		}
+		meta.latest = latest;
+		await this.storage.saveMetadata(this.blockId, meta);
+	}
+
+	private async ensureRevision(meta: BlockMetadata, rev: number): Promise<void> {
+		if (this.inRanges(rev, meta.ranges)) {
+			return;
+		}
+
+		const lockId = `BlockStorage.ensureRevision:${this.blockId}`;
+		const release = await Latches.acquire(lockId);
+		try {
+			const currentMeta = await this.storage.getMetadata(this.blockId);
+			if (!currentMeta) {
+				throw new Error(`Block ${this.blockId} metadata disappeared unexpectedly.`);
+			}
+			if (this.inRanges(rev, currentMeta.ranges)) {
+				return;
+			}
+
+			const restored = await this.restoreBlock(rev);
+			if (!restored) {
+				throw new Error(`Block ${this.blockId} revision ${rev} not found during restore attempt.`);
+			}
+			await this.saveRestored(restored);
+
+			currentMeta.ranges.unshift(restored.range);
+			currentMeta.ranges = mergeRanges(currentMeta.ranges);
+			await this.storage.saveMetadata(this.blockId, currentMeta);
+
+		} finally {
+			release();
+		}
+	}
+
+	private async materializeBlock(_meta: BlockMetadata, targetRev: number): Promise<{ block: IBlock, actionRev: ActionRev }> {
+		let block: IBlock | undefined;
+		let materializedActionRev: ActionRev | undefined;
+		const actions: ActionRev[] = [];
+
+		// Find the materialized block
+		for await (const actionRev of this.storage.listRevisions(this.blockId, targetRev, 1)) {
+			const materializedBlock = await this.storage.getMaterializedBlock(this.blockId, actionRev.actionId);
+			if (materializedBlock) {
+				block = materializedBlock;
+				materializedActionRev = actionRev;
+				break;
+			} else {
+				actions.push(actionRev);
+			}
+		}
+
+		if (!block || !materializedActionRev) {
+			// There is an implicit requirement that there must be a materialization of the block somewhere in it's history.  If the log is truncated, a materialization must be made at the truncation point..
+			throw new Error(`Failed to find materialized block ${this.blockId} for revision ${targetRev}`);
+		}
+
+		// Apply transforms in reverse order
+		for (let i = actions.length - 1; i >= 0; --i) {
+			const { actionId } = actions[i]!;
+			const transform = await this.storage.getTransaction(this.blockId, actionId);
+			if (!transform) {
+				throw new Error(`Missing action ${actionId} for block ${this.blockId}`);
+			}
+			block = applyTransform(block, transform);
+		}
+
+		if (!block) {
+			throw new Error(`Block ${this.blockId} has been deleted`);
+		}
+		if (actions.length) {
+			await this.storage.saveMaterializedBlock(this.blockId, actions[0]!.actionId, block);
+			return { block, actionRev: actions[0]! };
+		}
+		return { block, actionRev: materializedActionRev };
+	}
+
+	private async restoreBlock(rev: number): Promise<BlockArchive | undefined> {
+		if (!this.restoreCallback) return undefined;
+		return await this.restoreCallback(this.blockId, rev);
+	}
+
+	private async saveRestored(archive: BlockArchive) {
+		const revisions = Object.entries(archive.revisions)
+			.map(([rev, data]) => ({ rev: Number(rev), data }));
+
+		// Save all revisions, actions, and materializations
+		for (const { rev, data: { action, block } } of revisions) {
+			await Promise.all([
+				this.storage.saveRevision(this.blockId, rev, action.actionId),
+				this.storage.saveTransaction(this.blockId, action.actionId, action.transform),
+				block ? this.storage.saveMaterializedBlock(this.blockId, action.actionId, block) : Promise.resolve()
+			]);
+		}
+	}
+
+	private inRanges(rev: number, ranges: RevisionRange[]): boolean {
+		return ranges.some(range =>
+			rev >= range[0] && (range[1] === undefined || rev < range[1])
+		);
+	}
+}

package/src/storage/file-storage.ts

@@ -0,0 +1,163 @@
+import { promises as fs } from 'fs';
+import * as path from 'path';
+import type { BlockId, IBlock, Transform, ActionId, ActionRev } from "@optimystic/db-core";
+import type { BlockMetadata } from "./struct.js";
+import type { IRawStorage } from "./i-raw-storage.js";
+import { createLogger } from '../logger.js'
+
+const log = createLogger('storage:file')
+
+export class FileRawStorage implements IRawStorage {
+	constructor(private readonly basePath: string) {
+		// TODO: use https://www.npmjs.com/package/proper-lockfile to take a lock on the basePath, also introduce explicit dispose pattern
+	 }
+
+	async getMetadata(blockId: BlockId): Promise<BlockMetadata | undefined> {
+		return this.readIfExists<BlockMetadata>(this.getMetadataPath(blockId));
+	}
+
+	async saveMetadata(blockId: BlockId, metadata: BlockMetadata): Promise<void> {
+		await this.ensureAndWriteFile(
+			this.getMetadataPath(blockId),
+			JSON.stringify(metadata)
+		);
+	}
+
+	async getRevision(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
+		return this.readIfExists<ActionId>(this.getRevisionPath(blockId, rev));
+	}
+
+	async saveRevision(blockId: BlockId, rev: number, actionId: ActionId): Promise<void> {
+		await this.ensureAndWriteFile(
+			this.getRevisionPath(blockId, rev),
+			actionId
+		);
+	}
+
+	async getPendingTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
+		return this.readIfExists<Transform>(this.getPendingActionPath(blockId, actionId));
+	}
+
+	async savePendingTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
+		await this.ensureAndWriteFile(
+			this.getPendingActionPath(blockId, actionId),
+			JSON.stringify(transform)
+		);
+	}
+
+	async deletePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
+		const pendingPath = this.getPendingActionPath(blockId, actionId);
+		await fs.unlink(pendingPath)
+			.catch((err) => {
+				// Ignore if file doesn't exist
+				if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') log('deletePendingTransaction unlink failed for %s/%s - %o', blockId, actionId, err)
+			});
+	}
+
+	async *listPendingTransactions(blockId: BlockId): AsyncIterable<ActionId> {
+		const pendingPath = path.join(this.getBlockPath(blockId), 'pend');
+
+		const files = await fs.readdir(pendingPath).catch((err) => { log('listPendingTransactions readdir failed for %s - %o', blockId, err); return [] as string[] });
+		for (const file of files) {
+			if (!file.endsWith('.json')) continue;
+			const rawActionId = file.slice(0, -5);
+			if (!/^[\w\d]+-[\w\d]+-[\w\d]+-[\w\d]+-[\w\d]+$/.test(rawActionId)) continue;
+			yield rawActionId as ActionId;
+		}
+	}
+
+	async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
+		return this.readIfExists<Transform>(this.getActionPath(blockId, actionId));
+	}
+
+	async *listRevisions(blockId: BlockId, startRev: number, endRev: number): AsyncIterable<ActionRev> {
+		// TODO: Optimize this for sparse revs
+		for (let rev = startRev; startRev <= endRev ? rev <= endRev : rev >= endRev; startRev <= endRev ? ++rev : --rev) {
+			const actionId = await this.getRevision(blockId, rev);
+			if (actionId) {
+				yield { actionId, rev };
+			}
+		}
+	}
+
+	async saveTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
+		await this.ensureAndWriteFile(
+			this.getActionPath(blockId, actionId),
+			JSON.stringify(transform)
+		);
+	}
+
+	async getMaterializedBlock(blockId: BlockId, actionId: ActionId): Promise<IBlock | undefined> {
+		return this.readIfExists<IBlock>(this.getMaterializedPath(blockId, actionId));
+	}
+
+	async saveMaterializedBlock(blockId: BlockId, actionId: ActionId, block?: IBlock): Promise<void> {
+		if (block) {
+			await this.ensureAndWriteFile(
+				this.getMaterializedPath(blockId, actionId),
+				JSON.stringify(block)
+			);
+		} else {
+			await fs.unlink(this.getMaterializedPath(blockId, actionId))
+				.catch((err) => {
+					// Ignore if file doesn't exist
+					if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') log('saveMaterializedBlock unlink failed for %s/%s - %o', blockId, actionId, err)
+				});
+		}
+	}
+
+	async promotePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
+		const pendingPath = this.getPendingActionPath(blockId, actionId);
+		const actionPath = this.getActionPath(blockId, actionId);
+
+		// Ensure target directory exists
+		await fs.mkdir(path.dirname(actionPath), { recursive: true });
+
+		return fs.rename(pendingPath, actionPath)
+			.catch(err => {
+				if (err.code === 'ENOENT') {
+					throw new Error(`Pending action ${actionId} not found for block ${blockId}`);
+				}
+				log('promotePendingTransaction rename failed for %s/%s - %o', blockId, actionId, err)
+				throw err;
+			});
+	}
+
+	private getBlockPath(blockId: BlockId): string {
+		return path.join(this.basePath, blockId);
+	}
+
+	private getMetadataPath(blockId: BlockId): string {
+		return path.join(this.getBlockPath(blockId), 'meta.json');
+	}
+
+	private getRevisionPath(blockId: BlockId, rev: number): string {
+		return path.join(this.getBlockPath(blockId), 'revs', `${rev}.json`);
+	}
+
+	private getPendingActionPath(blockId: BlockId, actionId: ActionId): string {
+		return path.join(this.getBlockPath(blockId), 'pend', `${actionId}.json`);
+	}
+
+	private getActionPath(blockId: BlockId, actionId: ActionId): string {
+		return path.join(this.getBlockPath(blockId), 'actions', `${actionId}.json`);
+	}
+
+	private getMaterializedPath(blockId: BlockId, actionId: ActionId): string {
+		return path.join(this.getBlockPath(blockId), 'blocks', `${actionId}.json`);
+	}
+
+	private async readIfExists<T>(filePath: string): Promise<T | undefined> {
+		return fs.readFile(filePath, 'utf-8')
+			.then(content => JSON.parse(content) as T)
+			.catch(err => {
+				if (err.code === 'ENOENT') return undefined;
+				throw err;
+			});
+	}
+
+	private async ensureAndWriteFile(filePath: string, content: string): Promise<void> {
+		await fs.mkdir(path.dirname(filePath), { recursive: true });
+		await fs.writeFile(filePath, content);
+	}
+}

package/src/storage/helpers.ts

@@ -0,0 +1,29 @@
+import type { RevisionRange } from "./struct.js";
+
+export function mergeRanges(ranges: RevisionRange[]): RevisionRange[] {
+    if (ranges.length <= 1) return ranges;
+
+    ranges.sort((a, b) => a[0] - b[0]);
+    const merged: RevisionRange[] = [ranges[0]!];
+
+    for (const range of ranges.slice(1)) {
+        const last = merged[merged.length - 1]!;
+        // If last range is open-ended, it consumes all following ranges
+        if (last[1] === undefined) {
+            continue;
+        }
+        // If this range starts at or before last range's end (exclusive)
+        if (range[0] <= last[1]) {
+            // If this range is open-ended, make last range open-ended
+            if (range[1] === undefined) {
+                last[1] = undefined;
+            } else {
+                last[1] = Math.max(last[1], range[1]);
+            }
+        } else {
+            merged.push(range);
+        }
+    }
+
+    return merged;
+}

package/src/storage/i-block-storage.ts

@@ -0,0 +1,40 @@
+import type { IBlock, Transform, ActionId, ActionRev } from "@optimystic/db-core";
+
+/** Interface for block-level storage operations */
+export interface IBlockStorage {
+    /** Gets the latest revision information for this block */
+    getLatest(): Promise<ActionRev | undefined>;
+
+    /** Gets a materialized block at the given revision */
+    getBlock(rev?: number): Promise<{ block: IBlock, actionRev: ActionRev } | undefined>;
+
+    /** Gets an action by ID */
+    getTransaction(actionId: ActionId): Promise<Transform | undefined>;
+
+    /** Gets a pending action by ID */
+    getPendingTransaction(actionId: ActionId): Promise<Transform | undefined>;
+
+    /** Lists all pending action IDs */
+    listPendingTransactions(): AsyncIterable<ActionId>;
+
+    /** Saves a pending action */
+    savePendingTransaction(actionId: ActionId, transform: Transform): Promise<void>;
+
+    /** Deletes a pending action */
+    deletePendingTransaction(actionId: ActionId): Promise<void>;
+
+    /** Lists revisions in ascending or descending order between startRev and endRev (inclusive) */
+    listRevisions(startRev: number, endRev: number): AsyncIterable<ActionRev>;
+
+    /** Saves a materialized block */
+    saveMaterializedBlock(actionId: ActionId, block: IBlock | undefined): Promise<void>;
+
+    /** Saves a revision */
+    saveRevision(rev: number, actionId: ActionId): Promise<void>;
+
+    /** Promotes a pending action to committed */
+    promotePendingTransaction(actionId: ActionId): Promise<void>;
+
+    /** Sets the latest revision information */
+    setLatest(latest: ActionRev): Promise<void>;
+}

package/src/storage/i-raw-storage.ts

@@ -0,0 +1,30 @@
+import type { BlockId, ActionId, ActionRev, Transform, IBlock } from "@optimystic/db-core";
+import type { BlockMetadata } from "./struct.js";
+
+export interface IRawStorage {
+	// Metadata operations
+	getMetadata(blockId: BlockId): Promise<BlockMetadata | undefined>;
+	saveMetadata(blockId: BlockId, metadata: BlockMetadata): Promise<void>;
+
+	// Revision operations
+	getRevision(blockId: BlockId, rev: number): Promise<ActionId | undefined>;
+	saveRevision(blockId: BlockId, rev: number, actionId: ActionId): Promise<void>;
+	/** List revisions in ascending or descending order, depending on startRev and endRev - startRev and endRev are inclusive */
+	listRevisions(blockId: BlockId, startRev: number, endRev: number): AsyncIterable<ActionRev>;
+
+	// Action operations
+	getPendingTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined>;
+	savePendingTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void>;
+	deletePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void>;
+	listPendingTransactions(blockId: BlockId): AsyncIterable<ActionId>;
+
+	getTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined>;
+	saveTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void>;
+
+	// Block materialization operations
+	getMaterializedBlock(blockId: BlockId, actionId: ActionId): Promise<IBlock | undefined>;
+	saveMaterializedBlock(blockId: BlockId, actionId: ActionId, block?: IBlock): Promise<void>;
+
+	// Promote a pending action to a committed action
+	promotePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void>;
+}

package/src/storage/memory-storage.ts

@@ -0,0 +1,108 @@
+import type { BlockId, IBlock, Transform, ActionId, ActionRev } from "@optimystic/db-core";
+import type { BlockMetadata } from "./struct.js";
+import type { IRawStorage } from "./i-raw-storage.js";
+
+export class MemoryRawStorage implements IRawStorage {
+	private metadata = new Map<BlockId, BlockMetadata>();
+	private revisions = new Map<string, ActionId>(); // blockId:rev -> actionId
+	private pendingActions = new Map<string, Transform>(); // blockId:actionId -> transform
+	private actions = new Map<string, Transform>(); // blockId:actionId -> transform
+	private materializedBlocks = new Map<string, IBlock>(); // blockId:actionId -> block
+
+	private getRevisionKey(blockId: BlockId, rev: number): string {
+		return `${blockId}:${rev}`;
+	}
+
+	private getActionKey(blockId: BlockId, actionId: ActionId): string {
+		return `${blockId}:${actionId}`;
+	}
+
+	async getMetadata(blockId: BlockId): Promise<BlockMetadata | undefined> {
+		return this.metadata.get(blockId);
+	}
+
+	async saveMetadata(blockId: BlockId, metadata: BlockMetadata): Promise<void> {
+		this.metadata.set(blockId, metadata);
+	}
+
+	async getRevision(blockId: BlockId, rev: number): Promise<ActionId | undefined> {
+		return this.revisions.get(this.getRevisionKey(blockId, rev));
+	}
+
+	async saveRevision(blockId: BlockId, rev: number, actionId: ActionId): Promise<void> {
+		this.revisions.set(this.getRevisionKey(blockId, rev), actionId);
+	}
+
+	async *listRevisions(blockId: BlockId, startRev: number, endRev: number): AsyncIterable<ActionRev> {
+		const ascending = startRev <= endRev;
+		const actualStart = ascending ? startRev : endRev;
+		const actualEnd = ascending ? endRev : startRev;
+
+		const results: ActionRev[] = [];
+		for (let rev = actualStart; rev <= actualEnd; rev++) {
+			const actionId = this.revisions.get(this.getRevisionKey(blockId, rev));
+			if (actionId) {
+				results.push({ rev, actionId });
+			}
+		}
+
+		if (!ascending) {
+			results.reverse();
+		}
+
+		for (const result of results) {
+			yield result;
+		}
+	}
+
+	async getPendingTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
+		return this.pendingActions.get(this.getActionKey(blockId, actionId));
+	}
+
+	async savePendingTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
+		this.pendingActions.set(this.getActionKey(blockId, actionId), transform);
+	}
+
+	async deletePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
+		this.pendingActions.delete(this.getActionKey(blockId, actionId));
+	}
+
+	async *listPendingTransactions(blockId: BlockId): AsyncIterable<ActionId> {
+		const prefix = `${blockId}:`;
+		for (const [key] of Array.from(this.pendingActions.entries())) {
+			if (key.startsWith(prefix)) {
+				yield key.substring(prefix.length);
+			}
+		}
+	}
+
+	async getTransaction(blockId: BlockId, actionId: ActionId): Promise<Transform | undefined> {
+		return this.actions.get(this.getActionKey(blockId, actionId));
+	}
+
+	async saveTransaction(blockId: BlockId, actionId: ActionId, transform: Transform): Promise<void> {
+		this.actions.set(this.getActionKey(blockId, actionId), transform);
+	}
+
+	async getMaterializedBlock(blockId: BlockId, actionId: ActionId): Promise<IBlock | undefined> {
+		return this.materializedBlocks.get(this.getActionKey(blockId, actionId));
+	}
+
+	async saveMaterializedBlock(blockId: BlockId, actionId: ActionId, block?: IBlock): Promise<void> {
+		const key = this.getActionKey(blockId, actionId);
+		if (block) {
+			this.materializedBlocks.set(key, block);
+		} else {
+			this.materializedBlocks.delete(key);
+		}
+	}
+
+	async promotePendingTransaction(blockId: BlockId, actionId: ActionId): Promise<void> {
+		const key = this.getActionKey(blockId, actionId);
+		const transform = this.pendingActions.get(key);
+		if (transform) {
+			this.actions.set(key, transform);
+			this.pendingActions.delete(key);
+		}
+	}
+}