@optimystic/db-core 0.0.1

package/src/transform/helpers.ts

@@ -0,0 +1,117 @@
+import type { BlockId, BlockOperation, BlockOperations, BlockStore, IBlock, Transform, Transforms } from "../index.js";
+
+/** Mutates the given block with a copy of the given operation */
+export function applyOperation(block: IBlock, [entity, index, deleteCount, inserted]: BlockOperation) {
+	if (Array.isArray(inserted)) {
+		(block as unknown as any)[entity].splice(index, deleteCount, ...structuredClone(inserted));
+	} else {
+		(block as unknown as any)[entity] = structuredClone(inserted);
+	}
+}
+
+/** Mutates the given block with the given set of operations */
+export function applyOperations(block: IBlock, operations: BlockOperations) {
+	for (const op of operations) {
+		applyOperation(block, op);
+	}
+}
+
+/** Returns a copy of the block with the given operation applied */
+export function withOperation(block: IBlock, [entity, index, deleteCount, inserted]: BlockOperation) {
+	if (Array.isArray(inserted)) {
+		const source = (block as any)[entity];
+		return { ...block, [entity]: [...source.slice(0, index), ...structuredClone(inserted), ...source.slice(index + deleteCount)] };
+	} else {
+		return { ...block, [entity]: structuredClone(inserted) };
+	}
+}
+
+/** The set of distinct block ids affected by the transform */
+export function blockIdsForTransforms(transforms: Transforms | undefined) {
+	if (!transforms) return [];
+	const insertIds = Object.keys(transforms.inserts ?? {});
+	const updateIds = Object.keys(transforms.updates ?? {});
+	const deleteIds = transforms.deletes ?? [];
+	return [...new Set([...insertIds, ...updateIds, ...deleteIds])];
+}
+
+/** Returns an empty transform */
+export function emptyTransforms(): Transforms {
+	return { inserts: {}, updates: {}, deletes: [] };
+}
+
+export function copyTransforms(transform: Transforms): Transforms {
+	return { inserts: { ...transform.inserts }, updates: { ...transform.updates }, deletes: transform.deletes };
+}
+
+export function mergeTransforms(a: Transforms, b: Transforms): Transforms {
+	return {
+		inserts: { ...a.inserts, ...b.inserts },
+		updates: { ...a.updates, ...b.updates },
+		deletes: [...a.deletes, ...b.deletes]
+	};
+}
+
+export function isTransformsEmpty(transform: Transforms): boolean {
+	return Object.keys(transform.inserts).length === 0
+		&& Object.keys(transform.updates).length === 0
+		&& transform.deletes.length === 0;
+}
+
+export function concatTransforms(...transforms: Transforms[]): Transforms {
+	return transforms.reduce((acc, m) => mergeTransforms(acc, m), emptyTransforms());
+}
+
+
+export function transformForBlockId(transform: Transforms, blockId: BlockId): Transform {
+	return {
+		...(blockId in transform.inserts ? { insert: transform.inserts[blockId] } : {}),
+		...(blockId in transform.updates ? { updates: transform.updates[blockId] } : {}),
+		...(blockId in transform.deletes ? { delete: true } : {})
+	};
+}
+
+export function transformsFromTransform(transform: Transform, blockId: BlockId): Transforms {
+	return {
+		inserts: transform.insert ? { [blockId]: transform.insert } : {},
+		updates: transform.updates ? { [blockId]: transform.updates } : {},
+		deletes: transform.delete ? [blockId] : []
+	};
+}
+
+export function applyTransformToStore<T extends IBlock>(transform: Transforms, store: BlockStore<T>) {
+	for (const blockId of transform.deletes) {
+		store.delete(blockId);
+	}
+	for (const [, block] of Object.entries(transform.inserts)) {
+		store.insert(block as T);
+	}
+	for (const [blockId, operations] of Object.entries(transform.updates)) {
+		for (const op of operations) {
+			store.update(blockId, op);
+		}
+	}
+}
+
+/** Applies a transform to the given block */
+export function applyTransform(block: IBlock | undefined, transform: Transform): IBlock | undefined {
+	if (transform.insert) {
+		block = transform.insert;
+	}
+	if (block && transform.updates) {
+		applyOperations(block, transform.updates);
+	}
+	if (transform.delete) {
+		return undefined;
+	}
+	return block;
+}
+
+/** Concatenates a transform to the given transforms */
+export function concatTransform(transforms: Transforms, blockId: BlockId, transform: Transform): Transforms {
+	return {
+		inserts: { ...transforms.inserts, ...(transform.insert ? { [blockId]: transform.insert } : {}) },
+		updates: { ...transforms.updates, ...(transform.updates ? { [blockId]: transform.updates } : {}) },
+		deletes: [...transforms.deletes, ...(transform.delete ? [blockId] : [])]
+	};
+}

package/src/transform/index.ts

@@ -0,0 +1,5 @@
+export * from "./atomic.js";
+export * from "./cache-source.js";
+export * from "./helpers.js";
+export * from "./struct.js";
+export * from "./tracker.js";

package/src/transform/struct.ts

@@ -0,0 +1,22 @@
+import type { IBlock, BlockId, BlockOperations } from "../blocks/structs.js";
+
+// TODO: make each of these optional (assumes empty)
+
+/** A transform is a set of block mutations to be performed.
+ * If a block is present in more than one field, they are applied in order of: insert, update, delete. */
+export type Transforms = {
+	/** Inserted blocks by BlockId */
+	inserts: Record<BlockId, IBlock>;
+	/** Block update operations by BlockId */
+	updates: Record<BlockId, BlockOperations>;
+	/** Set of deleted BlockIds */
+	deletes: BlockId[];
+};
+
+/** A transform is a block-level mutation.
+ * If more than one field is set, they are applied in order of: insert, update, delete. */
+export type Transform = {
+	insert?: IBlock;
+	updates?: BlockOperations;
+	delete?: boolean;
+};

package/src/transform/tracker.ts

@@ -0,0 +1,70 @@
+import type { IBlock, BlockId, BlockStore as IBlockStore, BlockHeader, BlockOperation, BlockType, BlockSource as IBlockSource } from "../index.js";
+import { applyOperation, emptyTransforms, blockIdsForTransforms, ensured } from "../index.js";
+
+/** A block store that collects transformations, without applying them to the underlying source.
+ * Transformations are also applied to the retrieved blocks, making it seem like the source has been modified.
+ */
+export class Tracker<T extends IBlock> implements IBlockStore<T> {
+	constructor(
+		private readonly source: IBlockSource<T>,
+		/** The collected set of transformations to be applied. Treat as immutable */
+		public transforms = emptyTransforms(),
+	) { }
+
+	async tryGet(id: BlockId): Promise<T | undefined> {
+		const block = await this.source.tryGet(id);
+		if (block) {
+			const ops = this.transforms.updates[id] ?? [];
+			ops.forEach(op => applyOperation(block!, op));
+			if (this.transforms.deletes?.includes(id)) {
+				return undefined;
+			}
+		} else if (Object.hasOwn(this.transforms.inserts, id)) {
+			return structuredClone(this.transforms.inserts[id]) as T;
+		}
+
+		return block;
+	}
+
+	generateId(): BlockId {
+		return this.source.generateId();
+	}
+
+	createBlockHeader(type: BlockType, newId?: BlockId): BlockHeader {
+		return this.source.createBlockHeader(type, newId);
+	}
+
+	insert(block: T) {
+		this.transforms.inserts[block.header.id] = structuredClone(block);
+		this.transforms.deletes.splice(this.transforms.deletes.indexOf(block.header.id), 1);
+	}
+
+	update(blockId: BlockId, op: BlockOperation) {
+		const inserted = this.transforms.inserts[blockId];
+		if (inserted) {
+			applyOperation(inserted, op);
+		} else {
+			ensured(this.transforms.updates, blockId, () => []).push(structuredClone(op));
+		}
+	}
+
+	delete(blockId: BlockId) {
+		delete this.transforms.inserts[blockId];
+		delete this.transforms.updates[blockId];
+		this.transforms.deletes.push(blockId);
+	}
+
+	reset(newTransform = emptyTransforms()) {
+		const oldTransform = this.transforms;
+		this.transforms = newTransform;
+		return oldTransform;
+	}
+
+	transformedBlockIds(): BlockId[] {
+		return Array.from(new Set(blockIdsForTransforms(this.transforms)));
+	}
+
+	conflicts(blockIds: Set<BlockId>) {
+		return this.transformedBlockIds().filter(id => blockIds.has(id));
+	}
+}

package/src/utility/actor.ts

@@ -0,0 +1,62 @@
+type AsyncMethodKeys<T> = {
+  [K in keyof T]: T[K] extends (...args: any[]) => any ? K : never
+}[keyof T];
+
+type Actor<T> = {
+  [K in AsyncMethodKeys<T>]:
+    T[K] extends (...args: infer A) => infer R
+      ? (...args: A) => Promise<Awaited<R>>
+      : never;
+} & {
+  // pass through property reads
+  [K in Exclude<keyof T, AsyncMethodKeys<T>>]: T[K];
+};
+
+// makes an actor-like proxy around any object
+export function createActor<T extends object>(target: T): Actor<T> {
+  // a queue of calls waiting to run
+  const callQueue: Array<{
+    method: keyof T;
+    args: unknown[];
+    resolve: (val: unknown) => void;
+    reject: (err: unknown) => void;
+  }> = [];
+
+  let running = false;
+
+  async function runQueue() {
+    if (running) return;
+    running = true;
+
+    while (callQueue.length > 0) {
+      const { method, args, resolve, reject } = callQueue.shift()!;
+
+      try {
+        // call method on the original target
+        const result = (target[method] as any)(...args);
+        // if it’s a promise, await it; otherwise just pass it back
+        const awaited = result instanceof Promise ? await result : result;
+        resolve(awaited);
+      } catch (err) {
+        reject(err);
+      }
+    }
+
+    running = false;
+  }
+
+  return new Proxy(target, {
+    get(_obj, prop, _receiver) {
+      const value = (target as any)[prop];
+      // if it’s not a function, pass it through directly (optional)
+      if (typeof value !== 'function') return value;
+
+      // otherwise return a function that queues the call
+      return (...args: unknown[]) =>
+        new Promise((resolve, reject) => {
+          callQueue.push({ method: prop as keyof T, args, resolve, reject });
+          runQueue();
+        });
+    },
+  }) as Actor<T>;
+}

package/src/utility/batch-coordinator.ts

@@ -0,0 +1,174 @@
+import { type PeerId } from "@libp2p/interface";
+import type { BlockId } from "../index.js";
+import { Pending } from "./pending.js";
+
+/**
+ * Represents a batch of operations for a specific block coordinated by a peer
+ */
+export type CoordinatorBatch<TPayload, TResponse> = {
+	peerId: PeerId;
+	blockId: BlockId;
+	payload: TPayload;
+	request?: Pending<TResponse>;
+	/** Whether this batch has been subsumed by other successful batches */
+	subsumedBy?: CoordinatorBatch<TPayload, TResponse>[];
+	/** Peers that have already been tried (and failed) */
+	excludedPeers?: PeerId[];
+}
+
+/**
+ * Creates batches for a given payload, grouped by the coordinating peer for each block id
+ */
+export function makeBatchesByPeer<TPayload, TResponse>(
+	blockPeers: (readonly [BlockId, PeerId])[],
+	payload: TPayload,
+	getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
+	excludedPeers?: PeerId[]
+): CoordinatorBatch<TPayload, TResponse>[] {
+	const groups = blockPeers.reduce((acc, [blockId, peerId]) => {
+		const peerId_str = peerId.toString();
+		const coordinator = acc.get(peerId_str) ?? { peerId, blockId, excludedPeers } as Partial<CoordinatorBatch<TPayload, TResponse>>;
+		acc.set(peerId_str, { ...coordinator, payload: getBlockPayload(payload, blockId, coordinator.payload) } as CoordinatorBatch<TPayload, TResponse>);
+		return acc;
+	}, new Map<string, CoordinatorBatch<TPayload, TResponse>>());
+	return Array.from(groups.values());
+}
+
+/**
+ * Iterates over all batches that have not completed, whether subsumed or not
+ */
+export function* incompleteBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[]): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
+    const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
+    while (stack.length > 0) {
+        const batch = stack.pop()!;
+        if (!batch.request || !batch.request.isResponse) {
+            yield batch;
+        }
+        if (batch.subsumedBy && batch.subsumedBy.length) {
+            stack.push(...batch.subsumedBy);
+        }
+    }
+}
+
+/**
+ * Checks if all completed batches (ignoring failures) satisfy a predicate
+ */
+export function everyBatch<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): boolean {
+    // For each root batch require that SOME node in its retry tree satisfies the predicate.
+    // Use iterative DFS to avoid recursion depth and minimize allocations.
+    for (const root of batches) {
+        let found = false;
+        const stack: CoordinatorBatch<TPayload, TResponse>[] = [root];
+        while (stack.length > 0) {
+            const node = stack.pop()!;
+            if (predicate(node)) { found = true; break; }
+            if (node.subsumedBy && node.subsumedBy.length) {
+                for (let i = 0; i < node.subsumedBy.length; i++) stack.push(node.subsumedBy[i]!);
+            }
+        }
+        if (!found) return false;
+    }
+    return true;
+}
+
+/**
+ * Iterates over all batches that satisfy an optional predicate, whether subsumed or not
+ */
+export function* allBatches<TPayload, TResponse>(batches: CoordinatorBatch<TPayload, TResponse>[], predicate?: (batch: CoordinatorBatch<TPayload, TResponse>) => boolean): IterableIterator<CoordinatorBatch<TPayload, TResponse>> {
+    const stack: CoordinatorBatch<TPayload, TResponse>[] = [...batches];
+    while (stack.length > 0) {
+        const batch = stack.pop()!;
+        if (!predicate || predicate(batch)) {
+            yield batch;
+        }
+        if (batch.subsumedBy && batch.subsumedBy.length) {
+            stack.push(...batch.subsumedBy);
+        }
+    }
+}
+
+/**
+ * Returns a new blockId list payload with the given block id appended
+ */
+export function mergeBlocks(payload: BlockId[], blockId: BlockId, mergeWithPayload: BlockId[] | undefined): BlockId[] {
+	return [...(mergeWithPayload ?? []), blockId];
+}
+
+/**
+ * Processes a set of batches, retrying any failures until success or expiration
+ * @param batches - The batches to process - each represents a group of blocks centered on a coordinating peer
+ * @param process - The function to call for a given batch
+ * @param getBlockIds - The function to call to get the block ids for a given batch
+ * @param getBlockPayload - The function to call to get the payload given a parent payload and block id, and optionally merge with an existing payload
+ * @param expiration - The expiration time for the operation
+ * @param findCoordinator - The function to call to find a coordinator for a block id
+ */
+export async function processBatches<TPayload, TResponse>(
+	batches: CoordinatorBatch<TPayload, TResponse>[],
+	process: (batch: CoordinatorBatch<TPayload, TResponse>) => Promise<TResponse>,
+	getBlockIds: (batch: CoordinatorBatch<TPayload, TResponse>) => BlockId[],
+	getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
+	expiration: number,
+	findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
+): Promise<void> {
+    // Root-map ensures retries are recorded on the original batch to avoid deep trees
+    const rootOf = new WeakMap<CoordinatorBatch<TPayload, TResponse>, CoordinatorBatch<TPayload, TResponse>>();
+    for (const b of batches) rootOf.set(b, b);
+
+    // Process a set of batches concurrently and enqueue retries flatly onto the root's subsumedBy list
+    const processSet = async (set: CoordinatorBatch<TPayload, TResponse>[]) => {
+        await Promise.all(set.map(async (batch) => {
+            batch.request = new Pending(process(batch)
+                .catch(async e => {
+                    if (expiration > Date.now()) {
+                        const excludedPeers = [batch.peerId, ...(batch.excludedPeers ?? [])];
+                        const retries = await createBatchesForPayload<TPayload, TResponse>(
+                            getBlockIds(batch),
+                            batch.payload,
+                            getBlockPayload,
+                            excludedPeers,
+                            findCoordinator
+                        );
+                        if (retries.length > 0 && expiration > Date.now()) {
+                            const root = rootOf.get(batch) ?? batch;
+                            root.subsumedBy = [...(root.subsumedBy ?? []), ...retries];
+                            for (const r of retries) rootOf.set(r, root);
+                            // Process retries, but ensure further failures also attach to the same root
+                            await processSet(retries);
+                        }
+                    }
+                    throw e;
+                }));
+        }));
+
+        // Wait for all in this set to settle
+        await Promise.all(set.map(b => b.request?.result().catch(() => { /* ignore */ })));
+    };
+
+    await processSet(batches);
+}
+
+/**
+ * Creates batches for a given payload, grouped by the coordinating peer for each block id
+ * This is a placeholder function that will be implemented by the caller
+ */
+export async function createBatchesForPayload<TPayload, TResponse>(
+	blockIds: BlockId[],
+	payload: TPayload,
+	getBlockPayload: (payload: TPayload, blockId: BlockId, mergeWithPayload: TPayload | undefined) => TPayload,
+	excludedPeers: PeerId[],
+	findCoordinator: (blockId: BlockId, options: { excludedPeers: PeerId[] }) => Promise<PeerId>
+): Promise<CoordinatorBatch<TPayload, TResponse>[]> {
+	// Group by block id
+	const distinctBlockIds = new Set(blockIds);
+
+	// Find coordinator for each key
+	const blockIdPeerId = await Promise.all(
+		Array.from(distinctBlockIds).map(async (bid) =>
+			[bid, await findCoordinator(bid, { excludedPeers })] as const
+		)
+	);
+
+	// Group blocks around their coordinating peers
+	return makeBatchesByPeer<TPayload, TResponse>(blockIdPeerId, payload, getBlockPayload, excludedPeers);
+}

package/src/utility/block-id-to-bytes.ts

@@ -0,0 +1,8 @@
+import { sha256 } from 'multiformats/hashes/sha2'
+import type { BlockId } from '../index.js'
+
+export async function blockIdToBytes(blockId: BlockId): Promise<Uint8Array> {
+    const input = new TextEncoder().encode(blockId)
+    const mh = await sha256.digest(input)
+    return mh.digest
+}

package/src/utility/ensured.ts

@@ -0,0 +1,32 @@
+// Retrieves a value from a record, generating an entry if none exists
+export function ensured<K extends string | number | symbol, V>(
+	map: Record<K, V>,
+	key: K,
+	makeNew: () => Exclude<V, undefined>,
+	existing?: (existing: Exclude<V, undefined>) => void
+): Exclude<V, undefined> {
+	let v = map[key];
+	if (typeof v === 'undefined') {
+		v = makeNew();
+		map[key] = v;
+	} else if (existing) {
+		existing(v as Exclude<V, undefined>);
+	}
+	return v as Exclude<V, undefined>;
+}
+
+export function ensuredMap<K extends string | number | symbol, V>(
+	map: Map<K, V>,
+	key: K,
+	makeNew: () => Exclude<V, undefined>,
+	existing?: (existing: Exclude<V, undefined>) => void
+): Exclude<V, undefined> {
+	let v = map.get(key);
+	if (typeof v === 'undefined') {
+		v = makeNew();
+		map.set(key, v);
+	} else if (existing) {
+		existing(v as Exclude<V, undefined>);
+	}
+	return v as Exclude<V, undefined>;
+}

package/src/utility/groupby.ts

@@ -0,0 +1,18 @@
+// TODO: replace the usage of this with Object.groupBy in newer ES once more pervasive
+
+/**
+ * Groups an array of items by a key selector function.
+ * @param array - The array of items to group.
+ * @param keySelector - The function that selects the key for each item.
+ * @returns An object where each key is a unique value from the key selector function, and each value is an array of items that have that key.
+ */
+export function groupBy<T, K extends string | number | symbol>(
+	array: T[],
+	keySelector: (item: T) => K
+): Record<K, T[]> {
+	return array.reduce((acc, item) => {
+		const key = keySelector(item);
+		(acc[key] ??= []).push(item);
+		return acc;
+	}, {} as Record<K, T[]>);
+}

package/src/utility/is-record-empty.ts

@@ -0,0 +1,5 @@
+/** True if the given object has no keys.  This should not be used for classes or objects with proto fields. */
+export function isRecordEmpty<T>(record: Record<string, T>): boolean {
+	for (const key in record) return false;
+	return true;
+}

package/src/utility/latches.ts

@@ -0,0 +1,42 @@
+/** Lightweight implementation of a mutex lock queue. */
+export class Latches {
+    // Stores the promise representing the completion of the last queued operation for a key.
+    private static lockQueues = new Map<string, Promise<void>>();
+
+    /**
+     * Acquires a lock for the given key. Waits if another operation holds the lock.
+     * Returns a release function that must be called to release the lock.
+		 * WARNING: The key scope is global to the entire process, so follow the convention of using `ClassName.methodName:${id}` to avoid conflicts.
+     */
+    static async acquire(key: string): Promise<() => void> {
+        // Get the promise the current operation needs to wait for (if any)
+        const currentTail = this.lockQueues.get(key) ?? Promise.resolve();
+
+        let resolveNewTail: () => void;
+        // Create the promise that the *next* operation will wait for
+        const newTail = new Promise<void>(resolve => {
+            resolveNewTail = resolve;
+        });
+
+        // Immediately set the new promise as the tail for this key
+        this.lockQueues.set(key, newTail);
+
+        // Wait for the previous operation (if any) to complete
+        await currentTail;
+
+        // Lock acquired. Return the function to release *this* lock.
+        const release = () => {
+            // Signal that this operation is complete, allowing the next awaiter (if any)
+            resolveNewTail();
+
+            // Optimization: If this promise is still the current tail in the map,
+            // it means no other operation queued up behind this one while it was running.
+            // We can safely remove the entry from the map to prevent unbounded growth.
+            if (this.lockQueues.get(key) === newTail) {
+                this.lockQueues.delete(key);
+            }
+        };
+
+        return release;
+    }
+}

package/src/utility/nameof.ts

@@ -0,0 +1,7 @@
+/* eslint-disable no-redeclare, @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any */
+export function nameof<TObject>(obj: TObject, key: keyof TObject): string;
+export function nameof<TObject>(key: keyof TObject): string;
+export function nameof(key1: any, key2?: any): any {
+  return key2 ?? key1;
+}
+/* eslint-enable */

package/src/utility/pending.ts

@@ -0,0 +1,41 @@
+export class Pending<T> {
+	response?: T;
+	error?: unknown;
+	t1 = Date.now();
+	duration?: number;
+
+	get isResponse(): boolean {
+		return this.response !== undefined;
+	}
+
+	get isError(): boolean {
+		return this.error !== undefined;
+	}
+
+	get isComplete(): boolean {
+		return this.isResponse || this.isError;
+	}
+
+	async result(): Promise<T> {
+		if (this.isResponse) {
+			return this.response!;
+		}
+		if (this.isError) {
+			throw this.error!;
+		}
+		return await this.promise;
+	}
+
+	constructor(
+		public promise: Promise<T>
+	) {
+		promise.then(response => {
+			this.duration = Date.now() - this.t1;
+			this.response = response;
+			return response;
+		}, error => {
+			this.duration = Date.now() - this.t1;
+			this.error = error;
+		});
+	}
+}