@verdant-web/store 3.12.0 → 4.0.0-next.0

package/src/persistence/idb/idbPersistence.ts

@@ -0,0 +1,116 @@
+import { Context, InitialContext } from '../../context/context.js';
+import { PersistenceImplementation, PersistenceFileDb } from '../interfaces.js';
+import { IdbPersistenceFileDb } from './files/IdbPersistenceFileDb.js';
+import { IdbMetadataDb } from './metadata/IdbMetadataDb.js';
+import { openMetadataDatabase } from './metadata/openMetadataDatabase.js';
+import { IdbQueryDb } from './queries/IdbQueryDb.js';
+import { openQueryDatabase } from './queries/migration/openQueryDatabase.js';
+import { PersistenceMetadata } from '../PersistenceMetadata.js';
+import {
+	closeDatabase,
+	deleteDatabase,
+	getMetadataDbName,
+	getNamespaceFromDatabaseInfo,
+	getSizesOfAllObjectStores,
+} from './util.js';
+import { getDatabaseVersion } from './queries/migration/db.js';
+
+export class IdbPersistence implements PersistenceImplementation {
+	private metadataDb: IDBDatabase | undefined;
+	constructor(private indexedDB: IDBFactory = window.indexedDB) {}
+
+	getNamespaces = async (): Promise<string[]> => {
+		// list all idb database names
+		const dbs = await this.indexedDB.databases();
+		return Array.from(
+			new Set<string>(
+				dbs.map(getNamespaceFromDatabaseInfo).filter((n): n is string => !!n),
+			),
+		);
+	};
+
+	getNamespaceVersion = async (namespace: string): Promise<number> => {
+		return getDatabaseVersion(this.indexedDB, namespace);
+	};
+
+	deleteNamespace = async (
+		namespace: string,
+		ctx: InitialContext,
+	): Promise<void> => {
+		await Promise.all([
+			deleteDatabase(getMetadataDbName(namespace), this.indexedDB),
+			deleteDatabase([namespace, 'collections'].join('_'), this.indexedDB),
+		]);
+	};
+
+	openFiles(
+		ctx: Omit<Context, 'files' | 'queries'>,
+	): Promise<PersistenceFileDb> {
+		if (!this.metadataDb) {
+			throw new Error(
+				'Metadata database must be opened first. This is a bug in Verdant.',
+			);
+		}
+		return Promise.resolve(new IdbPersistenceFileDb(this.metadataDb, ctx));
+	}
+
+	openMetadata = async (ctx: InitialContext) => {
+		const { db } = await openMetadataDatabase({
+			indexedDB: this.indexedDB,
+			log: ctx.log,
+			namespace: ctx.namespace,
+		});
+		this.metadataDb = db;
+		return new IdbMetadataDb(db, ctx);
+	};
+
+	openQueries = async (ctx: InitialContext & { meta: PersistenceMetadata }) => {
+		const db = await openQueryDatabase({
+			version: ctx.schema.version,
+			indexedDB: this.indexedDB,
+			migrations: ctx.migrations,
+			context: ctx,
+		});
+		return new IdbQueryDb(db, ctx);
+	};
+
+	copyNamespace = async (
+		from: string,
+		to: string,
+		ctx: InitialContext,
+	): Promise<void> => {
+		const fromCtx = { ...ctx, namespace: from, originalNamespace: from };
+		const fromMetaDb = await this.openMetadata(fromCtx);
+		const fromMeta = new PersistenceMetadata(fromMetaDb, fromCtx);
+		const fromQueries = await this.openQueries({ ...fromCtx, meta: fromMeta });
+		ctx.log('info', `Copying data from ${from} to ${to}`);
+
+		const { db: toMetaDb } = await openMetadataDatabase({
+			indexedDB: this.indexedDB,
+			log: ctx.log,
+			namespace: to,
+		});
+		ctx.log('debug', 'Metadata database opened');
+		await fromMetaDb.cloneTo(toMetaDb);
+		ctx.log('debug', 'Metadata copied');
+
+		const toQueryDb = await openQueryDatabase({
+			version: ctx.schema.version,
+			indexedDB: this.indexedDB,
+			migrations: ctx.migrations,
+			context: { ...ctx, namespace: to, originalNamespace: to, meta: fromMeta },
+		});
+		await fromQueries.cloneTo(toQueryDb);
+		ctx.log('debug', 'Indexes copied');
+		ctx.log(
+			'debug',
+			'New document store stats:',
+			await getSizesOfAllObjectStores(toQueryDb),
+		);
+
+		await fromMetaDb.dispose();
+		await closeDatabase(toMetaDb);
+		await fromQueries.dispose();
+		await closeDatabase(toQueryDb);
+	};
+}

package/src/persistence/idb/metadata/IdbMetadataDb.ts

@@ -0,0 +1,460 @@
+import {
+	createCompoundIndexValue,
+	createLowerBoundIndexValue,
+	createUpperBoundIndexValue,
+	DocumentBaseline,
+	getLegacyDotOidSubIdRange,
+	getOidRoot,
+	getOidSubIdRange,
+	ObjectIdentifier,
+} from '@verdant-web/common';
+import {
+	AbstractTransaction,
+	AckInfo,
+	ClientOperation,
+	CommonQueryOptions,
+	Iterator,
+	LocalReplicaInfo,
+	PersistenceMetadataDb,
+} from '../../interfaces.js';
+import { IdbService } from '../IdbService.js';
+import cuid from 'cuid';
+import { closeDatabase, getSizeOfObjectStore } from '../util.js';
+import { Context } from '../../../context/context.js';
+
+export type StoredClientOperation = ClientOperation & {
+	/** This acts as the primary key */
+	oid_timestamp: string;
+	l_t: string;
+	d_t: string;
+};
+
+export type StoredSchema = {
+	type: 'schema';
+	schema: string;
+};
+
+export class IdbMetadataDb extends IdbService implements PersistenceMetadataDb {
+	constructor(
+		db: IDBDatabase,
+		private ctx: Pick<Context, 'log' | 'namespace'>,
+	) {
+		super(db, ctx);
+		this.addDispose(() => closeDatabase(db));
+	}
+
+	transaction = (opts: {
+		mode?: 'readwrite' | 'readonly';
+		storeNames: string[];
+		abort?: AbortSignal;
+	}) => {
+		return this.createTransaction(opts.storeNames, {
+			mode: opts.mode,
+			abort: opts.abort,
+		});
+	};
+
+	getAckInfo = async (): Promise<AckInfo> => {
+		const result = await this.run<AckInfo>('info', (store) => store.get('ack'));
+		if (result) {
+			return result;
+		} else {
+			return {
+				globalAckTimestamp: null,
+				type: 'ack',
+			};
+		}
+	};
+
+	setGlobalAck = async (ack: string): Promise<void> => {
+		await this.run(
+			'info',
+			(store) => store.put({ type: 'ack', globalAckTimestamp: ack }),
+			{ mode: 'readwrite' },
+		);
+	};
+
+	private _creatingLocalReplica: Promise<void> | undefined;
+	private cachedLocalReplica: LocalReplicaInfo | undefined;
+
+	getLocalReplica = async (
+		opts?: CommonQueryOptions,
+	): Promise<LocalReplicaInfo> => {
+		if (this.cachedLocalReplica) {
+			return this.cachedLocalReplica;
+		}
+
+		const lookup = await this.run<LocalReplicaInfo>(
+			'info',
+			(store) => store.get('localReplicaInfo'),
+			this.convertOpts(opts),
+		);
+
+		// not cached, not in db, create it
+		if (!lookup) {
+			// prevent a race condition if get() is called again while we are
+			// creating the replica info
+			if (!this._creatingLocalReplica) {
+				this._creatingLocalReplica = (async () => {
+					// create our own replica info now
+					const replicaId = cuid();
+					const replicaInfo: LocalReplicaInfo = {
+						type: 'localReplicaInfo',
+						id: replicaId,
+						userId: undefined,
+						ackedLogicalTime: null,
+						lastSyncedLogicalTime: null,
+					};
+					await this.run('info', (store) => store.put(replicaInfo), {
+						mode: 'readwrite',
+					});
+					this.cachedLocalReplica = replicaInfo;
+				})();
+			}
+			await this._creatingLocalReplica;
+
+			return this.getLocalReplica(opts);
+		}
+
+		this.cachedLocalReplica = lookup;
+		return lookup;
+	};
+
+	updateLocalReplica = async (
+		data: Partial<LocalReplicaInfo>,
+		opts: CommonQueryOptions = writeOpts,
+	): Promise<void> => {
+		const localReplicaInfo = await this.getLocalReplica(opts);
+		Object.assign(localReplicaInfo, data);
+		await this.run('info', (store) => store.put(localReplicaInfo), {
+			mode: 'readwrite',
+		});
+		this.cachedLocalReplica = localReplicaInfo;
+	};
+
+	iterateDocumentBaselines = async (
+		rootOid: string,
+		iterator: (baseline: DocumentBaseline) => void,
+		opts?: CommonQueryOptions,
+	): Promise<void> => {
+		await this.iterate(
+			'baselines',
+			(store) => {
+				const root = getOidRoot(rootOid);
+				const [start, end] = getOidSubIdRange(rootOid);
+				// FIXME: get rid of legacy dot OIDs...
+				const [dotStart, dotEnd] = getLegacyDotOidSubIdRange(rootOid);
+				return [
+					store.openCursor(IDBKeyRange.only(root)),
+					store.openCursor(IDBKeyRange.bound(start, end, false, false)),
+					store.openCursor(IDBKeyRange.bound(dotStart, dotEnd, false, false)),
+				];
+			},
+			iterator,
+			this.convertOpts(opts),
+		);
+	};
+
+	iterateCollectionBaselines = async (
+		collection: string,
+		iterator: (baseline: DocumentBaseline) => void,
+		opts?: CommonQueryOptions,
+	): Promise<void> => {
+		await this.iterate(
+			'baselines',
+			(store) => {
+				return [
+					store.openCursor(
+						IDBKeyRange.bound(collection, collection + '\uffff', false, false),
+					),
+				];
+			},
+			iterator,
+			this.convertOpts(opts),
+		);
+	};
+
+	iterateAllBaselines = async (
+		iterator: Iterator<DocumentBaseline>,
+		opts: CommonQueryOptions,
+	): Promise<void> => {
+		await this.iterate(
+			'baselines',
+			(store) => store.index('timestamp').openCursor(),
+			iterator,
+			this.convertOpts(opts),
+		);
+	};
+
+	getBaseline = (
+		oid: string,
+		opts?: CommonQueryOptions,
+	): Promise<DocumentBaseline> => {
+		return this.run<DocumentBaseline>(
+			'baselines',
+			(store) => store.get(oid),
+			this.convertOpts(opts),
+		);
+	};
+
+	setBaselines = async (
+		baselines: DocumentBaseline[],
+		opts: CommonQueryOptions = writeOpts,
+	): Promise<void> => {
+		await this.runAll<any>(
+			'baselines',
+			(store) => baselines.map((b) => store.put(b)),
+			this.convertOpts(opts),
+		);
+	};
+
+	deleteBaseline = async (
+		oid: string,
+		opts: CommonQueryOptions = writeOpts,
+	): Promise<void> => {
+		await this.run(
+			'baselines',
+			(store) => store.delete(oid),
+			this.convertOpts(opts),
+		);
+	};
+
+	iterateDocumentOperations = (
+		rootOid: string,
+		iterator: (op: StoredClientOperation) => void,
+		opts?: CommonQueryOptions & { to?: string | null },
+	): Promise<void> => {
+		return this.iterate<StoredClientOperation>(
+			'operations',
+			(store) => {
+				const index = store.index('d_t');
+				const start = createLowerBoundIndexValue(rootOid);
+				const end = opts?.to
+					? createCompoundIndexValue(rootOid, opts.to)
+					: createUpperBoundIndexValue(rootOid);
+
+				const range = IDBKeyRange.bound(start, end, false, false);
+				return index.openCursor(range);
+			},
+			iterator,
+			this.convertOpts(opts),
+		);
+	};
+
+	iterateEntityOperations = (
+		oid: string,
+		iterator: (op: StoredClientOperation) => void,
+		opts?: CommonQueryOptions & { to?: string | null },
+	): Promise<void> => {
+		// NOTE: this is simplified from original impl.
+		// perhaps I missed some nuance as to why it was
+		// so complex before?
+
+		return this.iterate<StoredClientOperation>(
+			'operations',
+			(store) => {
+				const start = createLowerBoundIndexValue(oid);
+				const end = opts?.to
+					? createCompoundIndexValue(oid, opts.to)
+					: createUpperBoundIndexValue(oid);
+
+				const range = IDBKeyRange.bound(start, end, false, false);
+				return store.openCursor(range);
+			},
+			iterator,
+			this.convertOpts(opts),
+		);
+	};
+
+	consumeEntityOperations = (
+		oid: string,
+		iterator: Iterator<ClientOperation>,
+		opts: CommonQueryOptions & { to?: string | null } = writeOpts,
+	): Promise<void> => {
+		return this.iterate<StoredClientOperation>(
+			'operations',
+			(store) => {
+				const start = createLowerBoundIndexValue(oid);
+				const end = opts?.to
+					? createCompoundIndexValue(oid, opts.to)
+					: createUpperBoundIndexValue(oid);
+
+				const range = IDBKeyRange.bound(start, end, false, false);
+				return store.openCursor(range);
+			},
+			(op, store) => {
+				iterator(op);
+				store.delete(op.oid_timestamp);
+			},
+			this.convertOpts(opts),
+		);
+	};
+
+	iterateCollectionOperations = (
+		collection: string,
+		iterator: (op: StoredClientOperation) => void,
+		opts?: CommonQueryOptions,
+	): Promise<void> => {
+		return this.iterate(
+			'operations',
+			(store) => {
+				return store.openCursor(
+					IDBKeyRange.bound(collection, collection + '\uffff', false, false),
+					'next',
+				);
+			},
+			iterator,
+			this.convertOpts(opts),
+		);
+	};
+
+	iterateLocalOperations = (
+		iterator: (op: StoredClientOperation) => void,
+		opts?: CommonQueryOptions & {
+			after?: string | null;
+		},
+	): Promise<void> => {
+		return this.iterate(
+			'operations',
+			(store) => {
+				const start = opts?.after
+					? createCompoundIndexValue(true, opts.after)
+					: createLowerBoundIndexValue(true);
+				const end = createUpperBoundIndexValue(true);
+				const index = store.index('l_t');
+				return index.openCursor(
+					// NOTE: differs from original impl -- last arg is 'false' instead of 'true'
+					IDBKeyRange.bound(start, end, !!opts?.after, false),
+					'next',
+				);
+			},
+			iterator,
+		);
+	};
+
+	iterateAllOperations = (
+		iterator: (op: StoredClientOperation) => void,
+		opts?: CommonQueryOptions & {
+			before?: string | null;
+			from?: string | null;
+		},
+	): Promise<void> => {
+		return this.iterate(
+			'operations',
+			(store) => {
+				const start = opts?.from
+					? createLowerBoundIndexValue(opts.from)
+					: undefined;
+				const end = opts?.before
+					? createUpperBoundIndexValue(opts.before)
+					: createLowerBoundIndexValue(true);
+				const range =
+					start && end
+						? IDBKeyRange.bound(start, end, false, true)
+						: start
+						? IDBKeyRange.lowerBound(start, false)
+						: end
+						? IDBKeyRange.upperBound(end, true)
+						: undefined;
+				return store.index('timestamp').openCursor(range, 'next');
+			},
+			iterator,
+			this.convertOpts(opts),
+		);
+	};
+
+	addOperations = async (
+		ops: StoredClientOperation[],
+		opts: CommonQueryOptions = writeOpts,
+	): Promise<ObjectIdentifier[]> => {
+		let affected = new Set<ObjectIdentifier>();
+		await this.runAll(
+			'operations',
+			(store) =>
+				ops.map((op) => {
+					affected.add(getOidRoot(op.oid));
+					return store.put(this.addOperationIndexes(op));
+				}),
+			this.convertOpts(opts),
+		);
+		return Array.from(affected);
+	};
+
+	reset = async ({
+		clearReplica,
+		transaction,
+	}: {
+		clearReplica?: boolean;
+		transaction?: AbstractTransaction;
+	} = {}): Promise<void> => {
+		const tx =
+			(transaction as IDBTransaction) ||
+			this.createTransaction(['info', 'operations', 'baselines'], {
+				mode: 'readwrite',
+			});
+		await Promise.all([
+			this.resetLocalReplica(tx, clearReplica),
+			this.resetBaselines(tx),
+			this.resetOperations(tx),
+		]);
+	};
+
+	stats = async (): Promise<{
+		operationsSize: { count: number; size: number };
+		baselinesSize: { count: number; size: number };
+	}> => {
+		const ops = await getSizeOfObjectStore(this.db, 'operations');
+		const baselines = await getSizeOfObjectStore(this.db, 'baselines');
+		return { operationsSize: ops, baselinesSize: baselines };
+	};
+
+	private resetLocalReplica = async (tx: IDBTransaction, clear = false) => {
+		if (clear) {
+			return this.run('info', (store) => store.delete('localReplicaInfo'), {
+				mode: 'readwrite',
+				transaction: tx,
+			});
+		}
+
+		const localInfo = await this.getLocalReplica();
+		localInfo.ackedLogicalTime = null;
+		localInfo.lastSyncedLogicalTime = null;
+		await this.run('info', (store) => store.put(localInfo), {
+			mode: 'readwrite',
+			transaction: tx,
+		});
+	};
+
+	private resetBaselines = async (tx: IDBTransaction) => {
+		return this.clear('baselines', tx);
+	};
+
+	private resetOperations = async (tx: IDBTransaction) => {
+		return this.clear('operations', tx);
+	};
+
+	private addOperationIndexes = (
+		op: ClientOperation,
+	): StoredClientOperation => {
+		return {
+			...op,
+			oid_timestamp: createCompoundIndexValue(op.oid, op.timestamp) as string,
+			l_t: createCompoundIndexValue(op.isLocal, op.timestamp) as string,
+			d_t: createCompoundIndexValue(getOidRoot(op.oid), op.timestamp) as string,
+		};
+	};
+
+	private convertOpts = (opts?: CommonQueryOptions) => {
+		if (opts?.transaction && !(opts.transaction instanceof IDBTransaction)) {
+			throw new Error(
+				`Invalid IndexedDB transaction. You cannot mix persistence providers. ${typeof opts.transaction}`,
+			);
+		}
+		return {
+			mode: opts?.mode,
+			transaction: opts?.transaction as IDBTransaction | undefined,
+		};
+	};
+}
+
+const writeOpts = { mode: 'readwrite' } as const;

package/src/{metadata → persistence/idb/metadata}/openMetadataDatabase.ts

@@ -1,22 +1,31 @@
 import { replaceLegacyOidsInObject } from '@verdant-web/common';
-import { closeDatabase, storeRequestPromise } from '../idb.js';
+import {
+	closeDatabase,
+	getMetadataDbName,
+	storeRequestPromise,
+} from '../util.js';
+import { Context } from '../../../context/context.js';
 
 const migrations = [version1, version2, version3, version4, version5, version6];
+export const CURRENT_METADATA_VERSION = migrations.length;
 
 export function openMetadataDatabase({
 	indexedDB = window.indexedDB,
 	namespace,
 	log,
-	metadataVersion = 5,
+	metadataVersion = CURRENT_METADATA_VERSION,
 }: {
 	indexedDB?: IDBFactory;
 	namespace: string;
-	log?: (...args: any[]) => void;
+	log?: Context['log'];
 	metadataVersion?: number;
 }): Promise<{ wasInitialized: boolean; db: IDBDatabase }> {
 	return new Promise<{ wasInitialized: boolean; db: IDBDatabase }>(
 		(resolve, reject) => {
-			const request = indexedDB.open(`${namespace}_meta`, metadataVersion);
+			const request = indexedDB.open(
+				getMetadataDbName(namespace),
+				metadataVersion,
+			);
 			let wasInitialized = false;
 			request.onupgradeneeded = async (event) => {
 				const db = request.result;
@@ -200,6 +209,10 @@ async function version3(db: IDBDatabase, tx: IDBTransaction) {
 	operations.createIndex('timestamp', 'timestamp');
 }
 
+/**
+ * 3 -> 4 changes:
+ * Add files store
+ */
 async function version4(db: IDBDatabase, tx: IDBTransaction) {
 	const files = db.createObjectStore('files', {
 		keyPath: 'id',
@@ -208,6 +221,10 @@ async function version4(db: IDBDatabase, tx: IDBTransaction) {
 	files.createIndex('deletedAt', 'deletedAt');
 }
 
+/**
+ * 4 -> 5 changes:
+ * replace legacy OIDs
+ */
 async function version5(db: IDBDatabase, tx: IDBTransaction) {
 	// rewrites all baselines and operations to replace legacy OIDs
 	// with new ones.