@verdant-web/store 2.8.5 → 3.0.0-next.0

package/src/metadata/BaselinesStore.ts

@@ -7,10 +7,11 @@ import {
 	getLegacyDotOidSubIdRange,
 } from '@verdant-web/common';
 import { IDBService } from '../IDBService.js';
+import { Context } from '../context.js';
 
 export class BaselinesStore extends IDBService {
-	constructor(db: IDBDatabase) {
-		super(db);
+	constructor(db: IDBDatabase, opts: { log?: Context['log'] }) {
+		super(db, opts);
 	}
 
 	getAllForDocument = async (
@@ -61,8 +62,7 @@ export class BaselinesStore extends IDBService {
 				// }
 			},
 			iterator,
-			mode,
-			transaction,
+			{ mode, transaction },
 		);
 	};
 
@@ -84,8 +84,7 @@ export class BaselinesStore extends IDBService {
 				];
 			},
 			iterator,
-			mode,
-			transaction,
+			{ mode, transaction },
 		);
 	};
 
@@ -115,7 +114,7 @@ export class BaselinesStore extends IDBService {
 					// }
 				});
 			},
-			mode,
+			{ mode },
 		);
 		return result.flat();
 	};
@@ -133,7 +132,7 @@ export class BaselinesStore extends IDBService {
 				const index = store.index('timestamp');
 				return index.getAll(range);
 			},
-			mode,
+			{ mode },
 		);
 	};
 
@@ -144,24 +143,20 @@ export class BaselinesStore extends IDBService {
 			mode = 'readonly',
 		}: { transaction?: IDBTransaction; mode?: 'readwrite' | 'readonly' } = {},
 	) => {
-		return this.run<DocumentBaseline>(
-			'baselines',
-			(store) => store.get(oid),
+		return this.run<DocumentBaseline>('baselines', (store) => store.get(oid), {
 			mode,
 			transaction,
-		);
+		});
 	};
 
 	set = async <T>(
 		baseline: DocumentBaseline<T>,
 		{ transaction }: { transaction?: IDBTransaction } = {},
 	) => {
-		await this.run(
-			'baselines',
-			(store) => store.put(baseline),
-			'readwrite',
+		await this.run('baselines', (store) => store.put(baseline), {
+			mode: 'readwrite',
 			transaction,
-		);
+		});
 	};
 
 	setAll = async <T>(
@@ -173,8 +168,7 @@ export class BaselinesStore extends IDBService {
 			(store) => {
 				return baselines.map((baseline) => store.put(baseline));
 			},
-			'readwrite',
-			transaction,
+			{ mode: 'readwrite', transaction },
 		);
 	};
 
@@ -186,11 +180,9 @@ export class BaselinesStore extends IDBService {
 		oid: ObjectIdentifier,
 		{ transaction }: { transaction?: IDBTransaction },
 	) => {
-		await this.run(
-			'baselines',
-			(store) => store.delete(oid),
-			'readwrite',
+		await this.run('baselines', (store) => store.delete(oid), {
+			mode: 'readwrite',
 			transaction,
-		);
+		});
 	};
 }

package/src/metadata/LocalReplicaStore.ts

@@ -12,7 +12,9 @@ export class LocalReplicaStore extends IDBService {
 	private _creating: Promise<void> | undefined;
 	private cached: LocalReplicaInfo | undefined;
 
-	get = async ({ transaction }: { transaction?: IDBTransaction } = {}): Promise<LocalReplicaInfo> => {
+	get = async ({
+		transaction,
+	}: { transaction?: IDBTransaction } = {}): Promise<LocalReplicaInfo> => {
 		if (this.cached) {
 			return this.cached;
 		}
@@ -20,8 +22,7 @@ export class LocalReplicaStore extends IDBService {
 		const lookup = await this.run<LocalReplicaInfo>(
 			'info',
 			(store) => store.get('localReplicaInfo'),
-			undefined,
-			transaction,
+			{ transaction },
 		);
 
 		// not cached, not in db, create it
@@ -38,7 +39,9 @@ export class LocalReplicaStore extends IDBService {
 						ackedLogicalTime: null,
 						lastSyncedLogicalTime: null,
 					};
-					await this.run('info', (store) => store.put(replicaInfo), 'readwrite');
+					await this.run('info', (store) => store.put(replicaInfo), {
+						mode: 'readwrite',
+					});
 					this.cached = replicaInfo;
 				})();
 			}
@@ -57,7 +60,9 @@ export class LocalReplicaStore extends IDBService {
 	) => {
 		const localReplicaInfo = await this.get({ transaction });
 		Object.assign(localReplicaInfo, data);
-		await this.run('info', (store) => store.put(localReplicaInfo), 'readwrite');
+		await this.run('info', (store) => store.put(localReplicaInfo), {
+			mode: 'readwrite',
+		});
 		this.cached = localReplicaInfo;
 	};
 
@@ -65,6 +70,8 @@ export class LocalReplicaStore extends IDBService {
 		const localInfo = await this.get();
 		localInfo.ackedLogicalTime = null;
 		localInfo.lastSyncedLogicalTime = null;
-		await this.run('info', (store) => store.put(localInfo), 'readwrite');
+		await this.run('info', (store) => store.put(localInfo), {
+			mode: 'readwrite',
+		});
 	};
 }

package/src/metadata/Metadata.ts

@@ -19,6 +19,7 @@ import {
 import { AckInfoStore } from './AckInfoStore.js';
 import { BaselinesStore } from './BaselinesStore.js';
 import {
+	createAbortableTransaction,
 	getAllFromObjectStores,
 	getSizeOfObjectStore,
 	storeRequestPromise,
@@ -57,20 +58,20 @@ export class Metadata extends EventSubscriber<{
 	 */
 	private _closing = false;
 
-	private context: Omit<Context, 'documentDb'>;
+	private context: Omit<Context, 'documentDb' | 'getNow'>;
 
 	constructor({
 		disableRebasing,
 		context,
 	}: {
 		disableRebasing?: boolean;
-		context: Omit<Context, 'documentDb'>;
+		context: Omit<Context, 'documentDb' | 'getNow'>;
 	}) {
 		super();
 		this.context = context;
 		this.schema = new SchemaStore(context.metaDb, context.schema.version);
-		this.operations = new OperationsStore(this.db);
-		this.baselines = new BaselinesStore(this.db);
+		this.operations = new OperationsStore(this.db, { log: context.log });
+		this.baselines = new BaselinesStore(this.db, { log: context.log });
 		this.localReplica = new LocalReplicaStore(this.db);
 		this.ackInfo = new AckInfoStore(this.db);
 		this.messageCreator = new MessageCreator(this);
@@ -102,8 +103,19 @@ export class Metadata extends EventSubscriber<{
 	 * Methods for accessing data
 	 */
 
-	createTransaction = (stores: ('operations' | 'baselines')[]) => {
-		return this.db.transaction(stores, 'readwrite');
+	createTransaction = (
+		stores: ('operations' | 'baselines')[],
+		opts: {
+			abort?: AbortSignal;
+		} = {},
+	) => {
+		return createAbortableTransaction(
+			this.db,
+			stores,
+			'readwrite',
+			opts.abort,
+			this.context.log,
+		);
 	};
 
 	/**
@@ -117,10 +129,7 @@ export class Metadata extends EventSubscriber<{
 		assert(documentOid === oid, 'Must be root document OID');
 		oids.add(documentOid);
 		// readwrite mode to block on other write transactions
-		const transaction = this.db.transaction(
-			['baselines', 'operations'],
-			'readwrite',
-		);
+		const transaction = this.createTransaction(['baselines', 'operations']);
 		await Promise.all([
 			this.baselines.iterateOverAllForDocument(
 				documentOid,
@@ -210,6 +219,43 @@ export class Metadata extends EventSubscriber<{
 		return root;
 	};
 
+	getDocumentData = async (
+		oid: ObjectIdentifier,
+		opts?: {
+			abort?: AbortSignal;
+		},
+	) => {
+		const transaction = this.createTransaction(
+			['baselines', 'operations'],
+			opts,
+		);
+		const baselines: DocumentBaseline[] = [];
+		const operations: Record<ObjectIdentifier, Operation[]> = {};
+		await Promise.all([
+			this.baselines.iterateOverAllForDocument(
+				oid,
+				(baseline) => {
+					baselines.push(baseline);
+				},
+				{
+					transaction,
+				},
+			),
+			this.operations.iterateOverAllOperationsForDocument(
+				oid,
+				(op) => {
+					operations[op.oid] ??= [];
+					operations[op.oid].push(op);
+				},
+				{ transaction },
+			),
+		]);
+		return {
+			baselines,
+			operations,
+		};
+	};
+
 	/**
 	 * Methods for writing data
 	 */
@@ -242,16 +288,23 @@ export class Metadata extends EventSubscriber<{
 	 * Applies a patch to the document and stores it in the database.
 	 * @returns the oldest local history timestamp
 	 */
-	insertLocalOperation = async (operations: Operation[]) => {
+	insertLocalOperations = async (
+		operations: Operation[],
+		opts?: { transaction?: IDBTransaction },
+	) => {
 		if (operations.length === 0) return;
 		// await this.rebaseLock;
-		this.log(`Inserting ${operations.length} local operations`);
+		this.log(
+			'debug',
+			`Inserting ${operations.length} local operations`,
+			operations,
+		);
 
 		// add local flag, in place.
 		for (const operation of operations) {
 			(operation as ClientOperation).isLocal = true;
 		}
-		await this.operations.addOperations(operations as ClientOperation[]);
+		await this.operations.addOperations(operations as ClientOperation[], opts);
 
 		const message = await this.messageCreator.createOperation({ operations });
 		this.emit('message', message);
@@ -264,16 +317,24 @@ export class Metadata extends EventSubscriber<{
 	 * Inserts remote operations. This does not affect local history.
 	 * @returns a list of affected document OIDs
 	 */
-	insertRemoteOperations = async (operations: Operation[]) => {
+	insertRemoteOperations = async (
+		operations: Operation[],
+		opts?: { transaction?: IDBTransaction },
+	) => {
 		if (operations.length === 0) return [];
 		// await this.rebaseLock;
-		this.log(`Inserting ${operations.length} remote operations`);
+		this.log(
+			'debug',
+			`Inserting ${operations.length} remote operations`,
+			operations,
+		);
 
 		const affectedDocumentOids = await this.operations.addOperations(
 			operations.map((patch) => ({
 				...patch,
 				isLocal: false,
 			})),
+			opts,
 		);
 
 		this.ack(operations[operations.length - 1].timestamp);
@@ -281,11 +342,14 @@ export class Metadata extends EventSubscriber<{
 		return affectedDocumentOids;
 	};
 
-	insertRemoteBaselines = async (baselines: DocumentBaseline[]) => {
+	insertRemoteBaselines = async (
+		baselines: DocumentBaseline[],
+		opts?: { transaction?: IDBTransaction },
+	) => {
 		if (baselines.length === 0) return [];
 		this.log(`Inserting ${baselines.length} remote baselines`);
 
-		await this.baselines.setAll(baselines);
+		await this.baselines.setAll(baselines, opts);
 
 		// this.ack(baselines[baselines.length - 1].timestamp);
 
@@ -297,6 +361,31 @@ export class Metadata extends EventSubscriber<{
 		return Array.from(affectedOidSet);
 	};
 
+	insertData = async (
+		data: {
+			baselines?: DocumentBaseline[];
+			operations?: Operation[];
+			isLocal?: boolean;
+		},
+		opts?: { abort: AbortSignal },
+	) => {
+		const transaction = this.createTransaction(
+			['baselines', 'operations'],
+			opts,
+		);
+		if (data.baselines) {
+			await this.insertRemoteBaselines(data.baselines, { transaction });
+		}
+		if (opts?.abort?.aborted) return;
+		if (data.operations) {
+			if (data.isLocal) {
+				await this.insertLocalOperations(data.operations, { transaction });
+			} else {
+				await this.insertRemoteOperations(data.operations, { transaction });
+			}
+		}
+	};
+
 	updateLastSynced = async (timestamp: string) => {
 		if (this._closing) return;
 
@@ -331,10 +420,7 @@ export class Metadata extends EventSubscriber<{
 		// find all operations before the global ack
 		let lastTimestamp;
 		const toRebase = new Set<ObjectIdentifier>();
-		const transaction = this.db.transaction(
-			['baselines', 'operations'],
-			'readwrite',
-		);
+		const transaction = this.createTransaction(['baselines', 'operations']);
 		let operationCount = 0;
 		await this.operations.iterateOverAllOperations(
 			(patch) => {
@@ -349,7 +435,6 @@ export class Metadata extends EventSubscriber<{
 		);
 
 		if (!toRebase.size) {
-			this.log('Cannot rebase, no operations prior to', globalAckTimestamp);
 			return;
 		}
 
@@ -381,8 +466,7 @@ export class Metadata extends EventSubscriber<{
 
 		this.log('[', replicaId, ']', 'Rebasing', oid, 'up to', upTo);
 		const transaction =
-			providedTx ||
-			this.db.transaction(['operations', 'baselines'], 'readwrite');
+			providedTx || this.createTransaction(['operations', 'baselines']);
 		const baseline = await this.baselines.get(oid, { transaction });
 		let current: any = baseline?.snapshot || undefined;
 		let operationsApplied = 0;
@@ -419,6 +503,7 @@ export class Metadata extends EventSubscriber<{
 		}
 
 		this.log(
+			'debug',
 			'successfully rebased',
 			oid,
 			'up to',

package/src/metadata/OperationsStore.ts

@@ -8,6 +8,7 @@ import {
 	assert,
 } from '@verdant-web/common';
 import { IDBService } from '../IDBService.js';
+import { isAbortError } from '../idb.js';
 
 export type ClientOperation = Operation & {
 	isLocal: boolean;
@@ -20,6 +21,9 @@ export type StoredClientOperation = ClientOperation & {
 };
 
 export class OperationsStore extends IDBService {
+	constructor(db: IDBDatabase, opts: { log?: (...args: any[]) => void }) {
+		super(db, opts);
+	}
 	/**
 	 * Iterates over every patch for the root and every sub-object
 	 * of a given document. Optionally limit by timestamp.
@@ -41,7 +45,11 @@ export class OperationsStore extends IDBService {
 			transaction?: IDBTransaction;
 		} = {},
 	): Promise<void> => {
-		const transaction = providedTx || this.db.transaction('operations', mode);
+		const transaction =
+			providedTx ||
+			this.createTransaction(['operations'], {
+				mode,
+			});
 		const store = transaction.objectStore('operations');
 		const index = store.index('d_t');
 
@@ -77,7 +85,12 @@ export class OperationsStore extends IDBService {
 				}
 			};
 			request.onerror = (event) => {
-				reject(event);
+				if (isAbortError(request.error)) {
+					resolve();
+					return;
+				} else {
+					reject(request.error);
+				}
 			};
 		});
 	};
@@ -97,7 +110,8 @@ export class OperationsStore extends IDBService {
 			transaction?: IDBTransaction;
 		},
 	): Promise<void> => {
-		const transaction = providedTx || this.db.transaction('operations', mode);
+		const transaction =
+			providedTx || this.createTransaction(['operations'], { mode });
 		const store = transaction.objectStore('operations');
 
 		const start = after
@@ -131,7 +145,11 @@ export class OperationsStore extends IDBService {
 				}
 			};
 			request.onerror = (event) => {
-				reject(event);
+				if (isAbortError(request.error)) {
+					resolve();
+				} else {
+					reject(request.error);
+				}
 			};
 		});
 	};
@@ -151,7 +169,8 @@ export class OperationsStore extends IDBService {
 			transaction?: IDBTransaction;
 		},
 	): Promise<void> => {
-		const transaction = providedTx || this.db.transaction('operations', mode);
+		const transaction =
+			providedTx || this.createTransaction(['operations'], { mode });
 
 		return this.iterate(
 			'operations',
@@ -162,8 +181,7 @@ export class OperationsStore extends IDBService {
 				);
 			},
 			iterator,
-			mode,
-			transaction,
+			{ mode, transaction },
 		);
 	};
 
@@ -181,7 +199,8 @@ export class OperationsStore extends IDBService {
 			transaction?: IDBTransaction;
 		},
 	): Promise<void> => {
-		const transaction = providedTx || this.db.transaction('operations', mode);
+		const transaction =
+			providedTx || this.createTransaction(['operations'], { mode });
 		const store = transaction.objectStore('operations');
 		const index = store.index('l_t');
 
@@ -215,7 +234,11 @@ export class OperationsStore extends IDBService {
 				}
 			};
 			request.onerror = (event) => {
-				reject(event);
+				if (isAbortError(request.error)) {
+					resolve();
+				} else {
+					reject(request.error);
+				}
 			};
 		});
 	};
@@ -244,18 +267,17 @@ export class OperationsStore extends IDBService {
 
 				const range =
 					start && end
-						? IDBKeyRange.bound(start, end, false, true)
+						? window.IDBKeyRange.bound(start, end, false, true)
 						: start
-						? IDBKeyRange.lowerBound(start, false)
+						? window.IDBKeyRange.lowerBound(start, false)
 						: end
-						? IDBKeyRange.upperBound(end, true)
+						? window.IDBKeyRange.upperBound(end, true)
 						: undefined;
 				const index = store.index('timestamp');
 				return index.openCursor(range, 'next');
 			},
 			iterator,
-			mode,
-			transaction,
+			{ mode, transaction },
 		);
 	};
 
@@ -299,8 +321,7 @@ export class OperationsStore extends IDBService {
 					affected.add(getOidRoot(op.oid));
 					return store.put(op);
 				}),
-			'readwrite',
-			transaction,
+			{ mode: 'readwrite', transaction },
 		);
 		return Array.from(affected);
 	};

package/src/migration/openDatabase.ts

@@ -14,6 +14,7 @@ import {
 	decomposeOid,
 	diffToPatches,
 	getIndexValues,
+	getOid,
 	getOidRoot,
 	hasOid,
 	initialToPatches,
@@ -258,7 +259,11 @@ async function runMigrations({
 						`Migration failed (${migration.oldSchema.version} -> ${migration.newSchema.version})`,
 						err,
 					);
-					throw err;
+					if (err instanceof Error) {
+						throw err;
+					} else {
+						throw new Error('Unknown error during migration');
+					}
 				}
 
 				// now we have to open the database again with the next version and
@@ -356,6 +361,25 @@ async function runMigrations({
 					}),
 				);
 
+				// add 'touch' operations to all root OIDs of all documents.
+				// this marks documents which have undergone a migration
+				// so that other clients know when they're working
+				// with unmigrated data - by seeing that there are no
+				// existing operations or baselines with a timestamp
+				// that matches the current version.
+				// UPDATE: no longer necessary now that pruning is a thing.
+				// await Promise.all(
+				// 	oids.map((oid) =>
+				// 		meta.insertLocalOperations([
+				// 			{
+				// 				oid,
+				// 				timestamp: meta.time.zero(migration.version),
+				// 				data: { op: 'touch' },
+				// 			},
+				// 		]),
+				// 	),
+				// );
+
 				const snapshots = await Promise.all(
 					oids.map(async (oid) => {
 						try {
@@ -396,10 +420,7 @@ async function runMigrations({
 				await Promise.all(
 					views.map(([oid, view]) => {
 						if (view) {
-							return putView(writeStore, view).catch((err) => {
-								view;
-								throw err;
-							});
+							return putView(writeStore, view);
 						} else {
 							const { id } = decomposeOid(oid);
 							return deleteView(writeStore, id);
@@ -453,7 +474,7 @@ function getMigrationMutations({
 					doc[migration.newSchema.collections[collectionName].primaryKey];
 				const oid = createOid(collectionName, primaryKey);
 				newOids.push(oid);
-				await meta.insertLocalOperation(
+				await meta.insertLocalOperations(
 					initialToPatches(doc, oid, getMigrationNow),
 				);
 				return doc;
@@ -461,7 +482,7 @@ function getMigrationMutations({
 			delete: async (id: string) => {
 				const rootOid = createOid(collectionName, id);
 				const allOids = await meta.getAllDocumentRelatedOids(rootOid);
-				return meta.insertLocalOperation(
+				return meta.insertLocalOperations(
 					allOids.map((oid) => ({
 						oid,
 						timestamp: getMigrationNow(),
@@ -556,7 +577,7 @@ function getMigrationEngine({
 	});
 	const deleteCollection = async (collection: string) => {
 		const allOids = await meta.getAllCollectionRelatedOids(collection);
-		return meta.insertLocalOperation(
+		return meta.insertLocalOperations(
 			allOids.map((oid) => ({
 				oid,
 				timestamp: getMigrationNow(),
@@ -574,8 +595,9 @@ function getMigrationEngine({
 
 			await Promise.all(
 				docs.filter(Boolean).map(async (doc: any) => {
+					const rootOid = getOid(doc);
 					assert(
-						hasOid(doc),
+						!!rootOid,
 						`Document is missing an OID: ${JSON.stringify(doc)}`,
 					);
 					const original = cloneDeep(doc);
@@ -599,7 +621,7 @@ function getMigrationEngine({
 							},
 						);
 						if (patches.length > 0) {
-							await meta.insertLocalOperation(patches);
+							await meta.insertLocalOperations(patches);
 						}
 					}
 				}),

package/src/queries/BaseQuery.ts

@@ -1,6 +1,6 @@
 import { EventSubscriber } from '@verdant-web/common';
 import { Context } from '../context.js';
-import { Entity } from '../entities/Entity.js';
+import { Entity } from '../entities/2/Entity.js';
 import { Disposable } from '../utils/Disposable.js';
 import { filterResultSet } from './utils.js';
 
@@ -70,11 +70,15 @@ export abstract class BaseQuery<T> extends Disposable {
 				(collections) => {
 					if (shouldUpdateFn(collections)) {
 						this.context.log('info', 'Updating query', this.key);
+						// immediately refilter the result set - deleted
+						// entities will already be nulled out
+						// this.refreshValue();
 						this.execute();
 					}
 				},
 			),
 		);
+		// TODO: subscribe to document changes and update if necessary.
 	}
 
 	get current() {
@@ -135,6 +139,13 @@ export abstract class BaseQuery<T> extends Disposable {
 		this._rawValue = value;
 		this.subscribeToDeleteAndRestore(this._rawValue);
 		this._value = filterResultSet(value);
+		// validate the value
+		if (
+			Array.isArray(this._value) &&
+			this._value.some((v) => v.getSnapshot() === null)
+		) {
+			debugger;
+		}
 		this._status = 'ready';
 		this._events.emit('change', this._value);
 	};
@@ -193,8 +204,10 @@ export abstract class BaseQuery<T> extends Disposable {
 						// possibly accessing db while it's closed. not much we can do.
 						return this._value;
 					}
+					throw err;
+				} else {
+					throw new Error('Unknown error executing query');
 				}
-				throw err;
 			});
 		return this._executionPromise;
 	};