@verdant-web/store 5.3.0 → 5.4.2

package/src/persistence/idb/metadata/openMetadataDatabase.ts

@@ -1,6 +1,7 @@
 import { replaceLegacyOidsInObject } from '@verdant-web/common';
-import { getMetadataDbName } from '../util.js';
 import { Context } from '../../../context/context.js';
+import { OPERATION_INDEX_NAMES } from '../indexNames.js';
+import { getMetadataDbName } from '../util.js';
 
 const migrations = [version1, version2, version3, version4, version5, version6];
 export const CURRENT_METADATA_VERSION = migrations.length;
@@ -55,7 +56,7 @@ async function version1(db: IDBDatabase, tx: IDBTransaction) {
 		keyPath: 'oid',
 	});
 	const operationsStore = db.createObjectStore('operations', {
-		keyPath: 'oid_timestamp',
+		keyPath: OPERATION_INDEX_NAMES.OID__TIMESTAMP,
 	});
 	const infoStore = db.createObjectStore('info', { keyPath: 'type' });
 	baselinesStore.createIndex('timestamp', 'timestamp');
@@ -84,8 +85,8 @@ async function version2(db: IDBDatabase, tx: IDBTransaction) {
 					cursor.value;
 				cursor.update({
 					...value,
-					l_t: isLocal_timestamp,
-					d_t: documentOid_timestamp,
+					[OPERATION_INDEX_NAMES.IS_LOCAL__TIMESTAMP]: isLocal_timestamp,
+					[OPERATION_INDEX_NAMES.ROOT_OID__TIMESTAMP]: documentOid_timestamp,
 				});
 				cursor.continue();
 			} else {
@@ -100,9 +101,16 @@ async function version2(db: IDBDatabase, tx: IDBTransaction) {
 	operations.deleteIndex('isLocal_timestamp');
 	operations.deleteIndex('documentOid_timestamp');
 	// create the new indexes
-	operations.createIndex('l_t', 'l_t', { unique: false });
-	operations.createIndex('o_t', 'o_t', { unique: false });
-	operations.createIndex('d_t', 'd_t', { unique: false });
+	operations.createIndex(
+		OPERATION_INDEX_NAMES.IS_LOCAL__TIMESTAMP,
+		OPERATION_INDEX_NAMES.IS_LOCAL__TIMESTAMP,
+		{ unique: false },
+	);
+	operations.createIndex(
+		OPERATION_INDEX_NAMES.ROOT_OID__TIMESTAMP,
+		OPERATION_INDEX_NAMES.ROOT_OID__TIMESTAMP,
+		{ unique: false },
+	);
 }
 
 /**

package/src/persistence/idb/queries/ranges.ts

@@ -46,15 +46,14 @@ const rangeIndexToIdbKeyRange = (filter: RangeCollectionIndexFilter) => {
 };
 
 const compoundIndexToIdbKeyRange = (
-	// FIXME:
-	schema: any,
+	schema: StorageSchema,
 	collection: string,
 	filter: CollectionCompoundIndexFilter,
 ) => {
 	// validate the usage of the compound index:
 	// - all match fields must be contiguous at the start of the compound order
 	const indexDefinition =
-		schema.collections[collection].compounds[filter.where];
+		schema.collections[collection].compounds?.[filter.where];
 	assert(
 		indexDefinition,
 		`Index ${filter.where} does not exist on collection ${collection}`,

package/src/persistence/interfaces.ts

@@ -105,7 +105,7 @@ export interface PersistenceMetadataDb<
 	iterateEntityOperations(
 		oid: string,
 		iterator: Iterator<ClientOperation>,
-		opts?: CommonQueryOptions<Tx> & { to?: string | null },
+		opts?: CommonQueryOptions<Tx> & { to?: string | null; reverse?: boolean },
 	): Promise<void>;
 	iterateCollectionOperations(
 		collection: string,
@@ -138,6 +138,10 @@ export interface PersistenceMetadataDb<
 		ops: ClientOperation[],
 		opts?: CommonQueryOptions<Tx>,
 	): Promise<ObjectIdentifier[]>;
+	purgeDocumentData(
+		oid: ObjectIdentifier,
+		opts?: CommonQueryOptions<Tx>,
+	): Promise<void>;
 
 	/* WARNING: deletes all data */
 	reset(opts?: { clearReplica?: boolean; transaction?: Tx }): Promise<void>;

package/src/persistence/migration/engine.ts

@@ -174,10 +174,9 @@ export async function getMigrationEngine({
 						!!rootOid,
 						`Document is missing an OID: ${JSON.stringify(doc)}`,
 					);
-					// FIXME: this could be optimized (making n queries for authz
-					// when the snapshots themselves are derived from the same data...)
-					// maybe don't use the findAll query, and instead go a level
-					// lower to retain access to lower level data here?
+					// while querying again for data we already discarded in the findAll
+					// above isn't ideal, in practice the get authz method is
+					// fairly quick because it short-circuits.
 					const authz = await meta.getDocumentAuthz(rootOid);
 					const original = cloneDeep(doc);
 					// @ts-ignore - excessive type resolution

package/src/queries/BaseQuery.ts

@@ -2,6 +2,7 @@ import { EventSubscriber } from '@verdant-web/common';
 import { Context } from '../context/context.js';
 import { Entity } from '../entities/Entity.js';
 import { Disposable } from '../utils/Disposable.js';
+import { getDiagnosticResult, QueryDiagnostic, QueryTiming } from './diagnostics.js';
 import { filterResultSet } from './utils.js';
 
 export type BaseQueryEvents = {
@@ -37,12 +38,18 @@ export abstract class BaseQuery<T> extends Disposable {
 	private _allUnsubscribedHandler?: (query: BaseQuery<T>) => void;
 	private _status: QueryStatus = 'initial';
 	private _executionPromise: Promise<T> | null = null;
+	private _timing: QueryTiming = {
+		sweep: null,
+		hydration: null,
+		total: null,
+	};
 
 	protected context;
 
 	readonly collection;
 	readonly key;
 	readonly isListQuery;
+	readonly type;
 
 	constructor({
 		initial,
@@ -65,6 +72,7 @@ export abstract class BaseQuery<T> extends Disposable {
 		this.context = context;
 		this.key = key;
 		this.collection = collection;
+		this.type = this.constructor.name;
 		const shouldUpdateFn =
 			shouldUpdate ||
 			((collections: string[]) => collections.includes(collection));
@@ -223,10 +231,9 @@ export abstract class BaseQuery<T> extends Disposable {
 	};
 
 	execute = () => {
-		const startTime = new Date();
+		const startTime = performance.now();
 		this.context.log(
 			'debug',
-			`[${startTime.toLocaleTimeString()}]`,
 			'Executing query',
 			this.key,
 		);
@@ -255,11 +262,10 @@ export abstract class BaseQuery<T> extends Disposable {
 				}
 			})
 			.finally(() => {
-				const endTime = new Date();
-				const duration = endTime.getTime() - startTime.getTime();
+				const duration = performance.now() - startTime;
+				this._timing.total = duration;
 				this.context.log(
 					'debug',
-					`[${endTime.toLocaleTimeString()}]`,
 					'Query executed',
 					this.key,
 					`Duration: ${duration}ms`,
@@ -267,6 +273,36 @@ export abstract class BaseQuery<T> extends Disposable {
 			});
 		return this._executionPromise;
 	};
+
+	protected measureSweep = async <V>(work: () => Promise<V>): Promise<V> => {
+		const start = performance.now();
+		try {
+			return await work();
+		} finally {
+			this._timing.sweep = performance.now() - start;
+		}
+	};
+
+	protected measureHydration = async <V>(
+		work: () => Promise<V>,
+	): Promise<V> => {
+		const start = performance.now();
+		try {
+			return await work();
+		} finally {
+			this._timing.hydration = performance.now() - start;
+		}
+	};
+
+	getDiagnostic = (active: boolean): QueryDiagnostic => ({
+		key: this.key,
+		collection: this.collection,
+		type: this.type,
+		status: this.status,
+		active,
+		result: getDiagnosticResult(this.current),
+		timing: { ...this._timing },
+	});
 	protected abstract run(): Promise<void>;
 
 	[ON_ALL_UNSUBSCRIBED] = (handler: (query: BaseQuery<T>) => void) => {

package/src/queries/CollectionQueries.ts

@@ -4,16 +4,16 @@ import {
 	hashObject,
 } from '@verdant-web/common';
 import { Context } from '../context/context.js';
-import { EntityStore } from '../entities/EntityStore.js';
-import { GetQuery } from './GetQuery.js';
-import { QueryCache } from './QueryCache.js';
-import { FindOneQuery } from './FindOneQuery.js';
-import { FindPageQuery } from './FindPageQuery.js';
-import { FindInfiniteQuery } from './FindInfiniteQuery.js';
-import { FindAllQuery } from './FindAllQuery.js';
 import { DocumentManager } from '../entities/DocumentManager.js';
+import { EntityStore } from '../entities/EntityStore.js';
 import { ObjectEntity } from '../index.js';
 import { UPDATE } from './BaseQuery.js';
+import { FindAllQuery } from './FindAllQuery.js';
+import { FindInfiniteQuery } from './FindInfiniteQuery.js';
+import { FindOneQuery } from './FindOneQuery.js';
+import { FindPageQuery } from './FindPageQuery.js';
+import { GetQuery } from './GetQuery.js';
+import { QueryCache } from './QueryCache.js';
 
 export class CollectionQueries<
 	T extends ObjectEntity<any, any>,
@@ -123,9 +123,11 @@ export class CollectionQueries<
 	findAll = ({
 		index,
 		key: providedKey,
-	}: { index?: Filter; key?: string } = {}) => {
+		limit,
+	}: { index?: Filter; key?: string; limit?: number } = {}) => {
 		const key =
-			providedKey || `findAll:${this.collection}:${this.serializeIndex(index)}`;
+			providedKey ||
+			`findAll:${this.collection}:${limit}:${this.serializeIndex(index)}`;
 		return this.cache.getOrSet(
 			key,
 			() =>
@@ -135,6 +137,7 @@ export class CollectionQueries<
 					hydrate: this.hydrate,
 					context: this.context,
 					key,
+					limit,
 				}),
 			(existing) => {
 				existing[UPDATE](index);

package/src/queries/FindAllQuery.ts

@@ -5,35 +5,42 @@ import { areIndexesEqual } from './utils.js';
 export class FindAllQuery<T> extends BaseQuery<T[]> {
 	private index;
 	private hydrate;
+	private limit;
 
 	constructor({
 		index,
 		hydrate,
+		limit,
 		...rest
 	}: {
 		index?: CollectionFilter;
 		hydrate: (oid: string) => Promise<T>;
+		limit?: number;
 	} & Omit<BaseQueryOptions<T[]>, 'initial'>) {
 		super({
 			initial: [],
 			...rest,
 		});
+		this.limit = limit;
 		this.index = index;
 		this.hydrate = hydrate;
 	}
 
 	protected run = async () => {
-		const { result: oids } = await (
-			await this.context.documents
-		).findAllOids({
-			collection: this.collection,
-			index: this.index,
-		});
+		const { result: oids } = await this.measureSweep(async () =>
+			(await this.context.documents).findAllOids({
+				collection: this.collection,
+				index: this.index,
+				limit: this.limit,
+			}),
+		);
 		this.context.log(
 			'debug',
 			`FindAllQuery: ${oids.length} oids found: ${oids}`,
 		);
-		this.setValue(await Promise.all(oids.map(this.hydrate)));
+		this.setValue(
+			await this.measureHydration(() => Promise.all(oids.map(this.hydrate))),
+		);
 	};
 
 	[UPDATE] = (index: CollectionFilter | undefined) => {

package/src/queries/FindInfiniteQuery.ts

@@ -37,32 +37,36 @@ export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await (
-			await this.context.documents
-		).findAllOids({
-			collection: this.collection,
-			limit: this._pageSize * this._upToPage,
-			offset: 0,
-			index: this.index,
-		});
+		const { result, hasNextPage } = await this.measureSweep(async () =>
+			(await this.context.documents).findAllOids({
+				collection: this.collection,
+				limit: this._pageSize * this._upToPage,
+				offset: 0,
+				index: this.index,
+			}),
+		);
 		this._hasNextPage = hasNextPage;
-		this.setValue(await Promise.all(result.map(this.hydrate)));
+		this.setValue(
+			await this.measureHydration(() => Promise.all(result.map(this.hydrate))),
+		);
 	};
 
 	public loadMore = async () => {
-		const { result, hasNextPage } = await (
-			await this.context.documents
-		).findAllOids({
-			collection: this.collection,
-			limit: this._pageSize,
-			offset: this._pageSize * this._upToPage,
-			index: this.index,
-		});
+		const { result, hasNextPage } = await this.measureSweep(async () =>
+			(await this.context.documents).findAllOids({
+				collection: this.collection,
+				limit: this._pageSize,
+				offset: this._pageSize * this._upToPage,
+				index: this.index,
+			}),
+		);
 		this._hasNextPage = hasNextPage;
 		this._upToPage++;
 		this.setValue([
 			...this.current,
-			...(await Promise.all(result.map(this.hydrate))),
+			...(await this.measureHydration(() =>
+				Promise.all(result.map(this.hydrate)),
+			)),
 		]);
 	};
 

package/src/queries/FindOneQuery.ts

@@ -23,13 +23,15 @@ export class FindOneQuery<T> extends BaseQuery<T | null> {
 	}
 
 	protected run = async () => {
-		const oid = await (
-			await this.context.documents
-		).findOneOid({
-			collection: this.collection,
-			index: this.index,
-		});
-		this.setValue(oid ? await this.hydrate(oid) : null);
+		const oid = await this.measureSweep(async () =>
+			(await this.context.documents).findOneOid({
+				collection: this.collection,
+				index: this.index,
+			}),
+		);
+		this.setValue(
+			oid ? await this.measureHydration(() => this.hydrate(oid)) : null,
+		);
 	};
 
 	[UPDATE] = (index: CollectionFilter | undefined) => {

package/src/queries/FindPageQuery.ts

@@ -48,16 +48,18 @@ export class FindPageQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await (
-			await this.context.documents
-		).findAllOids({
-			collection: this.collection,
-			index: this.index,
-			limit: this._pageSize,
-			offset: this._page * this._pageSize,
-		});
+		const { result, hasNextPage } = await this.measureSweep(async () =>
+			(await this.context.documents).findAllOids({
+				collection: this.collection,
+				index: this.index,
+				limit: this._pageSize,
+				offset: this._page * this._pageSize,
+			}),
+		);
 		this._hasNextPage = hasNextPage;
-		this.setValue(await Promise.all(result.map(this.hydrate)));
+		this.setValue(
+			await this.measureHydration(() => Promise.all(result.map(this.hydrate))),
+		);
 	};
 
 	nextPage = async () => {

package/src/queries/GetQuery.ts

@@ -22,7 +22,7 @@ export class GetQuery<T> extends BaseQuery<T | null> {
 	}
 
 	protected run = async () => {
-		const value = await this.hydrate(this.oid);
+		const value = await this.measureHydration(() => this.hydrate(this.oid));
 		this.setValue(value);
 	};
 }

package/src/queries/QueryCache.ts

@@ -1,6 +1,7 @@
 import { Context } from '../context/context.js';
 import { Disposable } from '../utils/Disposable.js';
 import { BaseQuery, ON_ALL_UNSUBSCRIBED } from './BaseQuery.js';
+import { QueryDiagnostic, QueryDiagnostics } from './diagnostics.js';
 
 export class QueryCache extends Disposable {
 	private _cache: Map<string, BaseQuery<any>> = new Map();
@@ -8,6 +9,7 @@ export class QueryCache extends Disposable {
 	private context;
 	/** A set of query keys to keep alive even if they unsubscribe */
 	private _holds = new Set<string>();
+	private _inactiveDiagnostics = new Map<string, QueryDiagnostic>();
 
 	constructor({
 		evictionTime = 5 * 1000,
@@ -38,6 +40,7 @@ export class QueryCache extends Disposable {
 
 	set<V extends BaseQuery<any>>(value: V) {
 		this._cache.set(value.key, value);
+		this._inactiveDiagnostics.delete(value.key);
 		value[ON_ALL_UNSUBSCRIBED](this.enqueueQueryEviction);
 		// immediately enqueue a check to see if this query should be evicted --
 		// this basically gives code X seconds to subscribe to the query before
@@ -47,6 +50,15 @@ export class QueryCache extends Disposable {
 		return value;
 	}
 
+	get diagnostics(): QueryDiagnostics {
+		return {
+			queries: [
+				...this._cache.values().map((query) => query.getDiagnostic(true)),
+				...this._inactiveDiagnostics.values(),
+			],
+		};
+	}
+
 	getOrSet<V extends BaseQuery<any>>(
 		key: string,
 		create: () => V,
@@ -78,19 +90,23 @@ export class QueryCache extends Disposable {
 			// got a different version of this query.
 			if (this._cache.get(query.key) === query) {
 				this._cache.delete(query.key);
+				this._inactiveDiagnostics.set(query.key, query.getDiagnostic(false));
 				this.context.log('debug', 'QueryCache: evicted query', query.key);
 			}
 		}, this._evictionTime);
 	};
 
-	dropAll = () => {
+		dropAll = () => {
 		this.context.log(
 			'debug',
 			'QueryCache: drop all',
 			this._cache.size,
 			'queries',
 		);
-		this._cache.forEach((query) => query.dispose());
+		this._cache.forEach((query) => {
+			this._inactiveDiagnostics.set(query.key, query.getDiagnostic(false));
+			query.dispose();
+		});
 		this._cache.clear();
 	};
 
@@ -135,4 +151,5 @@ export type PublicQueryCacheAPI = Pick<
 	| 'keepAlives'
 	| 'forceRefreshAll'
 	| 'activeKeys'
+	| 'diagnostics'
 >;

package/src/queries/diagnostics.ts

@@ -0,0 +1,32 @@
+import { Entity } from '../entities/Entity.js';
+import type { QueryStatus } from './BaseQuery.js';
+
+export type QueryTiming = {
+	sweep: number | null;
+	hydration: number | null;
+	total: number | null;
+};
+
+export type QueryDiagnostic = {
+	key: string;
+	collection: string;
+	type: string;
+	status: QueryStatus;
+	active: boolean;
+	result: unknown;
+	timing: QueryTiming;
+};
+
+export type QueryDiagnostics = {
+	queries: QueryDiagnostic[];
+};
+
+export const getDiagnosticResult = (value: unknown): unknown => {
+	if (Array.isArray(value)) {
+		return value.map(getDiagnosticResult);
+	}
+	if (value instanceof Entity) {
+		return value.getSnapshot();
+	}
+	return value;
+};

package/src/queries/ranges.ts

@@ -46,15 +46,14 @@ const rangeIndexToIdbKeyRange = (filter: RangeCollectionIndexFilter) => {
 };
 
 const compoundIndexToIdbKeyRange = (
-	// FIXME:
-	schema: any,
+	schema: StorageSchema,
 	collection: string,
 	filter: CollectionCompoundIndexFilter,
 ) => {
 	// validate the usage of the compound index:
 	// - all match fields must be contiguous at the start of the compound order
 	const indexDefinition =
-		schema.collections[collection].compounds[filter.where];
+		schema.collections[collection].compounds?.[filter.where];
 	assert(
 		indexDefinition,
 		`Index ${filter.where} does not exist on collection ${collection}`,

package/src/sync/WebSocketSync.ts

@@ -314,6 +314,8 @@ export class WebSocketSync
 		await this.sendDisconnecting();
 		this.socket?.removeEventListener('message', this.onMessage);
 		this.socket?.removeEventListener('close', this.onClose);
+		this.socket?.removeEventListener('error', this.onError);
+		this.socket?.removeEventListener('open', this.onOpen);
 		if (this.socket?.readyState === WEBSOCKET_OPEN) {
 			this.socket.close();
 		}