@verdant-web/store 2.0.1

package/src/queries2/dbQueries.ts

@@ -0,0 +1,154 @@
+import { CollectionFilter, createOid } from '@verdant-web/common';
+import { Context } from '../context.js';
+import { getRange } from './ranges.js';
+
+function getStore(db: IDBDatabase, collection: string, write?: boolean) {
+	return db
+		.transaction(collection, write ? 'readwrite' : 'readonly')
+		.objectStore(collection);
+}
+
+export async function findOneOid({
+	collection,
+	index,
+	context,
+}: {
+	collection: string;
+	index?: CollectionFilter;
+	context: Context;
+}) {
+	const store = getStore(context.documentDb, collection);
+	const source = index ? store.index(index.where) : store;
+	const range = getRange(context.schema, collection, index);
+	const direction = index?.order === 'desc' ? 'prev' : 'next';
+	const request = source.openCursor(range, direction);
+	const result = await new Promise<string | null>((resolve, reject) => {
+		request.onsuccess = () => {
+			const cursor = request.result;
+			if (cursor) {
+				resolve(createOid(collection, cursor.primaryKey.toString()));
+			} else {
+				resolve(null);
+			}
+		};
+		request.onerror = () => {
+			if (request.error?.name === 'InvalidStateError') {
+				context.log(
+					'error',
+					`findOne query failed with InvalidStateError`,
+					request.error,
+				);
+				resolve(null);
+			} else {
+				reject(request.error);
+			}
+		};
+	});
+	return result;
+}
+
+export async function findAllOids({
+	collection,
+	index,
+	context,
+}: {
+	collection: string;
+	index?: CollectionFilter;
+	context: Context;
+}) {
+	const store = getStore(context.documentDb, collection);
+	const source = index ? store.index(index.where) : store;
+	const range = getRange(context.schema, collection, index);
+	const direction = index?.order === 'desc' ? 'prev' : 'next';
+	const request = source.openCursor(range, direction);
+	const result = await new Promise<string[]>((resolve, reject) => {
+		const results = new Set<string>();
+		request.onsuccess = () => {
+			const cursor = request.result;
+			if (cursor) {
+				results.add(createOid(collection, cursor.primaryKey.toString()));
+				cursor.continue();
+			} else {
+				resolve(Array.from(results));
+			}
+		};
+		request.onerror = () => {
+			if (request.error?.name === 'InvalidStateError') {
+				context.log(
+					'error',
+					`findAll query failed with InvalidStateError`,
+					request.error,
+				);
+				resolve([]);
+			} else {
+				reject(request.error);
+			}
+		};
+	});
+	return result;
+}
+
+export async function findPageOfOids({
+	collection,
+	index,
+	context,
+	limit,
+	offset,
+}: {
+	collection: string;
+	index?: CollectionFilter;
+	context: Context;
+	limit?: number;
+	offset?: number;
+}) {
+	const store = getStore(context.documentDb, collection);
+	const source = index ? store.index(index.where) : store;
+	const range = getRange(context.schema, collection, index);
+	const direction = index?.order === 'desc' ? 'prev' : 'next';
+	const request = source.openCursor(range, direction);
+	let hasDoneOffset = !offset;
+	let hasNextPage = false;
+	let visited = 0;
+	const result = await new Promise<string[]>((resolve, reject) => {
+		const results = new Set<string>();
+		request.onsuccess = () => {
+			visited++;
+			const cursor = request.result;
+			if (cursor) {
+				if (offset && !hasDoneOffset) {
+					cursor.advance(offset);
+					hasDoneOffset = true;
+				} else {
+					if (limit && results.size < limit) {
+						results.add(createOid(collection, cursor.primaryKey.toString()));
+					}
+					if (limit && visited > limit) {
+						hasNextPage = true;
+						resolve(Array.from(results));
+					} else {
+						cursor.continue();
+					}
+				}
+			} else {
+				resolve(Array.from(results));
+			}
+		};
+		request.onerror = () => {
+			if (request.error?.name === 'InvalidStateError') {
+				context.log(
+					'error',
+					`find query failed with InvalidStateError`,
+					request.error,
+				);
+				resolve([]);
+			} else {
+				reject(request.error);
+			}
+		};
+	});
+
+	return {
+		result: result as string[],
+		hasNextPage,
+	};
+}

package/src/queries2/keys.ts

@@ -0,0 +1,27 @@
+import { hashObject } from '@verdant-web/common';
+
+export interface QueryParams {
+	collection: string;
+	range: IDBKeyRange | IDBValidKey | undefined;
+	index?: string;
+	direction?: IDBCursorDirection;
+	limit?: number;
+	single?: boolean;
+	write?: boolean;
+}
+
+export function getQueryKey({ range, ...rest }: QueryParams) {
+	let hashedRange;
+	if (range instanceof IDBKeyRange) {
+		hashedRange = hashObject({
+			includes: range.includes,
+			lower: range.lower,
+			lowerOpen: range.lowerOpen,
+			upper: range.upper,
+			upperOpen: range.upperOpen,
+		});
+	} else {
+		hashedRange = range;
+	}
+	return hashObject({ range: hashedRange, ...rest });
+}

package/src/queries2/ranges.ts

@@ -0,0 +1,107 @@
+import {
+	assert,
+	CollectionCompoundIndexFilter,
+	CollectionFilter,
+	createCompoundIndexValue,
+	createLowerBoundIndexValue,
+	createUpperBoundIndexValue,
+	isMatchIndexFilter,
+	isRangeIndexFilter,
+	isSortIndexFilter,
+	isStartsWithIndexFilter,
+	MatchCollectionIndexFilter,
+	RangeCollectionIndexFilter,
+	sanitizeIndexValue,
+	SortIndexFilter,
+	StartsWithIndexFilter,
+	StorageSchema,
+} from '@verdant-web/common';
+
+const matchIndexToIdbKeyRange = (filter: MatchCollectionIndexFilter) => {
+	return IDBKeyRange.only(sanitizeIndexValue(filter.equals));
+};
+
+const sortIndexToIdbKeyRange = (filter: SortIndexFilter) => {
+	return undefined;
+};
+
+const rangeIndexToIdbKeyRange = (filter: RangeCollectionIndexFilter) => {
+	const lower = filter.gte || filter.gt;
+	const upper = filter.lte || filter.lt;
+	if (lower === upper) {
+		return IDBKeyRange.only(sanitizeIndexValue(lower));
+	}
+	if (!lower) {
+		return IDBKeyRange.upperBound(sanitizeIndexValue(upper), !!filter.lt);
+	} else if (!upper) {
+		return IDBKeyRange.lowerBound(sanitizeIndexValue(lower), !!filter.gt);
+	} else {
+		return IDBKeyRange.bound(
+			sanitizeIndexValue(lower),
+			sanitizeIndexValue(upper),
+			!!filter.gt,
+			!!filter.lt,
+		);
+	}
+};
+
+const compoundIndexToIdbKeyRange = (
+	// FIXME:
+	schema: any,
+	collection: string,
+	filter: CollectionCompoundIndexFilter,
+) => {
+	// validate the usage of the compound index:
+	// - all match fields must be contiguous at the start of the compound order
+	const indexDefinition =
+		schema.collections[collection].compounds[filter.where];
+	assert(
+		indexDefinition,
+		`Index ${filter.where} does not exist on collection ${collection}`,
+	);
+	const matchedKeys = Object.keys(filter.match).sort(
+		(a, b) => indexDefinition.of.indexOf(a) - indexDefinition.of.indexOf(b),
+	);
+	for (const key of matchedKeys) {
+		if (indexDefinition.of.indexOf(key) !== matchedKeys.indexOf(key)) {
+			throw new Error(
+				`Compound index ${filter.where} does not have ${key} at the start of its order`,
+			);
+		}
+	}
+
+	const matchedValues = matchedKeys.map(
+		(key) => filter.match[key as keyof typeof filter.match] as string | number,
+	);
+
+	// special case: all match fields are specified - we don't need a range
+	// query, just a single key query
+	if (matchedKeys.length === indexDefinition.of.length) {
+		return IDBKeyRange.only(createCompoundIndexValue(...matchedValues));
+	}
+
+	// create our bounds for the matched values
+	const lower = createLowerBoundIndexValue(...matchedValues);
+	const upper = createUpperBoundIndexValue(...matchedValues);
+	return IDBKeyRange.bound(lower, upper);
+};
+
+function startsWithIndexToIdbKeyRange(filter: StartsWithIndexFilter) {
+	const lower = filter.startsWith;
+	const upper = filter.startsWith + '\uffff';
+	return IDBKeyRange.bound(lower, upper);
+}
+
+export function getRange(
+	schema: StorageSchema,
+	collection: string,
+	index?: CollectionFilter,
+) {
+	if (!index) return undefined;
+	if (isRangeIndexFilter(index)) return rangeIndexToIdbKeyRange(index);
+	if (isMatchIndexFilter(index)) return matchIndexToIdbKeyRange(index);
+	if (isSortIndexFilter(index)) return sortIndexToIdbKeyRange(index);
+	if (isStartsWithIndexFilter(index))
+		return startsWithIndexToIdbKeyRange(index);
+	return compoundIndexToIdbKeyRange(schema, collection, index);
+}

package/src/queries2/types.ts

@@ -0,0 +1,12 @@
+import type { FindAllQuery } from './FindAllQuery.js';
+import type { FindPageQuery } from './FindPageQuery.js';
+import type { FindOneQuery } from './FindOneQuery.js';
+import type { FindInfiniteQuery } from './FindInfiniteQuery.js';
+import type { GetQuery } from './GetQuery.js';
+
+export type Query<T> =
+	| FindAllQuery<T>
+	| FindPageQuery<T>
+	| FindOneQuery<T>
+	| FindInfiniteQuery<T>
+	| GetQuery<T>;

package/src/queries2/utils.ts

@@ -0,0 +1,15 @@
+import { Entity } from '../reactives/Entity.js';
+
+function existsFilter<T>(x: T | null): x is T {
+	return x !== null;
+}
+
+export function filterResultSet(results: any): any {
+	if (Array.isArray(results)) {
+		return results.map(filterResultSet).filter(existsFilter);
+	} else if (results instanceof Entity) {
+		return results.deleted ? null : results;
+	} else {
+		return results;
+	}
+}

package/src/reactives/DocumentFamiliyCache.ts

@@ -0,0 +1,289 @@
+import {
+	applyPatch,
+	assignOid,
+	cloneDeep,
+	DocumentBaseline,
+	EventSubscriber,
+	ObjectIdentifier,
+	Operation,
+	StorageFieldSchema,
+} from '@verdant-web/common';
+import { Entity, refreshEntity, StoreTools } from './Entity.js';
+import type { EntityStore } from './EntityStore.js';
+import { WeakRef } from './FakeWeakRef.js';
+import { Context } from '../context.js';
+
+export class DocumentFamilyCache extends EventSubscriber<
+	Record<`change:${string}`, () => void> & {
+		'change:*': (oid: ObjectIdentifier) => void;
+	}
+> {
+	readonly oid: ObjectIdentifier;
+	private operationsMap: Map<ObjectIdentifier, Operation[]>;
+	private unconfirmedOperationsMap: Map<ObjectIdentifier, Operation[]>;
+	private baselinesMap: Map<ObjectIdentifier, DocumentBaseline>;
+
+	private entities: Map<ObjectIdentifier, WeakRef<Entity>> = new Map();
+
+	private context;
+	private storeTools: StoreTools;
+
+	constructor({
+		oid,
+		store,
+		context,
+	}: {
+		oid: ObjectIdentifier;
+		store: EntityStore;
+		context: Context;
+	}) {
+		super();
+		this.oid = oid;
+		this.operationsMap = new Map();
+		this.unconfirmedOperationsMap = new Map();
+		this.baselinesMap = new Map();
+		this.storeTools = {
+			addLocalOperations: store.addLocalOperations,
+			patchCreator: store.meta.patchCreator,
+			addFile: store.files.add,
+			getFile: store.files.get,
+			time: store.meta.time,
+		};
+		this.context = context;
+	}
+
+	insertUnconfirmedOperations = (operations: Operation[]) => {
+		const oidSet = new Set<ObjectIdentifier>();
+		for (const operation of operations) {
+			const { oid } = operation;
+			oidSet.add(oid);
+			const existingOperations = this.unconfirmedOperationsMap.get(oid) || [];
+			existingOperations.push(operation);
+			this.unconfirmedOperationsMap.set(oid, existingOperations);
+		}
+		for (const oid of oidSet) {
+			const entityRef = this.entities.get(oid);
+			const entity = entityRef?.deref();
+			if (entity) {
+				refreshEntity(entity, { isLocal: true });
+				this.emit(`change:${oid}`);
+				this.emit('change:*', oid);
+			}
+		}
+	};
+
+	insertOperations = (
+		operations: Operation[],
+		info: { isLocal: boolean; affectedOids?: Set<ObjectIdentifier> },
+	) => {
+		for (const operation of operations) {
+			const { oid } = operation;
+			info.affectedOids?.add(oid);
+			const existingOperations = this.operationsMap.get(oid) || [];
+			// insert in order of timestamp
+			const index = existingOperations.findIndex(
+				(op) => op.timestamp >= operation.timestamp,
+			);
+			// ensure the operation doesn't already exist in this position
+			if (index !== -1) {
+				if (existingOperations[index].timestamp === operation.timestamp) {
+					continue;
+				}
+				existingOperations.splice(index, 0, operation);
+			} else {
+				existingOperations.push(operation);
+			}
+			this.operationsMap.set(oid, existingOperations);
+
+			// FIXME: seems inefficient
+			const unconfirmedOperations = this.unconfirmedOperationsMap.get(oid);
+			if (unconfirmedOperations) {
+				this.unconfirmedOperationsMap.set(
+					oid,
+					unconfirmedOperations.filter(
+						(op) => op.timestamp !== operation.timestamp,
+					),
+				);
+			}
+		}
+	};
+
+	insertBaselines = (
+		baselines: DocumentBaseline[],
+		{
+			affectedOids,
+		}: { isLocal: boolean; affectedOids?: Set<ObjectIdentifier> },
+	) => {
+		for (const baseline of baselines) {
+			const { oid } = baseline;
+			affectedOids?.add(oid);
+			this.baselinesMap.set(oid, baseline);
+			// drop operations before the baseline
+			const ops = this.operationsMap.get(oid) || [];
+			while (ops[0]?.timestamp < baseline.timestamp) {
+				ops.shift();
+			}
+		}
+	};
+
+	addConfirmedData = ({
+		operations,
+		baselines,
+		reset,
+		isLocal,
+	}: {
+		operations: Operation[];
+		baselines: DocumentBaseline[];
+		reset?: boolean;
+		isLocal?: boolean;
+	}) => {
+		if (reset) {
+			this.operationsMap.clear();
+			this.baselinesMap.clear();
+		}
+		const info = {
+			isLocal: isLocal || false,
+			affectedOids: new Set<ObjectIdentifier>(),
+		};
+		this.insertBaselines(baselines, info);
+		// for reset scenario, don't immediately update entities;
+		// we will update all of them in one go.
+		this.insertOperations(operations, info);
+		if (reset) {
+			for (const entityRef of this.entities.values()) {
+				const entity = entityRef.deref();
+				if (entity) {
+					refreshEntity(entity, info);
+				}
+			}
+		} else {
+			for (const oid of info.affectedOids) {
+				const entityRef = this.entities.get(oid);
+				const entity = entityRef?.deref();
+				if (entity) {
+					refreshEntity(entity, info);
+					this.emit(`change:${oid}`);
+					this.emit('change:*', oid);
+				}
+			}
+		}
+	};
+
+	private applyOperations = (
+		view: any,
+		deleted: boolean,
+		operations: Operation[],
+		after?: string,
+	) => {
+		for (const operation of operations) {
+			if (after && operation.timestamp <= after) {
+				continue;
+			}
+			if (operation.data.op === 'delete') {
+				deleted = true;
+			} else {
+				view = applyPatch(view, operation.data);
+				if (operation.data.op === 'initialize') {
+					deleted = false;
+				}
+			}
+		}
+		return { view, deleted };
+	};
+
+	computeView = (oid: ObjectIdentifier) => {
+		const confirmed = this.computeConfirmedView(oid);
+		const unconfirmedOperations = this.unconfirmedOperationsMap.get(oid) || [];
+		let { view, deleted } = this.applyOperations(
+			confirmed.view,
+			confirmed.deleted,
+			unconfirmedOperations,
+		);
+		if (view) {
+			assignOid(view, oid);
+		}
+		return { view, deleted, lastTimestamp: this.getLastTimestamp(oid) };
+	};
+
+	computeConfirmedView = (oid: ObjectIdentifier) => {
+		const baseline = this.baselinesMap.get(oid);
+		const operations = this.operationsMap.get(oid) || [];
+		let view = cloneDeep(baseline?.snapshot || undefined);
+		view = this.applyOperations(view, !view, operations, baseline?.timestamp);
+		if (view) {
+			assignOid(view, oid);
+		}
+		if (!baseline && !operations.length) {
+			this.context.log(
+				'debug',
+				`Entity ${oid} accessed with no confirmed data`,
+			);
+		}
+		return view;
+	};
+
+	getLastTimestamp = (oid: ObjectIdentifier) => {
+		let operations = this.unconfirmedOperationsMap.get(oid);
+		if (!operations?.length) {
+			operations = this.operationsMap.get(oid) || [];
+		}
+		let logicalTimestamp: string | null = null;
+		if (operations.length) {
+			logicalTimestamp = operations[operations.length - 1]?.timestamp;
+		} else {
+			logicalTimestamp = this.baselinesMap.get(oid)?.timestamp ?? null;
+		}
+		if (!logicalTimestamp) return null;
+		return this.storeTools.time.getWallClockTime(logicalTimestamp);
+	};
+
+	getEntity = (
+		oid: ObjectIdentifier,
+		schema: StorageFieldSchema,
+		parent?: Entity,
+	): Entity => {
+		let entityRef = this.entities.get(oid);
+		let entity = entityRef?.deref();
+		if (!entity) {
+			entity = new Entity({
+				oid,
+				cache: this,
+				fieldSchema: schema,
+				store: this.storeTools,
+				parent,
+			});
+
+			// immediately add to cache and queue a removal if nobody subscribed
+			this.entities.set(oid, new WeakRef(entity));
+		}
+
+		return entity as any;
+	};
+
+	hasOid = (oid: ObjectIdentifier) => {
+		return this.operationsMap.has(oid) || this.baselinesMap.has(oid);
+	};
+
+	dispose = () => {
+		this.entities.forEach((entity) => entity.deref()?.dispose());
+		this.entities.clear();
+	};
+
+	reset = (operations: Operation[], baselines: DocumentBaseline[]) => {
+		const info = { isLocal: false, affectedOids: new Set<ObjectIdentifier>() };
+		this.baselinesMap = new Map(
+			baselines.map((baseline) => [baseline.oid, baseline]),
+		);
+		this.operationsMap = new Map();
+		this.insertOperations(operations, info);
+		for (const oid of this.entities.keys()) {
+			const entityRef = this.entities.get(oid);
+			const entity = entityRef?.deref();
+			if (entity) {
+				refreshEntity(entity, info);
+				this.emit(`change:${oid}`);
+				this.emit('change:*', oid);
+			}
+		}
+	};
+}