@verdant-web/store 3.12.1 → 4.0.0-next.0

package/src/{migration → persistence/idb/queries/migration}/openQueryDatabase.ts

@@ -1,5 +1,4 @@
 import { Migration } from '@verdant-web/common';
-import { Metadata } from '../metadata/Metadata.js';
 import { getDatabaseVersion, openDatabase } from './db.js';
 import { runMigrations } from './migrations.js';
 import { getMigrationPath } from './paths.js';
@@ -12,32 +11,24 @@ export async function openQueryDatabase({
 	version,
 	indexedDB = globalIDB,
 	migrations,
-	meta,
 	context,
 }: {
 	version: number;
 	migrations: Migration<any>[];
 	indexedDB?: IDBFactory;
-	meta: Metadata;
 	context: OpenDocumentDbContext;
 }) {
-	if (context.schema.wip) {
-		throw new Error('Cannot open a production client with a WIP schema!');
-	}
-
-	const currentVersion = await getDatabaseVersion(
-		indexedDB,
-		context.namespace,
-		version,
-		context.log,
-	);
+	const currentVersion = await getDatabaseVersion(indexedDB, context.namespace);
 
 	context.log(
 		'debug',
+		'Opening index database',
+		context.namespace,
 		'Current database version:',
 		currentVersion,
 		'target version:',
 		version,
+		context.schema.wip ? '(wip)' : '',
 	);
 
 	const toRun = getMigrationPath({
@@ -52,7 +43,7 @@ export async function openQueryDatabase({
 			'Migrations to run:',
 			toRun.map((m) => m.version),
 		);
-		await runMigrations({ context, toRun, meta, indexedDB });
+		await runMigrations({ context, toRun, indexedDB });
 	}
 	return openDatabase({
 		indexedDB,

package/src/{migration → persistence/idb/queries/migration}/paths.ts

@@ -1,5 +1,4 @@
-import { Migration } from '@verdant-web/common';
-import { MigrationPathError } from './errors.js';
+import { Migration, VerdantError } from '@verdant-web/common';
 
 export function getMigrationPath({
 	currentVersion,
@@ -16,7 +15,9 @@ export function getMigrationPath({
 		migrations,
 	});
 	if (!path) {
-		throw new MigrationPathError(
+		throw new VerdantError(
+			VerdantError.Code.MigrationPathNotFound,
+			undefined,
 			`No migration path found from ${currentVersion} to ${targetVersion}! This is a bug. If you're seeing this, contact the developer and provide them with the full contents of this message.`,
 		);
 	}

package/src/persistence/idb/queries/migration/types.ts

@@ -0,0 +1,8 @@
+import { Context } from '../../../../context/context.js';
+
+/** During migration, only a partial context is available */
+export type OpenDocumentDbContext = Omit<Context, 'queries' | 'files'>;
+
+export type MigrationDbRef = {
+	current: IDBDatabase;
+};

package/src/persistence/idb/queries/ranges.ts

@@ -0,0 +1,107 @@
+import {
+	assert,
+	CollectionCompoundIndexFilter,
+	CollectionFilter,
+	createCompoundIndexValue,
+	createLowerBoundIndexValue,
+	createUpperBoundIndexValue,
+	isMatchIndexFilter,
+	isRangeIndexFilter,
+	isSortIndexFilter,
+	isStartsWithIndexFilter,
+	MatchCollectionIndexFilter,
+	RangeCollectionIndexFilter,
+	sanitizeIndexValue,
+	SortIndexFilter,
+	StartsWithIndexFilter,
+	StorageSchema,
+} from '@verdant-web/common';
+
+const matchIndexToIdbKeyRange = (filter: MatchCollectionIndexFilter) => {
+	return IDBKeyRange.only(sanitizeIndexValue(filter.equals));
+};
+
+const sortIndexToIdbKeyRange = (filter: SortIndexFilter) => {
+	return undefined;
+};
+
+const rangeIndexToIdbKeyRange = (filter: RangeCollectionIndexFilter) => {
+	const lower = filter.gte || filter.gt;
+	const upper = filter.lte || filter.lt;
+	if (lower === upper) {
+		return IDBKeyRange.only(sanitizeIndexValue(lower));
+	}
+	if (!lower) {
+		return IDBKeyRange.upperBound(sanitizeIndexValue(upper), !!filter.lt);
+	} else if (!upper) {
+		return IDBKeyRange.lowerBound(sanitizeIndexValue(lower), !!filter.gt);
+	} else {
+		return IDBKeyRange.bound(
+			sanitizeIndexValue(lower),
+			sanitizeIndexValue(upper),
+			!!filter.gt,
+			!!filter.lt,
+		);
+	}
+};
+
+const compoundIndexToIdbKeyRange = (
+	// FIXME:
+	schema: any,
+	collection: string,
+	filter: CollectionCompoundIndexFilter,
+) => {
+	// validate the usage of the compound index:
+	// - all match fields must be contiguous at the start of the compound order
+	const indexDefinition =
+		schema.collections[collection].compounds[filter.where];
+	assert(
+		indexDefinition,
+		`Index ${filter.where} does not exist on collection ${collection}`,
+	);
+	const matchedKeys = Object.keys(filter.match).sort(
+		(a, b) => indexDefinition.of.indexOf(a) - indexDefinition.of.indexOf(b),
+	);
+	for (const key of matchedKeys) {
+		if (indexDefinition.of.indexOf(key) !== matchedKeys.indexOf(key)) {
+			throw new Error(
+				`Compound index ${filter.where} does not have ${key} at the start of its order`,
+			);
+		}
+	}
+
+	const matchedValues = matchedKeys.map(
+		(key) => filter.match[key as keyof typeof filter.match] as string | number,
+	);
+
+	// special case: all match fields are specified - we don't need a range
+	// query, just a single key query
+	if (matchedKeys.length === indexDefinition.of.length) {
+		return IDBKeyRange.only(createCompoundIndexValue(...matchedValues));
+	}
+
+	// create our bounds for the matched values
+	const lower = createLowerBoundIndexValue(...matchedValues);
+	const upper = createUpperBoundIndexValue(...matchedValues);
+	return IDBKeyRange.bound(lower, upper);
+};
+
+function startsWithIndexToIdbKeyRange(filter: StartsWithIndexFilter) {
+	const lower = filter.startsWith;
+	const upper = filter.startsWith + '\uffff';
+	return IDBKeyRange.bound(lower, upper);
+}
+
+export function getRange(
+	schema: StorageSchema,
+	collection: string,
+	index?: CollectionFilter,
+) {
+	if (!index) return undefined;
+	if (isRangeIndexFilter(index)) return rangeIndexToIdbKeyRange(index);
+	if (isMatchIndexFilter(index)) return matchIndexToIdbKeyRange(index);
+	if (isSortIndexFilter(index)) return sortIndexToIdbKeyRange(index);
+	if (isStartsWithIndexFilter(index))
+		return startsWithIndexToIdbKeyRange(index);
+	return compoundIndexToIdbKeyRange(schema, collection, index);
+}

package/src/{idb.ts → persistence/idb/util.ts}

@@ -93,6 +93,20 @@ export function getSizeOfObjectStore(
 	});
 }
 
+export async function getSizesOfAllObjectStores(
+	database: IDBDatabase,
+): Promise<{ [storeName: string]: { count: number; size: number } }> {
+	const storeNames = Array.from(database.objectStoreNames);
+	const promises = storeNames.map(async (storeName) => {
+		const result = await getSizeOfObjectStore(database, storeName);
+		return { [storeName]: result };
+	});
+	const results = await Promise.all(promises);
+	return results.reduce((acc, result_1) => {
+		return { ...acc, ...result_1 };
+	}, {});
+}
+
 export function getAllFromObjectStores(db: IDBDatabase, stores: string[]) {
 	const transaction = db.transaction(stores, 'readonly');
 	const promises = stores.map((store) => {
@@ -182,3 +196,64 @@ export function emptyDatabase(db: IDBDatabase) {
 		tx.onerror = () => reject(tx.error);
 	});
 }
+
+export async function copyDatabase(from: IDBDatabase, to: IDBDatabase) {
+	await emptyDatabase(to);
+	const records = await getAllFromObjectStores(
+		from,
+		Array.from(from.objectStoreNames),
+	);
+	const writeTx = to.transaction(Array.from(to.objectStoreNames), 'readwrite');
+	for (let i = 0; i < records.length; i++) {
+		const store = writeTx.objectStore(from.objectStoreNames[i]);
+		for (const record of records[i]) {
+			store.add(record);
+		}
+	}
+	return new Promise<void>((resolve, reject) => {
+		writeTx.oncomplete = () => resolve();
+		writeTx.onerror = () => reject(writeTx.error);
+	});
+}
+
+export function openDatabase(
+	name: string,
+	expectedVersion: number,
+	indexedDB: IDBFactory = window.indexedDB,
+) {
+	return new Promise<IDBDatabase>((resolve, reject) => {
+		const req = indexedDB.open(name, expectedVersion);
+		req.onsuccess = () => {
+			resolve(req.result);
+		};
+		req.onerror = () => {
+			reject(req.error);
+		};
+		req.onblocked = () => {
+			reject(new Error('Database blocked'));
+		};
+		req.onupgradeneeded = (event) => {
+			const db = req.result;
+			if (db.version !== expectedVersion) {
+				db.close();
+				reject(
+					new Error(
+						`Migration error: database version changed unexpectedly when reading current data. Expected ${expectedVersion}, got ${db.version}`,
+					),
+				);
+			}
+		};
+	});
+}
+
+export function getMetadataDbName(namespace: string) {
+	return [namespace, 'meta'].join('_');
+}
+
+export function getDocumentDbName(namespace: string) {
+	return [namespace, 'collections'].join('_');
+}
+
+export function getNamespaceFromDatabaseInfo(info: IDBDatabaseInfo) {
+	return info.name?.split('_')[0];
+}

package/src/persistence/interfaces.ts

@@ -0,0 +1,240 @@
+import {
+	CollectionFilter,
+	DocumentBaseline,
+	FileData,
+	ObjectIdentifier,
+	Operation,
+} from '@verdant-web/common';
+import { Context, InitialContext } from '../context/context.js';
+
+export interface AckInfo {
+	type: 'ack';
+	globalAckTimestamp: string | null;
+}
+
+export interface LocalReplicaInfo {
+	type: 'localReplicaInfo';
+	id: string;
+	userId: string | undefined;
+	ackedLogicalTime: string | null;
+	lastSyncedLogicalTime: string | null;
+}
+
+export type ClientOperation = Operation & {
+	isLocal: boolean;
+};
+
+export interface MetadataExport {
+	operations: Operation[];
+	baselines: DocumentBaseline[];
+	localReplica?: LocalReplicaInfo;
+	schemaVersion: number;
+}
+
+export interface ExportedData {
+	data: MetadataExport;
+	fileData: Array<Omit<PersistedFileData, 'file'>>;
+	files: File[];
+}
+
+export type AbstractTransaction = unknown;
+export type QueryMode = 'readwrite' | 'readonly';
+export interface CommonQueryOptions {
+	transaction?: AbstractTransaction;
+	mode?: QueryMode;
+}
+export type Iterator<T> = (item: T) => void | boolean;
+
+export interface PersistenceMetadataDb {
+	transaction(opts: {
+		mode?: QueryMode;
+		storeNames: string[];
+		abort?: AbortSignal;
+	}): AbstractTransaction;
+	dispose(): void | Promise<void>;
+
+	// infos
+	getAckInfo(): Promise<AckInfo>;
+	setGlobalAck(ack: string): Promise<void>;
+	getLocalReplica(opts?: CommonQueryOptions): Promise<LocalReplicaInfo>;
+	updateLocalReplica(
+		data: Partial<LocalReplicaInfo>,
+		opts?: CommonQueryOptions,
+	): Promise<void>;
+
+	// baselines
+	iterateDocumentBaselines(
+		rootOid: string,
+		iterator: Iterator<DocumentBaseline>,
+		opts?: CommonQueryOptions,
+	): Promise<void>;
+	iterateCollectionBaselines(
+		collection: string,
+		iterator: Iterator<DocumentBaseline>,
+		opts?: CommonQueryOptions,
+	): Promise<void>;
+	iterateAllBaselines(
+		iterator: Iterator<DocumentBaseline>,
+		opts?: CommonQueryOptions,
+	): Promise<void>;
+	getBaseline(
+		oid: string,
+		opts?: CommonQueryOptions,
+	): Promise<DocumentBaseline>;
+	setBaselines(
+		baselines: DocumentBaseline[],
+		opts?: CommonQueryOptions,
+	): Promise<void>;
+	deleteBaseline(oid: string, opts?: CommonQueryOptions): Promise<void>;
+
+	// operations
+	iterateDocumentOperations(
+		rootOid: string,
+		iterator: Iterator<ClientOperation>,
+		opts?: CommonQueryOptions & {
+			to?: string | null;
+		},
+	): Promise<void>;
+	iterateEntityOperations(
+		oid: string,
+		iterator: Iterator<ClientOperation>,
+		opts?: CommonQueryOptions & { to?: string | null },
+	): Promise<void>;
+	iterateCollectionOperations(
+		collection: string,
+		iterator: Iterator<ClientOperation>,
+		opts?: CommonQueryOptions,
+	): Promise<void>;
+	iterateLocalOperations(
+		iterator: Iterator<ClientOperation>,
+		opts?: CommonQueryOptions & {
+			before?: string | null;
+			after?: string | null;
+		},
+	): Promise<void>;
+	/** Iterates over operations for an entity for processing and deletes them as it goes. */
+	consumeEntityOperations(
+		oid: string,
+		iterator: Iterator<ClientOperation>,
+		opts?: CommonQueryOptions & { to?: string | null },
+	): Promise<void>;
+	iterateAllOperations(
+		iterator: Iterator<ClientOperation>,
+		opts?: CommonQueryOptions & {
+			before?: string | null;
+			from?: string | null;
+		},
+	): Promise<void>;
+	/**
+	 * @returns a list of all document (root) OIDs affected by the adds.
+	 */
+	addOperations(
+		ops: ClientOperation[],
+		opts?: CommonQueryOptions,
+	): Promise<ObjectIdentifier[]>;
+
+	/* WARNING: deletes all data */
+	reset(opts?: {
+		clearReplica?: boolean;
+		transaction?: AbstractTransaction;
+	}): Promise<void>;
+
+	stats(): Promise<{
+		operationsSize: { count: number; size: number };
+		baselinesSize: { count: number; size: number };
+	}>;
+}
+
+export interface PersistenceQueryDb {
+	transaction(opts: {
+		mode?: QueryMode;
+		storeNames: string[];
+		abort?: AbortSignal;
+	}): AbstractTransaction;
+	dispose(): void | Promise<void>;
+
+	findOneOid(opts: {
+		collection: string;
+		index?: CollectionFilter;
+	}): Promise<ObjectIdentifier | null>;
+	findAllOids(opts: {
+		collection: string;
+		index?: CollectionFilter;
+		limit?: number;
+		offset?: number;
+	}): Promise<{ result: ObjectIdentifier[]; hasNextPage: boolean }>;
+
+	saveEntities(
+		entities: { oid: ObjectIdentifier; getSnapshot: () => any }[],
+		opts?: CommonQueryOptions & { abort?: AbortSignal },
+	): Promise<void>;
+
+	reset(opts?: { transaction?: AbstractTransaction }): Promise<void>;
+
+	stats(): Promise<Record<string, { count: number; size: number }>>;
+}
+
+export interface PersistedFileData extends FileData {
+	deletedAt: number | null;
+}
+
+export interface PersistenceFileDb {
+	transaction(opts: {
+		mode?: QueryMode;
+		storeNames: string[];
+		abort?: AbortSignal;
+	}): AbstractTransaction;
+	dispose(): void | Promise<void>;
+
+	add(
+		file: FileData,
+		options?: { transaction?: AbstractTransaction; downloadRemote?: boolean },
+	): Promise<void>;
+	markUploaded(
+		fileId: string,
+		options?: { transaction?: AbstractTransaction },
+	): Promise<void>;
+	get(
+		fileId: string,
+		options?: { transaction?: AbstractTransaction },
+	): Promise<PersistedFileData | null>;
+	delete(
+		fileId: string,
+		options?: { transaction?: AbstractTransaction },
+	): Promise<void>;
+	markPendingDelete(
+		fileId: string,
+		options?: { transaction?: AbstractTransaction },
+	): Promise<void>;
+	listUnsynced(options?: {
+		transaction?: AbstractTransaction;
+	}): Promise<PersistedFileData[]>;
+	resetSyncedStatusSince(
+		since: string | null,
+		options?: { transaction?: AbstractTransaction },
+	): Promise<void>;
+	iterateOverPendingDelete(
+		iterator: (file: PersistedFileData, store: IDBObjectStore) => void,
+		options?: { transaction?: IDBTransaction },
+	): Promise<void>;
+	getAll(options?: {
+		transaction?: AbstractTransaction;
+	}): Promise<PersistedFileData[]>;
+	stats(): Promise<{ size: { count: number; size: number } }>;
+}
+
+export interface PersistenceImplementation {
+	openMetadata(ctx: InitialContext): Promise<PersistenceMetadataDb>;
+	openQueries(ctx: Omit<Context, 'queries'>): Promise<PersistenceQueryDb>;
+	openFiles(
+		ctx: Omit<Context, 'files' | 'queries'>,
+	): Promise<PersistenceFileDb>;
+	/** Copies all data (metadata/document queries/files) from one namespace to another. */
+	copyNamespace(from: string, to: string, ctx: InitialContext): Promise<void>;
+	/** Returns a list of all persisted namespaces visible to this app. */
+	getNamespaces(): Promise<string[]>;
+	/** Deletes all data from a particular namespace. */
+	deleteNamespace(namespace: string, ctx: InitialContext): Promise<void>;
+	/** Gets the schema version of the given namespace */
+	getNamespaceVersion(namespace: string): Promise<number>;
+}