@verdant-web/store 4.6.1 → 5.0.1

package/src/persistence/migration/finalize.ts

@@ -4,17 +4,20 @@ import {
 	Migration,
 	MigrationEngine,
 } from '@verdant-web/common';
+import { ContextWithoutPersistence } from '../../context/context.js';
 import { ClientOperation, PersistenceDocumentDb } from '../interfaces.js';
-import { OpenDocumentDbContext } from './types.js';
+import { PersistenceMetadata } from '../PersistenceMetadata.js';
 
 export async function finalizeMigration({
 	ctx,
 	documents,
 	migration,
+	meta,
 	engine,
 }: {
-	ctx: OpenDocumentDbContext;
+	ctx: ContextWithoutPersistence;
 	documents: PersistenceDocumentDb;
+	meta: PersistenceMetadata;
 	migration: Migration<any>;
 	engine: MigrationEngine;
 }) {
@@ -37,6 +40,7 @@ export async function finalizeMigration({
 		currentVersion: migration.oldSchema.version,
 		newVersion: migration.newSchema.version,
 		ctx,
+		meta,
 	});
 
 	// once the schema is ready, we can write back the migrated documents
@@ -58,7 +62,7 @@ export async function finalizeMigration({
 		const snapshots = await Promise.all(
 			oids.map(async (oid) => {
 				try {
-					const snap = await ctx.meta.getDocumentSnapshot(oid);
+					const snap = await meta.getDocumentSnapshot(oid);
 					return [oid, snap];
 				} catch (e) {
 					// this seems to happen with baselines/ops which are not fully
@@ -103,16 +107,18 @@ async function getDocsWithUnappliedMigrations({
 	currentVersion,
 	newVersion: _,
 	ctx,
+	meta,
 }: {
 	currentVersion: number;
 	newVersion: number;
-	ctx: OpenDocumentDbContext;
+	ctx: ContextWithoutPersistence;
+	meta: PersistenceMetadata;
 }) {
 	// scan for all operations in metadata after the current version.
 	// this could be more efficient if also filtering below or equal newVersion but
 	// that seems so unlikely in practice...
 	const unappliedOperations: ClientOperation[] = [];
-	await ctx.meta.iterateAllOperations(
+	await meta.iterateAllOperations(
 		(op) => {
 			unappliedOperations.push(op);
 		},

package/src/persistence/migration/migrate.ts

@@ -1,16 +1,19 @@
 import { Migration } from '@verdant-web/common';
+import { ContextWithoutPersistence } from '../../context/context.js';
 import { ShutdownHandler } from '../../context/ShutdownHandler.js';
 import { PersistenceNamespace } from '../interfaces.js';
+import { PersistenceMetadata } from '../PersistenceMetadata.js';
 import { getMigrationEngine } from './engine.js';
 import { finalizeMigration } from './finalize.js';
 import { getMigrationPath } from './paths.js';
-import { OpenDocumentDbContext } from './types.js';
 
 export async function migrate({
 	context,
 	version,
+	meta,
 }: {
-	context: OpenDocumentDbContext;
+	context: ContextWithoutPersistence;
+	meta: PersistenceMetadata;
 	version: number;
 }) {
 	const ns = await context.persistence.openNamespace(
@@ -45,7 +48,7 @@ export async function migrate({
 				'Migrations to run:',
 				toRun.map((m) => m.version),
 			);
-			await runMigrations({ context, ns, toRun });
+			await runMigrations({ context, ns, toRun, meta });
 		}
 	});
 }
@@ -63,10 +66,12 @@ export async function runMigrations({
 	context,
 	toRun,
 	ns,
+	meta,
 }: {
-	context: OpenDocumentDbContext;
+	context: ContextWithoutPersistence;
 	toRun: Migration<any>[];
 	ns: PersistenceNamespace;
+	meta: PersistenceMetadata;
 }) {
 	// disable rebasing for the duration of migrations
 	context.pauseRebasing = true;
@@ -76,16 +81,16 @@ export async function runMigrations({
 			'info',
 			`🚀 Running migration v${migration.oldSchema.version} -> v${migration.newSchema.version}`,
 		);
-		const migrationContext = {
-			...context,
+		const migrationContext = context.cloneWithOptions({
 			schema: migration.oldSchema,
-			shutdownHandler: new ShutdownHandler(context.log),
-		};
+			persistenceShutdownHandler: new ShutdownHandler(context.log),
+		});
 		// this will only write to our metadata store via operations!
 		const engine = await getMigrationEngine({
 			migration,
 			context: migrationContext,
 			ns,
+			meta,
 		});
 		try {
 			context.log(
@@ -98,6 +103,7 @@ export async function runMigrations({
 				migration.newSchema.version,
 			);
 			await migration.migrate(engine);
+			context.log('debug', 'Awaiting remaining migration tasks');
 			// wait on any out-of-band async operations to complete
 			await Promise.all(engine.awaitables);
 		} catch (err) {
@@ -136,6 +142,7 @@ export async function runMigrations({
 			migration,
 			engine,
 			documents: upgradedDocuments,
+			meta,
 		});
 		await upgradedDocuments.close();
 

package/src/persistence/persistence.ts

@@ -1,5 +1,5 @@
-import { EventSubscriber, getOidRoot, VerdantError } from '@verdant-web/common';
-import { Context, InitialContext } from '../context/context.js';
+import { getOidRoot, VerdantError } from '@verdant-web/common';
+import { Context } from '../context/context.js';
 import { ShutdownHandler } from '../context/ShutdownHandler.js';
 import { getWipNamespace } from '../utils/wip.js';
 import { ExportedData } from './interfaces.js';
@@ -8,28 +8,26 @@ import { PersistenceFiles } from './PersistenceFiles.js';
 import { PersistenceMetadata } from './PersistenceMetadata.js';
 import { PersistenceDocuments } from './PersistenceQueries.js';
 
-export async function initializePersistence(
-	ctx: InitialContext,
-): Promise<Context> {
-	let context = ctx as any as Context;
+export async function initializePersistence(ctx: Context) {
+	const initialSchema = ctx.schema;
 	if (ctx.schema.wip) {
 		// this is a WIP database, so we need to create a new namespace for the WIP data.
-		context.namespace = getWipNamespace(ctx.originalNamespace, ctx.schema);
-		context.log('info', 'Switched to WIP namespace', context.namespace);
+		ctx.namespace = getWipNamespace(ctx.originalNamespace, ctx.schema);
+		ctx.log('info', '🔨', 'Switched to WIP namespace', ctx.namespace);
 		// check if this WIP database is already in use
-		const namespaces = await context.persistence.getNamespaces();
+		const namespaces = await ctx.persistence.getNamespaces();
 
-		if (!namespaces.includes(context.namespace)) {
+		if (!namespaces.includes(ctx.namespace)) {
 			// copy all data to WIP namespace -- from the current version of local
 			// data, not the WIP schema version. this may not be n-1, we might
 			// be loading a WIP schema over older data.
-			const currentVersion = await context.persistence.getNamespaceVersion(
-				context.originalNamespace,
+			const currentVersion = await ctx.persistence.getNamespaceVersion(
+				ctx.originalNamespace,
 			);
 
 			if (currentVersion === 0) {
 				// there is no existing data. nothing to copy.
-				context.log('debug', 'No existing data to copy to WIP namespace');
+				ctx.log('debug', 'No existing data to copy to WIP namespace');
 			} else {
 				const currentSchema = ctx.oldSchemas?.find(
 					(s) => s.version === currentVersion,
@@ -41,52 +39,62 @@ export async function initializePersistence(
 						`Trying to open WIP database for version ${ctx.schema.version}, but the current local data is version ${currentVersion} and a historical schema for that version is not available.`,
 					);
 				}
-				context.log(
+				ctx.log(
 					'info',
-					`Copying data from ${context.originalNamespace} to ${context.namespace}`,
+					`Copying data from ${ctx.originalNamespace} to ${ctx.namespace}`,
 				);
-				await context.persistence.copyNamespace(
-					context.originalNamespace,
-					context.namespace,
+				await ctx.persistence.copyNamespace(
+					ctx.originalNamespace,
+					ctx.namespace,
 					// needs to be the original schema; the copy should be of the original
 					// data and schema structure; the WIP schema migration application happens
 					// below.
-					{
-						...context,
+					ctx.cloneWithOptions({
 						schema: currentSchema,
-					},
+					}),
 				);
 			}
 		}
 	}
 
-	const namespace = await ctx.persistence.openNamespace(
-		context.namespace,
-		context,
-	);
+	const namespace = await ctx.persistence.openNamespace(ctx.namespace, ctx);
 
-	context.log('info', 'Opening persistence metadata');
-	context.meta = new PersistenceMetadata(
-		await namespace.openMetadata(ctx),
-		ctx,
-	);
+	ctx.log('info', 'Opening persistence metadata', ctx.namespace);
+	const meta = new PersistenceMetadata(await namespace.openMetadata(ctx), ctx);
 
-	context.log('info', 'Opening persistence files');
-	context.files = new PersistenceFiles(
-		await namespace.openFiles(context),
-		context,
-	);
+	ctx.log('info', 'Opening persistence files', ctx.namespace);
+	const files = new PersistenceFiles(await namespace.openFiles(ctx), ctx);
 
-	context.log('info', 'Migrating document database');
+	ctx.log('info', 'Migrating document database');
 	await migrate({
-		context,
+		context: ctx,
 		version: ctx.schema.version,
+		meta,
 	});
 
-	context.log('info', 'Opening persistence documents');
-	context.documents = new PersistenceDocuments(
-		await namespace.openDocuments(context),
-		context,
+	ctx.log('info', 'Opening persistence documents');
+	if (ctx.schema.version <= 0) {
+		// debugging....
+		if (ctx.schema !== initialSchema) {
+			ctx.log(
+				'critical',
+				'Schema at initialization does not match original schema. This is likely a bug in Verdant!',
+			);
+			throw new VerdantError(
+				VerdantError.Code.ConfigurationError,
+				undefined,
+				`Schema at initialization does not match original schema. This is likely a bug in Verdant!`,
+			);
+		}
+		throw new VerdantError(
+			VerdantError.Code.ConfigurationError,
+			undefined,
+			`Schema version must be greater than 0. Found version ${ctx.schema.version} with collections [${Object.keys(ctx.schema.collections).join(', ')}]\n${JSON.stringify(ctx.schema)}`,
+		);
+	}
+	const documents = new PersistenceDocuments(
+		await namespace.openDocuments(ctx),
+		ctx,
 	);
 
 	if (!ctx.schema.wip) {
@@ -100,7 +108,7 @@ export async function initializePersistence(
 		}
 	}
 
-	return context;
+	return { meta, files, documents };
 }
 
 export async function importPersistence(
@@ -123,26 +131,17 @@ export async function importPersistence(
 	// using a new namespace to put all of this into a temporary zone
 	const importedNamespace = `@@import_${Date.now()}`;
 
-	const importedContext = await initializePersistence({
-		...ctx,
+	const importedContext = ctx.cloneWithOptions({
 		schema: exportedSchema,
 		namespace: importedNamespace,
-		originalNamespace: importedNamespace,
-		// no-op entity events -- don't need to inform queries of changes.
-		entityEvents: new EventSubscriber(),
-		internalEvents: new EventSubscriber(),
-		globalEvents: new EventSubscriber(),
-		config: {
-			...ctx.config,
-			persistence: {
-				...ctx.config.persistence,
-				disableRebasing: true,
-			},
-		},
+		disableRebasing: true,
 		persistenceShutdownHandler: new ShutdownHandler(ctx.log),
 	});
+	await importedContext.reinitialize();
+	const importedMeta = await importedContext.meta;
+
 	// load imported data into persistence
-	await importedContext.meta.resetFrom(exportedData.data);
+	await importedMeta.resetFrom(exportedData.data);
 	// need to write indexes here!
 	const affectedOids = new Set<string>();
 	for (const baseline of exportedData.data.baselines) {
@@ -153,15 +152,15 @@ export async function importPersistence(
 	}
 	const toSave = await Promise.all(
 		Array.from(affectedOids).map(async (oid) => {
-			const snapshot = await importedContext.meta.getDocumentSnapshot(oid);
+			const snapshot = await importedMeta.getDocumentSnapshot(oid);
 			return {
 				oid,
 				getSnapshot: () => snapshot,
 			};
 		}),
 	);
-	await importedContext.documents.saveEntities(toSave);
-	await importedContext.files.import(exportedData);
+	await (await importedContext.documents).saveEntities(toSave);
+	await (await importedContext.files).import(exportedData);
 
 	ctx.log('debug', 'Imported data into temporary namespace', importedNamespace);
 
@@ -172,13 +171,14 @@ export async function importPersistence(
 		// an upgrade of the imported data is needed ; it's an older version
 		// of the schema.
 
-		// upgrade the imported data to the latest schema
-		const currentSchema = ctx.schema;
-		const upgradedContext = await initializePersistence({
-			...importedContext,
+		// upgrade the imported data to the latest schema by re-initializing
+		// a context at the latest version, pointing at the imported namespace
+		const upgradedContext = importedContext.cloneWithOptions({
 			persistenceShutdownHandler: new ShutdownHandler(ctx.log),
-			schema: currentSchema,
+			schema: ctx.schema,
+			oldSchemas: ctx.oldSchemas,
 		});
+		await upgradedContext.reinitialize();
 
 		ctx.log('debug', 'Upgraded imported data to current schema');
 
@@ -192,15 +192,17 @@ export async function importPersistence(
 
 	// copy the imported data into the current namespace
 	await ctx.persistence.copyNamespace(importedNamespace, ctx.namespace, ctx);
+	ctx.log('debug', 'Copied imported data to primary namespace');
 
 	// restart the persistence layer
-	await initializePersistence(ctx);
+	await ctx.reinitialize();
+	ctx.log('debug', 'Reinitialized primary persistence layer');
 
 	// verify integrity -- this can only be done if imported data was same
 	// version as current schema, because migrations could add or remove
 	// operations. still, it's a good sanity check.
 	if (exportedData.data.schemaVersion === ctx.schema.version) {
-		const stats = await ctx.meta.stats();
+		const stats = await (await ctx.meta).stats();
 		if (stats.operationsSize.count !== exportedData.data.operations.length) {
 			ctx.log(
 				'critical',
@@ -231,6 +233,15 @@ export async function importPersistence(
 				'Imported documents count mismatch',
 			);
 		}
+	} else {
+		ctx.log(
+			'debug',
+			'Skipping integrity check due to schema version mismatch (not an error)',
+			{
+				exportedVersion: exportedData.data.schemaVersion,
+				currentVersion: ctx.schema.version,
+			},
+		);
 	}
 
 	ctx.log('debug', 'Data copied to primary namespace');

package/src/queries/FindAllQuery.ts

@@ -23,7 +23,9 @@ export class FindAllQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result: oids } = await this.context.documents.findAllOids({
+		const { result: oids } = await (
+			await this.context.documents
+		).findAllOids({
 			collection: this.collection,
 			index: this.index,
 		});

package/src/queries/FindInfiniteQuery.ts

@@ -37,7 +37,9 @@ export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await this.context.documents.findAllOids({
+		const { result, hasNextPage } = await (
+			await this.context.documents
+		).findAllOids({
 			collection: this.collection,
 			limit: this._pageSize * this._upToPage,
 			offset: 0,
@@ -48,7 +50,9 @@ export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
 	};
 
 	public loadMore = async () => {
-		const { result, hasNextPage } = await this.context.documents.findAllOids({
+		const { result, hasNextPage } = await (
+			await this.context.documents
+		).findAllOids({
 			collection: this.collection,
 			limit: this._pageSize,
 			offset: this._pageSize * this._upToPage,

package/src/queries/FindOneQuery.ts

@@ -23,7 +23,9 @@ export class FindOneQuery<T> extends BaseQuery<T | null> {
 	}
 
 	protected run = async () => {
-		const oid = await this.context.documents.findOneOid({
+		const oid = await (
+			await this.context.documents
+		).findOneOid({
 			collection: this.collection,
 			index: this.index,
 		});

package/src/queries/FindPageQuery.ts

@@ -48,7 +48,9 @@ export class FindPageQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await this.context.documents.findAllOids({
+		const { result, hasNextPage } = await (
+			await this.context.documents
+		).findAllOids({
 			collection: this.collection,
 			index: this.index,
 			limit: this._pageSize,

package/src/sync/PresenceManager.ts

@@ -1,13 +1,13 @@
 import {
-	ServerMessage,
-	EventSubscriber,
-	Batcher,
 	Batch,
+	Batcher,
+	EventSubscriber,
+	ServerMessage,
 	VerdantInternalPresence,
 	initialInternalPresence,
 } from '@verdant-web/common';
-import type { UserInfo } from '../index.js';
 import { Context } from '../context/context.js';
+import type { UserInfo } from '../index.js';
 import { LocalReplicaInfo } from '../persistence/interfaces.js';
 
 export const HANDLE_MESSAGE = Symbol('handleMessage');
@@ -90,9 +90,12 @@ export class PresenceManager<
 		this.self.replicaId = '';
 
 		// set the local replica ID as soon as it's loaded
-		ctx.meta.getLocalReplica().then((info) => {
-			this.self.replicaId = info.id;
-		});
+		ctx.waitForInitialization
+			.then(() => ctx.meta)
+			.then((meta) => meta.getLocalReplica())
+			.then((info) => {
+				this.self.replicaId = info.id;
+			});
 
 		this._updateBatcher = new Batcher(this.flushPresenceUpdates);
 		this._updateBatch = this._updateBatcher.add({

package/src/sync/PushPullSync.ts

@@ -140,7 +140,9 @@ export class PushPullSync
 			if (message.ackThisNonce) {
 				this.ctx.log('debug', 'Sending sync ack', message.ackThisNonce);
 				await this.sendRequest([
-					await this.ctx.meta.messageCreator.createAck(message.ackThisNonce),
+					await (
+						await this.ctx.meta
+					).messageCreator.createAck(message.ackThisNonce),
 				]);
 			}
 		}
@@ -210,10 +212,10 @@ export class PushPullSync
 		// will include the client's own presence info and fill in missing profile
 		// data on the first request. otherwise it would have to wait for the second.
 		this.sendRequest([
-			await this.ctx.meta.messageCreator.createPresenceUpdate(
-				this.presence.self,
-			),
-			await this.ctx.meta.messageCreator.createSyncStep1(),
+			await (
+				await this.ctx.meta
+			).messageCreator.createPresenceUpdate(this.presence.self),
+			await (await this.ctx.meta).messageCreator.createSyncStep1(),
 		]);
 	};
 
@@ -227,7 +229,7 @@ export class PushPullSync
 
 	syncOnce = async () => {
 		await this.sendRequest([
-			await this.ctx.meta.messageCreator.createSyncStep1(),
+			await (await this.ctx.meta).messageCreator.createSyncStep1(),
 		]);
 	};
 

package/src/sync/Sync.ts

@@ -10,16 +10,16 @@ import {
 	VerdantError,
 	VerdantInternalPresence,
 } from '@verdant-web/common';
-import { HANDLE_MESSAGE, PresenceManager } from './PresenceManager.js';
+import { Context } from '../context/context.js';
+import { attemptToRegisterBackgroundSync } from './background.js';
 import { FilePullResult, FileSync, FileUploadResult } from './FileSync.js';
+import { HANDLE_MESSAGE, PresenceManager } from './PresenceManager.js';
 import { PushPullSync } from './PushPullSync.js';
 import {
 	ServerSyncEndpointProvider,
 	ServerSyncEndpointProviderConfig,
 } from './ServerSyncEndpointProvider.js';
 import { WebSocketSync } from './WebSocketSync.js';
-import { Context } from '../context/context.js';
-import { attemptToRegisterBackgroundSync } from './background.js';
 
 type SyncEvents = {
 	onlineChange: (isOnline: boolean) => void;
@@ -282,6 +282,11 @@ export class ServerSync<Presence = any, Profile = any>
 				this.handleBroadcastChannelMessage,
 			);
 		}
+		ctx.log(
+			'info',
+			'Sync initialized with transport:',
+			initialTransport ?? 'pull',
+		);
 		if (initialTransport === 'realtime') {
 			this.activeSync = this.webSocketSync;
 		} else {
@@ -290,8 +295,7 @@ export class ServerSync<Presence = any, Profile = any>
 
 		this.presence.subscribe('update', this.handlePresenceUpdate);
 
-		ctx.meta.events.subscribe('syncMessage', this.send);
-
+		ctx.internalEvents.subscribe('outgoingSyncMessage', this.send);
 		this.webSocketSync.subscribe('message', this.handleMessage);
 		this.webSocketSync.subscribe('onlineChange', this.handleOnlineChange);
 
@@ -384,11 +388,11 @@ export class ServerSync<Presence = any, Profile = any>
 					baselines: message.baselines,
 				});
 				if (message.globalAckTimestamp) {
-					await this.ctx.meta.setGlobalAck(message.globalAckTimestamp);
+					await (await this.ctx.meta).setGlobalAck(message.globalAckTimestamp);
 				}
 				break;
 			case 'global-ack':
-				await this.ctx.meta.setGlobalAck(message.timestamp);
+				await (await this.ctx.meta).setGlobalAck(message.timestamp);
 				break;
 			case 'sync-resp':
 				this._activelySyncing = true;
@@ -400,10 +404,10 @@ export class ServerSync<Presence = any, Profile = any>
 				});
 
 				if (message.globalAckTimestamp) {
-					await this.ctx.meta.setGlobalAck(message.globalAckTimestamp);
+					await (await this.ctx.meta).setGlobalAck(message.globalAckTimestamp);
 				}
 
-				await this.ctx.meta.updateLastSynced(message.ackedTimestamp);
+				await (await this.ctx.meta).updateLastSynced(message.ackedTimestamp);
 				this._activelySyncing = false;
 				this.emit('syncingChange', false);
 				this._hasSynced = true;
@@ -411,13 +415,15 @@ export class ServerSync<Presence = any, Profile = any>
 				break;
 			case 'need-since':
 				this.emit('serverReset', message.since);
-				this.ctx.files.onServerReset(message.since);
+				(await this.ctx.files).onServerReset(message.since);
 				this.activeSync.send(
-					await this.ctx.meta.messageCreator.createSyncStep1(message.since),
+					await (
+						await this.ctx.meta
+					).messageCreator.createSyncStep1(message.since),
 				);
 				break;
 			case 'server-ack':
-				await this.ctx.meta.updateLastSynced(message.timestamp);
+				await (await this.ctx.meta).updateLastSynced(message.timestamp);
 		}
 
 		// avoid rebroadcasting messages
@@ -430,7 +436,7 @@ export class ServerSync<Presence = any, Profile = any>
 
 		// update presence if necessary
 		this.presence[HANDLE_MESSAGE](
-			await this.ctx.meta.getLocalReplica(),
+			await (await this.ctx.meta).getLocalReplica(),
 			message,
 		);
 	};
@@ -439,7 +445,7 @@ export class ServerSync<Presence = any, Profile = any>
 
 		// if online, attempt to upload any unsynced files.
 		if (online) {
-			const unsyncedFiles = await this.ctx.files.listUnsynced();
+			const unsyncedFiles = await (await this.ctx.files).listUnsynced();
 			const results = await Promise.allSettled(
 				unsyncedFiles.map((file) => this.fileSync.uploadFile(file)),
 			);
@@ -458,7 +464,9 @@ export class ServerSync<Presence = any, Profile = any>
 		presence?: Presence;
 		internal?: VerdantInternalPresence;
 	}) => {
-		this.send(await this.ctx.meta.messageCreator.createPresenceUpdate(data));
+		this.send(
+			await (await this.ctx.meta).messageCreator.createPresenceUpdate(data),
+		);
 	};
 
 	setMode = (transport: SyncTransportMode) => {
@@ -512,7 +520,7 @@ export class ServerSync<Presence = any, Profile = any>
 			if (message.type === 'sync' || message.type === 'op') {
 				rewriteAuthzOriginator(message, userId);
 			}
-			await this.activeSync.send(message);
+			this.activeSync.send(message);
 			this.onOutgoingMessage?.(message);
 		}
 	};
@@ -570,10 +578,12 @@ export class ServerSync<Presence = any, Profile = any>
 	};
 
 	public start = () => {
+		this.ctx.log('info', 'Starting sync');
 		return this.activeSync.start();
 	};
 
 	public stop = () => {
+		this.ctx.log('info', 'Stopping sync');
 		return this.activeSync.stop();
 	};