@verdant-web/store 4.6.0 → 5.0.0

package/src/persistence/persistence.ts

@@ -1,5 +1,5 @@
-import { EventSubscriber, getOidRoot, VerdantError } from '@verdant-web/common';
-import { Context, InitialContext } from '../context/context.js';
+import { getOidRoot, VerdantError } from '@verdant-web/common';
+import { Context } from '../context/context.js';
 import { ShutdownHandler } from '../context/ShutdownHandler.js';
 import { getWipNamespace } from '../utils/wip.js';
 import { ExportedData } from './interfaces.js';
@@ -8,28 +8,26 @@ import { PersistenceFiles } from './PersistenceFiles.js';
 import { PersistenceMetadata } from './PersistenceMetadata.js';
 import { PersistenceDocuments } from './PersistenceQueries.js';
 
-export async function initializePersistence(
-	ctx: InitialContext,
-): Promise<Context> {
-	let context = ctx as any as Context;
+export async function initializePersistence(ctx: Context) {
+	const initialSchema = ctx.schema;
 	if (ctx.schema.wip) {
 		// this is a WIP database, so we need to create a new namespace for the WIP data.
-		context.namespace = getWipNamespace(ctx.originalNamespace, ctx.schema);
-		context.log('info', 'Switched to WIP namespace', context.namespace);
+		ctx.namespace = getWipNamespace(ctx.originalNamespace, ctx.schema);
+		ctx.log('info', '🔨', 'Switched to WIP namespace', ctx.namespace);
 		// check if this WIP database is already in use
-		const namespaces = await context.persistence.getNamespaces();
+		const namespaces = await ctx.persistence.getNamespaces();
 
-		if (!namespaces.includes(context.namespace)) {
+		if (!namespaces.includes(ctx.namespace)) {
 			// copy all data to WIP namespace -- from the current version of local
 			// data, not the WIP schema version. this may not be n-1, we might
 			// be loading a WIP schema over older data.
-			const currentVersion = await context.persistence.getNamespaceVersion(
-				context.originalNamespace,
+			const currentVersion = await ctx.persistence.getNamespaceVersion(
+				ctx.originalNamespace,
 			);
 
 			if (currentVersion === 0) {
 				// there is no existing data. nothing to copy.
-				context.log('debug', 'No existing data to copy to WIP namespace');
+				ctx.log('debug', 'No existing data to copy to WIP namespace');
 			} else {
 				const currentSchema = ctx.oldSchemas?.find(
 					(s) => s.version === currentVersion,
@@ -41,52 +39,62 @@ export async function initializePersistence(
 						`Trying to open WIP database for version ${ctx.schema.version}, but the current local data is version ${currentVersion} and a historical schema for that version is not available.`,
 					);
 				}
-				context.log(
+				ctx.log(
 					'info',
-					`Copying data from ${context.originalNamespace} to ${context.namespace}`,
+					`Copying data from ${ctx.originalNamespace} to ${ctx.namespace}`,
 				);
-				await context.persistence.copyNamespace(
-					context.originalNamespace,
-					context.namespace,
+				await ctx.persistence.copyNamespace(
+					ctx.originalNamespace,
+					ctx.namespace,
 					// needs to be the original schema; the copy should be of the original
 					// data and schema structure; the WIP schema migration application happens
 					// below.
-					{
-						...context,
+					ctx.cloneWithOptions({
 						schema: currentSchema,
-					},
+					}),
 				);
 			}
 		}
 	}
 
-	const namespace = await ctx.persistence.openNamespace(
-		context.namespace,
-		context,
-	);
+	const namespace = await ctx.persistence.openNamespace(ctx.namespace, ctx);
 
-	context.log('info', 'Opening persistence metadata');
-	context.meta = new PersistenceMetadata(
-		await namespace.openMetadata(ctx),
-		ctx,
-	);
+	ctx.log('info', 'Opening persistence metadata', ctx.namespace);
+	const meta = new PersistenceMetadata(await namespace.openMetadata(ctx), ctx);
 
-	context.log('info', 'Opening persistence files');
-	context.files = new PersistenceFiles(
-		await namespace.openFiles(context),
-		context,
-	);
+	ctx.log('info', 'Opening persistence files', ctx.namespace);
+	const files = new PersistenceFiles(await namespace.openFiles(ctx), ctx);
 
-	context.log('info', 'Migrating document database');
+	ctx.log('info', 'Migrating document database');
 	await migrate({
-		context,
+		context: ctx,
 		version: ctx.schema.version,
+		meta,
 	});
 
-	context.log('info', 'Opening persistence documents');
-	context.documents = new PersistenceDocuments(
-		await namespace.openDocuments(context),
-		context,
+	ctx.log('info', 'Opening persistence documents');
+	if (ctx.schema.version <= 0) {
+		// debugging....
+		if (ctx.schema !== initialSchema) {
+			ctx.log(
+				'critical',
+				'Schema at initialization does not match original schema. This is likely a bug in Verdant!',
+			);
+			throw new VerdantError(
+				VerdantError.Code.ConfigurationError,
+				undefined,
+				`Schema at initialization does not match original schema. This is likely a bug in Verdant!`,
+			);
+		}
+		throw new VerdantError(
+			VerdantError.Code.ConfigurationError,
+			undefined,
+			`Schema version must be greater than 0. Found version ${ctx.schema.version} with collections [${Object.keys(ctx.schema.collections).join(', ')}]\n${JSON.stringify(ctx.schema)}`,
+		);
+	}
+	const documents = new PersistenceDocuments(
+		await namespace.openDocuments(ctx),
+		ctx,
 	);
 
 	if (!ctx.schema.wip) {
@@ -100,7 +108,7 @@ export async function initializePersistence(
 		}
 	}
 
-	return context;
+	return { meta, files, documents };
 }
 
 export async function importPersistence(
@@ -123,26 +131,17 @@ export async function importPersistence(
 	// using a new namespace to put all of this into a temporary zone
 	const importedNamespace = `@@import_${Date.now()}`;
 
-	const importedContext = await initializePersistence({
-		...ctx,
+	const importedContext = ctx.cloneWithOptions({
 		schema: exportedSchema,
 		namespace: importedNamespace,
-		originalNamespace: importedNamespace,
-		// no-op entity events -- don't need to inform queries of changes.
-		entityEvents: new EventSubscriber(),
-		internalEvents: new EventSubscriber(),
-		globalEvents: new EventSubscriber(),
-		config: {
-			...ctx.config,
-			persistence: {
-				...ctx.config.persistence,
-				disableRebasing: true,
-			},
-		},
+		disableRebasing: true,
 		persistenceShutdownHandler: new ShutdownHandler(ctx.log),
 	});
+	await importedContext.reinitialize();
+	const importedMeta = await importedContext.meta;
+
 	// load imported data into persistence
-	await importedContext.meta.resetFrom(exportedData.data);
+	await importedMeta.resetFrom(exportedData.data);
 	// need to write indexes here!
 	const affectedOids = new Set<string>();
 	for (const baseline of exportedData.data.baselines) {
@@ -153,15 +152,15 @@ export async function importPersistence(
 	}
 	const toSave = await Promise.all(
 		Array.from(affectedOids).map(async (oid) => {
-			const snapshot = await importedContext.meta.getDocumentSnapshot(oid);
+			const snapshot = await importedMeta.getDocumentSnapshot(oid);
 			return {
 				oid,
 				getSnapshot: () => snapshot,
 			};
 		}),
 	);
-	await importedContext.documents.saveEntities(toSave);
-	await importedContext.files.import(exportedData);
+	await (await importedContext.documents).saveEntities(toSave);
+	await (await importedContext.files).import(exportedData);
 
 	ctx.log('debug', 'Imported data into temporary namespace', importedNamespace);
 
@@ -172,13 +171,14 @@ export async function importPersistence(
 		// an upgrade of the imported data is needed ; it's an older version
 		// of the schema.
 
-		// upgrade the imported data to the latest schema
-		const currentSchema = ctx.schema;
-		const upgradedContext = await initializePersistence({
-			...importedContext,
+		// upgrade the imported data to the latest schema by re-initializing
+		// a context at the latest version, pointing at the imported namespace
+		const upgradedContext = importedContext.cloneWithOptions({
 			persistenceShutdownHandler: new ShutdownHandler(ctx.log),
-			schema: currentSchema,
+			schema: ctx.schema,
+			oldSchemas: ctx.oldSchemas,
 		});
+		await upgradedContext.reinitialize();
 
 		ctx.log('debug', 'Upgraded imported data to current schema');
 
@@ -192,15 +192,17 @@ export async function importPersistence(
 
 	// copy the imported data into the current namespace
 	await ctx.persistence.copyNamespace(importedNamespace, ctx.namespace, ctx);
+	ctx.log('debug', 'Copied imported data to primary namespace');
 
 	// restart the persistence layer
-	await initializePersistence(ctx);
+	await ctx.reinitialize();
+	ctx.log('debug', 'Reinitialized primary persistence layer');
 
 	// verify integrity -- this can only be done if imported data was same
 	// version as current schema, because migrations could add or remove
 	// operations. still, it's a good sanity check.
 	if (exportedData.data.schemaVersion === ctx.schema.version) {
-		const stats = await ctx.meta.stats();
+		const stats = await (await ctx.meta).stats();
 		if (stats.operationsSize.count !== exportedData.data.operations.length) {
 			ctx.log(
 				'critical',
@@ -231,6 +233,15 @@ export async function importPersistence(
 				'Imported documents count mismatch',
 			);
 		}
+	} else {
+		ctx.log(
+			'debug',
+			'Skipping integrity check due to schema version mismatch (not an error)',
+			{
+				exportedVersion: exportedData.data.schemaVersion,
+				currentVersion: ctx.schema.version,
+			},
+		);
 	}
 
 	ctx.log('debug', 'Data copied to primary namespace');

package/src/queries/FindAllQuery.ts

@@ -23,7 +23,9 @@ export class FindAllQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result: oids } = await this.context.documents.findAllOids({
+		const { result: oids } = await (
+			await this.context.documents
+		).findAllOids({
 			collection: this.collection,
 			index: this.index,
 		});

package/src/queries/FindInfiniteQuery.ts

@@ -37,7 +37,9 @@ export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await this.context.documents.findAllOids({
+		const { result, hasNextPage } = await (
+			await this.context.documents
+		).findAllOids({
 			collection: this.collection,
 			limit: this._pageSize * this._upToPage,
 			offset: 0,
@@ -48,7 +50,9 @@ export class FindInfiniteQuery<T> extends BaseQuery<T[]> {
 	};
 
 	public loadMore = async () => {
-		const { result, hasNextPage } = await this.context.documents.findAllOids({
+		const { result, hasNextPage } = await (
+			await this.context.documents
+		).findAllOids({
 			collection: this.collection,
 			limit: this._pageSize,
 			offset: this._pageSize * this._upToPage,

package/src/queries/FindOneQuery.ts

@@ -23,7 +23,9 @@ export class FindOneQuery<T> extends BaseQuery<T | null> {
 	}
 
 	protected run = async () => {
-		const oid = await this.context.documents.findOneOid({
+		const oid = await (
+			await this.context.documents
+		).findOneOid({
 			collection: this.collection,
 			index: this.index,
 		});

package/src/queries/FindPageQuery.ts

@@ -48,7 +48,9 @@ export class FindPageQuery<T> extends BaseQuery<T[]> {
 	}
 
 	protected run = async () => {
-		const { result, hasNextPage } = await this.context.documents.findAllOids({
+		const { result, hasNextPage } = await (
+			await this.context.documents
+		).findAllOids({
 			collection: this.collection,
 			index: this.index,
 			limit: this._pageSize,

package/src/sync/Heartbeat.ts

@@ -55,9 +55,11 @@ export class Heartbeat extends EventSubscriber<{
 	stop = () => {
 		if (this.nextBeat) {
 			clearTimeout(this.nextBeat);
+			this.nextBeat = null;
 		}
 		if (this.deadline) {
 			clearTimeout(this.deadline);
+			this.deadline = null;
 		}
 	};
 

package/src/sync/PresenceManager.ts

@@ -1,13 +1,13 @@
 import {
-	ServerMessage,
-	EventSubscriber,
-	Batcher,
 	Batch,
+	Batcher,
+	EventSubscriber,
+	ServerMessage,
 	VerdantInternalPresence,
 	initialInternalPresence,
 } from '@verdant-web/common';
-import type { UserInfo } from '../index.js';
 import { Context } from '../context/context.js';
+import type { UserInfo } from '../index.js';
 import { LocalReplicaInfo } from '../persistence/interfaces.js';
 
 export const HANDLE_MESSAGE = Symbol('handleMessage');
@@ -90,9 +90,12 @@ export class PresenceManager<
 		this.self.replicaId = '';
 
 		// set the local replica ID as soon as it's loaded
-		ctx.meta.getLocalReplica().then((info) => {
-			this.self.replicaId = info.id;
-		});
+		ctx.waitForInitialization
+			.then(() => ctx.meta)
+			.then((meta) => meta.getLocalReplica())
+			.then((info) => {
+				this.self.replicaId = info.id;
+			});
 
 		this._updateBatcher = new Batcher(this.flushPresenceUpdates);
 		this._updateBatch = this._updateBatcher.add({

package/src/sync/PushPullSync.ts

@@ -140,7 +140,9 @@ export class PushPullSync
 			if (message.ackThisNonce) {
 				this.ctx.log('debug', 'Sending sync ack', message.ackThisNonce);
 				await this.sendRequest([
-					await this.ctx.meta.messageCreator.createAck(message.ackThisNonce),
+					await (
+						await this.ctx.meta
+					).messageCreator.createAck(message.ackThisNonce),
 				]);
 			}
 		}
@@ -154,6 +156,14 @@ export class PushPullSync
 	}, 3000);
 
 	send = (message: ClientMessage) => {
+		if (this.status !== 'active') {
+			this.ctx.log(
+				'warn',
+				'Attempted to send message while sync is not active',
+				message,
+			);
+			return;
+		}
 		// only certain messages are sent for pull-based sync.
 		switch (message.type) {
 			case 'presence-update':
@@ -173,11 +183,13 @@ export class PushPullSync
 		if (this.status === 'active') {
 			return;
 		}
+		this.ctx.log('debug', 'Starting push-pull sync');
 		await this.endpointProvider.getEndpoints();
 		this.heartbeat.start(true);
 		this._status = 'active';
 	};
 	stop(): void {
+		this.ctx.log('debug', 'Stopping push-pull sync');
 		this.heartbeat.stop();
 		this._status = 'paused';
 	}
@@ -200,10 +212,10 @@ export class PushPullSync
 		// will include the client's own presence info and fill in missing profile
 		// data on the first request. otherwise it would have to wait for the second.
 		this.sendRequest([
-			await this.ctx.meta.messageCreator.createPresenceUpdate(
-				this.presence.self,
-			),
-			await this.ctx.meta.messageCreator.createSyncStep1(),
+			await (
+				await this.ctx.meta
+			).messageCreator.createPresenceUpdate(this.presence.self),
+			await (await this.ctx.meta).messageCreator.createSyncStep1(),
 		]);
 	};
 
@@ -217,7 +229,7 @@ export class PushPullSync
 
 	syncOnce = async () => {
 		await this.sendRequest([
-			await this.ctx.meta.messageCreator.createSyncStep1(),
+			await (await this.ctx.meta).messageCreator.createSyncStep1(),
 		]);
 	};
 

package/src/sync/Sync.ts

@@ -10,16 +10,16 @@ import {
 	VerdantError,
 	VerdantInternalPresence,
 } from '@verdant-web/common';
-import { HANDLE_MESSAGE, PresenceManager } from './PresenceManager.js';
+import { Context } from '../context/context.js';
+import { attemptToRegisterBackgroundSync } from './background.js';
 import { FilePullResult, FileSync, FileUploadResult } from './FileSync.js';
+import { HANDLE_MESSAGE, PresenceManager } from './PresenceManager.js';
 import { PushPullSync } from './PushPullSync.js';
 import {
 	ServerSyncEndpointProvider,
 	ServerSyncEndpointProviderConfig,
 } from './ServerSyncEndpointProvider.js';
 import { WebSocketSync } from './WebSocketSync.js';
-import { Context } from '../context/context.js';
-import { attemptToRegisterBackgroundSync } from './background.js';
 
 type SyncEvents = {
 	onlineChange: (isOnline: boolean) => void;
@@ -282,6 +282,11 @@ export class ServerSync<Presence = any, Profile = any>
 				this.handleBroadcastChannelMessage,
 			);
 		}
+		ctx.log(
+			'info',
+			'Sync initialized with transport:',
+			initialTransport ?? 'pull',
+		);
 		if (initialTransport === 'realtime') {
 			this.activeSync = this.webSocketSync;
 		} else {
@@ -290,8 +295,7 @@ export class ServerSync<Presence = any, Profile = any>
 
 		this.presence.subscribe('update', this.handlePresenceUpdate);
 
-		ctx.meta.events.subscribe('syncMessage', this.send);
-
+		ctx.internalEvents.subscribe('outgoingSyncMessage', this.send);
 		this.webSocketSync.subscribe('message', this.handleMessage);
 		this.webSocketSync.subscribe('onlineChange', this.handleOnlineChange);
 
@@ -384,11 +388,11 @@ export class ServerSync<Presence = any, Profile = any>
 					baselines: message.baselines,
 				});
 				if (message.globalAckTimestamp) {
-					await this.ctx.meta.setGlobalAck(message.globalAckTimestamp);
+					await (await this.ctx.meta).setGlobalAck(message.globalAckTimestamp);
 				}
 				break;
 			case 'global-ack':
-				await this.ctx.meta.setGlobalAck(message.timestamp);
+				await (await this.ctx.meta).setGlobalAck(message.timestamp);
 				break;
 			case 'sync-resp':
 				this._activelySyncing = true;
@@ -400,10 +404,10 @@ export class ServerSync<Presence = any, Profile = any>
 				});
 
 				if (message.globalAckTimestamp) {
-					await this.ctx.meta.setGlobalAck(message.globalAckTimestamp);
+					await (await this.ctx.meta).setGlobalAck(message.globalAckTimestamp);
 				}
 
-				await this.ctx.meta.updateLastSynced(message.ackedTimestamp);
+				await (await this.ctx.meta).updateLastSynced(message.ackedTimestamp);
 				this._activelySyncing = false;
 				this.emit('syncingChange', false);
 				this._hasSynced = true;
@@ -411,13 +415,15 @@ export class ServerSync<Presence = any, Profile = any>
 				break;
 			case 'need-since':
 				this.emit('serverReset', message.since);
-				this.ctx.files.onServerReset(message.since);
+				(await this.ctx.files).onServerReset(message.since);
 				this.activeSync.send(
-					await this.ctx.meta.messageCreator.createSyncStep1(message.since),
+					await (
+						await this.ctx.meta
+					).messageCreator.createSyncStep1(message.since),
 				);
 				break;
 			case 'server-ack':
-				await this.ctx.meta.updateLastSynced(message.timestamp);
+				await (await this.ctx.meta).updateLastSynced(message.timestamp);
 		}
 
 		// avoid rebroadcasting messages
@@ -430,7 +436,7 @@ export class ServerSync<Presence = any, Profile = any>
 
 		// update presence if necessary
 		this.presence[HANDLE_MESSAGE](
-			await this.ctx.meta.getLocalReplica(),
+			await (await this.ctx.meta).getLocalReplica(),
 			message,
 		);
 	};
@@ -439,7 +445,7 @@ export class ServerSync<Presence = any, Profile = any>
 
 		// if online, attempt to upload any unsynced files.
 		if (online) {
-			const unsyncedFiles = await this.ctx.files.listUnsynced();
+			const unsyncedFiles = await (await this.ctx.files).listUnsynced();
 			const results = await Promise.allSettled(
 				unsyncedFiles.map((file) => this.fileSync.uploadFile(file)),
 			);
@@ -458,7 +464,9 @@ export class ServerSync<Presence = any, Profile = any>
 		presence?: Presence;
 		internal?: VerdantInternalPresence;
 	}) => {
-		this.send(await this.ctx.meta.messageCreator.createPresenceUpdate(data));
+		this.send(
+			await (await this.ctx.meta).messageCreator.createPresenceUpdate(data),
+		);
 	};
 
 	setMode = (transport: SyncTransportMode) => {
@@ -512,7 +520,7 @@ export class ServerSync<Presence = any, Profile = any>
 			if (message.type === 'sync' || message.type === 'op') {
 				rewriteAuthzOriginator(message, userId);
 			}
-			await this.activeSync.send(message);
+			this.activeSync.send(message);
 			this.onOutgoingMessage?.(message);
 		}
 	};
@@ -570,10 +578,12 @@ export class ServerSync<Presence = any, Profile = any>
 	};
 
 	public start = () => {
+		this.ctx.log('info', 'Starting sync');
 		return this.activeSync.start();
 	};
 
 	public stop = () => {
+		this.ctx.log('info', 'Stopping sync');
 		return this.activeSync.stop();
 	};
 

package/src/sync/WebSocketSync.ts

@@ -93,12 +93,12 @@ export class WebSocketSync
 			this.ctx.log('debug', 'Starting sync');
 			this.hasStartedSync = true;
 			this.synced = false;
+			const meta = await this.ctx.meta;
+			this.ctx.log('debug', 'HERE');
 			this.send(
-				await this.ctx.meta.messageCreator.createPresenceUpdate(
-					this.presence.self,
-				),
+				await meta.messageCreator.createPresenceUpdate(this.presence.self),
 			);
-			this.send(await this.ctx.meta.messageCreator.createSyncStep1());
+			this.send(await meta.messageCreator.createSyncStep1());
 			this.heartbeat.start();
 		}
 		this.emit('onlineChange', online);
@@ -122,7 +122,9 @@ export class WebSocketSync
 				if (message.ackThisNonce) {
 					// we need to send the ack to confirm we got the response
 					this.send(
-						await this.ctx.meta.messageCreator.createAck(message.ackThisNonce),
+						await (
+							await this.ctx.meta
+						).messageCreator.createAck(message.ackThisNonce),
 					);
 				}
 				this.hasStartedSync = true;
@@ -179,7 +181,7 @@ export class WebSocketSync
 	};
 
 	private onError = (event: Event) => {
-		this.ctx.log('error', 'Sync socket error', event);
+		this.ctx.log('error', 'Sync socket error', event, event.target);
 		if (this.disposed) return;
 		this.reconnectScheduler.next();
 
@@ -187,7 +189,7 @@ export class WebSocketSync
 	};
 
 	private onClose = (event: CloseEvent) => {
-		this.ctx.log('info', 'Sync socket disconnected');
+		this.ctx.log('info', 'Sync socket disconnected', event.code);
 		this.onOnlineChange(false);
 		if (this.disposed) return;
 		this.reconnectScheduler.next();
@@ -209,7 +211,7 @@ export class WebSocketSync
 	};
 
 	private sendHeartbeat = async () => {
-		this.send(await this.ctx.meta.messageCreator.createHeartbeat());
+		this.send(await (await this.ctx.meta).messageCreator.createHeartbeat());
 	};
 
 	reconnect = () => {
@@ -227,11 +229,25 @@ export class WebSocketSync
 	};
 
 	send = (message: ClientMessage) => {
-		if (this.status !== 'active') return;
+		if (this.status !== 'active') {
+			this.ctx.log(
+				'debug',
+				'Ignoring outgoing message',
+				message.type,
+				'sync is not active',
+			);
+			return;
+		}
 
 		// wait until a sync has started before doing anything other than sync.
 		// new "op" messages can arrive before sync has started, so we need to wait
 		if (!this.hasStartedSync && !this.canSkipSyncWait(message)) {
+			this.ctx.log(
+				'debug',
+				'Ignoring outgoing message',
+				message.type,
+				'still waiting to begin initial sync',
+			);
 			return;
 		}
 

package/src/sync/serviceWorker.ts

@@ -1,19 +1,17 @@
-import type { ClientDescriptor } from '../client/ClientDescriptor.js';
+import type { Client } from '../client/Client.js';
 
-export async function registerBackgroundSync(clientDesc: ClientDescriptor) {
+export async function registerBackgroundSync(client: Client) {
 	self.addEventListener('periodicsync', (event: any) => {
 		if (event.tag === 'verdant-sync') {
 			// See the "Think before you sync" section for
 			// checks you could perform before syncing.
-			event.waitUntil(sync(clientDesc));
+			event.waitUntil(sync(client));
 		}
 	});
 }
 
-async function sync(clientDesc: ClientDescriptor) {
+async function sync(client: Client) {
 	try {
-		const client = await clientDesc.open();
-
 		await client.sync.syncOnce();
 	} catch (err) {
 		console.error('Failed to sync:', err);