@verdant-web/store 2.0.1

package/src/sync/Sync.ts

@@ -0,0 +1,418 @@
+import {
+	ClientMessage,
+	DocumentBaseline,
+	EventSubscriber,
+	FileData,
+	Operation,
+	ReplicaType,
+	ServerMessage,
+} from '@verdant-web/common';
+import { Metadata } from '../metadata/Metadata.js';
+import { HANDLE_MESSAGE, PresenceManager } from './PresenceManager.js';
+import { EntityStore } from '../reactives/EntityStore.js';
+import { FilePullResult, FileSync, FileUploadResult } from './FileSync.js';
+import { PushPullSync } from './PushPullSync.js';
+import {
+	ServerSyncEndpointProvider,
+	ServerSyncEndpointProviderConfig,
+} from './ServerSyncEndpointProvider.js';
+import { WebSocketSync } from './WebSocketSync.js';
+
+type SyncEvents = {
+	onlineChange: (isOnline: boolean) => void;
+};
+
+export type SyncTransportEvents = SyncEvents & {
+	message: (message: ServerMessage) => void;
+};
+
+export interface SyncTransport {
+	subscribe(
+		event: 'onlineChange',
+		handler: (online: boolean) => void,
+	): () => void;
+
+	readonly presence: PresenceManager;
+
+	readonly mode: SyncTransportMode;
+
+	send(message: ClientMessage): void;
+
+	start(): void;
+	stop(): void;
+
+	dispose(): void;
+
+	reconnect(): void;
+
+	readonly isConnected: boolean;
+	readonly status: 'active' | 'paused';
+}
+
+export interface Sync extends SyncTransport {
+	setMode(mode: SyncTransportMode): void;
+	setPullInterval(interval: number): void;
+	readonly pullInterval: number;
+	uploadFile(data: FileData): Promise<FileUploadResult>;
+	getFile(fileId: string): Promise<FilePullResult>;
+}
+
+export class NoSync extends EventSubscriber<SyncEvents> implements Sync {
+	readonly mode = 'pull';
+
+	public send(): void {}
+
+	public start(): void {}
+
+	public stop(): void {}
+
+	public dispose = () => {};
+
+	public reconnect(): void {}
+
+	public setMode(): void {}
+	public setPullInterval(): void {}
+
+	public readonly isConnected = false;
+	public readonly status = 'paused';
+	public readonly pullInterval = 0;
+
+	public readonly presence = new PresenceManager({
+		initialPresence: {},
+		defaultProfile: {},
+	});
+
+	uploadFile = async () => {
+		return {
+			success: false,
+			retry: false,
+		};
+	};
+
+	getFile = async (): Promise<FilePullResult> => {
+		return {
+			success: false,
+			retry: false,
+		};
+	};
+}
+
+export type SyncTransportMode = 'realtime' | 'pull';
+
+export interface ServerSyncOptions<Profile = any, Presence = any>
+	extends ServerSyncEndpointProviderConfig {
+	/**
+	 * When a client first connects, it will use this presence value.
+	 */
+	initialPresence: Presence;
+	/**
+	 * Before connecting to the server, the local client will have
+	 * this value for their profile data. You can either cache and store
+	 * profile data from a previous connection or provide defaults like
+	 * empty strings.
+	 */
+	defaultProfile: Profile;
+
+	/**
+	 * Provide `false` to disable transport selection. Transport selection
+	 * automatically switches between HTTP and WebSocket based sync depending
+	 * on the number of peers connected. If a user is alone, they will use
+	 * HTTP push/pull to sync changes. If another user joins, both users will
+	 * be upgraded to websockets.
+	 *
+	 * Turning off this feature allows you more control over the transport
+	 * which can be useful for low-power devices or to save server traffic.
+	 * To modify transport modes manually, utilize `client.sync.setMode`.
+	 * The built-in behavior is essentially switching modes based on
+	 * the number of peers detected by client.sync.presence.
+	 */
+	automaticTransportSelection?: boolean;
+	initialTransport?: SyncTransportMode;
+	autoStart?: boolean;
+	/**
+	 * Optionally specify an interval, in milliseconds, to poll the server
+	 * when in pull mode.
+	 */
+	pullInterval?: number;
+	/**
+	 * Presence updates are batched to reduce number of requests / messages
+	 * sent to the server. You can specify the batching time slice, in milliseconds,
+	 */
+	presenceUpdateBatchTimeout?: number;
+}
+
+export class ServerSync<Profile = any, Presence = any>
+	extends EventSubscriber<SyncEvents>
+	implements Sync
+{
+	private webSocketSync: WebSocketSync;
+	private pushPullSync: PushPullSync;
+	private fileSync: FileSync;
+	private activeSync: SyncTransport;
+	private endpointProvider;
+	private onData: (data: {
+		operations: Operation[];
+		baselines: DocumentBaseline[];
+		reset?: boolean;
+	}) => Promise<void>;
+
+	private meta: Metadata;
+
+	readonly presence;
+
+	private log;
+
+	constructor(
+		{
+			authEndpoint,
+			fetchAuth,
+			initialPresence,
+			automaticTransportSelection = true,
+			autoStart,
+			initialTransport,
+			pullInterval,
+			presenceUpdateBatchTimeout,
+			defaultProfile,
+		}: ServerSyncOptions<Profile, Presence>,
+		{
+			meta,
+			log,
+			onData,
+		}: {
+			meta: Metadata;
+			log?: (...args: any[]) => void;
+			onData: (data: {
+				operations: Operation[];
+				baselines: DocumentBaseline[];
+				reset?: boolean;
+			}) => Promise<void>;
+		},
+	) {
+		super();
+		this.meta = meta;
+		this.onData = onData;
+		this.log = log || (() => {});
+		this.presence = new PresenceManager<Profile, Presence>({
+			initialPresence,
+			defaultProfile,
+			updateBatchTimeout: presenceUpdateBatchTimeout,
+		});
+		this.endpointProvider = new ServerSyncEndpointProvider({
+			authEndpoint,
+			fetchAuth,
+		});
+
+		this.webSocketSync = new WebSocketSync({
+			endpointProvider: this.endpointProvider,
+			meta,
+			presence: this.presence,
+			log: this.log,
+		});
+		this.pushPullSync = new PushPullSync({
+			endpointProvider: this.endpointProvider,
+			meta,
+			presence: this.presence,
+			log: this.log,
+			interval: pullInterval,
+		});
+		this.fileSync = new FileSync({
+			endpointProvider: this.endpointProvider,
+			log: this.log,
+		});
+		if (initialTransport === 'realtime') {
+			this.activeSync = this.webSocketSync;
+		} else {
+			this.activeSync = this.pushPullSync;
+		}
+
+		this.presence.subscribe('update', this.handlePresenceUpdate);
+
+		this.meta.subscribe('message', this.send);
+
+		this.webSocketSync.subscribe('message', this.handleMessage);
+		this.webSocketSync.subscribe('onlineChange', this.handleOnlineChange);
+
+		this.pushPullSync.subscribe('message', this.handleMessage);
+		this.pushPullSync.subscribe('onlineChange', this.handleOnlineChange);
+
+		if (automaticTransportSelection) {
+			// automatically shift between transport modes depending
+			// on whether any peers are present
+			let switchoverTimeout: NodeJS.Timer;
+			this.presence.subscribe('peersChanged', (peers) => {
+				if (switchoverTimeout) {
+					clearTimeout(switchoverTimeout);
+				}
+				if (Object.keys(peers).length > 0) {
+					// only upgrade if token allows it
+					if (this.canDoRealtime) {
+						if (this.mode === 'pull') {
+							this.setMode('realtime');
+						}
+					}
+				} else if (this.mode === 'realtime') {
+					// wait 1 second then switch to pull mode if still emtpy
+					switchoverTimeout = setTimeout(() => {
+						if (Object.keys(this.presence.peers).length === 0) {
+							this.setMode('pull');
+						}
+					}, 1000);
+				}
+			});
+		}
+
+		if (autoStart) {
+			this.start();
+		}
+	}
+
+	get canDoRealtime() {
+		return (
+			this.endpointProvider.type === ReplicaType.Realtime ||
+			this.endpointProvider.type === ReplicaType.PassiveRealtime ||
+			this.endpointProvider.type === ReplicaType.ReadOnlyRealtime
+		);
+	}
+
+	private handleMessage = async (message: ServerMessage) => {
+		// TODO: move this into metadata
+		if (message.type === 'op-re' || message.type === 'sync-resp') {
+			for (const op of message.operations) {
+				this.meta.time.update(op.timestamp);
+			}
+		}
+
+		this.log('sync message', JSON.stringify(message, null, 2));
+		switch (message.type) {
+			case 'op-re':
+				await this.onData({
+					operations: message.operations,
+					baselines: message.baselines,
+				});
+				if (message.globalAckTimestamp) {
+					await this.meta.setGlobalAck(message.globalAckTimestamp);
+				}
+				break;
+			case 'global-ack':
+				await this.meta.setGlobalAck(message.timestamp);
+				break;
+			case 'sync-resp':
+				await this.onData({
+					operations: message.operations,
+					baselines: message.baselines,
+					reset: message.overwriteLocalData,
+				});
+
+				if (message.globalAckTimestamp) {
+					await this.meta.setGlobalAck(message.globalAckTimestamp);
+				}
+
+				await this.meta.updateLastSynced(message.ackedTimestamp);
+				break;
+			case 'need-since':
+				this.activeSync.send(
+					await this.meta.messageCreator.createSyncStep1(message.since),
+				);
+				break;
+			case 'server-ack':
+				await this.meta.updateLastSynced(message.timestamp);
+		}
+
+		// update presence if necessary
+		this.presence[HANDLE_MESSAGE](await this.meta.localReplica.get(), message);
+	};
+	private handleOnlineChange = (online: boolean) => {
+		this.emit('onlineChange', online);
+	};
+	private handlePresenceUpdate = async (presence: any) => {
+		this.send(await this.meta.messageCreator.createPresenceUpdate(presence));
+	};
+
+	setMode = (transport: SyncTransportMode) => {
+		if (transport === 'realtime' && !this.canDoRealtime) {
+			throw new Error(
+				`Cannot switch to realtime mode, because the current auth token does not allow it`,
+			);
+		}
+
+		let newSync: SyncTransport;
+		if (transport === 'realtime') {
+			newSync = this.webSocketSync;
+		} else {
+			newSync = this.pushPullSync;
+		}
+
+		if (newSync === this.activeSync) return;
+		this.log('switching to', transport, 'mode');
+
+		// transfer state to new sync
+		if (this.activeSync.status === 'active') {
+			newSync.start();
+		}
+		this.activeSync.stop();
+		this.activeSync = newSync;
+	};
+
+	setPullInterval = (interval: number) => {
+		this.pushPullSync.setInterval(interval);
+	};
+
+	get pullInterval() {
+		return this.pushPullSync.interval;
+	}
+
+	send = (message: ClientMessage) => {
+		if (this.activeSync.status === 'active') {
+			return this.activeSync.send(message);
+		}
+	};
+
+	uploadFile = async (info: FileData) => {
+		if (this.activeSync.status === 'active') {
+			return this.fileSync.uploadFile(info);
+		} else {
+			return {
+				success: false,
+				retry: false,
+			};
+		}
+	};
+
+	getFile = async (id: string) => {
+		// TODO: should this error? or just try anyway?
+		if (this.activeSync.status === 'active') {
+			return this.fileSync.getFile(id);
+		} else {
+			throw new Error('Offline, cannot retrieve remote file details');
+		}
+	};
+
+	public start = () => {
+		return this.activeSync.start();
+	};
+
+	public stop = () => {
+		return this.activeSync.stop();
+	};
+
+	public dispose = () => {
+		this.webSocketSync.dispose();
+		this.pushPullSync.dispose();
+	};
+
+	public reconnect = () => {
+		return this.activeSync.reconnect();
+	};
+
+	public get isConnected(): boolean {
+		return this.activeSync.isConnected;
+	}
+
+	public get status() {
+		return this.activeSync.status;
+	}
+
+	public get mode() {
+		return this.activeSync.mode;
+	}
+}

package/src/sync/WebSocketSync.ts

@@ -0,0 +1,240 @@
+import {
+	ClientMessage,
+	ServerMessage,
+	EventSubscriber,
+	ObjectIdentifier,
+	assert,
+	ReplicaType,
+} from '@verdant-web/common';
+import { default as jwtDecode } from 'jwt-decode';
+import { Backoff, BackoffScheduler } from '../BackoffScheduler.js';
+import { EntityStore } from '../reactives/EntityStore.js';
+import { Metadata } from '../metadata/Metadata.js';
+import { PresenceManager } from './PresenceManager.js';
+import { SyncTransport, SyncTransportEvents } from './Sync.js';
+import { Heartbeat } from './Heartbeat.js';
+import { ServerSyncEndpointProvider } from './ServerSyncEndpointProvider.js';
+
+export class WebSocketSync
+	extends EventSubscriber<SyncTransportEvents>
+	implements SyncTransport
+{
+	private meta: Metadata;
+	readonly presence: PresenceManager;
+	private socket: WebSocket | null = null;
+	// messages awaiting websocket connection to send
+	private connectQueue: ClientMessage[] = [];
+	// messages awaiting sync response to send
+	private syncQueue: ClientMessage[] = [];
+	private endpointProvider;
+	private _status: 'active' | 'paused' = 'paused';
+	private synced = false;
+	private hasStartedSync = false;
+
+	readonly mode = 'realtime';
+	private log = (...args: any[]) => {};
+
+	private heartbeat = new Heartbeat();
+
+	private reconnectScheduler = new BackoffScheduler(
+		new Backoff(60 * 1000, 1.5),
+	);
+
+	constructor({
+		endpointProvider,
+		meta,
+		presence,
+		log,
+	}: {
+		endpointProvider: ServerSyncEndpointProvider;
+		meta: Metadata;
+		presence: PresenceManager;
+		log?: (...args: any[]) => any;
+	}) {
+		super();
+		this.log = log || this.log;
+		this.endpointProvider = endpointProvider;
+		this.meta = meta;
+		this.presence = presence;
+
+		this.reconnectScheduler.subscribe('trigger', this.initializeSocket);
+		this.heartbeat.subscribe('beat', this.sendHeartbeat);
+	}
+
+	private onOpen = () => {
+		if (!this.socket) {
+			throw new Error('Invalid sync state: online but socket is null');
+		}
+		this.synced = false;
+		if (this.connectQueue.length) {
+			for (const msg of this.connectQueue) {
+				this.log('Sending queued message', JSON.stringify(msg, null, 2));
+				this.socket.send(JSON.stringify(msg));
+			}
+			this.connectQueue = [];
+		}
+		this.log('Sync connected');
+		this.onOnlineChange(true);
+		this.reconnectScheduler.reset();
+	};
+
+	private onOnlineChange = async (online: boolean) => {
+		this.log('Socket online change', online);
+		if (!online) {
+			this.hasStartedSync = false;
+			this.synced = false;
+			this.heartbeat.stop();
+		} else {
+			this.log('Starting sync');
+			this.hasStartedSync = true;
+			this.synced = false;
+			this.send(
+				await this.meta.messageCreator.createPresenceUpdate(
+					this.presence.self.presence,
+				),
+			);
+			this.send(await this.meta.messageCreator.createSyncStep1());
+			this.heartbeat.start();
+		}
+		this.emit('onlineChange', online);
+	};
+
+	private onMessage = async (event: MessageEvent) => {
+		const message = JSON.parse(event.data) as ServerMessage;
+		switch (message.type) {
+			case 'sync-resp':
+				if (message.ackThisNonce) {
+					// we need to send the ack to confirm we got the response
+					this.send(
+						await this.meta.messageCreator.createAck(message.ackThisNonce),
+					);
+				}
+				this.hasStartedSync = true;
+				this.synced = true;
+				if (this.syncQueue.length) {
+					for (const msg of this.syncQueue) {
+						this.send(msg);
+					}
+					this.syncQueue = [];
+				}
+			case 'need-since':
+			case 'presence-changed':
+			case 'presence-offline':
+			case 'op-re':
+				this.emit('message', message);
+				break;
+			case 'heartbeat-response':
+				this.heartbeat.keepAlive();
+				this.emit('message', message);
+				break;
+			default:
+				if (this.synced) {
+					this.emit('message', message);
+				}
+				break;
+		}
+	};
+
+	private onError = (event: Event) => {
+		this.log(event);
+		this.reconnectScheduler.next();
+
+		this.log(`Attempting reconnect to websocket sync`);
+	};
+
+	private onClose = (event: CloseEvent) => {
+		this.log('Sync disconnected');
+		this.onOnlineChange(false);
+		this.onError(event);
+	};
+
+	private initializeSocket = async () => {
+		const endpoint = await this.endpointProvider.getEndpoints();
+		// abusing protocols to pass the auth token
+		this.socket = new WebSocket(endpoint.websocket, ['Bearer', endpoint.token]);
+		this.socket.addEventListener('message', this.onMessage);
+		this.socket.addEventListener('open', this.onOpen);
+		this.socket.addEventListener('error', this.onError);
+		this.socket.addEventListener('close', this.onClose);
+		return this.socket;
+	};
+
+	private sendHeartbeat = async () => {
+		this.send(await this.meta.messageCreator.createHeartbeat());
+	};
+
+	reconnect = () => {
+		this.stop();
+		this.start();
+	};
+
+	private canSkipSyncWait = (message: ClientMessage) => {
+		return (
+			message.type === 'sync' ||
+			message.type === 'presence-update' ||
+			message.type === 'sync-ack' ||
+			message.type === 'heartbeat'
+		);
+	};
+
+	send = (message: ClientMessage) => {
+		if (this.status !== 'active') return;
+
+		// wait until a sync has started before doing anything other than sync.
+		// new "op" messages can arrive before sync has started, so we need to wait
+		if (!this.hasStartedSync && !this.canSkipSyncWait(message)) {
+			return;
+		}
+
+		if (this.canSkipSyncWait(message)) {
+			if (this.socket?.readyState === WebSocket.OPEN) {
+				this.log('Sending message', JSON.stringify(message, null, 2));
+				this.socket!.send(JSON.stringify(message));
+			} else {
+				this.log(
+					'Enqueueing message until socket is open',
+					JSON.stringify(message, null, 2),
+				);
+				this.connectQueue.push(message);
+			}
+		} else if (this.synced) {
+			if (this.socket?.readyState === WebSocket.OPEN) {
+				this.log('Sending message', JSON.stringify(message, null, 2));
+				this.socket.send(JSON.stringify(message));
+			}
+		} else if (this.hasStartedSync) {
+			this.log(
+				'Enqueueing message until synced',
+				JSON.stringify(message, null, 2),
+			);
+			this.syncQueue.push(message);
+		}
+	};
+
+	dispose = () => {
+		this.socket?.removeEventListener('close', this.onClose);
+		this.socket?.close();
+	};
+
+	start = () => {
+		if (this.socket) {
+			return;
+		}
+		this.initializeSocket();
+		this._status = 'active';
+	};
+
+	stop = () => {
+		this.dispose();
+		this.socket = null;
+		this._status = 'paused';
+	};
+
+	get isConnected() {
+		return this.socket?.readyState === WebSocket.OPEN;
+	}
+
+	get status() {
+		return this._status;
+	}
+}

package/src/types.ts

@@ -0,0 +1,7 @@
+import { Operation, OperationPatch } from '@verdant-web/common';
+
+export interface LocalOperation extends Operation {
+	inverse: OperationPatch;
+}
+
+export type LogFunction = (...args: any[]) => void;

package/src/utils/Disposable.ts

@@ -0,0 +1,14 @@
+export class Disposable {
+	private _disposes: (() => void)[] = [];
+	protected disposed = false;
+
+	dispose = () => {
+		this.disposed = true;
+		this._disposes.forEach((dispose) => dispose());
+		this._disposes = [];
+	};
+
+	protected addDispose = (dispose: () => void) => {
+		this._disposes.push(dispose);
+	};
+}

package/src/utils/Resolvable.ts

@@ -0,0 +1,24 @@
+export class Resolvable<T> {
+	readonly promise: Promise<T>;
+	private _resolve: (value: T) => void;
+	private _reject: (reason?: any) => void;
+
+	constructor() {
+		let resolve: (value: T) => void;
+		let reject: (reason?: any) => void;
+		this.promise = new Promise((res, rej) => {
+			resolve = res;
+			reject = rej;
+		});
+		this._resolve = resolve!;
+		this._reject = reject!;
+	}
+
+	resolve(value: T) {
+		this._resolve(value);
+	}
+
+	reject(reason?: any) {
+		this._reject(reason);
+	}
+}