@verdant-web/store 2.0.1

package/src/sync/FileSync.ts

@@ -0,0 +1,124 @@
+import { FileData } from '@verdant-web/common';
+import { ServerSyncEndpointProvider } from './ServerSyncEndpointProvider.js';
+
+export interface FileUploadResult {
+	success: boolean;
+	retry: boolean;
+}
+
+export type FilePullResult =
+	| {
+			success: true;
+			data: FileData;
+	  }
+	| {
+			success: false;
+			retry: boolean;
+	  };
+
+export class FileSync {
+	private endpointProvider: ServerSyncEndpointProvider;
+	private log: (...args: any[]) => any;
+
+	constructor({
+		endpointProvider,
+		log,
+	}: {
+		endpointProvider: ServerSyncEndpointProvider;
+		log: (...args: any[]) => any;
+	}) {
+		this.endpointProvider = endpointProvider;
+		this.log = log;
+	}
+
+	uploadFile = async (data: FileData): Promise<FileUploadResult> => {
+		const file = data.file;
+
+		if (!file) {
+			throw new Error('Cannot upload a non-local file');
+		}
+
+		// multipart upload
+		const { files: fileEndpoint, token } =
+			await this.endpointProvider.getEndpoints();
+
+		const formData = new window.FormData();
+		formData.append('file', file);
+
+		try {
+			const response = await fetch(fileEndpoint + `/${data.id}`, {
+				method: 'POST',
+				body: formData,
+				credentials: 'include',
+				headers: {
+					Authorization: `Bearer ${token}`,
+				},
+			});
+
+			if (response.ok) {
+				return {
+					success: true,
+					retry: false,
+				};
+			} else {
+				this.log(
+					'error',
+					'File upload failed',
+					response.status,
+					await response.text(),
+				);
+				return {
+					success: false,
+					retry: response.status >= 500,
+				};
+			}
+		} catch (e) {
+			this.log('error', 'File upload failed', e);
+			return {
+				success: false,
+				retry: true,
+			};
+		}
+	};
+
+	getFile = async (id: string): Promise<FilePullResult> => {
+		const { files: fileEndpoint, token } =
+			await this.endpointProvider.getEndpoints();
+
+		try {
+			const response = await fetch(fileEndpoint + `/${id}`, {
+				method: 'GET',
+				credentials: 'include',
+				headers: {
+					'Content-Type': 'application/json',
+					Authorization: `Bearer ${token}`,
+				},
+			});
+
+			if (response.ok) {
+				const data = await response.json();
+				return {
+					success: true,
+					data,
+				};
+			} else {
+				this.log(
+					'error',
+					'File information fetch failed',
+					response.status,
+					await response.text(),
+				);
+				return {
+					success: false,
+					retry: response.status >= 500 || response.status === 404,
+				};
+			}
+		} catch (e) {
+			this.log('error', 'File information fetch failed', e);
+			return {
+				success: false,
+				retry: true,
+			};
+		}
+	};
+}

package/src/sync/Heartbeat.ts

@@ -0,0 +1,70 @@
+import { EventSubscriber } from '@verdant-web/common';
+
+export class Heartbeat extends EventSubscriber<{
+	missed: () => void;
+	beat: () => void;
+}> {
+	private _interval: number;
+	private deadlineLength: number;
+	private nextBeat: NodeJS.Timeout | null = null;
+	private deadline: NodeJS.Timeout | null = null;
+
+	get interval() {
+		return this._interval;
+	}
+
+	constructor({
+		interval = 15 * 1000,
+		deadlineLength = 3 * 1000,
+	}: {
+		interval?: number;
+		deadlineLength?: number;
+	} = {}) {
+		super();
+		this._interval = interval;
+		this.deadlineLength = deadlineLength;
+	}
+
+	keepAlive = () => {
+		if (this.deadline) {
+			clearTimeout(this.deadline);
+			this.deadline = null;
+			this.start();
+		}
+	};
+
+	start = (immediate = false) => {
+		this.stop();
+		if (immediate) {
+			this.beat();
+		} else {
+			this.nextBeat = setTimeout(this.beat, this._interval);
+		}
+	};
+
+	stop = () => {
+		if (this.nextBeat) {
+			clearTimeout(this.nextBeat);
+		}
+		if (this.deadline) {
+			clearTimeout(this.deadline);
+		}
+	};
+
+	private beat = async () => {
+		this.emit('beat');
+		this.deadline = setTimeout(this.onDeadline, this.deadlineLength);
+	};
+
+	private onDeadline = () => {
+		this.deadline = null;
+		this.emit('missed');
+	};
+
+	/**
+	 * Only takes affect after the next beat
+	 */
+	setInterval = (interval: number) => {
+		this._interval = interval;
+	};
+}

package/src/sync/PresenceManager.ts

@@ -0,0 +1,145 @@
+import {
+	ServerMessage,
+	EventSubscriber,
+	Batcher,
+	Batch,
+} from '@verdant-web/common';
+import type { UserInfo } from '../index.js';
+import { LocalReplicaInfo } from '../metadata/LocalReplicaStore.js';
+
+export const HANDLE_MESSAGE = Symbol('handleMessage');
+
+export class PresenceManager<
+	Profile = any,
+	Presence = any,
+> extends EventSubscriber<{
+	peerChanged: (userId: string, presence: UserInfo<Profile, Presence>) => void;
+	selfChanged: (presence: UserInfo<Profile, Presence>) => void;
+	peersChanged: (peers: Record<string, any>) => void;
+	peerLeft: (userId: string, lastPresence: UserInfo<Profile, Presence>) => void;
+	update: (presence: Partial<Presence>) => void;
+}> {
+	private _peers = {} as Record<string, UserInfo<Profile, Presence>>;
+	private _self = { profile: {} } as UserInfo<Profile, Presence>;
+	private _peerIds = new Array<string>();
+	private _updateBatcher;
+	private _updateBatch: Batch<Partial<Presence>>;
+
+	get self() {
+		return this._self;
+	}
+
+	get peers() {
+		return this._peers;
+	}
+
+	get peerIds() {
+		return this._peerIds;
+	}
+
+	get everyone() {
+		const everyone = { ...this._peers };
+		everyone[this.self.id] = this.self;
+		return everyone;
+	}
+
+	constructor({
+		initialPresence,
+		updateBatchTimeout = 200,
+		defaultProfile,
+	}: {
+		initialPresence: Presence;
+		defaultProfile: Profile;
+		updateBatchTimeout?: number;
+	}) {
+		super();
+		this.self.presence = initialPresence;
+		this.self.profile = defaultProfile;
+
+		this._updateBatcher = new Batcher(this.flushPresenceUpdates);
+		this._updateBatch = this._updateBatcher.add({
+			max: 25,
+			timeout: updateBatchTimeout,
+			items: [],
+			key: 'default',
+		});
+	}
+
+	/**
+	 * Decides if an update is for the local user or not. Even if it's a different replica
+	 * than the local one.
+	 *
+	 * If the replicaId matches, we use that first - we may not know the local replica's User ID yet,
+	 * e.g. on the first presence update.
+	 *
+	 * Otherwise, match the user ID to our local copy.
+	 */
+	private isSelf = (
+		localReplicaInfo: LocalReplicaInfo,
+		userInfo: UserInfo<Profile, Presence>,
+	) => {
+		return (
+			localReplicaInfo.id === userInfo.replicaId ||
+			this._self.id === userInfo.id
+		);
+	};
+
+	[HANDLE_MESSAGE] = async (
+		localReplicaInfo: LocalReplicaInfo,
+		message: ServerMessage,
+	) => {
+		let peersChanged = false;
+		const peerIdsSet = new Set<string>(this.peerIds);
+
+		if (message.type === 'presence-changed') {
+			if (this.isSelf(localReplicaInfo, message.userInfo)) {
+				this._self = message.userInfo;
+				this.emit('selfChanged', message.userInfo);
+			} else {
+				peerIdsSet.add(message.userInfo.id);
+				this._peers[message.userInfo.id] = message.userInfo;
+				peersChanged = true;
+				this.emit('peerChanged', message.userInfo.id, message.userInfo);
+			}
+		} else if (message.type === 'sync-resp') {
+			// reset to provided presence data, which includes all peers.
+			this._peers = {};
+			peerIdsSet.clear();
+
+			for (const [id, userInfo] of Object.entries(message.peerPresence)) {
+				if (this.isSelf(localReplicaInfo, userInfo)) {
+					this._self = userInfo;
+					this.emit('selfChanged', userInfo);
+				} else {
+					peersChanged = true;
+					peerIdsSet.add(id);
+					this._peers[id] = userInfo;
+					this.emit('peerChanged', id, userInfo);
+				}
+			}
+		} else if (message.type === 'presence-offline') {
+			peerIdsSet.delete(message.userId);
+			const lastPresence = this._peers[message.userId];
+			delete this._peers[message.userId];
+			peersChanged = true;
+			this.emit('peerLeft', message.userId, lastPresence);
+		}
+		if (peersChanged) {
+			this._peerIds = Array.from(peerIdsSet);
+			this.emit('peersChanged', this._peers);
+		}
+	};
+
+	update = async (presence: Partial<Presence>) => {
+		this._updateBatch.update({
+			items: [presence],
+		});
+	};
+
+	flushPresenceUpdates = (presenceUpdates: Partial<Presence>[]) => {
+		const presence = presenceUpdates.reduce((acc, update) => {
+			return { ...acc, ...update };
+		}, this.self.presence);
+		this.emit('update', presence);
+	};
+}

package/src/sync/PushPullSync.ts

@@ -0,0 +1,185 @@
+import {
+	ClientMessage,
+	EventSubscriber,
+	ServerMessage,
+} from '@verdant-web/common';
+import { Metadata } from '../metadata/Metadata.js';
+import { PresenceManager } from './PresenceManager.js';
+import { Heartbeat } from './Heartbeat.js';
+import { ServerSyncEndpointProvider } from './ServerSyncEndpointProvider.js';
+import { SyncTransport, SyncTransportEvents } from './Sync.js';
+
+export class PushPullSync
+	extends EventSubscriber<SyncTransportEvents>
+	implements SyncTransport
+{
+	readonly meta: Metadata;
+	readonly presence: PresenceManager;
+	private endpointProvider;
+	private heartbeat;
+
+	readonly mode = 'pull';
+	private log;
+
+	private _isConnected = false;
+	private _status: 'active' | 'paused' = 'paused';
+	private _hasSynced = false;
+
+	constructor({
+		endpointProvider,
+		meta,
+		presence,
+		interval = 15 * 1000,
+		log = () => {},
+	}: {
+		endpointProvider: ServerSyncEndpointProvider;
+		meta: Metadata;
+		presence: PresenceManager;
+		interval?: number;
+		log?: (...args: any[]) => any;
+	}) {
+		super();
+		this.log = log;
+		this.meta = meta;
+		this.presence = presence;
+		this.endpointProvider = endpointProvider;
+
+		this.heartbeat = new Heartbeat({
+			interval,
+		});
+		this.heartbeat.subscribe('beat', this.onHeartbeat);
+		this.heartbeat.subscribe('missed', this.onHeartbeatMissed);
+	}
+
+	setInterval = (interval: number) => {
+		this.heartbeat.setInterval(interval);
+	};
+
+	get interval() {
+		return this.heartbeat.interval;
+	}
+
+	private sendRequest = async (messages: ClientMessage[]) => {
+		this.log('Sending sync request', messages);
+		try {
+			const { http: host, token } = await this.endpointProvider.getEndpoints();
+			const response = await fetch(host, {
+				method: 'POST',
+				headers: {
+					'Content-Type': 'application/json',
+					Authorization: `Bearer ${token}`,
+				},
+				body: JSON.stringify({
+					messages,
+				}),
+				credentials: 'include',
+			});
+			if (response.ok) {
+				this.heartbeat.keepAlive();
+				const json = (await response.json()) as {
+					messages: ServerMessage[];
+				};
+				for (const message of json.messages) {
+					this.handleServerMessage(message);
+				}
+				if (!this._isConnected) {
+					this._isConnected = true;
+					this.emit('onlineChange', true);
+				}
+			} else {
+				this.log('Sync request failed', response.status, await response.text());
+
+				if (this._isConnected) {
+					this._isConnected = false;
+					this.emit('onlineChange', false);
+				}
+
+				// only keep trying if the error was not 4xx
+				if (response.status >= 500) {
+					this.heartbeat.keepAlive();
+				}
+			}
+		} catch (error) {
+			if (this._isConnected) {
+				this._isConnected = false;
+				this.emit('onlineChange', false);
+			}
+			this.log(error);
+
+			this.heartbeat.keepAlive();
+		}
+	};
+
+	private handleServerMessage = async (message: ServerMessage) => {
+		if (message.type === 'sync-resp') {
+			if (message.ackThisNonce) {
+				// we need to ack the nonce to confirm that we received the sync-resp
+				this.log('Sending sync ack', message.ackThisNonce);
+				this.sendRequest([
+					await this.meta.messageCreator.createAck(message.ackThisNonce),
+				]);
+			}
+			// but we can go ahead and preemptively allow ops to be sent
+			this._hasSynced = true;
+		}
+		this.emit('message', message);
+	};
+
+	send = (message: ClientMessage) => {
+		// only certain messages are sent for pull-based sync.
+		switch (message.type) {
+			case 'presence-update':
+			case 'sync':
+			case 'heartbeat':
+				return this.sendRequest([message]);
+			case 'op':
+				if (this._hasSynced) {
+					return this.sendRequest([message]);
+				}
+				break;
+		}
+	};
+
+	start(): void {
+		if (this.status === 'active') {
+			return;
+		}
+		this.heartbeat.start(true);
+		this._status = 'active';
+	}
+	stop(): void {
+		this.heartbeat.stop();
+		this._status = 'paused';
+	}
+
+	dispose = () => {};
+	reconnect(): void {}
+
+	// on a heartbeat, do a sync
+	private onHeartbeat = async () => {
+		// for HTTP sync we send presence first, so that the sync-resp message
+		// will include the client's own presence info and fill in missing profile
+		// data on the first request. otherwise it would have to wait for the second.
+		this.sendRequest([
+			await this.meta.messageCreator.createPresenceUpdate(
+				this.presence.self.presence,
+			),
+			await this.meta.messageCreator.createSyncStep1(),
+		]);
+	};
+
+	// if the server fails to respond in a certain amount of time, we assume
+	// the connection is lost and go offline.
+	private onHeartbeatMissed = async () => {
+		this.emit('onlineChange', false);
+		this.log('Missed heartbeat');
+		this._isConnected = false;
+	};
+
+	get isConnected(): boolean {
+		return this._isConnected;
+	}
+	get status() {
+		return this._status;
+	}
+}

package/src/sync/ServerSyncEndpointProvider.ts

@@ -0,0 +1,85 @@
+import { assert, ReplicaType } from '@verdant-web/common';
+import { default as jwtDecode } from 'jwt-decode';
+
+export interface ServerSyncEndpointProviderConfig {
+	/**
+	 * The location of the endpoint used to retrieve an
+	 * authorization token for the client.
+	 */
+	authEndpoint?: string;
+	/**
+	 * A custom fetch function to retrieve authorization
+	 * data.
+	 */
+	fetchAuth?: () => Promise<{
+		accessToken: string;
+	}>;
+}
+
+export class ServerSyncEndpointProvider {
+	private cached = null as {
+		http: string;
+		websocket: string;
+		files: string;
+		token: string;
+	} | null;
+	type: ReplicaType = ReplicaType.Realtime;
+
+	constructor(private config: ServerSyncEndpointProviderConfig) {
+		if (!config.authEndpoint && !config.fetchAuth) {
+			throw new Error(
+				'Either authEndpoint or fetchAuth must be provided to ServerSyncEndpointProvider',
+			);
+		}
+	}
+
+	getEndpoints = async () => {
+		if (this.cached) {
+			return this.cached;
+		}
+
+		let result: { accessToken: string };
+		if (this.config.fetchAuth) {
+			result = await this.config.fetchAuth();
+		} else {
+			result = await fetch(this.config.authEndpoint!, {
+				credentials: 'include',
+			}).then((res) => {
+				if (!res.ok) {
+					throw new Error(
+						`Auth endpoint returned non-200 response: ${res.status}`,
+					);
+				} else {
+					return res.json();
+				}
+			});
+		}
+		assert(result.accessToken, 'No access token provided from auth endpoint');
+		const decoded = (jwtDecode as any)(result.accessToken);
+		assert(decoded.url, 'No sync endpoint provided from auth endpoint');
+		assert(
+			decoded.type !== undefined,
+			'No replica type provided from auth endpoint',
+		);
+		this.type = parseInt(decoded.type + '');
+		const url = new URL(decoded.url);
+		url.protocol = url.protocol.replace('ws', 'http');
+		const httpEndpoint = url.toString();
+		url.protocol = url.protocol.replace('http', 'ws');
+		const websocketEndpoint = url.toString();
+		let fileEndpoint: string = decoded.file;
+		if (!fileEndpoint) {
+			// default to http endpoint + '/files';
+			const fileUrl = new URL(httpEndpoint);
+			fileUrl.pathname = fileUrl.pathname + '/files';
+			fileEndpoint = fileUrl.toString();
+		}
+		this.cached = {
+			http: httpEndpoint,
+			websocket: websocketEndpoint,
+			files: fileEndpoint,
+			token: result.accessToken,
+		};
+		return this.cached;
+	};
+}