@verdant-web/store 3.8.4 → 3.9.0-next.0

package/src/files/FileManager.ts

@@ -43,6 +43,9 @@ export class FileManager {
 	private config: Required<FileManagerConfig>;
 	private meta: Metadata;
 
+	private maxUploadRetries = 3;
+	private maxDownloadRetries = 3;
+
 	constructor({
 		db,
 		sync,
@@ -72,8 +75,20 @@ export class FileManager {
 		this.tryCleanupDeletedFiles();
 	}
 
-	add = async (fileInput: Omit<FileData, 'remote'>) => {
-		const file = fileInput as unknown as FileData;
+	add = async (file: FileData) => {
+		// this method accepts a FileData which refers to a remote
+		// file, as well as local files. in the case of a remote file,
+		// we actually re-download and upload the file again. this powers
+		// the cloning of documents with files; we clone their filedata
+		// and re-upload to a new file ID. otherwise, when the cloned
+		// filedata was marked deleted, the original file would be deleted
+		// and the clone would refer to a missing file.
+		if (file.url && !file.file) {
+			const blob = await this.downloadRemoteFile(file.url);
+			// convert blob to file with name and type
+			file.file = new File([blob], file.name, { type: file.type });
+		}
+
 		file.remote = false;
 		// immediately cache the file
 		if (!this.files.has(file.id)) {
@@ -86,8 +101,8 @@ export class FileManager {
 		// write to local storage and send to sync immediately
 		await this.storage.addFile(file);
 		// send to sync
-		if (file.file) {
-			await this.uploadFile(file, 1);
+		if (file.file && this.sync.status === 'active') {
+			await this.uploadFile(file);
 		}
 	};
 
@@ -99,20 +114,49 @@ export class FileManager {
 			if (cached) {
 				cached[MARK_UPLOADED]();
 			}
+			this.context.log('info', 'File uploaded', file.id);
 		} else {
-			if (result.retry && retries < 5) {
-				this.context.log('error', 'Error uploading file, retrying...');
+			if (result.retry && retries < this.maxUploadRetries) {
+				this.context.log(
+					'error',
+					`Error uploading file ${file.id}, retrying...`,
+					result.error,
+				);
 				// schedule a retry
 				setTimeout(this.uploadFile, 1000, file, retries + 1);
 			} else {
 				this.context.log(
 					'error',
-					'Failed to upload file. Not retrying until next sync.',
+					`Failed to upload file ${file.id}. Not retrying until next sync.`,
+					result.error,
 				);
 			}
 		}
 	};
 
+	private downloadRemoteFile = async (
+		url: string,
+		retries = 0,
+	): Promise<Blob> => {
+		const resp = await fetch(url, {
+			method: 'GET',
+			credentials: 'include',
+		});
+		if (!resp.ok) {
+			if (retries < this.maxDownloadRetries) {
+				return new Promise((resolve, reject) => {
+					setTimeout(() => {
+						this.downloadRemoteFile(url, retries + 1).then(resolve, reject);
+					}, 1000);
+				});
+			} else {
+				throw new Error(`Failed to download file: ${resp.status}`);
+			}
+		}
+		const blob = await resp.blob();
+		return blob;
+	};
+
 	/**
 	 * Immediately returns an EntityFile to use, then either loads
 	 * the file from cache, local database, or the server.
@@ -128,7 +172,7 @@ export class FileManager {
 	};
 
 	private load = async (file: EntityFile, retries = 0) => {
-		if (retries > 5) {
+		if (retries > this.maxDownloadRetries) {
 			this.context.log('error', 'Failed to load file after 5 retries');
 			file[MARK_FAILED]();
 			return;
@@ -205,7 +249,7 @@ export class FileManager {
 		}
 	};
 
-	private tryCleanupDeletedFiles = async () => {
+	tryCleanupDeletedFiles = async () => {
 		let count = 0;
 		let skipCount = 0;
 		await this.storage.iterateOverPendingDelete((fileData, store) => {
@@ -243,4 +287,8 @@ export class FileManager {
 	close = () => {
 		this.storage.dispose();
 	};
+
+	stats = () => {
+		return this.storage.stats();
+	};
 }

package/src/files/FileStorage.ts

@@ -1,7 +1,7 @@
 import { FileData } from '@verdant-web/common';
 import { IDBService } from '../IDBService.js';
 import { fileToArrayBuffer } from './utils.js';
-import { getAllFromObjectStores } from '../idb.js';
+import { getAllFromObjectStores, getSizeOfObjectStore } from '../idb.js';
 
 /**
  * When stored in IDB, replace the file blob with an array buffer
@@ -217,6 +217,12 @@ export class FileStorage extends IDBService {
 		const [files] = await getAllFromObjectStores(this.db, ['files']);
 		return files.map(this.hydrateFileData);
 	};
+
+	stats = async () => {
+		return {
+			size: await getSizeOfObjectStore(this.db, 'files'),
+		};
+	};
 }
 
 export function arrayBufferToBlob(buffer: ArrayBuffer, type: string) {

package/src/files/utils.ts

@@ -1,4 +1,9 @@
-import { createFileRef, FileData } from '@verdant-web/common';
+import {
+	createFileRef,
+	FileData,
+	isFile,
+	isFileData,
+} from '@verdant-web/common';
 import cuid from 'cuid';
 
 export function createFileData(file: File): FileData {
@@ -12,13 +17,6 @@ export function createFileData(file: File): FileData {
 	};
 }
 
-export function isFile(value: any): value is File {
-	return (
-		value instanceof File ||
-		(typeof Blob !== 'undefined' && value instanceof Blob)
-	);
-}
-
 /**
  * MUTATES the value.
  * Replaces File values with refs and returns the normalized value.
@@ -34,6 +32,13 @@ export function processValueFiles(
 		return createFileRef(data.id);
 	}
 
+	if (isFileData(value)) {
+		// create a new ID for the file
+		const cloned = { ...value, id: cuid() };
+		onFileIdentified(cloned);
+		return createFileRef(cloned.id);
+	}
+
 	if (Array.isArray(value)) {
 		for (let i = 0; i < value.length; i++) {
 			value[i] = processValueFiles(value[i], onFileIdentified);
@@ -52,6 +57,10 @@ export function processValueFiles(
 }
 
 export function fileToArrayBuffer(file: File | Blob) {
+	// special case for testing...
+	if ('__testReadBuffer' in file) {
+		return file.__testReadBuffer;
+	}
 	return new Promise<ArrayBuffer>((resolve, reject) => {
 		const reader = new FileReader();
 		reader.onload = () => {

package/src/index.ts

@@ -55,3 +55,4 @@ export type { CollectionQueries } from './queries/CollectionQueries.js';
 export { MigrationPathError } from './migration/errors.js';
 export * from './utils/id.js';
 export { UndoHistory } from './UndoHistory.js';
+export * from './authorization.js';

package/src/metadata/LocalReplicaStore.ts

@@ -4,6 +4,7 @@ import { IDBService } from '../IDBService.js';
 export type LocalReplicaInfo = {
 	type: 'localReplicaInfo';
 	id: string;
+	userId: string | undefined;
 	ackedLogicalTime: string | null;
 	lastSyncedLogicalTime: string | null;
 };
@@ -36,6 +37,7 @@ export class LocalReplicaStore extends IDBService {
 					const replicaInfo: LocalReplicaInfo = {
 						type: 'localReplicaInfo',
 						id: replicaId,
+						userId: undefined,
 						ackedLogicalTime: null,
 						lastSyncedLogicalTime: null,
 					};

package/src/metadata/MessageCreator.ts

@@ -6,6 +6,7 @@ import {
 	ObjectIdentifier,
 	Operation,
 	OperationMessage,
+	pickValidOperationKeys,
 	PresenceUpdateMessage,
 	SyncAckMessage,
 	SyncMessage,
@@ -26,11 +27,7 @@ export class MessageCreator {
 			type: 'op',
 			timestamp: this.meta.now,
 			replicaId: localInfo.id,
-			operations: init.operations.map((op) => ({
-				data: op.data,
-				oid: op.oid,
-				timestamp: op.timestamp,
-			})),
+			operations: init.operations.map(pickValidOperationKeys),
 		};
 	};
 
@@ -73,11 +70,7 @@ export class MessageCreator {
 		if (provideChangesSince) {
 			await this.meta.operations.iterateOverAllLocalOperations(
 				(patch) => {
-					operations.push({
-						data: patch.data,
-						oid: patch.oid,
-						timestamp: patch.timestamp,
-					});
+					operations.push(pickValidOperationKeys(patch));
 					affectedDocs.add(getOidRoot(patch.oid));
 				},
 				{
@@ -91,11 +84,7 @@ export class MessageCreator {
 			// operations
 			await this.meta.operations.iterateOverAllOperations(
 				(patch) => {
-					operations.push({
-						data: patch.data,
-						oid: patch.oid,
-						timestamp: patch.timestamp,
-					});
+					operations.push(pickValidOperationKeys(patch));
 					affectedDocs.add(getOidRoot(patch.oid));
 				},
 				{

package/src/metadata/Metadata.ts

@@ -262,6 +262,25 @@ export class Metadata extends EventSubscriber<{
 		};
 	};
 
+	getDocumentAuthz = async (oid: ObjectIdentifier) => {
+		const baseline = await this.baselines.get(oid);
+		if (baseline) {
+			return baseline.authz;
+		}
+		let authz;
+		await this.operations.iterateOverAllOperationsForEntity(
+			oid,
+			(op) => {
+				if (op.data.op === 'initialize') {
+					authz = op.authz;
+					return true;
+				}
+			},
+			{},
+		);
+		return authz;
+	};
+
 	/**
 	 * Methods for writing data
 	 */
@@ -487,6 +506,7 @@ export class Metadata extends EventSubscriber<{
 		const baseline = await this.baselines.get(oid, { transaction });
 		let current: any = baseline?.snapshot || undefined;
 		let operationsApplied = 0;
+		let authz = baseline?.authz;
 		const deletedRefs: Ref[] = [];
 		await this.operations.iterateOverAllOperationsForEntity(
 			oid,
@@ -495,6 +515,9 @@ export class Metadata extends EventSubscriber<{
 				// but it's here as a safety measure...
 				if (!baseline || patch.timestamp > baseline.timestamp) {
 					current = applyPatch(current, patch.data, deletedRefs);
+					if (patch.data.op === 'initialize') {
+						authz = patch.authz;
+					}
 				}
 				// delete all prior operations to the baseline
 				operationsApplied++;
@@ -512,6 +535,7 @@ export class Metadata extends EventSubscriber<{
 			oid,
 			snapshot: current,
 			timestamp: upTo,
+			authz,
 		};
 		if (newBaseline.snapshot) {
 			await this.baselines.set(newBaseline, { transaction });
@@ -583,6 +607,19 @@ export class Metadata extends EventSubscriber<{
 		}
 	};
 
+	/**
+	 * Manually triggers a storage rebase.
+	 * Rebases happen automatically as needed, so
+	 * you probably don't need this.
+	 */
+	manualRebase = async () => {
+		if (this._closing || this.disableRebasing) return;
+		const ackInfo = await this.ackInfo.getAckInfo();
+		if (ackInfo.globalAckTimestamp) {
+			this.runRebase(ackInfo.globalAckTimestamp);
+		}
+	};
+
 	export = async (): Promise<ExportData> => {
 		const db = this.db;
 		const [baselines, operations] = await getAllFromObjectStores(db, [

package/src/metadata/OperationsStore.ts

@@ -267,11 +267,11 @@ export class OperationsStore extends IDBService {
 
 				const range =
 					start && end
-						? window.IDBKeyRange.bound(start, end, false, true)
+						? IDBKeyRange.bound(start, end, false, true)
 						: start
-						? window.IDBKeyRange.lowerBound(start, false)
+						? IDBKeyRange.lowerBound(start, false)
 						: end
-						? window.IDBKeyRange.upperBound(end, true)
+						? IDBKeyRange.upperBound(end, true)
 						: undefined;
 				const index = store.index('timestamp');
 				return index.openCursor(range, 'next');

package/src/migration/engine.ts

@@ -12,6 +12,7 @@ import {
 	getOid,
 	initialToPatches,
 	removeOidPropertiesFromAllSubObjects,
+	AuthorizationKey,
 } from '@verdant-web/common';
 import { Context } from '../context.js';
 import { Metadata } from '../metadata/Metadata.js';
@@ -31,26 +32,31 @@ function getMigrationMutations({
 }) {
 	return migration.allCollections.reduce((acc, collectionName) => {
 		acc[collectionName] = {
-			put: async (doc: any) => {
+			put: async (doc: any, options?: { access?: AuthorizationKey }) => {
 				// add defaults
 				addFieldDefaults(migration.newSchema.collections[collectionName], doc);
 				const primaryKey =
 					doc[migration.newSchema.collections[collectionName].primaryKey];
 				const oid = createOid(collectionName, primaryKey);
 				newOids.push(oid);
+
 				await meta.insertLocalOperations(
-					initialToPatches(doc, oid, getMigrationNow),
+					initialToPatches(doc, oid, getMigrationNow, undefined, undefined, {
+						authz: options?.access,
+					}),
 				);
 				return doc;
 			},
 			delete: async (id: string) => {
 				const rootOid = createOid(collectionName, id);
+				const authz = await meta.getDocumentAuthz(rootOid);
 				const allOids = await meta.getAllDocumentRelatedOids(rootOid);
 				return meta.insertLocalOperations(
 					allOids.map((oid) => ({
 						oid,
 						timestamp: getMigrationNow(),
 						data: { op: 'delete' },
+						authz,
 					})),
 				);
 			},
@@ -164,6 +170,11 @@ export function getMigrationEngine({
 						!!rootOid,
 						`Document is missing an OID: ${JSON.stringify(doc)}`,
 					);
+					// FIXME: this could be optimized (making n queries for authz
+					// when the snapshots themselves are derived from the same data...)
+					// maybe don't use the findAll query, and instead go a level
+					// lower to retain access to lower level data here?
+					const authz = await meta.getDocumentAuthz(rootOid);
 					const original = cloneDeep(doc);
 					// @ts-ignore - excessive type resolution
 					const newValue = await strategy(doc);
@@ -182,6 +193,7 @@ export function getMigrationEngine({
 							[],
 							{
 								mergeUnknownObjects: true,
+								authz,
 							},
 						);
 						if (patches.length > 0) {

package/src/queries/CollectionQueries.ts

@@ -1,4 +1,8 @@
-import { CollectionFilter, hashObject } from '@verdant-web/common';
+import {
+	AuthorizationKey,
+	CollectionFilter,
+	hashObject,
+} from '@verdant-web/common';
 import { Context } from '../context.js';
 import { EntityStore } from '../entities/EntityStore.js';
 import { GetQuery } from './GetQuery.js';
@@ -7,8 +11,8 @@ import { FindOneQuery } from './FindOneQuery.js';
 import { FindPageQuery } from './FindPageQuery.js';
 import { FindInfiniteQuery } from './FindInfiniteQuery.js';
 import { FindAllQuery } from './FindAllQuery.js';
-import { DocumentManager } from '../DocumentManager.js';
-import { ObjectEntity } from '../index.js';
+import { DocumentManager } from '../entities/DocumentManager.js';
+import { Entity, ObjectEntity } from '../index.js';
 import { UPDATE } from './BaseQuery.js';
 
 export class CollectionQueries<
@@ -22,9 +26,20 @@ export class CollectionQueries<
 	private context;
 	private documentManager;
 
-	put: (init: Init, options?: { undoable?: boolean }) => Promise<T>;
+	put: (
+		init: Init,
+		options?: { undoable?: boolean; access?: AuthorizationKey },
+	) => Promise<T>;
 	delete: (id: string, options?: { undoable?: boolean }) => Promise<void>;
 	deleteAll: (ids: string[], options?: { undoable?: boolean }) => Promise<void>;
+	clone: (
+		entity: ObjectEntity<any, any>,
+		options?: {
+			undoable?: boolean;
+			access?: AuthorizationKey;
+			primaryKey?: string;
+		},
+	) => Promise<T>;
 
 	constructor({
 		collection,
@@ -57,6 +72,10 @@ export class CollectionQueries<
 			this.documentManager,
 			this.collection,
 		);
+		this.clone = this.documentManager.clone.bind(
+			this.documentManager,
+			this.collection,
+		);
 	}
 
 	private serializeIndex = (index?: CollectionFilter) => {

package/src/sync/FileSync.ts

@@ -4,6 +4,7 @@ import { ServerSyncEndpointProvider } from './ServerSyncEndpointProvider.js';
 export interface FileUploadResult {
 	success: boolean;
 	retry: boolean;
+	error?: string;
 }
 
 export type FilePullResult =
@@ -43,7 +44,7 @@ export class FileSync {
 		const { files: fileEndpoint, token } =
 			await this.endpointProvider.getEndpoints();
 
-		const formData = new window.FormData();
+		const formData = new FormData();
 		formData.append('file', file);
 
 		try {
@@ -63,15 +64,12 @@ export class FileSync {
 					retry: false,
 				};
 			} else {
-				this.log(
-					'error',
-					'File upload failed',
-					response.status,
-					await response.text(),
-				);
+				const responseText = await response.text();
+				this.log('error', 'File upload failed', response.status, responseText);
 				return {
 					success: false,
 					retry: response.status >= 500,
+					error: `Failed to upload file: ${response.status} ${responseText}`,
 				};
 			}
 		} catch (e) {
@@ -79,6 +77,7 @@ export class FileSync {
 			return {
 				success: false,
 				retry: true,
+				error: (e as Error).message,
 			};
 		}
 	};

package/src/sync/PushPullSync.ts

@@ -68,6 +68,10 @@ export class PushPullSync
 		return this.heartbeat.interval;
 	}
 
+	get hasSynced() {
+		return this._hasSynced;
+	}
+
 	private sendRequest = async (messages: ClientMessage[]) => {
 		this.log('Sending sync request', messages);
 		try {
@@ -169,13 +173,14 @@ export class PushPullSync
 		}
 	};
 
-	start(): void {
+	start = async () => {
 		if (this.status === 'active') {
 			return;
 		}
+		await this.endpointProvider.getEndpoints();
 		this.heartbeat.start(true);
 		this._status = 'active';
-	}
+	};
 	stop(): void {
 		this.heartbeat.stop();
 		this._status = 'paused';

package/src/sync/ServerSyncEndpointProvider.ts

@@ -22,6 +22,15 @@ export interface ServerSyncEndpointProviderConfig {
 	fetch?: typeof fetch;
 }
 
+export interface SyncTokenInfo {
+	url: string;
+	fileUrl: string;
+	type: ReplicaType;
+	userId: string;
+	libraryId: string;
+	role?: string;
+}
+
 export class ServerSyncEndpointProvider {
 	private cached = null as {
 		http: string;
@@ -29,7 +38,11 @@ export class ServerSyncEndpointProvider {
 		files: string;
 		token: string;
 	} | null;
-	type: ReplicaType = ReplicaType.Realtime;
+	tokenInfo: SyncTokenInfo | null = null;
+
+	get type() {
+		return this.tokenInfo?.type ?? ReplicaType.Realtime;
+	}
 
 	constructor(private config: ServerSyncEndpointProviderConfig) {
 		if (!config.authEndpoint && !config.fetchAuth) {
@@ -68,7 +81,14 @@ export class ServerSyncEndpointProvider {
 			decoded.type !== undefined,
 			'No replica type provided from auth endpoint',
 		);
-		this.type = parseInt(decoded.type + '');
+		this.tokenInfo = {
+			userId: decoded.sub,
+			libraryId: decoded.lib,
+			url: decoded.url,
+			fileUrl: decoded.file,
+			role: decoded.role,
+			type: parseInt(decoded.type + '') as ReplicaType,
+		};
 		const url = new URL(decoded.url);
 		url.protocol = url.protocol.replace('ws', 'http');
 		const httpEndpoint = url.toString();

package/src/sync/Sync.ts

@@ -5,7 +5,9 @@ import {
 	FileData,
 	Operation,
 	ReplicaType,
+	rewriteAuthzOriginator,
 	ServerMessage,
+	VerdantError,
 } from '@verdant-web/common';
 import { Metadata } from '../metadata/Metadata.js';
 import { HANDLE_MESSAGE, PresenceManager } from './PresenceManager.js';
@@ -33,10 +35,11 @@ export interface SyncTransport extends EventSubscriber<SyncTransportEvents> {
 	readonly presence: PresenceManager;
 
 	readonly mode: SyncTransportMode;
+	readonly hasSynced: boolean;
 
 	send(message: ClientMessage): void;
 
-	start(): void;
+	start(): Promise<void>;
 	ignoreIncoming(): void;
 	stop(): void;
 
@@ -57,7 +60,7 @@ export interface Sync<Presence = any, Profile = any>
 	getFile(fileId: string): Promise<FilePullResult>;
 	readonly presence: PresenceManager<Profile, Presence>;
 	send(message: ClientMessage): void;
-	start(): void;
+	start(): Promise<void>;
 	stop(): void;
 	ignoreIncoming(): void;
 	destroy(): void;
@@ -76,7 +79,7 @@ export class NoSync<Presence = any, Profile = any>
 
 	public send(): void {}
 
-	public start(): void {}
+	public async start(): Promise<void> {}
 
 	public stop(): void {}
 
@@ -257,19 +260,19 @@ export class ServerSync<Presence = any, Profile = any>
 			endpointProvider: this.endpointProvider,
 			meta,
 			presence: this.presence,
-			log: this.log,
+			log: ctx.log,
 		});
 		this.pushPullSync = new PushPullSync({
 			endpointProvider: this.endpointProvider,
 			meta,
 			presence: this.presence,
-			log: this.log,
+			log: ctx.log,
 			interval: pullInterval,
 			fetch,
 		});
 		this.fileSync = new FileSync({
 			endpointProvider: this.endpointProvider,
-			log: this.log,
+			log: ctx.log,
 		});
 		if (useBroadcastChannel && 'BroadcastChannel' in window) {
 			this.broadcastChannel = new BroadcastChannel(`verdant-${ctx.namespace}`);
@@ -458,6 +461,22 @@ export class ServerSync<Presence = any, Profile = any>
 
 	send = async (message: ClientMessage) => {
 		if (this.activeSync.status === 'active') {
+			// before sync, replace 'originator' authz subjects
+			// with token userId. This is the easiest place to
+			// do this and allows the rest of the system to be
+			// ambivalent about user identity when assigning
+			// authorization for the current user.
+			const userId = this.endpointProvider.tokenInfo?.userId;
+			if (!userId) {
+				throw new VerdantError(
+					VerdantError.Code.Unexpected,
+					undefined,
+					'Active sync has invalid token info',
+				);
+			}
+			if (message.type === 'sync' || message.type === 'op') {
+				rewriteAuthzOriginator(message, userId);
+			}
 			await this.activeSync.send(message);
 			this.onOutgoingMessage?.(message);
 		}
@@ -468,6 +487,7 @@ export class ServerSync<Presence = any, Profile = any>
 			name: info.name,
 			type: info.type,
 			id: info.id,
+			size: info.file?.size,
 		});
 		if (this.activeSync.status === 'active') {
 			return this.fileSync.uploadFile(info);
@@ -475,6 +495,7 @@ export class ServerSync<Presence = any, Profile = any>
 			return {
 				success: false,
 				retry: false,
+				error: 'Sync is not active',
 			};
 		}
 	};