@verdant-web/store 3.8.3 → 3.9.0-next.0

package/src/IDBService.ts

@@ -81,7 +81,7 @@ export class IDBService extends Disposable {
 	iterate = async <T>(
 		storeName: string,
 		getRequest: (store: IDBObjectStore) => IDBRequest | IDBRequest[],
-		iterator: (value: T, store: IDBObjectStore) => void,
+		iterator: (value: T, store: IDBObjectStore) => boolean | void,
 		opts?: {
 			mode?: 'readonly' | 'readwrite';
 			transaction?: IDBTransaction;
@@ -117,10 +117,14 @@ export class IDBService extends Disposable {
 		}
 		return new Promise<void>((resolve, reject) => {
 			request.onsuccess = () => {
-				const cursor = request.result;
+				const cursor = request.result as IDBCursorWithValue | null;
 				if (cursor) {
-					iterator(cursor.value, store);
-					cursor.continue();
+					const stop = iterator(cursor.value, store);
+					if (stop) {
+						resolve();
+					} else {
+						cursor.continue();
+					}
 				} else {
 					resolve();
 				}

package/src/__tests__/entities.test.ts

@@ -492,8 +492,7 @@ describe('entities', () => {
 			],
 		});
 
-		const { id, ...snapshot } = item1.getSnapshot();
-		const item2 = await storage.todos.put(snapshot);
+		const item2 = await storage.todos.clone(item1);
 
 		expect(item2.get('tags').length).toBe(1);
 		expect(item2.get('attachments').length).toBe(1);
@@ -501,6 +500,7 @@ describe('entities', () => {
 		item2.get('attachments').get(0).set('name', 'attachment 2');
 
 		expect(item1.get('attachments').get(0).get('name')).toBe('attachment 1');
+		expect(item1.uid).not.toBe(item2.uid);
 	});
 
 	it('should not allow modifying the primary key', async () => {
@@ -634,13 +634,37 @@ describe('entities', () => {
 
 	it('should error on invalid values passed to initialization', async () => {
 		const storage = await createTestStorage();
-		expect(() => {
-			storage.todos.put({
+		expect(async () => {
+			await storage.todos.put({
 				content: { invalid: 'value' },
 			});
+		}).rejects.toThrowErrorMatchingInlineSnapshot(
+			`[Error: Validation error: Expected string for field content, got [object Object]]`,
+		);
+	});
+
+	it('should only allow valid values for file fields', async () => {
+		const storage = await createTestStorage();
+		const weird = await storage.weirds.put({});
+
+		expect(() => {
+			weird.set('file', { invalid: 'value' });
 		}).toThrowErrorMatchingInlineSnapshot(
-			`[Error: Validation error: Expected string  for field content, got [object Object]]`,
+			`[Error: Validation error: Expected file or null for field file, got [object Object]]`,
 		);
+
+		// valid options
+		weird.set(
+			'file',
+			new window.File(['d(⌐□_□)b'], 'test.txt', { type: 'text/plain' }),
+		);
+		weird.set('file', {
+			id: 'abc',
+			type: 'text/plain',
+			name: 'foo.txt',
+			url: 'http://example.com/foo.txt',
+			remote: true,
+		});
 	});
 
 	it('should allow subscribing to one field', async () => {

package/src/__tests__/fixtures/testStorage.ts

@@ -83,6 +83,9 @@ export const weirdCollection = schema.collection({
 				items: schema.fields.string(),
 			}),
 		}),
+		file: schema.fields.file({
+			nullable: true,
+		}),
 	},
 });
 

package/src/authorization.ts

@@ -0,0 +1,6 @@
+import { authz } from '@verdant-web/common';
+
+export const authorization = {
+	private: authz.onlyMe(),
+	public: undefined,
+};

package/src/client/Client.ts

@@ -6,7 +6,7 @@ import {
 	Operation,
 } from '@verdant-web/common';
 import { Context } from '../context.js';
-import { DocumentManager } from '../DocumentManager.js';
+import { DocumentManager } from '../entities/DocumentManager.js';
 import { EntityStore } from '../entities/EntityStore.js';
 import { FileManager, FileManagerConfig } from '../files/FileManager.js';
 import { ReturnedFileData } from '../files/FileStorage.js';
@@ -44,6 +44,12 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 	 * This event may be called multiple times.
 	 */
 	futureSeen: () => void;
+	/**
+	 * The server requested this replica reset its state
+	 * completely. This can happen when the replica has
+	 * been offline for too long and reconnects.
+	 */
+	resetToServer: () => void;
 }> {
 	readonly meta: Metadata;
 	private _entities: EntityStore;
@@ -106,16 +112,15 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 		this._queryCache = new QueryCache({
 			context,
 		});
-		this._documentManager = new DocumentManager(
-			this.meta,
-			this.schema,
-			this._entities,
-		);
+		this._documentManager = new DocumentManager(this.schema, this._entities);
 
 		const notifyFutureSeen = debounce(() => {
 			this.emit('futureSeen');
 		}, 300);
 		this.context.globalEvents.subscribe('futureSeen', notifyFutureSeen);
+		this.context.globalEvents.subscribe('resetToServer', () => {
+			this.emit('resetToServer');
+		});
 
 		this.documentDb.addEventListener('versionchange', () => {
 			this.context.log?.(
@@ -222,6 +227,8 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 				? await navigator.storage.estimate()
 				: undefined;
 
+		const files = await this._fileManager.stats();
+
 		// determine data:metadata ratio for total size of all collections vs metadata
 		const totalCollectionsSize = Object.values(collections).reduce(
 			(acc, { size }) => acc + size,
@@ -237,6 +244,7 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 			totalMetaSize,
 			totalCollectionsSize,
 			metaToDataRatio,
+			files,
 			quotaUsage:
 				storage?.usage && storage?.quota
 					? storage.usage / storage.quota
@@ -424,6 +432,26 @@ export class Client<Presence = any, Profile = any> extends EventSubscriber<{
 		const exportData = await this.export();
 		await this.import(exportData);
 	};
+
+	/**
+	 * Immediately runs the file deletion process. This is useful
+	 * for testing, mostly. Or if your client is long-lived, since
+	 * normally this cleanup only runs on startup.
+	 *
+	 * Note this still follows the file deletion heuristic configured
+	 * on the client. So if you clean up files 3 days after delete,
+	 * invoking this manually will not skip that 3 day waiting period.
+	 */
+	__cleanupFilesImmediately = () => {
+		return this._fileManager.tryCleanupDeletedFiles();
+	};
+
+	/**
+	 * Manually triggers storage rebasing. Follows normal
+	 * rebasing rules. Rebases already happen automatically
+	 * during normal operation, so you probably don't need this.
+	 */
+	__manualRebase = () => this.meta.manualRebase();
 }
 
 export interface ClientStats {
@@ -432,6 +460,9 @@ export interface ClientStats {
 		baselinesSize: { count: number; size: number };
 		operationsSize: { count: number; size: number };
 	};
+	files: {
+		size: { count: number; size: number };
+	};
 	storage: StorageEstimate | undefined;
 	totalMetaSize: number;
 	totalCollectionsSize: number;

package/src/context.ts

@@ -36,6 +36,12 @@ export interface Context {
 		 * The parameter is the timestamp of the future change.
 		 */
 		futureSeen: (timestamp: string) => void;
+		/**
+		 * The server requested this replica reset its state
+		 * completely. This can happen when the replica has
+		 * been offline for too long and reconnects.
+		 */
+		resetToServer: () => void;
 	}>;
 	weakRef<T extends object>(value: T): WeakRef<T>;
 	migrations: Migration<any>[];

package/src/entities/DocumentManager.ts

@@ -0,0 +1,154 @@
+import {
+	addFieldDefaults,
+	constrainEntity,
+	assert,
+	createOid,
+	SchemaCollection,
+	StorageCollectionSchema,
+	StorageDocument,
+	StorageSchema,
+	isRootOid,
+	AuthorizationKey,
+} from '@verdant-web/common';
+import { EntityCreateOptions, EntityStore } from '../entities/EntityStore.js';
+import { Metadata } from '../metadata/Metadata.js';
+import { Sync } from '../sync/Sync.js';
+import { Context } from '../context.js';
+import { ObjectEntity } from '../index.js';
+
+/**
+ * Exposes functionality for creating documents,
+ * the only mutation which is available as an entry
+ * point in the storage system.
+ */
+export class DocumentManager<Schema extends StorageSchema<any>> {
+	constructor(
+		private schema: Schema,
+		private entities: EntityStore,
+	) {}
+
+	private getOid = (collection: string, init: any) => {
+		const primaryKeyName = this.schema.collections[collection]
+			.primaryKey as Exclude<
+			keyof StorageDocument<SchemaCollection<Schema, any>>,
+			symbol
+		>;
+		const primaryKey = init[primaryKeyName];
+		assert(
+			primaryKey,
+			`Document must have a primary key: ${primaryKeyName.toString()} (got: ${JSON.stringify(
+				init,
+			)})`,
+		);
+		return createOid(collection, primaryKey);
+	};
+
+	private addDefaults = (collectionName: string, init: any) => {
+		const collection = this.schema.collections[
+			collectionName
+		] as StorageCollectionSchema;
+		return addFieldDefaults(collection, init);
+	};
+
+	private validate = (collectionName: string, init: any) => {
+		const collection = this.schema.collections[
+			collectionName
+		] as StorageCollectionSchema;
+		return constrainEntity(collection.fields, init);
+	};
+
+	create = async (
+		collection: string,
+		init: any,
+		options: {
+			undoable?: boolean;
+			access?: AuthorizationKey;
+			silenceAccessControlWithPrimaryKeyWarning?: boolean;
+		} = {},
+	) => {
+		const widenedOptions = options as EntityCreateOptions;
+		const defaulted = this.addDefaults(collection, init);
+		const validated = this.validate(collection, defaulted);
+		const oid = this.getOid(collection, validated);
+
+		if (options.access) {
+			const collectionSchema = this.schema.collections[collection];
+			if (
+				options.access !== 'shared' &&
+				init[collectionSchema.primaryKey] &&
+				!options.silenceAccessControlWithPrimaryKeyWarning
+			) {
+				// using a custom primary key with access control is not supported.
+				// resulting docs could collide with existing docs with different permissions,
+				// leading to confusing results. this logs a warning.
+				console.warn(
+					`Using a custom primary key with access control is not supported. This may result in corrupted documents. Read more about why: https://verdant.dev/docs/sync/access#a-warning-about-custom-primaryKey`,
+				);
+			}
+			widenedOptions.access = options.access;
+		}
+
+		// documents are always objects at the root
+		return this.entities.create(validated, oid, widenedOptions) as any;
+	};
+
+	delete = async (
+		collection: string,
+		primaryKey: string,
+		options: { undoable?: boolean } = {},
+	) => {
+		const oid = createOid(collection, primaryKey);
+		return this.entities.delete(oid, options);
+	};
+
+	deleteAll = async (
+		ids: [string, string][],
+		options: { undoable?: boolean } = {},
+	) => {
+		return this.entities.deleteAll(
+			ids.map(([collection, primaryKey]) => createOid(collection, primaryKey)),
+			options,
+		);
+	};
+
+	deleteAllFromCollection = async (
+		collection: string,
+		ids: string[],
+		options: { undoable?: boolean } = {},
+	) => {
+		return this.entities.deleteAll(
+			ids.map((primaryKey) => createOid(collection, primaryKey)),
+			options,
+		);
+	};
+
+	clone = async (
+		collection: string,
+		entity: ObjectEntity<any, any>,
+		options: {
+			undoable?: boolean;
+			access?: AuthorizationKey;
+			primaryKey?: string;
+		} = {},
+	) => {
+		if (!isRootOid(entity.uid)) {
+			throw new Error('Cannot clone non-root documents');
+		}
+		// take the entity snapshot
+		const snapshot = entity.getSnapshot();
+		// remove the primary key
+		const collectionSchema = this.schema.collections[collection];
+		delete snapshot[collectionSchema.primaryKey];
+		// if collection schema's primary key doesn't have a default value,
+		// a user-supplied value is required
+		if (!collectionSchema.fields[collectionSchema.primaryKey].default) {
+			if (!options.primaryKey) {
+				throw new Error(
+					`Error cloning document from collection ${collection}: collection does not have a default on primary key. You must supply a value to options.primaryKey for the clone.`,
+				);
+			}
+			snapshot[collectionSchema.primaryKey] = options.primaryKey;
+		}
+		return this.create(collection, snapshot, options);
+	};
+}

package/src/entities/Entity.ts

@@ -17,6 +17,7 @@ import {
 	getDefault,
 	hasDefault,
 	isFileRef,
+	isFile,
 	isNullable,
 	isObject,
 	isObjectRef,
@@ -29,7 +30,7 @@ import {
 } from '@verdant-web/common';
 import { Context } from '../context.js';
 import { FileManager } from '../files/FileManager.js';
-import { isFile, processValueFiles } from '../files/utils.js';
+import { processValueFiles } from '../files/utils.js';
 import { EntityFile } from '../index.js';
 import { EntityCache } from './EntityCache.js';
 import { EntityFamilyMetadata, EntityMetadataView } from './EntityMetadata.js';
@@ -382,6 +383,19 @@ export class Entity<
 		return this.ctx.namespace;
 	}
 
+	/**
+	 * The authz string signifying the permissions this entity has.
+	 * On the client (where we are) it's only ever possible to see
+	 * an entity with either full access or access for the current
+	 * user.
+	 */
+	get access() {
+		return this.viewData.authz;
+	}
+	get isAuthorized() {
+		return !!this.access;
+	}
+
 	/**
 	 * Pruning - when entities have invalid children, we 'prune' that
 	 * data up to the nearest prunable point - a nullable field,
@@ -443,6 +457,14 @@ export class Entity<
 	// change management methods (internal use only)
 	private addPendingOperations = (operations: Operation[]) => {
 		this.ctx.log('debug', 'Entity: adding pending operations', this.oid);
+
+		// apply authz to all operations
+		if (this.access) {
+			for (const op of operations) {
+				op.authz = this.access;
+			}
+		}
+
 		const changes = this.metadataFamily.addPendingData(operations);
 		for (const change of changes) {
 			this.change(change);
@@ -708,7 +730,9 @@ export class Entity<
 			if (validationError) {
 				// TODO: is it a good idea to throw an error here? a runtime error won't be that helpful,
 				// but also we don't really want invalid data supplied.
-				throw new Error(validationError.message);
+				throw new Error(`Validation error: ${validationError.message}`, {
+					cause: validationError,
+				});
 			}
 		}
 		return processValueFiles(value, this.files.add);

package/src/entities/EntityMetadata.ts

@@ -20,6 +20,7 @@ export type EntityMetadataView = {
 	empty: boolean;
 	updatedAt: number;
 	latestTimestamp: string | null;
+	authz?: string;
 };
 
 export class EntityMetadata {
@@ -63,6 +64,11 @@ export class EntityMetadata {
 	computeView = (omitPending = false): EntityMetadataView => {
 		const base = cloneDeep(this.baseline?.snapshot ?? undefined);
 		const baselineTimestamp = this.baseline?.timestamp ?? null;
+
+		// start with the baseline authz, if any. further init ops
+		// may overwrite this.
+		let authz = this.baseline?.authz;
+
 		const confirmedResult = this.applyOperations(
 			// apply ops to baseline
 			base,
@@ -79,6 +85,10 @@ export class EntityMetadata {
 		if (confirmedResult.futureSeen) {
 			this.ctx.globalEvents.emit('futureSeen', confirmedResult.futureSeen);
 		}
+		if (confirmedResult.authz) {
+			authz = confirmedResult.authz;
+		}
+
 		const pendingResult = omitPending
 			? confirmedResult
 			: this.applyOperations(
@@ -92,6 +102,10 @@ export class EntityMetadata {
 					// logically in the future
 					null,
 			  );
+		if (pendingResult.authz) {
+			authz = pendingResult.authz;
+		}
+
 		// before letting this data out into the wild, we need
 		// to associate its oid
 		if (pendingResult.view) {
@@ -138,6 +152,7 @@ export class EntityMetadata {
 			fromOlderVersion,
 			updatedAt,
 			latestTimestamp: updatedAtTimestamp,
+			authz,
 		};
 	};
 
@@ -207,8 +222,11 @@ export class EntityMetadata {
 		latestTimestamp: string | null;
 		deleted: boolean;
 		futureSeen: string | undefined;
+		authz?: string;
 	} => {
 		let futureSeen: string | undefined = undefined;
+		let authz: string | undefined = undefined;
+
 		const now = this.ctx.getNow();
 		for (const op of operations) {
 			// ignore ops before our after cutoff
@@ -229,6 +247,9 @@ export class EntityMetadata {
 				base = applyPatch(base, op.data);
 				if (op.data.op === 'initialize') {
 					deleted = false;
+					if (op.authz) {
+						authz = op.authz;
+					}
 				}
 			}
 
@@ -242,6 +263,7 @@ export class EntityMetadata {
 			latestTimestamp: latestTimestamp ?? null,
 			deleted,
 			futureSeen,
+			authz,
 		};
 	};
 }

package/src/entities/EntityStore.ts

@@ -13,6 +13,7 @@ import {
 	groupPatchesByRootOid,
 	isRootOid,
 	removeOidsFromAllSubObjects,
+	AuthorizationKey,
 } from '@verdant-web/common';
 import { Context } from '../context.js';
 import { Metadata } from '../metadata/Metadata.js';
@@ -49,6 +50,11 @@ export interface IncomingData {
 	isLocal?: boolean;
 }
 
+export interface EntityCreateOptions {
+	undoable?: boolean;
+	access?: AuthorizationKey;
+}
+
 export class EntityStore extends Disposable {
 	private ctx;
 	private meta;
@@ -123,6 +129,7 @@ export class EntityStore extends Disposable {
 			this.abortDataQueueController = new AbortController();
 			this.ongoingResetPromise = this.resetData().finally(() => {
 				this.ongoingResetPromise = null;
+				this.ctx.globalEvents.emit('resetToServer');
 			});
 		}
 
@@ -319,7 +326,7 @@ export class EntityStore extends Disposable {
 	create = async (
 		initial: any,
 		oid: ObjectIdentifier,
-		{ undoable = true }: { undoable?: boolean } = {},
+		{ undoable = true, access }: EntityCreateOptions = {},
 	) => {
 		this.ctx.log('debug', 'Creating new entity', oid);
 		const { collection } = decomposeOid(oid);
@@ -339,17 +346,23 @@ export class EntityStore extends Disposable {
 		}
 
 		const operations = this.meta.patchCreator.createInitialize(processed, oid);
+		if (access) {
+			operations.forEach((op) => {
+				op.authz = access;
+			});
+		}
 		await this.batcher.commitOperations(operations, {
 			undoable: !!undoable,
 			source: entity,
 		});
 
-		// TODO: what happens if you create an entity with an OID that already
+		// TODONE: what happens if you create an entity with an OID that already
 		// exists?
+		// A: it will overwrite the existing entity
 
 		// we still need to synchronously add the initial operations to the Entity
 		// even though they are flowing through the system
-		// TODO: this could be better aligned to avoid grouping here
+		// FIXME: this could be better aligned to avoid grouping here
 		const operationsGroupedByOid = groupPatchesByOid(operations);
 		this.events.add.invoke(this, {
 			operations: operationsGroupedByOid,
@@ -371,12 +384,7 @@ export class EntityStore extends Disposable {
 			'Only root documents may be deleted via client methods',
 		);
 
-		const allOids = await Promise.all(
-			oids.flatMap(async (oid) => {
-				const entity = await this.hydrate(oid);
-				return entity?.__getFamilyOids__() ?? [];
-			}),
-		);
+		const entities = await Promise.all(oids.map((oid) => this.hydrate(oid)));
 
 		// remove the entities from cache
 		oids.forEach((oid) => {
@@ -384,8 +392,18 @@ export class EntityStore extends Disposable {
 			this.ctx.log('debug', 'Deleted document from cache', oid);
 		});
 
-		// create the delete patches and wait for them to be applied
-		const operations = this.meta.patchCreator.createDeleteAll(allOids.flat());
+		const operations: Operation[] = [];
+		for (const entity of entities) {
+			if (entity) {
+				const oids = entity.__getFamilyOids__();
+				const deletes = this.meta.patchCreator.createDeleteAll(oids);
+				for (const op of deletes) {
+					op.authz = entity.access;
+				}
+				operations.push(...deletes);
+			}
+		}
+
 		await this.batcher.commitOperations(operations, {
 			undoable: options?.undoable === undefined ? true : options.undoable,
 		});

package/src/entities/types.ts

@@ -78,6 +78,7 @@ export interface BaseEntity<
 	readonly deleted: boolean;
 	readonly updatedAt: number;
 	readonly uid: string;
+	readonly isAuthorized: boolean;
 }
 
 export type DeepPartial<T> = T extends object

package/src/files/EntityFile.ts

@@ -101,10 +101,14 @@ export class EntityFile extends EventSubscriber<EntityFileEvents> {
 		this.dispose();
 	};
 
-	getSnapshot(): EntityFileSnapshot {
+	getSnapshot(): FileData {
+		if (this._fileData) return this._fileData;
 		return {
 			id: this.id,
-			url: this.loading || this.failed ? undefined : this.url,
+			url: this._objectUrl ?? undefined,
+			name: this.name ?? 'unknown-file',
+			remote: false,
+			type: this.type ?? '',
 		};
 	}
 }