@verdant-web/store 4.1.2 → 4.1.3

package/src/client/ClientDescriptor.ts

@@ -6,17 +6,17 @@ import {
 	StorageSchema,
 	noop,
 } from '@verdant-web/common';
-import { FileConfig, InitialContext } from '../context/context.js';
+import { FileConfig, InitialContext, QueryConfig } from '../context/context.js';
+import { ShutdownHandler } from '../context/ShutdownHandler.js';
+import { Time } from '../context/Time.js';
+import { FakeWeakRef } from '../FakeWeakRef.js';
+import { IdbPersistence } from '../persistence/idb/idbPersistence.js';
+import { deleteAllDatabases } from '../persistence/idb/util.js';
+import { PersistenceImplementation } from '../persistence/interfaces.js';
+import { initializePersistence } from '../persistence/persistence.js';
 import { ServerSyncOptions } from '../sync/Sync.js';
 import { UndoHistory } from '../UndoHistory.js';
 import { Client } from './Client.js';
-import { deleteAllDatabases } from '../persistence/idb/util.js';
-import { FakeWeakRef } from '../FakeWeakRef.js';
-import { Time } from '../context/Time.js';
-import { initializePersistence } from '../persistence/persistence.js';
-import { PersistenceImplementation } from '../persistence/interfaces.js';
-import { IdbPersistence } from '../persistence/idb/idbPersistence.js';
-import { ShutdownHandler } from '../context/ShutdownHandler.js';
 
 export interface ClientDescriptorOptions<Presence = any, Profile = any> {
 	/** The schema used to create this client */
@@ -76,6 +76,11 @@ export interface ClientDescriptorOptions<Presence = any, Profile = any> {
 	 * before turning it on.
 	 */
 	EXPERIMENTAL_weakRefs?: boolean;
+
+	/**
+	 * Customize querying behavior.
+	 */
+	queries?: QueryConfig;
 }
 
 /**
@@ -155,6 +160,7 @@ export class ClientDescriptor<
 						disableRebasing: init.disableRebasing,
 						rebaseTimeout: init.rebaseTimeout,
 					},
+					queries: init.queries,
 				},
 				persistence:
 					init.persistence || new IdbPersistence(environment.indexedDB),

package/src/context/context.ts

@@ -10,15 +10,15 @@ import {
 	StorageSchema,
 } from '@verdant-web/common';
 import { UndoHistory } from '../UndoHistory.js';
-import { Time } from './Time.js';
-import type { PersistenceDocuments } from '../persistence/PersistenceQueries.js';
-import type { PersistenceMetadata } from '../persistence/PersistenceMetadata.js';
 import { PersistenceFiles } from '../persistence/PersistenceFiles.js';
+import type { PersistenceMetadata } from '../persistence/PersistenceMetadata.js';
+import type { PersistenceDocuments } from '../persistence/PersistenceQueries.js';
 import {
 	PersistedFileData,
 	PersistenceImplementation,
 } from '../persistence/interfaces.js';
 import { ShutdownHandler } from './ShutdownHandler.js';
+import { Time } from './Time.js';
 
 /**
  * Common components utilized across various client
@@ -57,6 +57,7 @@ export interface Context {
 		filesDeleted: (files: FileRef[]) => void;
 		fileAdded: (file: FileData) => void;
 		[ev: `fileUploaded:${string}`]: (file: FileData) => void;
+		fileUploaded: (file: FileData) => void;
 	}>;
 	globalEvents: EventSubscriber<{
 		/**
@@ -99,6 +100,7 @@ export interface Context {
 		files?: FileConfig;
 		sync?: SyncConfig;
 		persistence?: PersistenceConfig;
+		queries?: QueryConfig;
 	};
 
 	environment: {
@@ -209,4 +211,17 @@ export interface PersistenceConfig {
 	rebaseTimeout?: number;
 }
 
+export interface QueryConfig {
+	/**
+	 * Milliseconds to hold a query in memory after it is unsubscribed before
+	 * disposing of it. Once a query is disposed, it must be loaded fresh again
+	 * on next use. Queries are cached based on their `key`, which you can
+	 * manually override. By default keys are determined by the parameters
+	 * passed to the query.
+	 *
+	 * Defaults to 5 seconds.
+	 */
+	evictionTime?: number;
+}
+
 export type InitialContext = Omit<Context, 'documents' | 'meta' | 'files'>;

package/src/entities/Entity.ts

@@ -13,8 +13,8 @@ import {
 	getChildFieldSchema,
 	getDefault,
 	hasDefault,
-	isFileRef,
 	isFile,
+	isFileRef,
 	isNullable,
 	isObject,
 	isRef,
@@ -29,6 +29,8 @@ import { processValueFiles } from '../files/utils.js';
 import { EntityFile } from '../index.js';
 import { EntityCache } from './EntityCache.js';
 import { EntityFamilyMetadata, EntityMetadataView } from './EntityMetadata.js';
+import { EntityStoreEventData, EntityStoreEvents } from './EntityStore.js';
+import { entityFieldSubscriber } from './entityFieldSubscriber.js';
 import {
 	BaseEntityValue,
 	DataFromInit,
@@ -39,8 +41,6 @@ import {
 	ListItemValue,
 	ObjectEntity,
 } from './types.js';
-import { EntityStoreEventData, EntityStoreEvents } from './EntityStore.js';
-import { entityFieldSubscriber } from './entityFieldSubscriber.js';
 
 export interface EntityInit {
 	oid: ObjectIdentifier;
@@ -73,7 +73,7 @@ export class Entity<
 	private entityFamily: EntityCache;
 	private metadataFamily;
 
-	private schema;
+	readonly schema;
 	private parent: Entity | undefined;
 	private ctx;
 	private files;
@@ -657,6 +657,10 @@ export class Entity<
 				})
 			) {
 				if (hasDefault(fieldSchema)) {
+					// FIXME: this returns []/{} for arrays and objects, but the contract
+					// of this method should return an Entity for such object fields.
+					// I want to write a test case for this one before attempting to fix
+					// just to be sure the fix works.
 					return getDefault(fieldSchema);
 				}
 				if (isNullable(fieldSchema)) {

package/src/entities/types.ts

@@ -1,19 +1,16 @@
 import { ObjectIdentifier, StorageFieldSchema } from '@verdant-web/common';
 import type { Entity } from './Entity.js';
 
-export type AccessibleEntityProperty<T> = T extends Array<any>
-	? number
-	: T extends object
-	? keyof T
-	: never;
+export type AccessibleEntityProperty<T> =
+	T extends Array<any> ? number : T extends object ? keyof T : never;
 
 export type DataFromInit<Init> = Init extends { [key: string]: any }
 	? {
 			[Key in keyof Init]: Init[Key];
-	  }
+		}
 	: Init extends Array<any>
-	? Init
-	: any;
+		? Init
+		: any;
 
 // reduces keys of an object to only ones with an optional
 // value
@@ -23,15 +20,11 @@ export type DeletableKeys<T> = keyof {
 type IfNullableThen<T, Out> = undefined extends T
 	? Out
 	: null extends T
-	? Out
-	: never;
+		? Out
+		: never;
 
-export type EntityShape<E extends Entity<any, any>> = E extends Entity<
-	infer Value,
-	any
->
-	? Value
-	: never;
+export type EntityShape<E extends Entity<any, any>> =
+	E extends Entity<infer Value, any> ? Value : never;
 
 export type BaseEntityValue = { [Key: string]: any } | any[];
 
@@ -73,21 +66,62 @@ export interface BaseEntity<
 		) => void,
 	): () => void;
 	get<Key extends keyof Value>(key: Key): Value[Key];
+	/**
+	 * Returns a plain object or array containing sub-Entities and their data.
+	 * Equivalent to "destructuring" the entity. Unlike getSnapshot, sub-level
+	 * data is still reactive.
+	 */
 	getAll(): Readonly<Value>;
+	/**
+	 * Returns a plain Javascript object representing the current state of the entity.
+	 */
 	getSnapshot(): Snapshot;
+	/**
+	 * Returns the schema for the entity as specified in your Verdant schema.
+	 * For root Documents, this will be an Object schema with properties
+	 * representing each field in the document.
+	 */
+	readonly schema: StorageFieldSchema;
+	/**
+	 * Returns the schema for a field in the entity as specified in
+	 * your Verdant schema.
+	 */
 	getFieldSchema<FieldName extends keyof Value>(
 		key: FieldName,
 	): StorageFieldSchema;
+	/**
+	 * Will be marked true after an entity has been deleted. Any attempt to
+	 * access the entity's data will result in an error.
+	 */
 	readonly deleted: boolean;
+	/**
+	 * A Unix Epoch timestamp representing the last time this entity was updated.
+	 */
 	readonly updatedAt: number;
+	/**
+	 * A Unix Epoch timestamp representing the last time this entity or any of its
+	 * sub-entities were updated.
+	 *
+	 * NOTE: reading this property requires a bit of computation, but the result
+	 * is cached. If an entity is being frequently updated and this is frequently
+	 * read, it may result in mild performance degradation.
+	 */
+	readonly deepUpdatedAt: number;
+	/** A unique, opaque key for this Entity in the system. */
 	readonly uid: string;
+	/** If true, this Entity has authorization rules applied to it. */
 	readonly isAuthorized: boolean;
+	/** The authorization configuration string applied to this entity. */
+	readonly access: string | undefined;
+	readonly invalid: boolean;
+	/** The Verdant store namespace which contains this object */
+	readonly namespace: string;
 }
 
 export type DeepPartial<T> = T extends object
 	? {
 			[P in keyof T]?: DeepPartial<T[P]>;
-	  }
+		}
 	: T;
 
 export interface ObjectEntity<
@@ -200,22 +234,21 @@ export type AnyEntity<
 	| ListEntity<Init, KeyValue, Snapshot>
 	| ObjectEntity<Init, KeyValue, Snapshot>;
 
-export type ListItemValue<KeyValue> = KeyValue extends Array<infer T>
-	? T
-	: never;
+export type ListItemValue<KeyValue> =
+	KeyValue extends Array<infer T> ? T : never;
 export type ListItemInit<Init> = Init extends Array<infer T> ? T : never;
 
 export type EntityDestructured<T extends AnyEntity<any, any, any> | null> =
 	| (T extends ListEntity<any, infer KeyValue, any>
 			? KeyValue
 			: T extends ObjectEntity<any, infer KeyValue, any>
-			? KeyValue
-			: never)
+				? KeyValue
+				: never)
 	| (T extends null ? null : never);
 
 export type EntityInit<T extends AnyEntity<any, any, any>> =
 	T extends ListEntity<infer Init, any, any>
 		? Init
 		: T extends ObjectEntity<infer Init, any, any>
-		? Init
-		: never;
+			? Init
+			: never;

package/src/files/FileManager.ts

@@ -1,17 +1,25 @@
 import { FileData } from '@verdant-web/common';
 import { Context } from '../context/context.js';
+import { Disposable } from '../internal.js';
 import { Sync } from '../sync/Sync.js';
 import { EntityFile, MARK_FAILED, UPDATE } from './EntityFile.js';
 
-export class FileManager {
+export class FileManager extends Disposable {
 	private sync;
 	private context;
 
 	private cache = new Map<string, EntityFile>();
 
 	constructor({ sync, context }: { sync: Sync; context: Context }) {
+		super();
 		this.sync = sync;
 		this.context = context;
+		this.addDispose(
+			this.context.internalEvents.subscribe(
+				'fileUploaded',
+				this.onFileUploaded,
+			),
+		);
 	}
 
 	add = async (file: FileData) => {
@@ -72,8 +80,8 @@ export class FileManager {
 
 				const result = await this.sync.getFile(file.id);
 				if (result.success) {
-					file[UPDATE](result.data);
 					await this.context.files.add(result.data);
+					file[UPDATE](result.data);
 				} else {
 					this.context.log('error', 'Failed to load file', result);
 					file[MARK_FAILED]();
@@ -84,4 +92,9 @@ export class FileManager {
 			}
 		}
 	};
+
+	private onFileUploaded = (data: FileData) => {
+		this.context.log('debug', 'Marking file as uploaded', data.id);
+		this.context.files.onUploaded(data.id);
+	};
 }

package/src/persistence/PersistenceFiles.ts

@@ -65,10 +65,10 @@ export class PersistenceFiles {
 		// and must be uploaded, even if it is cloned from an uploaded file.
 		file.remote = false;
 
-		// fire event for processing immediately
-		this.context.internalEvents.emit('fileAdded', file);
 		// store in persistence db
 		await this.db.add(file);
+		// fire event for sync to pick up and upload the file
+		this.context.internalEvents.emit('fileAdded', file);
 		this.context.globalEvents.emit('fileSaved', file);
 		this.context.log(
 			'debug',
@@ -79,10 +79,10 @@ export class PersistenceFiles {
 			file.file
 				? 'with binary file'
 				: file.url
-				? 'with url'
-				: file.localPath
-				? 'with local path'
-				: 'with no data',
+					? 'with url'
+					: file.localPath
+						? 'with local path'
+						: 'with no data',
 		);
 		return file;
 	};
@@ -206,6 +206,11 @@ export class PersistenceFiles {
 					}, 1000);
 				});
 			} else {
+				this.context.log(
+					'error',
+					`Failed to download file after ${maxRetries} retries`,
+					err,
+				);
 				throw new Error(`Failed to download file after ${maxRetries} retries`, {
 					cause: err,
 				});

package/src/persistence/idb/files/IdbPersistenceFileDb.ts

@@ -1,4 +1,5 @@
 import { FileData } from '@verdant-web/common';
+import { Context } from '../../../internal.js';
 import {
 	AbstractTransaction,
 	PersistedFileData,
@@ -6,7 +7,6 @@ import {
 } from '../../interfaces.js';
 import { IdbService } from '../IdbService.js';
 import { getAllFromObjectStores, getSizeOfObjectStore } from '../util.js';
-import { Context } from '../../../internal.js';
 
 /**
  * When stored in IDB, replace the file blob with an array buffer
@@ -24,9 +24,7 @@ export class IdbPersistenceFileDb
 	extends IdbService
 	implements PersistenceFileDb
 {
-	add = async (
-		file: FileData,
-	): Promise<void> => {
+	add = async (file: FileData): Promise<void> => {
 		let buffer = file.file ? await fileToArrayBuffer(file.file) : undefined;
 
 		await this.run(
@@ -50,6 +48,10 @@ export class IdbPersistenceFileDb
 		);
 	};
 	markUploaded = async (id: string): Promise<void> => {
+		if (this.disposed) {
+			return;
+		}
+
 		const current = await this.getFileRaw(id);
 
 		if (!current) {
@@ -185,19 +187,21 @@ export class IdbPersistenceFileDb
 		if (file.url) {
 			const response = await ctx.environment.fetch(file.url);
 			if (!response.ok) {
-				throw new Error(`Failed to download file: ${response.statusText}`);
+				throw new Error(
+					`Failed to download file ${file.url}: ${response.statusText}`,
+				);
 			}
 			return response.blob();
 		}
 		throw new Error('File is missing url, file, and localPath');
-	}
+	};
 
 	private hydrateFileData = (raw: StoredFileData): PersistedFileData => {
 		(raw as any).remote = raw.remote === 'true';
 		const buffer = raw.buffer;
 		delete raw.buffer;
 		(raw as unknown as FileData).file = buffer
-			? arrayBufferToBlob(buffer, raw.type)
+			? arrayBufferToBlob(buffer, raw.type, raw.name)
 			: undefined;
 		return raw as unknown as PersistedFileData;
 	};
@@ -206,6 +210,9 @@ export class IdbPersistenceFileDb
 		id: string,
 		{ transaction }: { transaction?: AbstractTransaction } = {},
 	): Promise<StoredFileData | undefined> => {
+		if (this.disposed) {
+			return undefined;
+		}
 		const raw = await this.run<StoredFileData>(
 			'files',
 			(store) => {
@@ -220,8 +227,14 @@ export class IdbPersistenceFileDb
 	};
 }
 
-export function arrayBufferToBlob(buffer: ArrayBuffer, type: string) {
-	return new Blob([buffer], { type });
+export function arrayBufferToBlob(
+	buffer: ArrayBuffer,
+	type: string,
+	name?: string,
+) {
+	return new File([new Blob([buffer], { type })], name ?? 'blob', {
+		type,
+	});
 }
 
 function fileToArrayBuffer(file: File | Blob): Promise<ArrayBuffer> {

package/src/persistence/persistence.ts

@@ -1,12 +1,12 @@
 import { EventSubscriber, getOidRoot, VerdantError } from '@verdant-web/common';
 import { Context, InitialContext } from '../context/context.js';
+import { ShutdownHandler } from '../context/ShutdownHandler.js';
 import { getWipNamespace } from '../utils/wip.js';
 import { ExportedData } from './interfaces.js';
+import { migrate } from './migration/migrate.js';
 import { PersistenceFiles } from './PersistenceFiles.js';
 import { PersistenceMetadata } from './PersistenceMetadata.js';
 import { PersistenceDocuments } from './PersistenceQueries.js';
-import { migrate } from './migration/migrate.js';
-import { ShutdownHandler } from '../context/ShutdownHandler.js';
 
 export async function initializePersistence(
 	ctx: InitialContext,
@@ -130,6 +130,8 @@ export async function importPersistence(
 		originalNamespace: importedNamespace,
 		// no-op entity events -- don't need to inform queries of changes.
 		entityEvents: new EventSubscriber(),
+		internalEvents: new EventSubscriber(),
+		globalEvents: new EventSubscriber(),
 		config: {
 			...ctx.config,
 			persistence: {

package/src/queries/BaseQuery.ts

@@ -57,7 +57,9 @@ export abstract class BaseQuery<T> extends Disposable {
 		this.isListQuery = Array.isArray(initial);
 		this._events = new EventSubscriber<BaseQueryEvents>(
 			(event: keyof BaseQueryEvents) => {
-				if (event === 'change') this._allUnsubscribedHandler?.(this);
+				if (event === 'change') {
+					this._allUnsubscribedHandler?.(this);
+				}
 			},
 		);
 		this.context = context;

package/src/queries/QueryCache.ts

@@ -6,6 +6,8 @@ export class QueryCache extends Disposable {
 	private _cache: Map<string, BaseQuery<any>> = new Map();
 	private _evictionTime;
 	private context;
+	/** A set of query keys to keep alive even if they unsubscribe */
+	private _holds = new Set<string>();
 
 	constructor({
 		evictionTime = 5 * 1000,
@@ -26,13 +28,22 @@ export class QueryCache extends Disposable {
 		);
 	}
 
+	get activeKeys() {
+		return Array.from(this._cache.keys());
+	}
+
 	get<T extends BaseQuery<any>>(key: string): T | null {
 		return (this._cache.get(key) as T) || null;
 	}
 
 	set<V extends BaseQuery<any>>(value: V) {
 		this._cache.set(value.key, value);
-		value[ON_ALL_UNSUBSCRIBED](this.onQueryUnsubscribed);
+		value[ON_ALL_UNSUBSCRIBED](this.enqueueQueryEviction);
+		// immediately enqueue a check to see if this query should be evicted --
+		// this basically gives code X seconds to subscribe to the query before
+		// it gets evicted.
+		this.enqueueQueryEviction(value);
+
 		return value;
 	}
 
@@ -50,13 +61,24 @@ export class QueryCache extends Disposable {
 		return this.set(create());
 	}
 
-	private onQueryUnsubscribed = (query: BaseQuery<any>) => {
+	private enqueueQueryEviction = (query: BaseQuery<any>) => {
 		setTimeout(() => {
 			if (query.subscribed) return;
+
+			if (this._holds.has(query.key)) {
+				this.context.log(
+					'debug',
+					'QueryCache: keepAlive hold on query preserves after unsubscribe',
+					query.key,
+				);
+				return;
+			}
+
 			// double check before evicting... possible the cache
 			// got a different version of this query.
 			if (this._cache.get(query.key) === query) {
 				this._cache.delete(query.key);
+				this.context.log('debug', 'QueryCache: evicted query', query.key);
 			}
 		}, this._evictionTime);
 	};
@@ -81,4 +103,36 @@ export class QueryCache extends Disposable {
 		);
 		this._cache.forEach((q) => q.execute());
 	};
+
+	keepAlive(key: string) {
+		this._holds.add(key);
+		this.context.log('debug', 'QueryCache: keepAlive', key);
+	}
+
+	dropKeepAlive(key: string) {
+		this._holds.delete(key);
+		const cached = this.get(key);
+		if (!cached) return;
+		if (!cached.subscribed) {
+			this.context.log(
+				'debug',
+				'QueryCache: dropKeepAlive on unsubscribed query; queuing eviction',
+				key,
+			);
+			this.enqueueQueryEviction(cached);
+		}
+	}
+
+	get keepAlives() {
+		return this._holds;
+	}
 }
+
+export type PublicQueryCacheAPI = Pick<
+	QueryCache,
+	| 'keepAlive'
+	| 'dropKeepAlive'
+	| 'keepAlives'
+	| 'forceRefreshAll'
+	| 'activeKeys'
+>;

package/src/sync/FileSync.ts

@@ -1,7 +1,7 @@
 import { FileData } from '@verdant-web/common';
-import { ServerSyncEndpointProvider } from './ServerSyncEndpointProvider.js';
 import { Context } from '../context/context.js';
 import { Disposable } from '../utils/Disposable.js';
+import { ServerSyncEndpointProvider } from './ServerSyncEndpointProvider.js';
 
 export interface FileUploadResult {
 	success: boolean;
@@ -42,7 +42,6 @@ export class FileSync extends Disposable {
 		this.ctx.log('debug', 'Uploading file', data.id, data.name);
 		try {
 			await this.uploadFile(data);
-			this.ctx.internalEvents.emit(`fileUploaded:${data.id}`, data);
 		} catch (e) {
 			this.ctx.log('error', 'File upload failed', e);
 		}
@@ -83,6 +82,8 @@ export class FileSync extends Disposable {
 			);
 
 			if (response.ok) {
+				this.ctx.internalEvents.emit(`fileUploaded:${data.id}`, data);
+				this.ctx.internalEvents.emit('fileUploaded', data);
 				this.ctx.log('info', 'File upload successful');
 				return {
 					success: true,