@aws-amplify/datastore 3.4.8 → 3.4.9-in-app-messaging.48

package/src/datastore/datastore.ts

@@ -44,6 +44,7 @@ import {
 	SchemaNamespace,
 	SchemaNonModel,
 	SubscriptionMessage,
+	DataStoreSnapshot,
 	SyncConflict,
 	SyncError,
 	TypeConstructorMap,
@@ -52,6 +53,7 @@ import {
 	AuthModeStrategyType,
 	isNonModelFieldType,
 	isModelFieldType,
+	ObserveQueryOptions,
 } from '../types';
 import {
 	DATASTORE,
@@ -65,6 +67,8 @@ import {
 	USER,
 	isNullOrUndefined,
 	registerNonModelClass,
+	sortCompareFunction,
+	DeferredCallbackResolver,
 } from '../util';
 
 setAutoFreeze(true);
@@ -115,7 +119,7 @@ const isValidModelConstructor = <T extends PersistentModel>(
 	return isModelConstructor(obj) && modelNamespaceMap.has(obj);
 };
 
-const namespaceResolver: NamespaceResolver = modelConstructor =>
+const namespaceResolver: NamespaceResolver = (modelConstructor) =>
 	modelNamespaceMap.get(modelConstructor);
 
 // exporting syncClasses for testing outbox.test.ts
@@ -160,7 +164,7 @@ const initSchema = (userSchema: Schema) => {
 		version: userSchema.version,
 	};
 
-	Object.keys(schema.namespaces).forEach(namespace => {
+	Object.keys(schema.namespaces).forEach((namespace) => {
 		const [relations, keys] = establishRelationAndKeys(
 			schema.namespaces[namespace]
 		);
@@ -170,17 +174,17 @@ const initSchema = (userSchema: Schema) => {
 
 		const modelAssociations = new Map<string, string[]>();
 
-		Object.values(schema.namespaces[namespace].models).forEach(model => {
+		Object.values(schema.namespaces[namespace].models).forEach((model) => {
 			const connectedModels: string[] = [];
 
 			Object.values(model.fields)
 				.filter(
-					field =>
+					(field) =>
 						field.association &&
 						field.association.connectionType === 'BELONGS_TO' &&
 						(<ModelFieldType>field.type).model !== model.name
 				)
-				.forEach(field =>
+				.forEach((field) =>
 					connectedModels.push((<ModelFieldType>field.type).model)
 				);
 
@@ -204,12 +208,12 @@ const initSchema = (userSchema: Schema) => {
 			for (const modelName of Array.from(modelAssociations.keys())) {
 				const parents = modelAssociations.get(modelName);
 
-				if (parents.every(x => result.has(x))) {
+				if (parents.every((x) => result.has(x))) {
 					result.set(modelName, parents);
 				}
 			}
 
-			Array.from(result.keys()).forEach(x => modelAssociations.delete(x));
+			Array.from(result.keys()).forEach((x) => modelAssociations.delete(x));
 		}
 
 		schema.namespaces[namespace].modelTopologicalOrdering = result;
@@ -218,9 +222,9 @@ const initSchema = (userSchema: Schema) => {
 	return userClasses;
 };
 
-const createTypeClasses: (
-	namespace: SchemaNamespace
-) => TypeConstructorMap = namespace => {
+const createTypeClasses: (namespace: SchemaNamespace) => TypeConstructorMap = (
+	namespace
+) => {
 	const classes: TypeConstructorMap = {};
 
 	Object.entries(namespace.models).forEach(([modelName, modelDefinition]) => {
@@ -254,108 +258,101 @@ function modelInstanceCreator<T extends PersistentModel = PersistentModel>(
 	return <T>new modelConstructor(init);
 }
 
-const validateModelFields = (modelDefinition: SchemaModel | SchemaNonModel) => (
-	k: string,
-	v: any
-) => {
-	const fieldDefinition = modelDefinition.fields[k];
+const validateModelFields =
+	(modelDefinition: SchemaModel | SchemaNonModel) => (k: string, v: any) => {
+		const fieldDefinition = modelDefinition.fields[k];
 
-	if (fieldDefinition !== undefined) {
-		const {
-			type,
-			isRequired,
-			isArrayNullable,
-			name,
-			isArray,
-		} = fieldDefinition;
+		if (fieldDefinition !== undefined) {
+			const { type, isRequired, isArrayNullable, name, isArray } =
+				fieldDefinition;
 
-		if (
-			((!isArray && isRequired) || (isArray && !isArrayNullable)) &&
-			(v === null || v === undefined)
-		) {
-			throw new Error(`Field ${name} is required`);
-		}
+			if (
+				((!isArray && isRequired) || (isArray && !isArrayNullable)) &&
+				(v === null || v === undefined)
+			) {
+				throw new Error(`Field ${name} is required`);
+			}
 
-		if (isGraphQLScalarType(type)) {
-			const jsType = GraphQLScalarType.getJSType(type);
-			const validateScalar = GraphQLScalarType.getValidationFunction(type);
+			if (isGraphQLScalarType(type)) {
+				const jsType = GraphQLScalarType.getJSType(type);
+				const validateScalar = GraphQLScalarType.getValidationFunction(type);
 
-			if (type === 'AWSJSON') {
-				if (typeof v === jsType) {
-					return;
-				}
-				if (typeof v === 'string') {
-					try {
-						JSON.parse(v);
+				if (type === 'AWSJSON') {
+					if (typeof v === jsType) {
 						return;
-					} catch (error) {
-						throw new Error(`Field ${name} is an invalid JSON object. ${v}`);
+					}
+					if (typeof v === 'string') {
+						try {
+							JSON.parse(v);
+							return;
+						} catch (error) {
+							throw new Error(`Field ${name} is an invalid JSON object. ${v}`);
+						}
 					}
 				}
-			}
 
-			if (isArray) {
-				let errorTypeText: string = jsType;
-				if (!isRequired) {
-					errorTypeText = `${jsType} | null | undefined`;
-				}
+				if (isArray) {
+					let errorTypeText: string = jsType;
+					if (!isRequired) {
+						errorTypeText = `${jsType} | null | undefined`;
+					}
+
+					if (!Array.isArray(v) && !isArrayNullable) {
+						throw new Error(
+							`Field ${name} should be of type [${errorTypeText}], ${typeof v} received. ${v}`
+						);
+					}
+
+					if (
+						!isNullOrUndefined(v) &&
+						(<[]>v).some((e) =>
+							isNullOrUndefined(e) ? isRequired : typeof e !== jsType
+						)
+					) {
+						const elemTypes = (<[]>v)
+							.map((e) => (e === null ? 'null' : typeof e))
+							.join(',');
 
-				if (!Array.isArray(v) && !isArrayNullable) {
+						throw new Error(
+							`All elements in the ${name} array should be of type ${errorTypeText}, [${elemTypes}] received. ${v}`
+						);
+					}
+
+					if (validateScalar && !isNullOrUndefined(v)) {
+						const validationStatus = (<[]>v).map((e) => {
+							if (!isNullOrUndefined(e)) {
+								return validateScalar(e);
+							} else if (isNullOrUndefined(e) && !isRequired) {
+								return true;
+							} else {
+								return false;
+							}
+						});
+
+						if (!validationStatus.every((s) => s)) {
+							throw new Error(
+								`All elements in the ${name} array should be of type ${type}, validation failed for one or more elements. ${v}`
+							);
+						}
+					}
+				} else if (!isRequired && v === undefined) {
+					return;
+				} else if (typeof v !== jsType && v !== null) {
 					throw new Error(
-						`Field ${name} should be of type [${errorTypeText}], ${typeof v} received. ${v}`
+						`Field ${name} should be of type ${jsType}, ${typeof v} received. ${v}`
 					);
-				}
-
-				if (
+				} else if (
 					!isNullOrUndefined(v) &&
-					(<[]>v).some(e =>
-						isNullOrUndefined(e) ? isRequired : typeof e !== jsType
-					)
+					validateScalar &&
+					!validateScalar(v)
 				) {
-					const elemTypes = (<[]>v)
-						.map(e => (e === null ? 'null' : typeof e))
-						.join(',');
-
 					throw new Error(
-						`All elements in the ${name} array should be of type ${errorTypeText}, [${elemTypes}] received. ${v}`
+						`Field ${name} should be of type ${type}, validation failed. ${v}`
 					);
 				}
-
-				if (validateScalar && !isNullOrUndefined(v)) {
-					const validationStatus = (<[]>v).map(e => {
-						if (!isNullOrUndefined(e)) {
-							return validateScalar(e);
-						} else if (isNullOrUndefined(e) && !isRequired) {
-							return true;
-						} else {
-							return false;
-						}
-					});
-
-					if (!validationStatus.every(s => s)) {
-						throw new Error(
-							`All elements in the ${name} array should be of type ${type}, validation failed for one or more elements. ${v}`
-						);
-					}
-				}
-			} else if (!isRequired && v === undefined) {
-				return;
-			} else if (typeof v !== jsType && v !== null) {
-				throw new Error(
-					`Field ${name} should be of type ${jsType}, ${typeof v} received. ${v}`
-				);
-			} else if (
-				!isNullOrUndefined(v) &&
-				validateScalar &&
-				!validateScalar(v)
-			) {
-				throw new Error(
-					`Field ${name} should be of type ${type}, validation failed. ${v}`
-				);
 			}
 		}
-	}
-};
+	};
 
 const castInstanceType = (
 	modelDefinition: SchemaModel | SchemaNonModel,
@@ -410,11 +407,10 @@ const createModelClass = <T extends PersistentModel>(
 				(draft: Draft<T & ModelInstanceMetadata>) => {
 					initializeInstance(init, modelDefinition, draft);
 
-					const modelInstanceMetadata: ModelInstanceMetadata = instancesMetadata.has(
-						init
-					)
-						? <ModelInstanceMetadata>(<unknown>init)
-						: <ModelInstanceMetadata>{};
+					const modelInstanceMetadata: ModelInstanceMetadata =
+						instancesMetadata.has(init)
+							? <ModelInstanceMetadata>(<unknown>init)
+							: <ModelInstanceMetadata>{};
 					const {
 						id: _id,
 						_version,
@@ -459,7 +455,7 @@ const createModelClass = <T extends PersistentModel>(
 			let patches;
 			const model = produce(
 				source,
-				draft => {
+				(draft) => {
 					fn(<MutableModel<T>>(draft as unknown));
 					draft.id = source.id;
 					const modelValidator = validateModelFields(modelDefinition);
@@ -469,7 +465,7 @@ const createModelClass = <T extends PersistentModel>(
 						modelValidator(k, parsedValue);
 					});
 				},
-				p => (patches = p)
+				(p) => (patches = p)
 			);
 
 			if (patches.length) {
@@ -484,7 +480,7 @@ const createModelClass = <T extends PersistentModel>(
 		// to gain access to `modelInstanceCreator` and `clazz` for persisting IDs from server to client.
 		static fromJSON(json: T | T[]) {
 			if (Array.isArray(json)) {
-				return json.map(init => this.fromJSON(init));
+				return json.map((init) => this.fromJSON(init));
 			}
 
 			const instance = modelInstanceCreator(clazz, json);
@@ -512,7 +508,7 @@ const checkReadOnlyPropertyOnCreate = <T extends PersistentModel>(
 	const modelKeys = Object.keys(draft);
 	const { fields } = modelDefinition;
 
-	modelKeys.forEach(key => {
+	modelKeys.forEach((key) => {
 		if (fields[key] && fields[key].isReadOnly) {
 			throw new Error(`${key} is read-only.`);
 		}
@@ -523,7 +519,7 @@ const checkReadOnlyPropertyOnUpdate = (
 	patches: Patch[],
 	modelDefinition: SchemaModel
 ) => {
-	const patchArray = patches.map(p => [p.path[0], p.value]);
+	const patchArray = patches.map((p) => [p.path[0], p.value]);
 	const { fields } = modelDefinition;
 
 	patchArray.forEach(([key, val]) => {
@@ -610,16 +606,15 @@ async function checkSchemaVersion(
 	storage: Storage,
 	version: string
 ): Promise<void> {
-	const Setting = dataStoreClasses.Setting as PersistentModelConstructor<
-		Setting
-	>;
+	const Setting =
+		dataStoreClasses.Setting as PersistentModelConstructor<Setting>;
 
 	const modelDefinition = schema.namespaces[DATASTORE].models.Setting;
 
-	await storage.runExclusive(async s => {
+	await storage.runExclusive(async (s) => {
 		const [schemaVersionSetting] = await s.query(
 			Setting,
-			ModelPredicateCreator.createFromExisting(modelDefinition, c =>
+			ModelPredicateCreator.createFromExisting(modelDefinition, (c) =>
 				// @ts-ignore Argument of type '"eq"' is not assignable to parameter of type 'never'.
 				c.key('eq', SETTING_SCHEMA_VERSION)
 			),
@@ -700,10 +695,8 @@ class DataStore {
 	private sync: SyncEngine;
 	private syncPageSize: number;
 	private syncExpressions: SyncExpression[];
-	private syncPredicates: WeakMap<
-		SchemaModel,
-		ModelPredicate<any>
-	> = new WeakMap<SchemaModel, ModelPredicate<any>>();
+	private syncPredicates: WeakMap<SchemaModel, ModelPredicate<any>> =
+		new WeakMap<SchemaModel, ModelPredicate<any>>();
 	private sessionId: string;
 	private storageAdapter: Adapter;
 
@@ -781,7 +774,7 @@ class DataStore {
 							data,
 						});
 					},
-					error: err => {
+					error: (err) => {
 						logger.warn('Sync error', err);
 						this.initReject();
 					},
@@ -908,7 +901,7 @@ class DataStore {
 			condition
 		);
 
-		const [savedModel] = await this.storage.runExclusive(async s => {
+		const [savedModel] = await this.storage.runExclusive(async (s) => {
 			await s.save(model, producedCondition, undefined, patchesTuple);
 
 			return s.query(
@@ -1126,7 +1119,7 @@ class DataStore {
 				);
 		}
 
-		return new Observable<SubscriptionMessage<T>>(observer => {
+		return new Observable<SubscriptionMessage<T>>((observer) => {
 			let handle: ZenObservable.Subscription;
 
 			(async () => {
@@ -1146,6 +1139,150 @@ class DataStore {
 		});
 	};
 
+	observeQuery: {
+		<T extends PersistentModel>(
+			modelConstructor: PersistentModelConstructor<T>,
+			criteria?: ProducerModelPredicate<T> | typeof PredicateAll,
+			paginationProducer?: ObserveQueryOptions<T>
+		): Observable<DataStoreSnapshot<T>>;
+	} = <T extends PersistentModel = PersistentModel>(
+		model: PersistentModelConstructor<T>,
+		criteria?: ProducerModelPredicate<T> | typeof PredicateAll,
+		options?: ObserveQueryOptions<T>
+	): Observable<DataStoreSnapshot<T>> => {
+		return new Observable<DataStoreSnapshot<T>>((observer) => {
+			const items = new Map<string, T>();
+			const itemsChanged = new Map<string, T>();
+			let deletedItemIds: string[] = [];
+			let handle: ZenObservable.Subscription;
+
+			const generateAndEmitSnapshot = (): void => {
+				const snapshot = generateSnapshot();
+				emitSnapshot(snapshot);
+			};
+
+			// a mechanism to return data after X amount of seconds OR after the
+			// "limit" (itemsChanged >= this.syncPageSize) has been reached, whichever comes first
+			const limitTimerRace = new DeferredCallbackResolver({
+				callback: generateAndEmitSnapshot,
+				errorHandler: observer.error,
+				maxInterval: 2000,
+			});
+
+			const { sort } = options || {};
+			const sortOptions = sort ? { sort } : undefined;
+
+			(async () => {
+				try {
+					// first, query and return any locally-available records
+					(await this.query(model, criteria, sortOptions)).forEach((item) =>
+						items.set(item.id, item)
+					);
+
+					// observe the model and send a stream of updates (debounced)
+					handle = this.observe(
+						model,
+						// @ts-ignore TODO: fix this TSlint error
+						criteria
+					).subscribe(({ element, model, opType }) => {
+						// Flag items which have been recently deleted
+						// NOTE: Merging of separate operations to the same model instance is handled upstream
+						// in the `mergePage` method within src/sync/merger.ts. The final state of a model instance
+						// depends on the LATEST record (for a given id).
+						if (opType === 'DELETE') {
+							deletedItemIds.push(element.id);
+						} else {
+							itemsChanged.set(element.id, element);
+						}
+
+						const isSynced = this.sync.getModelSyncedStatus(model);
+
+						const limit =
+							itemsChanged.size - deletedItemIds.length >= this.syncPageSize;
+
+						if (limit || isSynced) {
+							limitTimerRace.resolve();
+						}
+
+						// kicks off every subsequent race as results sync down
+						limitTimerRace.start();
+					});
+
+					// returns a set of initial/locally-available results
+					generateAndEmitSnapshot();
+				} catch (err) {
+					observer.error(err);
+				}
+			})();
+
+			// TODO: abstract this function into a util file to be able to write better unit tests
+			const generateSnapshot = (): DataStoreSnapshot<T> => {
+				const isSynced = this.sync.getModelSyncedStatus(model);
+				const itemsArray = [
+					...Array.from(items.values()),
+					...Array.from(itemsChanged.values()),
+				];
+
+				if (options?.sort) {
+					sortItems(itemsArray);
+				}
+
+				items.clear();
+				itemsArray.forEach((item) => items.set(item.id, item));
+
+				// remove deleted items from the final result set
+				deletedItemIds.forEach((id) => items.delete(id));
+
+				return {
+					items: Array.from(items.values()),
+					isSynced,
+				};
+			};
+
+			const emitSnapshot = (snapshot: DataStoreSnapshot<T>): void => {
+				// send the generated snapshot to the primary subscription
+				observer.next(snapshot);
+
+				// reset the changed items sets
+				itemsChanged.clear();
+				deletedItemIds = [];
+			};
+
+			const sortItems = (itemsToSort: T[]): void => {
+				const modelDefinition = getModelDefinition(model);
+				const pagination = this.processPagination(modelDefinition, options);
+
+				const sortPredicates = ModelSortPredicateCreator.getPredicates(
+					pagination.sort
+				);
+
+				if (sortPredicates.length) {
+					const compareFn = sortCompareFunction(sortPredicates);
+					itemsToSort.sort(compareFn);
+				}
+			};
+
+			// send one last snapshot when the model is fully synced
+			const hubCallback = ({ payload }): void => {
+				const { event, data } = payload;
+				if (
+					event === ControlMessage.SYNC_ENGINE_MODEL_SYNCED &&
+					data?.model?.name === model.name
+				) {
+					generateAndEmitSnapshot();
+					Hub.remove('api', hubCallback);
+				}
+			};
+			Hub.listen('datastore', hubCallback);
+
+			return () => {
+				if (handle) {
+					handle.unsubscribe();
+				}
+			};
+		});
+	};
+
 	configure = (config: DataStoreConfig = {}) => {
 		const {
 			DataStore: configDataStore,
@@ -1161,7 +1298,10 @@ class DataStore {
 			...configFromAmplify
 		} = config;
 
-		this.amplifyConfig = { ...configFromAmplify, ...this.amplifyConfig };
+		this.amplifyConfig = {
+			...configFromAmplify,
+			...this.amplifyConfig,
+		};
 
 		this.conflictHandler = this.setConflictHandler(config);
 		this.errorHandler = this.setErrorHandler(config);
@@ -1194,13 +1334,19 @@ class DataStore {
 
 		this.maxRecordsToSync =
 			(configDataStore && configDataStore.maxRecordsToSync) ||
-			this.maxRecordsToSync ||
-			configMaxRecordsToSync;
+			configMaxRecordsToSync ||
+			10000;
+
+		// store on config object, so that Sync, Subscription, and Mutation processors can have access
+		this.amplifyConfig.maxRecordsToSync = this.maxRecordsToSync;
 
 		this.syncPageSize =
 			(configDataStore && configDataStore.syncPageSize) ||
-			this.syncPageSize ||
-			configSyncPageSize;
+			configSyncPageSize ||
+			1000;
+
+		// store on config object, so that Sync, Subscription, and Mutation processors can have access
+		this.amplifyConfig.syncPageSize = this.syncPageSize;
 
 		this.fullSyncInterval =
 			(configDataStore && configDataStore.fullSyncInterval) ||

package/src/sync/index.ts

@@ -22,7 +22,7 @@ import {
 	ModelPredicate,
 	AuthModeStrategy,
 } from '../types';
-import { exhaustiveCheck, getNow, SYNC } from '../util';
+import { exhaustiveCheck, getNow, SYNC, USER } from '../util';
 import DataStoreConnectivity from './datastoreConnectivity';
 import { ModelMerger } from './merger';
 import { MutationEventOutbox } from './outbox';
@@ -95,6 +95,16 @@ export class SyncEngine {
 	private readonly modelMerger: ModelMerger;
 	private readonly outbox: MutationEventOutbox;
 	private readonly datastoreConnectivity: DataStoreConnectivity;
+	private readonly modelSyncedStatus: WeakMap<
+		PersistentModelConstructor<any>,
+		boolean
+	> = new WeakMap();
+
+	public getModelSyncedStatus(
+		modelConstructor: PersistentModelConstructor<any>
+	): boolean {
+		return this.modelSyncedStatus.get(modelConstructor);
+	}
 
 	constructor(
 		private readonly schema: InternalSchema,
@@ -126,8 +136,6 @@ export class SyncEngine {
 
 		this.syncQueriesProcessor = new SyncProcessor(
 			this.schema,
-			this.maxRecordsToSync,
-			this.syncPageSize,
 			this.syncPredicates,
 			this.amplifyConfig,
 			this.authModeStrategy
@@ -615,6 +623,8 @@ export class SyncEngine {
 
 										const counts = count.get(modelConstructor);
 
+										this.modelSyncedStatus.set(modelConstructor, true);
+
 										observer.next({
 											type: ControlMessage.SYNC_ENGINE_MODEL_SYNCED,
 											data: {
@@ -708,6 +718,12 @@ export class SyncEngine {
 				.filter(({ syncable }) => syncable)
 				.forEach(model => {
 					models.push([namespace.name, model]);
+					if (namespace.name === USER) {
+						const modelConstructor = this.userModelClasses[
+							model.name
+						] as PersistentModelConstructor<any>;
+						this.modelSyncedStatus.set(modelConstructor, false);
+					}
 				});
 		});
 

package/src/sync/processors/subscription.ts

@@ -533,7 +533,7 @@ class SubscriptionProcessor {
 			})();
 
 			return () => {
-				Object.keys(subscriptions).map(modelName => {
+				Object.keys(subscriptions).forEach(modelName => {
 					subscriptions[modelName][
 						TransformerMutationType.CREATE
 					].forEach(subscription => subscription.unsubscribe());

package/src/sync/processors/sync.ts

@@ -25,9 +25,6 @@ import {
 } from '@aws-amplify/core';
 import { ModelPredicateCreator } from '../../predicates';
 
-const DEFAULT_PAGINATION_LIMIT = 1000;
-const DEFAULT_MAX_RECORDS_TO_SYNC = 10000;
-
 const opResultDefaults = {
 	items: [],
 	nextToken: null,
@@ -41,8 +38,6 @@ class SyncProcessor {
 
 	constructor(
 		private readonly schema: InternalSchema,
-		private readonly maxRecordsToSync: number = DEFAULT_MAX_RECORDS_TO_SYNC,
-		private readonly syncPageSize: number = DEFAULT_PAGINATION_LIMIT,
 		private readonly syncPredicates: WeakMap<SchemaModel, ModelPredicate<any>>,
 		private readonly amplifyConfig: Record<string, any> = {},
 		private readonly authModeStrategy: AuthModeStrategy
@@ -289,19 +284,8 @@ class SyncProcessor {
 		typesLastSync: Map<SchemaModel, [string, number]>
 	): Observable<SyncModelPage> {
 		let processing = true;
-
-		const maxRecordsToSync =
-			this.maxRecordsToSync !== undefined
-				? this.maxRecordsToSync
-				: DEFAULT_MAX_RECORDS_TO_SYNC;
-
-		const syncPageSize =
-			this.syncPageSize !== undefined
-				? this.syncPageSize
-				: DEFAULT_PAGINATION_LIMIT;
-
+		const { maxRecordsToSync, syncPageSize } = this.amplifyConfig;
 		const parentPromises = new Map<string, Promise<void>>();
-
 		const observable = new Observable<SyncModelPage>(observer => {
 			const sortedTypesLastSyncs = Object.values(this.schema.namespaces).reduce(
 				(map, namespace) => {