@aws-amplify/datastore 4.0.12 → 4.0.13-push-notification-dryrun.43

package/src/sync/index.ts

@@ -2,8 +2,13 @@ import {
 	browserOrNode,
 	ConsoleLogger as Logger,
 	BackgroundProcessManager,
+	Hub,
 } from '@aws-amplify/core';
-import { CONTROL_MSG as PUBSUB_CONTROL_MSG } from '@aws-amplify/pubsub';
+import {
+	CONTROL_MSG as PUBSUB_CONTROL_MSG,
+	CONNECTION_STATE_CHANGE as PUBSUB_CONNECTION_STATE_CHANGE,
+	ConnectionState,
+} from '@aws-amplify/pubsub';
 import Observable, { ZenObservable } from 'zen-observable-ts';
 import { ModelInstanceCreator } from '../datastore/datastore';
 import { ModelPredicateCreator } from '../predicates';
@@ -116,6 +121,13 @@ export class SyncEngine {
 		PersistentModelConstructor<any>,
 		boolean
 	> = new WeakMap();
+	private unsleepSyncQueriesObservable: (() => void) | null;
+	private waitForSleepState: Promise<void>;
+	private syncQueriesObservableStartSleeping: (
+		value?: void | PromiseLike<void>
+	) => void;
+	private stopDisruptionListener: () => void;
+	private connectionDisrupted = false;
 
 	private runningProcesses: BackgroundProcessManager;
 
@@ -134,13 +146,19 @@ export class SyncEngine {
 		private readonly modelInstanceCreator: ModelInstanceCreator,
 		conflictHandler: ConflictHandler,
 		errorHandler: ErrorHandler,
-		private readonly syncPredicates: WeakMap<SchemaModel, ModelPredicate<any>>,
+		private readonly syncPredicates: WeakMap<
+			SchemaModel,
+			ModelPredicate<any> | null
+		>,
 		private readonly amplifyConfig: Record<string, any> = {},
 		private readonly authModeStrategy: AuthModeStrategy,
 		private readonly amplifyContext: AmplifyContext,
 		private readonly connectivityMonitor?: DataStoreConnectivity
 	) {
 		this.runningProcesses = new BackgroundProcessManager();
+		this.waitForSleepState = new Promise(resolve => {
+			this.syncQueriesObservableStartSleeping = resolve;
+		});
 
 		const MutationEvent = this.modelClasses[
 			'MutationEvent'
@@ -234,6 +252,8 @@ export class SyncEngine {
 											'Realtime disabled when in a server-side environment'
 										);
 									} else {
+										this.stopDisruptionListener =
+											this.startDisruptionListener();
 										//#region GraphQL Subscriptions
 										[ctlSubsObservable, dataSubsObservable] =
 											this.subscriptionsProcessor.start();
@@ -448,6 +468,7 @@ export class SyncEngine {
 
 								await startPromise;
 
+								// Set by the this.datastoreConnectivity.status().subscribe() loop
 								if (this.online) {
 									this.mutationsProcessor.resume();
 								}
@@ -540,12 +561,12 @@ export class SyncEngine {
 						);
 						const paginatingModels = new Set(modelLastSync.keys());
 
-						let newestFullSyncStartedAt: number;
-						let theInterval: number;
+						let lastFullSyncStartedAt: number;
+						let syncInterval: number;
 
 						let start: number;
-						let duration: number;
-						let newestStartedAt: number;
+						let syncDuration: number;
+						let lastStartedAt: number;
 						await new Promise((resolve, reject) => {
 							if (!this.runningProcesses.isOpen) resolve();
 							onTerminate.then(() => resolve());
@@ -572,10 +593,10 @@ export class SyncEngine {
 											});
 
 											start = getNow();
-											newestStartedAt =
-												newestStartedAt === undefined
+											lastStartedAt =
+												lastStartedAt === undefined
 													? startedAt
-													: Math.max(newestStartedAt, startedAt);
+													: Math.max(lastStartedAt, startedAt);
 										}
 
 										/**
@@ -655,13 +676,13 @@ export class SyncEngine {
 
 											const { lastFullSync, fullSyncInterval } = modelMetadata;
 
-											theInterval = fullSyncInterval;
+											syncInterval = fullSyncInterval;
 
-											newestFullSyncStartedAt =
-												newestFullSyncStartedAt === undefined
+											lastFullSyncStartedAt =
+												lastFullSyncStartedAt === undefined
 													? lastFullSync!
 													: Math.max(
-															newestFullSyncStartedAt,
+															lastFullSyncStartedAt,
 															isFullSync ? startedAt : lastFullSync!
 													  );
 
@@ -699,7 +720,7 @@ export class SyncEngine {
 											paginatingModels.delete(modelDefinition);
 
 											if (paginatingModels.size === 0) {
-												duration = getNow() - start;
+												syncDuration = getNow() - start;
 												resolve();
 												observer.next({
 													type: ControlMessage.SYNC_ENGINE_SYNC_QUERIES_READY,
@@ -721,10 +742,19 @@ export class SyncEngine {
 							});
 						});
 
-						const msNextFullSync =
-							newestFullSyncStartedAt! +
-							theInterval! -
-							(newestStartedAt! + duration!);
+						// null is cast to 0 resulting in unexpected behavior.
+						// undefined in arithmetic operations results in NaN also resulting in unexpected behavior.
+						// If lastFullSyncStartedAt is null this is the first sync.
+						// Assume lastStartedAt is is also newest full sync.
+						let msNextFullSync;
+						if (!lastFullSyncStartedAt!) {
+							msNextFullSync = syncInterval! - syncDuration!;
+						} else {
+							msNextFullSync =
+								lastFullSyncStartedAt! +
+								syncInterval! -
+								(lastStartedAt! + syncDuration!);
+						}
 
 						logger.debug(
 							`Next fullSync in ${msNextFullSync / 1000} seconds. (${new Date(
@@ -756,11 +786,19 @@ export class SyncEngine {
 
 							onTerminate.then(() => {
 								terminated = true;
+								this.syncQueriesObservableStartSleeping();
 								unsleep();
 							});
 
+							this.unsleepSyncQueriesObservable = unsleep;
+							this.syncQueriesObservableStartSleeping();
 							return sleep;
 						}, 'syncQueriesObservable sleep');
+
+						this.unsleepSyncQueriesObservable = null;
+						this.waitForSleepState = new Promise(resolve => {
+							this.syncQueriesObservableStartSleeping = resolve;
+						});
 					}
 				}, 'syncQueriesObservable main');
 		});
@@ -795,6 +833,11 @@ export class SyncEngine {
 		 */
 		this.unsubscribeConnectivity();
 
+		/**
+		 * Stop listening for websocket connection disruption
+		 */
+		this.stopDisruptionListener && this.stopDisruptionListener();
+
 		/**
 		 * aggressively shut down any lingering background processes.
 		 * some of this might be semi-redundant with unsubscribing. however,
@@ -906,9 +949,9 @@ export class SyncEngine {
 		const ModelMetadata = this.modelClasses
 			.ModelMetadata as PersistentModelConstructor<ModelMetadata>;
 
-		const predicate = ModelPredicateCreator.createFromExisting<ModelMetadata>(
+		const predicate = ModelPredicateCreator.createFromAST<ModelMetadata>(
 			this.schema.namespaces[SYNC].models[ModelMetadata.name],
-			c => c.namespace('eq', namespace).model('eq', model)
+			{ and: [{ namespace: { eq: namespace } }, { model: { eq: model } }] }
 		);
 
 		const [modelMetadata] = await this.storage.query(ModelMetadata, predicate, {
@@ -1040,4 +1083,49 @@ export class SyncEngine {
 		};
 		return namespace;
 	}
+
+	/**
+	 * listen for websocket connection disruption
+	 *
+	 * May indicate there was a period of time where messages
+	 * from AppSync were missed. A sync needs to be triggered to
+	 * retrieve the missed data.
+	 */
+	private startDisruptionListener() {
+		return Hub.listen('api', (data: any) => {
+			if (
+				data.source === 'PubSub' &&
+				data.payload.event === PUBSUB_CONNECTION_STATE_CHANGE
+			) {
+				const connectionState = data.payload.data
+					.connectionState as ConnectionState;
+
+				switch (connectionState) {
+					// Do not need to listen for ConnectionDisruptedPendingNetwork
+					// Normal network reconnection logic will handle the sync
+					case ConnectionState.ConnectionDisrupted:
+						this.connectionDisrupted = true;
+						break;
+
+					case ConnectionState.Connected:
+						if (this.connectionDisrupted) {
+							this.scheduleSync();
+						}
+						this.connectionDisrupted = false;
+						break;
+				}
+			}
+		});
+	}
+
+	/*
+	 * Schedule a sync to start when syncQueriesObservable enters sleep state
+	 * Start sync immediately if syncQueriesObservable is already in sleep state
+	 */
+	private scheduleSync(): Promise<void> {
+		return this.waitForSleepState.then(() => {
+			// unsleepSyncQueriesObservable will be set if waitForSleepState has resolved
+			this.unsleepSyncQueriesObservable!();
+		});
+	}
 }

package/src/sync/outbox.ts

@@ -38,12 +38,14 @@ class MutationEventOutbox {
 
 			// `id` is the key for the record in the mutationEvent;
 			// `modelId` is the key for the actual record that was mutated
-			const predicate = ModelPredicateCreator.createFromExisting<MutationEvent>(
+			const predicate = ModelPredicateCreator.createFromAST<MutationEvent>(
 				mutationEventModelDefinition,
-				c =>
-					c
-						.modelId('eq', mutationEvent.modelId)
-						.id('ne', this.inProgressMutationEventId)
+				{
+					and: [
+						{ modelId: { eq: mutationEvent.modelId } },
+						{ id: { ne: this.inProgressMutationEventId } },
+					],
+				}
 			);
 
 			// Check if there are any other records with same id
@@ -138,10 +140,9 @@ class MutationEventOutbox {
 
 		const mutationEvents = await storage.query(
 			this.MutationEvent,
-			ModelPredicateCreator.createFromExisting(
-				mutationEventModelDefinition,
-				c => c.modelId('eq', modelId)
-			)
+			ModelPredicateCreator.createFromAST(mutationEventModelDefinition, {
+				and: { modelId: { eq: modelId } },
+			})
 		);
 
 		return mutationEvents;
@@ -201,9 +202,14 @@ class MutationEventOutbox {
 
 		const recordId = getIdentifierValue(userModelDefinition, record);
 
-		const predicate = ModelPredicateCreator.createFromExisting<MutationEvent>(
+		const predicate = ModelPredicateCreator.createFromAST<MutationEvent>(
 			mutationEventModelDefinition,
-			c => c.modelId('eq', recordId).id('ne', this.inProgressMutationEventId)
+			{
+				and: [
+					{ modelId: { eq: recordId } },
+					{ id: { ne: this.inProgressMutationEventId } },
+				],
+			}
 		);
 
 		const outdatedMutations = await storage.query(

package/src/sync/processors/mutation.ts

@@ -56,7 +56,15 @@ type MutationProcessorEvent = {
 };
 
 class MutationProcessor {
-	private observer!: ZenObservable.Observer<MutationProcessorEvent>;
+	/**
+	 * The observer that receives messages when mutations are successfully completed
+	 * against cloud storage.
+	 *
+	 * A value of `undefined` signals that the sync has either been stopped or has not
+	 * yet started. In this case, `isReady()` will be `false` and `resume()` will exit
+	 * early.
+	 */
+	private observer?: ZenObservable.Observer<MutationProcessorEvent>;
 	private readonly typeQuery = new WeakMap<
 		SchemaModel,
 		[TransformerMutationType, string, string][]
@@ -130,6 +138,8 @@ class MutationProcessor {
 			}
 
 			return this.runningProcesses.addCleaner(async () => {
+				// The observer has unsubscribed and/or `stop()` has been called.
+				this.removeObserver();
 				this.pause();
 			});
 		});
@@ -138,10 +148,16 @@ class MutationProcessor {
 	}
 
 	public async stop() {
+		this.removeObserver();
 		await this.runningProcesses.close();
 		await this.runningProcesses.open();
 	}
 
+	public removeObserver() {
+		this.observer?.complete?.();
+		this.observer = undefined;
+	}
+
 	public async resume(): Promise<void> {
 		await (this.runningProcesses.isOpen &&
 			this.runningProcesses.add(async onTerminate => {
@@ -195,7 +211,7 @@ class MutationProcessor {
 									operation,
 									data,
 									condition,
-									modelConstructor as any,
+									modelConstructor,
 									this.MutationEvent,
 									head,
 									operationAuthModes[authModeAttempts],
@@ -256,7 +272,7 @@ class MutationProcessor {
 						hasMore = (await this.outbox.peek(storage)) !== undefined;
 					});
 
-					this.observer.next!({
+					this.observer?.next?.({
 						operation,
 						modelDefinition,
 						model: record,
@@ -484,6 +500,11 @@ class MutationProcessor {
 		const modelDefinition = this.schema.namespaces[namespaceName].models[model];
 		const { primaryKey } = this.schema.namespaces[namespaceName].keys![model];
 
+		const auth = modelDefinition.attributes?.find(a => a.type === 'auth');
+		const ownerFields: string[] = auth?.properties?.rules
+			.map(rule => rule.ownerField)
+			.filter(f => f) || ['owner'];
+
 		const queriesTuples = this.typeQuery.get(modelDefinition);
 
 		const [, opName, query] = queriesTuples!.find(
@@ -512,7 +533,17 @@ class MutationProcessor {
 			mutationInput = {};
 			const modelFields = Object.values(modelDefinition.fields);
 
-			for (const { name, type, association } of modelFields) {
+			for (const { name, type, association, isReadOnly } of modelFields) {
+				// omit readonly fields. cloud storage doesn't need them and won't take them!
+				if (isReadOnly) {
+					continue;
+				}
+
+				// omit owner fields if it's `null`. cloud storage doesn't allow it.
+				if (ownerFields.includes(name) && parsedData[name] === null) {
+					continue;
+				}
+
 				// model fields should be stripped out from the input
 				if (isModelFieldType(type)) {
 					// except for belongs to relations - we need to replace them with the correct foreign key(s)

package/src/sync/processors/subscription.ts

@@ -28,6 +28,13 @@ import {
 	getUserGroupsFromToken,
 	TransformerMutationType,
 	getTokenForCustomAuth,
+	predicateToGraphQLFilter,
+	dynamicAuthFields,
+	filterFields,
+	repeatedFieldInGroup,
+	countFilterCombinations,
+	RTFError,
+	generateRTFRemediation,
 } from '../utils';
 import { ModelPredicateCreator } from '../../predicates';
 import { validatePredicate, USER_AGENT_SUFFIX_DATASTORE } from '../../util';
@@ -65,7 +72,10 @@ class SubscriptionProcessor {
 
 	constructor(
 		private readonly schema: InternalSchema,
-		private readonly syncPredicates: WeakMap<SchemaModel, ModelPredicate<any>>,
+		private readonly syncPredicates: WeakMap<
+			SchemaModel,
+			ModelPredicate<any> | null
+		>,
 		private readonly amplifyConfig: Record<string, any> = {},
 		private readonly authModeStrategy: AuthModeStrategy,
 		private readonly errorHandler: ErrorHandler,
@@ -79,7 +89,8 @@ class SubscriptionProcessor {
 		userCredentials: USER_CREDENTIALS,
 		cognitoTokenPayload: { [field: string]: any } | undefined,
 		oidcTokenPayload: { [field: string]: any } | undefined,
-		authMode: GRAPHQL_AUTH_MODE
+		authMode: GRAPHQL_AUTH_MODE,
+		filterArg: boolean = false
 	): {
 		opType: TransformerMutationType;
 		opName: string;
@@ -105,7 +116,8 @@ class SubscriptionProcessor {
 			model,
 			transformerMutationType,
 			isOwner,
-			ownerField!
+			ownerField!,
+			filterArg
 		);
 		return { authMode, opType, opName, query, isOwner, ownerField, ownerValue };
 	}
@@ -182,10 +194,18 @@ class SubscriptionProcessor {
 		cognitoOwnerAuthRules.forEach(ownerAuthRule => {
 			const ownerValue = cognitoTokenPayload[ownerAuthRule.identityClaim];
 
+			// AuthZ for "list of owners" is handled dynamically in the subscription auth request
+			// resolver. It doesn't rely on a subscription arg.
+			// Only pass a subscription arg for single owner auth
+			const singleOwner =
+				model.fields[ownerAuthRule.ownerField]?.isArray !== true;
+			const isOwnerArgRequired =
+				singleOwner && !ownerAuthRule.areSubscriptionsPublic;
+
 			if (ownerValue) {
 				ownerAuthInfo = {
 					authMode: GRAPHQL_AUTH_MODE.AMAZON_COGNITO_USER_POOLS,
-					isOwner: ownerAuthRule.areSubscriptionsPublic ? false : true,
+					isOwner: isOwnerArgRequired,
 					ownerField: ownerAuthRule.ownerField,
 					ownerValue,
 				};
@@ -209,10 +229,15 @@ class SubscriptionProcessor {
 		oidcOwnerAuthRules.forEach(ownerAuthRule => {
 			const ownerValue = oidcTokenPayload[ownerAuthRule.identityClaim];
 
+			const singleOwner =
+				model.fields[ownerAuthRule.ownerField]?.isArray !== true;
+			const isOwnerArgRequired =
+				singleOwner && !ownerAuthRule.areSubscriptionsPublic;
+
 			if (ownerValue) {
 				ownerAuthInfo = {
 					authMode: GRAPHQL_AUTH_MODE.OPENID_CONNECT,
-					isOwner: ownerAuthRule.areSubscriptionsPublic ? false : true,
+					isOwner: isOwnerArgRequired,
 					ownerField: ownerAuthRule.ownerField,
 					ownerValue,
 				};
@@ -356,8 +381,20 @@ class SubscriptionProcessor {
 										[TransformerMutationType.DELETE]: 0,
 									};
 
-									// Retry failed subscriptions with next auth mode (if available)
-									const authModeRetry = async operation => {
+									const predicatesGroup = ModelPredicateCreator.getPredicates(
+										this.syncPredicates.get(modelDefinition)!,
+										false
+									);
+
+									const addFilterArg = predicatesGroup !== undefined;
+
+									// Retry subscriptions that failed for one of the following reasons:
+									// 1. unauthorized - retry with next auth mode (if available)
+									// 2. RTF error - retry without sending filter arg. (filtering will fall back to clientside)
+									const subscriptionRetry = async (
+										operation,
+										addFilter = addFilterArg
+									) => {
 										const {
 											opType: transformerMutationType,
 											opName,
@@ -373,7 +410,8 @@ class SubscriptionProcessor {
 											userCredentials,
 											cognitoTokenPayload,
 											oidcTokenPayload,
-											readAuthModes[operationAuthModeAttempts[operation]]
+											readAuthModes[operationAuthModeAttempts[operation]],
+											addFilter
 										);
 
 										const authToken = await getTokenForCustomAuth(
@@ -383,6 +421,11 @@ class SubscriptionProcessor {
 
 										const variables = {};
 
+										if (addFilter && predicatesGroup) {
+											variables['filter'] =
+												predicateToGraphQLFilter(predicatesGroup);
+										}
+
 										if (isOwner) {
 											if (!ownerValue) {
 												observer.error(
@@ -478,6 +521,33 @@ class SubscriptionProcessor {
 															},
 														} = subscriptionError;
 
+														const isRTFError =
+															// only attempt catch if a filter variable was added to the subscription query
+															addFilter &&
+															this.catchRTFError(
+																message,
+																modelDefinition,
+																predicatesGroup
+															);
+
+														// Catch RTF errors
+														if (isRTFError) {
+															// Unsubscribe and clear subscription array for model/operation
+															subscriptions[modelDefinition.name][
+																transformerMutationType
+															].forEach(subscription =>
+																subscription.unsubscribe()
+															);
+
+															subscriptions[modelDefinition.name][
+																transformerMutationType
+															] = [];
+
+															// retry subscription connection without filter
+															subscriptionRetry(operation, false);
+															return;
+														}
+
 														if (
 															message.includes(
 																PUBSUB_CONTROL_MSG.REALTIME_SUBSCRIPTION_INIT_ERROR
@@ -523,10 +593,11 @@ class SubscriptionProcessor {
 																		]
 																	}`
 																);
-																authModeRetry(operation);
+																subscriptionRetry(operation);
 																return;
 															}
 														}
+
 														logger.warn('subscriptionError', message);
 
 														try {
@@ -586,7 +657,7 @@ class SubscriptionProcessor {
 										);
 									};
 
-									operations.forEach(op => authModeRetry(op));
+									operations.forEach(op => subscriptionRetry(op));
 								})
 						);
 				});
@@ -660,6 +731,49 @@ class SubscriptionProcessor {
 			this.buffer = [];
 		}
 	}
+
+	/**
+	 * @returns true if the service returned an RTF subscription error
+	 * @remarks logs a warning with remediation instructions
+	 *
+	 */
+	private catchRTFError(
+		message: string,
+		modelDefinition: SchemaModel,
+		predicatesGroup: PredicatesGroup<any> | undefined
+	): boolean {
+		const header =
+			'Backend subscriptions filtering error.\n' +
+			'Subscriptions filtering will be applied clientside.\n';
+
+		const messageErrorTypeMap = {
+			'UnknownArgument: Unknown field argument filter': RTFError.UnknownField,
+			'Filters exceed maximum attributes limit': RTFError.MaxAttributes,
+			'Filters combination exceed maximum limit': RTFError.MaxCombinations,
+			'filter uses same fieldName multiple time': RTFError.RepeatedFieldname,
+			"The variables input contains a field name 'not'": RTFError.NotGroup,
+			'The variables input contains a field that is not defined for input object type':
+				RTFError.FieldNotInType,
+		};
+
+		const [_errorMsg, errorType] =
+			Object.entries(messageErrorTypeMap).find(([errorMsg]) =>
+				message.includes(errorMsg)
+			) || [];
+
+		if (errorType !== undefined) {
+			const remediationMessage = generateRTFRemediation(
+				errorType,
+				modelDefinition,
+				predicatesGroup
+			);
+
+			logger.warn(`${header}\n${message}\n${remediationMessage}`);
+			return true;
+		}
+
+		return false;
+	}
 }
 
 export { SubscriptionProcessor };

package/src/sync/processors/sync.ts

@@ -45,7 +45,10 @@ class SyncProcessor {
 
 	constructor(
 		private readonly schema: InternalSchema,
-		private readonly syncPredicates: WeakMap<SchemaModel, ModelPredicate<any>>,
+		private readonly syncPredicates: WeakMap<
+			SchemaModel,
+			ModelPredicate<any> | null
+		>,
 		private readonly amplifyConfig: Record<string, any> = {},
 		private readonly authModeStrategy: AuthModeStrategy,
 		private readonly errorHandler: ErrorHandler,