@aws-amplify/datastore 3.12.6-next.13 → 3.12.6-next.32

package/src/sync/processors/sync.ts

@@ -1,4 +1,4 @@
-import API, { GraphQLResult, GRAPHQL_AUTH_MODE } from '@aws-amplify/api';
+import { API, GraphQLResult, GRAPHQL_AUTH_MODE } from '@aws-amplify/api';
 import Observable from 'zen-observable-ts';
 import {
 	InternalSchema,
@@ -26,6 +26,7 @@ import {
 	ConsoleLogger as Logger,
 	Hub,
 	NonRetryableError,
+	BackgroundProcessManager,
 } from '@aws-amplify/core';
 import { ModelPredicateCreator } from '../../predicates';
 import { getSyncErrorType } from './errorMaps';
@@ -40,6 +41,8 @@ const logger = new Logger('DataStore');
 class SyncProcessor {
 	private readonly typeQuery = new WeakMap<SchemaModel, [string, string]>();
 
+	private runningProcesses = new BackgroundProcessManager();
+
 	constructor(
 		private readonly schema: InternalSchema,
 		private readonly syncPredicates: WeakMap<SchemaModel, ModelPredicate<any>>,
@@ -85,14 +88,13 @@ class SyncProcessor {
 		return predicateToGraphQLFilter(predicatesGroup);
 	}
 
-	private async retrievePage<
-		T extends ModelInstanceMetadata = ModelInstanceMetadata
-	>(
+	private async retrievePage<T extends ModelInstanceMetadata>(
 		modelDefinition: SchemaModel,
 		lastSync: number,
 		nextToken: string,
 		limit: number = null,
-		filter: GraphQLFilter
+		filter: GraphQLFilter,
+		onTerminate: Promise<void>
 	): Promise<{ nextToken: string; startedAt: number; items: T[] }> {
 		const [opName, query] = this.typeQuery.get(modelDefinition);
 
@@ -115,6 +117,12 @@ class SyncProcessor {
 
 		let authModeAttempts = 0;
 		const authModeRetry = async () => {
+			if (!this.runningProcesses.isOpen) {
+				throw new Error(
+					'sync.retreievePage termination was requested. Exiting.'
+				);
+			}
+
 			try {
 				logger.debug(
 					`Attempting sync with authMode: ${readAuthModes[authModeAttempts]}`
@@ -125,6 +133,7 @@ class SyncProcessor {
 					opName,
 					modelDefinition,
 					authMode: readAuthModes[authModeAttempts],
+					onTerminate,
 				});
 				logger.debug(
 					`Sync successful with authMode: ${readAuthModes[authModeAttempts]}`
@@ -171,28 +180,20 @@ class SyncProcessor {
 		};
 	}
 
-	// Partial data private feature flag. Not a public API. This will be removed in a future release.
-	private partialDataFeatureFlagEnabled() {
-		try {
-			const flag = sessionStorage.getItem('datastorePartialData');
-			return Boolean(flag);
-		} catch (e) {
-			return false;
-		}
-	}
-
 	private async jitteredRetry<T>({
 		query,
 		variables,
 		opName,
 		modelDefinition,
 		authMode,
+		onTerminate,
 	}: {
 		query: string;
 		variables: { limit: number; lastSync: number; nextToken: string };
 		opName: string;
 		modelDefinition: SchemaModel;
 		authMode: GRAPHQL_AUTH_MODE;
+		onTerminate: Promise<void>;
 	}): Promise<
 		GraphQLResult<{
 			[opName: string]: {
@@ -217,6 +218,8 @@ class SyncProcessor {
 						authToken,
 						userAgentSuffix: USER_AGENT_SUFFIX_DATASTORE,
 					});
+
+					// TODO: onTerminate.then(() => API.cancel(...))
 				} catch (error) {
 					// Catch client-side (GraphQLAuthError) & 401/403 errors here so that we don't continue to retry
 					const clientOrForbiddenErrorMessage =
@@ -225,92 +228,90 @@ class SyncProcessor {
 						throw new NonRetryableError(clientOrForbiddenErrorMessage);
 					}
 
-					const hasItems = Boolean(
-						error &&
-							error.data &&
-							error.data[opName] &&
-							error.data[opName].items
-					);
-					if (this.partialDataFeatureFlagEnabled()) {
-						if (hasItems) {
-							const result = error;
-							result.data[opName].items = result.data[opName].items.filter(
-								item => item !== null
-							);
-							if (error.errors) {
-								await Promise.all(
-									error.errors.map(async err => {
-										try {
-											await this.errorHandler({
-												recoverySuggestion:
-													'Ensure app code is up to date, auth directives exist and are correct on each model, and that server-side data has not been invalidated by a schema change. If the problem persists, search for or create an issue: https://github.com/aws-amplify/amplify-js/issues',
-												localModel: null,
-												message: err.message,
-												model: modelDefinition.name,
-												operation: opName,
-												errorType: getSyncErrorType(err),
-												process: ProcessName.sync,
-												remoteModel: null,
-												cause: err,
-											});
-										} catch (e) {
-											logger.error('Sync error handler failed with:', e);
-										}
-									})
-								);
-								Hub.dispatch('datastore', {
-									event: 'syncQueriesPartialSyncError',
-									data: {
-										errors: error.errors,
-										modelName: modelDefinition.name,
-									},
-								});
-							}
-
-							return result;
-						} else {
-							throw error;
-						}
-					}
+					const hasItems = Boolean(error?.data?.[opName]?.items);
 
-					// If the error is unauthorized, filter out unauthorized items and return accessible items
 					const unauthorized =
-						error &&
-						error.errors &&
+						error?.errors &&
 						(error.errors as [any]).some(
 							err => err.errorType === 'Unauthorized'
 						);
-					if (unauthorized) {
-						const result = error;
 
-						if (hasItems) {
-							result.data[opName].items = result.data[opName].items.filter(
-								item => item !== null
-							);
-						} else {
-							result.data[opName] = {
-								...opResultDefaults,
-								...result.data[opName],
-							};
-						}
+					const otherErrors =
+						error?.errors &&
+						(error.errors as [any]).filter(
+							err => err.errorType !== 'Unauthorized'
+						);
+
+					const result = error;
+
+					if (hasItems) {
+						result.data[opName].items = result.data[opName].items.filter(
+							item => item !== null
+						);
+					}
+
+					if (hasItems && otherErrors?.length) {
+						await Promise.all(
+							otherErrors.map(async err => {
+								try {
+									await this.errorHandler({
+										recoverySuggestion:
+											'Ensure app code is up to date, auth directives exist and are correct on each model, and that server-side data has not been invalidated by a schema change. If the problem persists, search for or create an issue: https://github.com/aws-amplify/amplify-js/issues',
+										localModel: null,
+										message: err.message,
+										model: modelDefinition.name,
+										operation: opName,
+										errorType: getSyncErrorType(err),
+										process: ProcessName.sync,
+										remoteModel: null,
+										cause: err,
+									});
+								} catch (e) {
+									logger.error('Sync error handler failed with:', e);
+								}
+							})
+						);
+						Hub.dispatch('datastore', {
+							event: 'nonApplicableDataReceived',
+							data: {
+								errors: otherErrors,
+								modelName: modelDefinition.name,
+							},
+						});
+					}
+
+					if (unauthorized) {
 						logger.warn(
 							'queryError',
 							`User is unauthorized to query ${opName}, some items could not be returned.`
 						);
+
+						result.data = result.data || {};
+
+						result.data[opName] = {
+							...opResultDefaults,
+							...result.data[opName],
+						};
+
+						return result;
+					}
+
+					if (result.data?.[opName].items?.length) {
 						return result;
-					} else {
-						throw error;
 					}
+
+					throw error;
 				}
 			},
-			[query, variables]
+			[query, variables],
+			undefined,
+			onTerminate
 		);
 	}
 
 	start(
 		typesLastSync: Map<SchemaModel, [string, number]>
 	): Observable<SyncModelPage> {
-		let processing = true;
 		const { maxRecordsToSync, syncPageSize } = this.amplifyConfig;
 		const parentPromises = new Map<string, Promise<void>>();
 		const observable = new Observable<SyncModelPage>(observer => {
@@ -329,76 +330,88 @@ class SyncProcessor {
 
 			const allModelsReady = Array.from(sortedTypesLastSyncs.entries())
 				.filter(([{ syncable }]) => syncable)
-				.map(async ([modelDefinition, [namespace, lastSync]]) => {
-					let done = false;
-					let nextToken: string = null;
-					let startedAt: number = null;
-					let items: ModelInstanceMetadata[] = null;
-
-					let recordsReceived = 0;
-					const filter = this.graphqlFilterFromPredicate(modelDefinition);
-
-					const parents = this.schema.namespaces[
-						namespace
-					].modelTopologicalOrdering.get(modelDefinition.name);
-					const promises = parents.map(parent =>
-						parentPromises.get(`${namespace}_${parent}`)
-					);
-
-					const promise = new Promise<void>(async res => {
-						await Promise.all(promises);
-
-						do {
-							if (!processing) {
-								return;
-							}
-
-							const limit = Math.min(
-								maxRecordsToSync - recordsReceived,
-								syncPageSize
+				.map(
+					([modelDefinition, [namespace, lastSync]]) =>
+						this.runningProcesses.isOpen &&
+						this.runningProcesses.add(async onTerminate => {
+							let done = false;
+							let nextToken: string = null;
+							let startedAt: number = null;
+							let items: ModelInstanceMetadata[] = null;
+
+							let recordsReceived = 0;
+							const filter = this.graphqlFilterFromPredicate(modelDefinition);
+
+							const parents = this.schema.namespaces[
+								namespace
+							].modelTopologicalOrdering.get(modelDefinition.name);
+							const promises = parents.map(parent =>
+								parentPromises.get(`${namespace}_${parent}`)
 							);
 
-							({ items, nextToken, startedAt } = await this.retrievePage(
-								modelDefinition,
-								lastSync,
-								nextToken,
-								limit,
-								filter
-							));
-
-							recordsReceived += items.length;
-
-							done = nextToken === null || recordsReceived >= maxRecordsToSync;
-
-							observer.next({
-								namespace,
-								modelDefinition,
-								items,
-								done,
-								startedAt,
-								isFullSync: !lastSync,
+							const promise = new Promise<void>(async res => {
+								await Promise.all(promises);
+
+								do {
+									if (!this.runningProcesses.isOpen) {
+										return;
+									}
+
+									const limit = Math.min(
+										maxRecordsToSync - recordsReceived,
+										syncPageSize
+									);
+
+									({ items, nextToken, startedAt } = await this.retrievePage(
+										modelDefinition,
+										lastSync,
+										nextToken,
+										limit,
+										filter,
+										onTerminate
+									));
+
+									recordsReceived += items.length;
+
+									done =
+										nextToken === null || recordsReceived >= maxRecordsToSync;
+
+									observer.next({
+										namespace,
+										modelDefinition,
+										items,
+										done,
+										startedAt,
+										isFullSync: !lastSync,
+									});
+								} while (!done);
+
+								res();
 							});
-						} while (!done);
-
-						res();
-					});
 
-					parentPromises.set(`${namespace}_${modelDefinition.name}`, promise);
+							parentPromises.set(
+								`${namespace}_${modelDefinition.name}`,
+								promise
+							);
 
-					await promise;
-				});
+							await promise;
+						}, `adding model ${modelDefinition.name}`)
+				);
 
 			Promise.all(allModelsReady).then(() => {
 				observer.complete();
 			});
-
-			return () => {
-				processing = false;
-			};
 		});
 
 		return observable;
 	}
+
+	async stop() {
+		logger.debug('stopping sync processor');
+		await this.runningProcesses.close();
+		await this.runningProcesses.open();
+		logger.debug('sync processor stopped');
+	}
 }
 
 export type SyncModelPage = {

package/src/sync/utils.ts

@@ -11,6 +11,7 @@ import {
 	isGraphQLScalarType,
 	isPredicateObj,
 	isSchemaModel,
+	isSchemaModelWithAttributes,
 	isTargetNameAssociation,
 	isNonModelFieldType,
 	ModelFields,
@@ -27,7 +28,12 @@ import {
 	InternalSchema,
 	AuthModeStrategy,
 } from '../types';
-import { exhaustiveCheck } from '../util';
+import {
+	exhaustiveCheck,
+	extractPrimaryKeyFieldNames,
+	establishRelationAndKeys,
+	IDENTIFIER_KEY_SEPARATOR,
+} from '../util';
 import { MutationEvent } from './';
 
 const logger = new Logger('DataStore');
@@ -47,7 +53,7 @@ export enum TransformerMutationType {
 	GET = 'Get',
 }
 
-const dummyMetadata: Omit<ModelInstanceMetadata, 'id'> = {
+const dummyMetadata: ModelInstanceMetadata = {
 	_version: undefined,
 	_lastChangedAt: undefined,
 	_deleted: undefined,
@@ -79,7 +85,7 @@ export function generateSelectionSet(
 	if (isSchemaModel(modelDefinition)) {
 		scalarAndMetadataFields = scalarAndMetadataFields
 			.concat(getMetadataFields())
-			.concat(getConnectionFields(modelDefinition));
+			.concat(getConnectionFields(modelDefinition, namespace));
 	}
 
 	const result = scalarAndMetadataFields.join('\n');
@@ -103,7 +109,7 @@ function getOwnerFields(
 	modelDefinition: SchemaModel | SchemaNonModel
 ): string[] {
 	const ownerFields: string[] = [];
-	if (isSchemaModel(modelDefinition) && modelDefinition.attributes) {
+	if (isSchemaModelWithAttributes(modelDefinition)) {
 		modelDefinition.attributes.forEach(attr => {
 			if (attr.properties && attr.properties.rules) {
 				const rule = attr.properties.rules.find(rule => rule.allow === 'owner');
@@ -138,8 +144,12 @@ function getScalarFields(
 	return result;
 }
 
-function getConnectionFields(modelDefinition: SchemaModel): string[] {
-	const result = [];
+// Used for generating the selection set for queries and mutations
+function getConnectionFields(
+	modelDefinition: SchemaModel,
+	namespace: SchemaNamespace
+): string[] {
+	const result: string[] = [];
 
 	Object.values(modelDefinition.fields)
 		.filter(({ association }) => association && Object.keys(association).length)
@@ -153,7 +163,26 @@ function getConnectionFields(modelDefinition: SchemaModel): string[] {
 					break;
 				case 'BELONGS_TO':
 					if (isTargetNameAssociation(association)) {
-						result.push(`${name} { id _deleted }`);
+						// New codegen (CPK)
+						if (association.targetNames && association.targetNames.length > 0) {
+							// Need to retrieve relations in order to get connected model keys
+							const [relations] = establishRelationAndKeys(namespace);
+
+							const connectedModelName =
+								modelDefinition.fields[name].type['model'];
+
+							const byPkIndex = relations[connectedModelName].indexes.find(
+								([name]) => name === 'byPk'
+							);
+							const keyFields = byPkIndex && byPkIndex[1];
+							const keyFieldSelectionSet = keyFields?.join(' ');
+
+							// We rely on `_deleted` when we process the sync query (e.g. in batchSave in the adapters)
+							result.push(`${name} { ${keyFieldSelectionSet} _deleted }`);
+						} else {
+							// backwards-compatability for schema generated prior to custom primary key support
+							result.push(`${name} { id _deleted }`);
+						}
 					}
 					break;
 				default:
@@ -412,10 +441,13 @@ export function createMutationInstanceFromModelOperation<
 		return v;
 	};
 
+	const modelId = getIdentifierValue(modelDefinition, element);
+	const optionalId = OpType.INSERT && id ? { id } : {};
+
 	const mutationEvent = modelInstanceCreator(MutationEventConstructor, {
-		...(id ? { id } : {}),
+		...optionalId,
 		data: JSON.stringify(element, replacer),
-		modelId: element.id,
+		modelId,
 		model: model.name,
 		operation,
 		condition: JSON.stringify(condition),
@@ -425,7 +457,8 @@ export function createMutationInstanceFromModelOperation<
 }
 
 export function predicateToGraphQLCondition(
-	predicate: PredicatesGroup<any>
+	predicate: PredicatesGroup<any>,
+	modelDefinition: SchemaModel
 ): GraphQLCondition {
 	const result = {};
 
@@ -433,17 +466,27 @@ export function predicateToGraphQLCondition(
 		return result;
 	}
 
+	const keyFields = extractPrimaryKeyFieldNames(modelDefinition);
+
 	predicate.predicates.forEach(p => {
 		if (isPredicateObj(p)) {
 			const { field, operator, operand } = p;
 
-			if (field === 'id') {
+			// This is compatible with how the GQL Transform currently generates the Condition Input,
+			// i.e. any PK and SK fields are omitted and can't be used as conditions.
+			// However, I think this limits usability.
+			// What if we want to delete all records where SK > some value
+			// Or all records where PK = some value but SKs are different values
+
+			// TODO: if the Transform gets updated ^ we'll need to modify this logic to only omit
+			// key fields from the predicate/condition when ALL of the keyFields are present and using `eq` operators
+			if (keyFields.includes(field as string)) {
 				return;
 			}
 
 			result[field] = { [operator]: operand };
 		} else {
-			result[p.type] = predicateToGraphQLCondition(p);
+			result[p.type] = predicateToGraphQLCondition(p, modelDefinition);
 		}
 	});
 
@@ -610,3 +653,15 @@ export async function getTokenForCustomAuth(
 		}
 	}
 }
+
+// Util that takes a modelDefinition and model and returns either the id value(s) or the custom primary key value(s)
+export function getIdentifierValue(
+	modelDefinition: SchemaModel,
+	model: ModelInstanceMetadata | PersistentModel
+): string {
+	const pkFieldNames = extractPrimaryKeyFieldNames(modelDefinition);
+
+	const idOrPk = pkFieldNames.map(f => model[f]).join(IDENTIFIER_KEY_SEPARATOR);
+
+	return idOrPk;
+}