@aws-amplify/storage 6.6.5 → 6.6.6-storage-browser.4949269.0

package/src/providers/s3/apis/internal/remove.ts

@@ -14,7 +14,7 @@ import {
 	resolveS3ConfigAndInput,
 	validateStorageOperationInput,
 } from '../../utils';
-import { deleteObject } from '../../utils/client';
+import { deleteObject } from '../../utils/client/s3data';
 import { getStorageUserAgentValue } from '../../utils/userAgent';
 import { logger } from '../../../../utils';
 import { STORAGE_INPUT_KEY } from '../../utils/constants';
@@ -23,9 +23,8 @@ export const remove = async (
 	amplify: AmplifyClassV6,
 	input: RemoveInput | RemoveWithPathInput,
 ): Promise<RemoveOutput | RemoveWithPathOutput> => {
-	const { options = {} } = input ?? {};
 	const { s3Config, keyPrefix, bucket, identityId } =
-		await resolveS3ConfigAndInput(amplify, options);
+		await resolveS3ConfigAndInput(amplify, input);
 
 	const { inputType, objectKey } = validateStorageOperationInput(
 		input,

package/src/providers/s3/apis/uploadData/multipart/initialUpload.ts

@@ -5,8 +5,9 @@ import { StorageAccessLevel } from '@aws-amplify/core';
 
 import { ContentDisposition, ResolvedS3Config } from '../../../types/options';
 import { StorageUploadDataPayload } from '../../../../../types';
-import { Part, createMultipartUpload } from '../../../utils/client';
+import { Part, createMultipartUpload } from '../../../utils/client/s3data';
 import { logger } from '../../../../../utils';
+import { calculateContentCRC32 } from '../../../utils/crc32';
 import { constructContentDisposition } from '../../../utils/constructContentDisposition';
 
 import {
@@ -14,6 +15,7 @@ import {
 	findCachedUploadParts,
 	getUploadsCacheKey,
 } from './uploadCache';
+import { getDataChunker } from './getDataChunker';
 
 interface LoadOrCreateMultipartUploadOptions {
 	s3Config: ResolvedS3Config;
@@ -33,6 +35,7 @@ interface LoadOrCreateMultipartUploadOptions {
 interface LoadOrCreateMultipartUploadResult {
 	uploadId: string;
 	cachedParts: Part[];
+	finalCrc32?: string;
 }
 
 /**
@@ -62,6 +65,7 @@ export const loadOrCreateMultipartUpload = async ({
 				parts: Part[];
 				uploadId: string;
 				uploadCacheKey: string;
+				finalCrc32?: string;
 		  }
 		| undefined;
 	if (size === undefined) {
@@ -92,8 +96,11 @@ export const loadOrCreateMultipartUpload = async ({
 		return {
 			uploadId: cachedUpload.uploadId,
 			cachedParts: cachedUpload.parts,
+			finalCrc32: cachedUpload.finalCrc32,
 		};
 	} else {
+		const finalCrc32 = await getCombinedCrc32(data, size);
+
 		const { UploadId } = await createMultipartUpload(
 			{
 				...s3Config,
@@ -106,14 +113,17 @@ export const loadOrCreateMultipartUpload = async ({
 				ContentDisposition: constructContentDisposition(contentDisposition),
 				ContentEncoding: contentEncoding,
 				Metadata: metadata,
+				ChecksumAlgorithm: finalCrc32 ? 'CRC32' : undefined,
 			},
 		);
+
 		if (size === undefined) {
 			logger.debug('uploaded data size cannot be determined, skipping cache.');
 
 			return {
 				uploadId: UploadId!,
 				cachedParts: [],
+				finalCrc32,
 			};
 		}
 		const uploadCacheKey = getUploadsCacheKey({
@@ -128,12 +138,31 @@ export const loadOrCreateMultipartUpload = async ({
 			uploadId: UploadId!,
 			bucket,
 			key,
+			finalCrc32,
 			fileName: data instanceof File ? data.name : '',
 		});
 
 		return {
 			uploadId: UploadId!,
 			cachedParts: [],
+			finalCrc32,
 		};
 	}
 };
+
+const getCombinedCrc32 = async (
+	data: StorageUploadDataPayload,
+	size: number | undefined,
+) => {
+	const crc32List: ArrayBuffer[] = [];
+	const dataChunker = getDataChunker(data, size);
+	for (const { data: checkData } of dataChunker) {
+		const checksumArrayBuffer = (await calculateContentCRC32(checkData))
+			?.checksumArrayBuffer;
+		if (checksumArrayBuffer === undefined) return undefined;
+
+		crc32List.push(checksumArrayBuffer);
+	}
+
+	return `${(await calculateContentCRC32(new Blob(crc32List)))?.checksum}-${crc32List.length}`;
+};

package/src/providers/s3/apis/uploadData/multipart/uploadCache.ts

@@ -9,7 +9,7 @@ import {
 
 import { UPLOADS_STORAGE_KEY } from '../../../utils/constants';
 import { ResolvedS3Config } from '../../../types/options';
-import { Part, listParts } from '../../../utils/client';
+import { Part, listParts } from '../../../utils/client/s3data';
 import { logger } from '../../../../../utils';
 
 const ONE_HOUR = 1000 * 60 * 60;
@@ -33,6 +33,7 @@ export const findCachedUploadParts = async ({
 }: FindCachedUploadPartsOptions): Promise<{
 	parts: Part[];
 	uploadId: string;
+	finalCrc32?: string;
 } | null> => {
 	const cachedUploads = await listCachedUploadTasks(defaultStorage);
 	if (
@@ -60,6 +61,7 @@ export const findCachedUploadParts = async ({
 		return {
 			parts: Parts,
 			uploadId: cachedUpload.uploadId,
+			finalCrc32: cachedUpload.finalCrc32,
 		};
 	} catch (e) {
 		logger.debug('failed to list cached parts, removing cached upload.');
@@ -74,6 +76,7 @@ interface FileMetadata {
 	fileName: string;
 	key: string;
 	uploadId: string;
+	finalCrc32?: string;
 	// Unix timestamp in ms
 	lastTouched: number;
 }

package/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts

@@ -26,9 +26,10 @@ import {
 	abortMultipartUpload,
 	completeMultipartUpload,
 	headObject,
-} from '../../../utils/client';
+} from '../../../utils/client/s3data';
 import { getStorageUserAgentValue } from '../../../utils/userAgent';
 import { logger } from '../../../../../utils';
+import { validateObjectNotExists } from '../validateObjectNotExists';
 
 import { uploadPartExecutor } from './uploadPartExecutor';
 import { getUploadsCacheKey, removeCachedUpload } from './uploadCache';
@@ -54,6 +55,7 @@ export const getMultipartUploadHandlers = (
 		| {
 				uploadId: string;
 				completedParts: Part[];
+				finalCrc32?: string;
 		  }
 		| undefined;
 	let resolvedS3Config: ResolvedS3Config | undefined;
@@ -73,7 +75,7 @@ export const getMultipartUploadHandlers = (
 		const { options: uploadDataOptions, data } = uploadDataInput;
 		const resolvedS3Options = await resolveS3ConfigAndInput(
 			Amplify,
-			uploadDataOptions,
+			uploadDataInput,
 		);
 
 		abortController = new AbortController();
@@ -92,6 +94,7 @@ export const getMultipartUploadHandlers = (
 			contentEncoding,
 			contentType = 'application/octet-stream',
 			metadata,
+			preventOverwrite,
 			onProgress,
 		} = uploadDataOptions ?? {};
 
@@ -108,23 +111,25 @@ export const getMultipartUploadHandlers = (
 		}
 
 		if (!inProgressUpload) {
-			const { uploadId, cachedParts } = await loadOrCreateMultipartUpload({
-				s3Config: resolvedS3Config,
-				accessLevel: resolvedAccessLevel,
-				bucket: resolvedBucket,
-				keyPrefix: resolvedKeyPrefix,
-				key: objectKey,
-				contentType,
-				contentDisposition,
-				contentEncoding,
-				metadata,
-				data,
-				size,
-				abortSignal: abortController.signal,
-			});
+			const { uploadId, cachedParts, finalCrc32 } =
+				await loadOrCreateMultipartUpload({
+					s3Config: resolvedS3Config,
+					accessLevel: resolvedAccessLevel,
+					bucket: resolvedBucket,
+					keyPrefix: resolvedKeyPrefix,
+					key: objectKey,
+					contentType,
+					contentDisposition,
+					contentEncoding,
+					metadata,
+					data,
+					size,
+					abortSignal: abortController.signal,
+				});
 			inProgressUpload = {
 				uploadId,
 				completedParts: cachedParts,
+				finalCrc32,
 			};
 		}
 
@@ -143,10 +148,15 @@ export const getMultipartUploadHandlers = (
 		const completedPartNumberSet = new Set<number>(
 			inProgressUpload.completedParts.map(({ PartNumber }) => PartNumber!),
 		);
-		const onPartUploadCompletion = (partNumber: number, eTag: string) => {
+		const onPartUploadCompletion = (
+			partNumber: number,
+			eTag: string,
+			crc32: string | undefined,
+		) => {
 			inProgressUpload?.completedParts.push({
 				PartNumber: partNumber,
 				ETag: eTag,
+				ChecksumCRC32: crc32,
 			});
 		};
 		const concurrentUploadsProgressTracker =
@@ -169,12 +179,20 @@ export const getMultipartUploadHandlers = (
 					onPartUploadCompletion,
 					onProgress: concurrentUploadsProgressTracker.getOnProgressListener(),
 					isObjectLockEnabled: resolvedS3Options.isObjectLockEnabled,
+					useCRC32Checksum: Boolean(inProgressUpload.finalCrc32),
 				}),
 			);
 		}
 
 		await Promise.all(concurrentUploadPartExecutors);
 
+		if (preventOverwrite) {
+			await validateObjectNotExists(resolvedS3Config, {
+				Bucket: resolvedBucket,
+				Key: finalKey,
+			});
+		}
+
 		const { ETag: eTag } = await completeMultipartUpload(
 			{
 				...resolvedS3Config,
@@ -185,6 +203,7 @@ export const getMultipartUploadHandlers = (
 				Bucket: resolvedBucket,
 				Key: finalKey,
 				UploadId: inProgressUpload.uploadId,
+				ChecksumCRC32: inProgressUpload.finalCrc32,
 				MultipartUpload: {
 					Parts: inProgressUpload.completedParts.sort(
 						(partA, partB) => partA.PartNumber! - partB.PartNumber!,

package/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts

@@ -3,9 +3,10 @@
 
 import { TransferProgressEvent } from '../../../../../types';
 import { ResolvedS3Config } from '../../../types/options';
-import { calculateContentMd5 } from '../../../utils';
-import { uploadPart } from '../../../utils/client';
+import { uploadPart } from '../../../utils/client/s3data';
 import { logger } from '../../../../../utils';
+import { CRC32Checksum, calculateContentCRC32 } from '../../../utils/crc32';
+import { calculateContentMd5 } from '../../../utils';
 
 import { PartToUpload } from './getDataChunker';
 
@@ -18,7 +19,12 @@ interface UploadPartExecutorOptions {
 	finalKey: string;
 	uploadId: string;
 	isObjectLockEnabled?: boolean;
-	onPartUploadCompletion(partNumber: number, eTag: string): void;
+	useCRC32Checksum?: boolean;
+	onPartUploadCompletion(
+		partNumber: number,
+		eTag: string,
+		crc32: string | undefined,
+	): void;
 	onProgress?(event: TransferProgressEvent): void;
 }
 
@@ -33,6 +39,7 @@ export const uploadPartExecutor = async ({
 	onPartUploadCompletion,
 	onProgress,
 	isObjectLockEnabled,
+	useCRC32Checksum,
 }: UploadPartExecutorOptions) => {
 	let transferredBytes = 0;
 	for (const { data, partNumber, size } of dataChunkerGenerator) {
@@ -49,6 +56,16 @@ export const uploadPartExecutor = async ({
 			});
 		} else {
 			// handle cancel error
+			let checksumCRC32: CRC32Checksum | undefined;
+			if (useCRC32Checksum) {
+				checksumCRC32 = await calculateContentCRC32(data);
+			}
+			const contentMD5 =
+				// check if checksum exists. ex: should not exist in react native
+				!checksumCRC32 && isObjectLockEnabled
+					? await calculateContentMd5(data)
+					: undefined;
+
 			const { ETag: eTag } = await uploadPart(
 				{
 					...s3Config,
@@ -66,14 +83,13 @@ export const uploadPartExecutor = async ({
 					UploadId: uploadId,
 					Body: data,
 					PartNumber: partNumber,
-					ContentMD5: isObjectLockEnabled
-						? await calculateContentMd5(data)
-						: undefined,
+					ChecksumCRC32: checksumCRC32?.checksum,
+					ContentMD5: contentMD5,
 				},
 			);
 			transferredBytes += size;
 			// eTag will always be set even the S3 model interface marks it as optional.
-			onPartUploadCompletion(partNumber, eTag!);
+			onPartUploadCompletion(partNumber, eTag!, checksumCRC32?.checksum);
 		}
 	}
 };

package/src/providers/s3/apis/uploadData/putObjectJob.ts

@@ -11,11 +11,14 @@ import {
 	validateStorageOperationInput,
 } from '../../utils';
 import { ItemWithKey, ItemWithPath } from '../../types/outputs';
-import { putObject } from '../../utils/client';
+import { putObject } from '../../utils/client/s3data';
 import { getStorageUserAgentValue } from '../../utils/userAgent';
 import { STORAGE_INPUT_KEY } from '../../utils/constants';
+import { calculateContentCRC32 } from '../../utils/crc32';
 import { constructContentDisposition } from '../../utils/constructContentDisposition';
 
+import { validateObjectNotExists } from './validateObjectNotExists';
+
 /**
  * Get a function the returns a promise to call putObject API to S3.
  *
@@ -30,7 +33,7 @@ export const putObjectJob =
 	async (): Promise<ItemWithKey | ItemWithPath> => {
 		const { options: uploadDataOptions, data } = uploadDataInput;
 		const { bucket, keyPrefix, s3Config, isObjectLockEnabled, identityId } =
-			await resolveS3ConfigAndInput(Amplify, uploadDataOptions);
+			await resolveS3ConfigAndInput(Amplify, uploadDataInput);
 		const { inputType, objectKey } = validateStorageOperationInput(
 			uploadDataInput,
 			identityId,
@@ -42,10 +45,25 @@ export const putObjectJob =
 			contentDisposition,
 			contentEncoding,
 			contentType = 'application/octet-stream',
+			preventOverwrite,
 			metadata,
 			onProgress,
 		} = uploadDataOptions ?? {};
 
+		const checksumCRC32 = await calculateContentCRC32(data);
+		const contentMD5 =
+			// check if checksum exists. ex: should not exist in react native
+			!checksumCRC32 && isObjectLockEnabled
+				? await calculateContentMd5(data)
+				: undefined;
+
+		if (preventOverwrite) {
+			await validateObjectNotExists(s3Config, {
+				Bucket: bucket,
+				Key: finalKey,
+			});
+		}
+
 		const { ETag: eTag, VersionId: versionId } = await putObject(
 			{
 				...s3Config,
@@ -61,9 +79,8 @@ export const putObjectJob =
 				ContentDisposition: constructContentDisposition(contentDisposition),
 				ContentEncoding: contentEncoding,
 				Metadata: metadata,
-				ContentMD5: isObjectLockEnabled
-					? await calculateContentMd5(data)
-					: undefined,
+				ContentMD5: contentMD5,
+				ChecksumCRC32: checksumCRC32?.checksum,
 			},
 		);
 

package/src/providers/s3/apis/uploadData/validateObjectNotExists.ts

@@ -0,0 +1,25 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { StorageError } from '../../../../errors/StorageError';
+import { ResolvedS3Config } from '../../types/options';
+import { HeadObjectInput, headObject } from '../../utils/client/s3data';
+
+export const validateObjectNotExists = async (
+	s3Config: ResolvedS3Config,
+	input: HeadObjectInput,
+): Promise<void> => {
+	try {
+		await headObject(s3Config, input);
+
+		throw new StorageError({
+			name: 'PreconditionFailed',
+			message: 'At least one of the pre-conditions you specified did not hold',
+		});
+	} catch (error) {
+		const serviceError = error as StorageError;
+		if (serviceError.name !== 'NotFound') {
+			throw error;
+		}
+	}
+};

package/src/providers/s3/types/inputs.ts

@@ -17,6 +17,7 @@ import {
 	StorageUploadDataInputWithKey,
 	StorageUploadDataInputWithPath,
 } from '../../../types';
+import { StorageOperationOptionsInput } from '../../../types/inputs';
 import {
 	CopyDestinationOptionsWithKey,
 	CopySourceOptionsWithKey,
@@ -35,6 +36,8 @@ import {
 	UploadDataOptionsWithPath,
 } from '../types';
 
+import { LocationCredentialsProvider } from './options';
+
 // TODO: support use accelerate endpoint option
 /**
  * @deprecated Use {@link CopyWithPathInput} instead.
@@ -47,7 +50,10 @@ export type CopyInput = StorageCopyInputWithKey<
 /**
  * Input type with path for S3 copy API.
  */
-export type CopyWithPathInput = StorageCopyInputWithPath;
+export type CopyWithPathInput = StorageCopyInputWithPath &
+	StorageOperationOptionsInput<{
+		locationCredentialsProvider?: LocationCredentialsProvider;
+	}>;
 
 /**
  * @deprecated Use {@link GetPropertiesWithPathInput} instead.

package/src/providers/s3/types/options.ts

@@ -2,7 +2,11 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { StorageAccessLevel } from '@aws-amplify/core';
-import { SigningOptions } from '@aws-amplify/core/internals/aws-client-utils';
+import { AWSCredentials } from '@aws-amplify/core/internals/utils';
+import {
+	CredentialsProviderOptions,
+	SigningOptions,
+} from '@aws-amplify/core/internals/aws-client-utils';
 
 import { TransferProgressEvent } from '../../../types';
 import {
@@ -11,6 +15,23 @@ import {
 	StorageSubpathStrategy,
 } from '../../../types/options';
 
+/**
+ * @internal
+ */
+export type AWSTemporaryCredentials = Required<
+	Pick<
+		AWSCredentials,
+		'accessKeyId' | 'secretAccessKey' | 'sessionToken' | 'expiration'
+	>
+>;
+
+/**
+ * @internal
+ */
+export type LocationCredentialsProvider = (
+	options?: CredentialsProviderOptions,
+) => Promise<{ credentials: AWSTemporaryCredentials }>;
+
 export interface BucketInfo {
 	bucketName: string;
 	region: string;
@@ -23,6 +44,14 @@ interface CommonOptions {
 	 * @default false
 	 */
 	useAccelerateEndpoint?: boolean;
+
+	/**
+	 * Async function returning AWS credentials for an API call. This function
+	 * is invoked with S3 locations(bucket and path).
+	 * If omitted, the global credentials configured in Amplify Auth
+	 * would be used.
+	 */
+	locationCredentialsProvider?: LocationCredentialsProvider;
 	bucket?: StorageBucket;
 }
 
@@ -190,6 +219,11 @@ export type UploadDataOptions = CommonOptions &
 		 * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata
 		 */
 		metadata?: Record<string, string>;
+		/**
+		 * Enforces target key does not already exist in S3 before committing upload.
+		 * @default false
+		 */
+		preventOverwrite?: boolean;
 	};
 
 /** @deprecated Use {@link UploadDataOptionsWithPath} instead. */

package/src/providers/s3/utils/client/s3control/base.ts

@@ -0,0 +1,97 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+	AmplifyUrl,
+	getAmplifyUserAgent,
+} from '@aws-amplify/core/internals/utils';
+import {
+	EndpointResolverOptions,
+	getDnsSuffix,
+	jitteredBackoff,
+} from '@aws-amplify/core/internals/aws-client-utils';
+
+import { createRetryDecider, createXmlErrorParser } from '../utils';
+
+/**
+ * The service name used to sign requests if the API requires authentication.
+ */
+export const SERVICE_NAME = 's3';
+
+/**
+ * Options for endpoint resolver.
+ *
+ * @internal
+ */
+export type S3EndpointResolverOptions = EndpointResolverOptions & {
+	/**
+	 * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region.
+	 */
+	customEndpoint?: string;
+};
+
+/**
+ * The endpoint resolver function that returns the endpoint URL for a given region, and input parameters.
+ */
+const endpointResolver = (
+	options: S3EndpointResolverOptions,
+	apiInput?: { AccountId?: string },
+) => {
+	const { region, customEndpoint } = options;
+	const { AccountId: accountId } = apiInput || {};
+	let endpoint: URL;
+	// 1. get base endpoint
+	if (customEndpoint) {
+		endpoint = new AmplifyUrl(customEndpoint);
+	} else if (accountId) {
+		// Control plane operations
+		endpoint = new AmplifyUrl(
+			`https://${accountId}.s3-control.${region}.${getDnsSuffix(region)}`,
+		);
+	} else {
+		endpoint = new AmplifyUrl(
+			`https://s3-control.${region}.${getDnsSuffix(region)}`,
+		);
+	}
+
+	return { url: endpoint };
+};
+
+/**
+ * Error parser for the XML payload of S3 control plane error response. The
+ * error's `Code` and `Message` locates at the nested `Error` element instead of
+ * the XML root element.
+ *
+ * @example
+ * ```
+ * 	<?xml version="1.0" encoding="UTF-8"?>
+ * 	<ErrorResponse>
+ * 	  <Error>
+ * 		  <Code>AccessDenied</Code>
+ * 		  <Message>Access Denied</Message>
+ * 		</Error>
+ * 		<RequestId>656c76696e6727732072657175657374</RequestId>
+ * 		<HostId>Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg==</HostId>
+ * 	</ErrorResponse>
+ * 	```
+ *
+ * @internal
+ */
+export const parseXmlError = createXmlErrorParser();
+
+/**
+ * @internal
+ */
+export const retryDecider = createRetryDecider(parseXmlError);
+
+/**
+ * @internal
+ */
+export const defaultConfig = {
+	service: SERVICE_NAME,
+	endpointResolver,
+	retryDecider,
+	computeDelay: jitteredBackoff,
+	userAgentValue: getAmplifyUserAgent(),
+	uriEscapePath: false, // Required by S3. See https://github.com/aws/aws-sdk-js-v3/blob/9ba012dfa3a3429aa2db0f90b3b0b3a7a31f9bc3/packages/signature-v4/src/SignatureV4.ts#L76-L83
+};