@aws-amplify/storage 6.6.10 → 6.6.11-storage-browser-integrity.2a9053f.0

package/src/providers/s3/apis/{uploadData → internal/uploadData}/multipart/uploadHandlers.ts

@@ -4,31 +4,32 @@
 import { Amplify, StorageAccessLevel } from '@aws-amplify/core';
 import { StorageAction } from '@aws-amplify/core/internals/utils';
 
-import { UploadDataInput, UploadDataWithPathInput } from '../../../types';
+import { UploadDataInput, UploadDataWithPathInput } from '../../../../types';
 import {
 	resolveS3ConfigAndInput,
 	validateStorageOperationInput,
-} from '../../../utils';
-import { ItemWithKey, ItemWithPath } from '../../../types/outputs';
+} from '../../../../utils';
+import { ItemWithKey, ItemWithPath } from '../../../../types/outputs';
 import {
 	DEFAULT_ACCESS_LEVEL,
 	DEFAULT_QUEUE_SIZE,
 	STORAGE_INPUT_KEY,
-} from '../../../utils/constants';
+} from '../../../../utils/constants';
 import {
 	ResolvedS3Config,
 	UploadDataWithKeyOptions,
-} from '../../../types/options';
-import { StorageError } from '../../../../../errors/StorageError';
-import { CanceledError } from '../../../../../errors/CanceledError';
+} from '../../../../types/options';
+import { StorageError } from '../../../../../../errors/StorageError';
+import { CanceledError } from '../../../../../../errors/CanceledError';
 import {
 	Part,
 	abortMultipartUpload,
 	completeMultipartUpload,
 	headObject,
-} from '../../../utils/client';
-import { getStorageUserAgentValue } from '../../../utils/userAgent';
-import { logger } from '../../../../../utils';
+} from '../../../../utils/client/s3data';
+import { getStorageUserAgentValue } from '../../../../utils/userAgent';
+import { logger } from '../../../../../../utils';
+import { validateObjectNotExists } from '../validateObjectNotExists';
 
 import { uploadPartExecutor } from './uploadPartExecutor';
 import { getUploadsCacheKey, removeCachedUpload } from './uploadCache';
@@ -54,6 +55,7 @@ export const getMultipartUploadHandlers = (
 		| {
 				uploadId: string;
 				completedParts: Part[];
+				finalCrc32?: string;
 		  }
 		| undefined;
 	let resolvedS3Config: ResolvedS3Config | undefined;
@@ -64,6 +66,7 @@ export const getMultipartUploadHandlers = (
 	let resolvedIdentityId: string | undefined;
 	let uploadCacheKey: string | undefined;
 	let finalKey: string;
+	let expectedBucketOwner: string | undefined;
 	// Special flag that differentiates HTTP requests abort error caused by pause() from ones caused by cancel().
 	// The former one should NOT cause the upload job to throw, but cancels any pending HTTP requests.
 	// This should be replaced by a special abort reason. However,the support of this API is lagged behind.
@@ -73,7 +76,7 @@ export const getMultipartUploadHandlers = (
 		const { options: uploadDataOptions, data } = uploadDataInput;
 		const resolvedS3Options = await resolveS3ConfigAndInput(
 			Amplify,
-			uploadDataOptions,
+			uploadDataInput,
 		);
 
 		abortController = new AbortController();
@@ -81,6 +84,7 @@ export const getMultipartUploadHandlers = (
 		resolvedS3Config = resolvedS3Options.s3Config;
 		resolvedBucket = resolvedS3Options.bucket;
 		resolvedIdentityId = resolvedS3Options.identityId;
+		expectedBucketOwner = uploadDataOptions?.expectedBucketOwner;
 
 		const { inputType, objectKey } = validateStorageOperationInput(
 			uploadDataInput,
@@ -92,6 +96,7 @@ export const getMultipartUploadHandlers = (
 			contentEncoding,
 			contentType = 'application/octet-stream',
 			metadata,
+			preventOverwrite,
 			onProgress,
 		} = uploadDataOptions ?? {};
 
@@ -108,23 +113,26 @@ export const getMultipartUploadHandlers = (
 		}
 
 		if (!inProgressUpload) {
-			const { uploadId, cachedParts } = await loadOrCreateMultipartUpload({
-				s3Config: resolvedS3Config,
-				accessLevel: resolvedAccessLevel,
-				bucket: resolvedBucket,
-				keyPrefix: resolvedKeyPrefix,
-				key: objectKey,
-				contentType,
-				contentDisposition,
-				contentEncoding,
-				metadata,
-				data,
-				size,
-				abortSignal: abortController.signal,
-			});
+			const { uploadId, cachedParts, finalCrc32 } =
+				await loadOrCreateMultipartUpload({
+					s3Config: resolvedS3Config,
+					accessLevel: resolvedAccessLevel,
+					bucket: resolvedBucket,
+					keyPrefix: resolvedKeyPrefix,
+					key: objectKey,
+					contentType,
+					contentDisposition,
+					contentEncoding,
+					metadata,
+					data,
+					size,
+					abortSignal: abortController.signal,
+					expectedBucketOwner,
+				});
 			inProgressUpload = {
 				uploadId,
 				completedParts: cachedParts,
+				finalCrc32,
 			};
 		}
 
@@ -143,10 +151,16 @@ export const getMultipartUploadHandlers = (
 		const completedPartNumberSet = new Set<number>(
 			inProgressUpload.completedParts.map(({ PartNumber }) => PartNumber!),
 		);
-		const onPartUploadCompletion = (partNumber: number, eTag: string) => {
+		const onPartUploadCompletion = (
+			partNumber: number,
+			eTag: string,
+			crc32: string | undefined,
+		) => {
 			inProgressUpload?.completedParts.push({
 				PartNumber: partNumber,
 				ETag: eTag,
+				// TODO: crc32 can always be added once RN also has an implementation
+				...(crc32 ? { ChecksumCRC32: crc32 } : {}),
 			});
 		};
 		const concurrentUploadsProgressTracker =
@@ -169,12 +183,21 @@ export const getMultipartUploadHandlers = (
 					onPartUploadCompletion,
 					onProgress: concurrentUploadsProgressTracker.getOnProgressListener(),
 					isObjectLockEnabled: resolvedS3Options.isObjectLockEnabled,
+					useCRC32Checksum: Boolean(inProgressUpload.finalCrc32),
+					expectedBucketOwner,
 				}),
 			);
 		}
 
 		await Promise.all(concurrentUploadPartExecutors);
 
+		if (preventOverwrite) {
+			await validateObjectNotExists(resolvedS3Config, {
+				Bucket: resolvedBucket,
+				Key: finalKey,
+			});
+		}
+
 		const { ETag: eTag } = await completeMultipartUpload(
 			{
 				...resolvedS3Config,
@@ -185,11 +208,13 @@ export const getMultipartUploadHandlers = (
 				Bucket: resolvedBucket,
 				Key: finalKey,
 				UploadId: inProgressUpload.uploadId,
+				ChecksumCRC32: inProgressUpload.finalCrc32,
 				MultipartUpload: {
 					Parts: inProgressUpload.completedParts.sort(
 						(partA, partB) => partA.PartNumber! - partB.PartNumber!,
 					),
 				},
+				ExpectedBucketOwner: expectedBucketOwner,
 			},
 		);
 
@@ -199,6 +224,7 @@ export const getMultipartUploadHandlers = (
 				{
 					Bucket: resolvedBucket,
 					Key: finalKey,
+					ExpectedBucketOwner: expectedBucketOwner,
 				},
 			);
 			if (uploadedObjectSize && uploadedObjectSize !== size) {
@@ -264,6 +290,7 @@ export const getMultipartUploadHandlers = (
 				Bucket: resolvedBucket,
 				Key: finalKey,
 				UploadId: inProgressUpload?.uploadId,
+				ExpectedBucketOwner: expectedBucketOwner,
 			});
 		};
 		cancelUpload().catch(e => {

package/src/providers/s3/apis/{uploadData → internal/uploadData}/multipart/uploadPartExecutor.ts

@@ -1,11 +1,12 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { TransferProgressEvent } from '../../../../../types';
-import { ResolvedS3Config } from '../../../types/options';
-import { calculateContentMd5 } from '../../../utils';
-import { uploadPart } from '../../../utils/client';
-import { logger } from '../../../../../utils';
+import { TransferProgressEvent } from '../../../../../../types';
+import { ResolvedS3Config } from '../../../../types/options';
+import { uploadPart } from '../../../../utils/client/s3data';
+import { logger } from '../../../../../../utils';
+import { CRC32Checksum, calculateContentCRC32 } from '../../../../utils/crc32';
+import { calculateContentMd5 } from '../../../../utils';
 
 import { PartToUpload } from './getDataChunker';
 
@@ -18,8 +19,14 @@ interface UploadPartExecutorOptions {
 	finalKey: string;
 	uploadId: string;
 	isObjectLockEnabled?: boolean;
-	onPartUploadCompletion(partNumber: number, eTag: string): void;
+	useCRC32Checksum?: boolean;
+	onPartUploadCompletion(
+		partNumber: number,
+		eTag: string,
+		crc32: string | undefined,
+	): void;
 	onProgress?(event: TransferProgressEvent): void;
+	expectedBucketOwner?: string;
 }
 
 export const uploadPartExecutor = async ({
@@ -33,6 +40,8 @@ export const uploadPartExecutor = async ({
 	onPartUploadCompletion,
 	onProgress,
 	isObjectLockEnabled,
+	useCRC32Checksum,
+	expectedBucketOwner,
 }: UploadPartExecutorOptions) => {
 	let transferredBytes = 0;
 	for (const { data, partNumber, size } of dataChunkerGenerator) {
@@ -49,6 +58,16 @@ export const uploadPartExecutor = async ({
 			});
 		} else {
 			// handle cancel error
+			let checksumCRC32: CRC32Checksum | undefined;
+			if (useCRC32Checksum) {
+				checksumCRC32 = await calculateContentCRC32(data);
+			}
+			const contentMD5 =
+				// check if checksum exists. ex: should not exist in react native
+				!checksumCRC32 && isObjectLockEnabled
+					? await calculateContentMd5(data)
+					: undefined;
+
 			const { ETag: eTag } = await uploadPart(
 				{
 					...s3Config,
@@ -66,14 +85,14 @@ export const uploadPartExecutor = async ({
 					UploadId: uploadId,
 					Body: data,
 					PartNumber: partNumber,
-					ContentMD5: isObjectLockEnabled
-						? await calculateContentMd5(data)
-						: undefined,
+					ChecksumCRC32: checksumCRC32?.checksum,
+					ContentMD5: contentMD5,
+					ExpectedBucketOwner: expectedBucketOwner,
 				},
 			);
 			transferredBytes += size;
 			// eTag will always be set even the S3 model interface marks it as optional.
-			onPartUploadCompletion(partNumber, eTag!);
+			onPartUploadCompletion(partNumber, eTag!, checksumCRC32?.checksum);
 		}
 	}
 };

package/src/providers/s3/apis/{uploadData → internal/uploadData}/putObjectJob.ts

@@ -4,17 +4,21 @@
 import { Amplify } from '@aws-amplify/core';
 import { StorageAction } from '@aws-amplify/core/internals/utils';
 
-import { UploadDataInput, UploadDataWithPathInput } from '../../types';
+import { UploadDataInput, UploadDataWithPathInput } from '../../../types';
 import {
 	calculateContentMd5,
 	resolveS3ConfigAndInput,
+	validateBucketOwnerID,
 	validateStorageOperationInput,
-} from '../../utils';
-import { ItemWithKey, ItemWithPath } from '../../types/outputs';
-import { putObject } from '../../utils/client';
-import { getStorageUserAgentValue } from '../../utils/userAgent';
-import { STORAGE_INPUT_KEY } from '../../utils/constants';
-import { constructContentDisposition } from '../../utils/constructContentDisposition';
+} from '../../../utils';
+import { ItemWithKey, ItemWithPath } from '../../../types/outputs';
+import { putObject } from '../../../utils/client/s3data';
+import { getStorageUserAgentValue } from '../../../utils/userAgent';
+import { STORAGE_INPUT_KEY } from '../../../utils/constants';
+import { calculateContentCRC32 } from '../../../utils/crc32';
+import { constructContentDisposition } from '../../../utils/constructContentDisposition';
+
+import { validateObjectNotExists } from './validateObjectNotExists';
 
 /**
  * Get a function the returns a promise to call putObject API to S3.
@@ -30,11 +34,12 @@ export const putObjectJob =
 	async (): Promise<ItemWithKey | ItemWithPath> => {
 		const { options: uploadDataOptions, data } = uploadDataInput;
 		const { bucket, keyPrefix, s3Config, isObjectLockEnabled, identityId } =
-			await resolveS3ConfigAndInput(Amplify, uploadDataOptions);
+			await resolveS3ConfigAndInput(Amplify, uploadDataInput);
 		const { inputType, objectKey } = validateStorageOperationInput(
 			uploadDataInput,
 			identityId,
 		);
+		validateBucketOwnerID(uploadDataOptions?.expectedBucketOwner);
 
 		const finalKey =
 			inputType === STORAGE_INPUT_KEY ? keyPrefix + objectKey : objectKey;
@@ -42,10 +47,26 @@ export const putObjectJob =
 			contentDisposition,
 			contentEncoding,
 			contentType = 'application/octet-stream',
+			preventOverwrite,
 			metadata,
 			onProgress,
+			expectedBucketOwner,
 		} = uploadDataOptions ?? {};
 
+		const checksumCRC32 = await calculateContentCRC32(data);
+		const contentMD5 =
+			// check if checksum exists. ex: should not exist in react native
+			!checksumCRC32 && isObjectLockEnabled
+				? await calculateContentMd5(data)
+				: undefined;
+
+		if (preventOverwrite) {
+			await validateObjectNotExists(s3Config, {
+				Bucket: bucket,
+				Key: finalKey,
+			});
+		}
+
 		const { ETag: eTag, VersionId: versionId } = await putObject(
 			{
 				...s3Config,
@@ -61,9 +82,9 @@ export const putObjectJob =
 				ContentDisposition: constructContentDisposition(contentDisposition),
 				ContentEncoding: contentEncoding,
 				Metadata: metadata,
-				ContentMD5: isObjectLockEnabled
-					? await calculateContentMd5(data)
-					: undefined,
+				ContentMD5: contentMD5,
+				ChecksumCRC32: checksumCRC32?.checksum,
+				ExpectedBucketOwner: expectedBucketOwner,
 			},
 		);
 

package/src/providers/s3/apis/internal/uploadData/validateObjectNotExists.ts

@@ -0,0 +1,25 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { StorageError } from '../../../../../errors/StorageError';
+import { ResolvedS3Config } from '../../../types/options';
+import { HeadObjectInput, headObject } from '../../../utils/client/s3data';
+
+export const validateObjectNotExists = async (
+	s3Config: ResolvedS3Config,
+	input: HeadObjectInput,
+): Promise<void> => {
+	try {
+		await headObject(s3Config, input);
+
+		throw new StorageError({
+			name: 'PreconditionFailed',
+			message: 'At least one of the pre-conditions you specified did not hold',
+		});
+	} catch (error) {
+		const serviceError = error as StorageError;
+		if (serviceError.name !== 'NotFound') {
+			throw error;
+		}
+	}
+};

package/src/providers/s3/apis/{uploadData/index.ts → uploadData.ts}

@@ -6,15 +6,9 @@ import {
 	UploadDataOutput,
 	UploadDataWithPathInput,
 	UploadDataWithPathOutput,
-} from '../../types';
-import { createUploadTask } from '../../utils';
-import { assertValidationError } from '../../../../errors/utils/assertValidationError';
-import { StorageValidationErrorCode } from '../../../../errors/types/validation';
-import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../utils/constants';
+} from '../types';
 
-import { byteLength } from './byteLength';
-import { putObjectJob } from './putObjectJob';
-import { getMultipartUploadHandlers } from './multipart';
+import { uploadData as uploadDataInternal } from './internal/uploadData';
 
 /**
  * Upload data to the specified S3 object path. By default uses single PUT operation to upload if the payload is less than 5MB.
@@ -127,38 +121,5 @@ export function uploadData(
 export function uploadData(input: UploadDataInput): UploadDataOutput;
 
 export function uploadData(input: UploadDataInput | UploadDataWithPathInput) {
-	const { data } = input;
-
-	const dataByteLength = byteLength(data);
-	assertValidationError(
-		dataByteLength === undefined || dataByteLength <= MAX_OBJECT_SIZE,
-		StorageValidationErrorCode.ObjectIsTooLarge,
-	);
-
-	if (dataByteLength && dataByteLength <= DEFAULT_PART_SIZE) {
-		// Single part upload
-		const abortController = new AbortController();
-
-		return createUploadTask({
-			isMultipartUpload: false,
-			job: putObjectJob(input, abortController.signal, dataByteLength),
-			onCancel: (message?: string) => {
-				abortController.abort(message);
-			},
-		});
-	} else {
-		// Multipart upload
-		const { multipartUploadJob, onPause, onResume, onCancel } =
-			getMultipartUploadHandlers(input, dataByteLength);
-
-		return createUploadTask({
-			isMultipartUpload: true,
-			job: multipartUploadJob,
-			onCancel: (message?: string) => {
-				onCancel(message);
-			},
-			onPause,
-			onResume,
-		});
-	}
+	return uploadDataInternal(input);
 }

package/src/providers/s3/index.ts

@@ -48,3 +48,5 @@ export {
 	GetUrlOutput,
 	GetUrlWithPathOutput,
 } from './types/outputs';
+
+export { DEFAULT_PART_SIZE } from './utils/constants';

package/src/providers/s3/types/options.ts

@@ -2,7 +2,11 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { StorageAccessLevel } from '@aws-amplify/core';
-import { SigningOptions } from '@aws-amplify/core/internals/aws-client-utils';
+import {
+	CredentialsProviderOptions,
+	SigningOptions,
+} from '@aws-amplify/core/internals/aws-client-utils';
+import { AWSCredentials } from '@aws-amplify/core/internals/utils';
 
 import { TransferProgressEvent } from '../../../types';
 import {
@@ -11,6 +15,28 @@ import {
 	StorageSubpathStrategy,
 } from '../../../types/options';
 
+/**
+ * @internal
+ */
+export type AWSTemporaryCredentials = Required<
+	Pick<
+		AWSCredentials,
+		'accessKeyId' | 'secretAccessKey' | 'sessionToken' | 'expiration'
+	>
+>;
+
+/**
+ * Async function returning AWS credentials for an API call. This function
+ * is invoked with S3 locations(bucket and path).
+ * If omitted, the global credentials configured in Amplify Auth
+ * would be used.
+ *
+ * @internal
+ */
+export type LocationCredentialsProvider = (
+	options?: CredentialsProviderOptions,
+) => Promise<{ credentials: AWSTemporaryCredentials }>;
+
 export interface BucketInfo {
 	bucketName: string;
 	region: string;
@@ -23,7 +49,13 @@ interface CommonOptions {
 	 * @default false
 	 */
 	useAccelerateEndpoint?: boolean;
+
 	bucket?: StorageBucket;
+
+	/**
+	 * The expected owner of the target bucket.
+	 */
+	expectedBucketOwner?: string;
 }
 
 /**
@@ -190,6 +222,11 @@ export type UploadDataOptions = CommonOptions &
 		 * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata
 		 */
 		metadata?: Record<string, string>;
+		/**
+		 * Enforces target key does not already exist in S3 before committing upload.
+		 * @default false
+		 */
+		preventOverwrite?: boolean;
 	};
 
 /** @deprecated Use {@link UploadDataWithPathOptions} instead. */
@@ -201,6 +238,9 @@ export type CopySourceWithKeyOptions = ReadOptions & {
 	/** @deprecated This may be removed in the next major version. */
 	key: string;
 	bucket?: StorageBucket;
+	notModifiedSince?: Date;
+	eTag?: string;
+	expectedBucketOwner?: string;
 };
 
 /** @deprecated This may be removed in the next major version. */
@@ -208,13 +248,19 @@ export type CopyDestinationWithKeyOptions = WriteOptions & {
 	/** @deprecated This may be removed in the next major version. */
 	key: string;
 	bucket?: StorageBucket;
+	expectedBucketOwner?: string;
 };
 
 export interface CopyWithPathSourceOptions {
 	bucket?: StorageBucket;
+	notModifiedSince?: Date;
+	eTag?: string;
+	expectedBucketOwner?: string;
 }
+
 export interface CopyWithPathDestinationOptions {
 	bucket?: StorageBucket;
+	expectedBucketOwner?: string;
 }
 
 /**

package/src/providers/s3/utils/client/s3control/base.ts

@@ -0,0 +1,97 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+	AmplifyUrl,
+	getAmplifyUserAgent,
+} from '@aws-amplify/core/internals/utils';
+import {
+	EndpointResolverOptions,
+	getDnsSuffix,
+	jitteredBackoff,
+} from '@aws-amplify/core/internals/aws-client-utils';
+
+import { createRetryDecider, createXmlErrorParser } from '../utils';
+
+/**
+ * The service name used to sign requests if the API requires authentication.
+ */
+export const SERVICE_NAME = 's3';
+
+/**
+ * Options for endpoint resolver.
+ *
+ * @internal
+ */
+export type S3EndpointResolverOptions = EndpointResolverOptions & {
+	/**
+	 * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region.
+	 */
+	customEndpoint?: string;
+};
+
+/**
+ * The endpoint resolver function that returns the endpoint URL for a given region, and input parameters.
+ */
+const endpointResolver = (
+	options: S3EndpointResolverOptions,
+	apiInput?: { AccountId?: string },
+) => {
+	const { region, customEndpoint } = options;
+	const { AccountId: accountId } = apiInput || {};
+	let endpoint: URL;
+	// 1. get base endpoint
+	if (customEndpoint) {
+		endpoint = new AmplifyUrl(customEndpoint);
+	} else if (accountId) {
+		// Control plane operations
+		endpoint = new AmplifyUrl(
+			`https://${accountId}.s3-control.${region}.${getDnsSuffix(region)}`,
+		);
+	} else {
+		endpoint = new AmplifyUrl(
+			`https://s3-control.${region}.${getDnsSuffix(region)}`,
+		);
+	}
+
+	return { url: endpoint };
+};
+
+/**
+ * Error parser for the XML payload of S3 control plane error response. The
+ * error's `Code` and `Message` locates at the nested `Error` element instead of
+ * the XML root element.
+ *
+ * @example
+ * ```
+ * 	<?xml version="1.0" encoding="UTF-8"?>
+ * 	<ErrorResponse>
+ * 	  <Error>
+ * 		  <Code>AccessDenied</Code>
+ * 		  <Message>Access Denied</Message>
+ * 		</Error>
+ * 		<RequestId>656c76696e6727732072657175657374</RequestId>
+ * 		<HostId>Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg==</HostId>
+ * 	</ErrorResponse>
+ * 	```
+ *
+ * @internal
+ */
+export const parseXmlError = createXmlErrorParser();
+
+/**
+ * @internal
+ */
+export const retryDecider = createRetryDecider(parseXmlError);
+
+/**
+ * @internal
+ */
+export const defaultConfig = {
+	service: SERVICE_NAME,
+	endpointResolver,
+	retryDecider,
+	computeDelay: jitteredBackoff,
+	userAgentValue: getAmplifyUserAgent(),
+	uriEscapePath: false, // Required by S3. See https://github.com/aws/aws-sdk-js-v3/blob/9ba012dfa3a3429aa2db0f90b3b0b3a7a31f9bc3/packages/signature-v4/src/SignatureV4.ts#L76-L83
+};