@aws-amplify/storage 6.6.10 → 6.6.11-storage-browser-integrity.2a9053f.0

package/src/providers/s3/apis/internal/getProperties.ts

@@ -7,30 +7,34 @@ import { StorageAction } from '@aws-amplify/core/internals/utils';
 import {
 	GetPropertiesInput,
 	GetPropertiesOutput,
-	GetPropertiesWithPathInput,
 	GetPropertiesWithPathOutput,
 } from '../../types';
 import {
 	resolveS3ConfigAndInput,
+	validateBucketOwnerID,
 	validateStorageOperationInput,
 } from '../../utils';
-import { headObject } from '../../utils/client';
+import { headObject } from '../../utils/client/s3data';
 import { getStorageUserAgentValue } from '../../utils/userAgent';
 import { logger } from '../../../../utils';
 import { STORAGE_INPUT_KEY } from '../../utils/constants';
+// TODO: Remove this interface when we move to public advanced APIs.
+import { GetPropertiesInput as GetPropertiesWithPathInputWithAdvancedOptions } from '../../../../internals';
 
 export const getProperties = async (
 	amplify: AmplifyClassV6,
-	input: GetPropertiesInput | GetPropertiesWithPathInput,
+	input: GetPropertiesInput | GetPropertiesWithPathInputWithAdvancedOptions,
 	action?: StorageAction,
 ): Promise<GetPropertiesOutput | GetPropertiesWithPathOutput> => {
-	const { options: getPropertiesOptions } = input;
 	const { s3Config, bucket, keyPrefix, identityId } =
-		await resolveS3ConfigAndInput(amplify, getPropertiesOptions);
+		await resolveS3ConfigAndInput(amplify, input);
 	const { inputType, objectKey } = validateStorageOperationInput(
 		input,
 		identityId,
 	);
+
+	validateBucketOwnerID(input.options?.expectedBucketOwner);
+
 	const finalKey =
 		inputType === STORAGE_INPUT_KEY ? keyPrefix + objectKey : objectKey;
 
@@ -45,6 +49,7 @@ export const getProperties = async (
 		{
 			Bucket: bucket,
 			Key: finalKey,
+			ExpectedBucketOwner: input.options?.expectedBucketOwner,
 		},
 	);
 

package/src/providers/s3/apis/internal/getUrl.ts

@@ -4,16 +4,12 @@
 import { AmplifyClassV6 } from '@aws-amplify/core';
 import { StorageAction } from '@aws-amplify/core/internals/utils';
 
-import {
-	GetUrlInput,
-	GetUrlOutput,
-	GetUrlWithPathInput,
-	GetUrlWithPathOutput,
-} from '../../types';
+import { GetUrlInput, GetUrlOutput, GetUrlWithPathOutput } from '../../types';
 import { StorageValidationErrorCode } from '../../../../errors/types/validation';
-import { getPresignedGetObjectUrl } from '../../utils/client';
+import { getPresignedGetObjectUrl } from '../../utils/client/s3data';
 import {
 	resolveS3ConfigAndInput,
+	validateBucketOwnerID,
 	validateStorageOperationInput,
 } from '../../utils';
 import { assertValidationError } from '../../../../errors/utils/assertValidationError';
@@ -23,20 +19,23 @@ import {
 	STORAGE_INPUT_KEY,
 } from '../../utils/constants';
 import { constructContentDisposition } from '../../utils/constructContentDisposition';
+// TODO: Remove this interface when we move to public advanced APIs.
+import { GetUrlInput as GetUrlWithPathInputWithAdvancedOptions } from '../../../../internals';
 
 import { getProperties } from './getProperties';
 
 export const getUrl = async (
 	amplify: AmplifyClassV6,
-	input: GetUrlInput | GetUrlWithPathInput,
+	input: GetUrlInput | GetUrlWithPathInputWithAdvancedOptions,
 ): Promise<GetUrlOutput | GetUrlWithPathOutput> => {
 	const { options: getUrlOptions } = input;
 	const { s3Config, keyPrefix, bucket, identityId } =
-		await resolveS3ConfigAndInput(amplify, getUrlOptions);
+		await resolveS3ConfigAndInput(amplify, input);
 	const { inputType, objectKey } = validateStorageOperationInput(
 		input,
 		identityId,
 	);
+	validateBucketOwnerID(getUrlOptions?.expectedBucketOwner);
 
 	const finalKey =
 		inputType === STORAGE_INPUT_KEY ? keyPrefix + objectKey : objectKey;
@@ -83,6 +82,7 @@ export const getUrl = async (
 				...(getUrlOptions?.contentType && {
 					ResponseContentType: getUrlOptions.contentType,
 				}),
+				ExpectedBucketOwner: getUrlOptions?.expectedBucketOwner,
 			},
 		),
 		expiresAt: new Date(Date.now() + urlExpirationInSec * 1000),

package/src/providers/s3/apis/internal/list.ts

@@ -5,19 +5,16 @@ import { AmplifyClassV6 } from '@aws-amplify/core';
 import { StorageAction } from '@aws-amplify/core/internals/utils';
 
 import {
-	ListAllInput,
 	ListAllOutput,
-	ListAllWithPathInput,
 	ListAllWithPathOutput,
 	ListOutputItem,
 	ListOutputItemWithPath,
-	ListPaginateInput,
 	ListPaginateOutput,
-	ListPaginateWithPathInput,
 	ListPaginateWithPathOutput,
 } from '../../types';
 import {
 	resolveS3ConfigAndInput,
+	validateBucketOwnerID,
 	validateStorageOperationInputWithPrefix,
 } from '../../utils';
 import {
@@ -29,11 +26,15 @@ import {
 	ListObjectsV2Input,
 	ListObjectsV2Output,
 	listObjectsV2,
-} from '../../utils/client';
+} from '../../utils/client/s3data';
 import { getStorageUserAgentValue } from '../../utils/userAgent';
 import { logger } from '../../../../utils';
 import { DEFAULT_DELIMITER, STORAGE_INPUT_PREFIX } from '../../utils/constants';
-import { CommonPrefix } from '../../utils/client/types';
+import { CommonPrefix } from '../../utils/client/s3data/types';
+import { IntegrityError } from '../../../../errors/IntegrityError';
+import { ListAllInput, ListPaginateInput } from '../../types/inputs';
+// TODO: Remove this interface when we move to public advanced APIs.
+import { ListInput as ListWithPathInputAndAdvancedOptions } from '../../../../internals/types/inputs';
 
 const MAX_PAGE_SIZE = 1000;
 
@@ -45,11 +46,7 @@ interface ListInputArgs {
 
 export const list = async (
 	amplify: AmplifyClassV6,
-	input:
-		| ListAllInput
-		| ListPaginateInput
-		| ListAllWithPathInput
-		| ListPaginateWithPathInput,
+	input: ListAllInput | ListPaginateInput | ListWithPathInputAndAdvancedOptions,
 ): Promise<
 	| ListAllOutput
 	| ListPaginateOutput
@@ -62,12 +59,13 @@ export const list = async (
 		bucket,
 		keyPrefix: generatedPrefix,
 		identityId,
-	} = await resolveS3ConfigAndInput(amplify, options);
+	} = await resolveS3ConfigAndInput(amplify, input);
 
 	const { inputType, objectKey } = validateStorageOperationInputWithPrefix(
 		input,
 		identityId,
 	);
+	validateBucketOwnerID(options.expectedBucketOwner);
 	const isInputWithPrefix = inputType === STORAGE_INPUT_PREFIX;
 
 	// @ts-expect-error pageSize and nextToken should not coexist with listAll
@@ -86,6 +84,7 @@ export const list = async (
 		MaxKeys: options?.listAll ? undefined : options?.pageSize,
 		ContinuationToken: options?.listAll ? undefined : options?.nextToken,
 		Delimiter: getDelimiter(options),
+		ExpectedBucketOwner: options?.expectedBucketOwner,
 	};
 	logger.debug(`listing items from "${listParams.Prefix}"`);
 
@@ -160,6 +159,8 @@ const _listWithPrefix = async ({
 		listParamsClone,
 	);
 
+	validateEchoedElements(listParamsClone, response);
+
 	if (!response?.Contents) {
 		return {
 			items: [],
@@ -220,11 +221,7 @@ const _listWithPath = async ({
 		listParamsClone.MaxKeys = MAX_PAGE_SIZE;
 	}
 
-	const {
-		Contents: contents,
-		NextContinuationToken: nextContinuationToken,
-		CommonPrefixes: commonPrefixes,
-	}: ListObjectsV2Output = await listObjectsV2(
+	const listOutput = await listObjectsV2(
 		{
 			...s3Config,
 			userAgentValue: getStorageUserAgentValue(StorageAction.List),
@@ -232,6 +229,14 @@ const _listWithPath = async ({
 		listParamsClone,
 	);
 
+	validateEchoedElements(listParamsClone, listOutput);
+
+	const {
+		Contents: contents,
+		NextContinuationToken: nextContinuationToken,
+		CommonPrefixes: commonPrefixes,
+	}: ListObjectsV2Output = listOutput;
+
 	const excludedSubpaths =
 		commonPrefixes && mapCommonPrefixesToExcludedSubpaths(commonPrefixes);
 
@@ -273,3 +278,19 @@ const getDelimiter = (
 		return options?.subpathStrategy?.delimiter ?? DEFAULT_DELIMITER;
 	}
 };
+
+const validateEchoedElements = (
+	listInput: ListObjectsV2Input,
+	listOutput: ListObjectsV2Output,
+) => {
+	const validEchoedParameters =
+		listInput.Bucket === listOutput.Name &&
+		listInput.Delimiter === listOutput.Delimiter &&
+		listInput.MaxKeys === listOutput.MaxKeys &&
+		listInput.Prefix === listOutput.Prefix &&
+		listInput.ContinuationToken === listOutput.ContinuationToken;
+
+	if (!validEchoedParameters) {
+		throw new IntegrityError();
+	}
+};

package/src/providers/s3/apis/internal/remove.ts

@@ -4,33 +4,31 @@
 import { AmplifyClassV6 } from '@aws-amplify/core';
 import { StorageAction } from '@aws-amplify/core/internals/utils';
 
-import {
-	RemoveInput,
-	RemoveOutput,
-	RemoveWithPathInput,
-	RemoveWithPathOutput,
-} from '../../types';
+import { RemoveInput, RemoveOutput, RemoveWithPathOutput } from '../../types';
 import {
 	resolveS3ConfigAndInput,
+	validateBucketOwnerID,
 	validateStorageOperationInput,
 } from '../../utils';
-import { deleteObject } from '../../utils/client';
+import { deleteObject } from '../../utils/client/s3data';
 import { getStorageUserAgentValue } from '../../utils/userAgent';
 import { logger } from '../../../../utils';
 import { STORAGE_INPUT_KEY } from '../../utils/constants';
+// TODO: Remove this interface when we move to public advanced APIs.
+import { RemoveInput as RemoveWithPathInputWithAdvancedOptions } from '../../../../internals';
 
 export const remove = async (
 	amplify: AmplifyClassV6,
-	input: RemoveInput | RemoveWithPathInput,
+	input: RemoveInput | RemoveWithPathInputWithAdvancedOptions,
 ): Promise<RemoveOutput | RemoveWithPathOutput> => {
-	const { options = {} } = input ?? {};
 	const { s3Config, keyPrefix, bucket, identityId } =
-		await resolveS3ConfigAndInput(amplify, options);
+		await resolveS3ConfigAndInput(amplify, input);
 
 	const { inputType, objectKey } = validateStorageOperationInput(
 		input,
 		identityId,
 	);
+	validateBucketOwnerID(input.options?.expectedBucketOwner);
 
 	let finalKey;
 	if (inputType === STORAGE_INPUT_KEY) {
@@ -49,6 +47,7 @@ export const remove = async (
 		{
 			Bucket: bucket,
 			Key: finalKey,
+			ExpectedBucketOwner: input.options?.expectedBucketOwner,
 		},
 	);
 

package/src/providers/s3/apis/internal/uploadData/index.ts

@@ -0,0 +1,53 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { UploadDataInput } from '../../../types';
+// TODO: Remove this interface when we move to public advanced APIs.
+import { UploadDataInput as UploadDataWithPathInputWithAdvancedOptions } from '../../../../../internals/types/inputs';
+import { createUploadTask } from '../../../utils';
+import { assertValidationError } from '../../../../../errors/utils/assertValidationError';
+import { StorageValidationErrorCode } from '../../../../../errors/types/validation';
+import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../../utils/constants';
+
+import { byteLength } from './byteLength';
+import { putObjectJob } from './putObjectJob';
+import { getMultipartUploadHandlers } from './multipart';
+
+export const uploadData = (
+	input: UploadDataInput | UploadDataWithPathInputWithAdvancedOptions,
+) => {
+	const { data } = input;
+
+	const dataByteLength = byteLength(data);
+	assertValidationError(
+		dataByteLength === undefined || dataByteLength <= MAX_OBJECT_SIZE,
+		StorageValidationErrorCode.ObjectIsTooLarge,
+	);
+
+	if (dataByteLength !== undefined && dataByteLength <= DEFAULT_PART_SIZE) {
+		// Single part upload
+		const abortController = new AbortController();
+
+		return createUploadTask({
+			isMultipartUpload: false,
+			job: putObjectJob(input, abortController.signal, dataByteLength),
+			onCancel: (message?: string) => {
+				abortController.abort(message);
+			},
+		});
+	} else {
+		// Multipart upload
+		const { multipartUploadJob, onPause, onResume, onCancel } =
+			getMultipartUploadHandlers(input, dataByteLength);
+
+		return createUploadTask({
+			isMultipartUpload: true,
+			job: multipartUploadJob,
+			onCancel: (message?: string) => {
+				onCancel(message);
+			},
+			onPause,
+			onResume,
+		});
+	}
+};

package/src/providers/s3/apis/{uploadData → internal/uploadData}/multipart/calculatePartSize.ts

@@ -1,7 +1,10 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { DEFAULT_PART_SIZE, MAX_PARTS_COUNT } from '../../../utils/constants';
+import {
+	DEFAULT_PART_SIZE,
+	MAX_PARTS_COUNT,
+} from '../../../../utils/constants';
 
 export const calculatePartSize = (totalSize?: number): number => {
 	if (!totalSize) {

package/src/providers/s3/apis/{uploadData → internal/uploadData}/multipart/getDataChunker.ts

@@ -1,12 +1,12 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { StorageUploadDataPayload } from '../../../../../types';
+import { StorageUploadDataPayload } from '../../../../../../types';
 import {
 	StorageValidationErrorCode,
 	validationErrorMap,
-} from '../../../../../errors/types/validation';
-import { StorageError } from '../../../../../errors/StorageError';
+} from '../../../../../../errors/types/validation';
+import { StorageError } from '../../../../../../errors/StorageError';
 
 import { calculatePartSize } from './calculatePartSize';
 

package/src/providers/s3/apis/{uploadData → internal/uploadData}/multipart/initialUpload.ts

@@ -3,17 +3,22 @@
 
 import { StorageAccessLevel } from '@aws-amplify/core';
 
-import { ContentDisposition, ResolvedS3Config } from '../../../types/options';
-import { StorageUploadDataPayload } from '../../../../../types';
-import { Part, createMultipartUpload } from '../../../utils/client';
-import { logger } from '../../../../../utils';
-import { constructContentDisposition } from '../../../utils/constructContentDisposition';
+import {
+	ContentDisposition,
+	ResolvedS3Config,
+} from '../../../../types/options';
+import { StorageUploadDataPayload } from '../../../../../../types';
+import { Part, createMultipartUpload } from '../../../../utils/client/s3data';
+import { logger } from '../../../../../../utils';
+import { calculateContentCRC32 } from '../../../../utils/crc32';
+import { constructContentDisposition } from '../../../../utils/constructContentDisposition';
 
 import {
 	cacheMultipartUpload,
 	findCachedUploadParts,
 	getUploadsCacheKey,
 } from './uploadCache';
+import { getDataChunker } from './getDataChunker';
 
 interface LoadOrCreateMultipartUploadOptions {
 	s3Config: ResolvedS3Config;
@@ -28,11 +33,13 @@ interface LoadOrCreateMultipartUploadOptions {
 	metadata?: Record<string, string>;
 	size?: number;
 	abortSignal?: AbortSignal;
+	expectedBucketOwner?: string;
 }
 
 interface LoadOrCreateMultipartUploadResult {
 	uploadId: string;
 	cachedParts: Part[];
+	finalCrc32?: string;
 }
 
 /**
@@ -54,6 +61,7 @@ export const loadOrCreateMultipartUpload = async ({
 	contentEncoding,
 	metadata,
 	abortSignal,
+	expectedBucketOwner,
 }: LoadOrCreateMultipartUploadOptions): Promise<LoadOrCreateMultipartUploadResult> => {
 	const finalKey = keyPrefix !== undefined ? keyPrefix + key : key;
 
@@ -62,6 +70,7 @@ export const loadOrCreateMultipartUpload = async ({
 				parts: Part[];
 				uploadId: string;
 				uploadCacheKey: string;
+				finalCrc32?: string;
 		  }
 		| undefined;
 	if (size === undefined) {
@@ -92,8 +101,11 @@ export const loadOrCreateMultipartUpload = async ({
 		return {
 			uploadId: cachedUpload.uploadId,
 			cachedParts: cachedUpload.parts,
+			finalCrc32: cachedUpload.finalCrc32,
 		};
 	} else {
+		const finalCrc32 = await getCombinedCrc32(data, size);
+
 		const { UploadId } = await createMultipartUpload(
 			{
 				...s3Config,
@@ -106,14 +118,18 @@ export const loadOrCreateMultipartUpload = async ({
 				ContentDisposition: constructContentDisposition(contentDisposition),
 				ContentEncoding: contentEncoding,
 				Metadata: metadata,
+				ChecksumAlgorithm: finalCrc32 ? 'CRC32' : undefined,
+				ExpectedBucketOwner: expectedBucketOwner,
 			},
 		);
+
 		if (size === undefined) {
 			logger.debug('uploaded data size cannot be determined, skipping cache.');
 
 			return {
 				uploadId: UploadId!,
 				cachedParts: [],
+				finalCrc32,
 			};
 		}
 		const uploadCacheKey = getUploadsCacheKey({
@@ -128,12 +144,31 @@ export const loadOrCreateMultipartUpload = async ({
 			uploadId: UploadId!,
 			bucket,
 			key,
+			finalCrc32,
 			fileName: data instanceof File ? data.name : '',
 		});
 
 		return {
 			uploadId: UploadId!,
 			cachedParts: [],
+			finalCrc32,
 		};
 	}
 };
+
+const getCombinedCrc32 = async (
+	data: StorageUploadDataPayload,
+	size: number | undefined,
+) => {
+	const crc32List: ArrayBuffer[] = [];
+	const dataChunker = getDataChunker(data, size);
+	for (const { data: checkData } of dataChunker) {
+		const checksumArrayBuffer = (await calculateContentCRC32(checkData))
+			?.checksumArrayBuffer;
+		if (checksumArrayBuffer === undefined) return undefined;
+
+		crc32List.push(checksumArrayBuffer);
+	}
+
+	return `${(await calculateContentCRC32(new Blob(crc32List)))?.checksum}-${crc32List.length}`;
+};

package/src/providers/s3/apis/{uploadData → internal/uploadData}/multipart/progressTracker.ts

@@ -1,7 +1,7 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { TransferProgressEvent } from '../../../../../types';
+import { TransferProgressEvent } from '../../../../../../types';
 
 interface ConcurrentUploadsProgressTrackerOptions {
 	size?: number;

package/src/providers/s3/apis/{uploadData → internal/uploadData}/multipart/uploadCache.ts

@@ -7,10 +7,10 @@ import {
 	defaultStorage,
 } from '@aws-amplify/core';
 
-import { UPLOADS_STORAGE_KEY } from '../../../utils/constants';
-import { ResolvedS3Config } from '../../../types/options';
-import { Part, listParts } from '../../../utils/client';
-import { logger } from '../../../../../utils';
+import { UPLOADS_STORAGE_KEY } from '../../../../utils/constants';
+import { ResolvedS3Config } from '../../../../types/options';
+import { Part, listParts } from '../../../../utils/client/s3data';
+import { logger } from '../../../../../../utils';
 
 const ONE_HOUR = 1000 * 60 * 60;
 
@@ -33,6 +33,7 @@ export const findCachedUploadParts = async ({
 }: FindCachedUploadPartsOptions): Promise<{
 	parts: Part[];
 	uploadId: string;
+	finalCrc32?: string;
 } | null> => {
 	const cachedUploads = await listCachedUploadTasks(defaultStorage);
 	if (
@@ -60,6 +61,7 @@ export const findCachedUploadParts = async ({
 		return {
 			parts: Parts,
 			uploadId: cachedUpload.uploadId,
+			finalCrc32: cachedUpload.finalCrc32,
 		};
 	} catch (e) {
 		logger.debug('failed to list cached parts, removing cached upload.');
@@ -74,6 +76,7 @@ interface FileMetadata {
 	fileName: string;
 	key: string;
 	uploadId: string;
+	finalCrc32?: string;
 	// Unix timestamp in ms
 	lastTouched: number;
 }