@maas/payload-plugin-media-cloud 0.0.37 → 0.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -7,6 +7,8 @@ import { S3MetadataManager } from "./metadataManager.mjs";
|
|
|
7
7
|
import { S3PartsManager } from "./partsManager.mjs";
|
|
8
8
|
import stream from "node:stream";
|
|
9
9
|
import { NoSuchKey, NotFound, S3 } from "@aws-sdk/client-s3";
|
|
10
|
+
import { NodeHttpHandler } from "@smithy/node-http-handler";
|
|
11
|
+
import { Agent } from "https";
|
|
10
12
|
import { DataStore, ERRORS, TUS_RESUMABLE, Upload } from "@tus/utils";
|
|
11
13
|
|
|
12
14
|
//#region src/tus/stores/s3/s3Store.ts
|
|
@@ -29,9 +31,21 @@ var S3Store = class extends DataStore {
|
|
|
29
31
|
"termination",
|
|
30
32
|
"expiration"
|
|
31
33
|
];
|
|
34
|
+
const mappedS3ClientConfig = {
|
|
35
|
+
...restS3ClientConfig,
|
|
36
|
+
requestHandler: new NodeHttpHandler({
|
|
37
|
+
connectionTimeout: 6e4,
|
|
38
|
+
requestTimeout: 3e5,
|
|
39
|
+
httpsAgent: new Agent({
|
|
40
|
+
maxSockets: 1e3,
|
|
41
|
+
keepAlive: true,
|
|
42
|
+
keepAliveMsecs: 6e4
|
|
43
|
+
})
|
|
44
|
+
})
|
|
45
|
+
};
|
|
32
46
|
this.bucket = bucket;
|
|
33
47
|
this.acl = acl;
|
|
34
|
-
this.client = new S3(
|
|
48
|
+
this.client = new S3(mappedS3ClientConfig);
|
|
35
49
|
this.customEndpoint = String(restS3ClientConfig.endpoint);
|
|
36
50
|
this.partSize = partSize ?? this.partSize;
|
|
37
51
|
this.minPartSize = minPartSize ?? this.minPartSize;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3Store.mjs","names":["request: AWS.CreateMultipartUploadCommandInput","metadata: TusUploadMetadata","offset: number","lastError: Error | null","region: string"],"sources":["../../../../src/tus/stores/s3/s3Store.ts"],"sourcesContent":["import { NoSuchKey, NotFound, S3 } from '@aws-sdk/client-s3'\nimport { DataStore, ERRORS, TUS_RESUMABLE, Upload } from '@tus/utils'\nimport stream, { type Readable } from 'node:stream'\n\nimport { useErrorHandler } from '../../../hooks/useErrorHandler'\nimport { Semaphore } from './semaphore'\nimport { S3ExpirationManager } from './expirationManager'\nimport { S3FileOperations } from './fileOperations'\nimport { S3MetadataManager } from './metadataManager'\nimport { S3PartsManager } from './partsManager'\n\nimport { MediaCloudLogs } from '../../../types/errors'\n\nimport type AWS from '@aws-sdk/client-s3'\nimport type { AWSError, S3StoreConfig, TusUploadMetadata } from '../../../types'\n\nconst { log } = useErrorHandler()\n\nexport class S3Store extends DataStore {\n public client: S3\n public bucket: string\n public partSize = 8 * 1024 * 1024 // 8MB preferred part size\n public minPartSize = 5 * 1024 * 1024 // 5MB minimum part size\n public maxMultipartParts = 10_000\n public maxUploadSize = 5_497_558_138_880 as const // 5TiB\n public useTags = false\n public expirationPeriodInMilliseconds = 0\n protected acl?: string\n\n protected partUploadSemaphore: Semaphore\n protected metadataManager: S3MetadataManager\n protected fileOperations: S3FileOperations\n protected partsManager: S3PartsManager\n protected expirationManager: S3ExpirationManager\n protected customEndpoint: string\n\n constructor(options: S3StoreConfig) {\n super()\n const {\n maxMultipartParts,\n minPartSize,\n partSize,\n s3ClientConfig,\n maxConcurrentPartUploads,\n useTags,\n expirationPeriodInMilliseconds,\n } = options\n const { acl, bucket, ...restS3ClientConfig } = s3ClientConfig\n\n this.extensions = [\n 'creation',\n 'creation-with-upload',\n 'creation-defer-length',\n 'termination',\n 'expiration',\n ]\n\n this.bucket = bucket\n this.acl = acl\n this.client = new S3(restS3ClientConfig)\n this.customEndpoint = String(restS3ClientConfig.endpoint)\n\n this.partSize = partSize ?? this.partSize\n this.minPartSize = minPartSize ?? this.minPartSize\n this.maxMultipartParts = maxMultipartParts ?? this.maxMultipartParts\n\n this.useTags = useTags ?? this.useTags\n this.expirationPeriodInMilliseconds =\n expirationPeriodInMilliseconds ?? this.expirationPeriodInMilliseconds\n this.partUploadSemaphore = new Semaphore(maxConcurrentPartUploads ?? 60)\n\n // Initialize component managers\n this.metadataManager = new S3MetadataManager(\n this.client,\n this.bucket,\n this.shouldUseExpirationTags.bind(this),\n this.generateCompleteTag.bind(this)\n )\n\n this.fileOperations = new S3FileOperations(\n this.maxMultipartParts,\n this.maxUploadSize,\n this.minPartSize,\n this.partSize\n )\n\n this.partsManager = new S3PartsManager(\n this.client,\n this.bucket,\n this.minPartSize,\n this.partUploadSemaphore,\n this.metadataManager,\n this.fileOperations,\n this.generateCompleteTag.bind(this)\n )\n\n this.expirationManager = new S3ExpirationManager(\n this.client,\n this.bucket,\n this.expirationPeriodInMilliseconds,\n this.metadataManager.generateInfoKey.bind(this.metadataManager),\n this.metadataManager.generatePartKey.bind(this.metadataManager)\n )\n\n // Cleanup expired uploads when the store is initialized\n this.deleteExpired()\n }\n\n /**\n * Helper method to check if expiration tags should be used\n * @returns True if expiration tags should be used\n */\n protected shouldUseExpirationTags(): boolean {\n return this.expirationPeriodInMilliseconds !== 0 && this.useTags\n }\n\n /**\n * Generates a tag for marking complete/incomplete uploads\n * @param value - Either 'false' or 'true' to mark completion status\n * @returns The tag string or undefined if tags shouldn’t be used\n */\n protected generateCompleteTag(value: 'false' | 'true'): string | undefined {\n if (!this.shouldUseExpirationTags()) {\n return undefined\n }\n return `Tus-Completed=${value}`\n }\n\n /**\n * Creates a multipart upload on S3 attaching any metadata to it.\n * Also, a `${file_id}.info` file is created which holds some information\n * about the upload itself like: `upload-id`, `upload-length`, etc.\n * @param upload - The upload object to create\n * @return Promise that resolves to the created upload object with storage information\n */\n public async create(upload: Upload): Promise<Upload> {\n log(MediaCloudLogs.S3_STORE_MULTIPART_INIT)\n\n const request: AWS.CreateMultipartUploadCommandInput = {\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({\n id: upload.id,\n prefix: upload.metadata?.prefix ?? undefined,\n }),\n Metadata: { 'tus-version': TUS_RESUMABLE },\n }\n\n if (upload.metadata?.contentType) {\n request.ContentType = upload.metadata.contentType as string\n }\n\n if (upload.metadata?.cacheControl) {\n request.CacheControl = upload.metadata.cacheControl as string\n }\n\n if (this.acl) {\n request.ACL = this.acl as AWS.ObjectCannedACL\n }\n\n upload.creation_date = new Date().toISOString()\n\n const response = await this.client.createMultipartUpload(request)\n\n upload.storage = {\n type: 's3',\n bucket: this.bucket,\n path: response.Key as string,\n }\n await this.metadataManager.saveMetadata({\n upload,\n uploadId: response.UploadId as string,\n })\n log(MediaCloudLogs.S3_STORE_MULTIPART_CREATED)\n\n return upload\n }\n\n /**\n * Writes `buffer` to the file specified by the upload’s `id` at `offset`\n * @param readable - The readable stream to write\n * @param id - The upload ID\n * @param offset - The byte offset to write at\n * @return Promise that resolves to the new offset after writing the data\n */\n public async write(\n readable: stream.Readable,\n id: string,\n offset: number\n ): Promise<number> {\n const metadata = await this.metadataManager.getMetadata({ id })\n\n // TUS sends PATCH requests with an `upload-offset` header.\n // Offset the write by the offset in the PATCH request.\n const calculatedOffset = this.fileOperations.calculateOffsetFromParts({\n parts: await this.partsManager.retrieveParts({ id }),\n })\n const offsetDiff = offset - calculatedOffset\n const requestedOffset = offset\n\n let finalReadable = readable\n\n if (offsetDiff < 0) {\n throw ERRORS.FILE_WRITE_ERROR\n }\n\n // If the offset given in the PATCH request is higher than\n // the expected offset, we need to prepend an incomplete\n // part to the readable stream, if one exists.\n if (offsetDiff > 0) {\n const incompletePart = await this.partsManager.downloadIncompletePart({\n id,\n })\n\n if (!incompletePart) {\n throw ERRORS.FILE_WRITE_ERROR\n }\n\n if (incompletePart.size !== offsetDiff) {\n throw ERRORS.FILE_WRITE_ERROR\n }\n\n // Clear the incomplete part from S3 since it's going to be combined with the current request\n await this.partsManager.deleteIncompletePart({ id })\n\n // Adjust offset to account for the incomplete part\n offset = requestedOffset - incompletePart.size\n\n finalReadable = stream.Readable.from(\n (async function* () {\n yield* incompletePart.createReader({ cleanUpOnEnd: true })\n yield* readable\n })()\n )\n }\n\n const partNumber = this.fileOperations.calculatePartNumber({\n parts: await this.partsManager.retrieveParts({ id }),\n })\n\n const bytesUploaded = await this.partsManager.uploadParts({\n metadata,\n readStream: finalReadable,\n currentPartNumber: partNumber,\n offset,\n })\n\n // The size of the incomplete part should not be counted, because the\n // process of the incomplete part should be fully transparent to the user.\n const newOffset =\n requestedOffset + bytesUploaded - (offsetDiff > 0 ? offsetDiff : 0)\n\n // Check if the upload is complete\n if (metadata.file.size === newOffset) {\n try {\n const parts = await this.partsManager.retrieveParts({ id })\n await this.partsManager.finishMultipartUpload({ metadata, parts })\n\n // Update the metadata with completed state\n const completedUpload = new Upload({\n ...metadata.file,\n offset: newOffset,\n size: metadata.file.size,\n storage: metadata.file.storage,\n })\n\n await this.metadataManager.completeMetadata({ upload: completedUpload })\n } catch (error) {\n log(MediaCloudLogs.S3_STORE_UPLOAD_FAILED)\n throw error\n }\n }\n\n return newOffset\n }\n\n /**\n * Returns the current state of the upload, i.e how much data has been\n * uploaded and if the upload is complete.\n * @param id - The upload ID to retrieve\n * @returns Promise that resolves to the upload object with current offset and storage information\n */\n public async getUpload(id: string): Promise<Upload> {\n let metadata: TusUploadMetadata\n\n try {\n metadata = await this.metadataManager.getMetadata({ id })\n } catch (error) {\n if (\n error instanceof NoSuchKey ||\n error instanceof NotFound ||\n (error as AWSError)?.Code === 'NotFound' ||\n (error as AWSError)?.Code === 'NoSuchKey'\n ) {\n throw ERRORS.FILE_NOT_FOUND\n }\n throw error\n }\n\n let offset: number\n\n try {\n const parts = await this.partsManager.retrieveParts({ id })\n offset = this.fileOperations.calculateOffsetFromParts({ parts })\n } catch (error) {\n // Check if the error is caused by the upload not being found. This happens\n // when the multipart upload has already been completed or aborted.\n if (\n (error as AWSError)?.Code === 'NoSuchUpload' ||\n (error as AWSError)?.Code === 'NoSuchKey'\n ) {\n return new Upload({\n ...metadata.file,\n metadata: metadata.file.metadata,\n offset: metadata.file.size as number,\n size: metadata.file.size,\n storage: metadata.file.storage,\n })\n }\n\n log(MediaCloudLogs.S3_STORE_RETRIEVE_PARTS_ERROR)\n throw error\n }\n\n const incompletePartSize = await this.partsManager.getIncompletePartSize({\n id,\n })\n\n return new Upload({\n ...metadata.file,\n offset: offset + (incompletePartSize ?? 0),\n size: metadata.file.size,\n storage: metadata.file.storage,\n })\n }\n\n /**\n * Reads the file specified by the upload’s `id` and returns a readable stream\n * @param id - The upload ID to read\n * @returns Promise that resolves to a readable stream of the file's contents\n */\n async read(id: string): Promise<Readable> {\n log(MediaCloudLogs.S3_STORE_READ_ATTEMPT)\n let retries = 3\n let lastError: Error | null = null\n\n while (retries > 0) {\n try {\n const data = await this.client.getObject({\n Bucket: this.bucket,\n Key: id,\n })\n log(MediaCloudLogs.S3_STORE_READ_SUCCESS)\n return data.Body as Readable\n } catch (error) {\n log(MediaCloudLogs.S3_STORE_READ_RETRY)\n lastError = error as Error\n retries--\n\n if (retries > 0) {\n // Wait a bit before retrying, in case S3 needs time for consistency\n await new Promise((resolve) => setTimeout(resolve, 100))\n }\n }\n }\n\n log(MediaCloudLogs.S3_STORE_READ_FAILED)\n throw lastError ?? new Error(`Failed to read file ${id}`)\n }\n\n /**\n *\n * Moves the file specified by its `oldKey` to `newKey`.\n * @param oldKey - The current S3 key of the file to be moved\n * @param newKey - The new S3 key to move the file to\n * @return Promise that resolves when the file has been successfully moved, or rejects with an error if the move operation fails\n */\n public async copy(oldKey: string, newKey: string): Promise<void> {\n try {\n await this.client.copyObject({\n Bucket: this.bucket,\n CopySource: encodeURI(`${this.bucket}/${oldKey}`),\n Key: newKey,\n })\n\n await this.client.deleteObject({\n Bucket: this.bucket,\n Key: oldKey,\n })\n } catch (error) {\n if (\n (error as AWSError)?.code &&\n ['NoSuchKey', 'NoSuchUpload', 'NotFound'].includes(\n (error as AWSError).Code || ''\n )\n ) {\n log(MediaCloudLogs.S3_STORE_FILE_NOT_FOUND)\n throw ERRORS.FILE_NOT_FOUND\n }\n throw error as Error\n }\n }\n\n /**\n * Removes files specified by the upload’s `id`\n * @param id - The upload ID to remove\n * @returns Promise that resolves when the file and its metadata have been removed\n */\n public async remove(id: string): Promise<void> {\n try {\n const { 'upload-id': uploadId, file } =\n await this.metadataManager.getMetadata({ id })\n if (uploadId) {\n await this.client.abortMultipartUpload({\n Bucket: this.bucket,\n Key: id,\n UploadId: uploadId,\n })\n }\n\n await this.client.deleteObjects({\n Bucket: this.bucket,\n Delete: {\n Objects: [\n {\n Key: this.metadataManager.generatePartKey({\n id,\n prefix: file?.metadata?.prefix ?? undefined,\n }),\n },\n {\n Key: this.metadataManager.generatePartKey({\n id,\n prefix: file?.metadata?.prefix ?? undefined,\n isIncomplete: true,\n }),\n },\n {\n Key: this.metadataManager.generateInfoKey({ id }),\n },\n ],\n },\n })\n } catch (error) {\n if (\n (error as AWSError)?.code &&\n ['NoSuchKey', 'NoSuchUpload', 'NotFound'].includes(\n (error as AWSError).Code || ''\n )\n ) {\n log(MediaCloudLogs.S3_STORE_FILE_NOT_FOUND)\n throw ERRORS.FILE_NOT_FOUND\n }\n throw error\n }\n }\n\n /**\n * Removes the .info file specified by the upload’s `id`\n * @param id - The upload ID to clean up\n * @returns Promise that resolves when the metadata file has been removed\n */\n public async cleanup(id: string): Promise<void> {\n try {\n const { file } = await this.metadataManager.getMetadata({ id })\n\n await this.client.deleteObjects({\n Bucket: this.bucket,\n Delete: {\n Objects: [\n {\n Key: this.metadataManager.generatePartKey({\n id,\n prefix: file?.metadata?.prefix ?? undefined,\n isIncomplete: true,\n }),\n },\n {\n Key: this.metadataManager.generateInfoKey({ id }),\n },\n ],\n },\n })\n } catch (error) {\n if (\n (error as AWSError)?.code &&\n ['NoSuchKey', 'NoSuchUpload', 'NotFound'].includes(\n (error as AWSError).Code || ''\n )\n ) {\n log(MediaCloudLogs.S3_STORE_FILE_NOT_FOUND)\n throw ERRORS.FILE_NOT_FOUND\n }\n throw error\n }\n }\n\n /**\n * Combine all multipart uploads into a single object\n * @param id - The upload ID to complete\n * @returns Promise that resolves to the completed upload object with storage information\n */\n public async completeMultipartUpload(id: string): Promise<Upload> {\n const metadata = await this.metadataManager.getMetadata({ id })\n const parts = await this.partsManager.retrieveParts({ id })\n\n const incompletePartInfo = await this.partsManager.downloadIncompletePart({\n id,\n })\n\n if (incompletePartInfo) {\n // Upload the incomplete part as a regular part\n await this.partsManager.uploadPart({\n metadata,\n readStream: incompletePartInfo.createReader({ cleanUpOnEnd: true }),\n partNumber: parts.length + 1,\n })\n\n // Remove the incomplete part\n await this.partsManager.deleteIncompletePart({ id })\n\n // Re-fetch parts to include the newly uploaded part\n const updatedParts = await this.partsManager.retrieveParts({ id })\n await this.partsManager.finishMultipartUpload({\n metadata,\n parts: updatedParts,\n })\n } else {\n await this.partsManager.finishMultipartUpload({ metadata, parts })\n }\n\n const completedUpload = new Upload({\n ...metadata.file,\n offset: metadata.file.size ?? 0,\n size: metadata.file.size ?? 0,\n storage: metadata.file.storage,\n })\n\n await this.metadataManager.completeMetadata({ upload: completedUpload })\n\n return completedUpload\n }\n\n /**\n * Get the full S3 URL for an uploaded file\n * @param id - The upload ID to get the URL for\n * @returns The full URL to access the uploaded file on S3\n */\n public getUrl(id: string): string {\n // Use the custom endpoint if available\n if (this.customEndpoint) {\n return `${this.customEndpoint}/${this.bucket}/${id}`\n }\n\n // Fallback to standard AWS S3 URL format\n const regionConfig = this.client.config.region\n let region: string\n\n // If region is a function, we can't resolve it synchronously, so use a fallback\n if (typeof regionConfig === 'function') {\n region = 'us-east-1' // fallback for sync calls\n } else {\n region = regionConfig || 'us-east-1'\n }\n\n // Standard AWS S3 URL format\n if (region === 'us-east-1') {\n return `https://${this.bucket}.s3.amazonaws.com/${id}`\n } else {\n return `https://${this.bucket}.s3.${region}.amazonaws.com/${id}`\n }\n }\n\n /**\n * Deletes expired incomplete uploads.\n * Returns the number of deleted uploads.\n * @returns Promise that resolves to the number of deleted uploads\n */\n async deleteExpired(): Promise<number> {\n return this.expirationManager.deleteExpired()\n }\n\n /**\n * Returns the expiration period in milliseconds\n * @return The expiration period in milliseconds\n */\n getExpiration(): number {\n return this.expirationManager.getExpiration()\n }\n}\n"],"mappings":";;;;;;;;;;;;AAgBA,MAAM,EAAE,QAAQ,iBAAiB;AAEjC,IAAa,UAAb,cAA6B,UAAU;CAkBrC,YAAY,SAAwB;AAClC,SAAO;kBAhBS,IAAI,OAAO;qBACR,IAAI,OAAO;2BACL;uBACJ;iBACN;wCACuB;EAYtC,MAAM,EACJ,mBACA,aACA,UACA,gBACA,0BACA,SACA,mCACE;EACJ,MAAM,EAAE,KAAK,QAAQ,GAAG,uBAAuB;AAE/C,OAAK,aAAa;GAChB;GACA;GACA;GACA;GACA;GACD;AAED,OAAK,SAAS;AACd,OAAK,MAAM;AACX,OAAK,SAAS,IAAI,GAAG,mBAAmB;AACxC,OAAK,iBAAiB,OAAO,mBAAmB,SAAS;AAEzD,OAAK,WAAW,YAAY,KAAK;AACjC,OAAK,cAAc,eAAe,KAAK;AACvC,OAAK,oBAAoB,qBAAqB,KAAK;AAEnD,OAAK,UAAU,WAAW,KAAK;AAC/B,OAAK,iCACH,kCAAkC,KAAK;AACzC,OAAK,sBAAsB,IAAI,UAAU,4BAA4B,GAAG;AAGxE,OAAK,kBAAkB,IAAI,kBACzB,KAAK,QACL,KAAK,QACL,KAAK,wBAAwB,KAAK,KAAK,EACvC,KAAK,oBAAoB,KAAK,KAAK,CACpC;AAED,OAAK,iBAAiB,IAAI,iBACxB,KAAK,mBACL,KAAK,eACL,KAAK,aACL,KAAK,SACN;AAED,OAAK,eAAe,IAAI,eACtB,KAAK,QACL,KAAK,QACL,KAAK,aACL,KAAK,qBACL,KAAK,iBACL,KAAK,gBACL,KAAK,oBAAoB,KAAK,KAAK,CACpC;AAED,OAAK,oBAAoB,IAAI,oBAC3B,KAAK,QACL,KAAK,QACL,KAAK,gCACL,KAAK,gBAAgB,gBAAgB,KAAK,KAAK,gBAAgB,EAC/D,KAAK,gBAAgB,gBAAgB,KAAK,KAAK,gBAAgB,CAChE;AAGD,OAAK,eAAe;;;;;;CAOtB,AAAU,0BAAmC;AAC3C,SAAO,KAAK,mCAAmC,KAAK,KAAK;;;;;;;CAQ3D,AAAU,oBAAoB,OAA6C;AACzE,MAAI,CAAC,KAAK,yBAAyB,CACjC;AAEF,SAAO,iBAAiB;;;;;;;;;CAU1B,MAAa,OAAO,QAAiC;AACnD,MAAI,eAAe,wBAAwB;EAE3C,MAAMA,UAAiD;GACrD,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,gBAAgB;IACxC,IAAI,OAAO;IACX,QAAQ,OAAO,UAAU,UAAU;IACpC,CAAC;GACF,UAAU,EAAE,eAAe,eAAe;GAC3C;AAED,MAAI,OAAO,UAAU,YACnB,SAAQ,cAAc,OAAO,SAAS;AAGxC,MAAI,OAAO,UAAU,aACnB,SAAQ,eAAe,OAAO,SAAS;AAGzC,MAAI,KAAK,IACP,SAAQ,MAAM,KAAK;AAGrB,SAAO,iCAAgB,IAAI,MAAM,EAAC,aAAa;EAE/C,MAAM,WAAW,MAAM,KAAK,OAAO,sBAAsB,QAAQ;AAEjE,SAAO,UAAU;GACf,MAAM;GACN,QAAQ,KAAK;GACb,MAAM,SAAS;GAChB;AACD,QAAM,KAAK,gBAAgB,aAAa;GACtC;GACA,UAAU,SAAS;GACpB,CAAC;AACF,MAAI,eAAe,2BAA2B;AAE9C,SAAO;;;;;;;;;CAUT,MAAa,MACX,UACA,IACA,QACiB;EACjB,MAAM,WAAW,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;EAI/D,MAAM,mBAAmB,KAAK,eAAe,yBAAyB,EACpE,OAAO,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC,EACrD,CAAC;EACF,MAAM,aAAa,SAAS;EAC5B,MAAM,kBAAkB;EAExB,IAAI,gBAAgB;AAEpB,MAAI,aAAa,EACf,OAAM,OAAO;AAMf,MAAI,aAAa,GAAG;GAClB,MAAM,iBAAiB,MAAM,KAAK,aAAa,uBAAuB,EACpE,IACD,CAAC;AAEF,OAAI,CAAC,eACH,OAAM,OAAO;AAGf,OAAI,eAAe,SAAS,WAC1B,OAAM,OAAO;AAIf,SAAM,KAAK,aAAa,qBAAqB,EAAE,IAAI,CAAC;AAGpD,YAAS,kBAAkB,eAAe;AAE1C,mBAAgB,OAAO,SAAS,MAC7B,mBAAmB;AAClB,WAAO,eAAe,aAAa,EAAE,cAAc,MAAM,CAAC;AAC1D,WAAO;OACL,CACL;;EAGH,MAAM,aAAa,KAAK,eAAe,oBAAoB,EACzD,OAAO,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC,EACrD,CAAC;EAWF,MAAM,YACJ,kBAVoB,MAAM,KAAK,aAAa,YAAY;GACxD;GACA,YAAY;GACZ,mBAAmB;GACnB;GACD,CAAC,IAKmC,aAAa,IAAI,aAAa;AAGnE,MAAI,SAAS,KAAK,SAAS,UACzB,KAAI;GACF,MAAM,QAAQ,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC;AAC3D,SAAM,KAAK,aAAa,sBAAsB;IAAE;IAAU;IAAO,CAAC;GAGlE,MAAM,kBAAkB,IAAI,OAAO;IACjC,GAAG,SAAS;IACZ,QAAQ;IACR,MAAM,SAAS,KAAK;IACpB,SAAS,SAAS,KAAK;IACxB,CAAC;AAEF,SAAM,KAAK,gBAAgB,iBAAiB,EAAE,QAAQ,iBAAiB,CAAC;WACjE,OAAO;AACd,OAAI,eAAe,uBAAuB;AAC1C,SAAM;;AAIV,SAAO;;;;;;;;CAST,MAAa,UAAU,IAA6B;EAClD,IAAIC;AAEJ,MAAI;AACF,cAAW,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;WAClD,OAAO;AACd,OACE,iBAAiB,aACjB,iBAAiB,YAChB,OAAoB,SAAS,cAC7B,OAAoB,SAAS,YAE9B,OAAM,OAAO;AAEf,SAAM;;EAGR,IAAIC;AAEJ,MAAI;GACF,MAAM,QAAQ,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC;AAC3D,YAAS,KAAK,eAAe,yBAAyB,EAAE,OAAO,CAAC;WACzD,OAAO;AAGd,OACG,OAAoB,SAAS,kBAC7B,OAAoB,SAAS,YAE9B,QAAO,IAAI,OAAO;IAChB,GAAG,SAAS;IACZ,UAAU,SAAS,KAAK;IACxB,QAAQ,SAAS,KAAK;IACtB,MAAM,SAAS,KAAK;IACpB,SAAS,SAAS,KAAK;IACxB,CAAC;AAGJ,OAAI,eAAe,8BAA8B;AACjD,SAAM;;EAGR,MAAM,qBAAqB,MAAM,KAAK,aAAa,sBAAsB,EACvE,IACD,CAAC;AAEF,SAAO,IAAI,OAAO;GAChB,GAAG,SAAS;GACZ,QAAQ,UAAU,sBAAsB;GACxC,MAAM,SAAS,KAAK;GACpB,SAAS,SAAS,KAAK;GACxB,CAAC;;;;;;;CAQJ,MAAM,KAAK,IAA+B;AACxC,MAAI,eAAe,sBAAsB;EACzC,IAAI,UAAU;EACd,IAAIC,YAA0B;AAE9B,SAAO,UAAU,EACf,KAAI;GACF,MAAM,OAAO,MAAM,KAAK,OAAO,UAAU;IACvC,QAAQ,KAAK;IACb,KAAK;IACN,CAAC;AACF,OAAI,eAAe,sBAAsB;AACzC,UAAO,KAAK;WACL,OAAO;AACd,OAAI,eAAe,oBAAoB;AACvC,eAAY;AACZ;AAEA,OAAI,UAAU,EAEZ,OAAM,IAAI,SAAS,YAAY,WAAW,SAAS,IAAI,CAAC;;AAK9D,MAAI,eAAe,qBAAqB;AACxC,QAAM,6BAAa,IAAI,MAAM,uBAAuB,KAAK;;;;;;;;;CAU3D,MAAa,KAAK,QAAgB,QAA+B;AAC/D,MAAI;AACF,SAAM,KAAK,OAAO,WAAW;IAC3B,QAAQ,KAAK;IACb,YAAY,UAAU,GAAG,KAAK,OAAO,GAAG,SAAS;IACjD,KAAK;IACN,CAAC;AAEF,SAAM,KAAK,OAAO,aAAa;IAC7B,QAAQ,KAAK;IACb,KAAK;IACN,CAAC;WACK,OAAO;AACd,OACG,OAAoB,QACrB;IAAC;IAAa;IAAgB;IAAW,CAAC,SACvC,MAAmB,QAAQ,GAC7B,EACD;AACA,QAAI,eAAe,wBAAwB;AAC3C,UAAM,OAAO;;AAEf,SAAM;;;;;;;;CASV,MAAa,OAAO,IAA2B;AAC7C,MAAI;GACF,MAAM,EAAE,aAAa,UAAU,SAC7B,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;AAChD,OAAI,SACF,OAAM,KAAK,OAAO,qBAAqB;IACrC,QAAQ,KAAK;IACb,KAAK;IACL,UAAU;IACX,CAAC;AAGJ,SAAM,KAAK,OAAO,cAAc;IAC9B,QAAQ,KAAK;IACb,QAAQ,EACN,SAAS;KACP,EACE,KAAK,KAAK,gBAAgB,gBAAgB;MACxC;MACA,QAAQ,MAAM,UAAU,UAAU;MACnC,CAAC,EACH;KACD,EACE,KAAK,KAAK,gBAAgB,gBAAgB;MACxC;MACA,QAAQ,MAAM,UAAU,UAAU;MAClC,cAAc;MACf,CAAC,EACH;KACD,EACE,KAAK,KAAK,gBAAgB,gBAAgB,EAAE,IAAI,CAAC,EAClD;KACF,EACF;IACF,CAAC;WACK,OAAO;AACd,OACG,OAAoB,QACrB;IAAC;IAAa;IAAgB;IAAW,CAAC,SACvC,MAAmB,QAAQ,GAC7B,EACD;AACA,QAAI,eAAe,wBAAwB;AAC3C,UAAM,OAAO;;AAEf,SAAM;;;;;;;;CASV,MAAa,QAAQ,IAA2B;AAC9C,MAAI;GACF,MAAM,EAAE,SAAS,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;AAE/D,SAAM,KAAK,OAAO,cAAc;IAC9B,QAAQ,KAAK;IACb,QAAQ,EACN,SAAS,CACP,EACE,KAAK,KAAK,gBAAgB,gBAAgB;KACxC;KACA,QAAQ,MAAM,UAAU,UAAU;KAClC,cAAc;KACf,CAAC,EACH,EACD,EACE,KAAK,KAAK,gBAAgB,gBAAgB,EAAE,IAAI,CAAC,EAClD,CACF,EACF;IACF,CAAC;WACK,OAAO;AACd,OACG,OAAoB,QACrB;IAAC;IAAa;IAAgB;IAAW,CAAC,SACvC,MAAmB,QAAQ,GAC7B,EACD;AACA,QAAI,eAAe,wBAAwB;AAC3C,UAAM,OAAO;;AAEf,SAAM;;;;;;;;CASV,MAAa,wBAAwB,IAA6B;EAChE,MAAM,WAAW,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;EAC/D,MAAM,QAAQ,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC;EAE3D,MAAM,qBAAqB,MAAM,KAAK,aAAa,uBAAuB,EACxE,IACD,CAAC;AAEF,MAAI,oBAAoB;AAEtB,SAAM,KAAK,aAAa,WAAW;IACjC;IACA,YAAY,mBAAmB,aAAa,EAAE,cAAc,MAAM,CAAC;IACnE,YAAY,MAAM,SAAS;IAC5B,CAAC;AAGF,SAAM,KAAK,aAAa,qBAAqB,EAAE,IAAI,CAAC;GAGpD,MAAM,eAAe,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC;AAClE,SAAM,KAAK,aAAa,sBAAsB;IAC5C;IACA,OAAO;IACR,CAAC;QAEF,OAAM,KAAK,aAAa,sBAAsB;GAAE;GAAU;GAAO,CAAC;EAGpE,MAAM,kBAAkB,IAAI,OAAO;GACjC,GAAG,SAAS;GACZ,QAAQ,SAAS,KAAK,QAAQ;GAC9B,MAAM,SAAS,KAAK,QAAQ;GAC5B,SAAS,SAAS,KAAK;GACxB,CAAC;AAEF,QAAM,KAAK,gBAAgB,iBAAiB,EAAE,QAAQ,iBAAiB,CAAC;AAExE,SAAO;;;;;;;CAQT,AAAO,OAAO,IAAoB;AAEhC,MAAI,KAAK,eACP,QAAO,GAAG,KAAK,eAAe,GAAG,KAAK,OAAO,GAAG;EAIlD,MAAM,eAAe,KAAK,OAAO,OAAO;EACxC,IAAIC;AAGJ,MAAI,OAAO,iBAAiB,WAC1B,UAAS;MAET,UAAS,gBAAgB;AAI3B,MAAI,WAAW,YACb,QAAO,WAAW,KAAK,OAAO,oBAAoB;MAElD,QAAO,WAAW,KAAK,OAAO,MAAM,OAAO,iBAAiB;;;;;;;CAShE,MAAM,gBAAiC;AACrC,SAAO,KAAK,kBAAkB,eAAe;;;;;;CAO/C,gBAAwB;AACtB,SAAO,KAAK,kBAAkB,eAAe"}
|
|
1
|
+
{"version":3,"file":"s3Store.mjs","names":["mappedS3ClientConfig: AWS.S3ClientConfig","HttpsAgent","request: AWS.CreateMultipartUploadCommandInput","metadata: TusUploadMetadata","offset: number","lastError: Error | null","region: string"],"sources":["../../../../src/tus/stores/s3/s3Store.ts"],"sourcesContent":["import { NoSuchKey, NotFound, S3 } from '@aws-sdk/client-s3'\nimport { NodeHttpHandler } from '@smithy/node-http-handler'\nimport { Agent as HttpsAgent } from 'https'\nimport { DataStore, ERRORS, TUS_RESUMABLE, Upload } from '@tus/utils'\nimport stream, { type Readable } from 'node:stream'\n\nimport { useErrorHandler } from '../../../hooks/useErrorHandler'\nimport { Semaphore } from './semaphore'\nimport { S3ExpirationManager } from './expirationManager'\nimport { S3FileOperations } from './fileOperations'\nimport { S3MetadataManager } from './metadataManager'\nimport { S3PartsManager } from './partsManager'\n\nimport { MediaCloudLogs } from '../../../types/errors'\n\nimport type AWS from '@aws-sdk/client-s3'\nimport type { AWSError, S3StoreConfig, TusUploadMetadata } from '../../../types'\n\nconst { log } = useErrorHandler()\n\nexport class S3Store extends DataStore {\n public client: S3\n public bucket: string\n public partSize = 8 * 1024 * 1024 // 8MB preferred part size\n public minPartSize = 5 * 1024 * 1024 // 5MB minimum part size\n public maxMultipartParts = 10_000\n public maxUploadSize = 5_497_558_138_880 as const // 5TiB\n public useTags = false\n public expirationPeriodInMilliseconds = 0\n protected acl?: string\n\n protected partUploadSemaphore: Semaphore\n protected metadataManager: S3MetadataManager\n protected fileOperations: S3FileOperations\n protected partsManager: S3PartsManager\n protected expirationManager: S3ExpirationManager\n protected customEndpoint: string\n\n constructor(options: S3StoreConfig) {\n super()\n const {\n maxMultipartParts,\n minPartSize,\n partSize,\n s3ClientConfig,\n maxConcurrentPartUploads,\n useTags,\n expirationPeriodInMilliseconds,\n } = options\n const { acl, bucket, ...restS3ClientConfig } = s3ClientConfig\n\n this.extensions = [\n 'creation',\n 'creation-with-upload',\n 'creation-defer-length',\n 'termination',\n 'expiration',\n ]\n\n const mappedS3ClientConfig: AWS.S3ClientConfig = {\n ...restS3ClientConfig,\n requestHandler: new NodeHttpHandler({\n connectionTimeout: 60000,\n requestTimeout: 300000,\n httpsAgent: new HttpsAgent({\n maxSockets: 1000,\n keepAlive: true,\n keepAliveMsecs: 60000,\n }),\n }),\n }\n\n this.bucket = bucket\n this.acl = acl\n this.client = new S3(mappedS3ClientConfig)\n this.customEndpoint = String(restS3ClientConfig.endpoint)\n\n this.partSize = partSize ?? this.partSize\n this.minPartSize = minPartSize ?? this.minPartSize\n this.maxMultipartParts = maxMultipartParts ?? this.maxMultipartParts\n\n this.useTags = useTags ?? this.useTags\n this.expirationPeriodInMilliseconds =\n expirationPeriodInMilliseconds ?? this.expirationPeriodInMilliseconds\n this.partUploadSemaphore = new Semaphore(maxConcurrentPartUploads ?? 60)\n\n // Initialize component managers\n this.metadataManager = new S3MetadataManager(\n this.client,\n this.bucket,\n this.shouldUseExpirationTags.bind(this),\n this.generateCompleteTag.bind(this)\n )\n\n this.fileOperations = new S3FileOperations(\n this.maxMultipartParts,\n this.maxUploadSize,\n this.minPartSize,\n this.partSize\n )\n\n this.partsManager = new S3PartsManager(\n this.client,\n this.bucket,\n this.minPartSize,\n this.partUploadSemaphore,\n this.metadataManager,\n this.fileOperations,\n this.generateCompleteTag.bind(this)\n )\n\n this.expirationManager = new S3ExpirationManager(\n this.client,\n this.bucket,\n this.expirationPeriodInMilliseconds,\n this.metadataManager.generateInfoKey.bind(this.metadataManager),\n this.metadataManager.generatePartKey.bind(this.metadataManager)\n )\n\n // Cleanup expired uploads when the store is initialized\n this.deleteExpired()\n }\n\n /**\n * Helper method to check if expiration tags should be used\n * @returns True if expiration tags should be used\n */\n protected shouldUseExpirationTags(): boolean {\n return this.expirationPeriodInMilliseconds !== 0 && this.useTags\n }\n\n /**\n * Generates a tag for marking complete/incomplete uploads\n * @param value - Either 'false' or 'true' to mark completion status\n * @returns The tag string or undefined if tags shouldn’t be used\n */\n protected generateCompleteTag(value: 'false' | 'true'): string | undefined {\n if (!this.shouldUseExpirationTags()) {\n return undefined\n }\n return `Tus-Completed=${value}`\n }\n\n /**\n * Creates a multipart upload on S3 attaching any metadata to it.\n * Also, a `${file_id}.info` file is created which holds some information\n * about the upload itself like: `upload-id`, `upload-length`, etc.\n * @param upload - The upload object to create\n * @return Promise that resolves to the created upload object with storage information\n */\n public async create(upload: Upload): Promise<Upload> {\n log(MediaCloudLogs.S3_STORE_MULTIPART_INIT)\n\n const request: AWS.CreateMultipartUploadCommandInput = {\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({\n id: upload.id,\n prefix: upload.metadata?.prefix ?? undefined,\n }),\n Metadata: { 'tus-version': TUS_RESUMABLE },\n }\n\n if (upload.metadata?.contentType) {\n request.ContentType = upload.metadata.contentType as string\n }\n\n if (upload.metadata?.cacheControl) {\n request.CacheControl = upload.metadata.cacheControl as string\n }\n\n if (this.acl) {\n request.ACL = this.acl as AWS.ObjectCannedACL\n }\n\n upload.creation_date = new Date().toISOString()\n\n const response = await this.client.createMultipartUpload(request)\n\n upload.storage = {\n type: 's3',\n bucket: this.bucket,\n path: response.Key as string,\n }\n await this.metadataManager.saveMetadata({\n upload,\n uploadId: response.UploadId as string,\n })\n log(MediaCloudLogs.S3_STORE_MULTIPART_CREATED)\n\n return upload\n }\n\n /**\n * Writes `buffer` to the file specified by the upload’s `id` at `offset`\n * @param readable - The readable stream to write\n * @param id - The upload ID\n * @param offset - The byte offset to write at\n * @return Promise that resolves to the new offset after writing the data\n */\n public async write(\n readable: stream.Readable,\n id: string,\n offset: number\n ): Promise<number> {\n const metadata = await this.metadataManager.getMetadata({ id })\n\n // TUS sends PATCH requests with an `upload-offset` header.\n // Offset the write by the offset in the PATCH request.\n const calculatedOffset = this.fileOperations.calculateOffsetFromParts({\n parts: await this.partsManager.retrieveParts({ id }),\n })\n const offsetDiff = offset - calculatedOffset\n const requestedOffset = offset\n\n let finalReadable = readable\n\n if (offsetDiff < 0) {\n throw ERRORS.FILE_WRITE_ERROR\n }\n\n // If the offset given in the PATCH request is higher than\n // the expected offset, we need to prepend an incomplete\n // part to the readable stream, if one exists.\n if (offsetDiff > 0) {\n const incompletePart = await this.partsManager.downloadIncompletePart({\n id,\n })\n\n if (!incompletePart) {\n throw ERRORS.FILE_WRITE_ERROR\n }\n\n if (incompletePart.size !== offsetDiff) {\n throw ERRORS.FILE_WRITE_ERROR\n }\n\n // Clear the incomplete part from S3 since it's going to be combined with the current request\n await this.partsManager.deleteIncompletePart({ id })\n\n // Adjust offset to account for the incomplete part\n offset = requestedOffset - incompletePart.size\n\n finalReadable = stream.Readable.from(\n (async function* () {\n yield* incompletePart.createReader({ cleanUpOnEnd: true })\n yield* readable\n })()\n )\n }\n\n const partNumber = this.fileOperations.calculatePartNumber({\n parts: await this.partsManager.retrieveParts({ id }),\n })\n\n const bytesUploaded = await this.partsManager.uploadParts({\n metadata,\n readStream: finalReadable,\n currentPartNumber: partNumber,\n offset,\n })\n\n // The size of the incomplete part should not be counted, because the\n // process of the incomplete part should be fully transparent to the user.\n const newOffset =\n requestedOffset + bytesUploaded - (offsetDiff > 0 ? offsetDiff : 0)\n\n // Check if the upload is complete\n if (metadata.file.size === newOffset) {\n try {\n const parts = await this.partsManager.retrieveParts({ id })\n await this.partsManager.finishMultipartUpload({ metadata, parts })\n\n // Update the metadata with completed state\n const completedUpload = new Upload({\n ...metadata.file,\n offset: newOffset,\n size: metadata.file.size,\n storage: metadata.file.storage,\n })\n\n await this.metadataManager.completeMetadata({ upload: completedUpload })\n } catch (error) {\n log(MediaCloudLogs.S3_STORE_UPLOAD_FAILED)\n throw error\n }\n }\n\n return newOffset\n }\n\n /**\n * Returns the current state of the upload, i.e how much data has been\n * uploaded and if the upload is complete.\n * @param id - The upload ID to retrieve\n * @returns Promise that resolves to the upload object with current offset and storage information\n */\n public async getUpload(id: string): Promise<Upload> {\n let metadata: TusUploadMetadata\n\n try {\n metadata = await this.metadataManager.getMetadata({ id })\n } catch (error) {\n if (\n error instanceof NoSuchKey ||\n error instanceof NotFound ||\n (error as AWSError)?.Code === 'NotFound' ||\n (error as AWSError)?.Code === 'NoSuchKey'\n ) {\n throw ERRORS.FILE_NOT_FOUND\n }\n throw error\n }\n\n let offset: number\n\n try {\n const parts = await this.partsManager.retrieveParts({ id })\n offset = this.fileOperations.calculateOffsetFromParts({ parts })\n } catch (error) {\n // Check if the error is caused by the upload not being found. This happens\n // when the multipart upload has already been completed or aborted.\n if (\n (error as AWSError)?.Code === 'NoSuchUpload' ||\n (error as AWSError)?.Code === 'NoSuchKey'\n ) {\n return new Upload({\n ...metadata.file,\n metadata: metadata.file.metadata,\n offset: metadata.file.size as number,\n size: metadata.file.size,\n storage: metadata.file.storage,\n })\n }\n\n log(MediaCloudLogs.S3_STORE_RETRIEVE_PARTS_ERROR)\n throw error\n }\n\n const incompletePartSize = await this.partsManager.getIncompletePartSize({\n id,\n })\n\n return new Upload({\n ...metadata.file,\n offset: offset + (incompletePartSize ?? 0),\n size: metadata.file.size,\n storage: metadata.file.storage,\n })\n }\n\n /**\n * Reads the file specified by the upload’s `id` and returns a readable stream\n * @param id - The upload ID to read\n * @returns Promise that resolves to a readable stream of the file's contents\n */\n async read(id: string): Promise<Readable> {\n log(MediaCloudLogs.S3_STORE_READ_ATTEMPT)\n let retries = 3\n let lastError: Error | null = null\n\n while (retries > 0) {\n try {\n const data = await this.client.getObject({\n Bucket: this.bucket,\n Key: id,\n })\n log(MediaCloudLogs.S3_STORE_READ_SUCCESS)\n return data.Body as Readable\n } catch (error) {\n log(MediaCloudLogs.S3_STORE_READ_RETRY)\n lastError = error as Error\n retries--\n\n if (retries > 0) {\n // Wait a bit before retrying, in case S3 needs time for consistency\n await new Promise((resolve) => setTimeout(resolve, 100))\n }\n }\n }\n\n log(MediaCloudLogs.S3_STORE_READ_FAILED)\n throw lastError ?? new Error(`Failed to read file ${id}`)\n }\n\n /**\n *\n * Moves the file specified by its `oldKey` to `newKey`.\n * @param oldKey - The current S3 key of the file to be moved\n * @param newKey - The new S3 key to move the file to\n * @return Promise that resolves when the file has been successfully moved, or rejects with an error if the move operation fails\n */\n public async copy(oldKey: string, newKey: string): Promise<void> {\n try {\n await this.client.copyObject({\n Bucket: this.bucket,\n CopySource: encodeURI(`${this.bucket}/${oldKey}`),\n Key: newKey,\n })\n\n await this.client.deleteObject({\n Bucket: this.bucket,\n Key: oldKey,\n })\n } catch (error) {\n if (\n (error as AWSError)?.code &&\n ['NoSuchKey', 'NoSuchUpload', 'NotFound'].includes(\n (error as AWSError).Code || ''\n )\n ) {\n log(MediaCloudLogs.S3_STORE_FILE_NOT_FOUND)\n throw ERRORS.FILE_NOT_FOUND\n }\n throw error as Error\n }\n }\n\n /**\n * Removes files specified by the upload’s `id`\n * @param id - The upload ID to remove\n * @returns Promise that resolves when the file and its metadata have been removed\n */\n public async remove(id: string): Promise<void> {\n try {\n const { 'upload-id': uploadId, file } =\n await this.metadataManager.getMetadata({ id })\n if (uploadId) {\n await this.client.abortMultipartUpload({\n Bucket: this.bucket,\n Key: id,\n UploadId: uploadId,\n })\n }\n\n await this.client.deleteObjects({\n Bucket: this.bucket,\n Delete: {\n Objects: [\n {\n Key: this.metadataManager.generatePartKey({\n id,\n prefix: file?.metadata?.prefix ?? undefined,\n }),\n },\n {\n Key: this.metadataManager.generatePartKey({\n id,\n prefix: file?.metadata?.prefix ?? undefined,\n isIncomplete: true,\n }),\n },\n {\n Key: this.metadataManager.generateInfoKey({ id }),\n },\n ],\n },\n })\n } catch (error) {\n if (\n (error as AWSError)?.code &&\n ['NoSuchKey', 'NoSuchUpload', 'NotFound'].includes(\n (error as AWSError).Code || ''\n )\n ) {\n log(MediaCloudLogs.S3_STORE_FILE_NOT_FOUND)\n throw ERRORS.FILE_NOT_FOUND\n }\n throw error\n }\n }\n\n /**\n * Removes the .info file specified by the upload’s `id`\n * @param id - The upload ID to clean up\n * @returns Promise that resolves when the metadata file has been removed\n */\n public async cleanup(id: string): Promise<void> {\n try {\n const { file } = await this.metadataManager.getMetadata({ id })\n\n await this.client.deleteObjects({\n Bucket: this.bucket,\n Delete: {\n Objects: [\n {\n Key: this.metadataManager.generatePartKey({\n id,\n prefix: file?.metadata?.prefix ?? undefined,\n isIncomplete: true,\n }),\n },\n {\n Key: this.metadataManager.generateInfoKey({ id }),\n },\n ],\n },\n })\n } catch (error) {\n if (\n (error as AWSError)?.code &&\n ['NoSuchKey', 'NoSuchUpload', 'NotFound'].includes(\n (error as AWSError).Code || ''\n )\n ) {\n log(MediaCloudLogs.S3_STORE_FILE_NOT_FOUND)\n throw ERRORS.FILE_NOT_FOUND\n }\n throw error\n }\n }\n\n /**\n * Combine all multipart uploads into a single object\n * @param id - The upload ID to complete\n * @returns Promise that resolves to the completed upload object with storage information\n */\n public async completeMultipartUpload(id: string): Promise<Upload> {\n const metadata = await this.metadataManager.getMetadata({ id })\n const parts = await this.partsManager.retrieveParts({ id })\n\n const incompletePartInfo = await this.partsManager.downloadIncompletePart({\n id,\n })\n\n if (incompletePartInfo) {\n // Upload the incomplete part as a regular part\n await this.partsManager.uploadPart({\n metadata,\n readStream: incompletePartInfo.createReader({ cleanUpOnEnd: true }),\n partNumber: parts.length + 1,\n })\n\n // Remove the incomplete part\n await this.partsManager.deleteIncompletePart({ id })\n\n // Re-fetch parts to include the newly uploaded part\n const updatedParts = await this.partsManager.retrieveParts({ id })\n await this.partsManager.finishMultipartUpload({\n metadata,\n parts: updatedParts,\n })\n } else {\n await this.partsManager.finishMultipartUpload({ metadata, parts })\n }\n\n const completedUpload = new Upload({\n ...metadata.file,\n offset: metadata.file.size ?? 0,\n size: metadata.file.size ?? 0,\n storage: metadata.file.storage,\n })\n\n await this.metadataManager.completeMetadata({ upload: completedUpload })\n\n return completedUpload\n }\n\n /**\n * Get the full S3 URL for an uploaded file\n * @param id - The upload ID to get the URL for\n * @returns The full URL to access the uploaded file on S3\n */\n public getUrl(id: string): string {\n // Use the custom endpoint if available\n if (this.customEndpoint) {\n return `${this.customEndpoint}/${this.bucket}/${id}`\n }\n\n // Fallback to standard AWS S3 URL format\n const regionConfig = this.client.config.region\n let region: string\n\n // If region is a function, we can't resolve it synchronously, so use a fallback\n if (typeof regionConfig === 'function') {\n region = 'us-east-1' // fallback for sync calls\n } else {\n region = regionConfig || 'us-east-1'\n }\n\n // Standard AWS S3 URL format\n if (region === 'us-east-1') {\n return `https://${this.bucket}.s3.amazonaws.com/${id}`\n } else {\n return `https://${this.bucket}.s3.${region}.amazonaws.com/${id}`\n }\n }\n\n /**\n * Deletes expired incomplete uploads.\n * Returns the number of deleted uploads.\n * @returns Promise that resolves to the number of deleted uploads\n */\n async deleteExpired(): Promise<number> {\n return this.expirationManager.deleteExpired()\n }\n\n /**\n * Returns the expiration period in milliseconds\n * @return The expiration period in milliseconds\n */\n getExpiration(): number {\n return this.expirationManager.getExpiration()\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AAkBA,MAAM,EAAE,QAAQ,iBAAiB;AAEjC,IAAa,UAAb,cAA6B,UAAU;CAkBrC,YAAY,SAAwB;AAClC,SAAO;kBAhBS,IAAI,OAAO;qBACR,IAAI,OAAO;2BACL;uBACJ;iBACN;wCACuB;EAYtC,MAAM,EACJ,mBACA,aACA,UACA,gBACA,0BACA,SACA,mCACE;EACJ,MAAM,EAAE,KAAK,QAAQ,GAAG,uBAAuB;AAE/C,OAAK,aAAa;GAChB;GACA;GACA;GACA;GACA;GACD;EAED,MAAMA,uBAA2C;GAC/C,GAAG;GACH,gBAAgB,IAAI,gBAAgB;IAClC,mBAAmB;IACnB,gBAAgB;IAChB,YAAY,IAAIC,MAAW;KACzB,YAAY;KACZ,WAAW;KACX,gBAAgB;KACjB,CAAC;IACH,CAAC;GACH;AAED,OAAK,SAAS;AACd,OAAK,MAAM;AACX,OAAK,SAAS,IAAI,GAAG,qBAAqB;AAC1C,OAAK,iBAAiB,OAAO,mBAAmB,SAAS;AAEzD,OAAK,WAAW,YAAY,KAAK;AACjC,OAAK,cAAc,eAAe,KAAK;AACvC,OAAK,oBAAoB,qBAAqB,KAAK;AAEnD,OAAK,UAAU,WAAW,KAAK;AAC/B,OAAK,iCACH,kCAAkC,KAAK;AACzC,OAAK,sBAAsB,IAAI,UAAU,4BAA4B,GAAG;AAGxE,OAAK,kBAAkB,IAAI,kBACzB,KAAK,QACL,KAAK,QACL,KAAK,wBAAwB,KAAK,KAAK,EACvC,KAAK,oBAAoB,KAAK,KAAK,CACpC;AAED,OAAK,iBAAiB,IAAI,iBACxB,KAAK,mBACL,KAAK,eACL,KAAK,aACL,KAAK,SACN;AAED,OAAK,eAAe,IAAI,eACtB,KAAK,QACL,KAAK,QACL,KAAK,aACL,KAAK,qBACL,KAAK,iBACL,KAAK,gBACL,KAAK,oBAAoB,KAAK,KAAK,CACpC;AAED,OAAK,oBAAoB,IAAI,oBAC3B,KAAK,QACL,KAAK,QACL,KAAK,gCACL,KAAK,gBAAgB,gBAAgB,KAAK,KAAK,gBAAgB,EAC/D,KAAK,gBAAgB,gBAAgB,KAAK,KAAK,gBAAgB,CAChE;AAGD,OAAK,eAAe;;;;;;CAOtB,AAAU,0BAAmC;AAC3C,SAAO,KAAK,mCAAmC,KAAK,KAAK;;;;;;;CAQ3D,AAAU,oBAAoB,OAA6C;AACzE,MAAI,CAAC,KAAK,yBAAyB,CACjC;AAEF,SAAO,iBAAiB;;;;;;;;;CAU1B,MAAa,OAAO,QAAiC;AACnD,MAAI,eAAe,wBAAwB;EAE3C,MAAMC,UAAiD;GACrD,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,gBAAgB;IACxC,IAAI,OAAO;IACX,QAAQ,OAAO,UAAU,UAAU;IACpC,CAAC;GACF,UAAU,EAAE,eAAe,eAAe;GAC3C;AAED,MAAI,OAAO,UAAU,YACnB,SAAQ,cAAc,OAAO,SAAS;AAGxC,MAAI,OAAO,UAAU,aACnB,SAAQ,eAAe,OAAO,SAAS;AAGzC,MAAI,KAAK,IACP,SAAQ,MAAM,KAAK;AAGrB,SAAO,iCAAgB,IAAI,MAAM,EAAC,aAAa;EAE/C,MAAM,WAAW,MAAM,KAAK,OAAO,sBAAsB,QAAQ;AAEjE,SAAO,UAAU;GACf,MAAM;GACN,QAAQ,KAAK;GACb,MAAM,SAAS;GAChB;AACD,QAAM,KAAK,gBAAgB,aAAa;GACtC;GACA,UAAU,SAAS;GACpB,CAAC;AACF,MAAI,eAAe,2BAA2B;AAE9C,SAAO;;;;;;;;;CAUT,MAAa,MACX,UACA,IACA,QACiB;EACjB,MAAM,WAAW,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;EAI/D,MAAM,mBAAmB,KAAK,eAAe,yBAAyB,EACpE,OAAO,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC,EACrD,CAAC;EACF,MAAM,aAAa,SAAS;EAC5B,MAAM,kBAAkB;EAExB,IAAI,gBAAgB;AAEpB,MAAI,aAAa,EACf,OAAM,OAAO;AAMf,MAAI,aAAa,GAAG;GAClB,MAAM,iBAAiB,MAAM,KAAK,aAAa,uBAAuB,EACpE,IACD,CAAC;AAEF,OAAI,CAAC,eACH,OAAM,OAAO;AAGf,OAAI,eAAe,SAAS,WAC1B,OAAM,OAAO;AAIf,SAAM,KAAK,aAAa,qBAAqB,EAAE,IAAI,CAAC;AAGpD,YAAS,kBAAkB,eAAe;AAE1C,mBAAgB,OAAO,SAAS,MAC7B,mBAAmB;AAClB,WAAO,eAAe,aAAa,EAAE,cAAc,MAAM,CAAC;AAC1D,WAAO;OACL,CACL;;EAGH,MAAM,aAAa,KAAK,eAAe,oBAAoB,EACzD,OAAO,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC,EACrD,CAAC;EAWF,MAAM,YACJ,kBAVoB,MAAM,KAAK,aAAa,YAAY;GACxD;GACA,YAAY;GACZ,mBAAmB;GACnB;GACD,CAAC,IAKmC,aAAa,IAAI,aAAa;AAGnE,MAAI,SAAS,KAAK,SAAS,UACzB,KAAI;GACF,MAAM,QAAQ,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC;AAC3D,SAAM,KAAK,aAAa,sBAAsB;IAAE;IAAU;IAAO,CAAC;GAGlE,MAAM,kBAAkB,IAAI,OAAO;IACjC,GAAG,SAAS;IACZ,QAAQ;IACR,MAAM,SAAS,KAAK;IACpB,SAAS,SAAS,KAAK;IACxB,CAAC;AAEF,SAAM,KAAK,gBAAgB,iBAAiB,EAAE,QAAQ,iBAAiB,CAAC;WACjE,OAAO;AACd,OAAI,eAAe,uBAAuB;AAC1C,SAAM;;AAIV,SAAO;;;;;;;;CAST,MAAa,UAAU,IAA6B;EAClD,IAAIC;AAEJ,MAAI;AACF,cAAW,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;WAClD,OAAO;AACd,OACE,iBAAiB,aACjB,iBAAiB,YAChB,OAAoB,SAAS,cAC7B,OAAoB,SAAS,YAE9B,OAAM,OAAO;AAEf,SAAM;;EAGR,IAAIC;AAEJ,MAAI;GACF,MAAM,QAAQ,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC;AAC3D,YAAS,KAAK,eAAe,yBAAyB,EAAE,OAAO,CAAC;WACzD,OAAO;AAGd,OACG,OAAoB,SAAS,kBAC7B,OAAoB,SAAS,YAE9B,QAAO,IAAI,OAAO;IAChB,GAAG,SAAS;IACZ,UAAU,SAAS,KAAK;IACxB,QAAQ,SAAS,KAAK;IACtB,MAAM,SAAS,KAAK;IACpB,SAAS,SAAS,KAAK;IACxB,CAAC;AAGJ,OAAI,eAAe,8BAA8B;AACjD,SAAM;;EAGR,MAAM,qBAAqB,MAAM,KAAK,aAAa,sBAAsB,EACvE,IACD,CAAC;AAEF,SAAO,IAAI,OAAO;GAChB,GAAG,SAAS;GACZ,QAAQ,UAAU,sBAAsB;GACxC,MAAM,SAAS,KAAK;GACpB,SAAS,SAAS,KAAK;GACxB,CAAC;;;;;;;CAQJ,MAAM,KAAK,IAA+B;AACxC,MAAI,eAAe,sBAAsB;EACzC,IAAI,UAAU;EACd,IAAIC,YAA0B;AAE9B,SAAO,UAAU,EACf,KAAI;GACF,MAAM,OAAO,MAAM,KAAK,OAAO,UAAU;IACvC,QAAQ,KAAK;IACb,KAAK;IACN,CAAC;AACF,OAAI,eAAe,sBAAsB;AACzC,UAAO,KAAK;WACL,OAAO;AACd,OAAI,eAAe,oBAAoB;AACvC,eAAY;AACZ;AAEA,OAAI,UAAU,EAEZ,OAAM,IAAI,SAAS,YAAY,WAAW,SAAS,IAAI,CAAC;;AAK9D,MAAI,eAAe,qBAAqB;AACxC,QAAM,6BAAa,IAAI,MAAM,uBAAuB,KAAK;;;;;;;;;CAU3D,MAAa,KAAK,QAAgB,QAA+B;AAC/D,MAAI;AACF,SAAM,KAAK,OAAO,WAAW;IAC3B,QAAQ,KAAK;IACb,YAAY,UAAU,GAAG,KAAK,OAAO,GAAG,SAAS;IACjD,KAAK;IACN,CAAC;AAEF,SAAM,KAAK,OAAO,aAAa;IAC7B,QAAQ,KAAK;IACb,KAAK;IACN,CAAC;WACK,OAAO;AACd,OACG,OAAoB,QACrB;IAAC;IAAa;IAAgB;IAAW,CAAC,SACvC,MAAmB,QAAQ,GAC7B,EACD;AACA,QAAI,eAAe,wBAAwB;AAC3C,UAAM,OAAO;;AAEf,SAAM;;;;;;;;CASV,MAAa,OAAO,IAA2B;AAC7C,MAAI;GACF,MAAM,EAAE,aAAa,UAAU,SAC7B,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;AAChD,OAAI,SACF,OAAM,KAAK,OAAO,qBAAqB;IACrC,QAAQ,KAAK;IACb,KAAK;IACL,UAAU;IACX,CAAC;AAGJ,SAAM,KAAK,OAAO,cAAc;IAC9B,QAAQ,KAAK;IACb,QAAQ,EACN,SAAS;KACP,EACE,KAAK,KAAK,gBAAgB,gBAAgB;MACxC;MACA,QAAQ,MAAM,UAAU,UAAU;MACnC,CAAC,EACH;KACD,EACE,KAAK,KAAK,gBAAgB,gBAAgB;MACxC;MACA,QAAQ,MAAM,UAAU,UAAU;MAClC,cAAc;MACf,CAAC,EACH;KACD,EACE,KAAK,KAAK,gBAAgB,gBAAgB,EAAE,IAAI,CAAC,EAClD;KACF,EACF;IACF,CAAC;WACK,OAAO;AACd,OACG,OAAoB,QACrB;IAAC;IAAa;IAAgB;IAAW,CAAC,SACvC,MAAmB,QAAQ,GAC7B,EACD;AACA,QAAI,eAAe,wBAAwB;AAC3C,UAAM,OAAO;;AAEf,SAAM;;;;;;;;CASV,MAAa,QAAQ,IAA2B;AAC9C,MAAI;GACF,MAAM,EAAE,SAAS,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;AAE/D,SAAM,KAAK,OAAO,cAAc;IAC9B,QAAQ,KAAK;IACb,QAAQ,EACN,SAAS,CACP,EACE,KAAK,KAAK,gBAAgB,gBAAgB;KACxC;KACA,QAAQ,MAAM,UAAU,UAAU;KAClC,cAAc;KACf,CAAC,EACH,EACD,EACE,KAAK,KAAK,gBAAgB,gBAAgB,EAAE,IAAI,CAAC,EAClD,CACF,EACF;IACF,CAAC;WACK,OAAO;AACd,OACG,OAAoB,QACrB;IAAC;IAAa;IAAgB;IAAW,CAAC,SACvC,MAAmB,QAAQ,GAC7B,EACD;AACA,QAAI,eAAe,wBAAwB;AAC3C,UAAM,OAAO;;AAEf,SAAM;;;;;;;;CASV,MAAa,wBAAwB,IAA6B;EAChE,MAAM,WAAW,MAAM,KAAK,gBAAgB,YAAY,EAAE,IAAI,CAAC;EAC/D,MAAM,QAAQ,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC;EAE3D,MAAM,qBAAqB,MAAM,KAAK,aAAa,uBAAuB,EACxE,IACD,CAAC;AAEF,MAAI,oBAAoB;AAEtB,SAAM,KAAK,aAAa,WAAW;IACjC;IACA,YAAY,mBAAmB,aAAa,EAAE,cAAc,MAAM,CAAC;IACnE,YAAY,MAAM,SAAS;IAC5B,CAAC;AAGF,SAAM,KAAK,aAAa,qBAAqB,EAAE,IAAI,CAAC;GAGpD,MAAM,eAAe,MAAM,KAAK,aAAa,cAAc,EAAE,IAAI,CAAC;AAClE,SAAM,KAAK,aAAa,sBAAsB;IAC5C;IACA,OAAO;IACR,CAAC;QAEF,OAAM,KAAK,aAAa,sBAAsB;GAAE;GAAU;GAAO,CAAC;EAGpE,MAAM,kBAAkB,IAAI,OAAO;GACjC,GAAG,SAAS;GACZ,QAAQ,SAAS,KAAK,QAAQ;GAC9B,MAAM,SAAS,KAAK,QAAQ;GAC5B,SAAS,SAAS,KAAK;GACxB,CAAC;AAEF,QAAM,KAAK,gBAAgB,iBAAiB,EAAE,QAAQ,iBAAiB,CAAC;AAExE,SAAO;;;;;;;CAQT,AAAO,OAAO,IAAoB;AAEhC,MAAI,KAAK,eACP,QAAO,GAAG,KAAK,eAAe,GAAG,KAAK,OAAO,GAAG;EAIlD,MAAM,eAAe,KAAK,OAAO,OAAO;EACxC,IAAIC;AAGJ,MAAI,OAAO,iBAAiB,WAC1B,UAAS;MAET,UAAS,gBAAgB;AAI3B,MAAI,WAAW,YACb,QAAO,WAAW,KAAK,OAAO,oBAAoB;MAElD,QAAO,WAAW,KAAK,OAAO,MAAM,OAAO,iBAAiB;;;;;;;CAShE,MAAM,gBAAiC;AACrC,SAAO,KAAK,kBAAkB,eAAe;;;;;;CAO/C,gBAAwB;AACtB,SAAO,KAAK,kBAAkB,eAAe"}
|
package/dist/utils/mux.d.mts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { MediaCloudPluginOptions } from "../types/index.mjs";
|
|
2
2
|
import Mux from "@mux/mux-node";
|
|
3
|
-
import * as
|
|
3
|
+
import * as payload2 from "payload";
|
|
4
4
|
|
|
5
5
|
//#region src/utils/mux.d.ts
|
|
6
6
|
|
|
@@ -14,11 +14,11 @@ interface CreateMuxEndpointsArgs {
|
|
|
14
14
|
pluginOptions: MediaCloudPluginOptions;
|
|
15
15
|
}
|
|
16
16
|
declare function createMuxEndpoints(args: CreateMuxEndpointsArgs): ({
|
|
17
|
-
handler:
|
|
17
|
+
handler: payload2.PayloadHandler;
|
|
18
18
|
method: "post";
|
|
19
19
|
path: string;
|
|
20
20
|
} | {
|
|
21
|
-
handler:
|
|
21
|
+
handler: payload2.PayloadHandler;
|
|
22
22
|
method: "get";
|
|
23
23
|
path: string;
|
|
24
24
|
})[];
|
package/dist/utils/tus.d.mts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { MediaCloudPluginOptions } from "../types/index.mjs";
|
|
2
2
|
import { S3Store } from "../tus/stores/s3/s3Store.mjs";
|
|
3
3
|
import { Server } from "@tus/server";
|
|
4
|
-
import * as
|
|
4
|
+
import * as payload0 from "payload";
|
|
5
5
|
import { PayloadRequest } from "payload";
|
|
6
6
|
|
|
7
7
|
//#region src/utils/tus.d.ts
|
|
@@ -45,11 +45,11 @@ declare function createTusEndpoints(args: CreateTusEndpointsArgs): ({
|
|
|
45
45
|
method: "delete";
|
|
46
46
|
path: string;
|
|
47
47
|
} | {
|
|
48
|
-
handler:
|
|
48
|
+
handler: payload0.PayloadHandler;
|
|
49
49
|
method: "get";
|
|
50
50
|
path: string;
|
|
51
51
|
} | {
|
|
52
|
-
handler:
|
|
52
|
+
handler: payload0.PayloadHandler;
|
|
53
53
|
method: "post";
|
|
54
54
|
path: string;
|
|
55
55
|
})[];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@maas/payload-plugin-media-cloud",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.38",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"contributors": [
|
|
6
6
|
{
|
|
@@ -38,6 +38,7 @@
|
|
|
38
38
|
"@fastify/deepmerge": "3.1.0",
|
|
39
39
|
"@mux/mux-node": "^12.8.1",
|
|
40
40
|
"@mux/upchunk": "^3.5.0",
|
|
41
|
+
"@smithy/node-http-handler": "^4.4.10",
|
|
41
42
|
"@tus/server": "^2.3.0",
|
|
42
43
|
"@tus/utils": "^0.6.0",
|
|
43
44
|
"defu": "^6.1.4",
|