@maas/payload-plugin-media-cloud 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +8 -0
- package/dist/adapter/handleDelete.d.ts +20 -0
- package/dist/adapter/handleDelete.js +70 -0
- package/dist/adapter/handleDelete.js.map +1 -0
- package/dist/adapter/handleUpload.d.ts +12 -0
- package/dist/adapter/handleUpload.js +29 -0
- package/dist/adapter/handleUpload.js.map +1 -0
- package/dist/adapter/staticHandler.d.ts +17 -0
- package/dist/adapter/staticHandler.js +64 -0
- package/dist/adapter/staticHandler.js.map +1 -0
- package/dist/adapter/storageAdapter.d.ts +23 -0
- package/dist/adapter/storageAdapter.js +30 -0
- package/dist/adapter/storageAdapter.js.map +1 -0
- package/dist/collections/mediaCollection.d.ts +16 -0
- package/dist/collections/mediaCollection.js +139 -0
- package/dist/collections/mediaCollection.js.map +1 -0
- package/dist/components/index.d.ts +4 -0
- package/dist/components/index.js +5 -0
- package/dist/components/mux-preview/index.d.ts +2 -0
- package/dist/components/mux-preview/index.js +3 -0
- package/dist/components/mux-preview/mux-preview.d.ts +14 -0
- package/dist/components/mux-preview/mux-preview.js +38 -0
- package/dist/components/mux-preview/mux-preview.js.map +1 -0
- package/dist/components/upload-handler/index.d.ts +2 -0
- package/dist/components/upload-handler/index.js +3 -0
- package/dist/components/upload-handler/upload-handler.d.ts +22 -0
- package/dist/components/upload-handler/upload-handler.js +178 -0
- package/dist/components/upload-handler/upload-handler.js.map +1 -0
- package/dist/components/upload-manager/index.d.ts +2 -0
- package/dist/components/upload-manager/index.js +3 -0
- package/dist/components/upload-manager/upload-manager-DN4RrmYB.css +204 -0
- package/dist/components/upload-manager/upload-manager-DN4RrmYB.css.map +1 -0
- package/dist/components/upload-manager/upload-manager.css +201 -0
- package/dist/components/upload-manager/upload-manager.d.ts +42 -0
- package/dist/components/upload-manager/upload-manager.js +315 -0
- package/dist/components/upload-manager/upload-manager.js.map +1 -0
- package/dist/components/upload-manager/upload-manager2.js +0 -0
- package/dist/endpoints/muxAssetHandler.d.ts +11 -0
- package/dist/endpoints/muxAssetHandler.js +59 -0
- package/dist/endpoints/muxAssetHandler.js.map +1 -0
- package/dist/endpoints/muxCreateUploadHandler.d.ts +13 -0
- package/dist/endpoints/muxCreateUploadHandler.js +40 -0
- package/dist/endpoints/muxCreateUploadHandler.js.map +1 -0
- package/dist/endpoints/muxWebhookHandler.d.ts +11 -0
- package/dist/endpoints/muxWebhookHandler.js +49 -0
- package/dist/endpoints/muxWebhookHandler.js.map +1 -0
- package/dist/hooks/useEmitter.d.ts +48 -0
- package/dist/hooks/useEmitter.js +19 -0
- package/dist/hooks/useEmitter.js.map +1 -0
- package/dist/hooks/useErrorHandler.d.ts +11 -0
- package/dist/hooks/useErrorHandler.js +19 -0
- package/dist/hooks/useErrorHandler.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +3 -0
- package/dist/plugin.d.ts +15 -0
- package/dist/plugin.js +242 -0
- package/dist/plugin.js.map +1 -0
- package/dist/tus/stores/s3/expiration-manager.d.ts +36 -0
- package/dist/tus/stores/s3/expiration-manager.js +76 -0
- package/dist/tus/stores/s3/expiration-manager.js.map +1 -0
- package/dist/tus/stores/s3/file-operations.d.ts +66 -0
- package/dist/tus/stores/s3/file-operations.js +90 -0
- package/dist/tus/stores/s3/file-operations.js.map +1 -0
- package/dist/tus/stores/s3/log.d.ts +5 -0
- package/dist/tus/stores/s3/log.js +8 -0
- package/dist/tus/stores/s3/log.js.map +1 -0
- package/dist/tus/stores/s3/metadata-manager.d.ts +85 -0
- package/dist/tus/stores/s3/metadata-manager.js +135 -0
- package/dist/tus/stores/s3/metadata-manager.js.map +1 -0
- package/dist/tus/stores/s3/parts-manager.d.ts +130 -0
- package/dist/tus/stores/s3/parts-manager.js +328 -0
- package/dist/tus/stores/s3/parts-manager.js.map +1 -0
- package/dist/tus/stores/s3/s3-store.d.ts +110 -0
- package/dist/tus/stores/s3/s3-store.js +342 -0
- package/dist/tus/stores/s3/s3-store.js.map +1 -0
- package/dist/tus/stores/s3/semaphore.d.ts +16 -0
- package/dist/tus/stores/s3/semaphore.js +32 -0
- package/dist/tus/stores/s3/semaphore.js.map +1 -0
- package/dist/types/errors.d.ts +26 -0
- package/dist/types/errors.js +28 -0
- package/dist/types/errors.js.map +1 -0
- package/dist/types/index.d.ts +73 -0
- package/dist/types/index.js +0 -0
- package/dist/utils/file.d.ts +30 -0
- package/dist/utils/file.js +84 -0
- package/dist/utils/file.js.map +1 -0
- package/package.json +92 -0
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { TusUploadMetadata } from "../../../types/index.js";
|
|
2
|
+
import { S3 } from "@aws-sdk/client-s3";
|
|
3
|
+
import { KvStore, Upload } from "@tus/utils";
|
|
4
|
+
|
|
5
|
+
//#region src/tus/stores/s3/metadata-manager.d.ts
|
|
6
|
+
type GenerateInfoKeyArgs = {
|
|
7
|
+
id: string;
|
|
8
|
+
};
|
|
9
|
+
type GeneratePartKeyArgs = {
|
|
10
|
+
id: string;
|
|
11
|
+
isIncomplete?: boolean;
|
|
12
|
+
};
|
|
13
|
+
type GetMetadataArgs = {
|
|
14
|
+
id: string;
|
|
15
|
+
};
|
|
16
|
+
type SaveMetadataArgs = {
|
|
17
|
+
upload: Upload;
|
|
18
|
+
uploadId: string;
|
|
19
|
+
};
|
|
20
|
+
type CompleteMetadataArgs = {
|
|
21
|
+
upload: Upload;
|
|
22
|
+
};
|
|
23
|
+
type ClearCacheArgs = {
|
|
24
|
+
id: string;
|
|
25
|
+
};
|
|
26
|
+
declare class S3MetadataManager {
|
|
27
|
+
private client;
|
|
28
|
+
private bucket;
|
|
29
|
+
private cache;
|
|
30
|
+
private shouldUseExpirationTags;
|
|
31
|
+
private generateCompleteTag;
|
|
32
|
+
constructor(client: S3, bucket: string, cache: KvStore<TusUploadMetadata>, shouldUseExpirationTags: () => boolean, generateCompleteTag: (value: 'false' | 'true') => string | undefined);
|
|
33
|
+
/**
|
|
34
|
+
* Generates the S3 key for metadata info files
|
|
35
|
+
* @param args - The function arguments
|
|
36
|
+
* @param args.id - The file ID
|
|
37
|
+
* @returns The S3 key for the metadata file
|
|
38
|
+
*/
|
|
39
|
+
generateInfoKey(args: GenerateInfoKeyArgs): string;
|
|
40
|
+
/**
|
|
41
|
+
* Generates the S3 key for part files
|
|
42
|
+
* @param args - The function arguments
|
|
43
|
+
* @param args.id - The file ID
|
|
44
|
+
* @param args.isIncomplete - Whether this is an incomplete part
|
|
45
|
+
* @returns The S3 key for the part file
|
|
46
|
+
*/
|
|
47
|
+
generatePartKey(args: GeneratePartKeyArgs): string;
|
|
48
|
+
/**
|
|
49
|
+
* Retrieves upload metadata previously saved in `${file_id}.info`.
|
|
50
|
+
* There's a small and simple caching mechanism to avoid multiple
|
|
51
|
+
* HTTP calls to S3.
|
|
52
|
+
* @param args - The function arguments
|
|
53
|
+
* @param args.id - The file ID to retrieve metadata for
|
|
54
|
+
* @returns Promise that resolves to the upload metadata
|
|
55
|
+
*/
|
|
56
|
+
getMetadata(args: GetMetadataArgs): Promise<TusUploadMetadata>;
|
|
57
|
+
/**
|
|
58
|
+
* Saves upload metadata to a `${file_id}.info` file on S3.
|
|
59
|
+
* Please note that the file is empty and the metadata is saved
|
|
60
|
+
* on the S3 object's `Metadata` field, so that only a `headObject`
|
|
61
|
+
* is necessary to retrieve the data.
|
|
62
|
+
* @param args - The function arguments
|
|
63
|
+
* @param args.upload - The upload object containing metadata
|
|
64
|
+
* @param args.uploadId - The upload ID for the multipart upload
|
|
65
|
+
* @returns Promise that resolves when metadata is saved
|
|
66
|
+
*/
|
|
67
|
+
saveMetadata(args: SaveMetadataArgs): Promise<void>;
|
|
68
|
+
/**
|
|
69
|
+
* Completes metadata by updating it with completion tags
|
|
70
|
+
* @param args - The function arguments
|
|
71
|
+
* @param args.upload - The completed upload object
|
|
72
|
+
* @returns Promise that resolves when metadata is updated
|
|
73
|
+
*/
|
|
74
|
+
completeMetadata(args: CompleteMetadataArgs): Promise<void>;
|
|
75
|
+
/**
|
|
76
|
+
* Removes cached data for a given file
|
|
77
|
+
* @param args - The function arguments
|
|
78
|
+
* @param args.id - The file ID to remove from cache
|
|
79
|
+
* @returns Promise that resolves when cache is cleared
|
|
80
|
+
*/
|
|
81
|
+
clearCache(args: ClearCacheArgs): Promise<void>;
|
|
82
|
+
}
|
|
83
|
+
//#endregion
|
|
84
|
+
export { S3MetadataManager };
|
|
85
|
+
//# sourceMappingURL=metadata-manager.d.ts.map
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { log } from "./log.js";
|
|
2
|
+
import { TUS_RESUMABLE, Upload } from "@tus/utils";
|
|
3
|
+
|
|
4
|
+
//#region src/tus/stores/s3/metadata-manager.ts
|
|
5
|
+
var S3MetadataManager = class {
|
|
6
|
+
constructor(client, bucket, cache, shouldUseExpirationTags, generateCompleteTag) {
|
|
7
|
+
this.client = client;
|
|
8
|
+
this.bucket = bucket;
|
|
9
|
+
this.cache = cache;
|
|
10
|
+
this.shouldUseExpirationTags = shouldUseExpirationTags;
|
|
11
|
+
this.generateCompleteTag = generateCompleteTag;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Generates the S3 key for metadata info files
|
|
15
|
+
* @param args - The function arguments
|
|
16
|
+
* @param args.id - The file ID
|
|
17
|
+
* @returns The S3 key for the metadata file
|
|
18
|
+
*/
|
|
19
|
+
generateInfoKey(args) {
|
|
20
|
+
const { id } = args;
|
|
21
|
+
return `${id}.info`;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Generates the S3 key for part files
|
|
25
|
+
* @param args - The function arguments
|
|
26
|
+
* @param args.id - The file ID
|
|
27
|
+
* @param args.isIncomplete - Whether this is an incomplete part
|
|
28
|
+
* @returns The S3 key for the part file
|
|
29
|
+
*/
|
|
30
|
+
generatePartKey(args) {
|
|
31
|
+
const { id, isIncomplete = false } = args;
|
|
32
|
+
let key = id;
|
|
33
|
+
if (isIncomplete) key += ".part";
|
|
34
|
+
return key;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Retrieves upload metadata previously saved in `${file_id}.info`.
|
|
38
|
+
* There's a small and simple caching mechanism to avoid multiple
|
|
39
|
+
* HTTP calls to S3.
|
|
40
|
+
* @param args - The function arguments
|
|
41
|
+
* @param args.id - The file ID to retrieve metadata for
|
|
42
|
+
* @returns Promise that resolves to the upload metadata
|
|
43
|
+
*/
|
|
44
|
+
async getMetadata(args) {
|
|
45
|
+
const { id } = args;
|
|
46
|
+
const cached = await this.cache.get(id);
|
|
47
|
+
if (cached) return cached;
|
|
48
|
+
const { Body, Metadata } = await this.client.getObject({
|
|
49
|
+
Bucket: this.bucket,
|
|
50
|
+
Key: this.generateInfoKey({ id })
|
|
51
|
+
});
|
|
52
|
+
const file = JSON.parse(await Body?.transformToString());
|
|
53
|
+
const metadata = {
|
|
54
|
+
file: new Upload({
|
|
55
|
+
id,
|
|
56
|
+
creation_date: file.creation_date,
|
|
57
|
+
metadata: file.metadata,
|
|
58
|
+
offset: Number.parseInt(file.offset, 10),
|
|
59
|
+
size: Number.isFinite(file.size) ? Number.parseInt(file.size, 10) : void 0,
|
|
60
|
+
storage: file.storage
|
|
61
|
+
}),
|
|
62
|
+
"tus-version": Metadata?.["tus-version"],
|
|
63
|
+
"upload-id": Metadata?.["upload-id"]
|
|
64
|
+
};
|
|
65
|
+
await this.cache.set(id, metadata);
|
|
66
|
+
return metadata;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Saves upload metadata to a `${file_id}.info` file on S3.
|
|
70
|
+
* Please note that the file is empty and the metadata is saved
|
|
71
|
+
* on the S3 object's `Metadata` field, so that only a `headObject`
|
|
72
|
+
* is necessary to retrieve the data.
|
|
73
|
+
* @param args - The function arguments
|
|
74
|
+
* @param args.upload - The upload object containing metadata
|
|
75
|
+
* @param args.uploadId - The upload ID for the multipart upload
|
|
76
|
+
* @returns Promise that resolves when metadata is saved
|
|
77
|
+
*/
|
|
78
|
+
async saveMetadata(args) {
|
|
79
|
+
const { upload, uploadId } = args;
|
|
80
|
+
log(`[${upload.id}] saving metadata`);
|
|
81
|
+
await this.client.putObject({
|
|
82
|
+
Body: JSON.stringify(upload),
|
|
83
|
+
Bucket: this.bucket,
|
|
84
|
+
Key: this.generateInfoKey({ id: upload.id }),
|
|
85
|
+
Metadata: {
|
|
86
|
+
"tus-version": TUS_RESUMABLE,
|
|
87
|
+
"upload-id": uploadId
|
|
88
|
+
},
|
|
89
|
+
Tagging: this.generateCompleteTag("false")
|
|
90
|
+
});
|
|
91
|
+
log(`[${upload.id}] metadata file saved`);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Completes metadata by updating it with completion tags
|
|
95
|
+
* @param args - The function arguments
|
|
96
|
+
* @param args.upload - The completed upload object
|
|
97
|
+
* @returns Promise that resolves when metadata is updated
|
|
98
|
+
*/
|
|
99
|
+
async completeMetadata(args) {
|
|
100
|
+
const { upload } = args;
|
|
101
|
+
const metadata = await this.getMetadata({ id: upload.id });
|
|
102
|
+
const completedMetadata = {
|
|
103
|
+
...metadata,
|
|
104
|
+
file: upload
|
|
105
|
+
};
|
|
106
|
+
await this.cache.set(upload.id, completedMetadata);
|
|
107
|
+
if (!this.shouldUseExpirationTags()) return;
|
|
108
|
+
const { "upload-id": uploadId } = metadata;
|
|
109
|
+
await this.client.putObject({
|
|
110
|
+
Body: JSON.stringify(upload),
|
|
111
|
+
Bucket: this.bucket,
|
|
112
|
+
Key: this.generateInfoKey({ id: upload.id }),
|
|
113
|
+
Metadata: {
|
|
114
|
+
"tus-version": TUS_RESUMABLE,
|
|
115
|
+
"upload-id": uploadId
|
|
116
|
+
},
|
|
117
|
+
Tagging: this.generateCompleteTag("true")
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Removes cached data for a given file
|
|
122
|
+
* @param args - The function arguments
|
|
123
|
+
* @param args.id - The file ID to remove from cache
|
|
124
|
+
* @returns Promise that resolves when cache is cleared
|
|
125
|
+
*/
|
|
126
|
+
async clearCache(args) {
|
|
127
|
+
const { id } = args;
|
|
128
|
+
log(`[${id}] removing cached data`);
|
|
129
|
+
await this.cache.delete(id);
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
//#endregion
|
|
134
|
+
export { S3MetadataManager };
|
|
135
|
+
//# sourceMappingURL=metadata-manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metadata-manager.js","names":["client: S3","bucket: string","cache: KvStore<TusUploadMetadata>","shouldUseExpirationTags: () => boolean","generateCompleteTag: (value: 'false' | 'true') => string | undefined","args: GenerateInfoKeyArgs","args: GeneratePartKeyArgs","args: GetMetadataArgs","metadata: TusUploadMetadata","args: SaveMetadataArgs","args: CompleteMetadataArgs","completedMetadata: TusUploadMetadata","args: ClearCacheArgs"],"sources":["../../../../src/tus/stores/s3/metadata-manager.ts"],"sourcesContent":["import { TUS_RESUMABLE, Upload } from '@tus/utils'\nimport { log } from './log'\n\nimport type { S3 } from '@aws-sdk/client-s3'\nimport type { KvStore } from '@tus/utils'\nimport type { TusUploadMetadata } from '../../../types'\n\ntype GenerateInfoKeyArgs = {\n id: string\n}\n\ntype GeneratePartKeyArgs = {\n id: string\n isIncomplete?: boolean\n}\n\ntype GetMetadataArgs = {\n id: string\n}\n\ntype SaveMetadataArgs = {\n upload: Upload\n uploadId: string\n}\n\ntype CompleteMetadataArgs = {\n upload: Upload\n}\n\ntype ClearCacheArgs = {\n id: string\n}\n\nexport class S3MetadataManager {\n constructor(\n private client: S3,\n private bucket: string,\n private cache: KvStore<TusUploadMetadata>,\n private shouldUseExpirationTags: () => boolean,\n private generateCompleteTag: (value: 'false' | 'true') => string | undefined\n ) {}\n\n /**\n * Generates the S3 key for metadata info files\n * @param args - The function arguments\n * @param args.id - The file ID\n * @returns The S3 key for the metadata file\n */\n generateInfoKey(args: GenerateInfoKeyArgs): string {\n const { id } = args\n return `${id}.info`\n }\n\n /**\n * Generates the S3 key for part files\n * @param args - The function arguments\n * @param args.id - The file ID\n * @param args.isIncomplete - Whether this is an incomplete part\n * @returns The S3 key for the part file\n */\n generatePartKey(args: GeneratePartKeyArgs): string {\n const { id, isIncomplete = false } = args\n let key = id\n if (isIncomplete) {\n key += '.part'\n }\n\n // TODO: introduce ObjectPrefixing for parts and incomplete parts.\n // ObjectPrefix is prepended to the name of each S3 object that is created\n // to store uploaded files. It can be used to create a pseudo-directory\n // structure in the bucket, e.g. \"path/to/my/uploads\".\n return key\n }\n\n /**\n * Retrieves upload metadata previously saved in `${file_id}.info`.\n * There's a small and simple caching mechanism to avoid multiple\n * HTTP calls to S3.\n * @param args - The function arguments\n * @param args.id - The file ID to retrieve metadata for\n * @returns Promise that resolves to the upload metadata\n */\n async getMetadata(args: GetMetadataArgs): Promise<TusUploadMetadata> {\n const { id } = args\n const cached = await this.cache.get(id)\n if (cached) {\n return cached\n }\n\n const { Body, Metadata } = await this.client.getObject({\n Bucket: this.bucket,\n Key: this.generateInfoKey({ id }),\n })\n const file = JSON.parse((await Body?.transformToString()) as string)\n const metadata: TusUploadMetadata = {\n file: new Upload({\n id,\n creation_date: file.creation_date,\n metadata: file.metadata,\n offset: Number.parseInt(file.offset, 10),\n size: Number.isFinite(file.size)\n ? Number.parseInt(file.size, 10)\n : undefined,\n storage: file.storage,\n }),\n 'tus-version': Metadata?.['tus-version'] as string,\n 'upload-id': Metadata?.['upload-id'] as string,\n }\n await this.cache.set(id, metadata)\n return metadata\n }\n\n /**\n * Saves upload metadata to a `${file_id}.info` file on S3.\n * Please note that the file is empty and the metadata is saved\n * on the S3 object's `Metadata` field, so that only a `headObject`\n * is necessary to retrieve the data.\n * @param args - The function arguments\n * @param args.upload - The upload object containing metadata\n * @param args.uploadId - The upload ID for the multipart upload\n * @returns Promise that resolves when metadata is saved\n */\n async saveMetadata(args: SaveMetadataArgs): Promise<void> {\n const { upload, uploadId } = args\n log(`[${upload.id}] saving metadata`)\n await this.client.putObject({\n Body: JSON.stringify(upload),\n Bucket: this.bucket,\n Key: this.generateInfoKey({ id: upload.id }),\n Metadata: {\n 'tus-version': TUS_RESUMABLE,\n 'upload-id': uploadId,\n },\n Tagging: this.generateCompleteTag('false'),\n })\n log(`[${upload.id}] metadata file saved`)\n }\n\n /**\n * Completes metadata by updating it with completion tags\n * @param args - The function arguments\n * @param args.upload - The completed upload object\n * @returns Promise that resolves when metadata is updated\n */\n async completeMetadata(args: CompleteMetadataArgs): Promise<void> {\n const { upload } = args\n // Always update the cache with the completed upload\n const metadata = await this.getMetadata({ id: upload.id })\n const completedMetadata: TusUploadMetadata = {\n ...metadata,\n file: upload,\n }\n await this.cache.set(upload.id, completedMetadata)\n\n if (!this.shouldUseExpirationTags()) {\n return\n }\n\n const { 'upload-id': uploadId } = metadata\n await this.client.putObject({\n Body: JSON.stringify(upload),\n Bucket: this.bucket,\n Key: this.generateInfoKey({ id: upload.id }),\n Metadata: {\n 'tus-version': TUS_RESUMABLE,\n 'upload-id': uploadId,\n },\n Tagging: this.generateCompleteTag('true'),\n })\n }\n\n /**\n * Removes cached data for a given file\n * @param args - The function arguments\n * @param args.id - The file ID to remove from cache\n * @returns Promise that resolves when cache is cleared\n */\n async clearCache(args: ClearCacheArgs): Promise<void> {\n const { id } = args\n log(`[${id}] removing cached data`)\n await this.cache.delete(id)\n }\n}\n"],"mappings":";;;;AAiCA,IAAa,oBAAb,MAA+B;CAC7B,YACUA,QACAC,QACAC,OACAC,yBACAC,qBACR;EALQ;EACA;EACA;EACA;EACA;CACN;;;;;;;CAQJ,gBAAgBC,MAAmC;EACjD,MAAM,EAAE,IAAI,GAAG;AACf,SAAO,GAAG,GAAG,KAAK,CAAC;CACpB;;;;;;;;CASD,gBAAgBC,MAAmC;EACjD,MAAM,EAAE,IAAI,eAAe,OAAO,GAAG;EACrC,IAAI,MAAM;AACV,MAAI,cACF,OAAO;AAOT,SAAO;CACR;;;;;;;;;CAUD,MAAM,YAAYC,MAAmD;EACnE,MAAM,EAAE,IAAI,GAAG;EACf,MAAM,SAAS,MAAM,KAAK,MAAM,IAAI,GAAG;AACvC,MAAI,OACF,QAAO;EAGT,MAAM,EAAE,MAAM,UAAU,GAAG,MAAM,KAAK,OAAO,UAAU;GACrD,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,EAAE,GAAI,EAAC;EAClC,EAAC;EACF,MAAM,OAAO,KAAK,MAAO,MAAM,MAAM,mBAAmB,CAAY;EACpE,MAAMC,WAA8B;GAClC,MAAM,IAAI,OAAO;IACf;IACA,eAAe,KAAK;IACpB,UAAU,KAAK;IACf,QAAQ,OAAO,SAAS,KAAK,QAAQ,GAAG;IACxC,MAAM,OAAO,SAAS,KAAK,KAAK,GAC5B,OAAO,SAAS,KAAK,MAAM,GAAG,GAC9B;IACJ,SAAS,KAAK;GACf;GACD,eAAe,WAAW;GAC1B,aAAa,WAAW;EACzB;EACD,MAAM,KAAK,MAAM,IAAI,IAAI,SAAS;AAClC,SAAO;CACR;;;;;;;;;;;CAYD,MAAM,aAAaC,MAAuC;EACxD,MAAM,EAAE,QAAQ,UAAU,GAAG;EAC7B,IAAI,CAAC,CAAC,EAAE,OAAO,GAAG,iBAAiB,CAAC,CAAC;EACrC,MAAM,KAAK,OAAO,UAAU;GAC1B,MAAM,KAAK,UAAU,OAAO;GAC5B,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,EAAE,IAAI,OAAO,GAAI,EAAC;GAC5C,UAAU;IACR,eAAe;IACf,aAAa;GACd;GACD,SAAS,KAAK,oBAAoB,QAAQ;EAC3C,EAAC;EACF,IAAI,CAAC,CAAC,EAAE,OAAO,GAAG,qBAAqB,CAAC,CAAC;CAC1C;;;;;;;CAQD,MAAM,iBAAiBC,MAA2C;EAChE,MAAM,EAAE,QAAQ,GAAG;EAEnB,MAAM,WAAW,MAAM,KAAK,YAAY,EAAE,IAAI,OAAO,GAAI,EAAC;EAC1D,MAAMC,oBAAuC;GAC3C,GAAG;GACH,MAAM;EACP;EACD,MAAM,KAAK,MAAM,IAAI,OAAO,IAAI,kBAAkB;AAElD,MAAI,CAAC,KAAK,yBAAyB,CACjC;EAGF,MAAM,EAAE,aAAa,UAAU,GAAG;EAClC,MAAM,KAAK,OAAO,UAAU;GAC1B,MAAM,KAAK,UAAU,OAAO;GAC5B,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,EAAE,IAAI,OAAO,GAAI,EAAC;GAC5C,UAAU;IACR,eAAe;IACf,aAAa;GACd;GACD,SAAS,KAAK,oBAAoB,OAAO;EAC1C,EAAC;CACH;;;;;;;CAQD,MAAM,WAAWC,MAAqC;EACpD,MAAM,EAAE,IAAI,GAAG;EACf,IAAI,CAAC,CAAC,EAAE,GAAG,sBAAsB,CAAC,CAAC;EACnC,MAAM,KAAK,MAAM,OAAO,GAAG;CAC5B;AACF"}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { Semaphore } from "./semaphore.js";
|
|
2
|
+
import { S3FileOperations } from "./file-operations.js";
|
|
3
|
+
import { IncompletePartInfo, TusUploadMetadata } from "../../../types/index.js";
|
|
4
|
+
import { S3MetadataManager } from "./metadata-manager.js";
|
|
5
|
+
import stream, { Readable } from "node:stream";
|
|
6
|
+
import AWS, { S3 } from "@aws-sdk/client-s3";
|
|
7
|
+
import fs from "node:fs";
|
|
8
|
+
|
|
9
|
+
//#region src/tus/stores/s3/parts-manager.d.ts
|
|
10
|
+
type RetrievePartsArgs = {
|
|
11
|
+
id: string;
|
|
12
|
+
partNumberMarker?: string;
|
|
13
|
+
};
|
|
14
|
+
type FinishMultipartUploadArgs = {
|
|
15
|
+
metadata: TusUploadMetadata;
|
|
16
|
+
parts: Array<AWS.Part>;
|
|
17
|
+
};
|
|
18
|
+
type GetIncompletePartArgs = {
|
|
19
|
+
id: string;
|
|
20
|
+
};
|
|
21
|
+
type GetIncompletePartSizeArgs = {
|
|
22
|
+
id: string;
|
|
23
|
+
};
|
|
24
|
+
type DeleteIncompletePartArgs = {
|
|
25
|
+
id: string;
|
|
26
|
+
};
|
|
27
|
+
type DownloadIncompletePartArgs = {
|
|
28
|
+
id: string;
|
|
29
|
+
};
|
|
30
|
+
type UploadIncompletePartArgs = {
|
|
31
|
+
id: string;
|
|
32
|
+
readStream: fs.ReadStream | Readable;
|
|
33
|
+
};
|
|
34
|
+
type UploadPartArgs = {
|
|
35
|
+
metadata: TusUploadMetadata;
|
|
36
|
+
readStream: fs.ReadStream | Readable;
|
|
37
|
+
partNumber: number;
|
|
38
|
+
};
|
|
39
|
+
type UploadPartsArgs = {
|
|
40
|
+
metadata: TusUploadMetadata;
|
|
41
|
+
readStream: stream.Readable;
|
|
42
|
+
currentPartNumber: number;
|
|
43
|
+
offset: number;
|
|
44
|
+
};
|
|
45
|
+
declare class S3PartsManager {
|
|
46
|
+
private client;
|
|
47
|
+
private bucket;
|
|
48
|
+
private minPartSize;
|
|
49
|
+
private partUploadSemaphore;
|
|
50
|
+
private metadataManager;
|
|
51
|
+
private fileOperations;
|
|
52
|
+
private generateCompleteTag;
|
|
53
|
+
constructor(client: S3, bucket: string, minPartSize: number, partUploadSemaphore: Semaphore, metadataManager: S3MetadataManager, fileOperations: S3FileOperations, generateCompleteTag: (value: 'false' | 'true') => string | undefined);
|
|
54
|
+
/**
|
|
55
|
+
* Gets the number of complete parts/chunks already uploaded to S3.
|
|
56
|
+
* Retrieves only consecutive parts.
|
|
57
|
+
* @param args - The function arguments
|
|
58
|
+
* @param args.id - The upload ID
|
|
59
|
+
* @param args.partNumberMarker - Marker for pagination (optional)
|
|
60
|
+
* @returns Promise that resolves to array of uploaded parts
|
|
61
|
+
*/
|
|
62
|
+
retrieveParts(args: RetrievePartsArgs): Promise<Array<AWS.Part>>;
|
|
63
|
+
/**
|
|
64
|
+
* Completes a multipart upload on S3.
|
|
65
|
+
* This is where S3 concatenates all the uploaded parts.
|
|
66
|
+
* @param args - The function arguments
|
|
67
|
+
* @param args.metadata - The upload metadata
|
|
68
|
+
* @param args.parts - Array of uploaded parts to complete
|
|
69
|
+
* @returns Promise that resolves to the location URL (optional)
|
|
70
|
+
*/
|
|
71
|
+
finishMultipartUpload(args: FinishMultipartUploadArgs): Promise<string | undefined>;
|
|
72
|
+
/**
|
|
73
|
+
* Gets incomplete part from S3
|
|
74
|
+
* @param args - The function arguments
|
|
75
|
+
* @param args.id - The upload ID
|
|
76
|
+
* @returns Promise that resolves to readable stream or undefined if not found
|
|
77
|
+
*/
|
|
78
|
+
getIncompletePart(args: GetIncompletePartArgs): Promise<Readable | undefined>;
|
|
79
|
+
/**
|
|
80
|
+
* Gets the size of an incomplete part
|
|
81
|
+
* @param args - The function arguments
|
|
82
|
+
* @param args.id - The upload ID
|
|
83
|
+
* @returns Promise that resolves to part size or undefined if not found
|
|
84
|
+
*/
|
|
85
|
+
getIncompletePartSize(args: GetIncompletePartSizeArgs): Promise<number | undefined>;
|
|
86
|
+
/**
|
|
87
|
+
* Deletes an incomplete part
|
|
88
|
+
* @param args - The function arguments
|
|
89
|
+
* @param args.id - The upload ID
|
|
90
|
+
* @returns Promise that resolves when deletion is complete
|
|
91
|
+
*/
|
|
92
|
+
deleteIncompletePart(args: DeleteIncompletePartArgs): Promise<void>;
|
|
93
|
+
/**
|
|
94
|
+
* Downloads incomplete part to temporary file
|
|
95
|
+
* @param args - The function arguments
|
|
96
|
+
* @param args.id - The upload ID
|
|
97
|
+
* @returns Promise that resolves to incomplete part info or undefined if not found
|
|
98
|
+
*/
|
|
99
|
+
downloadIncompletePart(args: DownloadIncompletePartArgs): Promise<IncompletePartInfo | undefined>;
|
|
100
|
+
/**
|
|
101
|
+
* Uploads an incomplete part
|
|
102
|
+
* @param args - The function arguments
|
|
103
|
+
* @param args.id - The upload ID
|
|
104
|
+
* @param args.readStream - The stream to read data from
|
|
105
|
+
* @returns Promise that resolves to the ETag of the uploaded part
|
|
106
|
+
*/
|
|
107
|
+
uploadIncompletePart(args: UploadIncompletePartArgs): Promise<string>;
|
|
108
|
+
/**
|
|
109
|
+
* Uploads a single part
|
|
110
|
+
* @param args - The function arguments
|
|
111
|
+
* @param args.metadata - The upload metadata
|
|
112
|
+
* @param args.readStream - The stream to read data from
|
|
113
|
+
* @param args.partNumber - The part number to upload
|
|
114
|
+
* @returns Promise that resolves to the ETag of the uploaded part
|
|
115
|
+
*/
|
|
116
|
+
uploadPart(args: UploadPartArgs): Promise<AWS.Part>;
|
|
117
|
+
/**
|
|
118
|
+
* Uploads a stream to s3 using multiple parts
|
|
119
|
+
* @param args - The function arguments
|
|
120
|
+
* @param args.metadata - The upload metadata
|
|
121
|
+
* @param args.readStream - The stream to read data from
|
|
122
|
+
* @param args.currentPartNumber - The current part number to start from
|
|
123
|
+
* @param args.offset - The byte offset to start from
|
|
124
|
+
* @returns Promise that resolves to the number of bytes uploaded
|
|
125
|
+
*/
|
|
126
|
+
uploadParts(args: UploadPartsArgs): Promise<number>;
|
|
127
|
+
}
|
|
128
|
+
//#endregion
|
|
129
|
+
export { S3PartsManager };
|
|
130
|
+
//# sourceMappingURL=parts-manager.d.ts.map
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
import { MediaCloudError } from "../../../types/errors.js";
|
|
2
|
+
import { useErrorHandler } from "../../../hooks/useErrorHandler.js";
|
|
3
|
+
import { log } from "./log.js";
|
|
4
|
+
import stream from "node:stream";
|
|
5
|
+
import { NoSuchKey, NotFound } from "@aws-sdk/client-s3";
|
|
6
|
+
import { StreamSplitter } from "@tus/utils";
|
|
7
|
+
import fs from "node:fs";
|
|
8
|
+
import os from "node:os";
|
|
9
|
+
|
|
10
|
+
//#region src/tus/stores/s3/parts-manager.ts
|
|
11
|
+
const { throwError } = useErrorHandler();
|
|
12
|
+
var S3PartsManager = class {
|
|
13
|
+
constructor(client, bucket, minPartSize, partUploadSemaphore, metadataManager, fileOperations, generateCompleteTag) {
|
|
14
|
+
this.client = client;
|
|
15
|
+
this.bucket = bucket;
|
|
16
|
+
this.minPartSize = minPartSize;
|
|
17
|
+
this.partUploadSemaphore = partUploadSemaphore;
|
|
18
|
+
this.metadataManager = metadataManager;
|
|
19
|
+
this.fileOperations = fileOperations;
|
|
20
|
+
this.generateCompleteTag = generateCompleteTag;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Gets the number of complete parts/chunks already uploaded to S3.
|
|
24
|
+
* Retrieves only consecutive parts.
|
|
25
|
+
* @param args - The function arguments
|
|
26
|
+
* @param args.id - The upload ID
|
|
27
|
+
* @param args.partNumberMarker - Marker for pagination (optional)
|
|
28
|
+
* @returns Promise that resolves to array of uploaded parts
|
|
29
|
+
*/
|
|
30
|
+
async retrieveParts(args) {
|
|
31
|
+
const { id, partNumberMarker } = args;
|
|
32
|
+
const metadata = await this.metadataManager.getMetadata({ id });
|
|
33
|
+
if (!metadata["upload-id"]) {
|
|
34
|
+
throwError(MediaCloudError.MUX_UPLOAD_ID_MISSING);
|
|
35
|
+
throw new Error();
|
|
36
|
+
}
|
|
37
|
+
const params = {
|
|
38
|
+
Bucket: this.bucket,
|
|
39
|
+
Key: id,
|
|
40
|
+
PartNumberMarker: partNumberMarker,
|
|
41
|
+
UploadId: metadata["upload-id"]
|
|
42
|
+
};
|
|
43
|
+
const data = await this.client.listParts(params);
|
|
44
|
+
let parts = data.Parts ?? [];
|
|
45
|
+
if (data.IsTruncated) {
|
|
46
|
+
const rest = await this.retrieveParts({
|
|
47
|
+
id,
|
|
48
|
+
partNumberMarker: data.NextPartNumberMarker
|
|
49
|
+
});
|
|
50
|
+
parts = [...parts, ...rest];
|
|
51
|
+
}
|
|
52
|
+
if (!partNumberMarker) parts.sort((a, b) => (a.PartNumber || 0) - (b.PartNumber || 0));
|
|
53
|
+
return parts;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Completes a multipart upload on S3.
|
|
57
|
+
* This is where S3 concatenates all the uploaded parts.
|
|
58
|
+
* @param args - The function arguments
|
|
59
|
+
* @param args.metadata - The upload metadata
|
|
60
|
+
* @param args.parts - Array of uploaded parts to complete
|
|
61
|
+
* @returns Promise that resolves to the location URL (optional)
|
|
62
|
+
*/
|
|
63
|
+
async finishMultipartUpload(args) {
|
|
64
|
+
const { metadata, parts } = args;
|
|
65
|
+
const params = {
|
|
66
|
+
Bucket: this.bucket,
|
|
67
|
+
Key: metadata.file.id,
|
|
68
|
+
MultipartUpload: { Parts: parts.map((part) => {
|
|
69
|
+
return {
|
|
70
|
+
ETag: part.ETag,
|
|
71
|
+
PartNumber: part.PartNumber
|
|
72
|
+
};
|
|
73
|
+
}) },
|
|
74
|
+
UploadId: metadata["upload-id"]
|
|
75
|
+
};
|
|
76
|
+
try {
|
|
77
|
+
const result = await this.client.completeMultipartUpload(params);
|
|
78
|
+
return result.Location;
|
|
79
|
+
} catch (_error) {
|
|
80
|
+
throwError(MediaCloudError.TUS_UPLOAD_ERROR);
|
|
81
|
+
throw new Error();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Gets incomplete part from S3
|
|
86
|
+
* @param args - The function arguments
|
|
87
|
+
* @param args.id - The upload ID
|
|
88
|
+
* @returns Promise that resolves to readable stream or undefined if not found
|
|
89
|
+
*/
|
|
90
|
+
async getIncompletePart(args) {
|
|
91
|
+
const { id } = args;
|
|
92
|
+
try {
|
|
93
|
+
const data = await this.client.getObject({
|
|
94
|
+
Bucket: this.bucket,
|
|
95
|
+
Key: this.metadataManager.generatePartKey({
|
|
96
|
+
id,
|
|
97
|
+
isIncomplete: true
|
|
98
|
+
})
|
|
99
|
+
});
|
|
100
|
+
return data.Body;
|
|
101
|
+
} catch (error) {
|
|
102
|
+
if (error instanceof NoSuchKey) return void 0;
|
|
103
|
+
throw error;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Gets the size of an incomplete part
|
|
108
|
+
* @param args - The function arguments
|
|
109
|
+
* @param args.id - The upload ID
|
|
110
|
+
* @returns Promise that resolves to part size or undefined if not found
|
|
111
|
+
*/
|
|
112
|
+
async getIncompletePartSize(args) {
|
|
113
|
+
const { id } = args;
|
|
114
|
+
try {
|
|
115
|
+
const data = await this.client.headObject({
|
|
116
|
+
Bucket: this.bucket,
|
|
117
|
+
Key: this.metadataManager.generatePartKey({
|
|
118
|
+
id,
|
|
119
|
+
isIncomplete: true
|
|
120
|
+
})
|
|
121
|
+
});
|
|
122
|
+
return data.ContentLength;
|
|
123
|
+
} catch (error) {
|
|
124
|
+
if (error instanceof NotFound) return void 0;
|
|
125
|
+
throw error;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Deletes an incomplete part
|
|
130
|
+
* @param args - The function arguments
|
|
131
|
+
* @param args.id - The upload ID
|
|
132
|
+
* @returns Promise that resolves when deletion is complete
|
|
133
|
+
*/
|
|
134
|
+
async deleteIncompletePart(args) {
|
|
135
|
+
const { id } = args;
|
|
136
|
+
await this.client.deleteObject({
|
|
137
|
+
Bucket: this.bucket,
|
|
138
|
+
Key: this.metadataManager.generatePartKey({
|
|
139
|
+
id,
|
|
140
|
+
isIncomplete: true
|
|
141
|
+
})
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Downloads incomplete part to temporary file
|
|
146
|
+
* @param args - The function arguments
|
|
147
|
+
* @param args.id - The upload ID
|
|
148
|
+
* @returns Promise that resolves to incomplete part info or undefined if not found
|
|
149
|
+
*/
|
|
150
|
+
async downloadIncompletePart(args) {
|
|
151
|
+
const { id } = args;
|
|
152
|
+
const incompletePart = await this.getIncompletePart({ id });
|
|
153
|
+
if (!incompletePart) return;
|
|
154
|
+
const filePath = await this.fileOperations.generateUniqueTmpFileName({ template: "tus-s3-incomplete-part-" });
|
|
155
|
+
try {
|
|
156
|
+
let incompletePartSize = 0;
|
|
157
|
+
const byteCounterTransform = new stream.Transform({ transform(chunk, _, callback) {
|
|
158
|
+
incompletePartSize += chunk.length;
|
|
159
|
+
callback(null, chunk);
|
|
160
|
+
} });
|
|
161
|
+
await stream.promises.pipeline(incompletePart, byteCounterTransform, fs.createWriteStream(filePath));
|
|
162
|
+
const createReadStream = (options) => {
|
|
163
|
+
const fileReader = fs.createReadStream(filePath);
|
|
164
|
+
if (options.cleanUpOnEnd) {
|
|
165
|
+
fileReader.on("end", () => {
|
|
166
|
+
fs.unlink(filePath, () => {});
|
|
167
|
+
});
|
|
168
|
+
fileReader.on("error", (err) => {
|
|
169
|
+
fileReader.destroy(err);
|
|
170
|
+
fs.unlink(filePath, () => {});
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
return fileReader;
|
|
174
|
+
};
|
|
175
|
+
return {
|
|
176
|
+
createReader: createReadStream,
|
|
177
|
+
path: filePath,
|
|
178
|
+
size: incompletePartSize
|
|
179
|
+
};
|
|
180
|
+
} catch (err) {
|
|
181
|
+
fs.promises.rm(filePath).catch(() => {});
|
|
182
|
+
throw err;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Uploads an incomplete part
|
|
187
|
+
* @param args - The function arguments
|
|
188
|
+
* @param args.id - The upload ID
|
|
189
|
+
* @param args.readStream - The stream to read data from
|
|
190
|
+
* @returns Promise that resolves to the ETag of the uploaded part
|
|
191
|
+
*/
|
|
192
|
+
async uploadIncompletePart(args) {
|
|
193
|
+
const { id, readStream } = args;
|
|
194
|
+
const data = await this.client.putObject({
|
|
195
|
+
Body: readStream,
|
|
196
|
+
Bucket: this.bucket,
|
|
197
|
+
Key: this.metadataManager.generatePartKey({
|
|
198
|
+
id,
|
|
199
|
+
isIncomplete: true
|
|
200
|
+
}),
|
|
201
|
+
Tagging: this.generateCompleteTag("false")
|
|
202
|
+
});
|
|
203
|
+
log(`[${id}] finished uploading incomplete part`);
|
|
204
|
+
return data.ETag;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Uploads a single part
|
|
208
|
+
* @param args - The function arguments
|
|
209
|
+
* @param args.metadata - The upload metadata
|
|
210
|
+
* @param args.readStream - The stream to read data from
|
|
211
|
+
* @param args.partNumber - The part number to upload
|
|
212
|
+
* @returns Promise that resolves to the ETag of the uploaded part
|
|
213
|
+
*/
|
|
214
|
+
async uploadPart(args) {
|
|
215
|
+
const { metadata, readStream, partNumber } = args;
|
|
216
|
+
const permit = await this.partUploadSemaphore.acquire();
|
|
217
|
+
if (!metadata["upload-id"]) {
|
|
218
|
+
throwError(MediaCloudError.MUX_UPLOAD_ID_MISSING);
|
|
219
|
+
throw new Error();
|
|
220
|
+
}
|
|
221
|
+
const params = {
|
|
222
|
+
Body: readStream,
|
|
223
|
+
Bucket: this.bucket,
|
|
224
|
+
Key: metadata.file.id,
|
|
225
|
+
PartNumber: partNumber,
|
|
226
|
+
UploadId: metadata["upload-id"]
|
|
227
|
+
};
|
|
228
|
+
try {
|
|
229
|
+
const data = await this.client.uploadPart(params);
|
|
230
|
+
return {
|
|
231
|
+
ETag: data.ETag,
|
|
232
|
+
PartNumber: partNumber
|
|
233
|
+
};
|
|
234
|
+
} catch (_error) {
|
|
235
|
+
throwError(MediaCloudError.TUS_UPLOAD_ERROR);
|
|
236
|
+
throw new Error();
|
|
237
|
+
} finally {
|
|
238
|
+
permit();
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Uploads a stream to s3 using multiple parts
|
|
243
|
+
* @param args - The function arguments
|
|
244
|
+
* @param args.metadata - The upload metadata
|
|
245
|
+
* @param args.readStream - The stream to read data from
|
|
246
|
+
* @param args.currentPartNumber - The current part number to start from
|
|
247
|
+
* @param args.offset - The byte offset to start from
|
|
248
|
+
* @returns Promise that resolves to the number of bytes uploaded
|
|
249
|
+
*/
|
|
250
|
+
async uploadParts(args) {
|
|
251
|
+
const { metadata, readStream, offset: initialOffset } = args;
|
|
252
|
+
let { currentPartNumber } = args;
|
|
253
|
+
let offset = initialOffset;
|
|
254
|
+
const size = metadata.file.size;
|
|
255
|
+
const promises = [];
|
|
256
|
+
let pendingChunkFilepath = null;
|
|
257
|
+
let bytesUploaded = 0;
|
|
258
|
+
let permit = void 0;
|
|
259
|
+
const splitterStream = new StreamSplitter({
|
|
260
|
+
chunkSize: this.fileOperations.calculateOptimalPartSize({ size }),
|
|
261
|
+
directory: os.tmpdir()
|
|
262
|
+
}).on("beforeChunkStarted", async () => {
|
|
263
|
+
permit = await this.partUploadSemaphore.acquire();
|
|
264
|
+
}).on("chunkStarted", (filepath) => {
|
|
265
|
+
pendingChunkFilepath = filepath;
|
|
266
|
+
}).on("chunkFinished", ({ path, size: partSize }) => {
|
|
267
|
+
pendingChunkFilepath = null;
|
|
268
|
+
const acquiredPermit = permit;
|
|
269
|
+
const partNumber = currentPartNumber++;
|
|
270
|
+
offset += partSize;
|
|
271
|
+
const isFinalPart = size === offset;
|
|
272
|
+
const uploadChunk = async () => {
|
|
273
|
+
try {
|
|
274
|
+
const readable = fs.createReadStream(path);
|
|
275
|
+
readable.on("error", function(error) {
|
|
276
|
+
throw error;
|
|
277
|
+
});
|
|
278
|
+
switch (true) {
|
|
279
|
+
case partSize >= this.minPartSize || isFinalPart:
|
|
280
|
+
await this.uploadPart({
|
|
281
|
+
metadata,
|
|
282
|
+
readStream: readable,
|
|
283
|
+
partNumber
|
|
284
|
+
});
|
|
285
|
+
break;
|
|
286
|
+
default:
|
|
287
|
+
await this.uploadIncompletePart({
|
|
288
|
+
id: metadata.file.id,
|
|
289
|
+
readStream: readable
|
|
290
|
+
});
|
|
291
|
+
break;
|
|
292
|
+
}
|
|
293
|
+
bytesUploaded += partSize;
|
|
294
|
+
} catch (error) {
|
|
295
|
+
const mappedError = error instanceof Error ? error : new Error(String(error));
|
|
296
|
+
splitterStream.destroy(mappedError);
|
|
297
|
+
throw mappedError;
|
|
298
|
+
} finally {
|
|
299
|
+
fs.promises.rm(path).catch(function() {});
|
|
300
|
+
acquiredPermit?.();
|
|
301
|
+
}
|
|
302
|
+
};
|
|
303
|
+
const deferred = uploadChunk();
|
|
304
|
+
promises.push(deferred);
|
|
305
|
+
}).on("chunkError", () => {
|
|
306
|
+
permit?.();
|
|
307
|
+
});
|
|
308
|
+
try {
|
|
309
|
+
await stream.promises.pipeline(readStream, splitterStream);
|
|
310
|
+
} catch (error) {
|
|
311
|
+
if (pendingChunkFilepath !== null) try {
|
|
312
|
+
await fs.promises.rm(pendingChunkFilepath);
|
|
313
|
+
} catch {
|
|
314
|
+
log(`[${metadata.file.id}] failed to remove chunk ${String(pendingChunkFilepath)}`);
|
|
315
|
+
}
|
|
316
|
+
const mappedError = error instanceof Error ? error : new Error(String(error));
|
|
317
|
+
promises.push(Promise.reject(mappedError));
|
|
318
|
+
} finally {
|
|
319
|
+
await Promise.allSettled(promises);
|
|
320
|
+
await Promise.all(promises);
|
|
321
|
+
}
|
|
322
|
+
return bytesUploaded;
|
|
323
|
+
}
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
//#endregion
|
|
327
|
+
export { S3PartsManager };
|
|
328
|
+
//# sourceMappingURL=parts-manager.js.map
|