@maas/payload-plugin-media-cloud 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/LICENSE +8 -0
  2. package/dist/adapter/handleDelete.d.ts +20 -0
  3. package/dist/adapter/handleDelete.js +70 -0
  4. package/dist/adapter/handleDelete.js.map +1 -0
  5. package/dist/adapter/handleUpload.d.ts +12 -0
  6. package/dist/adapter/handleUpload.js +29 -0
  7. package/dist/adapter/handleUpload.js.map +1 -0
  8. package/dist/adapter/staticHandler.d.ts +17 -0
  9. package/dist/adapter/staticHandler.js +64 -0
  10. package/dist/adapter/staticHandler.js.map +1 -0
  11. package/dist/adapter/storageAdapter.d.ts +23 -0
  12. package/dist/adapter/storageAdapter.js +30 -0
  13. package/dist/adapter/storageAdapter.js.map +1 -0
  14. package/dist/collections/mediaCollection.d.ts +16 -0
  15. package/dist/collections/mediaCollection.js +139 -0
  16. package/dist/collections/mediaCollection.js.map +1 -0
  17. package/dist/components/index.d.ts +4 -0
  18. package/dist/components/index.js +5 -0
  19. package/dist/components/mux-preview/index.d.ts +2 -0
  20. package/dist/components/mux-preview/index.js +3 -0
  21. package/dist/components/mux-preview/mux-preview.d.ts +14 -0
  22. package/dist/components/mux-preview/mux-preview.js +38 -0
  23. package/dist/components/mux-preview/mux-preview.js.map +1 -0
  24. package/dist/components/upload-handler/index.d.ts +2 -0
  25. package/dist/components/upload-handler/index.js +3 -0
  26. package/dist/components/upload-handler/upload-handler.d.ts +22 -0
  27. package/dist/components/upload-handler/upload-handler.js +178 -0
  28. package/dist/components/upload-handler/upload-handler.js.map +1 -0
  29. package/dist/components/upload-manager/index.d.ts +2 -0
  30. package/dist/components/upload-manager/index.js +3 -0
  31. package/dist/components/upload-manager/upload-manager-DN4RrmYB.css +204 -0
  32. package/dist/components/upload-manager/upload-manager-DN4RrmYB.css.map +1 -0
  33. package/dist/components/upload-manager/upload-manager.css +201 -0
  34. package/dist/components/upload-manager/upload-manager.d.ts +42 -0
  35. package/dist/components/upload-manager/upload-manager.js +315 -0
  36. package/dist/components/upload-manager/upload-manager.js.map +1 -0
  37. package/dist/components/upload-manager/upload-manager2.js +0 -0
  38. package/dist/endpoints/muxAssetHandler.d.ts +11 -0
  39. package/dist/endpoints/muxAssetHandler.js +59 -0
  40. package/dist/endpoints/muxAssetHandler.js.map +1 -0
  41. package/dist/endpoints/muxCreateUploadHandler.d.ts +13 -0
  42. package/dist/endpoints/muxCreateUploadHandler.js +40 -0
  43. package/dist/endpoints/muxCreateUploadHandler.js.map +1 -0
  44. package/dist/endpoints/muxWebhookHandler.d.ts +11 -0
  45. package/dist/endpoints/muxWebhookHandler.js +49 -0
  46. package/dist/endpoints/muxWebhookHandler.js.map +1 -0
  47. package/dist/hooks/useEmitter.d.ts +48 -0
  48. package/dist/hooks/useEmitter.js +19 -0
  49. package/dist/hooks/useEmitter.js.map +1 -0
  50. package/dist/hooks/useErrorHandler.d.ts +11 -0
  51. package/dist/hooks/useErrorHandler.js +19 -0
  52. package/dist/hooks/useErrorHandler.js.map +1 -0
  53. package/dist/index.d.ts +3 -0
  54. package/dist/index.js +3 -0
  55. package/dist/plugin.d.ts +15 -0
  56. package/dist/plugin.js +242 -0
  57. package/dist/plugin.js.map +1 -0
  58. package/dist/tus/stores/s3/expiration-manager.d.ts +36 -0
  59. package/dist/tus/stores/s3/expiration-manager.js +76 -0
  60. package/dist/tus/stores/s3/expiration-manager.js.map +1 -0
  61. package/dist/tus/stores/s3/file-operations.d.ts +66 -0
  62. package/dist/tus/stores/s3/file-operations.js +90 -0
  63. package/dist/tus/stores/s3/file-operations.js.map +1 -0
  64. package/dist/tus/stores/s3/log.d.ts +5 -0
  65. package/dist/tus/stores/s3/log.js +8 -0
  66. package/dist/tus/stores/s3/log.js.map +1 -0
  67. package/dist/tus/stores/s3/metadata-manager.d.ts +85 -0
  68. package/dist/tus/stores/s3/metadata-manager.js +135 -0
  69. package/dist/tus/stores/s3/metadata-manager.js.map +1 -0
  70. package/dist/tus/stores/s3/parts-manager.d.ts +130 -0
  71. package/dist/tus/stores/s3/parts-manager.js +328 -0
  72. package/dist/tus/stores/s3/parts-manager.js.map +1 -0
  73. package/dist/tus/stores/s3/s3-store.d.ts +110 -0
  74. package/dist/tus/stores/s3/s3-store.js +342 -0
  75. package/dist/tus/stores/s3/s3-store.js.map +1 -0
  76. package/dist/tus/stores/s3/semaphore.d.ts +16 -0
  77. package/dist/tus/stores/s3/semaphore.js +32 -0
  78. package/dist/tus/stores/s3/semaphore.js.map +1 -0
  79. package/dist/types/errors.d.ts +26 -0
  80. package/dist/types/errors.js +28 -0
  81. package/dist/types/errors.js.map +1 -0
  82. package/dist/types/index.d.ts +73 -0
  83. package/dist/types/index.js +0 -0
  84. package/dist/utils/file.d.ts +30 -0
  85. package/dist/utils/file.js +84 -0
  86. package/dist/utils/file.js.map +1 -0
  87. package/package.json +92 -0
@@ -0,0 +1,85 @@
1
+ import { TusUploadMetadata } from "../../../types/index.js";
2
+ import { S3 } from "@aws-sdk/client-s3";
3
+ import { KvStore, Upload } from "@tus/utils";
4
+
5
+ //#region src/tus/stores/s3/metadata-manager.d.ts
6
+ type GenerateInfoKeyArgs = {
7
+ id: string;
8
+ };
9
+ type GeneratePartKeyArgs = {
10
+ id: string;
11
+ isIncomplete?: boolean;
12
+ };
13
+ type GetMetadataArgs = {
14
+ id: string;
15
+ };
16
+ type SaveMetadataArgs = {
17
+ upload: Upload;
18
+ uploadId: string;
19
+ };
20
+ type CompleteMetadataArgs = {
21
+ upload: Upload;
22
+ };
23
+ type ClearCacheArgs = {
24
+ id: string;
25
+ };
26
+ declare class S3MetadataManager {
27
+ private client;
28
+ private bucket;
29
+ private cache;
30
+ private shouldUseExpirationTags;
31
+ private generateCompleteTag;
32
+ constructor(client: S3, bucket: string, cache: KvStore<TusUploadMetadata>, shouldUseExpirationTags: () => boolean, generateCompleteTag: (value: 'false' | 'true') => string | undefined);
33
+ /**
34
+ * Generates the S3 key for metadata info files
35
+ * @param args - The function arguments
36
+ * @param args.id - The file ID
37
+ * @returns The S3 key for the metadata file
38
+ */
39
+ generateInfoKey(args: GenerateInfoKeyArgs): string;
40
+ /**
41
+ * Generates the S3 key for part files
42
+ * @param args - The function arguments
43
+ * @param args.id - The file ID
44
+ * @param args.isIncomplete - Whether this is an incomplete part
45
+ * @returns The S3 key for the part file
46
+ */
47
+ generatePartKey(args: GeneratePartKeyArgs): string;
48
+ /**
49
+ * Retrieves upload metadata previously saved in `${file_id}.info`.
50
+ * There's a small and simple caching mechanism to avoid multiple
51
+ * HTTP calls to S3.
52
+ * @param args - The function arguments
53
+ * @param args.id - The file ID to retrieve metadata for
54
+ * @returns Promise that resolves to the upload metadata
55
+ */
56
+ getMetadata(args: GetMetadataArgs): Promise<TusUploadMetadata>;
57
+ /**
58
+ * Saves upload metadata to a `${file_id}.info` file on S3.
59
+ * Please note that the file is empty and the metadata is saved
60
+ * on the S3 object's `Metadata` field, so that only a `headObject`
61
+ * is necessary to retrieve the data.
62
+ * @param args - The function arguments
63
+ * @param args.upload - The upload object containing metadata
64
+ * @param args.uploadId - The upload ID for the multipart upload
65
+ * @returns Promise that resolves when metadata is saved
66
+ */
67
+ saveMetadata(args: SaveMetadataArgs): Promise<void>;
68
+ /**
69
+ * Completes metadata by updating it with completion tags
70
+ * @param args - The function arguments
71
+ * @param args.upload - The completed upload object
72
+ * @returns Promise that resolves when metadata is updated
73
+ */
74
+ completeMetadata(args: CompleteMetadataArgs): Promise<void>;
75
+ /**
76
+ * Removes cached data for a given file
77
+ * @param args - The function arguments
78
+ * @param args.id - The file ID to remove from cache
79
+ * @returns Promise that resolves when cache is cleared
80
+ */
81
+ clearCache(args: ClearCacheArgs): Promise<void>;
82
+ }
83
+ //#endregion
84
+ export { S3MetadataManager };
85
+ //# sourceMappingURL=metadata-manager.d.ts.map
@@ -0,0 +1,135 @@
1
+ import { log } from "./log.js";
2
+ import { TUS_RESUMABLE, Upload } from "@tus/utils";
3
+
4
+ //#region src/tus/stores/s3/metadata-manager.ts
5
+ var S3MetadataManager = class {
6
+ constructor(client, bucket, cache, shouldUseExpirationTags, generateCompleteTag) {
7
+ this.client = client;
8
+ this.bucket = bucket;
9
+ this.cache = cache;
10
+ this.shouldUseExpirationTags = shouldUseExpirationTags;
11
+ this.generateCompleteTag = generateCompleteTag;
12
+ }
13
+ /**
14
+ * Generates the S3 key for metadata info files
15
+ * @param args - The function arguments
16
+ * @param args.id - The file ID
17
+ * @returns The S3 key for the metadata file
18
+ */
19
+ generateInfoKey(args) {
20
+ const { id } = args;
21
+ return `${id}.info`;
22
+ }
23
+ /**
24
+ * Generates the S3 key for part files
25
+ * @param args - The function arguments
26
+ * @param args.id - The file ID
27
+ * @param args.isIncomplete - Whether this is an incomplete part
28
+ * @returns The S3 key for the part file
29
+ */
30
+ generatePartKey(args) {
31
+ const { id, isIncomplete = false } = args;
32
+ let key = id;
33
+ if (isIncomplete) key += ".part";
34
+ return key;
35
+ }
36
+ /**
37
+ * Retrieves upload metadata previously saved in `${file_id}.info`.
38
+ * There's a small and simple caching mechanism to avoid multiple
39
+ * HTTP calls to S3.
40
+ * @param args - The function arguments
41
+ * @param args.id - The file ID to retrieve metadata for
42
+ * @returns Promise that resolves to the upload metadata
43
+ */
44
+ async getMetadata(args) {
45
+ const { id } = args;
46
+ const cached = await this.cache.get(id);
47
+ if (cached) return cached;
48
+ const { Body, Metadata } = await this.client.getObject({
49
+ Bucket: this.bucket,
50
+ Key: this.generateInfoKey({ id })
51
+ });
52
+ const file = JSON.parse(await Body?.transformToString());
53
+ const metadata = {
54
+ file: new Upload({
55
+ id,
56
+ creation_date: file.creation_date,
57
+ metadata: file.metadata,
58
+ offset: Number.parseInt(file.offset, 10),
59
+ size: Number.isFinite(file.size) ? Number.parseInt(file.size, 10) : void 0,
60
+ storage: file.storage
61
+ }),
62
+ "tus-version": Metadata?.["tus-version"],
63
+ "upload-id": Metadata?.["upload-id"]
64
+ };
65
+ await this.cache.set(id, metadata);
66
+ return metadata;
67
+ }
68
+ /**
69
+ * Saves upload metadata to a `${file_id}.info` file on S3.
70
+ * Please note that the file is empty and the metadata is saved
71
+ * on the S3 object's `Metadata` field, so that only a `headObject`
72
+ * is necessary to retrieve the data.
73
+ * @param args - The function arguments
74
+ * @param args.upload - The upload object containing metadata
75
+ * @param args.uploadId - The upload ID for the multipart upload
76
+ * @returns Promise that resolves when metadata is saved
77
+ */
78
+ async saveMetadata(args) {
79
+ const { upload, uploadId } = args;
80
+ log(`[${upload.id}] saving metadata`);
81
+ await this.client.putObject({
82
+ Body: JSON.stringify(upload),
83
+ Bucket: this.bucket,
84
+ Key: this.generateInfoKey({ id: upload.id }),
85
+ Metadata: {
86
+ "tus-version": TUS_RESUMABLE,
87
+ "upload-id": uploadId
88
+ },
89
+ Tagging: this.generateCompleteTag("false")
90
+ });
91
+ log(`[${upload.id}] metadata file saved`);
92
+ }
93
+ /**
94
+ * Completes metadata by updating it with completion tags
95
+ * @param args - The function arguments
96
+ * @param args.upload - The completed upload object
97
+ * @returns Promise that resolves when metadata is updated
98
+ */
99
+ async completeMetadata(args) {
100
+ const { upload } = args;
101
+ const metadata = await this.getMetadata({ id: upload.id });
102
+ const completedMetadata = {
103
+ ...metadata,
104
+ file: upload
105
+ };
106
+ await this.cache.set(upload.id, completedMetadata);
107
+ if (!this.shouldUseExpirationTags()) return;
108
+ const { "upload-id": uploadId } = metadata;
109
+ await this.client.putObject({
110
+ Body: JSON.stringify(upload),
111
+ Bucket: this.bucket,
112
+ Key: this.generateInfoKey({ id: upload.id }),
113
+ Metadata: {
114
+ "tus-version": TUS_RESUMABLE,
115
+ "upload-id": uploadId
116
+ },
117
+ Tagging: this.generateCompleteTag("true")
118
+ });
119
+ }
120
+ /**
121
+ * Removes cached data for a given file
122
+ * @param args - The function arguments
123
+ * @param args.id - The file ID to remove from cache
124
+ * @returns Promise that resolves when cache is cleared
125
+ */
126
+ async clearCache(args) {
127
+ const { id } = args;
128
+ log(`[${id}] removing cached data`);
129
+ await this.cache.delete(id);
130
+ }
131
+ };
132
+
133
+ //#endregion
134
+ export { S3MetadataManager };
135
+ //# sourceMappingURL=metadata-manager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metadata-manager.js","names":["client: S3","bucket: string","cache: KvStore<TusUploadMetadata>","shouldUseExpirationTags: () => boolean","generateCompleteTag: (value: 'false' | 'true') => string | undefined","args: GenerateInfoKeyArgs","args: GeneratePartKeyArgs","args: GetMetadataArgs","metadata: TusUploadMetadata","args: SaveMetadataArgs","args: CompleteMetadataArgs","completedMetadata: TusUploadMetadata","args: ClearCacheArgs"],"sources":["../../../../src/tus/stores/s3/metadata-manager.ts"],"sourcesContent":["import { TUS_RESUMABLE, Upload } from '@tus/utils'\nimport { log } from './log'\n\nimport type { S3 } from '@aws-sdk/client-s3'\nimport type { KvStore } from '@tus/utils'\nimport type { TusUploadMetadata } from '../../../types'\n\ntype GenerateInfoKeyArgs = {\n id: string\n}\n\ntype GeneratePartKeyArgs = {\n id: string\n isIncomplete?: boolean\n}\n\ntype GetMetadataArgs = {\n id: string\n}\n\ntype SaveMetadataArgs = {\n upload: Upload\n uploadId: string\n}\n\ntype CompleteMetadataArgs = {\n upload: Upload\n}\n\ntype ClearCacheArgs = {\n id: string\n}\n\nexport class S3MetadataManager {\n constructor(\n private client: S3,\n private bucket: string,\n private cache: KvStore<TusUploadMetadata>,\n private shouldUseExpirationTags: () => boolean,\n private generateCompleteTag: (value: 'false' | 'true') => string | undefined\n ) {}\n\n /**\n * Generates the S3 key for metadata info files\n * @param args - The function arguments\n * @param args.id - The file ID\n * @returns The S3 key for the metadata file\n */\n generateInfoKey(args: GenerateInfoKeyArgs): string {\n const { id } = args\n return `${id}.info`\n }\n\n /**\n * Generates the S3 key for part files\n * @param args - The function arguments\n * @param args.id - The file ID\n * @param args.isIncomplete - Whether this is an incomplete part\n * @returns The S3 key for the part file\n */\n generatePartKey(args: GeneratePartKeyArgs): string {\n const { id, isIncomplete = false } = args\n let key = id\n if (isIncomplete) {\n key += '.part'\n }\n\n // TODO: introduce ObjectPrefixing for parts and incomplete parts.\n // ObjectPrefix is prepended to the name of each S3 object that is created\n // to store uploaded files. It can be used to create a pseudo-directory\n // structure in the bucket, e.g. \"path/to/my/uploads\".\n return key\n }\n\n /**\n * Retrieves upload metadata previously saved in `${file_id}.info`.\n * There's a small and simple caching mechanism to avoid multiple\n * HTTP calls to S3.\n * @param args - The function arguments\n * @param args.id - The file ID to retrieve metadata for\n * @returns Promise that resolves to the upload metadata\n */\n async getMetadata(args: GetMetadataArgs): Promise<TusUploadMetadata> {\n const { id } = args\n const cached = await this.cache.get(id)\n if (cached) {\n return cached\n }\n\n const { Body, Metadata } = await this.client.getObject({\n Bucket: this.bucket,\n Key: this.generateInfoKey({ id }),\n })\n const file = JSON.parse((await Body?.transformToString()) as string)\n const metadata: TusUploadMetadata = {\n file: new Upload({\n id,\n creation_date: file.creation_date,\n metadata: file.metadata,\n offset: Number.parseInt(file.offset, 10),\n size: Number.isFinite(file.size)\n ? Number.parseInt(file.size, 10)\n : undefined,\n storage: file.storage,\n }),\n 'tus-version': Metadata?.['tus-version'] as string,\n 'upload-id': Metadata?.['upload-id'] as string,\n }\n await this.cache.set(id, metadata)\n return metadata\n }\n\n /**\n * Saves upload metadata to a `${file_id}.info` file on S3.\n * Please note that the file is empty and the metadata is saved\n * on the S3 object's `Metadata` field, so that only a `headObject`\n * is necessary to retrieve the data.\n * @param args - The function arguments\n * @param args.upload - The upload object containing metadata\n * @param args.uploadId - The upload ID for the multipart upload\n * @returns Promise that resolves when metadata is saved\n */\n async saveMetadata(args: SaveMetadataArgs): Promise<void> {\n const { upload, uploadId } = args\n log(`[${upload.id}] saving metadata`)\n await this.client.putObject({\n Body: JSON.stringify(upload),\n Bucket: this.bucket,\n Key: this.generateInfoKey({ id: upload.id }),\n Metadata: {\n 'tus-version': TUS_RESUMABLE,\n 'upload-id': uploadId,\n },\n Tagging: this.generateCompleteTag('false'),\n })\n log(`[${upload.id}] metadata file saved`)\n }\n\n /**\n * Completes metadata by updating it with completion tags\n * @param args - The function arguments\n * @param args.upload - The completed upload object\n * @returns Promise that resolves when metadata is updated\n */\n async completeMetadata(args: CompleteMetadataArgs): Promise<void> {\n const { upload } = args\n // Always update the cache with the completed upload\n const metadata = await this.getMetadata({ id: upload.id })\n const completedMetadata: TusUploadMetadata = {\n ...metadata,\n file: upload,\n }\n await this.cache.set(upload.id, completedMetadata)\n\n if (!this.shouldUseExpirationTags()) {\n return\n }\n\n const { 'upload-id': uploadId } = metadata\n await this.client.putObject({\n Body: JSON.stringify(upload),\n Bucket: this.bucket,\n Key: this.generateInfoKey({ id: upload.id }),\n Metadata: {\n 'tus-version': TUS_RESUMABLE,\n 'upload-id': uploadId,\n },\n Tagging: this.generateCompleteTag('true'),\n })\n }\n\n /**\n * Removes cached data for a given file\n * @param args - The function arguments\n * @param args.id - The file ID to remove from cache\n * @returns Promise that resolves when cache is cleared\n */\n async clearCache(args: ClearCacheArgs): Promise<void> {\n const { id } = args\n log(`[${id}] removing cached data`)\n await this.cache.delete(id)\n }\n}\n"],"mappings":";;;;AAiCA,IAAa,oBAAb,MAA+B;CAC7B,YACUA,QACAC,QACAC,OACAC,yBACAC,qBACR;EALQ;EACA;EACA;EACA;EACA;CACN;;;;;;;CAQJ,gBAAgBC,MAAmC;EACjD,MAAM,EAAE,IAAI,GAAG;AACf,SAAO,GAAG,GAAG,KAAK,CAAC;CACpB;;;;;;;;CASD,gBAAgBC,MAAmC;EACjD,MAAM,EAAE,IAAI,eAAe,OAAO,GAAG;EACrC,IAAI,MAAM;AACV,MAAI,cACF,OAAO;AAOT,SAAO;CACR;;;;;;;;;CAUD,MAAM,YAAYC,MAAmD;EACnE,MAAM,EAAE,IAAI,GAAG;EACf,MAAM,SAAS,MAAM,KAAK,MAAM,IAAI,GAAG;AACvC,MAAI,OACF,QAAO;EAGT,MAAM,EAAE,MAAM,UAAU,GAAG,MAAM,KAAK,OAAO,UAAU;GACrD,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,EAAE,GAAI,EAAC;EAClC,EAAC;EACF,MAAM,OAAO,KAAK,MAAO,MAAM,MAAM,mBAAmB,CAAY;EACpE,MAAMC,WAA8B;GAClC,MAAM,IAAI,OAAO;IACf;IACA,eAAe,KAAK;IACpB,UAAU,KAAK;IACf,QAAQ,OAAO,SAAS,KAAK,QAAQ,GAAG;IACxC,MAAM,OAAO,SAAS,KAAK,KAAK,GAC5B,OAAO,SAAS,KAAK,MAAM,GAAG,GAC9B;IACJ,SAAS,KAAK;GACf;GACD,eAAe,WAAW;GAC1B,aAAa,WAAW;EACzB;EACD,MAAM,KAAK,MAAM,IAAI,IAAI,SAAS;AAClC,SAAO;CACR;;;;;;;;;;;CAYD,MAAM,aAAaC,MAAuC;EACxD,MAAM,EAAE,QAAQ,UAAU,GAAG;EAC7B,IAAI,CAAC,CAAC,EAAE,OAAO,GAAG,iBAAiB,CAAC,CAAC;EACrC,MAAM,KAAK,OAAO,UAAU;GAC1B,MAAM,KAAK,UAAU,OAAO;GAC5B,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,EAAE,IAAI,OAAO,GAAI,EAAC;GAC5C,UAAU;IACR,eAAe;IACf,aAAa;GACd;GACD,SAAS,KAAK,oBAAoB,QAAQ;EAC3C,EAAC;EACF,IAAI,CAAC,CAAC,EAAE,OAAO,GAAG,qBAAqB,CAAC,CAAC;CAC1C;;;;;;;CAQD,MAAM,iBAAiBC,MAA2C;EAChE,MAAM,EAAE,QAAQ,GAAG;EAEnB,MAAM,WAAW,MAAM,KAAK,YAAY,EAAE,IAAI,OAAO,GAAI,EAAC;EAC1D,MAAMC,oBAAuC;GAC3C,GAAG;GACH,MAAM;EACP;EACD,MAAM,KAAK,MAAM,IAAI,OAAO,IAAI,kBAAkB;AAElD,MAAI,CAAC,KAAK,yBAAyB,CACjC;EAGF,MAAM,EAAE,aAAa,UAAU,GAAG;EAClC,MAAM,KAAK,OAAO,UAAU;GAC1B,MAAM,KAAK,UAAU,OAAO;GAC5B,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,EAAE,IAAI,OAAO,GAAI,EAAC;GAC5C,UAAU;IACR,eAAe;IACf,aAAa;GACd;GACD,SAAS,KAAK,oBAAoB,OAAO;EAC1C,EAAC;CACH;;;;;;;CAQD,MAAM,WAAWC,MAAqC;EACpD,MAAM,EAAE,IAAI,GAAG;EACf,IAAI,CAAC,CAAC,EAAE,GAAG,sBAAsB,CAAC,CAAC;EACnC,MAAM,KAAK,MAAM,OAAO,GAAG;CAC5B;AACF"}
@@ -0,0 +1,130 @@
1
+ import { Semaphore } from "./semaphore.js";
2
+ import { S3FileOperations } from "./file-operations.js";
3
+ import { IncompletePartInfo, TusUploadMetadata } from "../../../types/index.js";
4
+ import { S3MetadataManager } from "./metadata-manager.js";
5
+ import stream, { Readable } from "node:stream";
6
+ import AWS, { S3 } from "@aws-sdk/client-s3";
7
+ import fs from "node:fs";
8
+
9
+ //#region src/tus/stores/s3/parts-manager.d.ts
10
+ type RetrievePartsArgs = {
11
+ id: string;
12
+ partNumberMarker?: string;
13
+ };
14
+ type FinishMultipartUploadArgs = {
15
+ metadata: TusUploadMetadata;
16
+ parts: Array<AWS.Part>;
17
+ };
18
+ type GetIncompletePartArgs = {
19
+ id: string;
20
+ };
21
+ type GetIncompletePartSizeArgs = {
22
+ id: string;
23
+ };
24
+ type DeleteIncompletePartArgs = {
25
+ id: string;
26
+ };
27
+ type DownloadIncompletePartArgs = {
28
+ id: string;
29
+ };
30
+ type UploadIncompletePartArgs = {
31
+ id: string;
32
+ readStream: fs.ReadStream | Readable;
33
+ };
34
+ type UploadPartArgs = {
35
+ metadata: TusUploadMetadata;
36
+ readStream: fs.ReadStream | Readable;
37
+ partNumber: number;
38
+ };
39
+ type UploadPartsArgs = {
40
+ metadata: TusUploadMetadata;
41
+ readStream: stream.Readable;
42
+ currentPartNumber: number;
43
+ offset: number;
44
+ };
45
+ declare class S3PartsManager {
46
+ private client;
47
+ private bucket;
48
+ private minPartSize;
49
+ private partUploadSemaphore;
50
+ private metadataManager;
51
+ private fileOperations;
52
+ private generateCompleteTag;
53
+ constructor(client: S3, bucket: string, minPartSize: number, partUploadSemaphore: Semaphore, metadataManager: S3MetadataManager, fileOperations: S3FileOperations, generateCompleteTag: (value: 'false' | 'true') => string | undefined);
54
+ /**
55
+ * Gets the number of complete parts/chunks already uploaded to S3.
56
+ * Retrieves only consecutive parts.
57
+ * @param args - The function arguments
58
+ * @param args.id - The upload ID
59
+ * @param args.partNumberMarker - Marker for pagination (optional)
60
+ * @returns Promise that resolves to array of uploaded parts
61
+ */
62
+ retrieveParts(args: RetrievePartsArgs): Promise<Array<AWS.Part>>;
63
+ /**
64
+ * Completes a multipart upload on S3.
65
+ * This is where S3 concatenates all the uploaded parts.
66
+ * @param args - The function arguments
67
+ * @param args.metadata - The upload metadata
68
+ * @param args.parts - Array of uploaded parts to complete
69
+ * @returns Promise that resolves to the location URL (optional)
70
+ */
71
+ finishMultipartUpload(args: FinishMultipartUploadArgs): Promise<string | undefined>;
72
+ /**
73
+ * Gets incomplete part from S3
74
+ * @param args - The function arguments
75
+ * @param args.id - The upload ID
76
+ * @returns Promise that resolves to readable stream or undefined if not found
77
+ */
78
+ getIncompletePart(args: GetIncompletePartArgs): Promise<Readable | undefined>;
79
+ /**
80
+ * Gets the size of an incomplete part
81
+ * @param args - The function arguments
82
+ * @param args.id - The upload ID
83
+ * @returns Promise that resolves to part size or undefined if not found
84
+ */
85
+ getIncompletePartSize(args: GetIncompletePartSizeArgs): Promise<number | undefined>;
86
+ /**
87
+ * Deletes an incomplete part
88
+ * @param args - The function arguments
89
+ * @param args.id - The upload ID
90
+ * @returns Promise that resolves when deletion is complete
91
+ */
92
+ deleteIncompletePart(args: DeleteIncompletePartArgs): Promise<void>;
93
+ /**
94
+ * Downloads incomplete part to temporary file
95
+ * @param args - The function arguments
96
+ * @param args.id - The upload ID
97
+ * @returns Promise that resolves to incomplete part info or undefined if not found
98
+ */
99
+ downloadIncompletePart(args: DownloadIncompletePartArgs): Promise<IncompletePartInfo | undefined>;
100
+ /**
101
+ * Uploads an incomplete part
102
+ * @param args - The function arguments
103
+ * @param args.id - The upload ID
104
+ * @param args.readStream - The stream to read data from
105
+ * @returns Promise that resolves to the ETag of the uploaded part
106
+ */
107
+ uploadIncompletePart(args: UploadIncompletePartArgs): Promise<string>;
108
+ /**
109
+ * Uploads a single part
110
+ * @param args - The function arguments
111
+ * @param args.metadata - The upload metadata
112
+ * @param args.readStream - The stream to read data from
113
+ * @param args.partNumber - The part number to upload
114
+ * @returns Promise that resolves to the ETag of the uploaded part
115
+ */
116
+ uploadPart(args: UploadPartArgs): Promise<AWS.Part>;
117
+ /**
118
+ * Uploads a stream to s3 using multiple parts
119
+ * @param args - The function arguments
120
+ * @param args.metadata - The upload metadata
121
+ * @param args.readStream - The stream to read data from
122
+ * @param args.currentPartNumber - The current part number to start from
123
+ * @param args.offset - The byte offset to start from
124
+ * @returns Promise that resolves to the number of bytes uploaded
125
+ */
126
+ uploadParts(args: UploadPartsArgs): Promise<number>;
127
+ }
128
+ //#endregion
129
+ export { S3PartsManager };
130
+ //# sourceMappingURL=parts-manager.d.ts.map
@@ -0,0 +1,328 @@
1
+ import { MediaCloudError } from "../../../types/errors.js";
2
+ import { useErrorHandler } from "../../../hooks/useErrorHandler.js";
3
+ import { log } from "./log.js";
4
+ import stream from "node:stream";
5
+ import { NoSuchKey, NotFound } from "@aws-sdk/client-s3";
6
+ import { StreamSplitter } from "@tus/utils";
7
+ import fs from "node:fs";
8
+ import os from "node:os";
9
+
10
+ //#region src/tus/stores/s3/parts-manager.ts
11
+ const { throwError } = useErrorHandler();
12
+ var S3PartsManager = class {
13
+ constructor(client, bucket, minPartSize, partUploadSemaphore, metadataManager, fileOperations, generateCompleteTag) {
14
+ this.client = client;
15
+ this.bucket = bucket;
16
+ this.minPartSize = minPartSize;
17
+ this.partUploadSemaphore = partUploadSemaphore;
18
+ this.metadataManager = metadataManager;
19
+ this.fileOperations = fileOperations;
20
+ this.generateCompleteTag = generateCompleteTag;
21
+ }
22
+ /**
23
+ * Gets the number of complete parts/chunks already uploaded to S3.
24
+ * Retrieves only consecutive parts.
25
+ * @param args - The function arguments
26
+ * @param args.id - The upload ID
27
+ * @param args.partNumberMarker - Marker for pagination (optional)
28
+ * @returns Promise that resolves to array of uploaded parts
29
+ */
30
+ async retrieveParts(args) {
31
+ const { id, partNumberMarker } = args;
32
+ const metadata = await this.metadataManager.getMetadata({ id });
33
+ if (!metadata["upload-id"]) {
34
+ throwError(MediaCloudError.MUX_UPLOAD_ID_MISSING);
35
+ throw new Error();
36
+ }
37
+ const params = {
38
+ Bucket: this.bucket,
39
+ Key: id,
40
+ PartNumberMarker: partNumberMarker,
41
+ UploadId: metadata["upload-id"]
42
+ };
43
+ const data = await this.client.listParts(params);
44
+ let parts = data.Parts ?? [];
45
+ if (data.IsTruncated) {
46
+ const rest = await this.retrieveParts({
47
+ id,
48
+ partNumberMarker: data.NextPartNumberMarker
49
+ });
50
+ parts = [...parts, ...rest];
51
+ }
52
+ if (!partNumberMarker) parts.sort((a, b) => (a.PartNumber || 0) - (b.PartNumber || 0));
53
+ return parts;
54
+ }
55
+ /**
56
+ * Completes a multipart upload on S3.
57
+ * This is where S3 concatenates all the uploaded parts.
58
+ * @param args - The function arguments
59
+ * @param args.metadata - The upload metadata
60
+ * @param args.parts - Array of uploaded parts to complete
61
+ * @returns Promise that resolves to the location URL (optional)
62
+ */
63
+ async finishMultipartUpload(args) {
64
+ const { metadata, parts } = args;
65
+ const params = {
66
+ Bucket: this.bucket,
67
+ Key: metadata.file.id,
68
+ MultipartUpload: { Parts: parts.map((part) => {
69
+ return {
70
+ ETag: part.ETag,
71
+ PartNumber: part.PartNumber
72
+ };
73
+ }) },
74
+ UploadId: metadata["upload-id"]
75
+ };
76
+ try {
77
+ const result = await this.client.completeMultipartUpload(params);
78
+ return result.Location;
79
+ } catch (_error) {
80
+ throwError(MediaCloudError.TUS_UPLOAD_ERROR);
81
+ throw new Error();
82
+ }
83
+ }
84
+ /**
85
+ * Gets incomplete part from S3
86
+ * @param args - The function arguments
87
+ * @param args.id - The upload ID
88
+ * @returns Promise that resolves to readable stream or undefined if not found
89
+ */
90
+ async getIncompletePart(args) {
91
+ const { id } = args;
92
+ try {
93
+ const data = await this.client.getObject({
94
+ Bucket: this.bucket,
95
+ Key: this.metadataManager.generatePartKey({
96
+ id,
97
+ isIncomplete: true
98
+ })
99
+ });
100
+ return data.Body;
101
+ } catch (error) {
102
+ if (error instanceof NoSuchKey) return void 0;
103
+ throw error;
104
+ }
105
+ }
106
+ /**
107
+ * Gets the size of an incomplete part
108
+ * @param args - The function arguments
109
+ * @param args.id - The upload ID
110
+ * @returns Promise that resolves to part size or undefined if not found
111
+ */
112
+ async getIncompletePartSize(args) {
113
+ const { id } = args;
114
+ try {
115
+ const data = await this.client.headObject({
116
+ Bucket: this.bucket,
117
+ Key: this.metadataManager.generatePartKey({
118
+ id,
119
+ isIncomplete: true
120
+ })
121
+ });
122
+ return data.ContentLength;
123
+ } catch (error) {
124
+ if (error instanceof NotFound) return void 0;
125
+ throw error;
126
+ }
127
+ }
128
+ /**
129
+ * Deletes an incomplete part
130
+ * @param args - The function arguments
131
+ * @param args.id - The upload ID
132
+ * @returns Promise that resolves when deletion is complete
133
+ */
134
+ async deleteIncompletePart(args) {
135
+ const { id } = args;
136
+ await this.client.deleteObject({
137
+ Bucket: this.bucket,
138
+ Key: this.metadataManager.generatePartKey({
139
+ id,
140
+ isIncomplete: true
141
+ })
142
+ });
143
+ }
144
+ /**
145
+ * Downloads incomplete part to temporary file
146
+ * @param args - The function arguments
147
+ * @param args.id - The upload ID
148
+ * @returns Promise that resolves to incomplete part info or undefined if not found
149
+ */
150
+ async downloadIncompletePart(args) {
151
+ const { id } = args;
152
+ const incompletePart = await this.getIncompletePart({ id });
153
+ if (!incompletePart) return;
154
+ const filePath = await this.fileOperations.generateUniqueTmpFileName({ template: "tus-s3-incomplete-part-" });
155
+ try {
156
+ let incompletePartSize = 0;
157
+ const byteCounterTransform = new stream.Transform({ transform(chunk, _, callback) {
158
+ incompletePartSize += chunk.length;
159
+ callback(null, chunk);
160
+ } });
161
+ await stream.promises.pipeline(incompletePart, byteCounterTransform, fs.createWriteStream(filePath));
162
+ const createReadStream = (options) => {
163
+ const fileReader = fs.createReadStream(filePath);
164
+ if (options.cleanUpOnEnd) {
165
+ fileReader.on("end", () => {
166
+ fs.unlink(filePath, () => {});
167
+ });
168
+ fileReader.on("error", (err) => {
169
+ fileReader.destroy(err);
170
+ fs.unlink(filePath, () => {});
171
+ });
172
+ }
173
+ return fileReader;
174
+ };
175
+ return {
176
+ createReader: createReadStream,
177
+ path: filePath,
178
+ size: incompletePartSize
179
+ };
180
+ } catch (err) {
181
+ fs.promises.rm(filePath).catch(() => {});
182
+ throw err;
183
+ }
184
+ }
185
+ /**
186
+ * Uploads an incomplete part
187
+ * @param args - The function arguments
188
+ * @param args.id - The upload ID
189
+ * @param args.readStream - The stream to read data from
190
+ * @returns Promise that resolves to the ETag of the uploaded part
191
+ */
192
+ async uploadIncompletePart(args) {
193
+ const { id, readStream } = args;
194
+ const data = await this.client.putObject({
195
+ Body: readStream,
196
+ Bucket: this.bucket,
197
+ Key: this.metadataManager.generatePartKey({
198
+ id,
199
+ isIncomplete: true
200
+ }),
201
+ Tagging: this.generateCompleteTag("false")
202
+ });
203
+ log(`[${id}] finished uploading incomplete part`);
204
+ return data.ETag;
205
+ }
206
+ /**
207
+ * Uploads a single part
208
+ * @param args - The function arguments
209
+ * @param args.metadata - The upload metadata
210
+ * @param args.readStream - The stream to read data from
211
+ * @param args.partNumber - The part number to upload
212
+ * @returns Promise that resolves to the ETag of the uploaded part
213
+ */
214
+ async uploadPart(args) {
215
+ const { metadata, readStream, partNumber } = args;
216
+ const permit = await this.partUploadSemaphore.acquire();
217
+ if (!metadata["upload-id"]) {
218
+ throwError(MediaCloudError.MUX_UPLOAD_ID_MISSING);
219
+ throw new Error();
220
+ }
221
+ const params = {
222
+ Body: readStream,
223
+ Bucket: this.bucket,
224
+ Key: metadata.file.id,
225
+ PartNumber: partNumber,
226
+ UploadId: metadata["upload-id"]
227
+ };
228
+ try {
229
+ const data = await this.client.uploadPart(params);
230
+ return {
231
+ ETag: data.ETag,
232
+ PartNumber: partNumber
233
+ };
234
+ } catch (_error) {
235
+ throwError(MediaCloudError.TUS_UPLOAD_ERROR);
236
+ throw new Error();
237
+ } finally {
238
+ permit();
239
+ }
240
+ }
241
+ /**
242
+ * Uploads a stream to s3 using multiple parts
243
+ * @param args - The function arguments
244
+ * @param args.metadata - The upload metadata
245
+ * @param args.readStream - The stream to read data from
246
+ * @param args.currentPartNumber - The current part number to start from
247
+ * @param args.offset - The byte offset to start from
248
+ * @returns Promise that resolves to the number of bytes uploaded
249
+ */
250
+ async uploadParts(args) {
251
+ const { metadata, readStream, offset: initialOffset } = args;
252
+ let { currentPartNumber } = args;
253
+ let offset = initialOffset;
254
+ const size = metadata.file.size;
255
+ const promises = [];
256
+ let pendingChunkFilepath = null;
257
+ let bytesUploaded = 0;
258
+ let permit = void 0;
259
+ const splitterStream = new StreamSplitter({
260
+ chunkSize: this.fileOperations.calculateOptimalPartSize({ size }),
261
+ directory: os.tmpdir()
262
+ }).on("beforeChunkStarted", async () => {
263
+ permit = await this.partUploadSemaphore.acquire();
264
+ }).on("chunkStarted", (filepath) => {
265
+ pendingChunkFilepath = filepath;
266
+ }).on("chunkFinished", ({ path, size: partSize }) => {
267
+ pendingChunkFilepath = null;
268
+ const acquiredPermit = permit;
269
+ const partNumber = currentPartNumber++;
270
+ offset += partSize;
271
+ const isFinalPart = size === offset;
272
+ const uploadChunk = async () => {
273
+ try {
274
+ const readable = fs.createReadStream(path);
275
+ readable.on("error", function(error) {
276
+ throw error;
277
+ });
278
+ switch (true) {
279
+ case partSize >= this.minPartSize || isFinalPart:
280
+ await this.uploadPart({
281
+ metadata,
282
+ readStream: readable,
283
+ partNumber
284
+ });
285
+ break;
286
+ default:
287
+ await this.uploadIncompletePart({
288
+ id: metadata.file.id,
289
+ readStream: readable
290
+ });
291
+ break;
292
+ }
293
+ bytesUploaded += partSize;
294
+ } catch (error) {
295
+ const mappedError = error instanceof Error ? error : new Error(String(error));
296
+ splitterStream.destroy(mappedError);
297
+ throw mappedError;
298
+ } finally {
299
+ fs.promises.rm(path).catch(function() {});
300
+ acquiredPermit?.();
301
+ }
302
+ };
303
+ const deferred = uploadChunk();
304
+ promises.push(deferred);
305
+ }).on("chunkError", () => {
306
+ permit?.();
307
+ });
308
+ try {
309
+ await stream.promises.pipeline(readStream, splitterStream);
310
+ } catch (error) {
311
+ if (pendingChunkFilepath !== null) try {
312
+ await fs.promises.rm(pendingChunkFilepath);
313
+ } catch {
314
+ log(`[${metadata.file.id}] failed to remove chunk ${String(pendingChunkFilepath)}`);
315
+ }
316
+ const mappedError = error instanceof Error ? error : new Error(String(error));
317
+ promises.push(Promise.reject(mappedError));
318
+ } finally {
319
+ await Promise.allSettled(promises);
320
+ await Promise.all(promises);
321
+ }
322
+ return bytesUploaded;
323
+ }
324
+ };
325
+
326
+ //#endregion
327
+ export { S3PartsManager };
328
+ //# sourceMappingURL=parts-manager.js.map