@maas/payload-plugin-media-cloud 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/handleDelete.js +3 -3
- package/dist/adapter/handleDelete.js.map +1 -1
- package/dist/components/upload-handler/upload-handler.js +7 -9
- package/dist/components/upload-handler/upload-handler.js.map +1 -1
- package/dist/components/upload-manager/upload-manager.js +6 -6
- package/dist/components/upload-manager/upload-manager.js.map +1 -1
- package/dist/endpoints/muxCreateUploadHandler.js +3 -3
- package/dist/endpoints/muxCreateUploadHandler.js.map +1 -1
- package/dist/error-handler/dist/index.js +44 -27
- package/dist/error-handler/dist/index.js.map +1 -1
- package/dist/hooks/useEmitter.d.ts +2 -3
- package/dist/hooks/useEmitter.js.map +1 -1
- package/dist/hooks/useErrorHandler.d.ts +5 -5
- package/dist/hooks/useErrorHandler.js +7 -6
- package/dist/hooks/useErrorHandler.js.map +1 -1
- package/dist/plugin.js +11 -9
- package/dist/plugin.js.map +1 -1
- package/dist/tus/stores/s3/file-operations.js +2 -2
- package/dist/tus/stores/s3/file-operations.js.map +1 -1
- package/dist/tus/stores/s3/metadata-manager.js +5 -5
- package/dist/tus/stores/s3/metadata-manager.js.map +1 -1
- package/dist/tus/stores/s3/parts-manager.js +8 -8
- package/dist/tus/stores/s3/parts-manager.js.map +1 -1
- package/dist/tus/stores/s3/s3-store.js +11 -11
- package/dist/tus/stores/s3/s3-store.js.map +1 -1
- package/dist/types/errors.d.ts +5 -37
- package/dist/types/errors.js +99 -38
- package/dist/types/errors.js.map +1 -1
- package/dist/types/index.d.ts +4 -3
- package/dist/utils/file.js +3 -3
- package/dist/utils/file.js.map +1 -1
- package/package.json +1 -1
package/dist/plugin.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MediaCloudErrors } from "./types/errors.js";
|
|
2
2
|
import { useErrorHandler } from "./hooks/useErrorHandler.js";
|
|
3
3
|
import { getStorageAdapter } from "./adapter/storageAdapter.js";
|
|
4
4
|
import { getMediaCollection } from "./collections/mediaCollection.js";
|
|
@@ -22,7 +22,7 @@ let muxClient = null;
|
|
|
22
22
|
* @throws {Error} When required Mux configuration is missing
|
|
23
23
|
*/
|
|
24
24
|
function validateMuxConfig(muxConfig) {
|
|
25
|
-
if (!muxConfig?.tokenId || !muxConfig?.tokenSecret) throwError(
|
|
25
|
+
if (!muxConfig?.tokenId || !muxConfig?.tokenSecret) throwError(MediaCloudErrors.MUX_CONFIG_MISSING);
|
|
26
26
|
}
|
|
27
27
|
/**
|
|
28
28
|
* Creates a new Mux client instance
|
|
@@ -43,7 +43,7 @@ function createMuxClient(muxConfig) {
|
|
|
43
43
|
* @throws {Error} When required S3 configuration is missing
|
|
44
44
|
*/
|
|
45
45
|
function createS3Store(s3Config) {
|
|
46
|
-
if (!s3Config?.bucket || !s3Config?.region || !s3Config?.accessKeyId || !s3Config?.secretAccessKey) throwError(
|
|
46
|
+
if (!s3Config?.bucket || !s3Config?.region || !s3Config?.accessKeyId || !s3Config?.secretAccessKey) throwError(MediaCloudErrors.S3_CONFIG_MISSING);
|
|
47
47
|
return new S3Store({ s3ClientConfig: {
|
|
48
48
|
acl: "public-read",
|
|
49
49
|
bucket: s3Config.bucket,
|
|
@@ -62,7 +62,7 @@ function createS3Store(s3Config) {
|
|
|
62
62
|
* @returns A configured TusServer instance
|
|
63
63
|
*/
|
|
64
64
|
function createTusServer(args) {
|
|
65
|
-
const { config, s3Store } = args;
|
|
65
|
+
const { config, s3Store, pluginOptions } = args;
|
|
66
66
|
return new Server({
|
|
67
67
|
datastore: s3Store,
|
|
68
68
|
namingFunction: async (req, metadata) => {
|
|
@@ -75,11 +75,12 @@ function createTusServer(args) {
|
|
|
75
75
|
if (docs.length > 0) return generateUniqueFilename(metadata?.filename ?? "");
|
|
76
76
|
else return metadata?.filename || generateUniqueFilename("");
|
|
77
77
|
} catch (_error) {
|
|
78
|
-
logError(
|
|
78
|
+
logError(MediaCloudErrors.NAMING_FUNCTION_ERROR);
|
|
79
79
|
return metadata?.filename || generateUniqueFilename("");
|
|
80
80
|
}
|
|
81
81
|
},
|
|
82
|
-
path: "/api/uploads"
|
|
82
|
+
path: "/api/uploads",
|
|
83
|
+
respectForwardedHeaders: pluginOptions.s3.respectForwardedHeaders ?? true
|
|
83
84
|
});
|
|
84
85
|
}
|
|
85
86
|
/**
|
|
@@ -166,7 +167,7 @@ function createMuxEndpoints(args) {
|
|
|
166
167
|
function mediaCloudPlugin(pluginOptions) {
|
|
167
168
|
return function(config) {
|
|
168
169
|
if (!pluginOptions) {
|
|
169
|
-
logError(
|
|
170
|
+
logError(MediaCloudErrors.PLUGIN_NOT_CONFIGURED);
|
|
170
171
|
return config;
|
|
171
172
|
}
|
|
172
173
|
const isPluginDisabled = pluginOptions.enabled === false;
|
|
@@ -182,13 +183,14 @@ function mediaCloudPlugin(pluginOptions) {
|
|
|
182
183
|
validateMuxConfig(pluginOptions.mux);
|
|
183
184
|
muxClient = getMuxClient();
|
|
184
185
|
} else {
|
|
185
|
-
logError(
|
|
186
|
+
logError(MediaCloudErrors.MUX_CONFIG_INCOMPLETE);
|
|
186
187
|
muxClient = null;
|
|
187
188
|
}
|
|
188
189
|
const s3Store = createS3Store(pluginOptions.s3);
|
|
189
190
|
const tusServer = createTusServer({
|
|
190
191
|
config,
|
|
191
|
-
s3Store
|
|
192
|
+
s3Store,
|
|
193
|
+
pluginOptions
|
|
192
194
|
});
|
|
193
195
|
const mediaCollection = getMediaCollection({ s3Store });
|
|
194
196
|
if (isPluginDisabled) return config;
|
package/dist/plugin.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.js","names":["muxClient: Mux | null","muxConfig: MediaCloudPluginOptions['mux']","s3Config: MediaCloudPluginOptions['s3']","args: CreateTusServerArgs","TusServer","tusServer: TusServer","req: PayloadRequest","args: CreateMuxEndpointsArgs","pluginOptions: MediaCloudPluginOptions","config: Config","mappedConfig: Config"],"sources":["../src/plugin.ts"],"sourcesContent":["import Mux from '@mux/mux-node'\n\nimport { cloudStoragePlugin } from '@payloadcms/plugin-cloud-storage'\nimport { initClientUploads } from '@payloadcms/plugin-cloud-storage/utilities'\nimport { getPayload, sanitizeConfig } from 'payload'\nimport { Server as TusServer, type DataStore } from '@tus/server'\n\nimport { MediaCloudError } from './types/errors'\nimport { useErrorHandler } from './hooks/useErrorHandler'\nimport { getStorageAdapter } from './adapter/storageAdapter'\nimport { getMediaCollection } from './collections/mediaCollection'\nimport { getMuxAssetHandler } from './endpoints/muxAssetHandler'\nimport { getMuxCreateUploadHandler } from './endpoints/muxCreateUploadHandler'\nimport { getMuxWebhookHandler } from './endpoints/muxWebhookHandler'\nimport { S3Store } from './tus/stores/s3/s3-store'\nimport { generateUniqueFilename } from './utils/file'\n\nimport type { Config, PayloadRequest } from 'payload'\nimport type { MediaCloudPluginOptions } from './types'\n\ninterface CreateTusServerArgs {\n config: Config\n s3Store: DataStore\n}\n\ninterface CreateMuxEndpointsArgs {\n getMuxClient: () => Mux\n pluginOptions: MediaCloudPluginOptions\n}\n\nconst { logError, throwError } = useErrorHandler()\n\nlet muxClient: Mux | null = null\n\n/**\n * Validates Mux configuration options\n * @param muxConfig - The Mux configuration to validate\n * @throws {Error} When required Mux configuration is missing\n */\nfunction validateMuxConfig(muxConfig: MediaCloudPluginOptions['mux']): void {\n if (!muxConfig?.tokenId || !muxConfig?.tokenSecret) {\n throwError(MediaCloudError.MUX_CONFIG_MISSING)\n }\n}\n\n/**\n * Creates a new Mux client instance\n * @param muxConfig - The Mux configuration options\n * @returns A configured Mux client\n */\nfunction createMuxClient(muxConfig: MediaCloudPluginOptions['mux']): Mux {\n // Create and return Mux client instance\n return new Mux({\n tokenId: muxConfig.tokenId,\n tokenSecret: muxConfig.tokenSecret,\n webhookSecret: muxConfig.webhookSecret,\n })\n}\n\n/**\n * Creates and configures an S3Store instance\n * @param s3Config - The S3 configuration options\n * @returns A configured S3Store instance\n * @throws {Error} When required S3 configuration is missing\n */\nfunction createS3Store(s3Config: MediaCloudPluginOptions['s3']): S3Store {\n // Validate S3 configuration\n if (\n !s3Config?.bucket ||\n !s3Config?.region ||\n !s3Config?.accessKeyId ||\n !s3Config?.secretAccessKey\n ) {\n throwError(MediaCloudError.S3_CONFIG_MISSING)\n }\n\n // Create and return S3Store instance\n return new S3Store({\n s3ClientConfig: {\n acl: 'public-read',\n bucket: s3Config.bucket,\n credentials: {\n accessKeyId: s3Config.accessKeyId,\n secretAccessKey: s3Config.secretAccessKey,\n },\n endpoint: s3Config.endpoint,\n forcePathStyle: true,\n region: s3Config.region,\n },\n })\n}\n\n/**\n * Creates a TUS server instance with S3 storage\n * @param args - The arguments for creating the TUS server\n * @returns A configured TusServer instance\n */\nfunction createTusServer(args: CreateTusServerArgs): TusServer {\n const { config, s3Store } = args\n\n return new TusServer({\n datastore: s3Store,\n namingFunction: async (req, metadata) => {\n try {\n // Get Payload instance\n const payload = await getPayload({\n config: sanitizeConfig(config),\n })\n\n // Check if a file with the same name already exists in the media collection\n const { docs } = await payload.find({\n collection: 'media',\n where: {\n filename: {\n equals: metadata?.filename || '',\n },\n },\n })\n\n // If a file with the same name exists, generate a unique filename\n if (docs.length > 0) {\n return generateUniqueFilename(metadata?.filename ?? '')\n } else {\n // If no file with the same name exists, return the original filename\n return metadata?.filename || generateUniqueFilename('')\n }\n } catch (_error) {\n // If an error occurs, log it and return the original filename\n logError(MediaCloudError.NAMING_FUNCTION_ERROR)\n return metadata?.filename || generateUniqueFilename('')\n }\n },\n path: '/api/uploads',\n })\n}\n\n/**\n * Creates TUS upload endpoints for file handling\n * @param tusServer - The TUS server instance\n * @returns An array of endpoint configurations\n */\nfunction createTusEndpoints(tusServer: TusServer) {\n /**\n * Handles TUS requests through the server\n * @param req - The payload request object\n * @returns The server response\n */\n function tusHandler(req: PayloadRequest) {\n return tusServer.handleWeb(req as Request)\n }\n\n return [\n { handler: tusHandler, method: 'options' as const, path: '/uploads' },\n { handler: tusHandler, method: 'post' as const, path: '/uploads' },\n { handler: tusHandler, method: 'get' as const, path: '/uploads/:id' },\n { handler: tusHandler, method: 'put' as const, path: '/uploads/:id' },\n { handler: tusHandler, method: 'patch' as const, path: '/uploads/:id' },\n { handler: tusHandler, method: 'delete' as const, path: '/uploads/:id' },\n ]\n}\n\n/**\n * Creates Mux-related endpoints for asset handling\n * @param args - The arguments for creating Mux endpoints\n * @returns An array of Mux endpoint configurations\n */\nfunction createMuxEndpoints(args: CreateMuxEndpointsArgs) {\n const { getMuxClient, pluginOptions } = args\n return [\n {\n handler: getMuxWebhookHandler({ getMuxClient }),\n method: 'get' as const,\n path: '/mux/webhook',\n },\n {\n handler: getMuxCreateUploadHandler({ getMuxClient, pluginOptions }),\n method: 'post' as const,\n path: '/mux/upload',\n },\n {\n handler: getMuxAssetHandler({ getMuxClient }),\n method: 'get' as const,\n path: '/mux/asset',\n },\n ]\n}\n\n/**\n * Media Cloud Plugin for Payload CMS\n * Provides file upload capabilities using S3 storage and Mux video processing\n * @param pluginOptions - Configuration options for the plugin\n * @returns A function that configures the Payload config\n */\nexport function mediaCloudPlugin(pluginOptions: MediaCloudPluginOptions) {\n return function (config: Config): Config {\n if (!pluginOptions) {\n logError(MediaCloudError.PLUGIN_NOT_CONFIGURED)\n return config\n }\n\n const isPluginDisabled = pluginOptions.enabled === false\n\n /**\n * Gets or creates a Mux client instance\n * @returns The Mux client instance\n */\n function getMuxClient(): Mux {\n // Return existing Mux client if already created\n if (muxClient) {\n return muxClient\n }\n // Create and return a new Mux client\n return createMuxClient(pluginOptions.mux)\n }\n\n // Validate Mux configuration\n if (pluginOptions.mux) {\n validateMuxConfig(pluginOptions.mux)\n muxClient = getMuxClient()\n } else {\n logError(MediaCloudError.MUX_CONFIG_INCOMPLETE)\n muxClient = null\n }\n\n const s3Store = createS3Store(pluginOptions.s3)\n const tusServer = createTusServer({ config, s3Store })\n const mediaCollection = getMediaCollection({ s3Store })\n\n if (isPluginDisabled) {\n return config\n }\n\n // Initialize client uploads\n initClientUploads({\n clientHandler:\n '@maas/payload-plugin-media-cloud/components#UploadHandler',\n collections: {\n media: {\n clientUploads: true,\n disableLocalStorage: true,\n },\n },\n config,\n enabled: true,\n serverHandler: () => {\n return Response.json(\n { message: 'Server handler is not implemented' },\n { status: 501 }\n )\n },\n serverHandlerPath: '/media-cloud/upload',\n })\n\n const cloudStorageConfig = {\n collections: {\n media: {\n adapter: getStorageAdapter({ getMuxClient, pluginOptions, s3Store }),\n clientUploads: true,\n disableLocalStorage: true,\n },\n },\n }\n\n const mappedConfig: Config = {\n ...config,\n admin: {\n ...(config.admin ?? {}),\n components: {\n ...(config.admin?.components ?? {}),\n providers: [\n '@maas/payload-plugin-media-cloud/components#UploadManagerProvider',\n ...(config.admin?.components?.providers ?? []),\n ],\n },\n },\n collections: [...(config.collections ?? []), mediaCollection],\n endpoints: [\n ...(config.endpoints ?? []),\n ...createTusEndpoints(tusServer),\n ...createMuxEndpoints({ getMuxClient, pluginOptions }),\n ],\n }\n\n return cloudStoragePlugin(cloudStorageConfig)(mappedConfig)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA8BA,MAAM,EAAE,UAAU,YAAY,GAAG,iBAAiB;AAElD,IAAIA,YAAwB;;;;;;AAO5B,SAAS,kBAAkBC,WAAiD;AAC1E,KAAI,CAAC,WAAW,WAAW,CAAC,WAAW,aACrC,WAAW,gBAAgB,mBAAmB;AAEjD;;;;;;AAOD,SAAS,gBAAgBA,WAAgD;AAEvE,QAAO,IAAI,IAAI;EACb,SAAS,UAAU;EACnB,aAAa,UAAU;EACvB,eAAe,UAAU;CAC1B;AACF;;;;;;;AAQD,SAAS,cAAcC,UAAkD;AAEvE,KACE,CAAC,UAAU,UACX,CAAC,UAAU,UACX,CAAC,UAAU,eACX,CAAC,UAAU,iBAEX,WAAW,gBAAgB,kBAAkB;AAI/C,QAAO,IAAI,QAAQ,EACjB,gBAAgB;EACd,KAAK;EACL,QAAQ,SAAS;EACjB,aAAa;GACX,aAAa,SAAS;GACtB,iBAAiB,SAAS;EAC3B;EACD,UAAU,SAAS;EACnB,gBAAgB;EAChB,QAAQ,SAAS;CAClB,EACF;AACF;;;;;;AAOD,SAAS,gBAAgBC,MAAsC;CAC7D,MAAM,EAAE,QAAQ,SAAS,GAAG;AAE5B,QAAO,IAAIC,OAAU;EACnB,WAAW;EACX,gBAAgB,OAAO,KAAK,aAAa;AACvC,OAAI;IAEF,MAAM,UAAU,MAAM,WAAW,EAC/B,QAAQ,eAAe,OAAO,CAC/B,EAAC;IAGF,MAAM,EAAE,MAAM,GAAG,MAAM,QAAQ,KAAK;KAClC,YAAY;KACZ,OAAO,EACL,UAAU,EACR,QAAQ,UAAU,YAAY,GAC/B,EACF;IACF,EAAC;AAGF,QAAI,KAAK,SAAS,EAChB,QAAO,uBAAuB,UAAU,YAAY,GAAG;QAGvD,QAAO,UAAU,YAAY,uBAAuB,GAAG;GAE1D,SAAQ,QAAQ;IAEf,SAAS,gBAAgB,sBAAsB;AAC/C,WAAO,UAAU,YAAY,uBAAuB,GAAG;GACxD;EACF;EACD,MAAM;CACP;AACF;;;;;;AAOD,SAAS,mBAAmBC,WAAsB;;;;;;CAMhD,SAAS,WAAWC,KAAqB;AACvC,SAAO,UAAU,UAAU,IAAe;CAC3C;AAED,QAAO;EACL;GAAE,SAAS;GAAY,QAAQ;GAAoB,MAAM;EAAY;EACrE;GAAE,SAAS;GAAY,QAAQ;GAAiB,MAAM;EAAY;EAClE;GAAE,SAAS;GAAY,QAAQ;GAAgB,MAAM;EAAgB;EACrE;GAAE,SAAS;GAAY,QAAQ;GAAgB,MAAM;EAAgB;EACrE;GAAE,SAAS;GAAY,QAAQ;GAAkB,MAAM;EAAgB;EACvE;GAAE,SAAS;GAAY,QAAQ;GAAmB,MAAM;EAAgB;CACzE;AACF;;;;;;AAOD,SAAS,mBAAmBC,MAA8B;CACxD,MAAM,EAAE,cAAc,eAAe,GAAG;AACxC,QAAO;EACL;GACE,SAAS,qBAAqB,EAAE,aAAc,EAAC;GAC/C,QAAQ;GACR,MAAM;EACP;EACD;GACE,SAAS,0BAA0B;IAAE;IAAc;GAAe,EAAC;GACnE,QAAQ;GACR,MAAM;EACP;EACD;GACE,SAAS,mBAAmB,EAAE,aAAc,EAAC;GAC7C,QAAQ;GACR,MAAM;EACP;CACF;AACF;;;;;;;AAQD,SAAgB,iBAAiBC,eAAwC;AACvE,QAAO,SAAUC,QAAwB;AACvC,MAAI,CAAC,eAAe;GAClB,SAAS,gBAAgB,sBAAsB;AAC/C,UAAO;EACR;EAED,MAAM,mBAAmB,cAAc,YAAY;;;;;EAMnD,SAAS,eAAoB;AAE3B,OAAI,UACF,QAAO;AAGT,UAAO,gBAAgB,cAAc,IAAI;EAC1C;AAGD,MAAI,cAAc,KAAK;GACrB,kBAAkB,cAAc,IAAI;GACpC,YAAY,cAAc;EAC3B,OAAM;GACL,SAAS,gBAAgB,sBAAsB;GAC/C,YAAY;EACb;EAED,MAAM,UAAU,cAAc,cAAc,GAAG;EAC/C,MAAM,YAAY,gBAAgB;GAAE;GAAQ;EAAS,EAAC;EACtD,MAAM,kBAAkB,mBAAmB,EAAE,QAAS,EAAC;AAEvD,MAAI,iBACF,QAAO;EAIT,kBAAkB;GAChB,eACE;GACF,aAAa,EACX,OAAO;IACL,eAAe;IACf,qBAAqB;GACtB,EACF;GACD;GACA,SAAS;GACT,eAAe,MAAM;AACnB,WAAO,SAAS,KACd,EAAE,SAAS,oCAAqC,GAChD,EAAE,QAAQ,IAAK,EAChB;GACF;GACD,mBAAmB;EACpB,EAAC;EAEF,MAAM,qBAAqB,EACzB,aAAa,EACX,OAAO;GACL,SAAS,kBAAkB;IAAE;IAAc;IAAe;GAAS,EAAC;GACpE,eAAe;GACf,qBAAqB;EACtB,EACF,EACF;EAED,MAAMC,eAAuB;GAC3B,GAAG;GACH,OAAO;IACL,GAAI,OAAO,SAAS,CAAE;IACtB,YAAY;KACV,GAAI,OAAO,OAAO,cAAc,CAAE;KAClC,WAAW,CACT,qEACA,GAAI,OAAO,OAAO,YAAY,aAAa,CAAE,CAC9C;IACF;GACF;GACD,aAAa,CAAC,GAAI,OAAO,eAAe,CAAE,GAAG,eAAgB;GAC7D,WAAW;IACT,GAAI,OAAO,aAAa,CAAE;IAC1B,GAAG,mBAAmB,UAAU;IAChC,GAAG,mBAAmB;KAAE;KAAc;IAAe,EAAC;GACvD;EACF;AAED,SAAO,mBAAmB,mBAAmB,CAAC,aAAa;CAC5D;AACF"}
|
|
1
|
+
{"version":3,"file":"plugin.js","names":["muxClient: Mux | null","muxConfig: MediaCloudPluginOptions['mux']","s3Config: MediaCloudPluginOptions['s3']","args: CreateTusServerArgs","TusServer","tusServer: TusServer","req: PayloadRequest","args: CreateMuxEndpointsArgs","pluginOptions: MediaCloudPluginOptions","config: Config","mappedConfig: Config"],"sources":["../src/plugin.ts"],"sourcesContent":["import Mux from '@mux/mux-node'\n\nimport { cloudStoragePlugin } from '@payloadcms/plugin-cloud-storage'\nimport { initClientUploads } from '@payloadcms/plugin-cloud-storage/utilities'\nimport { getPayload, sanitizeConfig } from 'payload'\nimport { Server as TusServer, type DataStore } from '@tus/server'\n\nimport { MediaCloudErrors } from './types/errors'\nimport { useErrorHandler } from './hooks/useErrorHandler'\nimport { getStorageAdapter } from './adapter/storageAdapter'\nimport { getMediaCollection } from './collections/mediaCollection'\nimport { getMuxAssetHandler } from './endpoints/muxAssetHandler'\nimport { getMuxCreateUploadHandler } from './endpoints/muxCreateUploadHandler'\nimport { getMuxWebhookHandler } from './endpoints/muxWebhookHandler'\nimport { S3Store } from './tus/stores/s3/s3-store'\nimport { generateUniqueFilename } from './utils/file'\n\nimport type { Config, PayloadRequest } from 'payload'\nimport type { MediaCloudPluginOptions } from './types'\n\ninterface CreateTusServerArgs {\n config: Config\n s3Store: DataStore\n pluginOptions: MediaCloudPluginOptions\n}\n\ninterface CreateMuxEndpointsArgs {\n getMuxClient: () => Mux\n pluginOptions: MediaCloudPluginOptions\n}\n\nconst { logError, throwError } = useErrorHandler()\n\nlet muxClient: Mux | null = null\n\n/**\n * Validates Mux configuration options\n * @param muxConfig - The Mux configuration to validate\n * @throws {Error} When required Mux configuration is missing\n */\nfunction validateMuxConfig(muxConfig: MediaCloudPluginOptions['mux']): void {\n if (!muxConfig?.tokenId || !muxConfig?.tokenSecret) {\n throwError(MediaCloudErrors.MUX_CONFIG_MISSING)\n }\n}\n\n/**\n * Creates a new Mux client instance\n * @param muxConfig - The Mux configuration options\n * @returns A configured Mux client\n */\nfunction createMuxClient(muxConfig: MediaCloudPluginOptions['mux']): Mux {\n // Create and return Mux client instance\n return new Mux({\n tokenId: muxConfig.tokenId,\n tokenSecret: muxConfig.tokenSecret,\n webhookSecret: muxConfig.webhookSecret,\n })\n}\n\n/**\n * Creates and configures an S3Store instance\n * @param s3Config - The S3 configuration options\n * @returns A configured S3Store instance\n * @throws {Error} When required S3 configuration is missing\n */\nfunction createS3Store(s3Config: MediaCloudPluginOptions['s3']): S3Store {\n // Validate S3 configuration\n if (\n !s3Config?.bucket ||\n !s3Config?.region ||\n !s3Config?.accessKeyId ||\n !s3Config?.secretAccessKey\n ) {\n throwError(MediaCloudErrors.S3_CONFIG_MISSING)\n }\n\n // Create and return S3Store instance\n return new S3Store({\n s3ClientConfig: {\n acl: 'public-read',\n bucket: s3Config.bucket,\n credentials: {\n accessKeyId: s3Config.accessKeyId,\n secretAccessKey: s3Config.secretAccessKey,\n },\n endpoint: s3Config.endpoint,\n forcePathStyle: true,\n region: s3Config.region,\n },\n })\n}\n\n/**\n * Creates a TUS server instance with S3 storage\n * @param args - The arguments for creating the TUS server\n * @returns A configured TusServer instance\n */\nfunction createTusServer(args: CreateTusServerArgs): TusServer {\n const { config, s3Store, pluginOptions } = args\n\n return new TusServer({\n datastore: s3Store,\n namingFunction: async (req, metadata) => {\n try {\n // Get Payload instance\n const payload = await getPayload({\n config: sanitizeConfig(config),\n })\n\n // Check if a file with the same name already exists in the media collection\n const { docs } = await payload.find({\n collection: 'media',\n where: {\n filename: {\n equals: metadata?.filename || '',\n },\n },\n })\n\n // If a file with the same name exists, generate a unique filename\n if (docs.length > 0) {\n return generateUniqueFilename(metadata?.filename ?? '')\n } else {\n // If no file with the same name exists, return the original filename\n return metadata?.filename || generateUniqueFilename('')\n }\n } catch (_error) {\n // If an error occurs, log it and return the original filename\n logError(MediaCloudErrors.NAMING_FUNCTION_ERROR)\n return metadata?.filename || generateUniqueFilename('')\n }\n },\n path: '/api/uploads',\n respectForwardedHeaders: pluginOptions.s3.respectForwardedHeaders ?? true,\n })\n}\n\n/**\n * Creates TUS upload endpoints for file handling\n * @param tusServer - The TUS server instance\n * @returns An array of endpoint configurations\n */\nfunction createTusEndpoints(tusServer: TusServer) {\n /**\n * Handles TUS requests through the server\n * @param req - The payload request object\n * @returns The server response\n */\n function tusHandler(req: PayloadRequest) {\n return tusServer.handleWeb(req as Request)\n }\n\n return [\n { handler: tusHandler, method: 'options' as const, path: '/uploads' },\n { handler: tusHandler, method: 'post' as const, path: '/uploads' },\n { handler: tusHandler, method: 'get' as const, path: '/uploads/:id' },\n { handler: tusHandler, method: 'put' as const, path: '/uploads/:id' },\n { handler: tusHandler, method: 'patch' as const, path: '/uploads/:id' },\n { handler: tusHandler, method: 'delete' as const, path: '/uploads/:id' },\n ]\n}\n\n/**\n * Creates Mux-related endpoints for asset handling\n * @param args - The arguments for creating Mux endpoints\n * @returns An array of Mux endpoint configurations\n */\nfunction createMuxEndpoints(args: CreateMuxEndpointsArgs) {\n const { getMuxClient, pluginOptions } = args\n return [\n {\n handler: getMuxWebhookHandler({ getMuxClient }),\n method: 'get' as const,\n path: '/mux/webhook',\n },\n {\n handler: getMuxCreateUploadHandler({ getMuxClient, pluginOptions }),\n method: 'post' as const,\n path: '/mux/upload',\n },\n {\n handler: getMuxAssetHandler({ getMuxClient }),\n method: 'get' as const,\n path: '/mux/asset',\n },\n ]\n}\n\n/**\n * Media Cloud Plugin for Payload CMS\n * Provides file upload capabilities using S3 storage and Mux video processing\n * @param pluginOptions - Configuration options for the plugin\n * @returns A function that configures the Payload config\n */\nexport function mediaCloudPlugin(pluginOptions: MediaCloudPluginOptions) {\n return function (config: Config): Config {\n if (!pluginOptions) {\n logError(MediaCloudErrors.PLUGIN_NOT_CONFIGURED)\n return config\n }\n\n const isPluginDisabled = pluginOptions.enabled === false\n\n /**\n * Gets or creates a Mux client instance\n * @returns The Mux client instance\n */\n function getMuxClient(): Mux {\n // Return existing Mux client if already created\n if (muxClient) {\n return muxClient\n }\n // Create and return a new Mux client\n return createMuxClient(pluginOptions.mux)\n }\n\n // Validate Mux configuration\n if (pluginOptions.mux) {\n validateMuxConfig(pluginOptions.mux)\n muxClient = getMuxClient()\n } else {\n logError(MediaCloudErrors.MUX_CONFIG_INCOMPLETE)\n muxClient = null\n }\n\n const s3Store = createS3Store(pluginOptions.s3)\n const tusServer = createTusServer({ config, s3Store, pluginOptions })\n const mediaCollection = getMediaCollection({ s3Store })\n\n if (isPluginDisabled) {\n return config\n }\n\n // Initialize client uploads\n initClientUploads({\n clientHandler:\n '@maas/payload-plugin-media-cloud/components#UploadHandler',\n collections: {\n media: {\n clientUploads: true,\n disableLocalStorage: true,\n },\n },\n config,\n enabled: true,\n serverHandler: () => {\n return Response.json(\n { message: 'Server handler is not implemented' },\n { status: 501 }\n )\n },\n serverHandlerPath: '/media-cloud/upload',\n })\n\n const cloudStorageConfig = {\n collections: {\n media: {\n adapter: getStorageAdapter({ getMuxClient, pluginOptions, s3Store }),\n clientUploads: true,\n disableLocalStorage: true,\n },\n },\n }\n\n const mappedConfig: Config = {\n ...config,\n admin: {\n ...(config.admin ?? {}),\n components: {\n ...(config.admin?.components ?? {}),\n providers: [\n '@maas/payload-plugin-media-cloud/components#UploadManagerProvider',\n ...(config.admin?.components?.providers ?? []),\n ],\n },\n },\n collections: [...(config.collections ?? []), mediaCollection],\n endpoints: [\n ...(config.endpoints ?? []),\n ...createTusEndpoints(tusServer),\n ...createMuxEndpoints({ getMuxClient, pluginOptions }),\n ],\n }\n\n return cloudStoragePlugin(cloudStorageConfig)(mappedConfig)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA+BA,MAAM,EAAE,UAAU,YAAY,GAAG,iBAAiB;AAElD,IAAIA,YAAwB;;;;;;AAO5B,SAAS,kBAAkBC,WAAiD;AAC1E,KAAI,CAAC,WAAW,WAAW,CAAC,WAAW,aACrC,WAAW,iBAAiB,mBAAmB;AAElD;;;;;;AAOD,SAAS,gBAAgBA,WAAgD;AAEvE,QAAO,IAAI,IAAI;EACb,SAAS,UAAU;EACnB,aAAa,UAAU;EACvB,eAAe,UAAU;CAC1B;AACF;;;;;;;AAQD,SAAS,cAAcC,UAAkD;AAEvE,KACE,CAAC,UAAU,UACX,CAAC,UAAU,UACX,CAAC,UAAU,eACX,CAAC,UAAU,iBAEX,WAAW,iBAAiB,kBAAkB;AAIhD,QAAO,IAAI,QAAQ,EACjB,gBAAgB;EACd,KAAK;EACL,QAAQ,SAAS;EACjB,aAAa;GACX,aAAa,SAAS;GACtB,iBAAiB,SAAS;EAC3B;EACD,UAAU,SAAS;EACnB,gBAAgB;EAChB,QAAQ,SAAS;CAClB,EACF;AACF;;;;;;AAOD,SAAS,gBAAgBC,MAAsC;CAC7D,MAAM,EAAE,QAAQ,SAAS,eAAe,GAAG;AAE3C,QAAO,IAAIC,OAAU;EACnB,WAAW;EACX,gBAAgB,OAAO,KAAK,aAAa;AACvC,OAAI;IAEF,MAAM,UAAU,MAAM,WAAW,EAC/B,QAAQ,eAAe,OAAO,CAC/B,EAAC;IAGF,MAAM,EAAE,MAAM,GAAG,MAAM,QAAQ,KAAK;KAClC,YAAY;KACZ,OAAO,EACL,UAAU,EACR,QAAQ,UAAU,YAAY,GAC/B,EACF;IACF,EAAC;AAGF,QAAI,KAAK,SAAS,EAChB,QAAO,uBAAuB,UAAU,YAAY,GAAG;QAGvD,QAAO,UAAU,YAAY,uBAAuB,GAAG;GAE1D,SAAQ,QAAQ;IAEf,SAAS,iBAAiB,sBAAsB;AAChD,WAAO,UAAU,YAAY,uBAAuB,GAAG;GACxD;EACF;EACD,MAAM;EACN,yBAAyB,cAAc,GAAG,2BAA2B;CACtE;AACF;;;;;;AAOD,SAAS,mBAAmBC,WAAsB;;;;;;CAMhD,SAAS,WAAWC,KAAqB;AACvC,SAAO,UAAU,UAAU,IAAe;CAC3C;AAED,QAAO;EACL;GAAE,SAAS;GAAY,QAAQ;GAAoB,MAAM;EAAY;EACrE;GAAE,SAAS;GAAY,QAAQ;GAAiB,MAAM;EAAY;EAClE;GAAE,SAAS;GAAY,QAAQ;GAAgB,MAAM;EAAgB;EACrE;GAAE,SAAS;GAAY,QAAQ;GAAgB,MAAM;EAAgB;EACrE;GAAE,SAAS;GAAY,QAAQ;GAAkB,MAAM;EAAgB;EACvE;GAAE,SAAS;GAAY,QAAQ;GAAmB,MAAM;EAAgB;CACzE;AACF;;;;;;AAOD,SAAS,mBAAmBC,MAA8B;CACxD,MAAM,EAAE,cAAc,eAAe,GAAG;AACxC,QAAO;EACL;GACE,SAAS,qBAAqB,EAAE,aAAc,EAAC;GAC/C,QAAQ;GACR,MAAM;EACP;EACD;GACE,SAAS,0BAA0B;IAAE;IAAc;GAAe,EAAC;GACnE,QAAQ;GACR,MAAM;EACP;EACD;GACE,SAAS,mBAAmB,EAAE,aAAc,EAAC;GAC7C,QAAQ;GACR,MAAM;EACP;CACF;AACF;;;;;;;AAQD,SAAgB,iBAAiBC,eAAwC;AACvE,QAAO,SAAUC,QAAwB;AACvC,MAAI,CAAC,eAAe;GAClB,SAAS,iBAAiB,sBAAsB;AAChD,UAAO;EACR;EAED,MAAM,mBAAmB,cAAc,YAAY;;;;;EAMnD,SAAS,eAAoB;AAE3B,OAAI,UACF,QAAO;AAGT,UAAO,gBAAgB,cAAc,IAAI;EAC1C;AAGD,MAAI,cAAc,KAAK;GACrB,kBAAkB,cAAc,IAAI;GACpC,YAAY,cAAc;EAC3B,OAAM;GACL,SAAS,iBAAiB,sBAAsB;GAChD,YAAY;EACb;EAED,MAAM,UAAU,cAAc,cAAc,GAAG;EAC/C,MAAM,YAAY,gBAAgB;GAAE;GAAQ;GAAS;EAAe,EAAC;EACrE,MAAM,kBAAkB,mBAAmB,EAAE,QAAS,EAAC;AAEvD,MAAI,iBACF,QAAO;EAIT,kBAAkB;GAChB,eACE;GACF,aAAa,EACX,OAAO;IACL,eAAe;IACf,qBAAqB;GACtB,EACF;GACD;GACA,SAAS;GACT,eAAe,MAAM;AACnB,WAAO,SAAS,KACd,EAAE,SAAS,oCAAqC,GAChD,EAAE,QAAQ,IAAK,EAChB;GACF;GACD,mBAAmB;EACpB,EAAC;EAEF,MAAM,qBAAqB,EACzB,aAAa,EACX,OAAO;GACL,SAAS,kBAAkB;IAAE;IAAc;IAAe;GAAS,EAAC;GACpE,eAAe;GACf,qBAAqB;EACtB,EACF,EACF;EAED,MAAMC,eAAuB;GAC3B,GAAG;GACH,OAAO;IACL,GAAI,OAAO,SAAS,CAAE;IACtB,YAAY;KACV,GAAI,OAAO,OAAO,cAAc,CAAE;KAClC,WAAW,CACT,qEACA,GAAI,OAAO,OAAO,YAAY,aAAa,CAAE,CAC9C;IACF;GACF;GACD,aAAa,CAAC,GAAI,OAAO,eAAe,CAAE,GAAG,eAAgB;GAC7D,WAAW;IACT,GAAI,OAAO,aAAa,CAAE;IAC1B,GAAG,mBAAmB,UAAU;IAChC,GAAG,mBAAmB;KAAE;KAAc;IAAe,EAAC;GACvD;EACF;AAED,SAAO,mBAAmB,mBAAmB,CAAC,aAAa;CAC5D;AACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MediaCloudErrors } from "../../../types/errors.js";
|
|
2
2
|
import { useErrorHandler } from "../../../hooks/useErrorHandler.js";
|
|
3
3
|
import crypto from "node:crypto";
|
|
4
4
|
import fs from "node:fs";
|
|
@@ -70,7 +70,7 @@ var S3FileOperations = class {
|
|
|
70
70
|
if (nodeError.code === "ENOENT") return filePath;
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
|
-
throwError(
|
|
73
|
+
throwError(MediaCloudErrors.S3_UNIQUE_NAME_ERROR);
|
|
74
74
|
throw new Error();
|
|
75
75
|
}
|
|
76
76
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-operations.js","names":["maxMultipartParts: number","maxUploadSize: number","minPartSize: number","preferredPartSize: number","args: CalculateOptimalPartSizeArgs","optimalPartSize: number","args: CalculateOffsetFromPartsArgs","args: CalculatePartNumberArgs","args: GenerateUniqueTmpFileNameArgs","args: CalculateOffsetFromPartsExportArgs"],"sources":["../../../../src/tus/stores/s3/file-operations.ts"],"sourcesContent":["import crypto from 'node:crypto'\nimport fs from 'node:fs'\nimport os from 'node:os'\nimport path from 'node:path'\n\nimport { useErrorHandler } from '../../../hooks/useErrorHandler'\nimport {
|
|
1
|
+
{"version":3,"file":"file-operations.js","names":["maxMultipartParts: number","maxUploadSize: number","minPartSize: number","preferredPartSize: number","args: CalculateOptimalPartSizeArgs","optimalPartSize: number","args: CalculateOffsetFromPartsArgs","args: CalculatePartNumberArgs","args: GenerateUniqueTmpFileNameArgs","args: CalculateOffsetFromPartsExportArgs"],"sources":["../../../../src/tus/stores/s3/file-operations.ts"],"sourcesContent":["import crypto from 'node:crypto'\nimport fs from 'node:fs'\nimport os from 'node:os'\nimport path from 'node:path'\n\nimport { useErrorHandler } from '../../../hooks/useErrorHandler'\nimport { MediaCloudErrors, MediaCloudLogs } from '../../../types/errors'\n\nimport type { NodeFSError } from '../../../types'\nimport type AWS from '@aws-sdk/client-s3'\n\ntype CalculateOptimalPartSizeArgs = {\n size?: number\n}\n\ntype CalculateOffsetFromPartsArgs = {\n parts?: Array<AWS.Part>\n}\n\ntype CalculatePartNumberArgs = {\n parts: Array<AWS.Part>\n}\n\ntype GenerateUniqueTmpFileNameArgs = {\n template: string\n}\n\ntype CalculateOffsetFromPartsExportArgs = {\n parts?: Array<{ Size?: number }>\n}\n\nconst { throwError } = useErrorHandler()\n\nexport class S3FileOperations {\n constructor(\n private maxMultipartParts: number,\n private maxUploadSize: number,\n private minPartSize: number,\n private preferredPartSize: number\n ) {}\n\n /**\n * Calculates the optimal part size for S3 multipart upload\n * @param args - The function arguments\n * @param args.size - The upload size in bytes (optional)\n * @returns The optimal part size in bytes\n */\n calculateOptimalPartSize(args: CalculateOptimalPartSizeArgs): number {\n const { size } = args\n // When upload size is not known we assume largest possible value (`maxUploadSize`)\n let uploadSize = size\n if (uploadSize === undefined) {\n uploadSize = this.maxUploadSize\n }\n\n let optimalPartSize: number\n\n // When upload is smaller or equal to PreferredPartSize, we upload in just one part.\n if (uploadSize <= this.preferredPartSize) {\n optimalPartSize = uploadSize\n }\n // Does the upload fit in MaxMultipartParts parts or less with PreferredPartSize.\n else if (uploadSize <= this.preferredPartSize * this.maxMultipartParts) {\n optimalPartSize = this.preferredPartSize\n // The upload is too big for the preferred size.\n // We divide the size with the max amount of parts and round it up.\n } else {\n optimalPartSize = Math.ceil(uploadSize / this.maxMultipartParts)\n }\n\n // Always ensure the part size is at least minPartSize\n return Math.max(optimalPartSize, this.minPartSize)\n }\n\n /**\n * Calculates the offset based on uploaded parts\n * @param args - The function arguments\n * @param args.parts - Array of uploaded parts (optional)\n * @returns The total offset in bytes\n */\n calculateOffsetFromParts(args: CalculateOffsetFromPartsArgs): number {\n const { parts } = args\n return parts && parts.length > 0\n ? parts.reduce((a, b) => a + (b.Size ?? 0), 0)\n : 0\n }\n\n /**\n * Calculates the next part number based on existing parts\n * @param args - The function arguments\n * @param args.parts - Array of uploaded parts\n * @returns The next part number to use\n */\n calculatePartNumber(args: CalculatePartNumberArgs): number {\n const { parts } = args\n return parts.length > 0 ? parts[parts.length - 1].PartNumber! + 1 : 1\n }\n\n /**\n * Generates a unique temporary file name\n * @param args - The function arguments\n * @param args.template - The template string for the file name\n * @returns Promise that resolves to the unique file path\n * @throws Error if unable to find unique name after max tries\n */\n async generateUniqueTmpFileName(\n args: GenerateUniqueTmpFileNameArgs\n ): Promise<string> {\n const { template } = args\n const tries = 5\n for (let i = 0; i < tries; i++) {\n const randomId = crypto.randomBytes(16).toString('hex')\n const filePath = path.join(os.tmpdir(), `${template}${randomId}`)\n try {\n await fs.promises.access(filePath, fs.constants.F_OK)\n } catch (error) {\n const nodeError = error as NodeFSError\n if (nodeError.code === 'ENOENT') {\n return filePath\n }\n }\n }\n throwError(MediaCloudErrors.S3_UNIQUE_NAME_ERROR)\n throw new Error() // This will never execute but satisfies TypeScript\n }\n}\n\n/**\n * Calculates the total offset from uploaded parts\n * @param args - The function arguments\n * @param args.parts - Array of parts with size information (optional)\n * @returns The total offset in bytes\n */\nexport function calculateOffsetFromParts(\n args: CalculateOffsetFromPartsExportArgs\n) {\n const { parts } = args\n return parts && parts.length > 0\n ? parts.reduce((a, b) => a + (b.Size ?? 0), 0)\n : 0\n}\n"],"mappings":";;;;;;;;AA+BA,MAAM,EAAE,YAAY,GAAG,iBAAiB;AAExC,IAAa,mBAAb,MAA8B;CAC5B,YACUA,mBACAC,eACAC,aACAC,mBACR;EAJQ;EACA;EACA;EACA;CACN;;;;;;;CAQJ,yBAAyBC,MAA4C;EACnE,MAAM,EAAE,MAAM,GAAG;EAEjB,IAAI,aAAa;AACjB,MAAI,eAAe,QACjB,aAAa,KAAK;EAGpB,IAAIC;AAGJ,MAAI,cAAc,KAAK,mBACrB,kBAAkB;WAGX,cAAc,KAAK,oBAAoB,KAAK,mBACnD,kBAAkB,KAAK;OAIvB,kBAAkB,KAAK,KAAK,aAAa,KAAK,kBAAkB;AAIlE,SAAO,KAAK,IAAI,iBAAiB,KAAK,YAAY;CACnD;;;;;;;CAQD,yBAAyBC,MAA4C;EACnE,MAAM,EAAE,OAAO,GAAG;AAClB,SAAO,SAAS,MAAM,SAAS,IAC3B,MAAM,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,QAAQ,IAAI,EAAE,GAC5C;CACL;;;;;;;CAQD,oBAAoBC,MAAuC;EACzD,MAAM,EAAE,OAAO,GAAG;AAClB,SAAO,MAAM,SAAS,IAAI,MAAM,MAAM,SAAS,GAAG,aAAc,IAAI;CACrE;;;;;;;;CASD,MAAM,0BACJC,MACiB;EACjB,MAAM,EAAE,UAAU,GAAG;EACrB,MAAM,QAAQ;AACd,OAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK;GAC9B,MAAM,WAAW,OAAO,YAAY,GAAG,CAAC,SAAS,MAAM;GACvD,MAAM,WAAW,KAAK,KAAK,GAAG,QAAQ,EAAE,GAAG,WAAW,UAAU,CAAC;AACjE,OAAI;IACF,MAAM,GAAG,SAAS,OAAO,UAAU,GAAG,UAAU,KAAK;GACtD,SAAQ,OAAO;IACd,MAAM,YAAY;AAClB,QAAI,UAAU,SAAS,SACrB,QAAO;GAEV;EACF;EACD,WAAW,iBAAiB,qBAAqB;AACjD,QAAM,IAAI;CACX;AACF;;;;;;;AAQD,SAAgB,yBACdC,MACA;CACA,MAAM,EAAE,OAAO,GAAG;AAClB,QAAO,SAAS,MAAM,SAAS,IAC3B,MAAM,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,QAAQ,IAAI,EAAE,GAC5C;AACL"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MediaCloudLogs } from "../../../types/errors.js";
|
|
2
2
|
import { useErrorHandler } from "../../../hooks/useErrorHandler.js";
|
|
3
3
|
import { TUS_RESUMABLE, Upload } from "@tus/utils";
|
|
4
4
|
|
|
5
5
|
//#region src/tus/stores/s3/metadata-manager.ts
|
|
6
|
-
const {
|
|
6
|
+
const { log } = useErrorHandler();
|
|
7
7
|
var S3MetadataManager = class {
|
|
8
8
|
constructor(client, bucket, cache, shouldUseExpirationTags, generateCompleteTag) {
|
|
9
9
|
this.client = client;
|
|
@@ -79,7 +79,7 @@ var S3MetadataManager = class {
|
|
|
79
79
|
*/
|
|
80
80
|
async saveMetadata(args) {
|
|
81
81
|
const { upload, uploadId } = args;
|
|
82
|
-
|
|
82
|
+
log(MediaCloudLogs.S3_STORE_METADATA_SAVING);
|
|
83
83
|
await this.client.putObject({
|
|
84
84
|
Body: JSON.stringify(upload),
|
|
85
85
|
Bucket: this.bucket,
|
|
@@ -90,7 +90,7 @@ var S3MetadataManager = class {
|
|
|
90
90
|
},
|
|
91
91
|
Tagging: this.generateCompleteTag("false")
|
|
92
92
|
});
|
|
93
|
-
|
|
93
|
+
log(MediaCloudLogs.S3_STORE_METADATA_SAVED);
|
|
94
94
|
}
|
|
95
95
|
/**
|
|
96
96
|
* Completes metadata by updating it with completion tags
|
|
@@ -127,7 +127,7 @@ var S3MetadataManager = class {
|
|
|
127
127
|
*/
|
|
128
128
|
async clearCache(args) {
|
|
129
129
|
const { id } = args;
|
|
130
|
-
|
|
130
|
+
log(MediaCloudLogs.S3_STORE_METADATA_CACHE_CLEARED);
|
|
131
131
|
await this.cache.delete(id);
|
|
132
132
|
}
|
|
133
133
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metadata-manager.js","names":["client: S3","bucket: string","cache: KvStore<TusUploadMetadata>","shouldUseExpirationTags: () => boolean","generateCompleteTag: (value: 'false' | 'true') => string | undefined","args: GenerateInfoKeyArgs","args: GeneratePartKeyArgs","args: GetMetadataArgs","metadata: TusUploadMetadata","args: SaveMetadataArgs","args: CompleteMetadataArgs","completedMetadata: TusUploadMetadata","args: ClearCacheArgs"],"sources":["../../../../src/tus/stores/s3/metadata-manager.ts"],"sourcesContent":["import { TUS_RESUMABLE, Upload } from '@tus/utils'\n\nimport { useErrorHandler } from '../../../hooks/useErrorHandler'\nimport {
|
|
1
|
+
{"version":3,"file":"metadata-manager.js","names":["client: S3","bucket: string","cache: KvStore<TusUploadMetadata>","shouldUseExpirationTags: () => boolean","generateCompleteTag: (value: 'false' | 'true') => string | undefined","args: GenerateInfoKeyArgs","args: GeneratePartKeyArgs","args: GetMetadataArgs","metadata: TusUploadMetadata","args: SaveMetadataArgs","args: CompleteMetadataArgs","completedMetadata: TusUploadMetadata","args: ClearCacheArgs"],"sources":["../../../../src/tus/stores/s3/metadata-manager.ts"],"sourcesContent":["import { TUS_RESUMABLE, Upload } from '@tus/utils'\n\nimport { useErrorHandler } from '../../../hooks/useErrorHandler'\nimport { MediaCloudLogs } from '../../../types/errors'\n\nimport type { S3 } from '@aws-sdk/client-s3'\nimport type { KvStore } from '@tus/utils'\nimport type { TusUploadMetadata } from '../../../types'\n\ntype GenerateInfoKeyArgs = {\n id: string\n}\n\ntype GeneratePartKeyArgs = {\n id: string\n isIncomplete?: boolean\n}\n\ntype GetMetadataArgs = {\n id: string\n}\n\ntype SaveMetadataArgs = {\n upload: Upload\n uploadId: string\n}\n\ntype CompleteMetadataArgs = {\n upload: Upload\n}\n\ntype ClearCacheArgs = {\n id: string\n}\n\nconst { log } = useErrorHandler()\n\nexport class S3MetadataManager {\n constructor(\n private client: S3,\n private bucket: string,\n private cache: KvStore<TusUploadMetadata>,\n private shouldUseExpirationTags: () => boolean,\n private generateCompleteTag: (value: 'false' | 'true') => string | undefined\n ) {}\n /**\n * Generates the S3 key for metadata info files\n * @param args - The function arguments\n * @param args.id - The file ID\n * @returns The S3 key for the metadata file\n */\n generateInfoKey(args: GenerateInfoKeyArgs): string {\n const { id } = args\n return `${id}.info`\n }\n\n /**\n * Generates the S3 key for part files\n * @param args - The function arguments\n * @param args.id - The file ID\n * @param args.isIncomplete - Whether this is an incomplete part\n * @returns The S3 key for the part file\n */\n generatePartKey(args: GeneratePartKeyArgs): string {\n const { id, isIncomplete = false } = args\n let key = id\n if (isIncomplete) {\n key += '.part'\n }\n\n // TODO: introduce ObjectPrefixing for parts and incomplete parts.\n // ObjectPrefix is prepended to the name of each S3 object that is created\n // to store uploaded files. It can be used to create a pseudo-directory\n // structure in the bucket, e.g. \"path/to/my/uploads\".\n return key\n }\n\n /**\n * Retrieves upload metadata previously saved in `${file_id}.info`.\n * There's a small and simple caching mechanism to avoid multiple\n * HTTP calls to S3.\n * @param args - The function arguments\n * @param args.id - The file ID to retrieve metadata for\n * @returns Promise that resolves to the upload metadata\n */\n async getMetadata(args: GetMetadataArgs): Promise<TusUploadMetadata> {\n const { id } = args\n const cached = await this.cache.get(id)\n if (cached) {\n return cached\n }\n\n const { Body, Metadata } = await this.client.getObject({\n Bucket: this.bucket,\n Key: this.generateInfoKey({ id }),\n })\n const file = JSON.parse((await Body?.transformToString()) as string)\n const metadata: TusUploadMetadata = {\n file: new Upload({\n id,\n creation_date: file.creation_date,\n metadata: file.metadata,\n offset: Number.parseInt(file.offset, 10),\n size: Number.isFinite(file.size)\n ? Number.parseInt(file.size, 10)\n : undefined,\n storage: file.storage,\n }),\n 'tus-version': Metadata?.['tus-version'] as string,\n 'upload-id': Metadata?.['upload-id'] as string,\n }\n await this.cache.set(id, metadata)\n return metadata\n }\n\n /**\n * Saves upload metadata to a `${file_id}.info` file on S3.\n * Please note that the file is empty and the metadata is saved\n * on the S3 object's `Metadata` field, so that only a `headObject`\n * is necessary to retrieve the data.\n * @param args - The function arguments\n * @param args.upload - The upload object containing metadata\n * @param args.uploadId - The upload ID for the multipart upload\n * @returns Promise that resolves when metadata is saved\n */\n async saveMetadata(args: SaveMetadataArgs): Promise<void> {\n const { upload, uploadId } = args\n log(MediaCloudLogs.S3_STORE_METADATA_SAVING)\n await this.client.putObject({\n Body: JSON.stringify(upload),\n Bucket: this.bucket,\n Key: this.generateInfoKey({ id: upload.id }),\n Metadata: {\n 'tus-version': TUS_RESUMABLE,\n 'upload-id': uploadId,\n },\n Tagging: this.generateCompleteTag('false'),\n })\n log(MediaCloudLogs.S3_STORE_METADATA_SAVED)\n }\n\n /**\n * Completes metadata by updating it with completion tags\n * @param args - The function arguments\n * @param args.upload - The completed upload object\n * @returns Promise that resolves when metadata is updated\n */\n async completeMetadata(args: CompleteMetadataArgs): Promise<void> {\n const { upload } = args\n // Always update the cache with the completed upload\n const metadata = await this.getMetadata({ id: upload.id })\n const completedMetadata: TusUploadMetadata = {\n ...metadata,\n file: upload,\n }\n await this.cache.set(upload.id, completedMetadata)\n\n if (!this.shouldUseExpirationTags()) {\n return\n }\n\n const { 'upload-id': uploadId } = metadata\n await this.client.putObject({\n Body: JSON.stringify(upload),\n Bucket: this.bucket,\n Key: this.generateInfoKey({ id: upload.id }),\n Metadata: {\n 'tus-version': TUS_RESUMABLE,\n 'upload-id': uploadId,\n },\n Tagging: this.generateCompleteTag('true'),\n })\n }\n\n /**\n * Removes cached data for a given file\n * @param args - The function arguments\n * @param args.id - The file ID to remove from cache\n * @returns Promise that resolves when cache is cleared\n */\n async clearCache(args: ClearCacheArgs): Promise<void> {\n const { id } = args\n log(MediaCloudLogs.S3_STORE_METADATA_CACHE_CLEARED)\n await this.cache.delete(id)\n }\n}\n"],"mappings":";;;;;AAmCA,MAAM,EAAE,KAAK,GAAG,iBAAiB;AAEjC,IAAa,oBAAb,MAA+B;CAC7B,YACUA,QACAC,QACAC,OACAC,yBACAC,qBACR;EALQ;EACA;EACA;EACA;EACA;CACN;;;;;;;CAOJ,gBAAgBC,MAAmC;EACjD,MAAM,EAAE,IAAI,GAAG;AACf,SAAO,GAAG,GAAG,KAAK,CAAC;CACpB;;;;;;;;CASD,gBAAgBC,MAAmC;EACjD,MAAM,EAAE,IAAI,eAAe,OAAO,GAAG;EACrC,IAAI,MAAM;AACV,MAAI,cACF,OAAO;AAOT,SAAO;CACR;;;;;;;;;CAUD,MAAM,YAAYC,MAAmD;EACnE,MAAM,EAAE,IAAI,GAAG;EACf,MAAM,SAAS,MAAM,KAAK,MAAM,IAAI,GAAG;AACvC,MAAI,OACF,QAAO;EAGT,MAAM,EAAE,MAAM,UAAU,GAAG,MAAM,KAAK,OAAO,UAAU;GACrD,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,EAAE,GAAI,EAAC;EAClC,EAAC;EACF,MAAM,OAAO,KAAK,MAAO,MAAM,MAAM,mBAAmB,CAAY;EACpE,MAAMC,WAA8B;GAClC,MAAM,IAAI,OAAO;IACf;IACA,eAAe,KAAK;IACpB,UAAU,KAAK;IACf,QAAQ,OAAO,SAAS,KAAK,QAAQ,GAAG;IACxC,MAAM,OAAO,SAAS,KAAK,KAAK,GAC5B,OAAO,SAAS,KAAK,MAAM,GAAG,GAC9B;IACJ,SAAS,KAAK;GACf;GACD,eAAe,WAAW;GAC1B,aAAa,WAAW;EACzB;EACD,MAAM,KAAK,MAAM,IAAI,IAAI,SAAS;AAClC,SAAO;CACR;;;;;;;;;;;CAYD,MAAM,aAAaC,MAAuC;EACxD,MAAM,EAAE,QAAQ,UAAU,GAAG;EAC7B,IAAI,eAAe,yBAAyB;EAC5C,MAAM,KAAK,OAAO,UAAU;GAC1B,MAAM,KAAK,UAAU,OAAO;GAC5B,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,EAAE,IAAI,OAAO,GAAI,EAAC;GAC5C,UAAU;IACR,eAAe;IACf,aAAa;GACd;GACD,SAAS,KAAK,oBAAoB,QAAQ;EAC3C,EAAC;EACF,IAAI,eAAe,wBAAwB;CAC5C;;;;;;;CAQD,MAAM,iBAAiBC,MAA2C;EAChE,MAAM,EAAE,QAAQ,GAAG;EAEnB,MAAM,WAAW,MAAM,KAAK,YAAY,EAAE,IAAI,OAAO,GAAI,EAAC;EAC1D,MAAMC,oBAAuC;GAC3C,GAAG;GACH,MAAM;EACP;EACD,MAAM,KAAK,MAAM,IAAI,OAAO,IAAI,kBAAkB;AAElD,MAAI,CAAC,KAAK,yBAAyB,CACjC;EAGF,MAAM,EAAE,aAAa,UAAU,GAAG;EAClC,MAAM,KAAK,OAAO,UAAU;GAC1B,MAAM,KAAK,UAAU,OAAO;GAC5B,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,EAAE,IAAI,OAAO,GAAI,EAAC;GAC5C,UAAU;IACR,eAAe;IACf,aAAa;GACd;GACD,SAAS,KAAK,oBAAoB,OAAO;EAC1C,EAAC;CACH;;;;;;;CAQD,MAAM,WAAWC,MAAqC;EACpD,MAAM,EAAE,IAAI,GAAG;EACf,IAAI,eAAe,gCAAgC;EACnD,MAAM,KAAK,MAAM,OAAO,GAAG;CAC5B;AACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MediaCloudErrors, MediaCloudLogs } from "../../../types/errors.js";
|
|
2
2
|
import { useErrorHandler } from "../../../hooks/useErrorHandler.js";
|
|
3
3
|
import stream from "node:stream";
|
|
4
4
|
import { NoSuchKey, NotFound } from "@aws-sdk/client-s3";
|
|
@@ -7,7 +7,7 @@ import fs from "node:fs";
|
|
|
7
7
|
import os from "node:os";
|
|
8
8
|
|
|
9
9
|
//#region src/tus/stores/s3/parts-manager.ts
|
|
10
|
-
const { throwError,
|
|
10
|
+
const { throwError, log } = useErrorHandler();
|
|
11
11
|
var S3PartsManager = class {
|
|
12
12
|
constructor(client, bucket, minPartSize, partUploadSemaphore, metadataManager, fileOperations, generateCompleteTag) {
|
|
13
13
|
this.client = client;
|
|
@@ -30,7 +30,7 @@ var S3PartsManager = class {
|
|
|
30
30
|
const { id, partNumberMarker } = args;
|
|
31
31
|
const metadata = await this.metadataManager.getMetadata({ id });
|
|
32
32
|
if (!metadata["upload-id"]) {
|
|
33
|
-
throwError(
|
|
33
|
+
throwError(MediaCloudErrors.MUX_UPLOAD_ID_MISSING);
|
|
34
34
|
throw new Error();
|
|
35
35
|
}
|
|
36
36
|
const params = {
|
|
@@ -76,7 +76,7 @@ var S3PartsManager = class {
|
|
|
76
76
|
const result = await this.client.completeMultipartUpload(params);
|
|
77
77
|
return result.Location;
|
|
78
78
|
} catch (_error) {
|
|
79
|
-
throwError(
|
|
79
|
+
throwError(MediaCloudErrors.TUS_UPLOAD_ERROR);
|
|
80
80
|
throw new Error();
|
|
81
81
|
}
|
|
82
82
|
}
|
|
@@ -199,7 +199,7 @@ var S3PartsManager = class {
|
|
|
199
199
|
}),
|
|
200
200
|
Tagging: this.generateCompleteTag("false")
|
|
201
201
|
});
|
|
202
|
-
|
|
202
|
+
log(MediaCloudLogs.S3_STORE_INCOMPLETE_PART_UPLOADED);
|
|
203
203
|
return data.ETag;
|
|
204
204
|
}
|
|
205
205
|
/**
|
|
@@ -214,7 +214,7 @@ var S3PartsManager = class {
|
|
|
214
214
|
const { metadata, readStream, partNumber } = args;
|
|
215
215
|
const permit = await this.partUploadSemaphore.acquire();
|
|
216
216
|
if (!metadata["upload-id"]) {
|
|
217
|
-
throwError(
|
|
217
|
+
throwError(MediaCloudErrors.MUX_UPLOAD_ID_MISSING);
|
|
218
218
|
throw new Error();
|
|
219
219
|
}
|
|
220
220
|
const params = {
|
|
@@ -231,7 +231,7 @@ var S3PartsManager = class {
|
|
|
231
231
|
PartNumber: partNumber
|
|
232
232
|
};
|
|
233
233
|
} catch (_error) {
|
|
234
|
-
throwError(
|
|
234
|
+
throwError(MediaCloudErrors.TUS_UPLOAD_ERROR);
|
|
235
235
|
throw new Error();
|
|
236
236
|
} finally {
|
|
237
237
|
permit();
|
|
@@ -310,7 +310,7 @@ var S3PartsManager = class {
|
|
|
310
310
|
if (pendingChunkFilepath !== null) try {
|
|
311
311
|
await fs.promises.rm(pendingChunkFilepath);
|
|
312
312
|
} catch {
|
|
313
|
-
|
|
313
|
+
log(MediaCloudLogs.S3_STORE_CHUNK_REMOVAL_FAILED);
|
|
314
314
|
}
|
|
315
315
|
const mappedError = error instanceof Error ? error : new Error(String(error));
|
|
316
316
|
promises.push(Promise.reject(mappedError));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parts-manager.js","names":["client: S3","bucket: string","minPartSize: number","partUploadSemaphore: Semaphore","metadataManager: S3MetadataManager","fileOperations: S3FileOperations","generateCompleteTag: (value: 'false' | 'true') => string | undefined","args: RetrievePartsArgs","params: AWS.ListPartsCommandInput","args: FinishMultipartUploadArgs","args: GetIncompletePartArgs","args: GetIncompletePartSizeArgs","args: DeleteIncompletePartArgs","args: DownloadIncompletePartArgs","options: { cleanUpOnEnd: boolean }","args: UploadIncompletePartArgs","args: UploadPartArgs","params: AWS.UploadPartCommandInput","args: UploadPartsArgs","promises: Promise<void>[]","pendingChunkFilepath: null | string","permit: SemaphorePermit | undefined"],"sources":["../../../../src/tus/stores/s3/parts-manager.ts"],"sourcesContent":["import fs from 'node:fs'\nimport os from 'node:os'\nimport stream from 'node:stream'\n\nimport { NoSuchKey, NotFound, type S3 } from '@aws-sdk/client-s3'\nimport { StreamSplitter } from '@tus/utils'\n\nimport { useErrorHandler } from '../../../hooks/useErrorHandler'\nimport { MediaCloudError } from '../../../types/errors'\n\nimport type { Readable } from 'node:stream'\nimport type AWS from '@aws-sdk/client-s3'\nimport type { IncompletePartInfo, TusUploadMetadata } from '../../../types'\nimport type { S3FileOperations } from './file-operations'\nimport type { S3MetadataManager } from './metadata-manager'\nimport type { Semaphore, SemaphorePermit } from './semaphore'\n\ntype RetrievePartsArgs = {\n id: string\n partNumberMarker?: string\n}\n\ntype FinishMultipartUploadArgs = {\n metadata: TusUploadMetadata\n parts: Array<AWS.Part>\n}\n\ntype GetIncompletePartArgs = {\n id: string\n}\n\ntype GetIncompletePartSizeArgs = {\n id: string\n}\n\ntype DeleteIncompletePartArgs = {\n id: string\n}\n\ntype DownloadIncompletePartArgs = {\n id: string\n}\n\ntype UploadIncompletePartArgs = {\n id: string\n readStream: fs.ReadStream | Readable\n}\n\ntype UploadPartArgs = {\n metadata: TusUploadMetadata\n readStream: fs.ReadStream | Readable\n partNumber: number\n}\n\ntype UploadPartsArgs = {\n metadata: TusUploadMetadata\n readStream: stream.Readable\n currentPartNumber: number\n offset: number\n}\n\nconst { throwError, logConsole } = useErrorHandler()\n\nexport class S3PartsManager {\n constructor(\n private client: S3,\n private bucket: string,\n private minPartSize: number,\n private partUploadSemaphore: Semaphore,\n private metadataManager: S3MetadataManager,\n private fileOperations: S3FileOperations,\n private generateCompleteTag: (value: 'false' | 'true') => string | undefined\n ) {}\n\n /**\n * Gets the number of complete parts/chunks already uploaded to S3.\n * Retrieves only consecutive parts.\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @param args.partNumberMarker - Marker for pagination (optional)\n * @returns Promise that resolves to array of uploaded parts\n */\n async retrieveParts(args: RetrievePartsArgs): Promise<Array<AWS.Part>> {\n const { id, partNumberMarker } = args\n const metadata = await this.metadataManager.getMetadata({ id })\n\n if (!metadata['upload-id']) {\n throwError(MediaCloudError.MUX_UPLOAD_ID_MISSING)\n throw new Error() // This will never execute but satisfies TypeScript\n }\n\n const params: AWS.ListPartsCommandInput = {\n Bucket: this.bucket,\n Key: id,\n PartNumberMarker: partNumberMarker,\n UploadId: metadata['upload-id'],\n }\n\n const data = await this.client.listParts(params)\n\n let parts = data.Parts ?? []\n\n if (data.IsTruncated) {\n const rest = await this.retrieveParts({\n id,\n partNumberMarker: data.NextPartNumberMarker,\n })\n parts = [...parts, ...rest]\n }\n\n if (!partNumberMarker) {\n parts.sort((a, b) => (a.PartNumber || 0) - (b.PartNumber || 0))\n }\n\n return parts\n }\n\n /**\n * Completes a multipart upload on S3.\n * This is where S3 concatenates all the uploaded parts.\n * @param args - The function arguments\n * @param args.metadata - The upload metadata\n * @param args.parts - Array of uploaded parts to complete\n * @returns Promise that resolves to the location URL (optional)\n */\n async finishMultipartUpload(\n args: FinishMultipartUploadArgs\n ): Promise<string | undefined> {\n const { metadata, parts } = args\n const params = {\n Bucket: this.bucket,\n Key: metadata.file.id,\n MultipartUpload: {\n Parts: parts.map((part) => {\n return {\n ETag: part.ETag,\n PartNumber: part.PartNumber,\n }\n }),\n },\n UploadId: metadata['upload-id'],\n }\n\n try {\n const result = await this.client.completeMultipartUpload(params)\n return result.Location\n } catch (_error) {\n throwError(MediaCloudError.TUS_UPLOAD_ERROR)\n throw new Error() // This will never execute but satisfies TypeScript\n }\n }\n\n /**\n * Gets incomplete part from S3\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @returns Promise that resolves to readable stream or undefined if not found\n */\n async getIncompletePart(\n args: GetIncompletePartArgs\n ): Promise<Readable | undefined> {\n const { id } = args\n try {\n const data = await this.client.getObject({\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({ id, isIncomplete: true }),\n })\n return data.Body as Readable\n } catch (error) {\n if (error instanceof NoSuchKey) {\n return undefined\n }\n throw error\n }\n }\n\n /**\n * Gets the size of an incomplete part\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @returns Promise that resolves to part size or undefined if not found\n */\n async getIncompletePartSize(\n args: GetIncompletePartSizeArgs\n ): Promise<number | undefined> {\n const { id } = args\n try {\n const data = await this.client.headObject({\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({ id, isIncomplete: true }),\n })\n return data.ContentLength\n } catch (error) {\n if (error instanceof NotFound) {\n return undefined\n }\n throw error\n }\n }\n\n /**\n * Deletes an incomplete part\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @returns Promise that resolves when deletion is complete\n */\n async deleteIncompletePart(args: DeleteIncompletePartArgs): Promise<void> {\n const { id } = args\n await this.client.deleteObject({\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({ id, isIncomplete: true }),\n })\n }\n\n /**\n * Downloads incomplete part to temporary file\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @returns Promise that resolves to incomplete part info or undefined if not found\n */\n async downloadIncompletePart(\n args: DownloadIncompletePartArgs\n ): Promise<IncompletePartInfo | undefined> {\n const { id } = args\n const incompletePart = await this.getIncompletePart({ id })\n\n if (!incompletePart) {\n return\n }\n const filePath = await this.fileOperations.generateUniqueTmpFileName({\n template: 'tus-s3-incomplete-part-',\n })\n\n try {\n let incompletePartSize = 0\n\n const byteCounterTransform = new stream.Transform({\n transform(chunk, _, callback) {\n incompletePartSize += chunk.length\n callback(null, chunk)\n },\n })\n\n // Write to temporary file\n await stream.promises.pipeline(\n incompletePart,\n byteCounterTransform,\n fs.createWriteStream(filePath)\n )\n\n const createReadStream = (options: { cleanUpOnEnd: boolean }) => {\n const fileReader = fs.createReadStream(filePath)\n\n if (options.cleanUpOnEnd) {\n fileReader.on('end', () => {\n fs.unlink(filePath, () => {})\n })\n\n fileReader.on('error', (err) => {\n fileReader.destroy(err)\n fs.unlink(filePath, () => {})\n })\n }\n\n return fileReader\n }\n\n return {\n createReader: createReadStream,\n path: filePath,\n size: incompletePartSize,\n }\n } catch (err) {\n fs.promises.rm(filePath).catch(() => {})\n throw err\n }\n }\n\n /**\n * Uploads an incomplete part\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @param args.readStream - The stream to read data from\n * @returns Promise that resolves to the ETag of the uploaded part\n */\n async uploadIncompletePart(args: UploadIncompletePartArgs): Promise<string> {\n const { id, readStream } = args\n const data = await this.client.putObject({\n Body: readStream,\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({ id, isIncomplete: true }),\n Tagging: this.generateCompleteTag('false'),\n })\n logConsole(MediaCloudError.S3_STORE_INCOMPLETE_PART_UPLOADED)\n return data.ETag as string\n }\n\n /**\n * Uploads a single part\n * @param args - The function arguments\n * @param args.metadata - The upload metadata\n * @param args.readStream - The stream to read data from\n * @param args.partNumber - The part number to upload\n * @returns Promise that resolves to the ETag of the uploaded part\n */\n async uploadPart(args: UploadPartArgs): Promise<AWS.Part> {\n const { metadata, readStream, partNumber } = args\n const permit = await this.partUploadSemaphore.acquire()\n\n if (!metadata['upload-id']) {\n throwError(MediaCloudError.MUX_UPLOAD_ID_MISSING)\n throw new Error() // This will never execute but satisfies TypeScript\n }\n\n const params: AWS.UploadPartCommandInput = {\n Body: readStream,\n Bucket: this.bucket,\n Key: metadata.file.id,\n PartNumber: partNumber,\n UploadId: metadata['upload-id'],\n }\n\n try {\n const data = await this.client.uploadPart(params)\n return { ETag: data.ETag, PartNumber: partNumber }\n } catch (_error) {\n throwError(MediaCloudError.TUS_UPLOAD_ERROR)\n throw new Error() // This will never execute but satisfies TypeScript\n } finally {\n permit()\n }\n }\n\n /**\n * Uploads a stream to s3 using multiple parts\n * @param args - The function arguments\n * @param args.metadata - The upload metadata\n * @param args.readStream - The stream to read data from\n * @param args.currentPartNumber - The current part number to start from\n * @param args.offset - The byte offset to start from\n * @returns Promise that resolves to the number of bytes uploaded\n */\n async uploadParts(args: UploadPartsArgs): Promise<number> {\n const { metadata, readStream, offset: initialOffset } = args\n let { currentPartNumber } = args\n let offset = initialOffset\n const size = metadata.file.size\n const promises: Promise<void>[] = []\n let pendingChunkFilepath: null | string = null\n let bytesUploaded = 0\n let permit: SemaphorePermit | undefined = undefined\n\n const splitterStream = new StreamSplitter({\n chunkSize: this.fileOperations.calculateOptimalPartSize({ size }),\n directory: os.tmpdir(),\n })\n .on('beforeChunkStarted', async () => {\n permit = await this.partUploadSemaphore.acquire()\n })\n .on('chunkStarted', (filepath) => {\n pendingChunkFilepath = filepath\n })\n .on('chunkFinished', ({ path, size: partSize }) => {\n pendingChunkFilepath = null\n\n const acquiredPermit = permit\n const partNumber = currentPartNumber++\n\n offset += partSize\n\n const isFinalPart = size === offset\n\n const uploadChunk = async () => {\n try {\n // Only the first chunk of each PATCH request can prepend\n // an incomplete part (last chunk) from the previous request.\n const readable = fs.createReadStream(path)\n readable.on('error', function (error) {\n throw error\n })\n\n switch (true) {\n case partSize >= this.minPartSize || isFinalPart:\n await this.uploadPart({\n metadata,\n readStream: readable,\n partNumber,\n })\n break\n default:\n await this.uploadIncompletePart({\n id: metadata.file.id,\n readStream: readable,\n })\n break\n }\n\n bytesUploaded += partSize\n } catch (error) {\n // Destroy the splitter to stop processing more chunks\n const mappedError =\n error instanceof Error ? error : new Error(String(error))\n splitterStream.destroy(mappedError)\n throw mappedError\n } finally {\n fs.promises.rm(path).catch(function () {})\n acquiredPermit?.()\n }\n }\n\n const deferred = uploadChunk()\n\n promises.push(deferred)\n })\n .on('chunkError', () => {\n permit?.()\n })\n\n try {\n await stream.promises.pipeline(readStream, splitterStream)\n } catch (error) {\n if (pendingChunkFilepath !== null) {\n try {\n await fs.promises.rm(pendingChunkFilepath)\n } catch {\n logConsole(MediaCloudError.S3_STORE_CHUNK_REMOVAL_FAILED)\n }\n }\n const mappedError =\n error instanceof Error ? error : new Error(String(error))\n promises.push(Promise.reject(mappedError))\n } finally {\n // Wait for all promises\n await Promise.allSettled(promises)\n // Reject the promise if any of the promises reject\n await Promise.all(promises)\n }\n\n return bytesUploaded\n }\n}\n"],"mappings":";;;;;;;;;AA6DA,MAAM,EAAE,YAAY,YAAY,GAAG,iBAAiB;AAEpD,IAAa,iBAAb,MAA4B;CAC1B,YACUA,QACAC,QACAC,aACAC,qBACAC,iBACAC,gBACAC,qBACR;EAPQ;EACA;EACA;EACA;EACA;EACA;EACA;CACN;;;;;;;;;CAUJ,MAAM,cAAcC,MAAmD;EACrE,MAAM,EAAE,IAAI,kBAAkB,GAAG;EACjC,MAAM,WAAW,MAAM,KAAK,gBAAgB,YAAY,EAAE,GAAI,EAAC;AAE/D,MAAI,CAAC,SAAS,cAAc;GAC1B,WAAW,gBAAgB,sBAAsB;AACjD,SAAM,IAAI;EACX;EAED,MAAMC,SAAoC;GACxC,QAAQ,KAAK;GACb,KAAK;GACL,kBAAkB;GAClB,UAAU,SAAS;EACpB;EAED,MAAM,OAAO,MAAM,KAAK,OAAO,UAAU,OAAO;EAEhD,IAAI,QAAQ,KAAK,SAAS,CAAE;AAE5B,MAAI,KAAK,aAAa;GACpB,MAAM,OAAO,MAAM,KAAK,cAAc;IACpC;IACA,kBAAkB,KAAK;GACxB,EAAC;GACF,QAAQ,CAAC,GAAG,OAAO,GAAG,IAAK;EAC5B;AAED,MAAI,CAAC,kBACH,MAAM,KAAK,CAAC,GAAG,OAAO,EAAE,cAAc,MAAM,EAAE,cAAc,GAAG;AAGjE,SAAO;CACR;;;;;;;;;CAUD,MAAM,sBACJC,MAC6B;EAC7B,MAAM,EAAE,UAAU,OAAO,GAAG;EAC5B,MAAM,SAAS;GACb,QAAQ,KAAK;GACb,KAAK,SAAS,KAAK;GACnB,iBAAiB,EACf,OAAO,MAAM,IAAI,CAAC,SAAS;AACzB,WAAO;KACL,MAAM,KAAK;KACX,YAAY,KAAK;IAClB;GACF,EAAC,CACH;GACD,UAAU,SAAS;EACpB;AAED,MAAI;GACF,MAAM,SAAS,MAAM,KAAK,OAAO,wBAAwB,OAAO;AAChE,UAAO,OAAO;EACf,SAAQ,QAAQ;GACf,WAAW,gBAAgB,iBAAiB;AAC5C,SAAM,IAAI;EACX;CACF;;;;;;;CAQD,MAAM,kBACJC,MAC+B;EAC/B,MAAM,EAAE,IAAI,GAAG;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,OAAO,UAAU;IACvC,QAAQ,KAAK;IACb,KAAK,KAAK,gBAAgB,gBAAgB;KAAE;KAAI,cAAc;IAAM,EAAC;GACtE,EAAC;AACF,UAAO,KAAK;EACb,SAAQ,OAAO;AACd,OAAI,iBAAiB,UACnB,QAAO;AAET,SAAM;EACP;CACF;;;;;;;CAQD,MAAM,sBACJC,MAC6B;EAC7B,MAAM,EAAE,IAAI,GAAG;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,OAAO,WAAW;IACxC,QAAQ,KAAK;IACb,KAAK,KAAK,gBAAgB,gBAAgB;KAAE;KAAI,cAAc;IAAM,EAAC;GACtE,EAAC;AACF,UAAO,KAAK;EACb,SAAQ,OAAO;AACd,OAAI,iBAAiB,SACnB,QAAO;AAET,SAAM;EACP;CACF;;;;;;;CAQD,MAAM,qBAAqBC,MAA+C;EACxE,MAAM,EAAE,IAAI,GAAG;EACf,MAAM,KAAK,OAAO,aAAa;GAC7B,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,gBAAgB;IAAE;IAAI,cAAc;GAAM,EAAC;EACtE,EAAC;CACH;;;;;;;CAQD,MAAM,uBACJC,MACyC;EACzC,MAAM,EAAE,IAAI,GAAG;EACf,MAAM,iBAAiB,MAAM,KAAK,kBAAkB,EAAE,GAAI,EAAC;AAE3D,MAAI,CAAC,eACH;EAEF,MAAM,WAAW,MAAM,KAAK,eAAe,0BAA0B,EACnE,UAAU,0BACX,EAAC;AAEF,MAAI;GACF,IAAI,qBAAqB;GAEzB,MAAM,uBAAuB,IAAI,OAAO,UAAU,EAChD,UAAU,OAAO,GAAG,UAAU;IAC5B,sBAAsB,MAAM;IAC5B,SAAS,MAAM,MAAM;GACtB,EACF;GAGD,MAAM,OAAO,SAAS,SACpB,gBACA,sBACA,GAAG,kBAAkB,SAAS,CAC/B;GAED,MAAM,mBAAmB,CAACC,YAAuC;IAC/D,MAAM,aAAa,GAAG,iBAAiB,SAAS;AAEhD,QAAI,QAAQ,cAAc;KACxB,WAAW,GAAG,OAAO,MAAM;MACzB,GAAG,OAAO,UAAU,MAAM,CAAE,EAAC;KAC9B,EAAC;KAEF,WAAW,GAAG,SAAS,CAAC,QAAQ;MAC9B,WAAW,QAAQ,IAAI;MACvB,GAAG,OAAO,UAAU,MAAM,CAAE,EAAC;KAC9B,EAAC;IACH;AAED,WAAO;GACR;AAED,UAAO;IACL,cAAc;IACd,MAAM;IACN,MAAM;GACP;EACF,SAAQ,KAAK;GACZ,GAAG,SAAS,GAAG,SAAS,CAAC,MAAM,MAAM,CAAE,EAAC;AACxC,SAAM;EACP;CACF;;;;;;;;CASD,MAAM,qBAAqBC,MAAiD;EAC1E,MAAM,EAAE,IAAI,YAAY,GAAG;EAC3B,MAAM,OAAO,MAAM,KAAK,OAAO,UAAU;GACvC,MAAM;GACN,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,gBAAgB;IAAE;IAAI,cAAc;GAAM,EAAC;GACrE,SAAS,KAAK,oBAAoB,QAAQ;EAC3C,EAAC;EACF,WAAW,gBAAgB,kCAAkC;AAC7D,SAAO,KAAK;CACb;;;;;;;;;CAUD,MAAM,WAAWC,MAAyC;EACxD,MAAM,EAAE,UAAU,YAAY,YAAY,GAAG;EAC7C,MAAM,SAAS,MAAM,KAAK,oBAAoB,SAAS;AAEvD,MAAI,CAAC,SAAS,cAAc;GAC1B,WAAW,gBAAgB,sBAAsB;AACjD,SAAM,IAAI;EACX;EAED,MAAMC,SAAqC;GACzC,MAAM;GACN,QAAQ,KAAK;GACb,KAAK,SAAS,KAAK;GACnB,YAAY;GACZ,UAAU,SAAS;EACpB;AAED,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,OAAO,WAAW,OAAO;AACjD,UAAO;IAAE,MAAM,KAAK;IAAM,YAAY;GAAY;EACnD,SAAQ,QAAQ;GACf,WAAW,gBAAgB,iBAAiB;AAC5C,SAAM,IAAI;EACX,UAAS;GACR,QAAQ;EACT;CACF;;;;;;;;;;CAWD,MAAM,YAAYC,MAAwC;EACxD,MAAM,EAAE,UAAU,YAAY,QAAQ,eAAe,GAAG;EACxD,IAAI,EAAE,mBAAmB,GAAG;EAC5B,IAAI,SAAS;EACb,MAAM,OAAO,SAAS,KAAK;EAC3B,MAAMC,WAA4B,CAAE;EACpC,IAAIC,uBAAsC;EAC1C,IAAI,gBAAgB;EACpB,IAAIC,SAAsC;EAE1C,MAAM,iBAAiB,IAAI,eAAe;GACxC,WAAW,KAAK,eAAe,yBAAyB,EAAE,KAAM,EAAC;GACjE,WAAW,GAAG,QAAQ;EACvB,GACE,GAAG,sBAAsB,YAAY;GACpC,SAAS,MAAM,KAAK,oBAAoB,SAAS;EAClD,EAAC,CACD,GAAG,gBAAgB,CAAC,aAAa;GAChC,uBAAuB;EACxB,EAAC,CACD,GAAG,iBAAiB,CAAC,EAAE,MAAM,MAAM,UAAU,KAAK;GACjD,uBAAuB;GAEvB,MAAM,iBAAiB;GACvB,MAAM,aAAa;GAEnB,UAAU;GAEV,MAAM,cAAc,SAAS;GAE7B,MAAM,cAAc,YAAY;AAC9B,QAAI;KAGF,MAAM,WAAW,GAAG,iBAAiB,KAAK;KAC1C,SAAS,GAAG,SAAS,SAAU,OAAO;AACpC,YAAM;KACP,EAAC;AAEF,aAAQ,MAAR;MACE,KAAK,YAAY,KAAK,eAAe;OACnC,MAAM,KAAK,WAAW;QACpB;QACA,YAAY;QACZ;OACD,EAAC;AACF;MACF;OACE,MAAM,KAAK,qBAAqB;QAC9B,IAAI,SAAS,KAAK;QAClB,YAAY;OACb,EAAC;AACF;KACH;KAED,iBAAiB;IAClB,SAAQ,OAAO;KAEd,MAAM,cACJ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,MAAM;KAC1D,eAAe,QAAQ,YAAY;AACnC,WAAM;IACP,UAAS;KACR,GAAG,SAAS,GAAG,KAAK,CAAC,MAAM,WAAY,CAAE,EAAC;KAC1C,kBAAkB;IACnB;GACF;GAED,MAAM,WAAW,aAAa;GAE9B,SAAS,KAAK,SAAS;EACxB,EAAC,CACD,GAAG,cAAc,MAAM;GACtB,UAAU;EACX,EAAC;AAEJ,MAAI;GACF,MAAM,OAAO,SAAS,SAAS,YAAY,eAAe;EAC3D,SAAQ,OAAO;AACd,OAAI,yBAAyB,KAC3B,KAAI;IACF,MAAM,GAAG,SAAS,GAAG,qBAAqB;GAC3C,QAAO;IACN,WAAW,gBAAgB,8BAA8B;GAC1D;GAEH,MAAM,cACJ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,MAAM;GAC1D,SAAS,KAAK,QAAQ,OAAO,YAAY,CAAC;EAC3C,UAAS;GAER,MAAM,QAAQ,WAAW,SAAS;GAElC,MAAM,QAAQ,IAAI,SAAS;EAC5B;AAED,SAAO;CACR;AACF"}
|
|
1
|
+
{"version":3,"file":"parts-manager.js","names":["client: S3","bucket: string","minPartSize: number","partUploadSemaphore: Semaphore","metadataManager: S3MetadataManager","fileOperations: S3FileOperations","generateCompleteTag: (value: 'false' | 'true') => string | undefined","args: RetrievePartsArgs","params: AWS.ListPartsCommandInput","args: FinishMultipartUploadArgs","args: GetIncompletePartArgs","args: GetIncompletePartSizeArgs","args: DeleteIncompletePartArgs","args: DownloadIncompletePartArgs","options: { cleanUpOnEnd: boolean }","args: UploadIncompletePartArgs","args: UploadPartArgs","params: AWS.UploadPartCommandInput","args: UploadPartsArgs","promises: Promise<void>[]","pendingChunkFilepath: null | string","permit: SemaphorePermit | undefined"],"sources":["../../../../src/tus/stores/s3/parts-manager.ts"],"sourcesContent":["import fs from 'node:fs'\nimport os from 'node:os'\nimport stream from 'node:stream'\n\nimport { NoSuchKey, NotFound, type S3 } from '@aws-sdk/client-s3'\nimport { StreamSplitter } from '@tus/utils'\n\nimport { useErrorHandler } from '../../../hooks/useErrorHandler'\nimport { MediaCloudErrors, MediaCloudLogs } from '../../../types/errors'\n\nimport type { Readable } from 'node:stream'\nimport type AWS from '@aws-sdk/client-s3'\nimport type { IncompletePartInfo, TusUploadMetadata } from '../../../types'\nimport type { S3FileOperations } from './file-operations'\nimport type { S3MetadataManager } from './metadata-manager'\nimport type { Semaphore, SemaphorePermit } from './semaphore'\n\ntype RetrievePartsArgs = {\n id: string\n partNumberMarker?: string\n}\n\ntype FinishMultipartUploadArgs = {\n metadata: TusUploadMetadata\n parts: Array<AWS.Part>\n}\n\ntype GetIncompletePartArgs = {\n id: string\n}\n\ntype GetIncompletePartSizeArgs = {\n id: string\n}\n\ntype DeleteIncompletePartArgs = {\n id: string\n}\n\ntype DownloadIncompletePartArgs = {\n id: string\n}\n\ntype UploadIncompletePartArgs = {\n id: string\n readStream: fs.ReadStream | Readable\n}\n\ntype UploadPartArgs = {\n metadata: TusUploadMetadata\n readStream: fs.ReadStream | Readable\n partNumber: number\n}\n\ntype UploadPartsArgs = {\n metadata: TusUploadMetadata\n readStream: stream.Readable\n currentPartNumber: number\n offset: number\n}\n\nconst { throwError, log } = useErrorHandler()\n\nexport class S3PartsManager {\n constructor(\n private client: S3,\n private bucket: string,\n private minPartSize: number,\n private partUploadSemaphore: Semaphore,\n private metadataManager: S3MetadataManager,\n private fileOperations: S3FileOperations,\n private generateCompleteTag: (value: 'false' | 'true') => string | undefined\n ) {}\n\n /**\n * Gets the number of complete parts/chunks already uploaded to S3.\n * Retrieves only consecutive parts.\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @param args.partNumberMarker - Marker for pagination (optional)\n * @returns Promise that resolves to array of uploaded parts\n */\n async retrieveParts(args: RetrievePartsArgs): Promise<Array<AWS.Part>> {\n const { id, partNumberMarker } = args\n const metadata = await this.metadataManager.getMetadata({ id })\n\n if (!metadata['upload-id']) {\n throwError(MediaCloudErrors.MUX_UPLOAD_ID_MISSING)\n throw new Error() // This will never execute but satisfies TypeScript\n }\n\n const params: AWS.ListPartsCommandInput = {\n Bucket: this.bucket,\n Key: id,\n PartNumberMarker: partNumberMarker,\n UploadId: metadata['upload-id'],\n }\n\n const data = await this.client.listParts(params)\n\n let parts = data.Parts ?? []\n\n if (data.IsTruncated) {\n const rest = await this.retrieveParts({\n id,\n partNumberMarker: data.NextPartNumberMarker,\n })\n parts = [...parts, ...rest]\n }\n\n if (!partNumberMarker) {\n parts.sort((a, b) => (a.PartNumber || 0) - (b.PartNumber || 0))\n }\n\n return parts\n }\n\n /**\n * Completes a multipart upload on S3.\n * This is where S3 concatenates all the uploaded parts.\n * @param args - The function arguments\n * @param args.metadata - The upload metadata\n * @param args.parts - Array of uploaded parts to complete\n * @returns Promise that resolves to the location URL (optional)\n */\n async finishMultipartUpload(\n args: FinishMultipartUploadArgs\n ): Promise<string | undefined> {\n const { metadata, parts } = args\n const params = {\n Bucket: this.bucket,\n Key: metadata.file.id,\n MultipartUpload: {\n Parts: parts.map((part) => {\n return {\n ETag: part.ETag,\n PartNumber: part.PartNumber,\n }\n }),\n },\n UploadId: metadata['upload-id'],\n }\n\n try {\n const result = await this.client.completeMultipartUpload(params)\n return result.Location\n } catch (_error) {\n throwError(MediaCloudErrors.TUS_UPLOAD_ERROR)\n throw new Error() // This will never execute but satisfies TypeScript\n }\n }\n\n /**\n * Gets incomplete part from S3\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @returns Promise that resolves to readable stream or undefined if not found\n */\n async getIncompletePart(\n args: GetIncompletePartArgs\n ): Promise<Readable | undefined> {\n const { id } = args\n try {\n const data = await this.client.getObject({\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({ id, isIncomplete: true }),\n })\n return data.Body as Readable\n } catch (error) {\n if (error instanceof NoSuchKey) {\n return undefined\n }\n throw error\n }\n }\n\n /**\n * Gets the size of an incomplete part\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @returns Promise that resolves to part size or undefined if not found\n */\n async getIncompletePartSize(\n args: GetIncompletePartSizeArgs\n ): Promise<number | undefined> {\n const { id } = args\n try {\n const data = await this.client.headObject({\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({ id, isIncomplete: true }),\n })\n return data.ContentLength\n } catch (error) {\n if (error instanceof NotFound) {\n return undefined\n }\n throw error\n }\n }\n\n /**\n * Deletes an incomplete part\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @returns Promise that resolves when deletion is complete\n */\n async deleteIncompletePart(args: DeleteIncompletePartArgs): Promise<void> {\n const { id } = args\n await this.client.deleteObject({\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({ id, isIncomplete: true }),\n })\n }\n\n /**\n * Downloads incomplete part to temporary file\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @returns Promise that resolves to incomplete part info or undefined if not found\n */\n async downloadIncompletePart(\n args: DownloadIncompletePartArgs\n ): Promise<IncompletePartInfo | undefined> {\n const { id } = args\n const incompletePart = await this.getIncompletePart({ id })\n\n if (!incompletePart) {\n return\n }\n const filePath = await this.fileOperations.generateUniqueTmpFileName({\n template: 'tus-s3-incomplete-part-',\n })\n\n try {\n let incompletePartSize = 0\n\n const byteCounterTransform = new stream.Transform({\n transform(chunk, _, callback) {\n incompletePartSize += chunk.length\n callback(null, chunk)\n },\n })\n\n // Write to temporary file\n await stream.promises.pipeline(\n incompletePart,\n byteCounterTransform,\n fs.createWriteStream(filePath)\n )\n\n const createReadStream = (options: { cleanUpOnEnd: boolean }) => {\n const fileReader = fs.createReadStream(filePath)\n\n if (options.cleanUpOnEnd) {\n fileReader.on('end', () => {\n fs.unlink(filePath, () => {})\n })\n\n fileReader.on('error', (err) => {\n fileReader.destroy(err)\n fs.unlink(filePath, () => {})\n })\n }\n\n return fileReader\n }\n\n return {\n createReader: createReadStream,\n path: filePath,\n size: incompletePartSize,\n }\n } catch (err) {\n fs.promises.rm(filePath).catch(() => {})\n throw err\n }\n }\n\n /**\n * Uploads an incomplete part\n * @param args - The function arguments\n * @param args.id - The upload ID\n * @param args.readStream - The stream to read data from\n * @returns Promise that resolves to the ETag of the uploaded part\n */\n async uploadIncompletePart(args: UploadIncompletePartArgs): Promise<string> {\n const { id, readStream } = args\n const data = await this.client.putObject({\n Body: readStream,\n Bucket: this.bucket,\n Key: this.metadataManager.generatePartKey({ id, isIncomplete: true }),\n Tagging: this.generateCompleteTag('false'),\n })\n log(MediaCloudLogs.S3_STORE_INCOMPLETE_PART_UPLOADED)\n return data.ETag as string\n }\n\n /**\n * Uploads a single part\n * @param args - The function arguments\n * @param args.metadata - The upload metadata\n * @param args.readStream - The stream to read data from\n * @param args.partNumber - The part number to upload\n * @returns Promise that resolves to the ETag of the uploaded part\n */\n async uploadPart(args: UploadPartArgs): Promise<AWS.Part> {\n const { metadata, readStream, partNumber } = args\n const permit = await this.partUploadSemaphore.acquire()\n\n if (!metadata['upload-id']) {\n throwError(MediaCloudErrors.MUX_UPLOAD_ID_MISSING)\n throw new Error() // This will never execute but satisfies TypeScript\n }\n\n const params: AWS.UploadPartCommandInput = {\n Body: readStream,\n Bucket: this.bucket,\n Key: metadata.file.id,\n PartNumber: partNumber,\n UploadId: metadata['upload-id'],\n }\n\n try {\n const data = await this.client.uploadPart(params)\n return { ETag: data.ETag, PartNumber: partNumber }\n } catch (_error) {\n throwError(MediaCloudErrors.TUS_UPLOAD_ERROR)\n throw new Error() // This will never execute but satisfies TypeScript\n } finally {\n permit()\n }\n }\n\n /**\n * Uploads a stream to s3 using multiple parts\n * @param args - The function arguments\n * @param args.metadata - The upload metadata\n * @param args.readStream - The stream to read data from\n * @param args.currentPartNumber - The current part number to start from\n * @param args.offset - The byte offset to start from\n * @returns Promise that resolves to the number of bytes uploaded\n */\n async uploadParts(args: UploadPartsArgs): Promise<number> {\n const { metadata, readStream, offset: initialOffset } = args\n let { currentPartNumber } = args\n let offset = initialOffset\n const size = metadata.file.size\n const promises: Promise<void>[] = []\n let pendingChunkFilepath: null | string = null\n let bytesUploaded = 0\n let permit: SemaphorePermit | undefined = undefined\n\n const splitterStream = new StreamSplitter({\n chunkSize: this.fileOperations.calculateOptimalPartSize({ size }),\n directory: os.tmpdir(),\n })\n .on('beforeChunkStarted', async () => {\n permit = await this.partUploadSemaphore.acquire()\n })\n .on('chunkStarted', (filepath) => {\n pendingChunkFilepath = filepath\n })\n .on('chunkFinished', ({ path, size: partSize }) => {\n pendingChunkFilepath = null\n\n const acquiredPermit = permit\n const partNumber = currentPartNumber++\n\n offset += partSize\n\n const isFinalPart = size === offset\n\n const uploadChunk = async () => {\n try {\n // Only the first chunk of each PATCH request can prepend\n // an incomplete part (last chunk) from the previous request.\n const readable = fs.createReadStream(path)\n readable.on('error', function (error) {\n throw error\n })\n\n switch (true) {\n case partSize >= this.minPartSize || isFinalPart:\n await this.uploadPart({\n metadata,\n readStream: readable,\n partNumber,\n })\n break\n default:\n await this.uploadIncompletePart({\n id: metadata.file.id,\n readStream: readable,\n })\n break\n }\n\n bytesUploaded += partSize\n } catch (error) {\n // Destroy the splitter to stop processing more chunks\n const mappedError =\n error instanceof Error ? error : new Error(String(error))\n splitterStream.destroy(mappedError)\n throw mappedError\n } finally {\n fs.promises.rm(path).catch(function () {})\n acquiredPermit?.()\n }\n }\n\n const deferred = uploadChunk()\n\n promises.push(deferred)\n })\n .on('chunkError', () => {\n permit?.()\n })\n\n try {\n await stream.promises.pipeline(readStream, splitterStream)\n } catch (error) {\n if (pendingChunkFilepath !== null) {\n try {\n await fs.promises.rm(pendingChunkFilepath)\n } catch {\n log(MediaCloudLogs.S3_STORE_CHUNK_REMOVAL_FAILED)\n }\n }\n const mappedError =\n error instanceof Error ? error : new Error(String(error))\n promises.push(Promise.reject(mappedError))\n } finally {\n // Wait for all promises\n await Promise.allSettled(promises)\n // Reject the promise if any of the promises reject\n await Promise.all(promises)\n }\n\n return bytesUploaded\n }\n}\n"],"mappings":";;;;;;;;;AA6DA,MAAM,EAAE,YAAY,KAAK,GAAG,iBAAiB;AAE7C,IAAa,iBAAb,MAA4B;CAC1B,YACUA,QACAC,QACAC,aACAC,qBACAC,iBACAC,gBACAC,qBACR;EAPQ;EACA;EACA;EACA;EACA;EACA;EACA;CACN;;;;;;;;;CAUJ,MAAM,cAAcC,MAAmD;EACrE,MAAM,EAAE,IAAI,kBAAkB,GAAG;EACjC,MAAM,WAAW,MAAM,KAAK,gBAAgB,YAAY,EAAE,GAAI,EAAC;AAE/D,MAAI,CAAC,SAAS,cAAc;GAC1B,WAAW,iBAAiB,sBAAsB;AAClD,SAAM,IAAI;EACX;EAED,MAAMC,SAAoC;GACxC,QAAQ,KAAK;GACb,KAAK;GACL,kBAAkB;GAClB,UAAU,SAAS;EACpB;EAED,MAAM,OAAO,MAAM,KAAK,OAAO,UAAU,OAAO;EAEhD,IAAI,QAAQ,KAAK,SAAS,CAAE;AAE5B,MAAI,KAAK,aAAa;GACpB,MAAM,OAAO,MAAM,KAAK,cAAc;IACpC;IACA,kBAAkB,KAAK;GACxB,EAAC;GACF,QAAQ,CAAC,GAAG,OAAO,GAAG,IAAK;EAC5B;AAED,MAAI,CAAC,kBACH,MAAM,KAAK,CAAC,GAAG,OAAO,EAAE,cAAc,MAAM,EAAE,cAAc,GAAG;AAGjE,SAAO;CACR;;;;;;;;;CAUD,MAAM,sBACJC,MAC6B;EAC7B,MAAM,EAAE,UAAU,OAAO,GAAG;EAC5B,MAAM,SAAS;GACb,QAAQ,KAAK;GACb,KAAK,SAAS,KAAK;GACnB,iBAAiB,EACf,OAAO,MAAM,IAAI,CAAC,SAAS;AACzB,WAAO;KACL,MAAM,KAAK;KACX,YAAY,KAAK;IAClB;GACF,EAAC,CACH;GACD,UAAU,SAAS;EACpB;AAED,MAAI;GACF,MAAM,SAAS,MAAM,KAAK,OAAO,wBAAwB,OAAO;AAChE,UAAO,OAAO;EACf,SAAQ,QAAQ;GACf,WAAW,iBAAiB,iBAAiB;AAC7C,SAAM,IAAI;EACX;CACF;;;;;;;CAQD,MAAM,kBACJC,MAC+B;EAC/B,MAAM,EAAE,IAAI,GAAG;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,OAAO,UAAU;IACvC,QAAQ,KAAK;IACb,KAAK,KAAK,gBAAgB,gBAAgB;KAAE;KAAI,cAAc;IAAM,EAAC;GACtE,EAAC;AACF,UAAO,KAAK;EACb,SAAQ,OAAO;AACd,OAAI,iBAAiB,UACnB,QAAO;AAET,SAAM;EACP;CACF;;;;;;;CAQD,MAAM,sBACJC,MAC6B;EAC7B,MAAM,EAAE,IAAI,GAAG;AACf,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,OAAO,WAAW;IACxC,QAAQ,KAAK;IACb,KAAK,KAAK,gBAAgB,gBAAgB;KAAE;KAAI,cAAc;IAAM,EAAC;GACtE,EAAC;AACF,UAAO,KAAK;EACb,SAAQ,OAAO;AACd,OAAI,iBAAiB,SACnB,QAAO;AAET,SAAM;EACP;CACF;;;;;;;CAQD,MAAM,qBAAqBC,MAA+C;EACxE,MAAM,EAAE,IAAI,GAAG;EACf,MAAM,KAAK,OAAO,aAAa;GAC7B,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,gBAAgB;IAAE;IAAI,cAAc;GAAM,EAAC;EACtE,EAAC;CACH;;;;;;;CAQD,MAAM,uBACJC,MACyC;EACzC,MAAM,EAAE,IAAI,GAAG;EACf,MAAM,iBAAiB,MAAM,KAAK,kBAAkB,EAAE,GAAI,EAAC;AAE3D,MAAI,CAAC,eACH;EAEF,MAAM,WAAW,MAAM,KAAK,eAAe,0BAA0B,EACnE,UAAU,0BACX,EAAC;AAEF,MAAI;GACF,IAAI,qBAAqB;GAEzB,MAAM,uBAAuB,IAAI,OAAO,UAAU,EAChD,UAAU,OAAO,GAAG,UAAU;IAC5B,sBAAsB,MAAM;IAC5B,SAAS,MAAM,MAAM;GACtB,EACF;GAGD,MAAM,OAAO,SAAS,SACpB,gBACA,sBACA,GAAG,kBAAkB,SAAS,CAC/B;GAED,MAAM,mBAAmB,CAACC,YAAuC;IAC/D,MAAM,aAAa,GAAG,iBAAiB,SAAS;AAEhD,QAAI,QAAQ,cAAc;KACxB,WAAW,GAAG,OAAO,MAAM;MACzB,GAAG,OAAO,UAAU,MAAM,CAAE,EAAC;KAC9B,EAAC;KAEF,WAAW,GAAG,SAAS,CAAC,QAAQ;MAC9B,WAAW,QAAQ,IAAI;MACvB,GAAG,OAAO,UAAU,MAAM,CAAE,EAAC;KAC9B,EAAC;IACH;AAED,WAAO;GACR;AAED,UAAO;IACL,cAAc;IACd,MAAM;IACN,MAAM;GACP;EACF,SAAQ,KAAK;GACZ,GAAG,SAAS,GAAG,SAAS,CAAC,MAAM,MAAM,CAAE,EAAC;AACxC,SAAM;EACP;CACF;;;;;;;;CASD,MAAM,qBAAqBC,MAAiD;EAC1E,MAAM,EAAE,IAAI,YAAY,GAAG;EAC3B,MAAM,OAAO,MAAM,KAAK,OAAO,UAAU;GACvC,MAAM;GACN,QAAQ,KAAK;GACb,KAAK,KAAK,gBAAgB,gBAAgB;IAAE;IAAI,cAAc;GAAM,EAAC;GACrE,SAAS,KAAK,oBAAoB,QAAQ;EAC3C,EAAC;EACF,IAAI,eAAe,kCAAkC;AACrD,SAAO,KAAK;CACb;;;;;;;;;CAUD,MAAM,WAAWC,MAAyC;EACxD,MAAM,EAAE,UAAU,YAAY,YAAY,GAAG;EAC7C,MAAM,SAAS,MAAM,KAAK,oBAAoB,SAAS;AAEvD,MAAI,CAAC,SAAS,cAAc;GAC1B,WAAW,iBAAiB,sBAAsB;AAClD,SAAM,IAAI;EACX;EAED,MAAMC,SAAqC;GACzC,MAAM;GACN,QAAQ,KAAK;GACb,KAAK,SAAS,KAAK;GACnB,YAAY;GACZ,UAAU,SAAS;EACpB;AAED,MAAI;GACF,MAAM,OAAO,MAAM,KAAK,OAAO,WAAW,OAAO;AACjD,UAAO;IAAE,MAAM,KAAK;IAAM,YAAY;GAAY;EACnD,SAAQ,QAAQ;GACf,WAAW,iBAAiB,iBAAiB;AAC7C,SAAM,IAAI;EACX,UAAS;GACR,QAAQ;EACT;CACF;;;;;;;;;;CAWD,MAAM,YAAYC,MAAwC;EACxD,MAAM,EAAE,UAAU,YAAY,QAAQ,eAAe,GAAG;EACxD,IAAI,EAAE,mBAAmB,GAAG;EAC5B,IAAI,SAAS;EACb,MAAM,OAAO,SAAS,KAAK;EAC3B,MAAMC,WAA4B,CAAE;EACpC,IAAIC,uBAAsC;EAC1C,IAAI,gBAAgB;EACpB,IAAIC,SAAsC;EAE1C,MAAM,iBAAiB,IAAI,eAAe;GACxC,WAAW,KAAK,eAAe,yBAAyB,EAAE,KAAM,EAAC;GACjE,WAAW,GAAG,QAAQ;EACvB,GACE,GAAG,sBAAsB,YAAY;GACpC,SAAS,MAAM,KAAK,oBAAoB,SAAS;EAClD,EAAC,CACD,GAAG,gBAAgB,CAAC,aAAa;GAChC,uBAAuB;EACxB,EAAC,CACD,GAAG,iBAAiB,CAAC,EAAE,MAAM,MAAM,UAAU,KAAK;GACjD,uBAAuB;GAEvB,MAAM,iBAAiB;GACvB,MAAM,aAAa;GAEnB,UAAU;GAEV,MAAM,cAAc,SAAS;GAE7B,MAAM,cAAc,YAAY;AAC9B,QAAI;KAGF,MAAM,WAAW,GAAG,iBAAiB,KAAK;KAC1C,SAAS,GAAG,SAAS,SAAU,OAAO;AACpC,YAAM;KACP,EAAC;AAEF,aAAQ,MAAR;MACE,KAAK,YAAY,KAAK,eAAe;OACnC,MAAM,KAAK,WAAW;QACpB;QACA,YAAY;QACZ;OACD,EAAC;AACF;MACF;OACE,MAAM,KAAK,qBAAqB;QAC9B,IAAI,SAAS,KAAK;QAClB,YAAY;OACb,EAAC;AACF;KACH;KAED,iBAAiB;IAClB,SAAQ,OAAO;KAEd,MAAM,cACJ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,MAAM;KAC1D,eAAe,QAAQ,YAAY;AACnC,WAAM;IACP,UAAS;KACR,GAAG,SAAS,GAAG,KAAK,CAAC,MAAM,WAAY,CAAE,EAAC;KAC1C,kBAAkB;IACnB;GACF;GAED,MAAM,WAAW,aAAa;GAE9B,SAAS,KAAK,SAAS;EACxB,EAAC,CACD,GAAG,cAAc,MAAM;GACtB,UAAU;EACX,EAAC;AAEJ,MAAI;GACF,MAAM,OAAO,SAAS,SAAS,YAAY,eAAe;EAC3D,SAAQ,OAAO;AACd,OAAI,yBAAyB,KAC3B,KAAI;IACF,MAAM,GAAG,SAAS,GAAG,qBAAqB;GAC3C,QAAO;IACN,IAAI,eAAe,8BAA8B;GAClD;GAEH,MAAM,cACJ,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,MAAM;GAC1D,SAAS,KAAK,QAAQ,OAAO,YAAY,CAAC;EAC3C,UAAS;GAER,MAAM,QAAQ,WAAW,SAAS;GAElC,MAAM,QAAQ,IAAI,SAAS;EAC5B;AAED,SAAO;CACR;AACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MediaCloudLogs } from "../../../types/errors.js";
|
|
2
2
|
import { useErrorHandler } from "../../../hooks/useErrorHandler.js";
|
|
3
3
|
import { Semaphore } from "./semaphore.js";
|
|
4
4
|
import { S3ExpirationManager } from "./expiration-manager.js";
|
|
@@ -10,7 +10,7 @@ import { NoSuchKey, NotFound, S3 } from "@aws-sdk/client-s3";
|
|
|
10
10
|
import { DataStore, ERRORS, MemoryKvStore, TUS_RESUMABLE, Upload } from "@tus/utils";
|
|
11
11
|
|
|
12
12
|
//#region src/tus/stores/s3/s3-store.ts
|
|
13
|
-
const {
|
|
13
|
+
const { log } = useErrorHandler();
|
|
14
14
|
var S3Store = class extends DataStore {
|
|
15
15
|
client;
|
|
16
16
|
bucket;
|
|
@@ -97,7 +97,7 @@ var S3Store = class extends DataStore {
|
|
|
97
97
|
* @returns Promise that resolves to the created upload
|
|
98
98
|
*/
|
|
99
99
|
async create(upload) {
|
|
100
|
-
|
|
100
|
+
log(MediaCloudLogs.S3_STORE_MULTIPART_INIT);
|
|
101
101
|
const request = {
|
|
102
102
|
Bucket: this.bucket,
|
|
103
103
|
Key: upload.id,
|
|
@@ -117,7 +117,7 @@ var S3Store = class extends DataStore {
|
|
|
117
117
|
upload,
|
|
118
118
|
uploadId: response.UploadId
|
|
119
119
|
});
|
|
120
|
-
|
|
120
|
+
log(MediaCloudLogs.S3_STORE_MULTIPART_CREATED);
|
|
121
121
|
return upload;
|
|
122
122
|
}
|
|
123
123
|
/**
|
|
@@ -182,7 +182,7 @@ var S3Store = class extends DataStore {
|
|
|
182
182
|
});
|
|
183
183
|
await this.metadataManager.completeMetadata({ upload: completedUpload });
|
|
184
184
|
} catch (error) {
|
|
185
|
-
|
|
185
|
+
log(MediaCloudLogs.S3_STORE_UPLOAD_FAILED);
|
|
186
186
|
throw error;
|
|
187
187
|
}
|
|
188
188
|
return newOffset;
|
|
@@ -211,7 +211,7 @@ var S3Store = class extends DataStore {
|
|
|
211
211
|
size: metadata.file.size,
|
|
212
212
|
storage: metadata.file.storage
|
|
213
213
|
});
|
|
214
|
-
|
|
214
|
+
log(MediaCloudLogs.S3_STORE_RETRIEVE_PARTS_ERROR);
|
|
215
215
|
throw error;
|
|
216
216
|
}
|
|
217
217
|
const incompletePartSize = await this.partsManager.getIncompletePartSize({ id });
|
|
@@ -226,7 +226,7 @@ var S3Store = class extends DataStore {
|
|
|
226
226
|
* Reads the file specified by the upload's `id` and returns a readable stream
|
|
227
227
|
*/
|
|
228
228
|
async read(id) {
|
|
229
|
-
|
|
229
|
+
log(MediaCloudLogs.S3_STORE_READ_ATTEMPT);
|
|
230
230
|
let retries = 3;
|
|
231
231
|
let lastError = null;
|
|
232
232
|
while (retries > 0) try {
|
|
@@ -234,15 +234,15 @@ var S3Store = class extends DataStore {
|
|
|
234
234
|
Bucket: this.bucket,
|
|
235
235
|
Key: id
|
|
236
236
|
});
|
|
237
|
-
|
|
237
|
+
log(MediaCloudLogs.S3_STORE_READ_SUCCESS);
|
|
238
238
|
return data.Body;
|
|
239
239
|
} catch (error) {
|
|
240
|
-
|
|
240
|
+
log(MediaCloudLogs.S3_STORE_READ_RETRY);
|
|
241
241
|
lastError = error;
|
|
242
242
|
retries--;
|
|
243
243
|
if (retries > 0) await new Promise((resolve) => setTimeout(resolve, 100));
|
|
244
244
|
}
|
|
245
|
-
|
|
245
|
+
log(MediaCloudLogs.S3_STORE_READ_FAILED);
|
|
246
246
|
throw lastError || /* @__PURE__ */ new Error(`Failed to read file ${id} after retries`);
|
|
247
247
|
}
|
|
248
248
|
/**
|
|
@@ -262,7 +262,7 @@ var S3Store = class extends DataStore {
|
|
|
262
262
|
"NoSuchUpload",
|
|
263
263
|
"NotFound"
|
|
264
264
|
].includes(error.Code || "")) {
|
|
265
|
-
|
|
265
|
+
log(MediaCloudLogs.S3_STORE_FILE_NOT_FOUND);
|
|
266
266
|
throw ERRORS.FILE_NOT_FOUND;
|
|
267
267
|
}
|
|
268
268
|
throw error;
|