@empiricalrun/r2-uploader 0.3.0 → 0.3.2

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @empiricalrun/r2-uploader
 
+## 0.3.2
+
+### Patch Changes
+
+- f317919: feat: use multipart upload with file streams
+
+## 0.3.1
+
+### Patch Changes
+
+- fccb68a: chore: remove uploader logs for cleaner ci runs
+
 ## 0.3.0
 
 ### Minor Changes

package/dist/delete.d.ts

@@ -0,0 +1,3 @@
+import { R2DeleteConfig } from "./types";
+export declare function deleteFile(fileKey: string, config: R2DeleteConfig): Promise<import("@aws-sdk/client-s3").DeleteObjectCommandOutput>;
+//# sourceMappingURL=delete.d.ts.map

package/dist/delete.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../src/delete.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,wBAAsB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,mEAqBvE"}

package/dist/delete.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.deleteFile = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+async function deleteFile(fileKey, config) {
+    const s3Client = new client_s3_1.S3Client({
+        region: "auto",
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+        },
+    });
+    const params = {
+        Bucket: config.bucket,
+        Key: fileKey,
+    };
+    try {
+        const command = new client_s3_1.DeleteObjectCommand(params);
+        const response = await s3Client.send(command);
+        return response;
+    }
+    catch (error) {
+        console.error(`Error deleting file from R2: ${error}`);
+        throw error;
+    }
+}
+exports.deleteFile = deleteFile;

package/dist/index.d.ts

@@ -1,3 +1,4 @@
-export * from "./fetch";
-export * from "./upload";
+export { fetchFiles } from "./fetch";
+export type { FileMap } from "./types";
+export { uploadDirectory, uploadInMemoryFiles } from "./upload";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,YAAY,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC"}

package/dist/index.js

@@ -1,18 +1,8 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./fetch"), exports);
-__exportStar(require("./upload"), exports);
+exports.uploadInMemoryFiles = exports.uploadDirectory = exports.fetchFiles = void 0;
+var fetch_1 = require("./fetch");
+Object.defineProperty(exports, "fetchFiles", { enumerable: true, get: function () { return fetch_1.fetchFiles; } });
+var upload_1 = require("./upload");
+Object.defineProperty(exports, "uploadDirectory", { enumerable: true, get: function () { return upload_1.uploadDirectory; } });
+Object.defineProperty(exports, "uploadInMemoryFiles", { enumerable: true, get: function () { return upload_1.uploadInMemoryFiles; } });

package/dist/types.d.ts

@@ -5,7 +5,15 @@ interface R2BaseConfig {
     secretAccessKey: string;
     bucket: string;
 }
-export interface R2UploadConfig extends R2BaseConfig {
+export interface R2UploadStreamConfig extends R2BaseConfig {
+    destinationDir: string;
+    files: {
+        fullPath: string;
+        fileName: string;
+        mimeType?: string;
+    }[];
+}
+export interface R2UploadBufferConfig extends R2BaseConfig {
     destinationDir: string;
     files: {
         buffer: Buffer;
@@ -16,5 +24,10 @@ export interface R2UploadConfig extends R2BaseConfig {
 export interface R2FetchConfig extends R2BaseConfig {
     prefix: string;
 }
+export interface R2DeleteConfig extends R2BaseConfig {
+}
+export interface FileMap {
+    [file: string]: string;
+}
 export {};
 //# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA,UAAU,YAAY;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAClE;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,MAAM,EAAE,MAAM,CAAC;CAChB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA,UAAU,YAAY;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACpE;AAED,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAClE;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;CAAG;AAEvD,MAAM,WAAW,OAAO;IACtB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB"}

package/dist/upload/buffer.d.ts

@@ -0,0 +1,3 @@
+import { FileMap, R2UploadBufferConfig } from "../types";
+export declare const uploadFileBuffers: (config: R2UploadBufferConfig) => Promise<FileMap>;
+//# sourceMappingURL=buffer.d.ts.map

package/dist/upload/buffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buffer.d.ts","sourceRoot":"","sources":["../../src/upload/buffer.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEzD,eAAO,MAAM,iBAAiB,WAAkB,oBAAoB,qBA6FnE,CAAC"}

package/dist/upload/buffer.js

@@ -0,0 +1,87 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadFileBuffers = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
+const async_retry_1 = __importDefault(require("async-retry"));
+const md5_1 = __importDefault(require("md5"));
+const path_1 = __importDefault(require("path"));
+const uploadFileBuffers = async (config) => {
+    const map = new Map();
+    const urls = {};
+    const S3 = new client_s3_1.S3Client({
+        region: "auto",
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+        },
+    });
+    await Promise.all(config.files.map(async (file) => {
+        let fileKey = path_1.default.join(config.destinationDir, file.fileName);
+        if (fileKey.includes(".gitkeep")) {
+            return;
+        }
+        const mimeType = file.mimeType || "application/octet-stream";
+        const uploadParams = {
+            Bucket: config.bucket,
+            Key: fileKey,
+            Body: file.buffer,
+            ContentLength: file.buffer.length,
+            ContentType: mimeType ?? "application/octet-stream",
+        };
+        const cmd = new client_s3_1.PutObjectCommand(uploadParams);
+        const digest = (0, md5_1.default)(file.buffer);
+        cmd.middlewareStack.add((next) => async (args) => {
+            args.request.headers["if-none-match"] = `"${digest}"`;
+            return await next(args);
+        }, {
+            step: "build",
+            name: "addETag",
+        });
+        try {
+            await (0, async_retry_1.default)(async () => {
+                try {
+                    const data = await S3.send(cmd);
+                    map.set(file.fileName, data);
+                    const fileUrl = await (0, s3_request_presigner_1.getSignedUrl)(S3, cmd);
+                    urls[file.fileName] = fileUrl;
+                }
+                catch (err) {
+                    const error = err;
+                    if (error["$metadata"]) {
+                        // throw only those errors that are not 412 Precondition Failed
+                        // 412 errors are errors while accessing the asset, which is post upload and can be ignored
+                        if (error.$metadata.httpStatusCode !== 412) {
+                            throw error;
+                        }
+                    }
+                }
+            }, {
+                retries: 5,
+                factor: 3,
+                minTimeout: 1000,
+                maxTimeout: 60000,
+                randomize: true,
+                onRetry: (err, i) => {
+                    if (err) {
+                        const error = err;
+                        console.log("Upload retry attempt:", i, ":", file.fileName);
+                        console.log("Response status:", error.$response?.statusCode);
+                    }
+                },
+            });
+        }
+        catch (err) {
+            const error = err;
+            console.log(`R2 Error - ${file.fileName} \nError: ${error}`);
+            console.log("Upload response", error.$response, "with status", error.$metadata.httpStatusCode);
+        }
+        return;
+    }));
+    return urls;
+};
+exports.uploadFileBuffers = uploadFileBuffers;

package/dist/{upload.d.ts → upload/index.d.ts}

@@ -1,7 +1,5 @@
 /// <reference types="node" />
-export interface FileMap {
-    [file: string]: string;
-}
+import { FileMap } from "../types";
 export declare function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }: {
     sourceDir: string;
     fileList?: string[];
@@ -23,4 +21,4 @@ export declare function uploadInMemoryFiles({ files, destinationDir, uploadBucke
     accessKeyId?: string;
     secretAccessKey?: string;
 }): Promise<FileMap>;
-//# sourceMappingURL=upload.d.ts.map
+//# sourceMappingURL=index.d.ts.map

package/dist/upload/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/upload/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAA8C,MAAM,UAAU,CAAC;AAqB/E,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAiBnB;AAED,wBAAsB,mBAAmB,CAAC,EACxC,KAAK,EACL,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAUnB"}

package/dist/upload/index.js

@@ -0,0 +1,77 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadInMemoryFiles = exports.uploadDirectory = void 0;
+const fs = __importStar(require("fs"));
+const buffer_1 = require("./buffer");
+const stream_1 = require("./stream");
+const getFileList = (dir) => {
+    let files = [];
+    const items = fs.readdirSync(dir, {
+        withFileTypes: true,
+    });
+    for (const item of items) {
+        const isDir = item.isDirectory();
+        const absolutePath = `${dir}/${item.name}`;
+        if (isDir) {
+            files = [...files, ...getFileList(absolutePath)];
+        }
+        else {
+            files.push(absolutePath);
+        }
+    }
+    return files;
+};
+async function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
+    const filePaths = fileList || getFileList(sourceDir);
+    const files = filePaths.map((filePath) => ({
+        fullPath: filePath,
+        fileName: filePath.replace(sourceDir, ""), // relative path
+        mimeType: undefined,
+    }));
+    const config = {
+        accountId: accountId || process.env.R2_ACCOUNT_ID,
+        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
+        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
+        bucket: uploadBucket,
+        destinationDir,
+        files,
+    };
+    const uploadedFiles = await (0, stream_1.uploadFileStreams)(config);
+    return uploadedFiles;
+}
+exports.uploadDirectory = uploadDirectory;
+async function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
+    const config = {
+        accountId: accountId || process.env.R2_ACCOUNT_ID,
+        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
+        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
+        bucket: uploadBucket,
+        destinationDir,
+        files,
+    };
+    return await (0, buffer_1.uploadFileBuffers)(config);
+}
+exports.uploadInMemoryFiles = uploadInMemoryFiles;

package/dist/upload/stream.d.ts

@@ -0,0 +1,3 @@
+import { FileMap, R2UploadStreamConfig } from "../types";
+export declare const uploadFileStreams: (config: R2UploadStreamConfig) => Promise<FileMap>;
+//# sourceMappingURL=stream.d.ts.map

package/dist/upload/stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/upload/stream.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEzD,eAAO,MAAM,iBAAiB,WAAkB,oBAAoB,qBAkGnE,CAAC"}

package/dist/upload/stream.js

@@ -0,0 +1,89 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadFileStreams = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
+const async_retry_1 = __importDefault(require("async-retry"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const uploadFileStreams = async (config) => {
+    const urls = {};
+    const S3 = new client_s3_1.S3Client({
+        region: "auto",
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+        },
+    });
+    await Promise.all(config.files.map(async (file) => {
+        let fileKey = path_1.default.join(config.destinationDir, file.fileName);
+        if (fileKey.includes(".gitkeep")) {
+            return;
+        }
+        const mimeType = file.mimeType || "application/octet-stream";
+        const createMultipartUploadParams = {
+            Bucket: config.bucket,
+            Key: fileKey,
+            ContentType: mimeType,
+        };
+        const createMultipartUploadCommand = new client_s3_1.CreateMultipartUploadCommand(createMultipartUploadParams);
+        const { UploadId } = await S3.send(createMultipartUploadCommand);
+        const partSize = 5 * 1024 * 1024; // 5MB
+        const fileStream = fs_1.default.createReadStream(file.fullPath, {
+            highWaterMark: partSize,
+        });
+        let partNumber = 1;
+        const parts = [];
+        for await (const chunk of fileStream) {
+            const uploadPartParams = {
+                Bucket: config.bucket,
+                Key: fileKey,
+                PartNumber: partNumber,
+                UploadId,
+                Body: chunk,
+            };
+            const uploadPartCommand = new client_s3_1.UploadPartCommand(uploadPartParams);
+            try {
+                await (0, async_retry_1.default)(async () => {
+                    const { ETag } = await S3.send(uploadPartCommand);
+                    parts.push({ ETag: ETag, PartNumber: partNumber });
+                    partNumber++;
+                }, {
+                    retries: 5,
+                    factor: 3,
+                    minTimeout: 1000,
+                    maxTimeout: 60000,
+                    randomize: true,
+                    onRetry: (err, i) => {
+                        console.log("Upload part retry attempt:", i, ":", file.fileName);
+                    },
+                });
+            }
+            catch (err) {
+                console.log(`R2 Error - ${file.fileName} \nError: ${err}`);
+            }
+        }
+        const completeMultipartUploadParams = {
+            Bucket: config.bucket,
+            Key: fileKey,
+            UploadId,
+            MultipartUpload: {
+                Parts: parts,
+            },
+        };
+        const completeMultipartUploadCommand = new client_s3_1.CompleteMultipartUploadCommand(completeMultipartUploadParams);
+        await S3.send(completeMultipartUploadCommand);
+        const getObjectCommand = new client_s3_1.GetObjectCommand({
+            Bucket: config.bucket,
+            Key: fileKey,
+        });
+        const fileUrl = await (0, s3_request_presigner_1.getSignedUrl)(S3, getObjectCommand);
+        urls[file.fileName] = fileUrl;
+    }));
+    return urls;
+};
+exports.uploadFileStreams = uploadFileStreams;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/r2-uploader",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"

package/dist/upload.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../src/upload.ts"],"names":[],"mappings":";AAgBA,MAAM,WAAW,OAAO;IACtB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAuHD,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CA6BnB;AAED,wBAAsB,mBAAmB,CAAC,EACxC,KAAK,EACL,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAUnB"}

package/dist/upload.js

@@ -1,171 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadInMemoryFiles = exports.uploadDirectory = void 0;
-const client_s3_1 = require("@aws-sdk/client-s3");
-const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
-const async_retry_1 = __importDefault(require("async-retry"));
-const fs = __importStar(require("fs"));
-const md5_1 = __importDefault(require("md5"));
-const mime_1 = __importDefault(require("mime"));
-const path_1 = __importDefault(require("path"));
-const getFileList = (dir) => {
-    let files = [];
-    const items = fs.readdirSync(dir, {
-        withFileTypes: true,
-    });
-    for (const item of items) {
-        const isDir = item.isDirectory();
-        const absolutePath = `${dir}/${item.name}`;
-        if (isDir) {
-            files = [...files, ...getFileList(absolutePath)];
-        }
-        else {
-            files.push(absolutePath);
-        }
-    }
-    return files;
-};
-const run = async (config) => {
-    const map = new Map();
-    const urls = {};
-    const S3 = new client_s3_1.S3Client({
-        region: "auto",
-        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
-        credentials: {
-            accessKeyId: config.accessKeyId,
-            secretAccessKey: config.secretAccessKey,
-        },
-    });
-    await Promise.all(config.files.map(async (file) => {
-        let fileKey = path_1.default.join(config.destinationDir, file.fileName);
-        if (fileKey.includes(".gitkeep")) {
-            return;
-        }
-        const mimeType = file.mimeType || "application/octet-stream";
-        const uploadParams = {
-            Bucket: config.bucket,
-            Key: fileKey,
-            Body: file.buffer,
-            ContentLength: file.buffer.length,
-            ContentType: mimeType ?? "application/octet-stream",
-        };
-        const cmd = new client_s3_1.PutObjectCommand(uploadParams);
-        const digest = (0, md5_1.default)(file.buffer);
-        cmd.middlewareStack.add((next) => async (args) => {
-            args.request.headers["if-none-match"] = `"${digest}"`;
-            return await next(args);
-        }, {
-            step: "build",
-            name: "addETag",
-        });
-        try {
-            await (0, async_retry_1.default)(async () => {
-                try {
-                    console.log("\nStarting file upload for:", file.fileName, "\n");
-                    const data = await S3.send(cmd);
-                    map.set(file.fileName, data);
-                    const fileUrl = await (0, s3_request_presigner_1.getSignedUrl)(S3, cmd);
-                    urls[file.fileName] = fileUrl;
-                    console.log("\nFinished file upload for:", file.fileName, "\n");
-                }
-                catch (err) {
-                    console.log("\nError uploading file: ", file.fileName, err, "\n");
-                    const error = err;
-                    if (error["$metadata"]) {
-                        // throw only those errors that are not 412 Precondition Failed
-                        // 412 errors are errors while accessing the asset, which is post upload and can be ignored
-                        if (error.$metadata.httpStatusCode !== 412) {
-                            throw error;
-                        }
-                    }
-                }
-            }, {
-                retries: 5,
-                factor: 3,
-                minTimeout: 1000,
-                maxTimeout: 60000,
-                randomize: true,
-                onRetry: (err, i) => {
-                    if (err) {
-                        const error = err;
-                        console.log("Upload retry attempt:", i, ":", file.fileName);
-                        console.log("Response status:", error.$response?.statusCode);
-                    }
-                },
-            });
-        }
-        catch (err) {
-            const error = err;
-            console.log(`R2 Error - ${file.fileName} \nError: ${error}`);
-            console.log("Upload response", error.$response, "with status", error.$metadata.httpStatusCode);
-        }
-        return;
-    }));
-    return urls;
-};
-async function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
-    const filePaths = fileList || getFileList(sourceDir);
-    const files = filePaths
-        .map((filePath) => ({
-        path: filePath,
-        fileName: filePath.replace(sourceDir, ""),
-        mimeType: undefined,
-    }))
-        .map((file) => {
-        const mimeType = mime_1.default.getType(file.path) || "application/octet-stream";
-        return {
-            buffer: fs.readFileSync(file.path),
-            fileName: file.fileName,
-            mimeType,
-        };
-    });
-    let config = {
-        accountId: accountId || process.env.R2_ACCOUNT_ID,
-        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
-        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
-        bucket: uploadBucket,
-        destinationDir,
-        files,
-    };
-    const uploadedFiles = await run(config);
-    return uploadedFiles;
-}
-exports.uploadDirectory = uploadDirectory;
-async function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
-    const config = {
-        accountId: accountId || process.env.R2_ACCOUNT_ID,
-        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
-        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
-        bucket: uploadBucket,
-        destinationDir,
-        files,
-    };
-    return await run(config);
-}
-exports.uploadInMemoryFiles = uploadInMemoryFiles;