npm - @empiricalrun/r2-uploader - Versions diffs - 0.1.3 → 0.2.0 - Mend

@empiricalrun/r2-uploader 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (5) hide show

package/CHANGELOG.md CHANGED Viewed

@@ -1,5 +1,11 @@
 # @empiricalrun/r2-uploader
+## 0.2.0
+### Minor Changes
+- 2f520d3: feat: support in-memory file upload
 ## 0.1.3
 ### Patch Changes

package/dist/index.d.ts CHANGED Viewed

@@ -1,3 +1,4 @@
+/// <reference types="node" />
 export interface FileMap {
     [file: string]: string;
 }
@@ -10,4 +11,16 @@ export declare function uploadDirectory({ sourceDir, fileList, destinationDir, u
     accessKeyId?: string;
     secretAccessKey?: string;
 }): Promise<FileMap>;
+export declare function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }: {
+    files: {
+        buffer: Buffer;
+        fileName: string;
+        mimeType?: string;
+    }[];
+    destinationDir: string;
+    uploadBucket: string;
+    accountId?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+}): Promise<FileMap>;
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAuBA,MAAM,WAAW,OAAO;IACtB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;~~AA2ID~~,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,~~CAYnB~~"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAuBA,MAAM,WAAW,OAAO;IACtB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAuHD,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CA6BnB;AAED,wBAAsB,mBAAmB,CAAC,EACxC,KAAK,EACL,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAUnB"}

package/dist/index.js CHANGED Viewed

@@ -26,12 +26,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadDirectory = void 0;
+exports.uploadInMemoryFiles = exports.uploadDirectory = void 0;
 const client_s3_1 = require("@aws-sdk/client-s3");
 const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
 const async_retry_1 = __importDefault(require("async-retry"));
 const fs = __importStar(require("fs"));
 const md5_1 = __importDefault(require("md5"));
+const mime_1 = __importDefault(require("mime"));
 const path_1 = __importDefault(require("path"));
 const getFileList = (dir) => {
     let files = [];
@@ -61,38 +62,21 @@ const run = async (config) => {
             secretAccessKey: config.secretAccessKey,
         },
     });
-    let files;
-    if (config.fileList) {
-        files = config.fileList;
-    }
-    else {
-        files = getFileList(config.sourceDir);
-    }
-    await Promise.all(files.map(async (file) => {
-        // console.log("\nStarting file upload for : ", file);
-        const fileStream = fs.readFileSync(file);
-        const fileName = file.replace(config.sourceDir, "");
-        const fileKey = path_1.default.join(config.destinationDir !== "" ? config.destinationDir : config.sourceDir, fileName);
-        if (fileKey.includes(".gitkeep"))
+    await Promise.all(config.files.map(async (file) => {
+        let fileKey = path_1.default.join(config.destinationDir, file.fileName);
+        if (fileKey.includes(".gitkeep")) {
             return;
-        // console.log(fileKey);
-        let mimeType = "application/octet-stream";
-        try {
-            const mime = (await import("mime")).default;
-            mimeType = mime.getType(file) || "application/octet-stream";
-        }
-        catch (err) {
-            console.warn("Failed to get mime type for file", file, err);
         }
+        const mimeType = file.mimeType || "application/octet-stream";
         const uploadParams = {
             Bucket: config.bucket,
             Key: fileKey,
-            Body: fileStream,
-            ContentLength: fs.statSync(file).size,
+            Body: file.buffer,
+            ContentLength: file.buffer.length,
             ContentType: mimeType ?? "application/octet-stream",
         };
         const cmd = new client_s3_1.PutObjectCommand(uploadParams);
-        const digest = (0, md5_1.default)(fileStream);
+        const digest = (0, md5_1.default)(file.buffer);
         cmd.middlewareStack.add((next) => async (args) => {
             args.request.headers["if-none-match"] = `"${digest}"`;
             return await next(args);
@@ -103,16 +87,15 @@ const run = async (config) => {
         try {
             await (0, async_retry_1.default)(async () => {
                 try {
-                    console.log("\n\nStarting file upload for: ", file, "\n\n");
+                    console.log("\nStarting file upload for:", file.fileName, "\n");
                     const data = await S3.send(cmd);
-                    // console.log(`R2 Success - ${file}`);
-                    map.set(file, data);
+                    map.set(file.fileName, data);
                     const fileUrl = await (0, s3_request_presigner_1.getSignedUrl)(S3, cmd);
-                    urls[file] = fileUrl;
-                    console.log("\n\nFinished file upload for: ", file, "\n\n");
+                    urls[file.fileName] = fileUrl;
+                    console.log("\nFinished file upload for:", file.fileName, "\n");
                 }
                 catch (err) {
-                    console.log("\n\nError uploading file: ", file, err, "\n\n");
+                    console.log("\nError uploading file: ", file.fileName, err, "\n");
                     const error = err;
                     if (error["$metadata"]) {
                         // throw only those errors that are not 412 Precondition Failed
@@ -131,7 +114,7 @@ const run = async (config) => {
                 onRetry: (err, i) => {
                     if (err) {
                         const error = err;
-                        console.log("Upload retry attempt:", i, ":", file);
+                        console.log("Upload retry attempt:", i, ":", file.fileName);
                         console.log("Response status:", error.$response?.statusCode);
                     }
                 },
@@ -139,7 +122,7 @@ const run = async (config) => {
         }
         catch (err) {
             const error = err;
-            console.log(`R2 Error - ${file} ${config.sourceDir} ${config.destinationDir} \nError: ${error}`);
+            console.log(`R2 Error - ${file.fileName} \nError: ${error}`);
             console.log("Upload response", error.$response, "with status", error.$metadata.httpStatusCode);
         }
         return;
@@ -147,16 +130,42 @@ const run = async (config) => {
     return urls;
 };
 async function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
+    const filePaths = fileList || getFileList(sourceDir);
+    const files = filePaths
+        .map((filePath) => ({
+        path: filePath,
+        fileName: filePath.replace(sourceDir, ""),
+        mimeType: undefined,
+    }))
+        .map((file) => {
+        const mimeType = mime_1.default.getType(file.path) || "application/octet-stream";
+        return {
+            buffer: fs.readFileSync(file.path),
+            fileName: file.fileName,
+            mimeType,
+        };
+    });
     let config = {
         accountId: accountId || process.env.R2_ACCOUNT_ID,
         accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
         secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
         bucket: uploadBucket,
-        sourceDir,
         destinationDir,
-        fileList,
+        files,
     };
     const uploadedFiles = await run(config);
     return uploadedFiles;
 }
 exports.uploadDirectory = uploadDirectory;
+async function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
+    const config = {
+        accountId: accountId || process.env.R2_ACCOUNT_ID,
+        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
+        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
+        bucket: uploadBucket,
+        destinationDir,
+        files,
+    };
+    return await run(config);
+}
+exports.uploadInMemoryFiles = uploadInMemoryFiles;

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/r2-uploader",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"