@empiricalrun/r2-uploader 0.1.0

package/CHANGELOG.md

@@ -0,0 +1,7 @@
+# @empiricalrun/r2-uploader
+
+## 0.1.0
+
+### Minor Changes
+
+- 6a1288a: feat: create r2 uploader package and support for retry

package/README.md

@@ -0,0 +1,3 @@
+# r2-uploader
+
+Node.js library to upload files to R2

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+interface FileMap {
+    [file: string]: string;
+}
+export declare function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }: {
+    sourceDir: string;
+    fileList?: string[];
+    destinationDir: string;
+    uploadBucket: string;
+    accountId?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+}): Promise<FileMap>;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAuBA,UAAU,OAAO;IACf,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AA+HD,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAYnB"}

package/dist/index.js

@@ -0,0 +1,150 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadDirectory = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
+const async_retry_1 = __importDefault(require("async-retry"));
+const fs = __importStar(require("fs"));
+const md5_1 = __importDefault(require("md5"));
+const path_1 = __importDefault(require("path"));
+const getFileList = (dir) => {
+    let files = [];
+    const items = fs.readdirSync(dir, {
+        withFileTypes: true,
+    });
+    for (const item of items) {
+        const isDir = item.isDirectory();
+        const absolutePath = `${dir}/${item.name}`;
+        if (isDir) {
+            files = [...files, ...getFileList(absolutePath)];
+        }
+        else {
+            files.push(absolutePath);
+        }
+    }
+    return files;
+};
+const run = async (config) => {
+    const map = new Map();
+    const urls = {};
+    const S3 = new client_s3_1.S3Client({
+        region: "auto",
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+        },
+    });
+    let files;
+    if (config.fileList) {
+        files = config.fileList;
+    }
+    else {
+        files = getFileList(config.sourceDir);
+    }
+    await Promise.all(files.map(async (file) => {
+        const fileStream = fs.readFileSync(file);
+        const fileName = file.replace(config.sourceDir, "");
+        const fileKey = path_1.default.join(config.destinationDir !== "" ? config.destinationDir : config.sourceDir, fileName);
+        if (fileKey.includes(".gitkeep"))
+            return;
+        // console.log(fileKey);
+        let mimeType = "application/octet-stream";
+        try {
+            const mime = (await import("mime")).default;
+            mimeType = mime.getType(file) || "application/octet-stream";
+        }
+        catch (err) {
+            console.warn("Failed to get mime type for file", file, err);
+        }
+        const uploadParams = {
+            Bucket: config.bucket,
+            Key: fileKey,
+            Body: fileStream,
+            ContentLength: fs.statSync(file).size,
+            ContentType: mimeType ?? "application/octet-stream",
+        };
+        const cmd = new client_s3_1.PutObjectCommand(uploadParams);
+        const digest = (0, md5_1.default)(fileStream);
+        cmd.middlewareStack.add((next) => async (args) => {
+            args.request.headers["if-none-match"] = `"${digest}"`;
+            return await next(args);
+        }, {
+            step: "build",
+            name: "addETag",
+        });
+        try {
+            await (0, async_retry_1.default)(async () => {
+                try {
+                    const data = await S3.send(cmd);
+                    // console.log(`R2 Success - ${file}`);
+                    map.set(file, data);
+                    const fileUrl = await (0, s3_request_presigner_1.getSignedUrl)(S3, cmd);
+                    urls[file] = fileUrl;
+                }
+                catch (err) {
+                    const error = err;
+                    if (error["$metadata"]) {
+                        if (error.$metadata.httpStatusCode !== 412) {
+                            throw error;
+                        }
+                    }
+                    console.log("Error which is not retried", error);
+                }
+            }, {
+                retries: 5,
+                factor: 3,
+                minTimeout: 1000,
+                maxTimeout: 60000,
+                randomize: true,
+            });
+        }
+        catch (err) {
+            const error = err;
+            console.log(`R2 Error - ${file} ${config.sourceDir} ${config.destinationDir} \nError: ${error}`);
+            console.log("Upload response", error.$response, "with status", error.$metadata.httpStatusCode);
+        }
+        return;
+    }));
+    return urls;
+};
+async function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
+    let config = {
+        accountId: accountId || process.env.R2_ACCOUNT_ID,
+        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
+        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
+        bucket: uploadBucket,
+        sourceDir,
+        destinationDir,
+        fileList,
+    };
+    const uploadedFiles = await run(config);
+    return uploadedFiles;
+}
+exports.uploadDirectory = uploadDirectory;

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@empiricalrun/r2-uploader",
+  "version": "0.1.0",
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public"
+  },
+  "main": "dist/index.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/empirical-run/empirical.git"
+  },
+  "author": "Empirical Team <hey@empirical.run>",
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.614.0",
+    "@aws-sdk/s3-request-presigner": "^3.614.0",
+    "@types/async-retry": "^1.4.8",
+    "@types/md5": "^2.3.5",
+    "@types/mime": "3.0.0",
+    "async-retry": "^1.3.3",
+    "md5": "^2.3.0",
+    "mime": "3.0.0"
+  },
+  "scripts": {
+    "dev": "tsc --build --watch",
+    "build": "tsc --build",
+    "clean": "tsc --build --clean",
+    "lint": "eslint .",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}