@empiricalrun/r2-uploader 0.4.0 → 0.6.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @empiricalrun/r2-uploader
 
+## 0.6.0
+
+### Minor Changes
+
+- 79a4e0f: feat: add blob reporters for sharding
+
+## 0.5.0
+
+### Minor Changes
+
+- a94a7f7: fix: resolve multipart upload retry failures
+
 ## 0.4.0
 
 ### Minor Changes

package/dist/index.d.ts

@@ -2,4 +2,6 @@ export { fetchFiles } from "./fetch";
 export { sendTaskToQueue, waitForTaskQueueToFinish } from "./queue";
 export type { AsyncTask, FileMap } from "./types";
 export { uploadDirectory, uploadInMemoryFiles } from "./upload";
+export type { CreateUploadTaskOptions } from "./upload-task";
+export { createUploadTask } from "./upload-task";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAC;AACpE,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAC;AACpE,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAChE,YAAY,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC"}

package/dist/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadInMemoryFiles = exports.uploadDirectory = exports.waitForTaskQueueToFinish = exports.sendTaskToQueue = exports.fetchFiles = void 0;
+exports.createUploadTask = exports.uploadInMemoryFiles = exports.uploadDirectory = exports.waitForTaskQueueToFinish = exports.sendTaskToQueue = exports.fetchFiles = void 0;
 var fetch_1 = require("./fetch");
 Object.defineProperty(exports, "fetchFiles", { enumerable: true, get: function () { return fetch_1.fetchFiles; } });
 var queue_1 = require("./queue");
@@ -9,3 +9,5 @@ Object.defineProperty(exports, "waitForTaskQueueToFinish", { enumerable: true, g
 var upload_1 = require("./upload");
 Object.defineProperty(exports, "uploadDirectory", { enumerable: true, get: function () { return upload_1.uploadDirectory; } });
 Object.defineProperty(exports, "uploadInMemoryFiles", { enumerable: true, get: function () { return upload_1.uploadInMemoryFiles; } });
+var upload_task_1 = require("./upload-task");
+Object.defineProperty(exports, "createUploadTask", { enumerable: true, get: function () { return upload_task_1.createUploadTask; } });

package/dist/upload/stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/upload/stream.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEzD,eAAO,MAAM,iBAAiB,GAAU,QAAQ,oBAAoB,qBAkHnE,CAAC"}
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/upload/stream.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEzD,eAAO,MAAM,iBAAiB,GAAU,QAAQ,oBAAoB,qBA0GnE,CAAC"}

package/dist/upload/stream.js

@@ -42,34 +42,30 @@ const uploadFileStreams = async (config) => {
         let partNumber = 1;
         const parts = [];
         for await (const chunk of fileStream) {
-            const uploadPartParams = {
-                Bucket: config.bucket,
-                Key: fileKey,
-                PartNumber: partNumber,
-                UploadId,
-                Body: chunk,
-            };
-            const uploadPartCommand = new client_s3_1.UploadPartCommand(uploadPartParams);
-            try {
-                await (0, async_retry_1.default)(async () => {
-                    const { ETag } = await S3.send(uploadPartCommand);
-                    logger_1.logger.debug("upload part command sent successfully for file", file.fullPath);
-                    parts.push({ ETag: ETag, PartNumber: partNumber });
-                    partNumber++;
-                }, {
-                    retries: 5,
-                    factor: 3,
-                    minTimeout: 1_000,
-                    maxTimeout: 60_000,
-                    randomize: true,
-                    onRetry: (err, i) => {
-                        logger_1.logger.debug("Upload part retry attempt:", i, ":", file.fullPath);
-                    },
+            const currentPartNumber = partNumber;
+            const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            const { ETag } = await (0, async_retry_1.default)(async () => {
+                const uploadPartCommand = new client_s3_1.UploadPartCommand({
+                    Bucket: config.bucket,
+                    Key: fileKey,
+                    PartNumber: currentPartNumber,
+                    UploadId,
+                    Body: chunkBuffer,
                 });
-            }
-            catch (err) {
-                logger_1.logger.error(`R2 Error - ${file.fullPath} \nError: ${err}`);
-            }
+                return await S3.send(uploadPartCommand);
+            }, {
+                retries: 5,
+                factor: 3,
+                minTimeout: 1_000,
+                maxTimeout: 60_000,
+                randomize: true,
+                onRetry: (err, i) => {
+                    logger_1.logger.debug("Upload part retry attempt:", i, ":", file.fullPath);
+                },
+            });
+            logger_1.logger.debug("upload part command sent successfully for file", file.fullPath);
+            parts.push({ ETag: ETag, PartNumber: currentPartNumber });
+            partNumber++;
         }
         const completeMultipartUploadParams = {
             Bucket: config.bucket,

package/dist/upload-task.d.ts

@@ -0,0 +1,21 @@
+import { AsyncTask } from "./types";
+export interface CreateUploadTaskOptions {
+    sourceDir: string;
+    fileList?: string[];
+    destinationDir: string;
+    uploadBucket: string;
+    baseUrl: string;
+}
+/**
+ * Creates an async task for uploading files to R2 storage.
+ *
+ * @param {Object} params - The upload parameters
+ * @param {string} params.sourceDir - The source directory containing files to upload
+ * @param {string[]} [params.fileList] - Optional list of specific files to upload
+ * @param {string} params.destinationDir - The destination directory in R2 storage
+ * @param {string} params.uploadBucket - The R2 bucket to upload to
+ * @param {string} params.baseUrl - The base URL for generating file URLs
+ * @returns {AsyncTask} An async task that when executed will upload the files and return a map of file paths to URLs
+ */
+export declare function createUploadTask({ sourceDir, fileList, destinationDir, uploadBucket, baseUrl, }: CreateUploadTaskOptions): AsyncTask;
+//# sourceMappingURL=upload-task.d.ts.map

package/dist/upload-task.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload-task.d.ts","sourceRoot":"","sources":["../src/upload-task.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAW,MAAM,SAAS,CAAC;AAG7C,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,EAC/B,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,OAAO,GACR,EAAE,uBAAuB,GAAG,SAAS,CA4BrC"}

package/dist/upload-task.js

@@ -0,0 +1,48 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createUploadTask = createUploadTask;
+const path_1 = __importDefault(require("path"));
+const logger_1 = require("./logger");
+const upload_1 = require("./upload");
+/**
+ * Creates an async task for uploading files to R2 storage.
+ *
+ * @param {Object} params - The upload parameters
+ * @param {string} params.sourceDir - The source directory containing files to upload
+ * @param {string[]} [params.fileList] - Optional list of specific files to upload
+ * @param {string} params.destinationDir - The destination directory in R2 storage
+ * @param {string} params.uploadBucket - The R2 bucket to upload to
+ * @param {string} params.baseUrl - The base URL for generating file URLs
+ * @returns {AsyncTask} An async task that when executed will upload the files and return a map of file paths to URLs
+ */
+function createUploadTask({ sourceDir, fileList, destinationDir, uploadBucket, baseUrl, }) {
+    return async () => {
+        if (!process.env.R2_ACCOUNT_ID ||
+            !process.env.R2_ACCESS_KEY_ID ||
+            !process.env.R2_SECRET_ACCESS_KEY) {
+            return;
+        }
+        try {
+            const uploadedFiles = await (0, upload_1.uploadDirectory)({
+                sourceDir,
+                fileList,
+                destinationDir,
+                uploadBucket,
+            });
+            const urls = {};
+            Object.entries(uploadedFiles).forEach(([filePath]) => {
+                const fileUrl = new URL(path_1.default.join(destinationDir, filePath), baseUrl);
+                urls[filePath] = fileUrl.toString();
+            });
+            logger_1.logger.debug("Finished uploading files");
+            return urls;
+        }
+        catch (error) {
+            logger_1.logger.error("Error uploading files", error);
+            return {};
+        }
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/r2-uploader",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/delete.ts","./src/fetch.ts","./src/index.ts","./src/logger.ts","./src/queue.ts","./src/types.ts","./src/upload/buffer.ts","./src/upload/index.ts","./src/upload/stream.ts"],"version":"5.8.3"}
+{"root":["./src/delete.ts","./src/fetch.ts","./src/index.ts","./src/logger.ts","./src/queue.ts","./src/types.ts","./src/upload-task.ts","./src/upload/buffer.ts","./src/upload/index.ts","./src/upload/stream.ts"],"version":"5.8.3"}