@empiricalrun/r2-uploader 0.2.0 → 0.3.1

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @empiricalrun/r2-uploader
 
+## 0.3.1
+
+### Patch Changes
+
+- fccb68a: chore: remove uploader logs for cleaner ci runs
+
+## 0.3.0
+
+### Minor Changes
+
+- bac164e: feat: add report assets page which fetches from r2 bucket
+
 ## 0.2.0
 
 ### Minor Changes

package/dist/fetch.d.ts

@@ -0,0 +1,7 @@
+import { R2FetchConfig } from "./types";
+export declare const fetchFiles: (config: R2FetchConfig) => Promise<{
+    key: string;
+    lastModified: Date;
+    size: number;
+}[] | undefined>;
+//# sourceMappingURL=fetch.d.ts.map

package/dist/fetch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,eAAO,MAAM,UAAU,WAAkB,aAAa;;;;gBAsBrD,CAAC"}

package/dist/fetch.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchFiles = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const fetchFiles = async (config) => {
+    const S3 = new client_s3_1.S3Client({
+        region: "auto",
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+        },
+    });
+    const params = {
+        Bucket: config.bucket,
+        Prefix: config.prefix,
+    };
+    const cmd = new client_s3_1.ListObjectsV2Command(params);
+    const data = await S3.send(cmd);
+    return data.Contents?.map((c) => {
+        return {
+            key: c.Key,
+            lastModified: c.LastModified,
+            size: c.Size,
+        };
+    });
+};
+exports.fetchFiles = fetchFiles;

package/dist/index.d.ts

@@ -1,26 +1,3 @@
-/// <reference types="node" />
-export interface FileMap {
-    [file: string]: string;
-}
-export declare function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }: {
-    sourceDir: string;
-    fileList?: string[];
-    destinationDir: string;
-    uploadBucket: string;
-    accountId?: string;
-    accessKeyId?: string;
-    secretAccessKey?: string;
-}): Promise<FileMap>;
-export declare function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }: {
-    files: {
-        buffer: Buffer;
-        fileName: string;
-        mimeType?: string;
-    }[];
-    destinationDir: string;
-    uploadBucket: string;
-    accountId?: string;
-    accessKeyId?: string;
-    secretAccessKey?: string;
-}): Promise<FileMap>;
+export * from "./fetch";
+export * from "./upload";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAuBA,MAAM,WAAW,OAAO;IACtB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAuHD,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CA6BnB;AAED,wBAAsB,mBAAmB,CAAC,EACxC,KAAK,EACL,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAUnB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,UAAU,CAAC"}

package/dist/index.js

@@ -10,162 +10,9 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
 }));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadInMemoryFiles = exports.uploadDirectory = void 0;
-const client_s3_1 = require("@aws-sdk/client-s3");
-const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
-const async_retry_1 = __importDefault(require("async-retry"));
-const fs = __importStar(require("fs"));
-const md5_1 = __importDefault(require("md5"));
-const mime_1 = __importDefault(require("mime"));
-const path_1 = __importDefault(require("path"));
-const getFileList = (dir) => {
-    let files = [];
-    const items = fs.readdirSync(dir, {
-        withFileTypes: true,
-    });
-    for (const item of items) {
-        const isDir = item.isDirectory();
-        const absolutePath = `${dir}/${item.name}`;
-        if (isDir) {
-            files = [...files, ...getFileList(absolutePath)];
-        }
-        else {
-            files.push(absolutePath);
-        }
-    }
-    return files;
-};
-const run = async (config) => {
-    const map = new Map();
-    const urls = {};
-    const S3 = new client_s3_1.S3Client({
-        region: "auto",
-        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
-        credentials: {
-            accessKeyId: config.accessKeyId,
-            secretAccessKey: config.secretAccessKey,
-        },
-    });
-    await Promise.all(config.files.map(async (file) => {
-        let fileKey = path_1.default.join(config.destinationDir, file.fileName);
-        if (fileKey.includes(".gitkeep")) {
-            return;
-        }
-        const mimeType = file.mimeType || "application/octet-stream";
-        const uploadParams = {
-            Bucket: config.bucket,
-            Key: fileKey,
-            Body: file.buffer,
-            ContentLength: file.buffer.length,
-            ContentType: mimeType ?? "application/octet-stream",
-        };
-        const cmd = new client_s3_1.PutObjectCommand(uploadParams);
-        const digest = (0, md5_1.default)(file.buffer);
-        cmd.middlewareStack.add((next) => async (args) => {
-            args.request.headers["if-none-match"] = `"${digest}"`;
-            return await next(args);
-        }, {
-            step: "build",
-            name: "addETag",
-        });
-        try {
-            await (0, async_retry_1.default)(async () => {
-                try {
-                    console.log("\nStarting file upload for:", file.fileName, "\n");
-                    const data = await S3.send(cmd);
-                    map.set(file.fileName, data);
-                    const fileUrl = await (0, s3_request_presigner_1.getSignedUrl)(S3, cmd);
-                    urls[file.fileName] = fileUrl;
-                    console.log("\nFinished file upload for:", file.fileName, "\n");
-                }
-                catch (err) {
-                    console.log("\nError uploading file: ", file.fileName, err, "\n");
-                    const error = err;
-                    if (error["$metadata"]) {
-                        // throw only those errors that are not 412 Precondition Failed
-                        // 412 errors are errors while accessing the asset, which is post upload and can be ignored
-                        if (error.$metadata.httpStatusCode !== 412) {
-                            throw error;
-                        }
-                    }
-                }
-            }, {
-                retries: 5,
-                factor: 3,
-                minTimeout: 1000,
-                maxTimeout: 60000,
-                randomize: true,
-                onRetry: (err, i) => {
-                    if (err) {
-                        const error = err;
-                        console.log("Upload retry attempt:", i, ":", file.fileName);
-                        console.log("Response status:", error.$response?.statusCode);
-                    }
-                },
-            });
-        }
-        catch (err) {
-            const error = err;
-            console.log(`R2 Error - ${file.fileName} \nError: ${error}`);
-            console.log("Upload response", error.$response, "with status", error.$metadata.httpStatusCode);
-        }
-        return;
-    }));
-    return urls;
-};
-async function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
-    const filePaths = fileList || getFileList(sourceDir);
-    const files = filePaths
-        .map((filePath) => ({
-        path: filePath,
-        fileName: filePath.replace(sourceDir, ""),
-        mimeType: undefined,
-    }))
-        .map((file) => {
-        const mimeType = mime_1.default.getType(file.path) || "application/octet-stream";
-        return {
-            buffer: fs.readFileSync(file.path),
-            fileName: file.fileName,
-            mimeType,
-        };
-    });
-    let config = {
-        accountId: accountId || process.env.R2_ACCOUNT_ID,
-        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
-        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
-        bucket: uploadBucket,
-        destinationDir,
-        files,
-    };
-    const uploadedFiles = await run(config);
-    return uploadedFiles;
-}
-exports.uploadDirectory = uploadDirectory;
-async function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
-    const config = {
-        accountId: accountId || process.env.R2_ACCOUNT_ID,
-        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
-        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
-        bucket: uploadBucket,
-        destinationDir,
-        files,
-    };
-    return await run(config);
-}
-exports.uploadInMemoryFiles = uploadInMemoryFiles;
+__exportStar(require("./fetch"), exports);
+__exportStar(require("./upload"), exports);

package/dist/types.d.ts

@@ -0,0 +1,20 @@
+/// <reference types="node" />
+interface R2BaseConfig {
+    accountId: string;
+    accessKeyId: string;
+    secretAccessKey: string;
+    bucket: string;
+}
+export interface R2UploadConfig extends R2BaseConfig {
+    destinationDir: string;
+    files: {
+        buffer: Buffer;
+        fileName: string;
+        mimeType?: string;
+    }[];
+}
+export interface R2FetchConfig extends R2BaseConfig {
+    prefix: string;
+}
+export {};
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA,UAAU,YAAY;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;IAClD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAClE;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,MAAM,EAAE,MAAM,CAAC;CAChB"}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/upload.d.ts

@@ -0,0 +1,26 @@
+/// <reference types="node" />
+export interface FileMap {
+    [file: string]: string;
+}
+export declare function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }: {
+    sourceDir: string;
+    fileList?: string[];
+    destinationDir: string;
+    uploadBucket: string;
+    accountId?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+}): Promise<FileMap>;
+export declare function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }: {
+    files: {
+        buffer: Buffer;
+        fileName: string;
+        mimeType?: string;
+    }[];
+    destinationDir: string;
+    uploadBucket: string;
+    accountId?: string;
+    accessKeyId?: string;
+    secretAccessKey?: string;
+}): Promise<FileMap>;
+//# sourceMappingURL=upload.d.ts.map

package/dist/upload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../src/upload.ts"],"names":[],"mappings":";AAgBA,MAAM,WAAW,OAAO;IACtB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAmHD,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CA6BnB;AAED,wBAAsB,mBAAmB,CAAC,EACxC,KAAK,EACL,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAUnB"}

package/dist/upload.js

@@ -0,0 +1,168 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadInMemoryFiles = exports.uploadDirectory = void 0;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
+const async_retry_1 = __importDefault(require("async-retry"));
+const fs = __importStar(require("fs"));
+const md5_1 = __importDefault(require("md5"));
+const mime_1 = __importDefault(require("mime"));
+const path_1 = __importDefault(require("path"));
+const getFileList = (dir) => {
+    let files = [];
+    const items = fs.readdirSync(dir, {
+        withFileTypes: true,
+    });
+    for (const item of items) {
+        const isDir = item.isDirectory();
+        const absolutePath = `${dir}/${item.name}`;
+        if (isDir) {
+            files = [...files, ...getFileList(absolutePath)];
+        }
+        else {
+            files.push(absolutePath);
+        }
+    }
+    return files;
+};
+const run = async (config) => {
+    const map = new Map();
+    const urls = {};
+    const S3 = new client_s3_1.S3Client({
+        region: "auto",
+        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+        credentials: {
+            accessKeyId: config.accessKeyId,
+            secretAccessKey: config.secretAccessKey,
+        },
+    });
+    await Promise.all(config.files.map(async (file) => {
+        let fileKey = path_1.default.join(config.destinationDir, file.fileName);
+        if (fileKey.includes(".gitkeep")) {
+            return;
+        }
+        const mimeType = file.mimeType || "application/octet-stream";
+        const uploadParams = {
+            Bucket: config.bucket,
+            Key: fileKey,
+            Body: file.buffer,
+            ContentLength: file.buffer.length,
+            ContentType: mimeType ?? "application/octet-stream",
+        };
+        const cmd = new client_s3_1.PutObjectCommand(uploadParams);
+        const digest = (0, md5_1.default)(file.buffer);
+        cmd.middlewareStack.add((next) => async (args) => {
+            args.request.headers["if-none-match"] = `"${digest}"`;
+            return await next(args);
+        }, {
+            step: "build",
+            name: "addETag",
+        });
+        try {
+            await (0, async_retry_1.default)(async () => {
+                try {
+                    const data = await S3.send(cmd);
+                    map.set(file.fileName, data);
+                    const fileUrl = await (0, s3_request_presigner_1.getSignedUrl)(S3, cmd);
+                    urls[file.fileName] = fileUrl;
+                }
+                catch (err) {
+                    const error = err;
+                    if (error["$metadata"]) {
+                        // throw only those errors that are not 412 Precondition Failed
+                        // 412 errors are errors while accessing the asset, which is post upload and can be ignored
+                        if (error.$metadata.httpStatusCode !== 412) {
+                            throw error;
+                        }
+                    }
+                }
+            }, {
+                retries: 5,
+                factor: 3,
+                minTimeout: 1000,
+                maxTimeout: 60000,
+                randomize: true,
+                onRetry: (err, i) => {
+                    if (err) {
+                        const error = err;
+                        console.log("Upload retry attempt:", i, ":", file.fileName);
+                        console.log("Response status:", error.$response?.statusCode);
+                    }
+                },
+            });
+        }
+        catch (err) {
+            const error = err;
+            console.log(`R2 Error - ${file.fileName} \nError: ${error}`);
+            console.log("Upload response", error.$response, "with status", error.$metadata.httpStatusCode);
+        }
+        return;
+    }));
+    return urls;
+};
+async function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
+    const filePaths = fileList || getFileList(sourceDir);
+    const files = filePaths
+        .map((filePath) => ({
+        path: filePath,
+        fileName: filePath.replace(sourceDir, ""),
+        mimeType: undefined,
+    }))
+        .map((file) => {
+        const mimeType = mime_1.default.getType(file.path) || "application/octet-stream";
+        return {
+            buffer: fs.readFileSync(file.path),
+            fileName: file.fileName,
+            mimeType,
+        };
+    });
+    let config = {
+        accountId: accountId || process.env.R2_ACCOUNT_ID,
+        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
+        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
+        bucket: uploadBucket,
+        destinationDir,
+        files,
+    };
+    const uploadedFiles = await run(config);
+    return uploadedFiles;
+}
+exports.uploadDirectory = uploadDirectory;
+async function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
+    const config = {
+        accountId: accountId || process.env.R2_ACCOUNT_ID,
+        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
+        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
+        bucket: uploadBucket,
+        destinationDir,
+        files,
+    };
+    return await run(config);
+}
+exports.uploadInMemoryFiles = uploadInMemoryFiles;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/r2-uploader",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"