@empiricalrun/r2-uploader 0.6.0 → 0.8.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @empiricalrun/r2-uploader
 
+## 0.8.0
+
+### Minor Changes
+
+- 3ee1aec: feat: support aws s3
+
+## 0.7.0
+
+### Minor Changes
+
+- 2d9919d: feat: consolidate zip utils and move to streaming
+
 ## 0.6.0
 
 ### Minor Changes

package/dist/client.d.ts

@@ -0,0 +1,5 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import { StorageBaseConfig, StorageCredentials } from "./types";
+export declare function getStorageCredentialsFromEnv(): StorageCredentials | null;
+export declare function createStorageClient(config: StorageBaseConfig): S3Client;
+//# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEhE,wBAAgB,4BAA4B,IAAI,kBAAkB,GAAG,IAAI,CA4BxE;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,iBAAiB,GAAG,QAAQ,CAuBvE"}

package/dist/client.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getStorageCredentialsFromEnv = getStorageCredentialsFromEnv;
+exports.createStorageClient = createStorageClient;
+const client_s3_1 = require("@aws-sdk/client-s3");
+function getStorageCredentialsFromEnv() {
+    if (process.env.S3_REGION &&
+        process.env.S3_ACCESS_KEY_ID &&
+        process.env.S3_SECRET_ACCESS_KEY) {
+        return {
+            provider: "s3",
+            region: process.env.S3_REGION,
+            accessKeyId: process.env.S3_ACCESS_KEY_ID,
+            secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
+        };
+    }
+    if (process.env.R2_ACCOUNT_ID &&
+        process.env.R2_ACCESS_KEY_ID &&
+        process.env.R2_SECRET_ACCESS_KEY) {
+        return {
+            provider: "r2",
+            accountId: process.env.R2_ACCOUNT_ID,
+            accessKeyId: process.env.R2_ACCESS_KEY_ID,
+            secretAccessKey: process.env.R2_SECRET_ACCESS_KEY,
+        };
+    }
+    return null;
+}
+function createStorageClient(config) {
+    if (config.provider === "s3") {
+        return new client_s3_1.S3Client({
+            region: config.region,
+            credentials: {
+                accessKeyId: config.accessKeyId,
+                secretAccessKey: config.secretAccessKey,
+            },
+        });
+    }
+    if (config.provider === "r2" || "accountId" in config) {
+        return new client_s3_1.S3Client({
+            region: "auto",
+            endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
+            credentials: {
+                accessKeyId: config.accessKeyId,
+                secretAccessKey: config.secretAccessKey,
+            },
+        });
+    }
+    throw new Error("Invalid storage config: missing provider or accountId");
+}

package/dist/delete.d.ts

@@ -1,3 +1,3 @@
-import { R2DeleteConfig } from "./types";
-export declare function deleteFile(fileKey: string, config: R2DeleteConfig): Promise<import("@aws-sdk/client-s3").DeleteObjectCommandOutput>;
+import { DeleteConfig } from "./types";
+export declare function deleteFile(fileKey: string, config: DeleteConfig): Promise<import("@aws-sdk/client-s3").DeleteObjectCommandOutput>;
 //# sourceMappingURL=delete.d.ts.map

package/dist/delete.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../src/delete.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,wBAAsB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,mEAqBvE"}
+{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../src/delete.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,wBAAsB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,mEAcrE"}

package/dist/delete.js

@@ -2,16 +2,10 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.deleteFile = deleteFile;
 const client_s3_1 = require("@aws-sdk/client-s3");
+const client_1 = require("./client");
 const logger_1 = require("./logger");
 async function deleteFile(fileKey, config) {
-    const s3Client = new client_s3_1.S3Client({
-        region: "auto",
-        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
-        credentials: {
-            accessKeyId: config.accessKeyId,
-            secretAccessKey: config.secretAccessKey,
-        },
-    });
+    const s3Client = (0, client_1.createStorageClient)(config);
     const params = {
         Bucket: config.bucket,
         Key: fileKey,

package/dist/fetch.d.ts

@@ -1,5 +1,5 @@
-import { R2FetchConfig } from "./types";
-export declare const fetchFiles: (config: R2FetchConfig) => Promise<{
+import { FetchConfig } from "./types";
+export declare const fetchFiles: (config: FetchConfig) => Promise<{
     key: string;
     lastModified: Date;
     size: number;

package/dist/fetch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,eAAO,MAAM,UAAU,GAAU,QAAQ,aAAa;;;;;;;;;;IAmDrD,CAAC"}
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEtC,eAAO,MAAM,UAAU,GAAU,QAAQ,WAAW;;;;;;;;;;IA4CnD,CAAC"}

package/dist/fetch.js

@@ -3,15 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.fetchFiles = void 0;
 const client_s3_1 = require("@aws-sdk/client-s3");
 const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
+const client_1 = require("./client");
 const fetchFiles = async (config) => {
-    const S3 = new client_s3_1.S3Client({
-        region: "auto",
-        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
-        credentials: {
-            accessKeyId: config.accessKeyId,
-            secretAccessKey: config.secretAccessKey,
-        },
-    });
+    const S3 = (0, client_1.createStorageClient)(config);
     const params = {
         Bucket: config.bucket,
         Prefix: config.prefix,

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 export { fetchFiles } from "./fetch";
 export { sendTaskToQueue, waitForTaskQueueToFinish } from "./queue";
-export type { AsyncTask, FileMap } from "./types";
+export type { AsyncTask, DeleteConfig, FetchConfig, FileMap, StorageBaseConfig, StorageCredentials, StorageProvider, UploadBufferConfig, UploadStreamConfig, } from "./types";
 export { uploadDirectory, uploadInMemoryFiles } from "./upload";
 export type { CreateUploadTaskOptions } from "./upload-task";
 export { createUploadTask } from "./upload-task";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAC;AACpE,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAChE,YAAY,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,wBAAwB,EAAE,MAAM,SAAS,CAAC;AACpE,YAAY,EACV,SAAS,EACT,YAAY,EACZ,WAAW,EACX,OAAO,EACP,iBAAiB,EACjB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAChE,YAAY,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC"}

package/dist/types.d.ts

@@ -1,32 +1,61 @@
-interface R2BaseConfig {
+export type StorageProvider = "r2" | "s3";
+interface R2Credentials {
+    provider: "r2";
     accountId: string;
     accessKeyId: string;
     secretAccessKey: string;
+}
+interface S3Credentials {
+    provider: "s3";
+    region: string;
+    accessKeyId: string;
+    secretAccessKey: string;
+}
+/** @deprecated Legacy R2 credentials format without explicit provider */
+interface LegacyR2Credentials {
+    provider?: undefined;
+    accountId: string;
+    accessKeyId: string;
+    secretAccessKey: string;
+}
+export type StorageCredentials = R2Credentials | S3Credentials;
+interface BaseConfig {
     bucket: string;
 }
-export interface R2UploadStreamConfig extends R2BaseConfig {
+type R2BaseConfig = BaseConfig & R2Credentials;
+type S3BaseConfig = BaseConfig & S3Credentials;
+type LegacyR2BaseConfig = BaseConfig & LegacyR2Credentials;
+export type StorageBaseConfig = R2BaseConfig | S3BaseConfig | LegacyR2BaseConfig;
+export type UploadStreamConfig = StorageBaseConfig & {
     destinationDir: string;
     files: {
         fullPath: string;
         fileName: string;
         mimeType?: string;
     }[];
-}
-export interface R2UploadBufferConfig extends R2BaseConfig {
+};
+export type UploadBufferConfig = StorageBaseConfig & {
     destinationDir: string;
     files: {
         buffer: Buffer;
         fileName: string;
         mimeType?: string;
     }[];
-}
-export interface R2FetchConfig extends R2BaseConfig {
+};
+export type FetchConfig = StorageBaseConfig & {
     prefix: string;
     withSignedUrl?: boolean;
     suffix?: string;
-}
-export interface R2DeleteConfig extends R2BaseConfig {
-}
+};
+export type DeleteConfig = StorageBaseConfig;
+/** @deprecated Use UploadStreamConfig instead */
+export type R2UploadStreamConfig = UploadStreamConfig;
+/** @deprecated Use UploadBufferConfig instead */
+export type R2UploadBufferConfig = UploadBufferConfig;
+/** @deprecated Use FetchConfig instead */
+export type R2FetchConfig = FetchConfig;
+/** @deprecated Use DeleteConfig instead */
+export type R2DeleteConfig = DeleteConfig;
 export interface FileMap {
     [file: string]: string;
 }

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,UAAU,YAAY;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACpE;AAED,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAClE;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAe,SAAQ,YAAY;CAAG;AAEvD,MAAM,WAAW,OAAO;IACtB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,eAAe,GAAG,IAAI,GAAG,IAAI,CAAC;AAE1C,UAAU,aAAa;IACrB,QAAQ,EAAE,IAAI,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,UAAU,aAAa;IACrB,QAAQ,EAAE,IAAI,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,yEAAyE;AACzE,UAAU,mBAAmB;IAC3B,QAAQ,CAAC,EAAE,SAAS,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG,aAAa,CAAC;AAE/D,UAAU,UAAU;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,KAAK,YAAY,GAAG,UAAU,GAAG,aAAa,CAAC;AAC/C,KAAK,YAAY,GAAG,UAAU,GAAG,aAAa,CAAC;AAC/C,KAAK,kBAAkB,GAAG,UAAU,GAAG,mBAAmB,CAAC;AAE3D,MAAM,MAAM,iBAAiB,GACzB,YAAY,GACZ,YAAY,GACZ,kBAAkB,CAAC;AAEvB,MAAM,MAAM,kBAAkB,GAAG,iBAAiB,GAAG;IACnD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACpE,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,iBAAiB,GAAG;IACnD,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAClE,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,iBAAiB,GAAG;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG,iBAAiB,CAAC;AAE7C,iDAAiD;AACjD,MAAM,MAAM,oBAAoB,GAAG,kBAAkB,CAAC;AACtD,iDAAiD;AACjD,MAAM,MAAM,oBAAoB,GAAG,kBAAkB,CAAC;AACtD,0CAA0C;AAC1C,MAAM,MAAM,aAAa,GAAG,WAAW,CAAC;AACxC,2CAA2C;AAC3C,MAAM,MAAM,cAAc,GAAG,YAAY,CAAC;AAE1C,MAAM,WAAW,OAAO;IACtB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC"}

package/dist/upload/buffer.d.ts

@@ -1,3 +1,3 @@
-import { FileMap, R2UploadBufferConfig } from "../types";
-export declare const uploadFileBuffers: (config: R2UploadBufferConfig) => Promise<FileMap>;
+import { FileMap, UploadBufferConfig } from "../types";
+export declare const uploadFileBuffers: (config: UploadBufferConfig) => Promise<FileMap>;
 //# sourceMappingURL=buffer.d.ts.map

package/dist/upload/buffer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"buffer.d.ts","sourceRoot":"","sources":["../../src/upload/buffer.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEzD,eAAO,MAAM,iBAAiB,GAAU,QAAQ,oBAAoB,qBAyGnE,CAAC"}
+{"version":3,"file":"buffer.d.ts","sourceRoot":"","sources":["../../src/upload/buffer.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEvD,eAAO,MAAM,iBAAiB,GAAU,QAAQ,kBAAkB,qBAkGjE,CAAC"}

package/dist/upload/buffer.js

@@ -9,18 +9,12 @@ const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
 const async_retry_1 = __importDefault(require("async-retry"));
 const md5_1 = __importDefault(require("md5"));
 const path_1 = __importDefault(require("path"));
+const client_1 = require("../client");
 const logger_1 = require("../logger");
 const uploadFileBuffers = async (config) => {
     const map = new Map();
     const urls = {};
-    const S3 = new client_s3_1.S3Client({
-        region: "auto",
-        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
-        credentials: {
-            accessKeyId: config.accessKeyId,
-            secretAccessKey: config.secretAccessKey,
-        },
-    });
+    const S3 = (0, client_1.createStorageClient)(config);
     await Promise.all(config.files.map(async (file) => {
         let fileKey = path_1.default.join(config.destinationDir, file.fileName);
         if (fileKey.includes(".gitkeep")) {

package/dist/upload/index.d.ts

@@ -1,14 +1,18 @@
-import { FileMap } from "../types";
-export declare function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }: {
+import { FileMap, StorageCredentials } from "../types";
+export declare function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, credentials, accountId, accessKeyId, secretAccessKey, }: {
     sourceDir: string;
     fileList?: string[];
     destinationDir: string;
     uploadBucket: string;
+    credentials?: StorageCredentials;
+    /** @deprecated Use credentials instead */
     accountId?: string;
+    /** @deprecated Use credentials instead */
     accessKeyId?: string;
+    /** @deprecated Use credentials instead */
     secretAccessKey?: string;
 }): Promise<FileMap>;
-export declare function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }: {
+export declare function uploadInMemoryFiles({ files, destinationDir, uploadBucket, credentials, accountId, accessKeyId, secretAccessKey, }: {
     files: {
         buffer: Buffer;
         fileName: string;
@@ -16,8 +20,12 @@ export declare function uploadInMemoryFiles({ files, destinationDir, uploadBucke
     }[];
     destinationDir: string;
     uploadBucket: string;
+    credentials?: StorageCredentials;
+    /** @deprecated Use credentials instead */
     accountId?: string;
+    /** @deprecated Use credentials instead */
     accessKeyId?: string;
+    /** @deprecated Use credentials instead */
     secretAccessKey?: string;
 }): Promise<FileMap>;
 //# sourceMappingURL=index.d.ts.map

package/dist/upload/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/upload/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,OAAO,EAA8C,MAAM,UAAU,CAAC;AAqB/E,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAqBnB;AAED,wBAAsB,mBAAmB,CAAC,EACxC,KAAK,EACL,cAAc,EACd,YAAY,EACZ,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAUnB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/upload/index.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,OAAO,EACP,kBAAkB,EAGnB,MAAM,UAAU,CAAC;AAqBlB,wBAAsB,eAAe,CAAC,EACpC,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,WAAW,EACX,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAwCnB;AAED,wBAAsB,mBAAmB,CAAC,EACxC,KAAK,EACL,cAAc,EACd,YAAY,EACZ,WAAW,EACX,SAAS,EACT,WAAW,EACX,eAAe,GAChB,EAAE;IACD,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CA6BnB"}

package/dist/upload/index.js

@@ -40,6 +40,7 @@ exports.uploadDirectory = uploadDirectory;
 exports.uploadInMemoryFiles = uploadInMemoryFiles;
 const fs = __importStar(require("fs"));
 const mime_1 = __importDefault(require("mime"));
+const client_1 = require("../client");
 const logger_1 = require("../logger");
 const buffer_1 = require("./buffer");
 const stream_1 = require("./stream");
@@ -60,18 +61,32 @@ const getFileList = (dir) => {
     }
     return files;
 };
-async function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
+async function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBucket, credentials, accountId, accessKeyId, secretAccessKey, }) {
+    let creds = credentials;
+    if (!creds && accountId && accessKeyId && secretAccessKey) {
+        creds = {
+            provider: "r2",
+            accountId,
+            accessKeyId,
+            secretAccessKey,
+        };
+    }
+    if (!creds) {
+        const envCreds = (0, client_1.getStorageCredentialsFromEnv)();
+        if (!envCreds) {
+            throw new Error("No storage credentials provided. Set S3_* or R2_* environment variables.");
+        }
+        creds = envCreds;
+    }
     const filePaths = fileList || getFileList(sourceDir);
     const files = filePaths.map((filePath) => ({
         fullPath: filePath,
-        fileName: filePath.replace(sourceDir, ""), // relative path
+        fileName: filePath.replace(sourceDir, ""),
         mimeType: mime_1.default.getType(filePath) || "application/octet-stream",
     }));
     logger_1.logger.debug("Got request to upload following files:", JSON.stringify(files, null, 2));
     const config = {
-        accountId: accountId || process.env.R2_ACCOUNT_ID,
-        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
-        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
+        ...creds,
         bucket: uploadBucket,
         destinationDir,
         files,
@@ -79,11 +94,25 @@ async function uploadDirectory({ sourceDir, fileList, destinationDir, uploadBuck
     const uploadedFiles = await (0, stream_1.uploadFileStreams)(config);
     return uploadedFiles;
 }
-async function uploadInMemoryFiles({ files, destinationDir, uploadBucket, accountId, accessKeyId, secretAccessKey, }) {
+async function uploadInMemoryFiles({ files, destinationDir, uploadBucket, credentials, accountId, accessKeyId, secretAccessKey, }) {
+    let creds = credentials;
+    if (!creds && accountId && accessKeyId && secretAccessKey) {
+        creds = {
+            provider: "r2",
+            accountId,
+            accessKeyId,
+            secretAccessKey,
+        };
+    }
+    if (!creds) {
+        const envCreds = (0, client_1.getStorageCredentialsFromEnv)();
+        if (!envCreds) {
+            throw new Error("No storage credentials provided. Set S3_* or R2_* environment variables.");
+        }
+        creds = envCreds;
+    }
     const config = {
-        accountId: accountId || process.env.R2_ACCOUNT_ID,
-        accessKeyId: accessKeyId || process.env.R2_ACCESS_KEY_ID,
-        secretAccessKey: secretAccessKey || process.env.R2_SECRET_ACCESS_KEY,
+        ...creds,
         bucket: uploadBucket,
         destinationDir,
         files,

package/dist/upload/stream.d.ts

@@ -1,3 +1,3 @@
-import { FileMap, R2UploadStreamConfig } from "../types";
-export declare const uploadFileStreams: (config: R2UploadStreamConfig) => Promise<FileMap>;
+import { FileMap, UploadStreamConfig } from "../types";
+export declare const uploadFileStreams: (config: UploadStreamConfig) => Promise<FileMap>;
 //# sourceMappingURL=stream.d.ts.map

package/dist/upload/stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/upload/stream.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAEzD,eAAO,MAAM,iBAAiB,GAAU,QAAQ,oBAAoB,qBA0GnE,CAAC"}
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/upload/stream.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEvD,eAAO,MAAM,iBAAiB,GAAU,QAAQ,kBAAkB,qBAmGjE,CAAC"}

package/dist/upload/stream.js

@@ -9,17 +9,11 @@ const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
 const async_retry_1 = __importDefault(require("async-retry"));
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
+const client_1 = require("../client");
 const logger_1 = require("../logger");
 const uploadFileStreams = async (config) => {
     const urls = {};
-    const S3 = new client_s3_1.S3Client({
-        region: "auto",
-        endpoint: `https://${config.accountId}.r2.cloudflarestorage.com`,
-        credentials: {
-            accessKeyId: config.accessKeyId,
-            secretAccessKey: config.secretAccessKey,
-        },
-    });
+    const S3 = (0, client_1.createStorageClient)(config);
     await Promise.all(config.files.map(async (file) => {
         let fileKey = path_1.default.join(config.destinationDir, file.fileName);
         if (fileKey.includes(".gitkeep")) {

package/dist/upload-task.d.ts

@@ -7,13 +7,13 @@ export interface CreateUploadTaskOptions {
     baseUrl: string;
 }
 /**
- * Creates an async task for uploading files to R2 storage.
+ * Creates an async task for uploading files to R2 or S3 storage.
  *
  * @param {Object} params - The upload parameters
  * @param {string} params.sourceDir - The source directory containing files to upload
  * @param {string[]} [params.fileList] - Optional list of specific files to upload
- * @param {string} params.destinationDir - The destination directory in R2 storage
- * @param {string} params.uploadBucket - The R2 bucket to upload to
+ * @param {string} params.destinationDir - The destination directory in storage
+ * @param {string} params.uploadBucket - The bucket to upload to
  * @param {string} params.baseUrl - The base URL for generating file URLs
  * @returns {AsyncTask} An async task that when executed will upload the files and return a map of file paths to URLs
  */

package/dist/upload-task.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"upload-task.d.ts","sourceRoot":"","sources":["../src/upload-task.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAW,MAAM,SAAS,CAAC;AAG7C,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,EAC/B,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,OAAO,GACR,EAAE,uBAAuB,GAAG,SAAS,CA4BrC"}
+{"version":3,"file":"upload-task.d.ts","sourceRoot":"","sources":["../src/upload-task.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAW,MAAM,SAAS,CAAC;AAG7C,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,EAC/B,SAAS,EACT,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,OAAO,GACR,EAAE,uBAAuB,GAAG,SAAS,CAyBrC"}

package/dist/upload-task.js

@@ -5,24 +5,24 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createUploadTask = createUploadTask;
 const path_1 = __importDefault(require("path"));
+const client_1 = require("./client");
 const logger_1 = require("./logger");
 const upload_1 = require("./upload");
 /**
- * Creates an async task for uploading files to R2 storage.
+ * Creates an async task for uploading files to R2 or S3 storage.
  *
  * @param {Object} params - The upload parameters
  * @param {string} params.sourceDir - The source directory containing files to upload
  * @param {string[]} [params.fileList] - Optional list of specific files to upload
- * @param {string} params.destinationDir - The destination directory in R2 storage
- * @param {string} params.uploadBucket - The R2 bucket to upload to
+ * @param {string} params.destinationDir - The destination directory in storage
+ * @param {string} params.uploadBucket - The bucket to upload to
  * @param {string} params.baseUrl - The base URL for generating file URLs
  * @returns {AsyncTask} An async task that when executed will upload the files and return a map of file paths to URLs
  */
 function createUploadTask({ sourceDir, fileList, destinationDir, uploadBucket, baseUrl, }) {
     return async () => {
-        if (!process.env.R2_ACCOUNT_ID ||
-            !process.env.R2_ACCESS_KEY_ID ||
-            !process.env.R2_SECRET_ACCESS_KEY) {
+        const credentials = (0, client_1.getStorageCredentialsFromEnv)();
+        if (!credentials) {
             return;
         }
         try {

package/dist/zip-utils.d.ts

@@ -0,0 +1,14 @@
+export declare function readZipEntry(zipPath: string, entryName: string): Promise<Buffer | null>;
+export declare function getZipEntryNames(zipPath: string): Promise<string[]>;
+export declare function extractZipToDirectory(zipPath: string, destDir: string): Promise<void>;
+export declare function extractZipBufferToDirectory(buffer: Buffer, destDir: string): Promise<void>;
+export declare function createZipFromDirectory(sourceDir: string): Promise<Buffer>;
+export interface PatchZipOptions {
+    filterEntry?: (filename: string) => boolean;
+    newEntries?: {
+        filename: string;
+        content: Buffer;
+    }[];
+}
+export declare function patchZipFile(zipPath: string, options: PatchZipOptions): Promise<void>;
+//# sourceMappingURL=zip-utils.d.ts.map

package/dist/zip-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip-utils.d.ts","sourceRoot":"","sources":["../src/zip-utils.ts"],"names":[],"mappings":"AAMA,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAexB;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQzE;AAED,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAiBf;AAED,wBAAsB,2BAA2B,CAC/C,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAiBf;AAED,wBAAsB,sBAAsB,CAC1C,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,CAyBjB;AAED,MAAM,WAAW,eAAe;IAC9B,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;IAC5C,UAAU,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACtD;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,IAAI,CAAC,CAiCf"}

package/dist/zip-utils.js

@@ -0,0 +1,162 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readZipEntry = readZipEntry;
+exports.getZipEntryNames = getZipEntryNames;
+exports.extractZipToDirectory = extractZipToDirectory;
+exports.extractZipBufferToDirectory = extractZipBufferToDirectory;
+exports.createZipFromDirectory = createZipFromDirectory;
+exports.patchZipFile = patchZipFile;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const promises_1 = require("node:stream/promises");
+const yauzl = __importStar(require("yauzl-promise"));
+const yazl = __importStar(require("yazl"));
+async function readZipEntry(zipPath, entryName) {
+    const zipFile = await yauzl.open(zipPath);
+    for await (const entry of zipFile) {
+        if (entry.filename === entryName) {
+            const stream = await entry.openReadStream();
+            const chunks = [];
+            for await (const chunk of stream) {
+                chunks.push(chunk);
+            }
+            await zipFile.close();
+            return Buffer.concat(chunks);
+        }
+    }
+    await zipFile.close();
+    return null;
+}
+async function getZipEntryNames(zipPath) {
+    const zipFile = await yauzl.open(zipPath);
+    const names = [];
+    for await (const entry of zipFile) {
+        names.push(entry.filename);
+    }
+    await zipFile.close();
+    return names;
+}
+async function extractZipToDirectory(zipPath, destDir) {
+    const zipFile = await yauzl.open(zipPath);
+    try {
+        for await (const entry of zipFile) {
+            const destPath = node_path_1.default.join(destDir, entry.filename);
+            if (entry.filename.endsWith("/")) {
+                await node_fs_1.default.promises.mkdir(destPath, { recursive: true });
+            }
+            else {
+                await node_fs_1.default.promises.mkdir(node_path_1.default.dirname(destPath), { recursive: true });
+                const readStream = await entry.openReadStream();
+                const writeStream = node_fs_1.default.createWriteStream(destPath);
+                await (0, promises_1.pipeline)(readStream, writeStream);
+            }
+        }
+    }
+    finally {
+        await zipFile.close();
+    }
+}
+async function extractZipBufferToDirectory(buffer, destDir) {
+    const zipFile = await yauzl.fromBuffer(buffer);
+    try {
+        for await (const entry of zipFile) {
+            const destPath = node_path_1.default.join(destDir, entry.filename);
+            if (entry.filename.endsWith("/")) {
+                await node_fs_1.default.promises.mkdir(destPath, { recursive: true });
+            }
+            else {
+                await node_fs_1.default.promises.mkdir(node_path_1.default.dirname(destPath), { recursive: true });
+                const readStream = await entry.openReadStream();
+                const writeStream = node_fs_1.default.createWriteStream(destPath);
+                await (0, promises_1.pipeline)(readStream, writeStream);
+            }
+        }
+    }
+    finally {
+        await zipFile.close();
+    }
+}
+async function createZipFromDirectory(sourceDir) {
+    const zipFile = new yazl.ZipFile();
+    async function addDir(dir, prefix) {
+        const entries = await node_fs_1.default.promises.readdir(dir, { withFileTypes: true });
+        for (const entry of entries) {
+            const fullPath = node_path_1.default.join(dir, entry.name);
+            const zipPath = prefix ? `${prefix}/${entry.name}` : entry.name;
+            if (entry.isDirectory()) {
+                await addDir(fullPath, zipPath);
+            }
+            else {
+                zipFile.addFile(fullPath, zipPath);
+            }
+        }
+    }
+    await addDir(sourceDir, "");
+    zipFile.end();
+    return new Promise((resolve, reject) => {
+        const chunks = [];
+        zipFile.outputStream.on("data", (chunk) => chunks.push(chunk));
+        zipFile.outputStream.on("end", () => resolve(Buffer.concat(chunks)));
+        zipFile.outputStream.on("error", reject);
+    });
+}
+async function patchZipFile(zipPath, options) {
+    const { filterEntry, newEntries } = options;
+    const tempPath = `${zipPath}.tmp`;
+    const newZip = new yazl.ZipFile();
+    const oldZip = await yauzl.open(zipPath);
+    for await (const entry of oldZip) {
+        if (filterEntry && !filterEntry(entry.filename)) {
+            continue;
+        }
+        if (entry.filename.endsWith("/")) {
+            newZip.addEmptyDirectory(entry.filename);
+            continue;
+        }
+        const readStream = await entry.openReadStream();
+        newZip.addReadStream(readStream, entry.filename);
+    }
+    for (const { filename, content } of newEntries ?? []) {
+        newZip.addBuffer(content, filename);
+    }
+    newZip.end();
+    await (0, promises_1.pipeline)(newZip.outputStream, node_fs_1.default.createWriteStream(tempPath));
+    await oldZip.close();
+    node_fs_1.default.renameSync(tempPath, zipPath);
+}

package/package.json

@@ -1,11 +1,21 @@
 {
   "name": "@empiricalrun/r2-uploader",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
   },
   "main": "dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./zip": {
+      "types": "./dist/zip-utils.d.ts",
+      "default": "./dist/zip-utils.js"
+    }
+  },
   "repository": {
     "type": "git",
     "url": "https://github.com/empirical-run/empirical.git"
@@ -14,16 +24,20 @@
   "dependencies": {
     "@aws-sdk/client-s3": "3.614.0",
     "@aws-sdk/s3-request-presigner": "3.614.0",
-    "@types/async-retry": "^1.4.8",
-    "@types/md5": "^2.3.5",
-    "@types/mime": "3.0.0",
     "async-retry": "^1.3.3",
     "console-log-level": "^1.4.1",
     "md5": "^2.3.0",
-    "mime": "3.0.0"
+    "mime": "3.0.0",
+    "yazl": "^3.3.1",
+    "yauzl-promise": "^4.0.0"
   },
   "devDependencies": {
-    "@types/console-log-level": "^1.4.5"
+    "@types/async-retry": "^1.4.8",
+    "@types/console-log-level": "^1.4.5",
+    "@types/md5": "^2.3.5",
+    "@types/mime": "3.0.0",
+    "@types/yauzl-promise": "^4.0.1",
+    "@types/yazl": "^2.4.5"
   },
   "scripts": {
     "dev": "tsc --build --watch",

package/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/delete.ts","./src/fetch.ts","./src/index.ts","./src/logger.ts","./src/queue.ts","./src/types.ts","./src/upload-task.ts","./src/upload/buffer.ts","./src/upload/index.ts","./src/upload/stream.ts"],"version":"5.8.3"}
+{"root":["./src/client.ts","./src/delete.ts","./src/fetch.ts","./src/index.ts","./src/logger.ts","./src/queue.ts","./src/types.ts","./src/upload-task.ts","./src/zip-utils.ts","./src/upload/buffer.ts","./src/upload/index.ts","./src/upload/stream.ts"],"version":"5.8.3"}