@empiricalrun/playwright-utils 0.64.0 → 0.65.0

package/dist/reporter/presigned-upload.d.ts

@@ -0,0 +1,15 @@
+import { type FileMap } from "@empiricalrun/r2-uploader";
+import { ApiWorkerClient } from "../api-worker-client";
+export type PresignedUploadFile = {
+    filePath: string;
+    reportPath: string;
+};
+export declare function directoryUploadFiles(sourceDir: string, reportDir: string): PresignedUploadFile[];
+export declare function uploadPresigned({ client, runId, baseUrl, destinationDir, files, }: {
+    client: ApiWorkerClient;
+    runId: string;
+    baseUrl: string;
+    destinationDir: string;
+    files: PresignedUploadFile[];
+}): Promise<FileMap>;
+//# sourceMappingURL=presigned-upload.d.ts.map

package/dist/reporter/presigned-upload.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"presigned-upload.d.ts","sourceRoot":"","sources":["../../src/reporter/presigned-upload.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,OAAO,EAAkB,MAAM,2BAA2B,CAAC;AAOzE,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AASF,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,mBAAmB,EAAE,CAUvB;AA8CD,wBAAsB,eAAe,CAAC,EACpC,MAAM,EACN,KAAK,EACL,OAAO,EACP,cAAc,EACd,KAAK,GACN,EAAE;IACD,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,mBAAmB,EAAE,CAAC;CAC9B,GAAG,OAAO,CAAC,OAAO,CAAC,CAwCnB"}

package/dist/reporter/presigned-upload.js

@@ -0,0 +1,96 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.directoryUploadFiles = directoryUploadFiles;
+exports.uploadPresigned = uploadPresigned;
+const r2_uploader_1 = require("@empiricalrun/r2-uploader");
+const async_retry_1 = __importDefault(require("async-retry"));
+const fs_1 = __importDefault(require("fs"));
+const mime_1 = __importDefault(require("mime"));
+const path_1 = __importDefault(require("path"));
+const stream_1 = require("stream");
+const logger_1 = require("../logger");
+const MAX_FILES_PER_REQUEST = 100;
+function listFiles(dir) {
+    return fs_1.default.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+        const fullPath = path_1.default.join(dir, entry.name);
+        return entry.isDirectory() ? listFiles(fullPath) : [fullPath];
+    });
+}
+function directoryUploadFiles(sourceDir, reportDir) {
+    return listFiles(sourceDir)
+        .filter((filePath) => !filePath.includes(".gitkeep"))
+        .map((filePath) => ({
+        filePath,
+        reportPath: path_1.default.posix.join(reportDir, path_1.default.relative(sourceDir, filePath).split(path_1.default.sep).join("/")),
+    }));
+}
+async function putFile(uploadUrl, filePath, size, headers) {
+    await (0, async_retry_1.default)(async (bail) => {
+        const response = await fetch(uploadUrl, {
+            method: "PUT",
+            headers: { ...headers, "Content-Length": String(size) },
+            body: stream_1.Readable.toWeb(fs_1.default.createReadStream(filePath)),
+            duplex: "half",
+        });
+        if (response.ok) {
+            return;
+        }
+        const detail = (await response.text().catch(() => "")).slice(0, 500);
+        const error = new Error(`Report upload of ${filePath} failed: ${response.status} ${detail}`);
+        if (response.status < 500 &&
+            response.status !== 408 &&
+            response.status !== 429) {
+            bail(error);
+            return;
+        }
+        throw error;
+    }, {
+        retries: 5,
+        factor: 3,
+        minTimeout: 1_000,
+        maxTimeout: 60_000,
+        randomize: true,
+        onRetry: (_err, attempt) => {
+            logger_1.logger.debug(`[Uploader] upload retry ${attempt}: ${filePath}`);
+        },
+    });
+}
+async function uploadPresigned({ client, runId, baseUrl, destinationDir, files, }) {
+    const urls = {};
+    for (let i = 0; i < files.length; i += MAX_FILES_PER_REQUEST) {
+        const batch = await Promise.all(files.slice(i, i + MAX_FILES_PER_REQUEST).map(async (file) => {
+            const size = (await fs_1.default.promises.stat(file.filePath)).size;
+            return {
+                ...file,
+                size,
+                contentType: mime_1.default.getType(file.filePath) || "application/octet-stream",
+                storageClass: (0, r2_uploader_1.iaStorageClass)("r2", size)?.StorageClass,
+            };
+        }));
+        const { data } = await client.createTestReportUploadUrls({
+            run_id: runId,
+            files: batch.map((file) => ({
+                path: file.reportPath,
+                content_type: file.contentType,
+                storage_class: file.storageClass,
+            })),
+        });
+        await Promise.all(batch.map(async (file, index) => {
+            const signed = data.files[index];
+            if (!signed || signed.path !== file.reportPath) {
+                throw new Error(`No upload URL returned for ${file.reportPath}`);
+            }
+            await putFile(signed.upload_url, file.filePath, file.size, {
+                "Content-Type": file.contentType,
+                ...(file.storageClass && {
+                    "x-amz-storage-class": file.storageClass,
+                }),
+            });
+            urls[file.filePath] = `${baseUrl}/${destinationDir}/${file.reportPath}`;
+        }));
+    }
+    return urls;
+}

package/dist/reporter/uploader.d.ts

@@ -1,14 +1,21 @@
 import { StorageCredentials, UploadStorageClass } from "@empiricalrun/r2-uploader";
+import { ApiWorkerClient } from "../api-worker-client";
+type PresignedCredentials = {
+    provider: "r2-presigned";
+    client: ApiWorkerClient;
+};
+export type UploaderCredentials = StorageCredentials | PresignedCredentials;
 export interface UploaderConfig {
     projectSlug: string;
     runId: string;
     baseUrl: string;
     uploadBucket: string;
-    credentials: StorageCredentials;
+    credentials: UploaderCredentials;
 }
 export declare class Uploader {
     private _baseUrl;
     private _uploadBucket;
+    private _runId;
     private _destinationDir;
     private _credentials;
     constructor(config: UploaderConfig);
@@ -19,10 +26,12 @@ export declare class Uploader {
     get baseUrl(): string;
     get destinationDir(): string;
     get urlPrefix(): string;
+    private _presignedTask;
     uploadFile(filePath: string, destinationPath: string, storageClass?: UploadStorageClass): Promise<string | null>;
     uploadDirectory(sourceDir: string, destinationPrefix: string): Promise<void>;
     waitForUploads(): Promise<void>;
 }
 export declare function getUploader(): Uploader | null;
 export declare function createUploader(): Uploader | null;
+export {};
 //# sourceMappingURL=uploader.d.ts.map

package/dist/reporter/uploader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/reporter/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,kBAAkB,EAElB,kBAAkB,EAEnB,MAAM,2BAA2B,CAAC;AAKnC,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,kBAAkB,CAAC;CACjC;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,YAAY,CAAqB;IAEzC,YAAY,MAAM,EAAE,cAAc,EASjC;IAED,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAgBhC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAUhC,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAKzE;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAEK,UAAU,CACd,QAAQ,EAAE,MAAM,EAChB,eAAe,EAAE,MAAM,EACvB,YAAY,CAAC,EAAE,kBAAkB,GAChC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgCxB;IAEK,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,IAAI,CAAC,CAef;IAEK,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAyBpC;CACF;AAID,wBAAgB,WAAW,IAAI,QAAQ,GAAG,IAAI,CAM7C;AAED,wBAAgB,cAAc,IAAI,QAAQ,GAAG,IAAI,CA6DhD"}
+{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/reporter/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,kBAAkB,EAElB,kBAAkB,EAEnB,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAWvD,KAAK,oBAAoB,GAAG;IAC1B,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,eAAe,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,kBAAkB,GAAG,oBAAoB,CAAC;AAE5E,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,mBAAmB,CAAC;CAClC;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,eAAe,CAAS;IAChC,OAAO,CAAC,YAAY,CAAsB;IAE1C,YAAY,MAAM,EAAE,cAAc,EAUjC;IAED,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAgBhC,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAUhC,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,OAAO,GAAG,kBAAkB,GAAG,IAAI,CAKzE;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,OAAO,CAAC,cAAc;IAoBhB,UAAU,CACd,QAAQ,EAAE,MAAM,EAChB,eAAe,EAAE,MAAM,EACvB,YAAY,CAAC,EAAE,kBAAkB,GAChC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAyCxB;IAEK,eAAe,CACnB,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,IAAI,CAAC,CAyBf;IAEK,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAyBpC;CACF;AAID,wBAAgB,WAAW,IAAI,QAAQ,GAAG,IAAI,CAM7C;AAED,wBAAgB,cAAc,IAAI,QAAQ,GAAG,IAAI,CAmEhD"}

package/dist/reporter/uploader.js

@@ -8,15 +8,21 @@ exports.getUploader = getUploader;
 exports.createUploader = createUploader;
 const r2_uploader_1 = require("@empiricalrun/r2-uploader");
 const path_1 = __importDefault(require("path"));
+const api_worker_client_1 = require("../api-worker-client");
 const logger_1 = require("../logger");
+const presigned_upload_1 = require("./presigned-upload");
+const R2_REPORTS_BASE_URL = "https://reports-r2.empirical.run";
+const R2_REPORTS_BUCKET = "test-report";
 class Uploader {
     _baseUrl;
     _uploadBucket;
+    _runId;
     _destinationDir;
     _credentials;
     constructor(config) {
         this._baseUrl = config.baseUrl;
         this._uploadBucket = config.uploadBucket;
+        this._runId = config.runId;
         this._destinationDir = `${config.projectSlug}/${config.runId}`;
         this._credentials = config.credentials;
         logger_1.logger.debug(`[Uploader] Initialized with destination: ${this._destinationDir}, provider: ${this._credentials.provider}`);
@@ -61,8 +67,31 @@ class Uploader {
     get urlPrefix() {
         return `${this._baseUrl}/${this._destinationDir}`;
     }
+    _presignedTask(credentials, files) {
+        return async () => {
+            try {
+                return await (0, presigned_upload_1.uploadPresigned)({
+                    client: credentials.client,
+                    runId: this._runId,
+                    baseUrl: this._baseUrl,
+                    destinationDir: this._destinationDir,
+                    files,
+                });
+            }
+            catch (error) {
+                logger_1.logger.error("[Uploader] Error uploading files", error);
+                return {};
+            }
+        };
+    }
     async uploadFile(filePath, destinationPath, storageClass) {
         try {
+            if (this._credentials.provider === "r2-presigned") {
+                const result = await (0, r2_uploader_1.sendTaskToQueue)(this._presignedTask(this._credentials, [
+                    { filePath, reportPath: destinationPath },
+                ]));
+                return (result && result[filePath]) || null;
+            }
             const fullDestinationPath = path_1.default.join(this._destinationDir, destinationPath);
             const sourceDir = path_1.default.dirname(filePath);
             const uploadTask = (0, r2_uploader_1.createUploadTask)({
@@ -90,6 +119,10 @@ class Uploader {
         }
     }
     async uploadDirectory(sourceDir, destinationPrefix) {
+        if (this._credentials.provider === "r2-presigned") {
+            await (0, r2_uploader_1.sendTaskToQueue)(this._presignedTask(this._credentials, (0, presigned_upload_1.directoryUploadFiles)(sourceDir, destinationPrefix)));
+            return;
+        }
         const fullDestinationDir = path_1.default.join(this._destinationDir, destinationPrefix);
         const uploadTask = (0, r2_uploader_1.createUploadTask)({
             sourceDir,
@@ -145,6 +178,9 @@ function createUploader() {
         logger_1.logger.debug(`[Uploader] REPORT_STORAGE_BACKEND must be "s3" or "r2" (got ${backend ?? "unset"})`);
         return null;
     }
+    if (backend === "r2" && process.env.REPORT_UPLOAD_PRESIGNED === "true") {
+        return createPresignedUploader(process.env.PROJECT_NAME, process.env.TEST_RUN_GITHUB_ACTION_ID);
+    }
     const enableS3 = backend === "s3";
     const credentials = Uploader.getCredentialsFromEnv(enableS3);
     if (!credentials) {
@@ -168,12 +204,12 @@ function createUploader() {
     }
     else {
         const accountId = process.env.R2_ACCOUNT_ID;
-        const bucket = "test-report";
+        const bucket = R2_REPORTS_BUCKET;
         if (!accountId) {
             logger_1.logger.debug("[Uploader] R2_ACCOUNT_ID not configured");
             return null;
         }
-        baseUrl = `https://reports-r2.empirical.run`;
+        baseUrl = R2_REPORTS_BASE_URL;
         uploadBucket = bucket;
     }
     return new Uploader({
@@ -184,3 +220,16 @@ function createUploader() {
         credentials,
     });
 }
+function createPresignedUploader(projectSlug, runId) {
+    if (!process.env.EMPIRICALRUN_API_KEY) {
+        logger_1.logger.debug("[Uploader] EMPIRICALRUN_API_KEY is required for presigned uploads");
+        return null;
+    }
+    return new Uploader({
+        projectSlug,
+        runId,
+        baseUrl: R2_REPORTS_BASE_URL,
+        uploadBucket: R2_REPORTS_BUCKET,
+        credentials: { provider: "r2-presigned", client: new api_worker_client_1.ApiWorkerClient() },
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empiricalrun/playwright-utils",
-  "version": "0.64.0",
+  "version": "0.65.0",
   "engines": {
     "node": ">=22"
   },
@@ -45,7 +45,7 @@
     "playwright-core": "1.63.0",
     "serve-handler": "^6.1.6",
     "typescript": "7.0.2",
-    "@empiricalrun/shared-types": "0.22.0"
+    "@empiricalrun/shared-types": "0.23.0"
   },
   "dependencies": {
     "async-retry": "^1.3.3",
@@ -58,7 +58,7 @@
     "ts-morph": "^28.0.0",
     "@empiricalrun/cua": "^0.5.4",
     "@empiricalrun/llm": "^0.28.3",
-    "@empiricalrun/r2-uploader": "^0.11.4",
+    "@empiricalrun/r2-uploader": "^0.11.5",
     "@empiricalrun/reporter": "^0.30.3"
   },
   "scripts": {