@testdriverai/runner 7.11.62-canary → 7.11.63-canary

package/lib/automation.js

@@ -72,6 +72,128 @@ function readPngDimensions(buffer) {
   };
 }
 
+// ─── Transient HTTP retry ────────────────────────────────────────────────────
+// S3-compatible stores (we run on Tigris) answer backpressure with 503
+// SlowDown / ServiceUnavailable, and 429 when a rate limit is hit. Those are
+// retryable by contract — every AWS SDK retries them automatically. The
+// screenshot upload path below talks to the store over raw `fetch`, so it has
+// to do that itself or a single throttle blip fails the whole command.
+const RETRYABLE_STATUS = new Set([429, 500, 502, 503, 504]);
+const UPLOAD_MAX_ATTEMPTS = 4;
+
+// zlib level for encoded screenshots. Overridable so a runner on a CPU-starved
+// instance can trade bytes back for latency without a rebuild.
+const PNG_COMPRESSION_LEVEL = (() => {
+  const raw = parseInt(process.env.TD_PNG_COMPRESSION_LEVEL || '', 10);
+  return Number.isFinite(raw) && raw >= 0 && raw <= 9 ? raw : 6;
+})();
+
+/**
+ * Exponential backoff with full jitter. Jitter matters more than the base
+ * delay here: a fleet of runners throttled by the same bucket would otherwise
+ * retry in lockstep and re-trigger the throttle they're backing off from.
+ * @param {number} attempt - 1-based attempt number that just failed
+ * @returns {number} milliseconds to sleep
+ */
+function backoffMs(attempt) {
+  const base = Math.min(500 * 2 ** (attempt - 1), 8000);
+  return Math.round(base * (0.5 + Math.random() * 0.5));
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Fetch with retries on transient HTTP statuses and network/timeout errors.
+ *
+ * Returns the final Response even when it is an error status — the caller
+ * decides how to report it — but only after the retry budget is spent.
+ *
+ * Bounded by BOTH an attempt count and a wall-clock deadline. The deadline is
+ * what keeps this honest: retries have to fit inside the per-command timeout
+ * the SDK enforces (ably-service uses the command's own timeout, else 120s),
+ * or a throttled upload turns into a command timeout with no useful error.
+ * Throttles fail in milliseconds, so the deadline only ever bites on genuinely
+ * stuck connections — exactly the case where retrying further is pointless.
+ *
+ * @param {string} url
+ * @param {object} init - fetch init (a fresh AbortSignal is set per attempt)
+ * @param {object} opts
+ * @param {string} opts.label - short name used in logs
+ * @param {number} [opts.timeoutMs=20000] - per-attempt timeout
+ * @param {number} [opts.deadlineMs=45000] - total wall-clock budget
+ * @param {number} [opts.maxAttempts=UPLOAD_MAX_ATTEMPTS]
+ * @returns {Promise<Response>}
+ */
+async function fetchWithRetry(url, init, {
+  label,
+  timeoutMs = 20000,
+  deadlineMs = 45000,
+  maxAttempts = UPLOAD_MAX_ATTEMPTS,
+}) {
+  let lastError;
+  const startedAt = Date.now();
+  // True when there is no time left to both sleep and give the next attempt a
+  // fair chance at completing.
+  const outOfTime = (sleepMs) => Date.now() - startedAt + sleepMs >= deadlineMs;
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      // A fresh signal per attempt — an AbortSignal that already fired would
+      // abort every subsequent retry instantly. Clamp it to whatever is left
+      // of the overall budget so the last attempt can't overrun the deadline.
+      const remaining = Math.max(1000, deadlineMs - (Date.now() - startedAt));
+      const response = await fetch(url, {
+        ...init,
+        signal: AbortSignal.timeout(Math.min(timeoutMs, remaining)),
+      });
+
+      if (response.ok || !RETRYABLE_STATUS.has(response.status)) {
+        return response;
+      }
+
+      // Read the body so the reason (SlowDown vs. ServiceUnavailable vs. a
+      // gateway page) survives into the logs. Without this every throttle
+      // looks identical and is impossible to tell apart after the fact.
+      const body = await response.text().catch(() => '');
+      console.warn(
+        `[automation] ${label} attempt ${attempt}/${maxAttempts} failed: ` +
+        `status=${response.status} body=${body.slice(0, 300)}`
+      );
+      Sentry.addBreadcrumb({
+        category: 'upload.retry',
+        message: `${label} ${response.status}`,
+        level: 'warning',
+        data: { attempt, maxAttempts, status: response.status, body: body.slice(0, 300) },
+      });
+
+      const statusSleep = backoffMs(attempt);
+      if (attempt === maxAttempts || outOfTime(statusSleep)) {
+        // Re-wrap: the body is already consumed, so hand the caller a
+        // response-shaped object it can still read the status and text from.
+        return {
+          ok: false,
+          status: response.status,
+          text: async () => body,
+        };
+      }
+      lastError = new Error(`${label} failed: ${response.status} ${body.slice(0, 300)}`);
+      await sleep(statusSleep);
+      continue;
+    } catch (err) {
+      // Network errors and per-attempt timeouts land here.
+      lastError = err;
+      console.warn(
+        `[automation] ${label} attempt ${attempt}/${maxAttempts} errored: ${err.message}`
+      );
+      const errSleep = backoffMs(attempt);
+      if (attempt === maxAttempts || outOfTime(errSleep)) throw err;
+      await sleep(errSleep);
+    }
+  }
+
+  throw lastError;
+}
+
 // Log any pyautogui call that takes longer than this. Every call pays a fresh
 // interpreter start plus the `import pyautogui` graph, so the normal range is
 // tens to low-hundreds of ms — anything past this is drift worth seeing before
@@ -710,8 +832,17 @@ class Automation extends EventEmitter {
         // round-trip through base64 (PNG buffer → base64 → PNG buffer) that
         // doubles peak memory and adds ~10–30 ms of pure encoding cost on
         // large screenshots.
+        // The SDK sends targetWidth/targetHeight when it is going to downscale
+        // the frame on arrival anyway (redraw stability polling compares
+        // quarter-scale images). Doing that resize here instead means the
+        // bytes never make the round-trip at full resolution. Older SDKs omit
+        // it and get a full-size frame, exactly as before.
+        const target = (data.targetWidth && data.targetHeight)
+          ? { width: data.targetWidth, height: data.targetHeight }
+          : null;
+
         console.log('[automation] Step 1: Capturing screenshot...');
-        const buffer = await this._captureScreenshotBuffer();
+        const buffer = await this._captureScreenshotBuffer(target);
         console.log(`[automation] Step 2: Screenshot captured, size: ${buffer.length} bytes`);
 
         // Read the PNG dimensions from the IHDR header (bytes 16–24) without
@@ -749,8 +880,8 @@ class Automation extends EventEmitter {
     // Propagate Sentry trace context so this request appears under the active command span
     const traceHeaders = Sentry.getTraceData ? Sentry.getTraceData() : {};
 
-    // Get presigned URL from API (30s timeout)
-    const response = await fetch(`${apiRoot}/api/v7/runner/upload-url`, {
+    // Get presigned URL from API (30s per attempt)
+    const response = await fetchWithRetry(`${apiRoot}/api/v7/runner/upload-url`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json', ...traceHeaders },
       body: JSON.stringify({
@@ -758,8 +889,7 @@ class Automation extends EventEmitter {
         sandboxId,
         contentType,
       }),
-      signal: AbortSignal.timeout(30000),
-    });
+    }, { label: 'upload-url' });
 
     if (!response.ok) {
       const text = await response.text();
@@ -769,17 +899,22 @@ class Automation extends EventEmitter {
     const { uploadUrl, s3Key } = await response.json();
     console.log(`[automation] Got S3 upload URL, s3Key: ${s3Key}`);
 
-    // Upload to S3 (30s timeout)
-    const uploadResponse = await fetch(uploadUrl, {
+    // Upload to S3 (30s per attempt). The presigned URL is valid for 5
+    // minutes, so it comfortably outlives the retry budget and can be reused
+    // across attempts rather than re-signed.
+    const uploadResponse = await fetchWithRetry(uploadUrl, {
       method: 'PUT',
       headers: { 'Content-Type': contentType },
       body: buffer,
-      signal: AbortSignal.timeout(30000),
-    });
+    }, { label: 's3-upload' });
 
     if (!uploadResponse.ok) {
-      console.error(`[automation] S3 upload failed: status=${uploadResponse.status}`);
-      throw new Error(`S3 upload failed: ${uploadResponse.status}`);
+      const text = await uploadResponse.text().catch(() => '');
+      console.error(
+        `[automation] S3 upload failed after ${UPLOAD_MAX_ATTEMPTS} attempts: ` +
+        `status=${uploadResponse.status} body=${text.slice(0, 300)}`
+      );
+      throw new Error(`S3 upload failed: ${uploadResponse.status} ${text.slice(0, 300)}`);
     }
 
     console.log(`[automation] S3 upload successful, returning s3Key: ${s3Key}`);
@@ -795,8 +930,15 @@ class Automation extends EventEmitter {
    * PUT to S3): it avoids the wasteful PNG → base64 → PNG round-trip that
    * `_captureScreenshot()` does for backwards compatibility with callers
    * that still expect a base64 string.
+   *
+   * @param {object} [target] - optional final size, applied after the
+   *   physical → logical downscale. The SDK passes this when it is going to
+   *   downscale the frame anyway (redraw stability polling), so the bytes
+   *   never travel at full resolution just to be thrown away on arrival.
+   * @param {number} [target.width]
+   * @param {number} [target.height]
    */
-  async _captureScreenshotBuffer() {
+  async _captureScreenshotBuffer(target = null) {
     const sharp = require('sharp');
     const maxAttempts = 3;
     let lastError;
@@ -839,24 +981,47 @@ class Automation extends EventEmitter {
           SCREENSHOT_TIMEOUT_MS
         );
 
-        // Read the PNG and re-encode with sharp (lossless, no compression)
+        // Read the PNG and re-encode with sharp
         const pngBuffer = fs.readFileSync(tmpFile);
         const image = sharp(pngBuffer);
 
-        // Downscale to logical dimensions when the grab came back at physical
-        // resolution. `fit: 'fill'` forces the exact target size — the default
-        // ('cover') would crop, which would silently shift every coordinate
-        // the agent derives from this image. `stats()` below reads the input
-        // image, so queuing the resize here doesn't affect black detection.
+        // Work out the single final size, then resize once. Two candidates
+        // feed into it:
+        //   1. logical dimensions — when the grab came back at physical
+        //      resolution (Retina / DPI scaling), downscale to what pyautogui
+        //      reports, so coordinates line up with what the agent clicks.
+        //   2. an explicit `target` from the caller — the SDK asking for a
+        //      frame it is only going to shrink anyway.
+        // The caller's target wins when both apply. `fit: 'fill'` forces the
+        // exact size — the default ('cover') would crop, which would silently
+        // shift every coordinate the agent derives from this image. `stats()`
+        // below reads the input image, so queuing the resize here doesn't
+        // affect black detection.
         const [physW, physH, logW, logH] = sizeLine.split(/\s+/).map(Number);
         const haveSizes = [physW, physH, logW, logH].every(
           (n) => Number.isFinite(n) && n > 0
         );
-        if (haveSizes && (physW !== logW || physH !== logH)) {
+
+        const wantW = Number(target?.width);
+        const wantH = Number(target?.height);
+        const haveTarget =
+          Number.isFinite(wantW) && Number.isFinite(wantH) && wantW > 0 && wantH > 0;
+
+        let finalW = null;
+        let finalH = null;
+        if (haveTarget) {
+          finalW = Math.round(wantW);
+          finalH = Math.round(wantH);
+        } else if (haveSizes && (physW !== logW || physH !== logH)) {
+          finalW = logW;
+          finalH = logH;
+        }
+
+        if (finalW && finalH && !(finalW === physW && finalH === physH)) {
           console.log(
-            `[automation] screenshot: downscaling ${physW}x${physH} → ${logW}x${logH}`
+            `[automation] screenshot: resizing ${physW}x${physH} → ${finalW}x${finalH}`
           );
-          image.resize(logW, logH, { kernel: 'lanczos3', fit: 'fill' });
+          image.resize(finalW, finalH, { kernel: 'lanczos3', fit: 'fill' });
         }
 
         // Detect all-black screenshots (Xvfb/compositor issue)
@@ -895,7 +1060,19 @@ class Automation extends EventEmitter {
           }
         }
 
-        return await image.png({ compressionLevel: 0 }).toBuffer();
+        // PNG is lossless at every compression level — the level only trades
+        // encode CPU for bytes on the wire. Level 0 (store, no deflate) was
+        // leaving ~8MB frames to PUT to object storage on every capture,
+        // including the ~10 stability polls a single press-keys triggers.
+        // Screenshots are flat-region-heavy and compress roughly an order of
+        // magnitude at the default level, for tens of ms of libvips CPU
+        // against a capture that already costs hundreds.
+        const out = await image.png({ compressionLevel: PNG_COMPRESSION_LEVEL }).toBuffer();
+        console.log(
+          `[automation] screenshot: encoded ${(out.length / 1024).toFixed(0)}KB ` +
+          `(compressionLevel=${PNG_COMPRESSION_LEVEL})`
+        );
+        return out;
       } catch (err) {
         // Retry capture failures here, not just the all-black case above.
         // Without this a single timeout escapes on the first attempt and the
@@ -1024,4 +1201,13 @@ class Automation extends EventEmitter {
   }
 }
 
-module.exports = { Automation, getDiskSpace, sweepStaleScreenshots };
+module.exports = {
+  Automation,
+  getDiskSpace,
+  sweepStaleScreenshots,
+  // Exported for unit tests — the upload retry policy is the difference
+  // between a throttled object store and a failed command.
+  fetchWithRetry,
+  RETRYABLE_STATUS,
+  PNG_COMPRESSION_LEVEL,
+};

package/lib/upload-retry.test.mjs

@@ -0,0 +1,162 @@
+/**
+ * Retry policy for the runner's screenshot upload.
+ *
+ * The object store (Tigris) answers backpressure with 503 SlowDown, which is
+ * retryable by contract. Before this policy existed a single throttled PUT
+ * failed the whole command — most visibly during press-keys, where redraw
+ * stability polling fires a capture every 500ms.
+ */
+import { describe, it, expect } from "vitest";
+import http from "node:http";
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+const { fetchWithRetry, RETRYABLE_STATUS, PNG_COMPRESSION_LEVEL } = require("./automation.js");
+
+/** Start a throwaway HTTP server and return its URL plus a closer. */
+function serve(handler) {
+  return new Promise((resolve) => {
+    const server = http.createServer(handler);
+    server.listen(0, () => {
+      resolve({
+        url: `http://127.0.0.1:${server.address().port}/`,
+        close: () => new Promise((r) => server.close(r)),
+      });
+    });
+  });
+}
+
+/** Responds with `status` for the first `count` requests, then 200. */
+function failThen(status, count, state) {
+  return (req, res) => {
+    state.attempts++;
+    if (state.attempts <= count) {
+      res.writeHead(status);
+      res.end("<Error><Code>SlowDown</Code></Error>");
+    } else {
+      res.writeHead(200);
+      res.end("ok");
+    }
+  };
+}
+
+describe("fetchWithRetry", () => {
+  it("recovers from transient 503s instead of failing the command", async () => {
+    const state = { attempts: 0 };
+    const server = await serve(failThen(503, 2, state));
+
+    try {
+      const res = await fetchWithRetry(
+        server.url,
+        { method: "PUT", body: Buffer.from("png-bytes") },
+        { label: "s3-upload" },
+      );
+
+      expect(res.ok).toBe(true);
+      expect(state.attempts).toBe(3);
+    } finally {
+      await server.close();
+    }
+  });
+
+  it("re-sends the body intact on every attempt", async () => {
+    // A stream body would be consumed by the first attempt and every retry
+    // would upload zero bytes — silently storing truncated screenshots.
+    const received = [];
+    const server = await serve((req, res) => {
+      let len = 0;
+      req.on("data", (c) => { len += c.length; });
+      req.on("end", () => {
+        received.push(len);
+        if (received.length < 3) { res.writeHead(503); res.end("SlowDown"); }
+        else { res.writeHead(200); res.end("ok"); }
+      });
+    });
+
+    const res = await fetchWithRetry(
+      server.url,
+      { method: "PUT", body: Buffer.alloc(5000, 7) },
+      { label: "s3-upload" },
+    );
+
+    expect(res.ok).toBe(true);
+    expect(received).toEqual([5000, 5000, 5000]);
+    await server.close();
+  });
+
+  it("does not retry a non-retryable status", async () => {
+    // A 403 means the presigned URL is wrong — retrying only delays the error.
+    const state = { attempts: 0 };
+    const server = await serve((req, res) => {
+      state.attempts++;
+      res.writeHead(403);
+      res.end("denied");
+    });
+
+    const res = await fetchWithRetry(server.url, { method: "PUT" }, { label: "s3-upload" });
+
+    expect(res.ok).toBe(false);
+    expect(res.status).toBe(403);
+    expect(state.attempts).toBe(1);
+    await server.close();
+  });
+
+  it("surfaces the store's error body so throttles stay diagnosable", async () => {
+    const server = await serve((req, res) => {
+      res.writeHead(503);
+      res.end("<Error><Code>SlowDown</Code></Error>");
+    });
+
+    const res = await fetchWithRetry(
+      server.url,
+      { method: "PUT" },
+      { label: "s3-upload", deadlineMs: 1500 },
+    );
+
+    expect(res.ok).toBe(false);
+    expect(res.status).toBe(503);
+    expect(await res.text()).toContain("SlowDown");
+    await server.close();
+  });
+
+  it("stops retrying at the deadline so it fits inside the command timeout", async () => {
+    const state = { attempts: 0 };
+    const server = await serve((req, res) => {
+      state.attempts++;
+      res.writeHead(503);
+      res.end("SlowDown");
+    });
+
+    const startedAt = Date.now();
+    const res = await fetchWithRetry(
+      server.url,
+      { method: "PUT" },
+      { label: "s3-upload", deadlineMs: 1200 },
+    );
+    const elapsed = Date.now() - startedAt;
+
+    expect(res.ok).toBe(false);
+    // Gave up early rather than burning the full 4-attempt budget.
+    expect(state.attempts).toBeLessThan(4);
+    expect(elapsed).toBeLessThan(1200);
+    await server.close();
+  });
+
+  it("treats the statuses an object store uses for backpressure as retryable", () => {
+    for (const status of [429, 500, 502, 503, 504]) {
+      expect(RETRYABLE_STATUS.has(status)).toBe(true);
+    }
+    for (const status of [400, 403, 404]) {
+      expect(RETRYABLE_STATUS.has(status)).toBe(false);
+    }
+  });
+});
+
+describe("screenshot encoding", () => {
+  it("compresses PNGs rather than storing them raw", () => {
+    // Level 0 is store-only: it was putting ~8MB frames on the wire for every
+    // capture, including ~10 stability polls per press-keys.
+    expect(PNG_COMPRESSION_LEVEL).toBeGreaterThan(0);
+    expect(PNG_COMPRESSION_LEVEL).toBeLessThanOrEqual(9);
+  });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testdriverai/runner",
-  "version": "7.11.62-canary",
+  "version": "7.11.63-canary",
   "description": "TestDriver Runner - Ably-based remote automation agent with Node.js automation",
   "main": "index.js",
   "bin": {