@testdriverai/runner 7.11.27-test → 7.11.28-canary

package/index.js

@@ -35,7 +35,7 @@ const fs = require('fs');
 const Ably = require('ably');
 const environments = require('../shared/environments.json');
 
-const { Automation } = require('./lib/automation');
+const { Automation, getDiskSpace } = require('./lib/automation');
 const { AblyService } = require('./lib/ably-service');
 
 /**
@@ -338,11 +338,19 @@ class PresenceRunner {
     this.heartbeatInterval = setInterval(async () => {
       if (this.shuttingDown) return;
       try {
+        // Free space rides along on the heartbeat so every runner reports its
+        // own disk continuously — each machine has its own volume size and
+        // whatever the customer has installed or downloaded onto it, so a
+        // one-off check tells you nothing about the next session. statfs is a
+        // sub-millisecond syscall; omitted entirely when unavailable so
+        // "unknown" is never mistaken for "0 bytes free".
+        const disk = getDiskSpace();
         await this.runnerChannel.presence.update({
           ...this.presenceData,
           status: this.currentSession ? 'busy' : 'available',
           sandboxId: this.currentSession?.sandboxId || null,
           lastSeen: Date.now(),
+          ...(disk ? { freeBytes: disk.freeBytes, freePct: disk.freePct } : {}),
         });
       } catch (err) {
         log(`Warning: Heartbeat update failed: ${err.message}`);

package/lib/automation.js

@@ -72,6 +72,81 @@ function readPngDimensions(buffer) {
   };
 }
 
+// Log any pyautogui call that takes longer than this. Every call pays a fresh
+// interpreter start plus the `import pyautogui` graph, so the normal range is
+// tens to low-hundreds of ms — anything past this is drift worth seeing before
+// it grows into a timeout.
+const SLOW_PY_MS = 2000;
+
+/**
+ * Free space on the volume holding `dir`, in bytes.
+ *
+ * `fs.statfsSync` is built in and works on Windows as well as Linux/macOS, so
+ * this stays a sub-millisecond syscall — cheap enough to run on a heartbeat
+ * and in the screenshot preflight. Returns null if the platform or the path
+ * won't answer, so callers treat "unknown" differently from "full".
+ * @param {string} [dir=os.tmpdir()]
+ * @returns {{ freeBytes: number, totalBytes: number, freePct: number } | null}
+ */
+function getDiskSpace(dir = os.tmpdir()) {
+  try {
+    const { bsize, blocks, bavail } = fs.statfsSync(dir);
+    const totalBytes = blocks * bsize;
+    const freeBytes = bavail * bsize;
+    return {
+      freeBytes,
+      totalBytes,
+      freePct: totalBytes > 0 ? Math.round((freeBytes / totalBytes) * 100) : 0,
+    };
+  } catch {
+    return null;
+  }
+}
+
+// Screenshot capture budget. 20s was too tight for a cold interpreter on a
+// loaded Windows sandbox: the subprocess got SIGTERM'd and the caller saw a
+// bare "Command failed" with no stderr. Generous here, since a real hang is
+// now retried locally rather than costing the SDK four full-length attempts.
+const SCREENSHOT_TIMEOUT_MS = 45000;
+
+// Screenshots write a full-size PNG to the temp volume on every capture. Below
+// this much free space that write is at risk, and a stalled or failed write
+// looks exactly like a capture timeout from the caller's side.
+const LOW_DISK_BYTES = 500 * 1024 * 1024;
+
+// Only sweep screenshot temp files older than this, so a concurrent capture's
+// in-flight file is never deleted out from under it.
+const STALE_TMP_MS = 10 * 60 * 1000;
+
+/**
+ * Delete leftover td_screenshot_* files older than STALE_TMP_MS.
+ *
+ * The capture path unlinks its own temp file in a `finally`, but that doesn't
+ * run if the runner process itself is killed mid-capture — which is exactly
+ * what happens on the failure paths we're chasing, so they leak precisely when
+ * disk is already tight.
+ * @returns {{ removed: number, bytes: number }}
+ */
+function sweepStaleScreenshots() {
+  let removed = 0;
+  let bytes = 0;
+  const dir = os.tmpdir();
+  try {
+    for (const name of fs.readdirSync(dir)) {
+      if (!name.startsWith('td_screenshot_') || !name.endsWith('.png')) continue;
+      const file = path.join(dir, name);
+      try {
+        const stat = fs.statSync(file);
+        if (Date.now() - stat.mtimeMs < STALE_TMP_MS) continue;
+        fs.unlinkSync(file);
+        removed++;
+        bytes += stat.size;
+      } catch {}
+    }
+  } catch {}
+  return { removed, bytes };
+}
+
 /**
  * Run a pyautogui Python script via subprocess.
  * @param {string} script — Python code (pyautogui + sys are already imported via PY_IMPORT prefix)
@@ -86,15 +161,34 @@ function runPyAutoGUI(script, extraArgs = [], timeout = 15000) {
     if (IS_LINUX) {
       env.DISPLAY = env.DISPLAY || ':0';
     }
+    // First line only — enough to identify the op in timing logs without
+    // pasting the whole program into every message.
+    const label = script.split('\n', 1)[0].slice(0, 60);
+    const startedAt = Date.now();
     execFile(PYTHON, ['-c', PY_IMPORT + script, ...extraArgs.map(String)], {
       encoding: 'utf-8',
       timeout,
       env,
     }, (err, stdout, stderr) => {
+      const elapsed = Date.now() - startedAt;
       if (err) {
+        // execFile kills the child when `timeout` elapses, surfacing as
+        // killed/SIGTERM with an empty stderr and a generic "Command failed"
+        // message. That is indistinguishable from a Python crash unless we
+        // name it, so say plainly that it was a timeout and how long it ran.
+        if (err.killed || err.signal) {
+          reject(new Error(
+            `pyautogui timed out after ${elapsed}ms (limit ${timeout}ms, ` +
+            `signal ${err.signal || 'none'}, no stderr) — ${label}`
+          ));
+          return;
+        }
         const msg = (stderr || '').trim() || err.message;
-        reject(new Error(`pyautogui command failed: ${msg}`));
+        reject(new Error(`pyautogui command failed after ${elapsed}ms: ${msg}`));
       } else {
+        if (elapsed > SLOW_PY_MS) {
+          console.warn(`[automation] slow pyautogui: ${elapsed}ms — ${label}`);
+        }
         resolve((stdout || '').trim());
       }
     });
@@ -705,29 +799,66 @@ class Automation extends EventEmitter {
   async _captureScreenshotBuffer() {
     const sharp = require('sharp');
     const maxAttempts = 3;
+    let lastError;
+
+    // Preflight: a temp volume with no room turns the PNG write below into a
+    // stall or a failure that reads like a capture timeout. Check first so it
+    // fails loudly, and reclaim any leaked captures while we're here.
+    const disk = getDiskSpace();
+    if (disk && disk.freeBytes < LOW_DISK_BYTES) {
+      const freeMb = Math.round(disk.freeBytes / 1024 / 1024);
+      console.warn(
+        `[automation] LOW DISK: ${freeMb}MB free (${disk.freePct}%) on ${os.tmpdir()} — ` +
+        `screenshot writes may fail`
+      );
+      const swept = sweepStaleScreenshots();
+      if (swept.removed > 0) {
+        console.warn(
+          `[automation] swept ${swept.removed} stale screenshot temp files ` +
+          `(${Math.round(swept.bytes / 1024 / 1024)}MB reclaimed)`
+        );
+      }
+    }
 
     for (let attempt = 1; attempt <= maxAttempts; attempt++) {
       const tmpFile = path.join(os.tmpdir(), `td_screenshot_${Date.now()}.png`);
 
       try {
-        // Capture screenshot via pyautogui → saves to temp file
-        // Python handles Retina downscale: if physical size differs from logical,
-        // the image is resized to logical dimensions before saving.
-        await runPyAutoGUI(
+        // Capture screenshot via pyautogui → saves to temp file.
+        // Python only captures, saves, and reports sizes: physical (what the
+        // grab returned) then logical (what pyautogui reports). Any Retina /
+        // DPI-scaling downscale is done by sharp below — libvips is
+        // multithreaded, where PIL's LANCZOS resample is a single-threaded
+        // pass over a full-size frame inside the subprocess timeout budget.
+        const sizeLine = await runPyAutoGUI(
           'img = pyautogui.screenshot()\n' +
+          'img.save(sys.argv[1], format="PNG")\n' +
           'logical = pyautogui.size()\n' +
-          'if img.size[0] != logical[0] or img.size[1] != logical[1]:\n' +
-          '    from PIL import Image\n' +
-          '    img = img.resize((logical[0], logical[1]), Image.LANCZOS)\n' +
-          'img.save(sys.argv[1], format="PNG")',
+          'print(img.size[0], img.size[1], logical[0], logical[1])',
           [tmpFile],
-          20000
+          SCREENSHOT_TIMEOUT_MS
         );
 
         // Read the PNG and re-encode with sharp (lossless, no compression)
         const pngBuffer = fs.readFileSync(tmpFile);
         const image = sharp(pngBuffer);
 
+        // Downscale to logical dimensions when the grab came back at physical
+        // resolution. `fit: 'fill'` forces the exact target size — the default
+        // ('cover') would crop, which would silently shift every coordinate
+        // the agent derives from this image. `stats()` below reads the input
+        // image, so queuing the resize here doesn't affect black detection.
+        const [physW, physH, logW, logH] = sizeLine.split(/\s+/).map(Number);
+        const haveSizes = [physW, physH, logW, logH].every(
+          (n) => Number.isFinite(n) && n > 0
+        );
+        if (haveSizes && (physW !== logW || physH !== logH)) {
+          console.log(
+            `[automation] screenshot: downscaling ${physW}x${physH} → ${logW}x${logH}`
+          );
+          image.resize(logW, logH, { kernel: 'lanczos3', fit: 'fill' });
+        }
+
         // Detect all-black screenshots (Xvfb/compositor issue)
         if (IS_LINUX) {
           const { channels } = await image.stats();
@@ -765,11 +896,28 @@ class Automation extends EventEmitter {
         }
 
         return await image.png({ compressionLevel: 0 }).toBuffer();
+      } catch (err) {
+        // Retry capture failures here, not just the all-black case above.
+        // Without this a single timeout escapes on the first attempt and the
+        // SDK retries it at full cost — four capture timeouts back to back
+        // instead of one cheap local retry.
+        lastError = err;
+        console.warn(
+          `[automation] Screenshot attempt ${attempt}/${maxAttempts} failed: ${err.message}`
+        );
+        if (attempt < maxAttempts) {
+          await new Promise((r) => setTimeout(r, 1000 * attempt));
+          continue;
+        }
+        throw err;
       } finally {
         // Clean up temp file
         try { fs.unlinkSync(tmpFile); } catch {}
       }
     }
+
+    // Only reachable if the loop exits without returning or throwing.
+    throw lastError || new Error('Screenshot failed: no attempts produced an image');
   }
 
   async _captureScreenshot() {
@@ -876,4 +1024,4 @@ class Automation extends EventEmitter {
   }
 }
 
-module.exports = { Automation };
+module.exports = { Automation, getDiskSpace, sweepStaleScreenshots };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testdriverai/runner",
-  "version": "7.11.27-test",
+  "version": "7.11.28-canary",
   "description": "TestDriver Runner - Ably-based remote automation agent with Node.js automation",
   "main": "index.js",
   "bin": {