@testdriverai/runner 7.11.188-test → 7.11.189-canary

package/lib/ably-service.js

@@ -25,6 +25,15 @@ const https = require('https');
 const { EventEmitter } = require('events');
 const { maskCommandPayload } = require('./mask-command');
 
+// Coalescing ceiling for queued exec.output chunks. Ably's hard message limit
+// is 64KB; stay well under it to leave room for the JSON envelope + escaping.
+const EXEC_MESSAGE_MAX_BYTES = 48 * 1024;
+
+// Debug log batching: one message per interval, with a cap on how many lines
+// can pile up between flushes.
+const RUNNER_LOG_FLUSH_MS = 250;
+const RUNNER_LOG_MAX_PENDING = 200;
+
 /**
  * Get the local runner version from package.json.
  */
@@ -206,6 +215,19 @@ class AblyService extends EventEmitter {
     this._statsInterval = null;
     this._closed = false; // set to true in close() to suppress post-close events
     this._inFlight = new Map(); // requestId → { type, startedAt }; commands currently executing
+
+    // Ably enforces `channel.maxRate` (50 msg/sec) across *every* publisher on
+    // the session channel — the SDK and the API publish here too. The runner
+    // takes a 20 msg/sec slice and funnels all of its publishes through one
+    // queue so bursts of responses/logs/exec output can't blow the budget.
+    this._publishQueue = [];
+    this._publishDraining = false;
+    this._publishLastTime = 0;
+    this._publishMinIntervalMs = Number(process.env.TD_RUNNER_PUBLISH_INTERVAL_MS) || 50;
+
+    // Coalesced runner.log batching (debug mode only).
+    this._pendingLogs = [];
+    this._logFlushTimer = null;
   }
 
   /**
@@ -275,48 +297,25 @@ class AblyService extends EventEmitter {
       this.emit('log', `[stats] connection=${connState} | sandbox=${this._sandboxId} | channel=${chState}`);
     }, 10000);
 
-    // Forward debug logs from automation to SDK (only when debug mode is enabled)
+    // Forward debug logs from automation to SDK (only when debug mode is enabled).
+    // Log lines are coalesced per level so a burst becomes one message.
     this._debugMode = false;
-    this._automation.on('log', (message) => {
+    const forwardLog = (level) => (message) => {
       if (!this._debugMode) return;
-      this._sendResponse({
-        type: 'runner.log',
-        level: 'info',
-        message,
-        timestamp: Date.now(),
-      }).catch(() => { }); // best-effort
-    });
-    this._automation.on('warn', (message) => {
-      if (!this._debugMode) return;
-      this._sendResponse({
-        type: 'runner.log',
-        level: 'warn',
-        message,
-        timestamp: Date.now(),
-      }).catch(() => { }); // best-effort
-    });
-    this._automation.on('error', (message) => {
-      if (!this._debugMode) return;
-      this._sendResponse({
-        type: 'runner.log',
-        level: 'error',
-        message: typeof message === 'string' ? message : message.message || String(message),
-        timestamp: Date.now(),
-      }).catch(() => { }); // best-effort
-    });
-
-    // Forward exec streaming chunks to SDK with rate limiting.
-    // Exec output can produce many chunks rapidly (e.g. verbose commands);
-    // throttle to avoid hitting Ably's 50 msg/sec per-connection limit.
-    this._execOutputLastTime = 0;
-    this._execOutputMinIntervalMs = 50; // 20 msg/sec max for exec.output
-    this._execOutputQueue = [];         // queued chunks waiting to send
-    this._execOutputDraining = false;
+      this._sendRunnerLog(level, message);
+    };
+    this._automation.on('log', forwardLog('info'));
+    this._automation.on('warn', forwardLog('warn'));
+    this._automation.on('error', forwardLog('error'));
 
     this._automation.on('exec.output', ({ requestId, chunk }) => {
       if (this._closed) return; // don't enqueue once we're tearing down
-      this._execOutputQueue.push({ requestId, chunk });
-      this._drainExecOutputQueue();
+      this._sendResponse({
+        type: 'exec.output',
+        requestId,
+        chunk,
+        timestamp: Date.now(),
+      }).catch(() => { }); // best-effort
     });
 
     // Subscribe to commands — save subscription ref for historyBeforeSubscribe()
@@ -560,7 +559,7 @@ class AblyService extends EventEmitter {
         remoteVersion: this._updateInfo.remoteVersion,
       };
     }
-    await this._sessionChannel.publish('control', readyPayload);
+    await this._enqueuePublish('control', readyPayload);
     this.emit('log', 'Published runner.ready signal');
 
     // Re-publish runner.ready every 3s for up to 60s.
@@ -569,8 +568,7 @@ class AblyService extends EventEmitter {
     // the SDK catches at least one live runner.ready message.
     this._readyInterval = setInterval(async () => {
       try {
-        readyPayload.timestamp = Date.now();
-        await this._sessionChannel.publish('control', readyPayload);
+        await this._enqueuePublish('control', { ...readyPayload, timestamp: Date.now() });
         this.emit('log', 'Re-published runner.ready signal');
       } catch (err) {
         this.emit('log', `Failed to re-publish runner.ready: ${err.message}`);
@@ -598,62 +596,115 @@ class AblyService extends EventEmitter {
   }
 
   /**
-   * Drain the exec.output queue, respecting the rate limit interval.
-   * Coalesces queued chunks per-requestId into single messages to reduce
-   * message count when output arrives faster than we can send.
+   * Queue a publish on the session channel. All runner publishes go through
+   * here so the runner never exceeds its slice of the shared channel rate.
+   *
+   * Adjacent exec.output chunks for the same requestId are coalesced while
+   * they sit in the queue, so a burst of output costs one message instead of
+   * many (bounded by Ably's 64KB message limit).
+   */
+  _enqueuePublish(eventName, message) {
+    if (this._closed) return Promise.resolve();
+
+    if (message.type === 'exec.output') {
+      const tail = this._publishQueue[this._publishQueue.length - 1];
+      if (
+        tail &&
+        tail.message.type === 'exec.output' &&
+        tail.message.requestId === message.requestId &&
+        tail.message.chunk.length + message.chunk.length <= EXEC_MESSAGE_MAX_BYTES
+      ) {
+        tail.message.chunk += message.chunk;
+        return Promise.resolve();
+      }
+    }
+
+    return new Promise((resolve) => {
+      this._publishQueue.push({ eventName, message, resolve });
+      this._drainPublishQueue();
+    });
+  }
+
+  /**
+   * Drain the publish queue at the configured interval.
    */
-  async _drainExecOutputQueue() {
-    if (this._execOutputDraining) return; // already draining
-    this._execOutputDraining = true;
+  async _drainPublishQueue() {
+    if (this._publishDraining) return; // already draining
+    this._publishDraining = true;
 
     try {
-      while (this._execOutputQueue.length > 0) {
+      while (this._publishQueue.length > 0) {
         // Stop draining once close() has started tearing down the channel —
         // a publish landing after detach() can make Ably silently
         // re-attach the channel, doubling channel-open/close lifecycle
         // events for what should be one clean session teardown.
-        if (this._closed) {
-          this._execOutputQueue.length = 0;
-          break;
-        }
-
-        // Rate limit: wait if needed
-        const now = Date.now();
-        const elapsed = now - this._execOutputLastTime;
-        if (elapsed < this._execOutputMinIntervalMs) {
-          await new Promise((resolve) => setTimeout(resolve, this._execOutputMinIntervalMs - elapsed));
-        }
-
-        if (this._closed) {
-          this._execOutputQueue.length = 0;
-          break;
-        }
+        if (this._closed) break;
 
-        // Coalesce all queued chunks for the same requestId
-        const batch = {};
-        while (this._execOutputQueue.length > 0) {
-          const { requestId, chunk } = this._execOutputQueue.shift();
-          if (!batch[requestId]) batch[requestId] = '';
-          batch[requestId] += chunk;
+        const elapsed = Date.now() - this._publishLastTime;
+        if (elapsed < this._publishMinIntervalMs) {
+          await new Promise((resolve) => setTimeout(resolve, this._publishMinIntervalMs - elapsed));
         }
+        if (this._closed) break;
 
-        this._execOutputLastTime = Date.now();
+        const item = this._publishQueue.shift();
+        if (!item) break;
+        this._publishLastTime = Date.now();
 
-        // Send one message per requestId
-        for (const [requestId, chunk] of Object.entries(batch)) {
-          this._sendResponse({
-            type: 'exec.output',
-            requestId,
-            chunk,
-            timestamp: Date.now(),
-          }).catch(() => { }); // best-effort
+        try {
+          this.emit('log', `Publishing ${item.eventName}: type=${item.message.type || 'unknown'} (requestId=${item.message.requestId || 'none'})`);
+          await this._sessionChannel.publish(item.eventName, item.message);
+        } catch (err) {
+          this.emit('log', `Failed to publish ${item.eventName}: ${err.message}`);
         }
+        item.resolve();
       }
     } finally {
-      this._execOutputDraining = false;
+      this._publishDraining = false;
+      if (this._closed) {
+        for (const item of this._publishQueue) item.resolve();
+        this._publishQueue.length = 0;
+      }
     }
   }
 
+  /**
+   * Queue a runner.log line. Lines are batched into a single message per
+   * flush interval so verbose debug output can't swamp the channel budget.
+   */
+  _sendRunnerLog(level, message) {
+    const text = typeof message === 'string' ? message : (message && message.message) || String(message);
+
+    if (this._pendingLogs.length >= RUNNER_LOG_MAX_PENDING) {
+      this._pendingLogs[this._pendingLogs.length - 1] = {
+        level: 'warn',
+        message: `[runner.log dropped — more than ${RUNNER_LOG_MAX_PENDING} lines queued]`,
+        timestamp: Date.now(),
+      };
+      return;
+    }
+
+    this._pendingLogs.push({ level, message: text, timestamp: Date.now() });
+
+    if (this._logFlushTimer) return;
+    this._logFlushTimer = setTimeout(() => {
+      this._logFlushTimer = null;
+      this._flushRunnerLogs();
+    }, RUNNER_LOG_FLUSH_MS);
+    if (this._logFlushTimer.unref) this._logFlushTimer.unref();
+  }
+
+  _flushRunnerLogs() {
+    if (this._closed || this._pendingLogs.length === 0) return;
+    const lines = this._pendingLogs;
+    this._pendingLogs = [];
+    this._sendResponse({
+      type: 'runner.log',
+      level: lines.some((l) => l.level === 'error') ? 'error' : lines[0].level,
+      message: lines.map((l) => l.message).join('\n'),
+      timestamp: Date.now(),
+    }).catch(() => { }); // best-effort
+  }
+
   /**
    * Recover missed messages after a channel discontinuity.
    * Uses historyBeforeSubscribe() on each subscription, which guarantees
@@ -707,13 +758,7 @@ class AblyService extends EventEmitter {
       this.emit('log', 'Warning: session channel not available');
       return;
     }
-
-    try {
-      this.emit('log', `Publishing response: type=${message.type || 'unknown'} (requestId=${message.requestId || 'none'})`);
-      await this._sessionChannel.publish('response', message);
-    } catch (err) {
-      this.emit('log', `Failed to publish response: ${err.message}`);
-    }
+    await this._enqueuePublish('response', message);
   }
 
   /**
@@ -721,13 +766,7 @@ class AblyService extends EventEmitter {
    */
   async sendFile(message) {
     if (!this._sessionChannel) return;
-
-    try {
-      this.emit('log', `Publishing file: type=${message.type || 'unknown'} (requestId=${message.requestId || 'none'})`);
-      await this._sessionChannel.publish('file', message);
-    } catch (err) {
-      this.emit('log', `Failed to publish file: ${err.message}`);
-    }
+    await this._enqueuePublish('file', message);
   }
 
   /**
@@ -735,13 +774,7 @@ class AblyService extends EventEmitter {
    */
   async sendControl(message) {
     if (!this._sessionChannel) return;
-
-    try {
-      this.emit('log', `Publishing control: type=${message.type || 'unknown'}`);
-      await this._sessionChannel.publish('control', message);
-    } catch (err) {
-      this.emit('log', `Failed to publish control: ${err.message}`);
-    }
+    await this._enqueuePublish('control', message);
   }
 
   /**
@@ -767,6 +800,14 @@ class AblyService extends EventEmitter {
       this._statsInterval = null;
     }
 
+    if (this._logFlushTimer) {
+      clearTimeout(this._logFlushTimer);
+      this._logFlushTimer = null;
+    }
+    this._pendingLogs = [];
+    for (const item of this._publishQueue) item.resolve();
+    this._publishQueue.length = 0;
+
     // Null out subscription refs so _recoverFromDiscontinuity won't replay
     // messages on a channel that is about to be torn down
     this._commandSubscription = null;

package/lib/automation.js

@@ -37,6 +37,20 @@ if (IS_LINUX && !process.env.DISPLAY) {
 const API_ROOT = process.env.TD_API_ROOT || process.env.API_ROOT || 'https://api.testdriver.ai';
 const API_KEY = process.env.TD_API_KEY;
 
+// ─── Exec output limits ──────────────────────────────────────────────────────
+// Flush size keeps each streamed message well under Ably's 64KB hard limit
+// (leaves room for the JSON envelope + string escaping). The total cap stops a
+// runaway command (`tail -f`, an infinite loop) from streaming forever and
+// starving the session channel's shared publish budget.
+const EXEC_CHUNK_FLUSH_SIZE = 32 * 1024;
+const EXEC_OUTPUT_MAX_BYTES = Number(process.env.TD_EXEC_MAX_OUTPUT_BYTES) || 5 * 1024 * 1024;
+const EXEC_STDERR_MAX_BYTES = 256 * 1024;
+
+function truncateText(text, maxBytes) {
+  if (typeof text !== 'string' || text.length <= maxBytes) return text;
+  return text.slice(0, maxBytes) + `\n[truncated — exceeded ${maxBytes} bytes]`;
+}
+
 // ─── pyautogui via Python subprocess ─────────────────────────────────────────
 // Calls pyautogui functions via `python3 -c` (or `python -c` on Windows).
 // Each command spawns a short-lived Python process. The pyautogui library is
@@ -422,6 +436,7 @@ class ShellSession {
   constructor() {
     this._process = null;
     this._stderrLines = [];
+    this._stderrBytes = 0;
     this._lock = false;
   }
 
@@ -444,7 +459,10 @@ class ShellSession {
     });
 
     this._process.stderr.on('data', (chunk) => {
-      this._stderrLines.push(chunk.toString());
+      if (this._stderrBytes >= EXEC_STDERR_MAX_BYTES) return;
+      const text = chunk.toString();
+      this._stderrLines.push(text);
+      this._stderrBytes += text.length;
     });
 
     this._process.on('error', (err) => {
@@ -484,6 +502,7 @@ class ShellSession {
 
       // Clear stderr buffer
       this._stderrLines = [];
+      this._stderrBytes = 0;
 
       // Capture the process reference now — the exit handler may set
       // this._process to null before the Promise resolves/times out.
@@ -514,6 +533,7 @@ class ShellSession {
         }, timeoutMs);
 
         const stdoutLines = [];
+        let stdoutBytes = 0;
 
         const onData = (chunk) => {
           const text = chunk.toString();
@@ -531,7 +551,12 @@ class ShellSession {
               });
               return;
             }
-            stdoutLines.push(line);
+            // Keep the in-memory buffer bounded; a runaway command would
+            // otherwise grow it until the runner OOMs.
+            if (stdoutBytes < EXEC_OUTPUT_MAX_BYTES) {
+              stdoutLines.push(line);
+              stdoutBytes += line.length + 1;
+            }
           }
 
           // Stream chunk to caller if callback provided
@@ -891,15 +916,13 @@ class Automation extends EventEmitter {
         const timeout = Math.ceil((data.timeout || 300000) / 1000); // ms to seconds
         const requestId = data.requestId;
 
-        // Buffer stdout chunks to ~32KB before emitting over Ably.
-        // This reduces message count while keeping each message well under
-        // Ably's 64KB limit. 32KB leaves headroom for the JSON envelope +
-        // string escaping while halving the message count vs the previous
-        // 16KB size, helping avoid Ably's per-channel rate limit on verbose
-        // commands. The SDK accumulates these chunks and reconstructs the
-        // full stdout — the final response only carries returncode + stderr.
-        const CHUNK_FLUSH_SIZE = 32 * 1024; // 32KB
+        // Buffer stdout chunks before emitting over Ably. The SDK accumulates
+        // these chunks and reconstructs the full stdout — the final response
+        // only carries returncode + stderr.
         let chunkBuffer = '';
+        let streamedBytes = 0;
+        let truncated = false;
+
         const flushChunkBuffer = () => {
           if (chunkBuffer.length > 0) {
             this.emit('exec.output', { requestId, chunk: chunkBuffer });
@@ -907,15 +930,34 @@ class Automation extends EventEmitter {
           }
         };
 
+        // A runaway command (tail -f, a server log, an infinite loop) would
+        // otherwise stream forever and starve the session channel. Stop
+        // forwarding past the cap and tell the SDK once that it was cut off.
+        const onChunk = (text) => {
+          if (truncated) return;
+
+          const remaining = EXEC_OUTPUT_MAX_BYTES - streamedBytes;
+          if (text.length >= remaining) {
+            chunkBuffer += text.slice(0, Math.max(remaining, 0));
+            streamedBytes = EXEC_OUTPUT_MAX_BYTES;
+            truncated = true;
+            flushChunkBuffer();
+            this.emit('exec.output', {
+              requestId,
+              chunk: `\n[output truncated — exceeded ${EXEC_OUTPUT_MAX_BYTES} bytes]\n`,
+            });
+            return;
+          }
+
+          chunkBuffer += text;
+          streamedBytes += text.length;
+          if (chunkBuffer.length >= EXEC_CHUNK_FLUSH_SIZE) {
+            flushChunkBuffer();
+          }
+        };
+
         try {
-          const result = await this._shell.run(data.command, timeout, {
-            onChunk: (text) => {
-              chunkBuffer += text;
-              if (chunkBuffer.length >= CHUNK_FLUSH_SIZE) {
-                flushChunkBuffer();
-              }
-            },
-          });
+          const result = await this._shell.run(data.command, timeout, { onChunk });
 
           // Flush any remaining buffered output
           flushChunkBuffer();
@@ -928,7 +970,7 @@ class Automation extends EventEmitter {
             out: {
               returncode: result.returncode,
               stdout: '',
-              stderr: result.stderr,
+              stderr: truncateText(result.stderr, EXEC_STDERR_MAX_BYTES),
             },
           };
         } catch (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testdriverai/runner",
-  "version": "7.11.188-test",
+  "version": "7.11.189-canary",
   "description": "TestDriver Runner - Ably-based remote automation agent with Node.js automation",
   "main": "index.js",
   "bin": {