@testdriverai/runner 7.8.0-canary.10

package/lib/ably-service.js

@@ -0,0 +1,537 @@
+/**
+ * ably-service.js — Ably-based command listener for sandbox agents
+ *
+ * Runs on the sandbox machine (Windows EC2, self-hosted runner, etc.).
+ * Subscribes to an Ably commands channel, dispatches commands to the
+ * Automation module, and publishes responses back.
+ *
+ * Replaces the WebSocket server in pyautogui-cli.py. Uses a single
+ * Ably channel per session with event-name-based multiplexing:
+ *   - "command" event: receives commands from SDK/API
+ *   - "response" event: publishes command results
+ *   - "control" event: control messages (keepalive, status)
+ *   - "file" event: screenshot/file data (base64 or S3 references)
+ *
+ * The Ably token and channel name are provided by the API during
+ * sandbox provisioning.
+ *
+ * Screenshots are uploaded to S3 and the S3 key is returned instead of
+ * base64 data (Ably has a 64KB message limit).
+ */
+const Ably = require('ably');
+const Sentry = require('@sentry/node');
+const http = require('http');
+const https = require('https');
+const { EventEmitter } = require('events');
+
+/**
+ * Get the local runner version from package.json.
+ */
+function getLocalVersion() {
+  try {
+    if (process.env.RUNNER_VERSION) return process.env.RUNNER_VERSION;
+    const pkg = require('../package.json');
+    return pkg.version;
+  } catch { return null; }
+}
+
+// ─── S3 Upload Helper ────────────────────────────────────────────────────────
+
+/**
+ * Upload a screenshot to S3 via presigned URL from the API
+ * 
+ * @param {string} apiRoot - API base URL
+ * @param {string} apiKey - Team API key
+ * @param {string} sandboxId - Sandbox ID
+ * @param {string} base64 - Base64-encoded image data
+ * @param {string} label - Label for the screenshot (e.g., 'screenshot', 'before', 'after')
+ * @returns {Promise<{ s3Key: string, fileName: string } | null>}
+ */
+async function uploadToS3(apiRoot, apiKey, sandboxId, base64, label = 'screenshot') {
+  if (!base64 || !apiRoot || !apiKey) return null;
+
+  try {
+    const fileName = `${label}-${Date.now()}.png`;
+
+    // Get presigned upload URL from API
+    const uploadUrlResponse = await httpPost(apiRoot, '/api/v7/runner/upload-url', {
+      apiKey,
+      sandboxId,
+      fileName,
+    });
+
+    if (!uploadUrlResponse || !uploadUrlResponse.uploadUrl) {
+      console.warn('[ably-service] No upload URL returned for screenshot');
+      return null;
+    }
+
+    // Upload to S3 via presigned URL
+    const pngBuffer = Buffer.from(base64, 'base64');
+    const url = new URL(uploadUrlResponse.uploadUrl);
+    const transport = url.protocol === 'https:' ? https : http;
+
+    await new Promise((resolve, reject) => {
+      const req = transport.request(url, {
+        method: 'PUT',
+        headers: {
+          'Content-Type': 'image/png',
+          'Content-Length': pngBuffer.length,
+        },
+      }, (res) => {
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          resolve();
+        } else {
+          let data = '';
+          res.on('data', (chunk) => { data += chunk; });
+          res.on('end', () => reject(new Error(`S3 upload failed (${res.statusCode}): ${data}`)));
+        }
+      });
+      req.on('error', reject);
+      req.write(pngBuffer);
+      req.end();
+    });
+
+    return { s3Key: uploadUrlResponse.s3Key, fileName, size: pngBuffer.length };
+  } catch (err) {
+    console.error(`[ably-service] Screenshot upload failed (${label}): ${err.message}`);
+    return null;
+  }
+}
+
+/**
+ * HTTP POST helper
+ */
+async function httpPost(apiRoot, path, body) {
+  const url = new URL(path, apiRoot);
+  const transport = url.protocol === 'https:' ? https : http;
+
+  return new Promise((resolve, reject) => {
+    const req = transport.request(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    }, (res) => {
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => {
+        try {
+          resolve(JSON.parse(data));
+        } catch {
+          resolve(null);
+        }
+      });
+    });
+    req.on('error', reject);
+    req.write(JSON.stringify(body));
+    req.end();
+  });
+}
+
+// ─── Ably Service Class ──────────────────────────────────────────────────────
+
+class AblyService extends EventEmitter {
+  /**
+   * @param {object} options
+   * @param {object} options.automation - Automation instance (has dispatch method)
+   * @param {string} options.ablyToken - Token or tokenRequest from the API
+   * @param {string} options.channel - Single session channel name
+   * @param {string} options.sandboxId - This sandbox's ID
+   * @param {string} [options.clientId] - Ably client ID (default: 'sandbox-{sandboxId}')
+   * @param {string} [options.apiRoot] - API base URL (for S3 uploads)
+   * @param {string} [options.apiKey] - Team API key (for S3 uploads)
+   */
+  constructor(options) {
+    super();
+    this._automation = options.automation;
+    this._ablyToken = options.ablyToken;
+    this._channelName = options.channel;
+    this._sandboxId = options.sandboxId;
+    this._clientId = options.clientId || `agent-${options.sandboxId}`;
+    this._apiRoot = options.apiRoot;
+    this._apiKey = options.apiKey;
+    this._updateInfo = options.updateInfo || null;
+
+    this._ably = null;
+    this._sessionChannel = null;
+    this._connected = false;
+    this._statsInterval = null;
+  }
+
+  /**
+   * Connect to Ably and start listening for commands.
+   */
+  async connect() {
+    const self = this; // Capture this for use in callbacks
+    this.emit('log', `Connecting to Ably as ${this._clientId}...`);
+
+    this._ably = new Ably.Realtime({
+      authCallback: (tokenParams, callback) => {
+        callback(null, this._ablyToken);
+      },
+      clientId: this._clientId,
+      disconnectedRetryTimeout: 5000,   // retry reconnect every 5s (default 15s)
+      suspendedRetryTimeout: 15000,     // retry from suspended every 15s (default 30s)
+      logHandler: (msg) => {
+        const text = typeof msg === 'string' ? msg : String(msg);
+        const isError = /error/i.test(text) || /\[Level 1\]/i.test(text);
+        if (isError) {
+          Sentry.withScope((scope) => {
+            scope.setTag('ably.client', 'runner');
+            scope.setTag('sandbox.id', self._sandboxId);
+            scope.setContext('ably_log', { raw: text.slice(0, 2000) });
+            Sentry.captureMessage(`[Ably runner] ${text.slice(0, 500)}`, 'error');
+          });
+        }
+        Sentry.addBreadcrumb({
+          category: 'ably.log',
+          message: text.slice(0, 1000),
+          level: isError ? 'error' : 'info',
+          data: { client: 'runner', sandboxId: self._sandboxId },
+        });
+      },
+      logLevel: 2,
+    });
+
+    // Wait for connection
+    await new Promise((resolve, reject) => {
+      this._ably.connection.on('connected', () => {
+        this._connected = true;
+        resolve();
+      });
+      this._ably.connection.on('failed', () => {
+        reject(new Error('Ably connection failed'));
+      });
+      setTimeout(() => {
+        reject(new Error('Ably connection timeout (30s)'));
+      }, 30000);
+    });
+
+    this.emit('log', 'Ably connected');
+
+    // Get session channel
+    this._sessionChannel = this._ably.channels.get(this._channelName);
+
+    this.emit('log', `Ably session channel initialized: ${this._channelName}`);
+
+    // ─── Periodic stats logging ────────────────────────────────────────────
+    this._statsInterval = setInterval(() => {
+      const connState = this._ably ? this._ably.connection.state : 'no-client';
+      const chState = this._sessionChannel ? this._sessionChannel.state : 'null';
+      this.emit('log', `[stats] connection=${connState} | sandbox=${this._sandboxId} | channel=${chState}`);
+    }, 10000);
+
+    // Forward debug logs from automation to SDK (only when debug mode is enabled)
+    this._debugMode = false;
+    this._automation.on('log', (message) => {
+      if (!this._debugMode) return;
+      this._sendResponse({
+        type: 'runner.log',
+        level: 'info',
+        message,
+        timestamp: Date.now(),
+      }).catch(() => {}); // best-effort
+    });
+    this._automation.on('warn', (message) => {
+      if (!this._debugMode) return;
+      this._sendResponse({
+        type: 'runner.log',
+        level: 'warn',
+        message,
+        timestamp: Date.now(),
+      }).catch(() => {}); // best-effort
+    });
+    this._automation.on('error', (message) => {
+      if (!this._debugMode) return;
+      this._sendResponse({
+        type: 'runner.log',
+        level: 'error',
+        message: typeof message === 'string' ? message : message.message || String(message),
+        timestamp: Date.now(),
+      }).catch(() => {}); // best-effort
+    });
+
+    // Forward exec streaming chunks to SDK
+    this._automation.on('exec.output', ({ requestId, chunk }) => {
+      this._sendResponse({
+        type: 'exec.output',
+        requestId,
+        chunk,
+        timestamp: Date.now(),
+      }).catch(() => {}); // best-effort, don't block exec
+    });
+
+    // Subscribe to commands
+    this._sessionChannel.subscribe('command', async (msg) => {
+      const message = msg.data;
+      if (!message) return;
+
+      const requestId = message.requestId;
+      const type = message.type;
+
+      this.emit('log', `Command received: ${type} (requestId=${requestId})`);
+
+      // Per-command timeout: use message.timeout if provided, else default 120s
+      // Prevents hanging forever if screenshot capture or S3 upload stalls
+      const commandTimeout = (message.timeout && message.timeout > 0)
+        ? message.timeout + 5000  // Add 5s buffer over SDK-side timeout
+        : 120000;
+
+      // Wrap dispatch in Sentry distributed trace context
+      const sentryTrace = message.sentryTrace;
+      const baggage = message.baggage;
+      const executeCommand = async () => {
+        try {
+          let result = await Promise.race([
+            this._automation.dispatch(type, message),
+            new Promise((_, reject) =>
+              setTimeout(() => reject(new Error(
+                `Command '${type}' timed out on runner after ${commandTimeout}ms`
+              )), commandTimeout)
+            ),
+          ]);
+
+          // Screenshots are now handled by automation.js (returns { s3Key })
+          // No need to check type here - just pass through the result
+          
+          await this._sendResponse({
+            requestId,
+            type: `${type}.reply`,
+            result,
+            success: true,
+          });
+          this.emit('log', `Command completed: ${type} (requestId=${requestId})`);
+        } catch (err) {
+          this.emit('log', `Command failed: ${type} — ${err.message}`);
+          Sentry.captureException(err);
+          await this._sendResponse({
+            requestId,
+            type: `${type}.reply`,
+            error: true,
+            errorMessage: err.message,
+            success: false,
+          });
+        }
+      };
+
+      if (sentryTrace) {
+        await Sentry.continueTrace({ sentryTrace, baggage }, () => {
+          return Sentry.startSpan(
+            { name: `runner.command.${type}`, op: 'runner.dispatch' },
+            executeCommand,
+          );
+        });
+      } else {
+        await executeCommand();
+      }
+    });
+
+    // ─── Ably connection state monitoring → Sentry ─────────────────────────
+    this._ably.connection.on((stateChange) => {
+      const { current, previous, reason, retryIn } = stateChange;
+      const reasonMsg = reason ? (reason.message || reason.code || String(reason)) : undefined;
+
+      Sentry.addBreadcrumb({
+        category: 'ably.connection',
+        message: `runner: ${previous} → ${current}${reasonMsg ? ' — ' + reasonMsg : ''}`,
+        level: current === 'connected' ? 'info' : 'warning',
+        data: { client: 'runner', from: previous, to: current, reason: reasonMsg, retryIn, sandboxId: this._sandboxId },
+      });
+
+      // Preserve original behavior
+      if (current === 'disconnected') {
+        this._connected = false;
+        this.emit('log', `Ably connection: ${previous} → ${current}${reasonMsg ? ' — ' + reasonMsg : ''}${retryIn ? ' (retryIn=' + retryIn + 'ms)' : ''}`);
+        this.emit('log', 'Ably disconnected — will auto-reconnect');
+      } else if (current === 'connected' && previous !== 'initialized') {
+        if (!this._connected) {
+          this._connected = true;
+          this.emit('log', `Ably connection: ${previous} → ${current}`);
+          this.emit('log', 'Ably reconnected');
+        }
+      } else if (current === 'failed') {
+        this._connected = false;
+        this.emit('log', `Ably connection: ${previous} → ${current}${reasonMsg ? ' — ' + reasonMsg : ''}`);
+        this.emit('error', new Error('Ably connection failed'));
+      } else if (current === 'suspended') {
+        this._connected = false;
+        this.emit('log', `Ably connection: ${previous} → ${current}${reasonMsg ? ' — ' + reasonMsg : ''}`);
+        this.emit('log', 'Ably suspended — connection lost for extended period, will keep retrying');
+      } else if (current === 'closed') {
+        this._connected = false;
+        this.emit('log', `Ably connection: ${previous} → ${current}`);
+        this.emit('disconnected');
+      } else {
+        this.emit('log', `Ably connection: ${previous} → ${current}${reasonMsg ? ' — ' + reasonMsg : ''}`);
+      }
+
+      // Capture exceptions for bad states
+      if (current === 'failed' || current === 'suspended' || (current === 'disconnected' && reason)) {
+        Sentry.withScope((scope) => {
+          scope.setTag('ably.client', 'runner');
+          scope.setTag('ably.state', current);
+          scope.setTag('sandbox.id', this._sandboxId);
+          scope.setContext('ably_connection', { from: previous, to: current, reason: reasonMsg, retryIn });
+          const err = reason instanceof Error ? reason : new Error('Ably connection state error');
+          err.name = 'AblyConnectionError';
+          Sentry.captureException(err);
+        });
+      }
+    });
+
+    // ─── Ably channel state monitoring → Sentry ───────────────────────────
+    const sessionCh = this._sessionChannel;
+    if (sessionCh) {
+      sessionCh.on((stateChange) => {
+        const { current, previous, reason } = stateChange;
+        const reasonMsg = reason ? (reason.message || reason.code || String(reason)) : undefined;
+
+        Sentry.addBreadcrumb({
+          category: 'ably.channel',
+          message: `runner [session]: ${previous} → ${current}${reasonMsg ? ' — ' + reasonMsg : ''}`,
+          level: current === 'attached' ? 'info' : 'warning',
+          data: { client: 'runner', channel: sessionCh.name, from: previous, to: current, reason: reasonMsg, sandboxId: this._sandboxId },
+        });
+
+        this.emit('log', `Ably channel [session]: ${previous} → ${current}${reasonMsg ? ' — ' + reasonMsg : ''}`);
+
+        if (current === 'failed' || current === 'suspended') {
+          Sentry.withScope((scope) => {
+            scope.setTag('ably.client', 'runner');
+            scope.setTag('ably.channel', sessionCh.name);
+            scope.setTag('ably.channel_state', current);
+            scope.setTag('sandbox.id', this._sandboxId);
+            scope.setContext('ably_channel', { channel: sessionCh.name, from: previous, to: current, reason: reasonMsg, sandboxId: this._sandboxId });
+            const err = reason instanceof Error ? reason : new Error('Ably channel state error');
+            err.name = 'AblyChannelError';
+            Sentry.captureException(err);
+          });
+        }
+      });
+    }
+
+    // Subscribe to control messages
+    this._sessionChannel.subscribe('control', async (msg) => {
+      const message = msg.data;
+      if (!message) return;
+
+      this.emit('log', `Control message: ${message.type}`);
+
+      if (message.type === 'disconnect' || message.type === 'end-session') {
+        this.emit('log', 'Received disconnect request');
+        this.emit('disconnected');
+      }
+
+      if (message.type === 'keepalive') {
+        await this.sendControl({ type: 'keepalive.ack' });
+      }
+
+      if (message.type === 'debug') {
+        this._debugMode = !!message.enabled;
+        this.emit('log', `Debug mode ${this._debugMode ? 'enabled' : 'disabled'}`);
+      }
+    });
+
+    this.emit('log', 'Listening for commands on Ably');
+
+    // Signal readiness to SDK — commands sent before this would be lost
+    const readyPayload = {
+      type: 'runner.ready',
+      os: 'windows',
+      sandboxId: this._sandboxId,
+      runnerVersion: getLocalVersion() || 'unknown',
+      timestamp: Date.now(),
+    };
+    if (this._updateInfo) {
+      readyPayload.update = {
+        status: this._updateInfo.status,
+        localVersion: this._updateInfo.localVersion,
+        remoteVersion: this._updateInfo.remoteVersion,
+      };
+    }
+    await this._sessionChannel.publish('control', readyPayload);
+    this.emit('log', 'Published runner.ready signal');
+  }
+
+  /**
+   * Send a response on the session channel.
+   */
+  async _sendResponse(message) {
+    if (!this._sessionChannel) {
+      this.emit('log', 'Warning: session channel not available');
+      return;
+    }
+
+    try {
+      this.emit('log', `Publishing response: type=${message.type || 'unknown'} (requestId=${message.requestId || 'none'})`);
+      await this._sessionChannel.publish('response', message);
+    } catch (err) {
+      this.emit('log', `Failed to publish response: ${err.message}`);
+    }
+  }
+
+  /**
+   * Send a file/screenshot reference on the session channel.
+   */
+  async sendFile(message) {
+    if (!this._sessionChannel) return;
+
+    try {
+      this.emit('log', `Publishing file: type=${message.type || 'unknown'} (requestId=${message.requestId || 'none'})`);
+      await this._sessionChannel.publish('file', message);
+    } catch (err) {
+      this.emit('log', `Failed to publish file: ${err.message}`);
+    }
+  }
+
+  /**
+   * Send a control message.
+   */
+  async sendControl(message) {
+    if (!this._sessionChannel) return;
+
+    try {
+      this.emit('log', `Publishing control: type=${message.type || 'unknown'}`);
+      await this._sessionChannel.publish('control', message);
+    } catch (err) {
+      this.emit('log', `Failed to publish control: ${err.message}`);
+    }
+  }
+
+  /**
+   * Check if connected to Ably.
+   */
+  isConnected() {
+    return this._connected && this._ably && this._ably.connection.state === 'connected';
+  }
+
+  /**
+   * Disconnect from Ably and clean up.
+   */
+  async close() {
+    this.emit('log', 'Closing Ably service...');
+
+    if (this._statsInterval) {
+      clearInterval(this._statsInterval);
+      this._statsInterval = null;
+    }
+
+    try {
+      if (this._sessionChannel) this._sessionChannel.detach();
+    } catch {}
+
+    if (this._ably) {
+      try {
+        this._ably.close();
+      } catch {}
+      this._ably = null;
+    }
+
+    this._connected = false;
+    this._sessionChannel = null;
+
+    this.emit('log', 'Ably service closed');
+  }
+}
+
+module.exports = { AblyService };

package/lib/automation-bridge.js

@@ -0,0 +1,85 @@
+/**
+ * automation-bridge.js — Direct in-process automation bridge
+ *
+ * Replaces pyautogui-local.js. Instead of spawning a Python process and
+ * connecting via WebSocket, this bridge imports the Automation module
+ * directly and calls dispatch() in-process.
+ *
+ * Same public interface as PyAutoGUIBridge:
+ *   - extends EventEmitter (emits 'log', 'error')
+ *   - start() → ready
+ *   - stop()
+ *   - sendCommand(command, data) → Promise<result>
+ */
+const { EventEmitter } = require('events');
+const { Automation } = require('./automation');
+
+class AutomationBridge extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this._automation = null;
+    this._timeout = options.commandTimeout || 30000; // 30s default
+    this.started = false;
+  }
+
+  /**
+   * Initialize the automation module.
+   * No subprocess or network connection needed — just instantiate the class.
+   */
+  async start() {
+    this.emit('log', 'Starting automation bridge (in-process)');
+
+    this._automation = new Automation();
+    this.started = true;
+
+    this.emit('log', 'Automation bridge ready');
+  }
+
+  /**
+   * Execute a command via the automation module.
+   * @param {string} command - Command name (e.g. 'click', 'screenshot', 'exec')
+   * @param {object} data - Command parameters
+   * @returns {Promise<*>} Result value
+   */
+  async sendCommand(command, data = {}) {
+    if (!this.started || !this._automation) {
+      throw new Error('Automation bridge not started');
+    }
+
+    // Calculate timeout: use per-command timeout if provided, otherwise default
+    const timeoutMs = (data && data.timeout && data.timeout > 0)
+      ? (data.timeout * 1000) + 5000  // data.timeout is in seconds for exec; add 5s buffer
+      : this._timeout;
+
+    // Wrap dispatch in a timeout
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        reject(new Error(`Automation command timed out: ${command} (${timeoutMs}ms)`));
+      }, timeoutMs);
+
+      this._automation.dispatch(command, data)
+        .then((result) => {
+          clearTimeout(timer);
+          resolve(result);
+        })
+        .catch((err) => {
+          clearTimeout(timer);
+          reject(err);
+        });
+    });
+  }
+
+  /**
+   * Stop the automation bridge and clean up resources.
+   */
+  stop() {
+    this.started = false;
+    if (this._automation) {
+      this._automation.cleanup();
+      this._automation = null;
+    }
+    this.emit('log', 'Automation bridge stopped');
+  }
+}
+
+module.exports = { AutomationBridge };