@testdriverai/runner 7.8.0-canary.10

package/index.js

@@ -0,0 +1,556 @@
+#!/usr/bin/env node
+/**
+ * presence-runner.js — Presence-based runner for TestDriver
+ *
+ * This runner registers itself in a team's Ably presence channel and waits
+ * to be claimed by an SDK session. This enables:
+ *   - Discovery: SDK can see available runners
+ *   - Claiming: SDK claims a specific runner for a session
+ *   - Multi-runner: Multiple runners can be available for a team
+ *
+ * Flow:
+ *   1. Start with TD_API_KEY (required)
+ *   2. Call /api/v7/runner/register to get:
+ *      - Ably token with presence capability
+ *      - Runner channel name
+ *      - Team ID
+ *   3. Enter presence on the runner channel
+ *   4. Listen for 'claim' messages
+ *   5. When claimed:
+ *      - Exit presence (or mark as busy)
+ *      - Connect to session channel
+ *      - Handle commands until session ends
+ *   6. Re-enter presence when session ends (or exit)
+ *
+ * Environment variables:
+ *   TD_API_KEY      — Required: Team API key
+ *   TD_API_ROOT     — Optional: API URL (default: https://api.testdriver.ai)
+ *   TD_RUNNER_ID    — Optional: Runner ID (auto-generated if not set)
+ *   TD_RUNNER_OS    — Optional: OS capability (default: auto-detected)
+ *   TD_RUNNER_SINGLE — Optional: Exit after one session (default: false)
+ */
+require('../shared/load-env');
+const os = require('os');
+const fs = require('fs');
+const Ably = require('ably');
+const environments = require('../shared/environments.json');
+
+const { Automation } = require('./lib/automation');
+const { AblyService } = require('./lib/ably-service');
+
+/**
+ * Get the local runner version from package.json.
+ */
+function getLocalVersion() {
+  try {
+    if (process.env.RUNNER_VERSION) return process.env.RUNNER_VERSION;
+    const pkg = require('./package.json');
+    return pkg.version;
+  } catch { return null; }
+}
+
+const IS_WINDOWS = process.platform === 'win32';
+const IS_LINUX = process.platform === 'linux';
+const IS_MACOS = process.platform === 'darwin';
+
+// ─── Configuration ───────────────────────────────────────────────────────────
+
+const API_KEY = process.env.TD_API_KEY;
+const envName = process.env.TD_ENV || 'stable';
+const envConfig = environments[envName] || environments.stable;
+const API_ROOT = process.env.TD_API_ROOT || process.env.API_ROOT || envConfig.apiRoot;
+const RUNNER_ID = process.env.TD_RUNNER_ID;
+const SINGLE_SESSION = process.env.TD_RUNNER_SINGLE === 'true';
+
+if (!API_KEY) {
+  console.error('ERROR: TD_API_KEY environment variable is required');
+  console.error('Get your API key at: https://console.testdriver.ai/team');
+  process.exit(1);
+}
+
+// ─── Logging ─────────────────────────────────────────────────────────────────
+
+const logFile = IS_WINDOWS
+  ? 'C:\\Windows\\Temp\\testdriver-runner.log'
+  : '/tmp/testdriver-runner.log';
+
+function log(msg) {
+  const line = `[${new Date().toISOString()}] ${msg}`;
+  console.log(line);
+  try {
+    fs.appendFileSync(logFile, line + '\n');
+  } catch {}
+}
+
+// ─── Runner capabilities ─────────────────────────────────────────────────────
+
+function getCapabilities() {
+  const noVncPort = detectNoVncPort();
+  return {
+    os: process.env.TD_RUNNER_OS || process.platform,
+    arch: os.arch(),
+    hostname: os.hostname(),
+    nodeVersion: process.version,
+    ip: getLocalIP(),
+    cpus: os.cpus().length,
+    memory: Math.round(os.totalmem() / (1024 * 1024 * 1024)) + 'GB',
+    noVncPort: noVncPort,
+    vncUrl: buildVncUrl(noVncPort),
+  };
+}
+
+/**
+ * Build the publicly-reachable noVNC URL for this runner.
+ *
+ * Priority:
+ *   1. TD_VNC_URL env var — explicit override (any infra)
+ *   2. E2B detection — hostname is the sandbox ID, URL is predictable
+ *   3. Fallback — http://{localIP}:{noVncPort}/vnc_lite.html (direct/private access)
+ *
+ * Returns null if noVNC is not available.
+ */
+function buildVncUrl(noVncPort) {
+  // 1. Explicit override — works for any infrastructure
+  if (process.env.TD_VNC_URL) {
+    return process.env.TD_VNC_URL;
+  }
+
+  if (!noVncPort) return null;
+
+  // 2. E2B detection — check E2B_SANDBOX_ID env var first, then hostname heuristic
+  const e2bSandboxId = process.env.E2B_SANDBOX_ID;
+  if (e2bSandboxId) {
+    return `https://${noVncPort}-${e2bSandboxId}.e2b.dev/vnc_lite.html`;
+  }
+
+  // E2B hostnames can be various formats: "sb-xxxx-xxx", plain alphanumeric IDs, etc.
+  // Detect by checking for 169.254.x.x link-local IP (E2B sandbox networking) + non-standard hostname
+  const hostname = os.hostname();
+  const ip = getLocalIP();
+  if (ip && ip.startsWith('169.254.') && hostname && !/\./.test(hostname)) {
+    // Link-local IP + simple hostname strongly suggests E2B sandbox
+    return `https://${noVncPort}-${hostname}.e2b.dev/vnc_lite.html`;
+  }
+
+  // 3. Fallback: direct IP access (private networks, Docker, AWS with public IP, etc.)
+  return `http://${ip}:${noVncPort}/vnc_lite.html`;
+}
+
+/**
+ * Detect the noVNC websocket proxy port.
+ * Returns the port number (as a string) if noVNC is detected, or null.
+ *
+ * Detection strategy:
+ *   1. Check TD_NOVNC_PORT env var (explicit override)
+ *   2. On Linux/macOS: check if noVNC proxy process is listening (ss/lsof)
+ *   3. On Windows: check for noVNC or websockify on common ports (netstat)
+ *   4. Fallback: probe common noVNC ports (6080, 6081) via TCP connect
+ */
+function detectNoVncPort() {
+  // Allow explicit override
+  if (process.env.TD_NOVNC_PORT) {
+    return process.env.TD_NOVNC_PORT;
+  }
+
+  const { execSync } = require('child_process');
+
+  try {
+    if (IS_WINDOWS) {
+      // Windows: check netstat for common noVNC/websockify ports
+      const output = execSync('netstat -an 2>nul', { encoding: 'utf8', timeout: 5000 });
+      for (const port of ['6080', '6081']) {
+        if (output.includes(`:${port}`) && output.includes('LISTENING')) {
+          return port;
+        }
+      }
+    } else {
+      // Linux/macOS: try ss first (faster), then lsof
+      try {
+        const output = execSync('ss -tlnp 2>/dev/null || lsof -iTCP -sTCP:LISTEN -P -n 2>/dev/null', {
+          encoding: 'utf8',
+          timeout: 5000,
+        });
+        for (const port of ['6080', '6081']) {
+          if (output.includes(`:${port}`)) {
+            return port;
+          }
+        }
+      } catch {}
+
+      // Fallback: check if novnc_proxy process is running
+      try {
+        const output = execSync('pgrep -af "novnc_proxy|websockify" 2>/dev/null', {
+          encoding: 'utf8',
+          timeout: 5000,
+        });
+        // Try to extract port from command line args (e.g. --listen 6080)
+        const match = output.match(/(?:--listen\s+|:)(\d{4,5})/);
+        if (match) return match[1];
+        // Process is running but can't determine port — assume default
+        if (output.trim()) return '6080';
+      } catch {}
+    }
+  } catch (err) {
+    // Detection failed silently — that's fine
+  }
+
+  return null;
+}
+
+function getLocalIP() {
+  const interfaces = os.networkInterfaces();
+  for (const name of Object.keys(interfaces)) {
+    for (const iface of interfaces[name]) {
+      if (iface.family === 'IPv4' && !iface.internal) {
+        return iface.address;
+      }
+    }
+  }
+  return '127.0.0.1';
+}
+
+// ─── API registration ────────────────────────────────────────────────────────
+
+async function registerRunner() {
+  log(`Registering with API at ${API_ROOT}...`);
+
+  const capabilities = getCapabilities();
+  log(`Capabilities: ${JSON.stringify(capabilities)}`);
+
+  const response = await fetch(`${API_ROOT}/api/v7/runner/register`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      apiKey: API_KEY,
+      runnerId: RUNNER_ID,
+      capabilities,
+      version: require('./package.json').version,
+    }),
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Registration failed (${response.status}): ${text}`);
+  }
+
+  const data = await response.json();
+  if (!data.success) {
+    throw new Error(`Registration failed: ${data.errorMessage}`);
+  }
+
+  return data;
+}
+
+// ─── Presence management ─────────────────────────────────────────────────────
+
+class PresenceRunner {
+  constructor() {
+    this.ably = null;
+    this.runnerChannel = null;
+    this.runnerId = null;
+    this.teamId = null;
+    this.presenceData = null;
+    this.currentSession = null;
+    this.shuttingDown = false;
+    this.heartbeatInterval = null;
+    this.updateInfo = null;
+  }
+
+  async start() {
+    log('🚀 TestDriver Presence Runner starting...');
+
+    // Register with API first to get runner ID and team info
+    const registration = await registerRunner();
+    this.runnerId = registration.runnerId;
+    this.teamId = registration.teamId;
+    this.presenceData = registration.presenceData;
+
+    log(`Registered as runner: ${this.runnerId}`);
+    log(`Team ID: ${this.teamId}`);
+    log(`Runner channel: ${registration.ably.runnerChannel}`);
+
+    // Connect to Ably
+    this.ably = new Ably.Realtime({
+      authCallback: (tokenParams, callback) => {
+        callback(null, registration.ably.token);
+      },
+      clientId: `runner-${this.runnerId}`,
+    });
+
+    await new Promise((resolve, reject) => {
+      this.ably.connection.on('connected', resolve);
+      this.ably.connection.on('failed', (err) => {
+        reject(new Error(`Ably connection failed: ${err?.reason?.message || 'unknown'}`));
+      });
+      setTimeout(() => reject(new Error('Ably connection timeout')), 30000);
+    });
+
+    log('Connected to Ably');
+
+    // Monitor runner presence connection state
+    this.ably.connection.on((stateChange) => {
+      const { current, previous, reason, retryIn } = stateChange;
+      const reasonMsg = reason ? (reason.message || reason.code || String(reason)) : undefined;
+      log(`[ably] Presence connection: ${previous} → ${current}${reasonMsg ? ' — ' + reasonMsg : ''}${retryIn ? ' (retryIn=' + retryIn + 'ms)' : ''}`);
+    });
+
+    // Get runner channel and enter presence
+    this.runnerChannel = this.ably.channels.get(registration.ably.runnerChannel);
+
+    await this.enterPresence();
+
+    // Start heartbeat to update lastSeen for zombie detection
+    this.startHeartbeat();
+
+    // Listen for claim messages
+    this.runnerChannel.subscribe('claim', (msg) => this.handleClaim(msg));
+
+    log('✅ Runner ready — waiting for claims...');
+    log(`   Runner ID: ${this.runnerId}`);
+    log(`   IP: ${getLocalIP()}`);
+  }
+
+  async enterPresence() {
+    log('Entering presence...');
+    await this.runnerChannel.presence.enter({
+      ...this.presenceData,
+      status: 'available',
+      enteredAt: Date.now(),
+      lastSeen: Date.now(),
+      sandboxId: null,
+    });
+    log('Entered presence — runner is now available');
+  }
+
+  async leavePresence() {
+    log('Leaving presence...');
+    this.stopHeartbeat();
+    try {
+      await this.runnerChannel.presence.leave();
+    } catch (err) {
+      log(`Warning: Failed to leave presence: ${err.message}`);
+    }
+  }
+
+  startHeartbeat() {
+    // Update presence every 30 seconds with lastSeen timestamp
+    // This allows detection of zombie runners (present but not responding)
+    this.heartbeatInterval = setInterval(async () => {
+      if (this.shuttingDown) return;
+      try {
+        await this.runnerChannel.presence.update({
+          ...this.presenceData,
+          status: this.currentSession ? 'busy' : 'available',
+          sandboxId: this.currentSession?.sandboxId || null,
+          lastSeen: Date.now(),
+        });
+      } catch (err) {
+        log(`Warning: Heartbeat update failed: ${err.message}`);
+      }
+    }, 30000);
+  }
+
+  stopHeartbeat() {
+    if (this.heartbeatInterval) {
+      clearInterval(this.heartbeatInterval);
+      this.heartbeatInterval = null;
+    }
+  }
+
+  async updatePresenceStatus(status, extra = {}) {
+    try {
+      await this.runnerChannel.presence.update({
+        ...this.presenceData,
+        status,
+        lastSeen: Date.now(),
+        ...extra,
+      });
+    } catch (err) {
+      log(`Warning: Failed to update presence status: ${err.message}`);
+    }
+  }
+
+  async handleClaim(msg) {
+    const claim = msg.data;
+
+    // Check if this claim is for us
+    if (claim.targetRunner !== `runner-${this.runnerId}`) {
+      log(`Ignoring claim for ${claim.targetRunner} (we are runner-${this.runnerId})`);
+      return;
+    }
+
+    log(`📥 Received claim for sandbox: ${claim.sandboxId}`);
+    log(`[DEBUG] handleClaim started at ${Date.now()}`);
+
+    if (this.currentSession) {
+      log(`⚠️ Already in a session (${this.currentSession.sandboxId}), rejecting claim`);
+      // Publish rejection so the API can try another runner
+      await this.publishClaimResponse(claim.sandboxId, false, 'already_busy');
+      return;
+    }
+
+    // Mark as busy FIRST to prevent concurrent claims
+    this.currentSession = { sandboxId: claim.sandboxId, channel: claim.channel };
+
+    // Connect to session channel BEFORE acking so we're ready for commands
+    let ablyService, automation;
+    try {
+      log(`[DEBUG] Starting prepareSession at ${Date.now()}`);
+      ({ ablyService, automation } = await this.prepareSession(claim));
+      log(`[DEBUG] prepareSession completed at ${Date.now()}`);
+    } catch (err) {
+      log(`❌ Failed to prepare session: ${err.message}`);
+      this.currentSession = null;
+      await this.publishClaimResponse(claim.sandboxId, false, `prepare_failed: ${err.message}`);
+      return;
+    }
+
+    // Now that we're subscribed and ready, ack the claim
+    await this.publishClaimResponse(claim.sandboxId, true);
+
+    // Mark as busy in presence (stay in channel, just update status)
+    await this.updatePresenceStatus('busy', {
+      sandboxId: claim.sandboxId,
+      claimedAt: Date.now(),
+    });
+
+    // Run the session (waits for disconnect)
+    await this.runSession(claim.sandboxId, ablyService, automation);
+  }
+
+  /**
+   * Publish a claim-response on the runner channel so the API
+   * knows whether the claim was accepted or rejected.
+   */
+  async publishClaimResponse(sandboxId, accepted, reason) {
+    try {
+      log(`[DEBUG] About to publish claim-response. Channel state: ${this.runnerChannel.state}, Ably state: ${this.ably.connection.state}`);
+      const payload = {
+        runnerId: this.runnerId,
+        targetRunner: `runner-${this.runnerId}`,
+        sandboxId,
+        accepted,
+        reason: reason || (accepted ? 'ok' : 'rejected'),
+        timestamp: Date.now(),
+      };
+      log(`[DEBUG] claim-response payload: ${JSON.stringify(payload)}`);
+      await this.runnerChannel.publish('claim-response', payload);
+      log(`Published claim-response: ${accepted ? 'accepted' : 'rejected'} (sandbox=${sandboxId})`);
+    } catch (err) {
+      log(`⚠️ Failed to publish claim-response: ${err.message}`);
+      log(`[DEBUG] Publish error stack: ${err.stack}`);
+    }
+  }
+
+  /**
+   * Prepare a session — connect to Ably and subscribe to session channel.
+   * Returns the connected ablyService and automation, but does NOT
+   * wait for disconnect. Call runSession() after acking the claim.
+   */
+  async prepareSession(claim) {
+    const { sandboxId, channel, token, config } = claim;
+
+    log(`🔌 Starting session: ${sandboxId}`);
+
+    // Create automation instance for this session with sandbox context
+    const automation = new Automation({
+      sandboxId,
+      apiRoot: API_ROOT,
+      apiKey: API_KEY,
+    });
+
+    // Connect to session channel
+    const ablyService = new AblyService({
+      automation,
+      ablyToken: token,
+      channel,
+      sandboxId,
+      clientId: `agent-${sandboxId}`,
+      apiRoot: API_ROOT,
+      apiKey: API_KEY,
+      updateInfo: this.updateInfo,
+    });
+
+    ablyService.on('log', (msg) => log(`[session] ${msg}`));
+    ablyService.on('error', (err) => log(`[session] ERROR: ${err.message}`));
+
+    await ablyService.connect();
+    log(`✅ Session active: ${sandboxId}`);
+
+    return { ablyService, automation };
+  }
+
+  /**
+   * Run a session — wait for it to end (disconnect or error), then clean up.
+   */
+  async runSession(sandboxId, ablyService, automation) {
+    try {
+      // Wait for session to end (disconnect or error)
+      await new Promise((resolve) => {
+        ablyService.on('disconnected', resolve);
+        ablyService.on('error', resolve);
+      });
+
+      log(`📤 Session ended: ${sandboxId}`);
+    } catch (err) {
+      log(`❌ Session error: ${err.message}`);
+    } finally {
+      await ablyService.close();
+      automation.cleanup();
+      this.currentSession = null;
+
+      if (this.shuttingDown) {
+        return;
+      }
+
+      if (SINGLE_SESSION) {
+        log('Single session mode — exiting');
+        await this.shutdown();
+      } else {
+        // Update presence status back to available (stay in channel)
+        await this.updatePresenceStatus('available', {
+          sandboxId: null,
+          claimedAt: null,
+        });
+        log('🔄 Ready for next session');
+      }
+    }
+  }
+
+  async shutdown() {
+    this.shuttingDown = true;
+    log('Shutting down...');
+
+    await this.leavePresence();
+
+    if (this.ably) {
+      this.ably.close();
+    }
+
+    process.exit(0);
+  }
+}
+
+// ─── Main ────────────────────────────────────────────────────────────────────
+
+const runner = new PresenceRunner();
+
+async function main() {
+  log(`Runner version: ${getLocalVersion() || 'unknown'}`);
+  // Updates are applied via SSM before config is written to the instance.
+  // No self-update needed here.
+  await runner.start();
+
+  // Graceful shutdown
+  process.on('SIGINT', () => runner.shutdown());
+  process.on('SIGTERM', () => runner.shutdown());
+
+  // Keep alive
+  setInterval(() => {}, 60000);
+}
+
+main().catch((err) => {
+  log(`Fatal error: ${err.message}`);
+  console.error(err);
+  process.exit(1);
+});