agent-relay 1.3.0 → 1.3.2

package/dist/cloud/provisioner/index.js

@@ -4,12 +4,48 @@
  * One-click provisioning for compute resources (Fly.io, Railway, Docker).
  */
 import * as crypto from 'crypto';
+import { createHash } from 'node:crypto';
 import { getConfig } from '../config.js';
 import { db } from '../db/index.js';
 import { nangoService } from '../services/nango.js';
 import { canAutoScale, canScaleToTier, getResourceTierForPlan, } from '../services/planLimits.js';
 import { deriveSshPassword } from '../services/ssh-security.js';
+// ============================================================================
+// Daemon API Key Management
+// ============================================================================
+/**
+ * Generate a daemon API key in the format ar_live_<32 hex chars>
+ */
+function generateDaemonApiKey() {
+    const random = crypto.randomBytes(32).toString('hex');
+    return `ar_live_${random}`;
+}
+/**
+ * Hash an API key for secure storage
+ */
+function hashApiKey(apiKey) {
+    return createHash('sha256').update(apiKey).digest('hex');
+}
+/**
+ * Create a linked daemon record for a workspace during provisioning
+ * @param preGeneratedApiKey - Pre-generated API key (if not provided, one will be generated)
+ */
+async function createLinkedDaemon(userId, workspaceId, machineId, preGeneratedApiKey) {
+    const apiKey = preGeneratedApiKey ?? generateDaemonApiKey();
+    const apiKeyHash = hashApiKey(apiKey);
+    const daemon = await db.linkedDaemons.create({
+        userId,
+        workspaceId,
+        name: `auto-provisioned-${Date.now()}`,
+        machineId,
+        apiKeyHash,
+        status: 'offline',
+    });
+    return { daemonId: daemon.id, apiKey };
+}
 const WORKSPACE_PORT = 3888;
+const WORKSPACE_HEALTH_PORT = 3889; // Health check on separate thread - always responsive
+const WORKSPACE_SSH_PORT = 3022;
 const CODEX_OAUTH_PORT = 1455; // Codex CLI OAuth callback port - must be mapped for local dev
 const FETCH_TIMEOUT_MS = 10_000;
 const WORKSPACE_IMAGE = process.env.WORKSPACE_IMAGE || 'ghcr.io/agentworkforce/relay-workspace:latest';
@@ -355,12 +391,15 @@ class FlyProvisioner {
         updateProvisioningStage(workspace.id, 'networking');
         // Allocate IPs for the app (required for public DNS)
         // Must use GraphQL API - Machines REST API doesn't support IP allocation
-        // Shared IPv4 is free, IPv6 is free
+        // IMPORTANT: We use dedicated IPv4 ($2/mo) instead of shared because:
+        // - Shared IPv4 doesn't properly handle raw TCP on non-standard ports (like SSH on 3022)
+        // - SSH tunnel connections fail with "Connection closed by remote host" on shared IPs
+        // - Dedicated IPv4 is required for raw TCP services to work correctly
         console.log(`[fly] Allocating IPs for ${appName}...`);
         const allocateIP = async (type) => {
             try {
-                // Map our type to Fly GraphQL enum
-                const graphqlType = type === 'shared_v4' ? 'shared_v4' : 'v6';
+                // Map our type to Fly GraphQL enum (v4 = dedicated IPv4)
+                const graphqlType = type;
                 const res = await fetchWithRetry('https://api.fly.io/graphql', {
                     method: 'POST',
                     headers: {
@@ -412,11 +451,11 @@ class FlyProvisioner {
                 return false;
             }
         };
-        const [sharedV4Result, v6Result] = await Promise.all([
-            allocateIP('shared_v4'),
+        const [v4Result, v6Result] = await Promise.all([
+            allocateIP('v4'),
             allocateIP('v6'),
         ]);
-        console.log(`[fly] IP allocation results: shared_v4=${sharedV4Result}, v6=${v6Result}`);
+        console.log(`[fly] IP allocation results: v4=${v4Result}, v6=${v6Result}`);
         // Stage: Secrets
         updateProvisioningStage(workspace.id, 'secrets');
         // Set secrets (provider credentials)
@@ -447,6 +486,9 @@ class FlyProvisioner {
         }
         // Stage: Machine (includes volume creation)
         updateProvisioningStage(workspace.id, 'machine');
+        // Generate API key for cloud message sync BEFORE creating the machine
+        // The key is set as an env var on the machine and stored hashed in linkedDaemons
+        const machineApiKey = generateDaemonApiKey();
         // Create volume with automatic daily snapshots before machine
         // Fly.io takes daily snapshots automatically; we configure retention
         const volume = await this.createVolume(appName);
@@ -500,13 +542,20 @@ class FlyProvisioner {
                         PROVIDERS: (workspace.config.providers ?? []).join(','),
                         PORT: String(WORKSPACE_PORT),
                         AGENT_RELAY_DASHBOARD_PORT: String(WORKSPACE_PORT),
+                        // Store repos on persistent volume (/data) so they survive container restarts
+                        // Without this, repos are cloned to /workspace (ephemeral) and lost on restart
+                        WORKSPACE_DIR: '/data/repos',
                         // Git gateway configuration
                         CLOUD_API_URL: this.cloudApiUrl,
                         WORKSPACE_TOKEN: this.generateWorkspaceToken(workspace.id),
+                        // Daemon API key for cloud message sync
+                        // Auto-generated during provisioning, stored in linkedDaemons table
+                        AGENT_RELAY_API_KEY: machineApiKey,
                         // SSH for CLI tunneling (Codex OAuth callback forwarding)
                         // Each workspace gets a unique password derived from its ID + secret salt
                         ENABLE_SSH: 'true',
                         SSH_PASSWORD: deriveSshPassword(workspace.id),
+                        SSH_PORT: String(WORKSPACE_SSH_PORT),
                     },
                     services: [
                         {
@@ -538,16 +587,16 @@ class FlyProvisioner {
                             },
                         },
                         // SSH service for CLI tunneling (Codex OAuth callback forwarding)
-                        // Exposes port 2222 publicly for SSH connections from user's machine
+                        // Exposes port 3022 publicly for SSH connections from user's machine
                         {
                             ports: [
                                 {
-                                    port: 2222,
+                                    port: WORKSPACE_SSH_PORT,
                                     handlers: [], // Empty handlers = raw TCP passthrough
                                 },
                             ],
                             protocol: 'tcp',
-                            internal_port: 2222,
+                            internal_port: WORKSPACE_SSH_PORT,
                             // SSH connections should also wake the machine
                             auto_stop_machines: 'stop',
                             auto_start_machines: true,
@@ -557,11 +606,11 @@ class FlyProvisioner {
                     checks: {
                         health: {
                             type: 'http',
-                            port: WORKSPACE_PORT,
+                            port: WORKSPACE_HEALTH_PORT, // Health worker thread - responds even when main loop blocked
                             path: '/health',
                             interval: '30s',
-                            timeout: '5s',
-                            grace_period: '10s',
+                            timeout: '10s', // Increased timeout for safety
+                            grace_period: '30s', // Longer grace period for startup
                         },
                     },
                     // Instance size based on plan - free tier gets smaller instance
@@ -581,6 +630,11 @@ class FlyProvisioner {
             throw new Error(`Failed to create Fly machine: ${error}`);
         }
         const machine = (await machineResponse.json());
+        // Create linked daemon for cloud message sync
+        // Pass the pre-generated API key so it matches what was set in the machine env vars
+        const { daemonId } = await createLinkedDaemon(workspace.userId, workspace.id, machine.id, // Use Fly machine ID as daemon's machine ID
+        machineApiKey);
+        console.log(`[fly] Created linked daemon ${daemonId.substring(0, 8)} for workspace ${workspace.id.substring(0, 8)}`);
         // Return custom domain URL if configured, otherwise default fly.dev
         const publicUrl = customHostname
             ? `https://${customHostname}`
@@ -807,50 +861,114 @@ class FlyProvisioner {
         }
         console.log(`[fly] Updated machine image for workspace ${workspace.id.substring(0, 8)} to ${newImage}`);
     }
+    /**
+     * Set secrets as environment variables for a workspace.
+     */
+    async setSecrets(workspace, secrets) {
+        if (!workspace.computeId || Object.keys(secrets).length === 0)
+            return;
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/secrets`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(secrets),
+        });
+    }
     /**
      * Check if workspace has active agents by querying the daemon
+     * Retries up to 3 times with backoff to handle machines that are starting up
      */
     async checkActiveAgents(workspace) {
         if (!workspace.publicUrl) {
-            return { hasActiveAgents: false, agentCount: 0, agents: [] };
+            return { hasActiveAgents: false, agentCount: 0, agents: [], verified: true };
         }
-        try {
-            // Use internal Fly network URL if available (more reliable)
-            const appName = `ar-${workspace.id.substring(0, 8)}`;
-            const isOnFly = !!process.env.FLY_APP_NAME;
-            const baseUrl = isOnFly
-                ? `http://${appName}.internal:3888`
-                : workspace.publicUrl;
-            const controller = new AbortController();
-            const timer = setTimeout(() => controller.abort(), 10_000);
-            const response = await fetch(`${baseUrl}/api/agents`, {
-                method: 'GET',
-                headers: {
-                    'Accept': 'application/json',
-                },
-                signal: controller.signal,
-            });
-            clearTimeout(timer);
-            if (!response.ok) {
-                console.warn(`[fly] Failed to check agents for ${workspace.id.substring(0, 8)}: ${response.status}`);
-                return { hasActiveAgents: false, agentCount: 0, agents: [] };
+        // Use internal Fly network URL if available (more reliable)
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const isOnFly = !!process.env.FLY_APP_NAME;
+        const baseUrl = isOnFly
+            ? `http://${appName}.internal:3888`
+            : workspace.publicUrl;
+        const maxRetries = 3;
+        const retryDelays = [2000, 4000, 6000]; // 2s, 4s, 6s backoff
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
+            try {
+                const controller = new AbortController();
+                const timer = setTimeout(() => controller.abort(), 10_000);
+                // Use /api/data endpoint which returns { agents: [...], ... }
+                // Note: /api/agents doesn't exist on the workspace dashboard-server
+                const response = await fetch(`${baseUrl}/api/data`, {
+                    method: 'GET',
+                    headers: {
+                        'Accept': 'application/json',
+                    },
+                    signal: controller.signal,
+                });
+                clearTimeout(timer);
+                if (!response.ok) {
+                    console.warn(`[fly] Failed to check agents for ${workspace.id.substring(0, 8)}: HTTP ${response.status} (attempt ${attempt + 1}/${maxRetries})`);
+                    if (attempt < maxRetries - 1) {
+                        await new Promise(resolve => setTimeout(resolve, retryDelays[attempt]));
+                        continue;
+                    }
+                    return { hasActiveAgents: false, agentCount: 0, agents: [], verified: false };
+                }
+                const data = await response.json();
+                const agents = data.agents || [];
+                // Diagnostic logging: capture raw agent data before filtering
+                if (agents.length > 0) {
+                    console.log(`[fly] Workspace ${workspace.id.substring(0, 8)} raw agent data:`, agents.map(a => ({ name: a.name, status: a.status, activityState: a.activityState })));
+                }
+                // Treat any online agent as active unless explicitly disconnected/offline.
+                const activeAgents = agents.filter(a => {
+                    const status = (a.status ?? '').toLowerCase();
+                    const activityState = (a.activityState ?? '').toLowerCase();
+                    const isProcessing = a.isProcessing === true;
+                    if (activityState === 'active' || activityState === 'idle')
+                        return true;
+                    if (status && status !== 'disconnected' && status !== 'offline')
+                        return true;
+                    if (isProcessing)
+                        return true;
+                    return false;
+                });
+                // Log filtering results for diagnostics
+                if (agents.length > 0 && activeAgents.length !== agents.length) {
+                    const filteredOut = agents.filter(a => {
+                        const status = (a.status ?? '').toLowerCase();
+                        const activityState = (a.activityState ?? '').toLowerCase();
+                        const isProcessing = a.isProcessing === true;
+                        if (activityState === 'active' || activityState === 'idle')
+                            return false;
+                        if (status && status !== 'disconnected' && status !== 'offline')
+                            return false;
+                        if (isProcessing)
+                            return false;
+                        return true;
+                    });
+                    console.log(`[fly] Workspace ${workspace.id.substring(0, 8)} filtered out agents:`, filteredOut.map(a => ({ name: a.name, status: a.status, activityState: a.activityState })));
+                }
+                return {
+                    hasActiveAgents: activeAgents.length > 0,
+                    agentCount: activeAgents.length,
+                    agents: agents.map(a => ({ name: a.name, status: a.status || a.activityState || 'unknown' })),
+                    verified: true,
+                };
+            }
+            catch (error) {
+                // Workspace might be stopped or unreachable - retry with backoff
+                console.warn(`[fly] Could not reach workspace ${workspace.id.substring(0, 8)} (attempt ${attempt + 1}/${maxRetries}):`, error.message);
+                if (attempt < maxRetries - 1) {
+                    await new Promise(resolve => setTimeout(resolve, retryDelays[attempt]));
+                    continue;
+                }
             }
-            const data = await response.json();
-            const agents = data.agents || [];
-            // Consider agents with 'active' or 'idle' activity state as active
-            // 'disconnected' agents are not active
-            const activeAgents = agents.filter(a => a.status === 'running' || a.activityState === 'active' || a.activityState === 'idle');
-            return {
-                hasActiveAgents: activeAgents.length > 0,
-                agentCount: activeAgents.length,
-                agents: agents.map(a => ({ name: a.name, status: a.status || a.activityState || 'unknown' })),
-            };
-        }
-        catch (error) {
-            // Workspace might be stopped or unreachable - treat as no active agents
-            console.warn(`[fly] Could not reach workspace ${workspace.id.substring(0, 8)} to check agents:`, error.message);
-            return { hasActiveAgents: false, agentCount: 0, agents: [] };
         }
+        // All retries exhausted
+        console.warn(`[fly] Workspace ${workspace.id.substring(0, 8)} unreachable after ${maxRetries} attempts`);
+        return { hasActiveAgents: false, agentCount: 0, agents: [], verified: false };
     }
     /**
      * Get the current machine state
@@ -951,6 +1069,10 @@ class RailwayProvisioner {
         });
         const serviceData = await serviceResponse.json();
         const serviceId = serviceData.data.serviceCreate.id;
+        // Create linked daemon for cloud message sync
+        // This generates an API key and registers the daemon in the linkedDaemons table
+        const { daemonId, apiKey: railwayApiKey } = await createLinkedDaemon(workspace.userId, workspace.id, serviceId);
+        console.log(`[railway] Created linked daemon ${daemonId.substring(0, 8)} for workspace ${workspace.id.substring(0, 8)}`);
         // Set environment variables
         const envVars = {
             WORKSPACE_ID: workspace.id,
@@ -961,8 +1083,13 @@ class RailwayProvisioner {
             PROVIDERS: (workspace.config.providers ?? []).join(','),
             PORT: String(WORKSPACE_PORT),
             AGENT_RELAY_DASHBOARD_PORT: String(WORKSPACE_PORT),
+            // Store repos on persistent volume so they survive container restarts
+            WORKSPACE_DIR: '/data/repos',
             CLOUD_API_URL: this.cloudApiUrl,
             WORKSPACE_TOKEN: this.generateWorkspaceToken(workspace.id),
+            // Daemon API key for cloud message sync
+            // Auto-generated during provisioning, stored in linkedDaemons table
+            AGENT_RELAY_API_KEY: railwayApiKey,
         };
         for (const [provider, token] of credentials) {
             envVars[`${provider.toUpperCase()}_TOKEN`] = token;
@@ -1113,6 +1240,37 @@ class RailwayProvisioner {
             }),
         });
     }
+    async setEnvVars(workspace, envVars) {
+        if (!workspace.computeId || Object.keys(envVars).length === 0)
+            return;
+        const linkedDaemons = await db.linkedDaemons.findByWorkspaceId(workspace.id);
+        const serviceId = linkedDaemons[0]?.machineId;
+        if (!serviceId) {
+            console.warn(`[railway] No service ID found for workspace ${workspace.id}`);
+            return;
+        }
+        await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                query: `
+          mutation SetVariables($input: VariableCollectionUpsertInput!) {
+            variableCollectionUpsert(input: $input)
+          }
+        `,
+                variables: {
+                    input: {
+                        projectId: workspace.computeId,
+                        serviceId,
+                        variables: envVars,
+                    },
+                },
+            }),
+        });
+    }
 }
 /**
  * Local Docker provisioner (for development/self-hosted)
@@ -1170,6 +1328,11 @@ class DockerProvisioner {
     }
     async provision(workspace, credentials) {
         const containerName = `ar-${workspace.id.substring(0, 8)}`;
+        // Create linked daemon for cloud message sync
+        // This generates an API key and registers the daemon in the linkedDaemons table
+        // Use container name as daemon's machine ID (will be updated to actual container ID after creation)
+        const { daemonId, apiKey: dockerApiKey } = await createLinkedDaemon(workspace.userId, workspace.id, containerName);
+        console.log(`[docker] Created linked daemon ${daemonId.substring(0, 8)} for workspace ${workspace.id.substring(0, 8)}`);
         // Build environment variables
         const envArgs = [
             `-e WORKSPACE_ID=${workspace.id}`,
@@ -1180,8 +1343,13 @@ class DockerProvisioner {
             `-e PROVIDERS=${(workspace.config.providers ?? []).join(',')}`,
             `-e PORT=${WORKSPACE_PORT}`,
             `-e AGENT_RELAY_DASHBOARD_PORT=${WORKSPACE_PORT}`,
+            // Store repos on persistent volume so they survive container restarts
+            `-e WORKSPACE_DIR=/data/repos`,
             `-e CLOUD_API_URL=${this.cloudApiUrlForContainer}`,
             `-e WORKSPACE_TOKEN=${this.generateWorkspaceToken(workspace.id)}`,
+            // Daemon API key for cloud message sync
+            // Auto-generated during provisioning, stored in linkedDaemons table
+            `-e AGENT_RELAY_API_KEY=${dockerApiKey}`,
         ];
         for (const [provider, token] of credentials) {
             envArgs.push(`-e ${provider.toUpperCase()}_TOKEN=${token}`);
@@ -1214,12 +1382,13 @@ class DockerProvisioner {
             // Set CODEX_DIRECT_PORT=true to also map port 1455 directly (for debugging only)
             const directCodexPort = process.env.CODEX_DIRECT_PORT === 'true';
             const portMappings = directCodexPort
-                ? `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:2222 -p ${CODEX_OAUTH_PORT}:${CODEX_OAUTH_PORT}`
-                : `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:2222`;
+                ? `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:${WORKSPACE_SSH_PORT} -p ${CODEX_OAUTH_PORT}:${CODEX_OAUTH_PORT}`
+                : `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:${WORKSPACE_SSH_PORT}`;
             // Enable SSH in the container for tunneling
             // Each workspace gets a unique password derived from its ID + secret salt
             envArgs.push('-e ENABLE_SSH=true');
             envArgs.push(`-e SSH_PASSWORD=${deriveSshPassword(workspace.id)}`);
+            envArgs.push(`-e SSH_PORT=${WORKSPACE_SSH_PORT}`);
             execSync(`docker run -d --user root --name ${containerName} ${networkArg} ${volumeArgs} ${portMappings} ${envArgs.join(' ')} ${WORKSPACE_IMAGE}`, { stdio: 'pipe' });
             const publicUrl = `http://localhost:${hostPort}`;
             // Wait for container to be healthy before returning
@@ -1291,6 +1460,9 @@ class DockerProvisioner {
             throw new Error(`Failed to restart container: ${error}`);
         }
     }
+    async setEnvVars(_workspace, _envVars) {
+        console.warn('[docker] Updating environment variables for running containers is not supported.');
+    }
 }
 /**
  * Main Workspace Provisioner
@@ -1473,6 +1645,25 @@ export class WorkspaceProvisioner {
         }
         await this.provisioner.restart(workspace);
     }
+    /**
+     * Update environment variables for a workspace instance.
+     */
+    async setWorkspaceEnvVars(workspace, envVars) {
+        if (Object.keys(envVars).length === 0)
+            return;
+        if (this.provisioner instanceof FlyProvisioner) {
+            await this.provisioner.setSecrets(workspace, envVars);
+            return;
+        }
+        if (this.provisioner instanceof RailwayProvisioner) {
+            await this.provisioner.setEnvVars(workspace, envVars);
+            return;
+        }
+        if (this.provisioner instanceof DockerProvisioner) {
+            await this.provisioner.setEnvVars(workspace, envVars);
+            return;
+        }
+    }
     /**
      * Stop a workspace
      */
@@ -1693,6 +1884,7 @@ export class WorkspaceProvisioner {
         UPDATED: 'updated',
         UPDATED_PENDING_RESTART: 'updated_pending_restart',
         SKIPPED_ACTIVE_AGENTS: 'skipped_active_agents',
+        SKIPPED_VERIFICATION_FAILED: 'skipped_verification_failed',
         SKIPPED_NOT_RUNNING: 'skipped_not_running',
         NOT_SUPPORTED: 'not_supported',
         ERROR: 'error',
@@ -1744,6 +1936,19 @@ export class WorkspaceProvisioner {
             if (machineState === 'started') {
                 // Machine is running - check for active agents
                 const agentCheck = await flyProvisioner.checkActiveAgents(workspace);
+                // If we couldn't verify agent status and not forcing, skip to be safe
+                // This is expected behavior for workspaces that are waking up from auto-stop
+                // or experiencing temporary network issues - not an error condition
+                if (!agentCheck.verified && !options.force) {
+                    console.log(`[provisioner] Skipped workspace ${workspaceId.substring(0, 8)}: workspace unreachable (will update on next restart)`);
+                    return {
+                        result: WorkspaceProvisioner.UpdateResult.SKIPPED_VERIFICATION_FAILED,
+                        workspaceId,
+                        machineState,
+                        // Use 'reason' instead of 'error' - this is expected behavior, not an error
+                        reason: 'Workspace unreachable - will update on next restart or when accessible',
+                    };
+                }
                 if (agentCheck.hasActiveAgents && !options.force) {
                     // Has active agents and not forcing - skip
                     console.log(`[provisioner] Skipped workspace ${workspaceId.substring(0, 8)}: ${agentCheck.agentCount} active agents`);
@@ -1845,6 +2050,7 @@ export class WorkspaceProvisioner {
             updated: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.UPDATED).length,
             pendingRestart: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART).length,
             skippedActiveAgents: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_ACTIVE_AGENTS).length,
+            skippedVerificationFailed: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_VERIFICATION_FAILED).length,
             skippedNotRunning: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_NOT_RUNNING).length,
             errors: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.ERROR).length,
         };

package/dist/cloud/server.js

@@ -15,7 +15,7 @@ import { RedisStore } from 'connect-redis';
 import { WebSocketServer, WebSocket } from 'ws';
 import { getConfig } from './config.js';
 import { runMigrations } from './db/index.js';
-import { getScalingOrchestrator, getComputeEnforcementService, getIntroExpirationService } from './services/index.js';
+import { getScalingOrchestrator, getComputeEnforcementService, getIntroExpirationService, getWorkspaceKeepaliveService } from './services/index.js';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 // API routers
@@ -37,6 +37,7 @@ import { nangoAuthRouter } from './api/nango-auth.js';
 import { gitRouter } from './api/git.js';
 import { codexAuthHelperRouter } from './api/codex-auth-helper.js';
 import { adminRouter } from './api/admin.js';
+import { consensusRouter } from './api/consensus.js';
 import { db } from './db/index.js';
 import { validateSshSecurityConfig } from './services/ssh-security.js';
 /**
@@ -176,17 +177,19 @@ export async function createServer() {
     });
     // Lightweight CSRF protection using session token
     const SAFE_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']);
-    // Paths exempt from CSRF (webhooks from external services, workspace proxy, local auth callbacks)
+    // Paths exempt from CSRF (webhooks from external services, workspace proxy, local auth callbacks, admin API)
     const CSRF_EXEMPT_PATHS = [
         '/api/webhooks/',
         '/api/auth/nango/webhook',
         '/api/auth/codex-helper/callback',
+        '/api/admin/', // Admin API uses X-Admin-Secret header auth
     ];
     // Additional pattern for workspace proxy routes (contains /proxy/)
     const isWorkspaceProxyRoute = (path) => /^\/api\/workspaces\/[^/]+\/proxy\//.test(path);
     app.use((req, res, next) => {
         // Skip CSRF for webhook endpoints and workspace proxy routes
-        if (CSRF_EXEMPT_PATHS.some(path => req.path.startsWith(path)) || isWorkspaceProxyRoute(req.path)) {
+        const isExemptPath = CSRF_EXEMPT_PATHS.some(exemptPath => req.path.startsWith(exemptPath));
+        if (isExemptPath || isWorkspaceProxyRoute(req.path)) {
             return next();
         }
         if (!req.session)
@@ -212,6 +215,11 @@ export async function createServer() {
         if (authHeader?.startsWith('Bearer ')) {
             return next();
         }
+        // Skip CSRF for admin API key authenticated requests
+        const adminSecret = req.get('x-admin-secret');
+        if (adminSecret) {
+            return next();
+        }
         // Skip CSRF for test endpoints in non-production
         if (process.env.NODE_ENV !== 'production' && req.path.startsWith('/api/test/')) {
             return next();
@@ -247,6 +255,7 @@ export async function createServer() {
     // --- Routes with session auth ---
     app.use('/api/providers', providersRouter);
     app.use('/api/workspaces', workspacesRouter);
+    app.use('/api', consensusRouter); // Consensus API (nested under /api/workspaces/:id/consensus)
     app.use('/api/repos', reposRouter);
     app.use('/api/onboarding', onboardingRouter);
     app.use('/api/billing', billingRouter);
@@ -316,6 +325,8 @@ export async function createServer() {
     let scalingOrchestrator = null;
     let computeEnforcement = null;
     let introExpiration = null;
+    let workspaceKeepalive = null;
+    let daemonStaleCheckInterval = null;
     // Create HTTP server for WebSocket upgrade handling
     const httpServer = http.createServer(app);
     // ===== Presence WebSocket =====
@@ -644,7 +655,31 @@ export async function createServer() {
                 catch (error) {
                     console.warn('[cloud] Failed to start intro expiration:', error);
                 }
+                // Start workspace keepalive service (pings workspaces with active agents)
+                // This prevents Fly.io from idling machines that have running Claude agents
+                try {
+                    workspaceKeepalive = getWorkspaceKeepaliveService();
+                    workspaceKeepalive.start();
+                    console.log('[cloud] Workspace keepalive service started');
+                }
+                catch (error) {
+                    console.warn('[cloud] Failed to start workspace keepalive:', error);
+                }
             }
+            // Start daemon stale check (mark daemons offline if no heartbeat for 2+ minutes)
+            // Runs every 60 seconds regardless of RELAY_CLOUD_ENABLED
+            daemonStaleCheckInterval = setInterval(async () => {
+                try {
+                    const count = await db.linkedDaemons.markStale();
+                    if (count > 0) {
+                        console.log(`[cloud] Marked ${count} daemon(s) as offline (stale)`);
+                    }
+                }
+                catch (error) {
+                    console.error('[cloud] Failed to mark stale daemons:', error);
+                }
+            }, 60_000); // Every 60 seconds
+            console.log('[cloud] Daemon stale check started (60s interval)');
             return new Promise((resolve) => {
                 server = httpServer.listen(config.port, () => {
                     console.log(`Agent Relay Cloud running on port ${config.port}`);
@@ -667,6 +702,15 @@ export async function createServer() {
             if (introExpiration) {
                 introExpiration.stop();
             }
+            // Stop workspace keepalive service
+            if (workspaceKeepalive) {
+                workspaceKeepalive.stop();
+            }
+            // Stop daemon stale check
+            if (daemonStaleCheckInterval) {
+                clearInterval(daemonStaleCheckInterval);
+                daemonStaleCheckInterval = null;
+            }
             // Close WebSocket server
             wssPresence.close();
             if (server) {

package/dist/cloud/services/index.d.ts

@@ -11,4 +11,5 @@ export { spawnCIFixAgent, notifyAgentOfCIFailure, completeFixAttempt, getFailure
 export { handleMention, handleIssueAssignment, getPendingMentions, getPendingIssueAssignments, processPendingMentions, processPendingIssueAssignments, KNOWN_AGENTS, isKnownAgent, } from './mention-handler.js';
 export { ComputeEnforcementService, ComputeEnforcementConfig, EnforcementResult, getComputeEnforcementService, createComputeEnforcementService, } from './compute-enforcement.js';
 export { IntroExpirationService, IntroExpirationConfig, IntroStatus, ExpirationResult as IntroExpirationResult, INTRO_PERIOD_DAYS, getIntroStatus, getIntroExpirationService, startIntroExpirationService, stopIntroExpirationService, } from './intro-expiration.js';
+export { WorkspaceKeepaliveService, WorkspaceKeepaliveConfig, KeepaliveStats, getWorkspaceKeepaliveService, createWorkspaceKeepaliveService, } from './workspace-keepalive.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/cloud/services/index.js

@@ -16,4 +16,6 @@ export { handleMention, handleIssueAssignment, getPendingMentions, getPendingIss
 export { ComputeEnforcementService, getComputeEnforcementService, createComputeEnforcementService, } from './compute-enforcement.js';
 // Intro expiration (auto-resize after free tier intro period)
 export { IntroExpirationService, INTRO_PERIOD_DAYS, getIntroStatus, getIntroExpirationService, startIntroExpirationService, stopIntroExpirationService, } from './intro-expiration.js';
+// Workspace keepalive (prevent Fly.io from idling machines with active agents)
+export { WorkspaceKeepaliveService, getWorkspaceKeepaliveService, createWorkspaceKeepaliveService, } from './workspace-keepalive.js';
 //# sourceMappingURL=index.js.map

package/dist/cloud/services/nango.d.ts

@@ -38,10 +38,9 @@ declare class NangoService {
      * This is the user-level token (not the installation token).
      * Use this for operations that require user context (e.g., gh CLI).
      *
-     * The user token can be found in:
-     * 1. getToken() without installation flag
-     * 2. connection_config.access_token in github-app-oauth
-     * 3. Separate 'github' user connection
+     * The user token is stored in connection_config.userCredentials.access_token
+     * by Nango's GitHub App OAuth flow. This is a gho_* or ghu_* token that
+     * works for both git operations and gh CLI commands.
      */
     getGithubUserOAuthToken(connectionId: string): Promise<string>;
     /**