@agent-relay/cloud 0.1.0

package/dist/provisioner/index.js

@@ -0,0 +1,2114 @@
+/**
+ * Agent Relay Cloud - Workspace Provisioner
+ *
+ * One-click provisioning for compute resources (Fly.io, Railway, Docker).
+ */
+import * as crypto from 'crypto';
+import { createHash } from 'node:crypto';
+import { getConfig } from '../config.js';
+import { db } from '../db/index.js';
+import { nangoService } from '../services/nango.js';
+import { canAutoScale, canScaleToTier, getResourceTierForPlan, getPlanLimits, } from '../services/planLimits.js';
+import { deriveSshPassword } from '../services/ssh-security.js';
+// ============================================================================
+// Daemon API Key Management
+// ============================================================================
+/**
+ * Generate a daemon API key in the format ar_live_<32 hex chars>
+ */
+function generateDaemonApiKey() {
+    const random = crypto.randomBytes(32).toString('hex');
+    return `ar_live_${random}`;
+}
+/**
+ * Hash an API key for secure storage
+ */
+function hashApiKey(apiKey) {
+    return createHash('sha256').update(apiKey).digest('hex');
+}
+/**
+ * Create a linked daemon record for a workspace during provisioning
+ * @param preGeneratedApiKey - Pre-generated API key (if not provided, one will be generated)
+ */
+async function createLinkedDaemon(userId, workspaceId, machineId, preGeneratedApiKey) {
+    const apiKey = preGeneratedApiKey ?? generateDaemonApiKey();
+    const apiKeyHash = hashApiKey(apiKey);
+    const daemon = await db.linkedDaemons.create({
+        userId,
+        workspaceId,
+        name: `auto-provisioned-${Date.now()}`,
+        machineId,
+        apiKeyHash,
+        status: 'offline',
+    });
+    return { daemonId: daemon.id, apiKey };
+}
+const WORKSPACE_PORT = 3888;
+const WORKSPACE_HEALTH_PORT = 3889; // Health check on separate thread - always responsive
+const WORKSPACE_SSH_PORT = 3022;
+const CODEX_OAUTH_PORT = 1455; // Codex CLI OAuth callback port - must be mapped for local dev
+const FETCH_TIMEOUT_MS = 10_000;
+const WORKSPACE_IMAGE = process.env.WORKSPACE_IMAGE || 'ghcr.io/agentworkforce/relay-workspace:latest';
+// In-memory tracker for provisioning progress (workspace ID -> progress)
+const provisioningProgress = new Map();
+/**
+ * Update the provisioning stage for a workspace
+ */
+function updateProvisioningStage(workspaceId, stage) {
+    const existing = provisioningProgress.get(workspaceId);
+    provisioningProgress.set(workspaceId, {
+        stage,
+        startedAt: existing?.startedAt ?? Date.now(),
+        updatedAt: Date.now(),
+    });
+    console.log(`[provisioner] Workspace ${workspaceId.substring(0, 8)} stage: ${stage}`);
+}
+/**
+ * Get the current provisioning stage for a workspace
+ */
+export function getProvisioningStage(workspaceId) {
+    return provisioningProgress.get(workspaceId) ?? null;
+}
+/**
+ * Clear provisioning progress (call when complete or failed)
+ */
+function clearProvisioningProgress(workspaceId) {
+    provisioningProgress.delete(workspaceId);
+}
+/**
+ * Schedule cleanup of provisioning progress after a delay
+ * This gives the frontend time to poll and see the 'complete' stage
+ */
+function scheduleProgressCleanup(workspaceId, delayMs = 30_000) {
+    setTimeout(() => {
+        clearProvisioningProgress(workspaceId);
+        console.log(`[provisioner] Cleaned up provisioning progress for ${workspaceId.substring(0, 8)}`);
+    }, delayMs);
+}
+/**
+ * Get a fresh GitHub App installation token from Nango.
+ * Looks up the user's connected repositories to find a valid Nango connection.
+ */
+async function getGithubAppTokenForUser(userId) {
+    try {
+        // Find any repository with a Nango connection for this user
+        const repos = await db.repositories.findByUserId(userId);
+        const repoWithConnection = repos.find(r => r.nangoConnectionId);
+        if (!repoWithConnection?.nangoConnectionId) {
+            console.warn(`[provisioner] No Nango GitHub App connection found for user ${userId}`);
+            return null;
+        }
+        // Get fresh installation token from Nango (handles refresh automatically)
+        const token = await nangoService.getGithubAppToken(repoWithConnection.nangoConnectionId);
+        return token;
+    }
+    catch (error) {
+        console.error(`[provisioner] Failed to get GitHub App token for user ${userId}:`, error);
+        return null;
+    }
+}
+async function wait(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function fetchWithRetry(url, options = {}) {
+    const retries = options.retries ?? 2;
+    let attempt = 0;
+    while (attempt <= retries) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+        try {
+            const response = await fetch(url, { ...options, signal: controller.signal });
+            clearTimeout(timer);
+            if (!response.ok && response.status >= 500 && attempt < retries) {
+                attempt += 1;
+                await wait(500 * attempt);
+                continue;
+            }
+            return response;
+        }
+        catch (error) {
+            clearTimeout(timer);
+            if (attempt >= retries) {
+                throw error;
+            }
+            attempt += 1;
+            await wait(500 * attempt);
+        }
+    }
+    throw new Error('fetchWithRetry exhausted retries');
+}
+async function softHealthCheck(url) {
+    try {
+        const res = await fetchWithRetry(`${url.replace(/\/$/, '')}/health`, { method: 'GET', retries: 1 });
+        if (!res.ok) {
+            console.warn(`[health] Non-200 from ${url}/health: ${res.status}`);
+        }
+    }
+    catch (error) {
+        console.warn(`[health] Failed to reach ${url}/health`, error);
+    }
+}
+/**
+ * Wait for machine to be in "started" state using Fly.io's /wait endpoint
+ * This is more efficient than polling - the API blocks until the state is reached
+ * @see https://fly.io/docs/machines/api/machines-resource/#wait-for-a-machine-to-reach-a-specific-state
+ */
+async function waitForMachineStarted(apiToken, appName, machineId, timeoutSeconds = 120) {
+    console.log(`[provisioner] Waiting for machine ${machineId} to start (timeout: ${timeoutSeconds}s)...`);
+    // Fly.io /wait endpoint has max timeout of 60s, so we need to loop for longer waits
+    const maxSingleWait = 60;
+    const startTime = Date.now();
+    const deadline = startTime + timeoutSeconds * 1000;
+    while (Date.now() < deadline) {
+        const remainingMs = deadline - Date.now();
+        const waitSeconds = Math.min(maxSingleWait, Math.ceil(remainingMs / 1000));
+        if (waitSeconds <= 0)
+            break;
+        try {
+            // Use Fly.io's /wait endpoint - blocks until machine reaches target state
+            // timeout is an integer in seconds (max 60)
+            const res = await fetch(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}/wait?state=started&timeout=${waitSeconds}`, {
+                headers: { Authorization: `Bearer ${apiToken}` },
+            });
+            if (res.ok) {
+                console.log(`[provisioner] Machine ${machineId} is now started`);
+                return;
+            }
+            // 408 = timeout, machine didn't reach state in time - try again if we have time
+            if (res.status === 408) {
+                console.log(`[provisioner] Machine ${machineId} not ready yet, continuing to wait...`);
+                continue;
+            }
+            // Other error
+            const errorText = await res.text();
+            throw new Error(`Wait for machine failed: ${res.status} ${errorText}`);
+        }
+        catch (error) {
+            if (error instanceof Error && error.message.includes('Wait for machine failed')) {
+                throw error;
+            }
+            console.warn(`[provisioner] Error waiting for machine:`, error);
+            throw new Error(`Failed to wait for machine ${machineId}: ${error.message}`);
+        }
+    }
+    // Timeout reached - get current state for error message
+    const stateRes = await fetch(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}`, { headers: { Authorization: `Bearer ${apiToken}` } });
+    const machine = stateRes.ok ? (await stateRes.json()) : { state: 'unknown' };
+    throw new Error(`Machine ${machineId} did not start within ${timeoutSeconds}s (last state: ${machine.state})`);
+}
+/**
+ * Wait for health check to pass (with DNS propagation time)
+ * Tries internal Fly network first if available, then falls back to public URL
+ */
+async function waitForHealthy(url, appName, maxWaitMs = 60_000 // Reduced from 90s - health check is best-effort anyway
+) {
+    const startTime = Date.now();
+    // Build list of URLs to try - internal first (faster, more reliable from inside Fly)
+    const urlsToTry = [];
+    // If running on Fly and app name provided, try internal network first
+    const isOnFly = !!process.env.FLY_APP_NAME;
+    if (isOnFly && appName) {
+        urlsToTry.push(`http://${appName}.internal:8080/health`);
+    }
+    // Always add the public URL as fallback
+    urlsToTry.push(`${url.replace(/\/$/, '')}/health`);
+    console.log(`[provisioner] Waiting for workspace to become healthy (trying: ${urlsToTry.join(', ')})...`);
+    // Exponential backoff: start at 1s, max 5s (reduces unnecessary polling)
+    let retryDelayMs = 1000;
+    const maxRetryDelayMs = 5000;
+    while (Date.now() - startTime < maxWaitMs) {
+        // Try each URL in order
+        for (const healthUrl of urlsToTry) {
+            try {
+                const controller = new AbortController();
+                const timer = setTimeout(() => controller.abort(), 5_000);
+                const res = await fetch(healthUrl, {
+                    method: 'GET',
+                    signal: controller.signal,
+                });
+                clearTimeout(timer);
+                if (res.ok) {
+                    console.log(`[provisioner] Health check passed via ${healthUrl}`);
+                    return;
+                }
+                console.log(`[provisioner] Health check to ${healthUrl} returned ${res.status}`);
+            }
+            catch (error) {
+                const elapsed = Math.round((Date.now() - startTime) / 1000);
+                const errMsg = error.message;
+                // Only log detailed error for last URL attempt
+                if (healthUrl === urlsToTry[urlsToTry.length - 1]) {
+                    console.log(`[provisioner] Health check failed (${elapsed}s elapsed): ${errMsg}`);
+                }
+            }
+        }
+        await wait(retryDelayMs);
+        // Exponential backoff with cap
+        retryDelayMs = Math.min(retryDelayMs * 1.5, maxRetryDelayMs);
+    }
+    // Don't throw - workspace is provisioned, health check is best-effort
+    console.warn(`[provisioner] Health check did not pass within ${maxWaitMs}ms, continuing anyway`);
+}
+// Resource tiers sized for Claude Code agents (~1-2GB RAM per agent)
+// cpuKind: 'shared' = cheaper but can be throttled, 'performance' = dedicated
+// Note: Team tier (large) uses shared CPUs for better margins (~50% vs ~7% with perf)
+export const RESOURCE_TIERS = {
+    small: { name: 'small', cpuCores: 2, memoryMb: 2048, maxAgents: 2, cpuKind: 'shared' },
+    medium: { name: 'medium', cpuCores: 2, memoryMb: 4096, maxAgents: 5, cpuKind: 'shared' },
+    large: { name: 'large', cpuCores: 4, memoryMb: 8192, maxAgents: 10, cpuKind: 'shared' },
+    xlarge: { name: 'xlarge', cpuCores: 8, memoryMb: 16384, maxAgents: 20, cpuKind: 'performance' },
+};
+/**
+ * Fly.io provisioner
+ */
+class FlyProvisioner {
+    apiToken;
+    org;
+    region;
+    workspaceDomain;
+    cloudApiUrl;
+    sessionSecret;
+    registryAuth;
+    snapshotRetentionDays;
+    volumeSizeGb;
+    constructor() {
+        const config = getConfig();
+        if (!config.compute.fly) {
+            throw new Error('Fly.io configuration missing');
+        }
+        this.apiToken = config.compute.fly.apiToken;
+        this.org = config.compute.fly.org;
+        this.region = config.compute.fly.region || 'sjc';
+        this.workspaceDomain = config.compute.fly.workspaceDomain;
+        this.registryAuth = config.compute.fly.registryAuth;
+        this.cloudApiUrl = config.publicUrl;
+        this.sessionSecret = config.sessionSecret;
+        // Snapshot settings: default 14 days retention, 10GB volume
+        this.snapshotRetentionDays = Math.min(60, Math.max(1, config.compute.fly.snapshotRetentionDays ?? 14));
+        this.volumeSizeGb = config.compute.fly.volumeSizeGb ?? 10;
+    }
+    /**
+     * Generate a workspace token for API authentication
+     * This is a simple HMAC - in production, consider using JWTs
+     */
+    generateWorkspaceToken(workspaceId) {
+        return crypto
+            .createHmac('sha256', this.sessionSecret)
+            .update(`workspace:${workspaceId}`)
+            .digest('hex');
+    }
+    /**
+     * Create a volume with automatic snapshot settings
+     * Fly.io takes daily snapshots automatically; we configure retention
+     */
+    async createVolume(appName) {
+        const volumeName = 'workspace_data';
+        console.log(`[fly] Creating volume ${volumeName} with ${this.snapshotRetentionDays}-day snapshot retention...`);
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                name: volumeName,
+                region: this.region,
+                size_gb: this.volumeSizeGb,
+                // Enable automatic daily snapshots (default is true, but be explicit)
+                auto_backup_enabled: true,
+                // Retain snapshots for configured days (default 5, we use 14)
+                snapshot_retention: this.snapshotRetentionDays,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Failed to create volume: ${error}`);
+        }
+        const volume = await response.json();
+        console.log(`[fly] Volume ${volume.id} created with auto-snapshots (${this.snapshotRetentionDays} days retention)`);
+        return volume;
+    }
+    /**
+     * Create an on-demand snapshot of a workspace volume
+     * Use before risky operations or as manual backup
+     */
+    async createSnapshot(appName, volumeId) {
+        console.log(`[fly] Creating on-demand snapshot for volume ${volumeId}...`);
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes/${volumeId}/snapshots`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Failed to create snapshot: ${error}`);
+        }
+        const snapshot = await response.json();
+        console.log(`[fly] Snapshot ${snapshot.id} created`);
+        return snapshot;
+    }
+    /**
+     * List snapshots for a workspace volume
+     */
+    async listSnapshots(appName, volumeId) {
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes/${volumeId}/snapshots`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!response.ok) {
+            return [];
+        }
+        return await response.json();
+    }
+    /**
+     * Get volume info for a workspace
+     */
+    async getVolume(appName) {
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!response.ok) {
+            return null;
+        }
+        const volumes = await response.json();
+        return volumes.find(v => v.name === 'workspace_data') || null;
+    }
+    async provision(workspace, credentials) {
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        // Stage: Creating workspace
+        updateProvisioningStage(workspace.id, 'creating');
+        // Create Fly app
+        await fetchWithRetry('https://api.machines.dev/v1/apps', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                app_name: appName,
+                org_slug: this.org,
+            }),
+        });
+        // Stage: Networking
+        updateProvisioningStage(workspace.id, 'networking');
+        // Allocate IPs for the app (required for public DNS)
+        // Must use GraphQL API - Machines REST API doesn't support IP allocation
+        // IMPORTANT: We use dedicated IPv4 ($2/mo) instead of shared because:
+        // - Shared IPv4 doesn't properly handle raw TCP on non-standard ports (like SSH on 3022)
+        // - SSH tunnel connections fail with "Connection closed by remote host" on shared IPs
+        // - Dedicated IPv4 is required for raw TCP services to work correctly
+        console.log(`[fly] Allocating IPs for ${appName}...`);
+        const allocateIP = async (type) => {
+            try {
+                // Map our type to Fly GraphQL enum (v4 = dedicated IPv4)
+                const graphqlType = type;
+                const res = await fetchWithRetry('https://api.fly.io/graphql', {
+                    method: 'POST',
+                    headers: {
+                        Authorization: `Bearer ${this.apiToken}`,
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        query: `
+              mutation AllocateIPAddress($input: AllocateIPAddressInput!) {
+                allocateIpAddress(input: $input) {
+                  ipAddress {
+                    id
+                    address
+                    type
+                  }
+                }
+              }
+            `,
+                        variables: {
+                            input: {
+                                appId: appName,
+                                type: graphqlType,
+                            },
+                        },
+                    }),
+                });
+                if (!res.ok) {
+                    const errorText = await res.text();
+                    console.warn(`[fly] Failed to allocate ${type}: ${res.status} ${errorText}`);
+                    return false;
+                }
+                const data = await res.json();
+                if (data.errors?.length) {
+                    // Ignore "already allocated" errors
+                    const alreadyAllocated = data.errors.some(e => e.message.includes('already') || e.message.includes('exists'));
+                    if (!alreadyAllocated) {
+                        console.warn(`[fly] GraphQL error allocating ${type}: ${data.errors[0].message}`);
+                        return false;
+                    }
+                    console.log(`[fly] IP ${type} already allocated`);
+                    return true;
+                }
+                const address = data.data?.allocateIpAddress?.ipAddress?.address;
+                console.log(`[fly] Allocated ${type}: ${address}`);
+                return true;
+            }
+            catch (err) {
+                console.warn(`[fly] Failed to allocate ${type}: ${err.message}`);
+                return false;
+            }
+        };
+        const [v4Result, v6Result] = await Promise.all([
+            allocateIP('v4'),
+            allocateIP('v6'),
+        ]);
+        console.log(`[fly] IP allocation results: v4=${v4Result}, v6=${v6Result}`);
+        // Stage: Secrets
+        updateProvisioningStage(workspace.id, 'secrets');
+        // Set secrets (provider credentials)
+        const secrets = {};
+        for (const [provider, token] of credentials) {
+            secrets[`${provider.toUpperCase()}_TOKEN`] = token;
+            // Also set GH_TOKEN for gh CLI compatibility
+            if (provider === 'github') {
+                secrets['GH_TOKEN'] = token;
+            }
+        }
+        if (Object.keys(secrets).length > 0) {
+            await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/secrets`, {
+                method: 'POST',
+                headers: {
+                    Authorization: `Bearer ${this.apiToken}`,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(secrets),
+            });
+        }
+        // If custom workspace domain is configured, add certificate
+        const customHostname = this.workspaceDomain
+            ? `${appName}.${this.workspaceDomain}`
+            : null;
+        if (customHostname) {
+            await this.allocateCertificate(appName, customHostname);
+        }
+        // Stage: Machine (includes volume creation)
+        updateProvisioningStage(workspace.id, 'machine');
+        // Generate API key for cloud message sync BEFORE creating the machine
+        // The key is set as an env var on the machine and stored hashed in linkedDaemons
+        const machineApiKey = generateDaemonApiKey();
+        // Create volume with automatic daily snapshots before machine
+        // Fly.io takes daily snapshots automatically; we configure retention
+        const volume = await this.createVolume(appName);
+        // Determine instance size based on user's plan using RESOURCE_TIERS
+        const user = await db.users.findById(workspace.userId);
+        const userPlan = user?.plan || 'free';
+        const planLimits = getPlanLimits(userPlan);
+        const isFreeTier = userPlan === 'free';
+        // Check if user is in introductory period (first 14 days)
+        // Free users get Pro-level resources during intro period
+        const INTRO_PERIOD_DAYS = 14;
+        const userCreatedAt = user?.createdAt ? new Date(user.createdAt) : new Date();
+        const daysSinceSignup = (Date.now() - userCreatedAt.getTime()) / (1000 * 60 * 60 * 24);
+        const isIntroPeriod = isFreeTier && daysSinceSignup < INTRO_PERIOD_DAYS;
+        // Get the appropriate resource tier for this plan
+        // Intro period free users get 'pro' tier resources
+        const effectivePlan = isIntroPeriod ? 'pro' : userPlan;
+        const tierName = getResourceTierForPlan(effectivePlan);
+        const tier = RESOURCE_TIERS[tierName];
+        const guestConfig = {
+            cpu_kind: tier.cpuKind,
+            cpus: tier.cpuCores,
+            memory_mb: tier.memoryMb,
+        };
+        // Get max agents for the effective plan (intro users get pro limits)
+        const effectiveLimits = isIntroPeriod ? getPlanLimits('pro') : planLimits;
+        const maxAgents = effectiveLimits.maxConcurrentAgents === Infinity
+            ? 100 // Cap at 100 for practical purposes
+            : effectiveLimits.maxConcurrentAgents;
+        if (isIntroPeriod) {
+            const daysRemaining = Math.ceil(INTRO_PERIOD_DAYS - daysSinceSignup);
+            console.log(`[fly] Introductory bonus active (${daysRemaining} days remaining) - using ${tierName} tier`);
+        }
+        console.log(`[fly] Using ${tierName} tier: ${guestConfig.cpus} CPU / ${guestConfig.memory_mb}MB / max ${maxAgents} agents for ${userPlan} plan`);
+        // Create machine with auto-stop/start for cost optimization
+        const machineResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                region: this.region,
+                config: {
+                    image: WORKSPACE_IMAGE,
+                    // Registry auth for private ghcr.io images
+                    ...(this.registryAuth && {
+                        image_registry_auth: {
+                            registry: 'ghcr.io',
+                            username: this.registryAuth.username,
+                            password: this.registryAuth.password,
+                        },
+                    }),
+                    env: {
+                        WORKSPACE_ID: workspace.id,
+                        WORKSPACE_OWNER_USER_ID: workspace.userId,
+                        SUPERVISOR_ENABLED: String(workspace.config.supervisorEnabled ?? false),
+                        MAX_AGENTS: String(maxAgents),
+                        REPOSITORIES: (workspace.config.repositories ?? []).join(','),
+                        PROVIDERS: (workspace.config.providers ?? []).join(','),
+                        PORT: String(WORKSPACE_PORT),
+                        AGENT_RELAY_DASHBOARD_PORT: String(WORKSPACE_PORT),
+                        // Store repos on persistent volume (/data) so they survive container restarts
+                        // Without this, repos are cloned to /workspace (ephemeral) and lost on restart
+                        WORKSPACE_DIR: '/data/repos',
+                        // Git gateway configuration
+                        CLOUD_API_URL: this.cloudApiUrl,
+                        WORKSPACE_TOKEN: this.generateWorkspaceToken(workspace.id),
+                        // Daemon API key for cloud message sync
+                        // Auto-generated during provisioning, stored in linkedDaemons table
+                        AGENT_RELAY_API_KEY: machineApiKey,
+                        // SSH for CLI tunneling (Codex OAuth callback forwarding)
+                        // Each workspace gets a unique password derived from its ID + secret salt
+                        ENABLE_SSH: 'true',
+                        SSH_PASSWORD: deriveSshPassword(workspace.id),
+                        SSH_PORT: String(WORKSPACE_SSH_PORT),
+                        // Enable cloud persistence for agent sessions/summaries via API
+                        RELAY_CLOUD_ENABLED: 'true',
+                    },
+                    services: [
+                        {
+                            ports: [
+                                {
+                                    port: 443,
+                                    handlers: ['tls', 'http'],
+                                    // Force HTTP/1.1 to backend for WebSocket upgrade compatibility
+                                    // HTTP/2 doesn't support traditional WebSocket upgrade mechanism
+                                    http_options: {
+                                        h2_backend: false,
+                                    },
+                                },
+                                { port: 80, handlers: ['http'] },
+                            ],
+                            protocol: 'tcp',
+                            internal_port: WORKSPACE_PORT,
+                            // Auto-stop after inactivity to reduce costs
+                            // Fly Proxy automatically wakes machines on incoming requests
+                            auto_stop_machines: 'stop', // stop (not suspend) for faster wake
+                            auto_start_machines: true,
+                            min_machines_running: 0,
+                            // Idle timeout before auto-stop (in seconds)
+                            // Longer timeout = better UX, shorter = lower costs
+                            concurrency: {
+                                type: 'requests',
+                                soft_limit: 25,
+                                hard_limit: 50,
+                            },
+                        },
+                        // SSH service for CLI tunneling (Codex OAuth callback forwarding)
+                        // Exposes port 3022 publicly for SSH connections from user's machine
+                        {
+                            ports: [
+                                {
+                                    port: WORKSPACE_SSH_PORT,
+                                    handlers: [], // Empty handlers = raw TCP passthrough
+                                },
+                            ],
+                            protocol: 'tcp',
+                            internal_port: WORKSPACE_SSH_PORT,
+                            // SSH connections should also wake the machine
+                            auto_stop_machines: 'stop',
+                            auto_start_machines: true,
+                            min_machines_running: 0,
+                        },
+                    ],
+                    checks: {
+                        health: {
+                            type: 'http',
+                            port: WORKSPACE_HEALTH_PORT, // Health worker thread - responds even when main loop blocked
+                            path: '/health',
+                            interval: '30s',
+                            timeout: '10s', // Increased timeout for safety
+                            grace_period: '30s', // Longer grace period for startup
+                        },
+                    },
+                    // Instance size based on plan - free tier gets smaller instance
+                    guest: guestConfig,
+                    // Mount the volume we created with snapshot settings
+                    mounts: [
+                        {
+                            volume: volume.id,
+                            path: '/data',
+                        },
+                    ],
+                },
+            }),
+        });
+        if (!machineResponse.ok) {
+            const error = await machineResponse.text();
+            throw new Error(`Failed to create Fly machine: ${error}`);
+        }
+        const machine = (await machineResponse.json());
+        // Create linked daemon for cloud message sync
+        // Pass the pre-generated API key so it matches what was set in the machine env vars
+        const { daemonId } = await createLinkedDaemon(workspace.userId, workspace.id, machine.id, // Use Fly machine ID as daemon's machine ID
+        machineApiKey);
+        console.log(`[fly] Created linked daemon ${daemonId.substring(0, 8)} for workspace ${workspace.id.substring(0, 8)}`);
+        // Return custom domain URL if configured, otherwise default fly.dev
+        const publicUrl = customHostname
+            ? `https://${customHostname}`
+            : `https://${appName}.fly.dev`;
+        // Stage: Booting
+        updateProvisioningStage(workspace.id, 'booting');
+        // Wait for machine to be in started state
+        await waitForMachineStarted(this.apiToken, appName, machine.id);
+        // Stage: Health check
+        updateProvisioningStage(workspace.id, 'health');
+        // Wait for health check to pass (includes DNS propagation time)
+        // Pass appName to enable internal Fly network health checks
+        await waitForHealthy(publicUrl, appName);
+        // Stage: Complete
+        updateProvisioningStage(workspace.id, 'complete');
+        // Schedule cleanup of provisioning progress after 30s (gives frontend time to see 'complete')
+        scheduleProgressCleanup(workspace.id);
+        return {
+            computeId: machine.id,
+            publicUrl,
+        };
+    }
+    /**
+     * Allocate SSL certificate for custom domain
+     */
+    async allocateCertificate(appName, hostname) {
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/certificates`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ hostname }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            // Don't fail if cert already exists
+            if (!error.includes('already exists')) {
+                throw new Error(`Failed to allocate certificate for ${hostname}: ${error}`);
+            }
+        }
+    }
+    async deprovision(workspace) {
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}`, {
+            method: 'DELETE',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+    }
+    async getStatus(workspace) {
+        if (!workspace.computeId)
+            return 'error';
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!response.ok)
+            return 'error';
+        const machine = await response.json();
+        switch (machine.state) {
+            case 'started':
+                return 'running';
+            case 'stopped':
+                return 'stopped';
+            case 'created':
+            case 'starting':
+                return 'provisioning';
+            default:
+                return 'error';
+        }
+    }
+    async restart(workspace) {
+        if (!workspace.computeId)
+            return;
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}/restart`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+    }
+    /**
+     * Resize workspace - vertical scaling via Fly Machines API
+     * @param skipRestart - If true, config is saved but machine won't restart (changes apply on next start)
+     */
+    async resize(workspaceOrId, tier, skipRestart = false) {
+        const workspaceId = typeof workspaceOrId === 'string' ? workspaceOrId : workspaceOrId.id;
+        const computeId = typeof workspaceOrId === 'string' ? undefined : workspaceOrId.computeId;
+        // If passed just an ID, look up the workspace
+        let machineId = computeId;
+        if (!machineId) {
+            const workspace = await db.workspaces.findById(workspaceId);
+            if (!workspace?.computeId)
+                return;
+            machineId = workspace.computeId;
+        }
+        const appName = `ar-${workspaceId.substring(0, 8)}`;
+        // Get current machine config first to merge with new specs
+        // This is critical - Fly.io replaces the entire config, so we must preserve
+        // existing settings (image, services, auto_stop, other env vars, etc.)
+        const getResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!getResponse.ok) {
+            throw new Error(`Failed to get machine config for resize: ${await getResponse.text()}`);
+        }
+        const machine = await getResponse.json();
+        // Merge new specs into existing config, preserving everything else
+        const updatedConfig = {
+            ...machine.config,
+            guest: {
+                ...(machine.config.guest || {}),
+                // Use tier-specific CPU type (shared for cost, performance for power)
+                cpu_kind: tier.cpuKind,
+                cpus: tier.cpuCores,
+                memory_mb: tier.memoryMb,
+            },
+            env: {
+                ...(machine.config.env || {}),
+                MAX_AGENTS: String(tier.maxAgents),
+            },
+        };
+        // Update machine configuration
+        // If running: reboots with new specs (unless skip_launch: true)
+        // If stopped: config saved, applies on next start
+        const updateUrl = skipRestart
+            ? `https://api.machines.dev/v1/apps/${appName}/machines/${machineId}?skip_launch=true`
+            : `https://api.machines.dev/v1/apps/${appName}/machines/${machineId}`;
+        const updateResponse = await fetchWithRetry(updateUrl, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                config: updatedConfig,
+            }),
+        });
+        if (!updateResponse.ok) {
+            throw new Error(`Failed to resize machine: ${await updateResponse.text()}`);
+        }
+        const restartNote = skipRestart ? ' (will apply on next restart)' : ' (restarting)';
+        console.log(`[fly] Resized workspace ${workspaceId.substring(0, 8)} to ${tier.name} (${tier.cpuCores} CPU, ${tier.memoryMb}MB RAM)${restartNote}`);
+    }
+    /**
+     * Update the max agent limit for a workspace
+     */
+    async updateAgentLimit(workspace, newLimit) {
+        if (!workspace.computeId)
+            return;
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        // Update environment variable
+        await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                config: {
+                    env: {
+                        MAX_AGENTS: String(newLimit),
+                    },
+                },
+            }),
+        });
+        console.log(`[fly] Updated workspace ${workspace.id} agent limit to ${newLimit}`);
+    }
+    /**
+     * Get current resource tier for a workspace
+     */
+    async getCurrentTier(workspace) {
+        if (!workspace.computeId) {
+            return RESOURCE_TIERS.small;
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!response.ok) {
+            return RESOURCE_TIERS.small;
+        }
+        const machine = await response.json();
+        const _cpus = machine.config?.guest?.cpus || 1;
+        const memoryMb = machine.config?.guest?.memory_mb || 512;
+        // Map to nearest tier based on actual tier thresholds:
+        // small: 2048MB, medium: 4096MB, large: 8192MB, xlarge: 16384MB
+        if (memoryMb >= 16384)
+            return RESOURCE_TIERS.xlarge;
+        if (memoryMb >= 8192)
+            return RESOURCE_TIERS.large;
+        if (memoryMb >= 4096)
+            return RESOURCE_TIERS.medium;
+        return RESOURCE_TIERS.small;
+    }
+    /**
+     * Update machine image without restarting
+     * Note: The machine needs to be restarted later to use the new image
+     */
+    async updateMachineImage(workspace, newImage) {
+        if (!workspace.computeId)
+            return;
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        // Get current machine config first
+        const getResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!getResponse.ok) {
+            throw new Error(`Failed to get machine config: ${await getResponse.text()}`);
+        }
+        const machine = await getResponse.json();
+        // Update the image in the config
+        const updatedConfig = {
+            ...machine.config,
+            image: newImage,
+            // Include registry auth if configured
+            ...(this.registryAuth && {
+                image_registry_auth: {
+                    registry: 'ghcr.io',
+                    username: this.registryAuth.username,
+                    password: this.registryAuth.password,
+                },
+            }),
+        };
+        // Update machine with new image config (skip_launch keeps it in current state)
+        const updateResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}?skip_launch=true`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ config: updatedConfig }),
+        });
+        if (!updateResponse.ok) {
+            throw new Error(`Failed to update machine image: ${await updateResponse.text()}`);
+        }
+        console.log(`[fly] Updated machine image for workspace ${workspace.id.substring(0, 8)} to ${newImage}`);
+    }
+    /**
+     * Set secrets as environment variables for a workspace.
+     */
+    async setSecrets(workspace, secrets) {
+        if (!workspace.computeId || Object.keys(secrets).length === 0)
+            return;
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/secrets`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(secrets),
+        });
+    }
+    /**
+     * Check if workspace has active agents by querying the daemon
+     * Retries up to 3 times with backoff to handle machines that are starting up
+     */
+    async checkActiveAgents(workspace) {
+        if (!workspace.publicUrl) {
+            return { hasActiveAgents: false, agentCount: 0, agents: [], verified: true };
+        }
+        // Use internal Fly network URL if available (more reliable)
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const isOnFly = !!process.env.FLY_APP_NAME;
+        const baseUrl = isOnFly
+            ? `http://${appName}.internal:3888`
+            : workspace.publicUrl;
+        const maxRetries = 3;
+        const retryDelays = [2000, 4000, 6000]; // 2s, 4s, 6s backoff
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
+            try {
+                const controller = new AbortController();
+                const timer = setTimeout(() => controller.abort(), 10_000);
+                // Use /api/data endpoint which returns { agents: [...], ... }
+                // Note: /api/agents doesn't exist on the workspace dashboard-server
+                const response = await fetch(`${baseUrl}/api/data`, {
+                    method: 'GET',
+                    headers: {
+                        'Accept': 'application/json',
+                    },
+                    signal: controller.signal,
+                });
+                clearTimeout(timer);
+                if (!response.ok) {
+                    console.warn(`[fly] Failed to check agents for ${workspace.id.substring(0, 8)}: HTTP ${response.status} (attempt ${attempt + 1}/${maxRetries})`);
+                    if (attempt < maxRetries - 1) {
+                        await new Promise(resolve => setTimeout(resolve, retryDelays[attempt]));
+                        continue;
+                    }
+                    return { hasActiveAgents: false, agentCount: 0, agents: [], verified: false };
+                }
+                const data = await response.json();
+                const agents = data.agents || [];
+                // Diagnostic logging: capture raw agent data before filtering
+                if (agents.length > 0) {
+                    console.log(`[fly] Workspace ${workspace.id.substring(0, 8)} raw agent data:`, agents.map(a => ({ name: a.name, status: a.status, activityState: a.activityState })));
+                }
+                // Treat any online agent as active unless explicitly disconnected/offline.
+                const activeAgents = agents.filter(a => {
+                    const status = (a.status ?? '').toLowerCase();
+                    const activityState = (a.activityState ?? '').toLowerCase();
+                    const isProcessing = a.isProcessing === true;
+                    if (activityState === 'active' || activityState === 'idle')
+                        return true;
+                    if (status && status !== 'disconnected' && status !== 'offline')
+                        return true;
+                    if (isProcessing)
+                        return true;
+                    return false;
+                });
+                // Log filtering results for diagnostics
+                if (agents.length > 0 && activeAgents.length !== agents.length) {
+                    const filteredOut = agents.filter(a => {
+                        const status = (a.status ?? '').toLowerCase();
+                        const activityState = (a.activityState ?? '').toLowerCase();
+                        const isProcessing = a.isProcessing === true;
+                        if (activityState === 'active' || activityState === 'idle')
+                            return false;
+                        if (status && status !== 'disconnected' && status !== 'offline')
+                            return false;
+                        if (isProcessing)
+                            return false;
+                        return true;
+                    });
+                    console.log(`[fly] Workspace ${workspace.id.substring(0, 8)} filtered out agents:`, filteredOut.map(a => ({ name: a.name, status: a.status, activityState: a.activityState })));
+                }
+                return {
+                    hasActiveAgents: activeAgents.length > 0,
+                    agentCount: activeAgents.length,
+                    agents: agents.map(a => ({ name: a.name, status: a.status || a.activityState || 'unknown' })),
+                    verified: true,
+                };
+            }
+            catch (error) {
+                // Workspace might be stopped or unreachable - retry with backoff
+                console.warn(`[fly] Could not reach workspace ${workspace.id.substring(0, 8)} (attempt ${attempt + 1}/${maxRetries}):`, error.message);
+                if (attempt < maxRetries - 1) {
+                    await new Promise(resolve => setTimeout(resolve, retryDelays[attempt]));
+                    continue;
+                }
+            }
+        }
+        // All retries exhausted
+        console.warn(`[fly] Workspace ${workspace.id.substring(0, 8)} unreachable after ${maxRetries} attempts`);
+        return { hasActiveAgents: false, agentCount: 0, agents: [], verified: false };
+    }
+    /**
+     * Get the current machine state
+     */
+    async getMachineState(workspace) {
+        if (!workspace.computeId)
+            return 'unknown';
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        try {
+            const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+                headers: {
+                    Authorization: `Bearer ${this.apiToken}`,
+                },
+            });
+            if (!response.ok)
+                return 'unknown';
+            const machine = await response.json();
+            return machine.state;
+        }
+        catch {
+            return 'unknown';
+        }
+    }
+}
+/**
+ * Railway provisioner
+ */
+class RailwayProvisioner {
+    apiToken;
+    cloudApiUrl;
+    sessionSecret;
+    constructor() {
+        const config = getConfig();
+        if (!config.compute.railway) {
+            throw new Error('Railway configuration missing');
+        }
+        this.apiToken = config.compute.railway.apiToken;
+        this.cloudApiUrl = config.publicUrl;
+        this.sessionSecret = config.sessionSecret;
+    }
+    generateWorkspaceToken(workspaceId) {
+        return crypto
+            .createHmac('sha256', this.sessionSecret)
+            .update(`workspace:${workspaceId}`)
+            .digest('hex');
+    }
+    async provision(workspace, credentials) {
+        // Create project
+        const projectResponse = await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                query: `
+          mutation CreateProject($input: ProjectCreateInput!) {
+            projectCreate(input: $input) {
+              id
+              name
+            }
+          }
+        `,
+                variables: {
+                    input: {
+                        name: `agent-relay-${workspace.id.substring(0, 8)}`,
+                    },
+                },
+            }),
+        });
+        const projectData = await projectResponse.json();
+        const projectId = projectData.data.projectCreate.id;
+        // Deploy service
+        const serviceResponse = await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                query: `
+          mutation CreateService($input: ServiceCreateInput!) {
+            serviceCreate(input: $input) {
+              id
+            }
+          }
+        `,
+                variables: {
+                    input: {
+                        projectId,
+                        name: 'workspace',
+                        source: {
+                            image: WORKSPACE_IMAGE,
+                        },
+                    },
+                },
+            }),
+        });
+        const serviceData = await serviceResponse.json();
+        const serviceId = serviceData.data.serviceCreate.id;
+        // Create linked daemon for cloud message sync
+        // This generates an API key and registers the daemon in the linkedDaemons table
+        const { daemonId, apiKey: railwayApiKey } = await createLinkedDaemon(workspace.userId, workspace.id, serviceId);
+        console.log(`[railway] Created linked daemon ${daemonId.substring(0, 8)} for workspace ${workspace.id.substring(0, 8)}`);
+        // Set environment variables
+        const envVars = {
+            WORKSPACE_ID: workspace.id,
+            WORKSPACE_OWNER_USER_ID: workspace.userId,
+            SUPERVISOR_ENABLED: String(workspace.config.supervisorEnabled ?? false),
+            MAX_AGENTS: String(workspace.config.maxAgents ?? 10),
+            REPOSITORIES: (workspace.config.repositories ?? []).join(','),
+            PROVIDERS: (workspace.config.providers ?? []).join(','),
+            PORT: String(WORKSPACE_PORT),
+            AGENT_RELAY_DASHBOARD_PORT: String(WORKSPACE_PORT),
+            // Store repos on persistent volume so they survive container restarts
+            WORKSPACE_DIR: '/data/repos',
+            CLOUD_API_URL: this.cloudApiUrl,
+            WORKSPACE_TOKEN: this.generateWorkspaceToken(workspace.id),
+            // Daemon API key for cloud message sync
+            // Auto-generated during provisioning, stored in linkedDaemons table
+            AGENT_RELAY_API_KEY: railwayApiKey,
+            // Enable cloud persistence for agent sessions/summaries via API
+            RELAY_CLOUD_ENABLED: 'true',
+        };
+        for (const [provider, token] of credentials) {
+            envVars[`${provider.toUpperCase()}_TOKEN`] = token;
+            // Also set GH_TOKEN for gh CLI compatibility
+            if (provider === 'github') {
+                envVars['GH_TOKEN'] = token;
+            }
+        }
+        await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                query: `
+          mutation SetVariables($input: VariableCollectionUpsertInput!) {
+            variableCollectionUpsert(input: $input)
+          }
+        `,
+                variables: {
+                    input: {
+                        projectId,
+                        serviceId,
+                        variables: envVars,
+                    },
+                },
+            }),
+        });
+        // Generate domain
+        const domainResponse = await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                query: `
+          mutation CreateDomain($input: ServiceDomainCreateInput!) {
+            serviceDomainCreate(input: $input) {
+              domain
+            }
+          }
+        `,
+                variables: {
+                    input: {
+                        serviceId,
+                    },
+                },
+            }),
+        });
+        const domainData = await domainResponse.json();
+        const domain = domainData.data.serviceDomainCreate.domain;
+        await softHealthCheck(`https://${domain}`);
+        return {
+            computeId: projectId,
+            publicUrl: `https://${domain}`,
+        };
+    }
+    async deprovision(workspace) {
+        if (!workspace.computeId)
+            return;
+        await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                query: `
+          mutation DeleteProject($id: String!) {
+            projectDelete(id: $id)
+          }
+        `,
+                variables: {
+                    id: workspace.computeId,
+                },
+            }),
+        });
+    }
+    async getStatus(workspace) {
+        if (!workspace.computeId)
+            return 'error';
+        const response = await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                query: `
+          query GetProject($id: String!) {
+            project(id: $id) {
+              deployments {
+                edges {
+                  node {
+                    status
+                  }
+                }
+              }
+            }
+          }
+        `,
+                variables: {
+                    id: workspace.computeId,
+                },
+            }),
+        });
+        const data = await response.json();
+        const deployments = data.data?.project?.deployments?.edges;
+        if (!deployments || deployments.length === 0)
+            return 'provisioning';
+        const latestStatus = deployments[0].node.status;
+        switch (latestStatus) {
+            case 'SUCCESS':
+                return 'running';
+            case 'BUILDING':
+            case 'DEPLOYING':
+                return 'provisioning';
+            case 'CRASHED':
+            case 'FAILED':
+                return 'error';
+            default:
+                return 'stopped';
+        }
+    }
+    async restart(workspace) {
+        // Railway doesn't have a direct restart - redeploy instead
+        if (!workspace.computeId)
+            return;
+        await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                query: `
+          mutation RedeployService($input: DeploymentTriggerInput!) {
+            deploymentTrigger(input: $input)
+          }
+        `,
+                variables: {
+                    input: {
+                        projectId: workspace.computeId,
+                    },
+                },
+            }),
+        });
+    }
+    async setEnvVars(workspace, envVars) {
+        if (!workspace.computeId || Object.keys(envVars).length === 0)
+            return;
+        const linkedDaemons = await db.linkedDaemons.findByWorkspaceId(workspace.id);
+        const serviceId = linkedDaemons[0]?.machineId;
+        if (!serviceId) {
+            console.warn(`[railway] No service ID found for workspace ${workspace.id}`);
+            return;
+        }
+        await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                query: `
+          mutation SetVariables($input: VariableCollectionUpsertInput!) {
+            variableCollectionUpsert(input: $input)
+          }
+        `,
+                variables: {
+                    input: {
+                        projectId: workspace.computeId,
+                        serviceId,
+                        variables: envVars,
+                    },
+                },
+            }),
+        });
+    }
+}
+/**
+ * Local Docker provisioner (for development/self-hosted)
+ */
+class DockerProvisioner {
+    cloudApiUrl;
+    cloudApiUrlForContainer;
+    sessionSecret;
+    constructor() {
+        const config = getConfig();
+        this.cloudApiUrl = config.publicUrl;
+        this.sessionSecret = config.sessionSecret;
+        // For Docker containers, localhost won't work - they need to reach the host
+        // Convert localhost URLs to host.docker.internal for container access
+        if (this.cloudApiUrl.includes('localhost') || this.cloudApiUrl.includes('127.0.0.1')) {
+            this.cloudApiUrlForContainer = this.cloudApiUrl
+                .replace('localhost', 'host.docker.internal')
+                .replace('127.0.0.1', 'host.docker.internal');
+            console.log(`[docker] Container API URL: ${this.cloudApiUrlForContainer} (host: ${this.cloudApiUrl})`);
+        }
+        else {
+            this.cloudApiUrlForContainer = this.cloudApiUrl;
+        }
+    }
+    generateWorkspaceToken(workspaceId) {
+        return crypto
+            .createHmac('sha256', this.sessionSecret)
+            .update(`workspace:${workspaceId}`)
+            .digest('hex');
+    }
+    /**
+     * Wait for container to be healthy by polling the health endpoint
+     */
+    async waitForHealthy(publicUrl, timeoutMs = 60_000) {
+        const startTime = Date.now();
+        const pollInterval = 2000;
+        console.log(`[docker] Waiting for container to be healthy at ${publicUrl}...`);
+        while (Date.now() - startTime < timeoutMs) {
+            try {
+                const response = await fetch(`${publicUrl}/health`, {
+                    method: 'GET',
+                    signal: AbortSignal.timeout(5000),
+                });
+                if (response.ok) {
+                    console.log(`[docker] Container healthy after ${Date.now() - startTime}ms`);
+                    return;
+                }
+            }
+            catch {
+                // Container not ready yet, continue polling
+            }
+            await wait(pollInterval);
+        }
+        throw new Error(`Container did not become healthy within ${timeoutMs}ms`);
+    }
+    async provision(workspace, credentials) {
+        const containerName = `ar-${workspace.id.substring(0, 8)}`;
+        // Create linked daemon for cloud message sync
+        // This generates an API key and registers the daemon in the linkedDaemons table
+        // Use container name as daemon's machine ID (will be updated to actual container ID after creation)
+        const { daemonId, apiKey: dockerApiKey } = await createLinkedDaemon(workspace.userId, workspace.id, containerName);
+        console.log(`[docker] Created linked daemon ${daemonId.substring(0, 8)} for workspace ${workspace.id.substring(0, 8)}`);
+        // Build environment variables
+        const envArgs = [
+            `-e WORKSPACE_ID=${workspace.id}`,
+            `-e WORKSPACE_OWNER_USER_ID=${workspace.userId}`,
+            `-e SUPERVISOR_ENABLED=${workspace.config.supervisorEnabled ?? false}`,
+            `-e MAX_AGENTS=${workspace.config.maxAgents ?? 10}`,
+            `-e REPOSITORIES=${(workspace.config.repositories ?? []).join(',')}`,
+            `-e PROVIDERS=${(workspace.config.providers ?? []).join(',')}`,
+            `-e PORT=${WORKSPACE_PORT}`,
+            `-e AGENT_RELAY_DASHBOARD_PORT=${WORKSPACE_PORT}`,
+            // Store repos on persistent volume so they survive container restarts
+            `-e WORKSPACE_DIR=/data/repos`,
+            `-e CLOUD_API_URL=${this.cloudApiUrlForContainer}`,
+            `-e WORKSPACE_TOKEN=${this.generateWorkspaceToken(workspace.id)}`,
+            // Daemon API key for cloud message sync
+            // Auto-generated during provisioning, stored in linkedDaemons table
+            `-e AGENT_RELAY_API_KEY=${dockerApiKey}`,
+            // Enable cloud persistence for agent sessions/summaries via API
+            `-e RELAY_CLOUD_ENABLED=true`,
+        ];
+        for (const [provider, token] of credentials) {
+            envArgs.push(`-e ${provider.toUpperCase()}_TOKEN=${token}`);
+            // Also set GH_TOKEN for gh CLI compatibility
+            if (provider === 'github') {
+                envArgs.push(`-e GH_TOKEN=${token}`);
+            }
+        }
+        // Run container
+        const { execSync } = await import('child_process');
+        const hostPort = 3000 + Math.floor(Math.random() * 1000);
+        // SSH port for tunneling (Codex OAuth callback forwarding)
+        // Derive from hostPort to avoid collisions: API port 3500 -> SSH port 22500
+        const sshHostPort = 22000 + (hostPort - 3000);
+        // When running in Docker, connect to the same network for container-to-container communication
+        const runningInDocker = process.env.RUNNING_IN_DOCKER === 'true';
+        const networkArg = runningInDocker ? '--network agent-relay-dev' : '';
+        // In development, mount local dist and docs folders for faster iteration
+        // Set WORKSPACE_DEV_MOUNT=true to enable
+        const devMount = process.env.WORKSPACE_DEV_MOUNT === 'true';
+        const volumeArgs = devMount
+            ? `-v "${process.cwd()}/dist:/app/dist:ro" -v "${process.cwd()}/docs:/app/docs:ro"`
+            : '';
+        if (devMount) {
+            console.log('[provisioner] Dev mode: mounting local dist/ and docs/ folders into workspace container');
+        }
+        try {
+            // Map workspace API port and SSH port (for tunneling)
+            // SSH is used by CLI to forward localhost:1455 to workspace container for Codex OAuth
+            // Set CODEX_DIRECT_PORT=true to also map port 1455 directly (for debugging only)
+            const directCodexPort = process.env.CODEX_DIRECT_PORT === 'true';
+            const portMappings = directCodexPort
+                ? `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:${WORKSPACE_SSH_PORT} -p ${CODEX_OAUTH_PORT}:${CODEX_OAUTH_PORT}`
+                : `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:${WORKSPACE_SSH_PORT}`;
+            // Enable SSH in the container for tunneling
+            // Each workspace gets a unique password derived from its ID + secret salt
+            envArgs.push('-e ENABLE_SSH=true');
+            envArgs.push(`-e SSH_PASSWORD=${deriveSshPassword(workspace.id)}`);
+            envArgs.push(`-e SSH_PORT=${WORKSPACE_SSH_PORT}`);
+            execSync(`docker run -d --user root --name ${containerName} ${networkArg} ${volumeArgs} ${portMappings} ${envArgs.join(' ')} ${WORKSPACE_IMAGE}`, { stdio: 'pipe' });
+            const publicUrl = `http://localhost:${hostPort}`;
+            // Wait for container to be healthy before returning
+            // When running in Docker, use the internal container name for health check
+            const healthCheckUrl = runningInDocker
+                ? `http://${containerName}:${WORKSPACE_PORT}`
+                : publicUrl;
+            await this.waitForHealthy(healthCheckUrl);
+            return {
+                computeId: containerName,
+                publicUrl,
+                sshPort: sshHostPort,
+            };
+        }
+        catch (error) {
+            // Clean up container if it was created but health check failed
+            try {
+                const { execSync: execSyncCleanup } = await import('child_process');
+                execSyncCleanup(`docker rm -f ${containerName}`, { stdio: 'pipe' });
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+            throw new Error(`Failed to start Docker container: ${error}`);
+        }
+    }
+    async deprovision(workspace) {
+        if (!workspace.computeId)
+            return;
+        const { execSync } = await import('child_process');
+        try {
+            execSync(`docker rm -f ${workspace.computeId}`, { stdio: 'pipe' });
+        }
+        catch {
+            // Container may already be removed
+        }
+    }
+    async getStatus(workspace) {
+        if (!workspace.computeId)
+            return 'error';
+        const { execSync } = await import('child_process');
+        try {
+            const result = execSync(`docker inspect -f '{{.State.Status}}' ${workspace.computeId}`, { stdio: 'pipe' }).toString().trim();
+            switch (result) {
+                case 'running':
+                    return 'running';
+                case 'exited':
+                case 'dead':
+                    return 'stopped';
+                case 'created':
+                case 'restarting':
+                    return 'provisioning';
+                default:
+                    return 'error';
+            }
+        }
+        catch {
+            return 'error';
+        }
+    }
+    async restart(workspace) {
+        if (!workspace.computeId)
+            return;
+        const { execSync } = await import('child_process');
+        try {
+            execSync(`docker restart ${workspace.computeId}`, { stdio: 'pipe' });
+        }
+        catch (error) {
+            throw new Error(`Failed to restart container: ${error}`);
+        }
+    }
+    async setEnvVars(_workspace, _envVars) {
+        console.warn('[docker] Updating environment variables for running containers is not supported.');
+    }
+}
+/**
+ * Main Workspace Provisioner
+ */
+export class WorkspaceProvisioner {
+    provisioner;
+    constructor() {
+        const config = getConfig();
+        switch (config.compute.provider) {
+            case 'fly':
+                this.provisioner = new FlyProvisioner();
+                break;
+            case 'railway':
+                this.provisioner = new RailwayProvisioner();
+                break;
+            case 'docker':
+            default:
+                this.provisioner = new DockerProvisioner();
+        }
+    }
+    /**
+     * Provision a new workspace (one-click)
+     * Returns immediately with 'provisioning' status and runs actual provisioning in background
+     */
+    async provision(config) {
+        // Create workspace record
+        const workspace = await db.workspaces.create({
+            userId: config.userId,
+            name: config.name,
+            computeProvider: getConfig().compute.provider,
+            config: {
+                providers: config.providers,
+                repositories: config.repositories,
+                supervisorEnabled: config.supervisorEnabled ?? true,
+                maxAgents: config.maxAgents ?? 10,
+            },
+        });
+        // Add creator as owner in workspace_members for team collaboration support
+        await db.workspaceMembers.addMember({
+            workspaceId: workspace.id,
+            userId: config.userId,
+            role: 'owner',
+            invitedBy: config.userId, // Self-invited as creator
+        });
+        // Auto-accept the creator's membership
+        await db.workspaceMembers.acceptInvite(workspace.id, config.userId);
+        // Link repositories to this workspace
+        // This enables auto-access for users with GitHub access to these repos
+        for (const repoFullName of config.repositories) {
+            try {
+                // Find the user's repo record (may not exist if user didn't import it first)
+                const userRepos = await db.repositories.findByUserId(config.userId);
+                const repoRecord = userRepos.find(r => r.githubFullName.toLowerCase() === repoFullName.toLowerCase());
+                if (repoRecord) {
+                    await db.repositories.assignToWorkspace(repoRecord.id, workspace.id);
+                    console.log(`[provisioner] Linked repo ${repoFullName} to workspace ${workspace.id.substring(0, 8)}`);
+                }
+                else {
+                    // Create a placeholder repo record if it doesn't exist
+                    // This ensures the repo is tracked for workspace access checks
+                    console.log(`[provisioner] Creating repo record for ${repoFullName}`);
+                    const newRepo = await db.repositories.upsert({
+                        userId: config.userId,
+                        githubFullName: repoFullName,
+                        githubId: 0, // Will be updated when actually synced
+                        defaultBranch: 'main',
+                        isPrivate: true, // Assume private, will be updated
+                        workspaceId: workspace.id,
+                    });
+                    console.log(`[provisioner] Created and linked repo ${repoFullName} (id: ${newRepo.id.substring(0, 8)})`);
+                }
+            }
+            catch (err) {
+                console.warn(`[provisioner] Failed to link repo ${repoFullName}:`, err);
+                // Continue with other repos
+            }
+        }
+        // Initialize stage tracking immediately
+        updateProvisioningStage(workspace.id, 'creating');
+        // Run provisioning in the background so frontend can poll for stages
+        this.runProvisioningAsync(workspace, config).catch((error) => {
+            console.error(`[provisioner] Background provisioning failed for ${workspace.id}:`, error);
+        });
+        // Return immediately with 'provisioning' status
+        return {
+            workspaceId: workspace.id,
+            status: 'provisioning',
+        };
+    }
+    /**
+     * Run the actual provisioning work asynchronously
+     */
+    async runProvisioningAsync(workspace, config) {
+        // Build credentials map for workspace provisioning
+        // Note: Provider tokens (Claude, Codex, etc.) are no longer stored centrally.
+        // CLI tools authenticate directly on workspace instances.
+        // Only GitHub App tokens are obtained from Nango for repository cloning.
+        const credentials = new Map();
+        // GitHub token is required for cloning repositories
+        // Use direct token if provided (for testing), otherwise get from Nango
+        if (config.repositories.length > 0) {
+            if (config.githubToken) {
+                // Direct token provided (for testing)
+                credentials.set('github', config.githubToken);
+                console.log('[provisioner] Using provided GitHub token');
+            }
+            else {
+                // Get fresh installation token from Nango GitHub App
+                const githubToken = await getGithubAppTokenForUser(config.userId);
+                if (githubToken) {
+                    credentials.set('github', githubToken);
+                }
+                else {
+                    console.warn(`[provisioner] No GitHub App token for user ${config.userId}; repository cloning may fail.`);
+                }
+            }
+        }
+        // Provision compute
+        try {
+            const { computeId, publicUrl } = await this.provisioner.provision(workspace, credentials);
+            await db.workspaces.updateStatus(workspace.id, 'running', {
+                computeId,
+                publicUrl,
+            });
+            // Schedule cleanup of provisioning progress after 30s (gives frontend time to see 'complete')
+            setTimeout(() => {
+                clearProvisioningProgress(workspace.id);
+                console.log(`[provisioner] Cleaned up provisioning progress for ${workspace.id.substring(0, 8)}`);
+            }, 30_000);
+            console.log(`[provisioner] Workspace ${workspace.id} provisioned successfully at ${publicUrl}`);
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+            await db.workspaces.updateStatus(workspace.id, 'error', {
+                errorMessage,
+            });
+            // Clear provisioning progress on error
+            clearProvisioningProgress(workspace.id);
+            console.error(`[provisioner] Workspace ${workspace.id} provisioning failed:`, errorMessage);
+        }
+    }
+    /**
+     * Deprovision a workspace
+     */
+    async deprovision(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        await this.provisioner.deprovision(workspace);
+        await db.workspaces.delete(workspaceId);
+    }
+    /**
+     * Get workspace status
+     */
+    async getStatus(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // During early provisioning, computeId isn't set yet
+        // Return the database status instead of querying the provider
+        if (!workspace.computeId && workspace.status === 'provisioning') {
+            return 'provisioning';
+        }
+        const status = await this.provisioner.getStatus(workspace);
+        // Update database if status changed
+        if (status !== workspace.status) {
+            await db.workspaces.updateStatus(workspaceId, status);
+        }
+        return status;
+    }
+    /**
+     * Restart a workspace
+     */
+    async restart(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        await this.provisioner.restart(workspace);
+    }
+    /**
+     * Update environment variables for a workspace instance.
+     */
+    async setWorkspaceEnvVars(workspace, envVars) {
+        if (Object.keys(envVars).length === 0)
+            return;
+        if (this.provisioner instanceof FlyProvisioner) {
+            await this.provisioner.setSecrets(workspace, envVars);
+            return;
+        }
+        if (this.provisioner instanceof RailwayProvisioner) {
+            await this.provisioner.setEnvVars(workspace, envVars);
+            return;
+        }
+        if (this.provisioner instanceof DockerProvisioner) {
+            await this.provisioner.setEnvVars(workspace, envVars);
+            return;
+        }
+    }
+    /**
+     * Stop a workspace
+     */
+    async stop(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // For now, just deprovision to stop
+        await this.provisioner.deprovision(workspace);
+        await db.workspaces.updateStatus(workspaceId, 'stopped');
+    }
+    /**
+     * Resize a workspace (vertical scaling)
+     * @param skipRestart - If true, config is saved but machine won't restart (changes apply on next start)
+     */
+    async resize(workspaceId, tier, skipRestart = false) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        if (!this.provisioner.resize) {
+            throw new Error('Resize not supported by current compute provider');
+        }
+        await this.provisioner.resize(workspace, tier, skipRestart);
+        // Update workspace config with new limits
+        await db.workspaces.updateConfig(workspaceId, {
+            ...workspace.config,
+            maxAgents: tier.maxAgents,
+            resourceTier: tier.name,
+        });
+    }
+    /**
+     * Update the max agent limit for a workspace
+     */
+    async updateAgentLimit(workspaceId, newLimit) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        if (this.provisioner.updateAgentLimit) {
+            await this.provisioner.updateAgentLimit(workspace, newLimit);
+        }
+        // Update workspace config
+        await db.workspaces.updateConfig(workspaceId, {
+            ...workspace.config,
+            maxAgents: newLimit,
+        });
+    }
+    /**
+     * Get current resource tier for a workspace
+     */
+    async getCurrentTier(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        if (this.provisioner.getCurrentTier) {
+            return this.provisioner.getCurrentTier(workspace);
+        }
+        // Fallback: determine from config or default to small
+        const tierName = workspace.config.resourceTier || 'small';
+        return RESOURCE_TIERS[tierName] || RESOURCE_TIERS.small;
+    }
+    /**
+     * Get recommended tier based on agent count
+     * Uses 1.5-2GB per agent as baseline for Claude Code
+     */
+    getRecommendedTier(agentCount) {
+        // Find the smallest tier that supports this agent count
+        const tiers = Object.values(RESOURCE_TIERS).sort((a, b) => a.maxAgents - b.maxAgents);
+        for (const tier of tiers) {
+            if (tier.maxAgents >= agentCount) {
+                return tier;
+            }
+        }
+        // If agent count exceeds all tiers, return the largest
+        return RESOURCE_TIERS.xlarge;
+    }
+    /**
+     * Auto-scale workspace based on current agent count
+     * Respects plan limits - free tier cannot scale, others have max tier limits
+     * Returns { scaled: boolean, reason?: string }
+     */
+    async autoScale(workspaceId, currentAgentCount) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // Get user's plan
+        const user = await db.users.findById(workspace.userId);
+        const plan = user?.plan || 'free';
+        // Check if plan allows auto-scaling
+        if (!canAutoScale(plan)) {
+            return {
+                scaled: false,
+                reason: 'Auto-scaling requires Pro plan or higher',
+            };
+        }
+        const currentTier = await this.getCurrentTier(workspaceId);
+        const recommendedTier = this.getRecommendedTier(currentAgentCount);
+        // Only scale UP, never down (to avoid disruption)
+        if (recommendedTier.memoryMb <= currentTier.memoryMb) {
+            return {
+                scaled: false,
+                currentTier: currentTier.name,
+            };
+        }
+        // Check if plan allows scaling to the recommended tier
+        if (!canScaleToTier(plan, recommendedTier.name)) {
+            // Find the max tier allowed for this plan
+            const maxTierName = getResourceTierForPlan(plan);
+            const maxTier = RESOURCE_TIERS[maxTierName];
+            if (maxTier.memoryMb <= currentTier.memoryMb) {
+                return {
+                    scaled: false,
+                    reason: `Already at max tier (${currentTier.name}) for ${plan} plan`,
+                    currentTier: currentTier.name,
+                };
+            }
+            // Scale to max allowed tier instead
+            console.log(`[provisioner] Auto-scaling workspace ${workspaceId.substring(0, 8)} from ${currentTier.name} to ${maxTierName} (max for ${plan} plan)`);
+            await this.resize(workspaceId, maxTier);
+            return {
+                scaled: true,
+                currentTier: currentTier.name,
+                targetTier: maxTierName,
+                reason: `Scaled to max tier for ${plan} plan`,
+            };
+        }
+        console.log(`[provisioner] Auto-scaling workspace ${workspaceId.substring(0, 8)} from ${currentTier.name} to ${recommendedTier.name} (${currentAgentCount} agents)`);
+        await this.resize(workspaceId, recommendedTier);
+        return {
+            scaled: true,
+            currentTier: currentTier.name,
+            targetTier: recommendedTier.name,
+        };
+    }
+    // ============================================================================
+    // Snapshot Management
+    // ============================================================================
+    /**
+     * Create an on-demand snapshot of a workspace's volume
+     * Use before risky operations (e.g., major refactors, untrusted code execution)
+     */
+    async createSnapshot(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // Only Fly.io provisioner supports snapshots
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            console.warn('[provisioner] Snapshots only supported on Fly.io');
+            return null;
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const flyProvisioner = this.provisioner;
+        // Get the volume
+        const volume = await flyProvisioner.getVolume(appName);
+        if (!volume) {
+            throw new Error('No volume found for workspace');
+        }
+        // Create snapshot
+        const snapshot = await flyProvisioner.createSnapshot(appName, volume.id);
+        return { snapshotId: snapshot.id };
+    }
+    /**
+     * List available snapshots for a workspace
+     * Includes both automatic daily snapshots and on-demand snapshots
+     */
+    async listSnapshots(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // Only Fly.io provisioner supports snapshots
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            return [];
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const flyProvisioner = this.provisioner;
+        // Get the volume
+        const volume = await flyProvisioner.getVolume(appName);
+        if (!volume) {
+            return [];
+        }
+        // List snapshots
+        const snapshots = await flyProvisioner.listSnapshots(appName, volume.id);
+        return snapshots.map(s => ({
+            id: s.id,
+            createdAt: s.created_at,
+            sizeBytes: s.size,
+        }));
+    }
+    /**
+     * Get the volume ID for a workspace (needed for restore operations)
+     */
+    async getVolumeId(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            return null;
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const flyProvisioner = this.provisioner;
+        const volume = await flyProvisioner.getVolume(appName);
+        return volume?.id || null;
+    }
+    // ============================================================================
+    // Graceful Image Update
+    // ============================================================================
+    /**
+     * Result of a graceful update attempt
+     */
+    static UpdateResult = {
+        UPDATED: 'updated',
+        UPDATED_PENDING_RESTART: 'updated_pending_restart',
+        SKIPPED_ACTIVE_AGENTS: 'skipped_active_agents',
+        SKIPPED_VERIFICATION_FAILED: 'skipped_verification_failed',
+        SKIPPED_NOT_RUNNING: 'skipped_not_running',
+        NOT_SUPPORTED: 'not_supported',
+        ERROR: 'error',
+    };
+    /**
+     * Gracefully update a single workspace's image
+     *
+     * Behavior:
+     * - If workspace is stopped: Update config, will use new image on next wake
+     * - If workspace is running with no agents: Update config and restart
+     * - If workspace is running with active agents: Skip (or force if specified)
+     *
+     * @param workspaceId - Workspace to update
+     * @param newImage - New Docker image to use
+     * @param options - Update options
+     * @returns Update result with details
+     */
+    async gracefulUpdateImage(workspaceId, newImage, options = {}) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            return {
+                result: WorkspaceProvisioner.UpdateResult.ERROR,
+                workspaceId,
+                error: 'Workspace not found',
+            };
+        }
+        // Only Fly.io supports graceful updates
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            return {
+                result: WorkspaceProvisioner.UpdateResult.NOT_SUPPORTED,
+                workspaceId,
+                error: 'Graceful updates only supported on Fly.io',
+            };
+        }
+        const flyProvisioner = this.provisioner;
+        try {
+            // Check machine state
+            const machineState = await flyProvisioner.getMachineState(workspace);
+            if (machineState === 'stopped' || machineState === 'suspended') {
+                // Machine is not running - safe to update, will apply on next wake
+                await flyProvisioner.updateMachineImage(workspace, newImage);
+                console.log(`[provisioner] Updated stopped workspace ${workspaceId.substring(0, 8)} to ${newImage}`);
+                return {
+                    result: WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART,
+                    workspaceId,
+                    machineState,
+                };
+            }
+            if (machineState === 'started') {
+                // Machine is running - check for active agents
+                const agentCheck = await flyProvisioner.checkActiveAgents(workspace);
+                // If we couldn't verify agent status and not forcing, skip to be safe
+                // This is expected behavior for workspaces that are waking up from auto-stop
+                // or experiencing temporary network issues - not an error condition
+                if (!agentCheck.verified && !options.force) {
+                    console.log(`[provisioner] Skipped workspace ${workspaceId.substring(0, 8)}: workspace unreachable (will update on next restart)`);
+                    return {
+                        result: WorkspaceProvisioner.UpdateResult.SKIPPED_VERIFICATION_FAILED,
+                        workspaceId,
+                        machineState,
+                        // Use 'reason' instead of 'error' - this is expected behavior, not an error
+                        reason: 'Workspace unreachable - will update on next restart or when accessible',
+                    };
+                }
+                if (agentCheck.hasActiveAgents && !options.force) {
+                    // Has active agents and not forcing - skip
+                    console.log(`[provisioner] Skipped workspace ${workspaceId.substring(0, 8)}: ${agentCheck.agentCount} active agents`);
+                    return {
+                        result: WorkspaceProvisioner.UpdateResult.SKIPPED_ACTIVE_AGENTS,
+                        workspaceId,
+                        machineState,
+                        agentCount: agentCheck.agentCount,
+                        agents: agentCheck.agents,
+                    };
+                }
+                // Update the image config
+                await flyProvisioner.updateMachineImage(workspace, newImage);
+                if (options.skipRestart) {
+                    // Config updated but not restarting - will apply on next restart/auto-stop-wake
+                    console.log(`[provisioner] Updated workspace ${workspaceId.substring(0, 8)} config (restart skipped)`);
+                    return {
+                        result: WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART,
+                        workspaceId,
+                        machineState,
+                        agentCount: agentCheck.agentCount,
+                        agents: agentCheck.agents,
+                    };
+                }
+                // Restart to apply new image
+                await flyProvisioner.restart(workspace);
+                console.log(`[provisioner] Updated and restarted workspace ${workspaceId.substring(0, 8)}`);
+                return {
+                    result: WorkspaceProvisioner.UpdateResult.UPDATED,
+                    workspaceId,
+                    machineState,
+                    agentCount: agentCheck.agentCount,
+                };
+            }
+            // Unknown state
+            return {
+                result: WorkspaceProvisioner.UpdateResult.SKIPPED_NOT_RUNNING,
+                workspaceId,
+                machineState,
+            };
+        }
+        catch (error) {
+            console.error(`[provisioner] Error updating workspace ${workspaceId.substring(0, 8)}:`, error);
+            return {
+                result: WorkspaceProvisioner.UpdateResult.ERROR,
+                workspaceId,
+                error: error.message,
+            };
+        }
+    }
+    /**
+     * Gracefully update all workspaces to a new image
+     *
+     * Processes workspaces in batches, respecting active agents unless forced.
+     * Returns detailed results for each workspace.
+     *
+     * @param newImage - New Docker image to use
+     * @param options - Update options
+     * @returns Summary and per-workspace results
+     */
+    async gracefulUpdateAllImages(newImage, options = {}) {
+        // Get all workspaces to update
+        let workspaces;
+        if (options.workspaceIds?.length) {
+            // Specific workspaces
+            workspaces = (await Promise.all(options.workspaceIds.map(id => db.workspaces.findById(id)))).filter((w) => w !== null);
+        }
+        else if (options.userIds?.length) {
+            // Workspaces for specific users
+            const allWorkspaces = await Promise.all(options.userIds.map(userId => db.workspaces.findByUserId(userId)));
+            workspaces = allWorkspaces.flat();
+        }
+        else {
+            // All workspaces - need to query by status to get running ones
+            // For now, we'll get all workspaces from the provisioning provider
+            workspaces = await db.workspaces.findAll();
+        }
+        // Filter to only Fly.io workspaces
+        workspaces = workspaces.filter(w => w.computeProvider === 'fly' && w.computeId);
+        console.log(`[provisioner] Starting graceful update of ${workspaces.length} workspaces to ${newImage}`);
+        const batchSize = options.batchSize ?? 5;
+        const results = [];
+        // Process in batches
+        for (let i = 0; i < workspaces.length; i += batchSize) {
+            const batch = workspaces.slice(i, i + batchSize);
+            const batchResults = await Promise.all(batch.map(workspace => this.gracefulUpdateImage(workspace.id, newImage, {
+                force: options.force,
+                skipRestart: options.skipRestart,
+            })));
+            results.push(...batchResults);
+            // Small delay between batches to avoid overwhelming Fly API
+            if (i + batchSize < workspaces.length) {
+                await wait(1000);
+            }
+        }
+        // Compute summary
+        const summary = {
+            total: results.length,
+            updated: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.UPDATED).length,
+            pendingRestart: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART).length,
+            skippedActiveAgents: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_ACTIVE_AGENTS).length,
+            skippedVerificationFailed: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_VERIFICATION_FAILED).length,
+            skippedNotRunning: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_NOT_RUNNING).length,
+            errors: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.ERROR).length,
+        };
+        console.log(`[provisioner] Graceful update complete:`, summary);
+        return { summary, results };
+    }
+}
+// Singleton instance
+let _provisioner = null;
+export function getProvisioner() {
+    if (!_provisioner) {
+        _provisioner = new WorkspaceProvisioner();
+    }
+    return _provisioner;
+}
+//# sourceMappingURL=index.js.map