agent-relay 1.2.0 → 1.3.0

package/dist/cloud/provisioner/index.js

@@ -6,11 +6,49 @@
 import * as crypto from 'crypto';
 import { getConfig } from '../config.js';
 import { db } from '../db/index.js';
-import { vault } from '../vault/index.js';
 import { nangoService } from '../services/nango.js';
+import { canAutoScale, canScaleToTier, getResourceTierForPlan, } from '../services/planLimits.js';
+import { deriveSshPassword } from '../services/ssh-security.js';
 const WORKSPACE_PORT = 3888;
+const CODEX_OAUTH_PORT = 1455; // Codex CLI OAuth callback port - must be mapped for local dev
 const FETCH_TIMEOUT_MS = 10_000;
 const WORKSPACE_IMAGE = process.env.WORKSPACE_IMAGE || 'ghcr.io/agentworkforce/relay-workspace:latest';
+// In-memory tracker for provisioning progress (workspace ID -> progress)
+const provisioningProgress = new Map();
+/**
+ * Update the provisioning stage for a workspace
+ */
+function updateProvisioningStage(workspaceId, stage) {
+    const existing = provisioningProgress.get(workspaceId);
+    provisioningProgress.set(workspaceId, {
+        stage,
+        startedAt: existing?.startedAt ?? Date.now(),
+        updatedAt: Date.now(),
+    });
+    console.log(`[provisioner] Workspace ${workspaceId.substring(0, 8)} stage: ${stage}`);
+}
+/**
+ * Get the current provisioning stage for a workspace
+ */
+export function getProvisioningStage(workspaceId) {
+    return provisioningProgress.get(workspaceId) ?? null;
+}
+/**
+ * Clear provisioning progress (call when complete or failed)
+ */
+function clearProvisioningProgress(workspaceId) {
+    provisioningProgress.delete(workspaceId);
+}
+/**
+ * Schedule cleanup of provisioning progress after a delay
+ * This gives the frontend time to poll and see the 'complete' stage
+ */
+function scheduleProgressCleanup(workspaceId, delayMs = 30_000) {
+    setTimeout(() => {
+        clearProvisioningProgress(workspaceId);
+        console.log(`[provisioner] Cleaned up provisioning progress for ${workspaceId.substring(0, 8)}`);
+    }, delayMs);
+}
 /**
  * Get a fresh GitHub App installation token from Nango.
  * Looks up the user's connected repositories to find a valid Nango connection.
@@ -33,20 +71,6 @@ async function getGithubAppTokenForUser(userId) {
         return null;
     }
 }
-async function loadCredentialToken(userId, provider) {
-    try {
-        const cred = await vault.getCredential(userId, provider);
-        if (cred?.accessToken) {
-            return cred.accessToken;
-        }
-    }
-    catch (error) {
-        console.warn(`Failed to decrypt ${provider} credential from vault; trying raw storage fallback`, error);
-        const raw = await db.credentials.findByUserAndProvider(userId, provider);
-        return raw?.accessToken ?? null;
-    }
-    return null;
-}
 async function wait(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -88,11 +112,108 @@ async function softHealthCheck(url) {
         console.warn(`[health] Failed to reach ${url}/health`, error);
     }
 }
+/**
+ * Wait for machine to be in "started" state using Fly.io's /wait endpoint
+ * This is more efficient than polling - the API blocks until the state is reached
+ * @see https://fly.io/docs/machines/api/machines-resource/#wait-for-a-machine-to-reach-a-specific-state
+ */
+async function waitForMachineStarted(apiToken, appName, machineId, timeoutSeconds = 120) {
+    console.log(`[provisioner] Waiting for machine ${machineId} to start (timeout: ${timeoutSeconds}s)...`);
+    // Fly.io /wait endpoint has max timeout of 60s, so we need to loop for longer waits
+    const maxSingleWait = 60;
+    const startTime = Date.now();
+    const deadline = startTime + timeoutSeconds * 1000;
+    while (Date.now() < deadline) {
+        const remainingMs = deadline - Date.now();
+        const waitSeconds = Math.min(maxSingleWait, Math.ceil(remainingMs / 1000));
+        if (waitSeconds <= 0)
+            break;
+        try {
+            // Use Fly.io's /wait endpoint - blocks until machine reaches target state
+            // timeout is an integer in seconds (max 60)
+            const res = await fetch(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}/wait?state=started&timeout=${waitSeconds}`, {
+                headers: { Authorization: `Bearer ${apiToken}` },
+            });
+            if (res.ok) {
+                console.log(`[provisioner] Machine ${machineId} is now started`);
+                return;
+            }
+            // 408 = timeout, machine didn't reach state in time - try again if we have time
+            if (res.status === 408) {
+                console.log(`[provisioner] Machine ${machineId} not ready yet, continuing to wait...`);
+                continue;
+            }
+            // Other error
+            const errorText = await res.text();
+            throw new Error(`Wait for machine failed: ${res.status} ${errorText}`);
+        }
+        catch (error) {
+            if (error instanceof Error && error.message.includes('Wait for machine failed')) {
+                throw error;
+            }
+            console.warn(`[provisioner] Error waiting for machine:`, error);
+            throw new Error(`Failed to wait for machine ${machineId}: ${error.message}`);
+        }
+    }
+    // Timeout reached - get current state for error message
+    const stateRes = await fetch(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}`, { headers: { Authorization: `Bearer ${apiToken}` } });
+    const machine = stateRes.ok ? (await stateRes.json()) : { state: 'unknown' };
+    throw new Error(`Machine ${machineId} did not start within ${timeoutSeconds}s (last state: ${machine.state})`);
+}
+/**
+ * Wait for health check to pass (with DNS propagation time)
+ * Tries internal Fly network first if available, then falls back to public URL
+ */
+async function waitForHealthy(url, appName, maxWaitMs = 90_000) {
+    const startTime = Date.now();
+    // Build list of URLs to try - internal first (faster, more reliable from inside Fly)
+    const urlsToTry = [];
+    // If running on Fly and app name provided, try internal network first
+    const isOnFly = !!process.env.FLY_APP_NAME;
+    if (isOnFly && appName) {
+        urlsToTry.push(`http://${appName}.internal:8080/health`);
+    }
+    // Always add the public URL as fallback
+    urlsToTry.push(`${url.replace(/\/$/, '')}/health`);
+    console.log(`[provisioner] Waiting for workspace to become healthy (trying: ${urlsToTry.join(', ')})...`);
+    while (Date.now() - startTime < maxWaitMs) {
+        // Try each URL in order
+        for (const healthUrl of urlsToTry) {
+            try {
+                const controller = new AbortController();
+                const timer = setTimeout(() => controller.abort(), 5_000);
+                const res = await fetch(healthUrl, {
+                    method: 'GET',
+                    signal: controller.signal,
+                });
+                clearTimeout(timer);
+                if (res.ok) {
+                    console.log(`[provisioner] Health check passed via ${healthUrl}`);
+                    return;
+                }
+                console.log(`[provisioner] Health check to ${healthUrl} returned ${res.status}`);
+            }
+            catch (error) {
+                const elapsed = Math.round((Date.now() - startTime) / 1000);
+                const errMsg = error.message;
+                // Only log detailed error for last URL attempt
+                if (healthUrl === urlsToTry[urlsToTry.length - 1]) {
+                    console.log(`[provisioner] Health check failed (${elapsed}s elapsed): ${errMsg}`);
+                }
+            }
+        }
+        await wait(3000);
+    }
+    // Don't throw - workspace is provisioned, health check is best-effort
+    console.warn(`[provisioner] Health check did not pass within ${maxWaitMs}ms, continuing anyway`);
+}
+// Resource tiers sized for Claude Code agents (~1-2GB RAM per agent)
+// cpuKind: 'shared' = cheaper but can be throttled, 'performance' = dedicated
 export const RESOURCE_TIERS = {
-    small: { name: 'small', cpuCores: 1, memoryMb: 512, maxAgents: 5 },
-    medium: { name: 'medium', cpuCores: 2, memoryMb: 1024, maxAgents: 10 },
-    large: { name: 'large', cpuCores: 4, memoryMb: 2048, maxAgents: 20 },
-    xlarge: { name: 'xlarge', cpuCores: 8, memoryMb: 4096, maxAgents: 50 },
+    small: { name: 'small', cpuCores: 2, memoryMb: 2048, maxAgents: 2, cpuKind: 'shared' },
+    medium: { name: 'medium', cpuCores: 2, memoryMb: 4096, maxAgents: 5, cpuKind: 'shared' },
+    large: { name: 'large', cpuCores: 4, memoryMb: 8192, maxAgents: 10, cpuKind: 'performance' },
+    xlarge: { name: 'xlarge', cpuCores: 8, memoryMb: 16384, maxAgents: 20, cpuKind: 'performance' },
 };
 /**
  * Fly.io provisioner
@@ -105,6 +226,8 @@ class FlyProvisioner {
     cloudApiUrl;
     sessionSecret;
     registryAuth;
+    snapshotRetentionDays;
+    volumeSizeGb;
     constructor() {
         const config = getConfig();
         if (!config.compute.fly) {
@@ -117,6 +240,9 @@ class FlyProvisioner {
         this.registryAuth = config.compute.fly.registryAuth;
         this.cloudApiUrl = config.publicUrl;
         this.sessionSecret = config.sessionSecret;
+        // Snapshot settings: default 14 days retention, 10GB volume
+        this.snapshotRetentionDays = Math.min(60, Math.max(1, config.compute.fly.snapshotRetentionDays ?? 14));
+        this.volumeSizeGb = config.compute.fly.volumeSizeGb ?? 10;
     }
     /**
      * Generate a workspace token for API authentication
@@ -128,8 +254,91 @@ class FlyProvisioner {
             .update(`workspace:${workspaceId}`)
             .digest('hex');
     }
+    /**
+     * Create a volume with automatic snapshot settings
+     * Fly.io takes daily snapshots automatically; we configure retention
+     */
+    async createVolume(appName) {
+        const volumeName = 'workspace_data';
+        console.log(`[fly] Creating volume ${volumeName} with ${this.snapshotRetentionDays}-day snapshot retention...`);
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                name: volumeName,
+                region: this.region,
+                size_gb: this.volumeSizeGb,
+                // Enable automatic daily snapshots (default is true, but be explicit)
+                auto_backup_enabled: true,
+                // Retain snapshots for configured days (default 5, we use 14)
+                snapshot_retention: this.snapshotRetentionDays,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Failed to create volume: ${error}`);
+        }
+        const volume = await response.json();
+        console.log(`[fly] Volume ${volume.id} created with auto-snapshots (${this.snapshotRetentionDays} days retention)`);
+        return volume;
+    }
+    /**
+     * Create an on-demand snapshot of a workspace volume
+     * Use before risky operations or as manual backup
+     */
+    async createSnapshot(appName, volumeId) {
+        console.log(`[fly] Creating on-demand snapshot for volume ${volumeId}...`);
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes/${volumeId}/snapshots`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Failed to create snapshot: ${error}`);
+        }
+        const snapshot = await response.json();
+        console.log(`[fly] Snapshot ${snapshot.id} created`);
+        return snapshot;
+    }
+    /**
+     * List snapshots for a workspace volume
+     */
+    async listSnapshots(appName, volumeId) {
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes/${volumeId}/snapshots`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!response.ok) {
+            return [];
+        }
+        return await response.json();
+    }
+    /**
+     * Get volume info for a workspace
+     */
+    async getVolume(appName) {
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!response.ok) {
+            return null;
+        }
+        const volumes = await response.json();
+        return volumes.find(v => v.name === 'workspace_data') || null;
+    }
     async provision(workspace, credentials) {
         const appName = `ar-${workspace.id.substring(0, 8)}`;
+        // Stage: Creating workspace
+        updateProvisioningStage(workspace.id, 'creating');
         // Create Fly app
         await fetchWithRetry('https://api.machines.dev/v1/apps', {
             method: 'POST',
@@ -142,10 +351,82 @@ class FlyProvisioner {
                 org_slug: this.org,
             }),
         });
+        // Stage: Networking
+        updateProvisioningStage(workspace.id, 'networking');
+        // Allocate IPs for the app (required for public DNS)
+        // Must use GraphQL API - Machines REST API doesn't support IP allocation
+        // Shared IPv4 is free, IPv6 is free
+        console.log(`[fly] Allocating IPs for ${appName}...`);
+        const allocateIP = async (type) => {
+            try {
+                // Map our type to Fly GraphQL enum
+                const graphqlType = type === 'shared_v4' ? 'shared_v4' : 'v6';
+                const res = await fetchWithRetry('https://api.fly.io/graphql', {
+                    method: 'POST',
+                    headers: {
+                        Authorization: `Bearer ${this.apiToken}`,
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        query: `
+              mutation AllocateIPAddress($input: AllocateIPAddressInput!) {
+                allocateIpAddress(input: $input) {
+                  ipAddress {
+                    id
+                    address
+                    type
+                  }
+                }
+              }
+            `,
+                        variables: {
+                            input: {
+                                appId: appName,
+                                type: graphqlType,
+                            },
+                        },
+                    }),
+                });
+                if (!res.ok) {
+                    const errorText = await res.text();
+                    console.warn(`[fly] Failed to allocate ${type}: ${res.status} ${errorText}`);
+                    return false;
+                }
+                const data = await res.json();
+                if (data.errors?.length) {
+                    // Ignore "already allocated" errors
+                    const alreadyAllocated = data.errors.some(e => e.message.includes('already') || e.message.includes('exists'));
+                    if (!alreadyAllocated) {
+                        console.warn(`[fly] GraphQL error allocating ${type}: ${data.errors[0].message}`);
+                        return false;
+                    }
+                    console.log(`[fly] IP ${type} already allocated`);
+                    return true;
+                }
+                const address = data.data?.allocateIpAddress?.ipAddress?.address;
+                console.log(`[fly] Allocated ${type}: ${address}`);
+                return true;
+            }
+            catch (err) {
+                console.warn(`[fly] Failed to allocate ${type}: ${err.message}`);
+                return false;
+            }
+        };
+        const [sharedV4Result, v6Result] = await Promise.all([
+            allocateIP('shared_v4'),
+            allocateIP('v6'),
+        ]);
+        console.log(`[fly] IP allocation results: shared_v4=${sharedV4Result}, v6=${v6Result}`);
+        // Stage: Secrets
+        updateProvisioningStage(workspace.id, 'secrets');
         // Set secrets (provider credentials)
         const secrets = {};
         for (const [provider, token] of credentials) {
             secrets[`${provider.toUpperCase()}_TOKEN`] = token;
+            // Also set GH_TOKEN for gh CLI compatibility
+            if (provider === 'github') {
+                secrets['GH_TOKEN'] = token;
+            }
         }
         if (Object.keys(secrets).length > 0) {
             await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/secrets`, {
@@ -164,6 +445,33 @@ class FlyProvisioner {
         if (customHostname) {
             await this.allocateCertificate(appName, customHostname);
         }
+        // Stage: Machine (includes volume creation)
+        updateProvisioningStage(workspace.id, 'machine');
+        // Create volume with automatic daily snapshots before machine
+        // Fly.io takes daily snapshots automatically; we configure retention
+        const volume = await this.createVolume(appName);
+        // Determine instance size based on user's plan
+        // Free tier: 1 CPU, 2GB (~$10/mo) - Claude needs 2GB minimum
+        // Paid tiers: 2 CPU, 2GB (~$15/mo)
+        // Introductory bonus: Free users get Pro-level resources for first 14 days
+        const user = await db.users.findById(workspace.userId);
+        const userPlan = user?.plan || 'free';
+        const isFreeTier = userPlan === 'free';
+        // Check if user is in introductory period (first 14 days)
+        const INTRO_PERIOD_DAYS = 14;
+        const userCreatedAt = user?.createdAt ? new Date(user.createdAt) : new Date();
+        const daysSinceSignup = (Date.now() - userCreatedAt.getTime()) / (1000 * 60 * 60 * 24);
+        const isIntroPeriod = isFreeTier && daysSinceSignup < INTRO_PERIOD_DAYS;
+        const guestConfig = {
+            cpu_kind: 'shared',
+            cpus: isIntroPeriod ? 2 : (isFreeTier ? 1 : 2), // Intro gets 2 CPUs like Pro
+            memory_mb: isIntroPeriod ? 4096 : 2048, // Intro gets 4GB like Pro
+        };
+        if (isIntroPeriod) {
+            const daysRemaining = Math.ceil(INTRO_PERIOD_DAYS - daysSinceSignup);
+            console.log(`[fly] Introductory bonus active (${daysRemaining} days remaining) - 2 CPU / 4GB`);
+        }
+        console.log(`[fly] Using ${guestConfig.cpus} CPU / ${guestConfig.memory_mb}MB for ${userPlan} plan`);
         // Create machine with auto-stop/start for cost optimization
         const machineResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines`, {
             method: 'POST',
@@ -185,6 +493,7 @@ class FlyProvisioner {
                     }),
                     env: {
                         WORKSPACE_ID: workspace.id,
+                        WORKSPACE_OWNER_USER_ID: workspace.userId,
                         SUPERVISOR_ENABLED: String(workspace.config.supervisorEnabled ?? false),
                         MAX_AGENTS: String(workspace.config.maxAgents ?? 10),
                         REPOSITORIES: (workspace.config.repositories ?? []).join(','),
@@ -194,26 +503,76 @@ class FlyProvisioner {
                         // Git gateway configuration
                         CLOUD_API_URL: this.cloudApiUrl,
                         WORKSPACE_TOKEN: this.generateWorkspaceToken(workspace.id),
+                        // SSH for CLI tunneling (Codex OAuth callback forwarding)
+                        // Each workspace gets a unique password derived from its ID + secret salt
+                        ENABLE_SSH: 'true',
+                        SSH_PASSWORD: deriveSshPassword(workspace.id),
                     },
                     services: [
                         {
                             ports: [
-                                { port: 443, handlers: ['tls', 'http'] },
+                                {
+                                    port: 443,
+                                    handlers: ['tls', 'http'],
+                                    // Force HTTP/1.1 to backend for WebSocket upgrade compatibility
+                                    // HTTP/2 doesn't support traditional WebSocket upgrade mechanism
+                                    http_options: {
+                                        h2_backend: false,
+                                    },
+                                },
                                 { port: 80, handlers: ['http'] },
                             ],
                             protocol: 'tcp',
                             internal_port: WORKSPACE_PORT,
-                            // Auto-stop after 5 minutes of inactivity
-                            auto_stop_machines: true,
+                            // Auto-stop after inactivity to reduce costs
+                            // Fly Proxy automatically wakes machines on incoming requests
+                            auto_stop_machines: 'stop', // stop (not suspend) for faster wake
+                            auto_start_machines: true,
+                            min_machines_running: 0,
+                            // Idle timeout before auto-stop (in seconds)
+                            // Longer timeout = better UX, shorter = lower costs
+                            concurrency: {
+                                type: 'requests',
+                                soft_limit: 25,
+                                hard_limit: 50,
+                            },
+                        },
+                        // SSH service for CLI tunneling (Codex OAuth callback forwarding)
+                        // Exposes port 2222 publicly for SSH connections from user's machine
+                        {
+                            ports: [
+                                {
+                                    port: 2222,
+                                    handlers: [], // Empty handlers = raw TCP passthrough
+                                },
+                            ],
+                            protocol: 'tcp',
+                            internal_port: 2222,
+                            // SSH connections should also wake the machine
+                            auto_stop_machines: 'stop',
                             auto_start_machines: true,
                             min_machines_running: 0,
                         },
                     ],
-                    guest: {
-                        cpu_kind: 'shared',
-                        cpus: 1,
-                        memory_mb: 512,
+                    checks: {
+                        health: {
+                            type: 'http',
+                            port: WORKSPACE_PORT,
+                            path: '/health',
+                            interval: '30s',
+                            timeout: '5s',
+                            grace_period: '10s',
+                        },
                     },
+                    // Instance size based on plan - free tier gets smaller instance
+                    guest: guestConfig,
+                    // Mount the volume we created with snapshot settings
+                    mounts: [
+                        {
+                            volume: volume.id,
+                            path: '/data',
+                        },
+                    ],
                 },
             }),
         });
@@ -226,7 +585,19 @@ class FlyProvisioner {
         const publicUrl = customHostname
             ? `https://${customHostname}`
             : `https://${appName}.fly.dev`;
-        await softHealthCheck(publicUrl);
+        // Stage: Booting
+        updateProvisioningStage(workspace.id, 'booting');
+        // Wait for machine to be in started state
+        await waitForMachineStarted(this.apiToken, appName, machine.id);
+        // Stage: Health check
+        updateProvisioningStage(workspace.id, 'health');
+        // Wait for health check to pass (includes DNS propagation time)
+        // Pass appName to enable internal Fly network health checks
+        await waitForHealthy(publicUrl, appName);
+        // Stage: Complete
+        updateProvisioningStage(workspace.id, 'complete');
+        // Schedule cleanup of provisioning progress after 30s (gives frontend time to see 'complete')
+        scheduleProgressCleanup(workspace.id);
         return {
             computeId: machine.id,
             publicUrl,
@@ -298,13 +669,24 @@ class FlyProvisioner {
     }
     /**
      * Resize workspace - vertical scaling via Fly Machines API
+     * @param skipRestart - If true, config is saved but machine won't restart (changes apply on next start)
      */
-    async resize(workspace, tier) {
-        if (!workspace.computeId)
-            return;
-        const appName = `ar-${workspace.id.substring(0, 8)}`;
+    async resize(workspaceOrId, tier, skipRestart = false) {
+        const workspaceId = typeof workspaceOrId === 'string' ? workspaceOrId : workspaceOrId.id;
+        const computeId = typeof workspaceOrId === 'string' ? undefined : workspaceOrId.computeId;
+        // If passed just an ID, look up the workspace
+        let machineId = computeId;
+        if (!machineId) {
+            const workspace = await db.workspaces.findById(workspaceId);
+            if (!workspace?.computeId)
+                return;
+            machineId = workspace.computeId;
+        }
+        const appName = `ar-${workspaceId.substring(0, 8)}`;
         // Update machine configuration
-        await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+        // If running: reboots with new specs (unless skip_launch: true)
+        // If stopped: config saved, applies on next start
+        await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}`, {
             method: 'POST',
             headers: {
                 Authorization: `Bearer ${this.apiToken}`,
@@ -313,7 +695,8 @@ class FlyProvisioner {
             body: JSON.stringify({
                 config: {
                     guest: {
-                        cpu_kind: tier.cpuCores <= 2 ? 'shared' : 'performance',
+                        // Use tier-specific CPU type (shared for cost, performance for power)
+                        cpu_kind: tier.cpuKind,
                         cpus: tier.cpuCores,
                         memory_mb: tier.memoryMb,
                     },
@@ -321,9 +704,11 @@ class FlyProvisioner {
                         MAX_AGENTS: String(tier.maxAgents),
                     },
                 },
+                skip_launch: skipRestart, // If true, don't restart - changes apply on next start
             }),
         });
-        console.log(`[fly] Resized workspace ${workspace.id} to ${tier.name} (${tier.cpuCores} CPU, ${tier.memoryMb}MB RAM)`);
+        const restartNote = skipRestart ? ' (will apply on next restart)' : ' (restarting)';
+        console.log(`[fly] Resized workspace ${workspaceId.substring(0, 8)} to ${tier.name} (${tier.cpuCores} CPU, ${tier.memoryMb}MB RAM)${restartNote}`);
     }
     /**
      * Update the max agent limit for a workspace
@@ -377,6 +762,118 @@ class FlyProvisioner {
             return RESOURCE_TIERS.medium;
         return RESOURCE_TIERS.small;
     }
+    /**
+     * Update machine image without restarting
+     * Note: The machine needs to be restarted later to use the new image
+     */
+    async updateMachineImage(workspace, newImage) {
+        if (!workspace.computeId)
+            return;
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        // Get current machine config first
+        const getResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!getResponse.ok) {
+            throw new Error(`Failed to get machine config: ${await getResponse.text()}`);
+        }
+        const machine = await getResponse.json();
+        // Update the image in the config
+        const updatedConfig = {
+            ...machine.config,
+            image: newImage,
+            // Include registry auth if configured
+            ...(this.registryAuth && {
+                image_registry_auth: {
+                    registry: 'ghcr.io',
+                    username: this.registryAuth.username,
+                    password: this.registryAuth.password,
+                },
+            }),
+        };
+        // Update machine with new image config (skip_launch keeps it in current state)
+        const updateResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}?skip_launch=true`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ config: updatedConfig }),
+        });
+        if (!updateResponse.ok) {
+            throw new Error(`Failed to update machine image: ${await updateResponse.text()}`);
+        }
+        console.log(`[fly] Updated machine image for workspace ${workspace.id.substring(0, 8)} to ${newImage}`);
+    }
+    /**
+     * Check if workspace has active agents by querying the daemon
+     */
+    async checkActiveAgents(workspace) {
+        if (!workspace.publicUrl) {
+            return { hasActiveAgents: false, agentCount: 0, agents: [] };
+        }
+        try {
+            // Use internal Fly network URL if available (more reliable)
+            const appName = `ar-${workspace.id.substring(0, 8)}`;
+            const isOnFly = !!process.env.FLY_APP_NAME;
+            const baseUrl = isOnFly
+                ? `http://${appName}.internal:3888`
+                : workspace.publicUrl;
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), 10_000);
+            const response = await fetch(`${baseUrl}/api/agents`, {
+                method: 'GET',
+                headers: {
+                    'Accept': 'application/json',
+                },
+                signal: controller.signal,
+            });
+            clearTimeout(timer);
+            if (!response.ok) {
+                console.warn(`[fly] Failed to check agents for ${workspace.id.substring(0, 8)}: ${response.status}`);
+                return { hasActiveAgents: false, agentCount: 0, agents: [] };
+            }
+            const data = await response.json();
+            const agents = data.agents || [];
+            // Consider agents with 'active' or 'idle' activity state as active
+            // 'disconnected' agents are not active
+            const activeAgents = agents.filter(a => a.status === 'running' || a.activityState === 'active' || a.activityState === 'idle');
+            return {
+                hasActiveAgents: activeAgents.length > 0,
+                agentCount: activeAgents.length,
+                agents: agents.map(a => ({ name: a.name, status: a.status || a.activityState || 'unknown' })),
+            };
+        }
+        catch (error) {
+            // Workspace might be stopped or unreachable - treat as no active agents
+            console.warn(`[fly] Could not reach workspace ${workspace.id.substring(0, 8)} to check agents:`, error.message);
+            return { hasActiveAgents: false, agentCount: 0, agents: [] };
+        }
+    }
+    /**
+     * Get the current machine state
+     */
+    async getMachineState(workspace) {
+        if (!workspace.computeId)
+            return 'unknown';
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        try {
+            const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+                headers: {
+                    Authorization: `Bearer ${this.apiToken}`,
+                },
+            });
+            if (!response.ok)
+                return 'unknown';
+            const machine = await response.json();
+            return machine.state;
+        }
+        catch {
+            return 'unknown';
+        }
+    }
 }
 /**
  * Railway provisioner
@@ -457,6 +954,7 @@ class RailwayProvisioner {
         // Set environment variables
         const envVars = {
             WORKSPACE_ID: workspace.id,
+            WORKSPACE_OWNER_USER_ID: workspace.userId,
             SUPERVISOR_ENABLED: String(workspace.config.supervisorEnabled ?? false),
             MAX_AGENTS: String(workspace.config.maxAgents ?? 10),
             REPOSITORIES: (workspace.config.repositories ?? []).join(','),
@@ -468,6 +966,10 @@ class RailwayProvisioner {
         };
         for (const [provider, token] of credentials) {
             envVars[`${provider.toUpperCase()}_TOKEN`] = token;
+            // Also set GH_TOKEN for gh CLI compatibility
+            if (provider === 'github') {
+                envVars['GH_TOKEN'] = token;
+            }
         }
         await fetchWithRetry('https://backboard.railway.app/graphql/v2', {
             method: 'POST',
@@ -671,6 +1173,7 @@ class DockerProvisioner {
         // Build environment variables
         const envArgs = [
             `-e WORKSPACE_ID=${workspace.id}`,
+            `-e WORKSPACE_OWNER_USER_ID=${workspace.userId}`,
             `-e SUPERVISOR_ENABLED=${workspace.config.supervisorEnabled ?? false}`,
             `-e MAX_AGENTS=${workspace.config.maxAgents ?? 10}`,
             `-e REPOSITORIES=${(workspace.config.repositories ?? []).join(',')}`,
@@ -682,10 +1185,17 @@ class DockerProvisioner {
         ];
         for (const [provider, token] of credentials) {
             envArgs.push(`-e ${provider.toUpperCase()}_TOKEN=${token}`);
+            // Also set GH_TOKEN for gh CLI compatibility
+            if (provider === 'github') {
+                envArgs.push(`-e GH_TOKEN=${token}`);
+            }
         }
         // Run container
         const { execSync } = await import('child_process');
         const hostPort = 3000 + Math.floor(Math.random() * 1000);
+        // SSH port for tunneling (Codex OAuth callback forwarding)
+        // Derive from hostPort to avoid collisions: API port 3500 -> SSH port 22500
+        const sshHostPort = 22000 + (hostPort - 3000);
         // When running in Docker, connect to the same network for container-to-container communication
         const runningInDocker = process.env.RUNNING_IN_DOCKER === 'true';
         const networkArg = runningInDocker ? '--network agent-relay-dev' : '';
@@ -699,7 +1209,18 @@ class DockerProvisioner {
             console.log('[provisioner] Dev mode: mounting local dist/ and docs/ folders into workspace container');
         }
         try {
-            execSync(`docker run -d --user root --name ${containerName} ${networkArg} ${volumeArgs} -p ${hostPort}:${WORKSPACE_PORT} ${envArgs.join(' ')} ${WORKSPACE_IMAGE}`, { stdio: 'pipe' });
+            // Map workspace API port and SSH port (for tunneling)
+            // SSH is used by CLI to forward localhost:1455 to workspace container for Codex OAuth
+            // Set CODEX_DIRECT_PORT=true to also map port 1455 directly (for debugging only)
+            const directCodexPort = process.env.CODEX_DIRECT_PORT === 'true';
+            const portMappings = directCodexPort
+                ? `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:2222 -p ${CODEX_OAUTH_PORT}:${CODEX_OAUTH_PORT}`
+                : `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:2222`;
+            // Enable SSH in the container for tunneling
+            // Each workspace gets a unique password derived from its ID + secret salt
+            envArgs.push('-e ENABLE_SSH=true');
+            envArgs.push(`-e SSH_PASSWORD=${deriveSshPassword(workspace.id)}`);
+            execSync(`docker run -d --user root --name ${containerName} ${networkArg} ${volumeArgs} ${portMappings} ${envArgs.join(' ')} ${WORKSPACE_IMAGE}`, { stdio: 'pipe' });
             const publicUrl = `http://localhost:${hostPort}`;
             // Wait for container to be healthy before returning
             // When running in Docker, use the internal container name for health check
@@ -710,6 +1231,7 @@ class DockerProvisioner {
             return {
                 computeId: containerName,
                 publicUrl,
+                sshPort: sshHostPort,
             };
         }
         catch (error) {
@@ -791,6 +1313,7 @@ export class WorkspaceProvisioner {
     }
     /**
      * Provision a new workspace (one-click)
+     * Returns immediately with 'provisioning' status and runs actual provisioning in background
      */
     async provision(config) {
         // Create workspace record
@@ -814,14 +1337,58 @@ export class WorkspaceProvisioner {
         });
         // Auto-accept the creator's membership
         await db.workspaceMembers.acceptInvite(workspace.id, config.userId);
-        // Get credentials
-        const credentials = new Map();
-        for (const provider of config.providers) {
-            const token = await loadCredentialToken(config.userId, provider);
-            if (token) {
-                credentials.set(provider, token);
+        // Link repositories to this workspace
+        // This enables auto-access for users with GitHub access to these repos
+        for (const repoFullName of config.repositories) {
+            try {
+                // Find the user's repo record (may not exist if user didn't import it first)
+                const userRepos = await db.repositories.findByUserId(config.userId);
+                const repoRecord = userRepos.find(r => r.githubFullName.toLowerCase() === repoFullName.toLowerCase());
+                if (repoRecord) {
+                    await db.repositories.assignToWorkspace(repoRecord.id, workspace.id);
+                    console.log(`[provisioner] Linked repo ${repoFullName} to workspace ${workspace.id.substring(0, 8)}`);
+                }
+                else {
+                    // Create a placeholder repo record if it doesn't exist
+                    // This ensures the repo is tracked for workspace access checks
+                    console.log(`[provisioner] Creating repo record for ${repoFullName}`);
+                    const newRepo = await db.repositories.upsert({
+                        userId: config.userId,
+                        githubFullName: repoFullName,
+                        githubId: 0, // Will be updated when actually synced
+                        defaultBranch: 'main',
+                        isPrivate: true, // Assume private, will be updated
+                        workspaceId: workspace.id,
+                    });
+                    console.log(`[provisioner] Created and linked repo ${repoFullName} (id: ${newRepo.id.substring(0, 8)})`);
+                }
+            }
+            catch (err) {
+                console.warn(`[provisioner] Failed to link repo ${repoFullName}:`, err);
+                // Continue with other repos
             }
         }
+        // Initialize stage tracking immediately
+        updateProvisioningStage(workspace.id, 'creating');
+        // Run provisioning in the background so frontend can poll for stages
+        this.runProvisioningAsync(workspace, config).catch((error) => {
+            console.error(`[provisioner] Background provisioning failed for ${workspace.id}:`, error);
+        });
+        // Return immediately with 'provisioning' status
+        return {
+            workspaceId: workspace.id,
+            status: 'provisioning',
+        };
+    }
+    /**
+     * Run the actual provisioning work asynchronously
+     */
+    async runProvisioningAsync(workspace, config) {
+        // Build credentials map for workspace provisioning
+        // Note: Provider tokens (Claude, Codex, etc.) are no longer stored centrally.
+        // CLI tools authenticate directly on workspace instances.
+        // Only GitHub App tokens are obtained from Nango for repository cloning.
+        const credentials = new Map();
         // GitHub token is required for cloning repositories
         // Use direct token if provided (for testing), otherwise get from Nango
         if (config.repositories.length > 0) {
@@ -848,22 +1415,21 @@ export class WorkspaceProvisioner {
                 computeId,
                 publicUrl,
             });
-            return {
-                workspaceId: workspace.id,
-                status: 'running',
-                publicUrl,
-            };
+            // Schedule cleanup of provisioning progress after 30s (gives frontend time to see 'complete')
+            setTimeout(() => {
+                clearProvisioningProgress(workspace.id);
+                console.log(`[provisioner] Cleaned up provisioning progress for ${workspace.id.substring(0, 8)}`);
+            }, 30_000);
+            console.log(`[provisioner] Workspace ${workspace.id} provisioned successfully at ${publicUrl}`);
         }
         catch (error) {
             const errorMessage = error instanceof Error ? error.message : 'Unknown error';
             await db.workspaces.updateStatus(workspace.id, 'error', {
                 errorMessage,
             });
-            return {
-                workspaceId: workspace.id,
-                status: 'error',
-                error: errorMessage,
-            };
+            // Clear provisioning progress on error
+            clearProvisioningProgress(workspace.id);
+            console.error(`[provisioner] Workspace ${workspace.id} provisioning failed:`, errorMessage);
         }
     }
     /**
@@ -885,6 +1451,11 @@ export class WorkspaceProvisioner {
         if (!workspace) {
             throw new Error('Workspace not found');
         }
+        // During early provisioning, computeId isn't set yet
+        // Return the database status instead of querying the provider
+        if (!workspace.computeId && workspace.status === 'provisioning') {
+            return 'provisioning';
+        }
         const status = await this.provisioner.getStatus(workspace);
         // Update database if status changed
         if (status !== workspace.status) {
@@ -916,8 +1487,9 @@ export class WorkspaceProvisioner {
     }
     /**
      * Resize a workspace (vertical scaling)
+     * @param skipRestart - If true, config is saved but machine won't restart (changes apply on next start)
      */
-    async resize(workspaceId, tier) {
+    async resize(workspaceId, tier, skipRestart = false) {
         const workspace = await db.workspaces.findById(workspaceId);
         if (!workspace) {
             throw new Error('Workspace not found');
@@ -925,7 +1497,7 @@ export class WorkspaceProvisioner {
         if (!this.provisioner.resize) {
             throw new Error('Resize not supported by current compute provider');
         }
-        await this.provisioner.resize(workspace, tier);
+        await this.provisioner.resize(workspace, tier, skipRestart);
         // Update workspace config with new limits
         await db.workspaces.updateConfig(workspaceId, {
             ...workspace.config,
@@ -965,6 +1537,320 @@ export class WorkspaceProvisioner {
         const tierName = workspace.config.resourceTier || 'small';
         return RESOURCE_TIERS[tierName] || RESOURCE_TIERS.small;
     }
+    /**
+     * Get recommended tier based on agent count
+     * Uses 1.5-2GB per agent as baseline for Claude Code
+     */
+    getRecommendedTier(agentCount) {
+        // Find the smallest tier that supports this agent count
+        const tiers = Object.values(RESOURCE_TIERS).sort((a, b) => a.maxAgents - b.maxAgents);
+        for (const tier of tiers) {
+            if (tier.maxAgents >= agentCount) {
+                return tier;
+            }
+        }
+        // If agent count exceeds all tiers, return the largest
+        return RESOURCE_TIERS.xlarge;
+    }
+    /**
+     * Auto-scale workspace based on current agent count
+     * Respects plan limits - free tier cannot scale, others have max tier limits
+     * Returns { scaled: boolean, reason?: string }
+     */
+    async autoScale(workspaceId, currentAgentCount) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // Get user's plan
+        const user = await db.users.findById(workspace.userId);
+        const plan = user?.plan || 'free';
+        // Check if plan allows auto-scaling
+        if (!canAutoScale(plan)) {
+            return {
+                scaled: false,
+                reason: 'Auto-scaling requires Pro plan or higher',
+            };
+        }
+        const currentTier = await this.getCurrentTier(workspaceId);
+        const recommendedTier = this.getRecommendedTier(currentAgentCount);
+        // Only scale UP, never down (to avoid disruption)
+        if (recommendedTier.memoryMb <= currentTier.memoryMb) {
+            return {
+                scaled: false,
+                currentTier: currentTier.name,
+            };
+        }
+        // Check if plan allows scaling to the recommended tier
+        if (!canScaleToTier(plan, recommendedTier.name)) {
+            // Find the max tier allowed for this plan
+            const maxTierName = getResourceTierForPlan(plan);
+            const maxTier = RESOURCE_TIERS[maxTierName];
+            if (maxTier.memoryMb <= currentTier.memoryMb) {
+                return {
+                    scaled: false,
+                    reason: `Already at max tier (${currentTier.name}) for ${plan} plan`,
+                    currentTier: currentTier.name,
+                };
+            }
+            // Scale to max allowed tier instead
+            console.log(`[provisioner] Auto-scaling workspace ${workspaceId.substring(0, 8)} from ${currentTier.name} to ${maxTierName} (max for ${plan} plan)`);
+            await this.resize(workspaceId, maxTier);
+            return {
+                scaled: true,
+                currentTier: currentTier.name,
+                targetTier: maxTierName,
+                reason: `Scaled to max tier for ${plan} plan`,
+            };
+        }
+        console.log(`[provisioner] Auto-scaling workspace ${workspaceId.substring(0, 8)} from ${currentTier.name} to ${recommendedTier.name} (${currentAgentCount} agents)`);
+        await this.resize(workspaceId, recommendedTier);
+        return {
+            scaled: true,
+            currentTier: currentTier.name,
+            targetTier: recommendedTier.name,
+        };
+    }
+    // ============================================================================
+    // Snapshot Management
+    // ============================================================================
+    /**
+     * Create an on-demand snapshot of a workspace's volume
+     * Use before risky operations (e.g., major refactors, untrusted code execution)
+     */
+    async createSnapshot(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // Only Fly.io provisioner supports snapshots
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            console.warn('[provisioner] Snapshots only supported on Fly.io');
+            return null;
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const flyProvisioner = this.provisioner;
+        // Get the volume
+        const volume = await flyProvisioner.getVolume(appName);
+        if (!volume) {
+            throw new Error('No volume found for workspace');
+        }
+        // Create snapshot
+        const snapshot = await flyProvisioner.createSnapshot(appName, volume.id);
+        return { snapshotId: snapshot.id };
+    }
+    /**
+     * List available snapshots for a workspace
+     * Includes both automatic daily snapshots and on-demand snapshots
+     */
+    async listSnapshots(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // Only Fly.io provisioner supports snapshots
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            return [];
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const flyProvisioner = this.provisioner;
+        // Get the volume
+        const volume = await flyProvisioner.getVolume(appName);
+        if (!volume) {
+            return [];
+        }
+        // List snapshots
+        const snapshots = await flyProvisioner.listSnapshots(appName, volume.id);
+        return snapshots.map(s => ({
+            id: s.id,
+            createdAt: s.created_at,
+            sizeBytes: s.size,
+        }));
+    }
+    /**
+     * Get the volume ID for a workspace (needed for restore operations)
+     */
+    async getVolumeId(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            return null;
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const flyProvisioner = this.provisioner;
+        const volume = await flyProvisioner.getVolume(appName);
+        return volume?.id || null;
+    }
+    // ============================================================================
+    // Graceful Image Update
+    // ============================================================================
+    /**
+     * Result of a graceful update attempt
+     */
+    static UpdateResult = {
+        UPDATED: 'updated',
+        UPDATED_PENDING_RESTART: 'updated_pending_restart',
+        SKIPPED_ACTIVE_AGENTS: 'skipped_active_agents',
+        SKIPPED_NOT_RUNNING: 'skipped_not_running',
+        NOT_SUPPORTED: 'not_supported',
+        ERROR: 'error',
+    };
+    /**
+     * Gracefully update a single workspace's image
+     *
+     * Behavior:
+     * - If workspace is stopped: Update config, will use new image on next wake
+     * - If workspace is running with no agents: Update config and restart
+     * - If workspace is running with active agents: Skip (or force if specified)
+     *
+     * @param workspaceId - Workspace to update
+     * @param newImage - New Docker image to use
+     * @param options - Update options
+     * @returns Update result with details
+     */
+    async gracefulUpdateImage(workspaceId, newImage, options = {}) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            return {
+                result: WorkspaceProvisioner.UpdateResult.ERROR,
+                workspaceId,
+                error: 'Workspace not found',
+            };
+        }
+        // Only Fly.io supports graceful updates
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            return {
+                result: WorkspaceProvisioner.UpdateResult.NOT_SUPPORTED,
+                workspaceId,
+                error: 'Graceful updates only supported on Fly.io',
+            };
+        }
+        const flyProvisioner = this.provisioner;
+        try {
+            // Check machine state
+            const machineState = await flyProvisioner.getMachineState(workspace);
+            if (machineState === 'stopped' || machineState === 'suspended') {
+                // Machine is not running - safe to update, will apply on next wake
+                await flyProvisioner.updateMachineImage(workspace, newImage);
+                console.log(`[provisioner] Updated stopped workspace ${workspaceId.substring(0, 8)} to ${newImage}`);
+                return {
+                    result: WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART,
+                    workspaceId,
+                    machineState,
+                };
+            }
+            if (machineState === 'started') {
+                // Machine is running - check for active agents
+                const agentCheck = await flyProvisioner.checkActiveAgents(workspace);
+                if (agentCheck.hasActiveAgents && !options.force) {
+                    // Has active agents and not forcing - skip
+                    console.log(`[provisioner] Skipped workspace ${workspaceId.substring(0, 8)}: ${agentCheck.agentCount} active agents`);
+                    return {
+                        result: WorkspaceProvisioner.UpdateResult.SKIPPED_ACTIVE_AGENTS,
+                        workspaceId,
+                        machineState,
+                        agentCount: agentCheck.agentCount,
+                        agents: agentCheck.agents,
+                    };
+                }
+                // Update the image config
+                await flyProvisioner.updateMachineImage(workspace, newImage);
+                if (options.skipRestart) {
+                    // Config updated but not restarting - will apply on next restart/auto-stop-wake
+                    console.log(`[provisioner] Updated workspace ${workspaceId.substring(0, 8)} config (restart skipped)`);
+                    return {
+                        result: WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART,
+                        workspaceId,
+                        machineState,
+                        agentCount: agentCheck.agentCount,
+                        agents: agentCheck.agents,
+                    };
+                }
+                // Restart to apply new image
+                await flyProvisioner.restart(workspace);
+                console.log(`[provisioner] Updated and restarted workspace ${workspaceId.substring(0, 8)}`);
+                return {
+                    result: WorkspaceProvisioner.UpdateResult.UPDATED,
+                    workspaceId,
+                    machineState,
+                    agentCount: agentCheck.agentCount,
+                };
+            }
+            // Unknown state
+            return {
+                result: WorkspaceProvisioner.UpdateResult.SKIPPED_NOT_RUNNING,
+                workspaceId,
+                machineState,
+            };
+        }
+        catch (error) {
+            console.error(`[provisioner] Error updating workspace ${workspaceId.substring(0, 8)}:`, error);
+            return {
+                result: WorkspaceProvisioner.UpdateResult.ERROR,
+                workspaceId,
+                error: error.message,
+            };
+        }
+    }
+    /**
+     * Gracefully update all workspaces to a new image
+     *
+     * Processes workspaces in batches, respecting active agents unless forced.
+     * Returns detailed results for each workspace.
+     *
+     * @param newImage - New Docker image to use
+     * @param options - Update options
+     * @returns Summary and per-workspace results
+     */
+    async gracefulUpdateAllImages(newImage, options = {}) {
+        // Get all workspaces to update
+        let workspaces;
+        if (options.workspaceIds?.length) {
+            // Specific workspaces
+            workspaces = (await Promise.all(options.workspaceIds.map(id => db.workspaces.findById(id)))).filter((w) => w !== null);
+        }
+        else if (options.userIds?.length) {
+            // Workspaces for specific users
+            const allWorkspaces = await Promise.all(options.userIds.map(userId => db.workspaces.findByUserId(userId)));
+            workspaces = allWorkspaces.flat();
+        }
+        else {
+            // All workspaces - need to query by status to get running ones
+            // For now, we'll get all workspaces from the provisioning provider
+            workspaces = await db.workspaces.findAll();
+        }
+        // Filter to only Fly.io workspaces
+        workspaces = workspaces.filter(w => w.computeProvider === 'fly' && w.computeId);
+        console.log(`[provisioner] Starting graceful update of ${workspaces.length} workspaces to ${newImage}`);
+        const batchSize = options.batchSize ?? 5;
+        const results = [];
+        // Process in batches
+        for (let i = 0; i < workspaces.length; i += batchSize) {
+            const batch = workspaces.slice(i, i + batchSize);
+            const batchResults = await Promise.all(batch.map(workspace => this.gracefulUpdateImage(workspace.id, newImage, {
+                force: options.force,
+                skipRestart: options.skipRestart,
+            })));
+            results.push(...batchResults);
+            // Small delay between batches to avoid overwhelming Fly API
+            if (i + batchSize < workspaces.length) {
+                await wait(1000);
+            }
+        }
+        // Compute summary
+        const summary = {
+            total: results.length,
+            updated: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.UPDATED).length,
+            pendingRestart: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART).length,
+            skippedActiveAgents: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_ACTIVE_AGENTS).length,
+            skippedNotRunning: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_NOT_RUNNING).length,
+            errors: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.ERROR).length,
+        };
+        console.log(`[provisioner] Graceful update complete:`, summary);
+        return { summary, results };
+    }
 }
 // Singleton instance
 let _provisioner = null;