agent-relay 1.2.3 → 1.3.0

package/dist/cloud/provisioner/index.js

@@ -6,10 +6,11 @@
 import * as crypto from 'crypto';
 import { getConfig } from '../config.js';
 import { db } from '../db/index.js';
-import { vault } from '../vault/index.js';
 import { nangoService } from '../services/nango.js';
 import { canAutoScale, canScaleToTier, getResourceTierForPlan, } from '../services/planLimits.js';
+import { deriveSshPassword } from '../services/ssh-security.js';
 const WORKSPACE_PORT = 3888;
+const CODEX_OAUTH_PORT = 1455; // Codex CLI OAuth callback port - must be mapped for local dev
 const FETCH_TIMEOUT_MS = 10_000;
 const WORKSPACE_IMAGE = process.env.WORKSPACE_IMAGE || 'ghcr.io/agentworkforce/relay-workspace:latest';
 // In-memory tracker for provisioning progress (workspace ID -> progress)
@@ -70,20 +71,6 @@ async function getGithubAppTokenForUser(userId) {
         return null;
     }
 }
-async function loadCredentialToken(userId, provider) {
-    try {
-        const cred = await vault.getCredential(userId, provider);
-        if (cred?.accessToken) {
-            return cred.accessToken;
-        }
-    }
-    catch (error) {
-        console.warn(`Failed to decrypt ${provider} credential from vault; trying raw storage fallback`, error);
-        const raw = await db.credentials.findByUserAndProvider(userId, provider);
-        return raw?.accessToken ?? null;
-    }
-    return null;
-}
 async function wait(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -132,33 +119,46 @@ async function softHealthCheck(url) {
  */
 async function waitForMachineStarted(apiToken, appName, machineId, timeoutSeconds = 120) {
     console.log(`[provisioner] Waiting for machine ${machineId} to start (timeout: ${timeoutSeconds}s)...`);
-    try {
-        // Use Fly.io's /wait endpoint - blocks until machine reaches target state
-        const res = await fetch(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}/wait?state=started&timeout=${timeoutSeconds}`, {
-            headers: { Authorization: `Bearer ${apiToken}` },
-        });
-        if (res.ok) {
-            console.log(`[provisioner] Machine ${machineId} is now started`);
-            return;
-        }
-        // 408 = timeout, machine didn't reach state in time
-        if (res.status === 408) {
-            // Get current state for error message
-            const stateRes = await fetch(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}`, { headers: { Authorization: `Bearer ${apiToken}` } });
-            const machine = stateRes.ok ? (await stateRes.json()) : { state: 'unknown' };
-            throw new Error(`Machine ${machineId} did not start within ${timeoutSeconds}s (last state: ${machine.state})`);
+    // Fly.io /wait endpoint has max timeout of 60s, so we need to loop for longer waits
+    const maxSingleWait = 60;
+    const startTime = Date.now();
+    const deadline = startTime + timeoutSeconds * 1000;
+    while (Date.now() < deadline) {
+        const remainingMs = deadline - Date.now();
+        const waitSeconds = Math.min(maxSingleWait, Math.ceil(remainingMs / 1000));
+        if (waitSeconds <= 0)
+            break;
+        try {
+            // Use Fly.io's /wait endpoint - blocks until machine reaches target state
+            // timeout is an integer in seconds (max 60)
+            const res = await fetch(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}/wait?state=started&timeout=${waitSeconds}`, {
+                headers: { Authorization: `Bearer ${apiToken}` },
+            });
+            if (res.ok) {
+                console.log(`[provisioner] Machine ${machineId} is now started`);
+                return;
+            }
+            // 408 = timeout, machine didn't reach state in time - try again if we have time
+            if (res.status === 408) {
+                console.log(`[provisioner] Machine ${machineId} not ready yet, continuing to wait...`);
+                continue;
+            }
+            // Other error
+            const errorText = await res.text();
+            throw new Error(`Wait for machine failed: ${res.status} ${errorText}`);
         }
-        // Other error
-        const errorText = await res.text();
-        throw new Error(`Wait for machine failed: ${res.status} ${errorText}`);
-    }
-    catch (error) {
-        if (error instanceof Error && error.message.includes('did not start')) {
-            throw error;
+        catch (error) {
+            if (error instanceof Error && error.message.includes('Wait for machine failed')) {
+                throw error;
+            }
+            console.warn(`[provisioner] Error waiting for machine:`, error);
+            throw new Error(`Failed to wait for machine ${machineId}: ${error.message}`);
         }
-        console.warn(`[provisioner] Error waiting for machine:`, error);
-        throw new Error(`Failed to wait for machine ${machineId}: ${error.message}`);
     }
+    // Timeout reached - get current state for error message
+    const stateRes = await fetch(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}`, { headers: { Authorization: `Bearer ${apiToken}` } });
+    const machine = stateRes.ok ? (await stateRes.json()) : { state: 'unknown' };
+    throw new Error(`Machine ${machineId} did not start within ${timeoutSeconds}s (last state: ${machine.state})`);
 }
 /**
  * Wait for health check to pass (with DNS propagation time)
@@ -226,6 +226,8 @@ class FlyProvisioner {
     cloudApiUrl;
     sessionSecret;
     registryAuth;
+    snapshotRetentionDays;
+    volumeSizeGb;
     constructor() {
         const config = getConfig();
         if (!config.compute.fly) {
@@ -238,6 +240,9 @@ class FlyProvisioner {
         this.registryAuth = config.compute.fly.registryAuth;
         this.cloudApiUrl = config.publicUrl;
         this.sessionSecret = config.sessionSecret;
+        // Snapshot settings: default 14 days retention, 10GB volume
+        this.snapshotRetentionDays = Math.min(60, Math.max(1, config.compute.fly.snapshotRetentionDays ?? 14));
+        this.volumeSizeGb = config.compute.fly.volumeSizeGb ?? 10;
     }
     /**
      * Generate a workspace token for API authentication
@@ -249,6 +254,87 @@ class FlyProvisioner {
             .update(`workspace:${workspaceId}`)
             .digest('hex');
     }
+    /**
+     * Create a volume with automatic snapshot settings
+     * Fly.io takes daily snapshots automatically; we configure retention
+     */
+    async createVolume(appName) {
+        const volumeName = 'workspace_data';
+        console.log(`[fly] Creating volume ${volumeName} with ${this.snapshotRetentionDays}-day snapshot retention...`);
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                name: volumeName,
+                region: this.region,
+                size_gb: this.volumeSizeGb,
+                // Enable automatic daily snapshots (default is true, but be explicit)
+                auto_backup_enabled: true,
+                // Retain snapshots for configured days (default 5, we use 14)
+                snapshot_retention: this.snapshotRetentionDays,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Failed to create volume: ${error}`);
+        }
+        const volume = await response.json();
+        console.log(`[fly] Volume ${volume.id} created with auto-snapshots (${this.snapshotRetentionDays} days retention)`);
+        return volume;
+    }
+    /**
+     * Create an on-demand snapshot of a workspace volume
+     * Use before risky operations or as manual backup
+     */
+    async createSnapshot(appName, volumeId) {
+        console.log(`[fly] Creating on-demand snapshot for volume ${volumeId}...`);
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes/${volumeId}/snapshots`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Failed to create snapshot: ${error}`);
+        }
+        const snapshot = await response.json();
+        console.log(`[fly] Snapshot ${snapshot.id} created`);
+        return snapshot;
+    }
+    /**
+     * List snapshots for a workspace volume
+     */
+    async listSnapshots(appName, volumeId) {
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes/${volumeId}/snapshots`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!response.ok) {
+            return [];
+        }
+        return await response.json();
+    }
+    /**
+     * Get volume info for a workspace
+     */
+    async getVolume(appName) {
+        const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/volumes`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!response.ok) {
+            return null;
+        }
+        const volumes = await response.json();
+        return volumes.find(v => v.name === 'workspace_data') || null;
+    }
     async provision(workspace, credentials) {
         const appName = `ar-${workspace.id.substring(0, 8)}`;
         // Stage: Creating workspace
@@ -359,8 +445,33 @@ class FlyProvisioner {
         if (customHostname) {
             await this.allocateCertificate(appName, customHostname);
         }
-        // Stage: Machine
+        // Stage: Machine (includes volume creation)
         updateProvisioningStage(workspace.id, 'machine');
+        // Create volume with automatic daily snapshots before machine
+        // Fly.io takes daily snapshots automatically; we configure retention
+        const volume = await this.createVolume(appName);
+        // Determine instance size based on user's plan
+        // Free tier: 1 CPU, 2GB (~$10/mo) - Claude needs 2GB minimum
+        // Paid tiers: 2 CPU, 2GB (~$15/mo)
+        // Introductory bonus: Free users get Pro-level resources for first 14 days
+        const user = await db.users.findById(workspace.userId);
+        const userPlan = user?.plan || 'free';
+        const isFreeTier = userPlan === 'free';
+        // Check if user is in introductory period (first 14 days)
+        const INTRO_PERIOD_DAYS = 14;
+        const userCreatedAt = user?.createdAt ? new Date(user.createdAt) : new Date();
+        const daysSinceSignup = (Date.now() - userCreatedAt.getTime()) / (1000 * 60 * 60 * 24);
+        const isIntroPeriod = isFreeTier && daysSinceSignup < INTRO_PERIOD_DAYS;
+        const guestConfig = {
+            cpu_kind: 'shared',
+            cpus: isIntroPeriod ? 2 : (isFreeTier ? 1 : 2), // Intro gets 2 CPUs like Pro
+            memory_mb: isIntroPeriod ? 4096 : 2048, // Intro gets 4GB like Pro
+        };
+        if (isIntroPeriod) {
+            const daysRemaining = Math.ceil(INTRO_PERIOD_DAYS - daysSinceSignup);
+            console.log(`[fly] Introductory bonus active (${daysRemaining} days remaining) - 2 CPU / 4GB`);
+        }
+        console.log(`[fly] Using ${guestConfig.cpus} CPU / ${guestConfig.memory_mb}MB for ${userPlan} plan`);
         // Create machine with auto-stop/start for cost optimization
         const machineResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines`, {
             method: 'POST',
@@ -382,6 +493,7 @@ class FlyProvisioner {
                     }),
                     env: {
                         WORKSPACE_ID: workspace.id,
+                        WORKSPACE_OWNER_USER_ID: workspace.userId,
                         SUPERVISOR_ENABLED: String(workspace.config.supervisorEnabled ?? false),
                         MAX_AGENTS: String(workspace.config.maxAgents ?? 10),
                         REPOSITORIES: (workspace.config.repositories ?? []).join(','),
@@ -391,6 +503,10 @@ class FlyProvisioner {
                         // Git gateway configuration
                         CLOUD_API_URL: this.cloudApiUrl,
                         WORKSPACE_TOKEN: this.generateWorkspaceToken(workspace.id),
+                        // SSH for CLI tunneling (Codex OAuth callback forwarding)
+                        // Each workspace gets a unique password derived from its ID + secret salt
+                        ENABLE_SSH: 'true',
+                        SSH_PASSWORD: deriveSshPassword(workspace.id),
                     },
                     services: [
                         {
@@ -421,6 +537,22 @@ class FlyProvisioner {
                                 hard_limit: 50,
                             },
                         },
+                        // SSH service for CLI tunneling (Codex OAuth callback forwarding)
+                        // Exposes port 2222 publicly for SSH connections from user's machine
+                        {
+                            ports: [
+                                {
+                                    port: 2222,
+                                    handlers: [], // Empty handlers = raw TCP passthrough
+                                },
+                            ],
+                            protocol: 'tcp',
+                            internal_port: 2222,
+                            // SSH connections should also wake the machine
+                            auto_stop_machines: 'stop',
+                            auto_start_machines: true,
+                            min_machines_running: 0,
+                        },
                     ],
                     checks: {
                         health: {
@@ -432,13 +564,15 @@ class FlyProvisioner {
                             grace_period: '10s',
                         },
                     },
-                    // Start with small tier (shared CPUs) - scales up based on plan
-                    // Free tier uses shared CPUs for cost efficiency
-                    guest: {
-                        cpu_kind: 'shared',
-                        cpus: 2,
-                        memory_mb: 2048,
-                    },
+                    // Instance size based on plan - free tier gets smaller instance
+                    guest: guestConfig,
+                    // Mount the volume we created with snapshot settings
+                    mounts: [
+                        {
+                            volume: volume.id,
+                            path: '/data',
+                        },
+                    ],
                 },
             }),
         });
@@ -535,13 +669,24 @@ class FlyProvisioner {
     }
     /**
      * Resize workspace - vertical scaling via Fly Machines API
+     * @param skipRestart - If true, config is saved but machine won't restart (changes apply on next start)
      */
-    async resize(workspace, tier) {
-        if (!workspace.computeId)
-            return;
-        const appName = `ar-${workspace.id.substring(0, 8)}`;
+    async resize(workspaceOrId, tier, skipRestart = false) {
+        const workspaceId = typeof workspaceOrId === 'string' ? workspaceOrId : workspaceOrId.id;
+        const computeId = typeof workspaceOrId === 'string' ? undefined : workspaceOrId.computeId;
+        // If passed just an ID, look up the workspace
+        let machineId = computeId;
+        if (!machineId) {
+            const workspace = await db.workspaces.findById(workspaceId);
+            if (!workspace?.computeId)
+                return;
+            machineId = workspace.computeId;
+        }
+        const appName = `ar-${workspaceId.substring(0, 8)}`;
         // Update machine configuration
-        await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+        // If running: reboots with new specs (unless skip_launch: true)
+        // If stopped: config saved, applies on next start
+        await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${machineId}`, {
             method: 'POST',
             headers: {
                 Authorization: `Bearer ${this.apiToken}`,
@@ -559,9 +704,11 @@ class FlyProvisioner {
                         MAX_AGENTS: String(tier.maxAgents),
                     },
                 },
+                skip_launch: skipRestart, // If true, don't restart - changes apply on next start
             }),
         });
-        console.log(`[fly] Resized workspace ${workspace.id} to ${tier.name} (${tier.cpuCores} CPU, ${tier.memoryMb}MB RAM)`);
+        const restartNote = skipRestart ? ' (will apply on next restart)' : ' (restarting)';
+        console.log(`[fly] Resized workspace ${workspaceId.substring(0, 8)} to ${tier.name} (${tier.cpuCores} CPU, ${tier.memoryMb}MB RAM)${restartNote}`);
     }
     /**
      * Update the max agent limit for a workspace
@@ -615,6 +762,118 @@ class FlyProvisioner {
             return RESOURCE_TIERS.medium;
         return RESOURCE_TIERS.small;
     }
+    /**
+     * Update machine image without restarting
+     * Note: The machine needs to be restarted later to use the new image
+     */
+    async updateMachineImage(workspace, newImage) {
+        if (!workspace.computeId)
+            return;
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        // Get current machine config first
+        const getResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+            },
+        });
+        if (!getResponse.ok) {
+            throw new Error(`Failed to get machine config: ${await getResponse.text()}`);
+        }
+        const machine = await getResponse.json();
+        // Update the image in the config
+        const updatedConfig = {
+            ...machine.config,
+            image: newImage,
+            // Include registry auth if configured
+            ...(this.registryAuth && {
+                image_registry_auth: {
+                    registry: 'ghcr.io',
+                    username: this.registryAuth.username,
+                    password: this.registryAuth.password,
+                },
+            }),
+        };
+        // Update machine with new image config (skip_launch keeps it in current state)
+        const updateResponse = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}?skip_launch=true`, {
+            method: 'POST',
+            headers: {
+                Authorization: `Bearer ${this.apiToken}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({ config: updatedConfig }),
+        });
+        if (!updateResponse.ok) {
+            throw new Error(`Failed to update machine image: ${await updateResponse.text()}`);
+        }
+        console.log(`[fly] Updated machine image for workspace ${workspace.id.substring(0, 8)} to ${newImage}`);
+    }
+    /**
+     * Check if workspace has active agents by querying the daemon
+     */
+    async checkActiveAgents(workspace) {
+        if (!workspace.publicUrl) {
+            return { hasActiveAgents: false, agentCount: 0, agents: [] };
+        }
+        try {
+            // Use internal Fly network URL if available (more reliable)
+            const appName = `ar-${workspace.id.substring(0, 8)}`;
+            const isOnFly = !!process.env.FLY_APP_NAME;
+            const baseUrl = isOnFly
+                ? `http://${appName}.internal:3888`
+                : workspace.publicUrl;
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), 10_000);
+            const response = await fetch(`${baseUrl}/api/agents`, {
+                method: 'GET',
+                headers: {
+                    'Accept': 'application/json',
+                },
+                signal: controller.signal,
+            });
+            clearTimeout(timer);
+            if (!response.ok) {
+                console.warn(`[fly] Failed to check agents for ${workspace.id.substring(0, 8)}: ${response.status}`);
+                return { hasActiveAgents: false, agentCount: 0, agents: [] };
+            }
+            const data = await response.json();
+            const agents = data.agents || [];
+            // Consider agents with 'active' or 'idle' activity state as active
+            // 'disconnected' agents are not active
+            const activeAgents = agents.filter(a => a.status === 'running' || a.activityState === 'active' || a.activityState === 'idle');
+            return {
+                hasActiveAgents: activeAgents.length > 0,
+                agentCount: activeAgents.length,
+                agents: agents.map(a => ({ name: a.name, status: a.status || a.activityState || 'unknown' })),
+            };
+        }
+        catch (error) {
+            // Workspace might be stopped or unreachable - treat as no active agents
+            console.warn(`[fly] Could not reach workspace ${workspace.id.substring(0, 8)} to check agents:`, error.message);
+            return { hasActiveAgents: false, agentCount: 0, agents: [] };
+        }
+    }
+    /**
+     * Get the current machine state
+     */
+    async getMachineState(workspace) {
+        if (!workspace.computeId)
+            return 'unknown';
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        try {
+            const response = await fetchWithRetry(`https://api.machines.dev/v1/apps/${appName}/machines/${workspace.computeId}`, {
+                headers: {
+                    Authorization: `Bearer ${this.apiToken}`,
+                },
+            });
+            if (!response.ok)
+                return 'unknown';
+            const machine = await response.json();
+            return machine.state;
+        }
+        catch {
+            return 'unknown';
+        }
+    }
 }
 /**
  * Railway provisioner
@@ -695,6 +954,7 @@ class RailwayProvisioner {
         // Set environment variables
         const envVars = {
             WORKSPACE_ID: workspace.id,
+            WORKSPACE_OWNER_USER_ID: workspace.userId,
             SUPERVISOR_ENABLED: String(workspace.config.supervisorEnabled ?? false),
             MAX_AGENTS: String(workspace.config.maxAgents ?? 10),
             REPOSITORIES: (workspace.config.repositories ?? []).join(','),
@@ -913,6 +1173,7 @@ class DockerProvisioner {
         // Build environment variables
         const envArgs = [
             `-e WORKSPACE_ID=${workspace.id}`,
+            `-e WORKSPACE_OWNER_USER_ID=${workspace.userId}`,
             `-e SUPERVISOR_ENABLED=${workspace.config.supervisorEnabled ?? false}`,
             `-e MAX_AGENTS=${workspace.config.maxAgents ?? 10}`,
             `-e REPOSITORIES=${(workspace.config.repositories ?? []).join(',')}`,
@@ -932,6 +1193,9 @@ class DockerProvisioner {
         // Run container
         const { execSync } = await import('child_process');
         const hostPort = 3000 + Math.floor(Math.random() * 1000);
+        // SSH port for tunneling (Codex OAuth callback forwarding)
+        // Derive from hostPort to avoid collisions: API port 3500 -> SSH port 22500
+        const sshHostPort = 22000 + (hostPort - 3000);
         // When running in Docker, connect to the same network for container-to-container communication
         const runningInDocker = process.env.RUNNING_IN_DOCKER === 'true';
         const networkArg = runningInDocker ? '--network agent-relay-dev' : '';
@@ -945,7 +1209,18 @@ class DockerProvisioner {
             console.log('[provisioner] Dev mode: mounting local dist/ and docs/ folders into workspace container');
         }
         try {
-            execSync(`docker run -d --user root --name ${containerName} ${networkArg} ${volumeArgs} -p ${hostPort}:${WORKSPACE_PORT} ${envArgs.join(' ')} ${WORKSPACE_IMAGE}`, { stdio: 'pipe' });
+            // Map workspace API port and SSH port (for tunneling)
+            // SSH is used by CLI to forward localhost:1455 to workspace container for Codex OAuth
+            // Set CODEX_DIRECT_PORT=true to also map port 1455 directly (for debugging only)
+            const directCodexPort = process.env.CODEX_DIRECT_PORT === 'true';
+            const portMappings = directCodexPort
+                ? `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:2222 -p ${CODEX_OAUTH_PORT}:${CODEX_OAUTH_PORT}`
+                : `-p ${hostPort}:${WORKSPACE_PORT} -p ${sshHostPort}:2222`;
+            // Enable SSH in the container for tunneling
+            // Each workspace gets a unique password derived from its ID + secret salt
+            envArgs.push('-e ENABLE_SSH=true');
+            envArgs.push(`-e SSH_PASSWORD=${deriveSshPassword(workspace.id)}`);
+            execSync(`docker run -d --user root --name ${containerName} ${networkArg} ${volumeArgs} ${portMappings} ${envArgs.join(' ')} ${WORKSPACE_IMAGE}`, { stdio: 'pipe' });
             const publicUrl = `http://localhost:${hostPort}`;
             // Wait for container to be healthy before returning
             // When running in Docker, use the internal container name for health check
@@ -956,6 +1231,7 @@ class DockerProvisioner {
             return {
                 computeId: containerName,
                 publicUrl,
+                sshPort: sshHostPort,
             };
         }
         catch (error) {
@@ -1061,6 +1337,37 @@ export class WorkspaceProvisioner {
         });
         // Auto-accept the creator's membership
         await db.workspaceMembers.acceptInvite(workspace.id, config.userId);
+        // Link repositories to this workspace
+        // This enables auto-access for users with GitHub access to these repos
+        for (const repoFullName of config.repositories) {
+            try {
+                // Find the user's repo record (may not exist if user didn't import it first)
+                const userRepos = await db.repositories.findByUserId(config.userId);
+                const repoRecord = userRepos.find(r => r.githubFullName.toLowerCase() === repoFullName.toLowerCase());
+                if (repoRecord) {
+                    await db.repositories.assignToWorkspace(repoRecord.id, workspace.id);
+                    console.log(`[provisioner] Linked repo ${repoFullName} to workspace ${workspace.id.substring(0, 8)}`);
+                }
+                else {
+                    // Create a placeholder repo record if it doesn't exist
+                    // This ensures the repo is tracked for workspace access checks
+                    console.log(`[provisioner] Creating repo record for ${repoFullName}`);
+                    const newRepo = await db.repositories.upsert({
+                        userId: config.userId,
+                        githubFullName: repoFullName,
+                        githubId: 0, // Will be updated when actually synced
+                        defaultBranch: 'main',
+                        isPrivate: true, // Assume private, will be updated
+                        workspaceId: workspace.id,
+                    });
+                    console.log(`[provisioner] Created and linked repo ${repoFullName} (id: ${newRepo.id.substring(0, 8)})`);
+                }
+            }
+            catch (err) {
+                console.warn(`[provisioner] Failed to link repo ${repoFullName}:`, err);
+                // Continue with other repos
+            }
+        }
         // Initialize stage tracking immediately
         updateProvisioningStage(workspace.id, 'creating');
         // Run provisioning in the background so frontend can poll for stages
@@ -1077,14 +1384,11 @@ export class WorkspaceProvisioner {
      * Run the actual provisioning work asynchronously
      */
     async runProvisioningAsync(workspace, config) {
-        // Get credentials
+        // Build credentials map for workspace provisioning
+        // Note: Provider tokens (Claude, Codex, etc.) are no longer stored centrally.
+        // CLI tools authenticate directly on workspace instances.
+        // Only GitHub App tokens are obtained from Nango for repository cloning.
         const credentials = new Map();
-        for (const provider of config.providers) {
-            const token = await loadCredentialToken(config.userId, provider);
-            if (token) {
-                credentials.set(provider, token);
-            }
-        }
         // GitHub token is required for cloning repositories
         // Use direct token if provided (for testing), otherwise get from Nango
         if (config.repositories.length > 0) {
@@ -1183,8 +1487,9 @@ export class WorkspaceProvisioner {
     }
     /**
      * Resize a workspace (vertical scaling)
+     * @param skipRestart - If true, config is saved but machine won't restart (changes apply on next start)
      */
-    async resize(workspaceId, tier) {
+    async resize(workspaceId, tier, skipRestart = false) {
         const workspace = await db.workspaces.findById(workspaceId);
         if (!workspace) {
             throw new Error('Workspace not found');
@@ -1192,7 +1497,7 @@ export class WorkspaceProvisioner {
         if (!this.provisioner.resize) {
             throw new Error('Resize not supported by current compute provider');
         }
-        await this.provisioner.resize(workspace, tier);
+        await this.provisioner.resize(workspace, tier, skipRestart);
         // Update workspace config with new limits
         await db.workspaces.updateConfig(workspaceId, {
             ...workspace.config,
@@ -1306,6 +1611,246 @@ export class WorkspaceProvisioner {
             targetTier: recommendedTier.name,
         };
     }
+    // ============================================================================
+    // Snapshot Management
+    // ============================================================================
+    /**
+     * Create an on-demand snapshot of a workspace's volume
+     * Use before risky operations (e.g., major refactors, untrusted code execution)
+     */
+    async createSnapshot(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // Only Fly.io provisioner supports snapshots
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            console.warn('[provisioner] Snapshots only supported on Fly.io');
+            return null;
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const flyProvisioner = this.provisioner;
+        // Get the volume
+        const volume = await flyProvisioner.getVolume(appName);
+        if (!volume) {
+            throw new Error('No volume found for workspace');
+        }
+        // Create snapshot
+        const snapshot = await flyProvisioner.createSnapshot(appName, volume.id);
+        return { snapshotId: snapshot.id };
+    }
+    /**
+     * List available snapshots for a workspace
+     * Includes both automatic daily snapshots and on-demand snapshots
+     */
+    async listSnapshots(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        // Only Fly.io provisioner supports snapshots
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            return [];
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const flyProvisioner = this.provisioner;
+        // Get the volume
+        const volume = await flyProvisioner.getVolume(appName);
+        if (!volume) {
+            return [];
+        }
+        // List snapshots
+        const snapshots = await flyProvisioner.listSnapshots(appName, volume.id);
+        return snapshots.map(s => ({
+            id: s.id,
+            createdAt: s.created_at,
+            sizeBytes: s.size,
+        }));
+    }
+    /**
+     * Get the volume ID for a workspace (needed for restore operations)
+     */
+    async getVolumeId(workspaceId) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            throw new Error('Workspace not found');
+        }
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            return null;
+        }
+        const appName = `ar-${workspace.id.substring(0, 8)}`;
+        const flyProvisioner = this.provisioner;
+        const volume = await flyProvisioner.getVolume(appName);
+        return volume?.id || null;
+    }
+    // ============================================================================
+    // Graceful Image Update
+    // ============================================================================
+    /**
+     * Result of a graceful update attempt
+     */
+    static UpdateResult = {
+        UPDATED: 'updated',
+        UPDATED_PENDING_RESTART: 'updated_pending_restart',
+        SKIPPED_ACTIVE_AGENTS: 'skipped_active_agents',
+        SKIPPED_NOT_RUNNING: 'skipped_not_running',
+        NOT_SUPPORTED: 'not_supported',
+        ERROR: 'error',
+    };
+    /**
+     * Gracefully update a single workspace's image
+     *
+     * Behavior:
+     * - If workspace is stopped: Update config, will use new image on next wake
+     * - If workspace is running with no agents: Update config and restart
+     * - If workspace is running with active agents: Skip (or force if specified)
+     *
+     * @param workspaceId - Workspace to update
+     * @param newImage - New Docker image to use
+     * @param options - Update options
+     * @returns Update result with details
+     */
+    async gracefulUpdateImage(workspaceId, newImage, options = {}) {
+        const workspace = await db.workspaces.findById(workspaceId);
+        if (!workspace) {
+            return {
+                result: WorkspaceProvisioner.UpdateResult.ERROR,
+                workspaceId,
+                error: 'Workspace not found',
+            };
+        }
+        // Only Fly.io supports graceful updates
+        if (!(this.provisioner instanceof FlyProvisioner)) {
+            return {
+                result: WorkspaceProvisioner.UpdateResult.NOT_SUPPORTED,
+                workspaceId,
+                error: 'Graceful updates only supported on Fly.io',
+            };
+        }
+        const flyProvisioner = this.provisioner;
+        try {
+            // Check machine state
+            const machineState = await flyProvisioner.getMachineState(workspace);
+            if (machineState === 'stopped' || machineState === 'suspended') {
+                // Machine is not running - safe to update, will apply on next wake
+                await flyProvisioner.updateMachineImage(workspace, newImage);
+                console.log(`[provisioner] Updated stopped workspace ${workspaceId.substring(0, 8)} to ${newImage}`);
+                return {
+                    result: WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART,
+                    workspaceId,
+                    machineState,
+                };
+            }
+            if (machineState === 'started') {
+                // Machine is running - check for active agents
+                const agentCheck = await flyProvisioner.checkActiveAgents(workspace);
+                if (agentCheck.hasActiveAgents && !options.force) {
+                    // Has active agents and not forcing - skip
+                    console.log(`[provisioner] Skipped workspace ${workspaceId.substring(0, 8)}: ${agentCheck.agentCount} active agents`);
+                    return {
+                        result: WorkspaceProvisioner.UpdateResult.SKIPPED_ACTIVE_AGENTS,
+                        workspaceId,
+                        machineState,
+                        agentCount: agentCheck.agentCount,
+                        agents: agentCheck.agents,
+                    };
+                }
+                // Update the image config
+                await flyProvisioner.updateMachineImage(workspace, newImage);
+                if (options.skipRestart) {
+                    // Config updated but not restarting - will apply on next restart/auto-stop-wake
+                    console.log(`[provisioner] Updated workspace ${workspaceId.substring(0, 8)} config (restart skipped)`);
+                    return {
+                        result: WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART,
+                        workspaceId,
+                        machineState,
+                        agentCount: agentCheck.agentCount,
+                        agents: agentCheck.agents,
+                    };
+                }
+                // Restart to apply new image
+                await flyProvisioner.restart(workspace);
+                console.log(`[provisioner] Updated and restarted workspace ${workspaceId.substring(0, 8)}`);
+                return {
+                    result: WorkspaceProvisioner.UpdateResult.UPDATED,
+                    workspaceId,
+                    machineState,
+                    agentCount: agentCheck.agentCount,
+                };
+            }
+            // Unknown state
+            return {
+                result: WorkspaceProvisioner.UpdateResult.SKIPPED_NOT_RUNNING,
+                workspaceId,
+                machineState,
+            };
+        }
+        catch (error) {
+            console.error(`[provisioner] Error updating workspace ${workspaceId.substring(0, 8)}:`, error);
+            return {
+                result: WorkspaceProvisioner.UpdateResult.ERROR,
+                workspaceId,
+                error: error.message,
+            };
+        }
+    }
+    /**
+     * Gracefully update all workspaces to a new image
+     *
+     * Processes workspaces in batches, respecting active agents unless forced.
+     * Returns detailed results for each workspace.
+     *
+     * @param newImage - New Docker image to use
+     * @param options - Update options
+     * @returns Summary and per-workspace results
+     */
+    async gracefulUpdateAllImages(newImage, options = {}) {
+        // Get all workspaces to update
+        let workspaces;
+        if (options.workspaceIds?.length) {
+            // Specific workspaces
+            workspaces = (await Promise.all(options.workspaceIds.map(id => db.workspaces.findById(id)))).filter((w) => w !== null);
+        }
+        else if (options.userIds?.length) {
+            // Workspaces for specific users
+            const allWorkspaces = await Promise.all(options.userIds.map(userId => db.workspaces.findByUserId(userId)));
+            workspaces = allWorkspaces.flat();
+        }
+        else {
+            // All workspaces - need to query by status to get running ones
+            // For now, we'll get all workspaces from the provisioning provider
+            workspaces = await db.workspaces.findAll();
+        }
+        // Filter to only Fly.io workspaces
+        workspaces = workspaces.filter(w => w.computeProvider === 'fly' && w.computeId);
+        console.log(`[provisioner] Starting graceful update of ${workspaces.length} workspaces to ${newImage}`);
+        const batchSize = options.batchSize ?? 5;
+        const results = [];
+        // Process in batches
+        for (let i = 0; i < workspaces.length; i += batchSize) {
+            const batch = workspaces.slice(i, i + batchSize);
+            const batchResults = await Promise.all(batch.map(workspace => this.gracefulUpdateImage(workspace.id, newImage, {
+                force: options.force,
+                skipRestart: options.skipRestart,
+            })));
+            results.push(...batchResults);
+            // Small delay between batches to avoid overwhelming Fly API
+            if (i + batchSize < workspaces.length) {
+                await wait(1000);
+            }
+        }
+        // Compute summary
+        const summary = {
+            total: results.length,
+            updated: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.UPDATED).length,
+            pendingRestart: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.UPDATED_PENDING_RESTART).length,
+            skippedActiveAgents: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_ACTIVE_AGENTS).length,
+            skippedNotRunning: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.SKIPPED_NOT_RUNNING).length,
+            errors: results.filter(r => r.result === WorkspaceProvisioner.UpdateResult.ERROR).length,
+        };
+        console.log(`[provisioner] Graceful update complete:`, summary);
+        return { summary, results };
+    }
 }
 // Singleton instance
 let _provisioner = null;