@ash-ai/server 0.0.19 → 0.0.24

package/dist/routes/sessions.js

@@ -5,10 +5,12 @@ import { SSE_WRITE_TIMEOUT_MS, timingEnabled, startTimer, logTiming } from '@ash
 import { getAgent, insertSession, insertForkedSession, getSession, listSessions, updateSessionStatus, updateSessionSandbox, updateSessionConfig, touchSession, updateSessionRunner, insertMessage, listMessages, insertSessionEvent, insertSessionEvents, listSessionEvents } from '../db/index.js';
 import { classifyBridgeMessage, classifyToStreamEvents } from '@ash-ai/shared';
 import { VERSION } from '../version.js';
-import { restoreSessionState, hasPersistedState, restoreStateFromCloud } from '@ash-ai/sandbox';
+import { restoreSessionState, hasPersistedState, restoreStateFromCloud, restoreAgentFromCloud } from '@ash-ai/sandbox';
 import { decryptCredential } from './credentials.js';
 import { touchCredentialUsed } from '../db/index.js';
 import { recordUsageFromMessage } from '../usage/extractor.js';
+import { trace, context, propagation, SpanStatusCode } from '@opentelemetry/api';
+const tracer = trace.getTracer('ash-coordinator');
 /** Structured log line for every resume — always on, not gated by ASH_DEBUG_TIMING. */
 function logResume(path, sessionId, agentName, source) {
     process.stderr.write(JSON.stringify({
@@ -97,73 +99,101 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
                 201: sessionResponse,
                 400: { $ref: 'ApiError#' },
                 404: { $ref: 'ApiError#' },
+                422: { $ref: 'ApiError#' },
                 500: { $ref: 'ApiError#' },
                 503: { $ref: 'ApiError#' },
             },
         },
     }, async (req, reply) => {
         const { agent, credentialId, extraEnv: bodyExtraEnv, startupScript, model, mcpServers, systemPrompt, permissionMode, allowedTools, disallowedTools, betas, subagents, initialAgent } = req.body;
-        const agentRecord = await getAgent(agent, req.tenantId);
-        if (!agentRecord) {
-            return reply.status(404).send({ error: `Agent "${agent}" not found`, statusCode: 404 });
-        }
-        // Resolve credential to env vars if provided
-        let extraEnv;
-        if (credentialId) {
-            const cred = await decryptCredential(credentialId, req.tenantId);
-            if (!cred) {
-                return reply.status(400).send({ error: 'Invalid or inaccessible credential', statusCode: 400 });
+        return tracer.startActiveSpan('ash.session.create', { attributes: { 'ash.agent.name': agent } }, async (span) => {
+            try {
+                const agentRecord = await getAgent(agent, req.tenantId);
+                if (!agentRecord) {
+                    span.setStatus({ code: SpanStatusCode.ERROR, message: 'Agent not found' });
+                    return reply.status(404).send({ error: `Agent "${agent}" not found`, statusCode: 404 });
+                }
+                // Validate agent directory exists on disk — auto-restore from cloud if missing
+                if (!existsSync(agentRecord.path)) {
+                    const restored = await restoreAgentFromCloud(agentRecord.name, agentRecord.path, req.tenantId);
+                    if (!restored) {
+                        span.setStatus({ code: SpanStatusCode.ERROR, message: 'Agent directory not found' });
+                        return reply.status(422).send({ error: `Agent directory not found at "${agentRecord.path}". The agent "${agent}" may need to be re-deployed.`, statusCode: 422 });
+                    }
+                    console.log(`[sessions] Restored agent "${agent}" from cloud storage`);
+                }
+                // Resolve credential to env vars if provided
+                let extraEnv;
+                if (credentialId) {
+                    const cred = await decryptCredential(credentialId, req.tenantId);
+                    if (!cred) {
+                        span.setStatus({ code: SpanStatusCode.ERROR, message: 'Invalid credential' });
+                        return reply.status(400).send({ error: 'Invalid or inaccessible credential', statusCode: 400 });
+                    }
+                    const envKey = cred.type === 'anthropic' ? 'ANTHROPIC_API_KEY' : cred.type === 'openai' ? 'OPENAI_API_KEY' : 'ASH_CUSTOM_API_KEY';
+                    extraEnv = { [envKey]: cred.key };
+                    touchCredentialUsed(credentialId).catch(() => { });
+                }
+                // Merge body-level extraEnv (overrides credential env on conflict)
+                if (bodyExtraEnv) {
+                    extraEnv = { ...extraEnv, ...bodyExtraEnv };
+                }
+                // Inject permission mode into sandbox env (bridge reads ASH_PERMISSION_MODE)
+                if (permissionMode) {
+                    extraEnv = { ...extraEnv, ASH_PERMISSION_MODE: permissionMode };
+                }
+                const sessionId = randomUUID();
+                span.setAttribute('ash.session.id', sessionId);
+                try {
+                    span.addEvent('selectBackend.start');
+                    const { backend, runnerId } = await coordinator.selectBackend();
+                    span.addEvent('selectBackend.end');
+                    span.addEvent('createSandbox.start');
+                    const handle = await backend.createSandbox({
+                        sessionId,
+                        agentDir: agentRecord.path,
+                        agentName: agentRecord.name,
+                        sandboxId: sessionId,
+                        extraEnv,
+                        startupScript,
+                        mcpServers,
+                        systemPrompt,
+                        onOomKill: () => {
+                            updateSessionStatus(sessionId, 'paused').catch((err) => console.error(`Failed to update session status on OOM: ${err}`));
+                        },
+                    });
+                    span.addEvent('createSandbox.end');
+                    // Resolve effective model: explicit request > agent record > null (SDK default)
+                    const effectiveModel = model || agentRecord.model || undefined;
+                    if (effectiveModel)
+                        span.setAttribute('ash.model', effectiveModel);
+                    // Build session-level SDK config (persisted on session, injected into every query)
+                    const sessionConfig = (allowedTools || disallowedTools || betas || subagents || initialAgent)
+                        ? { allowedTools, disallowedTools, betas, subagents, initialAgent }
+                        : null;
+                    const session = await insertSession(sessionId, agentRecord.name, handle.sandboxId, req.tenantId, undefined, effectiveModel, sessionConfig);
+                    const effectiveRunnerId = runnerId === '__local__' ? null : runnerId;
+                    await updateSessionRunner(sessionId, effectiveRunnerId);
+                    await updateSessionStatus(sessionId, 'active');
+                    // Record lifecycle event
+                    insertSessionEvent(sessionId, 'lifecycle', JSON.stringify({ action: 'created', agentName: agentRecord.name, model: effectiveModel }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
+                    telemetry.emit({ sessionId, agentName: agentRecord.name, type: 'lifecycle', data: { status: 'active', action: 'created' } });
+                    return reply.status(201).send({ session: { ...session, status: 'active', runnerId: effectiveRunnerId } });
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    span.setStatus({ code: SpanStatusCode.ERROR, message: msg });
+                    span.recordException(err instanceof Error ? err : new Error(msg));
+                    if (msg.includes('capacity reached') || msg.includes('No runners available')) {
+                        return reply.status(503).send({ error: msg, statusCode: 503 });
+                    }
+                    return reply.status(500).send({ error: `Failed to create session: ${msg}`, statusCode: 500 });
+                }
             }
-            const envKey = cred.type === 'anthropic' ? 'ANTHROPIC_API_KEY' : cred.type === 'openai' ? 'OPENAI_API_KEY' : 'ASH_CUSTOM_API_KEY';
-            extraEnv = { [envKey]: cred.key };
-            touchCredentialUsed(credentialId).catch(() => { });
-        }
-        // Merge body-level extraEnv (overrides credential env on conflict)
-        if (bodyExtraEnv) {
-            extraEnv = { ...extraEnv, ...bodyExtraEnv };
-        }
-        // Inject permission mode into sandbox env (bridge reads ASH_PERMISSION_MODE)
-        if (permissionMode) {
-            extraEnv = { ...extraEnv, ASH_PERMISSION_MODE: permissionMode };
-        }
-        const sessionId = randomUUID();
-        try {
-            const { backend, runnerId } = await coordinator.selectBackend();
-            const handle = await backend.createSandbox({
-                sessionId,
-                agentDir: agentRecord.path,
-                agentName: agentRecord.name,
-                sandboxId: sessionId,
-                extraEnv,
-                startupScript,
-                mcpServers,
-                systemPrompt,
-                onOomKill: () => {
-                    updateSessionStatus(sessionId, 'paused').catch((err) => console.error(`Failed to update session status on OOM: ${err}`));
-                },
-            });
-            // Resolve effective model: explicit request > agent record > null (SDK default)
-            const effectiveModel = model || agentRecord.model || undefined;
-            // Build session-level SDK config (persisted on session, injected into every query)
-            const sessionConfig = (allowedTools || disallowedTools || betas || subagents || initialAgent)
-                ? { allowedTools, disallowedTools, betas, subagents, initialAgent }
-                : null;
-            const session = await insertSession(sessionId, agentRecord.name, handle.sandboxId, req.tenantId, undefined, effectiveModel, sessionConfig);
-            const effectiveRunnerId = runnerId === '__local__' ? null : runnerId;
-            await updateSessionRunner(sessionId, effectiveRunnerId);
-            await updateSessionStatus(sessionId, 'active');
-            // Record lifecycle event
-            insertSessionEvent(sessionId, 'lifecycle', JSON.stringify({ action: 'created', agentName: agentRecord.name, model: effectiveModel }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
-            telemetry.emit({ sessionId, agentName: agentRecord.name, type: 'lifecycle', data: { status: 'active', action: 'created' } });
-            return reply.status(201).send({ session: { ...session, status: 'active', runnerId: effectiveRunnerId } });
-        }
-        catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            if (msg.includes('capacity reached') || msg.includes('No runners available')) {
-                return reply.status(503).send({ error: msg, statusCode: 503 });
+            finally {
+                span.end();
             }
-            return reply.status(500).send({ error: `Failed to create session: ${msg}`, statusCode: 500 });
-        }
+        });
     });
     // List sessions (optional ?agent=name filter)
     app.get('/api/sessions', {
@@ -357,17 +387,31 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
             return reply.status(400).send({ error: `Session is ${session.status}`, statusCode: 400 });
         }
         const { content, includePartialMessages, model: messageModel, maxTurns, maxBudgetUsd, effort, thinking, outputFormat } = req.body;
+        const messageSpan = tracer.startSpan('ash.session.message', {
+            attributes: {
+                'ash.session.id': session.id,
+                'ash.agent.name': session.agentName,
+                'ash.sandbox.id': session.sandboxId,
+            },
+        });
+        const messageCtx = trace.setSpan(context.active(), messageSpan);
+        if (session.model)
+            messageSpan.setAttribute('ash.model', session.model);
         let backend;
         try {
             backend = await coordinator.getBackendForRunnerAsync(session.runnerId);
         }
         catch {
             await updateSessionStatus(session.id, 'error');
+            messageSpan.setStatus({ code: SpanStatusCode.ERROR, message: 'Runner not available' });
+            messageSpan.end();
             return reply.status(500).send({ error: 'Runner not available', statusCode: 500 });
         }
         const sandbox = backend.getSandbox(session.sandboxId);
         if (!sandbox) {
             await updateSessionStatus(session.id, 'error');
+            messageSpan.setStatus({ code: SpanStatusCode.ERROR, message: 'Sandbox not found' });
+            messageSpan.end();
             return reply.status(500).send({ error: 'Sandbox not found', statusCode: 500 });
         }
         const lookupMs = elapsed?.() ?? 0;
@@ -387,6 +431,10 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
         await writeSSE(reply.raw, `event: session_start\ndata: ${JSON.stringify({ sessionId: session.id, version: VERSION })}\n\n`);
         let eventCount = 0;
         let firstEventMs = 0;
+        // Inject trace context for bridge propagation
+        const carrier = {};
+        propagation.inject(messageCtx, carrier);
+        const traceContext = carrier['traceparent'];
         try {
             // Model precedence: per-message > session-level > agent default (.claude/settings.json)
             const queryModel = messageModel || session.model || undefined;
@@ -410,6 +458,8 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
                 ...(cfg?.betas && { betas: cfg.betas }),
                 ...(cfg?.subagents && { subagents: cfg.subagents }),
                 ...(cfg?.initialAgent && { initialAgent: cfg.initialAgent }),
+                // Distributed tracing context
+                ...(traceContext && { traceContext }),
             });
             for await (const event of events) {
                 eventCount++;
@@ -462,6 +512,8 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
         }
         catch (err) {
             const msg = err instanceof Error ? err.message : String(err);
+            messageSpan.setStatus({ code: SpanStatusCode.ERROR, message: msg });
+            messageSpan.recordException(err instanceof Error ? err : new Error(msg));
             reply.raw.write(`event: error\ndata: ${JSON.stringify({ error: msg })}\n\n`);
             // Signal stream completion — the session remains active (not ended)
             reply.raw.write(`event: done\ndata: ${JSON.stringify({ sessionId: session.id })}\n\n`);
@@ -469,6 +521,7 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
         finally {
             // Mark waiting after message processing completes
             backend.markWaiting(session.sandboxId);
+            messageSpan.end();
         }
         if (elapsed) {
             logTiming({
@@ -614,23 +667,31 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
             },
         },
     }, async (req, reply) => {
-        const session = await getSession(req.params.id);
-        if (!session || session.tenantId !== req.tenantId) {
-            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
-        }
-        if (session.status !== 'active') {
-            return reply.status(400).send({ error: `Cannot pause session with status "${session.status}"`, statusCode: 400 });
-        }
-        // Best-effort persist state before pausing
-        try {
-            const backend = await coordinator.getBackendForRunnerAsync(session.runnerId);
-            backend.persistState(session.sandboxId, session.id, session.agentName);
-        }
-        catch { /* runner may be gone */ }
-        await updateSessionStatus(session.id, 'paused');
-        insertSessionEvent(session.id, 'lifecycle', JSON.stringify({ action: 'paused' }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
-        telemetry.emit({ sessionId: session.id, agentName: session.agentName, type: 'lifecycle', data: { status: 'paused' } });
-        return reply.send({ session: { ...session, status: 'paused' } });
+        return tracer.startActiveSpan('ash.session.pause', async (span) => {
+            try {
+                const session = await getSession(req.params.id);
+                if (!session || session.tenantId !== req.tenantId) {
+                    return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+                }
+                span.setAttribute('ash.session.id', session.id);
+                if (session.status !== 'active') {
+                    return reply.status(400).send({ error: `Cannot pause session with status "${session.status}"`, statusCode: 400 });
+                }
+                // Best-effort persist state before pausing
+                try {
+                    const backend = await coordinator.getBackendForRunnerAsync(session.runnerId);
+                    backend.persistState(session.sandboxId, session.id, session.agentName);
+                }
+                catch { /* runner may be gone */ }
+                await updateSessionStatus(session.id, 'paused');
+                insertSessionEvent(session.id, 'lifecycle', JSON.stringify({ action: 'paused' }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
+                telemetry.emit({ sessionId: session.id, agentName: session.agentName, type: 'lifecycle', data: { status: 'paused' } });
+                return reply.send({ session: { ...session, status: 'paused' } });
+            }
+            finally {
+                span.end();
+            }
+        });
     });
     // Stop session — explicit user action (distinct from pause which is idle-based)
     app.post('/api/sessions/:id/stop', {
@@ -644,24 +705,32 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
             },
         },
     }, async (req, reply) => {
-        const session = await getSession(req.params.id);
-        if (!session || session.tenantId !== req.tenantId) {
-            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
-        }
-        if (session.status !== 'active' && session.status !== 'starting') {
-            return reply.status(400).send({ error: `Cannot stop session with status "${session.status}"`, statusCode: 400 });
-        }
-        // Persist state and destroy sandbox
-        try {
-            const backend = await coordinator.getBackendForRunnerAsync(session.runnerId);
-            backend.persistState(session.sandboxId, session.id, session.agentName);
-            await backend.destroySandbox(session.sandboxId);
-        }
-        catch { /* runner may be gone */ }
-        await updateSessionStatus(session.id, 'stopped');
-        insertSessionEvent(session.id, 'lifecycle', JSON.stringify({ action: 'stopped' }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
-        telemetry.emit({ sessionId: session.id, agentName: session.agentName, type: 'lifecycle', data: { status: 'stopped' } });
-        return reply.send({ session: { ...session, status: 'stopped' } });
+        return tracer.startActiveSpan('ash.session.stop', async (span) => {
+            try {
+                const session = await getSession(req.params.id);
+                if (!session || session.tenantId !== req.tenantId) {
+                    return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+                }
+                span.setAttribute('ash.session.id', session.id);
+                if (session.status !== 'active' && session.status !== 'starting') {
+                    return reply.status(400).send({ error: `Cannot stop session with status "${session.status}"`, statusCode: 400 });
+                }
+                // Persist state and destroy sandbox
+                try {
+                    const backend = await coordinator.getBackendForRunnerAsync(session.runnerId);
+                    backend.persistState(session.sandboxId, session.id, session.agentName);
+                    await backend.destroySandbox(session.sandboxId);
+                }
+                catch { /* runner may be gone */ }
+                await updateSessionStatus(session.id, 'stopped');
+                insertSessionEvent(session.id, 'lifecycle', JSON.stringify({ action: 'stopped' }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
+                telemetry.emit({ sessionId: session.id, agentName: session.agentName, type: 'lifecycle', data: { status: 'stopped' } });
+                return reply.send({ session: { ...session, status: 'stopped' } });
+            }
+            finally {
+                span.end();
+            }
+        });
     });
     // Fork session — create a new session branching from parent's state and messages
     app.post('/api/sessions/:id/fork', {
@@ -671,6 +740,7 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
             response: {
                 201: sessionResponse,
                 404: { $ref: 'ApiError#' },
+                422: { $ref: 'ApiError#' },
                 500: { $ref: 'ApiError#' },
                 503: { $ref: 'ApiError#' },
             },
@@ -684,6 +754,13 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
         if (!agent) {
             return reply.status(404).send({ error: `Agent "${parentSession.agentName}" not found`, statusCode: 404 });
         }
+        if (!existsSync(agent.path)) {
+            const restored = await restoreAgentFromCloud(agent.name, agent.path, req.tenantId);
+            if (!restored) {
+                return reply.status(422).send({ error: `Agent directory not found at "${agent.path}". The agent "${parentSession.agentName}" may need to be re-deployed.`, statusCode: 422 });
+            }
+            console.log(`[sessions] Restored agent "${parentSession.agentName}" from cloud storage`);
+        }
         // Persist parent workspace state if sandbox is still live
         try {
             const parentBackend = await coordinator.getBackendForRunnerAsync(parentSession.runnerId);
@@ -733,101 +810,123 @@ export function sessionRoutes(app, coordinator, dataDir, telemetry) {
                 200: sessionResponse,
                 404: { $ref: 'ApiError#' },
                 410: { $ref: 'ApiError#' },
+                422: { $ref: 'ApiError#' },
                 500: { $ref: 'ApiError#' },
                 503: { $ref: 'ApiError#' },
             },
         },
     }, async (req, reply) => {
-        const session = await getSession(req.params.id);
-        if (!session || session.tenantId !== req.tenantId) {
-            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
-        }
-        if (session.status === 'ended') {
-            return reply.status(410).send({ error: 'Session has ended — create a new session', statusCode: 410 });
-        }
-        if (session.status === 'active') {
-            return reply.send({ session });
-        }
-        // Resumable statuses: 'paused', 'stopped', 'error', 'starting'
-        const agentRecord = await getAgent(session.agentName, req.tenantId);
-        if (!agentRecord) {
-            return reply.status(404).send({ error: `Agent "${session.agentName}" not found`, statusCode: 404 });
-        }
-        // Fast path: try the same runner if sandbox is still alive
-        try {
-            const oldBackend = await coordinator.getBackendForRunnerAsync(session.runnerId);
-            if (oldBackend.isSandboxAlive(session.sandboxId)) {
-                oldBackend.recordWarmHit();
-                logResume('warm', session.id, session.agentName);
-                await updateSessionStatus(session.id, 'active');
-                insertSessionEvent(session.id, 'lifecycle', JSON.stringify({ action: 'resumed', path: 'warm' }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
-                telemetry.emit({ sessionId: session.id, agentName: session.agentName, type: 'lifecycle', data: { status: 'active', action: 'resumed', path: 'warm' } });
-                return reply.send({ session: { ...session, status: 'active' } });
-            }
-        }
-        catch { /* runner gone — cold path */ }
-        // Cold path: pick any healthy runner
-        try {
-            const oldWorkspaceDir = join(dataDir, 'sandboxes', session.id, 'workspace');
-            const workspaceExists = existsSync(oldWorkspaceDir);
-            let resumeSource = 'fresh';
-            if (!workspaceExists) {
-                if (hasPersistedState(dataDir, session.id, session.tenantId)) {
-                    restoreSessionState(dataDir, session.id, oldWorkspaceDir, session.tenantId);
-                    resumeSource = 'local';
+        return tracer.startActiveSpan('ash.session.resume', async (span) => {
+            try {
+                const session = await getSession(req.params.id);
+                if (!session || session.tenantId !== req.tenantId) {
+                    return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+                }
+                span.setAttribute('ash.session.id', session.id);
+                if (session.status === 'ended') {
+                    return reply.status(410).send({ error: 'Session has ended — create a new session', statusCode: 410 });
+                }
+                if (session.status === 'active') {
+                    return reply.send({ session });
+                }
+                // Resumable statuses: 'paused', 'stopped', 'error', 'starting'
+                const agentRecord = await getAgent(session.agentName, req.tenantId);
+                if (!agentRecord) {
+                    return reply.status(404).send({ error: `Agent "${session.agentName}" not found`, statusCode: 404 });
                 }
-                else {
-                    // Fall back to cloud storage
-                    const restored = await restoreStateFromCloud(dataDir, session.id, session.tenantId);
-                    if (restored) {
-                        restoreSessionState(dataDir, session.id, oldWorkspaceDir, session.tenantId);
-                        resumeSource = 'cloud';
+                // Validate agent directory exists — auto-restore from cloud if missing
+                if (!existsSync(agentRecord.path)) {
+                    const restored = await restoreAgentFromCloud(agentRecord.name, agentRecord.path, req.tenantId);
+                    if (!restored) {
+                        return reply.status(422).send({ error: `Agent directory not found at "${agentRecord.path}". The agent "${session.agentName}" may need to be re-deployed.`, statusCode: 422 });
                     }
+                    console.log(`[sessions] Restored agent "${session.agentName}" from cloud storage`);
+                }
+                // Fast path: try the same runner if sandbox is still alive
+                try {
+                    const oldBackend = await coordinator.getBackendForRunnerAsync(session.runnerId);
+                    if (oldBackend.isSandboxAlive(session.sandboxId)) {
+                        oldBackend.recordWarmHit();
+                        logResume('warm', session.id, session.agentName);
+                        span.setAttribute('ash.resume.path', 'warm');
+                        await updateSessionStatus(session.id, 'active');
+                        insertSessionEvent(session.id, 'lifecycle', JSON.stringify({ action: 'resumed', path: 'warm' }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
+                        telemetry.emit({ sessionId: session.id, agentName: session.agentName, type: 'lifecycle', data: { status: 'active', action: 'resumed', path: 'warm' } });
+                        return reply.send({ session: { ...session, status: 'active' } });
+                    }
+                }
+                catch { /* runner gone — cold path */ }
+                // Cold path: pick any healthy runner
+                try {
+                    const oldWorkspaceDir = join(dataDir, 'sandboxes', session.id, 'workspace');
+                    const workspaceExists = existsSync(oldWorkspaceDir);
+                    let resumeSource = 'fresh';
+                    if (!workspaceExists) {
+                        if (hasPersistedState(dataDir, session.id, session.tenantId)) {
+                            restoreSessionState(dataDir, session.id, oldWorkspaceDir, session.tenantId);
+                            resumeSource = 'local';
+                        }
+                        else {
+                            // Fall back to cloud storage
+                            const restored = await restoreStateFromCloud(dataDir, session.id, session.tenantId);
+                            if (restored) {
+                                restoreSessionState(dataDir, session.id, oldWorkspaceDir, session.tenantId);
+                                resumeSource = 'cloud';
+                            }
+                        }
+                    }
+                    else {
+                        resumeSource = 'local';
+                    }
+                    span.setAttribute('ash.resume.path', 'cold');
+                    span.setAttribute('ash.resume.source', resumeSource);
+                    const workspaceAvailable = existsSync(oldWorkspaceDir);
+                    const { backend, runnerId } = await coordinator.selectBackend();
+                    const handle = await backend.createSandbox({
+                        sessionId: session.id,
+                        agentDir: agentRecord.path,
+                        agentName: session.agentName,
+                        sandboxId: session.id,
+                        skipAgentCopy: workspaceAvailable,
+                        onOomKill: () => {
+                            updateSessionStatus(session.id, 'paused').catch((err) => console.error(`Failed to update session status on OOM: ${err}`));
+                        },
+                    });
+                    // Track resume source
+                    switch (resumeSource) {
+                        case 'local':
+                            backend.recordColdLocalHit();
+                            break;
+                        case 'cloud':
+                            backend.recordColdCloudHit();
+                            break;
+                        case 'fresh':
+                            backend.recordColdFreshHit();
+                            break;
+                    }
+                    logResume('cold', session.id, session.agentName, resumeSource);
+                    const effectiveRunnerId = runnerId === '__local__' ? null : runnerId;
+                    await updateSessionSandbox(session.id, handle.sandboxId);
+                    await updateSessionRunner(session.id, effectiveRunnerId);
+                    await updateSessionStatus(session.id, 'active');
+                    insertSessionEvent(session.id, 'lifecycle', JSON.stringify({ action: 'resumed', path: 'cold', source: resumeSource }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
+                    telemetry.emit({ sessionId: session.id, agentName: session.agentName, type: 'lifecycle', data: { status: 'active', action: 'resumed', path: 'cold', source: resumeSource } });
+                    return reply.send({ session: { ...session, sandboxId: handle.sandboxId, status: 'active', runnerId: effectiveRunnerId } });
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    span.setStatus({ code: SpanStatusCode.ERROR, message: msg });
+                    span.recordException(err instanceof Error ? err : new Error(msg));
+                    if (msg.includes('capacity reached') || msg.includes('No runners available')) {
+                        return reply.status(503).send({ error: msg, statusCode: 503 });
+                    }
+                    return reply.status(500).send({ error: `Failed to resume session: ${msg}`, statusCode: 500 });
                 }
             }
-            else {
-                resumeSource = 'local';
-            }
-            const workspaceAvailable = existsSync(oldWorkspaceDir);
-            const { backend, runnerId } = await coordinator.selectBackend();
-            const handle = await backend.createSandbox({
-                sessionId: session.id,
-                agentDir: agentRecord.path,
-                agentName: session.agentName,
-                sandboxId: session.id,
-                skipAgentCopy: workspaceAvailable,
-                onOomKill: () => {
-                    updateSessionStatus(session.id, 'paused').catch((err) => console.error(`Failed to update session status on OOM: ${err}`));
-                },
-            });
-            // Track resume source
-            switch (resumeSource) {
-                case 'local':
-                    backend.recordColdLocalHit();
-                    break;
-                case 'cloud':
-                    backend.recordColdCloudHit();
-                    break;
-                case 'fresh':
-                    backend.recordColdFreshHit();
-                    break;
-            }
-            logResume('cold', session.id, session.agentName, resumeSource);
-            const effectiveRunnerId = runnerId === '__local__' ? null : runnerId;
-            await updateSessionSandbox(session.id, handle.sandboxId);
-            await updateSessionRunner(session.id, effectiveRunnerId);
-            await updateSessionStatus(session.id, 'active');
-            insertSessionEvent(session.id, 'lifecycle', JSON.stringify({ action: 'resumed', path: 'cold', source: resumeSource }), req.tenantId).catch((err) => console.error(`Failed to persist lifecycle event: ${err}`));
-            telemetry.emit({ sessionId: session.id, agentName: session.agentName, type: 'lifecycle', data: { status: 'active', action: 'resumed', path: 'cold', source: resumeSource } });
-            return reply.send({ session: { ...session, sandboxId: handle.sandboxId, status: 'active', runnerId: effectiveRunnerId } });
-        }
-        catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            if (msg.includes('capacity reached') || msg.includes('No runners available')) {
-                return reply.status(503).send({ error: msg, statusCode: 503 });
+            finally {
+                span.end();
             }
-            return reply.status(500).send({ error: `Failed to resume session: ${msg}`, statusCode: 500 });
-        }
+        });
     });
     // End session
     app.delete('/api/sessions/:id', {