@ash-ai/server 0.0.1

package/dist/routes/sessions.js

@@ -0,0 +1,392 @@
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { SSE_WRITE_TIMEOUT_MS, timingEnabled, startTimer, logTiming } from '@ash-ai/shared';
+import { getAgent, insertSession, getSession, listSessions, updateSessionStatus, updateSessionSandbox, touchSession, updateSessionRunner } from '../db/index.js';
+import { restoreSessionState, hasPersistedState, restoreStateFromCloud } from '@ash-ai/sandbox';
+/** Structured log line for every resume — always on, not gated by ASH_DEBUG_TIMING. */
+function logResume(path, sessionId, agentName) {
+    process.stderr.write(JSON.stringify({
+        type: 'resume_hit',
+        path,
+        sessionId,
+        agentName,
+        ts: new Date().toISOString(),
+    }) + '\n');
+}
+const idParam = {
+    type: 'object',
+    properties: { id: { type: 'string', format: 'uuid' } },
+    required: ['id'],
+};
+const sessionResponse = {
+    type: 'object',
+    properties: { session: { $ref: 'Session#' } },
+    required: ['session'],
+};
+/**
+ * Write an SSE frame with backpressure. If the kernel TCP send buffer is full,
+ * waits for `drain` up to SSE_WRITE_TIMEOUT_MS before giving up.
+ */
+export async function writeSSE(raw, frame) {
+    const canWrite = raw.write(frame);
+    if (!canWrite) {
+        let timer;
+        let onDrain;
+        const drained = await Promise.race([
+            new Promise((resolve) => {
+                onDrain = () => resolve(true);
+                raw.once('drain', onDrain);
+            }),
+            new Promise((resolve) => {
+                timer = setTimeout(() => resolve(false), SSE_WRITE_TIMEOUT_MS);
+            }),
+        ]);
+        if (drained) {
+            clearTimeout(timer);
+        }
+        else {
+            raw.removeListener('drain', onDrain);
+            throw new Error('Client write timeout — closing stream');
+        }
+    }
+}
+export function sessionRoutes(app, coordinator, dataDir) {
+    // Create session — picks the best runner via coordinator
+    app.post('/api/sessions', {
+        schema: {
+            tags: ['sessions'],
+            body: {
+                type: 'object',
+                properties: {
+                    agent: { type: 'string' },
+                },
+                required: ['agent'],
+            },
+            response: {
+                201: sessionResponse,
+                400: { $ref: 'ApiError#' },
+                404: { $ref: 'ApiError#' },
+                500: { $ref: 'ApiError#' },
+                503: { $ref: 'ApiError#' },
+            },
+        },
+    }, async (req, reply) => {
+        const { agent } = req.body;
+        const agentRecord = await getAgent(agent);
+        if (!agentRecord) {
+            return reply.status(404).send({ error: `Agent "${agent}" not found`, statusCode: 404 });
+        }
+        const sessionId = randomUUID();
+        try {
+            const { backend, runnerId } = coordinator.selectBackend();
+            const handle = await backend.createSandbox({
+                sessionId,
+                agentDir: agentRecord.path,
+                agentName: agent,
+                sandboxId: sessionId,
+                onOomKill: () => {
+                    updateSessionStatus(sessionId, 'paused').catch((err) => console.error(`Failed to update session status on OOM: ${err}`));
+                },
+            });
+            const session = await insertSession(sessionId, agent, handle.sandboxId);
+            const effectiveRunnerId = runnerId === '__local__' ? null : runnerId;
+            await updateSessionRunner(sessionId, effectiveRunnerId);
+            await updateSessionStatus(sessionId, 'active');
+            return reply.status(201).send({ session: { ...session, status: 'active', runnerId: effectiveRunnerId } });
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (msg.includes('capacity reached') || msg.includes('No runners available')) {
+                return reply.status(503).send({ error: msg, statusCode: 503 });
+            }
+            return reply.status(500).send({ error: `Failed to create session: ${msg}`, statusCode: 500 });
+        }
+    });
+    // List sessions (optional ?agent=name filter)
+    app.get('/api/sessions', {
+        schema: {
+            tags: ['sessions'],
+            querystring: {
+                type: 'object',
+                properties: {
+                    agent: { type: 'string' },
+                },
+            },
+            response: {
+                200: {
+                    type: 'object',
+                    properties: {
+                        sessions: { type: 'array', items: { $ref: 'Session#' } },
+                    },
+                    required: ['sessions'],
+                },
+            },
+        },
+    }, async (req, reply) => {
+        const { agent } = req.query;
+        return reply.send({ sessions: await listSessions(agent || undefined) });
+    });
+    // Get session
+    app.get('/api/sessions/:id', {
+        schema: {
+            tags: ['sessions'],
+            params: idParam,
+            response: {
+                200: sessionResponse,
+                404: { $ref: 'ApiError#' },
+            },
+        },
+    }, async (req, reply) => {
+        const session = await getSession(req.params.id);
+        if (!session) {
+            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+        }
+        return reply.send({ session });
+    });
+    // Send message — routes to the correct runner for the session
+    app.post('/api/sessions/:id/messages', {
+        schema: {
+            tags: ['sessions'],
+            params: idParam,
+            body: {
+                type: 'object',
+                properties: {
+                    content: { type: 'string' },
+                    includePartialMessages: { type: 'boolean' },
+                },
+                required: ['content'],
+            },
+            response: {
+                200: {
+                    type: 'string',
+                    description: 'SSE stream. Events: `message` (SDK Message JSON), `error` ({error: string}), `done` ({sessionId: string})',
+                },
+                400: { $ref: 'ApiError#' },
+                404: { $ref: 'ApiError#' },
+                500: { $ref: 'ApiError#' },
+            },
+        },
+    }, async (req, reply) => {
+        const timing = timingEnabled();
+        const elapsed = timing ? startTimer() : null;
+        const session = await getSession(req.params.id);
+        if (!session) {
+            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+        }
+        if (session.status !== 'active') {
+            return reply.status(400).send({ error: `Session is ${session.status}`, statusCode: 400 });
+        }
+        const { content, includePartialMessages } = req.body;
+        let backend;
+        try {
+            backend = coordinator.getBackendForRunner(session.runnerId);
+        }
+        catch {
+            await updateSessionStatus(session.id, 'error');
+            return reply.status(500).send({ error: 'Runner not available', statusCode: 500 });
+        }
+        const sandbox = backend.getSandbox(session.sandboxId);
+        if (!sandbox) {
+            await updateSessionStatus(session.id, 'error');
+            return reply.status(500).send({ error: 'Sandbox not found', statusCode: 500 });
+        }
+        const lookupMs = elapsed?.() ?? 0;
+        // Mark running BEFORE any async work — prevents eviction
+        backend.markRunning(session.sandboxId);
+        await touchSession(session.id);
+        // SSE response
+        reply.raw.writeHead(200, {
+            'Content-Type': 'text/event-stream',
+            'Cache-Control': 'no-cache',
+            'Connection': 'keep-alive',
+        });
+        let eventCount = 0;
+        let firstEventMs = 0;
+        try {
+            const events = backend.sendCommand(session.sandboxId, {
+                cmd: 'query',
+                prompt: content,
+                sessionId: session.id,
+                ...(includePartialMessages && { includePartialMessages: true }),
+            });
+            for await (const event of events) {
+                eventCount++;
+                if (eventCount === 1 && elapsed) {
+                    firstEventMs = elapsed();
+                }
+                if (event.ev === 'message') {
+                    await writeSSE(reply.raw, `event: message\ndata: ${JSON.stringify(event.data)}\n\n`);
+                }
+                else if (event.ev === 'error') {
+                    await writeSSE(reply.raw, `event: error\ndata: ${JSON.stringify({ error: event.error })}\n\n`);
+                }
+                else if (event.ev === 'done') {
+                    await writeSSE(reply.raw, `event: done\ndata: ${JSON.stringify({ sessionId: event.sessionId })}\n\n`);
+                    // Best-effort state persistence after each completed turn
+                    backend.persistState(session.sandboxId, session.id, session.agentName);
+                }
+            }
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            reply.raw.write(`event: error\ndata: ${JSON.stringify({ error: msg })}\n\n`);
+        }
+        finally {
+            // Mark waiting after message processing completes
+            backend.markWaiting(session.sandboxId);
+        }
+        if (elapsed) {
+            logTiming({
+                type: 'timing',
+                source: 'server',
+                sessionId: session.id,
+                sandboxId: session.sandboxId,
+                lookupMs: Math.round(lookupMs * 100) / 100,
+                firstEventMs: Math.round(firstEventMs * 100) / 100,
+                totalMs: Math.round(elapsed() * 100) / 100,
+                eventCount,
+                timestamp: new Date().toISOString(),
+            });
+        }
+        reply.raw.end();
+    });
+    // Pause session
+    app.post('/api/sessions/:id/pause', {
+        schema: {
+            tags: ['sessions'],
+            params: idParam,
+            response: {
+                200: sessionResponse,
+                400: { $ref: 'ApiError#' },
+                404: { $ref: 'ApiError#' },
+            },
+        },
+    }, async (req, reply) => {
+        const session = await getSession(req.params.id);
+        if (!session) {
+            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+        }
+        if (session.status !== 'active') {
+            return reply.status(400).send({ error: `Cannot pause session with status "${session.status}"`, statusCode: 400 });
+        }
+        // Best-effort persist state before pausing
+        try {
+            const backend = coordinator.getBackendForRunner(session.runnerId);
+            backend.persistState(session.sandboxId, session.id, session.agentName);
+        }
+        catch { /* runner may be gone */ }
+        await updateSessionStatus(session.id, 'paused');
+        return reply.send({ session: { ...session, status: 'paused' } });
+    });
+    // Resume session — may route to a different runner
+    app.post('/api/sessions/:id/resume', {
+        schema: {
+            tags: ['sessions'],
+            params: idParam,
+            response: {
+                200: sessionResponse,
+                404: { $ref: 'ApiError#' },
+                410: { $ref: 'ApiError#' },
+                500: { $ref: 'ApiError#' },
+                503: { $ref: 'ApiError#' },
+            },
+        },
+    }, async (req, reply) => {
+        const session = await getSession(req.params.id);
+        if (!session) {
+            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+        }
+        if (session.status === 'ended') {
+            return reply.status(410).send({ error: 'Session has ended — create a new session', statusCode: 410 });
+        }
+        if (session.status === 'active') {
+            return reply.send({ session });
+        }
+        // Resumable statuses: 'paused', 'error', 'starting'
+        const agentRecord = await getAgent(session.agentName);
+        if (!agentRecord) {
+            return reply.status(404).send({ error: `Agent "${session.agentName}" not found`, statusCode: 404 });
+        }
+        // Fast path: try the same runner if sandbox is still alive
+        try {
+            const oldBackend = coordinator.getBackendForRunner(session.runnerId);
+            if (oldBackend.isSandboxAlive(session.sandboxId)) {
+                oldBackend.recordWarmHit();
+                logResume('warm', session.id, session.agentName);
+                await updateSessionStatus(session.id, 'active');
+                return reply.send({ session: { ...session, status: 'active' } });
+            }
+        }
+        catch { /* runner gone — cold path */ }
+        // Cold path: pick any healthy runner
+        try {
+            const oldWorkspaceDir = join(dataDir, 'sandboxes', session.id, 'workspace');
+            const workspaceExists = existsSync(oldWorkspaceDir);
+            if (!workspaceExists) {
+                if (hasPersistedState(dataDir, session.id)) {
+                    restoreSessionState(dataDir, session.id, oldWorkspaceDir);
+                }
+                else {
+                    // Fall back to cloud storage
+                    const restored = await restoreStateFromCloud(dataDir, session.id);
+                    if (restored) {
+                        restoreSessionState(dataDir, session.id, oldWorkspaceDir);
+                    }
+                }
+            }
+            const workspaceAvailable = existsSync(oldWorkspaceDir);
+            const { backend, runnerId } = coordinator.selectBackend();
+            const handle = await backend.createSandbox({
+                sessionId: session.id,
+                agentDir: agentRecord.path,
+                agentName: session.agentName,
+                sandboxId: session.id,
+                skipAgentCopy: workspaceAvailable,
+                onOomKill: () => {
+                    updateSessionStatus(session.id, 'paused').catch((err) => console.error(`Failed to update session status on OOM: ${err}`));
+                },
+            });
+            backend.recordColdHit();
+            logResume('cold', session.id, session.agentName);
+            const effectiveRunnerId = runnerId === '__local__' ? null : runnerId;
+            await updateSessionSandbox(session.id, handle.sandboxId);
+            await updateSessionRunner(session.id, effectiveRunnerId);
+            await updateSessionStatus(session.id, 'active');
+            return reply.send({ session: { ...session, sandboxId: handle.sandboxId, status: 'active', runnerId: effectiveRunnerId } });
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (msg.includes('capacity reached') || msg.includes('No runners available')) {
+                return reply.status(503).send({ error: msg, statusCode: 503 });
+            }
+            return reply.status(500).send({ error: `Failed to resume session: ${msg}`, statusCode: 500 });
+        }
+    });
+    // End session
+    app.delete('/api/sessions/:id', {
+        schema: {
+            tags: ['sessions'],
+            params: idParam,
+            response: {
+                200: sessionResponse,
+                404: { $ref: 'ApiError#' },
+            },
+        },
+    }, async (req, reply) => {
+        const session = await getSession(req.params.id);
+        if (!session) {
+            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+        }
+        // Persist state and destroy sandbox (best-effort — runner may be gone)
+        try {
+            const backend = coordinator.getBackendForRunner(session.runnerId);
+            backend.persistState(session.sandboxId, session.id, session.agentName);
+            await backend.destroySandbox(session.sandboxId);
+        }
+        catch { /* runner may be gone */ }
+        await updateSessionStatus(session.id, 'ended');
+        return reply.send({ session: { ...session, status: 'ended' } });
+    });
+}
+//# sourceMappingURL=sessions.js.map

package/dist/routes/sessions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sessions.js","sourceRoot":"","sources":["../../src/routes/sessions.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC5F,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAEjK,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAEhG,uFAAuF;AACvF,SAAS,SAAS,CAAC,IAAqB,EAAE,SAAiB,EAAE,SAAiB;IAC5E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;QAClC,IAAI,EAAE,YAAY;QAClB,IAAI;QACJ,SAAS;QACT,SAAS;QACT,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KAC7B,CAAC,GAAG,IAAI,CAAC,CAAC;AACb,CAAC;AAED,MAAM,OAAO,GAAG;IACd,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;IACtD,QAAQ,EAAE,CAAC,IAAI,CAAC;CACR,CAAC;AAEX,MAAM,eAAe,GAAG;IACtB,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE;IAC7C,QAAQ,EAAE,CAAC,SAAS,CAAC;CACb,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAmB,EAAE,KAAa;IAC/D,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,IAAI,KAAoC,CAAC;QACzC,IAAI,OAAmB,CAAC;QAExB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YACjC,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAC5B,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAC9B,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7B,CAAC,CAAC;YACF,IAAI,OAAO,CAAQ,CAAC,OAAO,EAAE,EAAE;gBAC7B,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,oBAAoB,CAAC,CAAC;YACjE,CAAC,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE,CAAC;YACZ,YAAY,CAAC,KAAM,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,cAAc,CAAC,OAAO,EAAE,OAAQ,CAAC,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAoB,EAAE,WAA8B,EAAE,OAAe;IACjG,yDAAyD;IACzD,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE;QACxB,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,IAAI,EAAE;gBACJ,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBAC1B;gBACD,QAAQ,EAAE,CAAC,OAAO,CAAC;aACpB;YACD,QAAQ,EAAE;gBACR,GAAG,EAAE,eAAe;gBACpB,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;aAC3B;SACF;KACF,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtB,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,IAAyB,CAAC;QAEhD,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,KAAK,aAAa,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,SAAS,GAAG,UAAU,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,aAAa,EAAE,CAAC;YAE1D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC;gBACzC,SAAS;gBACT,QAAQ,EAAE,WAAW,CAAC,IAAI;gBAC1B,SAAS,EAAE,KAAK;gBAChB,SAAS,EAAE,SAAS;gBACpB,SAAS,EAAE,GAAG,EAAE;oBACd,mBAAmB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CACrD,OAAO,CAAC,KAAK,CAAC,2CAA2C,GAAG,EAAE,CAAC,CAChE,CAAC;gBACJ,CAAC;aACF,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YACxE,MAAM,iBAAiB,GAAG,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;YACrE,MAAM,mBAAmB,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;YACxD,MAAM,mBAAmB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAE/C,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,EAAE,CAAC,CAAC;QAC5G,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,GAAG,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC;gBAC7E,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,6BAA6B,GAAG,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QAChG,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,8CAA8C;IAC9C,GAAG,CAAC,GAAG,CAAC,eAAe,EAAE;QACvB,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,WAAW,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBAC1B;aACF;YACD,QAAQ,EAAE;gBACR,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE;qBACzD;oBACD,QAAQ,EAAE,CAAC,UAAU,CAAC;iBACvB;aACF;SACF;KACF,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtB,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,KAA2B,CAAC;QAClD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC,KAAK,IAAI,SAAS,CAAC,EAAE,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,cAAc;IACd,GAAG,CAAC,GAAG,CAA6B,mBAAmB,EAAE;QACvD,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,MAAM,EAAE,OAAO;YACf,QAAQ,EAAE;gBACR,GAAG,EAAE,eAAe;gBACpB,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;aAC3B;SACF;KACF,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,8DAA8D;IAC9D,GAAG,CAAC,IAAI,CAA6B,4BAA4B,EAAE;QACjE,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,MAAM,EAAE,OAAO;YACf,IAAI,EAAE;gBACJ,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBAC3B,sBAAsB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;iBAC5C;gBACD,QAAQ,EAAE,CAAC,SAAS,CAAC;aACtB;YACD,QAAQ,EAAE;gBACR,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,2GAA2G;iBACzH;gBACD,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;aAC3B;SACF;KACF,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtB,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAE7C,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,cAAc,OAAO,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QAC5F,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,sBAAsB,EAAE,GAAG,GAAG,CAAC,IAA6D,CAAC;QAE9G,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,GAAG,WAAW,CAAC,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAC/C,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAC/C,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC;QAElC,yDAAyD;QACzD,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEvC,MAAM,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAE/B,eAAe;QACf,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE;YACvB,cAAc,EAAE,mBAAmB;YACnC,eAAe,EAAE,UAAU;YAC3B,YAAY,EAAE,YAAY;SAC3B,CAAC,CAAC;QAEH,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,EAAE;gBACpD,GAAG,EAAE,OAAO;gBACZ,MAAM,EAAE,OAAO;gBACf,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,GAAG,CAAC,sBAAsB,IAAI,EAAE,sBAAsB,EAAE,IAAI,EAAE,CAAC;aAChE,CAAC,CAAC;YAEH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,UAAU,EAAE,CAAC;gBACb,IAAI,UAAU,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;oBAChC,YAAY,GAAG,OAAO,EAAE,CAAC;gBAC3B,CAAC;gBAED,IAAI,KAAK,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;oBAC3B,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,yBAAyB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvF,CAAC;qBAAM,IAAI,KAAK,CAAC,EAAE,KAAK,OAAO,EAAE,CAAC;oBAChC,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC;gBACjG,CAAC;qBAAM,IAAI,KAAK,CAAC,EAAE,KAAK,MAAM,EAAE,CAAC;oBAC/B,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,sBAAsB,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,CAAC;oBACtG,0DAA0D;oBAC1D,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;QAC/E,CAAC;gBAAS,CAAC;YACT,kDAAkD;YAClD,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,SAAS,CAAC;gBACR,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,QAAQ;gBAChB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,GAAG;gBAC1C,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,GAAG,GAAG;gBAClD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,GAAG,GAAG;gBAC1C,UAAU;gBACV,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;QACL,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,gBAAgB;IAChB,GAAG,CAAC,IAAI,CAA6B,yBAAyB,EAAE;QAC9D,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,MAAM,EAAE,OAAO;YACf,QAAQ,EAAE;gBACR,GAAG,EAAE,eAAe;gBACpB,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;aAC3B;SACF;KACF,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qCAAqC,OAAO,CAAC,MAAM,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACpH,CAAC;QAED,2CAA2C;QAC3C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,WAAW,CAAC,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAClE,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACzE,CAAC;QAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;QAEpC,MAAM,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QAChD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;IAEH,mDAAmD;IACnD,GAAG,CAAC,IAAI,CAA6B,0BAA0B,EAAE;QAC/D,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,MAAM,EAAE,OAAO;YACf,QAAQ,EAAE;gBACR,GAAG,EAAE,eAAe;gBACpB,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;aAC3B;SACF;KACF,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,0CAA0C,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACxG,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;QACjC,CAAC;QAED,oDAAoD;QACpD,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,OAAO,CAAC,SAAS,aAAa,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACtG,CAAC;QAED,2DAA2D;QAC3D,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,WAAW,CAAC,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACrE,IAAI,UAAU,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjD,UAAU,CAAC,aAAa,EAAE,CAAC;gBAC3B,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;gBACjD,MAAM,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;gBAChD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,6BAA6B,CAAC,CAAC;QAEzC,qCAAqC;QACrC,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YAC5E,MAAM,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC;YAEpD,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,IAAI,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC3C,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;gBAC5D,CAAC;qBAAM,CAAC;oBACN,6BAA6B;oBAC7B,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;oBAClE,IAAI,QAAQ,EAAE,CAAC;wBACb,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC;oBAC5D,CAAC;gBACH,CAAC;YACH,CAAC;YAED,MAAM,kBAAkB,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC;YACvD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC,aAAa,EAAE,CAAC;YAE1D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC;gBACzC,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,QAAQ,EAAE,WAAW,CAAC,IAAI;gBAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,aAAa,EAAE,kBAAkB;gBACjC,SAAS,EAAE,GAAG,EAAE;oBACd,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CACtD,OAAO,CAAC,KAAK,CAAC,2CAA2C,GAAG,EAAE,CAAC,CAChE,CAAC;gBACJ,CAAC;aACF,CAAC,CAAC;YACH,OAAO,CAAC,aAAa,EAAE,CAAC;YACxB,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAEjD,MAAM,iBAAiB,GAAG,QAAQ,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;YACrE,MAAM,oBAAoB,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YACzD,MAAM,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAAC;YACzD,MAAM,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YAEhD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,iBAAiB,EAAE,EAAE,CAAC,CAAC;QAC7H,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,GAAG,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC;gBAC7E,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,6BAA6B,GAAG,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QAChG,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,cAAc;IACd,GAAG,CAAC,MAAM,CAA6B,mBAAmB,EAAE;QAC1D,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,MAAM,EAAE,OAAO;YACf,QAAQ,EAAE;gBACR,GAAG,EAAE,eAAe;gBACpB,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;aAC3B;SACF;KACF,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,uEAAuE;QACvE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,WAAW,CAAC,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAClE,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YACvE,MAAM,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;QAEpC,MAAM,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/runner/coordinator.d.ts

@@ -0,0 +1,52 @@
+import type { PoolStats } from '@ash-ai/shared';
+import type { RunnerBackend } from './types.js';
+/**
+ * Coordinates multiple runners. Routes session creation to the least-loaded runner.
+ * Detects dead runners and marks their sessions as paused.
+ */
+export declare class RunnerCoordinator {
+    private runners;
+    private localBackend;
+    private localRunnerId;
+    private livenessSweepTimer;
+    constructor(opts: {
+        localBackend?: RunnerBackend;
+    });
+    registerRunner(info: {
+        runnerId: string;
+        host: string;
+        port: number;
+        maxSandboxes: number;
+    }): void;
+    heartbeat(runnerId: string, stats: PoolStats): void;
+    /**
+     * Select the best backend for a new session.
+     * Returns the least-loaded runner, or local backend if no runners available.
+     */
+    selectBackend(): {
+        backend: RunnerBackend;
+        runnerId: string;
+    };
+    /**
+     * Get the backend for a specific runner. Used for routing messages to existing sessions.
+     */
+    getBackendForRunner(runnerId: string | null | undefined): RunnerBackend;
+    /**
+     * Start periodic liveness checks for remote runners.
+     */
+    startLivenessSweep(): void;
+    stopLivenessSweep(): void;
+    private checkLiveness;
+    handleDeadRunner(runnerId: string): Promise<void>;
+    get runnerCount(): number;
+    get hasLocalBackend(): boolean;
+    getRunnerInfo(): Array<{
+        runnerId: string;
+        host: string;
+        port: number;
+        active: number;
+        max: number;
+        lastHeartbeat: number;
+    }>;
+}
+//# sourceMappingURL=coordinator.d.ts.map

package/dist/runner/coordinator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinator.d.ts","sourceRoot":"","sources":["../../src/runner/coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAchD;;;GAGG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,kBAAkB,CAA+B;gBAE7C,IAAI,EAAE;QAAE,YAAY,CAAC,EAAE,aAAa,CAAA;KAAE;IAIlD,cAAc,CAAC,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAsBlG,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,GAAG,IAAI;IAUnD;;;OAGG;IACH,aAAa,IAAI;QAAE,OAAO,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;IAiC7D;;OAEG;IACH,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,aAAa;IAgBvE;;OAEG;IACH,kBAAkB,IAAI,IAAI;IAU1B,iBAAiB,IAAI,IAAI;YAOX,aAAa;IAUrB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBvD,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,eAAe,IAAI,OAAO,CAE7B;IAED,aAAa,IAAI,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;CAc7H"}

package/dist/runner/coordinator.js

@@ -0,0 +1,157 @@
+import { RUNNER_LIVENESS_TIMEOUT_MS } from '@ash-ai/shared';
+import { RemoteRunnerBackend } from './remote-backend.js';
+import { updateSessionStatus, listSessionsByRunner } from '../db/index.js';
+/**
+ * Coordinates multiple runners. Routes session creation to the least-loaded runner.
+ * Detects dead runners and marks their sessions as paused.
+ */
+export class RunnerCoordinator {
+    runners = new Map();
+    localBackend;
+    localRunnerId = '__local__';
+    livenessSweepTimer = null;
+    constructor(opts) {
+        this.localBackend = opts.localBackend ?? null;
+    }
+    registerRunner(info) {
+        const existing = this.runners.get(info.runnerId);
+        if (existing) {
+            // Re-registration — update connection info
+            existing.host = info.host;
+            existing.port = info.port;
+            existing.maxSandboxes = info.maxSandboxes;
+            existing.lastHeartbeat = Date.now();
+            console.log(`[coordinator] Runner ${info.runnerId} re-registered at ${info.host}:${info.port}`);
+            return;
+        }
+        const backend = new RemoteRunnerBackend({ host: info.host, port: info.port });
+        this.runners.set(info.runnerId, {
+            ...info,
+            backend,
+            lastHeartbeat: Date.now(),
+            stats: null,
+        });
+        console.log(`[coordinator] Runner ${info.runnerId} registered at ${info.host}:${info.port} (max ${info.maxSandboxes})`);
+    }
+    heartbeat(runnerId, stats) {
+        const runner = this.runners.get(runnerId);
+        if (!runner) {
+            console.warn(`[coordinator] Heartbeat from unknown runner ${runnerId}`);
+            return;
+        }
+        runner.lastHeartbeat = Date.now();
+        runner.stats = stats;
+    }
+    /**
+     * Select the best backend for a new session.
+     * Returns the least-loaded runner, or local backend if no runners available.
+     */
+    selectBackend() {
+        // If we have remote runners, pick the one with most available capacity
+        if (this.runners.size > 0) {
+            let bestRunner = null;
+            let bestAvailable = -1;
+            for (const runner of this.runners.values()) {
+                // Skip dead runners
+                if (Date.now() - runner.lastHeartbeat > RUNNER_LIVENESS_TIMEOUT_MS)
+                    continue;
+                const available = runner.stats
+                    ? runner.maxSandboxes - runner.stats.running - runner.stats.warming
+                    : runner.maxSandboxes;
+                if (available > bestAvailable) {
+                    bestAvailable = available;
+                    bestRunner = runner;
+                }
+            }
+            if (bestRunner && bestAvailable > 0) {
+                return { backend: bestRunner.backend, runnerId: bestRunner.runnerId };
+            }
+        }
+        // Fall back to local backend
+        if (this.localBackend) {
+            return { backend: this.localBackend, runnerId: this.localRunnerId };
+        }
+        throw new Error('No runners available and no local backend configured');
+    }
+    /**
+     * Get the backend for a specific runner. Used for routing messages to existing sessions.
+     */
+    getBackendForRunner(runnerId) {
+        if (!runnerId || runnerId === this.localRunnerId) {
+            if (this.localBackend)
+                return this.localBackend;
+            throw new Error('No local backend configured');
+        }
+        const runner = this.runners.get(runnerId);
+        if (!runner) {
+            // Runner gone — fall back to local if available
+            if (this.localBackend)
+                return this.localBackend;
+            throw new Error(`Runner ${runnerId} not found`);
+        }
+        return runner.backend;
+    }
+    /**
+     * Start periodic liveness checks for remote runners.
+     */
+    startLivenessSweep() {
+        if (this.livenessSweepTimer)
+            return;
+        this.livenessSweepTimer = setInterval(() => {
+            this.checkLiveness().catch((err) => console.error('[coordinator] Liveness sweep error:', err));
+        }, RUNNER_LIVENESS_TIMEOUT_MS);
+        this.livenessSweepTimer.unref();
+    }
+    stopLivenessSweep() {
+        if (this.livenessSweepTimer) {
+            clearInterval(this.livenessSweepTimer);
+            this.livenessSweepTimer = null;
+        }
+    }
+    async checkLiveness() {
+        const now = Date.now();
+        for (const [runnerId, runner] of this.runners) {
+            if (now - runner.lastHeartbeat > RUNNER_LIVENESS_TIMEOUT_MS) {
+                console.warn(`[coordinator] Runner ${runnerId} missed heartbeat — marking sessions paused`);
+                await this.handleDeadRunner(runnerId);
+            }
+        }
+    }
+    async handleDeadRunner(runnerId) {
+        // Mark all sessions on this runner as paused
+        const sessions = await listSessionsByRunner(runnerId);
+        for (const session of sessions) {
+            if (session.status === 'active' || session.status === 'starting') {
+                await updateSessionStatus(session.id, 'paused');
+                console.log(`[coordinator] Paused session ${session.id} (runner ${runnerId} dead)`);
+            }
+        }
+        // Remove the runner
+        const runner = this.runners.get(runnerId);
+        if (runner) {
+            runner.backend.close();
+            this.runners.delete(runnerId);
+        }
+    }
+    get runnerCount() {
+        return this.runners.size;
+    }
+    get hasLocalBackend() {
+        return this.localBackend !== null;
+    }
+    getRunnerInfo() {
+        const result = [];
+        for (const runner of this.runners.values()) {
+            result.push({
+                runnerId: runner.runnerId,
+                host: runner.host,
+                port: runner.port,
+                active: runner.stats?.running ?? 0,
+                max: runner.maxSandboxes,
+                lastHeartbeat: runner.lastHeartbeat,
+            });
+        }
+        return result;
+    }
+}
+//# sourceMappingURL=coordinator.js.map

package/dist/runner/coordinator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinator.js","sourceRoot":"","sources":["../../src/runner/coordinator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAE5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAY3E;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACpB,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IACxC,YAAY,CAAuB;IACnC,aAAa,GAAG,WAAW,CAAC;IAC5B,kBAAkB,GAA0B,IAAI,CAAC;IAEzD,YAAY,IAAsC;QAChD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;IAChD,CAAC;IAED,cAAc,CAAC,IAA4E;QACzF,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjD,IAAI,QAAQ,EAAE,CAAC;YACb,2CAA2C;YAC3C,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YAC1B,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YAC1B,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;YAC1C,QAAQ,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,QAAQ,qBAAqB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAChG,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE;YAC9B,GAAG,IAAI;YACP,OAAO;YACP,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;YACzB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,QAAQ,kBAAkB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IAC1H,CAAC;IAED,SAAS,CAAC,QAAgB,EAAE,KAAgB;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,+CAA+C,QAAQ,EAAE,CAAC,CAAC;YACxE,OAAO;QACT,CAAC;QACD,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,aAAa;QACX,uEAAuE;QACvE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,UAAU,GAAsB,IAAI,CAAC;YACzC,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC;YAEvB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC3C,oBAAoB;gBACpB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,aAAa,GAAG,0BAA0B;oBAAE,SAAS;gBAE7E,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK;oBAC5B,CAAC,CAAC,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO;oBACnE,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;gBAExB,IAAI,SAAS,GAAG,aAAa,EAAE,CAAC;oBAC9B,aAAa,GAAG,SAAS,CAAC;oBAC1B,UAAU,GAAG,MAAM,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,IAAI,UAAU,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACpC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxE,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,QAAmC;QACrD,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YACjD,IAAI,IAAI,CAAC,YAAY;gBAAE,OAAO,IAAI,CAAC,YAAY,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,gDAAgD;YAChD,IAAI,IAAI,CAAC,YAAY;gBAAE,OAAO,IAAI,CAAC,YAAY,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,UAAU,QAAQ,YAAY,CAAC,CAAC;QAClD,CAAC;QAED,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,CAAC,kBAAkB,GAAG,WAAW,CAAC,GAAG,EAAE;YACzC,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CACjC,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,GAAG,CAAC,CAC1D,CAAC;QACJ,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;IAED,iBAAiB;QACf,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACvC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QACjC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9C,IAAI,GAAG,GAAG,MAAM,CAAC,aAAa,GAAG,0BAA0B,EAAE,CAAC;gBAC5D,OAAO,CAAC,IAAI,CAAC,wBAAwB,QAAQ,6CAA6C,CAAC,CAAC;gBAC5F,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACtD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACjE,MAAM,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;gBAChD,OAAO,CAAC,GAAG,CAAC,gCAAgC,OAAO,CAAC,EAAE,YAAY,QAAQ,QAAQ,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC;IACpC,CAAC;IAED,aAAa;QACX,MAAM,MAAM,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,CAAC;gBAClC,GAAG,EAAE,MAAM,CAAC,YAAY;gBACxB,aAAa,EAAE,MAAM,CAAC,aAAa;aACpC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}