@ash-ai/server 0.0.2 → 0.0.4

package/dist/routes/workspace.js

@@ -0,0 +1,123 @@
+import { join } from 'node:path';
+import { existsSync, rmSync } from 'node:fs';
+import { getSession } from '../db/index.js';
+import { createBundle, extractBundle, hasPersistedState, restoreSessionState } from '@ash-ai/sandbox';
+/** Max body size for workspace uploads: ~134MB base64 ≈ 100MB binary. */
+const WORKSPACE_BODY_LIMIT = 134 * 1024 * 1024;
+export function workspaceRoutes(app, coordinator, dataDir) {
+    // Download workspace as tar.gz bundle
+    app.get('/api/sessions/:id/workspace', {
+        schema: {
+            tags: ['sessions'],
+            params: {
+                type: 'object',
+                properties: { id: { type: 'string', format: 'uuid' } },
+                required: ['id'],
+            },
+            response: {
+                404: { $ref: 'ApiError#' },
+            },
+        },
+    }, async (req, reply) => {
+        const session = await getSession(req.params.id);
+        if (!session || session.tenantId !== req.tenantId) {
+            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+        }
+        // Try live sandbox workspace first
+        let workspaceDir = null;
+        let tempDir = null;
+        try {
+            const backend = await coordinator.getBackendForRunnerAsync(session.runnerId);
+            const sandbox = backend.getSandbox(session.sandboxId);
+            if (sandbox) {
+                workspaceDir = sandbox.workspaceDir;
+            }
+        }
+        catch { /* runner may be gone */ }
+        // Fall back to persisted snapshot
+        if (!workspaceDir || !existsSync(workspaceDir)) {
+            const snapshotDir = join(dataDir, 'sessions', session.id, 'workspace');
+            if (existsSync(snapshotDir)) {
+                workspaceDir = snapshotDir;
+            }
+            else if (hasPersistedState(dataDir, session.id, session.tenantId)) {
+                tempDir = join(dataDir, 'tmp', `bundle-${session.id}-${Date.now()}`);
+                restoreSessionState(dataDir, session.id, tempDir, session.tenantId);
+                workspaceDir = tempDir;
+            }
+        }
+        if (!workspaceDir || !existsSync(workspaceDir)) {
+            return reply.status(404).send({ error: 'No workspace available for this session', statusCode: 404 });
+        }
+        try {
+            const bundle = createBundle(workspaceDir);
+            return reply
+                .header('Content-Type', 'application/gzip')
+                .header('Content-Disposition', `attachment; filename="${session.id}.tar.gz"`)
+                .send(bundle);
+        }
+        finally {
+            // Clean up temp directory if we created one
+            if (tempDir) {
+                try {
+                    rmSync(tempDir, { recursive: true, force: true });
+                }
+                catch { /* best effort */ }
+            }
+        }
+    });
+    // Upload/restore workspace from tar.gz bundle
+    app.post('/api/sessions/:id/workspace', {
+        bodyLimit: WORKSPACE_BODY_LIMIT,
+        schema: {
+            tags: ['sessions'],
+            params: {
+                type: 'object',
+                properties: { id: { type: 'string', format: 'uuid' } },
+                required: ['id'],
+            },
+            body: {
+                type: 'object',
+                properties: {
+                    bundle: { type: 'string', description: 'Base64-encoded tar.gz bundle' },
+                },
+                required: ['bundle'],
+            },
+            response: {
+                200: {
+                    type: 'object',
+                    properties: {
+                        message: { type: 'string' },
+                    },
+                },
+                400: { $ref: 'ApiError#' },
+                404: { $ref: 'ApiError#' },
+            },
+        },
+    }, async (req, reply) => {
+        const session = await getSession(req.params.id);
+        if (!session || session.tenantId !== req.tenantId) {
+            return reply.status(404).send({ error: 'Session not found', statusCode: 404 });
+        }
+        const { bundle: b64 } = req.body;
+        const bundleBuffer = Buffer.from(b64, 'base64');
+        if (bundleBuffer.length === 0) {
+            return reply.status(400).send({ error: 'Empty bundle', statusCode: 400 });
+        }
+        // Extract into live sandbox workspace if available
+        try {
+            const backend = await coordinator.getBackendForRunnerAsync(session.runnerId);
+            const sandbox = backend.getSandbox(session.sandboxId);
+            if (sandbox) {
+                extractBundle(bundleBuffer, sandbox.workspaceDir);
+                return reply.send({ message: 'Workspace restored to live sandbox' });
+            }
+        }
+        catch { /* runner may be gone */ }
+        // Fall back to persisting as a snapshot
+        const snapshotDir = join(dataDir, 'sessions', session.id, 'workspace');
+        extractBundle(bundleBuffer, snapshotDir);
+        return reply.send({ message: 'Workspace saved as snapshot' });
+    });
+}
+//# sourceMappingURL=workspace.js.map

package/dist/routes/workspace.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"workspace.js","sourceRoot":"","sources":["../../src/routes/workspace.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAGtG,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAE/C,MAAM,UAAU,eAAe,CAAC,GAAoB,EAAE,WAA8B,EAAE,OAAe;IACnG,sCAAsC;IACtC,GAAG,CAAC,GAAG,CAA6B,6BAA6B,EAAE;QACjE,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBACtD,QAAQ,EAAE,CAAC,IAAI,CAAC;aACjB;YACD,QAAQ,EAAE;gBACR,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;aAC3B;SACF;KACF,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,EAAE,CAAC;YAClD,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,mCAAmC;QACnC,IAAI,YAAY,GAAkB,IAAI,CAAC;QACvC,IAAI,OAAO,GAAkB,IAAI,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,wBAAwB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7E,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACtD,IAAI,OAAO,EAAE,CAAC;gBACZ,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;YACtC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;QAEpC,kCAAkC;QAClC,IAAI,CAAC,YAAY,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACvE,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5B,YAAY,GAAG,WAAW,CAAC;YAC7B,CAAC;iBAAM,IAAI,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACpE,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,OAAO,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACrE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;gBACpE,YAAY,GAAG,OAAO,CAAC;YACzB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC/C,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,yCAAyC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;YAC1C,OAAO,KAAK;iBACT,MAAM,CAAC,cAAc,EAAE,kBAAkB,CAAC;iBAC1C,MAAM,CAAC,qBAAqB,EAAE,yBAAyB,OAAO,CAAC,EAAE,UAAU,CAAC;iBAC5E,IAAI,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,4CAA4C;YAC5C,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC;oBAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,8CAA8C;IAC9C,GAAG,CAAC,IAAI,CAA6B,6BAA6B,EAAE;QAClE,SAAS,EAAE,oBAAoB;QAC/B,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,UAAU,CAAC;YAClB,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;gBACtD,QAAQ,EAAE,CAAC,IAAI,CAAC;aACjB;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8BAA8B,EAAE;iBACxE;gBACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;aACrB;YACD,QAAQ,EAAE;gBACR,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC5B;iBACF;gBACD,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;gBAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;aAC3B;SACF;KACF,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,EAAE,CAAC;YAClD,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,IAA0B,CAAC;QACvD,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAEhD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QAC5E,CAAC;QAED,mDAAmD;QACnD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,wBAAwB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7E,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACtD,IAAI,OAAO,EAAE,CAAC;gBACZ,aAAa,CAAC,YAAY,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;gBAClD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,oCAAoC,EAAE,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;QAEpC,wCAAwC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QACvE,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QACzC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,6BAA6B,EAAE,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/runner/coordinator.d.ts

@@ -3,50 +3,118 @@ import type { RunnerBackend } from './types.js';
 /**
  * Coordinates multiple runners. Routes session creation to the least-loaded runner.
  * Detects dead runners and marks their sessions as paused.
+ *
+ * In multi-coordinator mode, the runner registry lives in the shared database
+ * (Postgres/CRDB). Any coordinator can discover all healthy runners by querying
+ * the DB. The in-memory `backends` map is a local connection cache — each
+ * coordinator creates RemoteRunnerBackend instances on demand when it first
+ * needs to talk to a runner it hasn't seen before.
+ *
+ * Design principles:
+ * - DB is the source of truth for runner discovery (multi-coordinator safe)
+ * - Local Map is a connection cache only (avoids creating new HTTP clients per request)
+ * - All write operations (register, heartbeat, delete) go through DB
+ * - All read operations for routing go through DB (selectBestRunner)
+ * - Liveness sweep is idempotent (safe to run on multiple coordinators)
  */
 export declare class RunnerCoordinator {
-    private runners;
+    /** Local connection cache: runnerId -> RemoteRunnerBackend. Lazily populated from DB. */
+    private backends;
     private localBackend;
     private localRunnerId;
     private livenessSweepTimer;
     constructor(opts: {
         localBackend?: RunnerBackend;
     });
+    /**
+     * Register or re-register a runner. Persists to DB so all coordinators see it.
+     */
     registerRunner(info: {
         runnerId: string;
         host: string;
         port: number;
         maxSandboxes: number;
-    }): void;
-    heartbeat(runnerId: string, stats: PoolStats): void;
+    }): Promise<void>;
+    /**
+     * Process a heartbeat. Updates DB so all coordinators see fresh capacity stats.
+     */
+    heartbeat(runnerId: string, stats: PoolStats): Promise<void>;
+    /**
+     * Graceful deregistration. Called when a runner shuts down cleanly.
+     * Immediately pauses its sessions and removes it from the registry,
+     * rather than waiting 30s for the liveness sweep to notice.
+     */
+    deregisterRunner(runnerId: string): Promise<void>;
     /**
      * Select the best backend for a new session.
-     * Returns the least-loaded runner, or local backend if no runners available.
+     * Reads from DB to discover all healthy runners (multi-coordinator safe).
+     * Falls back to local backend if no remote runners available.
      */
-    selectBackend(): {
+    selectBackend(): Promise<{
         backend: RunnerBackend;
         runnerId: string;
-    };
+    }>;
     /**
      * Get the backend for a specific runner. Used for routing messages to existing sessions.
+     *
+     * Synchronous fast path: checks local cache first. If not found, falls back to local
+     * backend (standalone mode) or throws. For multi-coordinator mode where a runner was
+     * registered by a different coordinator, use getBackendForRunnerAsync().
      */
     getBackendForRunner(runnerId: string | null | undefined): RunnerBackend;
+    /**
+     * Async version: looks up runner from DB if not in local cache.
+     * Required in multi-coordinator mode where a different coordinator may have
+     * registered the runner. Creates a RemoteRunnerBackend on the fly from the
+     * DB record and caches it locally.
+     */
+    getBackendForRunnerAsync(runnerId: string | null | undefined): Promise<RunnerBackend>;
+    /**
+     * Get or lazily create a RemoteRunnerBackend from a DB record.
+     * The backends map is a local connection cache — avoids creating
+     * new HTTP clients on every request.
+     */
+    private getOrCreateBackend;
     /**
      * Start periodic liveness checks for remote runners.
+     * Safe to run on multiple coordinators — all operations are idempotent.
+     * Each coordinator runs independently; no leader election needed.
+     *
+     * Uses random jitter (0-5s) on each interval to prevent thundering herd
+     * when multiple coordinators run the same sweep at the same time.
      */
     startLivenessSweep(): void;
     stopLivenessSweep(): void;
+    /**
+     * Single-query dead runner detection. Queries directly for runners past
+     * the heartbeat cutoff instead of listing ALL runners then filtering in JS.
+     *
+     * Cache cleanup: stale backend cache entries are cleaned up in handleDeadRunner()
+     * (for this coordinator's kills) and lazily on next use via getOrCreateBackend()
+     * (for kills by other coordinators). No need to list all runners every sweep.
+     */
     private checkLiveness;
+    /**
+     * Handle a dead runner: pause its sessions and remove from registry.
+     * Idempotent — safe to call from multiple coordinators concurrently.
+     * Uses a single bulk UPDATE instead of per-session queries.
+     */
     handleDeadRunner(runnerId: string): Promise<void>;
     get runnerCount(): number;
     get hasLocalBackend(): boolean;
-    getRunnerInfo(): Array<{
+    /**
+     * Get runner info from DB (not just local cache).
+     * Any coordinator gets the same view. Use this for monitoring/admin.
+     */
+    getRunnerInfoFromDb(): Promise<Array<{
         runnerId: string;
         host: string;
         port: number;
         active: number;
         max: number;
-        lastHeartbeat: number;
-    }>;
+        lastHeartbeat: string;
+    }>>;
+    /** Number of locally cached backend connections. */
+    get cachedBackendCount(): number;
 }
 //# sourceMappingURL=coordinator.d.ts.map

package/dist/runner/coordinator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"coordinator.d.ts","sourceRoot":"","sources":["../../src/runner/coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAchD;;;GAGG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,kBAAkB,CAA+B;gBAE7C,IAAI,EAAE;QAAE,YAAY,CAAC,EAAE,aAAa,CAAA;KAAE;IAIlD,cAAc,CAAC,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAsBlG,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,GAAG,IAAI;IAUnD;;;OAGG;IACH,aAAa,IAAI;QAAE,OAAO,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;IAiC7D;;OAEG;IACH,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,aAAa;IAgBvE;;OAEG;IACH,kBAAkB,IAAI,IAAI;IAU1B,iBAAiB,IAAI,IAAI;YAOX,aAAa;IAUrB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBvD,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,eAAe,IAAI,OAAO,CAE7B;IAED,aAAa,IAAI,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC;CAc7H"}
+{"version":3,"file":"coordinator.d.ts","sourceRoot":"","sources":["../../src/runner/coordinator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAgB,MAAM,gBAAgB,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAIhD;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,iBAAiB;IAC5B,yFAAyF;IACzF,OAAO,CAAC,QAAQ,CAA0C;IAC1D,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,kBAAkB,CAA+B;gBAE7C,IAAI,EAAE;QAAE,YAAY,CAAC,EAAE,aAAa,CAAA;KAAE;IAIlD;;OAEG;IACG,cAAc,CAAC,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAajH;;OAEG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlE;;;;OAIG;IACG,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKvD;;;;OAIG;IACG,aAAa,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAmB5E;;;;;;OAMG;IACH,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,aAAa;IAevE;;;;;OAKG;IACG,wBAAwB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC;IAqB3F;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAS1B;;;;;;;OAOG;IACH,kBAAkB,IAAI,IAAI;IAe1B,iBAAiB,IAAI,IAAI;IAOzB;;;;;;;OAOG;YACW,aAAa;IAS3B;;;;OAIG;IACG,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBvD,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,eAAe,IAAI,OAAO,CAE7B;IAED;;;OAGG;IACG,mBAAmB,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAYjJ,oDAAoD;IACpD,IAAI,kBAAkB,IAAI,MAAM,CAE/B;CACF"}

package/dist/runner/coordinator.js

@@ -1,73 +1,76 @@
 import { RUNNER_LIVENESS_TIMEOUT_MS } from '@ash-ai/shared';
 import { RemoteRunnerBackend } from './remote-backend.js';
-import { updateSessionStatus, listSessionsByRunner } from '../db/index.js';
+import { bulkPauseSessionsByRunner, upsertRunner, heartbeatRunner, selectBestRunner, deleteRunner, listAllRunners, listDeadRunners, getRunner } from '../db/index.js';
 /**
  * Coordinates multiple runners. Routes session creation to the least-loaded runner.
  * Detects dead runners and marks their sessions as paused.
+ *
+ * In multi-coordinator mode, the runner registry lives in the shared database
+ * (Postgres/CRDB). Any coordinator can discover all healthy runners by querying
+ * the DB. The in-memory `backends` map is a local connection cache — each
+ * coordinator creates RemoteRunnerBackend instances on demand when it first
+ * needs to talk to a runner it hasn't seen before.
+ *
+ * Design principles:
+ * - DB is the source of truth for runner discovery (multi-coordinator safe)
+ * - Local Map is a connection cache only (avoids creating new HTTP clients per request)
+ * - All write operations (register, heartbeat, delete) go through DB
+ * - All read operations for routing go through DB (selectBestRunner)
+ * - Liveness sweep is idempotent (safe to run on multiple coordinators)
  */
 export class RunnerCoordinator {
-    runners = new Map();
+    /** Local connection cache: runnerId -> RemoteRunnerBackend. Lazily populated from DB. */
+    backends = new Map();
     localBackend;
     localRunnerId = '__local__';
     livenessSweepTimer = null;
     constructor(opts) {
         this.localBackend = opts.localBackend ?? null;
     }
-    registerRunner(info) {
-        const existing = this.runners.get(info.runnerId);
+    /**
+     * Register or re-register a runner. Persists to DB so all coordinators see it.
+     */
+    async registerRunner(info) {
+        // Persist to DB (upsert — idempotent, safe for concurrent coordinators)
+        await upsertRunner(info.runnerId, info.host, info.port, info.maxSandboxes);
+        // Update local backend cache
+        const existing = this.backends.get(info.runnerId);
         if (existing) {
-            // Re-registration — update connection info
-            existing.host = info.host;
-            existing.port = info.port;
-            existing.maxSandboxes = info.maxSandboxes;
-            existing.lastHeartbeat = Date.now();
-            console.log(`[coordinator] Runner ${info.runnerId} re-registered at ${info.host}:${info.port}`);
-            return;
+            existing.close();
         }
-        const backend = new RemoteRunnerBackend({ host: info.host, port: info.port });
-        this.runners.set(info.runnerId, {
-            ...info,
-            backend,
-            lastHeartbeat: Date.now(),
-            stats: null,
-        });
+        this.backends.set(info.runnerId, new RemoteRunnerBackend({ host: info.host, port: info.port }));
         console.log(`[coordinator] Runner ${info.runnerId} registered at ${info.host}:${info.port} (max ${info.maxSandboxes})`);
     }
-    heartbeat(runnerId, stats) {
-        const runner = this.runners.get(runnerId);
-        if (!runner) {
-            console.warn(`[coordinator] Heartbeat from unknown runner ${runnerId}`);
-            return;
-        }
-        runner.lastHeartbeat = Date.now();
-        runner.stats = stats;
+    /**
+     * Process a heartbeat. Updates DB so all coordinators see fresh capacity stats.
+     */
+    async heartbeat(runnerId, stats) {
+        await heartbeatRunner(runnerId, stats.running ?? 0, stats.warming ?? 0);
+    }
+    /**
+     * Graceful deregistration. Called when a runner shuts down cleanly.
+     * Immediately pauses its sessions and removes it from the registry,
+     * rather than waiting 30s for the liveness sweep to notice.
+     */
+    async deregisterRunner(runnerId) {
+        console.log(`[coordinator] Runner ${runnerId} deregistering gracefully`);
+        await this.handleDeadRunner(runnerId);
     }
     /**
      * Select the best backend for a new session.
-     * Returns the least-loaded runner, or local backend if no runners available.
+     * Reads from DB to discover all healthy runners (multi-coordinator safe).
+     * Falls back to local backend if no remote runners available.
      */
-    selectBackend() {
-        // If we have remote runners, pick the one with most available capacity
-        if (this.runners.size > 0) {
-            let bestRunner = null;
-            let bestAvailable = -1;
-            for (const runner of this.runners.values()) {
-                // Skip dead runners
-                if (Date.now() - runner.lastHeartbeat > RUNNER_LIVENESS_TIMEOUT_MS)
-                    continue;
-                const available = runner.stats
-                    ? runner.maxSandboxes - runner.stats.running - runner.stats.warming
-                    : runner.maxSandboxes;
-                if (available > bestAvailable) {
-                    bestAvailable = available;
-                    bestRunner = runner;
-                }
-            }
-            if (bestRunner && bestAvailable > 0) {
-                return { backend: bestRunner.backend, runnerId: bestRunner.runnerId };
-            }
+    async selectBackend() {
+        const cutoff = new Date(Date.now() - RUNNER_LIVENESS_TIMEOUT_MS).toISOString();
+        const bestRunner = await selectBestRunner(cutoff);
+        if (bestRunner) {
+            // selectBestRunner() already orders by available capacity DESC and only
+            // returns healthy runners. Trust the query — no redundant capacity check.
+            const backend = this.getOrCreateBackend(bestRunner);
+            return { backend, runnerId: bestRunner.id };
         }
-        // Fall back to local backend
+        // Fall back to local backend (standalone mode)
         if (this.localBackend) {
             return { backend: this.localBackend, runnerId: this.localRunnerId };
         }
@@ -75,6 +78,10 @@ export class RunnerCoordinator {
     }
     /**
      * Get the backend for a specific runner. Used for routing messages to existing sessions.
+     *
+     * Synchronous fast path: checks local cache first. If not found, falls back to local
+     * backend (standalone mode) or throws. For multi-coordinator mode where a runner was
+     * registered by a different coordinator, use getBackendForRunnerAsync().
      */
     getBackendForRunner(runnerId) {
         if (!runnerId || runnerId === this.localRunnerId) {
@@ -82,25 +89,76 @@ export class RunnerCoordinator {
                 return this.localBackend;
             throw new Error('No local backend configured');
         }
-        const runner = this.runners.get(runnerId);
-        if (!runner) {
-            // Runner gone — fall back to local if available
+        const cached = this.backends.get(runnerId);
+        if (cached && !cached.closed)
+            return cached;
+        // Not in cache. In standalone mode, fall back to local.
+        // In coordinator mode, caller should use getBackendForRunnerAsync().
+        if (this.localBackend)
+            return this.localBackend;
+        throw new Error(`Runner ${runnerId} not found in local cache — use getBackendForRunnerAsync() in multi-coordinator mode`);
+    }
+    /**
+     * Async version: looks up runner from DB if not in local cache.
+     * Required in multi-coordinator mode where a different coordinator may have
+     * registered the runner. Creates a RemoteRunnerBackend on the fly from the
+     * DB record and caches it locally.
+     */
+    async getBackendForRunnerAsync(runnerId) {
+        if (!runnerId || runnerId === this.localRunnerId) {
             if (this.localBackend)
                 return this.localBackend;
-            throw new Error(`Runner ${runnerId} not found`);
+            throw new Error('No local backend configured');
+        }
+        // Fast path: already cached
+        const cached = this.backends.get(runnerId);
+        if (cached && !cached.closed)
+            return cached;
+        // Slow path: look up from shared DB
+        const record = await getRunner(runnerId);
+        if (record) {
+            return this.getOrCreateBackend(record);
         }
-        return runner.backend;
+        // Runner gone — fall back to local if available
+        if (this.localBackend)
+            return this.localBackend;
+        throw new Error(`Runner ${runnerId} not found`);
+    }
+    /**
+     * Get or lazily create a RemoteRunnerBackend from a DB record.
+     * The backends map is a local connection cache — avoids creating
+     * new HTTP clients on every request.
+     */
+    getOrCreateBackend(record) {
+        let backend = this.backends.get(record.id);
+        if (backend && !backend.closed)
+            return backend;
+        backend = new RemoteRunnerBackend({ host: record.host, port: record.port });
+        this.backends.set(record.id, backend);
+        return backend;
     }
     /**
      * Start periodic liveness checks for remote runners.
+     * Safe to run on multiple coordinators — all operations are idempotent.
+     * Each coordinator runs independently; no leader election needed.
+     *
+     * Uses random jitter (0-5s) on each interval to prevent thundering herd
+     * when multiple coordinators run the same sweep at the same time.
      */
     startLivenessSweep() {
         if (this.livenessSweepTimer)
             return;
-        this.livenessSweepTimer = setInterval(() => {
-            this.checkLiveness().catch((err) => console.error('[coordinator] Liveness sweep error:', err));
-        }, RUNNER_LIVENESS_TIMEOUT_MS);
-        this.livenessSweepTimer.unref();
+        const scheduleNext = () => {
+            // Add 0-5s random jitter to prevent thundering herd across coordinators
+            const jitter = Math.floor(Math.random() * 5000);
+            this.livenessSweepTimer = setTimeout(() => {
+                this.checkLiveness()
+                    .catch((err) => console.error('[coordinator] Liveness sweep error:', err))
+                    .finally(() => scheduleNext());
+            }, RUNNER_LIVENESS_TIMEOUT_MS + jitter);
+            this.livenessSweepTimer.unref();
+        };
+        scheduleNext();
     }
     stopLivenessSweep() {
         if (this.livenessSweepTimer) {
@@ -108,50 +166,66 @@ export class RunnerCoordinator {
             this.livenessSweepTimer = null;
         }
     }
+    /**
+     * Single-query dead runner detection. Queries directly for runners past
+     * the heartbeat cutoff instead of listing ALL runners then filtering in JS.
+     *
+     * Cache cleanup: stale backend cache entries are cleaned up in handleDeadRunner()
+     * (for this coordinator's kills) and lazily on next use via getOrCreateBackend()
+     * (for kills by other coordinators). No need to list all runners every sweep.
+     */
     async checkLiveness() {
-        const now = Date.now();
-        for (const [runnerId, runner] of this.runners) {
-            if (now - runner.lastHeartbeat > RUNNER_LIVENESS_TIMEOUT_MS) {
-                console.warn(`[coordinator] Runner ${runnerId} missed heartbeat — marking sessions paused`);
-                await this.handleDeadRunner(runnerId);
-            }
+        const cutoff = new Date(Date.now() - RUNNER_LIVENESS_TIMEOUT_MS).toISOString();
+        const deadRunners = await listDeadRunners(cutoff);
+        for (const runner of deadRunners) {
+            console.warn(`[coordinator] Runner ${runner.id} missed heartbeat — marking sessions paused`);
+            await this.handleDeadRunner(runner.id);
         }
     }
+    /**
+     * Handle a dead runner: pause its sessions and remove from registry.
+     * Idempotent — safe to call from multiple coordinators concurrently.
+     * Uses a single bulk UPDATE instead of per-session queries.
+     */
     async handleDeadRunner(runnerId) {
-        // Mark all sessions on this runner as paused
-        const sessions = await listSessionsByRunner(runnerId);
-        for (const session of sessions) {
-            if (session.status === 'active' || session.status === 'starting') {
-                await updateSessionStatus(session.id, 'paused');
-                console.log(`[coordinator] Paused session ${session.id} (runner ${runnerId} dead)`);
-            }
+        // Single bulk UPDATE — no N+1
+        const pausedCount = await bulkPauseSessionsByRunner(runnerId);
+        if (pausedCount > 0) {
+            console.log(`[coordinator] Paused ${pausedCount} session(s) on runner ${runnerId}`);
         }
-        // Remove the runner
-        const runner = this.runners.get(runnerId);
-        if (runner) {
-            runner.backend.close();
-            this.runners.delete(runnerId);
+        // Remove from DB (all coordinators will see this)
+        await deleteRunner(runnerId);
+        // Clean up local cache
+        const backend = this.backends.get(runnerId);
+        if (backend) {
+            backend.close();
+            this.backends.delete(runnerId);
         }
     }
     get runnerCount() {
-        return this.runners.size;
+        return this.backends.size;
     }
     get hasLocalBackend() {
         return this.localBackend !== null;
     }
-    getRunnerInfo() {
-        const result = [];
-        for (const runner of this.runners.values()) {
-            result.push({
-                runnerId: runner.runnerId,
-                host: runner.host,
-                port: runner.port,
-                active: runner.stats?.running ?? 0,
-                max: runner.maxSandboxes,
-                lastHeartbeat: runner.lastHeartbeat,
-            });
-        }
-        return result;
+    /**
+     * Get runner info from DB (not just local cache).
+     * Any coordinator gets the same view. Use this for monitoring/admin.
+     */
+    async getRunnerInfoFromDb() {
+        const allRunners = await listAllRunners();
+        return allRunners.map((r) => ({
+            runnerId: r.id,
+            host: r.host,
+            port: r.port,
+            active: r.activeCount,
+            max: r.maxSandboxes,
+            lastHeartbeat: r.lastHeartbeatAt,
+        }));
+    }
+    /** Number of locally cached backend connections. */
+    get cachedBackendCount() {
+        return this.backends.size;
     }
 }
 //# sourceMappingURL=coordinator.js.map

package/dist/runner/coordinator.js.map

@@ -1 +1 @@
-{"version":3,"file":"coordinator.js","sourceRoot":"","sources":["../../src/runner/coordinator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAE5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAY3E;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACpB,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IACxC,YAAY,CAAuB;IACnC,aAAa,GAAG,WAAW,CAAC;IAC5B,kBAAkB,GAA0B,IAAI,CAAC;IAEzD,YAAY,IAAsC;QAChD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;IAChD,CAAC;IAED,cAAc,CAAC,IAA4E;QACzF,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjD,IAAI,QAAQ,EAAE,CAAC;YACb,2CAA2C;YAC3C,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YAC1B,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;YAC1B,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;YAC1C,QAAQ,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,QAAQ,qBAAqB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAChG,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE;YAC9B,GAAG,IAAI;YACP,OAAO;YACP,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;YACzB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,QAAQ,kBAAkB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IAC1H,CAAC;IAED,SAAS,CAAC,QAAgB,EAAE,KAAgB;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,+CAA+C,QAAQ,EAAE,CAAC,CAAC;YACxE,OAAO;QACT,CAAC;QACD,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,aAAa;QACX,uEAAuE;QACvE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,UAAU,GAAsB,IAAI,CAAC;YACzC,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC;YAEvB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC3C,oBAAoB;gBACpB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,aAAa,GAAG,0BAA0B;oBAAE,SAAS;gBAE7E,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK;oBAC5B,CAAC,CAAC,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO;oBACnE,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;gBAExB,IAAI,SAAS,GAAG,aAAa,EAAE,CAAC;oBAC9B,aAAa,GAAG,SAAS,CAAC;oBAC1B,UAAU,GAAG,MAAM,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,IAAI,UAAU,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACpC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxE,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,QAAmC;QACrD,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YACjD,IAAI,IAAI,CAAC,YAAY;gBAAE,OAAO,IAAI,CAAC,YAAY,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,gDAAgD;YAChD,IAAI,IAAI,CAAC,YAAY;gBAAE,OAAO,IAAI,CAAC,YAAY,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,UAAU,QAAQ,YAAY,CAAC,CAAC;QAClD,CAAC;QAED,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,IAAI,CAAC,kBAAkB,GAAG,WAAW,CAAC,GAAG,EAAE;YACzC,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CACjC,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,GAAG,CAAC,CAC1D,CAAC;QACJ,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;IAED,iBAAiB;QACf,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACvC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QACjC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9C,IAAI,GAAG,GAAG,MAAM,CAAC,aAAa,GAAG,0BAA0B,EAAE,CAAC;gBAC5D,OAAO,CAAC,IAAI,CAAC,wBAAwB,QAAQ,6CAA6C,CAAC,CAAC;gBAC5F,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACtD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;gBACjE,MAAM,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;gBAChD,OAAO,CAAC,GAAG,CAAC,gCAAgC,OAAO,CAAC,EAAE,YAAY,QAAQ,QAAQ,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC;IACpC,CAAC;IAED,aAAa;QACX,MAAM,MAAM,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,CAAC;gBAClC,GAAG,EAAE,MAAM,CAAC,YAAY;gBACxB,aAAa,EAAE,MAAM,CAAC,aAAa;aACpC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"coordinator.js","sourceRoot":"","sources":["../../src/runner/coordinator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AAE5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEtK;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,iBAAiB;IAC5B,yFAAyF;IACjF,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAC;IAClD,YAAY,CAAuB;IACnC,aAAa,GAAG,WAAW,CAAC;IAC5B,kBAAkB,GAA0B,IAAI,CAAC;IAEzD,YAAY,IAAsC;QAChD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAA4E;QAC/F,wEAAwE;QACxE,MAAM,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAE3E,6BAA6B;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,mBAAmB,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAChG,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,QAAQ,kBAAkB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IAC1H,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,QAAgB,EAAE,KAAgB;QAChD,MAAM,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,IAAI,CAAC,EAAE,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,2BAA2B,CAAC,CAAC;QACzE,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa;QACjB,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,0BAA0B,CAAC,CAAC,WAAW,EAAE,CAAC;QAC/E,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAElD,IAAI,UAAU,EAAE,CAAC;YACf,wEAAwE;YACxE,0EAA0E;YAC1E,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;YACpD,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;QAC9C,CAAC;QAED,+CAA+C;QAC/C,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED;;;;;;OAMG;IACH,mBAAmB,CAAC,QAAmC;QACrD,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YACjD,IAAI,IAAI,CAAC,YAAY;gBAAE,OAAO,IAAI,CAAC,YAAY,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,MAAM,CAAC;QAE5C,wDAAwD;QACxD,qEAAqE;QACrE,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,UAAU,QAAQ,sFAAsF,CAAC,CAAC;IAC5H,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,wBAAwB,CAAC,QAAmC;QAChE,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;YACjD,IAAI,IAAI,CAAC,YAAY;gBAAE,OAAO,IAAI,CAAC,YAAY,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,4BAA4B;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,MAAM,CAAC;QAE5C,oCAAoC;QACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,gDAAgD;QAChD,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,UAAU,QAAQ,YAAY,CAAC,CAAC;IAClD,CAAC;IAED;;;;OAIG;IACK,kBAAkB,CAAC,MAAoB;QAC7C,IAAI,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC3C,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC;QAE/C,OAAO,GAAG,IAAI,mBAAmB,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5E,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACtC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;;OAOG;IACH,kBAAkB;QAChB,IAAI,IAAI,CAAC,kBAAkB;YAAE,OAAO;QACpC,MAAM,YAAY,GAAG,GAAG,EAAE;YACxB,wEAAwE;YACxE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC;YAChD,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC,GAAG,EAAE;gBACxC,IAAI,CAAC,aAAa,EAAE;qBACjB,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,GAAG,CAAC,CAAC;qBACzE,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC;YACnC,CAAC,EAAE,0BAA0B,GAAG,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAClC,CAAC,CAAC;QACF,YAAY,EAAE,CAAC;IACjB,CAAC;IAED,iBAAiB;QACf,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACvC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QACjC,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,aAAa;QACzB,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,0BAA0B,CAAC,CAAC,WAAW,EAAE,CAAC;QAC/E,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;QAClD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,EAAE,6CAA6C,CAAC,CAAC;YAC7F,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,8BAA8B;QAC9B,MAAM,WAAW,GAAG,MAAM,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QAC9D,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,yBAAyB,QAAQ,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,kDAAkD;QAClD,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;QAE7B,uBAAuB;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,mBAAmB;QACvB,MAAM,UAAU,GAAG,MAAM,cAAc,EAAE,CAAC;QAC1C,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5B,QAAQ,EAAE,CAAC,CAAC,EAAE;YACd,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,WAAW;YACrB,GAAG,EAAE,CAAC,CAAC,YAAY;YACnB,aAAa,EAAE,CAAC,CAAC,eAAe;SACjC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,oDAAoD;IACpD,IAAI,kBAAkB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;CACF"}

package/dist/runner/local-backend.d.ts

@@ -13,6 +13,7 @@ export declare class LocalRunnerBackend implements RunnerBackend {
     destroySandbox(sandboxId: string): Promise<void>;
     destroyAll(): Promise<void>;
     sendCommand(sandboxId: string, cmd: BridgeCommand): AsyncGenerator<BridgeEvent>;
+    interrupt(sandboxId: string): void;
     getSandbox(sandboxId: string): SandboxHandle | undefined;
     isSandboxAlive(sandboxId: string): boolean;
     markRunning(sandboxId: string): void;