@inspectr/mcplab 1.28.5 → 1.30.0

package/dist/app-server/run-queue-domain.js

@@ -1,22 +1,195 @@
 import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
 import { emitQueueEvent, buildQueueState, closeJobClients } from './run-queue-events.js';
 import { admitQueuedJob, executeRunJob } from './run-queue-executor.js';
 import { createRunQueueState, currentWorkerUsage } from './run-queue-state.js';
+import { ROVER_LEASE_REQUEUE_REASONS, ROVER_LEASE_TERMINAL_REASONS, upsertQueueChildProgress } from '@inspectr/mcplab-core';
+import { appendExecutionEvent, readExecutionEvents } from './execution-journal.js';
+import { recordEvaluationTerminalExecution } from './evaluation-journal-writer.js';
+import { projectEvaluationJournal } from './evaluation-journal-projection.js';
 export function createRunQueueService(params) {
     const jobs = params.jobs ?? new Map();
     const state = params.state ?? createRunQueueState(params.settings.defaultQueueWorkers);
     const { settings, oauthSessionManager, deps } = params;
+    const leaseTimers = new Map();
+    let rejectedRoverRegistration;
+    function markRoverStarted(job) {
+        deps.addJobEvent(job, {
+            type: 'started',
+            ts: new Date().toISOString(),
+            payload: {
+                executionType: 'rover',
+                roverAgent: job.runParams.roverAgent,
+                agents: job.runParams.requestedAgents ?? null
+            }
+        });
+    }
+    function appendRoverTraceEvent(job, message, provider) {
+        if (!job?.runParams.evaluationRunId)
+            return;
+        const evaluationRunId = job.runParams.evaluationRunId;
+        const details = Object.fromEntries(Object.entries(message).filter(([key, value]) => !['type', 'prompt', 'text', 'response', 'result', 'content'].includes(key) &&
+            (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean')));
+        appendExecutionEvent(join(settings.runsDir, evaluationRunId), {
+            eventId: `rover-${job.id}-${randomUUID()}`,
+            type: 'rover_event',
+            ts: new Date().toISOString(),
+            evaluationRunId,
+            executionId: job.id,
+            roverType: message.type,
+            provider,
+            ...details
+        });
+    }
     function emit() {
         emitQueueEvent(jobs, state, deps.sendSseEvent);
     }
+    function scheduleLeaseExpiry(job) {
+        const lease = job.roverLease;
+        if (!lease)
+            return;
+        const existing = leaseTimers.get(job.id);
+        if (existing)
+            clearTimeout(existing);
+        const delay = Math.max(0, Date.parse(lease.expiresAt) - Date.now());
+        leaseTimers.set(job.id, setTimeout(() => {
+            leaseTimers.delete(job.id);
+            const current = jobs.get(job.id)?.roverLease;
+            if (!current ||
+                current.leaseId !== lease.leaseId ||
+                Date.parse(current.expiresAt) > Date.now())
+                return;
+            if (job.status === 'running' || job.status === 'paused_rover') {
+                finalizeRoverJob(job, { kind: 'requeue', reason: 'lease_expired' });
+                return;
+            }
+            deps.addJobEvent(job, {
+                type: 'log',
+                ts: new Date().toISOString(),
+                payload: { message: 'Rover assignment lease expired and was requeued.' }
+            });
+            emit();
+            if (job.runParams.executionType === 'rover')
+                params.onRoverJobReleased?.(job.runParams.roverAgent.provider, job.id);
+        }, delay));
+    }
     function addStopEvent(job, message) {
         deps.addJobEvent(job, {
             type: 'error',
             ts: new Date().toISOString(),
-            payload: { message }
+            payload: { message, status: 'stopped' }
+        });
+    }
+    function markEvaluationStopped(job, reason) {
+        const evaluationRunId = job.runParams.evaluationRunId;
+        if (!evaluationRunId)
+            return;
+        appendExecutionEvent(join(settings.runsDir, evaluationRunId), {
+            eventId: `evaluation-stopped-${job.id}`,
+            type: 'evaluation_stopped',
+            ts: new Date().toISOString(),
+            executionId: job.id,
+            evaluationRunId,
+            reason
+        });
+        projectEvaluationJournal({
+            runsDir: settings.runsDir,
+            evaluationRunId,
+            evaluationName: job.runParams.evaluationName,
+            runNote: job.runParams.runNote,
+            executionStatus: 'stopped'
+        });
+    }
+    function recordTerminalExecution(job, status, reason) {
+        const evaluationRunId = job.runParams.evaluationRunId;
+        if (!evaluationRunId)
+            return;
+        recordEvaluationTerminalExecution({
+            runsDir: settings.runsDir,
+            evaluationRunId,
+            evaluationName: job.runParams.evaluationName,
+            runNote: job.runParams.runNote,
+            executionId: job.id,
+            status,
+            reason
         });
     }
+    function clearRoverLease(job) {
+        const timer = leaseTimers.get(job.id);
+        if (timer)
+            clearTimeout(timer);
+        leaseTimers.delete(job.id);
+        job.roverLease = undefined;
+    }
+    function settleUnfinishedChildren(job, status) {
+        job.childProgress = job.childProgress?.map((child) => child.status === 'queued' || child.status === 'running' ? { ...child, status } : child);
+    }
+    function finalizeRoverJob(job, disposition) {
+        if (job.runParams.executionType !== 'rover')
+            return false;
+        state.activeJobIds.delete(job.id);
+        state.admittingJobIds.delete(job.id);
+        state.blockedJobIds.delete(job.id);
+        clearRoverLease(job);
+        const queueIndex = state.queue.indexOf(job.id);
+        if (queueIndex !== -1)
+            state.queue.splice(queueIndex, 1);
+        if (disposition.kind === 'requeue') {
+            job.status = 'waiting_for_rover';
+            state.queue.unshift(job.id);
+            deps.addJobEvent(job, {
+                type: 'log',
+                ts: new Date().toISOString(),
+                payload: { message: `Rover assignment requeued: ${disposition.reason}` }
+            });
+        }
+        else if (disposition.kind === 'completed') {
+            settleUnfinishedChildren(job, disposition.payload?.outcome === 'error'
+                ? 'error'
+                : disposition.payload?.outcome === 'incomplete'
+                    ? 'stopped'
+                    : 'completed');
+            job.status = 'completed';
+            deps.addJobEvent(job, {
+                type: 'completed',
+                ts: new Date().toISOString(),
+                payload: { ...disposition.payload, executionType: 'rover' }
+            });
+            closeJobClients(job);
+        }
+        else if (disposition.kind === 'stopped') {
+            settleUnfinishedChildren(job, 'stopped');
+            job.status = 'stopped';
+            addStopEvent(job, disposition.reason);
+            markEvaluationStopped(job, disposition.reason);
+            closeJobClients(job);
+        }
+        else {
+            settleUnfinishedChildren(job, 'error');
+            job.status = 'error';
+            deps.addJobEvent(job, {
+                type: 'error',
+                ts: new Date().toISOString(),
+                payload: { message: `Rover assignment failed: ${disposition.reason}` }
+            });
+            recordTerminalExecution(job, 'error', disposition.reason);
+            closeJobClients(job);
+        }
+        emit();
+        if (disposition.kind !== 'completed')
+            params.onRoverJobReleased?.(job.runParams.roverAgent.provider, job.id);
+        // Wake API runs after the caller has offered the next Rover assignment.
+        // That assignment must count against the shared worker limit before we claim another job.
+        queueMicrotask(() => {
+            void advance({ emitWhenIdle: true });
+        });
+        return true;
+    }
     function stopQueuedJob(job, message = 'Run stopped before it started') {
+        if (job.runParams.executionType === 'rover') {
+            clearRoverLease(job);
+            settleUnfinishedChildren(job, 'stopped');
+        }
         const idx = state.queue.indexOf(job.id);
         if (idx !== -1)
             state.queue.splice(idx, 1);
@@ -24,6 +197,7 @@ export function createRunQueueService(params) {
         state.blockedJobIds.delete(job.id);
         job.status = 'stopped';
         addStopEvent(job, message);
+        markEvaluationStopped(job, message);
         closeJobClients(job);
     }
     function shouldStartQueuedJobImmediately(jobId) {
@@ -70,6 +244,7 @@ export function createRunQueueService(params) {
         }
         if (job.status !== 'stopped') {
             job.status = 'error';
+            settleUnfinishedChildren(job, 'error');
             deps.addJobEvent(job, {
                 type: 'error',
                 ts: new Date().toISOString(),
@@ -105,6 +280,10 @@ export function createRunQueueService(params) {
         }
         state.blockedJobIds.delete(job.id);
         job.status = outcome.status;
+        if (outcome.status === 'error' || outcome.status === 'stopped') {
+            settleUnfinishedChildren(job, outcome.status);
+            recordTerminalExecution(job, outcome.status, outcome.status === 'stopped' ? 'Run stopped by user' : 'Run failed');
+        }
         closeJobClients(job);
         emit();
         pruneOldJobs();
@@ -187,6 +366,7 @@ export function createRunQueueService(params) {
                 state.queue.splice(queueIndex, 1);
             }
             job.status = 'running';
+            job.roverProgress = { completed: 0, total: job.runParams.roverScenarios?.length ?? 0 };
             state.activeJobIds.add(job.id);
             deps.addJobEvent(job, {
                 type: 'started',
@@ -238,12 +418,17 @@ export function createRunQueueService(params) {
                     index -= 1;
                     continue;
                 }
-                if (nextJob.status !== 'queued' && nextJob.status !== 'blocked_auth') {
+                if (nextJob.status !== 'queued' &&
+                    nextJob.status !== 'blocked_auth' &&
+                    nextJob.status !== 'waiting_for_rover' &&
+                    nextJob.status !== 'paused_rover') {
                     state.queue.splice(index, 1);
                     queueMutated = true;
                     index -= 1;
                     continue;
                 }
+                if (nextJob.status === 'waiting_for_rover' || nextJob.status === 'paused_rover')
+                    continue;
                 if (nextJob.status === 'blocked_auth' && !allowBlockedAuth) {
                     continue;
                 }
@@ -290,24 +475,54 @@ export function createRunQueueService(params) {
         state,
         enqueueRun(runParams, options) {
             const jobId = `run-${Date.now()}-${randomUUID().slice(0, 8)}`;
+            const isRover = runParams.executionType === 'rover';
             const job = {
                 id: jobId,
-                status: 'queued',
+                status: isRover ? 'waiting_for_rover' : 'queued',
                 events: [],
                 clients: new Set(),
                 abortController: new AbortController(),
-                runParams
+                runParams,
+                ...(runParams.executionType === 'rover'
+                    ? {
+                        childProgress: (runParams.roverScenarios ?? []).map((scenario) => ({
+                            scenarioId: scenario.id,
+                            agentName: runParams.roverAgent.name,
+                            completed: 0,
+                            total: 1,
+                            status: 'queued'
+                        }))
+                    }
+                    : {})
             };
             jobs.set(jobId, job);
+            if (runParams.evaluationRunId) {
+                const journalDir = join(settings.runsDir, runParams.evaluationRunId);
+                if (!readExecutionEvents(journalDir).some((event) => event.type === 'evaluation_started')) {
+                    appendExecutionEvent(journalDir, {
+                        eventId: `evaluation-started-${runParams.evaluationRunId}`,
+                        type: 'evaluation_started',
+                        ts: new Date().toISOString(),
+                        evaluationRunId: runParams.evaluationRunId,
+                        evaluationName: runParams.evaluationName,
+                        newConversationBeforeStart: runParams.newConversationBeforeStart,
+                        newConversationBetweenScenarios: runParams.newConversationBetweenScenarios
+                    });
+                }
+            }
             state.queue.push(jobId);
             const queuedPosition = state.queue.length;
-            const shouldAttemptAdvance = currentWorkerUsage(state) < state.queueWorkerCount;
+            const shouldAttemptAdvance = !isRover && currentWorkerUsage(state) < state.queueWorkerCount;
             const shouldStartImmediately = shouldStartQueuedJobImmediately(jobId);
-            if (!shouldStartImmediately) {
+            if (isRover && params.assignRoverJob) {
+                params.assignRoverJob(runParams.roverAgent?.provider ?? 'claude');
+            }
+            if (!shouldStartImmediately || isRover) {
                 deps.addJobEvent(job, {
                     type: 'queued',
                     ts: new Date().toISOString(),
                     payload: {
+                        evaluationRunId: runParams.evaluationRunId,
                         configPath: runParams.configPath,
                         runsPerScenario: runParams.runsPerScenario,
                         scenarioId: runParams.scenarioId ?? null,
@@ -316,7 +531,9 @@ export function createRunQueueService(params) {
                         runNote: runParams.runNote ?? null,
                         serverOverrideAll: runParams.serverOverrideAll ?? null,
                         scenarioServerOverrides: runParams.scenarioServerOverrides ?? null,
-                        position: queuedPosition
+                        position: queuedPosition,
+                        executionType: runParams.executionType ?? 'mcplab',
+                        roverAgent: runParams.roverAgent
                     }
                 });
                 emit();
@@ -332,7 +549,10 @@ export function createRunQueueService(params) {
             const job = jobs.get(jobId);
             if (!job)
                 return null;
-            if (job.status === 'queued' || job.status === 'blocked_auth') {
+            if (job.status === 'queued' ||
+                job.status === 'blocked_auth' ||
+                job.status === 'waiting_for_rover' ||
+                job.status === 'paused_rover') {
                 stopQueuedJob(job);
                 void advance({ emitWhenIdle: true, hostHeader: options?.hostHeader });
                 return { ok: true, status: 'stopped' };
@@ -341,9 +561,62 @@ export function createRunQueueService(params) {
                 return { ok: true, status: job.status };
             }
             job.abortController.abort();
+            if (job.runParams.executionType === 'rover') {
+                params.sendRoverMessage?.({ type: 'stop', jobId: job.id });
+                finalizeRoverJob(job, { kind: 'stopped', reason: 'Run stopped by user' });
+                return { ok: true, status: 'stopped' };
+            }
+            state.activeJobIds.delete(jobId);
             job.status = 'stopped';
+            markEvaluationStopped(job, 'Run stopped by user');
             return { ok: true, status: 'stopped' };
         },
+        stopEvaluationRun(evaluationRunId, options) {
+            const jobsForEvaluation = Array.from(jobs.values()).filter((job) => job.runParams.evaluationRunId === evaluationRunId);
+            if (jobsForEvaluation.length === 0)
+                return null;
+            let stopped = 0;
+            for (const job of jobsForEvaluation) {
+                if (['queued', 'blocked_auth', 'waiting_for_rover', 'paused_rover', 'running'].includes(job.status)) {
+                    if (this.stopJob(job.id, options)?.status === 'stopped')
+                        stopped += 1;
+                }
+            }
+            if (jobsForEvaluation.every((job) => ['completed', 'error', 'stopped'].includes(jobs.get(job.id)?.status ?? job.status))) {
+                for (const job of jobsForEvaluation)
+                    jobs.delete(job.id);
+            }
+            void advance({ emitWhenIdle: true, hostHeader: options?.hostHeader });
+            return { ok: true, stopped };
+        },
+        removeEvaluationRun(evaluationRunId, options) {
+            const jobsForEvaluation = Array.from(jobs.values()).filter((job) => job.runParams.evaluationRunId === evaluationRunId);
+            if (jobsForEvaluation.length === 0)
+                return null;
+            const terminalEvaluation = jobsForEvaluation.every((job) => ['completed', 'error', 'stopped'].includes(job.status));
+            if (!terminalEvaluation &&
+                jobsForEvaluation.some((job) => job.status !== 'queued' &&
+                    job.status !== 'blocked_auth' &&
+                    job.status !== 'waiting_for_rover' &&
+                    job.status !== 'stopped')) {
+                return {
+                    error: 'Evaluation run has already started. Use the stop action instead.',
+                    statusCode: 400
+                };
+            }
+            let removed = 0;
+            for (const job of jobsForEvaluation) {
+                if (terminalEvaluation || job.status === 'stopped') {
+                    jobs.delete(job.id);
+                    removed += 1;
+                }
+                else if (this.removeQueuedJob(job.id, options)) {
+                    removed += 1;
+                }
+            }
+            void advance({ emitWhenIdle: true, hostHeader: options?.hostHeader });
+            return { ok: true, removed };
+        },
         removeQueuedJob(jobId, options) {
             const job = jobs.get(jobId);
             if (!job)
@@ -354,7 +627,9 @@ export function createRunQueueService(params) {
                     statusCode: 400
                 };
             }
-            if (job.status !== 'queued' && job.status !== 'blocked_auth') {
+            if (job.status !== 'queued' &&
+                job.status !== 'blocked_auth' &&
+                job.status !== 'waiting_for_rover') {
                 return { error: 'Job is not queued', statusCode: 404 };
             }
             stopQueuedJob(job, 'Removed from queue by user');
@@ -367,6 +642,21 @@ export function createRunQueueService(params) {
         getQueueState() {
             return buildQueueState(jobs, state);
         },
+        getWaitingRoverJobs(provider) {
+            return state.queue.flatMap((jobId, index) => {
+                const job = jobs.get(jobId);
+                if (job?.status !== 'waiting_for_rover' || job.runParams.roverAgent?.provider !== provider)
+                    return [];
+                return [
+                    {
+                        jobId: job.id,
+                        evaluationName: job.runParams.evaluationName,
+                        provider,
+                        position: index + 1
+                    }
+                ];
+            });
+        },
         subscribeQueue(req, res) {
             if ('flushHeaders' in res && typeof res.flushHeaders === 'function') {
                 res.flushHeaders();
@@ -390,6 +680,430 @@ export function createRunQueueService(params) {
             for (const client of state.clients)
                 client.end();
             state.clients.clear();
+        },
+        assignRoverJob(provider, send, worker) {
+            if (worker && worker.provider !== provider)
+                return null;
+            // A page mismatch invalidates this registration. Wait for a fresh tab
+            // registration or reconnect instead of offering the same rejected job in a loop.
+            if (worker &&
+                rejectedRoverRegistration?.connectionId === worker.connectionId &&
+                rejectedRoverRegistration.version === (worker.registrationVersion ?? 0))
+                return null;
+            const roverAlreadyBusy = Array.from(state.activeJobIds).some((id) => jobs.get(id)?.runParams.executionType === 'rover');
+            if (roverAlreadyBusy)
+                return null;
+            const job = state.queue
+                .map((id) => jobs.get(id))
+                .find((candidate) => candidate?.status === 'waiting_for_rover' &&
+                candidate.runParams.roverAgent?.provider === provider &&
+                (candidate.runParams.roverAgent.providerRevision === undefined ||
+                    worker?.providerRevision === undefined ||
+                    candidate.runParams.roverAgent.providerRevision === worker.providerRevision));
+            if (!job)
+                return null;
+            const assignment = {
+                type: 'assignment',
+                jobId: job.id,
+                evaluationRunId: job.runParams.evaluationRunId,
+                configPath: job.runParams.configPath,
+                evaluationName: job.runParams.evaluationName,
+                agentName: job.runParams.roverAgent.name,
+                agent: job.runParams.roverAgent,
+                scenarios: job.runParams.roverScenarios ?? [],
+                newConversationBeforeStart: job.runParams.roverNewConversationBeforeStart !== false,
+                newConversationBetweenScenarios: job.runParams.roverNewConversationBetweenScenarios !== false
+            };
+            const leaseCapable = worker?.capabilities?.includes('assignment_lease') === true;
+            if (leaseCapable && worker) {
+                const now = new Date();
+                const leaseId = randomUUID();
+                const expiresAt = new Date(now.getTime() + 30_000).toISOString();
+                job.roverLease = {
+                    leaseId,
+                    connectionId: worker.connectionId,
+                    registrationVersion: worker.registrationVersion,
+                    state: 'offered',
+                    expiresAt,
+                    offeredAt: now.toISOString()
+                };
+                assignment.leaseId = leaseId;
+                assignment.leaseExpiresAt = expiresAt;
+                scheduleLeaseExpiry(job);
+            }
+            const index = state.queue.indexOf(job.id);
+            if (index !== -1)
+                state.queue.splice(index, 1);
+            job.status = 'running';
+            state.activeJobIds.add(job.id);
+            if (!send(assignment)) {
+                state.activeJobIds.delete(job.id);
+                clearRoverLease(job);
+                job.status = 'waiting_for_rover';
+                if (index !== -1)
+                    state.queue.splice(index, 0, job.id);
+                return null;
+            }
+            appendRoverTraceEvent(job, assignment, provider);
+            if (!leaseCapable)
+                markRoverStarted(job);
+            emit();
+            return job;
+        },
+        rebindRoverLeases(provider, worker) {
+            const now = Date.now();
+            for (const job of jobs.values()) {
+                if (job.runParams.executionType !== 'rover' ||
+                    job.runParams.roverAgent.provider !== provider ||
+                    !job.roverLease ||
+                    job.status === 'completed' ||
+                    job.status === 'stopped' ||
+                    job.status === 'error')
+                    continue;
+                job.roverLease.connectionId = worker.connectionId;
+                job.roverLease.registrationVersion = worker.registrationVersion;
+                job.roverLease.expiresAt = new Date(now + 30_000).toISOString();
+                scheduleLeaseExpiry(job);
+            }
+        },
+        pauseRoverJob(jobId, connectionId) {
+            const job = jobs.get(jobId);
+            if (!job || job.runParams.executionType !== 'rover' || job.status !== 'running')
+                return;
+            if (connectionId && job.roverLease && job.roverLease.connectionId !== connectionId)
+                return;
+            state.activeJobIds.delete(jobId);
+            job.status = 'paused_rover';
+            state.queue.unshift(jobId);
+            deps.addJobEvent(job, {
+                type: 'log',
+                ts: new Date().toISOString(),
+                payload: { message: 'Rover disconnected. Job paused until Rover reconnects.' }
+            });
+            emit();
+        },
+        resumeRoverJob(jobId) {
+            const job = jobs.get(jobId);
+            if (!job || job.runParams.executionType !== 'rover' || job.status !== 'paused_rover')
+                return false;
+            job.status = 'waiting_for_rover';
+            if (!state.queue.includes(jobId))
+                state.queue.push(jobId);
+            deps.addJobEvent(job, {
+                type: 'log',
+                ts: new Date().toISOString(),
+                payload: { message: 'Rover job resumed and waiting for Rover connection.' }
+            });
+            emit();
+            params.assignRoverJob?.(job.runParams.roverAgent.provider);
+            return true;
+        },
+        completeRoverJob(jobId, payload = {}) {
+            const job = jobs.get(jobId);
+            if (!job ||
+                job.runParams.executionType !== 'rover' ||
+                (job.status !== 'running' && job.status !== 'paused_rover'))
+                return;
+            finalizeRoverJob(job, { kind: 'completed', payload });
+        },
+        stopRoverScenario(jobId, scenarioId) {
+            const job = jobs.get(jobId);
+            if (!job || job.runParams.executionType !== 'rover')
+                return null;
+            if (!scenarioId.trim())
+                return { error: 'Scenario ID is required', statusCode: 400 };
+            const child = (job.childProgress ?? []).find((entry) => entry.scenarioId === scenarioId);
+            if (child?.status === 'completed' || child?.status === 'stopped') {
+                return { ok: true, status: 'stopped' };
+            }
+            if (job.status === 'queued' ||
+                job.status === 'waiting_for_rover' ||
+                job.status === 'paused_rover') {
+                // Revoke paused assignments before editing their scenario list. Reconnection
+                // must offer the remaining work, never resume a lease containing cancelled work.
+                if (!child)
+                    return { error: 'Scenario not found', statusCode: 404 };
+                if (job.status === 'paused_rover') {
+                    clearRoverLease(job);
+                    job.status = 'waiting_for_rover';
+                }
+                job.runParams.roverScenarios = job.runParams.roverScenarios?.filter((scenario) => scenario.id !== scenarioId &&
+                    !(job.childProgress ?? []).some((entry) => entry.scenarioId === scenario.id &&
+                        ['completed', 'error', 'stopped'].includes(entry.status)));
+            }
+            else {
+                const sent = params.sendRoverMessage?.({
+                    type: 'stop_scenario',
+                    jobId,
+                    scenarioId
+                });
+                if (sent === false)
+                    return { error: 'Rover is not connected', statusCode: 409 };
+            }
+            const progress = job.childProgress ?? [];
+            const next = {
+                scenarioId,
+                agentName: job.runParams.roverAgent.name,
+                completed: child?.completed ?? 0,
+                total: child?.total ?? 1,
+                status: 'stopped'
+            };
+            job.childProgress = upsertQueueChildProgress(progress, { ...next });
+            const children = job.childProgress ?? [];
+            const aggregateTerminal = children.length > 0 &&
+                children.every((entry) => ['completed', 'error', 'stopped'].includes(entry.status));
+            if (aggregateTerminal) {
+                finalizeRoverJob(job, { kind: 'stopped', reason: 'Run stopped by user' });
+                return { ok: true, status: 'stopped' };
+            }
+            emit();
+            return { ok: true, status: 'stopped' };
+        },
+        stopScenario(jobId, scenarioId, agentName) {
+            const job = jobs.get(jobId);
+            if (!job || !scenarioId.trim())
+                return null;
+            const matches = (job.childProgress ?? []).filter((entry) => entry.scenarioId === scenarioId);
+            if (matches.length > 1 && !agentName) {
+                return {
+                    error: 'Agent name is required when multiple agents run this scenario',
+                    statusCode: 400
+                };
+            }
+            const child = matches.find((entry) => !agentName || entry.agentName === agentName);
+            if (matches.length > 0 && !child)
+                return { error: 'Scenario-agent child not found', statusCode: 404 };
+            if (child?.status === 'completed' || child?.status === 'stopped') {
+                return { ok: true, status: 'stopped' };
+            }
+            const resolvedAgentName = child?.agentName ??
+                (job.runParams.executionType === 'rover' ? job.runParams.roverAgent.name : undefined);
+            if (job.runParams.executionType === 'rover') {
+                return this.stopRoverScenario(jobId, scenarioId);
+            }
+            if (!resolvedAgentName)
+                return { error: 'Scenario is not initialized', statusCode: 409 };
+            const controller = job.childAbortControllers?.get(`${scenarioId}:${resolvedAgentName}`);
+            if (!controller)
+                return { error: 'Scenario is not running', statusCode: 409 };
+            controller.abort();
+            const progress = job.childProgress ?? [];
+            job.childProgress = upsertQueueChildProgress(progress, {
+                ...child,
+                status: 'stopped'
+            });
+            emit();
+            return { ok: true, status: 'stopped' };
+        },
+        handleRoverMessage(message, provider, send, worker) {
+            const messageJob = typeof message.jobId === 'string' ? jobs.get(message.jobId) : undefined;
+            if (typeof message.jobId === 'string' && typeof message.leaseId === 'string') {
+                const knownJob = jobs.get(message.jobId);
+                const knownLease = knownJob?.roverLease;
+                if (!knownJob ||
+                    !knownLease ||
+                    knownLease.leaseId !== message.leaseId ||
+                    (worker && knownLease.connectionId !== worker.connectionId)) {
+                    send({
+                        type: 'lease_unknown',
+                        jobId: message.jobId,
+                        leaseId: message.leaseId,
+                        reason: 'unknown_lease'
+                    });
+                    return null;
+                }
+            }
+            if (messageJob?.roverLease &&
+                (message.leaseId !== messageJob.roverLease.leaseId ||
+                    (worker && messageJob.roverLease.connectionId !== worker.connectionId)))
+                return null;
+            appendRoverTraceEvent(messageJob, message, provider);
+            if ((message.type === 'assignment_accept' ||
+                message.type === 'assignment_reject' ||
+                message.type === 'lease_renew' ||
+                message.type === 'lease_release') &&
+                typeof message.jobId === 'string' &&
+                typeof message.leaseId === 'string') {
+                const job = jobs.get(message.jobId);
+                const lease = job?.roverLease;
+                if (!job ||
+                    !lease ||
+                    lease.leaseId !== message.leaseId ||
+                    (worker && lease.connectionId !== worker.connectionId))
+                    return null;
+                if (message.type === 'assignment_accept' && lease.state === 'offered') {
+                    lease.state = 'accepted';
+                    lease.acceptedAt = new Date().toISOString();
+                    if (typeof message.tabId === 'number')
+                        lease.tabId = message.tabId;
+                    scheduleLeaseExpiry(job);
+                    markRoverStarted(job);
+                    emit();
+                    return null;
+                }
+                if (message.type === 'lease_renew' && lease.state !== 'offered') {
+                    const requested = Date.parse(String(message.leaseExpiresAt ?? ''));
+                    if (!Number.isFinite(requested) || requested <= Date.now())
+                        return null;
+                    lease.expiresAt = new Date(Math.min(requested, Date.now() + 30_000)).toISOString();
+                    scheduleLeaseExpiry(job);
+                    emit();
+                    return null;
+                }
+                if (message.type === 'assignment_reject' && lease.state === 'offered') {
+                    const detail = typeof message.message === 'string' ? message.message.trim() : '';
+                    const providerUnavailable = message.reason === 'provider_mismatch' || message.reason === 'provider_unavailable';
+                    if (providerUnavailable && message.retryable !== false) {
+                        rejectedRoverRegistration = {
+                            connectionId: lease.connectionId,
+                            version: lease.registrationVersion ?? 0
+                        };
+                    }
+                    // A missing provider capability needs a configuration change, not another offer.
+                    // Older Rover versions omit retryable, so preserve their requeue behavior.
+                    finalizeRoverJob(job, {
+                        kind: message.retryable === false ? 'error' : 'requeue',
+                        reason: `assignment_rejected:${String(message.reason ?? 'unknown')}${detail ? `. ${detail}` : ''}${providerUnavailable && message.retryable !== false ? `. Waiting for Rover to confirm provider ${job.runParams.roverAgent?.provider}. Open its browser agent tab.` : ''}`
+                    });
+                    return null;
+                }
+                if (message.type === 'lease_release') {
+                    const reason = String(message.reason ?? 'error');
+                    const shouldRequeue = ROVER_LEASE_REQUEUE_REASONS.includes(reason) ||
+                        (reason === 'error' && job.status === 'running');
+                    if (shouldRequeue && job.status === 'running')
+                        finalizeRoverJob(job, { kind: 'requeue', reason });
+                    else if (ROVER_LEASE_TERMINAL_REASONS.includes(reason))
+                        finalizeRoverJob(job, { kind: 'error', reason });
+                    send({
+                        type: 'lease_action_ack',
+                        jobId: job.id,
+                        leaseId: message.leaseId,
+                        action: 'release'
+                    });
+                    return null;
+                }
+                return null;
+            }
+            if (typeof message.jobId === 'string' && typeof message.leaseId === 'string') {
+                const leasedJob = jobs.get(message.jobId);
+                if (leasedJob?.roverLease && leasedJob.roverLease.leaseId !== message.leaseId)
+                    return null;
+                if (leasedJob?.roverLease?.state === 'accepted' &&
+                    (message.type === 'progress' ||
+                        message.type === 'stage' ||
+                        (message.type === 'scenario_status' && message.status === 'running')) &&
+                    (!worker || leasedJob.roverLease.connectionId === worker.connectionId)) {
+                    leasedJob.roverLease.state = 'running';
+                    emit();
+                }
+            }
+            if (message.type === 'scenario_status' && typeof message.jobId === 'string') {
+                const job = jobs.get(message.jobId);
+                const roverAgent = job?.runParams.executionType === 'rover' ? job.runParams.roverAgent : undefined;
+                if (job?.runParams.executionType === 'rover' &&
+                    roverAgent &&
+                    typeof message.scenarioId === 'string') {
+                    const status = String(message.status ?? 'running');
+                    const allowed = new Set(['queued', 'running', 'completed', 'error', 'stopped']);
+                    if (!allowed.has(status))
+                        return null;
+                    const existing = job.childProgress ?? [];
+                    const current = existing.find((child) => child.scenarioId === message.scenarioId && child.agentName === roverAgent.name);
+                    const next = {
+                        scenarioId: message.scenarioId,
+                        agentName: roverAgent.name,
+                        completed: Math.max(0, Number(message.completed ?? 0)),
+                        total: Math.max(1, Number(message.total ?? 1)),
+                        status: status,
+                        ...(typeof message.lastDurationMs === 'number'
+                            ? { lastDurationMs: Math.max(0, message.lastDurationMs) }
+                            : {}),
+                        ...(typeof message.error === 'string' ? { error: message.error } : {})
+                    };
+                    const terminal = new Set(['completed', 'error', 'stopped']);
+                    if (current && terminal.has(current.status) && !terminal.has(next.status)) {
+                        return null;
+                    }
+                    if (current && next.completed < current.completed)
+                        return null;
+                    job.childProgress = upsertQueueChildProgress(existing, next);
+                    emit();
+                }
+                return null;
+            }
+            if (message.type === 'stage' && typeof message.jobId === 'string') {
+                const job = jobs.get(message.jobId);
+                const stage = typeof message.stage === 'string' ? message.stage : 'progress';
+                const scenario = typeof message.scenarioId === 'string' ? ` (${message.scenarioId})` : '';
+                const labels = {
+                    prompt_sent: 'Prompt sent to Rover',
+                    waiting_for_response: 'Waiting for the agent response',
+                    response_captured: 'Response captured from Rover',
+                    evaluating: 'Evaluating response in MCPLab',
+                    persisted: 'Result persisted in MCPLab'
+                };
+                if (job?.runParams.executionType === 'rover') {
+                    deps.addJobEvent(job, {
+                        type: 'log',
+                        ts: new Date().toISOString(),
+                        payload: {
+                            message: `${labels[stage] ?? String(message.message ?? 'Rover progress')}${scenario} [agent=${job.runParams.executionType === 'rover' ? job.runParams.roverAgent.name : 'unknown'}]`
+                        }
+                    });
+                    emit();
+                }
+                return null;
+            }
+            if (message.type === 'progress' && typeof message.jobId === 'string') {
+                const job = jobs.get(message.jobId);
+                if (job?.runParams.executionType === 'rover') {
+                    if (typeof message.completed === 'number' && typeof message.total === 'number') {
+                        const total = Math.max(0, message.total);
+                        job.roverProgress = {
+                            completed: Math.max(0, Math.min(message.completed, total)),
+                            total,
+                            ...(typeof message.currentScenarioId === 'string'
+                                ? { currentScenarioId: message.currentScenarioId }
+                                : {}),
+                            ...(typeof message.lastDurationMs === 'number'
+                                ? { lastDurationMs: Math.max(0, message.lastDurationMs) }
+                                : {}),
+                            ...(typeof message.error === 'string' ? { error: message.error } : {})
+                        };
+                    }
+                    deps.addJobEvent(job, {
+                        type: 'log',
+                        ts: new Date().toISOString(),
+                        payload: {
+                            message: String(message.message ??
+                                (message.error ? `Rover error: ${message.error}` : 'Rover progress'))
+                        }
+                    });
+                    if (typeof message.error === 'string' && job.status === 'running' && !job.roverLease) {
+                        finalizeRoverJob(job, { kind: 'error', reason: message.error });
+                    }
+                    emit();
+                }
+            }
+            if (message.type !== 'complete' || typeof message.jobId !== 'string')
+                return null;
+            const completedJob = jobs.get(message.jobId);
+            this.completeRoverJob(message.jobId, {
+                runId: completedJob?.runParams.evaluationRunId ?? message.runId,
+                outcome: message.outcome,
+                provider
+            });
+            if (typeof message.leaseId === 'string') {
+                send({
+                    type: 'lease_action_ack',
+                    jobId: message.jobId,
+                    leaseId: message.leaseId,
+                    action: 'complete'
+                });
+            }
+            const next = this.assignRoverJob(provider, send, worker);
+            return next?.id ?? null;
         }
     };
 }