@roj-ai/sdk 0.1.36 → 0.2.2

package/dist/plugins/workers/plugin.js

@@ -1,83 +1,186 @@
 import z from 'zod/v4';
 import { agentIdSchema } from '../../core/agents/schema.js';
 import { ValidationErrors } from '../../core/errors.js';
-import { createEventsFactory } from '../../core/events/types.js';
 import { definePlugin } from '../../core/plugins/plugin-builder.js';
 import { createTool } from '../../core/tools/definition.js';
 import { Err, Ok } from '../../lib/utils/result.js';
 import { WorkerContextImpl } from './context.js';
+import { workerEvents } from './state.js';
 import { generateWorkerId, WorkerId, workerIdSchema } from './worker.js';
-export const workerEvents = createEventsFactory({
-    events: {
-        worker_started: z.object({
-            workerId: workerIdSchema,
-            agentId: agentIdSchema,
-            workerType: z.string(),
-            config: z.unknown(),
-        }),
-        worker_sub_event: z.object({
-            workerId: workerIdSchema,
-            workerType: z.string(),
-            subEvent: z.record(z.string(), z.unknown()).and(z.object({
-                type: z.string(),
-            })),
-        }),
-        worker_status_changed: z.object({
-            workerId: workerIdSchema,
-            fromStatus: z.enum(['running', 'paused', 'completed', 'failed', 'cancelled']),
-            toStatus: z.enum(['running', 'paused', 'completed', 'failed', 'cancelled']),
-            reason: z.string().optional(),
-        }),
-        worker_completed: z.object({
-            workerId: workerIdSchema,
-            result: z.object({
-                status: z.string(),
-                resultsPath: z.string().optional(),
-                summary: z.string(),
-                data: z.unknown().optional(),
-            }),
-        }),
-        worker_failed: z.object({
-            workerId: workerIdSchema,
-            error: z.string(),
-            resumable: z.boolean(),
-        }),
-    },
-});
+/** Most workers a single session runtime runs at once. */
+export const MAX_CONCURRENT_WORKERS = 10;
+/** Default bound on waiting for a stopped worker's `execute()` to return. */
+export const WORKER_STOP_TIMEOUT_MS = 5000;
+/** Default bound on draining the context effects a stopped worker left in flight. */
+export const WORKER_EFFECT_DRAIN_TIMEOUT_MS = 5000;
 // ============================================================================
 // Worker execution helper
 // ============================================================================
-async function executeWorker(workerId, definition, config, context, emitEvent, runningWorkers, logger) {
+async function executeWorker(workerId, definition, config, context, runningWorkers, logger, releaseLease, executionControl, effectDrainTimeoutMs) {
     try {
         const result = await definition.execute(config, context);
+        if (executionControl.suppressTerminalEvent)
+            return;
         if (result.ok) {
-            await emitEvent(workerEvents.create('worker_completed', {
+            const persisted = await context.persistTerminal(workerEvents.create('worker_completed', {
                 workerId,
                 result: result.value,
             }));
-            logger.info('Worker completed', { workerId, result: result.value.summary });
+            if (persisted)
+                logger.info('Worker completed', { workerId, result: result.value.summary });
         }
         else {
-            await emitEvent(workerEvents.create('worker_failed', {
+            const persisted = await context.persistTerminal(workerEvents.create('worker_failed', {
                 workerId,
                 error: result.error.message,
                 resumable: result.error.resumable,
             }));
-            logger.warn('Worker failed', { workerId, error: result.error.message, resumable: result.error.resumable });
+            if (persisted)
+                logger.warn('Worker failed', { workerId, error: result.error.message, resumable: result.error.resumable });
         }
     }
     catch (error) {
+        if (executionControl.suppressTerminalEvent)
+            return;
         const message = error instanceof Error ? error.message : String(error);
-        await emitEvent(workerEvents.create('worker_failed', {
+        const persisted = await context.persistTerminal(workerEvents.create('worker_failed', {
             workerId,
             error: message,
             resumable: false,
         }));
-        logger.error('Worker threw exception', error instanceof Error ? error : undefined, { workerId });
+        if (persisted)
+            logger.error('Worker threw exception', error instanceof Error ? error : undefined, { workerId });
     }
     finally {
-        runningWorkers.delete(workerId);
+        // Bounded like the stop path: a stalled append must not hold the runtime lease forever.
+        const drained = await awaitWithin(context.waitForEffects(), effectDrainTimeoutMs);
+        if (!drained)
+            context.releaseEffectLeases();
+        // A stop may already have replaced this entry with a relaunch; only drop our own.
+        if (runningWorkers.get(workerId)?.context === context)
+            runningWorkers.delete(workerId);
+        releaseLease();
+    }
+}
+function resolveStopTimeouts(pluginConfig) {
+    return {
+        stopTimeoutMs: pluginConfig.stopTimeoutMs ?? WORKER_STOP_TIMEOUT_MS,
+        effectDrainTimeoutMs: pluginConfig.effectDrainTimeoutMs ?? WORKER_EFFECT_DRAIN_TIMEOUT_MS,
+    };
+}
+/** Await `work`, giving up after `timeoutMs`. Returns false when the wait timed out. */
+async function awaitWithin(work, timeoutMs) {
+    let timeoutId;
+    const timeout = new Promise((resolve) => {
+        timeoutId = setTimeout(() => resolve(false), timeoutMs);
+    });
+    try {
+        return await Promise.race([work.then(() => true, () => true), timeout]);
+    }
+    finally {
+        if (timeoutId)
+            clearTimeout(timeoutId);
+    }
+}
+/**
+ * Stop workers and give back their runtime leases, on a deadline.
+ *
+ * Shared by cancel, pause and session close so the three cannot drift: a worker
+ * that ignores cancellation must never pin the runtime, so both the execution
+ * wait and the effect drain are bounded and the leases are force-released once
+ * either bound expires.
+ *
+ * Giving up on the body does not stop it. A worker whose `execute()` outlived
+ * `stopTimeoutMs` keeps running with a live FileStore after its map entry is
+ * gone, so it is recorded in `abandonedWorkers` and `resume` refuses to launch a
+ * second body beside it until the first one finally returns.
+ */
+async function stopWorkers(workers, runningWorkers, abandonedWorkers, timeouts) {
+    if (workers.length === 0)
+        return;
+    for (const worker of workers) {
+        worker.executionControl.suppressTerminalEvent = true;
+        worker.context.invalidate();
+    }
+    const exited = await awaitWithin(Promise.allSettled(workers.map((worker) => worker.promise)), timeouts.stopTimeoutMs);
+    const drained = await awaitWithin(Promise.allSettled(workers.map((worker) => worker.context.waitForEffects())), timeouts.effectDrainTimeoutMs);
+    if (!exited || !drained) {
+        for (const worker of workers)
+            worker.releaseLease();
+    }
+    // The `worker:<id>` lease is not the only one a stalled write holds.
+    if (!drained) {
+        for (const worker of workers)
+            worker.context.releaseEffectLeases();
+    }
+    if (!exited) {
+        for (const worker of workers) {
+            abandonedWorkers.add(worker.workerId);
+            void worker.promise.then(() => abandonedWorkers.delete(worker.workerId));
+        }
+    }
+    for (const worker of workers) {
+        if (runningWorkers.get(worker.workerId) === worker)
+            runningWorkers.delete(worker.workerId);
+    }
+}
+/** Explain why a worker is not available for an operation that needs it running. */
+function describeUnavailableWorker(workerId, entry) {
+    if (!entry)
+        return `Worker not found: ${workerId}`;
+    if (entry.status === 'paused')
+        return `Worker is paused: ${workerId} — resume it first`;
+    if (entry.status !== 'running')
+        return `Worker is ${entry.status}: ${workerId}`;
+    // Projection says running but no body is resident — the relaunch has not landed yet.
+    return `Worker is stopped and pending relaunch: ${workerId}`;
+}
+/**
+ * Start a worker in the current runtime.
+ *
+ * The concurrency cap lives here so the unattended relaunch at boot cannot
+ * exceed what `spawn` allows.
+ */
+function launchWorker({ sessionId, agentId, workerId, definition, config, initialState, resumed, files, emitEvent, getSessionState, reserveMailboxMessageSequence, scheduleAgent, runtimeActivity, runningWorkers, effectDrainTimeoutMs, logger, }) {
+    if (runningWorkers.size >= MAX_CONCURRENT_WORKERS) {
+        return Err(`Max concurrent workers reached (${MAX_CONCURRENT_WORKERS})`);
     }
+    const releaseLease = runtimeActivity.tryAcquire(`worker:${workerId}`);
+    if (!releaseLease)
+        return Err('Session runtime is no longer accepting workers');
+    const workerLogger = logger.child({ workerId, workerType: definition.type });
+    const workerContext = new WorkerContextImpl({
+        sessionId,
+        workerId,
+        agentId,
+        workerType: definition.type,
+        files,
+        emitEvent,
+        getSessionState,
+        reserveMailboxMessageSequence,
+        acquireEffectLease: () => runtimeActivity.tryAcquire(`worker:${workerId}:effect`),
+        reducer: definition.reduce,
+        initialState,
+        resumed,
+        logger: workerLogger,
+        schedule: () => scheduleAgent(agentId),
+    });
+    const executionControl = { suppressTerminalEvent: false };
+    const promise = executeWorker(workerId, definition, config, workerContext, runningWorkers, logger, releaseLease, executionControl, effectDrainTimeoutMs).catch((error) => {
+        logger.error('Unhandled error in worker execution', error instanceof Error ? error : undefined, {
+            workerId,
+        });
+    });
+    runningWorkers.set(workerId, {
+        workerId,
+        workerType: definition.type,
+        definition,
+        context: workerContext,
+        promise,
+        releaseLease,
+        executionControl,
+    });
+    return Ok(undefined);
 }
 // ============================================================================
 // Worker Plugin
@@ -110,7 +213,7 @@ export const workerPlugin = definePlugin('workers')
             }
             case 'worker_sub_event': {
                 const worker = workers.get(event.workerId);
-                if (!worker)
+                if (!worker || !Object.hasOwn(worker, 'state'))
                     return workers;
                 const definition = workerDefs.find((w) => w.type === event.workerType);
                 const newState = definition ? definition.reduce(worker.state, event.subEvent) : worker.state;
@@ -127,11 +230,23 @@ export const workerPlugin = definePlugin('workers')
                 if (!worker)
                     return workers;
                 const newWorkers = new Map(workers);
-                newWorkers.set(event.workerId, {
-                    ...worker,
-                    status: event.toStatus,
-                    updatedAt: event.timestamp,
-                });
+                if (event.toStatus === 'running' || event.toStatus === 'paused') {
+                    newWorkers.set(event.workerId, {
+                        ...worker,
+                        status: event.toStatus,
+                        updatedAt: event.timestamp,
+                    });
+                }
+                else {
+                    newWorkers.set(event.workerId, {
+                        id: worker.id,
+                        agentId: worker.agentId,
+                        workerType: worker.workerType,
+                        status: event.toStatus,
+                        createdAt: worker.createdAt,
+                        updatedAt: event.timestamp,
+                    });
+                }
                 return newWorkers;
             }
             case 'worker_completed': {
@@ -140,8 +255,16 @@ export const workerPlugin = definePlugin('workers')
                     return workers;
                 const newWorkers = new Map(workers);
                 newWorkers.set(event.workerId, {
-                    ...worker,
+                    id: worker.id,
+                    agentId: worker.agentId,
+                    workerType: worker.workerType,
                     status: 'completed',
+                    result: {
+                        status: event.result.status,
+                        resultsPath: event.result.resultsPath,
+                        summary: event.result.summary,
+                    },
+                    createdAt: worker.createdAt,
                     updatedAt: event.timestamp,
                 });
                 return newWorkers;
@@ -152,8 +275,12 @@ export const workerPlugin = definePlugin('workers')
                     return workers;
                 const newWorkers = new Map(workers);
                 newWorkers.set(event.workerId, {
-                    ...worker,
+                    id: worker.id,
+                    agentId: worker.agentId,
+                    workerType: worker.workerType,
                     status: 'failed',
+                    error: { message: event.error, resumable: event.resumable },
+                    createdAt: worker.createdAt,
                     updatedAt: event.timestamp,
                 });
                 return newWorkers;
@@ -165,7 +292,12 @@ export const workerPlugin = definePlugin('workers')
 })
     .context(async (ctx) => {
     const runningWorkers = new Map();
-    return { runningWorkers, logger: ctx.logger };
+    // Claimed synchronously by `resume` so the emit it awaits cannot let a second
+    // resume through and overwrite the live entry, leaking its lease.
+    const pendingLaunches = new Set();
+    // Workers whose `execute()` outlived its stop deadline and may still be running.
+    const abandonedWorkers = new Set();
+    return { runningWorkers, pendingLaunches, abandonedWorkers, logger: ctx.logger };
 })
     .agentConfig()
     .method('spawn', {
@@ -194,6 +326,10 @@ export const workerPlugin = definePlugin('workers')
         const validConfig = validationResult.data;
         const workerId = generateWorkerId();
         const { runningWorkers, logger } = ctx.pluginContext;
+        // Refuse before persisting a start we already know we cannot honour.
+        if (runningWorkers.size >= MAX_CONCURRENT_WORKERS) {
+            return Err(ValidationErrors.invalid(`Max concurrent workers reached (${MAX_CONCURRENT_WORKERS})`));
+        }
         // Emit worker started event
         await ctx.emitEvent(workerEvents.create('worker_started', {
             workerId,
@@ -201,45 +337,105 @@ export const workerPlugin = definePlugin('workers')
             workerType: definition.type,
             config: validConfig,
         }));
-        // Create context
-        const workerContext = new WorkerContextImpl({
+        const sessionId = ctx.sessionId;
+        const agentId = input.agentId;
+        const files = ctx.files.session;
+        const emitEvent = ctx.emitEvent;
+        const getSessionState = ctx.getSessionState;
+        const reserveMailboxMessageSequence = ctx.reserveMailboxMessageSequence;
+        const scheduleAgent = ctx.scheduleAgent;
+        const runtimeActivity = ctx.runtimeActivity;
+        const launched = launchWorker({
             sessionId: ctx.sessionId,
             workerId,
-            agentId: input.agentId,
-            workerType: definition.type,
-            files: ctx.files.session,
-            emitEvent: ctx.emitEvent,
-            getSessionState: () => ctx.sessionState,
-            reducer: definition.reduce,
+            agentId,
+            definition,
+            config: validConfig,
             initialState: definition.initialState(validConfig),
-            logger: logger.child({ workerId, workerType: definition.type }),
-            schedule: () => ctx.scheduleAgent(input.agentId),
+            resumed: false,
+            files,
+            emitEvent,
+            getSessionState,
+            reserveMailboxMessageSequence,
+            scheduleAgent,
+            runtimeActivity,
+            runningWorkers,
+            effectDrainTimeoutMs: resolveStopTimeouts(ctx.pluginConfig).effectDrainTimeoutMs,
+            logger,
         });
-        // Enforce max concurrent workers (default: 10)
-        const MAX_CONCURRENT_WORKERS = 10;
-        if (runningWorkers.size >= MAX_CONCURRENT_WORKERS) {
-            return Err(ValidationErrors.invalid(`Max concurrent workers reached (${MAX_CONCURRENT_WORKERS})`));
+        if (!launched.ok) {
+            // The pre-check can go stale across the emit above; keep the projection truthful.
+            await ctx.emitEvent(workerEvents.create('worker_failed', {
+                workerId,
+                error: launched.error,
+                resumable: true,
+            }));
+            return Err(ValidationErrors.invalid(launched.error));
         }
-        // Start worker execution with error boundary
-        const promise = executeWorker(workerId, definition, validConfig, workerContext, ctx.emitEvent, runningWorkers, logger).catch((err) => {
-            logger.error('Unhandled error in worker execution', err instanceof Error ? err : undefined, { workerId });
-        });
-        // Track running worker
-        runningWorkers.set(workerId, {
-            workerId,
-            workerType: definition.type,
-            definition,
-            context: workerContext,
-            promise,
-        });
         logger.info('Worker started', {
-            sessionId: ctx.sessionId,
+            sessionId,
             workerId,
             workerType: definition.type,
             agentId: input.agentId,
         });
         return Ok({ workerId });
     },
+})
+    .sessionHook('onSessionReady', async (ctx) => {
+    const { runningWorkers, logger } = ctx.pluginContext;
+    const files = ctx.files.session;
+    const emitEvent = ctx.emitEvent;
+    const getSessionState = ctx.getSessionState;
+    const reserveMailboxMessageSequence = ctx.reserveMailboxMessageSequence;
+    const scheduleAgent = ctx.scheduleAgent;
+    const runtimeActivity = ctx.runtimeActivity;
+    const { effectDrainTimeoutMs } = resolveStopTimeouts(ctx.pluginConfig);
+    // A relaunch that does not happen must not leave the projection claiming `running`
+    // — park the worker instead, which keeps its state and lets `resume` pick it up.
+    const parkStoppedWorker = async (workerId) => {
+        await ctx.emitEvent(workerEvents.create('worker_status_changed', {
+            workerId,
+            fromStatus: 'running',
+            toStatus: 'paused',
+        }));
+    };
+    for (const worker of ctx.pluginState.values()) {
+        // Paused workers stay stopped — `workers.resume` relaunches them on demand.
+        if (worker.status !== 'running')
+            continue;
+        if (!Object.hasOwn(worker, 'config') || !Object.hasOwn(worker, 'state'))
+            continue;
+        if (runningWorkers.has(worker.id))
+            continue;
+        const definition = ctx.pluginConfig.workers.find((candidate) => candidate.type === worker.workerType);
+        if (!definition) {
+            logger.warn('Cannot resume worker with missing definition', { workerId: worker.id, workerType: worker.workerType });
+            await parkStoppedWorker(worker.id);
+            continue;
+        }
+        const launched = launchWorker({
+            sessionId: ctx.sessionId,
+            agentId: worker.agentId,
+            workerId: worker.id,
+            definition,
+            config: worker.config,
+            initialState: worker.state,
+            resumed: true,
+            files,
+            emitEvent,
+            getSessionState,
+            reserveMailboxMessageSequence,
+            scheduleAgent,
+            runtimeActivity,
+            runningWorkers,
+            effectDrainTimeoutMs,
+            logger,
+        });
+        if (!launched.ok) {
+            logger.warn('Cannot relaunch worker', { workerId: worker.id, workerType: worker.workerType, reason: launched.error });
+            await parkStoppedWorker(worker.id);
+        }
+    }
 })
     .method('getStatus', {
     input: z.object({
@@ -256,8 +452,14 @@ export const workerPlugin = definePlugin('workers')
         if (!worker) {
             return Err(ValidationErrors.invalid(`Worker not found: ${input.workerId}`));
         }
-        const definition = ctx.pluginConfig.workers.find((w) => w.type === worker.workerType);
-        const state = definition?.summarizeState ? definition.summarizeState(worker.state) : worker.state;
+        let state;
+        if (worker.status === 'running' || worker.status === 'paused') {
+            const definition = ctx.pluginConfig.workers.find((w) => w.type === worker.workerType);
+            state = definition?.summarizeState ? definition.summarizeState(worker.state) : worker.state;
+        }
+        else {
+            state = worker.result ?? worker.error;
+        }
         return Ok({
             status: worker.status,
             state,
@@ -277,7 +479,9 @@ export const workerPlugin = definePlugin('workers')
         const workerId = WorkerId(input.workerId);
         const worker = ctx.pluginContext.runningWorkers.get(workerId);
         if (!worker) {
-            return Err(ValidationErrors.invalid(`Worker not running: ${input.workerId}`));
+            // Commands run against a live context, and starting one here would silently
+            // undo an explicit pause — the caller resumes first.
+            return Err(ValidationErrors.invalid(describeUnavailableWorker(workerId, ctx.pluginState.get(workerId))));
         }
         if (!worker.definition.handleCommand) {
             return Err(ValidationErrors.invalid(`Worker type ${worker.workerType} does not support commands`));
@@ -297,16 +501,20 @@ export const workerPlugin = definePlugin('workers')
     output: z.object({}),
     handler: async (ctx, input) => {
         const workerId = WorkerId(input.workerId);
-        const worker = ctx.pluginContext.runningWorkers.get(workerId);
-        if (!worker) {
-            return Err(ValidationErrors.invalid(`Worker not running: ${input.workerId}`));
-        }
-        worker.context.cancel();
+        const { runningWorkers, abandonedWorkers } = ctx.pluginContext;
+        const worker = runningWorkers.get(workerId);
         const workerEntry = ctx.pluginState.get(workerId);
-        const fromStatus = workerEntry?.status ?? 'running';
+        // The projection decides, not the map: a completed worker lingers there for the
+        // whole of its effect drain, and cancelling it would overwrite its real result.
+        if (!workerEntry || (workerEntry.status !== 'running' && workerEntry.status !== 'paused')) {
+            return Err(ValidationErrors.invalid(describeUnavailableWorker(workerId, workerEntry)));
+        }
+        // Bounded, so a worker that ignores cancellation cannot hold the runtime open.
+        if (worker)
+            await stopWorkers([worker], runningWorkers, abandonedWorkers, resolveStopTimeouts(ctx.pluginConfig));
         await ctx.emitEvent(workerEvents.create('worker_status_changed', {
             workerId,
-            fromStatus,
+            fromStatus: workerEntry.status,
             toStatus: 'cancelled',
         }));
         return Ok({});
@@ -321,16 +529,20 @@ export const workerPlugin = definePlugin('workers')
     output: z.object({}),
     handler: async (ctx, input) => {
         const workerId = WorkerId(input.workerId);
-        const worker = ctx.pluginContext.runningWorkers.get(workerId);
+        const { runningWorkers, abandonedWorkers } = ctx.pluginContext;
+        const worker = runningWorkers.get(workerId);
         if (!worker) {
-            return Err(ValidationErrors.invalid(`Worker not running: ${input.workerId}`));
+            return Err(ValidationErrors.invalid(describeUnavailableWorker(workerId, ctx.pluginState.get(workerId))));
         }
         const workerEntry = ctx.pluginState.get(workerId);
         const fromStatus = workerEntry?.status ?? 'running';
         if (fromStatus !== 'running') {
             return Err(ValidationErrors.invalid(`Worker not in running state: ${fromStatus}`));
         }
+        // Pausing stops the run instead of spinning it: a parked worker must not keep
+        // the runtime resident. `resume` re-enters execute() with the replayed state.
         worker.context.pause();
+        await stopWorkers([worker], runningWorkers, abandonedWorkers, resolveStopTimeouts(ctx.pluginConfig));
         await ctx.emitEvent(workerEvents.create('worker_status_changed', {
             workerId,
             fromStatus,
@@ -348,39 +560,89 @@ export const workerPlugin = definePlugin('workers')
     output: z.object({}),
     handler: async (ctx, input) => {
         const workerId = WorkerId(input.workerId);
-        const worker = ctx.pluginContext.runningWorkers.get(workerId);
-        if (!worker) {
-            return Err(ValidationErrors.invalid(`Worker not running: ${input.workerId}`));
-        }
+        const { runningWorkers, pendingLaunches, abandonedWorkers, logger } = ctx.pluginContext;
         const workerEntry = ctx.pluginState.get(workerId);
-        const fromStatus = workerEntry?.status ?? 'paused';
-        if (fromStatus !== 'paused') {
-            return Err(ValidationErrors.invalid(`Worker not in paused state: ${fromStatus}`));
+        if (!workerEntry) {
+            return Err(ValidationErrors.invalid(`Worker not found: ${input.workerId}`));
+        }
+        if (workerEntry.status !== 'paused') {
+            return Err(ValidationErrors.invalid(`Worker not in paused state: ${workerEntry.status}`));
+        }
+        if (!Object.hasOwn(workerEntry, 'config') || !Object.hasOwn(workerEntry, 'state')) {
+            return Err(ValidationErrors.invalid(`Worker is no longer resumable: ${input.workerId}`));
+        }
+        const definition = ctx.pluginConfig.workers.find((candidate) => candidate.type === workerEntry.workerType);
+        if (!definition) {
+            return Err(ValidationErrors.invalid(`Unknown worker type: ${workerEntry.workerType}`));
+        }
+        // Nothing runs a paused worker any more, so resume relaunches it — and must
+        // never overwrite a live entry, whose lease would then leak. The claim below
+        // is therefore taken in the same synchronous block as the checks: everything
+        // from here to `pendingLaunches.add` must stay free of `await`.
+        if (runningWorkers.has(workerId)) {
+            return Err(ValidationErrors.invalid(`Worker is already running: ${input.workerId}`));
+        }
+        if (pendingLaunches.has(workerId)) {
+            return Err(ValidationErrors.invalid(`Worker is already resuming: ${input.workerId}`));
+        }
+        if (abandonedWorkers.has(workerId)) {
+            return Err(ValidationErrors.invalid(`Worker did not stop within its deadline and may still be running: ${input.workerId}`));
+        }
+        if (runningWorkers.size + pendingLaunches.size >= MAX_CONCURRENT_WORKERS) {
+            return Err(ValidationErrors.invalid(`Max concurrent workers reached (${MAX_CONCURRENT_WORKERS})`));
+        }
+        pendingLaunches.add(workerId);
+        try {
+            // The status event goes first: a worker that finishes instantly would otherwise
+            // have its terminal projection overwritten by this event.
+            await ctx.emitEvent(workerEvents.create('worker_status_changed', {
+                workerId,
+                fromStatus: 'paused',
+                toStatus: 'running',
+            }));
+            const launched = launchWorker({
+                sessionId: ctx.sessionId,
+                agentId: workerEntry.agentId,
+                workerId,
+                definition,
+                config: workerEntry.config,
+                initialState: workerEntry.state,
+                resumed: true,
+                files: ctx.files.session,
+                emitEvent: ctx.emitEvent,
+                getSessionState: ctx.getSessionState,
+                reserveMailboxMessageSequence: ctx.reserveMailboxMessageSequence,
+                scheduleAgent: ctx.scheduleAgent,
+                runtimeActivity: ctx.runtimeActivity,
+                runningWorkers,
+                effectDrainTimeoutMs: resolveStopTimeouts(ctx.pluginConfig).effectDrainTimeoutMs,
+                logger,
+            });
+            if (!launched.ok) {
+                // Park it again rather than fail it: both triggers are transient, and a
+                // `worker_failed` would drop the state and config that keep it resumable.
+                await ctx.emitEvent(workerEvents.create('worker_status_changed', {
+                    workerId,
+                    fromStatus: 'running',
+                    toStatus: 'paused',
+                    reason: launched.error,
+                }));
+                return Err(ValidationErrors.invalid(launched.error));
+            }
+            return Ok({});
+        }
+        finally {
+            pendingLaunches.delete(workerId);
         }
-        worker.context.resume();
-        await ctx.emitEvent(workerEvents.create('worker_status_changed', {
-            workerId,
-            fromStatus,
-            toStatus: 'running',
-        }));
-        return Ok({});
     },
 })
     .sessionHook('onSessionClose', async (ctx) => {
-    const { runningWorkers, logger } = ctx.pluginContext;
+    const { runningWorkers, abandonedWorkers, logger } = ctx.pluginContext;
     if (runningWorkers.size === 0)
         return;
-    logger.info('Cancelling running workers on session close', { count: runningWorkers.size });
-    // Cancel all running workers
-    for (const worker of runningWorkers.values()) {
-        worker.context.cancel();
-    }
-    // Await all worker promises with a timeout (5s)
-    const timeout = new Promise((resolve) => setTimeout(resolve, 5000));
-    await Promise.race([
-        Promise.allSettled([...runningWorkers.values()].map((w) => w.promise)),
-        timeout,
-    ]);
+    const closingWorkers = [...runningWorkers.values()];
+    logger.info('Cancelling running workers on session close', { count: closingWorkers.length });
+    await stopWorkers(closingWorkers, runningWorkers, abandonedWorkers, resolveStopTimeouts(ctx.pluginConfig));
     runningWorkers.clear();
 })
     .tools((ctx) => {
@@ -479,7 +741,7 @@ export const workerPlugin = definePlugin('workers')
         },
     }), createTool({
         name: 'worker_cancel',
-        description: 'Cancel a running worker. The worker will stop execution.',
+        description: 'Cancel a running or paused worker. It stops for good and cannot be resumed.',
         input: z.object({
             workerId: workerIdSchema,
         }),
@@ -495,7 +757,7 @@ export const workerPlugin = definePlugin('workers')
         },
     }), createTool({
         name: 'worker_pause',
-        description: 'Pause a running worker. The worker can be resumed later.',
+        description: 'Pause a running worker. It stops where it is; worker_resume restarts it from the progress it had recorded.',
         input: z.object({
             workerId: workerIdSchema,
         }),
@@ -511,7 +773,7 @@ export const workerPlugin = definePlugin('workers')
         },
     }), createTool({
         name: 'worker_resume',
-        description: 'Resume a paused worker.',
+        description: 'Resume a paused worker. It restarts from the progress it had recorded.',
         input: z.object({
             workerId: workerIdSchema,
         }),