libre-webui 0.21.3 → 0.22.0

package/backend/dist/services/workAgentService.js

@@ -22,7 +22,9 @@ import { isWebSearchAvailable, userCanUseWebSearch, webSearch, } from './webSear
 import { userModel } from '../models/userModel.js';
 import workTaskService, { WORK_MESSAGE_METADATA_MAX_BYTES, WorkConflictError, WorkNotFoundError, } from './workTaskService.js';
 import { createLogger } from '../utils/logger.js';
+import { getDurableJobRuntime } from '../platform/jobs/durableJobRuntime.js';
 import { boundedOpenAIResponsesOutputItems, OPENAI_RESPONSES_OUTPUT_ITEMS_METADATA_KEY, OPENAI_RESPONSES_STATE_DROPPED_METADATA_KEY, OPENAI_RESPONSES_STATE_SCOPE_METADATA_KEY, } from '../utils/openAIResponsesAdapter.js';
+import { OWNER_DELETE_CONTENT_JOB_TYPE, WORK_EXECUTE_IDEMPOTENCY_SCOPE, } from '../platform/jobs/domainJobContracts.js';
 const logger = createLogger('services:work-agent');
 export const WORK_PROVIDER_STATE_METADATA_KEY = 'workProviderState';
 export const WORK_EMPTY_MODEL_RESPONSE_METADATA_KEY = 'emptyModelResponse';
@@ -33,6 +35,7 @@ const WORK_EMPTY_ROUND_NUDGE_LIMIT = 2;
 // Cross-run replay only needs the shape of past tool calls, not full
 // write_file payloads; bound each persisted argument set.
 const WORK_CHAT_TOOL_CALL_ARGUMENTS_MAX_BYTES = 4096;
+const INTERRUPTED_TOOL_RESULT = 'Tool execution was interrupted; outcome unknown. Inspect the workspace before retrying.';
 export const WORK_TOOL_SCHEMAS = [
     functionTool('list_files', 'List direct children of a workspace directory.', {
         path: stringProperty('Relative directory path. Defaults to workspace root.'),
@@ -79,10 +82,10 @@ export const WORK_WEB_SEARCH_TOOL_SCHEMA = functionTool('web_search', 'Search th
         description: "Optional result count. Defaults to 5, capped by the server's configured limit.",
     },
 }, ['query']);
-export function workToolSchemasForTask(task) {
+export async function workToolSchemasForTask(task) {
     return task.networkEnabled &&
-        isWebSearchAvailable() &&
-        userCanUseWebSearch(userModel.getUserById(task.userId))
+        (await isWebSearchAvailable()) &&
+        (await userCanUseWebSearch(await userModel.getUserById(task.userId)))
         ? [...WORK_TOOL_SCHEMAS, WORK_WEB_SEARCH_TOOL_SCHEMA]
         : WORK_TOOL_SCHEMAS;
 }
@@ -105,8 +108,99 @@ export class WorkAgentService {
             this.executions.set(runId, execution);
         });
     }
+    /**
+     * Durable worker entrypoint. An interrupted active run is moved back to the
+     * queue while its persisted provider/tool transcript is retained. Missing
+     * tool results are restored as "outcome unknown", so the exact external
+     * call is not blindly replayed after a worker crash.
+     */
+    async executeDurable(taskId, runId, userId) {
+        const task = await workTaskService.requireTaskRecord(taskId, userId);
+        const run = await workTaskService.getRun(runId);
+        if (!run || run.taskId !== taskId) {
+            throw new WorkNotFoundError('Work run not found.');
+        }
+        if (!isActiveRunStatus(run.status))
+            return;
+        if (run.status !== 'queued') {
+            // A worker can disappear while its sandbox continues running. Fence all
+            // other lifecycle operations, stop that unknown execution, and only then
+            // reconcile the durable transcript. Replaying against the old container
+            // would allow both workers to mutate one workspace concurrently.
+            workRuntimeService.beginTaskSuspension(taskId);
+            try {
+                await workRuntimeService.interruptContainer(task);
+            }
+            finally {
+                workRuntimeService.releaseTaskSuspension(taskId);
+            }
+            await this.reconcileInterruptedToolCalls(taskId, runId);
+            await workTaskService.updateRun(runId, 'queued');
+            // Tasks deliberately have no queued state: idle means durable work is
+            // admitted but no container/provider side effect has started yet.
+            await workTaskService.updateTaskStatus(taskId, 'idle');
+        }
+        await this.execute(taskId, runId, userId);
+    }
+    /**
+     * Materialize every tool intent whose worker disappeared before its result
+     * commit. This must run while the caller owns the durable job lease. The
+     * persisted result is both user-visible and part of the next model request;
+     * checking the transcript first makes a crash after the insert safe to
+     * reclaim without duplicating the result.
+     */
+    async reconcileInterruptedToolCalls(taskId, runId) {
+        const messages = await workTaskService.getMessages(taskId);
+        const settled = new Set(messages
+            .filter(message => message.runId === runId &&
+            message.role === 'tool' &&
+            message.kind === 'tool_result')
+            .map(message => message.metadata?.toolCallId)
+            .filter((value) => Boolean(typeof value === 'string' && value)));
+        const pending = new Map();
+        for (const message of messages) {
+            if (message.runId !== runId || message.kind !== 'tool_call')
+                continue;
+            const toolCallId = message.metadata?.toolCallId;
+            const name = message.metadata?.name;
+            if (typeof toolCallId !== 'string' ||
+                !toolCallId ||
+                typeof name !== 'string' ||
+                !name ||
+                settled.has(toolCallId)) {
+                continue;
+            }
+            pending.set(toolCallId, { name, message });
+        }
+        let reconciled = 0;
+        for (const [toolCallId, { name }] of pending) {
+            if (settled.has(toolCallId))
+                continue;
+            const metadata = {
+                name,
+                toolName: name,
+                toolCallId,
+                error: true,
+                outcomeUnknown: true,
+                interrupted: true,
+            };
+            const result = await workTaskService.addMessage(taskId, runId, 'tool', 'tool_result', INTERRUPTED_TOOL_RESULT, metadata);
+            settled.add(toolCallId);
+            reconciled += 1;
+            await workEventService.publish(taskId, runId, 'tool_result', {
+                toolCallId,
+                name,
+                phase: 'failed',
+                content: INTERRUPTED_TOOL_RESULT,
+                error: true,
+                outcomeUnknown: true,
+                message: result,
+            }, `message:${result.id}`);
+        }
+        return reconciled;
+    }
     async cancel(taskId, userId) {
-        const task = workTaskService.requireMutableTaskRecord(taskId, userId);
+        const task = await workTaskService.requireMutableTaskRecord(taskId, userId);
         return this.cancelTask(task, userId);
     }
     async removeTask(taskId, userId) {
@@ -114,43 +208,112 @@ export class WorkAgentService {
     }
     async cancelTask(task, userId) {
         const taskId = task.id;
-        const run = workTaskService.getActiveRun(taskId);
+        const run = await workTaskService.getActiveRun(taskId);
         if (!run) {
             throw new WorkConflictError('This Work task has no active run.');
         }
         this.controllers.get(run.id)?.abort();
         await workRuntimeService.stopContainer(task);
         await this.executions.get(run.id);
-        if (isActiveRunStatus(workTaskService.getRun(run.id)?.status)) {
-            workTaskService.updateRun(run.id, 'cancelled', {
+        if (isActiveRunStatus((await workTaskService.getRun(run.id))?.status)) {
+            await workTaskService.updateRun(run.id, 'cancelled', {
                 error: 'Cancelled by user.',
                 finished: true,
             });
-            workTaskService.updateTaskStatus(taskId, 'cancelled');
-            workTaskService.updatePreview(taskId, 'stopped');
-            workEventService.publish(taskId, run.id, 'done', {
+            await workTaskService.updateTaskStatus(taskId, 'cancelled');
+            await workTaskService.updatePreview(taskId, 'stopped');
+            await workEventService.publish(taskId, run.id, 'done', {
                 status: 'cancelled',
                 error: 'Cancelled by user.',
-            });
+            }, 'terminal:cancelled');
         }
         return workTaskService.requireTaskDetail(taskId, userId);
     }
     async removeTasksForUser(userId) {
-        workTaskService.beginUserRetirement(userId);
-        try {
-            const tasks = workTaskService.listTaskRecords(userId);
-            for (const task of tasks) {
-                await this.removeTaskInternal(task.id, userId, true);
+        await workTaskService.withUserLifecycleLease(userId, async (assertHeld) => {
+            workTaskService.beginUserRetirement(userId);
+            try {
+                await this.removeTasksWithRetirementHeld(userId, assertHeld);
+            }
+            finally {
+                workTaskService.releaseUserRetirement(userId);
             }
+        });
+    }
+    /**
+     * Persist a cross-replica account fence before deleting external state.
+     * A failed drain deliberately leaves the account `retiring`, so a retry can
+     * continue while authentication and all new Work admission stay denied.
+     */
+    async retireAndDeleteUser(userId, actorUserId) {
+        return workTaskService.withUserLifecycleLease(userId, async (assertHeld) => {
+            workTaskService.beginUserRetirement(userId);
+            try {
+                await assertHeld();
+                if (!(await userModel.beginUserRetirement(userId)))
+                    return false;
+                await this.drainDurableJobsForUser(userId);
+                await this.removeTasksWithRetirementHeld(userId, assertHeld);
+                // Catch a request admitted before retirement that committed its job
+                // after the first zero observation. Retiring actors cannot be claimed,
+                // and this pass terminalizes any such late queue entry.
+                await this.drainDurableJobsForUser(userId);
+                await assertHeld();
+                return userModel.deleteUserAndEnqueueCleanup(userId, actorUserId);
+            }
+            finally {
+                workTaskService.releaseUserRetirement(userId);
+            }
+        });
+    }
+    async removeTasksWithRetirementHeld(userId, assertHeld = async () => undefined) {
+        const tasks = await workTaskService.listTaskRecords(userId);
+        for (const task of tasks) {
+            await assertHeld();
+            await this.removeTaskInternal(task.id, userId, true);
         }
-        catch (error) {
-            workTaskService.releaseUserRetirement(userId);
-            throw error;
+    }
+    async drainDurableJobsForUser(userId) {
+        const service = getDurableJobRuntime().service;
+        const deadline = Date.now() + 60000;
+        while (true) {
+            // Cleanup jobs initiated by this actor protect other already-deleted
+            // owners. They retain audit attribution to the actor and must reach
+            // success before this identity can disappear; cancelling them strands
+            // the other owner's vectors/blobs forever.
+            await service.cancelAllForActor(userId, 'actor-revoked', {
+                excludeJobTypes: [OWNER_DELETE_CONTENT_JOB_TYPE],
+            });
+            const ordinaryActive = await service.countActiveForActor(userId, {
+                excludeJobTypes: [OWNER_DELETE_CONTENT_JOB_TYPE],
+            });
+            const cleanupActive = await service.countActiveForActor(userId, {
+                jobTypes: [OWNER_DELETE_CONTENT_JOB_TYPE],
+            });
+            const cleanupNotSucceeded = await service.countNonSucceededForActor(userId, {
+                jobTypes: [OWNER_DELETE_CONTENT_JOB_TYPE],
+                excludeHandledLifecycleJobs: true,
+            });
+            if (ordinaryActive === 0 &&
+                cleanupActive === 0 &&
+                cleanupNotSucceeded === 0) {
+                return;
+            }
+            if (cleanupActive === 0 && cleanupNotSucceeded > 0) {
+                throw new WorkConflictError('This account remains retired because an initiated owner cleanup did not succeed.');
+            }
+            if (Date.now() >= deadline) {
+                throw new WorkConflictError('This account remains retired while its jobs and initiated owner cleanups finish draining.');
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
         }
     }
     async revokeWorkAccessForUser(userId, revoke) {
+        return workTaskService.withUserLifecycleLease(userId, assertHeld => this.revokeWorkAccessWithLeaseHeld(userId, revoke, assertHeld));
+    }
+    async revokeWorkAccessWithLeaseHeld(userId, revoke, assertHeld) {
         workTaskService.beginUserRetirement(userId);
-        const tasks = workTaskService.listTaskRecords(userId);
+        const tasks = await workTaskService.listTaskRecords(userId);
         const suspendedTaskIds = [];
         try {
             for (const task of tasks) {
@@ -160,8 +323,9 @@ export class WorkAgentService {
             // Persist revocation before depending on Docker cleanup. Even when a
             // daemon outage prevents teardown, current-role authorization denies
             // every subsequent Work request and a retry/restart can finish cleanup.
+            await assertHeld();
             const result = await revoke();
-            const activeRuns = new Map(tasks.map(task => [task.id, workTaskService.getActiveRun(task.id)]));
+            const activeRuns = new Map(await Promise.all(tasks.map(async (task) => [task.id, await workTaskService.getActiveRun(task.id)])));
             for (const run of activeRuns.values()) {
                 if (run)
                     this.controllers.get(run.id)?.abort();
@@ -179,18 +343,18 @@ export class WorkAgentService {
                 }
                 await workRuntimeService.stopContainer(task);
                 if (run &&
-                    isActiveRunStatus(workTaskService.getRun(run.id)?.status)) {
-                    workTaskService.updateRun(run.id, 'cancelled', {
+                    isActiveRunStatus((await workTaskService.getRun(run.id))?.status)) {
+                    await workTaskService.updateRun(run.id, 'cancelled', {
                         error: 'Administrator access was revoked.',
                         finished: true,
                     });
-                    workTaskService.updateTaskStatus(task.id, 'cancelled');
-                    workEventService.publish(task.id, run.id, 'done', {
+                    await workTaskService.updateTaskStatus(task.id, 'cancelled');
+                    await workEventService.publish(task.id, run.id, 'done', {
                         status: 'cancelled',
                         error: 'Administrator access was revoked.',
-                    });
+                    }, 'terminal:cancelled');
                 }
-                workTaskService.updatePreview(task.id, 'stopped');
+                await workTaskService.updatePreview(task.id, 'stopped');
                 if (run)
                     await this.executions.get(run.id);
             }));
@@ -211,14 +375,14 @@ export class WorkAgentService {
         workTaskService.releaseUserRetirement(userId);
     }
     async removeTaskInternal(taskId, userId, allowRetiringUser) {
-        const task = workTaskService.beginTaskRetirement(taskId, userId, allowRetiringUser);
+        const task = await workTaskService.beginTaskRetirement(taskId, userId, allowRetiringUser);
         try {
-            if (workTaskService.getActiveRun(taskId)) {
+            if (await workTaskService.getActiveRun(taskId)) {
                 await this.cancelTask(task, userId);
             }
             await workRuntimeService.removeTask(task, allowRetiringUser);
             try {
-                workTaskService.deleteTask(taskId, userId);
+                await workTaskService.deleteTask(taskId, userId);
             }
             finally {
                 workRuntimeService.finalizeTaskRemoval(taskId);
@@ -235,7 +399,7 @@ export class WorkAgentService {
         for (const controller of this.controllers.values()) {
             controller.abort();
         }
-        const tasks = workTaskService.listAllTaskRecords();
+        const tasks = await workTaskService.listAllTaskRecords();
         const results = await Promise.allSettled(tasks.map(task => workRuntimeService.stopContainer(task)));
         await Promise.allSettled(this.executions.values());
         return {
@@ -245,6 +409,7 @@ export class WorkAgentService {
     }
     async execute(taskId, runId, userId) {
         const controller = new AbortController();
+        const durableAttemptIdentity = String((await getDurableJobRuntime().service.getByIdempotency(userId, WORK_EXECUTE_IDEMPOTENCY_SCOPE, runId))?.attemptCount ?? 0);
         this.controllers.set(runId, controller);
         let task;
         let releaseExecutionLease;
@@ -256,8 +421,8 @@ export class WorkAgentService {
             executionContainerSettled = true;
         };
         try {
-            task = workTaskService.requireTaskRecord(taskId, userId);
-            const run = workTaskService.getRun(runId);
+            task = await workTaskService.requireTaskRecord(taskId, userId);
+            const run = await workTaskService.getRun(runId);
             if (!run || run.taskId !== taskId) {
                 throw new WorkNotFoundError('Work run not found.');
             }
@@ -266,32 +431,32 @@ export class WorkAgentService {
             if (run.status !== 'queued') {
                 return;
             }
-            workTaskService.updateRun(runId, 'preparing', { started: true });
-            workTaskService.updateTaskStatus(taskId, 'preparing');
-            workEventService.publish(taskId, runId, 'run_state', {
+            await workTaskService.updateRun(runId, 'preparing', { started: true });
+            await workTaskService.updateTaskStatus(taskId, 'preparing');
+            await workEventService.publish(taskId, runId, 'run_state', {
                 status: 'preparing',
                 phase: 'preparing',
-            });
+            }, 'transition:preparing', durableAttemptIdentity);
             for (const skill of WORK_AGENT_SKILLS) {
-                workEventService.publish(taskId, runId, 'skill_loaded', {
+                await workEventService.publish(taskId, runId, 'skill_loaded', {
                     id: skill.id,
                     name: skill.title,
                     description: skill.instructions[0],
-                });
+                }, `skill:${skill.id}`, durableAttemptIdentity);
             }
             releaseExecutionLease = await workRuntimeService.prepare(task, controller.signal);
-            this.throwIfCancelled(runId, controller);
-            workTaskService.updateRun(runId, 'running', { started: true });
-            workTaskService.updateTaskStatus(taskId, 'running');
+            await this.throwIfCancelled(runId, controller);
+            await workTaskService.updateRun(runId, 'running', { started: true });
+            await workTaskService.updateTaskStatus(taskId, 'running');
             const roundLimit = workRuntimeService.limits.maxRounds;
             const toolCallLimit = workToolCallBudget(roundLimit);
             const providerSelection = {
                 providerType: run.providerType,
                 providerId: run.providerId,
             };
-            const providerRoutingFingerprint = workModelProviderService.getRoutingFingerprint(run.model, providerSelection, userId);
-            const providerStateScope = workModelProviderService.getResponsesStateScope(run.model, providerSelection, userId);
-            const messages = this.contextMessages(task, roundLimit, providerStateScope, run);
+            const providerRoutingFingerprint = await workModelProviderService.getRoutingFingerprint(run.model, providerSelection, userId);
+            const providerStateScope = await workModelProviderService.getResponsesStateScope(run.model, providerSelection, userId);
+            const messages = await this.contextMessages(task, roundLimit, providerStateScope, run);
             let totalToolCalls = 0;
             let accumulatedInputTokens = 0;
             let accumulatedOutputTokens = 0;
@@ -300,25 +465,25 @@ export class WorkAgentService {
             let budgetReason = 'round';
             let emptyRoundNudges = 0;
             roundLoop: for (let round = 0; round < roundLimit; round++) {
-                this.throwIfCancelled(runId, controller);
-                workEventService.publish(taskId, runId, 'run_state', {
+                await this.throwIfCancelled(runId, controller);
+                await workEventService.publish(taskId, runId, 'run_state', {
                     status: 'running',
                     phase: 'thinking',
                     round: round + 1,
                     roundLimit,
-                });
-                const contentStream = new WorkDeltaPublisher(taskId, runId, 'assistant_delta', streamedAssistantTotal);
-                const reasoningStream = new WorkDeltaPublisher(taskId, runId, 'reasoning_delta', streamedReasoningTotal);
+                }, `round:${round + 1}:thinking`, durableAttemptIdentity);
+                const contentStream = new WorkDeltaPublisher(taskId, runId, 'assistant_delta', streamedAssistantTotal, `round:${round + 1}`, durableAttemptIdentity);
+                const reasoningStream = new WorkDeltaPublisher(taskId, runId, 'reasoning_delta', streamedReasoningTotal, `round:${round + 1}`, durableAttemptIdentity);
                 let roundInputTokens = 0;
                 let roundOutputTokens = 0;
                 const roundStartedAt = Date.now();
                 let response;
                 try {
-                    this.assertStableProviderRouting(run, userId, providerRoutingFingerprint);
+                    await this.assertStableProviderRouting(run, userId, providerRoutingFingerprint);
                     response = await workModelProviderService.generateChatStreamResponse({
                         model: run.model,
                         messages,
-                        tools: workToolSchemasForTask(task),
+                        tools: await workToolSchemasForTask(task),
                         stream: true,
                     }, providerSelection, userId, {
                         onContent: delta => contentStream.push(delta),
@@ -326,7 +491,7 @@ export class WorkAgentService {
                         onUsage: usage => {
                             roundInputTokens = usage.promptTokens ?? roundInputTokens;
                             roundOutputTokens = usage.completionTokens ?? roundOutputTokens;
-                            workEventService.publish(taskId, runId, 'usage', {
+                            void workEventService.publish(taskId, runId, 'usage', {
                                 inputTokens: accumulatedInputTokens + roundInputTokens,
                                 outputTokens: accumulatedOutputTokens + roundOutputTokens,
                                 totalTokens: accumulatedInputTokens +
@@ -344,7 +509,7 @@ export class WorkAgentService {
                     streamedAssistantTotal = contentStream.currentTotal;
                     streamedReasoningTotal = reasoningStream.currentTotal;
                 }
-                this.throwIfCancelled(runId, controller);
+                await this.throwIfCancelled(runId, controller);
                 assertCompleteProviderResponse(response);
                 accumulatedInputTokens += roundInputTokens;
                 accumulatedOutputTokens += roundOutputTokens;
@@ -377,7 +542,7 @@ export class WorkAgentService {
                 const assistantContent = boundUtf8(response.message?.content?.trim() || '', 100000);
                 const reasoningContent = boundUtf8(response.message?.thinking?.trim() || '', 100000);
                 if (reasoningContent) {
-                    workTaskService.addMessage(taskId, runId, 'assistant', 'reasoning', reasoningContent, { providerExposed: true, round: round + 1 });
+                    await workTaskService.addMessage(taskId, runId, 'assistant', 'reasoning', reasoningContent, { providerExposed: true, round: round + 1 });
                 }
                 if (toolCalls.length === 0) {
                     if (!assistantContent &&
@@ -394,7 +559,7 @@ export class WorkAgentService {
                             });
                         }
                         if (providerStateMetadata) {
-                            workTaskService.addMessage(taskId, runId, 'assistant', 'provider_state', '', providerStateMetadata);
+                            await workTaskService.addMessage(taskId, runId, 'assistant', 'provider_state', '', providerStateMetadata);
                         }
                         messages.push({
                             role: 'user',
@@ -404,7 +569,7 @@ export class WorkAgentService {
                     }
                     const finalContent = assistantContent ||
                         'The model completed without returning a text response.';
-                    workTaskService.addMessage(taskId, runId, 'assistant', 'message', finalContent, assistantContent
+                    await workTaskService.addMessage(taskId, runId, 'assistant', 'message', finalContent, assistantContent
                         ? providerStateMetadata
                         : {
                             // The placeholder tells the user what happened, but it is
@@ -417,15 +582,17 @@ export class WorkAgentService {
                     // stopped or been retained for a verified live preview. Consumers
                     // can then refresh files immediately without racing teardown.
                     await settleExecutionContainer();
-                    this.throwIfCancelled(runId, controller);
-                    if (!isActiveRunStatus(workTaskService.getRun(runId)?.status)) {
+                    await this.throwIfCancelled(runId, controller);
+                    if (!isActiveRunStatus((await workTaskService.getRun(runId))?.status)) {
                         return;
                     }
-                    workTaskService.updateRun(runId, 'completed', { finished: true });
-                    workTaskService.updateTaskStatus(taskId, 'completed');
-                    workEventService.publish(taskId, runId, 'done', {
-                        status: 'completed',
+                    await workTaskService.updateRun(runId, 'completed', {
+                        finished: true,
                     });
+                    await workTaskService.updateTaskStatus(taskId, 'completed');
+                    await workEventService.publish(taskId, runId, 'done', {
+                        status: 'completed',
+                    }, 'terminal:completed');
                     return;
                 }
                 const toolValidationErrors = new Map();
@@ -448,29 +615,29 @@ export class WorkAgentService {
                 // restored context keeps the tool history instead of dropping it.
                 const persistedAssistantMetadata = providerStateMetadata ?? toPersistedWorkChatToolCalls(run, toolCalls);
                 if (assistantContent) {
-                    workTaskService.addMessage(taskId, runId, 'assistant', 'message', assistantContent, persistedAssistantMetadata);
+                    await workTaskService.addMessage(taskId, runId, 'assistant', 'message', assistantContent, persistedAssistantMetadata);
                 }
                 else if (persistedAssistantMetadata) {
-                    workTaskService.addMessage(taskId, runId, 'assistant', 'provider_state', '', persistedAssistantMetadata);
+                    await workTaskService.addMessage(taskId, runId, 'assistant', 'provider_state', '', persistedAssistantMetadata);
                 }
                 for (const call of toolCalls) {
-                    this.throwIfCancelled(runId, controller);
+                    await this.throwIfCancelled(runId, controller);
                     const toolCallMetadata = summarizeToolCall(call);
-                    const toolCallMessage = workTaskService.addMessage(taskId, runId, 'assistant', 'tool_call', `Calling ${call.function.name}`, toolCallMetadata);
-                    workEventService.publish(taskId, runId, 'run_state', {
+                    const toolCallMessage = await workTaskService.addMessage(taskId, runId, 'assistant', 'tool_call', `Calling ${call.function.name}`, toolCallMetadata);
+                    await workEventService.publish(taskId, runId, 'run_state', {
                         status: 'running',
                         phase: 'using_tool',
                         round: round + 1,
                         roundLimit,
-                    });
-                    workEventService.publish(taskId, runId, 'tool_call', {
+                    }, `tool:${call.id}:state`, durableAttemptIdentity);
+                    await workEventService.publish(taskId, runId, 'tool_call', {
                         toolCallId: call.id,
                         name: call.function.name,
                         arguments: toolCallMetadata,
                         metadata: toolCallMetadata,
                         phase: 'running',
                         message: toolCallMessage,
-                    });
+                    }, `message:${toolCallMessage.id}`);
                     let toolOutput;
                     let toolMetadata = {
                         name: call.function.name,
@@ -501,15 +668,15 @@ export class WorkAgentService {
                         toolMetadata.outputTruncated = true;
                         toolOutput = boundedOutput;
                     }
-                    const toolResultMessage = workTaskService.addMessage(taskId, runId, 'tool', 'tool_result', toolOutput, toolMetadata);
-                    workEventService.publish(taskId, runId, 'tool_result', {
+                    const toolResultMessage = await workTaskService.addMessage(taskId, runId, 'tool', 'tool_result', toolOutput, toolMetadata);
+                    await workEventService.publish(taskId, runId, 'tool_result', {
                         toolCallId: call.id,
                         name: call.function.name,
                         phase: toolMetadata.error ? 'failed' : 'completed',
                         content: toolOutput,
                         error: toolMetadata.error === true,
                         message: toolResultMessage,
-                    });
+                    }, `message:${toolResultMessage.id}`);
                     messages.push({
                         role: 'tool',
                         content: toolOutput,
@@ -518,22 +685,22 @@ export class WorkAgentService {
                     });
                 }
             }
-            this.throwIfCancelled(runId, controller);
-            workEventService.publish(taskId, runId, 'run_state', {
+            await this.throwIfCancelled(runId, controller);
+            await workEventService.publish(taskId, runId, 'run_state', {
                 status: 'running',
                 phase: 'responding',
                 round: roundLimit,
                 roundLimit,
-            });
-            const handoffContentStream = new WorkDeltaPublisher(taskId, runId, 'assistant_delta', streamedAssistantTotal);
-            const handoffReasoningStream = new WorkDeltaPublisher(taskId, runId, 'reasoning_delta', streamedReasoningTotal);
+            }, 'handoff:responding', durableAttemptIdentity);
+            const handoffContentStream = new WorkDeltaPublisher(taskId, runId, 'assistant_delta', streamedAssistantTotal, 'handoff', durableAttemptIdentity);
+            const handoffReasoningStream = new WorkDeltaPublisher(taskId, runId, 'reasoning_delta', streamedReasoningTotal, 'handoff', durableAttemptIdentity);
             let handoffInputTokens = 0;
             let handoffOutputTokens = 0;
             const handoffStartedAt = Date.now();
             let handoffResponse;
             try {
                 try {
-                    this.assertStableProviderRouting(run, userId, providerRoutingFingerprint);
+                    await this.assertStableProviderRouting(run, userId, providerRoutingFingerprint);
                     handoffResponse =
                         await workModelProviderService.generateChatStreamResponse({
                             model: run.model,
@@ -556,7 +723,7 @@ export class WorkAgentService {
                                 handoffInputTokens = usage.promptTokens ?? handoffInputTokens;
                                 handoffOutputTokens =
                                     usage.completionTokens ?? handoffOutputTokens;
-                                workEventService.publish(taskId, runId, 'usage', {
+                                void workEventService.publish(taskId, runId, 'usage', {
                                     inputTokens: accumulatedInputTokens + handoffInputTokens,
                                     outputTokens: accumulatedOutputTokens + handoffOutputTokens,
                                     totalTokens: accumulatedInputTokens +
@@ -589,48 +756,48 @@ export class WorkAgentService {
                 handoffContentStream.flush();
                 handoffReasoningStream.flush();
             }
-            this.throwIfCancelled(runId, controller);
+            await this.throwIfCancelled(runId, controller);
             const handoffReasoning = boundUtf8(handoffResponse.message?.thinking?.trim() || '', 100000);
             if (handoffReasoning) {
-                workTaskService.addMessage(taskId, runId, 'assistant', 'reasoning', handoffReasoning, { providerExposed: true, budgetHandoff: true });
+                await workTaskService.addMessage(taskId, runId, 'assistant', 'reasoning', handoffReasoning, { providerExposed: true, budgetHandoff: true });
             }
             const handoffContent = boundUtf8(handoffResponse.message?.content?.trim() ||
                 `The run reached its ${budgetReason} budget. Start a follow-up run to continue in the same workspace.`, 100000);
             const handoffProviderStateMetadata = toPersistedWorkProviderState(run, handoffResponse.message.providerMetadata);
-            workTaskService.addMessage(taskId, runId, 'assistant', 'message', handoffContent, {
+            await workTaskService.addMessage(taskId, runId, 'assistant', 'message', handoffContent, {
                 budgetHandoff: true,
                 budgetReason,
                 ...handoffProviderStateMetadata,
             });
             await settleExecutionContainer();
-            this.throwIfCancelled(runId, controller);
-            if (!isActiveRunStatus(workTaskService.getRun(runId)?.status))
+            await this.throwIfCancelled(runId, controller);
+            if (!isActiveRunStatus((await workTaskService.getRun(runId))?.status))
                 return;
-            workTaskService.updateRun(runId, 'needs_input', { finished: true });
-            workTaskService.updateTaskStatus(taskId, 'needs_input');
-            workEventService.publish(taskId, runId, 'done', {
+            await workTaskService.updateRun(runId, 'needs_input', { finished: true });
+            await workTaskService.updateTaskStatus(taskId, 'needs_input');
+            await workEventService.publish(taskId, runId, 'done', {
                 status: 'needs_input',
                 budgetReason,
-            });
+            }, 'terminal:needs_input');
         }
         catch (error) {
-            const currentRun = workTaskService.getRun(runId);
+            const currentRun = await workTaskService.getRun(runId);
             if (currentRun?.status === 'cancelled' || controller.signal.aborted) {
                 if (task) {
                     try {
                         await workRuntimeService.stopContainer(task);
                         executionContainerSettled = true;
-                        if (isActiveRunStatus(workTaskService.getRun(runId)?.status)) {
-                            workTaskService.updateRun(runId, 'cancelled', {
+                        if (isActiveRunStatus((await workTaskService.getRun(runId))?.status)) {
+                            await workTaskService.updateRun(runId, 'cancelled', {
                                 error: 'Cancelled by user.',
                                 finished: true,
                             });
-                            workTaskService.updateTaskStatus(taskId, 'cancelled');
-                            workTaskService.updatePreview(taskId, 'stopped');
-                            workEventService.publish(taskId, runId, 'done', {
+                            await workTaskService.updateTaskStatus(taskId, 'cancelled');
+                            await workTaskService.updatePreview(taskId, 'stopped');
+                            await workEventService.publish(taskId, runId, 'done', {
                                 status: 'cancelled',
                                 error: 'Cancelled by user.',
-                            });
+                            }, 'terminal:cancelled');
                         }
                     }
                     catch (cleanupError) {
@@ -643,39 +810,39 @@ export class WorkAgentService {
                 ? error.message
                 : 'Work run failed unexpectedly.';
             await settleExecutionContainer();
-            const settledRun = workTaskService.getRun(runId);
+            const settledRun = await workTaskService.getRun(runId);
             if (controller.signal.aborted || settledRun?.status === 'cancelled') {
                 if (isActiveRunStatus(settledRun?.status)) {
-                    workTaskService.updateRun(runId, 'cancelled', {
+                    await workTaskService.updateRun(runId, 'cancelled', {
                         error: 'Cancelled by user.',
                         finished: true,
                     });
-                    workTaskService.updateTaskStatus(taskId, 'cancelled');
-                    workTaskService.updatePreview(taskId, 'stopped');
-                    workEventService.publish(taskId, runId, 'done', {
+                    await workTaskService.updateTaskStatus(taskId, 'cancelled');
+                    await workTaskService.updatePreview(taskId, 'stopped');
+                    await workEventService.publish(taskId, runId, 'done', {
                         status: 'cancelled',
                         error: 'Cancelled by user.',
-                    });
+                    }, 'terminal:cancelled');
                 }
                 return;
             }
             if (!isActiveRunStatus(settledRun?.status)) {
                 return;
             }
-            workTaskService.addMessage(taskId, runId, 'assistant', 'error', message);
-            workTaskService.updateRun(runId, 'failed', {
+            await workTaskService.addMessage(taskId, runId, 'assistant', 'error', message);
+            await workTaskService.updateRun(runId, 'failed', {
                 error: message,
                 finished: true,
             });
-            workTaskService.updateTaskStatus(taskId, 'failed');
-            workEventService.publish(taskId, runId, 'error', {
+            await workTaskService.updateTaskStatus(taskId, 'failed');
+            await workEventService.publish(taskId, runId, 'error', {
                 message,
                 code: error instanceof WorkAgentHttpError ? error.code : undefined,
-            });
-            workEventService.publish(taskId, runId, 'done', {
+            }, 'terminal:error');
+            await workEventService.publish(taskId, runId, 'done', {
                 status: 'failed',
                 error: message,
-            });
+            }, 'terminal:failed');
         }
         finally {
             this.controllers.delete(runId);
@@ -701,8 +868,8 @@ export class WorkAgentService {
         }
         try {
             await workRuntimeService.stopContainer(task);
-            if (workTaskService.getTaskRecord(task.id, task.userId)) {
-                workTaskService.updatePreview(task.id, 'stopped');
+            if (await workTaskService.getTaskRecord(task.id, task.userId)) {
+                await workTaskService.updatePreview(task.id, 'stopped');
             }
         }
         catch (error) {
@@ -711,8 +878,8 @@ export class WorkAgentService {
             logger.error(`Could not stop idle Work container ${task.containerName}:`, error);
         }
     }
-    assertStableProviderRouting(run, userId, expectedFingerprint) {
-        const currentFingerprint = workModelProviderService.getRoutingFingerprint(run.model, {
+    async assertStableProviderRouting(run, userId, expectedFingerprint) {
+        const currentFingerprint = await workModelProviderService.getRoutingFingerprint(run.model, {
             providerType: run.providerType,
             providerId: run.providerId,
         }, userId);
@@ -720,8 +887,8 @@ export class WorkAgentService {
             throw new WorkAgentHttpError('The provider routing changed while this Work run was active. Start a new run with the updated provider configuration.', 409, 'WORK_PROVIDER_ROUTING_CHANGED');
         }
     }
-    contextMessages(task, roundLimit, providerStateScope, provider = task) {
-        const persisted = restorePersistedWorkContext(workTaskService.getRecentModelContextMessages(task.id, 30), provider, providerStateScope);
+    async contextMessages(task, roundLimit, providerStateScope, provider = task) {
+        const persisted = restorePersistedWorkContext(await workTaskService.getRecentModelContextMessages(task.id, 30), provider, providerStateScope);
         return [
             {
                 role: 'system',
@@ -779,8 +946,8 @@ export class WorkAgentService {
             }
             case 'web_search': {
                 if (!task.networkEnabled ||
-                    !isWebSearchAvailable() ||
-                    !userCanUseWebSearch(userModel.getUserById(task.userId))) {
+                    !(await isWebSearchAvailable()) ||
+                    !(await userCanUseWebSearch(await userModel.getUserById(task.userId)))) {
                     throw new WorkAgentHttpError('Web search is not available for this task.', 403, 'WORK_WEB_SEARCH_UNAVAILABLE');
                 }
                 const query = requiredString(args.query, 'query');
@@ -803,7 +970,7 @@ export class WorkAgentService {
             case 'start_preview': {
                 const url = await workRuntimeService.startPreview(task, optionalString(args.command), {
                     onStarting: () => workTaskService.beginPreview(task.id, task.userId, true),
-                    onRunning: previewUrl => workTaskService.updatePreview(task.id, 'running', previewUrl),
+                    onRunning: (previewUrl, endpoint) => workTaskService.updatePreview(task.id, 'running', previewUrl, endpoint),
                     onFailed: () => workTaskService.updatePreview(task.id, 'failed'),
                 });
                 return {
@@ -820,21 +987,24 @@ export class WorkAgentService {
                 throw new WorkAgentHttpError(`Unknown tool: ${call.function.name}`, 400, 'WORK_UNKNOWN_TOOL');
         }
     }
-    throwIfCancelled(runId, controller) {
+    async throwIfCancelled(runId, controller) {
         if (controller.signal.aborted ||
-            workTaskService.getRun(runId)?.status === 'cancelled') {
+            (await workTaskService.getRun(runId))?.status === 'cancelled') {
             throw new WorkAgentHttpError('Work run was cancelled.', 409, 'WORK_RUN_CANCELLED');
         }
     }
 }
 class WorkDeltaPublisher {
-    constructor(taskId, runId, type, initialTotal = '') {
+    constructor(taskId, runId, type, initialTotal = '', occurrenceScope = 'stream', attemptIdentity = '0') {
         this.taskId = taskId;
         this.runId = runId;
         this.type = type;
+        this.occurrenceScope = occurrenceScope;
+        this.attemptIdentity = attemptIdentity;
         this.pending = '';
         this.total = '';
         this.separatorPending = '';
+        this.sequence = 0;
         this.total = initialTotal;
         this.separatorPending = initialTotal ? '\n\n' : '';
     }
@@ -862,9 +1032,7 @@ class WorkDeltaPublisher {
             return;
         const delta = this.pending;
         this.pending = '';
-        workEventService.publish(this.taskId, this.runId, this.type, {
-            delta,
-        });
+        workEventService.publish(this.taskId, this.runId, this.type, { delta, total: this.total }, `${this.occurrenceScope}:${++this.sequence}`, this.attemptIdentity);
     }
 }
 export class WorkAgentHttpError extends Error {
@@ -1009,7 +1177,7 @@ export function restorePersistedWorkContext(messages, provider, expectedStateSco
                 continue;
             restored.push({
                 role: 'tool',
-                content: 'Tool execution was interrupted; outcome unknown. Inspect the workspace before retrying.',
+                content: INTERRUPTED_TOOL_RESULT,
                 tool_name: call.name,
                 tool_call_id: call.id,
             });