libre-webui 0.21.2 → 0.22.0

package/backend/dist/services/workRuntimeService.js

@@ -15,12 +15,15 @@
  * limitations under the License.
  */
 import path from 'node:path';
-import { getDatabase } from '../db.js';
+import { randomUUID } from 'node:crypto';
+import { getWorkPersistence } from '../platform/workPersistence/index.js';
+import { getCoordinator, getPlatformRuntimeConfig, } from '../platform/coordination/service.js';
+import { combineAbortSignals, SHARED_COORDINATION_OPERATION_TIMEOUT_MS, withCoordinationTimeout, } from '../platform/coordination/sharedAdmission.js';
 import { createLogger } from '../utils/logger.js';
 import { buildWorkGitCommand, parseWorkGitBranches, parseWorkGitLog, parseWorkGitStatus, validateWorkGitBranchName, validateWorkGitRepositoryPaths, } from '../utils/workGit.js';
 import { userHasWorkAccess } from './workAccessService.js';
 import workPolicyService from './workPolicyService.js';
-import workPreviewProxyService from './workPreviewProxyService.js';
+import workPreviewProxyService, { normalizePreviewUpstreamHost, } from './workPreviewProxyService.js';
 import workTaskService from './workTaskService.js';
 import { KubernetesWorkRuntimeDriver } from './workKubernetesDriver.js';
 import { DockerWorkRuntimeDriver, } from './workRuntimeDriver.js';
@@ -49,28 +52,38 @@ export class WorkRuntimeService {
         this.retiringTasks = new Set();
         this.networkPolicies = new Map();
         this.activeCommands = new Set();
-        this.lastDockerUnavailableReason = null;
+        this.lastRuntimeUnavailableReason = null;
         this.runtimeLeases = new Map();
         this.previewLeaseReleases = new Map();
         this.terminalHolds = new Map();
         this.recoveryTasks = new Map();
         this.recoveryOrphans = new Map();
-        // Sweeps left for the empty-inventory case before giving up on a daemon
-        // that never appears (30 × 10s: covers a socket proxy starting late
-        // without probing a Docker-less deployment forever).
+        // Sweeps left for the empty-inventory case before giving up on a runtime
+        // that never appears (30 × 10s covers a late Docker socket proxy or
+        // Kubernetes API without probing an intentionally disabled backend forever).
         this.emptyInventorySweepsLeft = 30;
         this.shuttingDown = false;
+        this.activityMemberId = `${process.env.LIBRE_PROCESS_ROLE || 'standalone'}-${process.pid}-` +
+            randomUUID();
         // Last observed activity per task, feeding the idle sweep: a finished
         // command, a terminal session ending, or a preview fetch. In-memory only —
         // after a restart the clock restarts from the first sighting.
         this.taskActivity = new Map();
         this.driver = driver;
+        this.runtimeKind = driver.kind;
         // Authorized preview traffic keeps its task's idle clock fresh.
         workPreviewProxyService.onPreviewActivity(taskId => this.noteTaskActivity(taskId));
-        this.scheduleIdleSweep();
+        if (process.env.LIBRE_PROCESS_ROLE !== 'app-external') {
+            this.scheduleIdleSweep();
+        }
     }
     noteTaskActivity(taskId) {
         this.taskActivity.set(taskId, Date.now());
+        if (getPlatformRuntimeConfig().mode === 'team') {
+            void getCoordinator()
+                .setCache(`work-task-activity:${taskId}`, Date.now(), 86400000)
+                .catch(error => logger.warn(`Could not persist Work activity for ${taskId}:`, error));
+        }
     }
     get recoveryPending() {
         return this.recoveryPendingCount > 0;
@@ -90,25 +103,24 @@ export class WorkRuntimeService {
             throw new WorkRuntimeError(`Work is temporarily unavailable while ${this.recoveryPendingCount} container cleanup(s) are retried.`, 503, 'WORK_RUNTIME_RECOVERING');
         }
     }
-    async isDockerAvailable() {
+    async isRuntimeAvailable() {
         try {
             await this.driver.probe();
-            this.lastDockerUnavailableReason = null;
+            this.lastRuntimeUnavailableReason = null;
             return true;
         }
         catch (error) {
-            this.lastDockerUnavailableReason =
+            this.lastRuntimeUnavailableReason =
                 error instanceof Error ? error.message : String(error);
             return false;
         }
     }
     /**
-     * Why the last availability probe failed. "Docker is not available" cannot be
-     * acted on: a containerized deployment fails for a missing CLI, an unreadable
-     * socket, or a stopped daemon, and each needs a different change.
+     * Why the last availability probe failed. A generic "runtime unavailable"
+     * message cannot be acted on, so drivers return their operator-facing cause.
      */
-    get dockerUnavailableReason() {
-        return this.lastDockerUnavailableReason;
+    get runtimeUnavailableReason() {
+        return this.lastRuntimeUnavailableReason;
     }
     /**
      * Live runtime occupancy, so the interface can say "1 of 2 runtimes in use"
@@ -124,14 +136,25 @@ export class WorkRuntimeService {
         }
         return { global: this.runtimeLeases.size, user };
     }
-    async beginRecovery(tasks) {
+    async beginRecovery(tasks, assertRecoveryLease) {
         this.recoveryInventory = tasks;
-        const result = await this.sweepRecoveryTasks();
-        this.scheduleRecoverySweep();
+        const result = await this.sweepRecoveryTasks(assertRecoveryLease);
+        if (assertRecoveryLease) {
+            // An external worker must never hand unfinished destructive recovery to
+            // an unfenced background retry after releasing the global startup lease.
+            if (this.recoveryInventory !== undefined && result.failed === 0) {
+                return { ...result, failed: 1 };
+            }
+        }
+        else {
+            this.scheduleRecoverySweep();
+        }
         return result;
     }
-    acquireRuntimeLease(task) {
-        this.assertTaskIsActive(task);
+    async acquireRuntimeLease(task, signal) {
+        signal?.throwIfAborted();
+        await this.assertTaskIsActive(task);
+        signal?.throwIfAborted();
         const existing = this.runtimeLeases.get(task.id);
         if (existing) {
             if (existing.userId !== task.userId) {
@@ -147,25 +170,101 @@ export class WorkRuntimeService {
             if (this.runtimeLeases.size >= config.maxActiveRuntimesGlobal) {
                 throw new WorkRuntimeError(`This Libre WebUI instance already has ${config.maxActiveRuntimesGlobal} active Work runtime(s). Wait for another operation or preview to stop.`, 429, 'WORK_GLOBAL_RUNTIME_LIMIT');
             }
-            this.runtimeLeases.set(task.id, { userId: task.userId, holders: 1 });
+            const runtimeLease = {
+                userId: task.userId,
+                holders: 1,
+            };
+            if (getPlatformRuntimeConfig().mode === 'team') {
+                let abandoned = false;
+                const pendingSharedLease = getCoordinator().acquireLease(`work-task-runtime:${task.id}`, 60000);
+                void pendingSharedLease
+                    .then(lease => {
+                    if (!abandoned || !lease)
+                        return;
+                    void withCoordinationTimeout(lease.release(), SHARED_COORDINATION_OPERATION_TIMEOUT_MS).catch(() => undefined);
+                })
+                    .catch(() => undefined);
+                let sharedLease;
+                try {
+                    sharedLease = await waitForAbortSignal(withCoordinationTimeout(pendingSharedLease, SHARED_COORDINATION_OPERATION_TIMEOUT_MS), signal);
+                }
+                catch (error) {
+                    abandoned = true;
+                    throw error;
+                }
+                if (!sharedLease) {
+                    throw new WorkRuntimeError('This Work task is active on another replica.', 409, 'WORK_RUNTIME_LEASE_CONFLICT');
+                }
+                runtimeLease.sharedLease = sharedLease;
+                runtimeLease.sharedLeaseTimer = setInterval(() => {
+                    void withCoordinationTimeout(sharedLease.extend(60000), SHARED_COORDINATION_OPERATION_TIMEOUT_MS)
+                        .then(extended => {
+                        if (!extended)
+                            throw new Error('lease ownership was lost');
+                    })
+                        .catch(error => {
+                        if (runtimeLease.sharedLeaseLost)
+                            return;
+                        runtimeLease.sharedLeaseLost = true;
+                        logger.error(`Shared Work runtime lease was lost for task ${task.id}; stopping its sandbox:`, error);
+                        void this.driver
+                            .stopRuntime(task)
+                            .catch(stopError => logger.error(`Could not stop Work task ${task.id} after lease loss:`, stopError));
+                    });
+                }, 20000);
+                runtimeLease.sharedLeaseTimer.unref?.();
+                const refreshPresence = () => withCoordinationTimeout(getCoordinator().setPresence(`work-task-active:${task.id}`, this.activityMemberId, 30000), SHARED_COORDINATION_OPERATION_TIMEOUT_MS);
+                try {
+                    await waitForAbortSignal(refreshPresence(), signal);
+                }
+                catch (error) {
+                    if (runtimeLease.sharedLeaseTimer) {
+                        clearInterval(runtimeLease.sharedLeaseTimer);
+                    }
+                    await withCoordinationTimeout(sharedLease.release(), SHARED_COORDINATION_OPERATION_TIMEOUT_MS).catch(() => false);
+                    throw error;
+                }
+                runtimeLease.presenceTimer = setInterval(() => {
+                    void refreshPresence().catch(error => logger.warn(`Could not refresh Work activity for ${task.id}:`, error));
+                }, 10000);
+                runtimeLease.presenceTimer.unref?.();
+            }
+            this.runtimeLeases.set(task.id, runtimeLease);
         }
         let released = false;
         return () => {
             if (released)
                 return;
             released = true;
-            const lease = this.runtimeLeases.get(task.id);
-            if (!lease || lease.userId !== task.userId)
-                return;
-            lease.holders -= 1;
-            if (lease.holders <= 0) {
-                this.runtimeLeases.delete(task.id);
-            }
+            this.releaseRuntimeLease(task.id, task.userId);
         };
     }
+    releaseRuntimeLease(taskId, expectedUserId, force = false) {
+        const lease = this.runtimeLeases.get(taskId);
+        if (!lease || (expectedUserId && lease.userId !== expectedUserId))
+            return;
+        if (!force)
+            lease.holders -= 1;
+        if (!force && lease.holders > 0)
+            return;
+        if (lease.presenceTimer)
+            clearInterval(lease.presenceTimer);
+        if (lease.sharedLeaseTimer)
+            clearInterval(lease.sharedLeaseTimer);
+        this.runtimeLeases.delete(taskId);
+        void lease.sharedLease?.release().catch(() => false);
+        if (getPlatformRuntimeConfig().mode === 'team') {
+            void getCoordinator()
+                .clearPresence(`work-task-active:${taskId}`, this.activityMemberId)
+                .catch(error => logger.warn(`Could not clear Work activity for ${taskId}:`, error));
+        }
+    }
     assertRuntimeLease(task) {
         const lease = this.runtimeLeases.get(task.id);
-        if (!lease || lease.userId !== task.userId || lease.holders < 1) {
+        if (!lease ||
+            lease.userId !== task.userId ||
+            lease.holders < 1 ||
+            lease.sharedLeaseLost) {
             throw new WorkRuntimeError('Work container acquisition requires a runtime capacity lease.', 503, 'WORK_RUNTIME_LEASE_REQUIRED');
         }
     }
@@ -179,8 +278,8 @@ export class WorkRuntimeService {
         }
     }
     async prepare(task, signal) {
-        const releaseLease = this.acquireRuntimeLease(task);
-        this.assertTaskIsActive(task);
+        const releaseLease = await this.acquireRuntimeLease(task, signal);
+        await this.assertTaskIsActive(task);
         const active = this.preparations.get(task.id);
         if (active) {
             try {
@@ -195,11 +294,12 @@ export class WorkRuntimeService {
         let tracked;
         tracked = (async () => {
             await waitForAbortSignal(this.ensureImage(task), signal);
-            this.assertTaskIsActive(task);
-            await this.withLifecycleLock(task.id, async () => {
-                this.assertTaskIsActive(task);
-                await this.prepareWithLock(task);
-                this.assertTaskIsActive(task);
+            await this.assertTaskIsActive(task);
+            await this.withLifecycleLock(task.id, async (_assertHeld, leaseSignal) => {
+                const operationSignal = combineAbortSignals(signal, leaseSignal);
+                await this.assertTaskIsActive(task);
+                await this.prepareWithLock(task, operationSignal);
+                await this.assertTaskIsActive(task);
             });
         })().finally(() => {
             if (this.preparations.get(task.id) === tracked) {
@@ -222,11 +322,11 @@ export class WorkRuntimeService {
      * container preparation as every other runtime operation, and keeps the
      * idle-stop path from tearing the container down while a shell is attached.
      */
-    async acquireTerminalHold(task) {
+    async acquireTerminalHold(task, signal) {
         if (this.activeCommands.has(task.id)) {
             throw new WorkRuntimeError('A Work command is running in this container. Wait for it to finish, then reconnect the terminal.', 409, 'WORK_TERMINAL_COMMAND_ACTIVE');
         }
-        const releaseLease = await this.prepare(task);
+        const releaseLease = await this.prepare(task, signal);
         this.terminalHolds.set(task.id, (this.terminalHolds.get(task.id) ?? 0) + 1);
         let released = false;
         return async () => {
@@ -243,7 +343,7 @@ export class WorkRuntimeService {
             }
             try {
                 if (remaining <= 0 && !this.shuttingDown) {
-                    await this.withLifecycleLock(task.id, () => this.stopContainerIfIdleWithLock(task));
+                    await this.withLifecycleLock(task.id, (_assertHeld, signal) => this.stopContainerIfIdleWithLock(task, signal));
                 }
             }
             catch (error) {
@@ -267,6 +367,12 @@ export class WorkRuntimeService {
             clearTimeout(this.idleTimer);
             this.idleTimer = undefined;
         }
+        for (const taskId of [...this.previewLeaseReleases.keys()]) {
+            this.releasePreviewLease(taskId);
+        }
+        for (const taskId of [...this.runtimeLeases.keys()]) {
+            this.releaseRuntimeLease(taskId, undefined, true);
+        }
         this.driver.shutdown();
     }
     beginTaskSuspension(taskId) {
@@ -278,19 +384,18 @@ export class WorkRuntimeService {
     releaseTaskSuspension(taskId) {
         this.retiringTasks.delete(taskId);
     }
-    async sweepRecoveryTasks() {
+    async sweepRecoveryTasks(assertRecoveryLease = async () => undefined) {
         if (this.recoveryInventory === undefined &&
             this.recoveryTasks.size === 0 &&
             this.recoveryOrphans.size === 0) {
             return { stopped: 0, failed: 0 };
         }
-        if (this.shuttingDown || !(await this.isDockerAvailable())) {
+        if (this.shuttingDown || !(await this.isRuntimeAvailable())) {
             if (this.recoveryInventory?.length === 0) {
                 // No tasks to supervise, so nothing fail-closes — but leftover
-                // containers may still exist (a restored database, a daemon or
-                // socket proxy that comes up after the backend). Keep retrying the
-                // reconciliation quietly for a bounded window, then give up so a
-                // deployment without Docker is not probed forever.
+                // sandboxes may still exist after a database restore, or the runtime
+                // control plane may start after the backend. Retry for a bounded
+                // window without probing an intentionally disabled runtime forever.
                 if (this.shuttingDown || this.emptyInventorySweepsLeft <= 0) {
                     this.recoveryInventory = undefined;
                     return { stopped: 0, failed: 0 };
@@ -307,6 +412,7 @@ export class WorkRuntimeService {
             // by a task whose database row is gone.
             let discovered;
             try {
+                await assertRecoveryLease();
                 discovered = await this.driver.listManaged();
             }
             catch (error) {
@@ -331,8 +437,16 @@ export class WorkRuntimeService {
         const tasks = [...this.recoveryTasks.values()];
         const orphans = [...this.recoveryOrphans.values()];
         const [taskResults, orphanResults] = await Promise.all([
-            Promise.allSettled(tasks.map(task => this.stopContainer(task))),
-            Promise.allSettled(orphans.map(orphan => this.driver.removeOrphan(orphan.name))),
+            Promise.allSettled(tasks.map(async (task) => {
+                await assertRecoveryLease();
+                await this.stopContainer(task);
+                await assertRecoveryLease();
+            })),
+            Promise.allSettled(orphans.map(async (orphan) => {
+                await assertRecoveryLease();
+                await this.driver.removeOrphan(orphan.name);
+                await assertRecoveryLease();
+            })),
         ]);
         let stopped = 0;
         taskResults.forEach((result, index) => {
@@ -389,12 +503,14 @@ export class WorkRuntimeService {
      * on a guess.
      */
     async sweepIdleRuntimes(now = Date.now()) {
-        if (this.shuttingDown || this.recoveryPending) {
+        if (process.env.LIBRE_PROCESS_ROLE === 'app-external' ||
+            this.shuttingDown ||
+            this.recoveryPending) {
             return { stopped: 0 };
         }
         // Idle-stop can come from the global knob or from any named policy.
         if (config.idleTimeoutMs <= 0 &&
-            !workPolicyService.anyIdleTimeoutConfigured()) {
+            !(await workPolicyService.anyIdleTimeoutConfigured())) {
             return { stopped: 0 };
         }
         let discovered;
@@ -405,7 +521,7 @@ export class WorkRuntimeService {
             // The runtime is unreachable; nothing can be stopped anyway.
             return { stopped: 0 };
         }
-        const records = new Map(workTaskService.listAllTaskRecords().map(task => [task.id, task]));
+        const records = new Map((await workTaskService.listAllTaskRecords()).map(task => [task.id, task]));
         let stopped = 0;
         for (const entry of discovered) {
             if (!entry.running)
@@ -414,16 +530,25 @@ export class WorkRuntimeService {
             // Containers without a task row are startup reconciliation's business.
             if (!task)
                 continue;
+            if (getPlatformRuntimeConfig().mode === 'team' &&
+                (await getCoordinator().listPresence(`work-task-active:${task.id}`))
+                    .length > 0) {
+                continue;
+            }
             const busy = this.activeCommands.has(task.id) ||
                 (this.terminalHolds.get(task.id) ?? 0) > 0 ||
                 (this.runtimeLeases.has(task.id) &&
                     !this.previewLeaseReleases.has(task.id));
-            const lastActivity = this.taskActivity.get(task.id);
+            const sharedActivity = getPlatformRuntimeConfig().mode === 'team'
+                ? await getCoordinator().getCache(`work-task-activity:${task.id}`)
+                : null;
+            const lastActivity = sharedActivity ?? this.taskActivity.get(task.id);
             if (busy || lastActivity === undefined) {
                 this.noteTaskActivity(task.id);
                 continue;
             }
-            const idleAfterMs = workPolicyService.resolve(task.policyId).idleTimeoutMs;
+            const idleAfterMs = (await workPolicyService.resolve(task.policyId))
+                .idleTimeoutMs;
             if (idleAfterMs <= 0 || now - lastActivity < idleAfterMs)
                 continue;
             try {
@@ -499,80 +624,80 @@ export class WorkRuntimeService {
             logger.info('Pending Work container cleanup completed.');
         }
     }
-    markPreviewStopped(taskId) {
+    async markPreviewStopped(taskId) {
         try {
-            getDatabase()
-                .prepare(`UPDATE work_tasks
-           SET preview_status = 'stopped', preview_url = NULL, updated_at = ?
-           WHERE id = ?`)
-                .run(Date.now(), taskId);
+            await getWorkPersistence().updatePreview(taskId, 'stopped', null, null, null, Date.now());
         }
         catch (error) {
             logger.warn(`Could not persist stopped preview state for Work task ${taskId}:`, error);
         }
     }
-    async prepareWithLock(task) {
+    async prepareWithLock(task, signal) {
+        signal?.throwIfAborted();
         this.assertRuntimeLease(task);
         this.assertCurrentNetworkPolicy(task);
-        this.assertTaskIsActive(task);
-        await this.driver.ensureWorkspace(task);
-        this.assertTaskIsActive(task);
+        await this.assertTaskIsActive(task);
+        await this.driver.ensureWorkspace(task, signal);
+        await this.assertTaskIsActive(task);
         this.assertRuntimeLease(task);
-        await this.driver.ensureRuntime(task);
+        await this.driver.ensureRuntime(task, signal);
         if (this.shuttingDown) {
-            await this.stopContainerWithLock(task);
-            this.assertTaskIsActive(task);
+            await this.stopContainerWithLock(task, signal);
+            await this.assertTaskIsActive(task);
         }
     }
     async recreateContainer(task) {
-        const releaseLease = this.acquireRuntimeLease(task);
+        const releaseLease = await this.acquireRuntimeLease(task);
         try {
             await this.ensureImage(task);
-            this.assertTaskIsActive(task);
-            await this.withLifecycleLock(task.id, async () => {
-                this.assertTaskIsActive(task);
-                await this.recreateContainerWithLock(task);
-                await this.stopContainerWithLock(task);
+            await this.assertTaskIsActive(task);
+            await this.withLifecycleLock(task.id, async (_assertHeld, signal) => {
+                await this.assertTaskIsActive(task);
+                await this.recreateContainerWithLock(task, signal);
+                await this.stopContainerWithLock(task, signal);
                 this.networkPolicies.set(task.id, task.networkEnabled);
             });
             this.releasePreviewLease(task.id);
-            this.markPreviewStopped(task.id);
+            await this.markPreviewStopped(task.id);
         }
         finally {
             releaseLease();
         }
     }
     async changeNetworkPolicy(before, desired, commit) {
-        this.assertTaskIsActive(before);
+        await this.assertTaskIsActive(before);
         if (desired.networkEnabled === before.networkEnabled) {
-            return this.withLifecycleLock(before.id, async () => {
-                this.assertTaskIsActive(before);
+            return this.withLifecycleLock(before.id, async (assertHeld) => {
+                await this.assertTaskIsActive(before);
                 this.assertCurrentNetworkPolicy(before);
-                const result = commit();
+                await assertHeld();
+                const result = await commit();
                 this.networkPolicies.set(desired.id, desired.networkEnabled);
                 return result;
             });
         }
-        const releaseLease = this.acquireRuntimeLease(before);
+        const releaseLease = await this.acquireRuntimeLease(before);
         let previewStopped = false;
         try {
             await this.ensureImage(before);
-            this.assertTaskIsActive(before);
-            return await this.withLifecycleLock(before.id, async () => {
-                this.assertTaskIsActive(before);
+            await this.assertTaskIsActive(before);
+            return await this.withLifecycleLock(before.id, async (assertHeld, signal) => {
+                await this.assertTaskIsActive(before);
                 this.assertCurrentNetworkPolicy(before);
                 try {
-                    await this.recreateContainerWithLock(desired);
-                    await this.stopContainerWithLock(desired);
+                    await this.recreateContainerWithLock(desired, signal);
+                    await assertHeld();
+                    await this.stopContainerWithLock(desired, signal);
                     previewStopped = true;
-                    const result = commit();
+                    await assertHeld();
+                    const result = await commit();
                     this.networkPolicies.set(desired.id, desired.networkEnabled);
                     return result;
                 }
                 catch (error) {
                     try {
-                        await this.recreateContainerWithLock(before);
-                        await this.stopContainerWithLock(before);
+                        await this.recreateContainerWithLock(before, signal);
+                        await this.stopContainerWithLock(before, signal);
                         previewStopped = true;
                         this.networkPolicies.set(before.id, before.networkEnabled);
                     }
@@ -586,27 +711,27 @@ export class WorkRuntimeService {
         finally {
             if (previewStopped) {
                 this.releasePreviewLease(before.id);
-                this.markPreviewStopped(before.id);
+                await this.markPreviewStopped(before.id);
             }
             releaseLease();
         }
     }
-    async recreateContainerWithLock(task) {
+    async recreateContainerWithLock(task, signal) {
         this.assertRuntimeLease(task);
-        this.assertTaskIsActive(task);
-        await this.driver.ensureWorkspace(task);
-        this.assertTaskIsActive(task);
-        await this.driver.removeRuntime(task);
-        await this.driver.ensureRuntime(task);
+        await this.assertTaskIsActive(task);
+        await this.driver.ensureWorkspace(task, signal);
+        await this.assertTaskIsActive(task);
+        await this.driver.removeRuntime(task, signal);
+        await this.driver.ensureRuntime(task, signal);
         if (this.shuttingDown) {
-            await this.stopContainerWithLock(task);
-            this.assertTaskIsActive(task);
+            await this.stopContainerWithLock(task, signal);
+            await this.assertTaskIsActive(task);
         }
     }
     async stopContainer(task) {
-        await this.withLifecycleLock(task.id, () => this.stopContainerWithLock(task));
+        await this.withLifecycleLock(task.id, (_assertHeld, signal) => this.stopContainerWithLock(task, signal));
         this.releasePreviewLease(task.id);
-        this.markPreviewStopped(task.id);
+        await this.markPreviewStopped(task.id);
         this.completeRecoveryTask(task.id);
     }
     /**
@@ -618,12 +743,12 @@ export class WorkRuntimeService {
     async interruptContainer(task) {
         await this.stopContainerWithLock(task);
         this.releasePreviewLease(task.id);
-        this.markPreviewStopped(task.id);
+        await this.markPreviewStopped(task.id);
         this.completeRecoveryTask(task.id);
     }
-    async stopContainerWithLock(task) {
+    async stopContainerWithLock(task, signal) {
         try {
-            await this.driver.stopRuntime(task);
+            await this.driver.stopRuntime(task, signal);
         }
         catch (error) {
             this.queueFailedCleanup(task, error);
@@ -632,14 +757,14 @@ export class WorkRuntimeService {
     }
     async removeTask(task, allowRevokedOwner = false) {
         if (allowRevokedOwner) {
-            this.assertTaskStillOwned(task);
+            await this.assertTaskStillOwned(task);
         }
         else {
-            this.assertTaskOwnerHasWorkAccess(task);
+            await this.assertTaskOwnerHasWorkAccess(task);
         }
         this.retiringTasks.add(task.id);
         try {
-            await this.withLifecycleLock(task.id, () => this.driver.removeTaskResources(task));
+            await this.withLifecycleLock(task.id, (_assertHeld, signal) => this.driver.removeTaskResources(task, signal));
         }
         catch (error) {
             this.retiringTasks.delete(task.id);
@@ -652,9 +777,8 @@ export class WorkRuntimeService {
         this.preparations.delete(taskId);
         this.activeCommands.delete(taskId);
         this.releasePreviewLease(taskId);
-        this.runtimeLeases.delete(taskId);
+        this.releaseRuntimeLease(taskId, undefined, true);
         this.taskActivity.delete(taskId);
-        workPreviewProxyService.clearPreviewUpstream(taskId);
     }
     async listFiles(task, requestedPath = '.') {
         const workspacePath = validateWorkspacePath(requestedPath);
@@ -665,19 +789,19 @@ export class WorkRuntimeService {
         });
     }
     async withWorkspaceHelperContainer(task, operation) {
-        const releaseLease = this.acquireRuntimeLease(task);
+        const releaseLease = await this.acquireRuntimeLease(task);
         try {
             await this.ensureImage(task);
-            this.assertTaskIsActive(task);
-            return await this.withLifecycleLock(task.id, async () => {
-                this.assertTaskIsActive(task);
-                await this.prepareWithLock(task);
-                this.assertTaskIsActive(task);
+            await this.assertTaskIsActive(task);
+            return await this.withLifecycleLock(task.id, async (_assertHeld, signal) => {
+                await this.assertTaskIsActive(task);
+                await this.prepareWithLock(task, signal);
+                await this.assertTaskIsActive(task);
                 try {
                     return await operation();
                 }
                 finally {
-                    await this.stopContainerIfIdleWithLock(task);
+                    await this.stopContainerIfIdleWithLock(task, signal);
                 }
             });
         }
@@ -685,7 +809,7 @@ export class WorkRuntimeService {
             releaseLease();
         }
     }
-    async stopContainerIfIdleWithLock(task) {
+    async stopContainerIfIdleWithLock(task, signal) {
         if (this.activeCommands.has(task.id))
             return;
         // An attached terminal session owns the running container exactly like a
@@ -693,7 +817,7 @@ export class WorkRuntimeService {
         if ((this.terminalHolds.get(task.id) ?? 0) > 0)
             return;
         try {
-            if ((await this.previewProcessCheckWithLock(task)) === 'ready' &&
+            if ((await this.previewProcessCheckWithLock(task, signal)) === 'ready' &&
                 this.previewLeaseReleases.has(task.id)) {
                 return;
             }
@@ -701,9 +825,9 @@ export class WorkRuntimeService {
         catch (error) {
             logger.warn(`Could not verify preview state before idling Work container ${task.containerName}; stopping it:`, error);
         }
-        await this.stopContainerWithLock(task);
+        await this.stopContainerWithLock(task, signal);
         this.releasePreviewLease(task.id);
-        this.markPreviewStopped(task.id);
+        await this.markPreviewStopped(task.id);
         this.completeRecoveryTask(task.id);
     }
     async readFile(task, requestedPath) {
@@ -930,20 +1054,21 @@ export class WorkRuntimeService {
             throw new WorkRuntimeError('Command must contain 1 to 20,000 characters.', 400, 'WORK_INVALID_COMMAND');
         }
         const boundedTimeout = Math.min(Math.max(timeoutMs, 1000), 600000);
-        const releaseLease = this.acquireRuntimeLease(task);
+        const releaseLease = await this.acquireRuntimeLease(task);
         let commandRegistered = false;
+        let containerStopped = false;
         try {
             await this.ensureImage(task);
-            this.assertTaskIsActive(task);
-            await this.withLifecycleLock(task.id, async () => {
-                this.assertTaskIsActive(task);
+            await this.assertTaskIsActive(task);
+            return await this.withLifecycleLock(task.id, async (assertHeld, signal) => {
+                await this.assertTaskIsActive(task);
                 try {
-                    await this.prepareWithLock(task);
-                    this.assertTaskIsActive(task);
+                    await this.prepareWithLock(task, signal);
+                    await this.assertTaskIsActive(task);
                     if (this.activeCommands.has(task.id)) {
                         throw new WorkRuntimeError('This Work task already has an active command.', 409, 'WORK_COMMAND_ACTIVE');
                     }
-                    const previewState = await this.previewProcessCheckWithLock(task);
+                    const previewState = await this.previewProcessCheckWithLock(task, signal);
                     if (previewState === 'ready' || previewState === 'alive') {
                         throw new WorkRuntimeError('Stop the Work preview before running another command.', 409, 'WORK_COMMAND_REQUIRES_STOPPED_PREVIEW');
                     }
@@ -951,36 +1076,50 @@ export class WorkRuntimeService {
                     commandRegistered = true;
                 }
                 catch (error) {
-                    await this.stopContainerIfIdleWithLock(task);
+                    await this.stopContainerIfIdleWithLock(task, signal);
                     throw error;
                 }
-            });
-            return await this.exec(task, [
-                'timeout',
-                '--signal=TERM',
-                '--kill-after=3s',
-                `${Math.ceil(boundedTimeout / 1000)}s`,
-                '/bin/bash',
-                '-c',
-                MANAGED_COMMAND_SCRIPT,
-                '--',
-                command,
-            ], {
-                timeoutMs: boundedTimeout + 5000,
-                maxOutputChars: config.maxOutputChars,
-                acceptFailure: true,
+                try {
+                    const result = await this.exec(task, [
+                        'timeout',
+                        '--signal=TERM',
+                        '--kill-after=3s',
+                        `${Math.ceil(boundedTimeout / 1000)}s`,
+                        '/bin/bash',
+                        '-c',
+                        MANAGED_COMMAND_SCRIPT,
+                        '--',
+                        command,
+                    ], {
+                        timeoutMs: boundedTimeout + 5000,
+                        maxOutputChars: config.maxOutputChars,
+                        acceptFailure: true,
+                        abortSignal: signal,
+                    });
+                    await assertHeld();
+                    return result;
+                }
+                finally {
+                    if (commandRegistered) {
+                        // Keep the distributed lifecycle fence through teardown. A
+                        // second replica must never stop or recreate this sandbox in
+                        // the gap between preparation and Docker exec.
+                        await this.stopContainerWithLock(task, signal);
+                        containerStopped = true;
+                    }
+                }
             });
         }
         finally {
+            this.activeCommands.delete(task.id);
             try {
-                if (commandRegistered) {
-                    // This also kills intentionally detached descendants that escaped
-                    // the managed process group. The named volume remains durable.
-                    await this.stopContainer(task);
+                if (containerStopped) {
+                    this.releasePreviewLease(task.id);
+                    await this.markPreviewStopped(task.id);
+                    this.completeRecoveryTask(task.id);
                 }
             }
             finally {
-                this.activeCommands.delete(task.id);
                 releaseLease();
             }
         }
@@ -993,35 +1132,37 @@ export class WorkRuntimeService {
         if (previewCommand && previewCommand.length > 20000) {
             throw new WorkRuntimeError('Preview command is too long.', 400, 'WORK_INVALID_COMMAND');
         }
-        const releaseLease = this.acquireRuntimeLease(task);
+        const releaseLease = await this.acquireRuntimeLease(task);
         let leaseRetained = false;
         try {
             await this.ensureImage(task);
-            this.assertTaskIsActive(task);
-            const url = await this.withLifecycleLock(task.id, async () => {
-                this.assertTaskIsActive(task);
+            await this.assertTaskIsActive(task);
+            const url = await this.withLifecycleLock(task.id, async (assertHeld, signal) => {
+                await this.assertTaskIsActive(task);
                 this.assertCurrentNetworkPolicy(task);
                 if (this.activeCommands.has(task.id)) {
                     throw new WorkRuntimeError('Wait for the active command to finish before starting a preview.', 409, 'WORK_PREVIEW_COMMAND_ACTIVE');
                 }
-                hooks.onStarting?.();
+                await assertHeld();
+                await hooks.onStarting?.();
                 try {
-                    await this.prepareWithLock(task);
-                    this.assertTaskIsActive(task);
+                    await this.prepareWithLock(task, signal);
+                    await this.assertTaskIsActive(task);
                     const previewLaunch = previewCommand
                         ? {
                             kind: 'shell',
                             workdir: '/workspace',
                             command: previewCommand,
                         }
-                        : await this.detectPreviewLaunch(task);
-                    const previewUrl = await this.startPreviewPrepared(task, previewLaunch);
-                    this.assertTaskIsActive(task);
-                    hooks.onRunning?.(previewUrl);
-                    return previewUrl;
+                        : await this.detectPreviewLaunch(task, signal);
+                    const preview = await this.startPreviewPrepared(task, previewLaunch, signal);
+                    await this.assertTaskIsActive(task);
+                    await assertHeld();
+                    await hooks.onRunning?.(preview.url, preview.endpoint);
+                    return preview.url;
                 }
                 catch (error) {
-                    hooks.onFailed?.();
+                    await hooks.onFailed?.();
                     this.releasePreviewLease(task.id);
                     throw error;
                 }
@@ -1041,12 +1182,13 @@ export class WorkRuntimeService {
             }
         }
     }
-    async detectPreviewLaunch(task) {
+    async detectPreviewLaunch(task, signal) {
         try {
             const result = await this.driver.exec(task, ['node', '-e', PREVIEW_TARGET_SCRIPT, '--', '/workspace'], {
                 acceptFailure: true,
                 timeoutMs: 5000,
                 maxOutputChars: 10000,
+                abortSignal: signal,
             });
             if (result.exitCode !== 0) {
                 throw new WorkRuntimeError(`Could not inspect the workspace for a previewable app.${result.stderr.trim() ? `\n${result.stderr.trim()}` : ''}`, 422, 'WORK_PREVIEW_DETECTION_FAILED');
@@ -1084,7 +1226,7 @@ export class WorkRuntimeService {
         }
         catch (error) {
             try {
-                await this.stopPreviewPrepared(task);
+                await this.stopPreviewPrepared(task, signal);
             }
             catch (cleanupError) {
                 logger.warn(`Could not clean up failed preview detection for Work task ${task.id}:`, cleanupError);
@@ -1092,7 +1234,7 @@ export class WorkRuntimeService {
             throw error;
         }
     }
-    async startPreviewPrepared(task, previewLaunch) {
+    async startPreviewPrepared(task, previewLaunch, signal) {
         try {
             const launch = await this.driver.exec(task, [
                 '/bin/bash',
@@ -1121,6 +1263,7 @@ export class WorkRuntimeService {
                 workdir: previewLaunch.workdir,
                 acceptFailure: true,
                 timeoutMs: 5000,
+                abortSignal: signal,
             });
             if (launch.exitCode !== 0 && launch.exitCode !== 17) {
                 throw await this.previewStartError(task, 'Preview process could not be launched.', launch.stderr);
@@ -1133,7 +1276,12 @@ export class WorkRuntimeService {
                     PREVIEW_READY_SCRIPT,
                     '--',
                     String(config.previewPort),
-                ], { acceptFailure: true, timeoutMs: 2000, maxOutputChars: 2000 });
+                ], {
+                    acceptFailure: true,
+                    timeoutMs: 2000,
+                    maxOutputChars: 2000,
+                    abortSignal: signal,
+                });
                 if (readiness.exitCode === 0)
                     break;
                 if (readiness.exitCode !== 3) {
@@ -1142,17 +1290,24 @@ export class WorkRuntimeService {
                 if (Date.now() >= deadline) {
                     throw await this.previewStartError(task, `Preview did not listen on port ${config.previewPort} within ${PREVIEW_READY_TIMEOUT_MS / 1000} seconds.`);
                 }
-                await new Promise(resolve => setTimeout(resolve, PREVIEW_POLL_INTERVAL_MS));
+                await waitForAbortSignal(new Promise(resolve => setTimeout(resolve, PREVIEW_POLL_INTERVAL_MS)), signal);
             }
             const endpoint = await this.driver.previewEndpoint(task);
             if (!endpoint) {
                 throw new WorkRuntimeError('The runtime did not expose the preview port.', 503, 'WORK_PREVIEW_PORT_UNAVAILABLE');
             }
-            return workPreviewProxyService.createPreviewUrl(task.id, endpoint.port, endpoint.host);
+            const upstream = {
+                host: normalizePreviewUpstreamHost(endpoint.host),
+                port: endpoint.port,
+            };
+            return {
+                url: workPreviewProxyService.createPreviewUrl(task.id, endpoint.port),
+                endpoint: upstream,
+            };
         }
         catch (error) {
             try {
-                await this.stopPreviewPrepared(task);
+                await this.stopPreviewPrepared(task, signal);
             }
             catch (cleanupError) {
                 logger.warn(`Could not clean up failed preview for Work task ${task.id}:`, cleanupError);
@@ -1174,14 +1329,15 @@ export class WorkRuntimeService {
     async stopPreview(task, hooks = {}) {
         let stopped = false;
         try {
-            await this.withLifecycleLock(task.id, async () => {
+            await this.withLifecycleLock(task.id, async (assertHeld, signal) => {
                 this.assertCurrentNetworkPolicy(task);
                 if (this.activeCommands.has(task.id)) {
                     throw new WorkRuntimeError('Wait for the active command to finish before stopping a preview.', 409, 'WORK_PREVIEW_COMMAND_ACTIVE');
                 }
-                await this.stopPreviewPrepared(task);
+                await this.stopPreviewPrepared(task, signal);
                 stopped = true;
-                hooks.onStopped?.();
+                await assertHeld();
+                await hooks.onStopped?.();
             });
         }
         finally {
@@ -1191,30 +1347,35 @@ export class WorkRuntimeService {
         }
     }
     async isPreviewRunning(task) {
-        return this.withLifecycleLock(task.id, async () => {
+        return this.withLifecycleLock(task.id, async (_assertHeld, signal) => {
             this.assertCurrentNetworkPolicy(task);
             // An ordinary command owns this running container and will stop it when
             // it finishes. Preview reconciliation must not interrupt that command.
             if (this.activeCommands.has(task.id))
                 return false;
-            const state = await this.previewProcessCheckWithLock(task);
+            const state = await this.previewProcessCheckWithLock(task, signal);
             if (state === 'ready' && this.previewLeaseReleases.has(task.id)) {
                 return true;
             }
             if (state === 'dead' || state === 'alive') {
-                await this.stopPreviewPrepared(task);
+                await this.stopPreviewPrepared(task, signal);
             }
             if (state === 'ready') {
-                await this.stopPreviewPrepared(task);
+                await this.stopPreviewPrepared(task, signal);
             }
             this.releasePreviewLease(task.id);
             return false;
         });
     }
-    async previewProcessCheckWithLock(task) {
+    async previewProcessCheckWithLock(task, signal) {
         if ((await this.driver.runtimeState(task)) !== 'running')
             return 'absent';
-        const readiness = await this.driver.exec(task, ['node', '-e', PREVIEW_READY_SCRIPT, '--', String(config.previewPort)], { acceptFailure: true, timeoutMs: 2000, maxOutputChars: 2000 });
+        const readiness = await this.driver.exec(task, ['node', '-e', PREVIEW_READY_SCRIPT, '--', String(config.previewPort)], {
+            acceptFailure: true,
+            timeoutMs: 2000,
+            maxOutputChars: 2000,
+            abortSignal: signal,
+        });
         if (readiness.exitCode === 0)
             return 'ready';
         if (readiness.exitCode === 3)
@@ -1223,18 +1384,17 @@ export class WorkRuntimeService {
             return 'dead';
         throw new WorkRuntimeError(readiness.stderr.trim() || 'Could not inspect the Work preview.', 503, 'WORK_PREVIEW_INSPECT_FAILED');
     }
-    async stopPreviewPrepared(task) {
-        workPreviewProxyService.clearPreviewUpstream(task.id);
+    async stopPreviewPrepared(task, signal) {
         // Stop the container, not only the recorded process group. A custom
         // preview command can intentionally double-fork or create a new session;
         // Docker's container boundary guarantees those descendants are gone.
         // The named /workspace volume remains persistent across the restart.
-        await this.stopContainerWithLock(task);
+        await this.stopContainerWithLock(task, signal);
     }
     async ensureImage(task) {
         // Pulls are deduplicated per image, not globally: tasks under different
         // policies may run different images.
-        const image = workPolicyService.resolve(task.policyId).image;
+        const image = (await workPolicyService.resolve(task.policyId)).image;
         let preparation = this.imagePreparations.get(image);
         if (!preparation) {
             preparation = this.driver.ensureImage(image).catch(error => {
@@ -1245,34 +1405,29 @@ export class WorkRuntimeService {
         }
         await preparation;
     }
-    assertTaskIsActive(task) {
+    async assertTaskIsActive(task) {
         this.assertAcceptingWork();
         if (this.retiringTasks.has(task.id)) {
             throw new WorkRuntimeError('This Work task is being deleted.', 409, 'WORK_TASK_REMOVING');
         }
-        this.assertTaskOwnerHasWorkAccess(task);
+        await this.assertTaskOwnerHasWorkAccess(task);
     }
-    assertTaskOwnerHasWorkAccess(task) {
-        const access = getDatabase()
-            .prepare(`SELECT users.role AS role, users.account_status AS status
-         FROM work_tasks
-         JOIN users ON users.id = work_tasks.user_id
-         WHERE work_tasks.id = ? AND work_tasks.user_id = ?`)
-            .get(task.id, task.userId);
+    async assertTaskOwnerHasWorkAccess(task) {
+        const access = await getWorkPersistence().findTaskOwnerAccess(task.id, task.userId);
         if (!access) {
             throw new WorkRuntimeError('This Work task no longer exists.', 409, 'WORK_TASK_REMOVING');
         }
-        if (!userHasWorkAccess(access)) {
+        if (!(await userHasWorkAccess(access))) {
             throw new WorkRuntimeError('Work access for this task was revoked.', 403, 'WORK_ACCESS_REVOKED');
         }
     }
-    assertTaskStillOwned(task) {
-        const persisted = getDatabase()
-            .prepare(`SELECT 1
-         FROM work_tasks
-         WHERE id = ? AND user_id = ?
-           AND volume_name = ? AND container_name = ?`)
-            .get(task.id, task.userId, task.volumeName, task.containerName);
+    async assertTaskStillOwned(task) {
+        const persisted = await getWorkPersistence().taskStillOwnsResources({
+            taskId: task.id,
+            userId: task.userId,
+            volumeName: task.volumeName,
+            containerName: task.containerName,
+        });
         if (!persisted) {
             throw new WorkRuntimeError('This Work task no longer exists or its managed resources changed.', 409, 'WORK_TASK_REMOVING');
         }
@@ -1289,7 +1444,96 @@ export class WorkRuntimeService {
     }
     async withLifecycleLock(taskId, operation) {
         const previous = this.lifecycleTails.get(taskId) || Promise.resolve();
-        const current = previous.catch(() => undefined).then(operation);
+        const current = previous
+            .catch(() => undefined)
+            .then(async () => {
+            let sharedLease = null;
+            if (getPlatformRuntimeConfig().mode === 'team') {
+                const deadline = Date.now() + 10000;
+                do {
+                    let abandoned = false;
+                    const pendingLease = getCoordinator().acquireLease(`work-task-lifecycle:${taskId}`, 60000);
+                    void pendingLease
+                        .then(lease => {
+                        if (!abandoned || !lease)
+                            return;
+                        void withCoordinationTimeout(lease.release(), SHARED_COORDINATION_OPERATION_TIMEOUT_MS).catch(() => undefined);
+                    })
+                        .catch(() => undefined);
+                    try {
+                        sharedLease = await withCoordinationTimeout(pendingLease, SHARED_COORDINATION_OPERATION_TIMEOUT_MS);
+                    }
+                    catch (_error) {
+                        abandoned = true;
+                        throw new WorkRuntimeError('Work task lifecycle coordination is unavailable.', 503, 'WORK_RUNTIME_LEASE_CONFLICT');
+                    }
+                    if (!sharedLease) {
+                        await new Promise(resolve => setTimeout(resolve, 25));
+                    }
+                } while (!sharedLease && Date.now() < deadline);
+                if (!sharedLease) {
+                    throw new WorkRuntimeError('This Work task has a lifecycle operation on another replica.', 409, 'WORK_RUNTIME_LEASE_CONFLICT');
+                }
+            }
+            let closed = false;
+            let lost = false;
+            const lossController = new AbortController();
+            let renewalTimer;
+            const leaseLostError = () => new WorkRuntimeError('The shared Work task lifecycle lease was lost.', 503, 'WORK_RUNTIME_LEASE_CONFLICT');
+            const markLost = () => {
+                if (lost)
+                    return;
+                lost = true;
+                lossController.abort(leaseLostError());
+            };
+            const assertHeld = async () => {
+                if (!sharedLease)
+                    return;
+                if (closed || lost)
+                    throw leaseLostError();
+                try {
+                    if (await withCoordinationTimeout(sharedLease.extend(60000), SHARED_COORDINATION_OPERATION_TIMEOUT_MS)) {
+                        return;
+                    }
+                }
+                catch {
+                    // Report expiry and coordination outages through one safe fence.
+                }
+                markLost();
+                throw leaseLostError();
+            };
+            if (sharedLease) {
+                const renew = async () => {
+                    if (closed)
+                        return;
+                    try {
+                        if (!(await withCoordinationTimeout(sharedLease?.extend(60000) ?? Promise.resolve(false), SHARED_COORDINATION_OPERATION_TIMEOUT_MS))) {
+                            markLost();
+                        }
+                    }
+                    catch {
+                        markLost();
+                    }
+                    if (!closed && !lost)
+                        renewalTimer = setTimeout(renew, 20000);
+                };
+                renewalTimer = setTimeout(renew, 20000);
+                renewalTimer.unref?.();
+            }
+            try {
+                await assertHeld();
+                const result = await operation(assertHeld, lossController.signal);
+                return result;
+            }
+            finally {
+                closed = true;
+                if (renewalTimer)
+                    clearTimeout(renewalTimer);
+                if (sharedLease) {
+                    await withCoordinationTimeout(sharedLease.release(), SHARED_COORDINATION_OPERATION_TIMEOUT_MS).catch(() => false);
+                }
+            }
+        });
         const tail = current.then(() => undefined, () => undefined);
         this.lifecycleTails.set(taskId, tail);
         try {