libre-webui 0.39.0 → 0.40.0

package/backend/dist/services/workRuntimeService.js

@@ -346,6 +346,9 @@ export const acquireSharedRuntimeLeaseWithWait = async (attempt, options) => {
 };
 /** How long a preview may sit at `starting` before it counts as abandoned. */
 const PREVIEW_START_GRACE_MS = 5 * 60000;
+// Only this explicit protocol exempts a runtime hold from idle cleanup. Older
+// generic members and unrecognized members remain protected during upgrades.
+const isPreviewOnlyPresence = (member) => /^(?:standalone|app-external|app-embedded|external-worker)-\d+-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}:preview-only$/.test(member);
 export class WorkRuntimeService {
     constructor(driver = new DockerWorkRuntimeDriver()) {
         this.image = config.image;
@@ -365,6 +368,7 @@ export class WorkRuntimeService {
         this.activeCommands = new Set();
         this.lastRuntimeUnavailableReason = null;
         this.runtimeLeases = new Map();
+        this.runtimePresenceTails = new Map();
         this.previewLeaseReleases = new Map();
         this.previewUsageReleases = new Map();
         this.terminalHolds = new Map();
@@ -621,6 +625,15 @@ export class WorkRuntimeService {
                 throw new WorkRuntimeError('This Work runtime lease belongs to a different administrator.', 409, 'WORK_RUNTIME_LEASE_CONFLICT');
             }
             existing.holders += 1;
+            try {
+                // Publish protection before a helper or run can enter the lifecycle
+                // lock, even when its only previous holder was an idle preview.
+                await this.refreshRuntimePresence(task, existing);
+            }
+            catch (error) {
+                this.releaseRuntimeLease(task.id, task.userId);
+                throw error;
+            }
         }
         else {
             const perUser = [...this.runtimeLeases.values()].filter(lease => lease.userId === task.userId).length;
@@ -666,8 +679,10 @@ export class WorkRuntimeService {
                             throw new Error('lease ownership was lost');
                     })
                         .catch(error => {
-                        if (runtimeLease.sharedLeaseLost)
+                        if (runtimeLease.sharedLeaseLost ||
+                            this.runtimeLeases.get(task.id) !== runtimeLease) {
                             return;
+                        }
                         runtimeLease.sharedLeaseLost = true;
                         logger.error(`Shared Work runtime lease was lost for task ${task.id}; stopping its sandbox:`, error);
                         void this.driver
@@ -676,7 +691,7 @@ export class WorkRuntimeService {
                     });
                 }, 20000);
                 runtimeLease.sharedLeaseTimer.unref?.();
-                const refreshPresence = () => withCoordinationTimeout(getCoordinator().setPresence(`work-task-active:${task.id}`, this.activityMemberId, 30000), SHARED_COORDINATION_OPERATION_TIMEOUT_MS);
+                const refreshPresence = (reconcilePreview = false) => this.refreshRuntimePresence(task, runtimeLease, reconcilePreview);
                 try {
                     await waitForAbortSignal(refreshPresence(), signal);
                 }
@@ -688,7 +703,7 @@ export class WorkRuntimeService {
                     throw error;
                 }
                 runtimeLease.presenceTimer = setInterval(() => {
-                    void refreshPresence().catch(error => logger.warn(`Could not refresh Work activity for ${task.id}:`, error));
+                    void refreshPresence(true).catch(error => logger.warn(`Could not refresh Work activity for ${task.id}:`, error));
                 }, 10000);
                 runtimeLease.presenceTimer.unref?.();
             }
@@ -710,6 +725,7 @@ export class WorkRuntimeService {
             lease.holders -= 1;
         if (!force && lease.holders > 0)
             return;
+        lease.released = true;
         if (lease.presenceTimer)
             clearInterval(lease.presenceTimer);
         if (lease.sharedLeaseTimer)
@@ -717,10 +733,78 @@ export class WorkRuntimeService {
         this.runtimeLeases.delete(taskId);
         void lease.sharedLease?.release().catch(() => false);
         if (getPlatformRuntimeConfig().mode === 'team') {
-            void getCoordinator()
-                .clearPresence(`work-task-active:${taskId}`, this.activityMemberId)
-                .catch(error => logger.warn(`Could not clear Work activity for ${taskId}:`, error));
+            const clear = async () => {
+                await withCoordinationTimeout(getCoordinator().clearPresence(`work-task-active:${taskId}`, this.activityMemberId), SHARED_COORDINATION_OPERATION_TIMEOUT_MS);
+                await withCoordinationTimeout(getCoordinator().clearPresence(`work-task-active:${taskId}`, `${this.activityMemberId}:preview-only`), SHARED_COORDINATION_OPERATION_TIMEOUT_MS);
+            };
+            lease.presenceTail = this.queueRuntimePresence(taskId, clear);
+            void lease.presenceTail.catch(error => logger.warn(`Could not clear Work activity for ${taskId}:`, error));
+        }
+    }
+    refreshRuntimePresence(task, lease, reconcilePreview = false) {
+        if (getPlatformRuntimeConfig().mode !== 'team')
+            return Promise.resolve();
+        const refresh = async () => {
+            if (lease.released)
+                return;
+            if (reconcilePreview && this.previewLeaseReleases.has(task.id)) {
+                const generation = lease.previewGeneration;
+                const release = this.previewLeaseReleases.get(task.id);
+                const row = await getWorkPersistence().findTask(task.id, task.userId);
+                // A stopped-state snapshot must not release a preview reopened while
+                // this database read was in flight. Startup advances the generation
+                // under the lifecycle lock before changing its persisted state.
+                if ((!row || row.preview_status === 'stopped') &&
+                    lease.previewGeneration === generation &&
+                    this.previewLeaseReleases.get(task.id) === release) {
+                    this.releasePreviewLease(task.id);
+                }
+            }
+            if (lease.released)
+                return;
+            const activeMember = this.activityMemberId;
+            const previewMember = `${activeMember}:preview-only`;
+            const previewOnly = lease.holders === 1 && this.previewLeaseReleases.has(task.id);
+            // Set the new member before clearing the old one, and serialize all
+            // transitions so a racing heartbeat cannot erase a newly active hold.
+            await withCoordinationTimeout(getCoordinator().setPresence(`work-task-active:${task.id}`, previewOnly ? previewMember : activeMember, 30000), SHARED_COORDINATION_OPERATION_TIMEOUT_MS);
+            await withCoordinationTimeout(getCoordinator().clearPresence(`work-task-active:${task.id}`, previewOnly ? activeMember : previewMember), SHARED_COORDINATION_OPERATION_TIMEOUT_MS);
+        };
+        const pending = this.queueRuntimePresence(task.id, refresh);
+        lease.presenceTail = pending;
+        return pending;
+    }
+    queueRuntimePresence(taskId, operation) {
+        // Include releases and reacquisitions in one queue: an old lease's delayed
+        // clear must finish before a new lease publishes the same process member.
+        const pending = (this.runtimePresenceTails.get(taskId) ?? Promise.resolve())
+            .catch(() => undefined)
+            .then(operation);
+        this.runtimePresenceTails.set(taskId, pending);
+        void pending
+            .finally(() => {
+            if (this.runtimePresenceTails.get(taskId) === pending) {
+                this.runtimePresenceTails.delete(taskId);
+            }
+        })
+            .catch(() => undefined);
+        return pending;
+    }
+    hasLocalRuntimeActivity(taskId) {
+        return (this.activeCommands.has(taskId) ||
+            (this.terminalHolds.get(taskId) ?? 0) > 0 ||
+            (this.screenHolds.get(taskId) ?? 0) > 0 ||
+            (this.runtimeLeases.get(taskId)?.holders ?? 0) >
+                (this.previewLeaseReleases.has(taskId) ? 1 : 0));
+    }
+    async hasRuntimeActivity(taskId) {
+        if (this.hasLocalRuntimeActivity(taskId))
+            return true;
+        if (getPlatformRuntimeConfig().mode === 'team' &&
+            (await withCoordinationTimeout(getCoordinator().listPresence(`work-task-active:${taskId}`), SHARED_COORDINATION_OPERATION_TIMEOUT_MS)).some(member => !isPreviewOnlyPresence(member))) {
+            return true;
         }
+        return (await workUsageService.list(taskId)).some(entry => entry.kind !== 'preview');
     }
     assertRuntimeLease(task) {
         const lease = this.runtimeLeases.get(task.id);
@@ -866,7 +950,9 @@ export class WorkRuntimeService {
             releasePresence();
             try {
                 if (remaining <= 0 && !this.shuttingDown) {
-                    await this.withLifecycleLock(task.id, (_assertHeld, signal) => this.stopContainerIfIdleWithLock(task, signal));
+                    await this.withLifecycleLock(task.id, (_assertHeld, signal) => this.stopContainerIfIdleWithLock(task, signal, {
+                        ownsRuntimeLease: false,
+                    }));
                 }
             }
             catch (error) {
@@ -1164,19 +1250,7 @@ export class WorkRuntimeService {
             // Containers without a task row are startup reconciliation's business.
             if (!task)
                 continue;
-            if (getPlatformRuntimeConfig().mode === 'team' &&
-                (await getCoordinator().listPresence(`work-task-active:${task.id}`))
-                    .length > 0) {
-                continue;
-            }
-            const busy = this.activeCommands.has(task.id) ||
-                (this.terminalHolds.get(task.id) ?? 0) > 0 ||
-                (this.screenHolds.get(task.id) ?? 0) > 0 ||
-                (this.runtimeLeases.has(task.id) &&
-                    !this.previewLeaseReleases.has(task.id)) ||
-                // Holds from other processes: terminals, screens and commands
-                // record themselves in the usage registry.
-                (await workUsageService.list(task.id)).some(entry => entry.kind !== 'preview');
+            const busy = await this.hasRuntimeActivity(task.id);
             const sharedActivity = getPlatformRuntimeConfig().mode === 'team'
                 ? await getCoordinator().getCache(`work-task-activity:${task.id}`)
                 : null;
@@ -1190,14 +1264,29 @@ export class WorkRuntimeService {
             if (idleAfterMs <= 0 || now - lastActivity < idleAfterMs)
                 continue;
             try {
-                if (this.previewLeaseReleases.has(task.id)) {
-                    await this.stopPreview(task, {
-                        onStopped: () => workTaskService.updatePreview(task.id, 'stopped'),
-                    });
-                }
-                else {
-                    await this.stopContainer(task);
-                }
+                const didStop = await this.withLifecycleLock(task.id, async (assertHeld, signal) => {
+                    await assertHeld();
+                    // A run, viewer, or preview request may have arrived while the
+                    // sweep awaited policy resolution or the lifecycle lock.
+                    const currentActivity = getPlatformRuntimeConfig().mode === 'team'
+                        ? await getCoordinator().getCache(`work-task-activity:${task.id}`)
+                        : this.taskActivity.get(task.id);
+                    if ((await this.hasRuntimeActivity(task.id)) ||
+                        this.hasLocalRuntimeActivity(task.id) ||
+                        (currentActivity !== null &&
+                            currentActivity !== undefined &&
+                            now - currentActivity < idleAfterMs)) {
+                        return false;
+                    }
+                    await this.stopPreviewPrepared(task, signal);
+                    await assertHeld();
+                    await this.markPreviewStopped(task.id);
+                    this.releasePreviewLease(task.id);
+                    this.completeRecoveryTask(task.id);
+                    return true;
+                });
+                if (!didStop)
+                    continue;
                 this.taskActivity.delete(task.id);
                 stopped += 1;
                 logger.info(`Stopped idle Work sandbox ${task.containerName} after ${idleAfterMs}ms of inactivity.`);
@@ -1226,53 +1315,54 @@ export class WorkRuntimeService {
         const candidates = (await workTaskService.listAllTaskRecords()).filter(task => task.previewStatus === 'starting' || task.previewStatus === 'running');
         if (candidates.length === 0)
             return { stopped: 0 };
-        let discovered;
         try {
-            discovered = await this.driver.listManaged();
+            await this.driver.listManaged();
         }
         catch {
             // The runtime is unreachable; the rows cannot be judged either way.
             return { stopped: 0 };
         }
-        const running = new Set(discovered.filter(entry => entry.running).map(entry => entry.taskId));
         let stopped = 0;
         for (const task of candidates) {
-            // A preview that is still being brought up, or a task someone holds
-            // open, is not stale: leave the owner's own flows to settle it.
-            if (this.activeCommands.has(task.id) ||
-                (this.terminalHolds.get(task.id) ?? 0) > 0 ||
-                (this.screenHolds.get(task.id) ?? 0) > 0 ||
-                (this.runtimeLeases.has(task.id) &&
-                    !this.previewLeaseReleases.has(task.id))) {
-                continue;
-            }
-            if (task.previewStatus === 'starting' &&
-                now - task.updatedAt < PREVIEW_START_GRACE_MS) {
-                continue;
-            }
-            if ((await workUsageService.list(task.id)).some(entry => entry.kind !== 'preview')) {
+            if (await this.hasRuntimeActivity(task.id))
                 continue;
-            }
-            if (getPlatformRuntimeConfig().mode === 'team' &&
-                (await getCoordinator().listPresence(`work-task-active:${task.id}`))
-                    .length > 0) {
-                continue;
-            }
             try {
-                // Without a running container the preview cannot exist; with one,
-                // the same probe the Work list uses decides.
-                if (running.has(task.id) && (await this.isPreviewRunning(task))) {
+                const didReconcile = await this.withLifecycleLock(task.id, async (assertHeld, signal) => {
+                    await assertHeld();
+                    const row = await getWorkPersistence().findTask(task.id, task.userId);
+                    if (!row ||
+                        (row.preview_status !== 'starting' &&
+                            row.preview_status !== 'running') ||
+                        (row.preview_status === 'starting' &&
+                            now - row.updated_at < PREVIEW_START_GRACE_MS) ||
+                        (await this.hasRuntimeActivity(task.id)) ||
+                        this.hasLocalRuntimeActivity(task.id)) {
+                        return false;
+                    }
+                    const sharedPreview = getPlatformRuntimeConfig().mode === 'team' &&
+                        (await getCoordinator().listPresence(`work-task-active:${task.id}`)).some(isPreviewOnlyPresence);
+                    const state = await this.previewProcessCheckWithLock(task, signal);
+                    if (state === 'ready' &&
+                        (this.previewLeaseReleases.has(task.id) || sharedPreview))
+                        return false;
+                    if ((await this.hasRuntimeActivity(task.id)) ||
+                        this.hasLocalRuntimeActivity(task.id))
+                        return false;
+                    if (state !== 'absent')
+                        await this.stopPreviewPrepared(task, signal);
+                    await assertHeld();
+                    await this.markPreviewStopped(task.id);
+                    this.releasePreviewLease(task.id);
+                    return true;
+                });
+                if (!didReconcile)
                     continue;
-                }
+                stopped += 1;
+                logger.info(`Marked the stale ${task.previewStatus} preview of Work task ${task.id} as stopped.`);
             }
             catch (error) {
                 logger.warn(`Could not check the preview of Work task ${task.id}:`, error);
-                continue;
             }
-            await this.markPreviewStopped(task.id);
-            this.releasePreviewLease(task.id);
-            stopped += 1;
-            logger.info(`Marked the stale ${task.previewStatus} preview of Work task ${task.id} as stopped.`);
         }
         return { stopped };
     }
@@ -1535,9 +1625,23 @@ export class WorkRuntimeService {
             // transitions (start, stop, recreate, preview) stay lease-guarded.
             if (error instanceof WorkRuntimeError &&
                 error.code === 'WORK_RUNTIME_LEASE_CONFLICT') {
-                await this.assertTaskIsActive(task);
-                if ((await this.driver.runtimeState(task)) === 'running') {
-                    return operation();
+                const releasePresence = await this.acquireViewerPresence(task.id);
+                try {
+                    return await this.withLifecycleLock(task.id, async () => {
+                        await this.assertTaskIsActive(task);
+                        if ((await this.driver.runtimeState(task)) !== 'running') {
+                            throw error;
+                        }
+                        try {
+                            return await operation();
+                        }
+                        finally {
+                            this.noteTaskActivity(task.id);
+                        }
+                    });
+                }
+                finally {
+                    releasePresence();
                 }
             }
             throw error;
@@ -1566,6 +1670,16 @@ export class WorkRuntimeService {
         if (!options.ignoreActiveCommands && this.activeCommands.has(task.id)) {
             return false;
         }
+        // Commands and helpers still own their lease during teardown, but screen
+        // viewers do not. Preserve any other holder, including the run executor
+        // retaining the sandbox between tool calls, until explicit run cleanup.
+        const ownLeaseHolders = options.ownsRuntimeLease === false ? 0 : 1;
+        // A registered preview lease is checked for readiness below. Counting it
+        // as another hold would retain dead previews and their capacity forever.
+        const hasOtherRuntimeHolders = () => (this.runtimeLeases.get(task.id)?.holders ?? 0) >
+            ownLeaseHolders + (this.previewLeaseReleases.has(task.id) ? 1 : 0);
+        if (hasOtherRuntimeHolders())
+            return false;
         // An attached terminal session owns the running container exactly like a
         // ready preview does.
         if ((this.terminalHolds.get(task.id) ?? 0) > 0)
@@ -1602,10 +1716,11 @@ export class WorkRuntimeService {
         catch (error) {
             logger.warn(`Could not verify preview state before idling Work container ${task.containerName}; stopping it:`, error);
         }
-        // The preview probe above awaits a container exec; a terminal or screen
-        // hold can land during that window. Re-check synchronously so a viewer
-        // who just attached does not lose the session to a stale decision.
+        // The preview probe above awaits a container exec; a runtime lease or
+        // terminal/screen hold can land during that window. Re-check synchronously
+        // so newly acquired holds do not lose the sandbox to a stale decision.
         if ((!options.ignoreActiveCommands && this.activeCommands.has(task.id)) ||
+            hasOtherRuntimeHolders() ||
             (this.terminalHolds.get(task.id) ?? 0) > 0 ||
             (this.screenHolds.get(task.id) ?? 0) > 0) {
             return false;
@@ -2155,12 +2270,21 @@ export class WorkRuntimeService {
             // idempotent, and lifecycle transitions stay lease-guarded.
             if (error instanceof WorkRuntimeError &&
                 error.code === 'WORK_RUNTIME_LEASE_CONFLICT') {
-                await this.assertTaskIsActive(task);
-                if ((await this.driver.runtimeState(task)) === 'running') {
-                    await this.startComputerInContainer(task);
-                    const value = await fn();
-                    this.noteTaskActivity(task.id);
-                    return value;
+                const releasePresence = await this.acquireViewerPresence(task.id);
+                try {
+                    return await this.withLifecycleLock(task.id, async (_assertHeld, signal) => {
+                        await this.assertTaskIsActive(task);
+                        if ((await this.driver.runtimeState(task)) !== 'running') {
+                            throw error;
+                        }
+                        await this.startComputerInContainer(task, signal);
+                        const value = await fn(signal);
+                        this.noteTaskActivity(task.id);
+                        return value;
+                    });
+                }
+                finally {
+                    releasePresence();
                 }
             }
             throw error;
@@ -2213,7 +2337,13 @@ export class WorkRuntimeService {
                     throw new WorkRuntimeError('Wait for the active command to finish before starting a preview.', 409, 'WORK_PREVIEW_COMMAND_ACTIVE');
                 }
                 await assertHeld();
+                const startingLease = this.runtimeLeases.get(task.id);
+                if (startingLease) {
+                    startingLease.previewGeneration =
+                        (startingLease.previewGeneration ?? 0) + 1;
+                }
                 await hooks.onStarting?.();
+                let registeredPreview = false;
                 try {
                     await this.prepareWithLock(task, signal);
                     await this.assertTaskIsActive(task);
@@ -2228,28 +2358,53 @@ export class WorkRuntimeService {
                     await this.assertTaskIsActive(task);
                     await assertHeld();
                     await hooks.onRunning?.(preview.url, preview.endpoint);
+                    // Invalidate stopped snapshots begun during startup as well as
+                    // those begun before it, including a reused preview callback.
+                    if (startingLease) {
+                        startingLease.previewGeneration =
+                            (startingLease.previewGeneration ?? 0) + 1;
+                    }
+                    if (this.previewLeaseReleases.has(task.id)) {
+                        releaseLease();
+                    }
+                    else {
+                        this.previewLeaseReleases.set(task.id, releaseLease);
+                        registeredPreview = true;
+                        this.previewUsageReleases.set(task.id, await workUsageService.begin(task.id, 'preview', task.userId));
+                    }
+                    const lease = this.runtimeLeases.get(task.id);
+                    if (lease)
+                        await this.refreshRuntimePresence(task, lease);
+                    leaseRetained = true;
                     return preview.url;
                 }
                 catch (error) {
-                    await hooks.onFailed?.();
-                    this.releasePreviewLease(task.id);
+                    if (registeredPreview && !this.hasLocalRuntimeActivity(task.id)) {
+                        // Retain ownership and the startup lifecycle lock until rollback
+                        // has stopped the sandbox. Never stop a newly active run holder.
+                        try {
+                            await this.stopPreviewPrepared(task, signal);
+                            await assertHeld();
+                            await this.markPreviewStopped(task.id);
+                        }
+                        catch (cleanupError) {
+                            logger.warn(`Could not clean up failed Work preview ${task.id}:`, cleanupError);
+                        }
+                    }
+                    try {
+                        await hooks.onFailed?.();
+                    }
+                    finally {
+                        this.releasePreviewLease(task.id);
+                    }
                     throw error;
                 }
             });
-            if (this.previewLeaseReleases.has(task.id)) {
-                releaseLease();
-            }
-            else {
-                this.previewLeaseReleases.set(task.id, releaseLease);
-                this.previewUsageReleases.set(task.id, await workUsageService.begin(task.id, 'preview', task.userId));
-            }
-            leaseRetained = true;
             return url;
         }
         finally {
-            if (!leaseRetained) {
+            if (!leaseRetained)
                 releaseLease();
-            }
         }
     }
     async detectPreviewLaunch(task, signal) {
@@ -2424,6 +2579,10 @@ export class WorkRuntimeService {
             if (this.activeCommands.has(task.id))
                 return false;
             const state = await this.previewProcessCheckWithLock(task, signal);
+            const sharedPreview = getPlatformRuntimeConfig().mode === 'team' &&
+                (await withCoordinationTimeout(getCoordinator().listPresence(`work-task-active:${task.id}`), SHARED_COORDINATION_OPERATION_TIMEOUT_MS)).some(member => member !== this.activityMemberId);
+            if (sharedPreview)
+                return state === 'ready';
             if (state === 'ready' && this.previewLeaseReleases.has(task.id)) {
                 return true;
             }