agent-tower 0.6.0-beta.11 → 0.6.0-beta.13

package/dist/services/team-reconciler.service.js

@@ -2,6 +2,8 @@ import { TaskStatus } from '../types/index.js';
 import { prisma } from '../utils/index.js';
 import { emitTeamRunInvalidated } from './team-run-events.js';
 import { isTaskDeleted } from './deleted-task-guard.js';
+import { acquireTeamMemberAdmission } from './team-member-admission-barrier.js';
+import { evaluateSessionRuntimeCleanup, reportDueRuntimeCleanupQuarantines, } from './session-runtime-cleanup-gate.js';
 // 统一补催/唤醒退避：指数增长（×2）封顶 5min，共 10 档（约 42min 触达上限）。
 const DEFAULT_REMINDER_DELAYS_MS = [
     60_000, 120_000, 240_000, 300_000, 300_000, 300_000, 300_000, 300_000, 300_000, 300_000,
@@ -52,7 +54,7 @@ export class TeamReconcilerService {
         this.heartbeatIdleThresholdMs = dependencies.heartbeatIdleThresholdMs;
         this.absoluteNudgeBudgetMs = dependencies.absoluteNudgeBudgetMs ?? DEFAULT_ABSOLUTE_NUDGE_BUDGET_MS;
     }
-    async handleSessionExit(sessionId) {
+    async handleSessionExit(sessionId, expectedRuntimeInstanceId) {
         const invocation = await prisma.agentInvocation.findFirst({
             where: { sessionId },
             select: { id: true },
@@ -60,7 +62,7 @@ export class TeamReconcilerService {
         if (!invocation) {
             return false;
         }
-        await this.reconcileInvocation(invocation.id);
+        await this.reconcileInvocation(invocation.id, expectedRuntimeInstanceId);
         return true;
     }
     async handleSessionStopped(sessionId) {
@@ -69,49 +71,76 @@ export class TeamReconcilerService {
             select: {
                 id: true,
                 teamRunId: true,
+                memberId: true,
+                status: true,
                 teamRun: { select: { task: { select: { deletedAt: true } } } },
             },
         });
         if (!invocation) {
-            return false;
+            return [];
         }
-        const cancelled = await prisma.agentInvocation.updateMany({
-            where: {
-                id: invocation.id,
-                status: { in: ACTIVE_INVOCATION_STATUSES },
-            },
-            data: {
-                status: 'CANCELLED',
-                nextRoomReplyReminderAt: null,
-            },
+        if (TERMINAL_INVOCATION_STATUSES.includes(invocation.status)) {
+            return this.afterInvocationTerminal(invocation.teamRunId, invocation.id);
+        }
+        await prisma.agentInvocation.updateMany({
+            where: { id: invocation.id, status: { in: ACTIVE_INVOCATION_STATUSES } },
+            data: { dispatchRevokedAt: this.now(), nextRoomReplyReminderAt: null },
         });
-        this.clearReminderTimer(invocation.id);
-        if (cancelled.count !== 1) {
-            return true;
+        return this.terminalizeRevokedInvocation(invocation.id);
+    }
+    async isSessionRuntimeCleanupConfirmed(sessionId) {
+        return (await evaluateSessionRuntimeCleanup(sessionId, {
+            hasActiveTurn: (candidateId) => this.hasActiveTurn(candidateId),
+            hasRuntimeProcessOwner: this.sessionMessenger?.hasRuntimeProcessOwner
+                ? (runtimeInstanceId, ownerSessionId, launchClaimNumber) => this.sessionMessenger.hasRuntimeProcessOwner(runtimeInstanceId, ownerSessionId ?? sessionId, launchClaimNumber)
+                : undefined,
+        }, this.now())).confirmed;
+    }
+    async reconcileInvocation(invocationId, expectedRuntimeInstanceId) {
+        const candidate = await prisma.agentInvocation.findUnique({
+            where: { id: invocationId },
+            select: { teamRunId: true, memberId: true },
+        });
+        if (!candidate) {
+            return;
         }
-        await this.emitTeamRunInvalidated(invocation.teamRunId, ['agent-invocations', 'team-run'], 'agent-invocation-updated');
-        if (isTaskDeleted(invocation.teamRun.task)) {
-            await this.syncTerminalWorkRequest(invocation.id);
+        let result;
+        const releaseAdmission = await acquireTeamMemberAdmission(candidate.teamRunId, candidate.memberId);
+        try {
+            result = await this.reconcileInvocationUnderAdmission(invocationId);
+        }
+        finally {
+            releaseAdmission();
+        }
+        if (result.kind === 'task_deleted') {
             const scheduler = await this.getScheduler();
-            scheduler.releaseInvocationLocks(invocation.id);
-            return true;
+            scheduler.releaseInvocationLocks(invocationId);
+            return;
+        }
+        if (result.kind === 'terminal') {
+            await this.emitTeamRunInvalidated(result.teamRunId, ['agent-invocations', 'team-run'], 'agent-invocation-updated');
+            this.clearReminderTimer(result.invocationId);
+            await this.afterInvocationTerminal(result.teamRunId, result.invocationId, expectedRuntimeInstanceId);
         }
-        await this.afterInvocationTerminal(invocation.teamRunId, invocation.id);
-        return true;
     }
-    async reconcileInvocation(invocationId) {
+    async reconcileInvocationUnderAdmission(invocationId) {
         const invocation = await prisma.agentInvocation.findUnique({
             where: { id: invocationId },
             include: { teamRun: { select: { heartbeatTimeoutMinutes: true, task: { select: { deletedAt: true } } } } },
         });
-        if (!invocation) {
-            return;
-        }
+        if (!invocation)
+            return { kind: 'none' };
         if (isTaskDeleted(invocation.teamRun.task)) {
-            const scheduler = await this.getScheduler();
-            scheduler.releaseInvocationLocks(invocation.id);
             this.clearReminderTimer(invocation.id);
-            return;
+            return { kind: 'task_deleted' };
+        }
+        // Stop admission is authoritative for all reminder/follow-up paths. A
+        // revoked invocation is completed by the cleanup recovery boundary once
+        // its owned tree is confirmed; it must never be nudged back into a turn.
+        if (invocation.dispatchRevokedAt
+            && ACTIVE_INVOCATION_STATUSES.includes(invocation.status)) {
+            this.clearReminderTimer(invocation.id);
+            return { kind: 'none' };
         }
         const hasRoomReply = await prisma.roomMessage.count({
             where: {
@@ -122,8 +151,12 @@ export class TeamReconcilerService {
             },
         }) > 0;
         if (hasRoomReply) {
-            await prisma.agentInvocation.update({
-                where: { id: invocation.id },
+            const completed = await prisma.agentInvocation.updateMany({
+                where: {
+                    id: invocation.id,
+                    status: { in: ACTIVE_INVOCATION_STATUSES },
+                    dispatchRevokedAt: null,
+                },
                 data: {
                     status: 'COMPLETED',
                     roomReplyReminderCount: 0,
@@ -131,20 +164,23 @@ export class TeamReconcilerService {
                     firstNudgeAt: null,
                 },
             });
-            await this.emitTeamRunInvalidated(invocation.teamRunId, ['agent-invocations', 'team-run'], 'agent-invocation-updated');
-            this.clearReminderTimer(invocation.id);
-            await this.afterInvocationTerminal(invocation.teamRunId, invocation.id);
-            return;
+            return completed.count === 1
+                ? { kind: 'terminal', teamRunId: invocation.teamRunId, invocationId: invocation.id }
+                : { kind: 'none' };
         }
         if (invocation.status === 'WAITING_ROOM_REPLY'
             && invocation.nextRoomReplyReminderAt
             && invocation.nextRoomReplyReminderAt.getTime() > this.now().getTime()) {
             this.scheduleReminderTimer(invocation.id, invocation.nextRoomReplyReminderAt);
-            return;
+            return { kind: 'none' };
         }
         if (invocation.roomReplyReminderCount >= this.maxRoomReplyReminders) {
-            await prisma.agentInvocation.update({
-                where: { id: invocation.id },
+            const failed = await prisma.agentInvocation.updateMany({
+                where: {
+                    id: invocation.id,
+                    status: { in: ACTIVE_INVOCATION_STATUSES },
+                    dispatchRevokedAt: null,
+                },
                 data: {
                     status: 'FAILED',
                     roomReplyReminderCount: 0,
@@ -152,31 +188,38 @@ export class TeamReconcilerService {
                     firstNudgeAt: null,
                 },
             });
-            await this.emitTeamRunInvalidated(invocation.teamRunId, ['agent-invocations', 'team-run'], 'agent-invocation-updated');
-            this.clearReminderTimer(invocation.id);
-            await this.afterInvocationTerminal(invocation.teamRunId, invocation.id);
-            return;
+            return failed.count === 1
+                ? { kind: 'terminal', teamRunId: invocation.teamRunId, invocationId: invocation.id }
+                : { kind: 'none' };
         }
         const nextReminderCount = invocation.roomReplyReminderCount + 1;
         const nextReminderAt = this.addDelay(this.now(), this.getReminderDelayMs(nextReminderCount));
-        await prisma.agentInvocation.update({
-            where: { id: invocation.id },
+        const claimed = await prisma.agentInvocation.updateMany({
+            where: {
+                id: invocation.id,
+                status: { in: ACTIVE_INVOCATION_STATUSES },
+                dispatchRevokedAt: null,
+            },
             data: {
                 status: 'WAITING_ROOM_REPLY',
                 roomReplyReminderCount: nextReminderCount,
                 nextRoomReplyReminderAt: nextReminderAt,
             },
         });
+        if (claimed.count !== 1)
+            return { kind: 'none' };
         await this.emitTeamRunInvalidated(invocation.teamRunId, ['agent-invocations', 'team-run'], 'agent-invocation-updated');
         this.scheduleReminderTimer(invocation.id, nextReminderAt);
         if (invocation.sessionId) {
-            await this.sendRoomReplyReminder(invocation.sessionId);
+            await this.sendRoomReplyReminder(invocation.sessionId, invocation.id);
         }
+        return { kind: 'none' };
     }
     async reconcileDueRoomReplyReminders(limit = 50) {
         const dueInvocations = await prisma.agentInvocation.findMany({
             where: {
                 status: 'WAITING_ROOM_REPLY',
+                dispatchRevokedAt: null,
                 nextRoomReplyReminderAt: { lte: this.now() },
             },
             select: { id: true },
@@ -242,13 +285,35 @@ export class TeamReconcilerService {
      */
     async reconcileStalledInvocations() {
         const candidates = await prisma.agentInvocation.findMany({
-            where: { status: 'RUNNING', sessionId: { not: null } },
+            where: { status: 'RUNNING', dispatchRevokedAt: null, sessionId: { not: null } },
             include: { teamRun: { select: { heartbeatTimeoutMinutes: true, task: { select: { deletedAt: true } } } } },
         });
         const now = this.now();
         for (const invocation of candidates) {
             try {
-                await this.reconcileStalledCandidate(invocation, now);
+                let action = { kind: 'none' };
+                const releaseAdmission = await acquireTeamMemberAdmission(invocation.teamRunId, invocation.memberId);
+                try {
+                    const current = await prisma.agentInvocation.findUnique({
+                        where: { id: invocation.id },
+                        include: { teamRun: { select: { heartbeatTimeoutMinutes: true, task: { select: { deletedAt: true } } } } },
+                    });
+                    if (current) {
+                        action = await this.reconcileStalledCandidate(current, now);
+                    }
+                }
+                finally {
+                    releaseAdmission();
+                }
+                if (action.kind === 'nudge') {
+                    const nudgeSent = await this.sendHeartbeatNudge(action.sessionId, action.invocationId);
+                    if (!nudgeSent && this.isSessionPipelineMissing(action.sessionId)) {
+                        action = await this.claimStalledTerminal(action.invocationId, action.sessionId, now);
+                    }
+                }
+                if (action.kind === 'terminal') {
+                    await this.finishStalledTerminal(action);
+                }
             }
             catch (error) {
                 this.logCandidateFailure('stalled invocation', invocation.id, error);
@@ -261,13 +326,15 @@ export class TeamReconcilerService {
      */
     async reconcileOrphanInvocations() {
         const candidates = await prisma.agentInvocation.findMany({
-            where: { status: 'RUNNING', sessionId: { not: null } },
+            where: { status: 'RUNNING', dispatchRevokedAt: null, sessionId: { not: null } },
             include: { teamRun: { select: { heartbeatTimeoutMinutes: true, task: { select: { deletedAt: true } } } } },
         });
         const failures = [];
         for (const invocation of candidates) {
             try {
-                await this.reconcileOrphanCandidate(invocation);
+                const action = await this.reconcileOrphanCandidate(invocation);
+                if (action?.kind === 'terminal')
+                    await this.finishStalledTerminal(action);
             }
             catch (error) {
                 this.logCandidateFailure('orphan invocation', invocation.id, error);
@@ -288,6 +355,9 @@ export class TeamReconcilerService {
         const failures = [];
         for (const invocation of candidates) {
             try {
+                if (invocation.sessionId
+                    && !await this.isSessionRuntimeCleanupConfirmed(invocation.sessionId))
+                    continue;
                 await this.afterInvocationTerminal(invocation.teamRunId, invocation.id);
             }
             catch (error) {
@@ -299,6 +369,235 @@ export class TeamReconcilerService {
             throw new AggregateError(failures, `Failed to reconcile ${failures.length} incomplete terminal invocation candidate(s)`);
         }
     }
+    async reconcilePendingRuntimeCleanup(limit = 50) {
+        const now = this.now();
+        const quarantineCount = await reportDueRuntimeCleanupQuarantines(limit, now);
+        const retryable = this.sessionMessenger?.retryRuntimeProcessCleanup
+            ? await prisma.executionProcess.findMany({
+                where: {
+                    runtimeInstanceId: { not: null },
+                    ownershipToken: { not: null },
+                    cleanupState: { in: ['PENDING', 'FAILED'] },
+                    OR: [
+                        { nextCleanupRetryAt: null },
+                        { nextCleanupRetryAt: { lte: now } },
+                    ],
+                },
+                select: {
+                    sessionId: true,
+                    runtimeInstanceId: true,
+                    launchClaimNumber: true,
+                    pid: true,
+                    processGroupId: true,
+                    birthMarker: true,
+                    ownershipToken: true,
+                    cleanupState: true,
+                },
+                orderBy: [{ nextCleanupRetryAt: 'asc' }, { createdAt: 'asc' }, { id: 'asc' }],
+                take: limit,
+            })
+            : [];
+        // ACTIVE belongs to a live runtime in the current server generation. An
+        // ACTIVE row without an in-memory owner is from an abrupt prior shutdown
+        // and must enter the same persisted recovery path.
+        const staleActive = this.sessionMessenger?.retryRuntimeProcessCleanup && retryable.length < limit
+            ? (await prisma.executionProcess.findMany({
+                where: {
+                    runtimeInstanceId: { not: null },
+                    ownershipToken: { not: null },
+                    cleanupState: 'ACTIVE',
+                },
+                select: {
+                    sessionId: true,
+                    runtimeInstanceId: true,
+                    launchClaimNumber: true,
+                    pid: true,
+                    processGroupId: true,
+                    birthMarker: true,
+                    ownershipToken: true,
+                    cleanupState: true,
+                },
+                orderBy: [{ createdAt: 'asc' }, { id: 'asc' }],
+                take: limit - retryable.length,
+            })).filter((candidate) => (candidate.runtimeInstanceId
+                && Number.isInteger(candidate.launchClaimNumber)
+                && candidate.launchClaimNumber > 0
+                && this.sessionMessenger?.hasRuntimeProcessOwner != null
+                && !this.sessionMessenger.hasRuntimeProcessOwner(candidate.runtimeInstanceId, candidate.sessionId, candidate.launchClaimNumber)))
+            : [];
+        const candidates = [...retryable, ...staleActive];
+        const sessionMessenger = this.sessionMessenger;
+        const retryRuntimeProcessCleanup = sessionMessenger?.retryRuntimeProcessCleanup;
+        for (const candidate of candidates) {
+            if (!candidate.runtimeInstanceId
+                || !candidate.ownershipToken
+                || candidate.pid == null
+                || !Number.isInteger(candidate.launchClaimNumber)
+                || candidate.launchClaimNumber <= 0)
+                continue;
+            try {
+                if (!retryRuntimeProcessCleanup)
+                    continue;
+                // Keep the messenger as the receiver. SessionManager's cleanup method
+                // updates its own durable state through `this`, so invoking a detached
+                // function here would turn every restart recovery into a TypeError.
+                await sessionMessenger.retryRuntimeProcessCleanup({
+                    sessionId: candidate.sessionId,
+                    runtimeInstanceId: candidate.runtimeInstanceId,
+                    launchClaimNumber: candidate.launchClaimNumber,
+                    pid: candidate.pid,
+                    processGroupId: candidate.processGroupId,
+                    birthMarker: candidate.birthMarker,
+                    ownershipToken: candidate.ownershipToken,
+                });
+                const invocation = await prisma.agentInvocation.findFirst({
+                    where: { sessionId: candidate.sessionId },
+                    select: {
+                        id: true,
+                        teamRunId: true,
+                        status: true,
+                        dispatchRevokedAt: true,
+                    },
+                });
+                if (!invocation)
+                    continue;
+                if (invocation.dispatchRevokedAt
+                    && ACTIVE_INVOCATION_STATUSES.includes(invocation.status)) {
+                    const process = await prisma.executionProcess.findFirst({
+                        where: {
+                            sessionId: candidate.sessionId,
+                            runtimeInstanceId: candidate.runtimeInstanceId,
+                            launchClaimNumber: candidate.launchClaimNumber,
+                        },
+                        select: { cleanupState: true },
+                    });
+                    if (process?.cleanupState === 'CONFIRMED'
+                        && await this.canTerminalizeRevokedInvocation(invocation.id, candidate.sessionId)) {
+                        await this.terminalizeRevokedInvocation(invocation.id);
+                    }
+                    else {
+                        await this.sessionMessenger?.stop?.(candidate.sessionId, { skipTeamRunReconcile: true });
+                        if (await this.canTerminalizeRevokedInvocation(invocation.id, candidate.sessionId)) {
+                            await this.terminalizeRevokedInvocation(invocation.id);
+                        }
+                    }
+                }
+                else if (TERMINAL_INVOCATION_STATUSES.includes(invocation.status)) {
+                    await this.afterInvocationTerminal(invocation.teamRunId, invocation.id, candidate.runtimeInstanceId);
+                }
+            }
+            catch (error) {
+                this.logCandidateFailure('runtime cleanup', candidate.runtimeInstanceId, error);
+            }
+        }
+        // A tree can be confirmed immediately before a transient DB failure in
+        // terminalization. Such rows are no longer PENDING/FAILED, so scan
+        // confirmed ownership records for the revoked + active recovery case.
+        const confirmedCandidates = await this.findConfirmedRevokedCandidates(limit);
+        for (const candidate of confirmedCandidates) {
+            try {
+                if (await this.canTerminalizeRevokedInvocation(candidate.invocationId, candidate.sessionId)) {
+                    await this.terminalizeRevokedInvocation(candidate.invocationId);
+                }
+            }
+            catch (error) {
+                this.logCandidateFailure('revoked terminalization', candidate.invocationId, error);
+            }
+        }
+        return candidates.length + quarantineCount;
+    }
+    async findConfirmedRevokedCandidates(limit) {
+        const results = [];
+        const pageSize = Math.max(50, limit);
+        let cursor;
+        while (results.length < limit) {
+            const invocations = await prisma.agentInvocation.findMany({
+                where: {
+                    dispatchRevokedAt: { not: null },
+                    status: { in: ACTIVE_INVOCATION_STATUSES },
+                    sessionId: { not: null },
+                },
+                select: { id: true, sessionId: true },
+                orderBy: { id: 'asc' },
+                take: pageSize,
+                ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}),
+            });
+            for (const invocation of invocations) {
+                if (invocation.sessionId
+                    && await this.canTerminalizeRevokedInvocation(invocation.id, invocation.sessionId)) {
+                    results.push({ invocationId: invocation.id, sessionId: invocation.sessionId });
+                    if (results.length >= limit)
+                        break;
+                }
+            }
+            if (invocations.length < pageSize)
+                break;
+            cursor = invocations[invocations.length - 1].id;
+        }
+        return results;
+    }
+    async canTerminalizeRevokedInvocation(invocationId, sessionId) {
+        return this.canTerminalizeAfterRuntimeCleanup(invocationId, sessionId, true);
+    }
+    /** All terminal paths share the durable launch + ownership cleanup gate. */
+    async canTerminalizeAfterRuntimeCleanup(invocationId, sessionId, requireDispatchRevoked = false) {
+        if (!await this.isSessionRuntimeCleanupConfirmed(sessionId))
+            return false;
+        const invocation = await prisma.agentInvocation.findUnique({
+            where: { id: invocationId },
+            select: { status: true, dispatchRevokedAt: true, sessionId: true },
+        });
+        return invocation?.sessionId === sessionId
+            && (!requireDispatchRevoked || invocation.dispatchRevokedAt != null)
+            && ACTIVE_INVOCATION_STATUSES.includes(invocation.status);
+    }
+    async terminalizeRevokedInvocation(invocationId) {
+        const candidate = await prisma.agentInvocation.findUnique({
+            where: { id: invocationId },
+            select: { teamRunId: true, memberId: true, sessionId: true },
+        });
+        if (!candidate?.sessionId)
+            return [];
+        if (!await this.canTerminalizeAfterRuntimeCleanup(invocationId, candidate.sessionId, true))
+            return [];
+        const releaseAdmission = await acquireTeamMemberAdmission(candidate.teamRunId, candidate.memberId);
+        let result = null;
+        try {
+            if (!await this.canTerminalizeAfterRuntimeCleanup(invocationId, candidate.sessionId, true))
+                return [];
+            result = await prisma.$transaction(async (tx) => {
+                const invocation = await tx.agentInvocation.findUnique({
+                    where: { id: invocationId },
+                    select: { id: true, teamRunId: true, sessionId: true, workRequestId: true, status: true, dispatchRevokedAt: true },
+                });
+                if (!invocation
+                    || !invocation.dispatchRevokedAt
+                    || !ACTIVE_INVOCATION_STATUSES.includes(invocation.status))
+                    return null;
+                const updated = await tx.agentInvocation.updateMany({
+                    where: { id: invocation.id, status: { in: ACTIVE_INVOCATION_STATUSES }, dispatchRevokedAt: { not: null } },
+                    data: { status: 'CANCELLED', nextRoomReplyReminderAt: null, roomReplyReminderCount: 0, firstNudgeAt: null },
+                });
+                if (updated.count !== 1)
+                    return null;
+                if (invocation.sessionId) {
+                    await tx.session.updateMany({
+                        where: { id: invocation.sessionId, status: { notIn: ['COMPLETED', 'FAILED', 'CANCELLED'] } },
+                        data: { status: 'CANCELLED' },
+                    });
+                }
+                return { teamRunId: invocation.teamRunId };
+            });
+        }
+        finally {
+            releaseAdmission();
+        }
+        if (!result)
+            return [];
+        this.clearReminderTimer(invocationId);
+        await this.emitTeamRunInvalidated(result.teamRunId, ['agent-invocations', 'team-run'], 'agent-invocation-updated');
+        return this.afterInvocationTerminal(result.teamRunId, invocationId);
+    }
     async findIncompleteTerminalInvocationCandidates() {
         const startedWorkRequests = await prisma.workRequest.findMany({
             where: { status: 'STARTED' },
@@ -314,7 +613,7 @@ export class TeamReconcilerService {
                     workRequestId: { in: workRequestIds },
                     status: { in: [...ACTIVE_INVOCATION_STATUSES, ...TERMINAL_INVOCATION_STATUSES] },
                 },
-                select: { id: true, teamRunId: true, workRequestId: true, status: true },
+                select: { id: true, teamRunId: true, sessionId: true, workRequestId: true, status: true },
                 orderBy: [{ createdAt: 'desc' }, { id: 'desc' }],
             });
             const activeWorkRequestIds = new Set();
@@ -337,21 +636,25 @@ export class TeamReconcilerService {
         return candidates;
     }
     async reconcileStalledCandidate(invocation, now) {
+        if (invocation.dispatchRevokedAt) {
+            return { kind: 'none' };
+        }
         if (isTaskDeleted(invocation.teamRun.task)) {
-            return;
+            return { kind: 'none' };
         }
         const sessionId = invocation.sessionId;
         if (!sessionId) {
-            return;
+            return { kind: 'none' };
         }
         // Permission waits are an active turn controlled by the user. They do not
         // count as agent silence and must not trigger an automated nudge.
         if (this.sessionMessenger?.isAwaitingPermission?.(sessionId)) {
-            return;
+            return { kind: 'none' };
         }
-        // 进程已不在内存管理中：交给首扫 orphan 或正常 exit 流程，避免运行期对刚退出瞬态的误判。
+        // 进程已不在内存管理中：仍需通过 persisted cleanup gate，避免把
+        // hasActiveTurn=false 误当成 owned process tree 已退出。
         if (!this.hasActiveTurn(sessionId)) {
-            return;
+            return this.claimStalledTerminalUnderAdmission(invocation.id, sessionId, now);
         }
         const lastActivity = invocation.lastHeartbeatAt ?? invocation.createdAt;
         const idleMs = now.getTime() - lastActivity.getTime();
@@ -360,88 +663,184 @@ export class TeamReconcilerService {
             ?? (configuredThresholdMs > 0 ? configuredThresholdMs : DEFAULT_HEARTBEAT_IDLE_THRESHOLD_MS);
         if (idleMs < heartbeatIdleThresholdMs) {
             if (invocation.roomReplyReminderCount > 0 || invocation.nextRoomReplyReminderAt) {
-                await prisma.agentInvocation.update({
-                    where: { id: invocation.id },
+                await prisma.agentInvocation.updateMany({
+                    where: { id: invocation.id, status: 'RUNNING', dispatchRevokedAt: null },
                     data: { roomReplyReminderCount: 0, nextRoomReplyReminderAt: null },
                 });
                 await this.emitTeamRunInvalidated(invocation.teamRunId, ['agent-invocations'], 'agent-invocation-updated');
             }
-            return;
+            return { kind: 'none' };
         }
         if (invocation.firstNudgeAt && now.getTime() - invocation.firstNudgeAt.getTime() > this.absoluteNudgeBudgetMs) {
-            await this.releaseStalledInvocation(invocation);
-            return;
+            const claimed = await this.claimStalledRelease(invocation.id, now);
+            return claimed
+                ? { kind: 'terminal', teamRunId: invocation.teamRunId, memberId: invocation.memberId, invocationId: invocation.id, sessionId }
+                : { kind: 'none' };
         }
         if (invocation.roomReplyReminderCount >= this.maxRoomReplyReminders) {
-            await this.releaseStalledInvocation(invocation);
-            return;
+            const claimed = await this.claimStalledRelease(invocation.id, now);
+            return claimed
+                ? { kind: 'terminal', teamRunId: invocation.teamRunId, memberId: invocation.memberId, invocationId: invocation.id, sessionId }
+                : { kind: 'none' };
         }
         if (invocation.nextRoomReplyReminderAt && invocation.nextRoomReplyReminderAt.getTime() > now.getTime()) {
-            return;
+            return { kind: 'none' };
         }
         const nextCount = invocation.roomReplyReminderCount + 1;
         const nextAt = this.addDelay(now, this.getReminderDelayMs(nextCount));
-        await prisma.agentInvocation.update({
-            where: { id: invocation.id },
+        const claimed = await prisma.agentInvocation.updateMany({
+            where: { id: invocation.id, status: 'RUNNING', dispatchRevokedAt: null },
             data: {
                 roomReplyReminderCount: nextCount,
                 nextRoomReplyReminderAt: nextAt,
                 firstNudgeAt: invocation.firstNudgeAt ?? now,
             },
         });
+        if (claimed.count !== 1)
+            return { kind: 'none' };
         await this.emitTeamRunInvalidated(invocation.teamRunId, ['agent-invocations'], 'agent-invocation-updated');
-        const nudgeSent = await this.sendHeartbeatNudge(sessionId);
-        if (!nudgeSent && this.isSessionPipelineMissing(sessionId)) {
-            const current = await prisma.agentInvocation.findUnique({
-                where: { id: invocation.id },
-                select: { id: true, teamRunId: true, sessionId: true, status: true },
-            });
-            if (current?.status === 'RUNNING') {
-                await this.releaseStalledInvocation(current);
-            }
-        }
+        return { kind: 'nudge', sessionId, invocationId: invocation.id };
+    }
+    async claimStalledRelease(invocationId, now) {
+        const claimed = await prisma.agentInvocation.updateMany({
+            where: { id: invocationId, status: 'RUNNING', dispatchRevokedAt: null },
+            data: { dispatchRevokedAt: now, nextRoomReplyReminderAt: null },
+        });
+        return claimed.count === 1;
     }
     async reconcileOrphanCandidate(invocation) {
         if (isTaskDeleted(invocation.teamRun.task) || !invocation.sessionId) {
-            return;
+            return { kind: 'none' };
         }
         const alive = this.hasActiveTurn(invocation.sessionId);
         if (!alive) {
-            await this.releaseStalledInvocation(invocation);
+            const releaseAdmission = await acquireTeamMemberAdmission(invocation.teamRunId, invocation.memberId);
+            try {
+                return this.claimStalledTerminalUnderAdmission(invocation.id, invocation.sessionId, this.now());
+            }
+            finally {
+                releaseAdmission();
+            }
         }
+        return { kind: 'none' };
     }
     logCandidateFailure(kind, invocationId, error) {
         console.warn(`[TeamReconcilerService] Failed to reconcile ${kind} ${invocationId}:`, error instanceof Error ? error.message : error);
     }
-    async releaseStalledInvocation(invocation) {
-        const sessionId = invocation.sessionId;
-        const alive = sessionId ? !this.isSessionPipelineMissing(sessionId) : false;
-        // 进程仍存活：走 stop，由 handleSessionStopped 置 CANCELLED 并触发 afterInvocationTerminal 闭环。
-        if (sessionId && alive && this.sessionMessenger?.stop) {
-            this.clearReminderTimer(invocation.id);
-            await this.sessionMessenger.stop(sessionId);
-            return;
+    async claimStalledTerminal(invocationId, sessionId, now) {
+        const candidate = await prisma.agentInvocation.findUnique({
+            where: { id: invocationId },
+            select: { teamRunId: true, memberId: true },
+        });
+        if (!candidate)
+            return { kind: 'none' };
+        const releaseAdmission = await acquireTeamMemberAdmission(candidate.teamRunId, candidate.memberId);
+        try {
+            return await this.claimStalledTerminalUnderAdmission(invocationId, sessionId, now);
         }
-        // 无存活进程（已退出 / orphan）：直接置 FAILED 并手动走释放闭环。
-        await prisma.agentInvocation.update({
-            where: { id: invocation.id },
+        finally {
+            releaseAdmission();
+        }
+    }
+    async claimStalledTerminalUnderAdmission(invocationId, sessionId, now) {
+        const current = await prisma.agentInvocation.findUnique({
+            where: { id: invocationId },
+            select: { id: true, teamRunId: true, memberId: true, sessionId: true, status: true, dispatchRevokedAt: true },
+        });
+        if (!current || current.status !== 'RUNNING' || current.dispatchRevokedAt)
+            return { kind: 'none' };
+        if (!await this.canTerminalizeAfterRuntimeCleanup(invocationId, sessionId)) {
+            await prisma.agentInvocation.updateMany({
+                where: { id: invocationId, status: 'RUNNING', dispatchRevokedAt: null },
+                data: { dispatchRevokedAt: now, nextRoomReplyReminderAt: null },
+            });
+            return { kind: 'none' };
+        }
+        const failed = await prisma.agentInvocation.updateMany({
+            where: { id: invocationId, status: 'RUNNING', dispatchRevokedAt: null },
             data: {
                 status: 'FAILED',
-                roomReplyReminderCount: 0,
                 nextRoomReplyReminderAt: null,
+                roomReplyReminderCount: 0,
                 firstNudgeAt: null,
             },
         });
-        await this.emitTeamRunInvalidated(invocation.teamRunId, ['agent-invocations', 'team-run'], 'agent-invocation-updated');
-        this.clearReminderTimer(invocation.id);
-        await this.afterInvocationTerminal(invocation.teamRunId, invocation.id);
+        if (failed.count !== 1)
+            return { kind: 'none' };
+        await prisma.session.updateMany({
+            where: { id: sessionId, status: { notIn: ['COMPLETED', 'FAILED', 'CANCELLED'] } },
+            data: { status: 'FAILED' },
+        });
+        return { kind: 'terminal', teamRunId: current.teamRunId, memberId: current.memberId, invocationId, sessionId };
     }
-    async sendHeartbeatNudge(sessionId) {
+    async finishStalledTerminal(action) {
+        this.clearReminderTimer(action.invocationId);
+        const before = await prisma.agentInvocation.findUnique({
+            where: { id: action.invocationId },
+            select: { status: true, dispatchRevokedAt: true },
+        });
+        if (!before)
+            return;
+        if (ACTIVE_INVOCATION_STATUSES.includes(before.status)) {
+            if (action.sessionId && this.hasActiveTurn(action.sessionId) && this.sessionMessenger?.stop) {
+                await this.sessionMessenger.stop(action.sessionId, { skipTeamRunReconcile: true });
+            }
+            if (action.sessionId
+                && !await this.canTerminalizeAfterRuntimeCleanup(action.invocationId, action.sessionId, true)) {
+                return;
+            }
+        }
+        const releaseAdmission = await acquireTeamMemberAdmission(action.teamRunId, action.memberId);
+        let cancelled = false;
+        try {
+            const current = await prisma.agentInvocation.findUnique({
+                where: { id: action.invocationId },
+                select: { teamRunId: true, memberId: true, status: true, dispatchRevokedAt: true },
+            });
+            if (current?.status === 'FAILED') {
+                cancelled = true;
+            }
+            else if (current && current.dispatchRevokedAt && ACTIVE_INVOCATION_STATUSES.includes(current.status)) {
+                const updated = await prisma.agentInvocation.updateMany({
+                    where: { id: action.invocationId, status: { in: ACTIVE_INVOCATION_STATUSES }, dispatchRevokedAt: { not: null } },
+                    data: {
+                        status: 'CANCELLED',
+                        nextRoomReplyReminderAt: null,
+                        roomReplyReminderCount: 0,
+                        firstNudgeAt: null,
+                    },
+                });
+                cancelled = updated.count === 1;
+                if (cancelled && action.sessionId) {
+                    await prisma.session.updateMany({
+                        where: { id: action.sessionId, status: { notIn: ['COMPLETED', 'FAILED', 'CANCELLED'] } },
+                        data: { status: 'CANCELLED' },
+                    });
+                }
+            }
+        }
+        finally {
+            releaseAdmission();
+        }
+        if (!cancelled)
+            return;
+        await this.emitTeamRunInvalidated(action.teamRunId, ['agent-invocations', 'team-run'], 'agent-invocation-updated');
+        await this.afterInvocationTerminal(action.teamRunId, action.invocationId);
+    }
+    async sendHeartbeatNudge(sessionId, invocationId) {
         if (!this.sessionMessenger) {
             return false;
         }
         try {
-            await this.sessionMessenger.sendMessage(sessionId, TEAM_HEARTBEAT_NUDGE);
+            if (invocationId) {
+                const admitted = await prisma.agentInvocation.findFirst({
+                    where: { id: invocationId, sessionId, status: 'RUNNING', dispatchRevokedAt: null },
+                    select: { id: true },
+                });
+                if (!admitted)
+                    return false;
+            }
+            await this.sessionMessenger.sendMessage(sessionId, TEAM_HEARTBEAT_NUDGE, undefined, invocationId);
             return true;
         }
         catch (error) {
@@ -470,7 +869,6 @@ export class TeamReconcilerService {
         if (!teamRun) {
             return false;
         }
-        await this.releaseTerminalInvocationLocks(teamRun.invocations);
         if (teamRun.task.deletedAt) {
             return false;
         }
@@ -535,17 +933,43 @@ export class TeamReconcilerService {
             reason,
         }, this.eventBus);
     }
-    async afterInvocationTerminal(teamRunId, invocationId) {
+    async afterInvocationTerminal(teamRunId, invocationId, expectedRuntimeInstanceId) {
+        const terminalInvocation = await prisma.agentInvocation.findUnique({
+            where: { id: invocationId },
+            select: {
+                sessionId: true,
+                status: true,
+                teamRun: { select: { task: { select: { deletedAt: true } } } },
+            },
+        });
+        if (!terminalInvocation || !TERMINAL_INVOCATION_STATUSES.includes(terminalInvocation.status))
+            return [];
+        if (terminalInvocation.sessionId) {
+            // TeamRun sessions are one invocation per Tower Session. Dispose only
+            // after the invocation is genuinely terminal; WAITING_ROOM_REPLY keeps
+            // the runtime open for the legal reminder/follow-up path.
+            await this.sessionMessenger?.disposeRuntimeSession?.(terminalInvocation.sessionId, expectedRuntimeInstanceId);
+            if (!await this.isSessionRuntimeCleanupConfirmed(terminalInvocation.sessionId))
+                return [];
+        }
+        // WorkRequest STARTED -> terminal is the unique durable winner for all
+        // after-terminal side effects. Repeated heartbeat/startup reconciliation is
+        // therefore idempotent once cleanup has been confirmed.
+        if (!await this.syncTerminalWorkRequest(invocationId))
+            return [];
         const scheduler = await this.getScheduler();
         scheduler.releaseInvocationLocks(invocationId);
-        await this.syncTerminalWorkRequest(invocationId);
+        if (isTaskDeleted(terminalInvocation.teamRun.task))
+            return [];
+        let startedInvocations = [];
         try {
-            await scheduler.startNextSessions(teamRunId);
+            startedInvocations = await scheduler.startNextSessions(teamRunId);
         }
         catch (error) {
             console.warn(`[TeamReconcilerService] Failed to start queued TeamRun work for ${teamRunId}:`, error instanceof Error ? error.message : error);
         }
         await this.maybeAdvanceTeamRunToReview(teamRunId);
+        return startedInvocations;
     }
     async syncTerminalWorkRequest(invocationId) {
         const invocation = await prisma.agentInvocation.findUnique({
@@ -553,19 +977,20 @@ export class TeamReconcilerService {
             select: { status: true, workRequestId: true },
         });
         if (!invocation) {
-            return;
+            return false;
         }
         const terminalWorkRequestStatus = this.toTerminalWorkRequestStatus(invocation.status);
         if (!terminalWorkRequestStatus) {
-            return;
+            return false;
         }
-        await prisma.workRequest.updateMany({
+        const updated = await prisma.workRequest.updateMany({
             where: {
                 id: invocation.workRequestId,
                 status: 'STARTED',
             },
             data: { status: terminalWorkRequestStatus },
         });
+        return updated.count === 1;
     }
     toTerminalWorkRequestStatus(status) {
         if (status === 'COMPLETED' || status === 'FAILED' || status === 'CANCELLED') {
@@ -573,21 +998,13 @@ export class TeamReconcilerService {
         }
         return null;
     }
-    async releaseTerminalInvocationLocks(invocations) {
-        const scheduler = await this.getScheduler();
-        for (const invocation of invocations) {
-            if (TERMINAL_INVOCATION_STATUSES.includes(invocation.status)) {
-                scheduler.releaseInvocationLocks(invocation.id);
-            }
-        }
-    }
-    async sendRoomReplyReminder(sessionId) {
+    async sendRoomReplyReminder(sessionId, invocationId) {
         if (!this.sessionMessenger) {
             console.warn(`[TeamReconcilerService] No session messenger configured for room reply reminder: ${sessionId}`);
             return;
         }
         try {
-            await this.sessionMessenger.sendMessage(sessionId, TEAM_ROOM_REPLY_REMINDER);
+            await this.sessionMessenger.sendMessage(sessionId, TEAM_ROOM_REPLY_REMINDER, undefined, invocationId);
         }
         catch (error) {
             console.warn(`[TeamReconcilerService] Failed to send room reply reminder to session ${sessionId}:`, error instanceof Error ? error.message : error);