agent-tower 0.6.0-beta.11 → 0.6.0-beta.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/dist/app.d.ts.map +1 -1
  2. package/dist/app.js +4 -0
  3. package/dist/app.js.map +1 -1
  4. package/dist/cli.js +39 -19
  5. package/dist/cli.js.map +1 -1
  6. package/dist/executors/__tests__/base.executor.test.js +133 -1
  7. package/dist/executors/__tests__/base.executor.test.js.map +1 -1
  8. package/dist/executors/__tests__/stdin-prompt.executors.test.js +18 -0
  9. package/dist/executors/__tests__/stdin-prompt.executors.test.js.map +1 -1
  10. package/dist/executors/base.executor.d.ts +8 -0
  11. package/dist/executors/base.executor.d.ts.map +1 -1
  12. package/dist/executors/base.executor.js +336 -56
  13. package/dist/executors/base.executor.js.map +1 -1
  14. package/dist/executors/index.d.ts +2 -1
  15. package/dist/executors/index.d.ts.map +1 -1
  16. package/dist/executors/index.js +1 -1
  17. package/dist/executors/index.js.map +1 -1
  18. package/dist/executors/start-error.d.ts +17 -0
  19. package/dist/executors/start-error.d.ts.map +1 -1
  20. package/dist/executors/start-error.js +36 -1
  21. package/dist/executors/start-error.js.map +1 -1
  22. package/dist/index.js +39 -29
  23. package/dist/index.js.map +1 -1
  24. package/dist/mcp/http-client.test.js +20 -0
  25. package/dist/mcp/http-client.test.js.map +1 -1
  26. package/dist/mcp/tools/__tests__/sessions.test.d.ts +2 -0
  27. package/dist/mcp/tools/__tests__/sessions.test.d.ts.map +1 -0
  28. package/dist/mcp/tools/__tests__/sessions.test.js +33 -0
  29. package/dist/mcp/tools/__tests__/sessions.test.js.map +1 -0
  30. package/dist/routes/__tests__/sessions-runtime.test.js +75 -3
  31. package/dist/routes/__tests__/sessions-runtime.test.js.map +1 -1
  32. package/dist/routes/conversations.d.ts.map +1 -1
  33. package/dist/routes/conversations.js +5 -1
  34. package/dist/routes/conversations.js.map +1 -1
  35. package/dist/routes/sessions.d.ts.map +1 -1
  36. package/dist/routes/sessions.js +13 -2
  37. package/dist/routes/sessions.js.map +1 -1
  38. package/dist/routes/team-runs.d.ts.map +1 -1
  39. package/dist/routes/team-runs.js +13 -1
  40. package/dist/routes/team-runs.js.map +1 -1
  41. package/dist/runtime/__tests__/acp-driver-lifecycle.test.js +86 -0
  42. package/dist/runtime/__tests__/acp-driver-lifecycle.test.js.map +1 -1
  43. package/dist/runtime/__tests__/acp-process-manager.test.js +393 -6
  44. package/dist/runtime/__tests__/acp-process-manager.test.js.map +1 -1
  45. package/dist/runtime/__tests__/cli-driver-lifecycle.test.d.ts +2 -0
  46. package/dist/runtime/__tests__/cli-driver-lifecycle.test.d.ts.map +1 -0
  47. package/dist/runtime/__tests__/cli-driver-lifecycle.test.js +244 -0
  48. package/dist/runtime/__tests__/cli-driver-lifecycle.test.js.map +1 -0
  49. package/dist/runtime/__tests__/launch-cleanup-registry.test.d.ts +2 -0
  50. package/dist/runtime/__tests__/launch-cleanup-registry.test.d.ts.map +1 -0
  51. package/dist/runtime/__tests__/launch-cleanup-registry.test.js +154 -0
  52. package/dist/runtime/__tests__/launch-cleanup-registry.test.js.map +1 -0
  53. package/dist/runtime/__tests__/runtime-coordinator.test.js +117 -1
  54. package/dist/runtime/__tests__/runtime-coordinator.test.js.map +1 -1
  55. package/dist/runtime/__tests__/server-entry-shutdown.test.d.ts +2 -0
  56. package/dist/runtime/__tests__/server-entry-shutdown.test.d.ts.map +1 -0
  57. package/dist/runtime/__tests__/server-entry-shutdown.test.js +38 -0
  58. package/dist/runtime/__tests__/server-entry-shutdown.test.js.map +1 -0
  59. package/dist/runtime/__tests__/shutdown-coordinator.test.d.ts +2 -0
  60. package/dist/runtime/__tests__/shutdown-coordinator.test.d.ts.map +1 -0
  61. package/dist/runtime/__tests__/shutdown-coordinator.test.js +28 -0
  62. package/dist/runtime/__tests__/shutdown-coordinator.test.js.map +1 -0
  63. package/dist/runtime/acp/acp-driver.d.ts.map +1 -1
  64. package/dist/runtime/acp/acp-driver.js +108 -35
  65. package/dist/runtime/acp/acp-driver.js.map +1 -1
  66. package/dist/runtime/acp/agents/managed-directory.d.ts.map +1 -1
  67. package/dist/runtime/acp/agents/managed-directory.js +3 -1
  68. package/dist/runtime/acp/agents/managed-directory.js.map +1 -1
  69. package/dist/runtime/acp/agents/managed-directory.test.d.ts +2 -0
  70. package/dist/runtime/acp/agents/managed-directory.test.d.ts.map +1 -0
  71. package/dist/runtime/acp/agents/managed-directory.test.js +25 -0
  72. package/dist/runtime/acp/agents/managed-directory.test.js.map +1 -0
  73. package/dist/runtime/acp/launch-cleanup-registry.d.ts +32 -0
  74. package/dist/runtime/acp/launch-cleanup-registry.d.ts.map +1 -0
  75. package/dist/runtime/acp/launch-cleanup-registry.js +176 -0
  76. package/dist/runtime/acp/launch-cleanup-registry.js.map +1 -0
  77. package/dist/runtime/acp/process-manager.d.ts +58 -0
  78. package/dist/runtime/acp/process-manager.d.ts.map +1 -1
  79. package/dist/runtime/acp/process-manager.js +489 -33
  80. package/dist/runtime/acp/process-manager.js.map +1 -1
  81. package/dist/runtime/cli-driver.d.ts.map +1 -1
  82. package/dist/runtime/cli-driver.js +230 -46
  83. package/dist/runtime/cli-driver.js.map +1 -1
  84. package/dist/runtime/contracts.d.ts +21 -2
  85. package/dist/runtime/contracts.d.ts.map +1 -1
  86. package/dist/runtime/index.d.ts +1 -0
  87. package/dist/runtime/index.d.ts.map +1 -1
  88. package/dist/runtime/index.js +1 -0
  89. package/dist/runtime/index.js.map +1 -1
  90. package/dist/runtime/runtime-coordinator.d.ts +8 -1
  91. package/dist/runtime/runtime-coordinator.d.ts.map +1 -1
  92. package/dist/runtime/runtime-coordinator.js +118 -18
  93. package/dist/runtime/runtime-coordinator.js.map +1 -1
  94. package/dist/runtime/server-entry-shutdown.d.ts +12 -0
  95. package/dist/runtime/server-entry-shutdown.d.ts.map +1 -0
  96. package/dist/runtime/server-entry-shutdown.js +23 -0
  97. package/dist/runtime/server-entry-shutdown.js.map +1 -0
  98. package/dist/runtime/shutdown-coordinator.d.ts +17 -0
  99. package/dist/runtime/shutdown-coordinator.d.ts.map +1 -0
  100. package/dist/runtime/shutdown-coordinator.js +67 -0
  101. package/dist/runtime/shutdown-coordinator.js.map +1 -0
  102. package/dist/services/__tests__/conversation.service.test.js +160 -0
  103. package/dist/services/__tests__/conversation.service.test.js.map +1 -1
  104. package/dist/services/__tests__/database-maintenance.service.test.js +97 -0
  105. package/dist/services/__tests__/database-maintenance.service.test.js.map +1 -1
  106. package/dist/services/__tests__/member-heartbeat-scheduler.test.js +6 -0
  107. package/dist/services/__tests__/member-heartbeat-scheduler.test.js.map +1 -1
  108. package/dist/services/__tests__/member-heartbeat.test.js +7 -11
  109. package/dist/services/__tests__/member-heartbeat.test.js.map +1 -1
  110. package/dist/services/__tests__/session-manager.lifecycle.test.js +402 -12
  111. package/dist/services/__tests__/session-manager.lifecycle.test.js.map +1 -1
  112. package/dist/services/__tests__/session-manager.team-run.test.js +422 -29
  113. package/dist/services/__tests__/session-manager.team-run.test.js.map +1 -1
  114. package/dist/services/__tests__/team-queue-recovery.integration.test.js +1 -0
  115. package/dist/services/__tests__/team-queue-recovery.integration.test.js.map +1 -1
  116. package/dist/services/__tests__/team-reconciler.service.test.js +1282 -4
  117. package/dist/services/__tests__/team-reconciler.service.test.js.map +1 -1
  118. package/dist/services/__tests__/team-run.service.test.js +161 -1
  119. package/dist/services/__tests__/team-run.service.test.js.map +1 -1
  120. package/dist/services/__tests__/team-scheduler.service.test.js +334 -6
  121. package/dist/services/__tests__/team-scheduler.service.test.js.map +1 -1
  122. package/dist/services/conversation.service.d.ts.map +1 -1
  123. package/dist/services/conversation.service.js +21 -2
  124. package/dist/services/conversation.service.js.map +1 -1
  125. package/dist/services/database-maintenance.service.d.ts.map +1 -1
  126. package/dist/services/database-maintenance.service.js +68 -17
  127. package/dist/services/database-maintenance.service.js.map +1 -1
  128. package/dist/services/member-heartbeat-scheduler.d.ts +6 -5
  129. package/dist/services/member-heartbeat-scheduler.d.ts.map +1 -1
  130. package/dist/services/member-heartbeat-scheduler.js +7 -5
  131. package/dist/services/member-heartbeat-scheduler.js.map +1 -1
  132. package/dist/services/session-manager.d.ts +124 -14
  133. package/dist/services/session-manager.d.ts.map +1 -1
  134. package/dist/services/session-manager.js +1035 -116
  135. package/dist/services/session-manager.js.map +1 -1
  136. package/dist/services/session-runtime-cleanup-gate.d.ts +25 -0
  137. package/dist/services/session-runtime-cleanup-gate.d.ts.map +1 -0
  138. package/dist/services/session-runtime-cleanup-gate.js +284 -0
  139. package/dist/services/session-runtime-cleanup-gate.js.map +1 -0
  140. package/dist/services/task.service.d.ts +2 -2
  141. package/dist/services/team-member-admission-barrier.d.ts +3 -0
  142. package/dist/services/team-member-admission-barrier.d.ts.map +1 -0
  143. package/dist/services/team-member-admission-barrier.js +26 -0
  144. package/dist/services/team-member-admission-barrier.js.map +1 -0
  145. package/dist/services/team-reconciler.service.d.ts +30 -7
  146. package/dist/services/team-reconciler.service.d.ts.map +1 -1
  147. package/dist/services/team-reconciler.service.js +528 -111
  148. package/dist/services/team-reconciler.service.js.map +1 -1
  149. package/dist/services/team-run.service.d.ts.map +1 -1
  150. package/dist/services/team-run.service.js +37 -13
  151. package/dist/services/team-run.service.js.map +1 -1
  152. package/dist/services/team-scheduler.service.d.ts +8 -6
  153. package/dist/services/team-scheduler.service.d.ts.map +1 -1
  154. package/dist/services/team-scheduler.service.js +323 -120
  155. package/dist/services/team-scheduler.service.js.map +1 -1
  156. package/dist/services/workspace-background-process-manager.d.ts.map +1 -1
  157. package/dist/services/workspace-background-process-manager.js +2 -5
  158. package/dist/services/workspace-background-process-manager.js.map +1 -1
  159. package/dist/services/workspace.service.d.ts +15 -8
  160. package/dist/services/workspace.service.d.ts.map +1 -1
  161. package/dist/socket/index.d.ts.map +1 -1
  162. package/dist/socket/index.js +27 -14
  163. package/dist/socket/index.js.map +1 -1
  164. package/dist/utils/__tests__/unix-process-identity.test.d.ts +2 -0
  165. package/dist/utils/__tests__/unix-process-identity.test.d.ts.map +1 -0
  166. package/dist/utils/__tests__/unix-process-identity.test.js +73 -0
  167. package/dist/utils/__tests__/unix-process-identity.test.js.map +1 -0
  168. package/dist/utils/error-log.d.ts +2 -0
  169. package/dist/utils/error-log.d.ts.map +1 -1
  170. package/dist/utils/error-log.js +40 -3
  171. package/dist/utils/error-log.js.map +1 -1
  172. package/dist/utils/error-log.test.js +14 -2
  173. package/dist/utils/error-log.test.js.map +1 -1
  174. package/dist/utils/process-launch.d.ts +2 -2
  175. package/dist/utils/process-launch.d.ts.map +1 -1
  176. package/dist/utils/process-launch.js +339 -79
  177. package/dist/utils/process-launch.js.map +1 -1
  178. package/dist/utils/process-launch.test.js +168 -10
  179. package/dist/utils/process-launch.test.js.map +1 -1
  180. package/dist/utils/spawned-process-identity.d.ts +7 -0
  181. package/dist/utils/spawned-process-identity.d.ts.map +1 -0
  182. package/dist/utils/spawned-process-identity.js +36 -0
  183. package/dist/utils/spawned-process-identity.js.map +1 -0
  184. package/dist/utils/tree-cleanup-channel.d.ts +18 -0
  185. package/dist/utils/tree-cleanup-channel.d.ts.map +1 -0
  186. package/dist/utils/tree-cleanup-channel.js +16 -0
  187. package/dist/utils/tree-cleanup-channel.js.map +1 -0
  188. package/dist/utils/tree-cleanup-channel.test.d.ts +2 -0
  189. package/dist/utils/tree-cleanup-channel.test.d.ts.map +1 -0
  190. package/dist/utils/tree-cleanup-channel.test.js +15 -0
  191. package/dist/utils/tree-cleanup-channel.test.js.map +1 -0
  192. package/dist/utils/unix-process-identity.d.ts +13 -1
  193. package/dist/utils/unix-process-identity.d.ts.map +1 -1
  194. package/dist/utils/unix-process-identity.js +146 -25
  195. package/dist/utils/unix-process-identity.js.map +1 -1
  196. package/node_modules/@agent-tower/shared/dist/types.d.ts +1 -0
  197. package/node_modules/@agent-tower/shared/dist/types.d.ts.map +1 -1
  198. package/node_modules/@agent-tower/shared/dist/types.js.map +1 -1
  199. package/package.json +1 -1
  200. package/prisma/migrations/20260824000000_add_execution_process_ownership/migration.sql +10 -0
  201. package/prisma/migrations/20260824010000_add_runtime_cleanup_recovery/migration.sql +10 -0
  202. package/prisma/migrations/20260824020000_add_runtime_launch_admission/migration.sql +72 -0
  203. package/prisma/migrations/20260826000000_add_conversation_turn_queue/migration.sql +19 -0
  204. package/prisma/schema.prisma +50 -7
@@ -8,8 +8,10 @@ import { WorkspaceService } from './workspace.service.js';
8
8
  import { appendAttachmentMarkdownContext } from './attachment-context.js';
9
9
  import { emitTeamRunInvalidated } from './team-run-events.js';
10
10
  import { TeamReconcilerService } from './team-reconciler.service.js';
11
+ import { acquireTeamMemberAdmission } from './team-member-admission-barrier.js';
11
12
  import { ensureTaskNotDeleted, isTaskDeleted } from './deleted-task-guard.js';
12
13
  import { TEAM_ROOM_SYSTEM_SHARED_PROTOCOL } from '../prompts/team-room-system-shared-protocol.js';
14
+ import { evaluateSessionRuntimeCleanup } from './session-runtime-cleanup-gate.js';
13
15
  const ACTIVE_INVOCATION_STATUSES = [
14
16
  'QUEUED',
15
17
  'RUNNING',
@@ -22,6 +24,7 @@ const STOPPABLE_INVOCATION_STATUSES = [
22
24
  'SESSION_ENDED',
23
25
  'WAITING_ROOM_REPLY',
24
26
  ];
27
+ const TERMINAL_INVOCATION_STATUSES = ['COMPLETED', 'FAILED', 'CANCELLED'];
25
28
  const CANCELLABLE_QUEUED_WORK_REQUEST_STATUSES = [
26
29
  'PENDING_APPROVAL',
27
30
  'QUEUED',
@@ -168,7 +171,6 @@ function serializeTargetSyncStatus(value) {
168
171
  }
169
172
  export class TeamSchedulerService {
170
173
  lockService;
171
- static memberSchedulingLocks = new Set();
172
174
  static sharedWorkspaceClaims = new Map();
173
175
  workspaceService;
174
176
  sessionManager;
@@ -183,6 +185,7 @@ export class TeamSchedulerService {
183
185
  this.now = dependencies.now ?? (() => new Date());
184
186
  this.teamRunReviewAdvancer = dependencies.teamRunReviewAdvancer ?? new TeamReconcilerService({
185
187
  eventBus: getEventBus(),
188
+ sessionMessenger: this.sessionManager,
186
189
  scheduler: {
187
190
  releaseInvocationLocks: (invocationId) => this.releaseInvocationLocks(invocationId),
188
191
  startNextSessions: (nextTeamRunId) => this.startNextSessions(nextTeamRunId),
@@ -293,10 +296,7 @@ export class TeamSchedulerService {
293
296
  if (!member) {
294
297
  continue;
295
298
  }
296
- const memberLockKey = this.memberSchedulingLockKey(teamRunId, member.id);
297
- if (!this.acquireMemberSchedulingLock(memberLockKey)) {
298
- continue;
299
- }
299
+ const releaseMemberSchedulingLock = await acquireTeamMemberAdmission(teamRunId, member.id);
300
300
  let invocationId = null;
301
301
  try {
302
302
  if (await this.hasActiveInvocation(teamRunId, member.id)) {
@@ -332,7 +332,7 @@ export class TeamSchedulerService {
332
332
  throw error;
333
333
  }
334
334
  finally {
335
- this.releaseMemberSchedulingLock(memberLockKey);
335
+ releaseMemberSchedulingLock();
336
336
  }
337
337
  }
338
338
  return startedInvocations;
@@ -357,10 +357,7 @@ export class TeamSchedulerService {
357
357
  retryBlockedMemberIds.add(member.id);
358
358
  continue;
359
359
  }
360
- const memberLockKey = this.memberSchedulingLockKey(teamRunId, member.id);
361
- if (!this.acquireMemberSchedulingLock(memberLockKey)) {
362
- continue;
363
- }
360
+ const releaseMemberSchedulingLock = await acquireTeamMemberAdmission(teamRunId, member.id);
364
361
  let invocationId = null;
365
362
  try {
366
363
  if (await this.hasActiveInvocation(teamRunId, member.id)) {
@@ -457,6 +454,15 @@ export class TeamSchedulerService {
457
454
  invocationId = null;
458
455
  continue;
459
456
  }
457
+ // A stop/cancel can win after the initial queue claim but before the
458
+ // expensive provider/session spawn. Recheck the authoritative DB state
459
+ // immediately before crossing into the process-owning SessionManager.
460
+ if (!(await this.isInvocationAdmitted(createdInvocation.id, freshWorkRequest.id))) {
461
+ await this.cancelUnadmittedInvocation(createdInvocation.id, session.id);
462
+ this.lockService.releaseByOwner(invocationId);
463
+ invocationId = null;
464
+ continue;
465
+ }
460
466
  const resumeFromSessionId = await this.findResumeSourceSessionId(member, session.id, workspace.id, target?.targetHeadSha ?? null);
461
467
  try {
462
468
  if (resumeFromSessionId && this.sessionManager.startFollowUp) {
@@ -467,12 +473,19 @@ export class TeamSchedulerService {
467
473
  }
468
474
  }
469
475
  catch (error) {
476
+ if (error instanceof ServiceError && error.code === 'SESSION_NOT_ADMITTED') {
477
+ await this.cancelUnadmittedInvocation(createdInvocation.id, session.id);
478
+ this.lockService.releaseByOwner(invocationId);
479
+ invocationId = null;
480
+ continue;
481
+ }
470
482
  const failureResult = await this.markInvocationStartFailed(createdInvocation.id, session.id, error);
471
- this.lockService.releaseByOwner(createdInvocation.id);
472
483
  if (failureResult === 'terminal') {
484
+ this.lockService.releaseByOwner(createdInvocation.id);
473
485
  createdTerminalInvocation = true;
474
486
  }
475
487
  else if (failureResult === 'retry') {
488
+ this.lockService.releaseByOwner(createdInvocation.id);
476
489
  retryBlockedMemberIds.add(member.id);
477
490
  }
478
491
  await this.emitTeamRunInvalidated(context.teamRun, ['work-requests', 'agent-invocations'], 'agent-invocation-updated');
@@ -502,7 +515,7 @@ export class TeamSchedulerService {
502
515
  }
503
516
  }
504
517
  finally {
505
- this.releaseMemberSchedulingLock(memberLockKey);
518
+ releaseMemberSchedulingLock();
506
519
  }
507
520
  }
508
521
  if (createdTerminalInvocation) {
@@ -582,49 +595,67 @@ export class TeamSchedulerService {
582
595
  return workRequest;
583
596
  }
584
597
  async stopMemberWork(teamRunId, memberId, options = {}) {
585
- await this.getTeamMemberOrThrow(teamRunId, memberId);
586
- const activeInvocations = await prisma.agentInvocation.findMany({
587
- where: {
588
- teamRunId,
589
- memberId,
590
- status: { in: STOPPABLE_INVOCATION_STATUSES },
591
- },
592
- orderBy: [{ createdAt: 'asc' }, { id: 'asc' }],
593
- });
594
598
  const stoppedSessionIds = [];
595
- const queuedInvocationIds = activeInvocations
596
- .filter((invocation) => invocation.sessionId == null)
597
- .map((invocation) => invocation.id);
598
- const queuedCancellation = queuedInvocationIds.length > 0
599
- ? await this.cancelInvocationsWithoutSession(queuedInvocationIds)
600
- : { cancelledInvocationIds: [], cancelledWorkRequestIds: [] };
601
- if (options.cancelQueued) {
602
- const cancelledQueuedRequestIds = await this.cancelQueuedWorkRequestsForMember(teamRunId, memberId);
603
- queuedCancellation.cancelledWorkRequestIds.push(...cancelledQueuedRequestIds);
604
- }
605
- const sessionIds = activeInvocations
606
- .map((invocation) => invocation.sessionId)
607
- .filter((sessionId) => sessionId != null);
608
- if (sessionIds.length > 0) {
609
- if (!this.sessionManager.stop) {
610
- throw new ServiceError('Session stop is not available', 'SESSION_STOP_UNAVAILABLE', 500);
599
+ const queuedCancellation = {
600
+ cancelledInvocationIds: [],
601
+ cancelledWorkRequestIds: [],
602
+ };
603
+ let shouldStartNextForQueuedOnly = false;
604
+ let hadActiveInvocations = false;
605
+ const releaseMemberSchedulingLock = await acquireTeamMemberAdmission(teamRunId, memberId);
606
+ try {
607
+ // Re-read all admission state after acquiring the same barrier used by
608
+ // startNextSessions. A DB cancellation observed before this point must
609
+ // not be followed by a stale process stop/start decision.
610
+ await this.getTeamMemberOrThrow(teamRunId, memberId);
611
+ const stopAdmission = await this.establishMemberStopAdmissionGate(teamRunId, memberId, options.cancelQueued === true);
612
+ const activeInvocations = stopAdmission.activeInvocations;
613
+ hadActiveInvocations = activeInvocations.length > 0;
614
+ queuedCancellation.cancelledInvocationIds.push(...stopAdmission.cancelledInvocationIds);
615
+ queuedCancellation.cancelledWorkRequestIds.push(...stopAdmission.cancelledWorkRequestIds);
616
+ for (const invocationId of stopAdmission.cancelledInvocationIds) {
617
+ this.releaseInvocationLocks(invocationId);
611
618
  }
612
- for (const sessionId of sessionIds) {
613
- const stopped = await this.sessionManager.stop(sessionId);
614
- if (stopped) {
615
- stoppedSessionIds.push(sessionId);
619
+ const sessionIds = activeInvocations
620
+ .map((invocation) => invocation.sessionId)
621
+ .filter((sessionId) => sessionId != null);
622
+ if (sessionIds.length > 0) {
623
+ if (!this.sessionManager.stop) {
624
+ throw new ServiceError('Session stop is not available', 'SESSION_STOP_UNAVAILABLE', 500);
625
+ }
626
+ for (const sessionId of sessionIds) {
627
+ // SessionManager.stop normally reconciles TeamRun immediately. The
628
+ // scheduler already owns the member barrier here, so defer that
629
+ // reconciliation until after the barrier is released below.
630
+ const stopped = await this.sessionManager.stop(sessionId, { skipTeamRunReconcile: true });
631
+ if (stopped) {
632
+ stoppedSessionIds.push(sessionId);
633
+ }
616
634
  }
617
635
  }
636
+ const cancelledWorkRequestIds = Array.from(new Set(queuedCancellation.cancelledWorkRequestIds));
637
+ shouldStartNextForQueuedOnly = !hadActiveInvocations && (queuedCancellation.cancelledInvocationIds.length > 0
638
+ || cancelledWorkRequestIds.length > 0);
639
+ }
640
+ finally {
641
+ // Never call startNextSessions recursively while holding this barrier.
642
+ releaseMemberSchedulingLock();
618
643
  }
619
644
  const cancelledWorkRequestIds = Array.from(new Set(queuedCancellation.cancelledWorkRequestIds));
620
- const shouldStartNext = stoppedSessionIds.length > 0
645
+ const startedInvocations = [];
646
+ for (const sessionId of stoppedSessionIds) {
647
+ startedInvocations.push(...await (this.teamRunReviewAdvancer.handleSessionStopped?.(sessionId) ?? Promise.resolve([])));
648
+ }
649
+ if (shouldStartNextForQueuedOnly) {
650
+ startedInvocations.push(...await this.startNextSessions(teamRunId));
651
+ }
652
+ if (stoppedSessionIds.length > 0
621
653
  || queuedCancellation.cancelledInvocationIds.length > 0
622
- || cancelledWorkRequestIds.length > 0;
623
- const startedInvocations = shouldStartNext
624
- ? await this.startNextSessions(teamRunId)
625
- : [];
626
- if (shouldStartNext) {
654
+ || cancelledWorkRequestIds.length > 0) {
627
655
  await this.emitTeamRunInvalidatedById(teamRunId, ['work-requests', 'agent-invocations', 'team-run'], 'member-work-stopped');
656
+ if (shouldStartNextForQueuedOnly) {
657
+ await this.teamRunReviewAdvancer.maybeAdvanceTeamRunToReview(teamRunId);
658
+ }
628
659
  }
629
660
  return {
630
661
  stoppedSessionIds,
@@ -633,6 +664,59 @@ export class TeamSchedulerService {
633
664
  startedInvocations,
634
665
  };
635
666
  }
667
+ async stopSession(sessionId) {
668
+ const invocation = await prisma.agentInvocation.findFirst({
669
+ where: { sessionId },
670
+ select: { id: true, teamRunId: true, memberId: true, status: true },
671
+ });
672
+ if (!invocation) {
673
+ return this.sessionManager.stop?.(sessionId) ?? null;
674
+ }
675
+ const releaseMemberSchedulingLock = await acquireTeamMemberAdmission(invocation.teamRunId, invocation.memberId);
676
+ let stopped = null;
677
+ let shouldReconcileStop = false;
678
+ try {
679
+ const revoked = await prisma.agentInvocation.updateMany({
680
+ where: {
681
+ id: invocation.id,
682
+ sessionId,
683
+ status: { in: STOPPABLE_INVOCATION_STATUSES },
684
+ },
685
+ data: {
686
+ dispatchRevokedAt: this.now(),
687
+ nextRoomReplyReminderAt: null,
688
+ },
689
+ });
690
+ if (revoked.count === 0) {
691
+ const currentInvocation = await prisma.agentInvocation.findUnique({
692
+ where: { id: invocation.id },
693
+ select: { status: true },
694
+ });
695
+ if (currentInvocation
696
+ && TERMINAL_INVOCATION_STATUSES.includes(currentInvocation.status)) {
697
+ stopped = await this.sessionManager.stop?.(sessionId, { skipTeamRunReconcile: true }) ?? null;
698
+ shouldReconcileStop = stopped != null;
699
+ }
700
+ }
701
+ if (revoked.count > 0 && !this.sessionManager.stop) {
702
+ throw new ServiceError('Session stop is not available', 'SESSION_STOP_UNAVAILABLE', 500);
703
+ }
704
+ if (revoked.count > 0) {
705
+ stopped = await this.sessionManager.stop(sessionId, { skipTeamRunReconcile: true });
706
+ }
707
+ if (revoked.count > 0 && stopped) {
708
+ shouldReconcileStop = true;
709
+ }
710
+ }
711
+ finally {
712
+ releaseMemberSchedulingLock();
713
+ }
714
+ if (shouldReconcileStop) {
715
+ await this.teamRunReviewAdvancer.handleSessionStopped?.(sessionId);
716
+ await this.emitTeamRunInvalidatedById(invocation.teamRunId, ['work-requests', 'agent-invocations', 'team-run'], 'member-work-stopped');
717
+ }
718
+ return stopped;
719
+ }
636
720
  releaseInvocationLocks(invocationId) {
637
721
  this.lockService.releaseByOwner(invocationId);
638
722
  }
@@ -749,6 +833,17 @@ export class TeamSchedulerService {
749
833
  }
750
834
  async createRunningInvocationForClaimedWorkRequest(teamRun, member, workRequest, invocationId, workspaceId, sessionId, targetStartData = {}) {
751
835
  return prisma.$transaction(async (tx) => {
836
+ const session = await tx.session.findUnique({
837
+ where: { id: sessionId },
838
+ select: { status: true },
839
+ });
840
+ if (!session || session.status !== 'PENDING') {
841
+ await tx.workRequest.updateMany({
842
+ where: { id: workRequest.id, teamRunId: teamRun.id, status: 'QUEUED' },
843
+ data: { status: 'CANCELLED' },
844
+ });
845
+ return null;
846
+ }
752
847
  const claimed = await tx.workRequest.updateMany({
753
848
  where: {
754
849
  id: workRequest.id,
@@ -1002,71 +1097,74 @@ export class TeamSchedulerService {
1002
1097
  }
1003
1098
  return this.serializeWorkRequest(updated);
1004
1099
  }
1005
- async cancelInvocationsWithoutSession(invocationIds) {
1006
- if (invocationIds.length === 0) {
1007
- return { cancelledInvocationIds: [], cancelledWorkRequestIds: [] };
1008
- }
1009
- const cancelled = await prisma.$transaction(async (tx) => {
1010
- const invocations = await tx.agentInvocation.findMany({
1100
+ async establishMemberStopAdmissionGate(teamRunId, memberId, cancelQueued) {
1101
+ return prisma.$transaction(async (tx) => {
1102
+ const activeInvocations = await tx.agentInvocation.findMany({
1011
1103
  where: {
1012
- id: { in: invocationIds },
1013
- sessionId: null,
1104
+ teamRunId,
1105
+ memberId,
1014
1106
  status: { in: STOPPABLE_INVOCATION_STATUSES },
1015
1107
  },
1016
- select: { id: true, workRequestId: true },
1108
+ orderBy: [{ createdAt: 'asc' }, { id: 'asc' }],
1017
1109
  });
1018
- if (invocations.length === 0) {
1019
- return { invocationIds: [], workRequestIds: [] };
1110
+ const now = this.now();
1111
+ const sessionInvocations = activeInvocations.filter((invocation) => invocation.sessionId != null);
1112
+ const noSessionInvocations = activeInvocations.filter((invocation) => invocation.sessionId == null);
1113
+ if (activeInvocations.length > 0) {
1114
+ await tx.agentInvocation.updateMany({
1115
+ where: {
1116
+ id: { in: activeInvocations.map((invocation) => invocation.id) },
1117
+ status: { in: STOPPABLE_INVOCATION_STATUSES },
1118
+ },
1119
+ data: {
1120
+ dispatchRevokedAt: now,
1121
+ nextRoomReplyReminderAt: null,
1122
+ },
1123
+ });
1124
+ }
1125
+ const cancelledInvocationIds = noSessionInvocations.map((invocation) => invocation.id);
1126
+ const noSessionWorkRequestIds = noSessionInvocations.map((invocation) => invocation.workRequestId);
1127
+ if (cancelledInvocationIds.length > 0) {
1128
+ await tx.agentInvocation.updateMany({
1129
+ where: { id: { in: cancelledInvocationIds }, status: { in: STOPPABLE_INVOCATION_STATUSES } },
1130
+ data: { status: 'CANCELLED' },
1131
+ });
1132
+ await tx.workRequest.updateMany({
1133
+ where: {
1134
+ id: { in: noSessionWorkRequestIds },
1135
+ status: { in: ['PENDING_APPROVAL', 'QUEUED', 'STARTED'] },
1136
+ },
1137
+ data: { status: 'CANCELLED' },
1138
+ });
1139
+ }
1140
+ const queuedWorkRequests = cancelQueued
1141
+ ? await tx.workRequest.findMany({
1142
+ where: {
1143
+ teamRunId,
1144
+ targetMemberId: memberId,
1145
+ status: { in: CANCELLABLE_QUEUED_WORK_REQUEST_STATUSES },
1146
+ },
1147
+ select: { id: true },
1148
+ })
1149
+ : [];
1150
+ if (queuedWorkRequests.length > 0) {
1151
+ await tx.workRequest.updateMany({
1152
+ where: {
1153
+ id: { in: queuedWorkRequests.map((workRequest) => workRequest.id) },
1154
+ status: { in: CANCELLABLE_QUEUED_WORK_REQUEST_STATUSES },
1155
+ },
1156
+ data: { status: 'CANCELLED' },
1157
+ });
1020
1158
  }
1021
- const cancellableWorkRequestIds = invocations.map((invocation) => invocation.workRequestId);
1022
- await tx.agentInvocation.updateMany({
1023
- where: { id: { in: invocations.map((invocation) => invocation.id) } },
1024
- data: {
1025
- status: 'CANCELLED',
1026
- nextRoomReplyReminderAt: null,
1027
- },
1028
- });
1029
- await tx.workRequest.updateMany({
1030
- where: {
1031
- id: { in: cancellableWorkRequestIds },
1032
- status: { in: ['PENDING_APPROVAL', 'QUEUED', 'STARTED'] },
1033
- },
1034
- data: { status: 'CANCELLED' },
1035
- });
1036
1159
  return {
1037
- invocationIds: invocations.map((invocation) => invocation.id),
1038
- workRequestIds: cancellableWorkRequestIds,
1160
+ activeInvocations: sessionInvocations,
1161
+ cancelledInvocationIds,
1162
+ cancelledWorkRequestIds: [
1163
+ ...noSessionWorkRequestIds,
1164
+ ...queuedWorkRequests.map((workRequest) => workRequest.id),
1165
+ ],
1039
1166
  };
1040
1167
  });
1041
- for (const invocationId of cancelled.invocationIds) {
1042
- this.releaseInvocationLocks(invocationId);
1043
- }
1044
- return {
1045
- cancelledInvocationIds: cancelled.invocationIds,
1046
- cancelledWorkRequestIds: cancelled.workRequestIds,
1047
- };
1048
- }
1049
- async cancelQueuedWorkRequestsForMember(teamRunId, memberId) {
1050
- const workRequests = await prisma.workRequest.findMany({
1051
- where: {
1052
- teamRunId,
1053
- targetMemberId: memberId,
1054
- status: { in: CANCELLABLE_QUEUED_WORK_REQUEST_STATUSES },
1055
- },
1056
- select: { id: true },
1057
- });
1058
- if (workRequests.length === 0) {
1059
- return [];
1060
- }
1061
- const workRequestIds = workRequests.map((workRequest) => workRequest.id);
1062
- await prisma.workRequest.updateMany({
1063
- where: {
1064
- id: { in: workRequestIds },
1065
- status: { in: CANCELLABLE_QUEUED_WORK_REQUEST_STATUSES },
1066
- },
1067
- data: { status: 'CANCELLED' },
1068
- });
1069
- return workRequestIds;
1070
1168
  }
1071
1169
  async findActiveMemberIds(teamRunId) {
1072
1170
  const activeInvocations = await prisma.agentInvocation.findMany({
@@ -1086,7 +1184,50 @@ export class TeamSchedulerService {
1086
1184
  status: { in: ACTIVE_INVOCATION_STATUSES },
1087
1185
  },
1088
1186
  });
1089
- return count > 0;
1187
+ if (count > 0)
1188
+ return true;
1189
+ // A terminal DB status is not enough to admit the next WorkRequest when
1190
+ // the owned ACP tree still needs cleanup. The cleanup callback updates
1191
+ // this metadata only after the OS tree has been confirmed gone.
1192
+ const terminalInvocations = await prisma.agentInvocation.findMany({
1193
+ where: {
1194
+ teamRunId,
1195
+ memberId,
1196
+ status: { in: TERMINAL_INVOCATION_STATUSES },
1197
+ sessionId: { not: null },
1198
+ },
1199
+ select: { sessionId: true, workRequestId: true },
1200
+ });
1201
+ if (terminalInvocations.length === 0)
1202
+ return false;
1203
+ const terminalWorkRequests = await prisma.workRequest.findMany({
1204
+ where: { id: { in: terminalInvocations.map((invocation) => invocation.workRequestId) } },
1205
+ select: { id: true, status: true },
1206
+ });
1207
+ const workRequestStatusById = new Map(terminalWorkRequests.map((workRequest) => [workRequest.id, workRequest.status]));
1208
+ for (const invocation of terminalInvocations) {
1209
+ // STARTED is the durable after-terminal winner. Until the reconciler
1210
+ // consumes it, locks/queue admission remain blocked even if cleanup has
1211
+ // just become confirmable.
1212
+ if (workRequestStatusById.get(invocation.workRequestId) === 'STARTED')
1213
+ return true;
1214
+ const sessionId = invocation.sessionId;
1215
+ if (!sessionId)
1216
+ continue;
1217
+ const cleanupConfirmed = this.sessionManager.isRuntimeCleanupConfirmed
1218
+ ? await this.sessionManager.isRuntimeCleanupConfirmed(sessionId)
1219
+ : (await evaluateSessionRuntimeCleanup(sessionId, {
1220
+ hasActiveTurn: this.sessionManager.hasActiveTurn
1221
+ ? (candidateId) => this.sessionManager.hasActiveTurn(candidateId)
1222
+ : undefined,
1223
+ hasRuntimeProcessOwner: this.sessionManager.hasRuntimeProcessOwner
1224
+ ? (runtimeInstanceId) => this.sessionManager.hasRuntimeProcessOwner(runtimeInstanceId)
1225
+ : undefined,
1226
+ }, this.now())).confirmed;
1227
+ if (!cleanupConfirmed)
1228
+ return true;
1229
+ }
1230
+ return false;
1090
1231
  }
1091
1232
  async getOrCreateWorkspaceForMember(teamRun, member) {
1092
1233
  if (member.workspacePolicy === 'dedicated') {
@@ -1214,6 +1355,38 @@ export class TeamSchedulerService {
1214
1355
  return previousInvocation.sessionId;
1215
1356
  }
1216
1357
  async markInvocationStartFailed(invocationId, sessionId, error) {
1358
+ const cleanupConfirmed = this.sessionManager.isRuntimeCleanupConfirmed
1359
+ ? await this.sessionManager.isRuntimeCleanupConfirmed(sessionId)
1360
+ : (await evaluateSessionRuntimeCleanup(sessionId, {
1361
+ hasActiveTurn: this.sessionManager.hasActiveTurn
1362
+ ? (candidateId) => this.sessionManager.hasActiveTurn(candidateId)
1363
+ : undefined,
1364
+ hasRuntimeProcessOwner: this.sessionManager.hasRuntimeProcessOwner
1365
+ ? (runtimeInstanceId) => this.sessionManager.hasRuntimeProcessOwner(runtimeInstanceId)
1366
+ : undefined,
1367
+ }, this.now())).confirmed;
1368
+ if (!cleanupConfirmed) {
1369
+ await prisma.$transaction(async (tx) => {
1370
+ const invocation = await tx.agentInvocation.findUnique({
1371
+ where: { id: invocationId },
1372
+ select: { workRequestId: true },
1373
+ });
1374
+ if (!invocation)
1375
+ return;
1376
+ await tx.agentInvocation.updateMany({
1377
+ where: { id: invocationId, status: 'RUNNING' },
1378
+ data: { dispatchRevokedAt: this.now(), nextRoomReplyReminderAt: null },
1379
+ });
1380
+ await tx.workRequest.updateMany({
1381
+ where: { id: invocation.workRequestId, status: 'STARTED' },
1382
+ data: {
1383
+ lastStartError: startErrorMessage(error),
1384
+ nextStartRetryAt: null,
1385
+ },
1386
+ });
1387
+ });
1388
+ return 'pending';
1389
+ }
1217
1390
  return prisma.$transaction(async (tx) => {
1218
1391
  const invocation = await tx.agentInvocation.findUnique({
1219
1392
  where: { id: invocationId },
@@ -1272,11 +1445,53 @@ export class TeamSchedulerService {
1272
1445
  await emitTeamRunInvalidated({ teamRunId, scopes, reason });
1273
1446
  }
1274
1447
  async markSessionFailed(sessionId) {
1275
- await prisma.session.update({
1276
- where: { id: sessionId },
1448
+ await prisma.session.updateMany({
1449
+ where: { id: sessionId, status: 'PENDING' },
1277
1450
  data: { status: 'FAILED' },
1278
1451
  });
1279
1452
  }
1453
+ async isInvocationAdmitted(invocationId, workRequestId) {
1454
+ const state = await prisma.agentInvocation.findUnique({
1455
+ where: { id: invocationId },
1456
+ select: { status: true, workRequestId: true, dispatchRevokedAt: true, sessionId: true },
1457
+ });
1458
+ if (!state)
1459
+ return false;
1460
+ const workRequest = await prisma.workRequest.findUnique({
1461
+ where: { id: state.workRequestId },
1462
+ select: { id: true, status: true },
1463
+ });
1464
+ const session = state.sessionId
1465
+ ? await prisma.session.findUnique({ where: { id: state.sessionId }, select: { status: true } })
1466
+ : null;
1467
+ return state.status === 'RUNNING'
1468
+ && state.dispatchRevokedAt == null
1469
+ && session?.status === 'PENDING'
1470
+ && workRequest?.id === workRequestId
1471
+ && workRequest.status === 'STARTED';
1472
+ }
1473
+ async cancelUnadmittedInvocation(invocationId, sessionId) {
1474
+ const cancelled = await prisma.agentInvocation.updateMany({
1475
+ where: { id: invocationId, status: 'RUNNING' },
1476
+ data: { status: 'CANCELLED', nextRoomReplyReminderAt: null },
1477
+ });
1478
+ if (cancelled.count !== 1)
1479
+ return;
1480
+ const invocation = await prisma.agentInvocation.findUnique({
1481
+ where: { id: invocationId },
1482
+ select: { workRequestId: true },
1483
+ });
1484
+ if (invocation) {
1485
+ await prisma.workRequest.updateMany({
1486
+ where: { id: invocation.workRequestId, status: 'STARTED' },
1487
+ data: { status: 'CANCELLED' },
1488
+ });
1489
+ }
1490
+ await prisma.session.updateMany({
1491
+ where: { id: sessionId, status: 'PENDING' },
1492
+ data: { status: 'CANCELLED' },
1493
+ });
1494
+ }
1280
1495
  getRequiredLocks(teamRun, member) {
1281
1496
  const request = {
1282
1497
  teamRunId: teamRun.id,
@@ -1302,19 +1517,6 @@ export class TeamSchedulerService {
1302
1517
  }
1303
1518
  return `task:${teamRun.taskId}`;
1304
1519
  }
1305
- memberSchedulingLockKey(teamRunId, memberId) {
1306
- return `scheduling:${teamRunId}:member:${memberId}`;
1307
- }
1308
- acquireMemberSchedulingLock(lockKey) {
1309
- if (TeamSchedulerService.memberSchedulingLocks.has(lockKey)) {
1310
- return false;
1311
- }
1312
- TeamSchedulerService.memberSchedulingLocks.add(lockKey);
1313
- return true;
1314
- }
1315
- releaseMemberSchedulingLock(lockKey) {
1316
- TeamSchedulerService.memberSchedulingLocks.delete(lockKey);
1317
- }
1318
1520
  serializeWorkRequest(workRequest) {
1319
1521
  return {
1320
1522
  ...workRequest,
@@ -1342,6 +1544,7 @@ export class TeamSchedulerService {
1342
1544
  : null,
1343
1545
  lastHeartbeatAt: invocation.lastHeartbeatAt ? toIso(invocation.lastHeartbeatAt) : null,
1344
1546
  firstNudgeAt: invocation.firstNudgeAt ? toIso(invocation.firstNudgeAt) : null,
1547
+ dispatchRevokedAt: invocation.dispatchRevokedAt ? toIso(invocation.dispatchRevokedAt) : null,
1345
1548
  };
1346
1549
  }
1347
1550
  }