agent-tower 0.6.0-beta.11 → 0.6.0-beta.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/dist/app.d.ts.map +1 -1
  2. package/dist/app.js +4 -0
  3. package/dist/app.js.map +1 -1
  4. package/dist/cli.js +39 -19
  5. package/dist/cli.js.map +1 -1
  6. package/dist/executors/__tests__/base.executor.test.js +133 -1
  7. package/dist/executors/__tests__/base.executor.test.js.map +1 -1
  8. package/dist/executors/__tests__/stdin-prompt.executors.test.js +18 -0
  9. package/dist/executors/__tests__/stdin-prompt.executors.test.js.map +1 -1
  10. package/dist/executors/base.executor.d.ts +8 -0
  11. package/dist/executors/base.executor.d.ts.map +1 -1
  12. package/dist/executors/base.executor.js +336 -56
  13. package/dist/executors/base.executor.js.map +1 -1
  14. package/dist/executors/index.d.ts +2 -1
  15. package/dist/executors/index.d.ts.map +1 -1
  16. package/dist/executors/index.js +1 -1
  17. package/dist/executors/index.js.map +1 -1
  18. package/dist/executors/start-error.d.ts +17 -0
  19. package/dist/executors/start-error.d.ts.map +1 -1
  20. package/dist/executors/start-error.js +36 -1
  21. package/dist/executors/start-error.js.map +1 -1
  22. package/dist/index.js +39 -29
  23. package/dist/index.js.map +1 -1
  24. package/dist/mcp/http-client.test.js +20 -0
  25. package/dist/mcp/http-client.test.js.map +1 -1
  26. package/dist/mcp/tools/__tests__/sessions.test.d.ts +2 -0
  27. package/dist/mcp/tools/__tests__/sessions.test.d.ts.map +1 -0
  28. package/dist/mcp/tools/__tests__/sessions.test.js +33 -0
  29. package/dist/mcp/tools/__tests__/sessions.test.js.map +1 -0
  30. package/dist/routes/__tests__/sessions-runtime.test.js +75 -3
  31. package/dist/routes/__tests__/sessions-runtime.test.js.map +1 -1
  32. package/dist/routes/conversations.d.ts.map +1 -1
  33. package/dist/routes/conversations.js +5 -1
  34. package/dist/routes/conversations.js.map +1 -1
  35. package/dist/routes/sessions.d.ts.map +1 -1
  36. package/dist/routes/sessions.js +13 -2
  37. package/dist/routes/sessions.js.map +1 -1
  38. package/dist/routes/team-runs.d.ts.map +1 -1
  39. package/dist/routes/team-runs.js +13 -1
  40. package/dist/routes/team-runs.js.map +1 -1
  41. package/dist/runtime/__tests__/acp-driver-lifecycle.test.js +86 -0
  42. package/dist/runtime/__tests__/acp-driver-lifecycle.test.js.map +1 -1
  43. package/dist/runtime/__tests__/acp-process-manager.test.js +393 -6
  44. package/dist/runtime/__tests__/acp-process-manager.test.js.map +1 -1
  45. package/dist/runtime/__tests__/cli-driver-lifecycle.test.d.ts +2 -0
  46. package/dist/runtime/__tests__/cli-driver-lifecycle.test.d.ts.map +1 -0
  47. package/dist/runtime/__tests__/cli-driver-lifecycle.test.js +244 -0
  48. package/dist/runtime/__tests__/cli-driver-lifecycle.test.js.map +1 -0
  49. package/dist/runtime/__tests__/launch-cleanup-registry.test.d.ts +2 -0
  50. package/dist/runtime/__tests__/launch-cleanup-registry.test.d.ts.map +1 -0
  51. package/dist/runtime/__tests__/launch-cleanup-registry.test.js +154 -0
  52. package/dist/runtime/__tests__/launch-cleanup-registry.test.js.map +1 -0
  53. package/dist/runtime/__tests__/runtime-coordinator.test.js +117 -1
  54. package/dist/runtime/__tests__/runtime-coordinator.test.js.map +1 -1
  55. package/dist/runtime/__tests__/server-entry-shutdown.test.d.ts +2 -0
  56. package/dist/runtime/__tests__/server-entry-shutdown.test.d.ts.map +1 -0
  57. package/dist/runtime/__tests__/server-entry-shutdown.test.js +38 -0
  58. package/dist/runtime/__tests__/server-entry-shutdown.test.js.map +1 -0
  59. package/dist/runtime/__tests__/shutdown-coordinator.test.d.ts +2 -0
  60. package/dist/runtime/__tests__/shutdown-coordinator.test.d.ts.map +1 -0
  61. package/dist/runtime/__tests__/shutdown-coordinator.test.js +28 -0
  62. package/dist/runtime/__tests__/shutdown-coordinator.test.js.map +1 -0
  63. package/dist/runtime/acp/acp-driver.d.ts.map +1 -1
  64. package/dist/runtime/acp/acp-driver.js +108 -35
  65. package/dist/runtime/acp/acp-driver.js.map +1 -1
  66. package/dist/runtime/acp/agents/managed-directory.d.ts.map +1 -1
  67. package/dist/runtime/acp/agents/managed-directory.js +3 -1
  68. package/dist/runtime/acp/agents/managed-directory.js.map +1 -1
  69. package/dist/runtime/acp/agents/managed-directory.test.d.ts +2 -0
  70. package/dist/runtime/acp/agents/managed-directory.test.d.ts.map +1 -0
  71. package/dist/runtime/acp/agents/managed-directory.test.js +25 -0
  72. package/dist/runtime/acp/agents/managed-directory.test.js.map +1 -0
  73. package/dist/runtime/acp/launch-cleanup-registry.d.ts +32 -0
  74. package/dist/runtime/acp/launch-cleanup-registry.d.ts.map +1 -0
  75. package/dist/runtime/acp/launch-cleanup-registry.js +176 -0
  76. package/dist/runtime/acp/launch-cleanup-registry.js.map +1 -0
  77. package/dist/runtime/acp/process-manager.d.ts +58 -0
  78. package/dist/runtime/acp/process-manager.d.ts.map +1 -1
  79. package/dist/runtime/acp/process-manager.js +489 -33
  80. package/dist/runtime/acp/process-manager.js.map +1 -1
  81. package/dist/runtime/cli-driver.d.ts.map +1 -1
  82. package/dist/runtime/cli-driver.js +230 -46
  83. package/dist/runtime/cli-driver.js.map +1 -1
  84. package/dist/runtime/contracts.d.ts +21 -2
  85. package/dist/runtime/contracts.d.ts.map +1 -1
  86. package/dist/runtime/index.d.ts +1 -0
  87. package/dist/runtime/index.d.ts.map +1 -1
  88. package/dist/runtime/index.js +1 -0
  89. package/dist/runtime/index.js.map +1 -1
  90. package/dist/runtime/runtime-coordinator.d.ts +8 -1
  91. package/dist/runtime/runtime-coordinator.d.ts.map +1 -1
  92. package/dist/runtime/runtime-coordinator.js +118 -18
  93. package/dist/runtime/runtime-coordinator.js.map +1 -1
  94. package/dist/runtime/server-entry-shutdown.d.ts +12 -0
  95. package/dist/runtime/server-entry-shutdown.d.ts.map +1 -0
  96. package/dist/runtime/server-entry-shutdown.js +23 -0
  97. package/dist/runtime/server-entry-shutdown.js.map +1 -0
  98. package/dist/runtime/shutdown-coordinator.d.ts +17 -0
  99. package/dist/runtime/shutdown-coordinator.d.ts.map +1 -0
  100. package/dist/runtime/shutdown-coordinator.js +67 -0
  101. package/dist/runtime/shutdown-coordinator.js.map +1 -0
  102. package/dist/services/__tests__/conversation.service.test.js +160 -0
  103. package/dist/services/__tests__/conversation.service.test.js.map +1 -1
  104. package/dist/services/__tests__/database-maintenance.service.test.js +97 -0
  105. package/dist/services/__tests__/database-maintenance.service.test.js.map +1 -1
  106. package/dist/services/__tests__/member-heartbeat-scheduler.test.js +6 -0
  107. package/dist/services/__tests__/member-heartbeat-scheduler.test.js.map +1 -1
  108. package/dist/services/__tests__/member-heartbeat.test.js +7 -11
  109. package/dist/services/__tests__/member-heartbeat.test.js.map +1 -1
  110. package/dist/services/__tests__/session-manager.lifecycle.test.js +402 -12
  111. package/dist/services/__tests__/session-manager.lifecycle.test.js.map +1 -1
  112. package/dist/services/__tests__/session-manager.team-run.test.js +422 -29
  113. package/dist/services/__tests__/session-manager.team-run.test.js.map +1 -1
  114. package/dist/services/__tests__/team-queue-recovery.integration.test.js +1 -0
  115. package/dist/services/__tests__/team-queue-recovery.integration.test.js.map +1 -1
  116. package/dist/services/__tests__/team-reconciler.service.test.js +1282 -4
  117. package/dist/services/__tests__/team-reconciler.service.test.js.map +1 -1
  118. package/dist/services/__tests__/team-run.service.test.js +161 -1
  119. package/dist/services/__tests__/team-run.service.test.js.map +1 -1
  120. package/dist/services/__tests__/team-scheduler.service.test.js +334 -6
  121. package/dist/services/__tests__/team-scheduler.service.test.js.map +1 -1
  122. package/dist/services/conversation.service.d.ts.map +1 -1
  123. package/dist/services/conversation.service.js +21 -2
  124. package/dist/services/conversation.service.js.map +1 -1
  125. package/dist/services/database-maintenance.service.d.ts.map +1 -1
  126. package/dist/services/database-maintenance.service.js +68 -17
  127. package/dist/services/database-maintenance.service.js.map +1 -1
  128. package/dist/services/member-heartbeat-scheduler.d.ts +6 -5
  129. package/dist/services/member-heartbeat-scheduler.d.ts.map +1 -1
  130. package/dist/services/member-heartbeat-scheduler.js +7 -5
  131. package/dist/services/member-heartbeat-scheduler.js.map +1 -1
  132. package/dist/services/session-manager.d.ts +124 -14
  133. package/dist/services/session-manager.d.ts.map +1 -1
  134. package/dist/services/session-manager.js +1035 -116
  135. package/dist/services/session-manager.js.map +1 -1
  136. package/dist/services/session-runtime-cleanup-gate.d.ts +25 -0
  137. package/dist/services/session-runtime-cleanup-gate.d.ts.map +1 -0
  138. package/dist/services/session-runtime-cleanup-gate.js +284 -0
  139. package/dist/services/session-runtime-cleanup-gate.js.map +1 -0
  140. package/dist/services/task.service.d.ts +2 -2
  141. package/dist/services/team-member-admission-barrier.d.ts +3 -0
  142. package/dist/services/team-member-admission-barrier.d.ts.map +1 -0
  143. package/dist/services/team-member-admission-barrier.js +26 -0
  144. package/dist/services/team-member-admission-barrier.js.map +1 -0
  145. package/dist/services/team-reconciler.service.d.ts +30 -7
  146. package/dist/services/team-reconciler.service.d.ts.map +1 -1
  147. package/dist/services/team-reconciler.service.js +528 -111
  148. package/dist/services/team-reconciler.service.js.map +1 -1
  149. package/dist/services/team-run.service.d.ts.map +1 -1
  150. package/dist/services/team-run.service.js +37 -13
  151. package/dist/services/team-run.service.js.map +1 -1
  152. package/dist/services/team-scheduler.service.d.ts +8 -6
  153. package/dist/services/team-scheduler.service.d.ts.map +1 -1
  154. package/dist/services/team-scheduler.service.js +323 -120
  155. package/dist/services/team-scheduler.service.js.map +1 -1
  156. package/dist/services/workspace-background-process-manager.d.ts.map +1 -1
  157. package/dist/services/workspace-background-process-manager.js +2 -5
  158. package/dist/services/workspace-background-process-manager.js.map +1 -1
  159. package/dist/services/workspace.service.d.ts +15 -8
  160. package/dist/services/workspace.service.d.ts.map +1 -1
  161. package/dist/socket/index.d.ts.map +1 -1
  162. package/dist/socket/index.js +27 -14
  163. package/dist/socket/index.js.map +1 -1
  164. package/dist/utils/__tests__/unix-process-identity.test.d.ts +2 -0
  165. package/dist/utils/__tests__/unix-process-identity.test.d.ts.map +1 -0
  166. package/dist/utils/__tests__/unix-process-identity.test.js +73 -0
  167. package/dist/utils/__tests__/unix-process-identity.test.js.map +1 -0
  168. package/dist/utils/error-log.d.ts +2 -0
  169. package/dist/utils/error-log.d.ts.map +1 -1
  170. package/dist/utils/error-log.js +40 -3
  171. package/dist/utils/error-log.js.map +1 -1
  172. package/dist/utils/error-log.test.js +14 -2
  173. package/dist/utils/error-log.test.js.map +1 -1
  174. package/dist/utils/process-launch.d.ts +2 -2
  175. package/dist/utils/process-launch.d.ts.map +1 -1
  176. package/dist/utils/process-launch.js +339 -79
  177. package/dist/utils/process-launch.js.map +1 -1
  178. package/dist/utils/process-launch.test.js +168 -10
  179. package/dist/utils/process-launch.test.js.map +1 -1
  180. package/dist/utils/spawned-process-identity.d.ts +7 -0
  181. package/dist/utils/spawned-process-identity.d.ts.map +1 -0
  182. package/dist/utils/spawned-process-identity.js +36 -0
  183. package/dist/utils/spawned-process-identity.js.map +1 -0
  184. package/dist/utils/tree-cleanup-channel.d.ts +18 -0
  185. package/dist/utils/tree-cleanup-channel.d.ts.map +1 -0
  186. package/dist/utils/tree-cleanup-channel.js +16 -0
  187. package/dist/utils/tree-cleanup-channel.js.map +1 -0
  188. package/dist/utils/tree-cleanup-channel.test.d.ts +2 -0
  189. package/dist/utils/tree-cleanup-channel.test.d.ts.map +1 -0
  190. package/dist/utils/tree-cleanup-channel.test.js +15 -0
  191. package/dist/utils/tree-cleanup-channel.test.js.map +1 -0
  192. package/dist/utils/unix-process-identity.d.ts +13 -1
  193. package/dist/utils/unix-process-identity.d.ts.map +1 -1
  194. package/dist/utils/unix-process-identity.js +146 -25
  195. package/dist/utils/unix-process-identity.js.map +1 -1
  196. package/node_modules/@agent-tower/shared/dist/types.d.ts +1 -0
  197. package/node_modules/@agent-tower/shared/dist/types.d.ts.map +1 -1
  198. package/node_modules/@agent-tower/shared/dist/types.js.map +1 -1
  199. package/package.json +1 -1
  200. package/prisma/migrations/20260824000000_add_execution_process_ownership/migration.sql +10 -0
  201. package/prisma/migrations/20260824010000_add_runtime_cleanup_recovery/migration.sql +10 -0
  202. package/prisma/migrations/20260824020000_add_runtime_launch_admission/migration.sql +72 -0
  203. package/prisma/migrations/20260826000000_add_conversation_turn_queue/migration.sql +19 -0
  204. package/prisma/schema.prisma +50 -7
@@ -10,6 +10,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vites
10
10
  import { TaskStatus } from '../../types/index.js';
11
11
  import { EventBus } from '../../core/event-bus.js';
12
12
  import { TeamLockService } from '../team-lock.service.js';
13
+ import { acquireTeamMemberAdmission } from '../team-member-admission-barrier.js';
13
14
  const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-tower-team-reconciler-'));
14
15
  const dbPath = path.join(testDir, 'test.db');
15
16
  process.env.AGENT_TOWER_DATABASE_URL = `file:${dbPath}`;
@@ -367,6 +368,1223 @@ describe('TeamReconcilerService', () => {
367
368
  expect(lockService.listLocks()).toEqual([]);
368
369
  expect(scheduler.startNextSessions).toHaveBeenCalledWith(teamRun.id);
369
370
  });
371
+ it('retries persisted cleanup even when the WorkRequest is already terminal', async () => {
372
+ const { workspace, teamRun, members } = await createFixture();
373
+ const request = await createWorkRequest({
374
+ teamRunId: teamRun.id,
375
+ targetMemberId: members[0].id,
376
+ status: 'COMPLETED',
377
+ });
378
+ const invocation = await createRunningInvocation({
379
+ teamRunId: teamRun.id,
380
+ workRequestId: request.id,
381
+ memberId: members[0].id,
382
+ workspaceId: workspace.id,
383
+ status: 'COMPLETED',
384
+ });
385
+ await prisma.session.update({
386
+ where: { id: invocation.sessionId },
387
+ data: { runtimeType: 'ACP' },
388
+ });
389
+ const processRecord = await prisma.executionProcess.create({
390
+ data: {
391
+ sessionId: invocation.sessionId,
392
+ launchClaimNumber: 1,
393
+ runtimeInstanceId: 'runtime-restart',
394
+ pid: 8101,
395
+ processGroupId: '8101',
396
+ birthMarker: 'linux:101:owner-restart',
397
+ ownershipToken: 'owner-restart',
398
+ cleanupState: 'FAILED',
399
+ cleanupAttemptCount: 1,
400
+ nextCleanupRetryAt: new Date(Date.UTC(2025, 11, 31, 23, 59, 0)),
401
+ },
402
+ });
403
+ const retryRuntimeProcessCleanup = vi.fn(async () => {
404
+ await prisma.executionProcess.update({
405
+ where: { id: processRecord.id },
406
+ data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
407
+ });
408
+ });
409
+ const cleanupService = new TeamReconcilerService({
410
+ scheduler,
411
+ sessionMessenger: {
412
+ sendMessage: vi.fn(async () => null),
413
+ retryRuntimeProcessCleanup,
414
+ disposeRuntimeSession: vi.fn(async () => undefined),
415
+ },
416
+ now: () => new Date(Date.UTC(2026, 0, 1, 0, 0, 0)),
417
+ scheduleReminders: false,
418
+ });
419
+ await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
420
+ expect(retryRuntimeProcessCleanup).toHaveBeenCalledWith({
421
+ sessionId: invocation.sessionId,
422
+ runtimeInstanceId: 'runtime-restart',
423
+ launchClaimNumber: 1,
424
+ pid: 8101,
425
+ processGroupId: '8101',
426
+ birthMarker: 'linux:101:owner-restart',
427
+ ownershipToken: 'owner-restart',
428
+ });
429
+ expect(scheduler.startNextSessions).not.toHaveBeenCalled();
430
+ await expect(prisma.executionProcess.findUnique({ where: { id: processRecord.id } })).resolves.toMatchObject({
431
+ cleanupState: 'CONFIRMED',
432
+ });
433
+ });
434
+ it('preserves the cleanup messenger receiver during persisted recovery', async () => {
435
+ const { workspace, teamRun, members } = await createFixture();
436
+ const request = await createWorkRequest({
437
+ teamRunId: teamRun.id,
438
+ targetMemberId: members[0].id,
439
+ status: 'COMPLETED',
440
+ });
441
+ const invocation = await createRunningInvocation({
442
+ teamRunId: teamRun.id,
443
+ workRequestId: request.id,
444
+ memberId: members[0].id,
445
+ workspaceId: workspace.id,
446
+ status: 'COMPLETED',
447
+ });
448
+ await prisma.session.update({
449
+ where: { id: invocation.sessionId },
450
+ data: { runtimeType: 'ACP' },
451
+ });
452
+ const processRecord = await prisma.executionProcess.create({
453
+ data: {
454
+ sessionId: invocation.sessionId,
455
+ launchClaimNumber: 1,
456
+ runtimeInstanceId: 'runtime-receiver-check',
457
+ pid: 8102,
458
+ processGroupId: '8102',
459
+ birthMarker: 'linux:102:owner-receiver-check',
460
+ ownershipToken: 'owner-receiver-check',
461
+ cleanupState: 'FAILED',
462
+ cleanupAttemptCount: 1,
463
+ nextCleanupRetryAt: new Date(Date.UTC(2025, 11, 31, 23, 59, 0)),
464
+ },
465
+ });
466
+ const messenger = {
467
+ cleanupCalls: 0,
468
+ sendMessage: vi.fn(async () => null),
469
+ async retryRuntimeProcessCleanup() {
470
+ this.cleanupCalls += 1;
471
+ await prisma.executionProcess.update({
472
+ where: { id: processRecord.id },
473
+ data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
474
+ });
475
+ },
476
+ };
477
+ const cleanupService = new TeamReconcilerService({
478
+ scheduler,
479
+ sessionMessenger: messenger,
480
+ now: () => new Date(Date.UTC(2026, 0, 1, 0, 0, 0)),
481
+ scheduleReminders: false,
482
+ });
483
+ await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
484
+ expect(messenger.cleanupCalls).toBe(1);
485
+ await expect(prisma.executionProcess.findUnique({ where: { id: processRecord.id } })).resolves.toMatchObject({
486
+ cleanupState: 'CONFIRMED',
487
+ });
488
+ });
489
+ it('does not reclaim an ACTIVE runtime still owned by this server generation', async () => {
490
+ const { workspace, teamRun, members } = await createFixture();
491
+ const request = await createWorkRequest({
492
+ teamRunId: teamRun.id,
493
+ targetMemberId: members[0].id,
494
+ status: 'STARTED',
495
+ });
496
+ const invocation = await createRunningInvocation({
497
+ teamRunId: teamRun.id,
498
+ workRequestId: request.id,
499
+ memberId: members[0].id,
500
+ workspaceId: workspace.id,
501
+ status: 'RUNNING',
502
+ });
503
+ await prisma.executionProcess.create({
504
+ data: {
505
+ sessionId: invocation.sessionId,
506
+ launchClaimNumber: 1,
507
+ runtimeInstanceId: 'runtime-current',
508
+ pid: 8201,
509
+ processGroupId: '8201',
510
+ birthMarker: 'linux:201:owner-current',
511
+ ownershipToken: 'owner-current',
512
+ cleanupState: 'ACTIVE',
513
+ },
514
+ });
515
+ const retryRuntimeProcessCleanup = vi.fn(async () => undefined);
516
+ const cleanupService = new TeamReconcilerService({
517
+ scheduler,
518
+ sessionMessenger: {
519
+ sendMessage: vi.fn(async () => null),
520
+ retryRuntimeProcessCleanup,
521
+ hasRuntimeProcessOwner: (runtimeInstanceId) => runtimeInstanceId === 'runtime-current',
522
+ },
523
+ scheduleReminders: false,
524
+ });
525
+ await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(0);
526
+ expect(retryRuntimeProcessCleanup).not.toHaveBeenCalled();
527
+ });
528
+ it('recovers an ACTIVE runtime left by a prior server generation', async () => {
529
+ const { workspace, teamRun, members } = await createFixture();
530
+ const request = await createWorkRequest({
531
+ teamRunId: teamRun.id,
532
+ targetMemberId: members[0].id,
533
+ status: 'COMPLETED',
534
+ });
535
+ const invocation = await createRunningInvocation({
536
+ teamRunId: teamRun.id,
537
+ workRequestId: request.id,
538
+ memberId: members[0].id,
539
+ workspaceId: workspace.id,
540
+ status: 'COMPLETED',
541
+ });
542
+ const processRecord = await prisma.executionProcess.create({
543
+ data: {
544
+ sessionId: invocation.sessionId,
545
+ launchClaimNumber: 1,
546
+ runtimeInstanceId: 'runtime-stale-active',
547
+ pid: 8202,
548
+ processGroupId: '8202',
549
+ birthMarker: 'linux:202:owner-stale',
550
+ ownershipToken: 'owner-stale',
551
+ cleanupState: 'ACTIVE',
552
+ },
553
+ });
554
+ const retryRuntimeProcessCleanup = vi.fn(async () => {
555
+ await prisma.executionProcess.update({
556
+ where: { id: processRecord.id },
557
+ data: { cleanupState: 'CONFIRMED' },
558
+ });
559
+ });
560
+ const cleanupService = new TeamReconcilerService({
561
+ scheduler,
562
+ sessionMessenger: {
563
+ sendMessage: vi.fn(async () => null),
564
+ retryRuntimeProcessCleanup,
565
+ hasRuntimeProcessOwner: () => false,
566
+ disposeRuntimeSession: vi.fn(async () => undefined),
567
+ },
568
+ scheduleReminders: false,
569
+ });
570
+ await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
571
+ expect(retryRuntimeProcessCleanup).toHaveBeenCalledWith(expect.objectContaining({
572
+ runtimeInstanceId: 'runtime-stale-active',
573
+ ownershipToken: 'owner-stale',
574
+ }));
575
+ });
576
+ it('keeps stop admission revoked until a failed cleanup retry succeeds', async () => {
577
+ const { workspace, teamRun, members } = await createFixture();
578
+ const request = await createWorkRequest({
579
+ teamRunId: teamRun.id,
580
+ targetMemberId: members[0].id,
581
+ status: 'STARTED',
582
+ });
583
+ const invocation = await createRunningInvocation({
584
+ teamRunId: teamRun.id,
585
+ workRequestId: request.id,
586
+ memberId: members[0].id,
587
+ workspaceId: workspace.id,
588
+ status: 'RUNNING',
589
+ });
590
+ await prisma.agentInvocation.update({
591
+ where: { id: invocation.id },
592
+ data: { dispatchRevokedAt: new Date() },
593
+ });
594
+ const processRecord = await prisma.executionProcess.create({
595
+ data: {
596
+ sessionId: invocation.sessionId,
597
+ launchClaimNumber: 1,
598
+ runtimeInstanceId: 'runtime-stop-retry',
599
+ pid: 8203,
600
+ processGroupId: '8203',
601
+ birthMarker: 'linux:203:owner-stop-retry',
602
+ ownershipToken: 'owner-stop-retry',
603
+ cleanupState: 'FAILED',
604
+ nextCleanupRetryAt: new Date(0),
605
+ },
606
+ });
607
+ const cleanupService = new TeamReconcilerService({
608
+ scheduler,
609
+ sessionMessenger: {
610
+ sendMessage: vi.fn(async () => null),
611
+ retryRuntimeProcessCleanup: vi.fn(async () => {
612
+ await prisma.executionProcess.update({
613
+ where: { id: processRecord.id },
614
+ data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
615
+ });
616
+ }),
617
+ stop: vi.fn(async () => prisma.session.update({
618
+ where: { id: invocation.sessionId },
619
+ data: { status: 'CANCELLED' },
620
+ })),
621
+ hasRuntimeProcessOwner: () => false,
622
+ hasActiveTurn: () => false,
623
+ disposeRuntimeSession: vi.fn(async () => undefined),
624
+ },
625
+ scheduleReminders: false,
626
+ });
627
+ await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
628
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
629
+ status: 'CANCELLED',
630
+ dispatchRevokedAt: expect.any(Date),
631
+ });
632
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
633
+ status: 'CANCELLED',
634
+ });
635
+ expect(scheduler.startNextSessions).toHaveBeenCalledWith(teamRun.id);
636
+ });
637
+ it('terminalizes a revoked active invocation after confirmed cleanup when DB recovery previously failed', async () => {
638
+ const { workspace, teamRun, members } = await createFixture();
639
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
640
+ const invocation = await createRunningInvocation({
641
+ teamRunId: teamRun.id,
642
+ workRequestId: request.id,
643
+ memberId: members[0].id,
644
+ workspaceId: workspace.id,
645
+ status: 'RUNNING',
646
+ });
647
+ await prisma.session.update({ where: { id: invocation.sessionId }, data: { status: 'RUNNING' } });
648
+ await prisma.agentInvocation.update({
649
+ where: { id: invocation.id },
650
+ data: { dispatchRevokedAt: new Date() },
651
+ });
652
+ await prisma.executionProcess.create({
653
+ data: {
654
+ sessionId: invocation.sessionId,
655
+ launchClaimNumber: 1,
656
+ runtimeInstanceId: 'runtime-confirmed-recovery',
657
+ pid: 8204,
658
+ processGroupId: '8204',
659
+ birthMarker: 'linux:204:owner-confirmed-recovery',
660
+ ownershipToken: 'owner-confirmed-recovery',
661
+ cleanupState: 'CONFIRMED',
662
+ },
663
+ });
664
+ const cleanupService = new TeamReconcilerService({
665
+ scheduler,
666
+ sessionMessenger: {
667
+ sendMessage: vi.fn(async () => null),
668
+ hasRuntimeProcessOwner: () => false,
669
+ hasActiveTurn: () => false,
670
+ disposeRuntimeSession: vi.fn(async () => undefined),
671
+ },
672
+ scheduleReminders: false,
673
+ });
674
+ await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(0);
675
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
676
+ status: 'CANCELLED',
677
+ dispatchRevokedAt: expect.any(Date),
678
+ });
679
+ await expect(prisma.session.findUnique({ where: { id: invocation.sessionId } })).resolves.toMatchObject({
680
+ status: 'CANCELLED',
681
+ });
682
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
683
+ status: 'CANCELLED',
684
+ });
685
+ expect(scheduler.releaseInvocationLocks).toHaveBeenCalledWith(invocation.id);
686
+ expect(scheduler.startNextSessions).toHaveBeenCalledWith(teamRun.id);
687
+ });
688
+ it('recovers a revoked invocation beyond the old confirmed-process page limit', async () => {
689
+ const { workspace, teamRun, members } = await createFixture();
690
+ for (let index = 0; index < 55; index += 1) {
691
+ const historicalSession = await prisma.session.create({
692
+ data: {
693
+ workspaceId: workspace.id,
694
+ agentType: 'CODEX',
695
+ runtimeType: 'ACP',
696
+ prompt: `historical ${index}`,
697
+ status: 'COMPLETED',
698
+ },
699
+ });
700
+ await prisma.executionProcess.create({
701
+ data: {
702
+ sessionId: historicalSession.id,
703
+ launchClaimNumber: 1,
704
+ runtimeInstanceId: `historical-runtime-${index}`,
705
+ pid: 9000 + index,
706
+ processGroupId: String(9000 + index),
707
+ birthMarker: `test-birth:${9000 + index}`,
708
+ ownershipToken: `historical-owner-${index}`,
709
+ cleanupState: 'CONFIRMED',
710
+ },
711
+ });
712
+ }
713
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
714
+ const invocation = await createRunningInvocation({
715
+ teamRunId: teamRun.id,
716
+ workRequestId: request.id,
717
+ memberId: members[0].id,
718
+ workspaceId: workspace.id,
719
+ status: 'RUNNING',
720
+ });
721
+ await prisma.agentInvocation.update({
722
+ where: { id: invocation.id },
723
+ data: { dispatchRevokedAt: new Date() },
724
+ });
725
+ await prisma.executionProcess.create({
726
+ data: {
727
+ sessionId: invocation.sessionId,
728
+ launchClaimNumber: 1,
729
+ runtimeInstanceId: 'target-confirmed-runtime',
730
+ pid: 9999,
731
+ processGroupId: '9999',
732
+ birthMarker: 'test-birth:9999',
733
+ ownershipToken: 'target-confirmed-owner',
734
+ cleanupState: 'CONFIRMED',
735
+ },
736
+ });
737
+ const cleanupService = new TeamReconcilerService({
738
+ scheduler,
739
+ sessionMessenger: {
740
+ sendMessage: vi.fn(async () => null),
741
+ hasRuntimeProcessOwner: () => false,
742
+ hasActiveTurn: () => false,
743
+ },
744
+ scheduleReminders: false,
745
+ });
746
+ await expect(cleanupService.reconcilePendingRuntimeCleanup(50)).resolves.toBe(0);
747
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
748
+ status: 'CANCELLED',
749
+ });
750
+ });
751
+ it.each(['ACTIVE', 'PENDING', 'FAILED'])('does not terminalize a revoked invocation with an old CONFIRMED runtime and newer %s runtime', async (newerCleanupState) => {
752
+ const { workspace, teamRun, members } = await createFixture();
753
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
754
+ const invocation = await createRunningInvocation({
755
+ teamRunId: teamRun.id,
756
+ workRequestId: request.id,
757
+ memberId: members[0].id,
758
+ workspaceId: workspace.id,
759
+ status: 'RUNNING',
760
+ });
761
+ await prisma.agentInvocation.update({
762
+ where: { id: invocation.id },
763
+ data: { dispatchRevokedAt: new Date() },
764
+ });
765
+ await prisma.executionProcess.createMany({
766
+ data: [
767
+ {
768
+ sessionId: invocation.sessionId,
769
+ launchClaimNumber: 1,
770
+ runtimeInstanceId: `old-confirmed-${newerCleanupState}`,
771
+ pid: 9101,
772
+ processGroupId: '9101',
773
+ birthMarker: `test-birth:old-${newerCleanupState}`,
774
+ ownershipToken: `old-owner-${newerCleanupState}`,
775
+ cleanupState: 'CONFIRMED',
776
+ },
777
+ {
778
+ sessionId: invocation.sessionId,
779
+ launchClaimNumber: 2,
780
+ runtimeInstanceId: `new-${newerCleanupState}`,
781
+ pid: 9102,
782
+ processGroupId: '9102',
783
+ birthMarker: `test-birth:new-${newerCleanupState}`,
784
+ ownershipToken: `new-owner-${newerCleanupState}`,
785
+ cleanupState: newerCleanupState,
786
+ },
787
+ ],
788
+ });
789
+ const cleanupService = new TeamReconcilerService({
790
+ scheduler,
791
+ sessionMessenger: {
792
+ sendMessage: vi.fn(async () => null),
793
+ hasRuntimeProcessOwner: () => false,
794
+ hasActiveTurn: () => false,
795
+ },
796
+ scheduleReminders: false,
797
+ });
798
+ await cleanupService.reconcilePendingRuntimeCleanup();
799
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
800
+ status: 'RUNNING',
801
+ dispatchRevokedAt: expect.any(Date),
802
+ });
803
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
804
+ status: 'STARTED',
805
+ });
806
+ });
807
+ it('keeps post-confirm revocation recovery idempotent', async () => {
808
+ const { workspace, teamRun, members } = await createFixture();
809
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
810
+ const invocation = await createRunningInvocation({
811
+ teamRunId: teamRun.id,
812
+ workRequestId: request.id,
813
+ memberId: members[0].id,
814
+ workspaceId: workspace.id,
815
+ status: 'RUNNING',
816
+ });
817
+ await prisma.agentInvocation.update({
818
+ where: { id: invocation.id },
819
+ data: { dispatchRevokedAt: new Date() },
820
+ });
821
+ await prisma.executionProcess.create({
822
+ data: {
823
+ sessionId: invocation.sessionId,
824
+ launchClaimNumber: 1,
825
+ runtimeInstanceId: 'idempotent-confirmed-runtime',
826
+ pid: 9201,
827
+ processGroupId: '9201',
828
+ birthMarker: 'test-birth:9201',
829
+ ownershipToken: 'idempotent-owner',
830
+ cleanupState: 'CONFIRMED',
831
+ },
832
+ });
833
+ const cleanupService = new TeamReconcilerService({
834
+ scheduler,
835
+ sessionMessenger: {
836
+ sendMessage: vi.fn(async () => null),
837
+ hasRuntimeProcessOwner: () => false,
838
+ hasActiveTurn: () => false,
839
+ },
840
+ scheduleReminders: false,
841
+ });
842
+ await cleanupService.reconcilePendingRuntimeCleanup();
843
+ const releaseCount = vi.mocked(scheduler.releaseInvocationLocks).mock.calls.length;
844
+ const startCount = vi.mocked(scheduler.startNextSessions).mock.calls.length;
845
+ await cleanupService.reconcilePendingRuntimeCleanup();
846
+ expect(scheduler.releaseInvocationLocks).toHaveBeenCalledTimes(releaseCount);
847
+ expect(scheduler.startNextSessions).toHaveBeenCalledTimes(startCount);
848
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
849
+ status: 'CANCELLED',
850
+ });
851
+ });
852
+ it('proactively quarantines historical process rows with incomplete ownership', async () => {
853
+ vi.spyOn(console, 'warn').mockImplementation(() => undefined);
854
+ const { workspace } = await createFixture();
855
+ const session = await prisma.session.create({
856
+ data: {
857
+ workspaceId: workspace.id,
858
+ agentType: 'CODEX',
859
+ prompt: 'legacy incomplete ownership',
860
+ status: 'RUNNING',
861
+ },
862
+ });
863
+ const processRecord = await prisma.executionProcess.create({
864
+ data: {
865
+ sessionId: session.id,
866
+ launchClaimNumber: 1,
867
+ runtimeInstanceId: 'legacy-incomplete-runtime',
868
+ pid: 9251,
869
+ processGroupId: '9251',
870
+ birthMarker: null,
871
+ ownershipToken: 'legacy-incomplete-owner',
872
+ cleanupState: 'ACTIVE',
873
+ },
874
+ });
875
+ const retryRuntimeProcessCleanup = vi.fn(async () => undefined);
876
+ const cleanupService = new TeamReconcilerService({
877
+ scheduler,
878
+ sessionMessenger: {
879
+ sendMessage: vi.fn(async () => null),
880
+ hasActiveTurn: () => false,
881
+ hasRuntimeProcessOwner: () => false,
882
+ retryRuntimeProcessCleanup,
883
+ },
884
+ scheduleReminders: false,
885
+ });
886
+ await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
887
+ expect(retryRuntimeProcessCleanup).not.toHaveBeenCalled();
888
+ await expect(prisma.executionProcess.findUniqueOrThrow({ where: { id: processRecord.id } })).resolves.toMatchObject({
889
+ cleanupState: 'QUARANTINED',
890
+ cleanupError: expect.stringContaining('identity is incomplete'),
891
+ cleanupAttemptCount: 2,
892
+ nextCleanupRetryAt: expect.any(Date),
893
+ });
894
+ });
895
+ it('does not downgrade confirmed cleanup when a stale gate sees incomplete metadata', async () => {
896
+ const { workspace } = await createFixture();
897
+ const session = await prisma.session.create({
898
+ data: {
899
+ workspaceId: workspace.id,
900
+ agentType: 'CODEX',
901
+ prompt: 'confirmed legacy cleanup',
902
+ status: 'COMPLETED',
903
+ },
904
+ });
905
+ const processRecord = await prisma.executionProcess.create({
906
+ data: {
907
+ sessionId: session.id,
908
+ launchClaimNumber: 1,
909
+ runtimeInstanceId: 'confirmed-legacy-runtime',
910
+ pid: 9252,
911
+ processGroupId: '9252',
912
+ birthMarker: null,
913
+ ownershipToken: 'confirmed-legacy-owner',
914
+ cleanupState: 'CONFIRMED',
915
+ },
916
+ });
917
+ const cleanupService = new TeamReconcilerService({
918
+ scheduler,
919
+ sessionMessenger: {
920
+ sendMessage: vi.fn(async () => null),
921
+ hasRuntimeProcessOwner: () => false,
922
+ },
923
+ scheduleReminders: false,
924
+ });
925
+ await expect(cleanupService.isSessionRuntimeCleanupConfirmed(session.id)).resolves.toBe(false);
926
+ await expect(prisma.executionProcess.findUniqueOrThrow({ where: { id: processRecord.id } })).resolves.toMatchObject({
927
+ cleanupState: 'CONFIRMED',
928
+ cleanupAttemptCount: 0,
929
+ cleanupError: null,
930
+ });
931
+ });
932
+ it('re-reads a new started generation after a quarantine CAS loses the race', async () => {
933
+ const { workspace } = await createFixture();
934
+ const session = await prisma.session.create({
935
+ data: {
936
+ workspaceId: workspace.id,
937
+ agentType: 'CODEX',
938
+ prompt: 'CAS generation race',
939
+ status: 'RUNNING',
940
+ runtimeLaunchState: 'CLAIMED',
941
+ runtimeLaunchClaimCount: 1,
942
+ },
943
+ });
944
+ let injected = false;
945
+ prisma.$use(async (params, next) => {
946
+ const where = params.args?.where;
947
+ if (!injected
948
+ && params.model === 'Session'
949
+ && params.action === 'updateMany'
950
+ && where?.id === session.id
951
+ && where.runtimeLaunchState === 'CLAIMED') {
952
+ injected = true;
953
+ await prisma.session.update({
954
+ where: { id: session.id },
955
+ data: {
956
+ runtimeLaunchState: 'PROCESS_RECORDED',
957
+ runtimeLaunchResolvedCount: 1,
958
+ runtimeLaunchProcessCount: 1,
959
+ },
960
+ });
961
+ await prisma.executionProcess.create({
962
+ data: {
963
+ sessionId: session.id,
964
+ launchClaimNumber: 1,
965
+ runtimeInstanceId: 'cas-new-runtime',
966
+ pid: 9253,
967
+ processGroupId: '9253',
968
+ birthMarker: 'test-birth:cas-new',
969
+ ownershipToken: 'cas-new-owner',
970
+ cleanupState: 'CONFIRMED',
971
+ },
972
+ });
973
+ }
974
+ return next(params);
975
+ });
976
+ const cleanupService = new TeamReconcilerService({
977
+ scheduler,
978
+ sessionMessenger: {
979
+ sendMessage: vi.fn(async () => null),
980
+ hasRuntimeProcessOwner: () => false,
981
+ },
982
+ scheduleReminders: false,
983
+ });
984
+ await expect(cleanupService.isSessionRuntimeCleanupConfirmed(session.id)).resolves.toBe(true);
985
+ expect(injected).toBe(true);
986
+ await expect(prisma.session.findUniqueOrThrow({ where: { id: session.id } })).resolves.toMatchObject({
987
+ runtimeLaunchState: 'PROCESS_RECORDED',
988
+ runtimeLaunchClaimCount: 1,
989
+ runtimeLaunchResolvedCount: 1,
990
+ runtimeLaunchProcessCount: 1,
991
+ });
992
+ });
993
+ it.each([
994
+ ['PENDING cleanup', 'PENDING', false, false],
995
+ ['FAILED cleanup', 'FAILED', false, false],
996
+ ['ACTIVE cleanup', 'ACTIVE', false, false],
997
+ ['current runtime owner', 'CONFIRMED', true, false],
998
+ ['incomplete ownership identity', 'ACTIVE', false, true],
999
+ ])('keeps manual stop pending for %s until the unified cleanup gate confirms', async (_caseName, cleanupState, initialOwnerPresent, incompleteIdentity) => {
1000
+ const { workspace, task, teamRun, members } = await createFixture({
1001
+ taskStatus: TaskStatus.IN_PROGRESS,
1002
+ memberCount: 2,
1003
+ });
1004
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1005
+ const invocation = await createRunningInvocation({
1006
+ teamRunId: teamRun.id,
1007
+ workRequestId: request.id,
1008
+ memberId: members[0].id,
1009
+ workspaceId: workspace.id,
1010
+ status: 'RUNNING',
1011
+ });
1012
+ const queued = await createWorkRequest({
1013
+ teamRunId: teamRun.id,
1014
+ targetMemberId: members[1].id,
1015
+ status: 'QUEUED',
1016
+ });
1017
+ const processRecord = await prisma.executionProcess.create({
1018
+ data: {
1019
+ sessionId: invocation.sessionId,
1020
+ launchClaimNumber: 1,
1021
+ runtimeInstanceId: incompleteIdentity ? null : `manual-${cleanupState.toLowerCase()}`,
1022
+ pid: 9301,
1023
+ processGroupId: incompleteIdentity ? null : '9301',
1024
+ birthMarker: incompleteIdentity ? null : `test-birth:manual-${cleanupState.toLowerCase()}`,
1025
+ ownershipToken: incompleteIdentity ? null : `manual-owner-${cleanupState.toLowerCase()}`,
1026
+ cleanupState,
1027
+ },
1028
+ });
1029
+ expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
1030
+ let ownerPresent = initialOwnerPresent;
1031
+ const retryRuntimeProcessCleanup = vi.fn(async () => undefined);
1032
+ const stopService = new TeamReconcilerService({
1033
+ scheduler,
1034
+ sessionMessenger: {
1035
+ sendMessage: vi.fn(async () => null),
1036
+ hasActiveTurn: () => false,
1037
+ hasRuntimeProcessOwner: () => ownerPresent,
1038
+ retryRuntimeProcessCleanup,
1039
+ disposeRuntimeSession: vi.fn(async () => undefined),
1040
+ },
1041
+ now: () => new Date(Date.UTC(2026, 0, 1, 0, 0, 0)),
1042
+ scheduleReminders: false,
1043
+ });
1044
+ await expect(stopService.handleSessionStopped(invocation.sessionId)).resolves.toEqual([]);
1045
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1046
+ status: 'RUNNING',
1047
+ dispatchRevokedAt: expect.any(Date),
1048
+ });
1049
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
1050
+ status: 'STARTED',
1051
+ });
1052
+ await expect(prisma.workRequest.findUnique({ where: { id: queued.id } })).resolves.toMatchObject({
1053
+ status: 'QUEUED',
1054
+ });
1055
+ await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
1056
+ status: TaskStatus.IN_PROGRESS,
1057
+ });
1058
+ expect(lockService.listLocks()).toEqual([
1059
+ { key: 'workspace:task:write', ownerId: invocation.id },
1060
+ ]);
1061
+ expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
1062
+ expect(scheduler.startNextSessions).not.toHaveBeenCalled();
1063
+ if (incompleteIdentity) {
1064
+ await expect(prisma.executionProcess.findUnique({ where: { id: processRecord.id } })).resolves.toMatchObject({
1065
+ cleanupState: 'QUARANTINED',
1066
+ cleanupError: expect.stringContaining('identity is incomplete'),
1067
+ cleanupAttemptCount: 1,
1068
+ nextCleanupRetryAt: expect.any(Date),
1069
+ });
1070
+ await prisma.executionProcess.update({
1071
+ where: { id: processRecord.id },
1072
+ data: { nextCleanupRetryAt: new Date(0) },
1073
+ });
1074
+ await expect(stopService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
1075
+ expect(retryRuntimeProcessCleanup).not.toHaveBeenCalled();
1076
+ await expect(prisma.executionProcess.findUnique({ where: { id: processRecord.id } })).resolves.toMatchObject({
1077
+ cleanupState: 'QUARANTINED',
1078
+ cleanupAttemptCount: 2,
1079
+ });
1080
+ }
1081
+ ownerPresent = false;
1082
+ await prisma.executionProcess.update({
1083
+ where: { id: processRecord.id },
1084
+ data: {
1085
+ runtimeInstanceId: `manual-confirmed-${cleanupState.toLowerCase()}`,
1086
+ processGroupId: '9301',
1087
+ birthMarker: `test-birth:manual-confirmed-${cleanupState.toLowerCase()}`,
1088
+ ownershipToken: `manual-confirmed-owner-${cleanupState.toLowerCase()}`,
1089
+ cleanupState: 'CONFIRMED',
1090
+ cleanupError: null,
1091
+ nextCleanupRetryAt: null,
1092
+ },
1093
+ });
1094
+ await expect(stopService.handleSessionStopped(invocation.sessionId)).resolves.toEqual([]);
1095
+ const releaseCount = vi.mocked(scheduler.releaseInvocationLocks).mock.calls.length;
1096
+ const startCount = vi.mocked(scheduler.startNextSessions).mock.calls.length;
1097
+ await expect(stopService.handleSessionStopped(invocation.sessionId)).resolves.toEqual([]);
1098
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1099
+ status: 'CANCELLED',
1100
+ });
1101
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
1102
+ status: 'CANCELLED',
1103
+ });
1104
+ expect(lockService.listLocks()).toEqual([]);
1105
+ expect(vi.mocked(scheduler.releaseInvocationLocks).mock.calls).toEqual([[invocation.id]]);
1106
+ expect(scheduler.startNextSessions).toHaveBeenCalledTimes(1);
1107
+ expect(releaseCount).toBe(1);
1108
+ expect(startCount).toBe(1);
1109
+ });
1110
+ it.each([
1111
+ ['cleanup failure', 'FAILED', false],
1112
+ ['current owner', 'CONFIRMED', true],
1113
+ ])('keeps incomplete terminal recovery pending for %s and completes it exactly once after confirmation', async (_caseName, cleanupState, initialOwnerPresent) => {
1114
+ const { workspace, task, teamRun, members } = await createFixture({ taskStatus: TaskStatus.IN_PROGRESS });
1115
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1116
+ const invocation = await createRunningInvocation({
1117
+ teamRunId: teamRun.id,
1118
+ workRequestId: request.id,
1119
+ memberId: members[0].id,
1120
+ workspaceId: workspace.id,
1121
+ status: 'COMPLETED',
1122
+ });
1123
+ const processRecord = await prisma.executionProcess.create({
1124
+ data: {
1125
+ sessionId: invocation.sessionId,
1126
+ launchClaimNumber: 1,
1127
+ runtimeInstanceId: `incomplete-terminal-${cleanupState.toLowerCase()}`,
1128
+ pid: 9401,
1129
+ processGroupId: '9401',
1130
+ birthMarker: `test-birth:incomplete-terminal-${cleanupState.toLowerCase()}`,
1131
+ ownershipToken: `incomplete-terminal-owner-${cleanupState.toLowerCase()}`,
1132
+ cleanupState,
1133
+ },
1134
+ });
1135
+ expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
1136
+ let ownerPresent = initialOwnerPresent;
1137
+ const recoveryService = new TeamReconcilerService({
1138
+ scheduler,
1139
+ sessionMessenger: {
1140
+ sendMessage: vi.fn(async () => null),
1141
+ hasActiveTurn: () => false,
1142
+ hasRuntimeProcessOwner: () => ownerPresent,
1143
+ disposeRuntimeSession: vi.fn(async () => undefined),
1144
+ },
1145
+ scheduleReminders: false,
1146
+ });
1147
+ await recoveryService.reconcileIncompleteTerminalInvocations();
1148
+ await recoveryService.reconcileIncompleteTerminalInvocations();
1149
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
1150
+ status: 'STARTED',
1151
+ });
1152
+ await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
1153
+ status: TaskStatus.IN_PROGRESS,
1154
+ });
1155
+ expect(lockService.listLocks()).toEqual([
1156
+ { key: 'workspace:task:write', ownerId: invocation.id },
1157
+ ]);
1158
+ expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
1159
+ expect(scheduler.startNextSessions).not.toHaveBeenCalled();
1160
+ ownerPresent = false;
1161
+ await prisma.executionProcess.update({
1162
+ where: { id: processRecord.id },
1163
+ data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
1164
+ });
1165
+ await recoveryService.reconcileIncompleteTerminalInvocations();
1166
+ await recoveryService.reconcileIncompleteTerminalInvocations();
1167
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
1168
+ status: 'COMPLETED',
1169
+ });
1170
+ expect(lockService.listLocks()).toEqual([]);
1171
+ expect(vi.mocked(scheduler.releaseInvocationLocks).mock.calls).toEqual([[invocation.id]]);
1172
+ expect(scheduler.startNextSessions).toHaveBeenCalledTimes(1);
1173
+ });
1174
+ it('does not nudge a stalled invocation after stop revokes dispatch', async () => {
1175
+ const { workspace, teamRun, members } = await createFixture();
1176
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1177
+ const invocation = await createRunningInvocation({
1178
+ teamRunId: teamRun.id,
1179
+ workRequestId: request.id,
1180
+ memberId: members[0].id,
1181
+ workspaceId: workspace.id,
1182
+ status: 'RUNNING',
1183
+ });
1184
+ await prisma.agentInvocation.update({
1185
+ where: { id: invocation.id },
1186
+ data: { dispatchRevokedAt: new Date(), lastHeartbeatAt: new Date(0) },
1187
+ });
1188
+ const sendMessage = vi.fn(async () => null);
1189
+ const stalledService = new TeamReconcilerService({
1190
+ scheduler,
1191
+ sessionMessenger: {
1192
+ sendMessage,
1193
+ hasActiveTurn: () => true,
1194
+ },
1195
+ now: () => new Date(Date.UTC(2026, 0, 1, 1, 0, 0)),
1196
+ heartbeatIdleThresholdMs: 1,
1197
+ scheduleReminders: false,
1198
+ });
1199
+ await stalledService.reconcileStalledInvocations();
1200
+ expect(sendMessage).not.toHaveBeenCalled();
1201
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1202
+ dispatchRevokedAt: expect.any(Date),
1203
+ roomReplyReminderCount: 0,
1204
+ });
1205
+ });
1206
+ it('does not overwrite a manual CANCELLED winner during orphan recovery', async () => {
1207
+ const { workspace, teamRun, members } = await createFixture();
1208
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1209
+ const invocation = await createRunningInvocation({
1210
+ teamRunId: teamRun.id,
1211
+ workRequestId: request.id,
1212
+ memberId: members[0].id,
1213
+ workspaceId: workspace.id,
1214
+ status: 'RUNNING',
1215
+ });
1216
+ const orphanService = new TeamReconcilerService({
1217
+ scheduler,
1218
+ sessionMessenger: {
1219
+ sendMessage: vi.fn(async () => null),
1220
+ hasActiveTurn: () => false,
1221
+ },
1222
+ scheduleReminders: false,
1223
+ });
1224
+ const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
1225
+ const recovery = orphanService.reconcileOrphanInvocations();
1226
+ await Promise.resolve();
1227
+ await prisma.agentInvocation.update({
1228
+ where: { id: invocation.id },
1229
+ data: { status: 'CANCELLED', dispatchRevokedAt: new Date() },
1230
+ });
1231
+ releaseAdmission();
1232
+ await expect(recovery).resolves.toBeUndefined();
1233
+ await expect(orphanService.reconcileOrphanInvocations()).resolves.toBeUndefined();
1234
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1235
+ status: 'CANCELLED',
1236
+ dispatchRevokedAt: expect.any(Date),
1237
+ });
1238
+ expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalledWith(invocation.id);
1239
+ expect(scheduler.startNextSessions).not.toHaveBeenCalledWith(teamRun.id);
1240
+ });
1241
+ it.each(['PENDING', 'FAILED'])('keeps an orphan invocation pending until %s runtime cleanup is confirmed', async (cleanupState) => {
1242
+ const { workspace, task, teamRun, members } = await createFixture();
1243
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1244
+ const invocation = await createRunningInvocation({
1245
+ teamRunId: teamRun.id,
1246
+ workRequestId: request.id,
1247
+ memberId: members[0].id,
1248
+ workspaceId: workspace.id,
1249
+ status: 'RUNNING',
1250
+ });
1251
+ await prisma.executionProcess.create({
1252
+ data: {
1253
+ sessionId: invocation.sessionId,
1254
+ launchClaimNumber: 1,
1255
+ runtimeInstanceId: `orphan-${cleanupState.toLowerCase()}`,
1256
+ pid: 8301,
1257
+ processGroupId: '8301',
1258
+ birthMarker: `test-birth:orphan-${cleanupState.toLowerCase()}`,
1259
+ ownershipToken: `orphan-owner-${cleanupState.toLowerCase()}`,
1260
+ cleanupState,
1261
+ },
1262
+ });
1263
+ expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
1264
+ const orphanService = new TeamReconcilerService({
1265
+ scheduler,
1266
+ sessionMessenger: {
1267
+ sendMessage: vi.fn(async () => null),
1268
+ hasActiveTurn: () => false,
1269
+ hasRuntimeProcessOwner: () => false,
1270
+ },
1271
+ scheduleReminders: false,
1272
+ });
1273
+ await orphanService.reconcileOrphanInvocations();
1274
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1275
+ status: 'RUNNING',
1276
+ dispatchRevokedAt: expect.any(Date),
1277
+ });
1278
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
1279
+ status: 'STARTED',
1280
+ });
1281
+ await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
1282
+ status: TaskStatus.IN_PROGRESS,
1283
+ });
1284
+ expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
1285
+ expect(scheduler.startNextSessions).not.toHaveBeenCalled();
1286
+ await prisma.executionProcess.updateMany({
1287
+ where: { sessionId: invocation.sessionId },
1288
+ data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
1289
+ });
1290
+ await orphanService.reconcilePendingRuntimeCleanup();
1291
+ const releaseCount = scheduler.releaseInvocationLocks.mock.calls.length;
1292
+ const startCount = scheduler.startNextSessions.mock.calls.length;
1293
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1294
+ status: 'CANCELLED',
1295
+ });
1296
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
1297
+ status: 'CANCELLED',
1298
+ });
1299
+ await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
1300
+ status: TaskStatus.IN_REVIEW,
1301
+ });
1302
+ await orphanService.reconcilePendingRuntimeCleanup();
1303
+ expect(scheduler.releaseInvocationLocks).toHaveBeenCalledTimes(releaseCount);
1304
+ expect(scheduler.startNextSessions).toHaveBeenCalledTimes(startCount);
1305
+ });
1306
+ it.each(['PENDING', 'FAILED'])('keeps a stalled invocation pending until %s runtime cleanup is confirmed', async (cleanupState) => {
1307
+ const { workspace, task, teamRun, members } = await createFixture();
1308
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1309
+ const invocation = await createRunningInvocation({
1310
+ teamRunId: teamRun.id,
1311
+ workRequestId: request.id,
1312
+ memberId: members[0].id,
1313
+ workspaceId: workspace.id,
1314
+ status: 'RUNNING',
1315
+ });
1316
+ await prisma.executionProcess.create({
1317
+ data: {
1318
+ sessionId: invocation.sessionId,
1319
+ launchClaimNumber: 1,
1320
+ runtimeInstanceId: `stalled-${cleanupState.toLowerCase()}`,
1321
+ pid: 8401,
1322
+ processGroupId: '8401',
1323
+ birthMarker: `test-birth:stalled-${cleanupState.toLowerCase()}`,
1324
+ ownershipToken: `stalled-owner-${cleanupState.toLowerCase()}`,
1325
+ cleanupState,
1326
+ },
1327
+ });
1328
+ expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
1329
+ const stalledService = new TeamReconcilerService({
1330
+ scheduler,
1331
+ sessionMessenger: {
1332
+ sendMessage: vi.fn(async () => null),
1333
+ hasActiveTurn: () => false,
1334
+ hasRuntimeProcessOwner: () => false,
1335
+ },
1336
+ scheduleReminders: false,
1337
+ });
1338
+ await stalledService.reconcileStalledInvocations();
1339
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1340
+ status: 'RUNNING',
1341
+ dispatchRevokedAt: expect.any(Date),
1342
+ });
1343
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
1344
+ status: 'STARTED',
1345
+ });
1346
+ await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
1347
+ status: TaskStatus.IN_PROGRESS,
1348
+ });
1349
+ expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
1350
+ expect(scheduler.startNextSessions).not.toHaveBeenCalled();
1351
+ await prisma.executionProcess.updateMany({
1352
+ where: { sessionId: invocation.sessionId },
1353
+ data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
1354
+ });
1355
+ await stalledService.reconcilePendingRuntimeCleanup();
1356
+ const releaseCount = scheduler.releaseInvocationLocks.mock.calls.length;
1357
+ const startCount = scheduler.startNextSessions.mock.calls.length;
1358
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1359
+ status: 'CANCELLED',
1360
+ });
1361
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
1362
+ status: 'CANCELLED',
1363
+ });
1364
+ await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
1365
+ status: TaskStatus.IN_REVIEW,
1366
+ });
1367
+ await stalledService.reconcilePendingRuntimeCleanup();
1368
+ expect(scheduler.releaseInvocationLocks).toHaveBeenCalledTimes(releaseCount);
1369
+ expect(scheduler.startNextSessions).toHaveBeenCalledTimes(startCount);
1370
+ });
1371
+ it('terminalizes a never-launched orphan with no process record', async () => {
1372
+ const { workspace, teamRun, members } = await createFixture();
1373
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1374
+ const invocation = await createRunningInvocation({
1375
+ teamRunId: teamRun.id,
1376
+ workRequestId: request.id,
1377
+ memberId: members[0].id,
1378
+ workspaceId: workspace.id,
1379
+ status: 'RUNNING',
1380
+ });
1381
+ const orphanService = new TeamReconcilerService({
1382
+ scheduler,
1383
+ sessionMessenger: { sendMessage: vi.fn(async () => null), hasActiveTurn: () => false },
1384
+ scheduleReminders: false,
1385
+ });
1386
+ await orphanService.reconcileOrphanInvocations();
1387
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1388
+ status: 'FAILED',
1389
+ });
1390
+ await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
1391
+ status: 'FAILED',
1392
+ });
1393
+ expect(scheduler.releaseInvocationLocks).toHaveBeenCalledWith(invocation.id);
1394
+ expect(scheduler.startNextSessions).toHaveBeenCalledWith(teamRun.id);
1395
+ });
1396
+ it('does not duplicate terminal side effects when manual stop wins stalled cleanup', async () => {
1397
+ const { workspace, teamRun, members } = await createFixture();
1398
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1399
+ const invocation = await createRunningInvocation({
1400
+ teamRunId: teamRun.id,
1401
+ workRequestId: request.id,
1402
+ memberId: members[0].id,
1403
+ workspaceId: workspace.id,
1404
+ status: 'RUNNING',
1405
+ });
1406
+ await prisma.executionProcess.create({
1407
+ data: {
1408
+ sessionId: invocation.sessionId,
1409
+ launchClaimNumber: 1,
1410
+ runtimeInstanceId: 'manual-stop-pending-runtime',
1411
+ pid: 8501,
1412
+ processGroupId: '8501',
1413
+ birthMarker: 'test-birth:8501',
1414
+ ownershipToken: 'manual-stop-pending-owner',
1415
+ cleanupState: 'PENDING',
1416
+ },
1417
+ });
1418
+ await prisma.agentInvocation.update({
1419
+ where: { id: invocation.id },
1420
+ data: {
1421
+ lastHeartbeatAt: new Date(0),
1422
+ roomReplyReminderCount: 3,
1423
+ firstNudgeAt: new Date(0),
1424
+ },
1425
+ });
1426
+ let releaseAutomatedStop;
1427
+ const automatedStopStarted = new Promise((resolve) => {
1428
+ releaseAutomatedStop = resolve;
1429
+ });
1430
+ let notifyAutomatedStop;
1431
+ const automatedStopCalled = new Promise((resolve) => {
1432
+ notifyAutomatedStop = resolve;
1433
+ });
1434
+ const stop = vi.fn(async () => {
1435
+ notifyAutomatedStop();
1436
+ await automatedStopStarted;
1437
+ return { id: invocation.sessionId };
1438
+ });
1439
+ const stalledService = new TeamReconcilerService({
1440
+ scheduler,
1441
+ sessionMessenger: {
1442
+ sendMessage: vi.fn(async () => null),
1443
+ hasActiveTurn: () => true,
1444
+ stop,
1445
+ },
1446
+ now: () => new Date(Date.UTC(2026, 0, 1, 1, 0, 0)),
1447
+ heartbeatIdleThresholdMs: 1,
1448
+ scheduleReminders: false,
1449
+ });
1450
+ const recovery = stalledService.reconcileStalledInvocations();
1451
+ await automatedStopCalled;
1452
+ const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
1453
+ try {
1454
+ await prisma.agentInvocation.update({
1455
+ where: { id: invocation.id },
1456
+ data: { status: 'CANCELLED' },
1457
+ });
1458
+ }
1459
+ finally {
1460
+ releaseAdmission();
1461
+ }
1462
+ releaseAutomatedStop();
1463
+ await expect(recovery).resolves.toBeUndefined();
1464
+ await expect(stalledService.reconcileStalledInvocations()).resolves.toBeUndefined();
1465
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1466
+ status: 'CANCELLED',
1467
+ dispatchRevokedAt: expect.any(Date),
1468
+ });
1469
+ expect(stop).toHaveBeenCalledOnce();
1470
+ expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalledWith(invocation.id);
1471
+ expect(scheduler.startNextSessions).not.toHaveBeenCalledWith(teamRun.id);
1472
+ });
1473
+ it('does not claim or send a room reminder when revocation wins after the reminder query', async () => {
1474
+ const { workspace, teamRun, members } = await createFixture();
1475
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1476
+ const invocation = await createRunningInvocation({
1477
+ teamRunId: teamRun.id,
1478
+ workRequestId: request.id,
1479
+ memberId: members[0].id,
1480
+ workspaceId: workspace.id,
1481
+ status: 'WAITING_ROOM_REPLY',
1482
+ roomReplyReminderCount: 1,
1483
+ });
1484
+ await prisma.agentInvocation.update({
1485
+ where: { id: invocation.id },
1486
+ data: { nextRoomReplyReminderAt: new Date(0) },
1487
+ });
1488
+ const sendMessage = vi.fn(async () => null);
1489
+ const reminderService = new TeamReconcilerService({
1490
+ scheduler,
1491
+ sessionMessenger: { sendMessage },
1492
+ now: () => new Date(Date.UTC(2026, 0, 1, 1, 0, 0)),
1493
+ scheduleReminders: false,
1494
+ });
1495
+ const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
1496
+ const reconciling = reminderService.reconcileInvocation(invocation.id);
1497
+ await Promise.resolve();
1498
+ try {
1499
+ await prisma.agentInvocation.update({
1500
+ where: { id: invocation.id },
1501
+ data: { dispatchRevokedAt: new Date() },
1502
+ });
1503
+ }
1504
+ finally {
1505
+ releaseAdmission();
1506
+ }
1507
+ await reconciling;
1508
+ expect(sendMessage).not.toHaveBeenCalled();
1509
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1510
+ status: 'WAITING_ROOM_REPLY',
1511
+ roomReplyReminderCount: 1,
1512
+ dispatchRevokedAt: expect.any(Date),
1513
+ });
1514
+ });
1515
+ it('does not write the max-reminder terminal state when revocation wins before its claim', async () => {
1516
+ const { workspace, teamRun, members } = await createFixture();
1517
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1518
+ const invocation = await createRunningInvocation({
1519
+ teamRunId: teamRun.id,
1520
+ workRequestId: request.id,
1521
+ memberId: members[0].id,
1522
+ workspaceId: workspace.id,
1523
+ status: 'WAITING_ROOM_REPLY',
1524
+ roomReplyReminderCount: 3,
1525
+ });
1526
+ const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
1527
+ const reconciling = service.reconcileInvocation(invocation.id);
1528
+ await Promise.resolve();
1529
+ try {
1530
+ await prisma.agentInvocation.update({
1531
+ where: { id: invocation.id },
1532
+ data: { dispatchRevokedAt: new Date() },
1533
+ });
1534
+ }
1535
+ finally {
1536
+ releaseAdmission();
1537
+ }
1538
+ await reconciling;
1539
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1540
+ status: 'WAITING_ROOM_REPLY',
1541
+ roomReplyReminderCount: 3,
1542
+ dispatchRevokedAt: expect.any(Date),
1543
+ });
1544
+ expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalledWith(invocation.id);
1545
+ });
1546
+ it('does not claim or send a stalled nudge when revocation wins at the conditional claim', async () => {
1547
+ const { workspace, teamRun, members } = await createFixture();
1548
+ const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
1549
+ const invocation = await createRunningInvocation({
1550
+ teamRunId: teamRun.id,
1551
+ workRequestId: request.id,
1552
+ memberId: members[0].id,
1553
+ workspaceId: workspace.id,
1554
+ status: 'RUNNING',
1555
+ });
1556
+ await prisma.agentInvocation.update({
1557
+ where: { id: invocation.id },
1558
+ data: { lastHeartbeatAt: new Date(0) },
1559
+ });
1560
+ const sendMessage = vi.fn(async () => null);
1561
+ const stalledService = new TeamReconcilerService({
1562
+ scheduler,
1563
+ sessionMessenger: { sendMessage, hasActiveTurn: () => true },
1564
+ now: () => new Date(Date.UTC(2026, 0, 1, 1, 0, 0)),
1565
+ heartbeatIdleThresholdMs: 1,
1566
+ scheduleReminders: false,
1567
+ });
1568
+ const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
1569
+ const reconciling = stalledService.reconcileStalledInvocations();
1570
+ await Promise.resolve();
1571
+ try {
1572
+ await prisma.agentInvocation.update({
1573
+ where: { id: invocation.id },
1574
+ data: { dispatchRevokedAt: new Date() },
1575
+ });
1576
+ }
1577
+ finally {
1578
+ releaseAdmission();
1579
+ }
1580
+ await reconciling;
1581
+ expect(sendMessage).not.toHaveBeenCalled();
1582
+ await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
1583
+ status: 'RUNNING',
1584
+ roomReplyReminderCount: 0,
1585
+ dispatchRevokedAt: expect.any(Date),
1586
+ });
1587
+ });
370
1588
  it('marks invocation waiting, increments reminder count, and sends a reminder when no RoomMessage exists', async () => {
371
1589
  const { workspace, teamRun, members } = await createFixture();
372
1590
  const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
@@ -381,7 +1599,7 @@ describe('TeamReconcilerService', () => {
381
1599
  expect(reloaded.status).toBe('WAITING_ROOM_REPLY');
382
1600
  expect(reloaded.roomReplyReminderCount).toBe(1);
383
1601
  expect(reloaded.nextRoomReplyReminderAt?.toISOString()).toBe('2026-01-01T00:00:01.000Z');
384
- expect(messenger.sendMessage).toHaveBeenCalledWith(invocation.sessionId, TEAM_ROOM_REPLY_REMINDER);
1602
+ expect(messenger.sendMessage).toHaveBeenCalledWith(invocation.sessionId, TEAM_ROOM_REPLY_REMINDER, undefined, invocation.id);
385
1603
  expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
386
1604
  });
387
1605
  it('ignores forged non-agent RoomMessages with senderInvocationId during reconcile', async () => {
@@ -647,7 +1865,7 @@ describe('TeamReconcilerService', () => {
647
1865
  data: { nextRoomReplyReminderAt: new Date(Date.UTC(2026, 0, 1, 0, 1, 0)) },
648
1866
  });
649
1867
  expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
650
- await expect(service.handleSessionStopped(invocation.sessionId)).resolves.toBe(true);
1868
+ await expect(service.handleSessionStopped(invocation.sessionId)).resolves.toHaveLength(0);
651
1869
  await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
652
1870
  status: 'CANCELLED',
653
1871
  nextRoomReplyReminderAt: null,
@@ -683,7 +1901,7 @@ describe('TeamReconcilerService', () => {
683
1901
  targetMemberId: members[1].id,
684
1902
  status: 'QUEUED',
685
1903
  });
686
- await service.handleSessionStopped(invocation.sessionId);
1904
+ await expect(service.handleSessionStopped(invocation.sessionId)).resolves.toHaveLength(0);
687
1905
  await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
688
1906
  status: 'CANCELLED',
689
1907
  });
@@ -715,7 +1933,7 @@ describe('TeamReconcilerService', () => {
715
1933
  data: { deletedAt: new Date() },
716
1934
  });
717
1935
  expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
718
- await expect(service.handleSessionStopped(invocation.sessionId)).resolves.toBe(true);
1936
+ await expect(service.handleSessionStopped(invocation.sessionId)).resolves.toHaveLength(0);
719
1937
  await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
720
1938
  status: 'CANCELLED',
721
1939
  nextRoomReplyReminderAt: null,
@@ -1529,6 +2747,40 @@ describe('TeamReconcilerService', () => {
1529
2747
  await app.close();
1530
2748
  }
1531
2749
  });
2750
+ it('accepts a terminal REST private message without dispatching recipient work', async () => {
2751
+ const { workspace, teamRun, members } = await createFixture({ memberCount: 2 });
2752
+ const senderRequest = await createWorkRequest({
2753
+ teamRunId: teamRun.id,
2754
+ targetMemberId: members[0].id,
2755
+ status: 'COMPLETED',
2756
+ });
2757
+ const senderInvocation = await createRunningInvocation({
2758
+ teamRunId: teamRun.id,
2759
+ workRequestId: senderRequest.id,
2760
+ memberId: members[0].id,
2761
+ workspaceId: workspace.id,
2762
+ status: 'COMPLETED',
2763
+ });
2764
+ const app = Fastify({ logger: false });
2765
+ try {
2766
+ await app.register(teamRunRoutes, { prefix: '/api' });
2767
+ const response = await app.inject({
2768
+ method: 'POST',
2769
+ url: `/api/team-runs/${teamRun.id}/private-messages`,
2770
+ headers: { 'x-agent-tower-invocation-id': senderInvocation.id },
2771
+ payload: {
2772
+ content: 'Late terminal private result',
2773
+ recipientMemberIds: [members[1].id],
2774
+ },
2775
+ });
2776
+ expect(response.statusCode).toBe(201);
2777
+ expect(response.json()).toMatchObject({ workRequestIds: [] });
2778
+ await expect(prisma.workRequest.count({ where: { teamRunId: teamRun.id } })).resolves.toBe(1);
2779
+ }
2780
+ finally {
2781
+ await app.close();
2782
+ }
2783
+ });
1532
2784
  it('preserves REST user private message senderId for member visibility without leaking to non-participants', async () => {
1533
2785
  const { workspace, teamRun, members } = await createFixture({
1534
2786
  memberCount: 3,
@@ -1785,6 +3037,7 @@ describe('TeamReconcilerService', () => {
1785
3037
  memberCount: 2,
1786
3038
  memberCapabilities: [{ mentionMembers: false }],
1787
3039
  });
3040
+ const revoked = await createFixture({ memberCount: 2 });
1788
3041
  const noReadRequest = await createWorkRequest({
1789
3042
  teamRunId: noRead.teamRun.id,
1790
3043
  targetMemberId: noRead.members[0].id,
@@ -1818,6 +3071,21 @@ describe('TeamReconcilerService', () => {
1818
3071
  memberId: noMention.members[0].id,
1819
3072
  workspaceId: noMention.workspace.id,
1820
3073
  });
3074
+ const revokedRequest = await createWorkRequest({
3075
+ teamRunId: revoked.teamRun.id,
3076
+ targetMemberId: revoked.members[0].id,
3077
+ status: 'STARTED',
3078
+ });
3079
+ const revokedInvocation = await createRunningInvocation({
3080
+ teamRunId: revoked.teamRun.id,
3081
+ workRequestId: revokedRequest.id,
3082
+ memberId: revoked.members[0].id,
3083
+ workspaceId: revoked.workspace.id,
3084
+ });
3085
+ await prisma.agentInvocation.update({
3086
+ where: { id: revokedInvocation.id },
3087
+ data: { dispatchRevokedAt: new Date() },
3088
+ });
1821
3089
  const app = Fastify({ logger: false });
1822
3090
  try {
1823
3091
  await app.register(teamRunRoutes, { prefix: '/api' });
@@ -1879,6 +3147,16 @@ describe('TeamReconcilerService', () => {
1879
3147
  });
1880
3148
  expect(noMentionResult.isError).toBe(true);
1881
3149
  expect(getMcpToolText(noMentionResult)).toContain('mentionMembers');
3150
+ const revokedPrivateResult = await callToolForCurrentMember(revoked, revokedInvocation.id, {
3151
+ name: 'post_private_message',
3152
+ arguments: {
3153
+ recipient_member_ids: [revoked.members[1].id],
3154
+ content: 'Late private dispatch after stop',
3155
+ },
3156
+ });
3157
+ expect(revokedPrivateResult.isError).not.toBe(true);
3158
+ expect(JSON.parse(getMcpToolText(revokedPrivateResult))).toMatchObject({ workRequestIds: [] });
3159
+ await expect(prisma.workRequest.count({ where: { teamRunId: revoked.teamRun.id } })).resolves.toBe(1);
1882
3160
  const noStopResult = await callToolForCurrentMember(noRead, noReadInvocation.id, {
1883
3161
  name: 'stop_member_work',
1884
3162
  arguments: {