agent-tower 0.6.0-beta.11 → 0.6.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +4 -0
- package/dist/app.js.map +1 -1
- package/dist/cli.js +39 -19
- package/dist/cli.js.map +1 -1
- package/dist/executors/__tests__/base.executor.test.js +133 -1
- package/dist/executors/__tests__/base.executor.test.js.map +1 -1
- package/dist/executors/__tests__/stdin-prompt.executors.test.js +18 -0
- package/dist/executors/__tests__/stdin-prompt.executors.test.js.map +1 -1
- package/dist/executors/base.executor.d.ts +8 -0
- package/dist/executors/base.executor.d.ts.map +1 -1
- package/dist/executors/base.executor.js +336 -56
- package/dist/executors/base.executor.js.map +1 -1
- package/dist/executors/index.d.ts +2 -1
- package/dist/executors/index.d.ts.map +1 -1
- package/dist/executors/index.js +1 -1
- package/dist/executors/index.js.map +1 -1
- package/dist/executors/start-error.d.ts +17 -0
- package/dist/executors/start-error.d.ts.map +1 -1
- package/dist/executors/start-error.js +36 -1
- package/dist/executors/start-error.js.map +1 -1
- package/dist/index.js +39 -29
- package/dist/index.js.map +1 -1
- package/dist/mcp/http-client.test.js +20 -0
- package/dist/mcp/http-client.test.js.map +1 -1
- package/dist/mcp/tools/__tests__/sessions.test.d.ts +2 -0
- package/dist/mcp/tools/__tests__/sessions.test.d.ts.map +1 -0
- package/dist/mcp/tools/__tests__/sessions.test.js +33 -0
- package/dist/mcp/tools/__tests__/sessions.test.js.map +1 -0
- package/dist/routes/__tests__/sessions-runtime.test.js +75 -3
- package/dist/routes/__tests__/sessions-runtime.test.js.map +1 -1
- package/dist/routes/conversations.d.ts.map +1 -1
- package/dist/routes/conversations.js +5 -1
- package/dist/routes/conversations.js.map +1 -1
- package/dist/routes/sessions.d.ts.map +1 -1
- package/dist/routes/sessions.js +13 -2
- package/dist/routes/sessions.js.map +1 -1
- package/dist/routes/team-runs.d.ts.map +1 -1
- package/dist/routes/team-runs.js +13 -1
- package/dist/routes/team-runs.js.map +1 -1
- package/dist/runtime/__tests__/acp-driver-lifecycle.test.js +86 -0
- package/dist/runtime/__tests__/acp-driver-lifecycle.test.js.map +1 -1
- package/dist/runtime/__tests__/acp-process-manager.test.js +393 -6
- package/dist/runtime/__tests__/acp-process-manager.test.js.map +1 -1
- package/dist/runtime/__tests__/cli-driver-lifecycle.test.d.ts +2 -0
- package/dist/runtime/__tests__/cli-driver-lifecycle.test.d.ts.map +1 -0
- package/dist/runtime/__tests__/cli-driver-lifecycle.test.js +244 -0
- package/dist/runtime/__tests__/cli-driver-lifecycle.test.js.map +1 -0
- package/dist/runtime/__tests__/launch-cleanup-registry.test.d.ts +2 -0
- package/dist/runtime/__tests__/launch-cleanup-registry.test.d.ts.map +1 -0
- package/dist/runtime/__tests__/launch-cleanup-registry.test.js +154 -0
- package/dist/runtime/__tests__/launch-cleanup-registry.test.js.map +1 -0
- package/dist/runtime/__tests__/runtime-coordinator.test.js +117 -1
- package/dist/runtime/__tests__/runtime-coordinator.test.js.map +1 -1
- package/dist/runtime/__tests__/server-entry-shutdown.test.d.ts +2 -0
- package/dist/runtime/__tests__/server-entry-shutdown.test.d.ts.map +1 -0
- package/dist/runtime/__tests__/server-entry-shutdown.test.js +38 -0
- package/dist/runtime/__tests__/server-entry-shutdown.test.js.map +1 -0
- package/dist/runtime/__tests__/shutdown-coordinator.test.d.ts +2 -0
- package/dist/runtime/__tests__/shutdown-coordinator.test.d.ts.map +1 -0
- package/dist/runtime/__tests__/shutdown-coordinator.test.js +28 -0
- package/dist/runtime/__tests__/shutdown-coordinator.test.js.map +1 -0
- package/dist/runtime/acp/acp-driver.d.ts.map +1 -1
- package/dist/runtime/acp/acp-driver.js +108 -35
- package/dist/runtime/acp/acp-driver.js.map +1 -1
- package/dist/runtime/acp/agents/managed-directory.d.ts.map +1 -1
- package/dist/runtime/acp/agents/managed-directory.js +3 -1
- package/dist/runtime/acp/agents/managed-directory.js.map +1 -1
- package/dist/runtime/acp/agents/managed-directory.test.d.ts +2 -0
- package/dist/runtime/acp/agents/managed-directory.test.d.ts.map +1 -0
- package/dist/runtime/acp/agents/managed-directory.test.js +25 -0
- package/dist/runtime/acp/agents/managed-directory.test.js.map +1 -0
- package/dist/runtime/acp/launch-cleanup-registry.d.ts +32 -0
- package/dist/runtime/acp/launch-cleanup-registry.d.ts.map +1 -0
- package/dist/runtime/acp/launch-cleanup-registry.js +176 -0
- package/dist/runtime/acp/launch-cleanup-registry.js.map +1 -0
- package/dist/runtime/acp/process-manager.d.ts +58 -0
- package/dist/runtime/acp/process-manager.d.ts.map +1 -1
- package/dist/runtime/acp/process-manager.js +489 -33
- package/dist/runtime/acp/process-manager.js.map +1 -1
- package/dist/runtime/cli-driver.d.ts.map +1 -1
- package/dist/runtime/cli-driver.js +230 -46
- package/dist/runtime/cli-driver.js.map +1 -1
- package/dist/runtime/contracts.d.ts +21 -2
- package/dist/runtime/contracts.d.ts.map +1 -1
- package/dist/runtime/index.d.ts +1 -0
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +1 -0
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/runtime-coordinator.d.ts +8 -1
- package/dist/runtime/runtime-coordinator.d.ts.map +1 -1
- package/dist/runtime/runtime-coordinator.js +118 -18
- package/dist/runtime/runtime-coordinator.js.map +1 -1
- package/dist/runtime/server-entry-shutdown.d.ts +12 -0
- package/dist/runtime/server-entry-shutdown.d.ts.map +1 -0
- package/dist/runtime/server-entry-shutdown.js +23 -0
- package/dist/runtime/server-entry-shutdown.js.map +1 -0
- package/dist/runtime/shutdown-coordinator.d.ts +17 -0
- package/dist/runtime/shutdown-coordinator.d.ts.map +1 -0
- package/dist/runtime/shutdown-coordinator.js +67 -0
- package/dist/runtime/shutdown-coordinator.js.map +1 -0
- package/dist/services/__tests__/conversation.service.test.js +160 -0
- package/dist/services/__tests__/conversation.service.test.js.map +1 -1
- package/dist/services/__tests__/database-maintenance.service.test.js +97 -0
- package/dist/services/__tests__/database-maintenance.service.test.js.map +1 -1
- package/dist/services/__tests__/member-heartbeat-scheduler.test.js +6 -0
- package/dist/services/__tests__/member-heartbeat-scheduler.test.js.map +1 -1
- package/dist/services/__tests__/member-heartbeat.test.js +7 -11
- package/dist/services/__tests__/member-heartbeat.test.js.map +1 -1
- package/dist/services/__tests__/session-manager.lifecycle.test.js +402 -12
- package/dist/services/__tests__/session-manager.lifecycle.test.js.map +1 -1
- package/dist/services/__tests__/session-manager.team-run.test.js +422 -29
- package/dist/services/__tests__/session-manager.team-run.test.js.map +1 -1
- package/dist/services/__tests__/team-queue-recovery.integration.test.js +1 -0
- package/dist/services/__tests__/team-queue-recovery.integration.test.js.map +1 -1
- package/dist/services/__tests__/team-reconciler.service.test.js +1282 -4
- package/dist/services/__tests__/team-reconciler.service.test.js.map +1 -1
- package/dist/services/__tests__/team-run.service.test.js +161 -1
- package/dist/services/__tests__/team-run.service.test.js.map +1 -1
- package/dist/services/__tests__/team-scheduler.service.test.js +334 -6
- package/dist/services/__tests__/team-scheduler.service.test.js.map +1 -1
- package/dist/services/conversation.service.d.ts.map +1 -1
- package/dist/services/conversation.service.js +21 -2
- package/dist/services/conversation.service.js.map +1 -1
- package/dist/services/database-maintenance.service.d.ts.map +1 -1
- package/dist/services/database-maintenance.service.js +68 -17
- package/dist/services/database-maintenance.service.js.map +1 -1
- package/dist/services/member-heartbeat-scheduler.d.ts +6 -5
- package/dist/services/member-heartbeat-scheduler.d.ts.map +1 -1
- package/dist/services/member-heartbeat-scheduler.js +7 -5
- package/dist/services/member-heartbeat-scheduler.js.map +1 -1
- package/dist/services/session-manager.d.ts +124 -14
- package/dist/services/session-manager.d.ts.map +1 -1
- package/dist/services/session-manager.js +1035 -116
- package/dist/services/session-manager.js.map +1 -1
- package/dist/services/session-runtime-cleanup-gate.d.ts +25 -0
- package/dist/services/session-runtime-cleanup-gate.d.ts.map +1 -0
- package/dist/services/session-runtime-cleanup-gate.js +284 -0
- package/dist/services/session-runtime-cleanup-gate.js.map +1 -0
- package/dist/services/task.service.d.ts +2 -2
- package/dist/services/team-member-admission-barrier.d.ts +3 -0
- package/dist/services/team-member-admission-barrier.d.ts.map +1 -0
- package/dist/services/team-member-admission-barrier.js +26 -0
- package/dist/services/team-member-admission-barrier.js.map +1 -0
- package/dist/services/team-reconciler.service.d.ts +30 -7
- package/dist/services/team-reconciler.service.d.ts.map +1 -1
- package/dist/services/team-reconciler.service.js +528 -111
- package/dist/services/team-reconciler.service.js.map +1 -1
- package/dist/services/team-run.service.d.ts.map +1 -1
- package/dist/services/team-run.service.js +37 -13
- package/dist/services/team-run.service.js.map +1 -1
- package/dist/services/team-scheduler.service.d.ts +8 -6
- package/dist/services/team-scheduler.service.d.ts.map +1 -1
- package/dist/services/team-scheduler.service.js +323 -120
- package/dist/services/team-scheduler.service.js.map +1 -1
- package/dist/services/workspace-background-process-manager.d.ts.map +1 -1
- package/dist/services/workspace-background-process-manager.js +2 -5
- package/dist/services/workspace-background-process-manager.js.map +1 -1
- package/dist/services/workspace.service.d.ts +15 -8
- package/dist/services/workspace.service.d.ts.map +1 -1
- package/dist/socket/index.d.ts.map +1 -1
- package/dist/socket/index.js +27 -14
- package/dist/socket/index.js.map +1 -1
- package/dist/utils/__tests__/unix-process-identity.test.d.ts +2 -0
- package/dist/utils/__tests__/unix-process-identity.test.d.ts.map +1 -0
- package/dist/utils/__tests__/unix-process-identity.test.js +73 -0
- package/dist/utils/__tests__/unix-process-identity.test.js.map +1 -0
- package/dist/utils/error-log.d.ts +2 -0
- package/dist/utils/error-log.d.ts.map +1 -1
- package/dist/utils/error-log.js +40 -3
- package/dist/utils/error-log.js.map +1 -1
- package/dist/utils/error-log.test.js +14 -2
- package/dist/utils/error-log.test.js.map +1 -1
- package/dist/utils/process-launch.d.ts +2 -2
- package/dist/utils/process-launch.d.ts.map +1 -1
- package/dist/utils/process-launch.js +339 -79
- package/dist/utils/process-launch.js.map +1 -1
- package/dist/utils/process-launch.test.js +168 -10
- package/dist/utils/process-launch.test.js.map +1 -1
- package/dist/utils/spawned-process-identity.d.ts +7 -0
- package/dist/utils/spawned-process-identity.d.ts.map +1 -0
- package/dist/utils/spawned-process-identity.js +36 -0
- package/dist/utils/spawned-process-identity.js.map +1 -0
- package/dist/utils/tree-cleanup-channel.d.ts +18 -0
- package/dist/utils/tree-cleanup-channel.d.ts.map +1 -0
- package/dist/utils/tree-cleanup-channel.js +16 -0
- package/dist/utils/tree-cleanup-channel.js.map +1 -0
- package/dist/utils/tree-cleanup-channel.test.d.ts +2 -0
- package/dist/utils/tree-cleanup-channel.test.d.ts.map +1 -0
- package/dist/utils/tree-cleanup-channel.test.js +15 -0
- package/dist/utils/tree-cleanup-channel.test.js.map +1 -0
- package/dist/utils/unix-process-identity.d.ts +13 -1
- package/dist/utils/unix-process-identity.d.ts.map +1 -1
- package/dist/utils/unix-process-identity.js +146 -25
- package/dist/utils/unix-process-identity.js.map +1 -1
- package/node_modules/@agent-tower/shared/dist/types.d.ts +1 -0
- package/node_modules/@agent-tower/shared/dist/types.d.ts.map +1 -1
- package/node_modules/@agent-tower/shared/dist/types.js.map +1 -1
- package/package.json +1 -1
- package/prisma/migrations/20260824000000_add_execution_process_ownership/migration.sql +10 -0
- package/prisma/migrations/20260824010000_add_runtime_cleanup_recovery/migration.sql +10 -0
- package/prisma/migrations/20260824020000_add_runtime_launch_admission/migration.sql +72 -0
- package/prisma/migrations/20260826000000_add_conversation_turn_queue/migration.sql +19 -0
- package/prisma/schema.prisma +50 -7
|
@@ -10,6 +10,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vites
|
|
|
10
10
|
import { TaskStatus } from '../../types/index.js';
|
|
11
11
|
import { EventBus } from '../../core/event-bus.js';
|
|
12
12
|
import { TeamLockService } from '../team-lock.service.js';
|
|
13
|
+
import { acquireTeamMemberAdmission } from '../team-member-admission-barrier.js';
|
|
13
14
|
const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-tower-team-reconciler-'));
|
|
14
15
|
const dbPath = path.join(testDir, 'test.db');
|
|
15
16
|
process.env.AGENT_TOWER_DATABASE_URL = `file:${dbPath}`;
|
|
@@ -367,6 +368,1223 @@ describe('TeamReconcilerService', () => {
|
|
|
367
368
|
expect(lockService.listLocks()).toEqual([]);
|
|
368
369
|
expect(scheduler.startNextSessions).toHaveBeenCalledWith(teamRun.id);
|
|
369
370
|
});
|
|
371
|
+
it('retries persisted cleanup even when the WorkRequest is already terminal', async () => {
|
|
372
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
373
|
+
const request = await createWorkRequest({
|
|
374
|
+
teamRunId: teamRun.id,
|
|
375
|
+
targetMemberId: members[0].id,
|
|
376
|
+
status: 'COMPLETED',
|
|
377
|
+
});
|
|
378
|
+
const invocation = await createRunningInvocation({
|
|
379
|
+
teamRunId: teamRun.id,
|
|
380
|
+
workRequestId: request.id,
|
|
381
|
+
memberId: members[0].id,
|
|
382
|
+
workspaceId: workspace.id,
|
|
383
|
+
status: 'COMPLETED',
|
|
384
|
+
});
|
|
385
|
+
await prisma.session.update({
|
|
386
|
+
where: { id: invocation.sessionId },
|
|
387
|
+
data: { runtimeType: 'ACP' },
|
|
388
|
+
});
|
|
389
|
+
const processRecord = await prisma.executionProcess.create({
|
|
390
|
+
data: {
|
|
391
|
+
sessionId: invocation.sessionId,
|
|
392
|
+
launchClaimNumber: 1,
|
|
393
|
+
runtimeInstanceId: 'runtime-restart',
|
|
394
|
+
pid: 8101,
|
|
395
|
+
processGroupId: '8101',
|
|
396
|
+
birthMarker: 'linux:101:owner-restart',
|
|
397
|
+
ownershipToken: 'owner-restart',
|
|
398
|
+
cleanupState: 'FAILED',
|
|
399
|
+
cleanupAttemptCount: 1,
|
|
400
|
+
nextCleanupRetryAt: new Date(Date.UTC(2025, 11, 31, 23, 59, 0)),
|
|
401
|
+
},
|
|
402
|
+
});
|
|
403
|
+
const retryRuntimeProcessCleanup = vi.fn(async () => {
|
|
404
|
+
await prisma.executionProcess.update({
|
|
405
|
+
where: { id: processRecord.id },
|
|
406
|
+
data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
|
|
407
|
+
});
|
|
408
|
+
});
|
|
409
|
+
const cleanupService = new TeamReconcilerService({
|
|
410
|
+
scheduler,
|
|
411
|
+
sessionMessenger: {
|
|
412
|
+
sendMessage: vi.fn(async () => null),
|
|
413
|
+
retryRuntimeProcessCleanup,
|
|
414
|
+
disposeRuntimeSession: vi.fn(async () => undefined),
|
|
415
|
+
},
|
|
416
|
+
now: () => new Date(Date.UTC(2026, 0, 1, 0, 0, 0)),
|
|
417
|
+
scheduleReminders: false,
|
|
418
|
+
});
|
|
419
|
+
await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
|
|
420
|
+
expect(retryRuntimeProcessCleanup).toHaveBeenCalledWith({
|
|
421
|
+
sessionId: invocation.sessionId,
|
|
422
|
+
runtimeInstanceId: 'runtime-restart',
|
|
423
|
+
launchClaimNumber: 1,
|
|
424
|
+
pid: 8101,
|
|
425
|
+
processGroupId: '8101',
|
|
426
|
+
birthMarker: 'linux:101:owner-restart',
|
|
427
|
+
ownershipToken: 'owner-restart',
|
|
428
|
+
});
|
|
429
|
+
expect(scheduler.startNextSessions).not.toHaveBeenCalled();
|
|
430
|
+
await expect(prisma.executionProcess.findUnique({ where: { id: processRecord.id } })).resolves.toMatchObject({
|
|
431
|
+
cleanupState: 'CONFIRMED',
|
|
432
|
+
});
|
|
433
|
+
});
|
|
434
|
+
it('preserves the cleanup messenger receiver during persisted recovery', async () => {
|
|
435
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
436
|
+
const request = await createWorkRequest({
|
|
437
|
+
teamRunId: teamRun.id,
|
|
438
|
+
targetMemberId: members[0].id,
|
|
439
|
+
status: 'COMPLETED',
|
|
440
|
+
});
|
|
441
|
+
const invocation = await createRunningInvocation({
|
|
442
|
+
teamRunId: teamRun.id,
|
|
443
|
+
workRequestId: request.id,
|
|
444
|
+
memberId: members[0].id,
|
|
445
|
+
workspaceId: workspace.id,
|
|
446
|
+
status: 'COMPLETED',
|
|
447
|
+
});
|
|
448
|
+
await prisma.session.update({
|
|
449
|
+
where: { id: invocation.sessionId },
|
|
450
|
+
data: { runtimeType: 'ACP' },
|
|
451
|
+
});
|
|
452
|
+
const processRecord = await prisma.executionProcess.create({
|
|
453
|
+
data: {
|
|
454
|
+
sessionId: invocation.sessionId,
|
|
455
|
+
launchClaimNumber: 1,
|
|
456
|
+
runtimeInstanceId: 'runtime-receiver-check',
|
|
457
|
+
pid: 8102,
|
|
458
|
+
processGroupId: '8102',
|
|
459
|
+
birthMarker: 'linux:102:owner-receiver-check',
|
|
460
|
+
ownershipToken: 'owner-receiver-check',
|
|
461
|
+
cleanupState: 'FAILED',
|
|
462
|
+
cleanupAttemptCount: 1,
|
|
463
|
+
nextCleanupRetryAt: new Date(Date.UTC(2025, 11, 31, 23, 59, 0)),
|
|
464
|
+
},
|
|
465
|
+
});
|
|
466
|
+
const messenger = {
|
|
467
|
+
cleanupCalls: 0,
|
|
468
|
+
sendMessage: vi.fn(async () => null),
|
|
469
|
+
async retryRuntimeProcessCleanup() {
|
|
470
|
+
this.cleanupCalls += 1;
|
|
471
|
+
await prisma.executionProcess.update({
|
|
472
|
+
where: { id: processRecord.id },
|
|
473
|
+
data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
|
|
474
|
+
});
|
|
475
|
+
},
|
|
476
|
+
};
|
|
477
|
+
const cleanupService = new TeamReconcilerService({
|
|
478
|
+
scheduler,
|
|
479
|
+
sessionMessenger: messenger,
|
|
480
|
+
now: () => new Date(Date.UTC(2026, 0, 1, 0, 0, 0)),
|
|
481
|
+
scheduleReminders: false,
|
|
482
|
+
});
|
|
483
|
+
await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
|
|
484
|
+
expect(messenger.cleanupCalls).toBe(1);
|
|
485
|
+
await expect(prisma.executionProcess.findUnique({ where: { id: processRecord.id } })).resolves.toMatchObject({
|
|
486
|
+
cleanupState: 'CONFIRMED',
|
|
487
|
+
});
|
|
488
|
+
});
|
|
489
|
+
it('does not reclaim an ACTIVE runtime still owned by this server generation', async () => {
|
|
490
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
491
|
+
const request = await createWorkRequest({
|
|
492
|
+
teamRunId: teamRun.id,
|
|
493
|
+
targetMemberId: members[0].id,
|
|
494
|
+
status: 'STARTED',
|
|
495
|
+
});
|
|
496
|
+
const invocation = await createRunningInvocation({
|
|
497
|
+
teamRunId: teamRun.id,
|
|
498
|
+
workRequestId: request.id,
|
|
499
|
+
memberId: members[0].id,
|
|
500
|
+
workspaceId: workspace.id,
|
|
501
|
+
status: 'RUNNING',
|
|
502
|
+
});
|
|
503
|
+
await prisma.executionProcess.create({
|
|
504
|
+
data: {
|
|
505
|
+
sessionId: invocation.sessionId,
|
|
506
|
+
launchClaimNumber: 1,
|
|
507
|
+
runtimeInstanceId: 'runtime-current',
|
|
508
|
+
pid: 8201,
|
|
509
|
+
processGroupId: '8201',
|
|
510
|
+
birthMarker: 'linux:201:owner-current',
|
|
511
|
+
ownershipToken: 'owner-current',
|
|
512
|
+
cleanupState: 'ACTIVE',
|
|
513
|
+
},
|
|
514
|
+
});
|
|
515
|
+
const retryRuntimeProcessCleanup = vi.fn(async () => undefined);
|
|
516
|
+
const cleanupService = new TeamReconcilerService({
|
|
517
|
+
scheduler,
|
|
518
|
+
sessionMessenger: {
|
|
519
|
+
sendMessage: vi.fn(async () => null),
|
|
520
|
+
retryRuntimeProcessCleanup,
|
|
521
|
+
hasRuntimeProcessOwner: (runtimeInstanceId) => runtimeInstanceId === 'runtime-current',
|
|
522
|
+
},
|
|
523
|
+
scheduleReminders: false,
|
|
524
|
+
});
|
|
525
|
+
await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(0);
|
|
526
|
+
expect(retryRuntimeProcessCleanup).not.toHaveBeenCalled();
|
|
527
|
+
});
|
|
528
|
+
it('recovers an ACTIVE runtime left by a prior server generation', async () => {
|
|
529
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
530
|
+
const request = await createWorkRequest({
|
|
531
|
+
teamRunId: teamRun.id,
|
|
532
|
+
targetMemberId: members[0].id,
|
|
533
|
+
status: 'COMPLETED',
|
|
534
|
+
});
|
|
535
|
+
const invocation = await createRunningInvocation({
|
|
536
|
+
teamRunId: teamRun.id,
|
|
537
|
+
workRequestId: request.id,
|
|
538
|
+
memberId: members[0].id,
|
|
539
|
+
workspaceId: workspace.id,
|
|
540
|
+
status: 'COMPLETED',
|
|
541
|
+
});
|
|
542
|
+
const processRecord = await prisma.executionProcess.create({
|
|
543
|
+
data: {
|
|
544
|
+
sessionId: invocation.sessionId,
|
|
545
|
+
launchClaimNumber: 1,
|
|
546
|
+
runtimeInstanceId: 'runtime-stale-active',
|
|
547
|
+
pid: 8202,
|
|
548
|
+
processGroupId: '8202',
|
|
549
|
+
birthMarker: 'linux:202:owner-stale',
|
|
550
|
+
ownershipToken: 'owner-stale',
|
|
551
|
+
cleanupState: 'ACTIVE',
|
|
552
|
+
},
|
|
553
|
+
});
|
|
554
|
+
const retryRuntimeProcessCleanup = vi.fn(async () => {
|
|
555
|
+
await prisma.executionProcess.update({
|
|
556
|
+
where: { id: processRecord.id },
|
|
557
|
+
data: { cleanupState: 'CONFIRMED' },
|
|
558
|
+
});
|
|
559
|
+
});
|
|
560
|
+
const cleanupService = new TeamReconcilerService({
|
|
561
|
+
scheduler,
|
|
562
|
+
sessionMessenger: {
|
|
563
|
+
sendMessage: vi.fn(async () => null),
|
|
564
|
+
retryRuntimeProcessCleanup,
|
|
565
|
+
hasRuntimeProcessOwner: () => false,
|
|
566
|
+
disposeRuntimeSession: vi.fn(async () => undefined),
|
|
567
|
+
},
|
|
568
|
+
scheduleReminders: false,
|
|
569
|
+
});
|
|
570
|
+
await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
|
|
571
|
+
expect(retryRuntimeProcessCleanup).toHaveBeenCalledWith(expect.objectContaining({
|
|
572
|
+
runtimeInstanceId: 'runtime-stale-active',
|
|
573
|
+
ownershipToken: 'owner-stale',
|
|
574
|
+
}));
|
|
575
|
+
});
|
|
576
|
+
it('keeps stop admission revoked until a failed cleanup retry succeeds', async () => {
|
|
577
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
578
|
+
const request = await createWorkRequest({
|
|
579
|
+
teamRunId: teamRun.id,
|
|
580
|
+
targetMemberId: members[0].id,
|
|
581
|
+
status: 'STARTED',
|
|
582
|
+
});
|
|
583
|
+
const invocation = await createRunningInvocation({
|
|
584
|
+
teamRunId: teamRun.id,
|
|
585
|
+
workRequestId: request.id,
|
|
586
|
+
memberId: members[0].id,
|
|
587
|
+
workspaceId: workspace.id,
|
|
588
|
+
status: 'RUNNING',
|
|
589
|
+
});
|
|
590
|
+
await prisma.agentInvocation.update({
|
|
591
|
+
where: { id: invocation.id },
|
|
592
|
+
data: { dispatchRevokedAt: new Date() },
|
|
593
|
+
});
|
|
594
|
+
const processRecord = await prisma.executionProcess.create({
|
|
595
|
+
data: {
|
|
596
|
+
sessionId: invocation.sessionId,
|
|
597
|
+
launchClaimNumber: 1,
|
|
598
|
+
runtimeInstanceId: 'runtime-stop-retry',
|
|
599
|
+
pid: 8203,
|
|
600
|
+
processGroupId: '8203',
|
|
601
|
+
birthMarker: 'linux:203:owner-stop-retry',
|
|
602
|
+
ownershipToken: 'owner-stop-retry',
|
|
603
|
+
cleanupState: 'FAILED',
|
|
604
|
+
nextCleanupRetryAt: new Date(0),
|
|
605
|
+
},
|
|
606
|
+
});
|
|
607
|
+
const cleanupService = new TeamReconcilerService({
|
|
608
|
+
scheduler,
|
|
609
|
+
sessionMessenger: {
|
|
610
|
+
sendMessage: vi.fn(async () => null),
|
|
611
|
+
retryRuntimeProcessCleanup: vi.fn(async () => {
|
|
612
|
+
await prisma.executionProcess.update({
|
|
613
|
+
where: { id: processRecord.id },
|
|
614
|
+
data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
|
|
615
|
+
});
|
|
616
|
+
}),
|
|
617
|
+
stop: vi.fn(async () => prisma.session.update({
|
|
618
|
+
where: { id: invocation.sessionId },
|
|
619
|
+
data: { status: 'CANCELLED' },
|
|
620
|
+
})),
|
|
621
|
+
hasRuntimeProcessOwner: () => false,
|
|
622
|
+
hasActiveTurn: () => false,
|
|
623
|
+
disposeRuntimeSession: vi.fn(async () => undefined),
|
|
624
|
+
},
|
|
625
|
+
scheduleReminders: false,
|
|
626
|
+
});
|
|
627
|
+
await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
|
|
628
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
629
|
+
status: 'CANCELLED',
|
|
630
|
+
dispatchRevokedAt: expect.any(Date),
|
|
631
|
+
});
|
|
632
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
633
|
+
status: 'CANCELLED',
|
|
634
|
+
});
|
|
635
|
+
expect(scheduler.startNextSessions).toHaveBeenCalledWith(teamRun.id);
|
|
636
|
+
});
|
|
637
|
+
it('terminalizes a revoked active invocation after confirmed cleanup when DB recovery previously failed', async () => {
|
|
638
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
639
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
640
|
+
const invocation = await createRunningInvocation({
|
|
641
|
+
teamRunId: teamRun.id,
|
|
642
|
+
workRequestId: request.id,
|
|
643
|
+
memberId: members[0].id,
|
|
644
|
+
workspaceId: workspace.id,
|
|
645
|
+
status: 'RUNNING',
|
|
646
|
+
});
|
|
647
|
+
await prisma.session.update({ where: { id: invocation.sessionId }, data: { status: 'RUNNING' } });
|
|
648
|
+
await prisma.agentInvocation.update({
|
|
649
|
+
where: { id: invocation.id },
|
|
650
|
+
data: { dispatchRevokedAt: new Date() },
|
|
651
|
+
});
|
|
652
|
+
await prisma.executionProcess.create({
|
|
653
|
+
data: {
|
|
654
|
+
sessionId: invocation.sessionId,
|
|
655
|
+
launchClaimNumber: 1,
|
|
656
|
+
runtimeInstanceId: 'runtime-confirmed-recovery',
|
|
657
|
+
pid: 8204,
|
|
658
|
+
processGroupId: '8204',
|
|
659
|
+
birthMarker: 'linux:204:owner-confirmed-recovery',
|
|
660
|
+
ownershipToken: 'owner-confirmed-recovery',
|
|
661
|
+
cleanupState: 'CONFIRMED',
|
|
662
|
+
},
|
|
663
|
+
});
|
|
664
|
+
const cleanupService = new TeamReconcilerService({
|
|
665
|
+
scheduler,
|
|
666
|
+
sessionMessenger: {
|
|
667
|
+
sendMessage: vi.fn(async () => null),
|
|
668
|
+
hasRuntimeProcessOwner: () => false,
|
|
669
|
+
hasActiveTurn: () => false,
|
|
670
|
+
disposeRuntimeSession: vi.fn(async () => undefined),
|
|
671
|
+
},
|
|
672
|
+
scheduleReminders: false,
|
|
673
|
+
});
|
|
674
|
+
await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(0);
|
|
675
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
676
|
+
status: 'CANCELLED',
|
|
677
|
+
dispatchRevokedAt: expect.any(Date),
|
|
678
|
+
});
|
|
679
|
+
await expect(prisma.session.findUnique({ where: { id: invocation.sessionId } })).resolves.toMatchObject({
|
|
680
|
+
status: 'CANCELLED',
|
|
681
|
+
});
|
|
682
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
683
|
+
status: 'CANCELLED',
|
|
684
|
+
});
|
|
685
|
+
expect(scheduler.releaseInvocationLocks).toHaveBeenCalledWith(invocation.id);
|
|
686
|
+
expect(scheduler.startNextSessions).toHaveBeenCalledWith(teamRun.id);
|
|
687
|
+
});
|
|
688
|
+
it('recovers a revoked invocation beyond the old confirmed-process page limit', async () => {
|
|
689
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
690
|
+
for (let index = 0; index < 55; index += 1) {
|
|
691
|
+
const historicalSession = await prisma.session.create({
|
|
692
|
+
data: {
|
|
693
|
+
workspaceId: workspace.id,
|
|
694
|
+
agentType: 'CODEX',
|
|
695
|
+
runtimeType: 'ACP',
|
|
696
|
+
prompt: `historical ${index}`,
|
|
697
|
+
status: 'COMPLETED',
|
|
698
|
+
},
|
|
699
|
+
});
|
|
700
|
+
await prisma.executionProcess.create({
|
|
701
|
+
data: {
|
|
702
|
+
sessionId: historicalSession.id,
|
|
703
|
+
launchClaimNumber: 1,
|
|
704
|
+
runtimeInstanceId: `historical-runtime-${index}`,
|
|
705
|
+
pid: 9000 + index,
|
|
706
|
+
processGroupId: String(9000 + index),
|
|
707
|
+
birthMarker: `test-birth:${9000 + index}`,
|
|
708
|
+
ownershipToken: `historical-owner-${index}`,
|
|
709
|
+
cleanupState: 'CONFIRMED',
|
|
710
|
+
},
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
714
|
+
const invocation = await createRunningInvocation({
|
|
715
|
+
teamRunId: teamRun.id,
|
|
716
|
+
workRequestId: request.id,
|
|
717
|
+
memberId: members[0].id,
|
|
718
|
+
workspaceId: workspace.id,
|
|
719
|
+
status: 'RUNNING',
|
|
720
|
+
});
|
|
721
|
+
await prisma.agentInvocation.update({
|
|
722
|
+
where: { id: invocation.id },
|
|
723
|
+
data: { dispatchRevokedAt: new Date() },
|
|
724
|
+
});
|
|
725
|
+
await prisma.executionProcess.create({
|
|
726
|
+
data: {
|
|
727
|
+
sessionId: invocation.sessionId,
|
|
728
|
+
launchClaimNumber: 1,
|
|
729
|
+
runtimeInstanceId: 'target-confirmed-runtime',
|
|
730
|
+
pid: 9999,
|
|
731
|
+
processGroupId: '9999',
|
|
732
|
+
birthMarker: 'test-birth:9999',
|
|
733
|
+
ownershipToken: 'target-confirmed-owner',
|
|
734
|
+
cleanupState: 'CONFIRMED',
|
|
735
|
+
},
|
|
736
|
+
});
|
|
737
|
+
const cleanupService = new TeamReconcilerService({
|
|
738
|
+
scheduler,
|
|
739
|
+
sessionMessenger: {
|
|
740
|
+
sendMessage: vi.fn(async () => null),
|
|
741
|
+
hasRuntimeProcessOwner: () => false,
|
|
742
|
+
hasActiveTurn: () => false,
|
|
743
|
+
},
|
|
744
|
+
scheduleReminders: false,
|
|
745
|
+
});
|
|
746
|
+
await expect(cleanupService.reconcilePendingRuntimeCleanup(50)).resolves.toBe(0);
|
|
747
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
748
|
+
status: 'CANCELLED',
|
|
749
|
+
});
|
|
750
|
+
});
|
|
751
|
+
it.each(['ACTIVE', 'PENDING', 'FAILED'])('does not terminalize a revoked invocation with an old CONFIRMED runtime and newer %s runtime', async (newerCleanupState) => {
|
|
752
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
753
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
754
|
+
const invocation = await createRunningInvocation({
|
|
755
|
+
teamRunId: teamRun.id,
|
|
756
|
+
workRequestId: request.id,
|
|
757
|
+
memberId: members[0].id,
|
|
758
|
+
workspaceId: workspace.id,
|
|
759
|
+
status: 'RUNNING',
|
|
760
|
+
});
|
|
761
|
+
await prisma.agentInvocation.update({
|
|
762
|
+
where: { id: invocation.id },
|
|
763
|
+
data: { dispatchRevokedAt: new Date() },
|
|
764
|
+
});
|
|
765
|
+
await prisma.executionProcess.createMany({
|
|
766
|
+
data: [
|
|
767
|
+
{
|
|
768
|
+
sessionId: invocation.sessionId,
|
|
769
|
+
launchClaimNumber: 1,
|
|
770
|
+
runtimeInstanceId: `old-confirmed-${newerCleanupState}`,
|
|
771
|
+
pid: 9101,
|
|
772
|
+
processGroupId: '9101',
|
|
773
|
+
birthMarker: `test-birth:old-${newerCleanupState}`,
|
|
774
|
+
ownershipToken: `old-owner-${newerCleanupState}`,
|
|
775
|
+
cleanupState: 'CONFIRMED',
|
|
776
|
+
},
|
|
777
|
+
{
|
|
778
|
+
sessionId: invocation.sessionId,
|
|
779
|
+
launchClaimNumber: 2,
|
|
780
|
+
runtimeInstanceId: `new-${newerCleanupState}`,
|
|
781
|
+
pid: 9102,
|
|
782
|
+
processGroupId: '9102',
|
|
783
|
+
birthMarker: `test-birth:new-${newerCleanupState}`,
|
|
784
|
+
ownershipToken: `new-owner-${newerCleanupState}`,
|
|
785
|
+
cleanupState: newerCleanupState,
|
|
786
|
+
},
|
|
787
|
+
],
|
|
788
|
+
});
|
|
789
|
+
const cleanupService = new TeamReconcilerService({
|
|
790
|
+
scheduler,
|
|
791
|
+
sessionMessenger: {
|
|
792
|
+
sendMessage: vi.fn(async () => null),
|
|
793
|
+
hasRuntimeProcessOwner: () => false,
|
|
794
|
+
hasActiveTurn: () => false,
|
|
795
|
+
},
|
|
796
|
+
scheduleReminders: false,
|
|
797
|
+
});
|
|
798
|
+
await cleanupService.reconcilePendingRuntimeCleanup();
|
|
799
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
800
|
+
status: 'RUNNING',
|
|
801
|
+
dispatchRevokedAt: expect.any(Date),
|
|
802
|
+
});
|
|
803
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
804
|
+
status: 'STARTED',
|
|
805
|
+
});
|
|
806
|
+
});
|
|
807
|
+
it('keeps post-confirm revocation recovery idempotent', async () => {
|
|
808
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
809
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
810
|
+
const invocation = await createRunningInvocation({
|
|
811
|
+
teamRunId: teamRun.id,
|
|
812
|
+
workRequestId: request.id,
|
|
813
|
+
memberId: members[0].id,
|
|
814
|
+
workspaceId: workspace.id,
|
|
815
|
+
status: 'RUNNING',
|
|
816
|
+
});
|
|
817
|
+
await prisma.agentInvocation.update({
|
|
818
|
+
where: { id: invocation.id },
|
|
819
|
+
data: { dispatchRevokedAt: new Date() },
|
|
820
|
+
});
|
|
821
|
+
await prisma.executionProcess.create({
|
|
822
|
+
data: {
|
|
823
|
+
sessionId: invocation.sessionId,
|
|
824
|
+
launchClaimNumber: 1,
|
|
825
|
+
runtimeInstanceId: 'idempotent-confirmed-runtime',
|
|
826
|
+
pid: 9201,
|
|
827
|
+
processGroupId: '9201',
|
|
828
|
+
birthMarker: 'test-birth:9201',
|
|
829
|
+
ownershipToken: 'idempotent-owner',
|
|
830
|
+
cleanupState: 'CONFIRMED',
|
|
831
|
+
},
|
|
832
|
+
});
|
|
833
|
+
const cleanupService = new TeamReconcilerService({
|
|
834
|
+
scheduler,
|
|
835
|
+
sessionMessenger: {
|
|
836
|
+
sendMessage: vi.fn(async () => null),
|
|
837
|
+
hasRuntimeProcessOwner: () => false,
|
|
838
|
+
hasActiveTurn: () => false,
|
|
839
|
+
},
|
|
840
|
+
scheduleReminders: false,
|
|
841
|
+
});
|
|
842
|
+
await cleanupService.reconcilePendingRuntimeCleanup();
|
|
843
|
+
const releaseCount = vi.mocked(scheduler.releaseInvocationLocks).mock.calls.length;
|
|
844
|
+
const startCount = vi.mocked(scheduler.startNextSessions).mock.calls.length;
|
|
845
|
+
await cleanupService.reconcilePendingRuntimeCleanup();
|
|
846
|
+
expect(scheduler.releaseInvocationLocks).toHaveBeenCalledTimes(releaseCount);
|
|
847
|
+
expect(scheduler.startNextSessions).toHaveBeenCalledTimes(startCount);
|
|
848
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
849
|
+
status: 'CANCELLED',
|
|
850
|
+
});
|
|
851
|
+
});
|
|
852
|
+
it('proactively quarantines historical process rows with incomplete ownership', async () => {
|
|
853
|
+
vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
|
854
|
+
const { workspace } = await createFixture();
|
|
855
|
+
const session = await prisma.session.create({
|
|
856
|
+
data: {
|
|
857
|
+
workspaceId: workspace.id,
|
|
858
|
+
agentType: 'CODEX',
|
|
859
|
+
prompt: 'legacy incomplete ownership',
|
|
860
|
+
status: 'RUNNING',
|
|
861
|
+
},
|
|
862
|
+
});
|
|
863
|
+
const processRecord = await prisma.executionProcess.create({
|
|
864
|
+
data: {
|
|
865
|
+
sessionId: session.id,
|
|
866
|
+
launchClaimNumber: 1,
|
|
867
|
+
runtimeInstanceId: 'legacy-incomplete-runtime',
|
|
868
|
+
pid: 9251,
|
|
869
|
+
processGroupId: '9251',
|
|
870
|
+
birthMarker: null,
|
|
871
|
+
ownershipToken: 'legacy-incomplete-owner',
|
|
872
|
+
cleanupState: 'ACTIVE',
|
|
873
|
+
},
|
|
874
|
+
});
|
|
875
|
+
const retryRuntimeProcessCleanup = vi.fn(async () => undefined);
|
|
876
|
+
const cleanupService = new TeamReconcilerService({
|
|
877
|
+
scheduler,
|
|
878
|
+
sessionMessenger: {
|
|
879
|
+
sendMessage: vi.fn(async () => null),
|
|
880
|
+
hasActiveTurn: () => false,
|
|
881
|
+
hasRuntimeProcessOwner: () => false,
|
|
882
|
+
retryRuntimeProcessCleanup,
|
|
883
|
+
},
|
|
884
|
+
scheduleReminders: false,
|
|
885
|
+
});
|
|
886
|
+
await expect(cleanupService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
|
|
887
|
+
expect(retryRuntimeProcessCleanup).not.toHaveBeenCalled();
|
|
888
|
+
await expect(prisma.executionProcess.findUniqueOrThrow({ where: { id: processRecord.id } })).resolves.toMatchObject({
|
|
889
|
+
cleanupState: 'QUARANTINED',
|
|
890
|
+
cleanupError: expect.stringContaining('identity is incomplete'),
|
|
891
|
+
cleanupAttemptCount: 2,
|
|
892
|
+
nextCleanupRetryAt: expect.any(Date),
|
|
893
|
+
});
|
|
894
|
+
});
|
|
895
|
+
it('does not downgrade confirmed cleanup when a stale gate sees incomplete metadata', async () => {
|
|
896
|
+
const { workspace } = await createFixture();
|
|
897
|
+
const session = await prisma.session.create({
|
|
898
|
+
data: {
|
|
899
|
+
workspaceId: workspace.id,
|
|
900
|
+
agentType: 'CODEX',
|
|
901
|
+
prompt: 'confirmed legacy cleanup',
|
|
902
|
+
status: 'COMPLETED',
|
|
903
|
+
},
|
|
904
|
+
});
|
|
905
|
+
const processRecord = await prisma.executionProcess.create({
|
|
906
|
+
data: {
|
|
907
|
+
sessionId: session.id,
|
|
908
|
+
launchClaimNumber: 1,
|
|
909
|
+
runtimeInstanceId: 'confirmed-legacy-runtime',
|
|
910
|
+
pid: 9252,
|
|
911
|
+
processGroupId: '9252',
|
|
912
|
+
birthMarker: null,
|
|
913
|
+
ownershipToken: 'confirmed-legacy-owner',
|
|
914
|
+
cleanupState: 'CONFIRMED',
|
|
915
|
+
},
|
|
916
|
+
});
|
|
917
|
+
const cleanupService = new TeamReconcilerService({
|
|
918
|
+
scheduler,
|
|
919
|
+
sessionMessenger: {
|
|
920
|
+
sendMessage: vi.fn(async () => null),
|
|
921
|
+
hasRuntimeProcessOwner: () => false,
|
|
922
|
+
},
|
|
923
|
+
scheduleReminders: false,
|
|
924
|
+
});
|
|
925
|
+
await expect(cleanupService.isSessionRuntimeCleanupConfirmed(session.id)).resolves.toBe(false);
|
|
926
|
+
await expect(prisma.executionProcess.findUniqueOrThrow({ where: { id: processRecord.id } })).resolves.toMatchObject({
|
|
927
|
+
cleanupState: 'CONFIRMED',
|
|
928
|
+
cleanupAttemptCount: 0,
|
|
929
|
+
cleanupError: null,
|
|
930
|
+
});
|
|
931
|
+
});
|
|
932
|
+
it('re-reads a new started generation after a quarantine CAS loses the race', async () => {
|
|
933
|
+
const { workspace } = await createFixture();
|
|
934
|
+
const session = await prisma.session.create({
|
|
935
|
+
data: {
|
|
936
|
+
workspaceId: workspace.id,
|
|
937
|
+
agentType: 'CODEX',
|
|
938
|
+
prompt: 'CAS generation race',
|
|
939
|
+
status: 'RUNNING',
|
|
940
|
+
runtimeLaunchState: 'CLAIMED',
|
|
941
|
+
runtimeLaunchClaimCount: 1,
|
|
942
|
+
},
|
|
943
|
+
});
|
|
944
|
+
let injected = false;
|
|
945
|
+
prisma.$use(async (params, next) => {
|
|
946
|
+
const where = params.args?.where;
|
|
947
|
+
if (!injected
|
|
948
|
+
&& params.model === 'Session'
|
|
949
|
+
&& params.action === 'updateMany'
|
|
950
|
+
&& where?.id === session.id
|
|
951
|
+
&& where.runtimeLaunchState === 'CLAIMED') {
|
|
952
|
+
injected = true;
|
|
953
|
+
await prisma.session.update({
|
|
954
|
+
where: { id: session.id },
|
|
955
|
+
data: {
|
|
956
|
+
runtimeLaunchState: 'PROCESS_RECORDED',
|
|
957
|
+
runtimeLaunchResolvedCount: 1,
|
|
958
|
+
runtimeLaunchProcessCount: 1,
|
|
959
|
+
},
|
|
960
|
+
});
|
|
961
|
+
await prisma.executionProcess.create({
|
|
962
|
+
data: {
|
|
963
|
+
sessionId: session.id,
|
|
964
|
+
launchClaimNumber: 1,
|
|
965
|
+
runtimeInstanceId: 'cas-new-runtime',
|
|
966
|
+
pid: 9253,
|
|
967
|
+
processGroupId: '9253',
|
|
968
|
+
birthMarker: 'test-birth:cas-new',
|
|
969
|
+
ownershipToken: 'cas-new-owner',
|
|
970
|
+
cleanupState: 'CONFIRMED',
|
|
971
|
+
},
|
|
972
|
+
});
|
|
973
|
+
}
|
|
974
|
+
return next(params);
|
|
975
|
+
});
|
|
976
|
+
const cleanupService = new TeamReconcilerService({
|
|
977
|
+
scheduler,
|
|
978
|
+
sessionMessenger: {
|
|
979
|
+
sendMessage: vi.fn(async () => null),
|
|
980
|
+
hasRuntimeProcessOwner: () => false,
|
|
981
|
+
},
|
|
982
|
+
scheduleReminders: false,
|
|
983
|
+
});
|
|
984
|
+
await expect(cleanupService.isSessionRuntimeCleanupConfirmed(session.id)).resolves.toBe(true);
|
|
985
|
+
expect(injected).toBe(true);
|
|
986
|
+
await expect(prisma.session.findUniqueOrThrow({ where: { id: session.id } })).resolves.toMatchObject({
|
|
987
|
+
runtimeLaunchState: 'PROCESS_RECORDED',
|
|
988
|
+
runtimeLaunchClaimCount: 1,
|
|
989
|
+
runtimeLaunchResolvedCount: 1,
|
|
990
|
+
runtimeLaunchProcessCount: 1,
|
|
991
|
+
});
|
|
992
|
+
});
|
|
993
|
+
it.each([
|
|
994
|
+
['PENDING cleanup', 'PENDING', false, false],
|
|
995
|
+
['FAILED cleanup', 'FAILED', false, false],
|
|
996
|
+
['ACTIVE cleanup', 'ACTIVE', false, false],
|
|
997
|
+
['current runtime owner', 'CONFIRMED', true, false],
|
|
998
|
+
['incomplete ownership identity', 'ACTIVE', false, true],
|
|
999
|
+
])('keeps manual stop pending for %s until the unified cleanup gate confirms', async (_caseName, cleanupState, initialOwnerPresent, incompleteIdentity) => {
|
|
1000
|
+
const { workspace, task, teamRun, members } = await createFixture({
|
|
1001
|
+
taskStatus: TaskStatus.IN_PROGRESS,
|
|
1002
|
+
memberCount: 2,
|
|
1003
|
+
});
|
|
1004
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1005
|
+
const invocation = await createRunningInvocation({
|
|
1006
|
+
teamRunId: teamRun.id,
|
|
1007
|
+
workRequestId: request.id,
|
|
1008
|
+
memberId: members[0].id,
|
|
1009
|
+
workspaceId: workspace.id,
|
|
1010
|
+
status: 'RUNNING',
|
|
1011
|
+
});
|
|
1012
|
+
const queued = await createWorkRequest({
|
|
1013
|
+
teamRunId: teamRun.id,
|
|
1014
|
+
targetMemberId: members[1].id,
|
|
1015
|
+
status: 'QUEUED',
|
|
1016
|
+
});
|
|
1017
|
+
const processRecord = await prisma.executionProcess.create({
|
|
1018
|
+
data: {
|
|
1019
|
+
sessionId: invocation.sessionId,
|
|
1020
|
+
launchClaimNumber: 1,
|
|
1021
|
+
runtimeInstanceId: incompleteIdentity ? null : `manual-${cleanupState.toLowerCase()}`,
|
|
1022
|
+
pid: 9301,
|
|
1023
|
+
processGroupId: incompleteIdentity ? null : '9301',
|
|
1024
|
+
birthMarker: incompleteIdentity ? null : `test-birth:manual-${cleanupState.toLowerCase()}`,
|
|
1025
|
+
ownershipToken: incompleteIdentity ? null : `manual-owner-${cleanupState.toLowerCase()}`,
|
|
1026
|
+
cleanupState,
|
|
1027
|
+
},
|
|
1028
|
+
});
|
|
1029
|
+
expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
|
|
1030
|
+
let ownerPresent = initialOwnerPresent;
|
|
1031
|
+
const retryRuntimeProcessCleanup = vi.fn(async () => undefined);
|
|
1032
|
+
const stopService = new TeamReconcilerService({
|
|
1033
|
+
scheduler,
|
|
1034
|
+
sessionMessenger: {
|
|
1035
|
+
sendMessage: vi.fn(async () => null),
|
|
1036
|
+
hasActiveTurn: () => false,
|
|
1037
|
+
hasRuntimeProcessOwner: () => ownerPresent,
|
|
1038
|
+
retryRuntimeProcessCleanup,
|
|
1039
|
+
disposeRuntimeSession: vi.fn(async () => undefined),
|
|
1040
|
+
},
|
|
1041
|
+
now: () => new Date(Date.UTC(2026, 0, 1, 0, 0, 0)),
|
|
1042
|
+
scheduleReminders: false,
|
|
1043
|
+
});
|
|
1044
|
+
await expect(stopService.handleSessionStopped(invocation.sessionId)).resolves.toEqual([]);
|
|
1045
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1046
|
+
status: 'RUNNING',
|
|
1047
|
+
dispatchRevokedAt: expect.any(Date),
|
|
1048
|
+
});
|
|
1049
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
1050
|
+
status: 'STARTED',
|
|
1051
|
+
});
|
|
1052
|
+
await expect(prisma.workRequest.findUnique({ where: { id: queued.id } })).resolves.toMatchObject({
|
|
1053
|
+
status: 'QUEUED',
|
|
1054
|
+
});
|
|
1055
|
+
await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
|
|
1056
|
+
status: TaskStatus.IN_PROGRESS,
|
|
1057
|
+
});
|
|
1058
|
+
expect(lockService.listLocks()).toEqual([
|
|
1059
|
+
{ key: 'workspace:task:write', ownerId: invocation.id },
|
|
1060
|
+
]);
|
|
1061
|
+
expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
|
|
1062
|
+
expect(scheduler.startNextSessions).not.toHaveBeenCalled();
|
|
1063
|
+
if (incompleteIdentity) {
|
|
1064
|
+
await expect(prisma.executionProcess.findUnique({ where: { id: processRecord.id } })).resolves.toMatchObject({
|
|
1065
|
+
cleanupState: 'QUARANTINED',
|
|
1066
|
+
cleanupError: expect.stringContaining('identity is incomplete'),
|
|
1067
|
+
cleanupAttemptCount: 1,
|
|
1068
|
+
nextCleanupRetryAt: expect.any(Date),
|
|
1069
|
+
});
|
|
1070
|
+
await prisma.executionProcess.update({
|
|
1071
|
+
where: { id: processRecord.id },
|
|
1072
|
+
data: { nextCleanupRetryAt: new Date(0) },
|
|
1073
|
+
});
|
|
1074
|
+
await expect(stopService.reconcilePendingRuntimeCleanup()).resolves.toBe(1);
|
|
1075
|
+
expect(retryRuntimeProcessCleanup).not.toHaveBeenCalled();
|
|
1076
|
+
await expect(prisma.executionProcess.findUnique({ where: { id: processRecord.id } })).resolves.toMatchObject({
|
|
1077
|
+
cleanupState: 'QUARANTINED',
|
|
1078
|
+
cleanupAttemptCount: 2,
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
ownerPresent = false;
|
|
1082
|
+
await prisma.executionProcess.update({
|
|
1083
|
+
where: { id: processRecord.id },
|
|
1084
|
+
data: {
|
|
1085
|
+
runtimeInstanceId: `manual-confirmed-${cleanupState.toLowerCase()}`,
|
|
1086
|
+
processGroupId: '9301',
|
|
1087
|
+
birthMarker: `test-birth:manual-confirmed-${cleanupState.toLowerCase()}`,
|
|
1088
|
+
ownershipToken: `manual-confirmed-owner-${cleanupState.toLowerCase()}`,
|
|
1089
|
+
cleanupState: 'CONFIRMED',
|
|
1090
|
+
cleanupError: null,
|
|
1091
|
+
nextCleanupRetryAt: null,
|
|
1092
|
+
},
|
|
1093
|
+
});
|
|
1094
|
+
await expect(stopService.handleSessionStopped(invocation.sessionId)).resolves.toEqual([]);
|
|
1095
|
+
const releaseCount = vi.mocked(scheduler.releaseInvocationLocks).mock.calls.length;
|
|
1096
|
+
const startCount = vi.mocked(scheduler.startNextSessions).mock.calls.length;
|
|
1097
|
+
await expect(stopService.handleSessionStopped(invocation.sessionId)).resolves.toEqual([]);
|
|
1098
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1099
|
+
status: 'CANCELLED',
|
|
1100
|
+
});
|
|
1101
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
1102
|
+
status: 'CANCELLED',
|
|
1103
|
+
});
|
|
1104
|
+
expect(lockService.listLocks()).toEqual([]);
|
|
1105
|
+
expect(vi.mocked(scheduler.releaseInvocationLocks).mock.calls).toEqual([[invocation.id]]);
|
|
1106
|
+
expect(scheduler.startNextSessions).toHaveBeenCalledTimes(1);
|
|
1107
|
+
expect(releaseCount).toBe(1);
|
|
1108
|
+
expect(startCount).toBe(1);
|
|
1109
|
+
});
|
|
1110
|
+
it.each([
|
|
1111
|
+
['cleanup failure', 'FAILED', false],
|
|
1112
|
+
['current owner', 'CONFIRMED', true],
|
|
1113
|
+
])('keeps incomplete terminal recovery pending for %s and completes it exactly once after confirmation', async (_caseName, cleanupState, initialOwnerPresent) => {
|
|
1114
|
+
const { workspace, task, teamRun, members } = await createFixture({ taskStatus: TaskStatus.IN_PROGRESS });
|
|
1115
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1116
|
+
const invocation = await createRunningInvocation({
|
|
1117
|
+
teamRunId: teamRun.id,
|
|
1118
|
+
workRequestId: request.id,
|
|
1119
|
+
memberId: members[0].id,
|
|
1120
|
+
workspaceId: workspace.id,
|
|
1121
|
+
status: 'COMPLETED',
|
|
1122
|
+
});
|
|
1123
|
+
const processRecord = await prisma.executionProcess.create({
|
|
1124
|
+
data: {
|
|
1125
|
+
sessionId: invocation.sessionId,
|
|
1126
|
+
launchClaimNumber: 1,
|
|
1127
|
+
runtimeInstanceId: `incomplete-terminal-${cleanupState.toLowerCase()}`,
|
|
1128
|
+
pid: 9401,
|
|
1129
|
+
processGroupId: '9401',
|
|
1130
|
+
birthMarker: `test-birth:incomplete-terminal-${cleanupState.toLowerCase()}`,
|
|
1131
|
+
ownershipToken: `incomplete-terminal-owner-${cleanupState.toLowerCase()}`,
|
|
1132
|
+
cleanupState,
|
|
1133
|
+
},
|
|
1134
|
+
});
|
|
1135
|
+
expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
|
|
1136
|
+
let ownerPresent = initialOwnerPresent;
|
|
1137
|
+
const recoveryService = new TeamReconcilerService({
|
|
1138
|
+
scheduler,
|
|
1139
|
+
sessionMessenger: {
|
|
1140
|
+
sendMessage: vi.fn(async () => null),
|
|
1141
|
+
hasActiveTurn: () => false,
|
|
1142
|
+
hasRuntimeProcessOwner: () => ownerPresent,
|
|
1143
|
+
disposeRuntimeSession: vi.fn(async () => undefined),
|
|
1144
|
+
},
|
|
1145
|
+
scheduleReminders: false,
|
|
1146
|
+
});
|
|
1147
|
+
await recoveryService.reconcileIncompleteTerminalInvocations();
|
|
1148
|
+
await recoveryService.reconcileIncompleteTerminalInvocations();
|
|
1149
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
1150
|
+
status: 'STARTED',
|
|
1151
|
+
});
|
|
1152
|
+
await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
|
|
1153
|
+
status: TaskStatus.IN_PROGRESS,
|
|
1154
|
+
});
|
|
1155
|
+
expect(lockService.listLocks()).toEqual([
|
|
1156
|
+
{ key: 'workspace:task:write', ownerId: invocation.id },
|
|
1157
|
+
]);
|
|
1158
|
+
expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
|
|
1159
|
+
expect(scheduler.startNextSessions).not.toHaveBeenCalled();
|
|
1160
|
+
ownerPresent = false;
|
|
1161
|
+
await prisma.executionProcess.update({
|
|
1162
|
+
where: { id: processRecord.id },
|
|
1163
|
+
data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
|
|
1164
|
+
});
|
|
1165
|
+
await recoveryService.reconcileIncompleteTerminalInvocations();
|
|
1166
|
+
await recoveryService.reconcileIncompleteTerminalInvocations();
|
|
1167
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
1168
|
+
status: 'COMPLETED',
|
|
1169
|
+
});
|
|
1170
|
+
expect(lockService.listLocks()).toEqual([]);
|
|
1171
|
+
expect(vi.mocked(scheduler.releaseInvocationLocks).mock.calls).toEqual([[invocation.id]]);
|
|
1172
|
+
expect(scheduler.startNextSessions).toHaveBeenCalledTimes(1);
|
|
1173
|
+
});
|
|
1174
|
+
it('does not nudge a stalled invocation after stop revokes dispatch', async () => {
|
|
1175
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
1176
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1177
|
+
const invocation = await createRunningInvocation({
|
|
1178
|
+
teamRunId: teamRun.id,
|
|
1179
|
+
workRequestId: request.id,
|
|
1180
|
+
memberId: members[0].id,
|
|
1181
|
+
workspaceId: workspace.id,
|
|
1182
|
+
status: 'RUNNING',
|
|
1183
|
+
});
|
|
1184
|
+
await prisma.agentInvocation.update({
|
|
1185
|
+
where: { id: invocation.id },
|
|
1186
|
+
data: { dispatchRevokedAt: new Date(), lastHeartbeatAt: new Date(0) },
|
|
1187
|
+
});
|
|
1188
|
+
const sendMessage = vi.fn(async () => null);
|
|
1189
|
+
const stalledService = new TeamReconcilerService({
|
|
1190
|
+
scheduler,
|
|
1191
|
+
sessionMessenger: {
|
|
1192
|
+
sendMessage,
|
|
1193
|
+
hasActiveTurn: () => true,
|
|
1194
|
+
},
|
|
1195
|
+
now: () => new Date(Date.UTC(2026, 0, 1, 1, 0, 0)),
|
|
1196
|
+
heartbeatIdleThresholdMs: 1,
|
|
1197
|
+
scheduleReminders: false,
|
|
1198
|
+
});
|
|
1199
|
+
await stalledService.reconcileStalledInvocations();
|
|
1200
|
+
expect(sendMessage).not.toHaveBeenCalled();
|
|
1201
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1202
|
+
dispatchRevokedAt: expect.any(Date),
|
|
1203
|
+
roomReplyReminderCount: 0,
|
|
1204
|
+
});
|
|
1205
|
+
});
|
|
1206
|
+
it('does not overwrite a manual CANCELLED winner during orphan recovery', async () => {
|
|
1207
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
1208
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1209
|
+
const invocation = await createRunningInvocation({
|
|
1210
|
+
teamRunId: teamRun.id,
|
|
1211
|
+
workRequestId: request.id,
|
|
1212
|
+
memberId: members[0].id,
|
|
1213
|
+
workspaceId: workspace.id,
|
|
1214
|
+
status: 'RUNNING',
|
|
1215
|
+
});
|
|
1216
|
+
const orphanService = new TeamReconcilerService({
|
|
1217
|
+
scheduler,
|
|
1218
|
+
sessionMessenger: {
|
|
1219
|
+
sendMessage: vi.fn(async () => null),
|
|
1220
|
+
hasActiveTurn: () => false,
|
|
1221
|
+
},
|
|
1222
|
+
scheduleReminders: false,
|
|
1223
|
+
});
|
|
1224
|
+
const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
|
|
1225
|
+
const recovery = orphanService.reconcileOrphanInvocations();
|
|
1226
|
+
await Promise.resolve();
|
|
1227
|
+
await prisma.agentInvocation.update({
|
|
1228
|
+
where: { id: invocation.id },
|
|
1229
|
+
data: { status: 'CANCELLED', dispatchRevokedAt: new Date() },
|
|
1230
|
+
});
|
|
1231
|
+
releaseAdmission();
|
|
1232
|
+
await expect(recovery).resolves.toBeUndefined();
|
|
1233
|
+
await expect(orphanService.reconcileOrphanInvocations()).resolves.toBeUndefined();
|
|
1234
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1235
|
+
status: 'CANCELLED',
|
|
1236
|
+
dispatchRevokedAt: expect.any(Date),
|
|
1237
|
+
});
|
|
1238
|
+
expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalledWith(invocation.id);
|
|
1239
|
+
expect(scheduler.startNextSessions).not.toHaveBeenCalledWith(teamRun.id);
|
|
1240
|
+
});
|
|
1241
|
+
it.each(['PENDING', 'FAILED'])('keeps an orphan invocation pending until %s runtime cleanup is confirmed', async (cleanupState) => {
|
|
1242
|
+
const { workspace, task, teamRun, members } = await createFixture();
|
|
1243
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1244
|
+
const invocation = await createRunningInvocation({
|
|
1245
|
+
teamRunId: teamRun.id,
|
|
1246
|
+
workRequestId: request.id,
|
|
1247
|
+
memberId: members[0].id,
|
|
1248
|
+
workspaceId: workspace.id,
|
|
1249
|
+
status: 'RUNNING',
|
|
1250
|
+
});
|
|
1251
|
+
await prisma.executionProcess.create({
|
|
1252
|
+
data: {
|
|
1253
|
+
sessionId: invocation.sessionId,
|
|
1254
|
+
launchClaimNumber: 1,
|
|
1255
|
+
runtimeInstanceId: `orphan-${cleanupState.toLowerCase()}`,
|
|
1256
|
+
pid: 8301,
|
|
1257
|
+
processGroupId: '8301',
|
|
1258
|
+
birthMarker: `test-birth:orphan-${cleanupState.toLowerCase()}`,
|
|
1259
|
+
ownershipToken: `orphan-owner-${cleanupState.toLowerCase()}`,
|
|
1260
|
+
cleanupState,
|
|
1261
|
+
},
|
|
1262
|
+
});
|
|
1263
|
+
expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
|
|
1264
|
+
const orphanService = new TeamReconcilerService({
|
|
1265
|
+
scheduler,
|
|
1266
|
+
sessionMessenger: {
|
|
1267
|
+
sendMessage: vi.fn(async () => null),
|
|
1268
|
+
hasActiveTurn: () => false,
|
|
1269
|
+
hasRuntimeProcessOwner: () => false,
|
|
1270
|
+
},
|
|
1271
|
+
scheduleReminders: false,
|
|
1272
|
+
});
|
|
1273
|
+
await orphanService.reconcileOrphanInvocations();
|
|
1274
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1275
|
+
status: 'RUNNING',
|
|
1276
|
+
dispatchRevokedAt: expect.any(Date),
|
|
1277
|
+
});
|
|
1278
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
1279
|
+
status: 'STARTED',
|
|
1280
|
+
});
|
|
1281
|
+
await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
|
|
1282
|
+
status: TaskStatus.IN_PROGRESS,
|
|
1283
|
+
});
|
|
1284
|
+
expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
|
|
1285
|
+
expect(scheduler.startNextSessions).not.toHaveBeenCalled();
|
|
1286
|
+
await prisma.executionProcess.updateMany({
|
|
1287
|
+
where: { sessionId: invocation.sessionId },
|
|
1288
|
+
data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
|
|
1289
|
+
});
|
|
1290
|
+
await orphanService.reconcilePendingRuntimeCleanup();
|
|
1291
|
+
const releaseCount = scheduler.releaseInvocationLocks.mock.calls.length;
|
|
1292
|
+
const startCount = scheduler.startNextSessions.mock.calls.length;
|
|
1293
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1294
|
+
status: 'CANCELLED',
|
|
1295
|
+
});
|
|
1296
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
1297
|
+
status: 'CANCELLED',
|
|
1298
|
+
});
|
|
1299
|
+
await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
|
|
1300
|
+
status: TaskStatus.IN_REVIEW,
|
|
1301
|
+
});
|
|
1302
|
+
await orphanService.reconcilePendingRuntimeCleanup();
|
|
1303
|
+
expect(scheduler.releaseInvocationLocks).toHaveBeenCalledTimes(releaseCount);
|
|
1304
|
+
expect(scheduler.startNextSessions).toHaveBeenCalledTimes(startCount);
|
|
1305
|
+
});
|
|
1306
|
+
it.each(['PENDING', 'FAILED'])('keeps a stalled invocation pending until %s runtime cleanup is confirmed', async (cleanupState) => {
|
|
1307
|
+
const { workspace, task, teamRun, members } = await createFixture();
|
|
1308
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1309
|
+
const invocation = await createRunningInvocation({
|
|
1310
|
+
teamRunId: teamRun.id,
|
|
1311
|
+
workRequestId: request.id,
|
|
1312
|
+
memberId: members[0].id,
|
|
1313
|
+
workspaceId: workspace.id,
|
|
1314
|
+
status: 'RUNNING',
|
|
1315
|
+
});
|
|
1316
|
+
await prisma.executionProcess.create({
|
|
1317
|
+
data: {
|
|
1318
|
+
sessionId: invocation.sessionId,
|
|
1319
|
+
launchClaimNumber: 1,
|
|
1320
|
+
runtimeInstanceId: `stalled-${cleanupState.toLowerCase()}`,
|
|
1321
|
+
pid: 8401,
|
|
1322
|
+
processGroupId: '8401',
|
|
1323
|
+
birthMarker: `test-birth:stalled-${cleanupState.toLowerCase()}`,
|
|
1324
|
+
ownershipToken: `stalled-owner-${cleanupState.toLowerCase()}`,
|
|
1325
|
+
cleanupState,
|
|
1326
|
+
},
|
|
1327
|
+
});
|
|
1328
|
+
expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
|
|
1329
|
+
const stalledService = new TeamReconcilerService({
|
|
1330
|
+
scheduler,
|
|
1331
|
+
sessionMessenger: {
|
|
1332
|
+
sendMessage: vi.fn(async () => null),
|
|
1333
|
+
hasActiveTurn: () => false,
|
|
1334
|
+
hasRuntimeProcessOwner: () => false,
|
|
1335
|
+
},
|
|
1336
|
+
scheduleReminders: false,
|
|
1337
|
+
});
|
|
1338
|
+
await stalledService.reconcileStalledInvocations();
|
|
1339
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1340
|
+
status: 'RUNNING',
|
|
1341
|
+
dispatchRevokedAt: expect.any(Date),
|
|
1342
|
+
});
|
|
1343
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
1344
|
+
status: 'STARTED',
|
|
1345
|
+
});
|
|
1346
|
+
await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
|
|
1347
|
+
status: TaskStatus.IN_PROGRESS,
|
|
1348
|
+
});
|
|
1349
|
+
expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
|
|
1350
|
+
expect(scheduler.startNextSessions).not.toHaveBeenCalled();
|
|
1351
|
+
await prisma.executionProcess.updateMany({
|
|
1352
|
+
where: { sessionId: invocation.sessionId },
|
|
1353
|
+
data: { cleanupState: 'CONFIRMED', nextCleanupRetryAt: null },
|
|
1354
|
+
});
|
|
1355
|
+
await stalledService.reconcilePendingRuntimeCleanup();
|
|
1356
|
+
const releaseCount = scheduler.releaseInvocationLocks.mock.calls.length;
|
|
1357
|
+
const startCount = scheduler.startNextSessions.mock.calls.length;
|
|
1358
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1359
|
+
status: 'CANCELLED',
|
|
1360
|
+
});
|
|
1361
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
1362
|
+
status: 'CANCELLED',
|
|
1363
|
+
});
|
|
1364
|
+
await expect(prisma.task.findUnique({ where: { id: task.id } })).resolves.toMatchObject({
|
|
1365
|
+
status: TaskStatus.IN_REVIEW,
|
|
1366
|
+
});
|
|
1367
|
+
await stalledService.reconcilePendingRuntimeCleanup();
|
|
1368
|
+
expect(scheduler.releaseInvocationLocks).toHaveBeenCalledTimes(releaseCount);
|
|
1369
|
+
expect(scheduler.startNextSessions).toHaveBeenCalledTimes(startCount);
|
|
1370
|
+
});
|
|
1371
|
+
it('terminalizes a never-launched orphan with no process record', async () => {
|
|
1372
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
1373
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1374
|
+
const invocation = await createRunningInvocation({
|
|
1375
|
+
teamRunId: teamRun.id,
|
|
1376
|
+
workRequestId: request.id,
|
|
1377
|
+
memberId: members[0].id,
|
|
1378
|
+
workspaceId: workspace.id,
|
|
1379
|
+
status: 'RUNNING',
|
|
1380
|
+
});
|
|
1381
|
+
const orphanService = new TeamReconcilerService({
|
|
1382
|
+
scheduler,
|
|
1383
|
+
sessionMessenger: { sendMessage: vi.fn(async () => null), hasActiveTurn: () => false },
|
|
1384
|
+
scheduleReminders: false,
|
|
1385
|
+
});
|
|
1386
|
+
await orphanService.reconcileOrphanInvocations();
|
|
1387
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1388
|
+
status: 'FAILED',
|
|
1389
|
+
});
|
|
1390
|
+
await expect(prisma.workRequest.findUnique({ where: { id: request.id } })).resolves.toMatchObject({
|
|
1391
|
+
status: 'FAILED',
|
|
1392
|
+
});
|
|
1393
|
+
expect(scheduler.releaseInvocationLocks).toHaveBeenCalledWith(invocation.id);
|
|
1394
|
+
expect(scheduler.startNextSessions).toHaveBeenCalledWith(teamRun.id);
|
|
1395
|
+
});
|
|
1396
|
+
it('does not duplicate terminal side effects when manual stop wins stalled cleanup', async () => {
|
|
1397
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
1398
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1399
|
+
const invocation = await createRunningInvocation({
|
|
1400
|
+
teamRunId: teamRun.id,
|
|
1401
|
+
workRequestId: request.id,
|
|
1402
|
+
memberId: members[0].id,
|
|
1403
|
+
workspaceId: workspace.id,
|
|
1404
|
+
status: 'RUNNING',
|
|
1405
|
+
});
|
|
1406
|
+
await prisma.executionProcess.create({
|
|
1407
|
+
data: {
|
|
1408
|
+
sessionId: invocation.sessionId,
|
|
1409
|
+
launchClaimNumber: 1,
|
|
1410
|
+
runtimeInstanceId: 'manual-stop-pending-runtime',
|
|
1411
|
+
pid: 8501,
|
|
1412
|
+
processGroupId: '8501',
|
|
1413
|
+
birthMarker: 'test-birth:8501',
|
|
1414
|
+
ownershipToken: 'manual-stop-pending-owner',
|
|
1415
|
+
cleanupState: 'PENDING',
|
|
1416
|
+
},
|
|
1417
|
+
});
|
|
1418
|
+
await prisma.agentInvocation.update({
|
|
1419
|
+
where: { id: invocation.id },
|
|
1420
|
+
data: {
|
|
1421
|
+
lastHeartbeatAt: new Date(0),
|
|
1422
|
+
roomReplyReminderCount: 3,
|
|
1423
|
+
firstNudgeAt: new Date(0),
|
|
1424
|
+
},
|
|
1425
|
+
});
|
|
1426
|
+
let releaseAutomatedStop;
|
|
1427
|
+
const automatedStopStarted = new Promise((resolve) => {
|
|
1428
|
+
releaseAutomatedStop = resolve;
|
|
1429
|
+
});
|
|
1430
|
+
let notifyAutomatedStop;
|
|
1431
|
+
const automatedStopCalled = new Promise((resolve) => {
|
|
1432
|
+
notifyAutomatedStop = resolve;
|
|
1433
|
+
});
|
|
1434
|
+
const stop = vi.fn(async () => {
|
|
1435
|
+
notifyAutomatedStop();
|
|
1436
|
+
await automatedStopStarted;
|
|
1437
|
+
return { id: invocation.sessionId };
|
|
1438
|
+
});
|
|
1439
|
+
const stalledService = new TeamReconcilerService({
|
|
1440
|
+
scheduler,
|
|
1441
|
+
sessionMessenger: {
|
|
1442
|
+
sendMessage: vi.fn(async () => null),
|
|
1443
|
+
hasActiveTurn: () => true,
|
|
1444
|
+
stop,
|
|
1445
|
+
},
|
|
1446
|
+
now: () => new Date(Date.UTC(2026, 0, 1, 1, 0, 0)),
|
|
1447
|
+
heartbeatIdleThresholdMs: 1,
|
|
1448
|
+
scheduleReminders: false,
|
|
1449
|
+
});
|
|
1450
|
+
const recovery = stalledService.reconcileStalledInvocations();
|
|
1451
|
+
await automatedStopCalled;
|
|
1452
|
+
const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
|
|
1453
|
+
try {
|
|
1454
|
+
await prisma.agentInvocation.update({
|
|
1455
|
+
where: { id: invocation.id },
|
|
1456
|
+
data: { status: 'CANCELLED' },
|
|
1457
|
+
});
|
|
1458
|
+
}
|
|
1459
|
+
finally {
|
|
1460
|
+
releaseAdmission();
|
|
1461
|
+
}
|
|
1462
|
+
releaseAutomatedStop();
|
|
1463
|
+
await expect(recovery).resolves.toBeUndefined();
|
|
1464
|
+
await expect(stalledService.reconcileStalledInvocations()).resolves.toBeUndefined();
|
|
1465
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1466
|
+
status: 'CANCELLED',
|
|
1467
|
+
dispatchRevokedAt: expect.any(Date),
|
|
1468
|
+
});
|
|
1469
|
+
expect(stop).toHaveBeenCalledOnce();
|
|
1470
|
+
expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalledWith(invocation.id);
|
|
1471
|
+
expect(scheduler.startNextSessions).not.toHaveBeenCalledWith(teamRun.id);
|
|
1472
|
+
});
|
|
1473
|
+
it('does not claim or send a room reminder when revocation wins after the reminder query', async () => {
|
|
1474
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
1475
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1476
|
+
const invocation = await createRunningInvocation({
|
|
1477
|
+
teamRunId: teamRun.id,
|
|
1478
|
+
workRequestId: request.id,
|
|
1479
|
+
memberId: members[0].id,
|
|
1480
|
+
workspaceId: workspace.id,
|
|
1481
|
+
status: 'WAITING_ROOM_REPLY',
|
|
1482
|
+
roomReplyReminderCount: 1,
|
|
1483
|
+
});
|
|
1484
|
+
await prisma.agentInvocation.update({
|
|
1485
|
+
where: { id: invocation.id },
|
|
1486
|
+
data: { nextRoomReplyReminderAt: new Date(0) },
|
|
1487
|
+
});
|
|
1488
|
+
const sendMessage = vi.fn(async () => null);
|
|
1489
|
+
const reminderService = new TeamReconcilerService({
|
|
1490
|
+
scheduler,
|
|
1491
|
+
sessionMessenger: { sendMessage },
|
|
1492
|
+
now: () => new Date(Date.UTC(2026, 0, 1, 1, 0, 0)),
|
|
1493
|
+
scheduleReminders: false,
|
|
1494
|
+
});
|
|
1495
|
+
const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
|
|
1496
|
+
const reconciling = reminderService.reconcileInvocation(invocation.id);
|
|
1497
|
+
await Promise.resolve();
|
|
1498
|
+
try {
|
|
1499
|
+
await prisma.agentInvocation.update({
|
|
1500
|
+
where: { id: invocation.id },
|
|
1501
|
+
data: { dispatchRevokedAt: new Date() },
|
|
1502
|
+
});
|
|
1503
|
+
}
|
|
1504
|
+
finally {
|
|
1505
|
+
releaseAdmission();
|
|
1506
|
+
}
|
|
1507
|
+
await reconciling;
|
|
1508
|
+
expect(sendMessage).not.toHaveBeenCalled();
|
|
1509
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1510
|
+
status: 'WAITING_ROOM_REPLY',
|
|
1511
|
+
roomReplyReminderCount: 1,
|
|
1512
|
+
dispatchRevokedAt: expect.any(Date),
|
|
1513
|
+
});
|
|
1514
|
+
});
|
|
1515
|
+
it('does not write the max-reminder terminal state when revocation wins before its claim', async () => {
|
|
1516
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
1517
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1518
|
+
const invocation = await createRunningInvocation({
|
|
1519
|
+
teamRunId: teamRun.id,
|
|
1520
|
+
workRequestId: request.id,
|
|
1521
|
+
memberId: members[0].id,
|
|
1522
|
+
workspaceId: workspace.id,
|
|
1523
|
+
status: 'WAITING_ROOM_REPLY',
|
|
1524
|
+
roomReplyReminderCount: 3,
|
|
1525
|
+
});
|
|
1526
|
+
const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
|
|
1527
|
+
const reconciling = service.reconcileInvocation(invocation.id);
|
|
1528
|
+
await Promise.resolve();
|
|
1529
|
+
try {
|
|
1530
|
+
await prisma.agentInvocation.update({
|
|
1531
|
+
where: { id: invocation.id },
|
|
1532
|
+
data: { dispatchRevokedAt: new Date() },
|
|
1533
|
+
});
|
|
1534
|
+
}
|
|
1535
|
+
finally {
|
|
1536
|
+
releaseAdmission();
|
|
1537
|
+
}
|
|
1538
|
+
await reconciling;
|
|
1539
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1540
|
+
status: 'WAITING_ROOM_REPLY',
|
|
1541
|
+
roomReplyReminderCount: 3,
|
|
1542
|
+
dispatchRevokedAt: expect.any(Date),
|
|
1543
|
+
});
|
|
1544
|
+
expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalledWith(invocation.id);
|
|
1545
|
+
});
|
|
1546
|
+
it('does not claim or send a stalled nudge when revocation wins at the conditional claim', async () => {
|
|
1547
|
+
const { workspace, teamRun, members } = await createFixture();
|
|
1548
|
+
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
1549
|
+
const invocation = await createRunningInvocation({
|
|
1550
|
+
teamRunId: teamRun.id,
|
|
1551
|
+
workRequestId: request.id,
|
|
1552
|
+
memberId: members[0].id,
|
|
1553
|
+
workspaceId: workspace.id,
|
|
1554
|
+
status: 'RUNNING',
|
|
1555
|
+
});
|
|
1556
|
+
await prisma.agentInvocation.update({
|
|
1557
|
+
where: { id: invocation.id },
|
|
1558
|
+
data: { lastHeartbeatAt: new Date(0) },
|
|
1559
|
+
});
|
|
1560
|
+
const sendMessage = vi.fn(async () => null);
|
|
1561
|
+
const stalledService = new TeamReconcilerService({
|
|
1562
|
+
scheduler,
|
|
1563
|
+
sessionMessenger: { sendMessage, hasActiveTurn: () => true },
|
|
1564
|
+
now: () => new Date(Date.UTC(2026, 0, 1, 1, 0, 0)),
|
|
1565
|
+
heartbeatIdleThresholdMs: 1,
|
|
1566
|
+
scheduleReminders: false,
|
|
1567
|
+
});
|
|
1568
|
+
const releaseAdmission = await acquireTeamMemberAdmission(teamRun.id, members[0].id);
|
|
1569
|
+
const reconciling = stalledService.reconcileStalledInvocations();
|
|
1570
|
+
await Promise.resolve();
|
|
1571
|
+
try {
|
|
1572
|
+
await prisma.agentInvocation.update({
|
|
1573
|
+
where: { id: invocation.id },
|
|
1574
|
+
data: { dispatchRevokedAt: new Date() },
|
|
1575
|
+
});
|
|
1576
|
+
}
|
|
1577
|
+
finally {
|
|
1578
|
+
releaseAdmission();
|
|
1579
|
+
}
|
|
1580
|
+
await reconciling;
|
|
1581
|
+
expect(sendMessage).not.toHaveBeenCalled();
|
|
1582
|
+
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
1583
|
+
status: 'RUNNING',
|
|
1584
|
+
roomReplyReminderCount: 0,
|
|
1585
|
+
dispatchRevokedAt: expect.any(Date),
|
|
1586
|
+
});
|
|
1587
|
+
});
|
|
370
1588
|
it('marks invocation waiting, increments reminder count, and sends a reminder when no RoomMessage exists', async () => {
|
|
371
1589
|
const { workspace, teamRun, members } = await createFixture();
|
|
372
1590
|
const request = await createWorkRequest({ teamRunId: teamRun.id, targetMemberId: members[0].id });
|
|
@@ -381,7 +1599,7 @@ describe('TeamReconcilerService', () => {
|
|
|
381
1599
|
expect(reloaded.status).toBe('WAITING_ROOM_REPLY');
|
|
382
1600
|
expect(reloaded.roomReplyReminderCount).toBe(1);
|
|
383
1601
|
expect(reloaded.nextRoomReplyReminderAt?.toISOString()).toBe('2026-01-01T00:00:01.000Z');
|
|
384
|
-
expect(messenger.sendMessage).toHaveBeenCalledWith(invocation.sessionId, TEAM_ROOM_REPLY_REMINDER);
|
|
1602
|
+
expect(messenger.sendMessage).toHaveBeenCalledWith(invocation.sessionId, TEAM_ROOM_REPLY_REMINDER, undefined, invocation.id);
|
|
385
1603
|
expect(scheduler.releaseInvocationLocks).not.toHaveBeenCalled();
|
|
386
1604
|
});
|
|
387
1605
|
it('ignores forged non-agent RoomMessages with senderInvocationId during reconcile', async () => {
|
|
@@ -647,7 +1865,7 @@ describe('TeamReconcilerService', () => {
|
|
|
647
1865
|
data: { nextRoomReplyReminderAt: new Date(Date.UTC(2026, 0, 1, 0, 1, 0)) },
|
|
648
1866
|
});
|
|
649
1867
|
expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
|
|
650
|
-
await expect(service.handleSessionStopped(invocation.sessionId)).resolves.
|
|
1868
|
+
await expect(service.handleSessionStopped(invocation.sessionId)).resolves.toHaveLength(0);
|
|
651
1869
|
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
652
1870
|
status: 'CANCELLED',
|
|
653
1871
|
nextRoomReplyReminderAt: null,
|
|
@@ -683,7 +1901,7 @@ describe('TeamReconcilerService', () => {
|
|
|
683
1901
|
targetMemberId: members[1].id,
|
|
684
1902
|
status: 'QUEUED',
|
|
685
1903
|
});
|
|
686
|
-
await service.handleSessionStopped(invocation.sessionId);
|
|
1904
|
+
await expect(service.handleSessionStopped(invocation.sessionId)).resolves.toHaveLength(0);
|
|
687
1905
|
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
688
1906
|
status: 'CANCELLED',
|
|
689
1907
|
});
|
|
@@ -715,7 +1933,7 @@ describe('TeamReconcilerService', () => {
|
|
|
715
1933
|
data: { deletedAt: new Date() },
|
|
716
1934
|
});
|
|
717
1935
|
expect(lockService.acquire(invocation.id, ['workspace:task:write'])).toBe(true);
|
|
718
|
-
await expect(service.handleSessionStopped(invocation.sessionId)).resolves.
|
|
1936
|
+
await expect(service.handleSessionStopped(invocation.sessionId)).resolves.toHaveLength(0);
|
|
719
1937
|
await expect(prisma.agentInvocation.findUnique({ where: { id: invocation.id } })).resolves.toMatchObject({
|
|
720
1938
|
status: 'CANCELLED',
|
|
721
1939
|
nextRoomReplyReminderAt: null,
|
|
@@ -1529,6 +2747,40 @@ describe('TeamReconcilerService', () => {
|
|
|
1529
2747
|
await app.close();
|
|
1530
2748
|
}
|
|
1531
2749
|
});
|
|
2750
|
+
it('accepts a terminal REST private message without dispatching recipient work', async () => {
|
|
2751
|
+
const { workspace, teamRun, members } = await createFixture({ memberCount: 2 });
|
|
2752
|
+
const senderRequest = await createWorkRequest({
|
|
2753
|
+
teamRunId: teamRun.id,
|
|
2754
|
+
targetMemberId: members[0].id,
|
|
2755
|
+
status: 'COMPLETED',
|
|
2756
|
+
});
|
|
2757
|
+
const senderInvocation = await createRunningInvocation({
|
|
2758
|
+
teamRunId: teamRun.id,
|
|
2759
|
+
workRequestId: senderRequest.id,
|
|
2760
|
+
memberId: members[0].id,
|
|
2761
|
+
workspaceId: workspace.id,
|
|
2762
|
+
status: 'COMPLETED',
|
|
2763
|
+
});
|
|
2764
|
+
const app = Fastify({ logger: false });
|
|
2765
|
+
try {
|
|
2766
|
+
await app.register(teamRunRoutes, { prefix: '/api' });
|
|
2767
|
+
const response = await app.inject({
|
|
2768
|
+
method: 'POST',
|
|
2769
|
+
url: `/api/team-runs/${teamRun.id}/private-messages`,
|
|
2770
|
+
headers: { 'x-agent-tower-invocation-id': senderInvocation.id },
|
|
2771
|
+
payload: {
|
|
2772
|
+
content: 'Late terminal private result',
|
|
2773
|
+
recipientMemberIds: [members[1].id],
|
|
2774
|
+
},
|
|
2775
|
+
});
|
|
2776
|
+
expect(response.statusCode).toBe(201);
|
|
2777
|
+
expect(response.json()).toMatchObject({ workRequestIds: [] });
|
|
2778
|
+
await expect(prisma.workRequest.count({ where: { teamRunId: teamRun.id } })).resolves.toBe(1);
|
|
2779
|
+
}
|
|
2780
|
+
finally {
|
|
2781
|
+
await app.close();
|
|
2782
|
+
}
|
|
2783
|
+
});
|
|
1532
2784
|
it('preserves REST user private message senderId for member visibility without leaking to non-participants', async () => {
|
|
1533
2785
|
const { workspace, teamRun, members } = await createFixture({
|
|
1534
2786
|
memberCount: 3,
|
|
@@ -1785,6 +3037,7 @@ describe('TeamReconcilerService', () => {
|
|
|
1785
3037
|
memberCount: 2,
|
|
1786
3038
|
memberCapabilities: [{ mentionMembers: false }],
|
|
1787
3039
|
});
|
|
3040
|
+
const revoked = await createFixture({ memberCount: 2 });
|
|
1788
3041
|
const noReadRequest = await createWorkRequest({
|
|
1789
3042
|
teamRunId: noRead.teamRun.id,
|
|
1790
3043
|
targetMemberId: noRead.members[0].id,
|
|
@@ -1818,6 +3071,21 @@ describe('TeamReconcilerService', () => {
|
|
|
1818
3071
|
memberId: noMention.members[0].id,
|
|
1819
3072
|
workspaceId: noMention.workspace.id,
|
|
1820
3073
|
});
|
|
3074
|
+
const revokedRequest = await createWorkRequest({
|
|
3075
|
+
teamRunId: revoked.teamRun.id,
|
|
3076
|
+
targetMemberId: revoked.members[0].id,
|
|
3077
|
+
status: 'STARTED',
|
|
3078
|
+
});
|
|
3079
|
+
const revokedInvocation = await createRunningInvocation({
|
|
3080
|
+
teamRunId: revoked.teamRun.id,
|
|
3081
|
+
workRequestId: revokedRequest.id,
|
|
3082
|
+
memberId: revoked.members[0].id,
|
|
3083
|
+
workspaceId: revoked.workspace.id,
|
|
3084
|
+
});
|
|
3085
|
+
await prisma.agentInvocation.update({
|
|
3086
|
+
where: { id: revokedInvocation.id },
|
|
3087
|
+
data: { dispatchRevokedAt: new Date() },
|
|
3088
|
+
});
|
|
1821
3089
|
const app = Fastify({ logger: false });
|
|
1822
3090
|
try {
|
|
1823
3091
|
await app.register(teamRunRoutes, { prefix: '/api' });
|
|
@@ -1879,6 +3147,16 @@ describe('TeamReconcilerService', () => {
|
|
|
1879
3147
|
});
|
|
1880
3148
|
expect(noMentionResult.isError).toBe(true);
|
|
1881
3149
|
expect(getMcpToolText(noMentionResult)).toContain('mentionMembers');
|
|
3150
|
+
const revokedPrivateResult = await callToolForCurrentMember(revoked, revokedInvocation.id, {
|
|
3151
|
+
name: 'post_private_message',
|
|
3152
|
+
arguments: {
|
|
3153
|
+
recipient_member_ids: [revoked.members[1].id],
|
|
3154
|
+
content: 'Late private dispatch after stop',
|
|
3155
|
+
},
|
|
3156
|
+
});
|
|
3157
|
+
expect(revokedPrivateResult.isError).not.toBe(true);
|
|
3158
|
+
expect(JSON.parse(getMcpToolText(revokedPrivateResult))).toMatchObject({ workRequestIds: [] });
|
|
3159
|
+
await expect(prisma.workRequest.count({ where: { teamRunId: revoked.teamRun.id } })).resolves.toBe(1);
|
|
1882
3160
|
const noStopResult = await callToolForCurrentMember(noRead, noReadInvocation.id, {
|
|
1883
3161
|
name: 'stop_member_work',
|
|
1884
3162
|
arguments: {
|