crewly 1.20.234 → 1.20.236
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/backend/src/index.d.ts +17 -0
- package/dist/backend/backend/src/index.d.ts.map +1 -1
- package/dist/backend/backend/src/index.js +184 -125
- package/dist/backend/backend/src/index.js.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.js +30 -0
- package/dist/backend/backend/src/services/agent/agent-registration.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/idle-detection.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/idle-detection.service.js +5 -9
- package/dist/backend/backend/src/services/agent/idle-detection.service.js.map +1 -1
- package/dist/backend/backend/src/services/agent/resource-mode.service.d.ts +127 -0
- package/dist/backend/backend/src/services/agent/resource-mode.service.d.ts.map +1 -0
- package/dist/backend/backend/src/services/agent/resource-mode.service.js +264 -0
- package/dist/backend/backend/src/services/agent/resource-mode.service.js.map +1 -0
- package/dist/backend/backend/src/services/agent/restore-queue.d.ts +103 -0
- package/dist/backend/backend/src/services/agent/restore-queue.d.ts.map +1 -0
- package/dist/backend/backend/src/services/agent/restore-queue.js +184 -0
- package/dist/backend/backend/src/services/agent/restore-queue.js.map +1 -0
- package/dist/backend/backend/src/types/settings.types.d.ts +4 -0
- package/dist/backend/backend/src/types/settings.types.d.ts.map +1 -1
- package/dist/backend/backend/src/types/settings.types.js +8 -0
- package/dist/backend/backend/src/types/settings.types.js.map +1 -1
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/agent-registration.service.js +30 -0
- package/dist/cli/backend/src/services/agent/agent-registration.service.js.map +1 -1
- package/dist/cli/backend/src/services/agent/resource-mode.service.d.ts +127 -0
- package/dist/cli/backend/src/services/agent/resource-mode.service.d.ts.map +1 -0
- package/dist/cli/backend/src/services/agent/resource-mode.service.js +264 -0
- package/dist/cli/backend/src/services/agent/resource-mode.service.js.map +1 -0
- package/dist/cli/backend/src/services/agent/restore-queue.d.ts +103 -0
- package/dist/cli/backend/src/services/agent/restore-queue.d.ts.map +1 -0
- package/dist/cli/backend/src/services/agent/restore-queue.js +184 -0
- package/dist/cli/backend/src/services/agent/restore-queue.js.map +1 -0
- package/dist/cli/backend/src/services/core/system-health.util.d.ts +89 -0
- package/dist/cli/backend/src/services/core/system-health.util.d.ts.map +1 -0
- package/dist/cli/backend/src/services/core/system-health.util.js +220 -0
- package/dist/cli/backend/src/services/core/system-health.util.js.map +1 -0
- package/dist/cli/backend/src/types/settings.types.d.ts +4 -0
- package/dist/cli/backend/src/types/settings.types.d.ts.map +1 -1
- package/dist/cli/backend/src/types/settings.types.js +8 -0
- package/dist/cli/backend/src/types/settings.types.js.map +1 -1
- package/frontend/dist/assets/{index-bc4e2a93.js → index-a719c15a.js} +54 -54
- package/frontend/dist/index.html +1 -1
- package/package.json +1 -1
|
@@ -111,6 +111,23 @@ export declare class CrewlyServer {
|
|
|
111
111
|
* @param sessions - Entries loaded from session-state.json.
|
|
112
112
|
*/
|
|
113
113
|
private pruneUnboundPersistedSessions;
|
|
114
|
+
/** WorkItems to recover after boot, grouped by target session. */
|
|
115
|
+
private recoveryBySession;
|
|
116
|
+
/**
|
|
117
|
+
* In-progress WorkItems worth re-sending after a restart: active status,
|
|
118
|
+
* with a target, touched within the last hour of `referenceMs`.
|
|
119
|
+
*
|
|
120
|
+
* @param referenceMs - Boot time (age is measured at boot, not after the restore)
|
|
121
|
+
* @returns Items grouped by target session
|
|
122
|
+
*/
|
|
123
|
+
private collectRecoverableTasks;
|
|
124
|
+
/**
|
|
125
|
+
* Send the task-recovery messages of one session (once), e.g. right after
|
|
126
|
+
* that agent registered.
|
|
127
|
+
*
|
|
128
|
+
* @param sessionName - Target session
|
|
129
|
+
*/
|
|
130
|
+
private sendRecoveryFor;
|
|
114
131
|
/**
|
|
115
132
|
* Auto-restore agent sessions that were running before the last shutdown.
|
|
116
133
|
* Loads persisted session state and calls createAgentSession() for each
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../backend/src/index.ts"],"names":[],"mappings":";AA4CA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../backend/src/index.ts"],"names":[],"mappings":";AA4CA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAuTjD,qBAAa,YAAY;IACxB,OAAO,CAAC,GAAG,CAAsB;IACjC,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,EAAE,CAAiB;IAC3B,iFAAiF;IACjF,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,MAAM,CAAqE;IACnF,gFAAgF;IAChF,OAAO,CAAC,iBAAiB,CAAC,CAAyD;IAEnF,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,gBAAgB,CAAoB;IAC5C,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,sBAAsB,CAA0B;IACxD,OAAO,CAAC,4BAA4B,CAAgC;IACpE,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAmB;IAC1C,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,qBAAqB,CAAyB;IACtD,OAAO,CAAC,wBAAwB,CAA4B;IAC5D,OAAO,CAAC,eAAe,CAAmB;IAC1C,qEAAqE;IACrE,OAAO,CAAC,qBAAqB,CAAsC;IACnE,OAAO,CAAC,sBAAsB,CAAuC;IACrE,OAAO,CAAC,sBAAsB,CAAiD;IAC/E,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,kGAAkG;IAClG,OAAO,CAAC,sBAAsB,CAAuC;IAGrE,OAAO,CAAC,+BAA+B,CAAgD;IACvF,+FAA+F;IAC/F,OAAO,CAAC,oBAAoB,CAAqC;IACjE,8HAA8H;IAC9H,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,OAAO,CAAC,6BAA6B,CAA8C;IACnF,OAAO,CAAC,wBAAwB,CAAyC;IACzE,OAAO,CAAC,2BAA2B,CAA+B;IAClE,OAAO,CAAC,0BAA0B,CAA8B;IAChE,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,kBAAkB,CAA0C;IAKpE,OAAO,CAAC,aAAa,CAAuE;IAC5F,OAAO,CAAC,gBAAgB,CAET;IAGf,OAAO,CAAC,cAAc,CAAS;IAC/B,oEAAoE;IACpE,OAAO,CAAC,qBAAqB,CAA6C;IAC1E,OAAO,CAAC,qBAAqB,CAA2G;IACxI,0EAA0E;IAC1E,OAAO,CAAC,eAAe,CAAuC;IAC9D,OAAO,CAAC,gBAAgB,CAAiG;IACzH,oFAAoF;IACpF,OAAO,CAAC,oBAAoB,CAAK;IACjC,2EAA2E;IAC3E,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,yGAAyG;IACzG,OAAO,CAAC,sBAAsB,CAAuB;IACrD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAMhC;IACH,oGAAoG;IACpG,OAAO,CAAC,qBAAqB,CAAwB;IACrD,OAAO,CAAC,wBAAwB,CAA+B;gBAEnD,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC;IAsE3C,OAAO,CAAC,kBAAkB;IAkuC1B,OAAO,CAAC,mBAAmB;IAuE3B,OAAO,CAAC,eAAe;IA4MvB,OAAO,CAAC,kBAAkB;IA0BpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA+7D5B;;;OAGG;YACW,2BAA2B;IAuCzC;;;OAGG;YACW,8BAA8B;IAsB5C;;;OAGG;YACW,gCAAgC;IAsB9C;;;OAGG;YACW,8BAA8B;IAsB5C;;;;OAIG;YACW,8BAA8B;IA8M5C;;;;;;;;;;;;OAYG;YACW,6BAA6B;IAsB3C,kEAAkE;IAClE,OAAO,CAAC,iBAAiB,CAA4E;IAErG;;;;;;OAMG;YACW,uBAAuB;IAsCrC;;;;;OAKG;YACW,eAAe;IAuB7B;;;;;OAKG;YACW,iCAAiC;IA0K/C;;;;OAIG;YACW,2BAA2B;YA0B3B,qBAAqB;IAsBnC;;;;;;OAMG;IACH,OAAO,CAAC,0BAA0B;IAwBlC;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;YAqCZ,eAAe;YAoFf,kBAAkB;IAgBhC,OAAO,CAAC,WAAW,CAAK;IAExB,OAAO,CAAC,sBAAsB;IAsC9B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,oBAAoB;IAwB5B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAoBvB;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAyEvB;;;;;;;OAOG;YACW,2BAA2B;IAczC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IA+E1B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IA4DvB;;;;;;;OAOG;IACH,OAAO,CAAC,0BAA0B;IAuClC;;;;;OAKG;YACW,0BAA0B;IAoExC;;;;OAIG;YACW,qBAAqB;IA+DnC;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAuBlC;;;;;;OAMG;YACW,kBAAkB;IAmBhC;;;;;OAKG;YACW,8BAA8B;IA6C5C;;;;OAIG;YACW,+BAA+B;IA2C7C;;;;;;OAMG;YACW,kBAAkB;IA6BhC,OAAO,CAAC,qBAAqB;IAoC7B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAoE1B;;;;;;;;;OASG;IACH;;;;;;;;;;;;;;OAcG;IACH,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA6B;IAE1D;;;;;;;;OAQG;IACH;;OAEG;IACH;;;;;OAKG;YACW,sBAAsB;YA4CtB,eAAe;YAwCf,kBAAkB;YAyHlB,yBAAyB;IAyFvC;;;;;;;;;OASG;IACH,OAAO,CAAC,oBAAoB;IA8B5B;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,cAAc;IA2BtB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAmChC;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;YA+Ff,wBAAwB;IAWtC,OAAO,CAAC,qBAAqB;IA8H7B,OAAO,CAAC,cAAc;IActB;;;;;;;;OAQG;IACG,QAAQ,CAAC,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA6W1H,SAAS,IAAI,aAAa;CAG1B;AAmCD,eAAe,YAAY,CAAC"}
|
|
@@ -23,7 +23,7 @@ import { TerminalGateway, setTerminalGateway } from './websocket/terminal.gatewa
|
|
|
23
23
|
import { initializeChatGateway } from './websocket/chat.gateway.js';
|
|
24
24
|
import { LoggerService } from './services/core/logger.service.js';
|
|
25
25
|
import { retryWithBackoff } from './services/core/retry.util.js';
|
|
26
|
-
import { CREWLY_CONSTANTS, ORCHESTRATOR_SESSION_NAME, OWNER_MESSAGE_WATCHDOG_CONSTANTS, TURN_STATE_CONSTANTS, CLOUD_DISCONNECT_NOTICE_CONSTANTS, ORCHESTRATOR_ROLE, ORCHESTRATOR_WINDOW_NAME, MESSAGE_QUEUE_CONSTANTS, RUNTIME_TYPES, AUDITOR_CONSTANTS, AUDITOR_SCHEDULER_CONSTANTS, API_SECURITY_CONSTANTS, } from './constants.js';
|
|
26
|
+
import { CREWLY_CONSTANTS, ORCHESTRATOR_SESSION_NAME, AGENT_SUSPEND_CONSTANTS, OWNER_MESSAGE_WATCHDOG_CONSTANTS, TURN_STATE_CONSTANTS, CLOUD_DISCONNECT_NOTICE_CONSTANTS, ORCHESTRATOR_ROLE, ORCHESTRATOR_WINDOW_NAME, MESSAGE_QUEUE_CONSTANTS, RUNTIME_TYPES, AUDITOR_CONSTANTS, AUDITOR_SCHEDULER_CONSTANTS, API_SECURITY_CONSTANTS, TEAM_LEAD_CONSTANTS, } from './constants.js';
|
|
27
27
|
import { getSettingsService } from './services/settings/index.js';
|
|
28
28
|
import { MemoryService } from './services/memory/memory.service.js';
|
|
29
29
|
import { getImprovementStartupService } from './services/orchestrator/improvement-startup.service.js';
|
|
@@ -169,6 +169,8 @@ import { TaskPoolService } from './services/task-pool/task-pool.service.js';
|
|
|
169
169
|
import { PENDING_WORK_STATUSES } from './services/agent/idle-detection.service.js';
|
|
170
170
|
import { WorkItemWorktreeService } from './services/worktree/workitem-worktree.service.js';
|
|
171
171
|
import { WorkItemWorktreeSubscriber, createTerminalNotifier } from './services/worktree/workitem-worktree.subscriber.js';
|
|
172
|
+
import { ResourceModeService, RESOURCE_MODE_CONSTANTS } from './services/agent/resource-mode.service.js';
|
|
173
|
+
import { getRestoreQueue, RESTORE_TIER } from './services/agent/restore-queue.js';
|
|
172
174
|
import { sessionsToRestore } from './services/agent/restore-filter.js';
|
|
173
175
|
import { ProjectMemoryService } from './services/memory/project-memory.service.js';
|
|
174
176
|
import { TaskHistorySubscriber } from './services/memory/task-history.subscriber.js';
|
|
@@ -1701,6 +1703,10 @@ export class CrewlyServer {
|
|
|
1701
1703
|
// Agents whose deliveries the input guard keeps refusing (crewly#1028).
|
|
1702
1704
|
input_circuit: inputCircuitStats(),
|
|
1703
1705
|
cloud: cloudHealthBlock(),
|
|
1706
|
+
// Boot restore progress (staggered restore queue).
|
|
1707
|
+
restoreQueue: getRestoreQueue().stats(),
|
|
1708
|
+
// Adaptive agent limits: normal | pressure, running count, cap.
|
|
1709
|
+
...ResourceModeService.getInstance().stats(),
|
|
1704
1710
|
});
|
|
1705
1711
|
});
|
|
1706
1712
|
// H5 quick entry static page (served regardless of headless mode)
|
|
@@ -1926,6 +1932,44 @@ export class CrewlyServer {
|
|
|
1926
1932
|
return items.some((wi) => wi.target === sessionName && PENDING_WORK_STATUSES.has(wi.status));
|
|
1927
1933
|
});
|
|
1928
1934
|
idleDetection.start();
|
|
1935
|
+
// Adaptive agent limits (pressure mode cap + start queue).
|
|
1936
|
+
const resourceMode = ResourceModeService.getInstance();
|
|
1937
|
+
const agentRegistration = this.apiController.agentRegistrationService;
|
|
1938
|
+
resourceMode.setDeps({
|
|
1939
|
+
limits: async () => {
|
|
1940
|
+
const g = (await getSettingsService().getSettings()).general;
|
|
1941
|
+
return {
|
|
1942
|
+
maxRunning: Math.max(1, g.pressureMaxRunningAgents ?? RESOURCE_MODE_CONSTANTS.DEFAULT_MAX_RUNNING_AGENTS),
|
|
1943
|
+
idleTimeoutMinutes: g.pressureIdleTimeoutMinutes ?? RESOURCE_MODE_CONSTANTS.DEFAULT_IDLE_TIMEOUT_MINUTES,
|
|
1944
|
+
};
|
|
1945
|
+
},
|
|
1946
|
+
listRunning: async () => {
|
|
1947
|
+
const teams = await StorageService.getInstance().getTeams();
|
|
1948
|
+
const tracker = PtyActivityTrackerService.getInstance();
|
|
1949
|
+
const out = [];
|
|
1950
|
+
for (const t of teams) {
|
|
1951
|
+
for (const m of t.members || []) {
|
|
1952
|
+
if (m.agentStatus !== CREWLY_CONSTANTS.AGENT_STATUSES.ACTIVE && m.agentStatus !== CREWLY_CONSTANTS.AGENT_STATUSES.STARTED)
|
|
1953
|
+
continue;
|
|
1954
|
+
if (m.sessionName === ORCHESTRATOR_SESSION_NAME || AGENT_SUSPEND_CONSTANTS.ALWAYS_ON_ROLES.includes(m.role))
|
|
1955
|
+
continue;
|
|
1956
|
+
let busy = m.agentStatus === CREWLY_CONSTANTS.AGENT_STATUSES.STARTED;
|
|
1957
|
+
try {
|
|
1958
|
+
busy = busy || (await this.activityMonitorService.getWorkingStatusForSession(m.sessionName)) === 'in_progress';
|
|
1959
|
+
}
|
|
1960
|
+
catch { /* unknown status: treat as not busy */ }
|
|
1961
|
+
out.push({ sessionName: m.sessionName, role: m.role, idleMs: tracker.getIdleTimeMs(m.sessionName), busy });
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1964
|
+
return out;
|
|
1965
|
+
},
|
|
1966
|
+
hasOwnerMessage: (name) => SubAgentMessageQueue.getInstance().peek(name).some((m) => m.meta?.owner === true),
|
|
1967
|
+
stopAgent: async (name, role) => {
|
|
1968
|
+
await agentRegistration.terminateAgentSession(name, role);
|
|
1969
|
+
await StorageService.getInstance().updateAgentStatus(name, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE, 'idle_exit');
|
|
1970
|
+
},
|
|
1971
|
+
});
|
|
1972
|
+
resourceMode.start();
|
|
1929
1973
|
// Wire OrchestratorRestartService with dependencies for auto-restart
|
|
1930
1974
|
try {
|
|
1931
1975
|
const sessionBackend = getSessionBackendSync();
|
|
@@ -3492,68 +3536,21 @@ export class CrewlyServer {
|
|
|
3492
3536
|
// minutes, and measuring afterwards dropped work that was minutes old
|
|
3493
3537
|
// when the backend went down (CE-128 / CE-132, 2026-10-05).
|
|
3494
3538
|
const recoveryReferenceMs = Date.now();
|
|
3495
|
-
//
|
|
3539
|
+
// #166/#196: in-progress WorkItems to re-send after the restart (not
|
|
3540
|
+
// older than 1 hour at boot). Each restored agent gets its own recovery
|
|
3541
|
+
// messages right after it registers; agents outside the restore queue
|
|
3542
|
+
// (already live, or not restored) get theirs now.
|
|
3543
|
+
this.recoveryBySession = await this.collectRecoverableTasks(recoveryReferenceMs);
|
|
3544
|
+
// Auto-restore agent sessions that were running before the last shutdown.
|
|
3545
|
+
// Starts the staggered restore queue; it drains in the background.
|
|
3496
3546
|
await this.autoRestoreAgentSessionsIfEnabled();
|
|
3497
3547
|
// Re-deliver interrupted turns now that their agents are coming back.
|
|
3498
3548
|
// Background: the orchestrator may take minutes to register.
|
|
3499
3549
|
void this.resumeInterruptedTurnsAfterBoot();
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
// `TaskTrackingService.getAllInProgressTasks()` call).
|
|
3505
|
-
try {
|
|
3506
|
-
const TASK_RECOVERY_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
3507
|
-
const { TaskPoolService } = await import('./services/task-pool/task-pool.service.js');
|
|
3508
|
-
const allItems = await TaskPoolService.getInstance().getAllItems();
|
|
3509
|
-
const now = recoveryReferenceMs;
|
|
3510
|
-
const activeTasks = allItems.filter(wi => {
|
|
3511
|
-
if (wi.status !== 'queued' && wi.status !== 'accepted' && wi.status !== 'running')
|
|
3512
|
-
return false;
|
|
3513
|
-
if (!wi.target)
|
|
3514
|
-
return false;
|
|
3515
|
-
// Skip stale tasks — startedAt/createdAt older than threshold
|
|
3516
|
-
// Last sign of life: a re-claim / status change counts, not only creation.
|
|
3517
|
-
const taskTime = Math.max(new Date(wi.startedAt || wi.createdAt || 0).getTime() || 0, new Date(wi.statusChangedAt || 0).getTime() || 0);
|
|
3518
|
-
if (now - taskTime > TASK_RECOVERY_MAX_AGE_MS) {
|
|
3519
|
-
this.logger.info('Skipping stale task recovery (older than 1 hour)', {
|
|
3520
|
-
workItemId: wi.id,
|
|
3521
|
-
taskName: wi.title,
|
|
3522
|
-
age: `${Math.round((now - taskTime) / 60000)} minutes`,
|
|
3523
|
-
});
|
|
3524
|
-
return false;
|
|
3525
|
-
}
|
|
3526
|
-
return true;
|
|
3527
|
-
});
|
|
3528
|
-
if (activeTasks.length > 0) {
|
|
3529
|
-
this.logger.info('Found in-progress WorkItems to recover after restart', {
|
|
3530
|
-
count: activeTasks.length,
|
|
3531
|
-
});
|
|
3532
|
-
for (const wi of activeTasks) {
|
|
3533
|
-
try {
|
|
3534
|
-
const recoveryMessage = `[SYSTEM — TASK RECOVERY] You were working on this task before the server restarted. Please continue:\n\nTask: ${wi.title}\nWorkItem: ${wi.id}\n\nFetch full brief: bash config/skills/agent/core/read-task/execute.sh '{"workItemId":"${wi.id}"}'\n\nPlease check the current state and continue working.`;
|
|
3535
|
-
await this.apiController.agentRegistrationService.sendMessageToAgent(wi.target, recoveryMessage, undefined);
|
|
3536
|
-
this.logger.info('Task recovery message sent', {
|
|
3537
|
-
workItemId: wi.id,
|
|
3538
|
-
sessionName: wi.target,
|
|
3539
|
-
taskName: wi.title,
|
|
3540
|
-
});
|
|
3541
|
-
}
|
|
3542
|
-
catch (err) {
|
|
3543
|
-
// Agent might not be online yet — DLQ in scheduler will handle it
|
|
3544
|
-
this.logger.warn('Task recovery delivery deferred (agent may not be online yet)', {
|
|
3545
|
-
workItemId: wi.id,
|
|
3546
|
-
sessionName: wi.target,
|
|
3547
|
-
error: err instanceof Error ? err.message : String(err),
|
|
3548
|
-
});
|
|
3549
|
-
}
|
|
3550
|
-
}
|
|
3551
|
-
}
|
|
3552
|
-
}
|
|
3553
|
-
catch (err) {
|
|
3554
|
-
this.logger.warn('Task auto-recovery failed (non-critical)', {
|
|
3555
|
-
error: err instanceof Error ? err.message : String(err),
|
|
3556
|
-
});
|
|
3550
|
+
const queued = new Set(getRestoreQueue().stats().order);
|
|
3551
|
+
for (const session of [...this.recoveryBySession.keys()]) {
|
|
3552
|
+
if (!queued.has(session))
|
|
3553
|
+
await this.sendRecoveryFor(session);
|
|
3557
3554
|
}
|
|
3558
3555
|
// Start log rotation service (non-critical — logs cleanup)
|
|
3559
3556
|
try {
|
|
@@ -3983,6 +3980,77 @@ export class CrewlyServer {
|
|
|
3983
3980
|
});
|
|
3984
3981
|
}
|
|
3985
3982
|
}
|
|
3983
|
+
/** WorkItems to recover after boot, grouped by target session. */
|
|
3984
|
+
recoveryBySession = new Map();
|
|
3985
|
+
/**
|
|
3986
|
+
* In-progress WorkItems worth re-sending after a restart: active status,
|
|
3987
|
+
* with a target, touched within the last hour of `referenceMs`.
|
|
3988
|
+
*
|
|
3989
|
+
* @param referenceMs - Boot time (age is measured at boot, not after the restore)
|
|
3990
|
+
* @returns Items grouped by target session
|
|
3991
|
+
*/
|
|
3992
|
+
async collectRecoverableTasks(referenceMs) {
|
|
3993
|
+
const out = new Map();
|
|
3994
|
+
try {
|
|
3995
|
+
const TASK_RECOVERY_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
3996
|
+
const allItems = await TaskPoolService.getInstance().getAllItems();
|
|
3997
|
+
for (const wi of allItems) {
|
|
3998
|
+
if (wi.status !== 'queued' && wi.status !== 'accepted' && wi.status !== 'running')
|
|
3999
|
+
continue;
|
|
4000
|
+
if (!wi.target)
|
|
4001
|
+
continue;
|
|
4002
|
+
// Last sign of life: a re-claim / status change counts, not only creation.
|
|
4003
|
+
const taskTime = Math.max(new Date(wi.startedAt || wi.createdAt || 0).getTime() || 0, new Date(wi.statusChangedAt || 0).getTime() || 0);
|
|
4004
|
+
if (referenceMs - taskTime > TASK_RECOVERY_MAX_AGE_MS) {
|
|
4005
|
+
this.logger.info('Skipping stale task recovery (older than 1 hour)', {
|
|
4006
|
+
workItemId: wi.id,
|
|
4007
|
+
taskName: wi.title,
|
|
4008
|
+
age: `${Math.round((referenceMs - taskTime) / 60000)} minutes`,
|
|
4009
|
+
});
|
|
4010
|
+
continue;
|
|
4011
|
+
}
|
|
4012
|
+
const list = out.get(wi.target) ?? [];
|
|
4013
|
+
list.push(wi);
|
|
4014
|
+
out.set(wi.target, list);
|
|
4015
|
+
}
|
|
4016
|
+
if (out.size > 0) {
|
|
4017
|
+
this.logger.info('Found in-progress WorkItems to recover after restart', {
|
|
4018
|
+
count: [...out.values()].reduce((n, l) => n + l.length, 0),
|
|
4019
|
+
});
|
|
4020
|
+
}
|
|
4021
|
+
}
|
|
4022
|
+
catch (err) {
|
|
4023
|
+
this.logger.warn('Task auto-recovery failed (non-critical)', {
|
|
4024
|
+
error: err instanceof Error ? err.message : String(err),
|
|
4025
|
+
});
|
|
4026
|
+
}
|
|
4027
|
+
return out;
|
|
4028
|
+
}
|
|
4029
|
+
/**
|
|
4030
|
+
* Send the task-recovery messages of one session (once), e.g. right after
|
|
4031
|
+
* that agent registered.
|
|
4032
|
+
*
|
|
4033
|
+
* @param sessionName - Target session
|
|
4034
|
+
*/
|
|
4035
|
+
async sendRecoveryFor(sessionName) {
|
|
4036
|
+
const items = this.recoveryBySession.get(sessionName);
|
|
4037
|
+
this.recoveryBySession.delete(sessionName);
|
|
4038
|
+
for (const wi of items ?? []) {
|
|
4039
|
+
try {
|
|
4040
|
+
const recoveryMessage = `[SYSTEM — TASK RECOVERY] You were working on this task before the server restarted. Please continue:\n\nTask: ${wi.title}\nWorkItem: ${wi.id}\n\nFetch full brief: bash config/skills/agent/core/read-task/execute.sh '{"workItemId":"${wi.id}"}'\n\nPlease check the current state and continue working.`;
|
|
4041
|
+
await this.apiController.agentRegistrationService.sendMessageToAgent(sessionName, recoveryMessage, undefined);
|
|
4042
|
+
this.logger.info('Task recovery message sent', { workItemId: wi.id, sessionName, taskName: wi.title });
|
|
4043
|
+
}
|
|
4044
|
+
catch (err) {
|
|
4045
|
+
// Agent might not be online yet — DLQ in scheduler will handle it
|
|
4046
|
+
this.logger.warn('Task recovery delivery deferred (agent may not be online yet)', {
|
|
4047
|
+
workItemId: wi.id,
|
|
4048
|
+
sessionName,
|
|
4049
|
+
error: err instanceof Error ? err.message : String(err),
|
|
4050
|
+
});
|
|
4051
|
+
}
|
|
4052
|
+
}
|
|
4053
|
+
}
|
|
3986
4054
|
/**
|
|
3987
4055
|
* Auto-restore agent sessions that were running before the last shutdown.
|
|
3988
4056
|
* Loads persisted session state and calls createAgentSession() for each
|
|
@@ -4077,76 +4145,67 @@ export class CrewlyServer {
|
|
|
4077
4145
|
count: agentSessions.length,
|
|
4078
4146
|
sessions: agentSessions.map((s) => s.name),
|
|
4079
4147
|
});
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
const
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
}
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
role: session.role || 'developer',
|
|
4111
|
-
projectPath: session.cwd || process.cwd(),
|
|
4112
|
-
runtimeType: session.runtimeType,
|
|
4113
|
-
teamId: session.teamId,
|
|
4114
|
-
memberId: session.memberId,
|
|
4115
|
-
forceRecreate: true,
|
|
4116
|
-
});
|
|
4117
|
-
if (result.success) {
|
|
4118
|
-
restored++;
|
|
4119
|
-
this.logger.info('Restored agent session', {
|
|
4120
|
-
name: session.name,
|
|
4121
|
-
role: session.role,
|
|
4148
|
+
// Staggered restore: one agent at a time (CREWLY_RESTORE_CONCURRENCY,
|
|
4149
|
+
// max 3), the next starts when the previous one registered or timed out.
|
|
4150
|
+
// Owner-waiting agents first, then orchestrator, leads, everyone else.
|
|
4151
|
+
const queueMessages = SubAgentMessageQueue.getInstance();
|
|
4152
|
+
const owedSince = new Map();
|
|
4153
|
+
for (const c of this.openCommitmentsAtBoot) {
|
|
4154
|
+
const due = c.due ? Date.parse(c.due) : NaN;
|
|
4155
|
+
owedSince.set(c.sessionName, Math.min(owedSince.get(c.sessionName) ?? Infinity, Number.isFinite(due) ? due : 0));
|
|
4156
|
+
}
|
|
4157
|
+
const leadRoles = TEAM_LEAD_CONSTANTS.LEAD_ROLES;
|
|
4158
|
+
const entries = agentSessions.map((session) => {
|
|
4159
|
+
const ownerTimes = queueMessages.peek(session.name).filter((m) => m.meta?.owner === true).map((m) => m.queuedAt);
|
|
4160
|
+
if (owedSince.has(session.name))
|
|
4161
|
+
ownerTimes.push(owedSince.get(session.name));
|
|
4162
|
+
const base = {
|
|
4163
|
+
name: session.name,
|
|
4164
|
+
run: async (attempt) => {
|
|
4165
|
+
// PTYs live in this process, so none survive a restart: a session
|
|
4166
|
+
// that exists now was started this boot by another launcher (a
|
|
4167
|
+
// reconciler wake, a team start). forceRecreate would kill it and
|
|
4168
|
+
// resume the same conversation again — a second kickoff in one
|
|
4169
|
+
// chat (2026-09-25 startup-prompt loop). Leave it be.
|
|
4170
|
+
if (attempt === 0 && await this.apiController.agentRegistrationService.isSessionLiveOrLaunching(session.name)) {
|
|
4171
|
+
this.logger.info('Skipping restore — session already started this boot by another launcher', { name: session.name });
|
|
4172
|
+
return { success: true };
|
|
4173
|
+
}
|
|
4174
|
+
const result = await this.apiController.agentRegistrationService.createAgentSession({
|
|
4175
|
+
sessionName: session.name,
|
|
4176
|
+
role: session.role || 'developer',
|
|
4177
|
+
projectPath: session.cwd || process.cwd(),
|
|
4122
4178
|
runtimeType: session.runtimeType,
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
failed.push(session.name);
|
|
4128
|
-
this.logger.warn('Failed to restore agent session', {
|
|
4129
|
-
name: session.name,
|
|
4130
|
-
error: result.error,
|
|
4179
|
+
teamId: session.teamId,
|
|
4180
|
+
memberId: session.memberId,
|
|
4181
|
+
// A retry joins the launch still in flight instead of killing it.
|
|
4182
|
+
forceRecreate: attempt === 0,
|
|
4131
4183
|
});
|
|
4132
|
-
|
|
4133
|
-
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
this.
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
}
|
|
4184
|
+
if (!result.success)
|
|
4185
|
+
this.logger.warn('Failed to restore agent session', { name: session.name, error: result.error });
|
|
4186
|
+
return { success: result.success };
|
|
4187
|
+
},
|
|
4188
|
+
onReady: () => this.sendRecoveryFor(session.name),
|
|
4189
|
+
};
|
|
4190
|
+
if (ownerTimes.length > 0) {
|
|
4191
|
+
return { ...base, tier: RESTORE_TIER.OWNER, ownerSince: Math.min(...ownerTimes), reason: 'owner message waiting' };
|
|
4140
4192
|
}
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4193
|
+
if (session.role === ORCHESTRATOR_ROLE)
|
|
4194
|
+
return { ...base, tier: RESTORE_TIER.ORCHESTRATOR, reason: 'orchestrator' };
|
|
4195
|
+
if (leadRoles.includes(session.role ?? ''))
|
|
4196
|
+
return { ...base, tier: RESTORE_TIER.LEAD, reason: 'team lead with work in hand' };
|
|
4197
|
+
return { ...base, tier: RESTORE_TIER.OTHER, reason: 'work in hand' };
|
|
4198
|
+
});
|
|
4199
|
+
// Runs in the background; boot continues while agents come back.
|
|
4200
|
+
void getRestoreQueue()
|
|
4201
|
+
.start(entries)
|
|
4202
|
+
.then(async () => {
|
|
4203
|
+
// Clear persisted state after the restore to avoid double-restore
|
|
4204
|
+
await persistence.clearState();
|
|
4205
|
+
})
|
|
4206
|
+
.catch((error) => {
|
|
4207
|
+
this.logger.error('Restore queue failed', { error: error instanceof Error ? error.message : String(error) });
|
|
4147
4208
|
});
|
|
4148
|
-
// Clear persisted state after restore attempt to avoid double-restore
|
|
4149
|
-
await persistence.clearState();
|
|
4150
4209
|
}
|
|
4151
4210
|
catch (error) {
|
|
4152
4211
|
this.logger.error('Failed to auto-restore agent sessions', {
|