crewly 1.20.234 → 1.20.236

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/backend/backend/src/index.d.ts +17 -0
  2. package/dist/backend/backend/src/index.d.ts.map +1 -1
  3. package/dist/backend/backend/src/index.js +184 -125
  4. package/dist/backend/backend/src/index.js.map +1 -1
  5. package/dist/backend/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
  6. package/dist/backend/backend/src/services/agent/agent-registration.service.js +30 -0
  7. package/dist/backend/backend/src/services/agent/agent-registration.service.js.map +1 -1
  8. package/dist/backend/backend/src/services/agent/idle-detection.service.d.ts.map +1 -1
  9. package/dist/backend/backend/src/services/agent/idle-detection.service.js +5 -9
  10. package/dist/backend/backend/src/services/agent/idle-detection.service.js.map +1 -1
  11. package/dist/backend/backend/src/services/agent/resource-mode.service.d.ts +127 -0
  12. package/dist/backend/backend/src/services/agent/resource-mode.service.d.ts.map +1 -0
  13. package/dist/backend/backend/src/services/agent/resource-mode.service.js +264 -0
  14. package/dist/backend/backend/src/services/agent/resource-mode.service.js.map +1 -0
  15. package/dist/backend/backend/src/services/agent/restore-queue.d.ts +103 -0
  16. package/dist/backend/backend/src/services/agent/restore-queue.d.ts.map +1 -0
  17. package/dist/backend/backend/src/services/agent/restore-queue.js +184 -0
  18. package/dist/backend/backend/src/services/agent/restore-queue.js.map +1 -0
  19. package/dist/backend/backend/src/types/settings.types.d.ts +4 -0
  20. package/dist/backend/backend/src/types/settings.types.d.ts.map +1 -1
  21. package/dist/backend/backend/src/types/settings.types.js +8 -0
  22. package/dist/backend/backend/src/types/settings.types.js.map +1 -1
  23. package/dist/backend/build-info.json +2 -2
  24. package/dist/cli/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
  25. package/dist/cli/backend/src/services/agent/agent-registration.service.js +30 -0
  26. package/dist/cli/backend/src/services/agent/agent-registration.service.js.map +1 -1
  27. package/dist/cli/backend/src/services/agent/resource-mode.service.d.ts +127 -0
  28. package/dist/cli/backend/src/services/agent/resource-mode.service.d.ts.map +1 -0
  29. package/dist/cli/backend/src/services/agent/resource-mode.service.js +264 -0
  30. package/dist/cli/backend/src/services/agent/resource-mode.service.js.map +1 -0
  31. package/dist/cli/backend/src/services/agent/restore-queue.d.ts +103 -0
  32. package/dist/cli/backend/src/services/agent/restore-queue.d.ts.map +1 -0
  33. package/dist/cli/backend/src/services/agent/restore-queue.js +184 -0
  34. package/dist/cli/backend/src/services/agent/restore-queue.js.map +1 -0
  35. package/dist/cli/backend/src/services/core/system-health.util.d.ts +89 -0
  36. package/dist/cli/backend/src/services/core/system-health.util.d.ts.map +1 -0
  37. package/dist/cli/backend/src/services/core/system-health.util.js +220 -0
  38. package/dist/cli/backend/src/services/core/system-health.util.js.map +1 -0
  39. package/dist/cli/backend/src/types/settings.types.d.ts +4 -0
  40. package/dist/cli/backend/src/types/settings.types.d.ts.map +1 -1
  41. package/dist/cli/backend/src/types/settings.types.js +8 -0
  42. package/dist/cli/backend/src/types/settings.types.js.map +1 -1
  43. package/frontend/dist/assets/{index-bc4e2a93.js → index-a719c15a.js} +54 -54
  44. package/frontend/dist/index.html +1 -1
  45. package/package.json +1 -1
@@ -111,6 +111,23 @@ export declare class CrewlyServer {
111
111
  * @param sessions - Entries loaded from session-state.json.
112
112
  */
113
113
  private pruneUnboundPersistedSessions;
114
+ /** WorkItems to recover after boot, grouped by target session. */
115
+ private recoveryBySession;
116
+ /**
117
+ * In-progress WorkItems worth re-sending after a restart: active status,
118
+ * with a target, touched within the last hour of `referenceMs`.
119
+ *
120
+ * @param referenceMs - Boot time (age is measured at boot, not after the restore)
121
+ * @returns Items grouped by target session
122
+ */
123
+ private collectRecoverableTasks;
124
+ /**
125
+ * Send the task-recovery messages of one session (once), e.g. right after
126
+ * that agent registered.
127
+ *
128
+ * @param sessionName - Target session
129
+ */
130
+ private sendRecoveryFor;
114
131
  /**
115
132
  * Auto-restore agent sessions that were running before the last shutdown.
116
133
  * Loads persisted session state and calls createAgentSession() for each
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../backend/src/index.ts"],"names":[],"mappings":";AA4CA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAmTjD,qBAAa,YAAY;IACxB,OAAO,CAAC,GAAG,CAAsB;IACjC,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,EAAE,CAAiB;IAC3B,iFAAiF;IACjF,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,MAAM,CAAqE;IACnF,gFAAgF;IAChF,OAAO,CAAC,iBAAiB,CAAC,CAAyD;IAEnF,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,gBAAgB,CAAoB;IAC5C,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,sBAAsB,CAA0B;IACxD,OAAO,CAAC,4BAA4B,CAAgC;IACpE,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAmB;IAC1C,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,qBAAqB,CAAyB;IACtD,OAAO,CAAC,wBAAwB,CAA4B;IAC5D,OAAO,CAAC,eAAe,CAAmB;IAC1C,qEAAqE;IACrE,OAAO,CAAC,qBAAqB,CAAsC;IACnE,OAAO,CAAC,sBAAsB,CAAuC;IACrE,OAAO,CAAC,sBAAsB,CAAiD;IAC/E,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,kGAAkG;IAClG,OAAO,CAAC,sBAAsB,CAAuC;IAGrE,OAAO,CAAC,+BAA+B,CAAgD;IACvF,+FAA+F;IAC/F,OAAO,CAAC,oBAAoB,CAAqC;IACjE,8HAA8H;IAC9H,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,OAAO,CAAC,6BAA6B,CAA8C;IACnF,OAAO,CAAC,wBAAwB,CAAyC;IACzE,OAAO,CAAC,2BAA2B,CAA+B;IAClE,OAAO,CAAC,0BAA0B,CAA8B;IAChE,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,kBAAkB,CAA0C;IAKpE,OAAO,CAAC,aAAa,CAAuE;IAC5F,OAAO,CAAC,gBAAgB,CAET;IAGf,OAAO,CAAC,cAAc,CAAS;IAC/B,oEAAoE;IACpE,OAAO,CAAC,qBAAqB,CAA6C;IAC1E,OAAO,CAAC,qBAAqB,CAA2G;IACxI,0EAA0E;IAC1E,OAAO,CAAC,eAAe,CAAuC;IAC9D,OAAO,CAAC,gBAAgB,CAAiG;IACzH,oFAAoF;IACpF,OAAO,CAAC,oBAAoB,CAAK;IACjC,2EAA2E;IAC3E,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,yGAAyG;IACzG,OAAO,CAAC,sBAAsB,CAAuB;IACrD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAMhC;IACH,oGAAoG;IACpG,OAAO,CAAC,qBAAqB,CAAwB;IACrD,OAAO,CAAC,wBAAwB,CAA+B;gBAEnD,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC;IAsE3C,OAAO,CAAC,kBAAkB;IAkuC1B,OAAO,CAAC,mBAAmB;IAuE3B,OAAO,CAAC,eAAe;IAwMvB,OAAO,CAAC,kBAAkB;IA0BpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA68D5B;;;OAGG;YACW,2BAA2B;IAuCzC;;;OAGG;YACW,8BAA8B;IAsB5C;;;OAGG;YACW,gCAAgC;IAsB9C;;;OAGG;YACW,8BAA8B;IAsB5C;;;;OAIG;YACW,8BAA8B;IA8M5C;;;;;;;;;;;;OAYG;YACW,6BAA6B;IAsB3C;;;;;OAKG;YACW,iCAAiC;IA0L/C;;;;OAIG;YACW,2BAA2B;YA0B3B,qBAAqB;IAsBnC;;;;;;OAMG;IACH,OAAO,CAAC,0BAA0B;IAwBlC;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;YAqCZ,eAAe;YAoFf,kBAAkB;IAgBhC,OAAO,CAAC,WAAW,CAAK;IAExB,OAAO,CAAC,sBAAsB;IAsC9B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,oBAAoB;IAwB5B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAoBvB;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAyEvB;;;;;;;OAOG;YACW,2BAA2B;IAczC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IA+E1B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IA4DvB;;;;;;;OAOG;IACH,OAAO,CAAC,0BAA0B;IAuClC;;;;;OAKG;YACW,0BAA0B;IAoExC;;;;OAIG;YACW,qBAAqB;IA+DnC;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAuBlC;;;;;;OAMG;YACW,kBAAkB;IAmBhC;;;;;OAKG;YACW,8BAA8B;IA6C5C;;;;OAIG;YACW,+BAA+B;IA2C7C;;;;;;OAMG;YACW,kBAAkB;IA6BhC,OAAO,CAAC,qBAAqB;IAoC7B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAoE1B;;;;;;;;;OASG;IACH;;;;;;;;;;;;;;OAcG;IACH,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA6B;IAE1D;;;;;;;;OAQG;IACH;;OAEG;IACH;;;;;OAKG;YACW,sBAAsB;YA4CtB,eAAe;YAwCf,kBAAkB;YAyHlB,yBAAyB;IAyFvC;;;;;;;;;OASG;IACH,OAAO,CAAC,oBAAoB;IA8B5B;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,cAAc;IA2BtB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAmChC;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;YA+Ff,wBAAwB;IAWtC,OAAO,CAAC,qBAAqB;IA8H7B,OAAO,CAAC,cAAc;IActB;;;;;;;;OAQG;IACG,QAAQ,CAAC,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA6W1H,SAAS,IAAI,aAAa;CAG1B;AAmCD,eAAe,YAAY,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../backend/src/index.ts"],"names":[],"mappings":";AA4CA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAuTjD,qBAAa,YAAY;IACxB,OAAO,CAAC,GAAG,CAAsB;IACjC,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,EAAE,CAAiB;IAC3B,iFAAiF;IACjF,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,MAAM,CAAqE;IACnF,gFAAgF;IAChF,OAAO,CAAC,iBAAiB,CAAC,CAAyD;IAEnF,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,gBAAgB,CAAoB;IAC5C,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,sBAAsB,CAA0B;IACxD,OAAO,CAAC,4BAA4B,CAAgC;IACpE,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAmB;IAC1C,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,qBAAqB,CAAyB;IACtD,OAAO,CAAC,wBAAwB,CAA4B;IAC5D,OAAO,CAAC,eAAe,CAAmB;IAC1C,qEAAqE;IACrE,OAAO,CAAC,qBAAqB,CAAsC;IACnE,OAAO,CAAC,sBAAsB,CAAuC;IACrE,OAAO,CAAC,sBAAsB,CAAiD;IAC/E,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,kGAAkG;IAClG,OAAO,CAAC,sBAAsB,CAAuC;IAGrE,OAAO,CAAC,+BAA+B,CAAgD;IACvF,+FAA+F;IAC/F,OAAO,CAAC,oBAAoB,CAAqC;IACjE,8HAA8H;IAC9H,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,OAAO,CAAC,6BAA6B,CAA8C;IACnF,OAAO,CAAC,wBAAwB,CAAyC;IACzE,OAAO,CAAC,2BAA2B,CAA+B;IAClE,OAAO,CAAC,0BAA0B,CAA8B;IAChE,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,kBAAkB,CAA0C;IAKpE,OAAO,CAAC,aAAa,CAAuE;IAC5F,OAAO,CAAC,gBAAgB,CAET;IAGf,OAAO,CAAC,cAAc,CAAS;IAC/B,oEAAoE;IACpE,OAAO,CAAC,qBAAqB,CAA6C;IAC1E,OAAO,CAAC,qBAAqB,CAA2G;IACxI,0EAA0E;IAC1E,OAAO,CAAC,eAAe,CAAuC;IAC9D,OAAO,CAAC,gBAAgB,CAAiG;IACzH,oFAAoF;IACpF,OAAO,CAAC,oBAAoB,CAAK;IACjC,2EAA2E;IAC3E,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,yGAAyG;IACzG,OAAO,CAAC,sBAAsB,CAAuB;IACrD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAMhC;IACH,oGAAoG;IACpG,OAAO,CAAC,qBAAqB,CAAwB;IACrD,OAAO,CAAC,wBAAwB,CAA+B;gBAEnD,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC;IAsE3C,OAAO,CAAC,kBAAkB;IAkuC1B,OAAO,CAAC,mBAAmB;IAuE3B,OAAO,CAAC,eAAe;IA4MvB,OAAO,CAAC,kBAAkB;IA0BpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA+7D5B;;;OAGG;YACW,2BAA2B;IAuCzC;;;OAGG;YACW,8BAA8B;IAsB5C;;;OAGG;YACW,gCAAgC;IAsB9C;;;OAGG;YACW,8BAA8B;IAsB5C;;;;OAIG;YACW,8BAA8B;IA8M5C;;;;;;;;;;;;OAYG;YACW,6BAA6B;IAsB3C,kEAAkE;IAClE,OAAO,CAAC,iBAAiB,CAA4E;IAErG;;;;;;OAMG;YACW,uBAAuB;IAsCrC;;;;;OAKG;YACW,eAAe;IAuB7B;;;;;OAKG;YACW,iCAAiC;IA0K/C;;;;OAIG;YACW,2BAA2B;YA0B3B,qBAAqB;IAsBnC;;;;;;OAMG;IACH,OAAO,CAAC,0BAA0B;IAwBlC;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;YAqCZ,eAAe;YAoFf,kBAAkB;IAgBhC,OAAO,CAAC,WAAW,CAAK;IAExB,OAAO,CAAC,sBAAsB;IAsC9B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,oBAAoB;IAwB5B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAoBvB;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAyEvB;;;;;;;OAOG;YACW,2BAA2B;IAczC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IA+E1B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IA4DvB;;;;;;;OAOG;IACH,OAAO,CAAC,0BAA0B;IAuClC;;;;;OAKG;YACW,0BAA0B;IAoExC;;;;OAIG;YACW,qBAAqB;IA+DnC;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAuBlC;;;;;;OAMG;YACW,kBAAkB;IAmBhC;;;;;OAKG;YACW,8BAA8B;IA6C5C;;;;OAIG;YACW,+BAA+B;IA2C7C;;;;;;OAMG;YACW,kBAAkB;IA6BhC,OAAO,CAAC,qBAAqB;IAoC7B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAoE1B;;;;;;;;;OASG;IACH;;;;;;;;;;;;;;OAcG;IACH,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA6B;IAE1D;;;;;;;;OAQG;IACH;;OAEG;IACH;;;;;OAKG;YACW,sBAAsB;YA4CtB,eAAe;YAwCf,kBAAkB;YAyHlB,yBAAyB;IAyFvC;;;;;;;;;OASG;IACH,OAAO,CAAC,oBAAoB;IA8B5B;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,cAAc;IA2BtB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAmChC;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;YA+Ff,wBAAwB;IAWtC,OAAO,CAAC,qBAAqB;IA8H7B,OAAO,CAAC,cAAc;IActB;;;;;;;;OAQG;IACG,QAAQ,CAAC,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA6W1H,SAAS,IAAI,aAAa;CAG1B;AAmCD,eAAe,YAAY,CAAC"}
@@ -23,7 +23,7 @@ import { TerminalGateway, setTerminalGateway } from './websocket/terminal.gatewa
23
23
  import { initializeChatGateway } from './websocket/chat.gateway.js';
24
24
  import { LoggerService } from './services/core/logger.service.js';
25
25
  import { retryWithBackoff } from './services/core/retry.util.js';
26
- import { CREWLY_CONSTANTS, ORCHESTRATOR_SESSION_NAME, OWNER_MESSAGE_WATCHDOG_CONSTANTS, TURN_STATE_CONSTANTS, CLOUD_DISCONNECT_NOTICE_CONSTANTS, ORCHESTRATOR_ROLE, ORCHESTRATOR_WINDOW_NAME, MESSAGE_QUEUE_CONSTANTS, RUNTIME_TYPES, AUDITOR_CONSTANTS, AUDITOR_SCHEDULER_CONSTANTS, API_SECURITY_CONSTANTS, } from './constants.js';
26
+ import { CREWLY_CONSTANTS, ORCHESTRATOR_SESSION_NAME, AGENT_SUSPEND_CONSTANTS, OWNER_MESSAGE_WATCHDOG_CONSTANTS, TURN_STATE_CONSTANTS, CLOUD_DISCONNECT_NOTICE_CONSTANTS, ORCHESTRATOR_ROLE, ORCHESTRATOR_WINDOW_NAME, MESSAGE_QUEUE_CONSTANTS, RUNTIME_TYPES, AUDITOR_CONSTANTS, AUDITOR_SCHEDULER_CONSTANTS, API_SECURITY_CONSTANTS, TEAM_LEAD_CONSTANTS, } from './constants.js';
27
27
  import { getSettingsService } from './services/settings/index.js';
28
28
  import { MemoryService } from './services/memory/memory.service.js';
29
29
  import { getImprovementStartupService } from './services/orchestrator/improvement-startup.service.js';
@@ -169,6 +169,8 @@ import { TaskPoolService } from './services/task-pool/task-pool.service.js';
169
169
  import { PENDING_WORK_STATUSES } from './services/agent/idle-detection.service.js';
170
170
  import { WorkItemWorktreeService } from './services/worktree/workitem-worktree.service.js';
171
171
  import { WorkItemWorktreeSubscriber, createTerminalNotifier } from './services/worktree/workitem-worktree.subscriber.js';
172
+ import { ResourceModeService, RESOURCE_MODE_CONSTANTS } from './services/agent/resource-mode.service.js';
173
+ import { getRestoreQueue, RESTORE_TIER } from './services/agent/restore-queue.js';
172
174
  import { sessionsToRestore } from './services/agent/restore-filter.js';
173
175
  import { ProjectMemoryService } from './services/memory/project-memory.service.js';
174
176
  import { TaskHistorySubscriber } from './services/memory/task-history.subscriber.js';
@@ -1701,6 +1703,10 @@ export class CrewlyServer {
1701
1703
  // Agents whose deliveries the input guard keeps refusing (crewly#1028).
1702
1704
  input_circuit: inputCircuitStats(),
1703
1705
  cloud: cloudHealthBlock(),
1706
+ // Boot restore progress (staggered restore queue).
1707
+ restoreQueue: getRestoreQueue().stats(),
1708
+ // Adaptive agent limits: normal | pressure, running count, cap.
1709
+ ...ResourceModeService.getInstance().stats(),
1704
1710
  });
1705
1711
  });
1706
1712
  // H5 quick entry static page (served regardless of headless mode)
@@ -1926,6 +1932,44 @@ export class CrewlyServer {
1926
1932
  return items.some((wi) => wi.target === sessionName && PENDING_WORK_STATUSES.has(wi.status));
1927
1933
  });
1928
1934
  idleDetection.start();
1935
+ // Adaptive agent limits (pressure mode cap + start queue).
1936
+ const resourceMode = ResourceModeService.getInstance();
1937
+ const agentRegistration = this.apiController.agentRegistrationService;
1938
+ resourceMode.setDeps({
1939
+ limits: async () => {
1940
+ const g = (await getSettingsService().getSettings()).general;
1941
+ return {
1942
+ maxRunning: Math.max(1, g.pressureMaxRunningAgents ?? RESOURCE_MODE_CONSTANTS.DEFAULT_MAX_RUNNING_AGENTS),
1943
+ idleTimeoutMinutes: g.pressureIdleTimeoutMinutes ?? RESOURCE_MODE_CONSTANTS.DEFAULT_IDLE_TIMEOUT_MINUTES,
1944
+ };
1945
+ },
1946
+ listRunning: async () => {
1947
+ const teams = await StorageService.getInstance().getTeams();
1948
+ const tracker = PtyActivityTrackerService.getInstance();
1949
+ const out = [];
1950
+ for (const t of teams) {
1951
+ for (const m of t.members || []) {
1952
+ if (m.agentStatus !== CREWLY_CONSTANTS.AGENT_STATUSES.ACTIVE && m.agentStatus !== CREWLY_CONSTANTS.AGENT_STATUSES.STARTED)
1953
+ continue;
1954
+ if (m.sessionName === ORCHESTRATOR_SESSION_NAME || AGENT_SUSPEND_CONSTANTS.ALWAYS_ON_ROLES.includes(m.role))
1955
+ continue;
1956
+ let busy = m.agentStatus === CREWLY_CONSTANTS.AGENT_STATUSES.STARTED;
1957
+ try {
1958
+ busy = busy || (await this.activityMonitorService.getWorkingStatusForSession(m.sessionName)) === 'in_progress';
1959
+ }
1960
+ catch { /* unknown status: treat as not busy */ }
1961
+ out.push({ sessionName: m.sessionName, role: m.role, idleMs: tracker.getIdleTimeMs(m.sessionName), busy });
1962
+ }
1963
+ }
1964
+ return out;
1965
+ },
1966
+ hasOwnerMessage: (name) => SubAgentMessageQueue.getInstance().peek(name).some((m) => m.meta?.owner === true),
1967
+ stopAgent: async (name, role) => {
1968
+ await agentRegistration.terminateAgentSession(name, role);
1969
+ await StorageService.getInstance().updateAgentStatus(name, CREWLY_CONSTANTS.AGENT_STATUSES.INACTIVE, 'idle_exit');
1970
+ },
1971
+ });
1972
+ resourceMode.start();
1929
1973
  // Wire OrchestratorRestartService with dependencies for auto-restart
1930
1974
  try {
1931
1975
  const sessionBackend = getSessionBackendSync();
@@ -3492,68 +3536,21 @@ export class CrewlyServer {
3492
3536
  // minutes, and measuring afterwards dropped work that was minutes old
3493
3537
  // when the backend went down (CE-128 / CE-132, 2026-10-05).
3494
3538
  const recoveryReferenceMs = Date.now();
3495
- // Auto-restore agent sessions that were running before the last shutdown
3539
+ // #166/#196: in-progress WorkItems to re-send after the restart (not
3540
+ // older than 1 hour at boot). Each restored agent gets its own recovery
3541
+ // messages right after it registers; agents outside the restore queue
3542
+ // (already live, or not restored) get theirs now.
3543
+ this.recoveryBySession = await this.collectRecoverableTasks(recoveryReferenceMs);
3544
+ // Auto-restore agent sessions that were running before the last shutdown.
3545
+ // Starts the staggered restore queue; it drains in the background.
3496
3546
  await this.autoRestoreAgentSessionsIfEnabled();
3497
3547
  // Re-deliver interrupted turns now that their agents are coming back.
3498
3548
  // Background: the orchestrator may take minutes to register.
3499
3549
  void this.resumeInterruptedTurnsAfterBoot();
3500
- // #166: Auto-recover in-progress tasks after restart.
3501
- // #196: Skip tasks older than 1 hour to avoid re-sending stale work.
3502
- // V3-only as of spec 2026-05-06-task-management-v1-deprecation.md —
3503
- // reads WorkItems from TaskPoolService (replaces the prior
3504
- // `TaskTrackingService.getAllInProgressTasks()` call).
3505
- try {
3506
- const TASK_RECOVERY_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
3507
- const { TaskPoolService } = await import('./services/task-pool/task-pool.service.js');
3508
- const allItems = await TaskPoolService.getInstance().getAllItems();
3509
- const now = recoveryReferenceMs;
3510
- const activeTasks = allItems.filter(wi => {
3511
- if (wi.status !== 'queued' && wi.status !== 'accepted' && wi.status !== 'running')
3512
- return false;
3513
- if (!wi.target)
3514
- return false;
3515
- // Skip stale tasks — startedAt/createdAt older than threshold
3516
- // Last sign of life: a re-claim / status change counts, not only creation.
3517
- const taskTime = Math.max(new Date(wi.startedAt || wi.createdAt || 0).getTime() || 0, new Date(wi.statusChangedAt || 0).getTime() || 0);
3518
- if (now - taskTime > TASK_RECOVERY_MAX_AGE_MS) {
3519
- this.logger.info('Skipping stale task recovery (older than 1 hour)', {
3520
- workItemId: wi.id,
3521
- taskName: wi.title,
3522
- age: `${Math.round((now - taskTime) / 60000)} minutes`,
3523
- });
3524
- return false;
3525
- }
3526
- return true;
3527
- });
3528
- if (activeTasks.length > 0) {
3529
- this.logger.info('Found in-progress WorkItems to recover after restart', {
3530
- count: activeTasks.length,
3531
- });
3532
- for (const wi of activeTasks) {
3533
- try {
3534
- const recoveryMessage = `[SYSTEM — TASK RECOVERY] You were working on this task before the server restarted. Please continue:\n\nTask: ${wi.title}\nWorkItem: ${wi.id}\n\nFetch full brief: bash config/skills/agent/core/read-task/execute.sh '{"workItemId":"${wi.id}"}'\n\nPlease check the current state and continue working.`;
3535
- await this.apiController.agentRegistrationService.sendMessageToAgent(wi.target, recoveryMessage, undefined);
3536
- this.logger.info('Task recovery message sent', {
3537
- workItemId: wi.id,
3538
- sessionName: wi.target,
3539
- taskName: wi.title,
3540
- });
3541
- }
3542
- catch (err) {
3543
- // Agent might not be online yet — DLQ in scheduler will handle it
3544
- this.logger.warn('Task recovery delivery deferred (agent may not be online yet)', {
3545
- workItemId: wi.id,
3546
- sessionName: wi.target,
3547
- error: err instanceof Error ? err.message : String(err),
3548
- });
3549
- }
3550
- }
3551
- }
3552
- }
3553
- catch (err) {
3554
- this.logger.warn('Task auto-recovery failed (non-critical)', {
3555
- error: err instanceof Error ? err.message : String(err),
3556
- });
3550
+ const queued = new Set(getRestoreQueue().stats().order);
3551
+ for (const session of [...this.recoveryBySession.keys()]) {
3552
+ if (!queued.has(session))
3553
+ await this.sendRecoveryFor(session);
3557
3554
  }
3558
3555
  // Start log rotation service (non-critical — logs cleanup)
3559
3556
  try {
@@ -3983,6 +3980,77 @@ export class CrewlyServer {
3983
3980
  });
3984
3981
  }
3985
3982
  }
3983
+ /** WorkItems to recover after boot, grouped by target session. */
3984
+ recoveryBySession = new Map();
3985
+ /**
3986
+ * In-progress WorkItems worth re-sending after a restart: active status,
3987
+ * with a target, touched within the last hour of `referenceMs`.
3988
+ *
3989
+ * @param referenceMs - Boot time (age is measured at boot, not after the restore)
3990
+ * @returns Items grouped by target session
3991
+ */
3992
+ async collectRecoverableTasks(referenceMs) {
3993
+ const out = new Map();
3994
+ try {
3995
+ const TASK_RECOVERY_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
3996
+ const allItems = await TaskPoolService.getInstance().getAllItems();
3997
+ for (const wi of allItems) {
3998
+ if (wi.status !== 'queued' && wi.status !== 'accepted' && wi.status !== 'running')
3999
+ continue;
4000
+ if (!wi.target)
4001
+ continue;
4002
+ // Last sign of life: a re-claim / status change counts, not only creation.
4003
+ const taskTime = Math.max(new Date(wi.startedAt || wi.createdAt || 0).getTime() || 0, new Date(wi.statusChangedAt || 0).getTime() || 0);
4004
+ if (referenceMs - taskTime > TASK_RECOVERY_MAX_AGE_MS) {
4005
+ this.logger.info('Skipping stale task recovery (older than 1 hour)', {
4006
+ workItemId: wi.id,
4007
+ taskName: wi.title,
4008
+ age: `${Math.round((referenceMs - taskTime) / 60000)} minutes`,
4009
+ });
4010
+ continue;
4011
+ }
4012
+ const list = out.get(wi.target) ?? [];
4013
+ list.push(wi);
4014
+ out.set(wi.target, list);
4015
+ }
4016
+ if (out.size > 0) {
4017
+ this.logger.info('Found in-progress WorkItems to recover after restart', {
4018
+ count: [...out.values()].reduce((n, l) => n + l.length, 0),
4019
+ });
4020
+ }
4021
+ }
4022
+ catch (err) {
4023
+ this.logger.warn('Task auto-recovery failed (non-critical)', {
4024
+ error: err instanceof Error ? err.message : String(err),
4025
+ });
4026
+ }
4027
+ return out;
4028
+ }
4029
+ /**
4030
+ * Send the task-recovery messages of one session (once), e.g. right after
4031
+ * that agent registered.
4032
+ *
4033
+ * @param sessionName - Target session
4034
+ */
4035
+ async sendRecoveryFor(sessionName) {
4036
+ const items = this.recoveryBySession.get(sessionName);
4037
+ this.recoveryBySession.delete(sessionName);
4038
+ for (const wi of items ?? []) {
4039
+ try {
4040
+ const recoveryMessage = `[SYSTEM — TASK RECOVERY] You were working on this task before the server restarted. Please continue:\n\nTask: ${wi.title}\nWorkItem: ${wi.id}\n\nFetch full brief: bash config/skills/agent/core/read-task/execute.sh '{"workItemId":"${wi.id}"}'\n\nPlease check the current state and continue working.`;
4041
+ await this.apiController.agentRegistrationService.sendMessageToAgent(sessionName, recoveryMessage, undefined);
4042
+ this.logger.info('Task recovery message sent', { workItemId: wi.id, sessionName, taskName: wi.title });
4043
+ }
4044
+ catch (err) {
4045
+ // Agent might not be online yet — DLQ in scheduler will handle it
4046
+ this.logger.warn('Task recovery delivery deferred (agent may not be online yet)', {
4047
+ workItemId: wi.id,
4048
+ sessionName,
4049
+ error: err instanceof Error ? err.message : String(err),
4050
+ });
4051
+ }
4052
+ }
4053
+ }
3986
4054
  /**
3987
4055
  * Auto-restore agent sessions that were running before the last shutdown.
3988
4056
  * Loads persisted session state and calls createAgentSession() for each
@@ -4077,76 +4145,67 @@ export class CrewlyServer {
4077
4145
  count: agentSessions.length,
4078
4146
  sessions: agentSessions.map((s) => s.name),
4079
4147
  });
4080
- let restored = 0;
4081
- let alreadyRunning = 0;
4082
- const failed = [];
4083
- const RESTORE_DELAY_MS = 10_000; // 10 seconds between each session restore to avoid resource pressure
4084
- for (let i = 0; i < agentSessions.length; i++) {
4085
- const session = agentSessions[i];
4086
- // Wait between session restores to avoid SIGTERM from resource pressure
4087
- if (i > 0) {
4088
- this.logger.info('Waiting before restoring next session to avoid resource pressure', {
4089
- delayMs: RESTORE_DELAY_MS,
4090
- nextSession: session.name,
4091
- progress: `${i}/${agentSessions.length}`,
4092
- });
4093
- await new Promise((resolve) => setTimeout(resolve, RESTORE_DELAY_MS));
4094
- }
4095
- try {
4096
- // PTYs live in this process, so none survive a restart: a session
4097
- // that exists now was started this boot by another launcher (a
4098
- // reconciler wake, a team start). forceRecreate would kill it and
4099
- // resume the same conversation again — a second kickoff in one
4100
- // chat (2026-09-25 startup-prompt loop). Leave it be.
4101
- if (await this.apiController.agentRegistrationService.isSessionLiveOrLaunching(session.name)) {
4102
- alreadyRunning++;
4103
- this.logger.info('Skipping restore — session already started this boot by another launcher', {
4104
- name: session.name,
4105
- });
4106
- continue;
4107
- }
4108
- const result = await this.apiController.agentRegistrationService.createAgentSession({
4109
- sessionName: session.name,
4110
- role: session.role || 'developer',
4111
- projectPath: session.cwd || process.cwd(),
4112
- runtimeType: session.runtimeType,
4113
- teamId: session.teamId,
4114
- memberId: session.memberId,
4115
- forceRecreate: true,
4116
- });
4117
- if (result.success) {
4118
- restored++;
4119
- this.logger.info('Restored agent session', {
4120
- name: session.name,
4121
- role: session.role,
4148
+ // Staggered restore: one agent at a time (CREWLY_RESTORE_CONCURRENCY,
4149
+ // max 3), the next starts when the previous one registered or timed out.
4150
+ // Owner-waiting agents first, then orchestrator, leads, everyone else.
4151
+ const queueMessages = SubAgentMessageQueue.getInstance();
4152
+ const owedSince = new Map();
4153
+ for (const c of this.openCommitmentsAtBoot) {
4154
+ const due = c.due ? Date.parse(c.due) : NaN;
4155
+ owedSince.set(c.sessionName, Math.min(owedSince.get(c.sessionName) ?? Infinity, Number.isFinite(due) ? due : 0));
4156
+ }
4157
+ const leadRoles = TEAM_LEAD_CONSTANTS.LEAD_ROLES;
4158
+ const entries = agentSessions.map((session) => {
4159
+ const ownerTimes = queueMessages.peek(session.name).filter((m) => m.meta?.owner === true).map((m) => m.queuedAt);
4160
+ if (owedSince.has(session.name))
4161
+ ownerTimes.push(owedSince.get(session.name));
4162
+ const base = {
4163
+ name: session.name,
4164
+ run: async (attempt) => {
4165
+ // PTYs live in this process, so none survive a restart: a session
4166
+ // that exists now was started this boot by another launcher (a
4167
+ // reconciler wake, a team start). forceRecreate would kill it and
4168
+ // resume the same conversation again — a second kickoff in one
4169
+ // chat (2026-09-25 startup-prompt loop). Leave it be.
4170
+ if (attempt === 0 && await this.apiController.agentRegistrationService.isSessionLiveOrLaunching(session.name)) {
4171
+ this.logger.info('Skipping restore — session already started this boot by another launcher', { name: session.name });
4172
+ return { success: true };
4173
+ }
4174
+ const result = await this.apiController.agentRegistrationService.createAgentSession({
4175
+ sessionName: session.name,
4176
+ role: session.role || 'developer',
4177
+ projectPath: session.cwd || process.cwd(),
4122
4178
  runtimeType: session.runtimeType,
4123
- progress: `${restored}/${agentSessions.length}`,
4124
- });
4125
- }
4126
- else {
4127
- failed.push(session.name);
4128
- this.logger.warn('Failed to restore agent session', {
4129
- name: session.name,
4130
- error: result.error,
4179
+ teamId: session.teamId,
4180
+ memberId: session.memberId,
4181
+ // A retry joins the launch still in flight instead of killing it.
4182
+ forceRecreate: attempt === 0,
4131
4183
  });
4132
- }
4133
- }
4134
- catch (error) {
4135
- failed.push(session.name);
4136
- this.logger.error('Error restoring agent session', {
4137
- name: session.name,
4138
- error: error instanceof Error ? error.message : String(error),
4139
- });
4184
+ if (!result.success)
4185
+ this.logger.warn('Failed to restore agent session', { name: session.name, error: result.error });
4186
+ return { success: result.success };
4187
+ },
4188
+ onReady: () => this.sendRecoveryFor(session.name),
4189
+ };
4190
+ if (ownerTimes.length > 0) {
4191
+ return { ...base, tier: RESTORE_TIER.OWNER, ownerSince: Math.min(...ownerTimes), reason: 'owner message waiting' };
4140
4192
  }
4141
- }
4142
- this.logger.info('Agent session restore complete', {
4143
- restored,
4144
- alreadyRunning,
4145
- total: agentSessions.length,
4146
- failed: failed.length > 0 ? failed : undefined,
4193
+ if (session.role === ORCHESTRATOR_ROLE)
4194
+ return { ...base, tier: RESTORE_TIER.ORCHESTRATOR, reason: 'orchestrator' };
4195
+ if (leadRoles.includes(session.role ?? ''))
4196
+ return { ...base, tier: RESTORE_TIER.LEAD, reason: 'team lead with work in hand' };
4197
+ return { ...base, tier: RESTORE_TIER.OTHER, reason: 'work in hand' };
4198
+ });
4199
+ // Runs in the background; boot continues while agents come back.
4200
+ void getRestoreQueue()
4201
+ .start(entries)
4202
+ .then(async () => {
4203
+ // Clear persisted state after the restore to avoid double-restore
4204
+ await persistence.clearState();
4205
+ })
4206
+ .catch((error) => {
4207
+ this.logger.error('Restore queue failed', { error: error instanceof Error ? error.message : String(error) });
4147
4208
  });
4148
- // Clear persisted state after restore attempt to avoid double-restore
4149
- await persistence.clearState();
4150
4209
  }
4151
4210
  catch (error) {
4152
4211
  this.logger.error('Failed to auto-restore agent sessions', {