crewly 1.20.359 → 1.20.361
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/roles/developer/prompt.md +3 -1
- package/config/skills/registry.json +125 -1
- package/dist/backend/backend/src/constants.d.ts +132 -2
- package/dist/backend/backend/src/constants.d.ts.map +1 -1
- package/dist/backend/backend/src/constants.js +143 -3
- package/dist/backend/backend/src/constants.js.map +1 -1
- package/dist/backend/backend/src/index.d.ts +37 -0
- package/dist/backend/backend/src/index.d.ts.map +1 -1
- package/dist/backend/backend/src/index.js +106 -10
- package/dist/backend/backend/src/index.js.map +1 -1
- package/dist/backend/backend/src/services/agent/boot-pressure.d.ts +163 -0
- package/dist/backend/backend/src/services/agent/boot-pressure.d.ts.map +1 -0
- package/dist/backend/backend/src/services/agent/boot-pressure.js +235 -0
- package/dist/backend/backend/src/services/agent/boot-pressure.js.map +1 -0
- package/dist/backend/backend/src/services/agent/restore-queue.d.ts +50 -1
- package/dist/backend/backend/src/services/agent/restore-queue.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/restore-queue.js +100 -5
- package/dist/backend/backend/src/services/agent/restore-queue.js.map +1 -1
- package/dist/backend/backend/src/services/memory/agent-memory-refresh.service.d.ts +11 -0
- package/dist/backend/backend/src/services/memory/agent-memory-refresh.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/memory/agent-memory-refresh.service.js +23 -2
- package/dist/backend/backend/src/services/memory/agent-memory-refresh.service.js.map +1 -1
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.d.ts +8 -0
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.d.ts.map +1 -1
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.js +8 -0
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-entries.d.ts +7 -0
- package/dist/backend/backend/src/services/wiki/wiki-log-entries.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-entries.js +10 -0
- package/dist/backend/backend/src/services/wiki/wiki-log-entries.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-rotation.service.d.ts +24 -0
- package/dist/backend/backend/src/services/wiki/wiki-log-rotation.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-rotation.service.js +75 -9
- package/dist/backend/backend/src/services/wiki/wiki-log-rotation.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-maintenance.service.d.ts +19 -0
- package/dist/backend/backend/src/services/wiki/wiki-maintenance.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-maintenance.service.js +23 -2
- package/dist/backend/backend/src/services/wiki/wiki-maintenance.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/work-record.service.d.ts +23 -1
- package/dist/backend/backend/src/services/wiki/work-record.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/work-record.service.js +33 -5
- package/dist/backend/backend/src/services/wiki/work-record.service.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/low-disk-guard.d.ts +92 -8
- package/dist/backend/backend/src/services/worktree/low-disk-guard.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/low-disk-guard.js +98 -14
- package/dist/backend/backend/src/services/worktree/low-disk-guard.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/tmp-janitor.d.ts +138 -0
- package/dist/backend/backend/src/services/worktree/tmp-janitor.d.ts.map +1 -0
- package/dist/backend/backend/src/services/worktree/tmp-janitor.js +199 -0
- package/dist/backend/backend/src/services/worktree/tmp-janitor.js.map +1 -0
- package/dist/backend/backend/src/services/worktree/transcript-janitor.d.ts +154 -0
- package/dist/backend/backend/src/services/worktree/transcript-janitor.d.ts.map +1 -0
- package/dist/backend/backend/src/services/worktree/transcript-janitor.js +357 -0
- package/dist/backend/backend/src/services/worktree/transcript-janitor.js.map +1 -0
- package/dist/backend/backend/src/services/worktree/workitem-worktree.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/workitem-worktree.service.js +7 -2
- package/dist/backend/backend/src/services/worktree/workitem-worktree.service.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-dirty.d.ts +16 -4
- package/dist/backend/backend/src/services/worktree/worktree-dirty.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-dirty.js +21 -5
- package/dist/backend/backend/src/services/worktree/worktree-dirty.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-git.d.ts +18 -0
- package/dist/backend/backend/src/services/worktree/worktree-git.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-git.js +60 -0
- package/dist/backend/backend/src/services/worktree/worktree-git.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.d.ts +164 -25
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.js +494 -55
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-tiers.d.ts +186 -0
- package/dist/backend/backend/src/services/worktree/worktree-tiers.d.ts.map +1 -0
- package/dist/backend/backend/src/services/worktree/worktree-tiers.js +279 -0
- package/dist/backend/backend/src/services/worktree/worktree-tiers.js.map +1 -0
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/constants.d.ts +132 -2
- package/dist/cli/backend/src/constants.d.ts.map +1 -1
- package/dist/cli/backend/src/constants.js +143 -3
- package/dist/cli/backend/src/constants.js.map +1 -1
- package/dist/cli/backend/src/services/agent/boot-pressure.d.ts +163 -0
- package/dist/cli/backend/src/services/agent/boot-pressure.d.ts.map +1 -0
- package/dist/cli/backend/src/services/agent/boot-pressure.js +235 -0
- package/dist/cli/backend/src/services/agent/boot-pressure.js.map +1 -0
- package/dist/cli/backend/src/services/agent/restore-queue.d.ts +50 -1
- package/dist/cli/backend/src/services/agent/restore-queue.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/restore-queue.js +100 -5
- package/dist/cli/backend/src/services/agent/restore-queue.js.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-log-entries.d.ts +7 -0
- package/dist/cli/backend/src/services/wiki/wiki-log-entries.d.ts.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-log-entries.js +10 -0
- package/dist/cli/backend/src/services/wiki/wiki-log-entries.js.map +1 -1
- package/dist/cli/backend/src/services/worktree/low-disk-guard.d.ts +92 -8
- package/dist/cli/backend/src/services/worktree/low-disk-guard.d.ts.map +1 -1
- package/dist/cli/backend/src/services/worktree/low-disk-guard.js +98 -14
- package/dist/cli/backend/src/services/worktree/low-disk-guard.js.map +1 -1
- package/dist/cli/backend/src/services/worktree/worktree-git.d.ts +18 -0
- package/dist/cli/backend/src/services/worktree/worktree-git.d.ts.map +1 -1
- package/dist/cli/backend/src/services/worktree/worktree-git.js +60 -0
- package/dist/cli/backend/src/services/worktree/worktree-git.js.map +1 -1
- package/package.json +1 -1
|
@@ -71,6 +71,16 @@ export declare class CrewlyServer {
|
|
|
71
71
|
private readonly deferredIdleSettle;
|
|
72
72
|
/** Owner promises past due and never nudged at boot; their agents are restored and reminded once */
|
|
73
73
|
private openCommitmentsAtBoot;
|
|
74
|
+
/**
|
|
75
|
+
* The boot restore is over: the queue drained, or there was nothing to
|
|
76
|
+
* restore. The boot maintenance passes wait for this (crewly#1353).
|
|
77
|
+
*/
|
|
78
|
+
private bootRestoreDone;
|
|
79
|
+
/** When this backend booted (the memory hold on boot retries is time-boxed from here) */
|
|
80
|
+
private readonly bootAt;
|
|
81
|
+
/** Agents the boot restore handed to the retry because memory was tight (crewly#1353) */
|
|
82
|
+
private heldBackAtBoot;
|
|
83
|
+
private heldBackNoticeScheduled;
|
|
74
84
|
private healthMonitoringInterval;
|
|
75
85
|
private backgroundWatchdog;
|
|
76
86
|
constructor(config?: Partial<StartupConfig>);
|
|
@@ -164,6 +174,32 @@ export declare class CrewlyServer {
|
|
|
164
174
|
* @param sessionName - The agent whose restore was deferred
|
|
165
175
|
*/
|
|
166
176
|
private retryRestoreWhileOwed;
|
|
177
|
+
/**
|
|
178
|
+
* Why a boot launch outside the restore queue must wait this round
|
|
179
|
+
* (crewly#1353): memory is tight (within the first hour), the restore queue
|
|
180
|
+
* is still launching, or another boot launch began less than the launch
|
|
181
|
+
* spacing ago. Null = go.
|
|
182
|
+
*
|
|
183
|
+
* @returns Reason to hold, or null
|
|
184
|
+
*/
|
|
185
|
+
private bootLaunchHold;
|
|
186
|
+
/**
|
|
187
|
+
* Gate for the boot maintenance passes (crewly#1353): resolves when the
|
|
188
|
+
* boot restore is over and memory is calm, or after the wait cap with
|
|
189
|
+
* `reduced: true`.
|
|
190
|
+
*
|
|
191
|
+
* @returns A gate function for the service constructors
|
|
192
|
+
*/
|
|
193
|
+
private bootQuietGate;
|
|
194
|
+
/**
|
|
195
|
+
* The boot restore handed an agent to the retry because memory was tight.
|
|
196
|
+
* Once, when the first one is held back, schedule the one-line owner note
|
|
197
|
+
* that is sent only if some of them are still down after
|
|
198
|
+
* OWNER_NOTE_AFTER_MS (crewly#1353).
|
|
199
|
+
*
|
|
200
|
+
* @param sessionName - The agent held back
|
|
201
|
+
*/
|
|
202
|
+
private noteHeldBackAtBoot;
|
|
167
203
|
/**
|
|
168
204
|
* Check for and handle pending self-improvement from hot-reload.
|
|
169
205
|
* This runs at startup to validate or rollback any changes made
|
|
@@ -450,6 +486,7 @@ export declare class CrewlyServer {
|
|
|
450
486
|
*
|
|
451
487
|
* @param sessionName - The agent to start
|
|
452
488
|
* @param firstPauseMs - Pause before the first attempt (default: the retry's own)
|
|
489
|
+
* @param opts - `boot`: a boot launch, held while memory is tight and paced with the restore queue (crewly#1353)
|
|
453
490
|
*/
|
|
454
491
|
private retryStartWhileQueued;
|
|
455
492
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../backend/src/index.ts"],"names":[],"mappings":";AA4CA,OAAO,EAAE,aAAa,EAAa,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../backend/src/index.ts"],"names":[],"mappings":";AA4CA,OAAO,EAAE,aAAa,EAAa,MAAM,kBAAkB,CAAC;AAuV5D,qBAAa,YAAY;IACxB,OAAO,CAAC,GAAG,CAAsB;IACjC,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,EAAE,CAAiB;IAC3B,iFAAiF;IACjF,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,MAAM,CAAqE;IACnF,gFAAgF;IAChF,OAAO,CAAC,iBAAiB,CAAC,CAAyD;IAEnF,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,gBAAgB,CAAoB;IAC5C,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,sBAAsB,CAA0B;IACxD,OAAO,CAAC,4BAA4B,CAAgC;IACpE,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAmB;IAC1C,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,qBAAqB,CAAyB;IACtD,OAAO,CAAC,wBAAwB,CAA4B;IAC5D,OAAO,CAAC,eAAe,CAAmB;IAC1C,qEAAqE;IACrE,OAAO,CAAC,qBAAqB,CAAsC;IACnE,OAAO,CAAC,sBAAsB,CAAuC;IACrE,OAAO,CAAC,sBAAsB,CAAiD;IAC/E,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,kGAAkG;IAClG,OAAO,CAAC,sBAAsB,CAAuC;IAGrE,OAAO,CAAC,+BAA+B,CAAgD;IACvF,+FAA+F;IAC/F,OAAO,CAAC,oBAAoB,CAAqC;IACjE,8HAA8H;IAC9H,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,OAAO,CAAC,6BAA6B,CAA8C;IACnF,OAAO,CAAC,wBAAwB,CAAyC;IACzE,OAAO,CAAC,2BAA2B,CAA+B;IAClE,OAAO,CAAC,0BAA0B,CAA8B;IAChE,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,kBAAkB,CAA0C;IACpE,8DAA8D;IAC9D,OAAO,CAAC,iBAAiB,CAAmG;IAK5H,OAAO,CAAC,aAAa,CAAuE;IAC5F,OAAO,CAAC,gBAAgB,CAET;IAGf,OAAO,CAAC,cAAc,CAAS;IAC/B,oEAAoE;IACpE,OAAO,CAAC,qBAAqB,CAA6C;IAC1E,OAAO,CAAC,qBAAqB,CAA2G;IACxI,0EAA0E;IAC1E,OAAO,CAAC,eAAe,CAAuC;IAC9D,OAAO,CAAC,gBAAgB,CAAiG;IACzH,OAAO,CAAC,UAAU,CAAyE;IAC3F,wFAAwF;IACxF,OAAO,CAAC,UAAU,CAAkG;IACpH,qEAAqE;IACrE,OAAO,CAAC,cAAc,CAAmG;IACzH,oFAAoF;IACpF,OAAO,CAAC,oBAAoB,CAAK;IACjC,2EAA2E;IAC3E,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,yGAAyG;IACzG,OAAO,CAAC,sBAAsB,CAAuB;IACrD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAMhC;IACH,oGAAoG;IACpG,OAAO,CAAC,qBAAqB,CAAwB;IACrD;;;OAGG;IACH,OAAO,CAAC,eAAe,CAAS;IAChC,yFAAyF;IACzF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,yFAAyF;IACzF,OAAO,CAAC,cAAc,CAAqB;IAC3C,OAAO,CAAC,uBAAuB,CAAS;IACxC,OAAO,CAAC,wBAAwB,CAA+B;IAC/D,OAAO,CAAC,kBAAkB,CAAuC;gBAErD,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC;IAsE3C,OAAO,CAAC,kBAAkB;IA62C1B,OAAO,CAAC,mBAAmB;IAuE3B,OAAO,CAAC,eAAe;IA+MvB,OAAO,CAAC,kBAAkB;IA0BpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAijF5B;;;OAGG;YACW,2BAA2B;IAuCzC;;;OAGG;YACW,8BAA8B;IAsB5C;;;OAGG;YACW,gCAAgC;IAsB9C;;;OAGG;YACW,8BAA8B;IAsB5C;;;;OAIG;YACW,8BAA8B;IA8M5C;;;;;;;;;;;;OAYG;YACW,6BAA6B;IAsB3C,kEAAkE;IAClE,OAAO,CAAC,iBAAiB,CAA4E;IAErG;;;;;;OAMG;YACW,uBAAuB;IAsCrC;;;;;OAKG;YACW,eAAe;IAuB7B;;;;;OAKG;YACW,iCAAiC;IAuO/C;;;;;;;;;OASG;YACW,4BAA4B;IAgC1C;;;;;;;;OAQG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAStB;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IAIrB;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAmB1B;;;;OAIG;YACW,2BAA2B;YA0B3B,qBAAqB;IAsBnC;;;;;;OAMG;IACH,OAAO,CAAC,0BAA0B;IAwBlC;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;YAqCZ,eAAe;YAoFf,kBAAkB;IAgBhC,OAAO,CAAC,WAAW,CAAK;IAExB,OAAO,CAAC,sBAAsB;IAsC9B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,oBAAoB;IAwB5B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IA0BvB,4DAA4D;IAC5D,OAAO,CAAC,eAAe,CAAyD;IAEhF;;;;OAIG;YACW,iBAAiB;IAQ/B;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IA6B5B;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAyEvB;;;;;;;OAOG;YACW,2BAA2B;IAczC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAsG1B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IA4DvB;;;;;;OAMG;IACH,OAAO,CAAC,yBAAyB;IA4DjC;;;;;;;OAOG;IACH,OAAO,CAAC,0BAA0B;IAuClC;;;;;OAKG;YACW,0BAA0B;IAoExC;;;;OAIG;YACW,qBAAqB;IA+DnC;;;OAGG;YACW,eAAe;IAkD7B;;;OAGG;YACW,eAAe;IAqC7B,gEAAgE;IAChE,OAAO,CAAC,eAAe,CAAyD;IAEhF,OAAO,CAAC,iBAAiB;IAazB;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAuBlC;;;;;;OAMG;IACH;;;;;;;OAOG;YACW,yBAAyB;YAoBzB,kBAAkB;IAmBhC;;;;;OAKG;YACW,8BAA8B;IA6C5C;;;;OAIG;YACW,+BAA+B;IA+C7C;;;;;;OAMG;YACW,kBAAkB;IA6BhC,OAAO,CAAC,qBAAqB;IAoC7B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAoE1B;;;;;;;;;OASG;IACH;;;;;;;;;;;;;;OAcG;IACH,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA6B;IAE1D;;;;;;;;OAQG;IACH;;OAEG;IACH;;;;;OAKG;YACW,sBAAsB;YA4CtB,eAAe;YA6Cf,kBAAkB;IA2IhC;;;;;;OAMG;YACW,0BAA0B;IA6FxC;;;;;;;OAOG;YACW,wBAAwB;YA2CxB,yBAAyB;IA2FvC;;;;;OAKG;YACW,8BAA8B;IAY5C;;;;;;;OAOG;IACH,OAAO,CAAC,mBAAmB;IAsB3B;;;;;;;;;OASG;IACH,OAAO,CAAC,oBAAoB;IAkC5B;;;;;;;OAOG;IACH,OAAO,CAAC,qBAAqB;IAsB7B;;;;;OAKG;IACH,OAAO,CAAC,yBAAyB;IAqBjC;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,cAAc;IA2BtB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAmChC;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;YAmIf,wBAAwB;IAWtC,OAAO,CAAC,qBAAqB;IA8H7B,OAAO,CAAC,cAAc;IActB;;;;;;;;OAQG;IACG,QAAQ,CAAC,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA4X1H,SAAS,IAAI,aAAa;CAG1B;AAyDD,eAAe,YAAY,CAAC"}
|
|
@@ -194,6 +194,7 @@ import { WorkItemWorktreeService } from './services/worktree/workitem-worktree.s
|
|
|
194
194
|
import { WorkItemWorktreeSubscriber, createTerminalNotifier } from './services/worktree/workitem-worktree.subscriber.js';
|
|
195
195
|
import { ResourceModeService, RESOURCE_MODE_CONSTANTS } from './services/agent/resource-mode.service.js';
|
|
196
196
|
import { getRestoreQueue, RESTORE_TIER } from './services/agent/restore-queue.js';
|
|
197
|
+
import { bootLaunchHoldReason, getBootLaunchPacer, readBootPressureSnapshot, scheduleHeldBackNotice, waitForBootQuiet, } from './services/agent/boot-pressure.js';
|
|
197
198
|
import { sessionsToRestore } from './services/agent/restore-filter.js';
|
|
198
199
|
import { ProjectMemoryService } from './services/memory/project-memory.service.js';
|
|
199
200
|
import { TaskHistorySubscriber } from './services/memory/task-history.subscriber.js';
|
|
@@ -369,6 +370,16 @@ export class CrewlyServer {
|
|
|
369
370
|
});
|
|
370
371
|
/** Owner promises past due and never nudged at boot; their agents are restored and reminded once */
|
|
371
372
|
openCommitmentsAtBoot = [];
|
|
373
|
+
/**
|
|
374
|
+
* The boot restore is over: the queue drained, or there was nothing to
|
|
375
|
+
* restore. The boot maintenance passes wait for this (crewly#1353).
|
|
376
|
+
*/
|
|
377
|
+
bootRestoreDone = false;
|
|
378
|
+
/** When this backend booted (the memory hold on boot retries is time-boxed from here) */
|
|
379
|
+
bootAt = Date.now();
|
|
380
|
+
/** Agents the boot restore handed to the retry because memory was tight (crewly#1353) */
|
|
381
|
+
heldBackAtBoot = new Set();
|
|
382
|
+
heldBackNoticeScheduled = false;
|
|
372
383
|
healthMonitoringInterval = null;
|
|
373
384
|
backgroundWatchdog = null;
|
|
374
385
|
constructor(config) {
|
|
@@ -1045,7 +1056,7 @@ export class CrewlyServer {
|
|
|
1045
1056
|
// digest page compiled from the log. No LLM. Non-fatal.
|
|
1046
1057
|
try {
|
|
1047
1058
|
const { WikiMaintenanceService } = await import('./services/wiki/wiki-maintenance.service.js');
|
|
1048
|
-
const maintenance = new WikiMaintenanceService();
|
|
1059
|
+
const maintenance = new WikiMaintenanceService({ bootGate: this.bootQuietGate() });
|
|
1049
1060
|
WikiMaintenanceService.setInstance(maintenance);
|
|
1050
1061
|
maintenance.start();
|
|
1051
1062
|
}
|
|
@@ -1078,6 +1089,7 @@ export class CrewlyServer {
|
|
|
1078
1089
|
}
|
|
1079
1090
|
return agents;
|
|
1080
1091
|
},
|
|
1092
|
+
bootGate: this.bootQuietGate(),
|
|
1081
1093
|
});
|
|
1082
1094
|
AgentMemoryRefreshService.setInstance(refresh);
|
|
1083
1095
|
refresh.start();
|
|
@@ -1122,6 +1134,7 @@ export class CrewlyServer {
|
|
|
1122
1134
|
if (settled && change.before.status !== change.ticket.status && change.ticket.workItemId)
|
|
1123
1135
|
listener(change.ticket.workItemId);
|
|
1124
1136
|
}),
|
|
1137
|
+
bootGate: this.bootQuietGate(),
|
|
1125
1138
|
});
|
|
1126
1139
|
WorkRecordService.setInstance(records);
|
|
1127
1140
|
records.start();
|
|
@@ -4333,10 +4346,11 @@ export class CrewlyServer {
|
|
|
4333
4346
|
return r.success;
|
|
4334
4347
|
});
|
|
4335
4348
|
// Finished work cleans up after itself: a WorkItem / ticket reaching
|
|
4336
|
-
//
|
|
4337
|
-
// unchanged — only landed, clean, unattended worktrees go)
|
|
4338
|
-
|
|
4339
|
-
|
|
4349
|
+
// a terminal state schedules one debounced janitor pass (safe rules
|
|
4350
|
+
// unchanged — only landed, clean, unattended worktrees go) and notes
|
|
4351
|
+
// the item's worktree as finished (its node_modules go after 1 day).
|
|
4352
|
+
this.eventBusService.onInProcess(['task:done', 'task:verified', 'task:cancelled', 'task:failed'], (event) => {
|
|
4353
|
+
WorktreeJanitorService.getInstance().requestTidy(event.workItemId);
|
|
4340
4354
|
});
|
|
4341
4355
|
if (WorktreeJanitorService.getInstance().start()) {
|
|
4342
4356
|
this.logger.info('WorktreeJanitorService scheduled');
|
|
@@ -4829,12 +4843,14 @@ export class CrewlyServer {
|
|
|
4829
4843
|
const settings = await settingsService.getSettings();
|
|
4830
4844
|
if (!settings.general.autoResumeOnRestart) {
|
|
4831
4845
|
this.logger.info('Auto-resume on restart is disabled, skipping agent session restore');
|
|
4846
|
+
this.bootRestoreDone = true;
|
|
4832
4847
|
return;
|
|
4833
4848
|
}
|
|
4834
4849
|
const persistence = getSessionStatePersistence();
|
|
4835
4850
|
const state = await persistence.loadState();
|
|
4836
4851
|
if (!state || state.sessions.length === 0) {
|
|
4837
4852
|
this.logger.debug('No persisted agent sessions to restore');
|
|
4853
|
+
this.bootRestoreDone = true;
|
|
4838
4854
|
return;
|
|
4839
4855
|
}
|
|
4840
4856
|
// Filter out orchestrator sessions (already auto-started separately)
|
|
@@ -4871,6 +4887,8 @@ export class CrewlyServer {
|
|
|
4871
4887
|
// Why each target other than a WorkItem is owed something; the reason
|
|
4872
4888
|
// is logged in the restore plan for agents that were idle at shutdown.
|
|
4873
4889
|
const owedReason = new Map();
|
|
4890
|
+
// Agents an owner thread waits on: launched even while memory is tight (crewly#1353).
|
|
4891
|
+
const ownerThreadWaits = new Set();
|
|
4874
4892
|
const noteOwed = (names, reason) => {
|
|
4875
4893
|
const out = [];
|
|
4876
4894
|
for (const name of names) {
|
|
@@ -4900,7 +4918,8 @@ export class CrewlyServer {
|
|
|
4900
4918
|
// card they asked (2026-10-08: Atlas waited on a browser
|
|
4901
4919
|
// approval in the owner's thread and was skipped as "no
|
|
4902
4920
|
// work in hand").
|
|
4903
|
-
...noteOwed(await this.ownerThreadSessionsAtBoot()
|
|
4921
|
+
...noteOwed(await this.ownerThreadSessionsAtBoot().then((names) => { for (const n of names)
|
|
4922
|
+
ownerThreadWaits.add(n); return names; }), 'owner thread waits on it'),
|
|
4904
4923
|
]);
|
|
4905
4924
|
}
|
|
4906
4925
|
catch (poolErr) {
|
|
@@ -4945,6 +4964,7 @@ export class CrewlyServer {
|
|
|
4945
4964
|
const agentSessions = [...selection.sessions, ...idleAtShutdown];
|
|
4946
4965
|
if (agentSessions.length === 0) {
|
|
4947
4966
|
this.logger.info('No persisted agent sessions to restore (all idle, no pending WorkItems)');
|
|
4967
|
+
this.bootRestoreDone = true;
|
|
4948
4968
|
return;
|
|
4949
4969
|
}
|
|
4950
4970
|
this.logger.info('Auto-restoring agent sessions from persisted state', {
|
|
@@ -4997,6 +5017,13 @@ export class CrewlyServer {
|
|
|
4997
5017
|
return { success: result.success };
|
|
4998
5018
|
},
|
|
4999
5019
|
onReady: () => this.sendRecoveryFor(session.name),
|
|
5020
|
+
// Memory is tight and nobody waits on it: the retry brings it back
|
|
5021
|
+
// when the machine eases (held by bootLaunchHold), never dropped.
|
|
5022
|
+
onDeferred: () => {
|
|
5023
|
+
this.noteHeldBackAtBoot(session.name);
|
|
5024
|
+
this.retryRestoreWhileOwed(session.name);
|
|
5025
|
+
},
|
|
5026
|
+
essential: ownerTimes.length > 0 || session.role === ORCHESTRATOR_ROLE || ownerThreadWaits.has(session.name),
|
|
5000
5027
|
};
|
|
5001
5028
|
// An agent that was idle at shutdown says in the plan what it is owed.
|
|
5002
5029
|
const idle = wasIdle.has(session.name) ? ` (${owedReason.get(session.name) ?? 'work in hand'}; was idle at shutdown)` : '';
|
|
@@ -5018,9 +5045,13 @@ export class CrewlyServer {
|
|
|
5018
5045
|
})
|
|
5019
5046
|
.catch((error) => {
|
|
5020
5047
|
this.logger.error('Restore queue failed', { error: error instanceof Error ? error.message : String(error) });
|
|
5048
|
+
})
|
|
5049
|
+
.finally(() => {
|
|
5050
|
+
this.bootRestoreDone = true;
|
|
5021
5051
|
});
|
|
5022
5052
|
}
|
|
5023
5053
|
catch (error) {
|
|
5054
|
+
this.bootRestoreDone = true;
|
|
5024
5055
|
this.logger.error('Failed to auto-restore agent sessions', {
|
|
5025
5056
|
error: error instanceof Error ? error.message : String(error),
|
|
5026
5057
|
});
|
|
@@ -5082,8 +5113,12 @@ export class CrewlyServer {
|
|
|
5082
5113
|
const { scheduleDeferredStartRetry } = await import('./services/messaging/deferred-start-retry.js');
|
|
5083
5114
|
const { activateAgentBySession } = await import('./controllers/team/team.controller.js');
|
|
5084
5115
|
await scheduleDeferredStartRetry(sessionName, {
|
|
5085
|
-
activate: (n) =>
|
|
5116
|
+
activate: (n) => {
|
|
5117
|
+
getBootLaunchPacer().note();
|
|
5118
|
+
return activateAgentBySession(this.apiController, n);
|
|
5119
|
+
},
|
|
5086
5120
|
hasPending: (n) => !isOwnerStopped(n),
|
|
5121
|
+
hold: () => this.bootLaunchHold(),
|
|
5087
5122
|
});
|
|
5088
5123
|
})().catch((err) => {
|
|
5089
5124
|
this.logger.warn('Could not retry a deferred boot restore', {
|
|
@@ -5092,6 +5127,60 @@ export class CrewlyServer {
|
|
|
5092
5127
|
});
|
|
5093
5128
|
});
|
|
5094
5129
|
}
|
|
5130
|
+
/**
|
|
5131
|
+
* Why a boot launch outside the restore queue must wait this round
|
|
5132
|
+
* (crewly#1353): memory is tight (within the first hour), the restore queue
|
|
5133
|
+
* is still launching, or another boot launch began less than the launch
|
|
5134
|
+
* spacing ago. Null = go.
|
|
5135
|
+
*
|
|
5136
|
+
* @returns Reason to hold, or null
|
|
5137
|
+
*/
|
|
5138
|
+
bootLaunchHold() {
|
|
5139
|
+
return bootLaunchHoldReason({
|
|
5140
|
+
snapshot: readBootPressureSnapshot,
|
|
5141
|
+
restoreRunning: () => getRestoreQueue().stats().running,
|
|
5142
|
+
pacer: getBootLaunchPacer(),
|
|
5143
|
+
bootAt: this.bootAt,
|
|
5144
|
+
});
|
|
5145
|
+
}
|
|
5146
|
+
/**
|
|
5147
|
+
* Gate for the boot maintenance passes (crewly#1353): resolves when the
|
|
5148
|
+
* boot restore is over and memory is calm, or after the wait cap with
|
|
5149
|
+
* `reduced: true`.
|
|
5150
|
+
*
|
|
5151
|
+
* @returns A gate function for the service constructors
|
|
5152
|
+
*/
|
|
5153
|
+
bootQuietGate() {
|
|
5154
|
+
return () => waitForBootQuiet({ restoreDone: () => this.bootRestoreDone });
|
|
5155
|
+
}
|
|
5156
|
+
/**
|
|
5157
|
+
* The boot restore handed an agent to the retry because memory was tight.
|
|
5158
|
+
* Once, when the first one is held back, schedule the one-line owner note
|
|
5159
|
+
* that is sent only if some of them are still down after
|
|
5160
|
+
* OWNER_NOTE_AFTER_MS (crewly#1353).
|
|
5161
|
+
*
|
|
5162
|
+
* @param sessionName - The agent held back
|
|
5163
|
+
*/
|
|
5164
|
+
noteHeldBackAtBoot(sessionName) {
|
|
5165
|
+
this.heldBackAtBoot.add(sessionName);
|
|
5166
|
+
if (this.heldBackNoticeScheduled)
|
|
5167
|
+
return;
|
|
5168
|
+
this.heldBackNoticeScheduled = true;
|
|
5169
|
+
void scheduleHeldBackNotice({
|
|
5170
|
+
stillHeld: async () => {
|
|
5171
|
+
const out = [];
|
|
5172
|
+
for (const name of this.heldBackAtBoot) {
|
|
5173
|
+
if (await this.apiController.agentRegistrationService.isSessionLiveOrLaunching(name))
|
|
5174
|
+
continue;
|
|
5175
|
+
// The owner reads agent names, not session ids.
|
|
5176
|
+
const member = (await this.storageService.findMemberBySessionName(name).catch(() => null))?.member;
|
|
5177
|
+
out.push(member?.name || name);
|
|
5178
|
+
}
|
|
5179
|
+
return out;
|
|
5180
|
+
},
|
|
5181
|
+
notify: (line) => slackOwnerAlertNotifier({ title: 'Agents waiting on memory', message: line, urgent: false }),
|
|
5182
|
+
});
|
|
5183
|
+
}
|
|
5095
5184
|
/**
|
|
5096
5185
|
* Check for and handle pending self-improvement from hot-reload.
|
|
5097
5186
|
* This runs at startup to validate or rollback any changes made
|
|
@@ -7025,14 +7114,21 @@ export class CrewlyServer {
|
|
|
7025
7114
|
*
|
|
7026
7115
|
* @param sessionName - The agent to start
|
|
7027
7116
|
* @param firstPauseMs - Pause before the first attempt (default: the retry's own)
|
|
7117
|
+
* @param opts - `boot`: a boot launch, held while memory is tight and paced with the restore queue (crewly#1353)
|
|
7028
7118
|
*/
|
|
7029
|
-
retryStartWhileQueued(sessionName, firstPauseMs) {
|
|
7119
|
+
retryStartWhileQueued(sessionName, firstPauseMs, opts = {}) {
|
|
7030
7120
|
void (async () => {
|
|
7031
7121
|
const { scheduleDeferredStartRetry } = await import('./services/messaging/deferred-start-retry.js');
|
|
7032
7122
|
const { activateAgentBySession } = await import('./controllers/team/team.controller.js');
|
|
7033
7123
|
await scheduleDeferredStartRetry(sessionName, {
|
|
7034
|
-
activate: (n) =>
|
|
7124
|
+
activate: (n) => {
|
|
7125
|
+
if (opts.boot)
|
|
7126
|
+
getBootLaunchPacer().note();
|
|
7127
|
+
return activateAgentBySession(this.apiController, n);
|
|
7128
|
+
},
|
|
7035
7129
|
hasPending: (n) => SubAgentMessageQueue.getInstance().hasPending(n),
|
|
7130
|
+
// Boot launches outside the restore queue keep its spacing and its memory gate (crewly#1353).
|
|
7131
|
+
...(opts.boot ? { hold: () => this.bootLaunchHold() } : {}),
|
|
7036
7132
|
...(firstPauseMs !== undefined ? { firstPauseMs } : {}),
|
|
7037
7133
|
});
|
|
7038
7134
|
})().catch((err) => {
|
|
@@ -7065,7 +7161,7 @@ export class CrewlyServer {
|
|
|
7065
7161
|
return;
|
|
7066
7162
|
this.logger.info('Starting agents that were down with messages queued for them', { sessions });
|
|
7067
7163
|
for (const name of sessions)
|
|
7068
|
-
this.retryStartWhileQueued(name, SUB_AGENT_QUEUE_CONSTANTS.BOOT_QUEUED_WAKE_DELAY_MS);
|
|
7164
|
+
this.retryStartWhileQueued(name, SUB_AGENT_QUEUE_CONSTANTS.BOOT_QUEUED_WAKE_DELAY_MS, { boot: true });
|
|
7069
7165
|
}
|
|
7070
7166
|
/**
|
|
7071
7167
|
* Write a system note into the orchestrator's own chat (the conversation
|