crewly 1.20.358 → 1.20.360
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/roles/developer/prompt.md +3 -1
- package/config/skills/registry.json +125 -1
- package/dist/backend/backend/src/constants.d.ts +189 -2
- package/dist/backend/backend/src/constants.d.ts.map +1 -1
- package/dist/backend/backend/src/constants.js +200 -3
- package/dist/backend/backend/src/constants.js.map +1 -1
- package/dist/backend/backend/src/index.d.ts +58 -0
- package/dist/backend/backend/src/index.d.ts.map +1 -1
- package/dist/backend/backend/src/index.js +263 -19
- package/dist/backend/backend/src/index.js.map +1 -1
- package/dist/backend/backend/src/services/agent/boot-pressure.d.ts +147 -0
- package/dist/backend/backend/src/services/agent/boot-pressure.d.ts.map +1 -0
- package/dist/backend/backend/src/services/agent/boot-pressure.js +212 -0
- package/dist/backend/backend/src/services/agent/boot-pressure.js.map +1 -0
- package/dist/backend/backend/src/services/agent/restore-queue.d.ts +50 -1
- package/dist/backend/backend/src/services/agent/restore-queue.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/restore-queue.js +100 -5
- package/dist/backend/backend/src/services/agent/restore-queue.js.map +1 -1
- package/dist/backend/backend/src/services/memory/agent-memory-refresh.service.d.ts +11 -0
- package/dist/backend/backend/src/services/memory/agent-memory-refresh.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/memory/agent-memory-refresh.service.js +28 -2
- package/dist/backend/backend/src/services/memory/agent-memory-refresh.service.js.map +1 -1
- package/dist/backend/backend/src/services/memory/startup-briefing.service.d.ts +10 -0
- package/dist/backend/backend/src/services/memory/startup-briefing.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/memory/startup-briefing.service.js +65 -22
- package/dist/backend/backend/src/services/memory/startup-briefing.service.js.map +1 -1
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.d.ts +8 -0
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.d.ts.map +1 -1
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.js +8 -0
- package/dist/backend/backend/src/services/messaging/deferred-start-retry.js.map +1 -1
- package/dist/backend/backend/src/services/session/session-binding.d.ts +25 -2
- package/dist/backend/backend/src/services/session/session-binding.d.ts.map +1 -1
- package/dist/backend/backend/src/services/session/session-binding.js +32 -4
- package/dist/backend/backend/src/services/session/session-binding.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-ingest.service.d.ts +3 -1
- package/dist/backend/backend/src/services/wiki/wiki-ingest.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-ingest.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-compile.service.d.ts +27 -2
- package/dist/backend/backend/src/services/wiki/wiki-log-compile.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-compile.service.js +105 -8
- package/dist/backend/backend/src/services/wiki/wiki-log-compile.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-entries.d.ts +7 -0
- package/dist/backend/backend/src/services/wiki/wiki-log-entries.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-entries.js +10 -0
- package/dist/backend/backend/src/services/wiki/wiki-log-entries.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-rotation.service.d.ts +24 -0
- package/dist/backend/backend/src/services/wiki/wiki-log-rotation.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-log-rotation.service.js +75 -9
- package/dist/backend/backend/src/services/wiki/wiki-log-rotation.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-maintenance.service.d.ts +19 -0
- package/dist/backend/backend/src/services/wiki/wiki-maintenance.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-maintenance.service.js +23 -2
- package/dist/backend/backend/src/services/wiki/wiki-maintenance.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/work-record-transcript.d.ts +64 -0
- package/dist/backend/backend/src/services/wiki/work-record-transcript.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/work-record-transcript.js +194 -0
- package/dist/backend/backend/src/services/wiki/work-record-transcript.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/work-record.d.ts +239 -0
- package/dist/backend/backend/src/services/wiki/work-record.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/work-record.js +631 -0
- package/dist/backend/backend/src/services/wiki/work-record.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/work-record.service.d.ts +231 -0
- package/dist/backend/backend/src/services/wiki/work-record.service.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/work-record.service.js +542 -0
- package/dist/backend/backend/src/services/wiki/work-record.service.js.map +1 -0
- package/dist/backend/backend/src/services/worktree/low-disk-guard.d.ts +92 -8
- package/dist/backend/backend/src/services/worktree/low-disk-guard.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/low-disk-guard.js +98 -14
- package/dist/backend/backend/src/services/worktree/low-disk-guard.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/tmp-janitor.d.ts +138 -0
- package/dist/backend/backend/src/services/worktree/tmp-janitor.d.ts.map +1 -0
- package/dist/backend/backend/src/services/worktree/tmp-janitor.js +199 -0
- package/dist/backend/backend/src/services/worktree/tmp-janitor.js.map +1 -0
- package/dist/backend/backend/src/services/worktree/transcript-janitor.d.ts +154 -0
- package/dist/backend/backend/src/services/worktree/transcript-janitor.d.ts.map +1 -0
- package/dist/backend/backend/src/services/worktree/transcript-janitor.js +357 -0
- package/dist/backend/backend/src/services/worktree/transcript-janitor.js.map +1 -0
- package/dist/backend/backend/src/services/worktree/workitem-worktree.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/workitem-worktree.service.js +7 -2
- package/dist/backend/backend/src/services/worktree/workitem-worktree.service.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-dirty.d.ts +16 -4
- package/dist/backend/backend/src/services/worktree/worktree-dirty.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-dirty.js +21 -5
- package/dist/backend/backend/src/services/worktree/worktree-dirty.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-git.d.ts +18 -0
- package/dist/backend/backend/src/services/worktree/worktree-git.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-git.js +60 -0
- package/dist/backend/backend/src/services/worktree/worktree-git.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.d.ts +164 -25
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.js +494 -55
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-tiers.d.ts +186 -0
- package/dist/backend/backend/src/services/worktree/worktree-tiers.d.ts.map +1 -0
- package/dist/backend/backend/src/services/worktree/worktree-tiers.js +279 -0
- package/dist/backend/backend/src/services/worktree/worktree-tiers.js.map +1 -0
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/constants.d.ts +189 -2
- package/dist/cli/backend/src/constants.d.ts.map +1 -1
- package/dist/cli/backend/src/constants.js +200 -3
- package/dist/cli/backend/src/constants.js.map +1 -1
- package/dist/cli/backend/src/services/agent/boot-pressure.d.ts +147 -0
- package/dist/cli/backend/src/services/agent/boot-pressure.d.ts.map +1 -0
- package/dist/cli/backend/src/services/agent/boot-pressure.js +212 -0
- package/dist/cli/backend/src/services/agent/boot-pressure.js.map +1 -0
- package/dist/cli/backend/src/services/agent/restore-queue.d.ts +50 -1
- package/dist/cli/backend/src/services/agent/restore-queue.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/restore-queue.js +100 -5
- package/dist/cli/backend/src/services/agent/restore-queue.js.map +1 -1
- package/dist/cli/backend/src/services/memory/startup-briefing.service.d.ts +10 -0
- package/dist/cli/backend/src/services/memory/startup-briefing.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/memory/startup-briefing.service.js +65 -22
- package/dist/cli/backend/src/services/memory/startup-briefing.service.js.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-ingest.service.d.ts +3 -1
- package/dist/cli/backend/src/services/wiki/wiki-ingest.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-ingest.service.js.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-log-entries.d.ts +7 -0
- package/dist/cli/backend/src/services/wiki/wiki-log-entries.d.ts.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-log-entries.js +10 -0
- package/dist/cli/backend/src/services/wiki/wiki-log-entries.js.map +1 -1
- package/dist/cli/backend/src/services/wiki/work-record-transcript.d.ts +64 -0
- package/dist/cli/backend/src/services/wiki/work-record-transcript.d.ts.map +1 -0
- package/dist/cli/backend/src/services/wiki/work-record-transcript.js +194 -0
- package/dist/cli/backend/src/services/wiki/work-record-transcript.js.map +1 -0
- package/dist/cli/backend/src/services/wiki/work-record.d.ts +239 -0
- package/dist/cli/backend/src/services/wiki/work-record.d.ts.map +1 -0
- package/dist/cli/backend/src/services/wiki/work-record.js +631 -0
- package/dist/cli/backend/src/services/wiki/work-record.js.map +1 -0
- package/dist/cli/backend/src/services/worktree/low-disk-guard.d.ts +92 -8
- package/dist/cli/backend/src/services/worktree/low-disk-guard.d.ts.map +1 -1
- package/dist/cli/backend/src/services/worktree/low-disk-guard.js +98 -14
- package/dist/cli/backend/src/services/worktree/low-disk-guard.js.map +1 -1
- package/dist/cli/backend/src/services/worktree/worktree-git.d.ts +18 -0
- package/dist/cli/backend/src/services/worktree/worktree-git.d.ts.map +1 -1
- package/dist/cli/backend/src/services/worktree/worktree-git.js +60 -0
- package/dist/cli/backend/src/services/worktree/worktree-git.js.map +1 -1
- package/package.json +1 -1
|
@@ -71,6 +71,16 @@ export declare class CrewlyServer {
|
|
|
71
71
|
private readonly deferredIdleSettle;
|
|
72
72
|
/** Owner promises past due and never nudged at boot; their agents are restored and reminded once */
|
|
73
73
|
private openCommitmentsAtBoot;
|
|
74
|
+
/**
|
|
75
|
+
* The boot restore is over: the queue drained, or there was nothing to
|
|
76
|
+
* restore. The boot maintenance passes wait for this (crewly#1353).
|
|
77
|
+
*/
|
|
78
|
+
private bootRestoreDone;
|
|
79
|
+
/** When this backend booted (the memory hold on boot retries is time-boxed from here) */
|
|
80
|
+
private readonly bootAt;
|
|
81
|
+
/** Agents the boot restore handed to the retry because memory was tight (crewly#1353) */
|
|
82
|
+
private heldBackAtBoot;
|
|
83
|
+
private heldBackNoticeScheduled;
|
|
74
84
|
private healthMonitoringInterval;
|
|
75
85
|
private backgroundWatchdog;
|
|
76
86
|
constructor(config?: Partial<StartupConfig>);
|
|
@@ -143,6 +153,53 @@ export declare class CrewlyServer {
|
|
|
143
153
|
* Runs after orchestrator auto-start so the orchestrator is available.
|
|
144
154
|
*/
|
|
145
155
|
private autoRestoreAgentSessionsIfEnabled;
|
|
156
|
+
/**
|
|
157
|
+
* Restore entries for owed agents that have no persisted session: built from
|
|
158
|
+
* the team config the way a team start would (member role and runtime, the
|
|
159
|
+
* team's first project as the working directory). A binding whose member is
|
|
160
|
+
* gone from the config by now is skipped.
|
|
161
|
+
*
|
|
162
|
+
* @param teams - Teams from storage
|
|
163
|
+
* @param bindings - Targets resolved to members (see selectAutoRestoreSessions)
|
|
164
|
+
* @returns Session entries in the shape the restore queue runs
|
|
165
|
+
*/
|
|
166
|
+
private persistedSessionsForBindings;
|
|
167
|
+
/**
|
|
168
|
+
* A boot restore was deferred by the running-agent cap. The agent is owed
|
|
169
|
+
* something that does not expire on its own (an owner thread, a promise,
|
|
170
|
+
* queued messages), so keep trying until a slot frees (crewly#1337's retry;
|
|
171
|
+
* the queue check is replaced because not every debt is a queued message).
|
|
172
|
+
* Never throws.
|
|
173
|
+
*
|
|
174
|
+
* @param sessionName - The agent whose restore was deferred
|
|
175
|
+
*/
|
|
176
|
+
private retryRestoreWhileOwed;
|
|
177
|
+
/**
|
|
178
|
+
* Why a boot launch outside the restore queue must wait this round
|
|
179
|
+
* (crewly#1353): memory is tight (within the first hour), the restore queue
|
|
180
|
+
* is still launching, or another boot launch began less than the launch
|
|
181
|
+
* spacing ago. Null = go.
|
|
182
|
+
*
|
|
183
|
+
* @returns Reason to hold, or null
|
|
184
|
+
*/
|
|
185
|
+
private bootLaunchHold;
|
|
186
|
+
/**
|
|
187
|
+
* Gate for the boot maintenance passes (crewly#1353): resolves when the
|
|
188
|
+
* boot restore is over and memory is calm, or after the wait cap with
|
|
189
|
+
* `reduced: true`.
|
|
190
|
+
*
|
|
191
|
+
* @returns A gate function for the service constructors
|
|
192
|
+
*/
|
|
193
|
+
private bootQuietGate;
|
|
194
|
+
/**
|
|
195
|
+
* The boot restore handed an agent to the retry because memory was tight.
|
|
196
|
+
* Once, when the first one is held back, schedule the one-line owner note
|
|
197
|
+
* that is sent only if some of them are still down after
|
|
198
|
+
* OWNER_NOTE_AFTER_MS (crewly#1353).
|
|
199
|
+
*
|
|
200
|
+
* @param sessionName - The agent held back
|
|
201
|
+
*/
|
|
202
|
+
private noteHeldBackAtBoot;
|
|
146
203
|
/**
|
|
147
204
|
* Check for and handle pending self-improvement from hot-reload.
|
|
148
205
|
* This runs at startup to validate or rollback any changes made
|
|
@@ -429,6 +486,7 @@ export declare class CrewlyServer {
|
|
|
429
486
|
*
|
|
430
487
|
* @param sessionName - The agent to start
|
|
431
488
|
* @param firstPauseMs - Pause before the first attempt (default: the retry's own)
|
|
489
|
+
* @param opts - `boot`: a boot launch, held while memory is tight and paced with the restore queue (crewly#1353)
|
|
432
490
|
*/
|
|
433
491
|
private retryStartWhileQueued;
|
|
434
492
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../backend/src/index.ts"],"names":[],"mappings":";AA4CA,OAAO,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../backend/src/index.ts"],"names":[],"mappings":";AA4CA,OAAO,EAAE,aAAa,EAAa,MAAM,kBAAkB,CAAC;AAuV5D,qBAAa,YAAY;IACxB,OAAO,CAAC,GAAG,CAAsB;IACjC,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,EAAE,CAAiB;IAC3B,iFAAiF;IACjF,OAAO,CAAC,cAAc,CAAuB;IAC7C,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,MAAM,CAAqE;IACnF,gFAAgF;IAChF,OAAO,CAAC,iBAAiB,CAAC,CAAyD;IAEnF,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,WAAW,CAAe;IAClC,OAAO,CAAC,gBAAgB,CAAoB;IAC5C,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,sBAAsB,CAA0B;IACxD,OAAO,CAAC,4BAA4B,CAAgC;IACpE,OAAO,CAAC,uBAAuB,CAA2B;IAC1D,OAAO,CAAC,aAAa,CAAiB;IACtC,OAAO,CAAC,eAAe,CAAmB;IAC1C,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,qBAAqB,CAAyB;IACtD,OAAO,CAAC,wBAAwB,CAA4B;IAC5D,OAAO,CAAC,eAAe,CAAmB;IAC1C,qEAAqE;IACrE,OAAO,CAAC,qBAAqB,CAAsC;IACnE,OAAO,CAAC,sBAAsB,CAAuC;IACrE,OAAO,CAAC,sBAAsB,CAAiD;IAC/E,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,kGAAkG;IAClG,OAAO,CAAC,sBAAsB,CAAuC;IAGrE,OAAO,CAAC,+BAA+B,CAAgD;IACvF,+FAA+F;IAC/F,OAAO,CAAC,oBAAoB,CAAqC;IACjE,8HAA8H;IAC9H,OAAO,CAAC,0BAA0B,CAA2C;IAC7E,OAAO,CAAC,6BAA6B,CAA8C;IACnF,OAAO,CAAC,wBAAwB,CAAyC;IACzE,OAAO,CAAC,2BAA2B,CAA+B;IAClE,OAAO,CAAC,0BAA0B,CAA8B;IAChE,OAAO,CAAC,iBAAiB,CAAkC;IAC3D,OAAO,CAAC,kBAAkB,CAA0C;IACpE,8DAA8D;IAC9D,OAAO,CAAC,iBAAiB,CAAmG;IAK5H,OAAO,CAAC,aAAa,CAAuE;IAC5F,OAAO,CAAC,gBAAgB,CAET;IAGf,OAAO,CAAC,cAAc,CAAS;IAC/B,oEAAoE;IACpE,OAAO,CAAC,qBAAqB,CAA6C;IAC1E,OAAO,CAAC,qBAAqB,CAA2G;IACxI,0EAA0E;IAC1E,OAAO,CAAC,eAAe,CAAuC;IAC9D,OAAO,CAAC,gBAAgB,CAAiG;IACzH,OAAO,CAAC,UAAU,CAAyE;IAC3F,wFAAwF;IACxF,OAAO,CAAC,UAAU,CAAkG;IACpH,qEAAqE;IACrE,OAAO,CAAC,cAAc,CAAmG;IACzH,oFAAoF;IACpF,OAAO,CAAC,oBAAoB,CAAK;IACjC,2EAA2E;IAC3E,OAAO,CAAC,sBAAsB,CAA8B;IAC5D,yGAAyG;IACzG,OAAO,CAAC,sBAAsB,CAAuB;IACrD;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAMhC;IACH,oGAAoG;IACpG,OAAO,CAAC,qBAAqB,CAAwB;IACrD;;;OAGG;IACH,OAAO,CAAC,eAAe,CAAS;IAChC,yFAAyF;IACzF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,yFAAyF;IACzF,OAAO,CAAC,cAAc,CAAqB;IAC3C,OAAO,CAAC,uBAAuB,CAAS;IACxC,OAAO,CAAC,wBAAwB,CAA+B;IAC/D,OAAO,CAAC,kBAAkB,CAAuC;gBAErD,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC;IAsE3C,OAAO,CAAC,kBAAkB;IA62C1B,OAAO,CAAC,mBAAmB;IAuE3B,OAAO,CAAC,eAAe;IA+MvB,OAAO,CAAC,kBAAkB;IA0BpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAijF5B;;;OAGG;YACW,2BAA2B;IAuCzC;;;OAGG;YACW,8BAA8B;IAsB5C;;;OAGG;YACW,gCAAgC;IAsB9C;;;OAGG;YACW,8BAA8B;IAsB5C;;;;OAIG;YACW,8BAA8B;IA8M5C;;;;;;;;;;;;OAYG;YACW,6BAA6B;IAsB3C,kEAAkE;IAClE,OAAO,CAAC,iBAAiB,CAA4E;IAErG;;;;;;OAMG;YACW,uBAAuB;IAsCrC;;;;;OAKG;YACW,eAAe;IAuB7B;;;;;OAKG;YACW,iCAAiC;IAuO/C;;;;;;;;;OASG;YACW,4BAA4B;IAgC1C;;;;;;;;OAQG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAStB;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IAIrB;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAgB1B;;;;OAIG;YACW,2BAA2B;YA0B3B,qBAAqB;IAsBnC;;;;;;OAMG;IACH,OAAO,CAAC,0BAA0B;IAwBlC;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;YAqCZ,eAAe;YAoFf,kBAAkB;IAgBhC,OAAO,CAAC,WAAW,CAAK;IAExB,OAAO,CAAC,sBAAsB;IAsC9B;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,oBAAoB;IAwB5B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IA0BvB,4DAA4D;IAC5D,OAAO,CAAC,eAAe,CAAyD;IAEhF;;;;OAIG;YACW,iBAAiB;IAQ/B;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IA6B5B;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAyEvB;;;;;;;OAOG;YACW,2BAA2B;IAczC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAsG1B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IA4DvB;;;;;;OAMG;IACH,OAAO,CAAC,yBAAyB;IA4DjC;;;;;;;OAOG;IACH,OAAO,CAAC,0BAA0B;IAuClC;;;;;OAKG;YACW,0BAA0B;IAoExC;;;;OAIG;YACW,qBAAqB;IA+DnC;;;OAGG;YACW,eAAe;IAkD7B;;;OAGG;YACW,eAAe;IAqC7B,gEAAgE;IAChE,OAAO,CAAC,eAAe,CAAyD;IAEhF,OAAO,CAAC,iBAAiB;IAazB;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAuBlC;;;;;;OAMG;IACH;;;;;;;OAOG;YACW,yBAAyB;YAoBzB,kBAAkB;IAmBhC;;;;;OAKG;YACW,8BAA8B;IA6C5C;;;;OAIG;YACW,+BAA+B;IA+C7C;;;;;;OAMG;YACW,kBAAkB;IA6BhC,OAAO,CAAC,qBAAqB;IAoC7B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAoE1B;;;;;;;;;OASG;IACH;;;;;;;;;;;;;;OAcG;IACH,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA6B;IAE1D;;;;;;;;OAQG;IACH;;OAEG;IACH;;;;;OAKG;YACW,sBAAsB;YA4CtB,eAAe;YA6Cf,kBAAkB;IA2IhC;;;;;;OAMG;YACW,0BAA0B;IA6FxC;;;;;;;OAOG;YACW,wBAAwB;YA2CxB,yBAAyB;IA2FvC;;;;;OAKG;YACW,8BAA8B;IAY5C;;;;;;;OAOG;IACH,OAAO,CAAC,mBAAmB;IAsB3B;;;;;;;;;OASG;IACH,OAAO,CAAC,oBAAoB;IAkC5B;;;;;;;OAOG;IACH,OAAO,CAAC,qBAAqB;IAsB7B;;;;;OAKG;IACH,OAAO,CAAC,yBAAyB;IAqBjC;;;;;;;OAOG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,cAAc;IA2BtB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAkB5B;;;;;OAKG;IACH,OAAO,CAAC,wBAAwB;IAmChC;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;YAmIf,wBAAwB;IAWtC,OAAO,CAAC,qBAAqB;IA8H7B,OAAO,CAAC,cAAc;IActB;;;;;;;;OAQG;IACG,QAAQ,CAAC,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA4X1H,SAAS,IAAI,aAAa;CAG1B;AAyDD,eAAe,YAAY,CAAC"}
|
|
@@ -194,6 +194,7 @@ import { WorkItemWorktreeService } from './services/worktree/workitem-worktree.s
|
|
|
194
194
|
import { WorkItemWorktreeSubscriber, createTerminalNotifier } from './services/worktree/workitem-worktree.subscriber.js';
|
|
195
195
|
import { ResourceModeService, RESOURCE_MODE_CONSTANTS } from './services/agent/resource-mode.service.js';
|
|
196
196
|
import { getRestoreQueue, RESTORE_TIER } from './services/agent/restore-queue.js';
|
|
197
|
+
import { bootLaunchHoldReason, getBootLaunchPacer, readBootPressureSnapshot, scheduleHeldBackNotice, waitForBootQuiet, } from './services/agent/boot-pressure.js';
|
|
197
198
|
import { sessionsToRestore } from './services/agent/restore-filter.js';
|
|
198
199
|
import { ProjectMemoryService } from './services/memory/project-memory.service.js';
|
|
199
200
|
import { TaskHistorySubscriber } from './services/memory/task-history.subscriber.js';
|
|
@@ -369,6 +370,16 @@ export class CrewlyServer {
|
|
|
369
370
|
});
|
|
370
371
|
/** Owner promises past due and never nudged at boot; their agents are restored and reminded once */
|
|
371
372
|
openCommitmentsAtBoot = [];
|
|
373
|
+
/**
|
|
374
|
+
* The boot restore is over: the queue drained, or there was nothing to
|
|
375
|
+
* restore. The boot maintenance passes wait for this (crewly#1353).
|
|
376
|
+
*/
|
|
377
|
+
bootRestoreDone = false;
|
|
378
|
+
/** When this backend booted (the memory hold on boot retries is time-boxed from here) */
|
|
379
|
+
bootAt = Date.now();
|
|
380
|
+
/** Agents the boot restore handed to the retry because memory was tight (crewly#1353) */
|
|
381
|
+
heldBackAtBoot = new Set();
|
|
382
|
+
heldBackNoticeScheduled = false;
|
|
372
383
|
healthMonitoringInterval = null;
|
|
373
384
|
backgroundWatchdog = null;
|
|
374
385
|
constructor(config) {
|
|
@@ -1045,7 +1056,7 @@ export class CrewlyServer {
|
|
|
1045
1056
|
// digest page compiled from the log. No LLM. Non-fatal.
|
|
1046
1057
|
try {
|
|
1047
1058
|
const { WikiMaintenanceService } = await import('./services/wiki/wiki-maintenance.service.js');
|
|
1048
|
-
const maintenance = new WikiMaintenanceService();
|
|
1059
|
+
const maintenance = new WikiMaintenanceService({ bootGate: this.bootQuietGate() });
|
|
1049
1060
|
WikiMaintenanceService.setInstance(maintenance);
|
|
1050
1061
|
maintenance.start();
|
|
1051
1062
|
}
|
|
@@ -1078,6 +1089,7 @@ export class CrewlyServer {
|
|
|
1078
1089
|
}
|
|
1079
1090
|
return agents;
|
|
1080
1091
|
},
|
|
1092
|
+
bootGate: this.bootQuietGate(),
|
|
1081
1093
|
});
|
|
1082
1094
|
AgentMemoryRefreshService.setInstance(refresh);
|
|
1083
1095
|
refresh.start();
|
|
@@ -1087,6 +1099,51 @@ export class CrewlyServer {
|
|
|
1087
1099
|
error: refreshErr.message,
|
|
1088
1100
|
});
|
|
1089
1101
|
}
|
|
1102
|
+
// Work records (#1349): one structured `(record)` entry per finished
|
|
1103
|
+
// (or long-blocked) WorkItem in the project vault log — who asked, who
|
|
1104
|
+
// did it, outputs, errors, helpers, outcome — from the WorkItem, its
|
|
1105
|
+
// ticket, its children, the trace and the doer's transcript tail. The
|
|
1106
|
+
// digest, search and the startup briefing read them. No LLM. Non-fatal.
|
|
1107
|
+
try {
|
|
1108
|
+
const [{ WorkRecordService }, { ProjectTicketService }, { getTraceStore }] = await Promise.all([
|
|
1109
|
+
import('./services/wiki/work-record.service.js'),
|
|
1110
|
+
import('./services/project-tickets/project-ticket.service.js'),
|
|
1111
|
+
import('./services/trace/trace-store.js'),
|
|
1112
|
+
]);
|
|
1113
|
+
const pool = TaskPoolService.getInstance();
|
|
1114
|
+
const tickets = ProjectTicketService.getInstance();
|
|
1115
|
+
const records = new WorkRecordService({
|
|
1116
|
+
listWorkItems: () => pool.getAllItems(),
|
|
1117
|
+
findWorkItem: (id) => pool.findWorkItem(id),
|
|
1118
|
+
listTeams: () => this.storageService.getTeams(),
|
|
1119
|
+
listProjects: () => this.storageService.getProjects(),
|
|
1120
|
+
getTicket: (projectPath, id) => tickets.get(projectPath, id),
|
|
1121
|
+
getRequest: (id) => RequestService.getInstance().getById(id),
|
|
1122
|
+
readTrace: async (wi) => {
|
|
1123
|
+
const store = getTraceStore();
|
|
1124
|
+
const traceId = wi.traceId ?? store.traceByRef('workItem', wi.id);
|
|
1125
|
+
if (!traceId)
|
|
1126
|
+
return [];
|
|
1127
|
+
const all = await store.readAll(traceId);
|
|
1128
|
+
return all?.events ?? [];
|
|
1129
|
+
},
|
|
1130
|
+
locateTranscript: this.askTranscriptLocator(),
|
|
1131
|
+
eventBus: this.eventBusService,
|
|
1132
|
+
onTicketSettled: (listener) => tickets.onChange((change) => {
|
|
1133
|
+
const settled = change.ticket.status === 'done' || change.ticket.status === 'cancelled';
|
|
1134
|
+
if (settled && change.before.status !== change.ticket.status && change.ticket.workItemId)
|
|
1135
|
+
listener(change.ticket.workItemId);
|
|
1136
|
+
}),
|
|
1137
|
+
bootGate: this.bootQuietGate(),
|
|
1138
|
+
});
|
|
1139
|
+
WorkRecordService.setInstance(records);
|
|
1140
|
+
records.start();
|
|
1141
|
+
}
|
|
1142
|
+
catch (recordErr) {
|
|
1143
|
+
this.logger.warn('Work records failed to start (non-fatal)', {
|
|
1144
|
+
error: recordErr.message,
|
|
1145
|
+
});
|
|
1146
|
+
}
|
|
1090
1147
|
// LLM-wiki → WorkItem bridge (2026-05-27): pending wiki queue
|
|
1091
1148
|
// items + legacy migration candidates become claimable
|
|
1092
1149
|
// WorkItems in the V3 pool. Replaces the bookkeep/reflect
|
|
@@ -4289,10 +4346,11 @@ export class CrewlyServer {
|
|
|
4289
4346
|
return r.success;
|
|
4290
4347
|
});
|
|
4291
4348
|
// Finished work cleans up after itself: a WorkItem / ticket reaching
|
|
4292
|
-
//
|
|
4293
|
-
// unchanged — only landed, clean, unattended worktrees go)
|
|
4294
|
-
|
|
4295
|
-
|
|
4349
|
+
// a terminal state schedules one debounced janitor pass (safe rules
|
|
4350
|
+
// unchanged — only landed, clean, unattended worktrees go) and notes
|
|
4351
|
+
// the item's worktree as finished (its node_modules go after 1 day).
|
|
4352
|
+
this.eventBusService.onInProcess(['task:done', 'task:verified', 'task:cancelled', 'task:failed'], (event) => {
|
|
4353
|
+
WorktreeJanitorService.getInstance().requestTidy(event.workItemId);
|
|
4296
4354
|
});
|
|
4297
4355
|
if (WorktreeJanitorService.getInstance().start()) {
|
|
4298
4356
|
this.logger.info('WorktreeJanitorService scheduled');
|
|
@@ -4785,12 +4843,14 @@ export class CrewlyServer {
|
|
|
4785
4843
|
const settings = await settingsService.getSettings();
|
|
4786
4844
|
if (!settings.general.autoResumeOnRestart) {
|
|
4787
4845
|
this.logger.info('Auto-resume on restart is disabled, skipping agent session restore');
|
|
4846
|
+
this.bootRestoreDone = true;
|
|
4788
4847
|
return;
|
|
4789
4848
|
}
|
|
4790
4849
|
const persistence = getSessionStatePersistence();
|
|
4791
4850
|
const state = await persistence.loadState();
|
|
4792
4851
|
if (!state || state.sessions.length === 0) {
|
|
4793
4852
|
this.logger.debug('No persisted agent sessions to restore');
|
|
4853
|
+
this.bootRestoreDone = true;
|
|
4794
4854
|
return;
|
|
4795
4855
|
}
|
|
4796
4856
|
// Filter out orchestrator sessions (already auto-started separately)
|
|
@@ -4824,33 +4884,70 @@ export class CrewlyServer {
|
|
|
4824
4884
|
});
|
|
4825
4885
|
}
|
|
4826
4886
|
let targets = null;
|
|
4887
|
+
// Why each target other than a WorkItem is owed something; the reason
|
|
4888
|
+
// is logged in the restore plan for agents that were idle at shutdown.
|
|
4889
|
+
const owedReason = new Map();
|
|
4890
|
+
// Agents an owner thread waits on: launched even while memory is tight (crewly#1353).
|
|
4891
|
+
const ownerThreadWaits = new Set();
|
|
4892
|
+
const noteOwed = (names, reason) => {
|
|
4893
|
+
const out = [];
|
|
4894
|
+
for (const name of names) {
|
|
4895
|
+
if (!owedReason.has(name))
|
|
4896
|
+
owedReason.set(name, reason);
|
|
4897
|
+
out.push(name);
|
|
4898
|
+
}
|
|
4899
|
+
return out;
|
|
4900
|
+
};
|
|
4827
4901
|
try {
|
|
4828
4902
|
const allItems = await TaskPoolService.getInstance().getAllItems();
|
|
4829
4903
|
// Work in hand only: active statuses, touched recently (see restore-filter),
|
|
4830
4904
|
// plus agents whose turn the last restart cut off.
|
|
4831
4905
|
const owed = await this.loadOwedWorkAtBoot();
|
|
4832
4906
|
targets = sessionsToRestore(allItems, [
|
|
4833
|
-
...this.interruptedTurnsAtBoot.map((t) => t.sessionName),
|
|
4907
|
+
...noteOwed(this.interruptedTurnsAtBoot.map((t) => t.sessionName), 'interrupted turn'),
|
|
4834
4908
|
// Agents with messages still waiting for them (restored from
|
|
4835
4909
|
// disk by the queue) — someone is owed an answer.
|
|
4836
|
-
...SubAgentMessageQueue.getInstance().sessionsWithPending(),
|
|
4910
|
+
...noteOwed(SubAgentMessageQueue.getInstance().sessionsWithPending(), 'queued messages'),
|
|
4837
4911
|
// Agents whose owner promise is past due and was never
|
|
4838
4912
|
// nudged: idle or not, they come back to be reminded once
|
|
4839
4913
|
// (2026-10-02, Eve). Not-yet-due and already-nudged
|
|
4840
4914
|
// promises are left to the open-items sweep.
|
|
4841
|
-
...owed.map((c) => c.sessionName),
|
|
4915
|
+
...noteOwed(owed.map((c) => c.sessionName), 'owed promise'),
|
|
4842
4916
|
// Agents an owner Slack thread waits on: a promised
|
|
4843
4917
|
// follow-up, an unanswered owner message, or an owner
|
|
4844
4918
|
// card they asked (2026-10-08: Atlas waited on a browser
|
|
4845
4919
|
// approval in the owner's thread and was skipped as "no
|
|
4846
4920
|
// work in hand").
|
|
4847
|
-
...(await this.ownerThreadSessionsAtBoot())
|
|
4921
|
+
...noteOwed(await this.ownerThreadSessionsAtBoot().then((names) => { for (const n of names)
|
|
4922
|
+
ownerThreadWaits.add(n); return names; }), 'owner thread waits on it'),
|
|
4848
4923
|
]);
|
|
4849
4924
|
}
|
|
4850
4925
|
catch (poolErr) {
|
|
4851
4926
|
this.logger.warn('Auto-restore could not query task pool; falling back to restoring every persisted session', { error: poolErr instanceof Error ? poolErr.message : String(poolErr) });
|
|
4852
4927
|
}
|
|
4853
|
-
|
|
4928
|
+
// A target that was not running at shutdown (idle-stopped earlier) is
|
|
4929
|
+
// still owed something and comes back from its team binding — unless
|
|
4930
|
+
// the owner stopped it, its team is paused, or its team moved to
|
|
4931
|
+
// another machine (crewly#1348: Pia, 2026-10-16).
|
|
4932
|
+
const { liveAgentSessions, movedAwayTeamIds } = await import('./services/messaging/owner-thread-sentinel.wiring.js');
|
|
4933
|
+
const live = teams ? liveAgentSessions(teams, movedAwayTeamIds(this.config.crewlyHome)) : new Set();
|
|
4934
|
+
const selection = selectAutoRestoreSessions({
|
|
4935
|
+
sessions: baselineSessions,
|
|
4936
|
+
teams,
|
|
4937
|
+
targets,
|
|
4938
|
+
heldBack: (name) => name === ORCHESTRATOR_SESSION_NAME || !live.has(name) || isOwnerStopped(name) || !!pausedTeamOfSession(name),
|
|
4939
|
+
});
|
|
4940
|
+
if (selection.droppedTargets.length > 0) {
|
|
4941
|
+
this.logger.info('Not restoring owed agents that were idle at shutdown: no live team member is bound to the name, or it is held back', {
|
|
4942
|
+
dropped: selection.droppedTargets,
|
|
4943
|
+
});
|
|
4944
|
+
}
|
|
4945
|
+
const idleAtShutdown = await this.persistedSessionsForBindings(teams ?? [], selection.fromTargets);
|
|
4946
|
+
if (idleAtShutdown.length > 0) {
|
|
4947
|
+
this.logger.info('Restoring owed agents that were idle at shutdown (not in the persisted state)', {
|
|
4948
|
+
sessions: idleAtShutdown.map((s) => `${s.name} (${owedReason.get(s.name) ?? 'work in hand'})`),
|
|
4949
|
+
});
|
|
4950
|
+
}
|
|
4854
4951
|
if (selection.unbound.length > 0) {
|
|
4855
4952
|
this.logger.warn('Not auto-restoring sessions whose name no team member is bound to', {
|
|
4856
4953
|
unbound: selection.unbound,
|
|
@@ -4864,9 +4961,10 @@ export class CrewlyServer {
|
|
|
4864
4961
|
truncated: selection.skippedNoWork.length > 20,
|
|
4865
4962
|
});
|
|
4866
4963
|
}
|
|
4867
|
-
const agentSessions = selection.sessions;
|
|
4964
|
+
const agentSessions = [...selection.sessions, ...idleAtShutdown];
|
|
4868
4965
|
if (agentSessions.length === 0) {
|
|
4869
4966
|
this.logger.info('No persisted agent sessions to restore (all idle, no pending WorkItems)');
|
|
4967
|
+
this.bootRestoreDone = true;
|
|
4870
4968
|
return;
|
|
4871
4969
|
}
|
|
4872
4970
|
this.logger.info('Auto-restoring agent sessions from persisted state', {
|
|
@@ -4883,6 +4981,7 @@ export class CrewlyServer {
|
|
|
4883
4981
|
owedSince.set(c.sessionName, Math.min(owedSince.get(c.sessionName) ?? Infinity, Number.isFinite(due) ? due : 0));
|
|
4884
4982
|
}
|
|
4885
4983
|
const leadRoles = TEAM_LEAD_CONSTANTS.LEAD_ROLES;
|
|
4984
|
+
const wasIdle = new Set(idleAtShutdown.map((s) => s.name));
|
|
4886
4985
|
const entries = agentSessions.map((session) => {
|
|
4887
4986
|
const ownerTimes = queueMessages.peek(session.name).filter((m) => m.meta?.owner === true).map((m) => m.queuedAt);
|
|
4888
4987
|
if (owedSince.has(session.name))
|
|
@@ -4911,18 +5010,31 @@ export class CrewlyServer {
|
|
|
4911
5010
|
});
|
|
4912
5011
|
if (!result.success)
|
|
4913
5012
|
this.logger.warn('Failed to restore agent session', { name: session.name, error: result.error });
|
|
5013
|
+
// Every slot was taken: keep trying until one frees instead of
|
|
5014
|
+
// dropping the agent (the owner's thread is still waiting).
|
|
5015
|
+
if (!result.success && result.errorCode === RESOURCE_MODE_CONSTANTS.START_DEFERRED_ERROR_CODE)
|
|
5016
|
+
this.retryRestoreWhileOwed(session.name);
|
|
4914
5017
|
return { success: result.success };
|
|
4915
5018
|
},
|
|
4916
5019
|
onReady: () => this.sendRecoveryFor(session.name),
|
|
5020
|
+
// Memory is tight and nobody waits on it: the retry brings it back
|
|
5021
|
+
// when the machine eases (held by bootLaunchHold), never dropped.
|
|
5022
|
+
onDeferred: () => {
|
|
5023
|
+
this.noteHeldBackAtBoot(session.name);
|
|
5024
|
+
this.retryRestoreWhileOwed(session.name);
|
|
5025
|
+
},
|
|
5026
|
+
essential: ownerTimes.length > 0 || session.role === ORCHESTRATOR_ROLE || ownerThreadWaits.has(session.name),
|
|
4917
5027
|
};
|
|
5028
|
+
// An agent that was idle at shutdown says in the plan what it is owed.
|
|
5029
|
+
const idle = wasIdle.has(session.name) ? ` (${owedReason.get(session.name) ?? 'work in hand'}; was idle at shutdown)` : '';
|
|
4918
5030
|
if (ownerTimes.length > 0) {
|
|
4919
|
-
return { ...base, tier: RESTORE_TIER.OWNER, ownerSince: Math.min(...ownerTimes), reason:
|
|
5031
|
+
return { ...base, tier: RESTORE_TIER.OWNER, ownerSince: Math.min(...ownerTimes), reason: `owner message waiting${idle}` };
|
|
4920
5032
|
}
|
|
4921
5033
|
if (session.role === ORCHESTRATOR_ROLE)
|
|
4922
5034
|
return { ...base, tier: RESTORE_TIER.ORCHESTRATOR, reason: 'orchestrator' };
|
|
4923
5035
|
if (leadRoles.includes(session.role ?? ''))
|
|
4924
|
-
return { ...base, tier: RESTORE_TIER.LEAD, reason:
|
|
4925
|
-
return { ...base, tier: RESTORE_TIER.OTHER, reason:
|
|
5036
|
+
return { ...base, tier: RESTORE_TIER.LEAD, reason: `team lead with work in hand${idle}` };
|
|
5037
|
+
return { ...base, tier: RESTORE_TIER.OTHER, reason: `work in hand${idle}` };
|
|
4926
5038
|
});
|
|
4927
5039
|
// Runs in the background; boot continues while agents come back.
|
|
4928
5040
|
void getRestoreQueue()
|
|
@@ -4933,15 +5045,139 @@ export class CrewlyServer {
|
|
|
4933
5045
|
})
|
|
4934
5046
|
.catch((error) => {
|
|
4935
5047
|
this.logger.error('Restore queue failed', { error: error instanceof Error ? error.message : String(error) });
|
|
5048
|
+
})
|
|
5049
|
+
.finally(() => {
|
|
5050
|
+
this.bootRestoreDone = true;
|
|
4936
5051
|
});
|
|
4937
5052
|
}
|
|
4938
5053
|
catch (error) {
|
|
5054
|
+
this.bootRestoreDone = true;
|
|
4939
5055
|
this.logger.error('Failed to auto-restore agent sessions', {
|
|
4940
5056
|
error: error instanceof Error ? error.message : String(error),
|
|
4941
5057
|
});
|
|
4942
5058
|
// Don't fail startup if auto-restore fails
|
|
4943
5059
|
}
|
|
4944
5060
|
}
|
|
5061
|
+
/**
|
|
5062
|
+
* Restore entries for owed agents that have no persisted session: built from
|
|
5063
|
+
* the team config the way a team start would (member role and runtime, the
|
|
5064
|
+
* team's first project as the working directory). A binding whose member is
|
|
5065
|
+
* gone from the config by now is skipped.
|
|
5066
|
+
*
|
|
5067
|
+
* @param teams - Teams from storage
|
|
5068
|
+
* @param bindings - Targets resolved to members (see selectAutoRestoreSessions)
|
|
5069
|
+
* @returns Session entries in the shape the restore queue runs
|
|
5070
|
+
*/
|
|
5071
|
+
async persistedSessionsForBindings(teams, bindings) {
|
|
5072
|
+
if (bindings.length === 0)
|
|
5073
|
+
return [];
|
|
5074
|
+
let projectPaths = new Map();
|
|
5075
|
+
try {
|
|
5076
|
+
projectPaths = new Map((await this.storageService.getProjects()).map((p) => [p.id, p.path]));
|
|
5077
|
+
}
|
|
5078
|
+
catch (err) {
|
|
5079
|
+
this.logger.warn('Could not read projects for the owed agents; restoring them in the server directory', {
|
|
5080
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5081
|
+
});
|
|
5082
|
+
}
|
|
5083
|
+
const out = [];
|
|
5084
|
+
for (const binding of bindings) {
|
|
5085
|
+
const team = teams.find((t) => t.id === binding.teamId);
|
|
5086
|
+
const member = team?.members.find((m) => m.id === binding.memberId);
|
|
5087
|
+
if (!team || !member)
|
|
5088
|
+
continue;
|
|
5089
|
+
out.push({
|
|
5090
|
+
name: binding.name,
|
|
5091
|
+
cwd: (team.projectIds[0] && projectPaths.get(team.projectIds[0])) || process.cwd(),
|
|
5092
|
+
command: '',
|
|
5093
|
+
args: [],
|
|
5094
|
+
runtimeType: member.runtimeType,
|
|
5095
|
+
role: member.role,
|
|
5096
|
+
teamId: team.id,
|
|
5097
|
+
memberId: member.id,
|
|
5098
|
+
});
|
|
5099
|
+
}
|
|
5100
|
+
return out;
|
|
5101
|
+
}
|
|
5102
|
+
/**
|
|
5103
|
+
* A boot restore was deferred by the running-agent cap. The agent is owed
|
|
5104
|
+
* something that does not expire on its own (an owner thread, a promise,
|
|
5105
|
+
* queued messages), so keep trying until a slot frees (crewly#1337's retry;
|
|
5106
|
+
* the queue check is replaced because not every debt is a queued message).
|
|
5107
|
+
* Never throws.
|
|
5108
|
+
*
|
|
5109
|
+
* @param sessionName - The agent whose restore was deferred
|
|
5110
|
+
*/
|
|
5111
|
+
retryRestoreWhileOwed(sessionName) {
|
|
5112
|
+
void (async () => {
|
|
5113
|
+
const { scheduleDeferredStartRetry } = await import('./services/messaging/deferred-start-retry.js');
|
|
5114
|
+
const { activateAgentBySession } = await import('./controllers/team/team.controller.js');
|
|
5115
|
+
await scheduleDeferredStartRetry(sessionName, {
|
|
5116
|
+
activate: (n) => {
|
|
5117
|
+
getBootLaunchPacer().note();
|
|
5118
|
+
return activateAgentBySession(this.apiController, n);
|
|
5119
|
+
},
|
|
5120
|
+
hasPending: (n) => !isOwnerStopped(n),
|
|
5121
|
+
hold: () => this.bootLaunchHold(),
|
|
5122
|
+
});
|
|
5123
|
+
})().catch((err) => {
|
|
5124
|
+
this.logger.warn('Could not retry a deferred boot restore', {
|
|
5125
|
+
sessionName,
|
|
5126
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5127
|
+
});
|
|
5128
|
+
});
|
|
5129
|
+
}
|
|
5130
|
+
/**
|
|
5131
|
+
* Why a boot launch outside the restore queue must wait this round
|
|
5132
|
+
* (crewly#1353): memory is tight (within the first hour), the restore queue
|
|
5133
|
+
* is still launching, or another boot launch began less than the launch
|
|
5134
|
+
* spacing ago. Null = go.
|
|
5135
|
+
*
|
|
5136
|
+
* @returns Reason to hold, or null
|
|
5137
|
+
*/
|
|
5138
|
+
bootLaunchHold() {
|
|
5139
|
+
return bootLaunchHoldReason({
|
|
5140
|
+
snapshot: readBootPressureSnapshot,
|
|
5141
|
+
restoreRunning: () => getRestoreQueue().stats().running,
|
|
5142
|
+
pacer: getBootLaunchPacer(),
|
|
5143
|
+
bootAt: this.bootAt,
|
|
5144
|
+
});
|
|
5145
|
+
}
|
|
5146
|
+
/**
|
|
5147
|
+
* Gate for the boot maintenance passes (crewly#1353): resolves when the
|
|
5148
|
+
* boot restore is over and memory is calm, or after the wait cap with
|
|
5149
|
+
* `reduced: true`.
|
|
5150
|
+
*
|
|
5151
|
+
* @returns A gate function for the service constructors
|
|
5152
|
+
*/
|
|
5153
|
+
bootQuietGate() {
|
|
5154
|
+
return () => waitForBootQuiet({ restoreDone: () => this.bootRestoreDone });
|
|
5155
|
+
}
|
|
5156
|
+
/**
|
|
5157
|
+
* The boot restore handed an agent to the retry because memory was tight.
|
|
5158
|
+
* Once, when the first one is held back, schedule the one-line owner note
|
|
5159
|
+
* that is sent only if some of them are still down after
|
|
5160
|
+
* OWNER_NOTE_AFTER_MS (crewly#1353).
|
|
5161
|
+
*
|
|
5162
|
+
* @param sessionName - The agent held back
|
|
5163
|
+
*/
|
|
5164
|
+
noteHeldBackAtBoot(sessionName) {
|
|
5165
|
+
this.heldBackAtBoot.add(sessionName);
|
|
5166
|
+
if (this.heldBackNoticeScheduled)
|
|
5167
|
+
return;
|
|
5168
|
+
this.heldBackNoticeScheduled = true;
|
|
5169
|
+
void scheduleHeldBackNotice({
|
|
5170
|
+
stillHeld: async () => {
|
|
5171
|
+
const out = [];
|
|
5172
|
+
for (const name of this.heldBackAtBoot) {
|
|
5173
|
+
if (!(await this.apiController.agentRegistrationService.isSessionLiveOrLaunching(name)))
|
|
5174
|
+
out.push(name);
|
|
5175
|
+
}
|
|
5176
|
+
return out;
|
|
5177
|
+
},
|
|
5178
|
+
notify: (line) => slackOwnerAlertNotifier({ title: 'Agents waiting on memory', message: line, urgent: false }),
|
|
5179
|
+
});
|
|
5180
|
+
}
|
|
4945
5181
|
/**
|
|
4946
5182
|
* Check for and handle pending self-improvement from hot-reload.
|
|
4947
5183
|
* This runs at startup to validate or rollback any changes made
|
|
@@ -6875,14 +7111,21 @@ export class CrewlyServer {
|
|
|
6875
7111
|
*
|
|
6876
7112
|
* @param sessionName - The agent to start
|
|
6877
7113
|
* @param firstPauseMs - Pause before the first attempt (default: the retry's own)
|
|
7114
|
+
* @param opts - `boot`: a boot launch, held while memory is tight and paced with the restore queue (crewly#1353)
|
|
6878
7115
|
*/
|
|
6879
|
-
retryStartWhileQueued(sessionName, firstPauseMs) {
|
|
7116
|
+
retryStartWhileQueued(sessionName, firstPauseMs, opts = {}) {
|
|
6880
7117
|
void (async () => {
|
|
6881
7118
|
const { scheduleDeferredStartRetry } = await import('./services/messaging/deferred-start-retry.js');
|
|
6882
7119
|
const { activateAgentBySession } = await import('./controllers/team/team.controller.js');
|
|
6883
7120
|
await scheduleDeferredStartRetry(sessionName, {
|
|
6884
|
-
activate: (n) =>
|
|
7121
|
+
activate: (n) => {
|
|
7122
|
+
if (opts.boot)
|
|
7123
|
+
getBootLaunchPacer().note();
|
|
7124
|
+
return activateAgentBySession(this.apiController, n);
|
|
7125
|
+
},
|
|
6885
7126
|
hasPending: (n) => SubAgentMessageQueue.getInstance().hasPending(n),
|
|
7127
|
+
// Boot launches outside the restore queue keep its spacing and its memory gate (crewly#1353).
|
|
7128
|
+
...(opts.boot ? { hold: () => this.bootLaunchHold() } : {}),
|
|
6886
7129
|
...(firstPauseMs !== undefined ? { firstPauseMs } : {}),
|
|
6887
7130
|
});
|
|
6888
7131
|
})().catch((err) => {
|
|
@@ -6899,13 +7142,14 @@ export class CrewlyServer {
|
|
|
6899
7142
|
* (with its messages persisted) was never started again (Pia, 2026-10-16).
|
|
6900
7143
|
*/
|
|
6901
7144
|
wakeQueuedAgentsAfterBoot() {
|
|
6902
|
-
const
|
|
7145
|
+
const restoreQueue = getRestoreQueue();
|
|
6903
7146
|
const backend = getSessionBackendSync();
|
|
6904
7147
|
const registration = this.apiController.agentRegistrationService;
|
|
6905
7148
|
const sessions = SubAgentMessageQueue.getInstance()
|
|
6906
7149
|
.sessionsWithPending()
|
|
6907
7150
|
.filter((name) => name !== ORCHESTRATOR_SESSION_NAME &&
|
|
6908
|
-
|
|
7151
|
+
// Waiting in the restore queue or being started by it right now
|
|
7152
|
+
!restoreQueue.isRestoring(name) &&
|
|
6909
7153
|
!isOwnerStopped(name) &&
|
|
6910
7154
|
!pausedTeamOfSession(name) &&
|
|
6911
7155
|
!backend?.sessionExists(name) &&
|
|
@@ -6914,7 +7158,7 @@ export class CrewlyServer {
|
|
|
6914
7158
|
return;
|
|
6915
7159
|
this.logger.info('Starting agents that were down with messages queued for them', { sessions });
|
|
6916
7160
|
for (const name of sessions)
|
|
6917
|
-
this.retryStartWhileQueued(name, SUB_AGENT_QUEUE_CONSTANTS.BOOT_QUEUED_WAKE_DELAY_MS);
|
|
7161
|
+
this.retryStartWhileQueued(name, SUB_AGENT_QUEUE_CONSTANTS.BOOT_QUEUED_WAKE_DELAY_MS, { boot: true });
|
|
6918
7162
|
}
|
|
6919
7163
|
/**
|
|
6920
7164
|
* Write a system note into the orchestrator's own chat (the conversation
|