crewly 1.20.359 → 1.20.360

package/dist/backend/backend/src/services/agent/boot-pressure.d.ts

@@ -0,0 +1,147 @@
+/**
+ * Memory-aware boot (crewly#1353): the snapshot the boot restore and the boot
+ * maintenance passes read, the gate that holds a maintenance pass until the
+ * restore queue is drained and the machine is calm, the pacer that spaces
+ * launches across the restore queue and the deferred-start retry, and the
+ * one-line owner note when boot restores were held back for long.
+ *
+ * Pressure mode (ResourceModeService) needs two tight 30 s samples before it
+ * engages; the first minute after a boot is exactly when everything starts at
+ * once, so the boot reads the raw memory reading as well.
+ *
+ * @module services/agent/boot-pressure
+ */
+/** What the boot looks at before a launch or a maintenance pass. */
+export interface BootPressureSnapshot {
+    usedPercent: number;
+    freeMB: number;
+    swapUsedPercent: number;
+    osPressure: boolean;
+    /** ResourceModeService is in pressure mode */
+    pressureMode: boolean;
+}
+/** Thresholds, with the optional env overrides applied. */
+export interface BootPressureThresholds {
+    swapUsedPercent: number;
+    minFreeMB: number;
+    launchSpacingMs: number;
+}
+/**
+ * Thresholds from the constants block, with env overrides.
+ *
+ * @param env - Environment (tests inject one)
+ * @returns Effective thresholds
+ */
+export declare function bootPressureThresholds(env?: NodeJS.ProcessEnv): BootPressureThresholds;
+/**
+ * Why the snapshot is tight for the boot, or null when it is calm. Tight:
+ * pressure mode, swap at or above the threshold, or free memory under the
+ * floor. An unreadable snapshot (null) is calm: a boot must not stall on a
+ * reading that failed.
+ *
+ * @param snapshot - Reading, or null when unavailable
+ * @param thresholds - Effective thresholds
+ * @returns One-line reason, or null
+ */
+export declare function bootPressureReason(snapshot: BootPressureSnapshot | null, thresholds?: BootPressureThresholds): string | null;
+/**
+ * The live snapshot: ResourceModeService's memory reading and mode. Null when
+ * the reading throws or memory is unknown (totalMB 0).
+ *
+ * @returns Snapshot, or null
+ */
+export declare function readBootPressureSnapshot(): BootPressureSnapshot | null;
+/**
+ * Remembers when the last boot launch began, so the restore queue and the
+ * deferred-start retry keep the launch spacing between them.
+ */
+export declare class BootLaunchPacer {
+    private readonly now;
+    private lastAt;
+    constructor(now?: () => number);
+    /** A launch began now. */
+    note(): void;
+    /** Milliseconds since the last launch began (Infinity when none yet). */
+    msSinceLast(): number;
+}
+/** The process-wide pacer. */
+export declare function getBootLaunchPacer(): BootLaunchPacer;
+/** Test hook: drop the pacer. */
+export declare function resetBootLaunchPacer(): void;
+/** What {@link bootLaunchHoldReason} looks at. */
+export interface BootLaunchHoldDeps {
+    snapshot: () => BootPressureSnapshot | null;
+    /** The restore queue is still launching agents */
+    restoreRunning: () => boolean;
+    pacer: BootLaunchPacer;
+    /** Epoch ms of the boot */
+    bootAt: number;
+    thresholds?: BootPressureThresholds;
+    now?: () => number;
+    holdMaxMs?: number;
+}
+/**
+ * Why a boot launch outside the restore queue (the deferred-start retry) must
+ * wait this round: memory is tight (only within `holdMaxMs` of the boot), the
+ * restore queue is still launching, or another boot launch began less than
+ * the launch spacing ago. Null = go.
+ *
+ * @param deps - Snapshot, restore state, pacer, boot time
+ * @returns Reason to hold, or null
+ */
+export declare function bootLaunchHoldReason(deps: BootLaunchHoldDeps): string | null;
+/** What {@link waitForBootQuiet} needs. */
+export interface BootQuietGateDeps {
+    /** The boot restore is over (queue drained, or there was nothing to restore). */
+    restoreDone: () => boolean;
+    /** Current reading (default: the live one). */
+    snapshot?: () => BootPressureSnapshot | null;
+    thresholds?: BootPressureThresholds;
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+    pollMs?: number;
+    maxWaitMs?: number;
+}
+/** Outcome of the gate. */
+export interface BootQuietGateResult {
+    waitedMs: number;
+    /** The gate gave up waiting; the pass should run reduced. */
+    reduced: boolean;
+    /** Why it was still waiting when it gave up */
+    reason?: string;
+}
+/**
+ * Hold a boot maintenance pass until the restore queue is drained and the
+ * machine is not tight, polling every `pollMs`. After `maxWaitMs` it gives
+ * up and returns `reduced: true` so the pass runs with a smaller batch.
+ *
+ * @param deps - Restore state, snapshot, clock
+ * @returns How long it waited and whether the pass should run reduced
+ */
+export declare function waitForBootQuiet(deps: BootQuietGateDeps): Promise<BootQuietGateResult>;
+/**
+ * Names as a sentence fragment: "Ella", "Ella and Pia", "Ella, Pia and Owen".
+ *
+ * @param names - Agent names
+ * @returns Joined names
+ */
+export declare function joinNames(names: readonly string[]): string;
+/** What {@link scheduleHeldBackNotice} needs. */
+export interface HeldBackNoticeDeps {
+    /** Agents held back by the boot that are still not running */
+    stillHeld: () => Promise<string[]>;
+    /** Sends one line to the owner; false when nothing is connected */
+    notify: (line: string) => Promise<boolean>;
+    afterMs?: number;
+    setTimer?: (fn: () => void, ms: number) => void;
+}
+/**
+ * One line to the owner, once, if agents the boot held back for memory are
+ * still down `afterMs` later. Nothing is sent when they came back in time.
+ * The line names the agents and says they will return; no mechanics.
+ *
+ * @param deps - Held-back check and notifier
+ * @returns Resolves when the check ran (tests); callers do not await it
+ */
+export declare function scheduleHeldBackNotice(deps: HeldBackNoticeDeps): Promise<void>;
+//# sourceMappingURL=boot-pressure.d.ts.map

package/dist/backend/backend/src/services/agent/boot-pressure.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"boot-pressure.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/agent/boot-pressure.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,oEAAoE;AACpE,MAAM,WAAW,oBAAoB;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,OAAO,CAAC;IACpB,8CAA8C;IAC9C,YAAY,EAAE,OAAO,CAAC;CACtB;AAED,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB;IACtC,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;CACxB;AAcD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,sBAAsB,CAMnG;AAED;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CACjC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,EACrC,UAAU,GAAE,sBAAiD,GAC3D,MAAM,GAAG,IAAI,CAQf;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,IAAI,oBAAoB,GAAG,IAAI,CAetE;AAED;;;GAGG;AACH,qBAAa,eAAe;IAGf,OAAO,CAAC,QAAQ,CAAC,GAAG;IAFhC,OAAO,CAAC,MAAM,CAAuB;gBAER,GAAG,GAAE,MAAM,MAAiB;IAEzD,0BAA0B;IAC1B,IAAI,IAAI,IAAI;IAIZ,yEAAyE;IACzE,WAAW,IAAI,MAAM;CAGrB;AAID,8BAA8B;AAC9B,wBAAgB,kBAAkB,IAAI,eAAe,CAEpD;AAED,iCAAiC;AACjC,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C;AAED,kDAAkD;AAClD,MAAM,WAAW,kBAAkB;IAClC,QAAQ,EAAE,MAAM,oBAAoB,GAAG,IAAI,CAAC;IAC5C,kDAAkD;IAClD,cAAc,EAAE,MAAM,OAAO,CAAC;IAC9B,KAAK,EAAE,eAAe,CAAC;IACvB,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,GAAG,IAAI,CAY5E;AAED,2CAA2C;AAC3C,MAAM,WAAW,iBAAiB;IACjC,iFAAiF;IACjF,WAAW,EAAE,MAAM,OAAO,CAAC;IAC3B,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,oBAAoB,GAAG,IAAI,CAAC;IAC7C,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,2BAA2B;AAC3B,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,6DAA6D;IAC7D,OAAO,EAAE,OAAO,CAAC;IACjB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAID;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAc5F;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAG1D;AAED,iDAAiD;AACjD,MAAM,WAAW,kBAAkB;IAClC,8DAA8D;IAC9D,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACnC,mEAAmE;IACnE,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,IAAI,EAAE,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;CAChD;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB9E"}

package/dist/backend/backend/src/services/agent/boot-pressure.js

@@ -0,0 +1,212 @@
+/**
+ * Memory-aware boot (crewly#1353): the snapshot the boot restore and the boot
+ * maintenance passes read, the gate that holds a maintenance pass until the
+ * restore queue is drained and the machine is calm, the pacer that spaces
+ * launches across the restore queue and the deferred-start retry, and the
+ * one-line owner note when boot restores were held back for long.
+ *
+ * Pressure mode (ResourceModeService) needs two tight 30 s samples before it
+ * engages; the first minute after a boot is exactly when everything starts at
+ * once, so the boot reads the raw memory reading as well.
+ *
+ * @module services/agent/boot-pressure
+ */
+import { BOOT_PRESSURE_CONSTANTS } from '../../constants.js';
+import { LoggerService } from '../core/logger.service.js';
+import { ResourceModeService } from './resource-mode.service.js';
+/**
+ * Positive integer from an env var, or the fallback.
+ *
+ * @param raw - Env value
+ * @param fallback - Default
+ * @returns The parsed value or the fallback
+ */
+function envInt(raw, fallback) {
+    const n = Number.parseInt(raw ?? '', 10);
+    return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+/**
+ * Thresholds from the constants block, with env overrides.
+ *
+ * @param env - Environment (tests inject one)
+ * @returns Effective thresholds
+ */
+export function bootPressureThresholds(env = process.env) {
+    return {
+        swapUsedPercent: envInt(env[BOOT_PRESSURE_CONSTANTS.ENV_SWAP_PERCENT], BOOT_PRESSURE_CONSTANTS.SWAP_USED_PERCENT),
+        minFreeMB: envInt(env[BOOT_PRESSURE_CONSTANTS.ENV_MIN_FREE_MB], BOOT_PRESSURE_CONSTANTS.MIN_FREE_MB),
+        launchSpacingMs: envInt(env[BOOT_PRESSURE_CONSTANTS.ENV_LAUNCH_SPACING_MS], BOOT_PRESSURE_CONSTANTS.MIN_LAUNCH_SPACING_MS),
+    };
+}
+/**
+ * Why the snapshot is tight for the boot, or null when it is calm. Tight:
+ * pressure mode, swap at or above the threshold, or free memory under the
+ * floor. An unreadable snapshot (null) is calm: a boot must not stall on a
+ * reading that failed.
+ *
+ * @param snapshot - Reading, or null when unavailable
+ * @param thresholds - Effective thresholds
+ * @returns One-line reason, or null
+ */
+export function bootPressureReason(snapshot, thresholds = bootPressureThresholds()) {
+    if (!snapshot)
+        return null;
+    const why = [];
+    if (snapshot.pressureMode)
+        why.push('pressure mode');
+    if (snapshot.swapUsedPercent >= thresholds.swapUsedPercent)
+        why.push(`swap ${snapshot.swapUsedPercent}% >= ${thresholds.swapUsedPercent}%`);
+    if (snapshot.freeMB < thresholds.minFreeMB)
+        why.push(`free ${snapshot.freeMB}MB < ${thresholds.minFreeMB}MB`);
+    if (why.length === 0)
+        return null;
+    return `${why.join(', ')} (used=${snapshot.usedPercent}% free=${snapshot.freeMB}MB swap=${snapshot.swapUsedPercent}% osPressure=${snapshot.osPressure})`;
+}
+/**
+ * The live snapshot: ResourceModeService's memory reading and mode. Null when
+ * the reading throws or memory is unknown (totalMB 0).
+ *
+ * @returns Snapshot, or null
+ */
+export function readBootPressureSnapshot() {
+    try {
+        const rm = ResourceModeService.getInstance();
+        const mem = rm.memoryStats();
+        if (!mem.totalMB)
+            return null;
+        return {
+            usedPercent: mem.usedPercent,
+            freeMB: mem.freeMB,
+            swapUsedPercent: mem.swapUsedPercent ?? 0,
+            osPressure: mem.pressureElevated === true,
+            pressureMode: rm.getMode() === 'pressure',
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Remembers when the last boot launch began, so the restore queue and the
+ * deferred-start retry keep the launch spacing between them.
+ */
+export class BootLaunchPacer {
+    now;
+    lastAt = null;
+    constructor(now = Date.now) {
+        this.now = now;
+    }
+    /** A launch began now. */
+    note() {
+        this.lastAt = this.now();
+    }
+    /** Milliseconds since the last launch began (Infinity when none yet). */
+    msSinceLast() {
+        return this.lastAt === null ? Infinity : this.now() - this.lastAt;
+    }
+}
+let pacer = null;
+/** The process-wide pacer. */
+export function getBootLaunchPacer() {
+    return (pacer ??= new BootLaunchPacer());
+}
+/** Test hook: drop the pacer. */
+export function resetBootLaunchPacer() {
+    pacer = null;
+}
+/**
+ * Why a boot launch outside the restore queue (the deferred-start retry) must
+ * wait this round: memory is tight (only within `holdMaxMs` of the boot), the
+ * restore queue is still launching, or another boot launch began less than
+ * the launch spacing ago. Null = go.
+ *
+ * @param deps - Snapshot, restore state, pacer, boot time
+ * @returns Reason to hold, or null
+ */
+export function bootLaunchHoldReason(deps) {
+    const now = deps.now ?? Date.now;
+    const thresholds = deps.thresholds ?? bootPressureThresholds();
+    const holdMaxMs = deps.holdMaxMs ?? BOOT_PRESSURE_CONSTANTS.RETRY_HOLD_MAX_MS;
+    if (now() - deps.bootAt < holdMaxMs) {
+        const tight = bootPressureReason(deps.snapshot(), thresholds);
+        if (tight)
+            return `memory tight: ${tight}`;
+    }
+    if (deps.restoreRunning())
+        return 'boot restore queue still launching';
+    const since = deps.pacer.msSinceLast();
+    if (since < thresholds.launchSpacingMs)
+        return `a boot launch began ${Math.round(since / 1000)} s ago`;
+    return null;
+}
+const defaultSleep = (ms) => new Promise((resolve) => { const t = setTimeout(resolve, ms); t.unref?.(); });
+/**
+ * Hold a boot maintenance pass until the restore queue is drained and the
+ * machine is not tight, polling every `pollMs`. After `maxWaitMs` it gives
+ * up and returns `reduced: true` so the pass runs with a smaller batch.
+ *
+ * @param deps - Restore state, snapshot, clock
+ * @returns How long it waited and whether the pass should run reduced
+ */
+export async function waitForBootQuiet(deps) {
+    const snapshot = deps.snapshot ?? readBootPressureSnapshot;
+    const sleep = deps.sleep ?? defaultSleep;
+    const now = deps.now ?? Date.now;
+    const pollMs = deps.pollMs ?? BOOT_PRESSURE_CONSTANTS.MAINTENANCE_POLL_MS;
+    const maxWaitMs = deps.maxWaitMs ?? BOOT_PRESSURE_CONSTANTS.MAINTENANCE_WAIT_MAX_MS;
+    const started = now();
+    for (;;) {
+        const tight = bootPressureReason(snapshot(), deps.thresholds);
+        const reason = !deps.restoreDone() ? `restore queue still running${tight ? `; ${tight}` : ''}` : tight;
+        if (!reason)
+            return { waitedMs: now() - started, reduced: false };
+        if (now() - started >= maxWaitMs)
+            return { waitedMs: now() - started, reduced: true, reason };
+        await sleep(pollMs);
+    }
+}
+/**
+ * Names as a sentence fragment: "Ella", "Ella and Pia", "Ella, Pia and Owen".
+ *
+ * @param names - Agent names
+ * @returns Joined names
+ */
+export function joinNames(names) {
+    if (names.length <= 1)
+        return names[0] ?? '';
+    return `${names.slice(0, -1).join(', ')} and ${names[names.length - 1]}`;
+}
+/**
+ * One line to the owner, once, if agents the boot held back for memory are
+ * still down `afterMs` later. Nothing is sent when they came back in time.
+ * The line names the agents and says they will return; no mechanics.
+ *
+ * @param deps - Held-back check and notifier
+ * @returns Resolves when the check ran (tests); callers do not await it
+ */
+export function scheduleHeldBackNotice(deps) {
+    const logger = LoggerService.getInstance().createComponentLogger('BootPressure');
+    const afterMs = deps.afterMs ?? BOOT_PRESSURE_CONSTANTS.OWNER_NOTE_AFTER_MS;
+    const setTimer = deps.setTimer ?? ((fn, ms) => { const t = setTimeout(fn, ms); t.unref?.(); });
+    return new Promise((resolve) => {
+        setTimer(() => {
+            void (async () => {
+                try {
+                    const held = await deps.stillHeld();
+                    if (held.length === 0)
+                        return;
+                    const line = `Memory is tight on this machine, so ${joinNames(held)} ${held.length === 1 ? 'has' : 'have'} not been brought back yet; ${held.length === 1 ? 'it' : 'they'} will start as soon as it eases.`;
+                    const sent = await deps.notify(line);
+                    logger.info('Told the owner about boot restores held back for memory', { held, sent });
+                }
+                catch (err) {
+                    logger.warn('Could not tell the owner about held-back boot restores', { error: err instanceof Error ? err.message : String(err) });
+                }
+                finally {
+                    resolve();
+                }
+            })();
+        }, afterMs);
+    });
+}
+//# sourceMappingURL=boot-pressure.js.map

package/dist/backend/backend/src/services/agent/boot-pressure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"boot-pressure.js","sourceRoot":"","sources":["../../../../../../backend/src/services/agent/boot-pressure.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAmBjE;;;;;;GAMG;AACH,SAAS,MAAM,CAAC,GAAuB,EAAE,QAAgB;IACxD,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACzC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AACnD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC1E,OAAO;QACN,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,gBAAgB,CAAC,EAAE,uBAAuB,CAAC,iBAAiB,CAAC;QACjH,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,eAAe,CAAC,EAAE,uBAAuB,CAAC,WAAW,CAAC;QACpG,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,qBAAqB,CAAC,EAAE,uBAAuB,CAAC,qBAAqB,CAAC;KAC1H,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CACjC,QAAqC,EACrC,aAAqC,sBAAsB,EAAE;IAE7D,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,QAAQ,CAAC,YAAY;QAAE,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACrD,IAAI,QAAQ,CAAC,eAAe,IAAI,UAAU,CAAC,eAAe;QAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,QAAQ,CAAC,eAAe,QAAQ,UAAU,CAAC,eAAe,GAAG,CAAC,CAAC;IAC5I,IAAI,QAAQ,CAAC,MAAM,GAAG,UAAU,CAAC,SAAS;QAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,QAAQ,CAAC,MAAM,QAAQ,UAAU,CAAC,SAAS,IAAI,CAAC,CAAC;IAC9G,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,WAAW,UAAU,QAAQ,CAAC,MAAM,WAAW,QAAQ,CAAC,eAAe,gBAAgB,QAAQ,CAAC,UAAU,GAAG,CAAC;AAC1J,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,wBAAwB;IACvC,IAAI,CAAC;QACJ,MAAM,EAAE,GAAG,mBAAmB,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC9B,OAAO;YACN,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,eAAe,EAAE,GAAG,CAAC,eAAe,IAAI,CAAC;YACzC,UAAU,EAAE,GAAG,CAAC,gBAAgB,KAAK,IAAI;YACzC,YAAY,EAAE,EAAE,CAAC,OAAO,EAAE,KAAK,UAAU;SACzC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAC;IACb,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,eAAe;IAGE;IAFrB,MAAM,GAAkB,IAAI,CAAC;IAErC,YAA6B,MAAoB,IAAI,CAAC,GAAG;QAA5B,QAAG,GAAH,GAAG,CAAyB;IAAG,CAAC;IAE7D,0BAA0B;IAC1B,IAAI;QACH,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC1B,CAAC;IAED,yEAAyE;IACzE,WAAW;QACV,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;IACnE,CAAC;CACD;AAED,IAAI,KAAK,GAA2B,IAAI,CAAC;AAEzC,8BAA8B;AAC9B,MAAM,UAAU,kBAAkB;IACjC,OAAO,CAAC,KAAK,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;AAC1C,CAAC;AAED,iCAAiC;AACjC,MAAM,UAAU,oBAAoB;IACnC,KAAK,GAAG,IAAI,CAAC;AACd,CAAC;AAeD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAwB;IAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACjC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,sBAAsB,EAAE,CAAC;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,uBAAuB,CAAC,iBAAiB,CAAC;IAC9E,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,UAAU,CAAC,CAAC;QAC9D,IAAI,KAAK;YAAE,OAAO,iBAAiB,KAAK,EAAE,CAAC;IAC5C,CAAC;IACD,IAAI,IAAI,CAAC,cAAc,EAAE;QAAE,OAAO,oCAAoC,CAAC;IACvE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,KAAK,GAAG,UAAU,CAAC,eAAe;QAAE,OAAO,uBAAuB,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;IACvG,OAAO,IAAI,CAAC;AACb,CAAC;AAwBD,MAAM,YAAY,GAAG,CAAC,EAAU,EAAiB,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,GAAG,MAAM,CAAC,GAAG,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAExI;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAuB;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,wBAAwB,CAAC;IAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,YAAY,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACjC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,uBAAuB,CAAC,mBAAmB,CAAC;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,uBAAuB,CAAC,uBAAuB,CAAC;IACpF,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC;IACtB,SAAS,CAAC;QACT,MAAM,KAAK,GAAG,kBAAkB,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,8BAA8B,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QACvG,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAClE,IAAI,GAAG,EAAE,GAAG,OAAO,IAAI,SAAS;YAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC9F,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,KAAwB;IACjD,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7C,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;AAC1E,CAAC;AAYD;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAwB;IAC9D,MAAM,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,qBAAqB,CAAC,cAAc,CAAC,CAAC;IACjF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,uBAAuB,CAAC,mBAAmB,CAAC;IAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,GAAG,UAAU,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/F,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QACpC,QAAQ,CAAC,GAAG,EAAE;YACb,KAAK,CAAC,KAAK,IAAI,EAAE;gBAChB,IAAI,CAAC;oBACJ,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;oBACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;wBAAE,OAAO;oBAC9B,MAAM,IAAI,GAAG,uCAAuC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,+BAA+B,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,kCAAkC,CAAC;oBAC5M,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,yDAAyD,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxF,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,MAAM,CAAC,IAAI,CAAC,wDAAwD,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACpI,CAAC;wBAAS,CAAC;oBACV,OAAO,EAAE,CAAC;gBACX,CAAC;YACF,CAAC,CAAC,EAAE,CAAC;QACN,CAAC,EAAE,OAAO,CAAC,CAAC;IACb,CAAC,CAAC,CAAC;AACJ,CAAC"}

package/dist/backend/backend/src/services/agent/restore-queue.d.ts

@@ -6,6 +6,7 @@
  *
  * @module services/agent/restore-queue
  */
+import { type BootPressureSnapshot, type BootPressureThresholds } from './boot-pressure.js';
 /** Defaults for the restore queue. */
 export declare const RESTORE_QUEUE_CONSTANTS: {
     readonly DEFAULT_CONCURRENCY: 1;
@@ -36,6 +37,28 @@ export interface RestoreEntry {
     }>;
     /** Called right after the agent registered */
     onReady?: () => Promise<void>;
+    /**
+     * Launched even while memory is tight (crewly#1353): the owner waits on it
+     * (owner message, owed promise, owner thread) or it is the orchestrator.
+     * Everything else is deferred to the retry when the machine is tight.
+     */
+    essential?: boolean;
+    /** The queue handed the agent to the deferred-start retry instead of launching it (tight memory) */
+    onDeferred?: (snapshot: BootPressureSnapshot | null, reason: string) => void;
+}
+/** Memory awareness and pacing of the queue (tests inject fakes). */
+export interface RestoreQueueOptions {
+    /** Current memory reading (default: the live one; null = unknown, treated as calm) */
+    snapshot?: () => BootPressureSnapshot | null;
+    thresholds?: BootPressureThresholds;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+    /** A launch began (default: the process-wide pacer) */
+    onLaunch?: () => void;
+    settleMs?: number;
+    pollMs?: number;
+    holdMaxMs?: number;
+    pressureConcurrency?: number;
 }
 /** Progress counters exposed in /health. */
 export interface RestoreQueueStats {
@@ -45,6 +68,8 @@ export interface RestoreQueueStats {
     ready: number;
     failed: number;
     timedOut: number;
+    /** Handed to the deferred-start retry because memory was tight */
+    deferred: number;
     order: string[];
 }
 /**
@@ -71,7 +96,18 @@ export declare class RestoreQueue {
     private running;
     private done;
     private readonly logger;
-    constructor(concurrency?: number, timeoutMs?: number);
+    private readonly snapshot;
+    private readonly thresholds;
+    private readonly now;
+    private readonly sleep;
+    private readonly onLaunch;
+    private readonly settleMs;
+    private readonly pollMs;
+    private readonly holdMaxMs;
+    private readonly pressureConcurrency;
+    /** When the last launch settled (registered, failed or timed out); spacing counts from here */
+    private lastSettledAt;
+    constructor(concurrency?: number, timeoutMs?: number, opts?: RestoreQueueOptions);
     /**
      * Plan and run the restore. Resolves when every entry is ready, failed or
      * out of retries. Returns the same promise while already running.
@@ -96,6 +132,19 @@ export declare class RestoreQueue {
     /** Progress for /health. */
     stats(): RestoreQueueStats;
     private worker;
+    /**
+     * Memory is tight and nobody waits on this agent: hand it to the
+     * deferred-start retry (it is owed something, so it is not dropped) and
+     * log the reading it was deferred on.
+     */
+    private defer;
+    /**
+     * Wait before a launch: always the launch spacing since the previous one
+     * settled (RSS growth is visible before the next); under pressure also the
+     * settle time, the pressure concurrency, and the snapshot clearing (at most
+     * `holdMaxMs`, then the owner-waiting agent launches anyway).
+     */
+    private paceBeforeLaunch;
     private runSlot;
 }
 /** The process-wide restore queue. */

package/dist/backend/backend/src/services/agent/restore-queue.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"restore-queue.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/agent/restore-queue.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,sCAAsC;AACtC,eAAO,MAAM,uBAAuB;;;IAGnC,sEAAsE;;IAEtE,6EAA6E;;CAEpE,CAAC;AAEX,4CAA4C;AAC5C,eAAO,MAAM,YAAY;;;;;CAA4D,CAAC;AAEtF,4BAA4B;AAC5B,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wFAAwF;IACxF,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACxD,8CAA8C;IAC9C,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAED,4CAA4C;AAC5C,MAAM,WAAW,iBAAiB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CAChB;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,GAAG,GAAE,MAAM,GAAG,SAAkD,GAAG,MAAM,CAIlH;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,YAAY,EAAE,CAQpF;AAID,wBAAwB;AACxB,qBAAa,YAAY;IASvB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAT3B,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,QAAQ,CAAoD;IACpE,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,IAAI,CAAoC;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;gBAGvB,WAAW,GAAE,MAAoC,EACjD,SAAS,GAAE,MAAoG;IAKjI;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBtD,yEAAyE;IACzE,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIhC,kFAAkF;IAClF,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIlC;;;;;;;OAOG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,GAAG,OAAO;IAepD,4BAA4B;IAC5B,KAAK,IAAI,iBAAiB;YAIZ,MAAM;YAQN,OAAO;CA2CrB;AAID,sCAAsC;AACtC,wBAAgB,eAAe,IAAI,YAAY,CAE9C;AAED,qCAAqC;AACrC,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}
+{"version":3,"file":"restore-queue.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/agent/restore-queue.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,EAKN,KAAK,oBAAoB,EACzB,KAAK,sBAAsB,EAC3B,MAAM,oBAAoB,CAAC;AAE5B,sCAAsC;AACtC,eAAO,MAAM,uBAAuB;;;IAGnC,sEAAsE;;IAEtE,6EAA6E;;CAEpE,CAAC;AAEX,4CAA4C;AAC5C,eAAO,MAAM,YAAY;;;;;CAA4D,CAAC;AAEtF,4BAA4B;AAC5B,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wFAAwF;IACxF,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IACxD,8CAA8C;IAC9C,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,oGAAoG;IACpG,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7E;AAED,qEAAqE;AACrE,MAAM,WAAW,mBAAmB;IACnC,sFAAsF;IACtF,QAAQ,CAAC,EAAE,MAAM,oBAAoB,GAAG,IAAI,CAAC;IAC7C,UAAU,CAAC,EAAE,sBAAsB,CAAC;IACpC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,uDAAuD;IACvD,QAAQ,CAAC,EAAE,MAAM,IAAI,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,4CAA4C;AAC5C,MAAM,WAAW,iBAAiB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CAChB;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,GAAG,GAAE,MAAM,GAAG,SAAkD,GAAG,MAAM,CAIlH;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,YAAY,EAAE,CAQpF;AAID,wBAAwB;AACxB,qBAAa,YAAY;IAoBvB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,SAAS;IApB3B,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,QAAQ,CAAiE;IACjF,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,IAAI,CAAoC;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoC;IAC7D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;IACpD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgC;IACtD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAa;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,+FAA+F;IAC/F,OAAO,CAAC,aAAa,CAAuB;gBAG1B,WAAW,GAAE,MAAoC,EACjD,SAAS,GAAE,MAAoG,EAChI,IAAI,GAAE,mBAAwB;IAc/B;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,EAAE,SAAS,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBtD,yEAAyE;IACzE,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIhC,kFAAkF;IAClF,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAIlC;;;;;;;OAOG;IACH,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,GAAG,OAAO;IAepD,4BAA4B;IAC5B,KAAK,IAAI,iBAAiB;YAIZ,MAAM;IAsBpB;;;;OAIG;IACH,OAAO,CAAC,KAAK;IAYb;;;;;OAKG;YACW,gBAAgB;YA6BhB,OAAO;CA4CrB;AAID,sCAAsC;AACtC,wBAAgB,eAAe,IAAI,YAAY,CAE9C;AAED,qCAAqC;AACrC,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}

package/dist/backend/backend/src/services/agent/restore-queue.js

@@ -7,6 +7,8 @@
  * @module services/agent/restore-queue
  */
 import { LoggerService } from '../core/logger.service.js';
+import { BOOT_PRESSURE_CONSTANTS } from '../../constants.js';
+import { bootPressureReason, bootPressureThresholds, getBootLaunchPacer, readBootPressureSnapshot, } from './boot-pressure.js';
 /** Defaults for the restore queue. */
 export const RESTORE_QUEUE_CONSTANTS = {
     DEFAULT_CONCURRENCY: 1,
@@ -50,14 +52,34 @@ export class RestoreQueue {
     timeoutMs;
     pending = [];
     active = new Set();
-    counters = { started: 0, ready: 0, failed: 0, timedOut: 0 };
+    counters = { started: 0, ready: 0, failed: 0, timedOut: 0, deferred: 0 };
     running = false;
     done = Promise.resolve();
     logger;
-    constructor(concurrency = restoreConcurrencyFromEnv(), timeoutMs = Number(process.env.CREWLY_RESTORE_TIMEOUT_MS) || RESTORE_QUEUE_CONSTANTS.DEFAULT_TIMEOUT_MS) {
+    snapshot;
+    thresholds;
+    now;
+    sleep;
+    onLaunch;
+    settleMs;
+    pollMs;
+    holdMaxMs;
+    pressureConcurrency;
+    /** When the last launch settled (registered, failed or timed out); spacing counts from here */
+    lastSettledAt = null;
+    constructor(concurrency = restoreConcurrencyFromEnv(), timeoutMs = Number(process.env.CREWLY_RESTORE_TIMEOUT_MS) || RESTORE_QUEUE_CONSTANTS.DEFAULT_TIMEOUT_MS, opts = {}) {
         this.concurrency = concurrency;
         this.timeoutMs = timeoutMs;
         this.logger = LoggerService.getInstance().createComponentLogger('RestoreQueue');
+        this.snapshot = opts.snapshot ?? readBootPressureSnapshot;
+        this.thresholds = opts.thresholds ?? bootPressureThresholds();
+        this.now = opts.now ?? Date.now;
+        this.sleep = opts.sleep ?? ((ms) => new Promise((resolve) => { const t = setTimeout(resolve, ms); t.unref?.(); }));
+        this.onLaunch = opts.onLaunch ?? (() => getBootLaunchPacer().note());
+        this.settleMs = opts.settleMs ?? BOOT_PRESSURE_CONSTANTS.SETTLE_AFTER_READY_MS;
+        this.pollMs = opts.pollMs ?? BOOT_PRESSURE_CONSTANTS.PRESSURE_POLL_MS;
+        this.holdMaxMs = opts.holdMaxMs ?? BOOT_PRESSURE_CONSTANTS.PRESSURE_HOLD_MAX_MS;
+        this.pressureConcurrency = opts.pressureConcurrency ?? BOOT_PRESSURE_CONSTANTS.PRESSURE_MAX_CONCURRENCY;
     }
     /**
      * Plan and run the restore. Resolves when every entry is ready, failed or
@@ -70,11 +92,13 @@ export class RestoreQueue {
         if (this.running)
             return this.done;
         this.pending = orderRestoreEntries(entries).map((entry) => ({ entry, attempt: 0, promoted: false }));
-        this.counters = { started: 0, ready: 0, failed: 0, timedOut: 0 };
+        this.counters = { started: 0, ready: 0, failed: 0, timedOut: 0, deferred: 0 };
         this.running = true;
+        this.lastSettledAt = null;
         this.logger.info('Restore plan', {
             concurrency: this.concurrency,
-            order: this.pending.map((s, i) => `${i + 1}. ${s.entry.name} (${s.entry.reason})`),
+            launchSpacingMs: this.thresholds.launchSpacingMs,
+            order: this.pending.map((s, i) => `${i + 1}. ${s.entry.name} (${s.entry.reason}${s.entry.essential ? '; launched even under memory pressure' : ''})`),
         });
         const workers = Array.from({ length: this.concurrency }, () => this.worker());
         this.done = Promise.all(workers).then(() => {
@@ -124,13 +148,84 @@ export class RestoreQueue {
             const slot = this.pending.shift();
             if (!slot)
                 return;
-            await this.runSlot(slot);
+            const snap = this.snapshot();
+            const tight = bootPressureReason(snap, this.thresholds);
+            if (tight && !slot.entry.essential) {
+                this.defer(slot, snap, tight);
+                continue;
+            }
+            // In flight from here (isRestoring, the pressure concurrency), pacing included.
+            this.active.add(slot.entry.name);
+            try {
+                await this.paceBeforeLaunch(slot, tight);
+                await this.runSlot(slot);
+            }
+            finally {
+                this.active.delete(slot.entry.name);
+                this.lastSettledAt = this.now();
+            }
+        }
+    }
+    /**
+     * Memory is tight and nobody waits on this agent: hand it to the
+     * deferred-start retry (it is owed something, so it is not dropped) and
+     * log the reading it was deferred on.
+     */
+    defer(slot, snapshot, reason) {
+        this.counters.deferred++;
+        this.logger.warn('Deferring boot restore: memory is tight and the owner is not waiting on this agent', {
+            name: slot.entry.name, reason: slot.entry.reason, memory: reason, remaining: this.pending.length,
+        });
+        try {
+            slot.entry.onDeferred?.(snapshot, reason);
+        }
+        catch (err) {
+            this.logger.warn('Deferral hook failed', { name: slot.entry.name, error: err instanceof Error ? err.message : String(err) });
+        }
+    }
+    /**
+     * Wait before a launch: always the launch spacing since the previous one
+     * settled (RSS growth is visible before the next); under pressure also the
+     * settle time, the pressure concurrency, and the snapshot clearing (at most
+     * `holdMaxMs`, then the owner-waiting agent launches anyway).
+     */
+    async paceBeforeLaunch(slot, tight) {
+        const { name } = slot.entry;
+        const first = this.lastSettledAt === null;
+        if (!first) {
+            const gap = tight ? Math.max(this.thresholds.launchSpacingMs, this.settleMs) : this.thresholds.launchSpacingMs;
+            const wait = gap - (this.now() - this.lastSettledAt);
+            if (wait > 0)
+                await this.sleep(wait);
+        }
+        if (!tight)
+            return;
+        // The first launches under pressure are the owner's agents: nothing has
+        // settled yet, so they go now (within the pressure concurrency; this
+        // launch itself is already counted in `active`).
+        if (first && this.active.size <= this.pressureConcurrency)
+            return;
+        const heldSince = this.now();
+        for (let poll = 0;; poll++) {
+            const reason = bootPressureReason(this.snapshot(), this.thresholds);
+            const crowded = this.active.size > this.pressureConcurrency;
+            if (!reason && !crowded)
+                return;
+            if (this.now() - heldSince >= this.holdMaxMs) {
+                this.logger.warn('Launching an owner-waiting agent although memory is still tight: held long enough', { name, memory: reason, heldMs: this.now() - heldSince });
+                return;
+            }
+            if (poll % BOOT_PRESSURE_CONSTANTS.PRESSURE_HOLD_LOG_EVERY_POLLS === 0) {
+                this.logger.info('Holding an owner-waiting boot restore until memory eases', { name, memory: reason, crowded, heldMs: this.now() - heldSince });
+            }
+            await this.sleep(this.pollMs);
         }
     }
     async runSlot({ entry, attempt, promoted }) {
         const { name } = entry;
         this.active.add(name);
         this.counters.started++;
+        this.onLaunch();
         this.logger.info('Restoring agent', { name, attempt, reason: entry.reason, remaining: this.pending.length });
         let timer;
         const TIMEOUT = Symbol('timeout');