@memberjunction/server 6.2.0-edge.3 → 6.2.0-edge.4

package/dist/agentSessions/SessionJanitor.js

@@ -1,36 +1,81 @@
 import { BaseSingleton, ShutdownRegistry } from '@memberjunction/global';
 import { CompositeKey, RunView, LogError, LogStatus, } from '@memberjunction/core';
-import { GetHostInstanceID, GetHostNamePrefix } from './HostInstance.js';
+import { RecoverRealtimeRecordingFromSegments, ResolveRecordingStorageAccountID, } from '@memberjunction/ai-agents';
+import { GetCurrentHostInstance, GetHostInstanceID } from './HostInstance.js';
 import { SessionManager } from './SessionManager.js';
 /** Entity name for session rows (kept in sync with {@link SessionManager}). */
 const SESSION_ENTITY = 'MJ: AI Agent Sessions';
 /** Rows fetched per keyset page during a sweep. Small enough to bound memory on a large backlog. */
 const SWEEP_PAGE_SIZE = 200;
+/** Entity name for agent rows (the recording storage account is configured on the agent). */
+const AGENT_ENTITY = 'MJ: AI Agents';
+/** A session whose recording recovery has failed this many times is given up on for the life of the process. */
+const MAX_RECOVERY_ATTEMPTS = 8;
+/** Recovery attempts (calls into the recovery routine) per sweep; bounds the I/O one tick can trigger. */
+const MAX_RECOVERIES_PER_SWEEP = 5;
+/**
+ * Abandoned (timed-out, still running) recovery attempts allowed at once. Each keeps its socket and any
+ * shard buffers it had read (up to the 384 MiB recovery cap) until it settles, so recovery pauses at this
+ * many instead of adding one more every timeout while storage keeps hanging.
+ */
+const MAX_ABANDONED_RECOVERIES = 2;
+/** Backoff after the k-th failed recovery is `2^k * base`, capped at the max. */
+const RECOVERY_BACKOFF_BASE_MS = 5 * 60_000;
+const RECOVERY_BACKOFF_MAX_MS = 6 * 60 * 60_000;
+/**
+ * How long {@link SessionJanitor.Shutdown} waits for an in-flight recording recovery pass. A pass can
+ * spend minutes on large downloads/uploads, and graceful shutdown must not hang on it indefinitely.
+ */
+const RECOVERY_SHUTDOWN_WAIT_MS = 10_000;
 const DEFAULT_CONFIG = {
     closeThresholdMinutes: 15,
     sweepIntervalMs: 60_000,
+    recordingRecoveryGraceMinutes: 10,
+    recordingRecoveryLookbackHours: 168,
+    recordingRecoveryAttemptTimeoutMinutes: 10,
 };
+/** Every knob {@link SessionJanitor.Configure} accepts; each must be a finite number greater than zero. */
+const CONFIG_KEYS = [
+    'closeThresholdMinutes',
+    'sweepIntervalMs',
+    'recordingRecoveryGraceMinutes',
+    'recordingRecoveryLookbackHours',
+    'recordingRecoveryAttemptTimeoutMinutes',
+];
 /**
  * `BaseSingleton` background reconciler that keeps the durable `AIAgentSession` state from drifting
  * away from volatile process reality. A crash/redeploy vaporizes a host's in-memory sockets but
  * leaves its session rows reading `Active`/`Idle` forever; the janitor force-closes those orphans.
  *
- * Two sweeps, both writing through {@link SessionManager.CloseSession} so Record Changes captures
+ * Three close sweeps, all writing through {@link SessionManager.CloseSession} so Record Changes captures
  * each transition and channel rows are disconnected consistently:
  *
- * 1. **Own-host recovery ({@link RunStartupRecovery})** — run once at boot. Closes any `Active`/`Idle`
- *    session whose `HostInstanceID` belongs to a *previous* boot of *this* host (same hostname prefix,
- *    different `bootId`). Primary defense against the "Active forever" leak after a restart.
- * 2. **Global staleness sweep ({@link RunStalenessSweep})** — run periodically on every instance.
- *    Closes any `Active`/`Idle` session whose `LastActiveAt` is older than `closeThresholdMinutes`,
- *    regardless of host. Catches sessions whose owner died without a clean reboot (OOM, scaled-down pod).
+ * 1. **Own-instance recovery ({@link RunStartupRecovery})** — run once at boot. Closes any `Active`/`Idle`
+ *    session whose `HostInstanceID` belongs to an *earlier* boot of *this* MJAPI instance (same host and port,
+ *    different boot; see {@link HostInstanceIdentity}). Another MJAPI on the same host serves on another port,
+ *    so its live sessions are left alone. Primary defense against the "Active forever" leak after a restart.
+ * 2. **Global staleness sweep ({@link RunStalenessSweep})** — run on every periodic tick, on every
+ *    instance. Closes any `Active`/`Idle` session whose `LastActiveAt` is older than
+ *    `closeThresholdMinutes`, regardless of host. Catches sessions whose owner died without a clean
+ *    reboot (OOM, scaled-down pod).
+ * 3. **Max-duration sweep ({@link RunMaxDurationSweep})** — run on every periodic tick after the
+ *    staleness sweep. Closes any session past its stored hard deadline (the public voice cost cap).
+ *
+ * The close sweeps stamp `CloseReason = 'Janitor'`. A shutdown-time path — {@link RunShutdownDrain},
+ * invoked from {@link Shutdown} during the graceful ShutdownRegistry drain — closes this exact host
+ * instance's own live sessions with `CloseReason = 'Shutdown'`.
  *
- * Both sweeps stamp `CloseReason = 'Janitor'`. A third, shutdown-time path —
- * {@link RunShutdownDrain}, invoked from {@link Shutdown} during the graceful ShutdownRegistry
- * drain — closes this exact host instance's own live sessions with `CloseReason = 'Shutdown'`.
+ * Separately from closing, {@link RunRecordingRecoverySweep} runs after the close sweeps on every
+ * periodic tick, without blocking them, and rebuilds the recording of already-`Closed` sessions whose
+ * end-of-call upload never arrived (it closes nothing).
  *
- * Both sweeps are **idempotent and safe to run concurrently** on every instance: closing an
- * already-`Closed` session is a no-op, and the close path is last-writer-wins. Both page with keyset
+ * The close sweeps are **idempotent and safe to run concurrently** on every instance: closing an
+ * already-`Closed` session is a no-op, and the close path is last-writer-wins. Recording recovery is not
+ * strictly so: two instances can recover the same session at once (there is no claim column). Each
+ * attempt uploads its own file and defers to an already-stamped recording on a best-effort basis, so
+ * the cost is duplicate work and an extra file, not a lost recording. The extra file is an orphan when the
+ * second attempt's re-check sees the first stamp; when both re-checks pass first, both files are stamped in
+ * turn and both are linked to the session (the later stamp wins `RecordingFileID`). Every sweep pages with keyset
  * (`AfterKey`) pagination per the deep-pagination guide so a large backlog can't blow up memory.
  */
 export class SessionJanitor extends BaseSingleton {
@@ -39,10 +84,26 @@ export class SessionJanitor extends BaseSingleton {
         this._config = DEFAULT_CONFIG;
         this._sweepTimer = null;
         this._sweepRunning = false;
+        /**
+         * The in-flight recording recovery pass, or null. Separate from {@link _sweepRunning}: slow recovery
+         * must never block the close sweeps. Kept as a promise so {@link Shutdown} can wait for it.
+         */
+        this._recoveryPass = null;
+        /** Set by {@link Stop}, cleared by {@link Start}: no new recovery pass starts while stopped. */
+        this._stopped = false;
         this._registered = false;
         this._provider = null;
         this._systemUser = null;
         this.sessionManager = new SessionManager();
+        /** Recovery retry state keyed by lowercase session id. Process-local: a restart gets fresh attempts. */
+        this.recoveryBackoff = new Map();
+        /**
+         * Lowercase ids of sessions whose recovery attempt timed out but has not settled. Skipped by later
+         * sweeps so a second attempt never runs alongside the abandoned one; removed when it settles.
+         */
+        this.abandonedRecoveries = new Set();
+        /** True once the "recovery paused" error has been logged for the current pause; reset when it lifts. */
+        this._recoveryPauseLogged = false;
         /** Identifier surfaced in graceful-shutdown logs. */
         this.ShutdownName = 'SessionJanitor';
     }
@@ -50,35 +111,60 @@ export class SessionJanitor extends BaseSingleton {
     static get Instance() {
         return super.getInstance();
     }
-    /** Overrides default tuning. Safe to call before or after {@link Start}. */
+    /**
+     * Overrides default tuning. Safe to call before or after {@link Start}. Every value given must be a
+     * finite number greater than zero: a NaN or non-positive duration would otherwise build an invalid
+     * cutoff date and make every later sweep throw. All-or-nothing: on an invalid value it throws, naming
+     * the field and value, and applies nothing. `undefined` values are ignored.
+     */
     Configure(config) {
-        this._config = { ...this._config, ...config };
+        const next = { ...this._config };
+        for (const key of CONFIG_KEYS) {
+            const value = config[key];
+            if (value === undefined) {
+                continue;
+            }
+            if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
+                throw new Error(`SessionJanitor.Configure: ${key} must be a finite number greater than 0, got ${String(value)}`);
+            }
+            next[key] = value;
+        }
+        this._config = next;
     }
     /**
-     * Run own-host startup recovery once, then schedule the periodic staleness sweep. Idempotent:
-     * a second call does not stack a second timer. Captures the provider + system user so the timer
-     * callback can run without re-supplying them.
+     * Run own-host startup recovery once, then schedule the periodic tick (staleness and max-duration
+     * close sweeps, then recording recovery). Idempotent: a second call does not stack a second timer.
+     * Captures the provider + system user so the timer callback can run without re-supplying them, and
+     * re-enables recording recovery after a {@link Stop}. Throws, before doing any work, when
+     * `intervalMs` is invalid (see {@link Configure}).
      */
     async Start(provider, systemUser, intervalMs) {
-        this._provider = provider;
-        this._systemUser = systemUser;
         if (intervalMs != null) {
-            this._config = { ...this._config, sweepIntervalMs: intervalMs };
+            this.Configure({ sweepIntervalMs: intervalMs });
         }
+        this._provider = provider;
+        this._systemUser = systemUser;
+        this._stopped = false;
         this.ensureRegistered();
         await this.RunStartupRecovery(provider, systemUser);
         this.scheduleSweep();
     }
-    /** Stops the periodic sweep timer. Idempotent. Part of {@link IShutdownable}. */
+    /**
+     * Stops the periodic tick and keeps any new recording recovery pass from starting until the next
+     * {@link Start}, including one a tick already in progress would have started. A pass that is already
+     * running is not interrupted ({@link Shutdown} waits for it). Idempotent. Part of {@link IShutdownable}.
+     */
     Stop() {
+        this._stopped = true;
         if (this._sweepTimer) {
             clearInterval(this._sweepTimer);
             this._sweepTimer = null;
         }
     }
     /**
-     * {@link IShutdownable}: clear the timer, then drain this host's own live sessions so a
-     * graceful stop never strands `Active`/`Idle` rows for the next boot's janitor to mop up.
+     * {@link IShutdownable}: stop the timer, then drain this host's own live sessions so a
+     * graceful stop never strands `Active`/`Idle` rows for the next boot's janitor to mop up, then wait
+     * (at most {@link RECOVERY_SHUTDOWN_WAIT_MS}) for an in-flight recording recovery pass.
      * Drained sessions are stamped `CloseReason = 'Shutdown'` (vs. `'Janitor'` for crash orphans),
      * so the dashboards can tell a clean redeploy from a reconciled crash. Never throws; the drain
      * is skipped when {@link Start} was never called (no captured provider/user).
@@ -93,6 +179,7 @@ export class SessionJanitor extends BaseSingleton {
                 LogError(`SessionJanitor shutdown drain failed: ${err instanceof Error ? err.message : String(err)}`);
             }
         }
+        await this.waitForRecoveryPass();
     }
     /**
      * Close every `Active`/`Idle` session owned by **this exact host instance** (current boot),
@@ -110,18 +197,27 @@ export class SessionJanitor extends BaseSingleton {
         return closed;
     }
     /**
-     * Force-close any `Active`/`Idle` session left behind by a *previous boot of this host*
-     * (matching hostname prefix, differing `bootId`). Returns the number of sessions closed.
+     * Force-close any `Active`/`Idle` session left behind by an earlier boot of this MJAPI instance: its
+     * `HostInstanceID` has this instance's prefix (`hostname:port:`) and another boot. The sessions of another
+     * MJAPI on the same host carry another port and stay open. Returns the number of sessions closed.
+     *
+     * @param provider The metadata provider for the reads and writes.
+     * @param systemUser The user the closes run as.
+     * @param hostInstance The instance whose earlier boots to close. Defaults to this process's
+     *   ({@link GetCurrentHostInstance}).
      */
-    async RunStartupRecovery(provider, systemUser) {
-        const prefix = GetHostNamePrefix().replace(/'/g, "''");
-        const current = GetHostInstanceID().replace(/'/g, "''");
+    async RunStartupRecovery(provider, systemUser, hostInstance = GetCurrentHostInstance()) {
+        const prefix = hostInstance.GetInstancePrefix().replace(/'/g, "''");
+        const current = hostInstance.GetHostInstanceID().replace(/'/g, "''");
         const filter = `Status IN ('Active','Idle') ` +
             `AND HostInstanceID LIKE '${prefix}%' ` +
             `AND HostInstanceID <> '${current}'`;
-        const closed = await this.sweepAndClose(filter, provider, systemUser, 'Janitor');
+        // The LIKE narrows the read; the exact check decides, since LIKE reads a `_` in a host name as any character.
+        const isPriorBoot = (session) => hostInstance.IsPriorBoot(session.HostInstanceID);
+        const closed = await this.sweepAndClose(filter, provider, systemUser, 'Janitor', isPriorBoot);
         if (closed > 0) {
-            LogStatus(`[SessionJanitor] Startup recovery closed ${closed} orphaned session(s) from a prior boot of this host`);
+            LogStatus(`[SessionJanitor] Startup recovery closed ${closed} orphaned session(s) from an earlier boot of this instance ` +
+                `(${hostInstance.GetInstancePrefix()})`);
         }
         return closed;
     }
@@ -163,6 +259,57 @@ export class SessionJanitor extends BaseSingleton {
         }
         return closed;
     }
+    /**
+     * Rebuilds the recording of sessions whose end-of-call upload never arrived (browser died, tab
+     * closed). A session is a candidate when it is `Closed`, has `RecordingStartedAt` (consent is implied:
+     * recording only starts after the guest agreed), still has no `RecordingFileID`, and closed between
+     * the lookback window and the grace period (see {@link SessionJanitorConfig.recordingRecoveryGraceMinutes}).
+     *
+     * Work is bounded: candidates are handled sequentially, at most {@link MAX_RECOVERIES_PER_SWEEP}
+     * recovery attempts per call, and a failing session backs off exponentially (`min(2^k * 5 min, 6 h)`)
+     * and is abandoned after {@link MAX_RECOVERY_ATTEMPTS} failures with one `LogError`. Backoff entries of
+     * sessions that left the candidate set are pruned, but only after a pass that saw every candidate, so
+     * a capped pass never forgets sessions it did not reach.
+     *
+     * Never throws for a single bad session. Returns the number of recordings recovered.
+     */
+    async RunRecordingRecoverySweep(provider, systemUser) {
+        if (this.isRecoveryPausedForAbandonedAttempts()) {
+            return 0;
+        }
+        const nowMs = Date.now();
+        const graceCutoff = new Date(nowMs - this._config.recordingRecoveryGraceMinutes * 60_000).toISOString();
+        const lookbackCutoff = new Date(nowMs - this._config.recordingRecoveryLookbackHours * 3_600_000).toISOString();
+        const filter = `Status = 'Closed' AND RecordingStartedAt IS NOT NULL AND RecordingFileID IS NULL ` +
+            `AND ClosedAt < '${graceCutoff}' AND ClosedAt >= '${lookbackCutoff}'`;
+        const context = { provider, systemUser, storage: new Map() };
+        const seen = new Set();
+        let attempts = 0;
+        let recovered = 0;
+        const walk = await this.forEachSessionPage(filter, provider, systemUser, async (page) => {
+            for (const session of page) {
+                seen.add(session.ID.toLowerCase());
+                if (this.isRecoveryBackedOff(session.ID) || this.abandonedRecoveries.has(session.ID.toLowerCase())) {
+                    continue;
+                }
+                if (attempts >= MAX_RECOVERIES_PER_SWEEP || this.abandonedRecoveries.size >= MAX_ABANDONED_RECOVERIES) {
+                    return false; // per-sweep cap, or too many abandoned attempts still running: stop paging
+                }
+                const visit = await this.recoverSessionRecording(session, context);
+                attempts += visit.Attempted ? 1 : 0;
+                recovered += visit.Recovered ? 1 : 0;
+            }
+        });
+        // Only a walk that reached the end saw every candidate. A capped or failed-load walk has an
+        // incomplete `seen`; pruning on it would wipe backoff and give-up state for unseen sessions.
+        if (walk === 'complete') {
+            this.pruneRecoveryBackoff(seen);
+        }
+        if (recovered > 0) {
+            LogStatus(`[SessionJanitor] Recording recovery rebuilt ${recovered} recording(s)`);
+        }
+        return recovered;
+    }
     /** Parses the absolute deadline (ms) from a session's `Config_` JSON, or null when absent/malformed. */
     parseSessionDeadlineMs(configJson) {
         if (!configJson) {
@@ -188,7 +335,7 @@ export class SessionJanitor extends BaseSingleton {
             this._registered = true;
         }
     }
-    /** Schedule the periodic staleness sweep, exactly once. Timer is unref'd so it never blocks exit. */
+    /** Schedule the periodic sweep tick (close sweeps + recording recovery), exactly once. Timer is unref'd so it never blocks exit. */
     scheduleSweep() {
         if (this._sweepTimer) {
             return;
@@ -196,7 +343,16 @@ export class SessionJanitor extends BaseSingleton {
         this._sweepTimer = setInterval(() => void this.periodicSweep(), this._config.sweepIntervalMs);
         this._sweepTimer.unref?.();
     }
-    /** Timer tick: run the staleness sweep under the captured provider/user, guarding overlap. */
+    /**
+     * Timer tick: run the close sweeps (staleness, then max-duration) under the captured
+     * provider/user, guarding overlap, then kick off recording recovery.
+     *
+     * Recovery is started WITHOUT awaiting and has its own overlap guard: it can spend minutes on
+     * large downloads/uploads, and awaiting it under the close sweeps' guard would delay the
+     * max-duration cost cap and make every later tick return early. Each tick skips recovery while a pass
+     * is still running; because every session's attempt is bounded by
+     * `recordingRecoveryAttemptTimeoutMinutes`, a hung storage call delays later passes, never stops them.
+     */
     async periodicSweep() {
         if (this._sweepRunning || !this._provider || !this._systemUser) {
             return;
@@ -214,6 +370,233 @@ export class SessionJanitor extends BaseSingleton {
         finally {
             this._sweepRunning = false;
         }
+        this.startRecoveryPass(this._provider, this._systemUser);
+    }
+    /**
+     * Fire-and-forget recording recovery, skipped when the previous pass is still running or the janitor
+     * has been stopped (a tick that was mid-close-sweep when {@link Stop} ran must not start one).
+     */
+    startRecoveryPass(provider, systemUser) {
+        if (this._stopped || this._recoveryPass) {
+            return;
+        }
+        this._recoveryPass = this.RunRecordingRecoverySweep(provider, systemUser)
+            .then(() => undefined, err => {
+            LogError(`SessionJanitor recording recovery sweep failed: ${err instanceof Error ? err.message : String(err)}`);
+        })
+            .finally(() => {
+            this._recoveryPass = null;
+        });
+    }
+    /**
+     * Waits for the in-flight recovery pass, if any, for at most {@link RECOVERY_SHUTDOWN_WAIT_MS}; logs
+     * once when it gives up. Never throws: the pass promise already logs and absorbs its own errors.
+     */
+    async waitForRecoveryPass() {
+        const pass = this._recoveryPass;
+        if (!pass) {
+            return;
+        }
+        let timer;
+        const timedOut = new Promise(resolve => {
+            timer = setTimeout(() => resolve('timeout'), RECOVERY_SHUTDOWN_WAIT_MS);
+        });
+        try {
+            const outcome = await Promise.race([pass.then(() => 'finished'), timedOut]);
+            if (outcome === 'timeout') {
+                LogError(`SessionJanitor shutdown (host instance ${GetHostInstanceID()}): recording recovery pass still running after ` +
+                    `${RECOVERY_SHUTDOWN_WAIT_MS} ms; shutting down without it. A session it had not yet stamped is retried by a later sweep.`);
+            }
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    /**
+     * Recovers one candidate's recording and updates its backoff. Never throws: any exception is logged
+     * with the session id and counted as a failure.
+     */
+    async recoverSessionRecording(candidate, context) {
+        let attempted = false;
+        // The sweep filter requires RecordingStartedAt, so this only fires on drifted data; recovery would
+        // otherwise stamp a recording with no t0. Counted as a failure so it backs off rather than logging every tick.
+        const startedAt = candidate.RecordingStartedAt;
+        if (!startedAt) {
+            LogError(`[SessionJanitor] Recording recovery skipped for session ${candidate.ID}: RecordingStartedAt is empty`);
+            return this.recordRecoveryFailure(candidate.ID, false);
+        }
+        try {
+            // Re-read: the late upload may have landed since the candidate page was fetched.
+            const session = await context.provider.GetEntityObject(SESSION_ENTITY, context.systemUser);
+            if (!(await session.Load(candidate.ID))) {
+                LogError(`[SessionJanitor] Recording recovery could not reload session ${candidate.ID}`);
+                return this.recordRecoveryFailure(candidate.ID, false);
+            }
+            if (session.RecordingFileID) {
+                return { Attempted: false, Recovered: false };
+            }
+            const storage = await this.resolveRecoveryStorage(candidate.AgentID, context);
+            if (storage.Kind === 'NoAccount') {
+                return this.deferRecovery(candidate.ID, `no recording storage account for agent ${candidate.AgentID}`);
+            }
+            if (storage.Kind === 'AgentUnavailable') {
+                LogStatus(`[SessionJanitor] Recording recovery skipped for session ${candidate.ID}: agent ${candidate.AgentID} could not be loaded`);
+                return this.recordRecoveryFailure(candidate.ID, false);
+            }
+            const accountID = storage.AccountID;
+            attempted = true;
+            const result = await this.recoverWithinTimeout({
+                SessionID: candidate.ID,
+                StorageAccountID: accountID,
+                StartedAt: startedAt,
+                ContextUser: context.systemUser,
+                Provider: context.provider,
+            });
+            if (result === 'TimedOut') {
+                LogError(`[SessionJanitor] Recording recovery for session ${candidate.ID} did not finish within ` +
+                    `${this._config.recordingRecoveryAttemptTimeoutMinutes} minutes; abandoning this attempt (its storage call may ` +
+                    `still complete) and continuing the sweep. The session is not retried until that call settles.`);
+                return this.recordRecoveryFailure(candidate.ID, true);
+            }
+            switch (result.Outcome) {
+                case 'Recovered':
+                    this.recoveryBackoff.delete(candidate.ID.toLowerCase());
+                    LogStatus(`[SessionJanitor] Recovered recording for session ${candidate.ID} from ${result.SegmentCount} segment(s) (file ${result.FileID})`);
+                    return { Attempted: true, Recovered: true };
+                case 'Superseded':
+                    this.recoveryBackoff.delete(candidate.ID.toLowerCase());
+                    LogStatus(`[SessionJanitor] Session ${candidate.ID} was given a recording by another writer; recovery left it alone`);
+                    return { Attempted: true, Recovered: false };
+                case 'NoSegments':
+                    LogStatus(`[SessionJanitor] No recoverable segments for session ${candidate.ID} (agent ${candidate.AgentID}, storage account ${accountID})`);
+                    // Retried with backoff: a storage outage can also read as an empty folder (drivers that swallow
+                    // list errors), but a session that never has shards is not an error when checking stops.
+                    return this.recordRecoveryFailure(candidate.ID, true, 'NoSegments');
+                default:
+                    LogError(`[SessionJanitor] Recording recovery failed for session ${candidate.ID}: ${result.ErrorMessage ?? 'unknown error'}`);
+                    return this.recordRecoveryFailure(candidate.ID, true);
+            }
+        }
+        catch (err) {
+            LogError(`[SessionJanitor] Recording recovery threw for session ${candidate.ID}: ${err instanceof Error ? err.message : String(err)}`);
+            return this.recordRecoveryFailure(candidate.ID, attempted);
+        }
+    }
+    /**
+     * Runs one session's recovery, giving up on waiting after `recordingRecoveryAttemptTimeoutMinutes`.
+     * A call cannot be cancelled, so a timed-out one keeps running: the session is marked abandoned until
+     * it settles (later sweeps skip it), and its late outcome is logged then.
+     */
+    async recoverWithinTimeout(input) {
+        const key = input.SessionID.toLowerCase();
+        const attempt = RecoverRealtimeRecordingFromSegments(input);
+        let timer;
+        const timedOut = new Promise(resolve => {
+            timer = setTimeout(() => resolve('TimedOut'), this._config.recordingRecoveryAttemptTimeoutMinutes * 60_000);
+            timer.unref?.();
+        });
+        try {
+            const outcome = await Promise.race([attempt, timedOut]);
+            if (outcome === 'TimedOut') {
+                this.abandonedRecoveries.add(key);
+                attempt.then(late => LogStatus(`[SessionJanitor] Abandoned recording recovery for session ${input.SessionID} finished late: ${late.Outcome}`), err => LogError(`[SessionJanitor] Abandoned recording recovery for session ${input.SessionID} failed late: ${err instanceof Error ? err.message : String(err)}`)).finally(() => this.abandonedRecoveries.delete(key));
+            }
+            return outcome;
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    /**
+     * True while {@link MAX_ABANDONED_RECOVERIES} timed-out attempts are still running: no new attempt
+     * starts until one settles. Logs one `LogError` per pause.
+     */
+    isRecoveryPausedForAbandonedAttempts() {
+        if (this.abandonedRecoveries.size < MAX_ABANDONED_RECOVERIES) {
+            this._recoveryPauseLogged = false;
+            return false;
+        }
+        if (!this._recoveryPauseLogged) {
+            LogError(`[SessionJanitor] Recording recovery paused: ${this.abandonedRecoveries.size} abandoned attempts are still waiting on storage ` +
+                `(sessions ${[...this.abandonedRecoveries].join(', ')}); no new attempt starts until one of them settles`);
+            this._recoveryPauseLogged = true;
+        }
+        return true;
+    }
+    /**
+     * Defers a session that cannot be recovered until configuration changes (its agent has no recording
+     * storage account). That is not a failed attempt, so it never counts toward the give-up: the session
+     * is re-checked every {@link RECOVERY_BACKOFF_MAX_MS} and logged only the first time.
+     */
+    deferRecovery(sessionID, reason) {
+        const key = sessionID.toLowerCase();
+        const state = this.recoveryBackoff.get(key);
+        if (!state?.Deferred) {
+            LogStatus(`[SessionJanitor] Recording recovery skipped for session ${sessionID}: ${reason}; re-checked every ${RECOVERY_BACKOFF_MAX_MS / 3_600_000} h`);
+        }
+        this.recoveryBackoff.set(key, { Failures: state?.Failures ?? 0, NextAttemptAt: Date.now() + RECOVERY_BACKOFF_MAX_MS, Deferred: true });
+        return { Attempted: false, Recovered: false };
+    }
+    /**
+     * Loads the agent carrying a session's recording storage config and resolves its storage account,
+     * memoized per agent for the sweep: the resolve may force a storage-engine reload, so it must not
+     * repeat for every session of the same agent.
+     */
+    async resolveRecoveryStorage(agentID, context) {
+        const key = agentID.toLowerCase();
+        const cached = context.storage.get(key);
+        if (cached) {
+            return cached;
+        }
+        const agent = await context.provider.GetEntityObject(AGENT_ENTITY, context.systemUser);
+        let storage;
+        if (!(await agent.Load(agentID))) {
+            LogError(`[SessionJanitor] Recording recovery could not load agent ${agentID}`);
+            storage = { Kind: 'AgentUnavailable' };
+        }
+        else {
+            const accountID = await ResolveRecordingStorageAccountID(agent, context.systemUser, context.provider);
+            storage = accountID ? { Kind: 'Resolved', AccountID: accountID } : { Kind: 'NoAccount' };
+        }
+        context.storage.set(key, storage);
+        return storage;
+    }
+    /** True while a session's backoff (or give-up) window has not elapsed. */
+    isRecoveryBackedOff(sessionID) {
+        const state = this.recoveryBackoff.get(sessionID.toLowerCase());
+        return state != null && Date.now() < state.NextAttemptAt;
+    }
+    /**
+     * Counts one more unsuccessful recovery for a session and schedules its next attempt
+     * (`min(2^k * 5 min, 6 h)`); at {@link MAX_RECOVERY_ATTEMPTS} it stops checking the session for this
+     * process with a single log line: a `LogError` when the last attempt failed, a `LogStatus` when it
+     * simply found no shards (nothing to recover is not an error).
+     */
+    recordRecoveryFailure(sessionID, attempted, kind = 'Failed') {
+        const key = sessionID.toLowerCase();
+        const failures = (this.recoveryBackoff.get(key)?.Failures ?? 0) + 1;
+        if (failures >= MAX_RECOVERY_ATTEMPTS) {
+            this.recoveryBackoff.set(key, { Failures: failures, NextAttemptAt: Infinity, Deferred: false });
+            if (kind === 'NoSegments') {
+                LogStatus(`[SessionJanitor] Session ${sessionID} had no recoverable segments in ${failures} checks; not checking it again until restart`);
+            }
+            else {
+                LogError(`[SessionJanitor] Giving up on recording recovery for session ${sessionID} after ${failures} failed attempts`);
+            }
+        }
+        else {
+            const waitMs = Math.min(2 ** failures * RECOVERY_BACKOFF_BASE_MS, RECOVERY_BACKOFF_MAX_MS);
+            this.recoveryBackoff.set(key, { Failures: failures, NextAttemptAt: Date.now() + waitMs, Deferred: false });
+        }
+        return { Attempted: attempted, Recovered: false };
+    }
+    /** Drops backoff state for sessions that were not in the latest complete candidate set. */
+    pruneRecoveryBackoff(candidateKeys) {
+        for (const key of this.recoveryBackoff.keys()) {
+            if (!candidateKeys.has(key)) {
+                this.recoveryBackoff.delete(key);
+            }
+        }
     }
     /**
      * Page through every session matching `filter` with keyset (`AfterKey`) pagination and close each
@@ -223,28 +606,11 @@ export class SessionJanitor extends BaseSingleton {
      */
     async sweepAndClose(filter, provider, systemUser, closeReason, shouldClose) {
         let closedCount = 0;
-        let afterKey;
-        // eslint-disable-next-line no-constant-condition
-        while (true) {
-            const fetched = await this.fetchPage(filter, afterKey, provider, systemUser);
-            if (fetched == null) {
-                break; // load failure already logged
-            }
-            if (fetched.length === 0) {
-                break;
-            }
-            // Advance the keyset BEFORE the optional JS predicate narrows the page (we must page by the
-            // SQL-matched set, not the post-filtered subset, or pagination would stall/skip).
-            const lastFetchedId = fetched[fetched.length - 1].ID;
-            const fetchedCount = fetched.length;
+        await this.forEachSessionPage(filter, provider, systemUser, async (fetched) => {
             // Optional in-JS narrowing (e.g. exact deadline check the SQL pre-filter can't express).
             const page = shouldClose ? fetched.filter(shouldClose) : fetched;
             if (page.length === 0) {
-                if (fetchedCount < SWEEP_PAGE_SIZE) {
-                    break;
-                }
-                afterKey = CompositeKey.FromID(lastFetchedId); // first-pk-ok: keyset AfterKey on SESSION_ENTITY = MJ: AI Agent Sessions, a core entity keyed by ID
-                continue;
+                return;
             }
             // Batch-load every channel for this whole page of sessions in ONE query, then hand each
             // session its own slice to CloseSession — avoids the N+1 channel read (one RunView per
@@ -256,12 +622,44 @@ export class SessionJanitor extends BaseSingleton {
                     closedCount++;
                 }
             }
+        });
+        return closedCount;
+    }
+    /**
+     * Keyset-page (`AfterKey`) through every session matching `filter`, handing each full fetched page
+     * to `visit`. Stops on a load failure (already logged), an empty page, a partial page, or when
+     * `visit` returns `false`.
+     *
+     * `visit` receives the page exactly as the SQL filter matched it. The keyset advances from the
+     * LAST FETCHED row, so any narrowing `visit` does in JS can never stall or skip pagination.
+     *
+     * Returns how the walk ended: `'complete'` (reached the end of the matching set), `'stopped'`
+     * (`visit` returned `false`) or `'failed'` (a page failed to load). Callers that reason about the
+     * WHOLE set (e.g. pruning state for absent rows) must act only on `'complete'`.
+     */
+    async forEachSessionPage(filter, provider, systemUser, visit) {
+        let afterKey;
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const fetched = await this.fetchPage(filter, afterKey, provider, systemUser);
+            if (fetched == null) {
+                return 'failed'; // load failure already logged
+            }
+            if (fetched.length === 0) {
+                return 'complete';
+            }
+            // Advance from the last FETCHED row (before `visit` narrows anything): we must page by the
+            // SQL-matched set, not a post-filtered subset, or pagination would stall/skip.
+            const lastFetchedId = fetched[fetched.length - 1].ID;
+            const fetchedCount = fetched.length;
+            if ((await visit(fetched)) === false) {
+                return 'stopped';
+            }
             if (fetchedCount < SWEEP_PAGE_SIZE) {
-                break; // partial page (by the SQL-matched set) => end of data
+                return 'complete'; // partial page (by the SQL-matched set) => end of data
             }
             afterKey = CompositeKey.FromID(lastFetchedId); // first-pk-ok: keyset AfterKey on SESSION_ENTITY = MJ: AI Agent Sessions, a core entity keyed by ID
         }
-        return closedCount;
     }
     /** Fetch one keyset page of matching sessions, or null on a load failure (logged). */
     async fetchPage(filter, afterKey, provider, systemUser) {