@memberjunction/server 6.2.0-edge.4 → 6.2.0-edge.6

package/dist/agentSessions/SessionJanitor.js

@@ -1,6 +1,6 @@
 import { BaseSingleton, ShutdownRegistry } from '@memberjunction/global';
 import { CompositeKey, RunView, LogError, LogStatus, } from '@memberjunction/core';
-import { RecoverRealtimeRecordingFromSegments, ResolveRecordingStorageAccountID, } from '@memberjunction/ai-agents';
+import { AssembleRealtimeRecordingFromLedger, RecoverRealtimeRecordingFromSegments, ResolveRecordingStorageAccountID, SqlRealtimeRecordingLedger, } from '@memberjunction/ai-agents';
 import { GetCurrentHostInstance, GetHostInstanceID } from './HostInstance.js';
 import { SessionManager } from './SessionManager.js';
 /** Entity name for session rows (kept in sync with {@link SessionManager}). */
@@ -11,8 +11,16 @@ const SWEEP_PAGE_SIZE = 200;
 const AGENT_ENTITY = 'MJ: AI Agents';
 /** A session whose recording recovery has failed this many times is given up on for the life of the process. */
 const MAX_RECOVERY_ATTEMPTS = 8;
-/** Recovery attempts (calls into the recovery routine) per sweep; bounds the I/O one tick can trigger. */
+/**
+ * Recovery attempts (calls into the recovery routine) per sweep; bounds the I/O one tick can trigger.
+ * The ledger sweep and the legacy segment sweep each get this many: a backlog in one must not starve
+ * the other, so a pass makes at most twice this many attempts.
+ */
 const MAX_RECOVERIES_PER_SWEEP = 5;
+/** The shard-ledger view (core schema); a session with rows in it is assembled by the ledger sweep. */
+const RECORDING_SHARD_VIEW = 'vwAIAgentSessionRecordingShards';
+/** Ledger shard rows each recovery pass deletes, at most, for sessions closed before the lookback window. */
+const MAX_PURGED_SHARDS_PER_PASS = 500;
 /**
  * Abandoned (timed-out, still running) recovery attempts allowed at once. Each keeps its socket and any
  * shard buffers it had read (up to the 384 MiB recovery cap) until it settles, so recovery pauses at this
@@ -42,6 +50,69 @@ const CONFIG_KEYS = [
     'recordingRecoveryLookbackHours',
     'recordingRecoveryAttemptTimeoutMinutes',
 ];
+/**
+ * Per-session retry state of one recovery sweep, keyed by lowercase session id (process-local: a restart
+ * gets fresh attempts). A failing session backs off exponentially (`min(2^k * 5 min, 6 h)`) and is given
+ * up on after {@link MAX_RECOVERY_ATTEMPTS} failures with one log line.
+ */
+class RecoveryBackoffTracker {
+    constructor() {
+        this.states = new Map();
+    }
+    /** True while a session's backoff (or give-up) window has not elapsed. */
+    IsBackedOff(sessionID) {
+        const state = this.states.get(sessionID.toLowerCase());
+        return state != null && Date.now() < state.NextAttemptAt;
+    }
+    /** Forgets a session's failures (it succeeded, or another writer took care of it). */
+    Clear(sessionID) {
+        this.states.delete(sessionID.toLowerCase());
+    }
+    /**
+     * Defers a session that cannot be recovered until configuration changes (its agent has no recording
+     * storage account). That is not a failed attempt, so it never counts toward the give-up: the session
+     * is re-checked every {@link RECOVERY_BACKOFF_MAX_MS} and logged only the first time.
+     */
+    Defer(sessionID, reason) {
+        const key = sessionID.toLowerCase();
+        const state = this.states.get(key);
+        if (!state?.Deferred) {
+            LogStatus(`[SessionJanitor] Recording recovery skipped for session ${sessionID}: ${reason}; re-checked every ${RECOVERY_BACKOFF_MAX_MS / 3_600_000} h`);
+        }
+        this.states.set(key, { Failures: state?.Failures ?? 0, NextAttemptAt: Date.now() + RECOVERY_BACKOFF_MAX_MS, Deferred: true });
+    }
+    /**
+     * Counts one more unsuccessful recovery for a session and schedules its next attempt
+     * (`min(2^k * 5 min, 6 h)`); at {@link MAX_RECOVERY_ATTEMPTS} it stops checking the session for this
+     * process with a single log line: a `LogError` when the last attempt failed, a `LogStatus` when it
+     * simply found no shards (nothing to recover is not an error).
+     */
+    RecordFailure(sessionID, kind) {
+        const key = sessionID.toLowerCase();
+        const failures = (this.states.get(key)?.Failures ?? 0) + 1;
+        if (failures >= MAX_RECOVERY_ATTEMPTS) {
+            this.states.set(key, { Failures: failures, NextAttemptAt: Infinity, Deferred: false });
+            if (kind === 'NoSegments') {
+                LogStatus(`[SessionJanitor] Session ${sessionID} had no recoverable segments in ${failures} checks; not checking it again until restart`);
+            }
+            else {
+                LogError(`[SessionJanitor] Giving up on recording recovery for session ${sessionID} after ${failures} failed attempts`);
+            }
+        }
+        else {
+            const waitMs = Math.min(2 ** failures * RECOVERY_BACKOFF_BASE_MS, RECOVERY_BACKOFF_MAX_MS);
+            this.states.set(key, { Failures: failures, NextAttemptAt: Date.now() + waitMs, Deferred: false });
+        }
+    }
+    /** Drops state for sessions that were not in the latest complete candidate set (lowercase ids). */
+    Prune(candidateKeys) {
+        for (const key of this.states.keys()) {
+            if (!candidateKeys.has(key)) {
+                this.states.delete(key);
+            }
+        }
+    }
+}
 /**
  * `BaseSingleton` background reconciler that keeps the durable `AIAgentSession` state from drifting
  * away from volatile process reality. A crash/redeploy vaporizes a host's in-memory sockets but
@@ -65,13 +136,17 @@ const CONFIG_KEYS = [
  * invoked from {@link Shutdown} during the graceful ShutdownRegistry drain — closes this exact host
  * instance's own live sessions with `CloseReason = 'Shutdown'`.
  *
- * Separately from closing, {@link RunRecordingRecoverySweep} runs after the close sweeps on every
- * periodic tick, without blocking them, and rebuilds the recording of already-`Closed` sessions whose
- * end-of-call upload never arrived (it closes nothing).
+ * Separately from closing, a recording recovery pass runs after the close sweeps on every periodic tick,
+ * without blocking them, and closes nothing. It purges shard-ledger rows past the lookback window, then
+ * {@link RunRecordingLedgerSweep} assembles the recordings of `Closed` sessions on the shard ledger that
+ * no terminal signal (or a dead assembler) left unassembled, then {@link RunRecordingRecoverySweep}
+ * rebuilds, from legacy storage segments, the recording of `Closed` sessions not on the ledger whose
+ * end-of-call upload never arrived.
  *
  * The close sweeps are **idempotent and safe to run concurrently** on every instance: closing an
- * already-`Closed` session is a no-op, and the close path is last-writer-wins. Recording recovery is not
- * strictly so: two instances can recover the same session at once (there is no claim column). Each
+ * already-`Closed` session is a no-op, and the close path is last-writer-wins. Ledger assembly is
+ * arbitrated by the ledger's atomic claim. Legacy recovery is not: two instances can recover the same
+ * session at once (there is no claim column for it). Each
  * attempt uploads its own file and defers to an already-stamped recording on a best-effort basis, so
  * the cost is duplicate work and an extra file, not a lost recording. The extra file is an orphan when the
  * second attempt's re-check sees the first stamp; when both re-checks pass first, both files are stamped in
@@ -95,8 +170,13 @@ export class SessionJanitor extends BaseSingleton {
         this._provider = null;
         this._systemUser = null;
         this.sessionManager = new SessionManager();
-        /** Recovery retry state keyed by lowercase session id. Process-local: a restart gets fresh attempts. */
-        this.recoveryBackoff = new Map();
+        /** Retry state of the legacy storage-segment recovery. */
+        this.recoveryBackoff = new RecoveryBackoffTracker();
+        /**
+         * Retry state of the ledger sweep, kept apart from {@link recoveryBackoff}: each sweep prunes its
+         * tracker by its own candidate set, so a shared one would forget the other sweep's failures.
+         */
+        this.ledgerBackoff = new RecoveryBackoffTracker();
         /**
          * Lowercase ids of sessions whose recovery attempt timed out but has not settled. Skipped by later
          * sweeps so a second attempt never runs alongside the abandoned one; removed when it settles.
@@ -136,7 +216,8 @@ export class SessionJanitor extends BaseSingleton {
      * close sweeps, then recording recovery). Idempotent: a second call does not stack a second timer.
      * Captures the provider + system user so the timer callback can run without re-supplying them, and
      * re-enables recording recovery after a {@link Stop}. Throws, before doing any work, when
-     * `intervalMs` is invalid (see {@link Configure}).
+     * `intervalMs` is invalid (see {@link Configure}). Takes the server's database provider because
+     * recording recovery calls the shard ledger's stored procedures.
      */
     async Start(provider, systemUser, intervalMs) {
         if (intervalMs != null) {
@@ -271,31 +352,96 @@ export class SessionJanitor extends BaseSingleton {
      * sessions that left the candidate set are pruned, but only after a pass that saw every candidate, so
      * a capped pass never forgets sessions it did not reach.
      *
+     * Sessions on the shard ledger (a `RecordingStatus`, or any ledger row) are left to
+     * {@link RunRecordingLedgerSweep}: their audio is in SQL, not in storage segments.
+     *
      * Never throws for a single bad session. Returns the number of recordings recovered.
      */
     async RunRecordingRecoverySweep(provider, systemUser) {
         if (this.isRecoveryPausedForAbandonedAttempts()) {
             return 0;
         }
-        const nowMs = Date.now();
-        const graceCutoff = new Date(nowMs - this._config.recordingRecoveryGraceMinutes * 60_000).toISOString();
-        const lookbackCutoff = new Date(nowMs - this._config.recordingRecoveryLookbackHours * 3_600_000).toISOString();
+        const { graceCutoff, lookbackCutoff } = this.recoveryWindow();
         const filter = `Status = 'Closed' AND RecordingStartedAt IS NOT NULL AND RecordingFileID IS NULL ` +
-            `AND ClosedAt < '${graceCutoff}' AND ClosedAt >= '${lookbackCutoff}'`;
+            `AND ClosedAt < '${graceCutoff.toISOString()}' AND ClosedAt >= '${lookbackCutoff.toISOString()}' ` +
+            `AND RecordingStatus IS NULL AND ID NOT IN (SELECT AgentSessionID FROM ${this.shardViewName(provider)})`;
         const context = { provider, systemUser, storage: new Map() };
+        const recovered = await this.sweepRecoveryCandidates(filter, provider, systemUser, this.recoveryBackoff, session => this.recoverSessionRecording(session, context));
+        if (recovered > 0) {
+            LogStatus(`[SessionJanitor] Recording recovery rebuilt ${recovered} recording(s)`);
+        }
+        return recovered;
+    }
+    /**
+     * Backstop for client-direct recordings on the shard ledger: assembles the recording of `Closed`
+     * sessions that closed between the lookback window and the grace period and are still
+     * - `Pending` (shards, or a terminal signal, with no assembly since: the browser died before its
+     *   terminal request, or the in-process assembly request was lost with its host),
+     * - `Assembling` with no write for `recordingRecoveryAttemptTimeoutMinutes` (the assembler died, or
+     *   it hit a transient error and left the claim for a re-claim), or
+     * - status-less while the ledger holds shards for them (see {@link restoreLostRecordingStatus}).
+     *
+     * Each candidate is claimed with the ledger's atomic, stale-aware claim (a lost claim is skipped: another
+     * assembler has it) and assembled with `ForcePartial`, so missing shards become silence. An attempt is
+     * bounded by `recordingRecoveryAttemptTimeoutMinutes` and shares the legacy sweep's abandoned-attempt
+     * pause; failures back off and are given up on exactly like legacy recovery, in their own tracker.
+     * At most {@link MAX_RECOVERIES_PER_SWEEP} assemblies per call. Never throws for a single bad session.
+     * Returns the number of recordings assembled (`Complete` or `Partial`).
+     */
+    async RunRecordingLedgerSweep(provider, systemUser) {
+        if (this.isRecoveryPausedForAbandonedAttempts()) {
+            return 0;
+        }
+        const { graceCutoff, lookbackCutoff } = this.recoveryWindow();
+        const staleCutoff = new Date(Date.now() - this._config.recordingRecoveryAttemptTimeoutMinutes * 60_000);
+        // Every Assembling session is fetched, not only stale ones, and a fresh one is skipped below: a
+        // session whose assembly left a Retry claim is fresh for a while, and dropping it from the walk
+        // would prune its backoff, so the give-up cap would never be reached.
+        const filter = `Status = 'Closed' AND ClosedAt < '${graceCutoff.toISOString()}' AND ClosedAt >= '${lookbackCutoff.toISOString()}' ` +
+            `AND (RecordingStatus IN ('Pending','Assembling') ` +
+            `OR (RecordingStatus IS NULL AND ID IN (SELECT AgentSessionID FROM ${this.shardViewName(provider)})))`;
+        const ledger = new SqlRealtimeRecordingLedger(provider, systemUser);
+        const assembled = await this.sweepRecoveryCandidates(filter, provider, systemUser, this.ledgerBackoff, session => this.assembleSessionFromLedger(session, ledger, staleCutoff, provider, systemUser), 
+        // The claim proc re-checks staleness atomically; this only spares a claim that would lose.
+        session => session.RecordingStatus === 'Assembling' && session.__mj_UpdatedAt.getTime() >= staleCutoff.getTime());
+        if (assembled > 0) {
+            LogStatus(`[SessionJanitor] Recording ledger sweep assembled ${assembled} recording(s)`);
+        }
+        return assembled;
+    }
+    /** The closed-at window recording recovery considers: older than the grace, newer than the lookback. */
+    recoveryWindow() {
+        const nowMs = Date.now();
+        return {
+            graceCutoff: new Date(nowMs - this._config.recordingRecoveryGraceMinutes * 60_000),
+            lookbackCutoff: new Date(nowMs - this._config.recordingRecoveryLookbackHours * 3_600_000),
+        };
+    }
+    /** The schema-qualified, dialect-quoted shard-ledger view, for filter sub-queries. */
+    shardViewName(provider) {
+        return provider.QuoteSchemaAndView(provider.MJCoreSchemaName, RECORDING_SHARD_VIEW);
+    }
+    /**
+     * Walks every session matching `filter` and runs `recover` on each one that is not backed off, not
+     * abandoned, and not `isBusy`, sequentially, stopping at {@link MAX_RECOVERIES_PER_SWEEP} attempts or
+     * {@link MAX_ABANDONED_RECOVERIES} abandoned attempts. Backoff entries of sessions that left the
+     * candidate set are pruned, but only after a walk that saw every candidate, so a capped walk never
+     * forgets sessions it did not reach. Returns the number recovered.
+     */
+    async sweepRecoveryCandidates(filter, provider, systemUser, backoff, recover, isBusy = () => false) {
         const seen = new Set();
         let attempts = 0;
         let recovered = 0;
         const walk = await this.forEachSessionPage(filter, provider, systemUser, async (page) => {
             for (const session of page) {
                 seen.add(session.ID.toLowerCase());
-                if (this.isRecoveryBackedOff(session.ID) || this.abandonedRecoveries.has(session.ID.toLowerCase())) {
+                if (backoff.IsBackedOff(session.ID) || this.abandonedRecoveries.has(session.ID.toLowerCase()) || isBusy(session)) {
                     continue;
                 }
                 if (attempts >= MAX_RECOVERIES_PER_SWEEP || this.abandonedRecoveries.size >= MAX_ABANDONED_RECOVERIES) {
                     return false; // per-sweep cap, or too many abandoned attempts still running: stop paging
                 }
-                const visit = await this.recoverSessionRecording(session, context);
+                const visit = await recover(session);
                 attempts += visit.Attempted ? 1 : 0;
                 recovered += visit.Recovered ? 1 : 0;
             }
@@ -303,13 +449,120 @@ export class SessionJanitor extends BaseSingleton {
         // Only a walk that reached the end saw every candidate. A capped or failed-load walk has an
         // incomplete `seen`; pruning on it would wipe backoff and give-up state for unseen sessions.
         if (walk === 'complete') {
-            this.pruneRecoveryBackoff(seen);
-        }
-        if (recovered > 0) {
-            LogStatus(`[SessionJanitor] Recording recovery rebuilt ${recovered} recording(s)`);
+            backoff.Prune(seen);
         }
         return recovered;
     }
+    /**
+     * Claims and assembles one ledger candidate and updates its backoff. Never throws: any exception is
+     * logged with the session id and counted as a failure.
+     */
+    async assembleSessionFromLedger(candidate, ledger, staleCutoff, provider, systemUser) {
+        const sessionID = candidate.ID;
+        let attempted = false;
+        try {
+            if (candidate.RecordingStatus == null) {
+                // The restore is a write; a failing one counts toward the per-sweep cap like an assembly.
+                attempted = true;
+                if (!(await this.restoreLostRecordingStatus(candidate, ledger))) {
+                    return this.recordLedgerFailure(sessionID, true);
+                }
+                attempted = false;
+            }
+            const version = await ledger.ClaimAssembly(sessionID, staleCutoff);
+            if (version === null) {
+                LogStatus(`[SessionJanitor] Recording ledger sweep skipped session ${sessionID}: another assembler holds its claim`);
+                return { Attempted: false, Recovered: false };
+            }
+            attempted = true;
+            const outcome = await this.runWithinAttemptTimeout(sessionID, AssembleRealtimeRecordingFromLedger({
+                SessionID: sessionID, Version: version, Ledger: ledger,
+                ContextUser: systemUser, Provider: provider, ForcePartial: true,
+            }), late => late);
+            switch (outcome) {
+                case 'Complete':
+                case 'Partial':
+                    this.ledgerBackoff.Clear(sessionID);
+                    LogStatus(`[SessionJanitor] Assembled the recording of session ${sessionID} from its shard ledger: ${outcome} (version ${version})`);
+                    return { Attempted: true, Recovered: true };
+                case 'Superseded':
+                    this.ledgerBackoff.Clear(sessionID);
+                    LogStatus(`[SessionJanitor] Ledger assembly of session ${sessionID} (version ${version}) was Superseded by a newer claim; left to it`);
+                    return { Attempted: true, Recovered: false };
+                case 'Retry':
+                    // The assembler already logged the error; the claim stays Assembling for a re-claim after the backoff.
+                    LogStatus(`[SessionJanitor] Ledger assembly of session ${sessionID} (version ${version}) hit a transient error; retried after its backoff`);
+                    return this.recordLedgerFailure(sessionID, true);
+                case 'Deferred':
+                    LogError(`[SessionJanitor] Forced ledger assembly of session ${sessionID} (version ${version}) returned Deferred, which only an unforced assembly should; backing off`);
+                    return this.recordLedgerFailure(sessionID, true);
+                case 'Failed':
+                    LogError(`[SessionJanitor] Ledger assembly of session ${sessionID} (version ${version}) Failed; its recording status is now Failed`);
+                    return this.recordLedgerFailure(sessionID, true);
+                case 'TimedOut':
+                    LogError(`[SessionJanitor] Ledger assembly of session ${sessionID} (version ${version}) did not finish within ` +
+                        `${this._config.recordingRecoveryAttemptTimeoutMinutes} minutes; abandoning this attempt (it may still finish) ` +
+                        `and continuing the sweep. The session is not retried until it settles.`);
+                    return this.recordLedgerFailure(sessionID, true);
+            }
+        }
+        catch (err) {
+            LogError(`[SessionJanitor] Recording ledger sweep failed for session ${sessionID}: ${err instanceof Error ? err.message : String(err)}`);
+            return this.recordLedgerFailure(sessionID, attempted);
+        }
+    }
+    /**
+     * Gives a status-less session with ledger shards back a `Pending` status so it can be claimed (the
+     * claim only takes Pending/Partial/stale Assembling). That is how statuses were lost before #6010
+     * made the ledger columns not updatable through an entity save (a stale `Save` of the session, e.g. its
+     * close, put back the NULL status and counts). The restore stays for rows hit before then.
+     *
+     * The terminal proc is the existing way to set a NULL status to `Pending`. It is stamped with the
+     * counts the session still stores, or 0 for a NULL count (the proc accepts any count over NULL), and
+     * never with counts invented from the ledger: a made-up terminal could make a recording whose tail
+     * never arrived assemble as Complete, which deletes its shards. With no real terminal the stored shard
+     * count differs from the stamped one, so the assembly is Partial and keeps the shards; a later real
+     * terminal (higher counts) or late shard re-opens it intact.
+     * Returns false (logged) when the status could not be restored.
+     */
+    async restoreLostRecordingStatus(candidate, ledger) {
+        const sessionID = candidate.ID;
+        const expectedShards = candidate.RecordingExpectedShards ?? 0;
+        // Number(): the candidate comes from a RunView entity page, which keeps the driver's value for a
+        // BIGINT column, and the SQL Server driver returns BIGINT as a string (the ledger refuses strings).
+        const capturedSamples = Number(candidate.RecordingCapturedSamples ?? 0);
+        const stamped = await ledger.StampTerminal(sessionID, expectedShards, capturedSamples);
+        if (stamped.RecordingStatus == null) {
+            LogError(`[SessionJanitor] Could not restore the recording status of session ${sessionID} ` +
+                `(terminal stamp of ${expectedShards} shard(s), ${capturedSamples} sample(s) was ${stamped.Outcome}); backing off`);
+            return false;
+        }
+        LogStatus(`[SessionJanitor] Restored recording status ${stamped.RecordingStatus} for session ${sessionID} ` +
+            `(stamped its stored counts: ${expectedShards} shard(s), ${capturedSamples} sample(s))`);
+        return true;
+    }
+    /** Counts a failed ledger assembly in its backoff (see {@link RecoveryBackoffTracker.RecordFailure}). */
+    recordLedgerFailure(sessionID, attempted) {
+        this.ledgerBackoff.RecordFailure(sessionID, 'Failed');
+        return { Attempted: attempted, Recovered: false };
+    }
+    /**
+     * Deletes up to {@link MAX_PURGED_SHARDS_PER_PASS} ledger shards of sessions closed before the lookback
+     * window, which no sweep assembles any more. Never throws: a failure is logged and retried next pass.
+     */
+    async purgeExpiredLedgerShards(provider, systemUser) {
+        const { lookbackCutoff } = this.recoveryWindow();
+        try {
+            const purged = await new SqlRealtimeRecordingLedger(provider, systemUser).PurgeClosedBefore(lookbackCutoff, MAX_PURGED_SHARDS_PER_PASS);
+            if (purged > 0) {
+                LogStatus(`[SessionJanitor] Purged ${purged} recording shard(s) of sessions closed before ${lookbackCutoff.toISOString()}`);
+            }
+        }
+        catch (err) {
+            LogError(`[SessionJanitor] Recording shard purge (sessions closed before ${lookbackCutoff.toISOString()}) failed: ` +
+                `${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
     /** Parses the absolute deadline (ms) from a session's `Config_` JSON, or null when absent/malformed. */
     parseSessionDeadlineMs(configJson) {
         if (!configJson) {
@@ -372,6 +625,29 @@ export class SessionJanitor extends BaseSingleton {
         }
         this.startRecoveryPass(this._provider, this._systemUser);
     }
+    /**
+     * One recording recovery pass: purge expired ledger shards, then the ledger sweep, then legacy
+     * segment recovery. The ledger sweep goes first because its sessions are the current recording path;
+     * a failed ledger sweep is logged here so legacy recovery still runs. A legacy failure rejects (logged
+     * by {@link startRecoveryPass}).
+     */
+    async runRecoveryPass(provider, systemUser) {
+        await this.purgeExpiredLedgerShards(provider, systemUser);
+        // A Stop() during a step ends the pass before the next one, so Shutdown waits on less work.
+        if (this._stopped) {
+            return;
+        }
+        try {
+            await this.RunRecordingLedgerSweep(provider, systemUser);
+        }
+        catch (err) {
+            LogError(`SessionJanitor recording ledger sweep failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        if (this._stopped) {
+            return;
+        }
+        await this.RunRecordingRecoverySweep(provider, systemUser);
+    }
     /**
      * Fire-and-forget recording recovery, skipped when the previous pass is still running or the janitor
      * has been stopped (a tick that was mid-close-sweep when {@link Stop} ran must not start one).
@@ -380,7 +656,7 @@ export class SessionJanitor extends BaseSingleton {
         if (this._stopped || this._recoveryPass) {
             return;
         }
-        this._recoveryPass = this.RunRecordingRecoverySweep(provider, systemUser)
+        this._recoveryPass = this.runRecoveryPass(provider, systemUser)
             .then(() => undefined, err => {
             LogError(`SessionJanitor recording recovery sweep failed: ${err instanceof Error ? err.message : String(err)}`);
         })
@@ -437,7 +713,8 @@ export class SessionJanitor extends BaseSingleton {
             }
             const storage = await this.resolveRecoveryStorage(candidate.AgentID, context);
             if (storage.Kind === 'NoAccount') {
-                return this.deferRecovery(candidate.ID, `no recording storage account for agent ${candidate.AgentID}`);
+                this.recoveryBackoff.Defer(candidate.ID, `no recording storage account for agent ${candidate.AgentID}`);
+                return { Attempted: false, Recovered: false };
             }
             if (storage.Kind === 'AgentUnavailable') {
                 LogStatus(`[SessionJanitor] Recording recovery skipped for session ${candidate.ID}: agent ${candidate.AgentID} could not be loaded`);
@@ -445,13 +722,13 @@ export class SessionJanitor extends BaseSingleton {
             }
             const accountID = storage.AccountID;
             attempted = true;
-            const result = await this.recoverWithinTimeout({
+            const result = await this.runWithinAttemptTimeout(candidate.ID, RecoverRealtimeRecordingFromSegments({
                 SessionID: candidate.ID,
                 StorageAccountID: accountID,
                 StartedAt: startedAt,
                 ContextUser: context.systemUser,
                 Provider: context.provider,
-            });
+            }), late => late.Outcome);
             if (result === 'TimedOut') {
                 LogError(`[SessionJanitor] Recording recovery for session ${candidate.ID} did not finish within ` +
                     `${this._config.recordingRecoveryAttemptTimeoutMinutes} minutes; abandoning this attempt (its storage call may ` +
@@ -460,11 +737,11 @@ export class SessionJanitor extends BaseSingleton {
             }
             switch (result.Outcome) {
                 case 'Recovered':
-                    this.recoveryBackoff.delete(candidate.ID.toLowerCase());
+                    this.recoveryBackoff.Clear(candidate.ID);
                     LogStatus(`[SessionJanitor] Recovered recording for session ${candidate.ID} from ${result.SegmentCount} segment(s) (file ${result.FileID})`);
                     return { Attempted: true, Recovered: true };
                 case 'Superseded':
-                    this.recoveryBackoff.delete(candidate.ID.toLowerCase());
+                    this.recoveryBackoff.Clear(candidate.ID);
                     LogStatus(`[SessionJanitor] Session ${candidate.ID} was given a recording by another writer; recovery left it alone`);
                     return { Attempted: true, Recovered: false };
                 case 'NoSegments':
@@ -483,13 +760,12 @@ export class SessionJanitor extends BaseSingleton {
         }
     }
     /**
-     * Runs one session's recovery, giving up on waiting after `recordingRecoveryAttemptTimeoutMinutes`.
-     * A call cannot be cancelled, so a timed-out one keeps running: the session is marked abandoned until
-     * it settles (later sweeps skip it), and its late outcome is logged then.
+     * Waits for one session's recovery attempt for at most `recordingRecoveryAttemptTimeoutMinutes`.
+     * An attempt cannot be cancelled, so a timed-out one keeps running: the session is marked abandoned
+     * until it settles (later sweeps skip it), and its late outcome (`describeLate`) is logged then.
      */
-    async recoverWithinTimeout(input) {
-        const key = input.SessionID.toLowerCase();
-        const attempt = RecoverRealtimeRecordingFromSegments(input);
+    async runWithinAttemptTimeout(sessionID, attempt, describeLate) {
+        const key = sessionID.toLowerCase();
         let timer;
         const timedOut = new Promise(resolve => {
             timer = setTimeout(() => resolve('TimedOut'), this._config.recordingRecoveryAttemptTimeoutMinutes * 60_000);
@@ -499,7 +775,7 @@ export class SessionJanitor extends BaseSingleton {
             const outcome = await Promise.race([attempt, timedOut]);
             if (outcome === 'TimedOut') {
                 this.abandonedRecoveries.add(key);
-                attempt.then(late => LogStatus(`[SessionJanitor] Abandoned recording recovery for session ${input.SessionID} finished late: ${late.Outcome}`), err => LogError(`[SessionJanitor] Abandoned recording recovery for session ${input.SessionID} failed late: ${err instanceof Error ? err.message : String(err)}`)).finally(() => this.abandonedRecoveries.delete(key));
+                attempt.then(late => LogStatus(`[SessionJanitor] Abandoned recording recovery for session ${sessionID} finished late: ${describeLate(late)}`), err => LogError(`[SessionJanitor] Abandoned recording recovery for session ${sessionID} failed late: ${err instanceof Error ? err.message : String(err)}`)).finally(() => this.abandonedRecoveries.delete(key));
             }
             return outcome;
         }
@@ -523,20 +799,6 @@ export class SessionJanitor extends BaseSingleton {
         }
         return true;
     }
-    /**
-     * Defers a session that cannot be recovered until configuration changes (its agent has no recording
-     * storage account). That is not a failed attempt, so it never counts toward the give-up: the session
-     * is re-checked every {@link RECOVERY_BACKOFF_MAX_MS} and logged only the first time.
-     */
-    deferRecovery(sessionID, reason) {
-        const key = sessionID.toLowerCase();
-        const state = this.recoveryBackoff.get(key);
-        if (!state?.Deferred) {
-            LogStatus(`[SessionJanitor] Recording recovery skipped for session ${sessionID}: ${reason}; re-checked every ${RECOVERY_BACKOFF_MAX_MS / 3_600_000} h`);
-        }
-        this.recoveryBackoff.set(key, { Failures: state?.Failures ?? 0, NextAttemptAt: Date.now() + RECOVERY_BACKOFF_MAX_MS, Deferred: true });
-        return { Attempted: false, Recovered: false };
-    }
     /**
      * Loads the agent carrying a session's recording storage config and resolves its storage account,
      * memoized per agent for the sweep: the resolve may force a storage-engine reload, so it must not
@@ -561,43 +823,11 @@ export class SessionJanitor extends BaseSingleton {
         context.storage.set(key, storage);
         return storage;
     }
-    /** True while a session's backoff (or give-up) window has not elapsed. */
-    isRecoveryBackedOff(sessionID) {
-        const state = this.recoveryBackoff.get(sessionID.toLowerCase());
-        return state != null && Date.now() < state.NextAttemptAt;
-    }
-    /**
-     * Counts one more unsuccessful recovery for a session and schedules its next attempt
-     * (`min(2^k * 5 min, 6 h)`); at {@link MAX_RECOVERY_ATTEMPTS} it stops checking the session for this
-     * process with a single log line: a `LogError` when the last attempt failed, a `LogStatus` when it
-     * simply found no shards (nothing to recover is not an error).
-     */
+    /** Counts a failed legacy recovery in its backoff (see {@link RecoveryBackoffTracker.RecordFailure}). */
     recordRecoveryFailure(sessionID, attempted, kind = 'Failed') {
-        const key = sessionID.toLowerCase();
-        const failures = (this.recoveryBackoff.get(key)?.Failures ?? 0) + 1;
-        if (failures >= MAX_RECOVERY_ATTEMPTS) {
-            this.recoveryBackoff.set(key, { Failures: failures, NextAttemptAt: Infinity, Deferred: false });
-            if (kind === 'NoSegments') {
-                LogStatus(`[SessionJanitor] Session ${sessionID} had no recoverable segments in ${failures} checks; not checking it again until restart`);
-            }
-            else {
-                LogError(`[SessionJanitor] Giving up on recording recovery for session ${sessionID} after ${failures} failed attempts`);
-            }
-        }
-        else {
-            const waitMs = Math.min(2 ** failures * RECOVERY_BACKOFF_BASE_MS, RECOVERY_BACKOFF_MAX_MS);
-            this.recoveryBackoff.set(key, { Failures: failures, NextAttemptAt: Date.now() + waitMs, Deferred: false });
-        }
+        this.recoveryBackoff.RecordFailure(sessionID, kind);
         return { Attempted: attempted, Recovered: false };
     }
-    /** Drops backoff state for sessions that were not in the latest complete candidate set. */
-    pruneRecoveryBackoff(candidateKeys) {
-        for (const key of this.recoveryBackoff.keys()) {
-            if (!candidateKeys.has(key)) {
-                this.recoveryBackoff.delete(key);
-            }
-        }
-    }
     /**
      * Page through every session matching `filter` with keyset (`AfterKey`) pagination and close each
      * via {@link SessionManager.CloseSession}, stamping `closeReason` on every row it transitions.