@memberjunction/server 6.2.0-edge.4 → 6.2.0-edge.5

package/dist/agentSessions/SessionJanitor.d.ts

@@ -1,5 +1,5 @@
 import { BaseSingleton, IShutdownable } from '@memberjunction/global';
-import { IMetadataProvider, UserInfo } from '@memberjunction/core';
+import { DatabaseProviderBase, IMetadataProvider, UserInfo } from '@memberjunction/core';
 import { HostInstanceIdentity } from './HostInstance.js';
 /** Tuning knobs for the janitor. All durations in their named units. */
 export interface SessionJanitorConfig {
@@ -45,13 +45,17 @@ export interface SessionJanitorConfig {
  * invoked from {@link Shutdown} during the graceful ShutdownRegistry drain — closes this exact host
  * instance's own live sessions with `CloseReason = 'Shutdown'`.
  *
- * Separately from closing, {@link RunRecordingRecoverySweep} runs after the close sweeps on every
- * periodic tick, without blocking them, and rebuilds the recording of already-`Closed` sessions whose
- * end-of-call upload never arrived (it closes nothing).
+ * Separately from closing, a recording recovery pass runs after the close sweeps on every periodic tick,
+ * without blocking them, and closes nothing. It purges shard-ledger rows past the lookback window, then
+ * {@link RunRecordingLedgerSweep} assembles the recordings of `Closed` sessions on the shard ledger that
+ * no terminal signal (or a dead assembler) left unassembled, then {@link RunRecordingRecoverySweep}
+ * rebuilds, from legacy storage segments, the recording of `Closed` sessions not on the ledger whose
+ * end-of-call upload never arrived.
  *
  * The close sweeps are **idempotent and safe to run concurrently** on every instance: closing an
- * already-`Closed` session is a no-op, and the close path is last-writer-wins. Recording recovery is not
- * strictly so: two instances can recover the same session at once (there is no claim column). Each
+ * already-`Closed` session is a no-op, and the close path is last-writer-wins. Ledger assembly is
+ * arbitrated by the ledger's atomic claim. Legacy recovery is not: two instances can recover the same
+ * session at once (there is no claim column for it). Each
  * attempt uploads its own file and defers to an already-stamped recording on a best-effort basis, so
  * the cost is duplicate work and an extra file, not a lost recording. The extra file is an orphan when the
  * second attempt's re-check sees the first stamp; when both re-checks pass first, both files are stamped in
@@ -73,8 +77,13 @@ export declare class SessionJanitor extends BaseSingleton<SessionJanitor> implem
     private _provider;
     private _systemUser;
     private readonly sessionManager;
-    /** Recovery retry state keyed by lowercase session id. Process-local: a restart gets fresh attempts. */
+    /** Retry state of the legacy storage-segment recovery. */
     private readonly recoveryBackoff;
+    /**
+     * Retry state of the ledger sweep, kept apart from {@link recoveryBackoff}: each sweep prunes its
+     * tracker by its own candidate set, so a shared one would forget the other sweep's failures.
+     */
+    private readonly ledgerBackoff;
     /**
      * Lowercase ids of sessions whose recovery attempt timed out but has not settled. Skipped by later
      * sweeps so a second attempt never runs alongside the abandoned one; removed when it settles.
@@ -99,9 +108,10 @@ export declare class SessionJanitor extends BaseSingleton<SessionJanitor> implem
      * close sweeps, then recording recovery). Idempotent: a second call does not stack a second timer.
      * Captures the provider + system user so the timer callback can run without re-supplying them, and
      * re-enables recording recovery after a {@link Stop}. Throws, before doing any work, when
-     * `intervalMs` is invalid (see {@link Configure}).
+     * `intervalMs` is invalid (see {@link Configure}). Takes the server's database provider because
+     * recording recovery calls the shard ledger's stored procedures.
      */
-    Start(provider: IMetadataProvider, systemUser: UserInfo, intervalMs?: number): Promise<void>;
+    Start(provider: DatabaseProviderBase, systemUser: UserInfo, intervalMs?: number): Promise<void>;
     /**
      * Stops the periodic tick and keeps any new recording recovery pass from starting until the next
      * {@link Start}, including one a tick already in progress would have started. A pass that is already
@@ -163,9 +173,68 @@ export declare class SessionJanitor extends BaseSingleton<SessionJanitor> implem
      * sessions that left the candidate set are pruned, but only after a pass that saw every candidate, so
      * a capped pass never forgets sessions it did not reach.
      *
+     * Sessions on the shard ledger (a `RecordingStatus`, or any ledger row) are left to
+     * {@link RunRecordingLedgerSweep}: their audio is in SQL, not in storage segments.
+     *
      * Never throws for a single bad session. Returns the number of recordings recovered.
      */
-    RunRecordingRecoverySweep(provider: IMetadataProvider, systemUser: UserInfo): Promise<number>;
+    RunRecordingRecoverySweep(provider: DatabaseProviderBase, systemUser: UserInfo): Promise<number>;
+    /**
+     * Backstop for client-direct recordings on the shard ledger: assembles the recording of `Closed`
+     * sessions that closed between the lookback window and the grace period and are still
+     * - `Pending` (shards, or a terminal signal, with no assembly since: the browser died before its
+     *   terminal request, or the in-process assembly request was lost with its host),
+     * - `Assembling` with no write for `recordingRecoveryAttemptTimeoutMinutes` (the assembler died, or
+     *   it hit a transient error and left the claim for a re-claim), or
+     * - status-less while the ledger holds shards for them (see {@link restoreLostRecordingStatus}).
+     *
+     * Each candidate is claimed with the ledger's atomic, stale-aware claim (a lost claim is skipped: another
+     * assembler has it) and assembled with `ForcePartial`, so missing shards become silence. An attempt is
+     * bounded by `recordingRecoveryAttemptTimeoutMinutes` and shares the legacy sweep's abandoned-attempt
+     * pause; failures back off and are given up on exactly like legacy recovery, in their own tracker.
+     * At most {@link MAX_RECOVERIES_PER_SWEEP} assemblies per call. Never throws for a single bad session.
+     * Returns the number of recordings assembled (`Complete` or `Partial`).
+     */
+    RunRecordingLedgerSweep(provider: DatabaseProviderBase, systemUser: UserInfo): Promise<number>;
+    /** The closed-at window recording recovery considers: older than the grace, newer than the lookback. */
+    private recoveryWindow;
+    /** The schema-qualified, dialect-quoted shard-ledger view, for filter sub-queries. */
+    private shardViewName;
+    /**
+     * Walks every session matching `filter` and runs `recover` on each one that is not backed off, not
+     * abandoned, and not `isBusy`, sequentially, stopping at {@link MAX_RECOVERIES_PER_SWEEP} attempts or
+     * {@link MAX_ABANDONED_RECOVERIES} abandoned attempts. Backoff entries of sessions that left the
+     * candidate set are pruned, but only after a walk that saw every candidate, so a capped walk never
+     * forgets sessions it did not reach. Returns the number recovered.
+     */
+    private sweepRecoveryCandidates;
+    /**
+     * Claims and assembles one ledger candidate and updates its backoff. Never throws: any exception is
+     * logged with the session id and counted as a failure.
+     */
+    private assembleSessionFromLedger;
+    /**
+     * Gives a status-less session with ledger shards back a `Pending` status so it can be claimed (the
+     * claim only takes Pending/Partial/stale Assembling). The status is lost when a `BaseEntity.Save` of
+     * the session (e.g. its close) loaded the row before a shard or terminal proc wrote it and saved
+     * after: `spUpdate` rewrites every column, so it puts back the stale NULL status and counts.
+     *
+     * The terminal proc is the existing way to set a NULL status to `Pending`. It is stamped with the
+     * counts the session still stores, or 0 for a NULL count (the proc accepts any count over NULL), and
+     * never with counts invented from the ledger: a made-up terminal could make a recording whose tail
+     * never arrived assemble as Complete, which deletes its shards. With no real terminal the stored shard
+     * count differs from the stamped one, so the assembly is Partial and keeps the shards; a later real
+     * terminal (higher counts) or late shard re-opens it intact.
+     * Returns false (logged) when the status could not be restored.
+     */
+    private restoreLostRecordingStatus;
+    /** Counts a failed ledger assembly in its backoff (see {@link RecoveryBackoffTracker.RecordFailure}). */
+    private recordLedgerFailure;
+    /**
+     * Deletes up to {@link MAX_PURGED_SHARDS_PER_PASS} ledger shards of sessions closed before the lookback
+     * window, which no sweep assembles any more. Never throws: a failure is logged and retried next pass.
+     */
+    private purgeExpiredLedgerShards;
     /** Parses the absolute deadline (ms) from a session's `Config_` JSON, or null when absent/malformed. */
     private parseSessionDeadlineMs;
     /** Register for graceful shutdown exactly once. */
@@ -183,6 +252,13 @@ export declare class SessionJanitor extends BaseSingleton<SessionJanitor> implem
      * `recordingRecoveryAttemptTimeoutMinutes`, a hung storage call delays later passes, never stops them.
      */
     private periodicSweep;
+    /**
+     * One recording recovery pass: purge expired ledger shards, then the ledger sweep, then legacy
+     * segment recovery. The ledger sweep goes first because its sessions are the current recording path;
+     * a failed ledger sweep is logged here so legacy recovery still runs. A legacy failure rejects (logged
+     * by {@link startRecoveryPass}).
+     */
+    private runRecoveryPass;
     /**
      * Fire-and-forget recording recovery, skipped when the previous pass is still running or the janitor
      * has been stopped (a tick that was mid-close-sweep when {@link Stop} ran must not start one).
@@ -199,39 +275,24 @@ export declare class SessionJanitor extends BaseSingleton<SessionJanitor> implem
      */
     private recoverSessionRecording;
     /**
-     * Runs one session's recovery, giving up on waiting after `recordingRecoveryAttemptTimeoutMinutes`.
-     * A call cannot be cancelled, so a timed-out one keeps running: the session is marked abandoned until
-     * it settles (later sweeps skip it), and its late outcome is logged then.
+     * Waits for one session's recovery attempt for at most `recordingRecoveryAttemptTimeoutMinutes`.
+     * An attempt cannot be cancelled, so a timed-out one keeps running: the session is marked abandoned
+     * until it settles (later sweeps skip it), and its late outcome (`describeLate`) is logged then.
      */
-    private recoverWithinTimeout;
+    private runWithinAttemptTimeout;
     /**
      * True while {@link MAX_ABANDONED_RECOVERIES} timed-out attempts are still running: no new attempt
      * starts until one settles. Logs one `LogError` per pause.
      */
     private isRecoveryPausedForAbandonedAttempts;
-    /**
-     * Defers a session that cannot be recovered until configuration changes (its agent has no recording
-     * storage account). That is not a failed attempt, so it never counts toward the give-up: the session
-     * is re-checked every {@link RECOVERY_BACKOFF_MAX_MS} and logged only the first time.
-     */
-    private deferRecovery;
     /**
      * Loads the agent carrying a session's recording storage config and resolves its storage account,
      * memoized per agent for the sweep: the resolve may force a storage-engine reload, so it must not
      * repeat for every session of the same agent.
      */
     private resolveRecoveryStorage;
-    /** True while a session's backoff (or give-up) window has not elapsed. */
-    private isRecoveryBackedOff;
-    /**
-     * Counts one more unsuccessful recovery for a session and schedules its next attempt
-     * (`min(2^k * 5 min, 6 h)`); at {@link MAX_RECOVERY_ATTEMPTS} it stops checking the session for this
-     * process with a single log line: a `LogError` when the last attempt failed, a `LogStatus` when it
-     * simply found no shards (nothing to recover is not an error).
-     */
+    /** Counts a failed legacy recovery in its backoff (see {@link RecoveryBackoffTracker.RecordFailure}). */
     private recordRecoveryFailure;
-    /** Drops backoff state for sessions that were not in the latest complete candidate set. */
-    private pruneRecoveryBackoff;
     /**
      * Page through every session matching `filter` with keyset (`AfterKey`) pagination and close each
      * via {@link SessionManager.CloseSession}, stamping `closeReason` on every row it transitions.

package/dist/agentSessions/SessionJanitor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SessionJanitor.d.ts","sourceRoot":"","sources":["../../src/agentSessions/SessionJanitor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAoB,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EACH,iBAAiB,EACjB,QAAQ,EAMX,MAAM,sBAAsB,CAAC;AAQ9B,OAAO,EAA6C,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAmCpG,wEAAwE;AACxE,MAAM,WAAW,oBAAoB;IACjC,0GAA0G;IAC1G,qBAAqB,EAAE,MAAM,CAAC;IAC9B,4HAA4H;IAC5H,eAAe,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,6BAA6B,EAAE,MAAM,CAAC;IACtC,2FAA2F;IAC3F,8BAA8B,EAAE,MAAM,CAAC;IACvC;;;;OAIG;IACH,sCAAsC,EAAE,MAAM,CAAC;CAClD;AAgDD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,cAAe,SAAQ,aAAa,CAAC,cAAc,CAAE,YAAW,aAAa;IACtF,OAAO,CAAC,OAAO,CAAwC;IACvD,OAAO,CAAC,WAAW,CAA+C;IAClE,OAAO,CAAC,aAAa,CAAS;IAC9B;;;OAGG;IACH,OAAO,CAAC,aAAa,CAA8B;IACnD,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAkC;IACnD,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,wGAAwG;IACxG,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsC;IACtE;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAqB;IACzD,wGAAwG;IACxG,OAAO,CAAC,oBAAoB,CAAS;IAErC,SAAS;IAIT,8FAA8F;IAC9F,WAAkB,QAAQ,IAAI,cAAc,CAE3C;IAED,qDAAqD;IACrD,SAAgB,YAAY,oBAAoB;IAEhD;;;;;OAKG;IACI,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,IAAI;IAe7D;;;;;;OAMG;IACU,KAAK,CAAC,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYzG;;;;OAIG;IACI,IAAI,IAAI,IAAI;IAQnB;;;;;;;OAOG;IACU,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAYtC;;;;;OAKG;IACU,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAUjG;;;;;;;;;OASG;IACU,kBAAkB,CAC3B,QAAQ,EAAE,iBAAiB,EAC3B,UAAU,EAAE,QAAQ,EACpB,YAAY,GAAE,oBAA+C,GAC9D,OAAO,CAAC,MAAM,CAAC;IAmBlB;;;OAGG;IACU,iBAAiB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAUlG;;;;;;;;;OASG;IACU,mBAAmB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBpG;;;;;;;;;;;;;OAaG;IACU,yBAAyB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IA0C1G,wGAAwG;IACxG,OAAO,CAAC,sBAAsB;IAkB9B,mDAAmD;IACnD,OAAO,CAAC,gBAAgB;IAOxB,oIAAoI;IACpI,OAAO,CAAC,aAAa;IAQrB;;;;;;;;;OASG;YACW,aAAa;IAkB3B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAgBzB;;;OAGG;YACW,mBAAmB;IAsBjC;;;OAGG;YACW,uBAAuB;IAoErC;;;;OAIG;YACW,oBAAoB;IAuBlC;;;OAGG;IACH,OAAO,CAAC,oCAAoC;IAe5C;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAUrB;;;;OAIG;YACW,sBAAsB;IAmBpC,0EAA0E;IAC1E,OAAO,CAAC,mBAAmB;IAK3B;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAiB7B,2FAA2F;IAC3F,OAAO,CAAC,oBAAoB;IAQ5B;;;;;OAKG;YACW,aAAa;IAsC3B;;;;;;;;;;;OAWG;YACW,kBAAkB;IA+BhC,sFAAsF;YACxE,SAAS;CAqB1B"}
+{"version":3,"file":"SessionJanitor.d.ts","sourceRoot":"","sources":["../../src/agentSessions/SessionJanitor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAoB,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EACH,oBAAoB,EACpB,iBAAiB,EACjB,QAAQ,EAMX,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAA6C,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AA6CpG,wEAAwE;AACxE,MAAM,WAAW,oBAAoB;IACjC,0GAA0G;IAC1G,qBAAqB,EAAE,MAAM,CAAC;IAC9B,4HAA4H;IAC5H,eAAe,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,6BAA6B,EAAE,MAAM,CAAC;IACtC,2FAA2F;IAC3F,8BAA8B,EAAE,MAAM,CAAC;IACvC;;;;OAIG;IACH,sCAAsC,EAAE,MAAM,CAAC;CAClD;AAiHD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,cAAe,SAAQ,aAAa,CAAC,cAAc,CAAE,YAAW,aAAa;IACtF,OAAO,CAAC,OAAO,CAAwC;IACvD,OAAO,CAAC,WAAW,CAA+C;IAClE,OAAO,CAAC,aAAa,CAAS;IAC9B;;;OAGG;IACH,OAAO,CAAC,aAAa,CAA8B;IACnD,gGAAgG;IAChG,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAqC;IACtD,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAwB;IACvD,0DAA0D;IAC1D,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAChE;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgC;IAC9D;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAqB;IACzD,wGAAwG;IACxG,OAAO,CAAC,oBAAoB,CAAS;IAErC,SAAS;IAIT,8FAA8F;IAC9F,WAAkB,QAAQ,IAAI,cAAc,CAE3C;IAED,qDAAqD;IACrD,SAAgB,YAAY,oBAAoB;IAEhD;;;;;OAKG;IACI,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,GAAG,IAAI;IAe7D;;;;;;;OAOG;IACU,KAAK,CAAC,QAAQ,EAAE,oBAAoB,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY5G;;;;OAIG;IACI,IAAI,IAAI,IAAI;IAQnB;;;;;;;OAOG;IACU,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAYtC;;;;;OAKG;IACU,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAUjG;;;;;;;;;OASG;IACU,kBAAkB,CAC3B,QAAQ,EAAE,iBAAiB,EAC3B,UAAU,EAAE,QAAQ,EACpB,YAAY,GAAE,oBAA+C,GAC9D,OAAO,CAAC,MAAM,CAAC;IAmBlB;;;OAGG;IACU,iBAAiB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAUlG;;;;;;;;;OASG;IACU,mBAAmB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBpG;;;;;;;;;;;;;;;;OAgBG;IACU,yBAAyB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,UAAU,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IAmB7G;;;;;;;;;;;;;;;OAeG;IACU,uBAAuB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,UAAU,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;IA2B3G,wGAAwG;IACxG,OAAO,CAAC,cAAc;IAQtB,sFAAsF;IACtF,OAAO,CAAC,aAAa;IAIrB;;;;;;OAMG;YACW,uBAAuB;IAmCrC;;;OAGG;YACW,yBAAyB;IAkEvC;;;;;;;;;;;;;OAaG;YACW,0BAA0B;IAqBxC,yGAAyG;IACzG,OAAO,CAAC,mBAAmB;IAK3B;;;OAGG;YACW,wBAAwB;IAetC,wGAAwG;IACxG,OAAO,CAAC,sBAAsB;IAkB9B,mDAAmD;IACnD,OAAO,CAAC,gBAAgB;IAOxB,oIAAoI;IACpI,OAAO,CAAC,aAAa;IAQrB;;;;;;;;;OASG;YACW,aAAa;IAkB3B;;;;;OAKG;YACW,eAAe;IAiB7B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAgBzB;;;OAGG;YACW,mBAAmB;IAsBjC;;;OAGG;YACW,uBAAuB;IAyErC;;;;OAIG;YACW,uBAAuB;IAsBrC;;;OAGG;IACH,OAAO,CAAC,oCAAoC;IAe5C;;;;OAIG;YACW,sBAAsB;IAmBpC,yGAAyG;IACzG,OAAO,CAAC,qBAAqB;IAK7B;;;;;OAKG;YACW,aAAa;IAsC3B;;;;;;;;;;;OAWG;YACW,kBAAkB;IA+BhC,sFAAsF;YACxE,SAAS;CAqB1B"}