@memberjunction/server 6.2.0-edge.5 → 6.2.0-edge.6

package/dist/index.js

@@ -35,7 +35,7 @@ import { BuildPostgreSQLConnectionConfig, DescribeReadOnlyLoginOverreach, Postgr
 import { default as jwt } from 'jsonwebtoken';
 import { contextFunction, CreateReadOnlyProvider, CreateUnifiedAuthMiddleware, getUserPayload } from './context.js';
 import { requireSystemUserDirective, publicDirective } from './directives/index.js';
-import { variablesLoggingMiddleware } from './logging/variablesLoggingMiddleware.js';
+import { ApplyRootVariablesLoggingMiddleware } from './logging/variablesLoggingMiddleware.js';
 import { auditResolversForUndecoratedArgs } from './logging/bootAudit.js';
 import { StartupLogger } from './logging/StartupLogger.js';
 import { AuthProviderFactory } from '@memberjunction/auth-providers';
@@ -43,7 +43,7 @@ import createMSSQLConfig from './orm.js';
 import { setupRESTEndpoints } from './rest/setupRESTEndpoints.js';
 import { CreateRealtimeRecordingRouter, REALTIME_RECORDING_MOUNT_PATH } from './rest/RealtimeRecordingHandler.js';
 import { LoadOwnedRealtimeSession } from './realtimeSessions/ownedRealtimeSession.js';
-import { LoadSessionConversationExternalID } from './realtimeSessions/sessionAccess.js';
+import { AuthorizeAnonymousRoomSessions, LoadSessionConversationScope } from './realtimeSessions/sessionAccess.js';
 import { RecordingAssemblyScheduler } from './realtimeSessions/RecordingAssemblyScheduler.js';
 import { RealtimeSessionEventService, ParseReplicatedRealtimeSessionEvent, } from './realtimeSessions/index.js';
 import { resolve } from 'node:path';
@@ -59,8 +59,8 @@ import { CacheManagerConfigFromSettings, CreateSharedCacheFromEnvironment, Start
 import { PubSubManager } from './generic/PubSubManager.js';
 import { ReconcileOrphanedConversationDetails } from './generic/OrphanedConversationDetailReconciler.js';
 import { HANDOFF_OFFER_FANOUT_CHANNEL, HANDOFF_OFFER_TOPIC, SetHandoffOfferPublishHook, ParseReplicatedHandoffOfferUpdate, } from './resolvers/HumanHandoffResolver.js';
-import { RoomHandoffEngine } from '@memberjunction/livekit-room-server';
-import { IntegrationProgressEmitter } from '@memberjunction/integration-progress-artifacts';
+import { RoomAuthorizationService, RoomHandoffEngine } from '@memberjunction/livekit-room-server';
+import { IntegrationProgressEmitter, SweepInterruptedRuns } from '@memberjunction/integration-progress-artifacts';
 import { PublishIntegrationProgress } from './resolvers/IntegrationProgressResolver.js';
 import { RegisterRSUProgressBridge } from './integration/RSUProgressBridge.js';
 import { ClientToolRequestManager, AgentRunWatchdog, SqlRealtimeRecordingLedger } from '@memberjunction/ai-agents';
@@ -79,8 +79,13 @@ import { ApiRoute, ApiRootPath, ConfiguredPublicUrl } from './apiRoot.js';
 import { MountHealthcheckRoute, MountPublicCoreRoutes, MountExtensionHealthRoute, MountAuthCatalogRoute, MountAuthenticatedCoreRoutes } from './coreRoutes.js';
 import { MetadataCacheRefreshIntervalSeconds } from './providerConfigUnits.js';
 import { CreateMetadataRefreshSignalHandler, METADATA_REFRESH_SIGNAL } from './metadataRefreshSignal.js';
+import { CacheInvalidationBroadcastManager } from './broadcast/index.js';
+import { ResolveServerRole, ShouldRunBackgroundWorkers } from './serverRoleConfigUnits.js';
 const cacheRefreshInterval = configInfo.databaseSettings.metadataCacheRefreshInterval;
 export { MetadataCacheRefreshIntervalSeconds } from './providerConfigUnits.js';
+export * from './serverRoleConfigUnits.js';
+export * from './broadcast/index.js';
+export { ApplyRootVariablesLoggingMiddleware } from './logging/variablesLoggingMiddleware.js';
 /**
  * Returns the configured database platform from the `DB_PLATFORM` environment
  * variable, falling back to `'sqlserver'` when the env var is unset. An
@@ -359,6 +364,9 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
     SetHostInstancePort(graphqlPort);
     // The realtime bridge engine stamps its bridge rows with the same identity and reconciles orphans by it (#5310).
     BindBridgeEngineHostInstance();
+    // An anonymous visitor may join only the LiveKit agent-session room it owns by scope; the room gate cannot import
+    // this package's session-access decision, so it is handed in here, before any resolver runs (MJ#6044).
+    RoomAuthorizationService.Instance.SetAnonymousSessionOwnerCheck(AuthorizeAnonymousRoomSessions);
     // Level-gated startup logger. Resolves verbosity from telemetry.level (single
     // operator knob). At `standard` (default), per-phase timings are collapsed into
     // the one-line summary; at `verbose`+ each phase prints inline as before.
@@ -411,6 +419,8 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
         // Every API pool carries the statement and idle-in-transaction timeouts from connection #1.
         const pgConnectionConfig = BuildPostgreSQLConnectionConfig(pgEndpoint, configInfo.databaseSettings, 'api');
         const pgPool = new pg.default.Pool(ToPGPoolConfig(pgConnectionConfig));
+        // An idle connection the server drops is emitted on the pool; without a listener Node exits.
+        pgPool.on('error', (err) => console.error('[PostgreSQL pool] Idle connection error (connection evicted):', err.message));
         // Verify connection
         const testClient = await pgPool.connect();
         await testClient.query('SELECT 1');
@@ -426,6 +436,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
                 ...ToPGPoolConfig(BuildPostgreSQLConnectionConfig({ ...pgEndpoint, ...pgReadOnlyCredentials }, configInfo.databaseSettings, 'read-only')),
                 types: MJPostgresTypes,
             });
+            readOnlyPgPool.on('error', (err) => console.error('[PostgreSQL read-only pool] Idle connection error (connection evicted):', err.message));
             const readOnlyTestClient = await readOnlyPgPool.connect();
             try {
                 for (const warning of await DescribeReadOnlyLoginOverreach(readOnlyTestClient, mj_core_schema)) {
@@ -496,6 +507,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
                 // long CodeGen timeout rather than the API one, as the SQL Server CodeGen pool does.
                 const codegenPgConnectionConfig = BuildPostgreSQLConnectionConfig({ ...pgEndpoint, User: pgCodegenUser, Password: pgCodegenPass }, configInfo.databaseSettings, 'codegen');
                 const codegenPgPool = new pg.default.Pool(ToPGPoolConfig(codegenPgConnectionConfig));
+                codegenPgPool.on('error', (err) => console.error('[PostgreSQL CodeGen pool] Idle connection error (connection evicted):', err.message));
                 const codegenTestClient = await codegenPgPool.connect();
                 await codegenTestClient.query('SELECT 1');
                 codegenTestClient.release();
@@ -523,7 +535,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
                     });
                     startupLog.LogIf('verbose', 'RSU in-process CodeGen runner initialized (PostgreSQL).');
                     // Inject CodeGen output paths for targeted git staging
-                    const { initializeConfig } = await import('@memberjunction/codegen-lib');
+                    const { initializeConfig, ResolveAdditionalSchemaInfoPath } = await import('@memberjunction/codegen-lib');
                     const cgConfig = initializeConfig(rsuWorkDir);
                     const outputPaths = (cgConfig.output ?? []).map((o) => o.directory);
                     RuntimeSchemaManager.Instance.SetCodeGenOutputPaths(outputPaths);
@@ -531,9 +543,12 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
                     // Point RSU's soft PK/FK writer at the SAME file CodeGen reads (mj.config.cjs
                     // `additionalSchemaInfo`), or RSU writes soft PKs to its own default path while
                     // CodeGen reads a different one and skips integration tables with "No primary key found".
-                    if (cgConfig.additionalSchemaInfo) {
-                        RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(cgConfig.additionalSchemaInfo);
-                        startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${cgConfig.additionalSchemaInfo}`);
+                    // Inject the ABSOLUTE path CodeGen resolves (env override, else relative to the mj.config
+                    // file's directory) so RSU, the resolvers and CodeGen agree whatever their cwd (#5858).
+                    const resolvedSchemaInfoPath = ResolveAdditionalSchemaInfoPath();
+                    if (resolvedSchemaInfoPath) {
+                        RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(resolvedSchemaInfoPath);
+                        startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${resolvedSchemaInfoPath}`);
                     }
                     else if (RuntimeSchemaManager.Instance.IsEnabled) {
                         // U5 — boot-time assert: RSU is enabled but CodeGen has NO additionalSchemaInfo path
@@ -541,7 +556,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
                         // resolved soft PK would be silently lost ("No primary key found" per table). LOUD,
                         // named, and actionable — grep for RSU_ADDITIONAL_SCHEMA_INFO_DIVERGENCE.
                         console.error(`[RSU_ADDITIONAL_SCHEMA_INFO_DIVERGENCE] Runtime Schema Update is ENABLED but mj.config.cjs has no ` +
-                            `'additionalSchemaInfo' setting — RSU writes soft PK/FK definitions to '${process.env.RSU_ADDITIONAL_SCHEMA_INFO_PATH ?? 'additionalSchemaInfo.json'}' ` +
+                            `'additionalSchemaInfo' setting (and RSU_ADDITIONAL_SCHEMA_INFO_PATH is unset) — RSU writes soft PK/FK definitions to '${RuntimeSchemaManager.Instance.AdditionalSchemaInfoPath}' ` +
                             `while CodeGen reads none, so resolved soft primary keys will be silently LOST at codegen time. ` +
                             `Set 'additionalSchemaInfo' in mj.config.cjs to the same file so the write path and read path agree.`);
                     }
@@ -651,7 +666,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
                     });
                     startupLog.LogIf('verbose', 'RSU in-process CodeGen runner initialized.');
                     // Inject CodeGen output paths for targeted git staging
-                    const { initializeConfig } = await import('@memberjunction/codegen-lib');
+                    const { initializeConfig, ResolveAdditionalSchemaInfoPath } = await import('@memberjunction/codegen-lib');
                     const codegenConfig = initializeConfig(rsuWorkDir);
                     const outputPaths = (codegenConfig.output ?? []).map((o) => o.directory);
                     RuntimeSchemaManager.Instance.SetCodeGenOutputPaths(outputPaths);
@@ -660,14 +675,17 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
                     // `additionalSchemaInfo`). Without this, RSU writes soft PKs to its own default
                     // path while CodeGen reads a different one and skips every integration table
                     // with "No primary key found".
-                    if (codegenConfig.additionalSchemaInfo) {
-                        RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(codegenConfig.additionalSchemaInfo);
-                        startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${codegenConfig.additionalSchemaInfo}`);
+                    // Inject the ABSOLUTE path CodeGen resolves (env override, else relative to the mj.config
+                    // file's directory) so RSU, the resolvers and CodeGen agree whatever their cwd (#5858).
+                    const resolvedSchemaInfoPath = ResolveAdditionalSchemaInfoPath();
+                    if (resolvedSchemaInfoPath) {
+                        RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(resolvedSchemaInfoPath);
+                        startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${resolvedSchemaInfoPath}`);
                     }
                     else if (RuntimeSchemaManager.Instance.IsEnabled) {
                         // U5 — boot-time assert: write path ≠ read path (see the PostgreSQL branch for detail).
                         console.error(`[RSU_ADDITIONAL_SCHEMA_INFO_DIVERGENCE] Runtime Schema Update is ENABLED but mj.config.cjs has no ` +
-                            `'additionalSchemaInfo' setting — RSU writes soft PK/FK definitions to '${process.env.RSU_ADDITIONAL_SCHEMA_INFO_PATH ?? 'additionalSchemaInfo.json'}' ` +
+                            `'additionalSchemaInfo' setting (and RSU_ADDITIONAL_SCHEMA_INFO_PATH is unset) — RSU writes soft PK/FK definitions to '${RuntimeSchemaManager.Instance.AdditionalSchemaInfoPath}' ` +
                             `while CodeGen reads none, so resolved soft primary keys will be silently LOST at codegen time. ` +
                             `Set 'additionalSchemaInfo' in mj.config.cjs to the same file so the write path and read path agree.`);
                     }
@@ -868,6 +886,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
     // config.ts loads and validates at module scope, so importing it there would pull full config
     // validation into every import chain that touches the resolver, unit tests included.
     ConfigureRecordDataBroadcast(configInfo.cacheSettings?.recordDataBroadcastEntities);
+    CacheInvalidationBroadcastManager.Instance.Configure(configInfo.cacheSettings?.broadcast);
     // Global listener: broadcast CACHE_INVALIDATION to all browser clients whenever
     // ANY BaseEntity save/delete occurs on this server — regardless of whether it
     // originated from a GraphQL mutation or internal server-side code (agents, actions,
@@ -888,7 +907,8 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
                     // Opt-in only: this event reaches every connected client unfiltered, and this listener
                     // fires for server-internal saves too (agents, actions, orchestrator), which are exactly
                     // the ones no browser session asked for.
-                    recordData: beEvent.type === 'save' && MayBroadcastRecordData(entityName)
+                    recordData: beEvent.type === 'save' && MayBroadcastRecordData(entityName, beEvent.baseEntity.EntityInfo)
+                        // fls-guard: MayBroadcastRecordData refuses FLS entities and Record Changes
                         ? JSON.stringify(beEvent.baseEntity.GetAll())
                         : undefined,
                 });
@@ -936,7 +956,6 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
         scalarsMap: [{ type: Date, scalar: GraphQLTimestamp }],
         emitSchemaFile: websiteRunFromPackage !== 1,
         pubSub,
-        globalMiddlewares: [variablesLoggingMiddleware],
     });
     const buildMs = performance.now() - tBuild;
     if (cpuProf && profSession) {
@@ -957,6 +976,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
     // and aren't @NoLog-marked. No-op in default config (logVariables=false).
     auditResolversForUndecoratedArgs();
     const tTransform = performance.now();
+    schema = ApplyRootVariablesLoggingMiddleware(schema);
     schema = requireSystemUserDirective.transformer(schema);
     schema = publicDirective.transformer(schema);
     // Apply middleware-contributed schema transformers (after built-in directive transformers)
@@ -1303,11 +1323,29 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
     // No per-route authMiddleware needed — unified auth middleware already ran
     setupRESTEndpoints(app, restApiConfig);
     // ─── GraphQL middleware (contextFunction reads req.userPayload, no re-auth) ─────
-    app.use(graphqlRootPath, cors(), BodyParser.json({ limit: '50mb' }), 
+    app.use(graphqlRootPath, cors({
+        exposedHeaders: ['X-MJ-Cache-Seq', 'X-MJ-Server-ID'],
+    }), BodyParser.json({ limit: '50mb' }), 
     // Express 5 leaves req.body as undefined for non-JSON or empty bodies;
     // Apollo Server's expressMiddleware requires req.body to be defined.
-    (req, _res, next) => { if (req.body === undefined)
-        req.body = {}; next(); }, expressMiddleware(apolloServer, {
+    // Also piggyback the latest monotonic cache invalidation sequence and server process ID
+    // onto every GraphQL HTTP response so clients can detect and recover missed invalidations.
+    (req, res, next) => {
+        if (req.body === undefined)
+            req.body = {};
+        const broadcastProvider = CacheInvalidationBroadcastManager.Instance.Provider;
+        res.setHeader('X-MJ-Cache-Seq', String(broadcastProvider.CurrentSequence));
+        res.setHeader('X-MJ-Server-ID', broadcastProvider.ServerProcessID);
+        const originalWriteHead = res.writeHead.bind(res);
+        res.writeHead = ((...args) => {
+            if (!res.headersSent) {
+                res.setHeader('X-MJ-Cache-Seq', String(broadcastProvider.CurrentSequence));
+                res.setHeader('X-MJ-Server-ID', broadcastProvider.ServerProcessID);
+            }
+            return originalWriteHead(...args);
+        });
+        next();
+    }, expressMiddleware(apolloServer, {
         context: contextFunction({
             setupComplete$,
             dataSource: extendConnectionPoolWithQuery(dataSources[0].dataSource), // default read-write data source
@@ -1318,9 +1356,24 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
     for (const mw of mwPostRoute) {
         app.use(mw);
     }
+    const serverRole = ResolveServerRole(configInfo.serverRole ?? process.env.MJ_SERVER_ROLE);
+    const runBackgroundWorkers = ShouldRunBackgroundWorkers(serverRole);
+    if (!runBackgroundWorkers) {
+        LogStatus(`[ServerRole] Running in '${serverRole}' mode — background worker services are disabled on this replica.`);
+    }
+    // Close out run artifacts a dead process left open, BEFORE anything below can start a new run.
+    //
+    // A killed process (OOM killer, SIGKILL, host reboot) never writes its run's terminal result, so
+    // the run reads as in-flight forever: the wizard spins, the active-operations panel shows a step
+    // that will never advance, and no failure is reported anywhere. Sound only before the first new
+    // run: artifacts are process-local, so anything still in flight as THIS process starts belongs
+    // to a process that no longer exists. That is why it runs here, ahead of the scheduler, the
+    // integration sync worker (its Start() polls immediately), the HTTP listener (a request can
+    // start a discovery) and pending RSU work, and why it is awaited. It needs none of them.
+    await SweepInterruptedRuns(undefined, msg => console.warn(msg));
     // Initialize and start scheduled jobs service if enabled
     let scheduledJobsService = null;
-    if (configInfo.scheduledJobs?.enabled) {
+    if (runBackgroundWorkers && configInfo.scheduledJobs?.enabled) {
         try {
             scheduledJobsService = new ScheduledJobsService(configInfo.scheduledJobs);
             await scheduledJobsService.Initialize();
@@ -1334,7 +1387,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
     // Initialize and start the integration sync worker if enabled (PR 1 item 8).
     // Off by default — deployments that don't opt in keep running syncs inline.
     let integrationSyncWorker = null;
-    if (configInfo.integrationSyncWorker?.enabled) {
+    if (runBackgroundWorkers && configInfo.integrationSyncWorker?.enabled) {
         try {
             integrationSyncWorker = new IntegrationSyncWorkerService(configInfo.integrationSyncWorker);
             await integrationSyncWorker.Initialize();
@@ -1368,47 +1421,49 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
         startupLog.SetScheduledJobCount(scheduledJobsService.GetStatus().activeJobs);
     }
     startupLog.PrintSummary();
-    // Process pending RSU work from pre-restart (entity maps, field maps, sync)
-    processRSUPendingWork().catch(err => console.warn(`RSU pending work processing failed: ${err}`));
-    // Resume any integration syncs that were orphaned by the previous process restart
     const resumeUser = UserCache.Instance.GetSystemUser();
-    if (resumeUser) {
-        IntegrationEngine.Instance.ResumeOrphanedSyncs(resumeUser)
-            .catch(err => console.warn(`[IntegrationEngine] Orphaned sync resume failed: ${err}`));
-    }
-    // Force-fail any agent runs left 'Running' by a process that died (restart/crash/OOM) or whose
-    // terminal-state write never landed. Staleness-based, so it never touches runs another healthy
-    // instance is still heart-beating. The watchdog also self-registers for graceful-shutdown
-    // cancellation (via ShutdownRegistry) once it begins tracking this process's first live run.
-    if (resumeUser && Metadata.Provider instanceof DatabaseProviderBase) { // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
-        const sweepUser = resumeUser;
-        const sweepProvider = Metadata.Provider; // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
-        AgentRunWatchdog.SweepOrphanedRuns(sweepProvider, sweepUser)
-            .catch(err => console.warn(`[AgentRunWatchdog] Startup sweep failed: ${err}`));
-        // The watchdog repairs the RUN; nothing repaired the conversation detail, which is the row the
-        // chat actually renders from (MJ #4222). A process that dies mid-run leaves a terminal run
-        // beside a message that still claims to be generating, and it spins forever for anyone who
-        // opens it. Runs at boot (closes restart orphans) and on a timer (closes mid-life orphans),
-        // mirroring the watchdog's own two-phase shape.
-        const reconcileOrphans = () => ReconcileOrphanedConversationDetails(sweepProvider, sweepUser)
-            .catch(err => console.warn(`[OrphanDetailReconciler] Pass failed: ${err}`));
-        void reconcileOrphans();
-        const orphanDetailTimer = setInterval(() => void reconcileOrphans(), ORPHAN_DETAIL_SWEEP_INTERVAL_MS);
-        ShutdownRegistry.Instance.Register({
-            ShutdownName: 'OrphanDetailReconciler',
-            Shutdown: () => { clearInterval(orphanDetailTimer); },
-        });
-    }
-    // Launch the AI Agent Session janitor: run own-host orphan recovery once at boot, then a periodic tick
-    // that closes stale and over-deadline sessions and rebuilds lost realtime recordings from their
-    // crash-recovery shards (storage reads and writes). Self-registers with ShutdownRegistry, so its timer
-    // is cleared by the gracefulShutdown drain below (no explicit Stop() wiring needed here).
-    if (resumeUser && Metadata.Provider instanceof DatabaseProviderBase) { // global-provider-ok: server startup recovery — boot-time session janitor uses the server's own provider
-        SessionJanitor.Instance.Start(Metadata.Provider, resumeUser) // global-provider-ok: server-owned background reconciler runs under the server's provider + system user
-            .catch(err => console.warn(`[SessionJanitor] Startup failed: ${err}`));
-        // The bridge engine's orphan reconcile likewise starts only now that this process holds its port, so it can't
-        // take the live bridges of another MJAPI still serving that port for its own orphans (#5310).
-        StartBridgeOrphanReconciliation(Metadata.Provider, resumeUser); // global-provider-ok: server-owned background reconciler runs under the server's provider + system user
+    if (runBackgroundWorkers) {
+        // Process pending RSU work from pre-restart (entity maps, field maps, sync)
+        processRSUPendingWork().catch(err => console.warn(`RSU pending work processing failed: ${err}`));
+        // Resume any integration syncs that were orphaned by the previous process restart
+        if (resumeUser) {
+            IntegrationEngine.Instance.ResumeOrphanedSyncs(resumeUser)
+                .catch(err => console.warn(`[IntegrationEngine] Orphaned sync resume failed: ${err}`));
+        }
+        // Force-fail any agent runs left 'Running' by a process that died (restart/crash/OOM) or whose
+        // terminal-state write never landed. Staleness-based, so it never touches runs another healthy
+        // instance is still heart-beating. The watchdog also self-registers for graceful-shutdown
+        // cancellation (via ShutdownRegistry) once it begins tracking this process's first live run.
+        if (resumeUser && Metadata.Provider instanceof DatabaseProviderBase) { // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
+            const sweepUser = resumeUser;
+            const sweepProvider = Metadata.Provider; // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
+            AgentRunWatchdog.SweepOrphanedRuns(sweepProvider, sweepUser)
+                .catch(err => console.warn(`[AgentRunWatchdog] Startup sweep failed: ${err}`));
+            // The watchdog repairs the RUN; nothing repaired the conversation detail, which is the row the
+            // chat actually renders from (MJ #4222). A process that dies mid-run leaves a terminal run
+            // beside a message that still claims to be generating, and it spins forever for anyone who
+            // opens it. Runs at boot (closes restart orphans) and on a timer (closes mid-life orphans),
+            // mirroring the watchdog's own two-phase shape.
+            const reconcileOrphans = () => ReconcileOrphanedConversationDetails(sweepProvider, sweepUser)
+                .catch(err => console.warn(`[OrphanDetailReconciler] Pass failed: ${err}`));
+            void reconcileOrphans();
+            const orphanDetailTimer = setInterval(() => void reconcileOrphans(), ORPHAN_DETAIL_SWEEP_INTERVAL_MS);
+            ShutdownRegistry.Instance.Register({
+                ShutdownName: 'OrphanDetailReconciler',
+                Shutdown: () => { clearInterval(orphanDetailTimer); },
+            });
+        }
+        // Launch the AI Agent Session janitor: run own-host orphan recovery once at boot, then a periodic tick
+        // that closes stale and over-deadline sessions and rebuilds lost realtime recordings from their
+        // crash-recovery shards (storage reads and writes). Self-registers with ShutdownRegistry, so its timer
+        // is cleared by the gracefulShutdown drain below (no explicit Stop() wiring needed here).
+        if (resumeUser && Metadata.Provider instanceof DatabaseProviderBase) { // global-provider-ok: server startup recovery — boot-time session janitor uses the server's own provider
+            SessionJanitor.Instance.Start(Metadata.Provider, resumeUser) // global-provider-ok: server-owned background reconciler runs under the server's provider + system user
+                .catch(err => console.warn(`[SessionJanitor] Startup failed: ${err}`));
+            // The bridge engine's orphan reconcile likewise starts only now that this process holds its port, so it can't
+            // take the live bridges of another MJAPI still serving that port for its own orphans (#5310).
+            StartBridgeOrphanReconciliation(Metadata.Provider, resumeUser); // global-provider-ok: server-owned background reconciler runs under the server's provider + system user
+        }
     }
     // Launch the durable task-graph dispatcher: claim Pending tasks, execute them with a fresh
     // provider each, and reconcile claims orphaned by a crash. Without this nothing ever picks up a
@@ -1437,7 +1492,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
         SuppressTaskGraphSubmission('MJ_DISABLE_TASK_GRAPH_DISPATCHER=1 is set on this host');
         LogStatus('[TaskGraphDispatcher] Disabled by MJ_DISABLE_TASK_GRAPH_DISPATCHER=1 — this process will neither accept nor execute task graphs.');
     }
-    else if (resumeUser && taskGraphPool instanceof sql.ConnectionPool) {
+    else if (runBackgroundWorkers && resumeUser && taskGraphPool instanceof sql.ConnectionPool) {
         StartTaskGraphDispatcher(taskGraphPool, resumeUser)
             .catch(err => console.warn(`[TaskGraphDispatcher] Startup failed: ${err}`));
     }
@@ -1446,7 +1501,7 @@ export const Serve = async (resolverPaths, app = CreateApp(), options) => {
     // ShutdownAll() drains it (up to 2 × shutdownDrainMs) before the HTTP server closes.
     // Not awaited: a slow engine load must not delay readiness, and a failure never stops the API.
     const workQueueProvider = Metadata.Provider; // global-provider-ok: server startup — the work-queue host runs on the server's own provider
-    if (configInfo.workQueue?.enabled && workQueueProvider instanceof DatabaseProviderBase) {
+    if (runBackgroundWorkers && configInfo.workQueue?.enabled && workQueueProvider instanceof DatabaseProviderBase) {
         const workQueuePool = dataSources[0]?.dataSource;
         const providerSource = new MJServerWorkQueueProviderSource(workQueuePool instanceof sql.ConnectionPool ? workQueuePool : null, workQueueProvider);
         StartWorkQueueHost(configInfo.workQueue, workQueueProvider, providerSource)
@@ -1911,10 +1966,11 @@ function mountRealtimeRecordingRoutes(app) {
     }
     const graceMs = () => configInfo.realtime.recordingAssemblyGraceSeconds * 1000;
     const systemUser = UserCache.Instance.GetSystemUser();
-    if (systemUser) {
+    const runWorkers = ShouldRunBackgroundWorkers(ResolveServerRole(configInfo.serverRole ?? process.env.MJ_SERVER_ROLE));
+    if (systemUser && runWorkers) {
         RecordingAssemblyScheduler.Instance.Start({ Provider: provider, SystemUser: systemUser, GraceMs: graceMs });
     }
-    else {
+    else if (!systemUser) {
         LogError('[Realtime] Recording assembly scheduler not started: no system user; the session janitor assembles recordings instead');
     }
     app.use(ApiRoute(REALTIME_RECORDING_MOUNT_PATH), cors(), CreateRealtimeRecordingRouter({
@@ -1923,7 +1979,7 @@ function mountRealtimeRecordingRoutes(app) {
             return user ? { ContextUser: user, Provider: provider } : null;
         },
         LoadOwnedSession: LoadOwnedRealtimeSession,
-        LoadConversationExternalID: LoadSessionConversationExternalID,
+        LoadConversationScope: LoadSessionConversationScope,
         CreateLedger: (ledgerProvider, contextUser) => new SqlRealtimeRecordingLedger(ledgerProvider, contextUser),
         Scheduler: RecordingAssemblyScheduler.Instance,
         GraceMs: graceMs,