@memberjunction/server 6.2.0-edge.5 → 6.2.0-edge.6

package/src/index.ts

@@ -38,7 +38,7 @@ import { default as jwt } from 'jsonwebtoken';
 import { contextFunction, CreateReadOnlyProvider, CreateUnifiedAuthMiddleware, getUserPayload } from './context.js';
 import { UserPayload } from './types.js';
 import { requireSystemUserDirective, publicDirective } from './directives/index.js';
-import { variablesLoggingMiddleware } from './logging/variablesLoggingMiddleware.js';
+import { ApplyRootVariablesLoggingMiddleware, variablesLoggingMiddleware } from './logging/variablesLoggingMiddleware.js';
 import { auditResolversForUndecoratedArgs } from './logging/bootAudit.js';
 import { StartupLogger } from './logging/StartupLogger.js';
 import { AuthProviderFactory } from '@memberjunction/auth-providers';
@@ -46,7 +46,7 @@ import createMSSQLConfig from './orm.js';
 import { setupRESTEndpoints } from './rest/setupRESTEndpoints.js';
 import { CreateRealtimeRecordingRouter, REALTIME_RECORDING_MOUNT_PATH } from './rest/RealtimeRecordingHandler.js';
 import { LoadOwnedRealtimeSession } from './realtimeSessions/ownedRealtimeSession.js';
-import { LoadSessionConversationExternalID } from './realtimeSessions/sessionAccess.js';
+import { AuthorizeAnonymousRoomSessions, LoadSessionConversationScope } from './realtimeSessions/sessionAccess.js';
 import { RecordingAssemblyScheduler } from './realtimeSessions/RecordingAssemblyScheduler.js';
 import {
   RealtimeSessionEventService,
@@ -73,11 +73,11 @@ import {
   SetHandoffOfferPublishHook,
   ParseReplicatedHandoffOfferUpdate,
 } from './resolvers/HumanHandoffResolver.js';
-import { RoomHandoffEngine } from '@memberjunction/livekit-room-server';
+import { RoomAuthorizationService, RoomHandoffEngine } from '@memberjunction/livekit-room-server';
 // Type-only: the provider is built in ./sharedCache.ts. Still needed here because the handoff-offer
 // fan-out below stays in this file — see the note at its call site.
 import type { RedisLocalStorageProvider } from '@memberjunction/redis-provider';
-import { IntegrationProgressEmitter } from '@memberjunction/integration-progress-artifacts';
+import { IntegrationProgressEmitter, SweepInterruptedRuns } from '@memberjunction/integration-progress-artifacts';
 import { PublishIntegrationProgress } from './resolvers/IntegrationProgressResolver.js';
 import { RegisterRSUProgressBridge } from './integration/RSUProgressBridge.js';
 import { ClientToolRequestManager, AgentRunWatchdog, SqlRealtimeRecordingLedger } from '@memberjunction/ai-agents';
@@ -103,10 +103,15 @@ import { ApiRoute, ApiRootPath, ConfiguredPublicUrl } from './apiRoot.js';
 import { MountHealthcheckRoute, MountPublicCoreRoutes, MountExtensionHealthRoute, MountAuthCatalogRoute, MountAuthenticatedCoreRoutes } from './coreRoutes.js';
 import { MetadataCacheRefreshIntervalSeconds } from './providerConfigUnits.js';
 import { CreateMetadataRefreshSignalHandler, METADATA_REFRESH_SIGNAL } from './metadataRefreshSignal.js';
+import { CacheInvalidationBroadcastManager } from './broadcast/index.js';
+import { ResolveServerRole, ShouldRunBackgroundWorkers } from './serverRoleConfigUnits.js';
 
 const cacheRefreshInterval = configInfo.databaseSettings.metadataCacheRefreshInterval;
 
 export { MetadataCacheRefreshIntervalSeconds } from './providerConfigUnits.js';
+export * from './serverRoleConfigUnits.js';
+export * from './broadcast/index.js';
+export { ApplyRootVariablesLoggingMiddleware } from './logging/variablesLoggingMiddleware.js';
 
 /**
  * Returns the configured database platform from the `DB_PLATFORM` environment
@@ -432,6 +437,9 @@ export const Serve = async (resolverPaths: Array<string>, app: Application = Cre
   SetHostInstancePort(graphqlPort);
   // The realtime bridge engine stamps its bridge rows with the same identity and reconciles orphans by it (#5310).
   BindBridgeEngineHostInstance();
+  // An anonymous visitor may join only the LiveKit agent-session room it owns by scope; the room gate cannot import
+  // this package's session-access decision, so it is handed in here, before any resolver runs (MJ#6044).
+  RoomAuthorizationService.Instance.SetAnonymousSessionOwnerCheck(AuthorizeAnonymousRoomSessions);
   // Level-gated startup logger. Resolves verbosity from telemetry.level (single
   // operator knob). At `standard` (default), per-phase timings are collapsed into
   // the one-line summary; at `verbose`+ each phase prints inline as before.
@@ -491,6 +499,8 @@ const setupComplete$ = new ReplaySubject(1);
     // Every API pool carries the statement and idle-in-transaction timeouts from connection #1.
     const pgConnectionConfig = BuildPostgreSQLConnectionConfig(pgEndpoint, configInfo.databaseSettings, 'api');
     const pgPool = new pg.default.Pool(ToPGPoolConfig(pgConnectionConfig));
+    // An idle connection the server drops is emitted on the pool; without a listener Node exits.
+    pgPool.on('error', (err) => console.error('[PostgreSQL pool] Idle connection error (connection evicted):', err.message));
 
     // Verify connection
     const testClient = await pgPool.connect();
@@ -508,6 +518,7 @@ const setupComplete$ = new ReplaySubject(1);
         ...ToPGPoolConfig(BuildPostgreSQLConnectionConfig({ ...pgEndpoint, ...pgReadOnlyCredentials }, configInfo.databaseSettings, 'read-only')),
         types: MJPostgresTypes,
       });
+      readOnlyPgPool.on('error', (err) => console.error('[PostgreSQL read-only pool] Idle connection error (connection evicted):', err.message));
       const readOnlyTestClient = await readOnlyPgPool.connect();
       try {
         for (const warning of await DescribeReadOnlyLoginOverreach(readOnlyTestClient, mj_core_schema)) {
@@ -594,6 +605,7 @@ const setupComplete$ = new ReplaySubject(1);
           'codegen',
         );
         const codegenPgPool = new pg.default.Pool(ToPGPoolConfig(codegenPgConnectionConfig));
+        codegenPgPool.on('error', (err) => console.error('[PostgreSQL CodeGen pool] Idle connection error (connection evicted):', err.message));
         const codegenTestClient = await codegenPgPool.connect();
         await codegenTestClient.query('SELECT 1');
         codegenTestClient.release();
@@ -632,7 +644,7 @@ const setupComplete$ = new ReplaySubject(1);
           startupLog.LogIf('verbose', 'RSU in-process CodeGen runner initialized (PostgreSQL).');
 
           // Inject CodeGen output paths for targeted git staging
-          const { initializeConfig } = await import('@memberjunction/codegen-lib');
+          const { initializeConfig, ResolveAdditionalSchemaInfoPath } = await import('@memberjunction/codegen-lib');
           const cgConfig = initializeConfig(rsuWorkDir);
           const outputPaths = (cgConfig.output ?? []).map((o: { directory: string }) => o.directory);
           RuntimeSchemaManager.Instance.SetCodeGenOutputPaths(outputPaths);
@@ -641,9 +653,12 @@ const setupComplete$ = new ReplaySubject(1);
           // Point RSU's soft PK/FK writer at the SAME file CodeGen reads (mj.config.cjs
           // `additionalSchemaInfo`), or RSU writes soft PKs to its own default path while
           // CodeGen reads a different one and skips integration tables with "No primary key found".
-          if (cgConfig.additionalSchemaInfo) {
-            RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(cgConfig.additionalSchemaInfo);
-            startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${cgConfig.additionalSchemaInfo}`);
+          // Inject the ABSOLUTE path CodeGen resolves (env override, else relative to the mj.config
+          // file's directory) so RSU, the resolvers and CodeGen agree whatever their cwd (#5858).
+          const resolvedSchemaInfoPath = ResolveAdditionalSchemaInfoPath();
+          if (resolvedSchemaInfoPath) {
+            RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(resolvedSchemaInfoPath);
+            startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${resolvedSchemaInfoPath}`);
           } else if (RuntimeSchemaManager.Instance.IsEnabled) {
             // U5 — boot-time assert: RSU is enabled but CodeGen has NO additionalSchemaInfo path
             // configured, so RSU's soft-PK/FK writes land in a file CodeGen never reads. Every
@@ -651,7 +666,7 @@ const setupComplete$ = new ReplaySubject(1);
             // named, and actionable — grep for RSU_ADDITIONAL_SCHEMA_INFO_DIVERGENCE.
             console.error(
               `[RSU_ADDITIONAL_SCHEMA_INFO_DIVERGENCE] Runtime Schema Update is ENABLED but mj.config.cjs has no ` +
-              `'additionalSchemaInfo' setting — RSU writes soft PK/FK definitions to '${process.env.RSU_ADDITIONAL_SCHEMA_INFO_PATH ?? 'additionalSchemaInfo.json'}' ` +
+              `'additionalSchemaInfo' setting (and RSU_ADDITIONAL_SCHEMA_INFO_PATH is unset) — RSU writes soft PK/FK definitions to '${RuntimeSchemaManager.Instance.AdditionalSchemaInfoPath}' ` +
               `while CodeGen reads none, so resolved soft primary keys will be silently LOST at codegen time. ` +
               `Set 'additionalSchemaInfo' in mj.config.cjs to the same file so the write path and read path agree.`
             );
@@ -777,7 +792,7 @@ const setupComplete$ = new ReplaySubject(1);
           startupLog.LogIf('verbose', 'RSU in-process CodeGen runner initialized.');
 
           // Inject CodeGen output paths for targeted git staging
-          const { initializeConfig } = await import('@memberjunction/codegen-lib');
+          const { initializeConfig, ResolveAdditionalSchemaInfoPath } = await import('@memberjunction/codegen-lib');
           const codegenConfig = initializeConfig(rsuWorkDir);
           const outputPaths = (codegenConfig.output ?? []).map((o: { directory: string }) => o.directory);
           RuntimeSchemaManager.Instance.SetCodeGenOutputPaths(outputPaths);
@@ -787,14 +802,17 @@ const setupComplete$ = new ReplaySubject(1);
           // `additionalSchemaInfo`). Without this, RSU writes soft PKs to its own default
           // path while CodeGen reads a different one and skips every integration table
           // with "No primary key found".
-          if (codegenConfig.additionalSchemaInfo) {
-            RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(codegenConfig.additionalSchemaInfo);
-            startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${codegenConfig.additionalSchemaInfo}`);
+          // Inject the ABSOLUTE path CodeGen resolves (env override, else relative to the mj.config
+          // file's directory) so RSU, the resolvers and CodeGen agree whatever their cwd (#5858).
+          const resolvedSchemaInfoPath = ResolveAdditionalSchemaInfoPath();
+          if (resolvedSchemaInfoPath) {
+            RuntimeSchemaManager.Instance.SetAdditionalSchemaInfoPath(resolvedSchemaInfoPath);
+            startupLog.LogIf('verbose', `RSU additionalSchemaInfo path: ${resolvedSchemaInfoPath}`);
           } else if (RuntimeSchemaManager.Instance.IsEnabled) {
             // U5 — boot-time assert: write path ≠ read path (see the PostgreSQL branch for detail).
             console.error(
               `[RSU_ADDITIONAL_SCHEMA_INFO_DIVERGENCE] Runtime Schema Update is ENABLED but mj.config.cjs has no ` +
-              `'additionalSchemaInfo' setting — RSU writes soft PK/FK definitions to '${process.env.RSU_ADDITIONAL_SCHEMA_INFO_PATH ?? 'additionalSchemaInfo.json'}' ` +
+              `'additionalSchemaInfo' setting (and RSU_ADDITIONAL_SCHEMA_INFO_PATH is unset) — RSU writes soft PK/FK definitions to '${RuntimeSchemaManager.Instance.AdditionalSchemaInfoPath}' ` +
               `while CodeGen reads none, so resolved soft primary keys will be silently LOST at codegen time. ` +
               `Set 'additionalSchemaInfo' in mj.config.cjs to the same file so the write path and read path agree.`
             );
@@ -1030,6 +1048,7 @@ const setupComplete$ = new ReplaySubject(1);
   // config.ts loads and validates at module scope, so importing it there would pull full config
   // validation into every import chain that touches the resolver, unit tests included.
   ConfigureRecordDataBroadcast(configInfo.cacheSettings?.recordDataBroadcastEntities);
+  CacheInvalidationBroadcastManager.Instance.Configure(configInfo.cacheSettings?.broadcast);
 
   // Global listener: broadcast CACHE_INVALIDATION to all browser clients whenever
   // ANY BaseEntity save/delete occurs on this server — regardless of whether it
@@ -1052,7 +1071,8 @@ const setupComplete$ = new ReplaySubject(1);
           // fires for server-internal saves too (agents, actions, orchestrator), which are exactly
           // the ones no browser session asked for.
           recordData:
-            beEvent.type === 'save' && MayBroadcastRecordData(entityName)
+            beEvent.type === 'save' && MayBroadcastRecordData(entityName, beEvent.baseEntity.EntityInfo)
+              // fls-guard: MayBroadcastRecordData refuses FLS entities and Record Changes
               ? JSON.stringify(beEvent.baseEntity.GetAll())
               : undefined,
         });
@@ -1103,7 +1123,6 @@ const setupComplete$ = new ReplaySubject(1);
     scalarsMap: [{ type: Date, scalar: GraphQLTimestamp }],
     emitSchemaFile: websiteRunFromPackage !== 1,
     pubSub,
-    globalMiddlewares: [variablesLoggingMiddleware],
   });
   const buildMs = performance.now() - tBuild;
 
@@ -1130,6 +1149,7 @@ const setupComplete$ = new ReplaySubject(1);
   auditResolversForUndecoratedArgs();
 
   const tTransform = performance.now();
+  schema = ApplyRootVariablesLoggingMiddleware(schema);
   schema = requireSystemUserDirective.transformer(schema);
   schema = publicDirective.transformer(schema);
 
@@ -1530,11 +1550,29 @@ const setupComplete$ = new ReplaySubject(1);
   // ─── GraphQL middleware (contextFunction reads req.userPayload, no re-auth) ─────
   app.use(
     graphqlRootPath,
-    cors<cors.CorsRequest>(),
+    cors<cors.CorsRequest>({
+      exposedHeaders: ['X-MJ-Cache-Seq', 'X-MJ-Server-ID'],
+    }),
     BodyParser.json({ limit: '50mb' }),
     // Express 5 leaves req.body as undefined for non-JSON or empty bodies;
     // Apollo Server's expressMiddleware requires req.body to be defined.
-    (req, _res, next) => { if (req.body === undefined) req.body = {}; next(); },
+    // Also piggyback the latest monotonic cache invalidation sequence and server process ID
+    // onto every GraphQL HTTP response so clients can detect and recover missed invalidations.
+    (req, res, next) => {
+      if (req.body === undefined) req.body = {};
+      const broadcastProvider = CacheInvalidationBroadcastManager.Instance.Provider;
+      res.setHeader('X-MJ-Cache-Seq', String(broadcastProvider.CurrentSequence));
+      res.setHeader('X-MJ-Server-ID', broadcastProvider.ServerProcessID);
+      const originalWriteHead = res.writeHead.bind(res);
+      res.writeHead = ((...args: Parameters<typeof res.writeHead>) => {
+        if (!res.headersSent) {
+          res.setHeader('X-MJ-Cache-Seq', String(broadcastProvider.CurrentSequence));
+          res.setHeader('X-MJ-Server-ID', broadcastProvider.ServerProcessID);
+        }
+        return originalWriteHead(...args);
+      }) as typeof res.writeHead;
+      next();
+    },
     expressMiddleware(apolloServer, {
       context: contextFunction({
                                  setupComplete$,
@@ -1549,9 +1587,26 @@ const setupComplete$ = new ReplaySubject(1);
     app.use(mw);
   }
 
+  const serverRole = ResolveServerRole(configInfo.serverRole ?? process.env.MJ_SERVER_ROLE);
+  const runBackgroundWorkers = ShouldRunBackgroundWorkers(serverRole);
+  if (!runBackgroundWorkers) {
+    LogStatus(`[ServerRole] Running in '${serverRole}' mode — background worker services are disabled on this replica.`);
+  }
+
+  // Close out run artifacts a dead process left open, BEFORE anything below can start a new run.
+  //
+  // A killed process (OOM killer, SIGKILL, host reboot) never writes its run's terminal result, so
+  // the run reads as in-flight forever: the wizard spins, the active-operations panel shows a step
+  // that will never advance, and no failure is reported anywhere. Sound only before the first new
+  // run: artifacts are process-local, so anything still in flight as THIS process starts belongs
+  // to a process that no longer exists. That is why it runs here, ahead of the scheduler, the
+  // integration sync worker (its Start() polls immediately), the HTTP listener (a request can
+  // start a discovery) and pending RSU work, and why it is awaited. It needs none of them.
+  await SweepInterruptedRuns(undefined, msg => console.warn(msg));
+
   // Initialize and start scheduled jobs service if enabled
   let scheduledJobsService: ScheduledJobsService | null = null;
-  if (configInfo.scheduledJobs?.enabled) {
+  if (runBackgroundWorkers && configInfo.scheduledJobs?.enabled) {
     try {
       scheduledJobsService = new ScheduledJobsService(configInfo.scheduledJobs);
       await scheduledJobsService.Initialize();
@@ -1565,7 +1620,7 @@ const setupComplete$ = new ReplaySubject(1);
   // Initialize and start the integration sync worker if enabled (PR 1 item 8).
   // Off by default — deployments that don't opt in keep running syncs inline.
   let integrationSyncWorker: IntegrationSyncWorkerService | null = null;
-  if (configInfo.integrationSyncWorker?.enabled) {
+  if (runBackgroundWorkers && configInfo.integrationSyncWorker?.enabled) {
     try {
       integrationSyncWorker = new IntegrationSyncWorkerService(configInfo.integrationSyncWorker);
       await integrationSyncWorker.Initialize();
@@ -1603,52 +1658,55 @@ const setupComplete$ = new ReplaySubject(1);
   }
   startupLog.PrintSummary();
 
-  // Process pending RSU work from pre-restart (entity maps, field maps, sync)
-  processRSUPendingWork().catch(err => console.warn(`RSU pending work processing failed: ${err}`));
-
-  // Resume any integration syncs that were orphaned by the previous process restart
   const resumeUser = UserCache.Instance.GetSystemUser();
-  if (resumeUser) {
-    IntegrationEngine.Instance.ResumeOrphanedSyncs(resumeUser)
-      .catch(err => console.warn(`[IntegrationEngine] Orphaned sync resume failed: ${err}`));
-  }
 
-  // Force-fail any agent runs left 'Running' by a process that died (restart/crash/OOM) or whose
-  // terminal-state write never landed. Staleness-based, so it never touches runs another healthy
-  // instance is still heart-beating. The watchdog also self-registers for graceful-shutdown
-  // cancellation (via ShutdownRegistry) once it begins tracking this process's first live run.
-  if (resumeUser && Metadata.Provider instanceof DatabaseProviderBase) { // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
-    const sweepUser = resumeUser;
-    const sweepProvider = Metadata.Provider; // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
-    AgentRunWatchdog.SweepOrphanedRuns(sweepProvider, sweepUser)
-      .catch(err => console.warn(`[AgentRunWatchdog] Startup sweep failed: ${err}`));
-
-    // The watchdog repairs the RUN; nothing repaired the conversation detail, which is the row the
-    // chat actually renders from (MJ #4222). A process that dies mid-run leaves a terminal run
-    // beside a message that still claims to be generating, and it spins forever for anyone who
-    // opens it. Runs at boot (closes restart orphans) and on a timer (closes mid-life orphans),
-    // mirroring the watchdog's own two-phase shape.
-    const reconcileOrphans = () =>
-      ReconcileOrphanedConversationDetails(sweepProvider, sweepUser)
-        .catch(err => console.warn(`[OrphanDetailReconciler] Pass failed: ${err}`));
-    void reconcileOrphans();
-    const orphanDetailTimer = setInterval(() => void reconcileOrphans(), ORPHAN_DETAIL_SWEEP_INTERVAL_MS);
-    ShutdownRegistry.Instance.Register({
-      ShutdownName: 'OrphanDetailReconciler',
-      Shutdown: () => { clearInterval(orphanDetailTimer); },
-    });
-  }
+  if (runBackgroundWorkers) {
+    // Process pending RSU work from pre-restart (entity maps, field maps, sync)
+    processRSUPendingWork().catch(err => console.warn(`RSU pending work processing failed: ${err}`));
 
-  // Launch the AI Agent Session janitor: run own-host orphan recovery once at boot, then a periodic tick
-  // that closes stale and over-deadline sessions and rebuilds lost realtime recordings from their
-  // crash-recovery shards (storage reads and writes). Self-registers with ShutdownRegistry, so its timer
-  // is cleared by the gracefulShutdown drain below (no explicit Stop() wiring needed here).
-  if (resumeUser && Metadata.Provider instanceof DatabaseProviderBase) { // global-provider-ok: server startup recovery — boot-time session janitor uses the server's own provider
-    SessionJanitor.Instance.Start(Metadata.Provider, resumeUser) // global-provider-ok: server-owned background reconciler runs under the server's provider + system user
-      .catch(err => console.warn(`[SessionJanitor] Startup failed: ${err}`));
-    // The bridge engine's orphan reconcile likewise starts only now that this process holds its port, so it can't
-    // take the live bridges of another MJAPI still serving that port for its own orphans (#5310).
-    StartBridgeOrphanReconciliation(Metadata.Provider, resumeUser); // global-provider-ok: server-owned background reconciler runs under the server's provider + system user
+    // Resume any integration syncs that were orphaned by the previous process restart
+    if (resumeUser) {
+      IntegrationEngine.Instance.ResumeOrphanedSyncs(resumeUser)
+        .catch(err => console.warn(`[IntegrationEngine] Orphaned sync resume failed: ${err}`));
+    }
+
+    // Force-fail any agent runs left 'Running' by a process that died (restart/crash/OOM) or whose
+    // terminal-state write never landed. Staleness-based, so it never touches runs another healthy
+    // instance is still heart-beating. The watchdog also self-registers for graceful-shutdown
+    // cancellation (via ShutdownRegistry) once it begins tracking this process's first live run.
+    if (resumeUser && Metadata.Provider instanceof DatabaseProviderBase) { // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
+      const sweepUser = resumeUser;
+      const sweepProvider = Metadata.Provider; // global-provider-ok: server startup recovery — one-shot orphaned-run sweep at boot
+      AgentRunWatchdog.SweepOrphanedRuns(sweepProvider, sweepUser)
+        .catch(err => console.warn(`[AgentRunWatchdog] Startup sweep failed: ${err}`));
+
+      // The watchdog repairs the RUN; nothing repaired the conversation detail, which is the row the
+      // chat actually renders from (MJ #4222). A process that dies mid-run leaves a terminal run
+      // beside a message that still claims to be generating, and it spins forever for anyone who
+      // opens it. Runs at boot (closes restart orphans) and on a timer (closes mid-life orphans),
+      // mirroring the watchdog's own two-phase shape.
+      const reconcileOrphans = () =>
+        ReconcileOrphanedConversationDetails(sweepProvider, sweepUser)
+          .catch(err => console.warn(`[OrphanDetailReconciler] Pass failed: ${err}`));
+      void reconcileOrphans();
+      const orphanDetailTimer = setInterval(() => void reconcileOrphans(), ORPHAN_DETAIL_SWEEP_INTERVAL_MS);
+      ShutdownRegistry.Instance.Register({
+        ShutdownName: 'OrphanDetailReconciler',
+        Shutdown: () => { clearInterval(orphanDetailTimer); },
+      });
+    }
+
+    // Launch the AI Agent Session janitor: run own-host orphan recovery once at boot, then a periodic tick
+    // that closes stale and over-deadline sessions and rebuilds lost realtime recordings from their
+    // crash-recovery shards (storage reads and writes). Self-registers with ShutdownRegistry, so its timer
+    // is cleared by the gracefulShutdown drain below (no explicit Stop() wiring needed here).
+    if (resumeUser && Metadata.Provider instanceof DatabaseProviderBase) { // global-provider-ok: server startup recovery — boot-time session janitor uses the server's own provider
+      SessionJanitor.Instance.Start(Metadata.Provider, resumeUser) // global-provider-ok: server-owned background reconciler runs under the server's provider + system user
+        .catch(err => console.warn(`[SessionJanitor] Startup failed: ${err}`));
+      // The bridge engine's orphan reconcile likewise starts only now that this process holds its port, so it can't
+      // take the live bridges of another MJAPI still serving that port for its own orphans (#5310).
+      StartBridgeOrphanReconciliation(Metadata.Provider, resumeUser); // global-provider-ok: server-owned background reconciler runs under the server's provider + system user
+    }
   }
 
   // Launch the durable task-graph dispatcher: claim Pending tasks, execute them with a fresh
@@ -1677,7 +1735,7 @@ const setupComplete$ = new ReplaySubject(1);
     // StartTaskGraphDispatcher); this gives the agent seam the same.
     SuppressTaskGraphSubmission('MJ_DISABLE_TASK_GRAPH_DISPATCHER=1 is set on this host');
     LogStatus('[TaskGraphDispatcher] Disabled by MJ_DISABLE_TASK_GRAPH_DISPATCHER=1 — this process will neither accept nor execute task graphs.');
-  } else if (resumeUser && taskGraphPool instanceof sql.ConnectionPool) {
+  } else if (runBackgroundWorkers && resumeUser && taskGraphPool instanceof sql.ConnectionPool) {
     StartTaskGraphDispatcher(taskGraphPool, resumeUser)
       .catch(err => console.warn(`[TaskGraphDispatcher] Startup failed: ${err}`));
   }
@@ -1687,7 +1745,7 @@ const setupComplete$ = new ReplaySubject(1);
   // ShutdownAll() drains it (up to 2 × shutdownDrainMs) before the HTTP server closes.
   // Not awaited: a slow engine load must not delay readiness, and a failure never stops the API.
   const workQueueProvider = Metadata.Provider; // global-provider-ok: server startup — the work-queue host runs on the server's own provider
-  if (configInfo.workQueue?.enabled && workQueueProvider instanceof DatabaseProviderBase) {
+  if (runBackgroundWorkers && configInfo.workQueue?.enabled && workQueueProvider instanceof DatabaseProviderBase) {
     const workQueuePool = dataSources[0]?.dataSource;
     const providerSource = new MJServerWorkQueueProviderSource(workQueuePool instanceof sql.ConnectionPool ? workQueuePool : null, workQueueProvider);
     StartWorkQueueHost(configInfo.workQueue, workQueueProvider, providerSource)
@@ -2193,9 +2251,10 @@ function mountRealtimeRecordingRoutes(app: Application): boolean {
   }
   const graceMs = (): number => configInfo.realtime.recordingAssemblyGraceSeconds * 1000;
   const systemUser = UserCache.Instance.GetSystemUser();
-  if (systemUser) {
+  const runWorkers = ShouldRunBackgroundWorkers(ResolveServerRole(configInfo.serverRole ?? process.env.MJ_SERVER_ROLE));
+  if (systemUser && runWorkers) {
     RecordingAssemblyScheduler.Instance.Start({ Provider: provider, SystemUser: systemUser, GraceMs: graceMs });
-  } else {
+  } else if (!systemUser) {
     LogError('[Realtime] Recording assembly scheduler not started: no system user; the session janitor assembles recordings instead');
   }
   app.use(ApiRoute(REALTIME_RECORDING_MOUNT_PATH), cors<cors.CorsRequest>(), CreateRealtimeRecordingRouter({
@@ -2204,7 +2263,7 @@ function mountRealtimeRecordingRoutes(app: Application): boolean {
       return user ? { ContextUser: user, Provider: provider } : null;
     },
     LoadOwnedSession: LoadOwnedRealtimeSession,
-    LoadConversationExternalID: LoadSessionConversationExternalID,
+    LoadConversationScope: LoadSessionConversationScope,
     CreateLedger: (ledgerProvider, contextUser) => new SqlRealtimeRecordingLedger(ledgerProvider, contextUser),
     Scheduler: RecordingAssemblyScheduler.Instance,
     GraceMs: graceMs,

package/src/integration/CustomColumnPromoter.ts

@@ -300,7 +300,12 @@ export class IntegrationCustomColumnPromoter {
         // step with the database.
         let batchResults: RSUPipelineResult[] = [];
         if (batchInputs.length > 0) {
-            const batch = await RuntimeSchemaManager.Instance.RunPipelineBatch(batchInputs);
+            // Retrying variant. This runs UNATTENDED, mid-sync, with no operator watching: a
+            // transient ExecuteMigration / RunCodeGen / CompileTypeScript / RestartMJAPI failure
+            // drops the whole promotion, and the columns simply stay in overflow until something
+            // triggers a promotion again. The wrapper replays only those four steps and only while
+            // nothing in the batch was applied, so a partially-applied batch is still returned as-is.
+            const batch = await RuntimeSchemaManager.Instance.RunPipelineBatchWithRetry(batchInputs);
             batchResults = batch.Results ?? [];
             // M3: make the freshly-created EntityFields + regenerated sprocs visible in-process
             // ONCE for the whole batch. CRITICAL: the spread below builds sproc calls from this

package/src/logging/variablesLoggingMiddleware.ts

@@ -1,7 +1,7 @@
-import type { MiddlewareFn } from 'type-graphql';
+import type { MiddlewareFn, ResolverData } from 'type-graphql';
 import { getMetadataStorage } from 'type-graphql';
-import { getNamedType } from 'graphql';
-import type { GraphQLArgument } from 'graphql';
+import { defaultFieldResolver, getNamedType } from 'graphql';
+import type { GraphQLArgument, GraphQLField, GraphQLObjectType, GraphQLSchema } from 'graphql';
 import { configInfo } from '../config.js';
 import type { AppContext } from '../types.js';
 import { RedactArg } from './secretRedactor.js';
@@ -191,6 +191,62 @@ export const VariablesLoggingMiddleware: MiddlewareFn<AppContext> = async (actio
 /** @deprecated Use {@link VariablesLoggingMiddleware}. */
 export const variablesLoggingMiddleware: MiddlewareFn<AppContext> = VariablesLoggingMiddleware;
 
+/**
+ * Attaches {@link VariablesLoggingMiddleware} ONLY to root `Query`, `Mutation`, and `Subscription`
+ * fields on the compiled GraphQLSchema instead of registering it via TypeGraphQL `globalMiddlewares`.
+ *
+ * Registering a global middleware in TypeGraphQL wraps every scalar and child field resolver across
+ * all 300+ `@ObjectType` entities in an async middleware chain, allocating $O(\text{Rows} \times \text{Fields})$
+ * Promises per query even when `logVariables` is false or `info.path.prev !== undefined`.
+ * Wrapping only the root operation fields reduces middleware invocations from ~15,000 per 500-row
+ * `RunView` response down to 1.
+ */
+export function ApplyRootVariablesLoggingMiddleware(
+  schema: GraphQLSchema,
+  middleware: MiddlewareFn<AppContext> = VariablesLoggingMiddleware
+): GraphQLSchema {
+  const rootTypes: Array<GraphQLObjectType | null | undefined> = [
+    schema.getQueryType(),
+    schema.getMutationType(),
+    schema.getSubscriptionType(),
+  ];
+
+  for (const rootType of rootTypes) {
+    if (!rootType) continue;
+    const fields = rootType.getFields();
+    for (const field of Object.values(fields) as Array<GraphQLField<unknown, AppContext>>) {
+      if (field.resolve) {
+        const originalResolve = field.resolve;
+        field.resolve = (root, args, context, info) => {
+          const action: ResolverData<AppContext> = {
+            root,
+            args: args as Record<string, unknown>,
+            context,
+            info,
+          };
+          return middleware(action, async () => originalResolve(root, args, context, info));
+        };
+      }
+      if (field.subscribe) {
+        const originalSubscribe = field.subscribe;
+        field.subscribe = (root, args, context, info) => {
+          const action: ResolverData<AppContext> = {
+            root,
+            args: args as Record<string, unknown>,
+            context,
+            info,
+          };
+          return middleware(action, async () =>
+            originalSubscribe ? originalSubscribe(root, args, context, info) : defaultFieldResolver(root, args, context, info)
+          );
+        };
+      }
+    }
+  }
+
+  return schema;
+}
+
 /**
  * Heuristic: an arg is "custom" (i.e. needs `@NoLog` discipline because metadata can't redact it)
  * when its GraphQL input type is not a known `Create<X>Input` / `Update<X>Input` for an entity

package/src/realtimeSessions/README.md

@@ -64,9 +64,13 @@ The principal comes from the **server-authenticated** context (`userPayload.user
 argument. An id argument only selects *which of the caller's own sessions*.
 
 A caller may use a session when `AIAgentSession.UserID` equals their user (`UUIDsEqual`). Widget guests all
-share one Anonymous user, so for an anonymous/magic-link principal that is **not enough**: the guest's
-signed per-session scope (`MagicLinkScope.ResourceID`) must equal the session's conversation `ExternalID`.
-An anonymous principal with no scope is refused outright.
+share one Anonymous user, so for an anonymous/magic-link principal that is **not enough**: the session's
+conversation `ExternalID` must equal the principal's scope identity (`ResolveSessionScopeIdentity`): the
+signed `MagicLinkScope.ResourceID` when the token carries one, else the anonymous visitor's `mj_sid`.
+`SessionManager` stamps that same value when it creates the conversation. An anonymous principal with
+neither is refused outright. A *named* user with a resource scope must match too, which keeps one person's
+different links apart; but it may also use its own session on a conversation that carries no stamp (one
+started from the Conversations app) when its own row-level security let it read that conversation.
 
 - Refusals are uniform ("session not found") whether the session is missing or simply not yours, so ids
   cannot be probed.

package/src/realtimeSessions/SessionVerificationPortsBase.ts

@@ -54,6 +54,7 @@ function toSnapshot(loaded: LoadedSessionForAccess): SessionSnapshot {
         AgentSessionID: Session.ID,
         OwnerUserID: Session.UserID,
         ConversationExternalID: loaded.ConversationExternalID,
+        ConversationReadable: loaded.ConversationReadable,
         Status: Session.Status,
         ConfigRaw: Session.Config_,
         // `__mj_CreatedAt` is the row's creation instant — i.e. the session start. It is read-only and

package/src/realtimeSessions/ownedRealtimeSession.ts

@@ -8,6 +8,7 @@
 import { IMetadataProvider, LogError, UserInfo } from '@memberjunction/core';
 import { UUIDsEqual } from '@memberjunction/global';
 import type { MJAIAgentSessionEntity } from '@memberjunction/core-entities';
+import { AuthorizeSessionAccess, LoadSessionConversationScope } from './sessionAccess.js';
 
 const SESSION_ENTITY = 'MJ: AI Agent Sessions';
 
@@ -19,7 +20,7 @@ export class RealtimeSessionNotFoundError extends Error {
     }
 }
 
-/** The session exists but belongs to another user. */
+/** The session exists but the caller may not use it: another user owns it, or (for a scoped or anonymous caller) its conversation carries another scope identity. */
 export class RealtimeSessionOwnershipError extends Error {
     constructor(public readonly AgentSessionID: string) {
         super('Not authorized: you do not own this realtime session');
@@ -28,13 +29,15 @@ export class RealtimeSessionOwnershipError extends Error {
 }
 
 /**
- * Loads a session under the caller's identity and enforces ownership (`UserID === contextUser.ID`). It
- * deliberately does NOT reject `Closed` sessions: the final channel-state flush and the recording terminal
- * signal legitimately arrive after the session closed. A scoped-anonymous widget guest owns the session it
- * minted, so the same check covers it; loading as the caller also applies the entity's row-level security.
+ * Loads a session under the caller's identity and enforces ownership: `UserID === contextUser.ID`, then the
+ * scope rule of {@link AuthorizeSessionAccess} (every anonymous visitor shares one Anonymous user, so the
+ * session's conversation `ExternalID` must equal the caller's scope identity). It deliberately does NOT
+ * reject `Closed` sessions: the final channel-state flush and the recording terminal signal legitimately
+ * arrive after the session closed. Loading as the caller also applies the entity's row-level security.
  *
  * @throws RealtimeSessionNotFoundError when the row does not load
- * @throws RealtimeSessionOwnershipError when the caller is not the owner (logged with both user ids)
+ * @throws RealtimeSessionOwnershipError when another user owns the session (logged with both user ids) or
+ *   the scope rule refuses the caller (logged with the refusal reason)
  */
 export async function LoadOwnedRealtimeSession(
     agentSessionId: string,
@@ -51,5 +54,16 @@ export async function LoadOwnedRealtimeSession(
         );
         throw new RealtimeSessionOwnershipError(agentSessionId);
     }
+    const conversation = await LoadSessionConversationScope(session, contextUser, provider);
+    const access = AuthorizeSessionAccess(
+        { SessionUserID: session.UserID, ConversationExternalID: conversation.ExternalID, ConversationReadable: conversation.Readable },
+        contextUser,
+    );
+    if (!access.Allowed) {
+        LogError(
+            `LoadOwnedRealtimeSession: session access refused (${access.Reason}) — user ${contextUser.ID} attempted to operate on session ${agentSessionId}`,
+        );
+        throw new RealtimeSessionOwnershipError(agentSessionId);
+    }
     return session;
 }