@memberjunction/server 6.1.0-edge.2 → 6.1.0-edge.3

package/src/generic/ResolverBase.ts

@@ -408,7 +408,8 @@ export class ResolverBase {
           viewInput.AfterKey
             ? CompositeKey.FromKeyValuePairs((viewInput.AfterKey as { KeyValuePairs: { FieldName: string; Value: string }[] }).KeyValuePairs)
             : undefined,
-          viewInput.BypassCache
+          viewInput.BypassCache,
+          viewInput.DataSource
         );
       }
       else {
@@ -451,7 +452,8 @@ export class ResolverBase {
         viewInput.StartRow,
         viewInput.Aggregates,
         undefined,
-        viewInput.BypassCache
+        viewInput.BypassCache,
+        viewInput.DataSource
       );
     } catch (err) {
       console.log(err);
@@ -497,7 +499,8 @@ export class ResolverBase {
         viewInput.StartRow,
         viewInput.Aggregates,
         undefined,
-        viewInput.BypassCache
+        viewInput.BypassCache,
+        viewInput.DataSource
       );
     } catch (err) {
       console.log(err);
@@ -571,6 +574,7 @@ export class ResolverBase {
           userPayload,
           aggregates: viewInput.Aggregates,
           bypassCache: viewInput.BypassCache,
+          dataSource: viewInput.DataSource,
         });
       } catch (err) {
         LogError(err);
@@ -767,7 +771,8 @@ export class ResolverBase {
     startRow: number | undefined,
     aggregates?: AggregateExpression[],
     afterKey?: CompositeKey,
-    bypassCache?: boolean
+    bypassCache?: boolean,
+    dataSource?: 'Live' | 'Materialized'
   ) {
     try {
       if (!viewInfo || !userPayload) return null;
@@ -837,6 +842,7 @@ export class ResolverBase {
           ResultType: rt,
           Aggregates: aggregates,
           BypassCache: bypassCache,
+          DataSource: dataSource,
         },
         user
       );
@@ -972,6 +978,7 @@ export class ResolverBase {
           ResultType: rt,
           Aggregates: param.aggregates,
           BypassCache: param.bypassCache,
+          DataSource: param.dataSource,
         });
       }
 

package/src/generic/RunViewResolver.ts

@@ -177,6 +177,13 @@ export class RunViewByIDInput {
       'Optional, when true bypasses ALL server-side caching for this view run — the pre-check cache lookup is skipped and the result is not stored in the cache. Use for maintenance/audit queries that must see true database state, or to force-refresh views whose filters reference rows the server cache invalidator cannot follow (e.g., cross-entity subqueries against vwListDetails).',
   })
   BypassCache?: boolean;
+
+  @Field(() => String, {
+    nullable: true,
+    description:
+      "Optional source-of-truth selector for a base-view materialization: 'Live' (default) reads the entity's live base view; 'Materialized' reads its materialized snapshot wrapper view (materialized_vw<CodeName>). Only request 'Materialized' for an entity that actually has a base-view materialization: it resolves to the wrapper view by naming convention with NO existence check or fallback, so requesting it for a non-materialized entity targets a view that does not exist and the query errors.",
+  })
+  DataSource?: 'Live' | 'Materialized';
 }
 
 @InputType()
@@ -291,6 +298,13 @@ export class RunViewByNameInput {
       'Optional, when true bypasses ALL server-side caching for this view run — the pre-check cache lookup is skipped and the result is not stored in the cache. Use for maintenance/audit queries that must see true database state, or to force-refresh views whose filters reference rows the server cache invalidator cannot follow (e.g., cross-entity subqueries against vwListDetails).',
   })
   BypassCache?: boolean;
+
+  @Field(() => String, {
+    nullable: true,
+    description:
+      "Optional source-of-truth selector for a base-view materialization: 'Live' (default) reads the entity's live base view; 'Materialized' reads its materialized snapshot wrapper view (materialized_vw<CodeName>). Only request 'Materialized' for an entity that actually has a base-view materialization: it resolves to the wrapper view by naming convention with NO existence check or fallback, so requesting it for a non-materialized entity targets a view that does not exist and the query errors.",
+  })
+  DataSource?: 'Live' | 'Materialized';
 }
 
 @InputType()
@@ -391,6 +405,13 @@ export class RunDynamicViewInput {
       'Optional, when true bypasses ALL server-side caching for this view run — the pre-check cache lookup is skipped and the result is not stored in the cache. Use for maintenance/audit queries that must see true database state, or to force-refresh views whose filters reference rows the server cache invalidator cannot follow (e.g., cross-entity subqueries against vwListDetails).',
   })
   BypassCache?: boolean;
+
+  @Field(() => String, {
+    nullable: true,
+    description:
+      "Optional source-of-truth selector for a base-view materialization: 'Live' (default) reads the entity's live base view; 'Materialized' reads its materialized snapshot wrapper view (materialized_vw<CodeName>). Only request 'Materialized' for an entity that actually has a base-view materialization: it resolves to the wrapper view by naming convention with NO existence check or fallback, so requesting it for a non-materialized entity targets a view that does not exist and the query errors.",
+  })
+  DataSource?: 'Live' | 'Materialized';
 }
 
 @InputType()
@@ -520,6 +541,13 @@ export class RunViewGenericInput {
       'Optional, when true bypasses ALL server-side caching for this view run — the pre-check cache lookup is skipped and the result is not stored in the cache. Use for maintenance/audit queries that must see true database state, or to force-refresh views whose filters reference rows the server cache invalidator cannot follow (e.g., cross-entity subqueries against vwListDetails).',
   })
   BypassCache?: boolean;
+
+  @Field(() => String, {
+    nullable: true,
+    description:
+      "Optional source-of-truth selector for a base-view materialization: 'Live' (default) reads the entity's live base view; 'Materialized' reads its materialized snapshot wrapper view (materialized_vw<CodeName>). Only request 'Materialized' for an entity that actually has a base-view materialization: it resolves to the wrapper view by naming convention with NO existence check or fallback, so requesting it for a non-materialized entity targets a view that does not exist and the query errors.",
+  })
+  DataSource?: 'Live' | 'Materialized';
 }
 
 //****************************************************************************
@@ -1082,6 +1110,9 @@ export class RunViewResolver extends ResolverBase {
           // Forward the aggregate request to the engine (B40) — omitted here as well as in the
           // client's input map, so aggregates never reached InternalRunView on this transport.
           Aggregates: item.params.Aggregates,
+          // DataSource must be forwarded on this transport too (same as the InternalRunView/RunViews maps):
+          // otherwise a CacheLocal batch's DataSource:'Materialized' request silently reads the live view.
+          DataSource: item.params.DataSource,
           AfterKey: item.params.AfterKey
             ? CompositeKey.FromKeyValuePairs(item.params.AfterKey.KeyValuePairs)
             : undefined,

package/src/index.ts

@@ -57,14 +57,17 @@ import { DataSourceInfo, raiseEvent } from './types.js';
 
 import { ExternalChangeDetectorEngine } from '@memberjunction/external-change-detection';
 import { ScheduledJobsService } from './services/ScheduledJobsService.js';
+import { IntegrationSyncWorkerService } from './services/IntegrationSyncWorkerService.js';
 import { LocalCacheManager, StartupManager, TelemetryManager, TelemetryLevel, LogStatus, SetVerboseLogging } from '@memberjunction/core';
-import { getSystemUser } from './auth/index.js';
+import { getSystemUser, validateAuthProvidersRegistered } from './auth/index.js';
+import { createAuthProviderCatalogRouter, AUTH_CATALOG_MOUNT_PATH } from './auth/AuthProviderCatalogRouter.js';
 import { GetAPIKeyEngine } from '@memberjunction/api-keys';
 import { RedisLocalStorageProvider } from '@memberjunction/redis-provider';
 import { GenericDatabaseProvider } from '@memberjunction/generic-database-provider';
 import { PubSubManager } from './generic/PubSubManager.js';
 import { IntegrationProgressEmitter } from '@memberjunction/integration-progress-artifacts';
 import { PublishIntegrationProgress } from './resolvers/IntegrationProgressResolver.js';
+import { RegisterRSUProgressBridge } from './integration/RSUProgressBridge.js';
 import { ClientToolRequestManager, AgentRunWatchdog } from '@memberjunction/ai-agents';
 import { SessionJanitor } from './agentSessions/index.js';
 import { StartTaskGraphDispatcher } from './services/StartTaskGraphDispatcher.js';
@@ -79,9 +82,12 @@ import {
   MJScheduledJobEntity,
 } from '@memberjunction/core-entities';
 import { ServerExtensionLoader, ServerExtensionConfig } from '@memberjunction/server-extensions-core';
+import { MetadataCacheRefreshIntervalSeconds } from './providerConfigUnits.js';
 
 const cacheRefreshInterval = configInfo.databaseSettings.metadataCacheRefreshInterval;
 
+export { MetadataCacheRefreshIntervalSeconds } from './providerConfigUnits.js';
+
 /**
  * Returns the configured database platform from the `DB_PLATFORM` environment
  * variable, falling back to `'sqlserver'` when the env var is unset. An
@@ -100,6 +106,10 @@ export function getDbType(): DatabasePlatform {
 
 export { MaxLength } from 'class-validator';
 export * from 'type-graphql';
+// Named re-export so Open App generated resolvers get a live ESM binding for
+// Int/Float/ID. `export *` from type-graphql can leave these undefined for
+// later importers, which makes schema build fail on ViewResult.RowCount.
+export { Int, Float, ID } from 'type-graphql';
 export { NewUserBase } from './auth/newUsers.js';
 export { configInfo, DEFAULT_SERVER_CONFIG } from './config.js';
 export { ServerExtensionLoader, BaseServerExtension } from '@memberjunction/server-extensions-core';
@@ -204,6 +214,7 @@ import type { RequestHandler, ErrorRequestHandler } from 'express';
 import type { ApolloServerPlugin } from '@apollo/server';
 import type { GraphQLSchema } from 'graphql';
 import { BaseServerMiddleware } from './middleware/BaseServerMiddleware.js';
+import { SuppressTaskGraphSubmission } from '@memberjunction/ai-core-plus';
 
 export type MJServerOptions = {
   onBeforeServe?: () => void | Promise<void>;
@@ -339,6 +350,10 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
     const pgStartupMode = ResolveStartupMode({ configValue: configInfo.startup?.mode, defaultMode: 'full' });
     await StartupManagerImport.Instance.Startup(false, sysUser || backupSysUser, provider, { mode: pgStartupMode.mode });
 
+    // Both provider sources have now had their turn — config/env at module load, metadata via
+    // AuthProviderEngine's startup hook — so "no providers at all" is finally a meaningful check.
+    validateAuthProvidersRegistered();
+
     // Monkey-patch SQLServerDataProvider.ExecuteSQLWithPool to support PostgreSQL
     // Generated resolvers call this static method with bracket-quoted SQL.
     // When the pool is our PG-compat wrapper, translate and execute via pg.Pool.
@@ -489,11 +504,16 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
       startupLog.LogIf('verbose', 'Read-only Connection Pool has been initialized.');
     }
 
-    const config = new SQLServerProviderConfigData(pool, mj_core_schema, cacheRefreshInterval / 1000); // convert ms to seconds (checkRefreshIntervalSeconds)
+    // cacheRefreshInterval is configured in ms; checkRefreshIntervalSeconds declares seconds — see providerConfigUnits.ts
+    const config = new SQLServerProviderConfigData(pool, mj_core_schema, MetadataCacheRefreshIntervalSeconds(cacheRefreshInterval));
     // MJAPI is a long-running server, so entry-point default is 'full' engine pre-warm;
     // MJ_STARTUP_MODE / mj.config.cjs startup.mode can override per the shared precedence chain
     const startupMode = ResolveStartupMode({ configValue: configInfo.startup?.mode, defaultMode: 'full' });
     await setupSQLServerClient(config, { mode: startupMode.mode });
+
+    // See the note on the PostgreSQL path above: this is the first point at which both the
+    // config/env providers and the metadata catalog have been registered.
+    validateAuthProvidersRegistered();
     lap('Metadata + Provider Setup', tPhase);
     startupLog.BeginPhase('Initializing data provider');
     const md = new Metadata(); // global-provider-ok: bootstrap
@@ -524,7 +544,8 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
         await codegenPool.connect();
 
         const { RuntimeSchemaManager } = await import('@memberjunction/schema-engine');
-        const codegenConfig = new SQLServerProviderConfigData(codegenPool, mj_core_schema, cacheRefreshInterval / 1000); // convert ms to seconds (checkRefreshIntervalSeconds)
+        // Same ms→seconds seam as the main provider config above — see providerConfigUnits.ts
+        const codegenConfig = new SQLServerProviderConfigData(codegenPool, mj_core_schema, MetadataCacheRefreshIntervalSeconds(cacheRefreshInterval));
         const codegenProvider = new SQLServerDataProvider();
         await codegenProvider.Config(codegenConfig);
         RuntimeSchemaManager.Instance.SetDDLProvider(codegenProvider);
@@ -835,6 +856,14 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
     })
   );
 
+  // Give Runtime Schema Update runs the same durable, tailable event stream as syncs and connector
+  // builds. `IntegrationRunKind` has always had an 'RSU' kind and RUN_KIND_TO_TOPIC has always
+  // mapped it to an 'RSU' channel, but nothing published to it — the only live signal was polling
+  // RuntimeSchemaUpdateStatus, which reports the current step and keeps no history, and which goes
+  // silent entirely across the API restart the pipeline performs on itself. Registered AFTER the
+  // publish hook above so the first RSU event also reaches live subscribers.
+  RegisterRSUProgressBridge();
+
   // Global listener: broadcast CACHE_INVALIDATION to all browser clients whenever
   // ANY BaseEntity save/delete occurs on this server — regardless of whether it
   // originated from a GraphQL mutation or internal server-side code (agents, actions,
@@ -1107,7 +1136,9 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
     const { callbackRouter, authenticatedRouter } = createOAuthCallbackHandler({
       publicUrl: oauthPublicUrl,
       successRedirectUrl: `${oauthPublicUrl}/oauth/success`,
-      errorRedirectUrl: `${oauthPublicUrl}/oauth/error`
+      errorRedirectUrl: `${oauthPublicUrl}/oauth/error`,
+      // Constrains where a caller-supplied frontendReturnUrl may point (open-redirect guard).
+      allowedFrontendOrigins: configInfo.cors?.allowedOrigins ?? ['*']
     });
     oauthAuthenticatedRouter = authenticatedRouter;
 
@@ -1256,6 +1287,14 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
     res.status(allHealthy ? 200 : 503).json({ extensions: results });
   });
 
+  // ─── Public authentication-provider catalog (PUBLIC, before auth mw) ──────
+  // The browser needs the provider list BEFORE it holds a token, so this is necessarily
+  // unauthenticated and must mount ahead of the auth middleware. It publishes only the
+  // non-secret allow-list (see AuthProviderEngine.GetPublicCatalog) — the same values a
+  // single-provider SPA already compiled into its bundle.
+  app.use(AUTH_CATALOG_MOUNT_PATH, cors<cors.CorsRequest>(), createAuthProviderCatalogRouter());
+  startupLog.LogIf('verbose', `[Auth] Public provider catalog registered at ${AUTH_CATALOG_MOUNT_PATH}/providers`);
+
   // ─── Unified auth middleware (replaces both REST authMiddleware and contextFunction auth) ─────
   app.use(createUnifiedAuthMiddleware(dataSources));
 
@@ -1354,6 +1393,20 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
     }
   }
 
+  // Initialize and start the integration sync worker if enabled (PR 1 item 8).
+  // Off by default — deployments that don't opt in keep running syncs inline.
+  let integrationSyncWorker: IntegrationSyncWorkerService | null = null;
+  if (configInfo.integrationSyncWorker?.enabled) {
+    try {
+      integrationSyncWorker = new IntegrationSyncWorkerService(configInfo.integrationSyncWorker);
+      await integrationSyncWorker.Initialize();
+      integrationSyncWorker.Start();
+    } catch (error) {
+      console.error('❌ Failed to start integration sync worker:', error);
+      // Don't throw — an unavailable worker must not prevent the API from serving
+    }
+  }
+
   // Data hooks are now registered via BaseServerMiddleware classes above
   // (e.g., MJTenantFilterMiddleware registers PreRunView and PreSave hooks).
   // No config-bag hook registration needed.
@@ -1425,7 +1478,15 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
   const taskGraphPool = dataSources[0]?.dataSource;
   const taskGraphDispatcherDisabled = process.env.MJ_DISABLE_TASK_GRAPH_DISPATCHER === '1';
   if (taskGraphDispatcherDisabled) {
-    LogStatus('[TaskGraphDispatcher] Disabled by MJ_DISABLE_TASK_GRAPH_DISPATCHER=1 — submitted graphs will not be executed by this process.');
+    // AND REFUSE SUBMISSIONS, not just execution (R3-11). The durable submitter registers through
+    // the generated manifest unconditionally, so without this the host went on ACCEPTING graphs it
+    // had no intention of running: the agent submitted, promised the user a follow-up, and parked
+    // its run `Paused` — with the graph `Pending` and the run parked forever, no per-submission
+    // diagnostics anywhere, and the stale graph executing hours later if anyone unset the flag.
+    // The entity-action seam already had this treatment (its submitter registers inside
+    // StartTaskGraphDispatcher); this gives the agent seam the same.
+    SuppressTaskGraphSubmission('MJ_DISABLE_TASK_GRAPH_DISPATCHER=1 is set on this host');
+    LogStatus('[TaskGraphDispatcher] Disabled by MJ_DISABLE_TASK_GRAPH_DISPATCHER=1 — this process will neither accept nor execute task graphs.');
   } else if (resumeUser && taskGraphPool instanceof sql.ConnectionPool) {
     StartTaskGraphDispatcher(taskGraphPool, resumeUser)
       .catch(err => console.warn(`[TaskGraphDispatcher] Startup failed: ${err}`));
@@ -1472,6 +1533,17 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
       }
     }
 
+    // Stop the integration sync worker's polling. In-flight runs finish on their own —
+    // their leases are heartbeat-renewed, and killing them here would only strand rows.
+    if (integrationSyncWorker?.IsRunning) {
+      try {
+        integrationSyncWorker.Stop();
+        console.log(`✅ Integration sync worker stopped (${integrationSyncWorker.InFlightCount} run(s) still in flight)`);
+      } catch (error) {
+        console.error('❌ Error stopping integration sync worker:', error);
+      }
+    }
+
     // Drain anything self-registered with ShutdownRegistry — QueueManager,
     // future engines/services with timers/intervals/listeners. Each is
     // responsible for being idempotent and not throwing.
@@ -1510,15 +1582,32 @@ export const serve = async (resolverPaths: Array<string>, app: Application = cre
   });
 };
 
+/**
+ * Age at which an unprocessed `MJ: RSU Pending Works` row is reported as stranded.
+ * A row older than this survived at least one full restart cycle without being completed.
+ */
+const RSU_PENDING_WORK_STALE_MINUTES = 30;
+
 /**
  * Process pending RSU work left from a pre-restart Apply All.
- * Reads pending work files, creates entity maps + field maps, starts sync.
+ * Reads the durable `MJ: RSU Pending Works` queue, creates entity maps + field maps,
+ * starts sync, and marks each row Completed only AFTER its work actually succeeded —
+ * so a crash mid-processing leaves the row Pending and re-processable on the next boot.
  */
 async function processRSUPendingWork(): Promise<void> {
   // Dynamic import — schema-engine is not yet published to npm, only exists as a workspace package
   const { RuntimeSchemaManager } = await import('@memberjunction/schema-engine');
   const rsm = RuntimeSchemaManager.Instance;
-  const pendingItems = await rsm.ReadAndClearPendingWork();
+
+  // Get system user for server-side operations — needed to read the queue itself
+  const systemUser = UserCache.Instance.Users.find(u => u.Type?.trim().toLowerCase() === 'owner') ?? UserCache.Instance.Users[0];
+  if (!systemUser) {
+    console.warn(`[RSU] No system user found — cannot process pending work`);
+    return;
+  }
+
+  // Rows older than this were left behind by an earlier process and are surfaced as stale.
+  const pendingItems = await rsm.ReadPendingWork(systemUser, RSU_PENDING_WORK_STALE_MINUTES);
   if (pendingItems.length === 0) return;
 
   console.log(`[RSU] Processing ${pendingItems.length} pending work item(s) from pre-restart...`);
@@ -1526,15 +1615,11 @@ async function processRSUPendingWork(): Promise<void> {
   // Wait a moment for metadata to be fully loaded
   await new Promise(resolve => setTimeout(resolve, 3000));
 
-  for (const item of pendingItems) {
+  for (const pending of pendingItems) {
+    const pendingWorkID = pending.ID;
+    const item = pending.Work;
     try {
       const md = new Metadata(); // global-provider-ok: server startup recovery — runs once before any per-request context exists
-      // Get system user for server-side operations
-      const systemUser = UserCache.Instance.Users.find(u => u.Type?.trim().toLowerCase() === 'owner') ?? UserCache.Instance.Users[0];
-      if (!systemUser) {
-        console.warn(`[RSU] No system user found, skipping pending work for ${item.CompanyIntegrationID}`);
-        continue;
-      }
 
       await Metadata.Provider.Refresh(); // global-provider-ok: server startup recovery — one-shot global cache refresh
 
@@ -1547,6 +1632,7 @@ async function processRSUPendingWork(): Promise<void> {
       }, systemUser);
       const companyIntegration = ciResult.Results[0];
       if (!companyIntegration) {
+        await rsm.FailPendingWork(pendingWorkID, `CompanyIntegration ${item.CompanyIntegrationID} not found`, systemUser);
         console.warn(`[RSU] CompanyIntegration ${item.CompanyIntegrationID} not found`);
         continue;
       }
@@ -1559,11 +1645,13 @@ async function processRSUPendingWork(): Promise<void> {
       }, systemUser);
       const integrationEntity = integrationResult.Results[0];
       if (!integrationEntity) {
+        await rsm.FailPendingWork(pendingWorkID, `Integration entity for ${integrationName} not found`, systemUser);
         console.warn(`[RSU] Integration entity for ${integrationName} not found`);
         continue;
       }
       const connector = ConnectorFactory.Resolve(integrationEntity);
       if (!connector) {
+        await rsm.FailPendingWork(pendingWorkID, `Connector for ${integrationName} not found`, systemUser);
         console.warn(`[RSU] Connector for ${integrationName} not found`);
         continue;
       }
@@ -1788,8 +1876,13 @@ async function processRSUPendingWork(): Promise<void> {
           console.warn(`[RSU] Schedule creation failed: ${schedErr}`);
         }
       }
+
+      // Only NOW is the work durably done — close the row out.
+      await rsm.CompletePendingWork(pendingWorkID, systemUser);
     } catch (err) {
-      console.error(`[RSU] Failed to process pending work for ${item.CompanyIntegrationID}: ${err}`);
+      const message = err instanceof Error ? err.message : String(err);
+      console.error(`[RSU] Failed to process pending work for ${item.CompanyIntegrationID}: ${message}`);
+      await rsm.FailPendingWork(pendingWorkID, message, systemUser);
     }
   }
 

package/src/integration/RSUProgressBridge.ts

@@ -0,0 +1,248 @@
+/**
+ * @fileoverview Bridges RSU pipeline lifecycle events onto the integration progress artifact
+ * stream, so a Runtime Schema Update is observable the same way a sync or a connector build is.
+ *
+ * The gap this closes: `IntegrationRunKind` has always included `'RSU'` and
+ * `RUN_KIND_TO_TOPIC` has always mapped it to an `'RSU'` subscription channel — but nothing in
+ * production ever constructed an emitter with that kind, so the channel carried no traffic and
+ * `IntegrationTailRunEvents` had no RSU runs to tail. The only live signal was polling
+ * `RuntimeSchemaUpdateStatus`, which reports the CURRENT step and nothing else: no history, no
+ * durable record, and nothing at all to read after a mid-run API restart — which the RSU pipeline
+ * does to itself, by design, as one of its own steps.
+ *
+ * Layering: `@memberjunction/schema-engine` must not depend on the progress artifacts package, so
+ * it publishes plain {@link RSUObserverEvent}s and this module — one layer up, where both are
+ * already dependencies — owns the emitter lifecycle.
+ */
+import { IntegrationProgressEmitter } from '@memberjunction/integration-progress-artifacts';
+import type { EmitterOptions } from '@memberjunction/integration-progress-artifacts';
+import { RuntimeSchemaManager } from '@memberjunction/schema-engine';
+import type { RSUObserverEvent } from '@memberjunction/schema-engine';
+import { LogError } from '@memberjunction/core';
+import { BaseSingleton } from '@memberjunction/global';
+
+/** The emitter stage name used for run-level (non-step) events. */
+const RUN_STAGE = 'RSUPipeline';
+
+/**
+ * How often an in-flight step reports that it is still alive.
+ *
+ * RSU's expensive steps — CodeGen, TypeScript compile, npm install — run for minutes with no
+ * intermediate observer event, so without this the stream goes silent for the entire step and a
+ * watcher cannot tell "compiling" from "hung". 30s matches the interval the agent notification
+ * conventions ask for on long phases.
+ */
+const HEARTBEAT_INTERVAL_MS = 30_000;
+
+/**
+ * Owns one emitter per RSU pipeline run and translates observer events into progress events.
+ *
+ * A {@link BaseSingleton} because the state here is inherently process-wide AND has to be readable
+ * from elsewhere. `RuntimeSchemaManager` is itself a singleton and serializes its runs behind the
+ * pipeline lock, so at most one RSU run is ever in flight and one current-emitter slot is sufficient
+ * — but the reason for the singleton is {@link CurrentRunID}: a resolver that has just triggered an
+ * RSU needs to hand its client a run to tail, and it can only do that if it can reach this instance.
+ * Constructing the bridge and keeping the only reference inside the observer closure made that
+ * accessor unreachable from any caller.
+ *
+ * NOT a `BaseEngine`: there is no metadata to load and nothing to `Config()`. The distinction is
+ * "single shared instance" (this) versus "cached metadata with a load lifecycle" (that).
+ */
+export class RSUProgressBridge extends BaseSingleton<RSUProgressBridge> {
+    private emitter: IntegrationProgressEmitter | null = null;
+    private currentRunID: string | null = null;
+    /** Liveness timer for the step currently in flight — see {@link HEARTBEAT_INTERVAL_MS}. */
+    private heartbeat: ReturnType<typeof setInterval> | null = null;
+    private emitterOptions: EmitterOptions = {};
+
+    /** The one bridge for this process. */
+    public static get Instance(): RSUProgressBridge {
+        return super.getInstance<RSUProgressBridge>();
+    }
+
+    /**
+     * Options passed through to each run's emitter. Set once, before the first run.
+     *
+     * The only field that matters in practice is `rootDir` — left unset in production so RSU runs
+     * land in the same `logs/integration-runs/` tree as every other run kind and
+     * `IntegrationTailRunEvents` finds them without configuration. Tests point it at a temp dir.
+     *
+     * A settable property rather than a constructor argument because `BaseSingleton.getInstance`
+     * requires a zero-argument constructor.
+     */
+    public Configure(emitterOptions: EmitterOptions): void {
+        this.emitterOptions = emitterOptions;
+    }
+
+    /**
+     * The run ID of the in-flight RSU run, or null when idle. Lets a caller that triggered an RSU
+     * hand a client something to tail via `IntegrationTailRunEvents` / the `RSU` subscription
+     * channel.
+     */
+    public get CurrentRunID(): string | null {
+        return this.currentRunID;
+    }
+
+    /**
+     * Abandons any in-flight run state and stops its heartbeat, without emitting a terminal event.
+     *
+     * Needed because singleton state outlives a single caller: a test (or a server tearing down)
+     * must be able to return the bridge to idle without the next run inheriting a stale emitter or
+     * leaking an interval that keeps the process alive.
+     */
+    public Reset(): void {
+        this.stopHeartbeat();
+        this.emitter = null;
+        this.currentRunID = null;
+    }
+
+    /** The observer to hand to {@link RuntimeSchemaManager.PipelineObserver}. */
+    public readonly Observe = (event: RSUObserverEvent): void => {
+        switch (event.Kind) {
+            case 'run.start':
+                return this.onRunStart(event);
+            case 'step.start':
+                return this.onStepStart(event);
+            case 'step.end':
+                return this.onStepEnd(event);
+            case 'run.end':
+                return this.onRunEnd(event);
+        }
+    };
+
+    private onRunStart(event: Extract<RSUObserverEvent, { Kind: 'run.start' }>): void {
+        // A retry re-enters RunPipelineBatch, so it legitimately starts a NEW run. If a previous
+        // emitter is still open (a throw escaped before run.end), close it rather than leak it.
+        if (this.emitter) this.closeCurrent('Superseded by a new RSU run');
+
+        const runID = IntegrationProgressEmitter.newRunID('rsu');
+        this.currentRunID = runID;
+        this.emitter = new IntegrationProgressEmitter(
+            {
+                runID,
+                runKind: 'RSU',
+                triggerType: 'Pipeline',
+                startedAt: new Date().toISOString(),
+                context: {
+                    itemCount: event.ItemCount,
+                    descriptions: event.Descriptions,
+                    affectedTables: event.AffectedTables,
+                    stepTotal: event.StepTotal,
+                },
+            },
+            this.emitterOptions,
+        );
+        this.emitter.runStart(
+            `RSU pipeline started — ${event.ItemCount} migration(s), ${event.StepTotal} expected step(s): ` +
+            event.Descriptions.join('; ')
+        );
+    }
+
+    private onStepStart(event: Extract<RSUObserverEvent, { Kind: 'step.start' }>): void {
+        if (!this.emitter) return;
+        this.emitter.stageStart(event.Name, this.positionLabel(event.StepIndex, event.StepTotal));
+        this.startHeartbeat(event.Name);
+    }
+
+    private onStepEnd(event: Extract<RSUObserverEvent, { Kind: 'step.end' }>): void {
+        this.stopHeartbeat();
+        if (!this.emitter) return;
+        // A failed step is a stage error, not a stage completion — the run may still continue (a
+        // per-item migration failure does not abort the batch), so this is deliberately not fatal.
+        if (event.Status === 'failed') {
+            this.emitter.stageError(event.Name, event.Message, { durationMs: event.DurationMs, code: 'RSU_STEP_FAILED' });
+            return;
+        }
+        // Deliberately no `counts` here. Counts roll up into the run aggregate, and the countable
+        // unit of an RSU run is MIGRATIONS, not steps — reporting "processed: 12" for a 12-step
+        // single-migration run would read as 12 migrations. The run-level quartet is emitted once,
+        // in onRunEnd. Step position lives in the stage name + the step.start message.
+        this.emitter.stageComplete(event.Name);
+    }
+
+    private onRunEnd(event: Extract<RSUObserverEvent, { Kind: 'run.end' }>): void {
+        this.stopHeartbeat();
+        const emitter = this.emitter;
+        if (!emitter) return;
+        this.emitter = null;
+        this.currentRunID = null;
+        const summary = `${event.SuccessCount}/${event.TotalCount} migration(s) succeeded`;
+        // The one authoritative applied quartet for the run, in migrations — this is what feeds
+        // the reader's aggregateCounts.
+        emitter.stageComplete(RUN_STAGE, {
+            processed: event.TotalCount,
+            succeeded: event.SuccessCount,
+            failed: event.FailureCount,
+            totalKnown: event.TotalCount,
+        });
+        // Terminal writes are async (they flush the JSONL + write result.json). The observer
+        // contract is synchronous, so this is intentionally fire-and-forget with a logged catch —
+        // a progress-artifact write must never surface as a pipeline failure.
+        const terminal = event.Success
+            ? emitter.complete(`RSU pipeline complete — ${summary}`)
+            : emitter.fail(
+                `RSU pipeline failed at '${event.ErrorStep ?? 'unknown step'}' — ${summary}: ` +
+                `${event.ErrorMessage ?? 'no error message'}`,
+                'rsu-pipeline-failed'
+            );
+        void terminal.catch(err => LogError(`RSUProgressBridge: terminal write failed — ${err}`));
+    }
+
+    /**
+     * Starts reporting liveness for the step that just opened. Any previous timer is cleared first,
+     * so a missed `step.end` can never leave two timers running.
+     */
+    private startHeartbeat(stage: string): void {
+        this.stopHeartbeat();
+        const startedAt = Date.now();
+        this.heartbeat = setInterval(() => {
+            const elapsedSec = Math.round((Date.now() - startedAt) / 1000);
+            this.emitter?.heartbeat(stage, `${stage} still running — ${elapsedSec}s elapsed`);
+        }, HEARTBEAT_INTERVAL_MS);
+        // Never hold the process open on a progress timer.
+        this.heartbeat.unref?.();
+    }
+
+    /** Stops the liveness timer. Safe to call when none is running. */
+    private stopHeartbeat(): void {
+        if (!this.heartbeat) return;
+        clearInterval(this.heartbeat);
+        this.heartbeat = null;
+    }
+
+    /** Closes an orphaned emitter so a leaked run does not stay in-flight forever. */
+    private closeCurrent(reason: string): void {
+        this.stopHeartbeat();
+        const emitter = this.emitter;
+        this.emitter = null;
+        this.currentRunID = null;
+        void emitter?.fail(reason, 'rsu-run-abandoned')
+            .catch(err => LogError(`RSUProgressBridge: abandon write failed — ${err}`));
+    }
+
+    /**
+     * "step 4 of 12" — omitted entirely when the determinate counter isn't armed.
+     *
+     * Explicit null/undefined checks rather than truthiness: a 0 index is falsy, and silently
+     * dropping the label for the first step of a 0-based counter would be a bug that only appears
+     * if the pipeline's numbering ever changes.
+     */
+    private positionLabel(stepIndex?: number, stepTotal?: number): string | undefined {
+        if (stepIndex == null || stepTotal == null) return undefined;
+        return `step ${stepIndex} of ${stepTotal}`;
+    }
+}
+
+/**
+ * Attaches a fresh {@link RSUProgressBridge} to the RuntimeSchemaManager singleton. Call once at
+ * server startup, after the progress publish hook is registered so the very first RSU event also
+ * reaches live subscribers.
+ *
+ * @returns the bridge, so callers can read {@link RSUProgressBridge.CurrentRunID}.
+ */
+export function RegisterRSUProgressBridge(emitterOptions: EmitterOptions = {}): RSUProgressBridge {
+    const bridge = RSUProgressBridge.Instance;
+    bridge.Configure(emitterOptions);
+    RuntimeSchemaManager.Instance.PipelineObserver = bridge.Observe;
+    return bridge;
+}