@memberjunction/core 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/generic/baseEngine.js

@@ -3,7 +3,8 @@ import { TelemetryManager } from "./telemetryManager.js";
 import { BehaviorSubject, Subject } from "rxjs";
 import { buffer, debounceTime, filter } from "rxjs/operators";
 import { RunView } from "../views/runView.js";
-import { LogError, LogStatus } from "./logging.js";
+import { LogError, LogStatus, LogStatusEx, LogWarning } from "./logging.js";
+import { EntityInfo } from "./entityInfo.js";
 import { Metadata } from "./metadata.js";
 import { ProviderType } from "./interfaces.js";
 import { BaseInfo } from "./baseInfo.js";
@@ -38,11 +39,47 @@ export class BaseEnginePropertyConfig extends BaseInfo {
          * environment, and the same is true for MJAPI/Server based environments. If you need network based notification, additional infrastructure will be needed to implement that.
          */
         this.AutoRefresh = true;
+        /**
+         * When set to true, the engine will use the LocalCacheManager to cache results locally.
+         * On subsequent loads (unless forceRefresh is true), cached results will be validated
+         * via smart cache check and returned from local storage if still current.
+         *
+         * @default false
+         */
+        this.CacheLocal = false;
         // now copy the values from init to this object
         if (init)
             Object.assign(this, init);
     }
 }
+/**
+ * Timestamps closer than this are treated as equal, as the smart-cache check does: the database
+ * and JavaScript round datetimes differently.
+ */
+const SWEEP_TIMESTAMP_TOLERANCE_MS = 1000;
+/** Resolves on a later macrotask, after I/O callbacks already queued have run. Works in browsers too. */
+function yieldToEventLoop() {
+    return new Promise(resolve => setTimeout(resolve, 0));
+}
+/**
+ * 64-bit FNV-1a over a list of strings, as hex. Synchronous and dependency-free so it runs in the
+ * browser too; for comparing snapshots, not for security.
+ * @internal
+ */
+function hashStrings(values) {
+    let h1 = 0x811c9dc5;
+    let h2 = 0x01000193 ^ 0x5bd1e995;
+    for (const value of values) {
+        for (let i = 0; i < value.length; i++) {
+            const c = value.charCodeAt(i);
+            h1 = Math.imul(h1 ^ c, 0x01000193);
+            h2 = Math.imul(h2 ^ c, 0x01000193);
+        }
+        h1 = Math.imul(h1 ^ 0x0a, 0x01000193); // separator, so ['ab','c'] != ['a','bc']
+        h2 = Math.imul(h2 ^ 0x0b, 0x01000193);
+    }
+    return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');
+}
 /**
  * Thrown when engine data is accessed but was never loaded because the
  * current user lacks read permissions on the underlying entities.
@@ -161,8 +198,16 @@ export class BaseEngine extends BaseSingleton {
          * reconciles it. Keyed by config PropertyName. See {@link beginConfigRefresh}.
          */
         this._configRefreshGeneration = new Map();
+        /**
+         * Reloads started by {@link OnExternalCacheChange}, keyed by config PropertyName. While one is
+         * running, further events for that property set `Again` instead of starting another, so a
+         * burst of peer events costs at most two reloads per property. See {@link reloadAfterExternalChange}.
+         */
+        this._externalChangeReloads = new Map();
         this._entityEventDebounceTime = 1500; // Default debounce time in milliseconds (1.5 seconds)
         this._eventRefreshRetryTimers = new Map();
+        /** Configs waiting for the cache manager to initialize before their callbacks can register. */
+        this._deferredCacheCallbackConfigs = null;
     }
     /**
      * Observable that emits when any data property changes due to a refresh.
@@ -486,9 +531,9 @@ export class BaseEngine extends BaseSingleton {
         }
         else if (!BaseEngine._systemUserFallbackWarned.has(this.constructor.name)) {
             BaseEngine._systemUserFallbackWarned.add(this.constructor.name);
-            LogStatus(`${this.constructor.name}: could not resolve the MJ system user for server-side loading — ` +
+            LogWarning(`${this.constructor.name}: could not resolve the MJ system user for server-side loading — ` +
                 `falling back to the calling user. This engine's data will reflect that user's permissions ` +
-                `and is cached process-wide, so a restricted caller can leave incomplete data cached for everyone.`);
+                `and is cached process-wide, so a restricted caller can leave incomplete data cached for everyone.`, 'Cache');
         }
     }
     /**********************************************************************
@@ -628,7 +673,7 @@ export class BaseEngine extends BaseSingleton {
             }
             if (event.type === 'delete' || event.type === 'save') {
                 const eName = event.baseEntity.EntityInfo.Name.toLowerCase().trim();
-                const matchingConfigs = this.Configs.filter((config) => {
+                const matchingConfigs = this._metadataConfigs.filter((config) => {
                     return config.AutoRefresh && config.EntityName && config.EntityName.trim().toLowerCase() === eName;
                 });
                 if (matchingConfigs.length === 0) {
@@ -695,7 +740,7 @@ export class BaseEngine extends BaseSingleton {
             if (!entityName) {
                 return true; // No entity name, nothing to do
             }
-            const matchingConfigs = this.Configs.filter((config) => {
+            const matchingConfigs = this._metadataConfigs.filter((config) => {
                 return config.AutoRefresh && config.EntityName && config.EntityName.trim().toLowerCase() === entityName;
             });
             if (matchingConfigs.length === 0) {
@@ -844,7 +889,7 @@ export class BaseEngine extends BaseSingleton {
             const entityName = event.baseEntity.EntityInfo.Name.toLowerCase().trim();
             if (!this._entityEventSubjects.has(entityName)) {
                 // Find matching config to get custom debounce time
-                const matchingConfig = this.Configs.find(c => c.EntityName?.trim().toLowerCase() === entityName);
+                const matchingConfig = this._metadataConfigs.find(c => c.EntityName?.trim().toLowerCase() === entityName);
                 // Use config-specific debounce time or fall back to default
                 const debounceTimeValue = matchingConfig?.DebounceTime ?? this.EntityEventDebounceTime;
                 const subject = new Subject();
@@ -916,7 +961,7 @@ export class BaseEngine extends BaseSingleton {
         try {
             const entityName = events[0].baseEntity.EntityInfo.Name.toLowerCase().trim();
             let refreshCount = 0;
-            for (const config of this.Configs) {
+            for (const config of this._metadataConfigs) {
                 if (config.AutoRefresh && config.Type === 'entity' && config.EntityName?.trim().toLowerCase() === entityName) {
                     const classified = events.map(e => ({ event: e, disposition: this.classifyEventForConfig(config, e) }));
                     const needingWork = classified.filter(c => c.disposition.action === 'refresh');
@@ -1308,7 +1353,7 @@ export class BaseEngine extends BaseSingleton {
         if (config.CacheLocal) {
             this.syncLocalCacheForConfig(config, event).catch(e => {
                 // Log status but don't fail - cache will self-correct on next fetch
-                LogStatus(`BaseEngine: Failed to sync local cache for ${config.EntityName}: ${e}`);
+                LogWarning(`BaseEngine: Failed to sync local cache for ${config.EntityName}: ${e}`, 'Cache');
             });
         }
     }
@@ -1351,6 +1396,14 @@ export class BaseEngine extends BaseSingleton {
             return;
         }
         const entity = event.baseEntity;
+        // Inside a batch the cache manager records this change and rewrites every slot indexed for
+        // the entity — this config's included — once, when the batch closes. The event is checked
+        // as well as the provider because this runs after an await, by which time a batch that
+        // took the event may already have closed.
+        const cache = LocalCacheManager.Instance;
+        if (cache.IsBatchingEntityEvents(entity?.ProviderToUse) || cache.WasTakenByEntityEventBatch(event)) {
+            return;
+        }
         // Get the connection string from the provider for fingerprint generation
         // The provider is needed because fingerprints include connection prefix
         const provider = this.ProviderToUse;
@@ -1469,26 +1522,37 @@ export class BaseEngine extends BaseSingleton {
     CheckPermissionsOrSkipAll(configs, contextUser) {
         if (configs.length === 0)
             return configs;
+        // First check whether any configured entity is missing from metadata altogether
+        // (e.g., an optional OpenApp engine registered in the bundle whose database schema
+        // is not installed in the target environment). Skip cleanly instead of firing N
+        // guaranteed-to-fail RunView calls and logging N error stack traces at startup.
+        const missingEntities = this.findMissingEntities(configs);
+        if (missingEntities.length > 0) {
+            LogStatusEx({
+                message: `${this.constructor.name}: Skipping ${configs.length} entity config(s) — entity metadata not installed for: ${missingEntities.join(', ')}`,
+                verboseOnly: true,
+                category: 'Cache',
+            });
+            for (const config of configs) {
+                if (config.AddToObject !== false) {
+                    this[config.PropertyName] = [];
+                }
+                this._dataMap.set(config.PropertyName, {
+                    entityName: config.EntityName,
+                    data: [],
+                    loadedSuccessfully: true,
+                });
+            }
+            return [];
+        }
         // Determine the user to check permissions for — on the client side contextUser
         // may be undefined, so fall back to the current logged-in user from Metadata.
         const user = contextUser || this.ProviderToUse?.CurrentUser;
         if (!user)
             return configs; // Can't check without a user — proceed with normal loading
-        const md = this.ProviderToUse;
-        const deniedEntities = [];
-        for (const config of configs) {
-            if (!config.EntityName)
-                continue;
-            const entityInfo = md.EntityByName(config.EntityName);
-            if (!entityInfo)
-                continue; // Entity not in metadata — let RunView handle it
-            const perms = entityInfo.GetUserPermisions(user);
-            if (!perms || !perms.CanRead) {
-                deniedEntities.push(config.EntityName);
-            }
-        }
+        const deniedEntities = this.findUnreadableEntities(configs, user);
         if (deniedEntities.length > 0) {
-            LogStatus(`${this.constructor.name}: Skipping ${configs.length} entity config(s) — user ${user.Email} lacks read permission on: ${deniedEntities.join(', ')}`);
+            LogWarning(`${this.constructor.name}: Skipping ${configs.length} entity config(s) — user ${user.Email} lacks read permission on: ${deniedEntities.join(', ')}`, 'Cache');
             // Mark all entity configs as successfully loaded with empty data.
             // This is not a failure — it's expected behavior for limited-permission users.
             // Marking as successful allows the engine to set _loaded = true and avoid
@@ -1510,17 +1574,54 @@ export class BaseEngine extends BaseSingleton {
         }
         return configs;
     }
+    /**
+     * Returns entity names among `configs` that do not exist in the loaded metadata provider.
+     * Only checks when the metadata provider actually has entities loaded (`md.Entities.length > 0`)
+     * so unit tests with empty mock providers are unaffected.
+     */
+    findMissingEntities(configs) {
+        const md = this.ProviderToUse;
+        if (!md?.Entities || md.Entities.length === 0) {
+            return [];
+        }
+        const missing = [];
+        for (const config of configs) {
+            if (!config.EntityName)
+                continue;
+            if (!md.EntityByName(config.EntityName)) {
+                missing.push(config.EntityName);
+            }
+        }
+        return missing;
+    }
+    /** The entity names among `configs` that `user` cannot read, per current metadata. */
+    findUnreadableEntities(configs, user) {
+        const md = this.ProviderToUse;
+        const denied = [];
+        for (const config of configs) {
+            if (!config.EntityName)
+                continue;
+            const entityInfo = md.EntityByName(config.EntityName);
+            if (!entityInfo)
+                continue; // Entity not in metadata — let RunView handle it
+            const perms = entityInfo.GetUserPermisions(user);
+            if (!perms || !perms.CanRead) {
+                denied.push(config.EntityName);
+            }
+        }
+        return denied;
+    }
     /**
      * Loads a single metadata configuration.
      * @param config - The metadata configuration to load
      * @param contextUser - The context user information
      * @param bypassCache - When true, bypasses server-side cache to get fresh data from the database
      */
-    async LoadSingleConfig(config, contextUser, bypassCache = false) {
+    async LoadSingleConfig(config, contextUser, bypassCache = false, suppressEmit = false) {
         if (config.Type === 'dataset')
             return await this.LoadSingleDatasetConfig(config, contextUser, bypassCache);
         else
-            return await this.LoadSingleEntityConfig(config, contextUser, bypassCache);
+            return await this.LoadSingleEntityConfig(config, contextUser, bypassCache, suppressEmit);
     }
     /**
      * Opens a new full-refresh "generation" for a property and returns its token. Each call
@@ -1557,6 +1658,15 @@ export class BaseEngine extends BaseSingleton {
      * to ensure the fingerprint-affecting params (EntityName, ExtraFilter, OrderBy,
      * IgnoreMaxRows, IncludeBinaryFields) are always consistent — preventing cache key mismatches that break
      * cross-server invalidation via Redis pub/sub and local cache upsert/remove operations.
+     *
+     * **Row-level security is not part of the fingerprints built from these params.** The engine
+     * builds them with `GenerateRunViewFingerprint(params, connection)`, without the RLS clause the
+     * provider adds for the reading user. On a server that is correct: engines load as the system
+     * user, whose RLS clause is empty, so the provider's fingerprint for the same read matches. An
+     * engine loaded by an RLS-scoped user (possible on a client) would compute an unscoped
+     * fingerprint for a slot the provider stored under a scoped one, so its cache callbacks and
+     * `syncLocalCacheForConfig` would target a different slot. Pass the RLS clause here if such an
+     * engine is ever needed.
      */
     BuildRunViewParamsForConfig(config, bypassCache = false) {
         return {
@@ -1581,7 +1691,7 @@ export class BaseEngine extends BaseSingleton {
      * @param contextUser
      * @param bypassCache - When true, bypasses server-side cache to get fresh data from the database
      */
-    async LoadSingleEntityConfig(config, contextUser, bypassCache = false) {
+    async LoadSingleEntityConfig(config, contextUser, bypassCache = false, suppressEmit = false) {
         // Claim a refresh generation BEFORE the awaited RunView. If another full refresh for
         // this same property starts while our RunView is in flight, ours becomes stale and must
         // not commit — otherwise concurrent refreshes (the filtered/OrderBy path, e.g.
@@ -1600,7 +1710,12 @@ export class BaseEngine extends BaseSingleton {
             return;
         }
         this.HandleSingleViewResult(config, result, contextUser);
-        this.emitPropertyChange(config.PropertyName);
+        if (!suppressEmit) {
+            // A caller that rebuilds derived state afterwards emits itself, once, AFTER the
+            // rebuild — subscribers must never observe a property whose derived state is still
+            // the previous load's (the invariant stated on OnExternalCacheChange).
+            this.emitPropertyChange(config.PropertyName);
+        }
     }
     /**
      * Handles the result of a single view load.
@@ -1627,7 +1742,7 @@ export class BaseEngine extends BaseSingleton {
             // the user still gets no data. Logged at status level, not error, because
             // for a restricted role this is expected, not a fault.
             this.MarkConfigEmptyLoaded(config);
-            LogStatus(`BaseEngine: ${config.EntityName} not readable by current role — loaded empty (restricted-role degradation).`);
+            LogWarning(`BaseEngine: ${config.EntityName} not readable by current role — loaded empty (restricted-role degradation).`, 'Cache');
         }
         else {
             // TRANSIENT failure (network, MJAPI restart, etc.) — leave loadedSuccessfully
@@ -1687,7 +1802,7 @@ export class BaseEngine extends BaseSingleton {
         if (config.AddToObject !== false) {
             this[config.PropertyName] = [];
         }
-        this._dataMap.set(config.PropertyName, { entityName: config.EntityName, data: [], loadedSuccessfully: true });
+        this._dataMap.set(config.PropertyName, { entityName: config.EntityName, data: [], loadedSuccessfully: true, readDenied: true });
         this.NotifyDataChange(config, []);
     }
     /**
@@ -1810,17 +1925,35 @@ export class BaseEngine extends BaseSingleton {
         }
         this._cacheChangeUnsubscribers = [];
         if (!LocalCacheManager.Instance.IsInitialized) {
+            this.deferCacheChangeRegistration(entityConfigs);
             return;
         }
-        const connectionPrefix = this.RunViewProviderToUse instanceof ProviderBase
-            ? this.RunViewProviderToUse.InstanceConnectionString
-            : undefined;
         for (const config of entityConfigs) {
-            const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(this.BuildRunViewParamsForConfig(config), connectionPrefix);
+            const fingerprint = this.configFingerprint(config);
             const unsubscribe = LocalCacheManager.Instance.RegisterChangeCallback(fingerprint, (event) => this.OnExternalCacheChange(config, event));
             this._cacheChangeUnsubscribers.push(unsubscribe);
         }
     }
+    /**
+     * An engine that loads before `LocalCacheManager` is initialized (a custom host, a script that
+     * touches an engine before `StartupManager` runs) would otherwise never hear another server's
+     * cache changes: nothing registers its callbacks later except a forced reload. Registration is
+     * deferred to initialization instead; only the latest config list is kept.
+     */
+    deferCacheChangeRegistration(entityConfigs) {
+        const alreadyWaiting = this._deferredCacheCallbackConfigs !== null;
+        this._deferredCacheCallbackConfigs = entityConfigs;
+        if (alreadyWaiting) {
+            return;
+        }
+        LocalCacheManager.Instance.WhenInitialized(() => {
+            const configs = this._deferredCacheCallbackConfigs;
+            this._deferredCacheCallbackConfigs = null;
+            if (configs) {
+                this.RegisterCacheChangeCallbacks(configs);
+            }
+        });
+    }
     /**
      * Called when another server instance updates cached data that this engine
      * is tracking. Default behavior: reload the affected config from the database.
@@ -1832,16 +1965,43 @@ export class BaseEngine extends BaseSingleton {
      * @param event - The cache change event from the other server
      */
     async OnExternalCacheChange(config, event) {
+        // A config the current user cannot read stays empty: neither a peer's rows nor a reload
+        // may fill it. The per-record event path applies the same rule.
+        if (this._isPermissionConstrained || this._dataMap.get(config.PropertyName)?.readDenied) {
+            return;
+        }
         // If the event includes the new data, try to apply it directly
         if (event.Data && event.Action === 'set') {
             try {
                 const parsed = JSON.parse(event.Data);
                 if (parsed?.results && Array.isArray(parsed.results)) {
+                    // A payload carrying exactly the rows this engine already holds tells it nothing.
+                    // Skipping it leaves the engine as if the event had not arrived: no generation is
+                    // claimed, so a reload already in flight still commits its (possibly newer) rows.
+                    if (this.payloadMatchesHeldRows(config, parsed.results)) {
+                        LogStatusEx({ message: `BaseEngine.OnExternalCacheChange: '${config.PropertyName}' already holds the payload's rows — skipped`, verboseOnly: true });
+                        return;
+                    }
+                    // Rows written under a different field list (a server on the other side of a
+                    // migration) lack columns this process has, or carry ones it does not. Loading
+                    // them would blank the missing columns on every row, so read the rows instead.
+                    if (this.payloadSchemaDiffers(config, parsed.schemaHash)) {
+                        LogStatusEx({ message: `BaseEngine.OnExternalCacheChange: payload for '${config.PropertyName}' was written under a different schema — reloading instead`, verboseOnly: true });
+                        await this.reloadAfterExternalChange(config);
+                        return;
+                    }
                     // Claim a refresh generation BEFORE the awaited materialization — the same
                     // protocol LoadSingleConfig uses around its awaited RunView. Without it, two
                     // overlapping cache events (or an event racing a full reload) can resolve out
                     // of order and the stale result would be the one that assigns last.
                     const generation = this.beginConfigRefresh(config.PropertyName);
+                    // A burst of payloads for one slot usually arrives together. Let the rest of the
+                    // burst claim its generations first, so only the newest payload is materialized
+                    // instead of every one of them (receive side).
+                    await yieldToEventLoop();
+                    if (!this.isLatestConfigRefresh(config.PropertyName, generation)) {
+                        return;
+                    }
                     const rows = await this.materializeCacheEventRows(config, parsed.results);
                     if (!this.isLatestConfigRefresh(config.PropertyName, generation)) {
                         return; // superseded while materializing — the newer refresh owns the property
@@ -1879,15 +2039,364 @@ export class BaseEngine extends BaseSingleton {
                 // materialization failures (e.g. an entity class that cannot construct in
                 // this process), and without a trace the only symptom is every cache event
                 // turning into a database reload.
-                LogStatus(`BaseEngine.OnExternalCacheChange: payload for '${config.PropertyName}' could not be applied (${e instanceof Error ? e.message : String(e)}) — falling back to full reload`);
+                LogWarning(`BaseEngine.OnExternalCacheChange: payload for '${config.PropertyName}' could not be applied (${e instanceof Error ? e.message : String(e)}) — falling back to full reload`, 'Cache');
             }
         }
-        // Fallback: reload this config from the database. This also replaces the property,
-        // so the derived state has to be rebuilt here too.
-        await this.LoadSingleConfig(config, this._contextUser);
+        await this.reloadAfterExternalChange(config);
+    }
+    /**
+     * True when a cache-change payload carries a schema hash and it differs from this engine's
+     * hash for the config's entity. A payload without a hash, or an entity this process cannot
+     * resolve, does not count as differing.
+     */
+    payloadSchemaDiffers(config, payloadSchemaHash) {
+        if (typeof payloadSchemaHash !== 'string' || !payloadSchemaHash || !config.EntityName) {
+            return false;
+        }
+        const ownHash = LocalCacheManager.Instance.ComputeSchemaHash(this.ProviderToUse, config.EntityName);
+        return ownHash !== undefined && ownHash !== payloadSchemaHash;
+    }
+    /**
+     * Reloads a config after a peer's cache change, coalescing overlapping requests: an event that
+     * arrives while a reload for the same property is running asks for one more reload after it
+     * instead of starting its own. Every caller's promise settles only after a reload that began
+     * after its event, so no change is missed, but N events in a burst cost at most two reloads.
+     */
+    reloadAfterExternalChange(config) {
+        const running = this._externalChangeReloads.get(config.PropertyName);
+        if (running) {
+            running.Again = true;
+            return running.Done;
+        }
+        const reload = { Again: false, Done: Promise.resolve() };
+        this._externalChangeReloads.set(config.PropertyName, reload);
+        reload.Done = this.runExternalChangeReloads(config, reload);
+        return reload.Done;
+    }
+    /** Runs reloads for {@link reloadAfterExternalChange} until no further event asked for one. */
+    async runExternalChangeReloads(config, reload) {
+        try {
+            do {
+                reload.Again = false;
+                // The reload replaces the property, so the derived state has to be rebuilt too —
+                // and the reload must NOT emit on its own, or a subscriber wakes between the new
+                // rows and the rebuild and reads derived state still pointing at the instances
+                // this reload discarded.
+                await this.LoadSingleConfig(config, this._contextUser, /*bypassCache*/ false, /*suppressEmit*/ true);
+                await this.RebuildDerivedState(this._contextUser);
+                this.emitPropertyChange(config.PropertyName);
+            } while (reload.Again);
+        }
+        finally {
+            this._externalChangeReloads.delete(config.PropertyName);
+        }
+    }
+    /**
+     * True when `payloadRows` are the same row versions this config currently holds: the same
+     * number of rows, and the same multiset of (primary key, `__mj_UpdatedAt`) pairs.
+     *
+     * Identity is derived from what the engine holds right now, never remembered, so no other
+     * assignment path has to keep anything in step with it. (An earlier version remembered the last
+     * payload applied and went stale whenever another path replaced the property.) Comparing rows,
+     * not the writer's `maxUpdatedAt`, keeps it independent of how the writer stamped the slot, and
+     * catches a delete paired with an insert.
+     *
+     * Returns false — apply the payload — whenever identity cannot be established: the config is not
+     * loaded, the entity has no `__mj_UpdatedAt` column, or a timestamp cannot be read.
+     * Known limit: a change that does not bump `__mj_UpdatedAt` (direct SQL) is invisible here, as it
+     * is to the client smart-cache check.
+     */
+    payloadMatchesHeldRows(config, payloadRows) {
+        try {
+            return this.compareHeldRows(config, payloadRows);
+        }
+        catch (e) {
+            // The skip is an optimization; failing to decide must never stop the payload applying.
+            LogStatusEx({ message: `BaseEngine.payloadMatchesHeldRows: could not compare '${config.PropertyName}' (${e instanceof Error ? e.message : String(e)}) — applying the payload`, verboseOnly: true });
+            return false;
+        }
+    }
+    compareHeldRows(config, payloadRows) {
+        const entry = this._dataMap.get(config.PropertyName);
+        if (!entry?.loadedSuccessfully || !config.EntityName || !Array.isArray(entry.data)) {
+            return false;
+        }
+        if (entry.data.length !== payloadRows.length) {
+            return false;
+        }
+        const entityInfo = this.ProviderToUse?.EntityByName(config.EntityName);
+        if (!entityInfo || !entityInfo.Fields.some(f => f.IsUpdatedAtField)) {
+            return false;
+        }
+        const keyFields = entityInfo.PrimaryKeys.map(pk => pk.Name);
+        const held = this.rowVersionKeys(entry.data, keyFields, (row, field) => row instanceof BaseEntity ? row.Get(field) : row[field]);
+        const incoming = this.rowVersionKeys(payloadRows, keyFields, (row, field) => row[field]);
+        if (!held || !incoming) {
+            return false;
+        }
+        return held.every((key, i) => key === incoming[i]);
+    }
+    /**
+     * Sorted `pk|epochMs` strings for a row set, or null when any row lacks a readable timestamp.
+     * Key values are lower-cased so GUID casing differences between sources do not matter.
+     */
+    rowVersionKeys(rows, keyFields, read) {
+        const keys = [];
+        for (const row of rows) {
+            const raw = read(row, EntityInfo.UpdatedAtFieldName);
+            const stamp = raw instanceof Date ? raw.getTime() : new Date(String(raw)).getTime();
+            if (raw == null || Number.isNaN(stamp)) {
+                return null;
+            }
+            const pk = keyFields.map(f => String(read(row, f) ?? '').toLowerCase()).join('
');
+            keys.push(`${pk}${stamp}`);
+        }
+        return keys.sort();
+    }
+    // ========================================================================
+    // RECONCILIATION WITH THE DATABASE (plan Phase 3.1)
+    // ========================================================================
+    /**
+     * Compares each loaded entity config with the database — row count and newest
+     * `__mj_UpdatedAt` — and reloads the ones that differ. Finds changes that never raised an MJ
+     * event: direct SQL, other applications, restores. Everything else reaches engines through
+     * events and does not need this.
+     *
+     * One batched status query for the whole engine. A reloaded config is written to the cache
+     * once, so on a shared cache every other server adopts the fresh rows from that one write.
+     * Derived state is rebuilt once, after all reloads.
+     *
+     * Limits: an UPDATE that does not change `__mj_UpdatedAt` or the row count is invisible, as it
+     * is to the smart-cache check. Entities without `__mj_UpdatedAt` are compared by row count
+     * only; dataset configs are not checked.
+     *
+     * @returns What was checked and reloaded. Never throws.
+     */
+    async SweepAgainstDatabase() {
+        const result = { EngineClass: this.constructor.name, Checked: 0, Reloaded: [], Errors: [] };
+        const provider = this.RunViewProviderToUse;
+        const configs = this.sweepableConfigs();
+        if (!this.Loaded || this._isPermissionConstrained || !provider?.GetRunViewsDatabaseStatus || configs.length === 0) {
+            return result;
+        }
+        try {
+            const statuses = await provider.GetRunViewsDatabaseStatus(configs.map(c => this.BuildRunViewParamsForConfig(c)), this._contextUser);
+            const stale = this.staleConfigs(configs, statuses, result);
+            await this.reloadStaleConfigs(stale, result);
+        }
+        catch (e) {
+            result.Errors.push(e instanceof Error ? e.message : String(e));
+        }
+        if (result.Errors.length > 0) {
+            LogWarning(`BaseEngine.SweepAgainstDatabase(${result.EngineClass}): ${result.Errors.join('; ')}`, 'Cache');
+        }
+        return result;
+    }
+    /**
+     * Whether this engine holds any config a sweep could act on — an entity config, loaded and
+     * readable, whose entity declares `TrustServerCacheCompletely = false`.
+     *
+     * Exposed so a caller can decide whether the engine is worth the cost of asking. The fleet
+     * sweeper takes a cross-process lease per engine per interval; on an installation where nothing
+     * declares out-of-band writes, every one of those leases would be claimed only to discover there
+     * is nothing to compare. Reading this first is an in-memory check.
+     */
+    HasSweepableConfigs() {
+        return this.sweepableConfigs().length > 0;
+    }
+    /** Entity configs holding a successful, readable load whose entity can drift outside MJ. */
+    sweepableConfigs() {
+        return this._metadataConfigs.filter(config => {
+            const entry = this._dataMap.get(config.PropertyName);
+            if (config.Type !== 'entity' || !entry?.loadedSuccessfully || entry.readDenied || entry.permissionDenied) {
+                return false;
+            }
+            return this.entityMayChangeWithoutAnEvent(config.EntityName);
+        });
+    }
+    /**
+     * Whether this entity has **declared** that its rows can change without firing an event — the
+     * only situation the sweep exists for.
+     *
+     * `Entity.TrustServerCacheCompletely` is that declaration, in its own words: when true (the
+     * default) the cache trusts "that all mutations flow through BaseEntity.Save() which fires cache
+     * invalidation events", and it is set false "for entities whose rows are created as side-effects
+     * of other operations via raw SQL … since those inserts bypass BaseEntity and never trigger
+     * cache invalidation". An entity that still trusts its cache has nothing for a sweep to find, so
+     * querying it every interval buys nothing.
+     *
+     * It also subsumes the `AllowDirectSQLInsert`/`Update`/`Delete` flags: a database CHECK requires
+     * `TrustServerCacheCompletely = false` whenever any of those is set, so this one flag already
+     * covers every entity that sanctions writes outside `BaseEntity`.
+     *
+     * **Why gate at all.** The sweep is a periodic database query, and a periodic query is not free:
+     * on Azure SQL serverless it prevents auto-pause outright, which several installs pay for, and
+     * the interval does not help — auto-pause needs sustained inactivity, so a long interval is no
+     * better than a short one. Gating here makes the cost proportional to the declared risk: an
+     * engine whose entities are all trusted is never swept and issues no queries. Operators who do
+     * write out of band mark those entities — which they must do anyway for the cache to be correct
+     * at all — and get the backstop exactly there.
+     *
+     * An entity absent from metadata answers false: the sweep cannot establish that it drifts, and
+     * guessing in favour of a recurring query is the expensive guess.
+     */
+    entityMayChangeWithoutAnEvent(entityName) {
+        if (!entityName) {
+            return false;
+        }
+        return this.ProviderToUse?.EntityByName(entityName)?.TrustServerCacheCompletely === false;
+    }
+    /** The configs whose held rows disagree with what the database reports. */
+    staleConfigs(configs, statuses, result) {
+        const stale = [];
+        configs.forEach((config, i) => {
+            const status = statuses[i];
+            if (!status?.Success) {
+                result.Errors.push(`${config.PropertyName}: ${status?.ErrorMessage ?? 'no status returned'}`);
+                return;
+            }
+            result.Checked++;
+            if (!this.heldRowsMatchDatabase(config, status)) {
+                stale.push(config);
+            }
+        });
+        return stale;
+    }
+    /**
+     * Row count and newest `__mj_UpdatedAt` of the held rows against the database's.
+     *
+     * The held stamp is computed the way SQL's `MAX` computes the database's — **ignoring rows
+     * whose timestamp is null** — rather than taken from the census, whose stamp is null unless
+     * EVERY row carries one (it also backs an identity hash, which needs them all). Comparing a
+     * census stamp with a SQL `MAX` made one un-stamped row in a config mean "stale" on every
+     * sweep, forever: a reload every interval, each one rewriting the shared slot and making every
+     * peer reload too. Two sides that both have no stamp at all agree; one with
+     * and one without is a real difference.
+     */
+    heldRowsMatchDatabase(config, status) {
+        const entry = this._dataMap.get(config.PropertyName);
+        if (!entry) {
+            return true;
+        }
+        const rows = Array.isArray(entry.data) ? entry.data : [];
+        if (rows.length !== status.RowCount) {
+            return false;
+        }
+        const entityInfo = this.ProviderToUse?.EntityByName(config.EntityName);
+        if (!entityInfo?.Fields.some(f => f.IsUpdatedAtField) || rows.length === 0) {
+            return true; // nothing to compare beyond the count
+        }
+        const heldMax = LocalCacheManager.MaxUpdatedAtOfRows(rows.map(row => row instanceof BaseEntity ? row.GetAll() : row));
+        if (!heldMax && !status.MaxUpdatedAt) {
+            return true; // neither side has a usable stamp; the counts already agreed
+        }
+        if (!heldMax || !status.MaxUpdatedAt) {
+            return false;
+        }
+        return Math.abs(Date.parse(heldMax) - Date.parse(status.MaxUpdatedAt)) < SWEEP_TIMESTAMP_TOLERANCE_MS;
+    }
+    /** Reloads each stale config from the database, stores it once, then rebuilds derived state. */
+    async reloadStaleConfigs(stale, result) {
+        for (const config of stale) {
+            await this.LoadSingleConfig(config, this._contextUser, true, /*suppressEmit*/ true);
+            if (!this._dataMap.get(config.PropertyName)?.loadedSuccessfully) {
+                result.Errors.push(`${config.PropertyName}: reload failed`);
+                continue;
+            }
+            await this.storeConfigRows(config);
+            result.Reloaded.push(config.PropertyName);
+        }
+        if (result.Reloaded.length === 0) {
+            return;
+        }
         await this.RebuildDerivedState(this._contextUser);
+        for (const propertyName of result.Reloaded) {
+            this.emitPropertyChange(propertyName);
+        }
+        LogStatusEx({ message: `BaseEngine.SweepAgainstDatabase(${result.EngineClass}): reloaded ${result.Reloaded.join(', ')}`, verboseOnly: false });
+    }
+    /**
+     * Writes a config's rows to the cache slot the engine reads and listens on. The reload
+     * bypassed the cache, so without this the slot — and every server reading it — would keep the
+     * old rows.
+     */
+    async storeConfigRows(config) {
+        const cache = LocalCacheManager.Instance;
+        const rows = this._dataMap.get(config.PropertyName)?.data;
+        if (!cache.IsInitialized || !Array.isArray(rows)) {
+            return;
+        }
+        const plainRows = rows.map(row => row instanceof BaseEntity ? row.GetAll() : row);
+        const params = this.BuildRunViewParamsForConfig(config);
+        // Never the clock: a stamp the database cannot reproduce makes the slot permanently
+        // "stale" — every currency check sees a different value and refetches, and the next sweep
+        // stamps it again with a newer one. `''` is what the database's own MAX reports for rows
+        // with no timestamp, and what the other two write funnels store.
+        const maxUpdatedAt = LocalCacheManager.MaxUpdatedAtOfRows(plainRows) ?? '';
+        // Under the slot's cross-process lock: a peer may be part-way through its own
+        // read-modify-write of this slot, and an unlocked replace would clobber it.
+        await cache.ReplaceRunViewResultLocked(this.configFingerprint(config), params, plainRows, maxUpdatedAt, plainRows.length, this.ProviderToUse);
+    }
+    /** The cache fingerprint of a config's slot, as {@link RegisterCacheChangeCallbacks} computes it. */
+    configFingerprint(config) {
+        const provider = this.RunViewProviderToUse;
+        const connectionPrefix = provider instanceof ProviderBase ? provider.InstanceConnectionString : undefined;
+        return LocalCacheManager.Instance.GenerateRunViewFingerprint(this.BuildRunViewParamsForConfig(config), connectionPrefix);
+    }
+    /**
+     * Snapshot of what this engine holds: per property, the row count, newest `__mj_UpdatedAt`,
+     * and a hash of the row versions; plus the engine's own derived-state counts. Read-only and
+     * cheap enough for diagnostics and startup checks (O(rows)).
+     */
+    GetStateCensus() {
+        const properties = [];
+        for (const [propertyName, entry] of this._dataMap) {
+            properties.push(this.censusForProperty(propertyName, entry));
+        }
+        properties.sort((a, b) => a.PropertyName.localeCompare(b.PropertyName));
+        return {
+            EngineClass: this.constructor.name,
+            Loaded: this.Loaded,
+            PermissionConstrained: this._isPermissionConstrained,
+            Properties: properties,
+            Derived: this.GetDerivedStateCensus(),
+        };
     }
     /**
+     * Counts that describe the state a subclass derives in {@link AdditionalLoading} — grouped
+     * child collections, lookup maps. Override to expose them; the default reports nothing.
+     * Must be read-only and must not throw.
+     */
+    GetDerivedStateCensus() {
+        return {};
+    }
+    censusForProperty(propertyName, entry) {
+        const rows = Array.isArray(entry.data) ? entry.data : [];
+        const census = {
+            PropertyName: propertyName,
+            EntityName: entry.entityName,
+            DatasetName: entry.datasetName,
+            RowCount: rows.length,
+            MaxUpdatedAt: null,
+            IdentityHash: null,
+            LoadedSuccessfully: entry.loadedSuccessfully,
+            PermissionDenied: entry.permissionDenied === true || entry.readDenied === true,
+        };
+        const entityInfo = entry.entityName ? this.ProviderToUse?.EntityByName(entry.entityName) : undefined;
+        if (!entityInfo) {
+            return census;
+        }
+        const read = (row, field) => row instanceof BaseEntity ? row.Get(field) : row?.[field];
+        const keys = this.rowVersionKeys(rows, entityInfo.PrimaryKeys.map(pk => pk.Name), read);
+        if (keys) {
+            census.IdentityHash = hashStrings(keys);
+            const newest = keys.reduce((max, k) => Math.max(max, Number(k.slice(k.lastIndexOf('\u0002') + 1))), Number.NEGATIVE_INFINITY);
+            census.MaxUpdatedAt = Number.isFinite(newest) ? new Date(newest).toISOString() : null;
+        }
+        return census;
+    }
+    /**
+     * Converts rows from a cache-change payload into the shape this config's property expects.    /**
      * Converts rows from a cache-change payload into the shape this config's property expects.
      *
      * Cache payloads are JSON, so their rows are plain objects with string dates. A config loaded
@@ -1946,12 +2455,21 @@ export class BaseEngine extends BaseSingleton {
     }
     /**
      * Refreshes a specific item.
+     *
+     * Rebuilds derived state before notifying subscribers: this replaces the property's array, so
+     * anything a subclass derived from the previous instances — grouped child collections, memoized
+     * lookups — refers to objects the engine has just discarded. This is also the callback the
+     * expiration timer fires, so without the rebuild a config with an expiry quietly served stale
+     * derived state until something else happened to rebuild it.
+     *
      * @param propertyName - The name of the property to refresh
      */
     async RefreshItem(propertyName) {
         const config = this._metadataConfigs.find(c => c.PropertyName === propertyName) || this._dynamicConfigs.get(propertyName);
         if (config) {
-            await this.LoadSingleConfig(config, this._contextUser);
+            await this.LoadSingleConfig(config, this._contextUser, /*bypassCache*/ false, /*suppressEmit*/ true);
+            await this.RebuildDerivedState(this._contextUser);
+            this.emitPropertyChange(config.PropertyName);
         }
     }
     /**