@memberjunction/core 5.9.0 → 5.10.1

package/dist/generic/providerBase.js

@@ -97,8 +97,65 @@ export class ProviderBase {
         this._localMetadata = new AllMetadata();
         this._entityRecordNameCache = new Map();
         this._refresh = false;
+        this._lastRefreshCheckAt = 0;
+        /**
+         * In-flight + linger cache keyed by a deterministic fingerprint of
+         * the RunViewParams batch.  While a request is in-flight, concurrent
+         * identical calls share the same promise.  After resolution the entry
+         * lingers so that near-sequential identical calls return immediately.
+         */
+        this._inflightViews = new Map();
         this._cachedVisibleExplorerNavigationItems = null;
     }
+    // ── Metadata Refresh Check Debounce ────────────────────────────────
+    /**
+     * Minimum interval (ms) between metadata refresh checks to prevent
+     * redundant network calls when Config()/RefreshIfNeeded() fire in
+     * quick succession (e.g., multiple engines during startup).
+     * Does NOT affect forced Refresh() calls. Default: 30 000 ms.
+     */
+    static { this.MinRefreshCheckIntervalMs = 30000; }
+    // ── Server-Side Auto-Cache ────────────────────────────────────────
+    /**
+     * Maximum row count for auto-caching on the server side. When
+     * `TrustLocalCacheCompletely` is true and a RunView result has no
+     * ExtraFilter, no OrderBy, and the result count is at or below this
+     * threshold, the result is automatically stored in LocalCacheManager
+     * even without an explicit `CacheLocal` flag.
+     *
+     * This captures small reference/lookup tables that are repeatedly
+     * queried by multiple clients while avoiding caching large ad-hoc
+     * result sets. Invalidation is handled by the standard BaseEntity
+     * event-driven upsert (safe because unfiltered caches can be updated
+     * in-place).
+     *
+     * Set to 0 to disable auto-caching. Default 250.
+     */
+    static { this.ServerAutoCacheMaxRows = 250; }
+    // ── Request Deduplication + Linger Window ──────────────────────────
+    /**
+     * How long (ms) a resolved RunViews result stays available for instant
+     * replay.  Set to 0 to disable the linger window (in-flight dedup
+     * still applies).  Default 5 000 ms.
+     */
+    static { this.DedupLingerMs = 5000; }
+    /******** ABSTRACT SECTION ****************************************************************** */
+    /**
+     * When true, cached RunView/RunQuery results are returned immediately on a
+     * cache hit without any server-side validation round-trip.
+     *
+     * Server-side providers (DatabaseProviderBase and its subclasses) override
+     * this to return `true` because the cache is kept in perfect sync via
+     * BaseEntity save/delete events and cross-server Redis pub/sub — the DB
+     * validation query is unnecessary overhead.
+     *
+     * Client-side providers (e.g. GraphQLDataProvider) keep the default `false`
+     * so that the lightweight smart cache check (maxUpdatedAt + rowCount) is
+     * still performed against the server before trusting the browser cache.
+     */
+    get TrustLocalCacheCompletely() {
+        return false;
+    }
     /**
      * Helper to generate cache key for entity record names
      */
@@ -231,20 +288,119 @@ export class ProviderBase {
      * @returns The view results
      */
     async RunView(params, contextUser) {
-        // Delegate to RunViews with a single-element array to ensure smart cache check is used
-        // This guarantees that CacheLocal uses server-side validation (maxUpdatedAt + rowCount check)
-        // rather than blindly accepting stale local cache
+        if (this.TrustLocalCacheCompletely) {
+            // Server-side: use direct Pre → Internal → Post pipeline.
+            // Cache is kept in sync via BaseEntity events + Redis pub/sub,
+            // so PreRunView cache hits are returned immediately with no DB round-trip.
+            const preResult = await this.PreRunView(params, contextUser);
+            if (preResult.cachedResult) {
+                // Cache hit — transform and return directly
+                LogStatusEx({ message: `  ✅ [Cache HIT] RunView "${params.EntityName || params.ViewName || 'unknown'}" — ${preResult.cachedResult.Results?.length ?? 0} rows from cache, no DB query`, verboseOnly: true });
+                await this.TransformSimpleObjectToEntityObject(params, preResult.cachedResult, contextUser);
+                TelemetryManager.Instance.EndEvent(preResult.telemetryEventId, {
+                    cacheHit: true,
+                    cacheStatus: preResult.cacheStatus,
+                    resultCount: preResult.cachedResult.Results?.length ?? 0
+                });
+                if (params.OnDataChanged && preResult.fingerprint) {
+                    preResult.cachedResult.Unsubscribe = LocalCacheManager.Instance.RegisterChangeCallback(preResult.fingerprint, params.OnDataChanged);
+                }
+                return preResult.cachedResult;
+            }
+            // Cache miss — execute query, then post-process (stores in cache)
+            LogStatusEx({ message: `  🔍 [Cache MISS] RunView "${params.EntityName || params.ViewName || 'unknown'}" — querying database`, verboseOnly: true });
+            const result = await this.InternalRunView(params, contextUser);
+            await this.PostRunView(result, params, preResult, contextUser);
+            return result;
+        }
+        // Client-side: delegate to RunViews which uses the smart cache check
+        // (lightweight maxUpdatedAt + rowCount validation against the server)
         const results = await this.RunViews([params], contextUser);
         return results[0];
     }
     /**
      * Runs multiple views based on the provided parameters.
-     * This method orchestrates the full execution flow for batch operations.
+     * Wraps the execution pipeline with request deduplication and a linger
+     * window so that concurrent (and near-sequential) identical calls share
+     * a single server round-trip.  Every caller receives a shallow-copied
+     * Results array to protect against cross-caller mutations (push/sort/splice).
+     *
      * @param params - Array of view parameters
      * @param contextUser - Optional user context for permissions (required server-side)
-     * @returns Array of view results
+     * @returns Array of view results (shallow-copied Results per caller)
      */
     async RunViews(params, contextUser) {
+        // Bypass dedup for side-effect calls (SaveViewResults creates DB records)
+        if (this.ShouldBypassDedup(params)) {
+            return this.ExecuteRunViewsPipeline(params, contextUser);
+        }
+        const key = this.GenerateDedupKey(params, contextUser);
+        const existing = this._inflightViews.get(key);
+        // ── Linger hit: resolved result still within the linger window ──
+        if (existing?.resolvedResults && existing.resolvedAt) {
+            const age = Date.now() - existing.resolvedAt;
+            if (age < ProviderBase.DedupLingerMs) {
+                const entities = params.map(p => p.EntityName || p.ViewName || 'unknown').join(', ');
+                LogStatusEx({
+                    message: `[Dedup] Linger hit for [${entities}] — returning cached result (age ${age}ms, window ${ProviderBase.DedupLingerMs}ms)`,
+                    verboseOnly: true
+                });
+                return existing.resolvedResults.map(r => this.ShallowCopyResult(r));
+            }
+            // Linger expired — fall through to fresh execution
+            this._inflightViews.delete(key);
+        }
+        // ── In-flight hit: another caller is already executing this exact request ──
+        if (existing && !existing.resolvedResults) {
+            const entities = params.map(p => p.EntityName || p.ViewName || 'unknown').join(', ');
+            LogStatusEx({
+                message: `[Dedup] In-flight hit for [${entities}] — sharing pending execution`,
+                verboseOnly: true
+            });
+            const results = await existing.promise;
+            return results.map(r => this.ShallowCopyResult(r));
+        }
+        // ── Fresh execution ──
+        const promise = this.ExecuteRunViewsPipeline(params, contextUser)
+            .then(results => {
+            // Stash resolved results for the linger window
+            const entry = this._inflightViews.get(key);
+            if (entry && entry.promise === promise) {
+                entry.resolvedResults = results;
+                entry.resolvedAt = Date.now();
+                // Schedule cleanup after linger expires
+                if (ProviderBase.DedupLingerMs > 0) {
+                    setTimeout(() => {
+                        const current = this._inflightViews.get(key);
+                        if (current && current.promise === promise) {
+                            this._inflightViews.delete(key);
+                        }
+                    }, ProviderBase.DedupLingerMs);
+                }
+                else {
+                    this._inflightViews.delete(key);
+                }
+            }
+            return results;
+        })
+            .catch(err => {
+            // Clean up so retries aren't stuck on a failed entry
+            const entry = this._inflightViews.get(key);
+            if (entry && entry.promise === promise) {
+                this._inflightViews.delete(key);
+            }
+            throw err;
+        });
+        this._inflightViews.set(key, { promise });
+        const results = await promise;
+        return results.map(r => this.ShallowCopyResult(r));
+    }
+    // ── Dedup helpers ──────────────────────────────────────────────────
+    /**
+     * The original RunViews execution pipeline (pre-processing, cache,
+     * internal execution, post-processing).
+     */
+    async ExecuteRunViewsPipeline(params, contextUser) {
         // Pre-processing for batch
         const preResult = await this.PreRunViews(params, contextUser);
         // Check for smart cache check mode
@@ -253,7 +409,9 @@ export class ProviderBase {
         }
         // Check for cached results - if all are cached, end telemetry and return early
         if (preResult.allCached && preResult.cachedResults) {
+            const entities = params.map(p => p.EntityName || p.ViewName || 'unknown').join(', ');
             const totalResults = preResult.cachedResults.reduce((sum, r) => sum + (r.Results?.length ?? 0), 0);
+            LogStatusEx({ message: `  ✅ [Cache HIT] RunViews batch [${entities}] — all ${params.length} views served from cache (${totalResults} total rows), no DB queries`, verboseOnly: true });
             TelemetryManager.Instance.EndEvent(preResult.telemetryEventId, {
                 cacheHit: true,
                 allCached: true,
@@ -263,6 +421,8 @@ export class ProviderBase {
             return preResult.cachedResults;
         }
         // Execute the internal implementation for non-cached items
+        const uncachedEntities = (preResult.uncachedParams || params).map(p => p.EntityName || p.ViewName || 'unknown').join(', ');
+        LogStatusEx({ message: `  🔍 [Cache MISS] RunViews batch [${uncachedEntities}] — querying database for ${(preResult.uncachedParams || params).length} view(s)`, verboseOnly: true });
         const results = await this.InternalRunViews(preResult.uncachedParams || params, contextUser);
         // Merge cached and fresh results if needed
         const finalResults = preResult.cachedResults
@@ -272,6 +432,45 @@ export class ProviderBase {
         await this.PostRunViews(finalResults, params, preResult, contextUser);
         return finalResults;
     }
+    /**
+     * Generates a deterministic dedup key for a batch of RunViewParams.
+     * Extends the local-cache fingerprint with additional fields that
+     * affect result identity (Fields, UserSearchString, ViewID, ViewName,
+     * contextUser).
+     */
+    GenerateDedupKey(params, contextUser) {
+        const parts = params.map(p => {
+            const base = LocalCacheManager.Instance.GenerateRunViewFingerprint(p, this.InstanceConnectionString);
+            const extras = [
+                p.Fields?.join(',') ?? '',
+                p.UserSearchString ?? '',
+                p.ViewID ?? '',
+                p.ViewName ?? '',
+                contextUser?.ID ?? ''
+            ].join('|');
+            return `${base}|${extras}`;
+        });
+        return parts.join('||');
+    }
+    /**
+     * Returns true if any param in the batch has SaveViewResults set,
+     * which means the call has a side effect (creating UserViewRun records)
+     * and must not be deduplicated.
+     */
+    ShouldBypassDedup(params) {
+        return params.some(p => p.SaveViewResults === true);
+    }
+    /**
+     * Returns a shallow copy of a RunViewResult: the Results array is a
+     * new array instance (protecting against push/sort/splice by other
+     * callers) but the individual row objects inside are shared references.
+     */
+    ShallowCopyResult(result) {
+        return {
+            ...result,
+            Results: [...result.Results]
+        };
+    }
     /**
      * Runs a query based on the provided parameters.
      * This method orchestrates the full execution flow: pre-processing, cache check,
@@ -441,7 +640,7 @@ export class ProviderBase {
         let cacheStatus = 'disabled';
         let cachedResult;
         let fingerprint;
-        if (params.CacheLocal && LocalCacheManager.Instance.IsInitialized) {
+        if ((params.CacheLocal || this.TrustLocalCacheCompletely) && LocalCacheManager.Instance.IsInitialized) {
             fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params, this.InstanceConnectionString);
             const cached = await LocalCacheManager.Instance.GetRunViewResult(fingerprint);
             if (cached) {
@@ -457,6 +656,9 @@ export class ProviderBase {
                     AggregateResults: cached.aggregateResults // Include cached aggregate results
                 };
                 cacheStatus = 'hit';
+                if (!params.CacheLocal && this.TrustLocalCacheCompletely) {
+                    LogStatusEx({ message: `  ✅ [Server Cache HIT] RunView "${params.EntityName || params.ViewName || 'unknown'}" — ${cached.results.length} rows served from server cache (no DB query)`, verboseOnly: true });
+                }
             }
             else {
                 cacheStatus = 'miss';
@@ -497,11 +699,14 @@ export class ProviderBase {
             Entities: params.map(p => p.EntityName || p.ViewName || p.ViewID).filter(Boolean),
             _fromEngine: fromEngine
         }, contextUser?.ID);
-        // Check if any params have CacheLocal enabled - smart caching is always used when caching locally
-        const useSmartCacheCheck = params.some(p => p.CacheLocal);
-        // If local caching is enabled, use smart cache check flow
-        if (useSmartCacheCheck && LocalCacheManager.Instance.IsInitialized) {
-            return this.prepareSmartCacheCheckParams(params, telemetryEventId, contextUser);
+        // Client-side providers use smart cache check (lightweight server validation)
+        // Server-side providers trust the cache completely and fall through to
+        // the traditional flow which returns cached data immediately on hit.
+        if (!this.TrustLocalCacheCompletely) {
+            const useSmartCacheCheck = params.some(p => p.CacheLocal);
+            if (useSmartCacheCheck && LocalCacheManager.Instance.IsInitialized) {
+                return this.prepareSmartCacheCheckParams(params, telemetryEventId, contextUser);
+            }
         }
         // Traditional caching flow
         const cacheStatusMap = new Map();
@@ -520,8 +725,8 @@ export class ProviderBase {
                 }
                 param.Fields = entity.Fields.map(f => f.Name);
             }
-            // Check local cache if enabled
-            if (param.CacheLocal && LocalCacheManager.Instance.IsInitialized) {
+            // Check local cache if enabled or if server trusts its cache completely
+            if ((param.CacheLocal || this.TrustLocalCacheCompletely) && LocalCacheManager.Instance.IsInitialized) {
                 const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString);
                 const cached = await LocalCacheManager.Instance.GetRunViewResult(fingerprint);
                 if (cached) {
@@ -537,10 +742,15 @@ export class ProviderBase {
                     };
                     // if needed this will transform each result into an entity object
                     await this.TransformSimpleObjectToEntityObject(param, cachedViewResult, contextUser);
+                    if (!param.CacheLocal && this.TrustLocalCacheCompletely) {
+                        LogStatusEx({ message: `    ✅ [Server Cache HIT] RunViews "${param.EntityName || param.ViewName || 'unknown'}" — ${cached.results.length} rows served from server cache (no DB query)`, verboseOnly: true });
+                    }
+                    LogStatusEx({ message: `    ✅ [Cache HIT] "${param.EntityName || param.ViewName || 'unknown'}" — ${cached.results.length} rows from cache`, verboseOnly: true });
                     cacheStatusMap.set(i, { status: 'hit', result: cachedViewResult });
                     cachedResults.push(cachedViewResult);
                     continue;
                 }
+                LogStatusEx({ message: `    🔍 [Cache MISS] "${param.EntityName || param.ViewName || 'unknown'}" — will query database`, verboseOnly: true });
                 cacheStatusMap.set(i, { status: 'miss' });
             }
             else {
@@ -550,10 +760,13 @@ export class ProviderBase {
             uncachedParams.push(param);
             cachedResults.push(null); // Placeholder for uncached
         }
+        const hasCacheHits = cacheStatusMap.size > 0 && [...cacheStatusMap.values()].some(v => v.status === 'hit');
         return {
             telemetryEventId,
             allCached,
-            cachedResults: allCached ? cachedResults.filter(r => r !== null) : undefined,
+            cachedResults: allCached
+                ? cachedResults.filter(r => r !== null)
+                : (hasCacheHits ? cachedResults : undefined),
             uncachedParams: allCached ? undefined : uncachedParams,
             cacheStatusMap
         };
@@ -847,17 +1060,28 @@ export class ProviderBase {
      * @param contextUser - Optional user context
      */
     async PostRunView(result, params, preResult, contextUser) {
+        // Store in local cache BEFORE entity transformation — the cache needs
+        // plain JSON-serializable objects. BaseEntity objects contain RxJS Subjects
+        // with circular subscriber references that break JSON.stringify.
+        // On cache read, TransformSimpleObjectToEntityObject is called to restore
+        // entity objects when ResultType === 'entity_object'.
+        if (params.CacheLocal && result.Success && preResult.fingerprint && LocalCacheManager.Instance.IsInitialized) {
+            const maxUpdatedAt = this.extractMaxUpdatedAt(result.Results);
+            await LocalCacheManager.Instance.SetRunViewResult(preResult.fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults);
+        }
+        else if (this.shouldAutoCache(params, result)) {
+            // Server-side auto-cache: small, unfiltered, unsorted results are
+            // automatically cached even without explicit CacheLocal. These are
+            // safe for in-place upsert on entity changes (no filter to evaluate).
+            const fingerprint = preResult.fingerprint || LocalCacheManager.Instance.GenerateRunViewFingerprint(params, this.InstanceConnectionString);
+            const maxUpdatedAt = this.extractMaxUpdatedAt(result.Results);
+            await LocalCacheManager.Instance.SetRunViewResult(fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults);
+            LogStatusEx({ message: `  📦 [Auto-Cache] RunView "${params.EntityName || params.ViewName || 'unknown'}" — ${result.Results.length} rows auto-cached (small + unfiltered)`, verboseOnly: true });
+        }
         // Transform the result set into BaseEntity-derived objects, if needed
         await this.TransformSimpleObjectToEntityObject(params, result, contextUser);
         // Run registered PostRunView hooks (e.g., data masking, audit logging)
         result = await this.RunPostRunViewHooks(params, result, contextUser);
-        // Store in local cache if enabled and we have a successful result
-        if (params.CacheLocal && result.Success && preResult.fingerprint && LocalCacheManager.Instance.IsInitialized) {
-            // Extract maxUpdatedAt from results if available
-            const maxUpdatedAt = this.extractMaxUpdatedAt(result.Results);
-            await LocalCacheManager.Instance.SetRunViewResult(preResult.fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults // Include aggregate results in cache
-            );
-        }
         // Register OnDataChanged callback if provided and we have a fingerprint
         if (params.OnDataChanged && preResult.fingerprint) {
             result.Unsubscribe = LocalCacheManager.Instance.RegisterChangeCallback(preResult.fingerprint, params.OnDataChanged);
@@ -881,25 +1105,26 @@ export class ProviderBase {
      * @param contextUser - Optional user context
      */
     async PostRunViews(results, params, preResult, contextUser) {
-        // Transform results in parallel
-        const transformPromises = [];
-        for (let i = 0; i < results.length; i++) {
-            transformPromises.push(this.TransformSimpleObjectToEntityObject(params[i], results[i], contextUser));
-        }
-        await Promise.all(transformPromises);
-        // Run registered PostRunView hooks on each result in the batch
-        for (let i = 0; i < results.length; i++) {
-            results[i] = await this.RunPostRunViewHooks(params[i], results[i], contextUser);
-        }
-        // Store in local cache if enabled
+        // Store in local cache BEFORE entity transformation — the cache needs
+        // plain JSON-serializable objects. BaseEntity objects contain RxJS Subjects
+        // with circular subscriber references that break JSON.stringify.
         const cachePromises = [];
         for (let i = 0; i < results.length; i++) {
-            // Store in local cache if enabled
+            // Skip results that came from cache hits — they're already cached and
+            // already transformed to entity objects (would break JSON.stringify).
+            const cacheInfo = preResult.cacheStatusMap?.get(i);
+            if (cacheInfo?.status === 'hit') {
+                continue;
+            }
             const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params[i], this.InstanceConnectionString);
             if (params[i].CacheLocal && results[i].Success && LocalCacheManager.Instance.IsInitialized) {
                 const maxUpdatedAt = this.extractMaxUpdatedAt(results[i].Results);
-                cachePromises.push(LocalCacheManager.Instance.SetRunViewResult(fingerprint, params[i], results[i].Results, maxUpdatedAt, results[i].AggregateResults // Include aggregate results in cache
-                ));
+                cachePromises.push(LocalCacheManager.Instance.SetRunViewResult(fingerprint, params[i], results[i].Results, maxUpdatedAt, results[i].AggregateResults));
+            }
+            else if (this.shouldAutoCache(params[i], results[i])) {
+                const maxUpdatedAt = this.extractMaxUpdatedAt(results[i].Results);
+                cachePromises.push(LocalCacheManager.Instance.SetRunViewResult(fingerprint, params[i], results[i].Results, maxUpdatedAt, results[i].AggregateResults));
+                LogStatusEx({ message: `    📦 [Auto-Cache] RunViews "${params[i].EntityName || params[i].ViewName || 'unknown'}" — ${results[i].Results.length} rows auto-cached (small + unfiltered)`, verboseOnly: true });
             }
             // Register OnDataChanged callback if provided
             if (params[i].OnDataChanged && fingerprint) {
@@ -907,6 +1132,21 @@ export class ProviderBase {
             }
         }
         await Promise.all(cachePromises);
+        // Transform results to entity objects AFTER caching plain objects.
+        // Skip results that came from cache hits — they're already entity objects.
+        const transformPromises = [];
+        for (let i = 0; i < results.length; i++) {
+            const cacheInfo = preResult.cacheStatusMap?.get(i);
+            if (cacheInfo?.status === 'hit') {
+                continue;
+            }
+            transformPromises.push(this.TransformSimpleObjectToEntityObject(params[i], results[i], contextUser));
+        }
+        await Promise.all(transformPromises);
+        // Run registered PostRunView hooks on each result in the batch
+        for (let i = 0; i < results.length; i++) {
+            results[i] = await this.RunPostRunViewHooks(params[i], results[i], contextUser);
+        }
         // End telemetry tracking with batch info
         if (preResult.telemetryEventId) {
             const totalResults = results.reduce((sum, r) => sum + (r.Results?.length ?? 0), 0);
@@ -1000,6 +1240,42 @@ export class ProviderBase {
      * @param results - Array of result objects that may contain __mj_UpdatedAt
      * @returns ISO string of the max timestamp, or current time if none found
      */
+    /**
+     * Determines if a RunView result should be automatically cached on the
+     * server side. Auto-caching is limited to small, unfiltered, unsorted
+     * result sets that are safe for in-place upsert on entity changes.
+     *
+     * Criteria (all must be true):
+     * - `TrustLocalCacheCompletely` is true (server-side only)
+     * - `CacheLocal` is NOT already set (already handled by explicit path)
+     * - `LocalCacheManager` is initialized
+     * - Result was successful
+     * - Result row count is at or below `ServerAutoCacheMaxRows`
+     * - No `ExtraFilter` (empty or undefined)
+     * - No `OrderBy` (empty or undefined)
+     */
+    shouldAutoCache(params, result) {
+        if (!this.TrustLocalCacheCompletely)
+            return false;
+        if (params.CacheLocal)
+            return false; // already handled
+        if (!LocalCacheManager.Instance.IsInitialized)
+            return false;
+        if (!result.Success)
+            return false;
+        if (ProviderBase.ServerAutoCacheMaxRows <= 0)
+            return false;
+        if ((result.Results?.length ?? 0) > ProviderBase.ServerAutoCacheMaxRows)
+            return false;
+        // Only auto-cache unfiltered, unsorted queries — these are safe for
+        // in-place upsert because LocalCacheManager doesn't need to evaluate
+        // SQL predicates or sort expressions.
+        const filter = typeof params.ExtraFilter === 'string' ? params.ExtraFilter.trim() : '';
+        const orderBy = typeof params.OrderBy === 'string' ? params.OrderBy.trim() : '';
+        if (filter.length > 0 || orderBy.length > 0)
+            return false;
+        return true;
+    }
     extractMaxUpdatedAt(results) {
         let maxDate = null;
         for (const item of results) {
@@ -1213,6 +1489,12 @@ export class ProviderBase {
      */
     async Config(data, providerToUse) {
         this._ConfigData = data;
+        // Initialize LocalCacheManager early so dataset loading can use the cache.
+        // Initialize() is idempotent — subsequent calls (e.g. from StartupManager) are no-ops.
+        if (!LocalCacheManager.Instance.IsInitialized) {
+            const storageProvider = this.LocalStorageProvider;
+            await LocalCacheManager.Instance.Initialize(storageProvider);
+        }
         // first, let's check to see if we have an existing Metadata.Provider registered, if so
         // unless our data.IgnoreExistingMetadata is set to true, we will not refresh the metadata
         if (Metadata.Provider && !data.IgnoreExistingMetadata) {
@@ -1318,13 +1600,13 @@ export class ProviderBase {
     async GetAllMetadata(providerToUse) {
         try {
             // we are now using datasets instead of the custom metadata to GraphQL to simplify GraphQL's work as it was very slow preivously
-            //const start1 = new Date().getTime();
-            const f = this.BuildDatasetFilterFromConfig();
+            // NOTE: Schema filters (IncludeSchemas/ExcludeSchemas) are for CodeGen only, not for runtime
+            // metadata loading. We always load all schemas — there are no sys/staging entities in metadata anyway.
             // Get the dataset and cache it for anyone else who wants to use it
-            const d = await this.GetDatasetByName(ProviderBase._mjMetadataDatasetName, f.length > 0 ? f : null, this.CurrentUser, providerToUse);
+            const d = await this.GetDatasetByName(ProviderBase._mjMetadataDatasetName, null, this.CurrentUser, providerToUse);
             if (d && d.Success) {
                 // cache the dataset for anyone who wants to use it
-                await this.CacheDataset(ProviderBase._mjMetadataDatasetName, f.length > 0 ? f : null, d);
+                await this.CacheDataset(ProviderBase._mjMetadataDatasetName, null, d);
                 // got the results, let's build our response in the format we need
                 const simpleMetadata = {};
                 for (let r of d.Results) {
@@ -1552,13 +1834,20 @@ export class ProviderBase {
      * @returns True if refresh is needed, false otherwise
      */
     async CheckToSeeIfRefreshNeeded(providerToUse) {
-        if (this.AllowRefresh) {
-            await this.RefreshRemoteMetadataTimestamps(providerToUse); // get the latest timestamps from the server first
-            await this.LoadLocalMetadataFromStorage(); // then, attempt to load before we check to see if it is obsolete
-            return this.LocalMetadataObsolete();
-        }
-        else //subclass is telling us not to do any refresh ops right now
+        if (!this.AllowRefresh)
             return false;
+        const now = Date.now();
+        if ((now - this._lastRefreshCheckAt) < ProviderBase.MinRefreshCheckIntervalMs) {
+            LogStatusEx({
+                message: `[RefreshCheck] Skipped — last check was ${now - this._lastRefreshCheckAt}ms ago (min interval ${ProviderBase.MinRefreshCheckIntervalMs}ms)`,
+                verboseOnly: true
+            });
+            return false;
+        }
+        this._lastRefreshCheckAt = now;
+        await this.RefreshRemoteMetadataTimestamps(providerToUse);
+        await this.LoadLocalMetadataFromStorage();
+        return this.LocalMetadataObsolete();
     }
     /**
      * Refreshes metadata only if needed based on timestamp comparison.
@@ -1852,8 +2141,8 @@ export class ProviderBase {
      * @returns Array of metadata update information
      */
     async GetLatestMetadataUpdates(providerToUse) {
-        const f = this.BuildDatasetFilterFromConfig();
-        const d = await this.GetDatasetStatusByName(ProviderBase._mjMetadataDatasetName, f.length > 0 ? f : null, this.CurrentUser, providerToUse);
+        // No schema filters for metadata — see comment in GetAllMetadata
+        const d = await this.GetDatasetStatusByName(ProviderBase._mjMetadataDatasetName, null, this.CurrentUser, providerToUse);
         if (d && d.Success) {
             const ret = d.EntityUpdateDates.map(e => {
                 return {