@memberjunction/generic-database-provider 5.8.0 → 5.10.0

package/dist/GenericDatabaseProvider.js

@@ -15,7 +15,7 @@
  *
  * @module @memberjunction/generic-database-provider
  */
-import { DatabaseProviderBase, EntityFieldTSType, EntityPermissionType, Metadata, QueryInfo, LogError, LogStatus, StripStopWords, } from '@memberjunction/core';
+import { DatabaseProviderBase, EntityFieldTSType, EntityPermissionType, InMemoryLocalStorageProvider, Metadata, QueryInfo, LocalCacheManager, LogError, LogStatus, LogStatusEx, StripStopWords, } from '@memberjunction/core';
 import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
 import { v4 as uuidv4 } from 'uuid';
 import { SqlLoggingSessionImpl } from './SqlLogger.js';
@@ -32,6 +32,44 @@ import { EncryptionEngine } from '@memberjunction/encryption';
  * to inherit these shared behaviors.
  */
 export class GenericDatabaseProvider extends DatabaseProviderBase {
+    /**
+     * Returns the active local storage provider, lazily creating an
+     * {@link InMemoryLocalStorageProvider} if none has been set.
+     *
+     * This fulfills the abstract `LocalStorageProvider` requirement from
+     * {@link ProviderBase} and is shared by all database providers
+     * (SQL Server, PostgreSQL, and any future platforms).
+     */
+    get LocalStorageProvider() {
+        if (!this._localStorageProvider) {
+            this._localStorageProvider = new InMemoryLocalStorageProvider();
+        }
+        return this._localStorageProvider;
+    }
+    /**
+     * Replaces the active local storage provider at runtime.
+     *
+     * Use this to swap from the default in-memory provider to a Redis-backed
+     * provider (or any other {@link ILocalStorageProvider}) after reading
+     * application configuration.
+     *
+     * @param provider - The new storage provider to use for all caching operations.
+     *
+     * @example
+     * ```typescript
+     * import { RedisLocalStorageProvider } from '@memberjunction/redis-provider';
+     *
+     * // During server startup, after config is loaded:
+     * const redis = new RedisLocalStorageProvider({
+     *     url: process.env.REDIS_URL,
+     *     defaultTTLSeconds: 300
+     * });
+     * (Metadata.Provider as GenericDatabaseProvider).SetLocalStorageProvider(redis);
+     * ```
+     */
+    SetLocalStorageProvider(provider) {
+        this._localStorageProvider = provider;
+    }
     /**************************************************************************/
     // SQL Logging — Session Management & Statement Logging
     /**************************************************************************/
@@ -948,9 +986,9 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
             if (!user) {
                 return { success: false, results: [], errorMessage: 'No user context available' };
             }
-            // Separate items that need cache check from those that don't
-            const itemsNeedingCacheCheck = [];
+            // Separate items by type: no cache check, needs validation
             const itemsWithoutCacheCheck = [];
+            const itemsNeedingValidation = [];
             const errorResults = [];
             for (let i = 0; i < params.length; i++) {
                 const item = params[i];
@@ -965,49 +1003,110 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
                 }
                 try {
                     this.CheckUserReadPermissions(entityInfo.Name, user);
-                    const whereSQL = await this.buildWhereClauseForCacheCheck(item.params, entityInfo, user);
-                    itemsNeedingCacheCheck.push({ index: i, item, entityInfo, whereSQL });
+                    itemsNeedingValidation.push({ index: i, item, entityInfo });
                 }
                 catch (e) {
                     errorResults.push({ viewIndex: i, status: 'error', errorMessage: e instanceof Error ? e.message : String(e) });
                 }
             }
-            // Execute batched cache status check
-            const cacheStatusResults = await this.getBatchedServerCacheStatus(itemsNeedingCacheCheck, contextUser);
-            // Determine which items are current vs stale, and whether they support differential updates
-            const differentialItems = [];
-            const staleItemsNoTracking = [];
+            // Phase 1: Check server's LocalCacheManager first (zero DB hits)
             const currentResults = [];
-            for (const { index, item, entityInfo, whereSQL } of itemsNeedingCacheCheck) {
-                const serverStatus = cacheStatusResults.get(index);
-                if (!serverStatus || !serverStatus.success) {
-                    errorResults.push({ viewIndex: index, status: 'error', errorMessage: serverStatus?.errorMessage || 'Failed to get cache status' });
-                    continue;
+            const serverCacheStaleItems = [];
+            const serverCacheMissItems = [];
+            for (const { index, item, entityInfo } of itemsNeedingValidation) {
+                const entityLabel = item.params.EntityName || 'unknown';
+                const resolved = await this.resolveFromServerCache(item, index, entityLabel);
+                if (resolved) {
+                    if (resolved.status === 'current') {
+                        currentResults.push(resolved.result);
+                    }
+                    else {
+                        serverCacheStaleItems.push({ index, item, entityInfo, serverCached: resolved.serverCached });
+                    }
                 }
-                if (this.isCacheCurrent(item.cacheStatus, serverStatus)) {
-                    currentResults.push({ viewIndex: index, status: 'current' });
+                else {
+                    serverCacheMissItems.push({ index, item, entityInfo });
                 }
-                else if (entityInfo.TrackRecordChanges) {
-                    differentialItems.push({
-                        index, params: item.params, entityInfo, whereSQL,
-                        clientMaxUpdatedAt: item.cacheStatus.maxUpdatedAt,
-                        clientRowCount: item.cacheStatus.rowCount,
-                        serverStatus,
-                    });
+            }
+            // Phase 2: For server cache misses, fall back to DB validation
+            const differentialItems = [];
+            const staleItemsNoTracking = [];
+            if (serverCacheMissItems.length > 0) {
+                // Build WHERE clauses and run batched DB status check only for cache misses
+                const itemsForDBCheck = [];
+                for (const { index, item, entityInfo } of serverCacheMissItems) {
+                    try {
+                        const whereSQL = await this.buildWhereClauseForCacheCheck(item.params, entityInfo, user);
+                        itemsForDBCheck.push({ index, item, entityInfo, whereSQL });
+                    }
+                    catch (e) {
+                        errorResults.push({ viewIndex: index, status: 'error', errorMessage: e instanceof Error ? e.message : String(e) });
+                    }
                 }
-                else {
-                    staleItemsNoTracking.push({ index, params: item.params });
+                const cacheStatusResults = await this.getBatchedServerCacheStatus(itemsForDBCheck, contextUser);
+                for (const { index, item, entityInfo, whereSQL } of itemsForDBCheck) {
+                    const serverStatus = cacheStatusResults.get(index);
+                    if (!serverStatus || !serverStatus.success) {
+                        errorResults.push({ viewIndex: index, status: 'error', errorMessage: serverStatus?.errorMessage || 'Failed to get cache status' });
+                        continue;
+                    }
+                    const entityLabel = item.params.EntityName || 'unknown';
+                    if (this.isCacheCurrent(item.cacheStatus, serverStatus)) {
+                        LogStatusEx({ message: `    ✅ [SmartCache CURRENT] "${entityLabel}" — client cache matches DB (server cache miss)`, verboseOnly: true });
+                        currentResults.push({ viewIndex: index, status: 'current' });
+                    }
+                    else if (entityInfo.TrackRecordChanges) {
+                        LogStatusEx({ message: `    🔄 [SmartCache DIFFERENTIAL] "${entityLabel}" — sending only changed rows (from DB)`, verboseOnly: true });
+                        differentialItems.push({
+                            index, params: item.params, entityInfo, whereSQL,
+                            clientMaxUpdatedAt: item.cacheStatus.maxUpdatedAt,
+                            clientRowCount: item.cacheStatus.rowCount,
+                            serverStatus,
+                        });
+                    }
+                    else {
+                        LogStatusEx({ message: `    🔍 [SmartCache STALE] "${entityLabel}" — full refresh from DB (no change tracking)`, verboseOnly: true });
+                        staleItemsNoTracking.push({ index, params: item.params });
+                    }
+                }
+            }
+            // Phase 3: For items without cacheStatus (client has nothing), check server cache before hitting DB
+            const noCacheStatusServedFromCache = [];
+            const noCacheStatusNeedsDB = [];
+            for (const entry of itemsWithoutCacheCheck) {
+                if (LocalCacheManager.Instance.IsInitialized) {
+                    const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(entry.item.params, this.InstanceConnectionString);
+                    const cached = await LocalCacheManager.Instance.GetRunViewResult(fingerprint);
+                    if (cached) {
+                        const entityLabel = entry.item.params.EntityName || 'unknown';
+                        LogStatusEx({ message: `    📦 [SmartCache SERVE-FROM-CACHE] "${entityLabel}" — client has no cache, serving ${cached.rowCount} rows from server cache, no DB hit`, verboseOnly: true });
+                        noCacheStatusServedFromCache.push({ index: entry.index, serverCached: cached });
+                        continue;
+                    }
                 }
+                noCacheStatusNeedsDB.push(entry);
             }
-            // Run queries in parallel
+            // Phase 4: Run queries in parallel for items needing data
             const queryPromises = [
-                ...itemsWithoutCacheCheck.map(({ index, item }) => this.runFullQueryAndReturn(item.params, index, contextUser)),
-                ...staleItemsNoTracking.map(({ index, params: viewParams }) => this.runFullQueryAndReturn(viewParams, index, contextUser)),
+                // Items without cache check — served from server cache
+                ...noCacheStatusServedFromCache.map(({ index, serverCached }) => this.serveFromServerCache(index, serverCached)),
+                // Items without cache check — server cache miss, must hit DB
+                ...noCacheStatusNeedsDB.map(({ index, item }) => this.runFullQueryAndCacheResult(item.params, index, contextUser)),
+                // Server cache stale — serve from server's cached data (zero DB)
+                ...serverCacheStaleItems.map(({ index, serverCached }) => this.serveFromServerCache(index, serverCached)),
+                // DB-validated stale items (no change tracking)
+                ...staleItemsNoTracking.map(({ index, params: viewParams }) => this.runFullQueryAndCacheResult(viewParams, index, contextUser)),
+                // DB-validated differential items
                 ...differentialItems.map(({ index, params: viewParams, entityInfo, whereSQL, clientMaxUpdatedAt, clientRowCount, serverStatus }) => this.runDifferentialQueryAndReturn(viewParams, entityInfo, clientMaxUpdatedAt, clientRowCount, serverStatus, whereSQL, index, contextUser)),
             ];
             const fullQueryResults = await Promise.all(queryPromises);
             const allResults = [...errorResults, ...currentResults, ...fullQueryResults];
             allResults.sort((a, b) => a.viewIndex - b.viewIndex);
+            const entities = params.map(p => p.params.EntityName || 'unknown').join(', ');
+            const totalServerCacheHits = (itemsNeedingValidation.length - serverCacheMissItems.length) + noCacheStatusServedFromCache.length;
+            const totalChecked = itemsNeedingValidation.length + itemsWithoutCacheCheck.length;
+            const totalDBQueries = noCacheStatusNeedsDB.length + staleItemsNoTracking.length + differentialItems.length;
+            LogStatusEx({ message: `  📊 [SmartCache] Batch [${entities}] — ${currentResults.length} current, ${serverCacheStaleItems.length + noCacheStatusServedFromCache.length} served-from-cache, ${differentialItems.length} differential, ${totalDBQueries} full-query, ${errorResults.length} errors (server cache: ${totalServerCacheHits}/${totalChecked} hits)`, verboseOnly: true });
             return { success: true, results: allResults };
         }
         catch (e) {
@@ -1090,6 +1189,53 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
         const maxUpdatedAt = this.extractMaxUpdatedAt(result.Results);
         return { viewIndex, status: 'stale', results: result.Results, maxUpdatedAt, rowCount: result.Results.length };
     }
+    /**
+     * Runs a full query and stores the result in the server's LocalCacheManager.
+     * Used by RunViewsWithCacheCheck to populate the server cache for future requests.
+     */
+    async runFullQueryAndCacheResult(params, viewIndex, contextUser) {
+        const result = await this.runFullQueryAndReturn(params, viewIndex, contextUser);
+        // Cache the result so subsequent RunViewsWithCacheCheck calls can skip DB
+        if (result.status !== 'error' && result.results && LocalCacheManager.Instance.IsInitialized) {
+            const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params, this.InstanceConnectionString);
+            const maxUpdatedAt = result.maxUpdatedAt || new Date().toISOString();
+            await LocalCacheManager.Instance.SetRunViewResult(fingerprint, params, result.results, maxUpdatedAt);
+        }
+        return result;
+    }
+    /**
+     * Checks the server's LocalCacheManager for cached data matching the client's request.
+     * Returns the resolution if found (either 'current' or server-cached data to serve),
+     * or null if the server cache doesn't have this data.
+     */
+    async resolveFromServerCache(item, index, entityLabel) {
+        if (!LocalCacheManager.Instance.IsInitialized)
+            return null;
+        const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(item.params, this.InstanceConnectionString);
+        const cached = await LocalCacheManager.Instance.GetRunViewResult(fingerprint);
+        if (!cached)
+            return null;
+        const serverStatus = { maxUpdatedAt: cached.maxUpdatedAt, rowCount: cached.rowCount };
+        if (this.isCacheCurrent(item.cacheStatus, serverStatus)) {
+            LogStatusEx({ message: `    ✅ [SmartCache CURRENT] "${entityLabel}" — client cache matches server cache, no DB hit`, verboseOnly: true });
+            return { status: 'current', result: { viewIndex: index, status: 'current' } };
+        }
+        // Server has newer data than client — we can serve it directly from cache
+        LogStatusEx({ message: `    📦 [SmartCache SERVE-FROM-CACHE] "${entityLabel}" — serving ${cached.rowCount} rows from server cache, no DB hit`, verboseOnly: true });
+        return { status: 'stale', serverCached: cached };
+    }
+    /**
+     * Packages server-cached data as a RunViewWithCacheCheckResult for return to the client.
+     */
+    async serveFromServerCache(viewIndex, serverCached) {
+        return {
+            viewIndex,
+            status: 'stale',
+            results: serverCached.results,
+            maxUpdatedAt: serverCached.maxUpdatedAt,
+            rowCount: serverCached.rowCount,
+        };
+    }
     /**
      * Runs a differential query and returns only changes since the client's cached state.
      * Includes updated/created rows and deleted record IDs.
@@ -1312,6 +1458,25 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
         }
         return null;
     }
+    /**
+     * Validates that a query can be executed by the given user. Checks both permissions
+     * and approval status. Permission failures throw an error. Non-approved status
+     * emits a console warning but allows execution to proceed, enabling query testing
+     * before formal approval.
+     *
+     * @param query - The resolved QueryInfo to validate
+     * @param contextUser - The user attempting to execute the query
+     * @throws Error if the user does not have permission to run the query
+     */
+    ValidateQueryForExecution(query, contextUser) {
+        const user = contextUser || this.CurrentUser;
+        if (user && !query.UserHasRunPermissions(user)) {
+            throw new Error(`User does not have permission to run query '${query.Name}' (ID: ${query.ID})`);
+        }
+        if (query.Status !== 'Approved') {
+            LogStatus(`WARNING: Executing query '${query.Name}' (ID: ${query.ID}) with status '${query.Status}'. Query has not been approved.`);
+        }
+    }
     /**
      * Creates a fresh QueryInfo from a MJQueryEntity and patches the ProviderBase cache.
      */
@@ -1414,7 +1579,15 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
             const quotes = pk.NeedsQuotes ? "'" : '';
             return `${this.QuoteIdentifier(pk.CodeName)}=${quotes}${val.Value}${quotes}`;
         }).join(' AND ');
-        const sql = `SELECT * FROM ${this.QuoteSchemaAndView(entityInfo.SchemaName, entityInfo.BaseView)} WHERE ${where}`;
+        // Append Read RLS filter if user is not exempt
+        let fullWhere = where;
+        if (user && !entityInfo.UserExemptFromRowLevelSecurity(user, EntityPermissionType.Read)) {
+            const rlsWhereClause = entityInfo.GetUserRowLevelSecurityWhereClause(user, EntityPermissionType.Read, '');
+            if (rlsWhereClause && rlsWhereClause.length > 0) {
+                fullWhere = `${where} AND (${rlsWhereClause})`;
+            }
+        }
+        const sql = `SELECT * FROM ${this.QuoteSchemaAndView(entityInfo.SchemaName, entityInfo.BaseView)} WHERE ${fullWhere}`;
         const rawData = await this.ExecuteSQL(sql, undefined, undefined, user);
         const d = await this.PostProcessRows(rawData, entityInfo, user);
         if (d && d.length > 0) {
@@ -1459,6 +1632,76 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
         return null;
     }
     /**************************************************************************/
+    // Row-Level Security Checks
+    /**************************************************************************/
+    /**
+     * Checks whether an existing record passes the RLS filter for a given permission type.
+     * Executes: SELECT COUNT(*) AS cnt FROM view WHERE PK=value AND (RLS filter)
+     * Returns true if the record matches (cnt > 0), false otherwise.
+     */
+    async CheckRecordRLS(entity, user, type) {
+        const entityInfo = entity.EntityInfo;
+        if (entityInfo.UserExemptFromRowLevelSecurity(user, type)) {
+            return true;
+        }
+        const rlsWhereClause = entityInfo.GetUserRowLevelSecurityWhereClause(user, type, '');
+        if (!rlsWhereClause || rlsWhereClause.length === 0) {
+            return true;
+        }
+        const pkWhere = entity.PrimaryKeys.map(pk => {
+            const fieldInfo = entityInfo.Fields.find(f => f.Name === pk.Name);
+            const quotes = fieldInfo?.NeedsQuotes ? "'" : '';
+            return `${this.QuoteIdentifier(pk.Name)}=${quotes}${pk.Value}${quotes}`;
+        }).join(' AND ');
+        const sql = `SELECT COUNT(*) AS cnt FROM ${this.QuoteSchemaAndView(entityInfo.SchemaName, entityInfo.BaseView)} WHERE ${pkWhere} AND (${rlsWhereClause})`;
+        const result = await this.ExecuteSQL(sql, undefined, undefined, user);
+        return result && result.length > 0 && Number(result[0]['cnt']) > 0;
+    }
+    /**
+     * Checks whether a new record's field values pass the Create RLS filter.
+     * Builds a synthetic single-row subquery from entity field values, then tests the RLS filter against it.
+     */
+    async CheckCreateRLS(entity, user) {
+        const entityInfo = entity.EntityInfo;
+        if (entityInfo.UserExemptFromRowLevelSecurity(user, EntityPermissionType.Create)) {
+            return true;
+        }
+        const rlsWhereClause = entityInfo.GetUserRowLevelSecurityWhereClause(user, EntityPermissionType.Create, '');
+        if (!rlsWhereClause || rlsWhereClause.length === 0) {
+            return true;
+        }
+        const projections = this.BuildCreateRLSProjections(entity, entityInfo);
+        const sql = `SELECT CASE WHEN (${rlsWhereClause}) THEN 1 ELSE 0 END AS pass FROM (SELECT ${projections}) AS newrow`;
+        const result = await this.ExecuteSQL(sql, undefined, undefined, user);
+        return result && result.length > 0 && Number(result[0]['pass']) === 1;
+    }
+    /**
+     * Builds field projections for the Create RLS synthetic row subquery.
+     * Only includes non-virtual fields that have non-null values.
+     */
+    BuildCreateRLSProjections(entity, entityInfo) {
+        const parts = [];
+        for (const field of entityInfo.Fields) {
+            if (field.IsVirtual)
+                continue;
+            const val = entity.Get(field.Name);
+            if (val == null)
+                continue;
+            let sqlVal;
+            if (typeof val === 'boolean') {
+                sqlVal = val ? '1' : '0';
+            }
+            else if (field.NeedsQuotes) {
+                sqlVal = `'${String(val).replace(/'/g, "''")}'`;
+            }
+            else {
+                sqlVal = String(val);
+            }
+            parts.push(`${sqlVal} AS ${this.QuoteIdentifier(field.Name)}`);
+        }
+        return parts.join(', ');
+    }
+    /**************************************************************************/
     // GetDatasetByName — Shared Implementation
     /**************************************************************************/
     /**
@@ -1499,10 +1742,18 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
                 Results: [],
             };
         }
-        // Phase 1: Build all item SQL queries
-        const queries = [];
-        const validItems = [];
+        // Phase 1: Check LocalCacheManager for each item, build SQL only for cache misses
+        const overallStart = performance.now();
+        const cache = LocalCacheManager.Instance;
+        const cacheAvailable = cache.IsInitialized && this.TrustLocalCacheCompletely;
         const errorResults = [];
+        const cachedResults = [];
+        const uncachedQueries = [];
+        const uncachedItems = [];
+        // Track fingerprints for uncached items so we can write-through after SQL
+        const uncachedFingerprints = [];
+        let cacheHitCount = 0;
+        let cacheMissCount = 0;
         for (const item of items) {
             const entitySchemaName = String(item['EntitySchemaName'] ?? schema);
             const entityBaseView = String(item['EntityBaseView']);
@@ -1510,12 +1761,37 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
             const entityName = String(item['Entity']);
             const entityID = String(item['EntityID']);
             const whereClause = item['WhereClause'] ? String(item['WhereClause']) : '';
-            let filterSQL = '';
+            // Build effective filter (WhereClause + optional runtime ItemFilter)
+            let effectiveFilter = whereClause;
             if (itemFilters && itemFilters.length > 0) {
                 const filter = itemFilters.find(f => f.ItemCode === code);
-                if (filter)
-                    filterSQL = (whereClause ? ' AND ' : ' WHERE ') + '(' + filter.Filter + ')';
+                if (filter) {
+                    effectiveFilter = whereClause
+                        ? `${whereClause} AND (${filter.Filter})`
+                        : filter.Filter;
+                }
             }
+            // Try cache first
+            if (cacheAvailable) {
+                const fingerprint = cache.GenerateRunViewFingerprint({ EntityName: entityName, ExtraFilter: effectiveFilter }, this.InstanceConnectionString);
+                const cached = await cache.GetRunViewResult(fingerprint);
+                if (cached) {
+                    cacheHitCount++;
+                    const dateFieldToCheck = String(item['DateFieldToCheck'] ?? '__mj_UpdatedAt');
+                    const latestUpdateDate = this.computeLatestUpdateDate(cached.results, dateFieldToCheck, item);
+                    cachedResults.push({
+                        EntityID: entityID,
+                        EntityName: entityName,
+                        Code: code,
+                        Results: cached.results,
+                        LatestUpdateDate: latestUpdateDate,
+                        Success: true,
+                    });
+                    continue; // Skip SQL for this item
+                }
+            }
+            // Cache miss — validate columns and build SQL
+            cacheMissCount++;
             const columns = provider.getColumnsForDatasetItem(item, datasetName);
             if (!columns) {
                 errorResults.push({
@@ -1529,22 +1805,30 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
                 });
                 continue;
             }
-            queries.push(`SELECT ${columns} FROM ${provider.QuoteSchemaAndView(entitySchemaName, entityBaseView)} ${whereClause ? 'WHERE ' + whereClause : ''}${filterSQL}`);
-            validItems.push(item);
+            const filterSQL = effectiveFilter ? 'WHERE ' + effectiveFilter : '';
+            uncachedQueries.push(`SELECT ${columns} FROM ${provider.QuoteSchemaAndView(entitySchemaName, entityBaseView)} ${filterSQL}`);
+            uncachedItems.push(item);
+            // Store fingerprint for write-through caching after SQL
+            const fp = cacheAvailable
+                ? cache.GenerateRunViewFingerprint({ EntityName: entityName, ExtraFilter: effectiveFilter }, this.InstanceConnectionString)
+                : '';
+            uncachedFingerprints.push(fp);
         }
-        // Phase 2: Execute all queries via ExecuteSQLBatch (true batch on SQL Server, parallel on PG)
+        // Phase 2: Execute SQL only for cache misses
         let batchResults = [];
-        try {
-            batchResults = await provider.ExecuteSQLBatch(queries, undefined, undefined, contextUser);
-        }
-        catch (err) {
-            LogError(`GetDatasetByName: Batch execution failed: ${err instanceof Error ? err.message : String(err)}`);
-            // Fall through with empty results
+        if (uncachedQueries.length > 0) {
+            try {
+                batchResults = await provider.ExecuteSQLBatch(uncachedQueries, undefined, undefined, contextUser);
+            }
+            catch (err) {
+                LogError(`GetDatasetByName: Batch execution failed: ${err instanceof Error ? err.message : String(err)}`);
+                // Fall through with empty results
+            }
         }
-        // Phase 3: Process results per item
-        const results = [...errorResults];
-        for (let i = 0; i < validItems.length; i++) {
-            const item = validItems[i];
+        // Phase 3: Process SQL results and write-through to cache
+        const sqlResults = [];
+        for (let i = 0; i < uncachedItems.length; i++) {
+            const item = uncachedItems[i];
             const entityName = String(item['Entity']);
             const entityID = String(item['EntityID']);
             const code = String(item['Code']);
@@ -1557,20 +1841,16 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
                     itemData = await provider.PostProcessRows(itemData, entityInfo, contextUser);
                 }
             }
-            const itemUpdatedAt = new Date(String(item['DatasetItemUpdatedAt']));
-            const datasetUpdatedAt = new Date(String(item['DatasetUpdatedAt']));
-            const datasetMaxUpdatedAt = new Date(Math.max(itemUpdatedAt.getTime(), datasetUpdatedAt.getTime()));
-            let latestUpdateDate = new Date(1900, 1, 1);
-            if (itemData && itemData.length > 0) {
-                for (const data of itemData) {
-                    if (data[dateFieldToCheck] && new Date(String(data[dateFieldToCheck])) > latestUpdateDate) {
-                        latestUpdateDate = new Date(String(data[dateFieldToCheck]));
-                    }
-                }
+            const latestUpdateDate = this.computeLatestUpdateDate(itemData, dateFieldToCheck, item);
+            // Write-through: cache this result for future requests (including empty results)
+            if (cacheAvailable && uncachedFingerprints[i]) {
+                const maxUpdatedAt = itemData.length > 0
+                    ? this.extractMaxUpdatedAtFromRows(itemData, dateFieldToCheck)
+                    : new Date(0).toISOString();
+                const syntheticParams = { EntityName: entityName };
+                await cache.SetRunViewResult(uncachedFingerprints[i], syntheticParams, itemData, maxUpdatedAt);
             }
-            if (datasetMaxUpdatedAt > latestUpdateDate)
-                latestUpdateDate = datasetMaxUpdatedAt;
-            results.push({
+            sqlResults.push({
                 EntityID: entityID,
                 EntityName: entityName,
                 Code: code,
@@ -1579,6 +1859,21 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
                 Success: itemData !== null && itemData !== undefined,
             });
         }
+        // Merge results: errors + cached + SQL (maintain original item order via code matching)
+        const results = [];
+        for (const item of items) {
+            const code = String(item['Code']);
+            const found = errorResults.find(r => r.Code === code)
+                ?? cachedResults.find(r => r.Code === code)
+                ?? sqlResults.find(r => r.Code === code);
+            if (found)
+                results.push(found);
+        }
+        const elapsedMs = (performance.now() - overallStart).toFixed(1);
+        LogStatusEx({
+            message: `📊 [Dataset] GetDatasetByName("${datasetName}"): ${cacheHitCount} cache hits, ${cacheMissCount} cache misses, ${errorResults.length} errors — ${elapsedMs}ms`,
+            verboseOnly: true
+        });
         // Aggregate results
         const bSuccess = results.every(result => result.Success);
         const latestUpdateDate = results.reduce((acc, result) => {
@@ -1606,9 +1901,12 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
      * latest update date. Uses ExecuteSQLBatch for per-item status queries.
      */
     async GetDatasetStatusByName(datasetName, itemFilters, contextUser, providerToUse) {
+        const overallStart = performance.now();
         const provider = (providerToUse ?? this);
         const schema = provider.MJCoreSchemaName;
-        // Fetch dataset items metadata
+        const cache = LocalCacheManager.Instance;
+        const cacheAvailable = cache.IsInitialized && this.TrustLocalCacheCompletely;
+        // Fetch dataset items metadata (lightweight — just the dataset definition, not entity data)
         const sSQL = `SELECT di.*, ` +
             `e.${provider.QuoteIdentifier('BaseView')} AS ${provider.QuoteIdentifier('EntityBaseView')}, ` +
             `e.${provider.QuoteIdentifier('SchemaName')} AS ${provider.QuoteIdentifier('EntitySchemaName')}, ` +
@@ -1629,61 +1927,117 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
                 EntityUpdateDates: [],
             };
         }
-        // Build per-item status queries
-        const queries = [];
-        const itemMeta = [];
+        // Phase 1: Try to derive status from cached data for each item
+        const updateDates = [];
+        let overallLatestDate = new Date(1900, 1, 1);
+        let cacheHitCount = 0;
+        let cacheMissCount = 0;
+        // Collect items that need SQL fallback
+        const uncachedItems = [];
+        const uncachedItemMeta = [];
         for (const item of items) {
-            const entitySchemaName = String(item['EntitySchemaName'] ?? schema);
-            const entityBaseView = String(item['EntityBaseView']);
             const entityID = String(item['EntityID']);
             const entityName = String(item['Entity']);
+            const code = String(item['Code']);
             const dateFieldToCheck = String(item['DateFieldToCheck'] ?? '__mj_UpdatedAt');
-            let filterSQL = '';
+            const whereClause = item['WhereClause'] ? String(item['WhereClause']) : '';
+            // Build effective filter for fingerprint
+            let effectiveFilter = whereClause;
             if (itemFilters && itemFilters.length > 0) {
-                const filter = itemFilters.find(f => f.ItemCode === String(item['Code']));
-                if (filter)
-                    filterSQL = ' WHERE ' + filter.Filter;
+                const filter = itemFilters.find(f => f.ItemCode === code);
+                if (filter) {
+                    effectiveFilter = whereClause
+                        ? `${whereClause} AND (${filter.Filter})`
+                        : filter.Filter;
+                }
             }
             const itemUpdatedAt = new Date(String(item['DatasetItemUpdatedAt']));
             const datasetUpdatedAt = new Date(String(item['DatasetUpdatedAt']));
-            const datasetMaxUpdatedAt = new Date(Math.max(itemUpdatedAt.getTime(), datasetUpdatedAt.getTime())).toISOString();
-            const statusSQL = `SELECT ` +
-                `CASE ` +
-                `WHEN MAX(${provider.QuoteIdentifier(dateFieldToCheck)}) > '${datasetMaxUpdatedAt}' THEN MAX(${provider.QuoteIdentifier(dateFieldToCheck)}) ` +
-                `ELSE '${datasetMaxUpdatedAt}' ` +
-                `END AS ${provider.QuoteIdentifier('UpdateDate')}, ` +
-                `COUNT(*) AS ${provider.QuoteIdentifier('TheRowCount')} ` +
-                `FROM ${provider.QuoteSchemaAndView(entitySchemaName, entityBaseView)}${filterSQL}`;
-            queries.push(statusSQL);
-            itemMeta.push({ entityID, entityName, datasetMaxUpdatedAt });
-        }
-        // Execute all status queries via ExecuteSQLBatch
-        let batchResults = [];
-        try {
-            batchResults = await provider.ExecuteSQLBatch(queries, undefined, undefined, contextUser);
-        }
-        catch (err) {
-            LogError(`GetDatasetStatusByName: Batch execution failed: ${err instanceof Error ? err.message : String(err)}`);
+            const datasetMaxUpdatedAt = new Date(Math.max(itemUpdatedAt.getTime(), datasetUpdatedAt.getTime()));
+            // Try to derive status from cached data
+            if (cacheAvailable) {
+                const fingerprint = cache.GenerateRunViewFingerprint({ EntityName: entityName, ExtraFilter: effectiveFilter }, this.InstanceConnectionString);
+                const cached = await cache.GetRunViewResult(fingerprint);
+                if (cached) {
+                    cacheHitCount++;
+                    // Derive MAX(dateField) and COUNT(*) directly from cached rows
+                    let maxDateFromRows = new Date(1900, 1, 1);
+                    for (const row of cached.results) {
+                        const record = row;
+                        if (record[dateFieldToCheck]) {
+                            const d = new Date(String(record[dateFieldToCheck]));
+                            if (d > maxDateFromRows)
+                                maxDateFromRows = d;
+                        }
+                    }
+                    const updateDate = maxDateFromRows > datasetMaxUpdatedAt ? maxDateFromRows : datasetMaxUpdatedAt;
+                    updateDates.push({
+                        EntityID: entityID,
+                        EntityName: entityName,
+                        RowCount: cached.results.length,
+                        UpdateDate: updateDate,
+                    });
+                    if (updateDate > overallLatestDate)
+                        overallLatestDate = updateDate;
+                    continue; // No SQL needed for this item
+                }
+            }
+            // Cache miss — need SQL fallback
+            cacheMissCount++;
+            uncachedItems.push(item);
+            uncachedItemMeta.push({ entityID, entityName, datasetMaxUpdatedAt: datasetMaxUpdatedAt.toISOString() });
         }
-        // Process results
-        const updateDates = [];
-        let overallLatestDate = new Date(1900, 1, 1);
-        for (let i = 0; i < itemMeta.length; i++) {
-            const meta = itemMeta[i];
-            const statusRows = batchResults[i];
-            if (statusRows && statusRows.length > 0) {
-                const updateDate = new Date(String(statusRows[0]['UpdateDate']));
-                updateDates.push({
-                    EntityID: meta.entityID,
-                    EntityName: meta.entityName,
-                    RowCount: Number(statusRows[0]['TheRowCount']),
-                    UpdateDate: updateDate,
-                });
-                if (updateDate > overallLatestDate) {
-                    overallLatestDate = updateDate;
+        // Phase 2: Execute SQL only for cache misses
+        if (uncachedItems.length > 0) {
+            const queries = uncachedItems.map((item, idx) => {
+                const entitySchemaName = String(item['EntitySchemaName'] ?? schema);
+                const entityBaseView = String(item['EntityBaseView']);
+                const code = String(item['Code']);
+                const dateFieldToCheck = String(item['DateFieldToCheck'] ?? '__mj_UpdatedAt');
+                const meta = uncachedItemMeta[idx];
+                let filterSQL = '';
+                if (itemFilters && itemFilters.length > 0) {
+                    const filter = itemFilters.find(f => f.ItemCode === code);
+                    if (filter)
+                        filterSQL = ' WHERE ' + filter.Filter;
+                }
+                return `SELECT ` +
+                    `CASE ` +
+                    `WHEN MAX(${provider.QuoteIdentifier(dateFieldToCheck)}) > '${meta.datasetMaxUpdatedAt}' THEN MAX(${provider.QuoteIdentifier(dateFieldToCheck)}) ` +
+                    `ELSE '${meta.datasetMaxUpdatedAt}' ` +
+                    `END AS ${provider.QuoteIdentifier('UpdateDate')}, ` +
+                    `COUNT(*) AS ${provider.QuoteIdentifier('TheRowCount')} ` +
+                    `FROM ${provider.QuoteSchemaAndView(entitySchemaName, entityBaseView)}${filterSQL}`;
+            });
+            let batchResults = [];
+            try {
+                batchResults = await provider.ExecuteSQLBatch(queries, undefined, undefined, contextUser);
+            }
+            catch (err) {
+                LogError(`GetDatasetStatusByName: Batch execution failed: ${err instanceof Error ? err.message : String(err)}`);
+            }
+            for (let i = 0; i < uncachedItemMeta.length; i++) {
+                const meta = uncachedItemMeta[i];
+                const statusRows = batchResults[i];
+                if (statusRows && statusRows.length > 0) {
+                    const updateDate = new Date(String(statusRows[0]['UpdateDate']));
+                    updateDates.push({
+                        EntityID: meta.entityID,
+                        EntityName: meta.entityName,
+                        RowCount: Number(statusRows[0]['TheRowCount']),
+                        UpdateDate: updateDate,
+                    });
+                    if (updateDate > overallLatestDate) {
+                        overallLatestDate = updateDate;
+                    }
                 }
             }
         }
+        const elapsedMs = (performance.now() - overallStart).toFixed(1);
+        LogStatusEx({
+            message: `📊 [Dataset Status] GetDatasetStatusByName("${datasetName}"): ${cacheHitCount} cache-derived, ${cacheMissCount} SQL queries — ${elapsedMs}ms`,
+            verboseOnly: true
+        });
         if (updateDates.length === 0) {
             return {
                 DatasetID: String(items[0]['DatasetID']),
@@ -1739,5 +2093,56 @@ export class GenericDatabaseProvider extends DatabaseProviderBase {
         }
         return specifiedColumns.length > 0 ? specifiedColumns.map(col => this.QuoteIdentifier(col.trim())).join(',') : '*';
     }
+    /**************************************************************************/
+    // Dataset Cache Helpers
+    /**************************************************************************/
+    /**
+     * Computes the latest update date for a dataset item from its result rows and dataset metadata.
+     * Used by both the cache-hit and cache-miss paths in GetDatasetByName.
+     * @param rows - The result rows (from cache or SQL)
+     * @param dateFieldToCheck - The field name to scan for latest date
+     * @param item - The dataset item metadata row (contains DatasetItemUpdatedAt, DatasetUpdatedAt)
+     * @returns The latest date across all rows and dataset metadata
+     */
+    computeLatestUpdateDate(rows, dateFieldToCheck, item) {
+        const itemUpdatedAt = new Date(String(item['DatasetItemUpdatedAt']));
+        const datasetUpdatedAt = new Date(String(item['DatasetUpdatedAt']));
+        const datasetMaxUpdatedAt = new Date(Math.max(itemUpdatedAt.getTime(), datasetUpdatedAt.getTime()));
+        let latestUpdateDate = new Date(1900, 1, 1);
+        if (rows && rows.length > 0) {
+            for (const data of rows) {
+                const record = data;
+                if (record[dateFieldToCheck]) {
+                    const d = new Date(String(record[dateFieldToCheck]));
+                    if (d > latestUpdateDate)
+                        latestUpdateDate = d;
+                }
+            }
+        }
+        if (datasetMaxUpdatedAt > latestUpdateDate)
+            latestUpdateDate = datasetMaxUpdatedAt;
+        return latestUpdateDate;
+    }
+    /**
+     * Extracts the MAX value of a specified date field from result rows as an ISO string.
+     * Used for write-through caching of dataset item results.
+     * @param rows - The result rows
+     * @param dateFieldToCheck - The field name to scan
+     * @returns ISO string of the max date, or current time if no dates found
+     */
+    extractMaxUpdatedAtFromRows(rows, dateFieldToCheck) {
+        let maxDate = null;
+        for (const row of rows) {
+            const record = row;
+            const val = record[dateFieldToCheck];
+            if (val) {
+                const d = val instanceof Date ? val : new Date(val);
+                if (!isNaN(d.getTime()) && (!maxDate || d > maxDate)) {
+                    maxDate = d;
+                }
+            }
+        }
+        return maxDate ? maxDate.toISOString() : new Date().toISOString();
+    }
 }
 //# sourceMappingURL=GenericDatabaseProvider.js.map