@memberjunction/core 5.40.2 → 5.42.0

package/dist/generic/providerBase.js

@@ -113,6 +113,63 @@ export const AllMetadataArrays = [
     { key: 'AllLibraries', class: LibraryInfo },
     { key: 'AllExplorerNavigationItems', class: ExplorerNavigationItem }
 ];
+/**
+ * Projects plain-object rows down to a caller-requested field subset, matching
+ * field names case-insensitively (and ignoring surrounding whitespace).
+ *
+ * Used by the RunView caching pipeline: when a query is cacheable, the provider
+ * widens `params.Fields` to ALL entity fields so the cache entry is a universal
+ * superset that satisfies any future field subset. This helper restores the
+ * caller's originally requested shape — on cache hits (filtering the cached
+ * superset) AND on cache misses (filtering the widened DB result) — so callers
+ * always receive the same columns regardless of cache temperature.
+ *
+ * Returns the original array untouched when no projection is requested
+ * (`requestedFields` null/empty) or there are no rows. Never mutates input rows.
+ *
+ * @param rows - Plain-object result rows (NOT BaseEntity objects)
+ * @param requestedFields - The caller's original Fields list, or null for "all fields"
+ */
+export function ProjectRowsToFields(rows, requestedFields) {
+    if (!requestedFields || requestedFields.length === 0 || !rows || rows.length === 0) {
+        return rows;
+    }
+    const requestedFieldSet = new Set(requestedFields.map(f => f.trim().toLowerCase()));
+    // No-op probe: SQL result rows are uniform (same SELECT list), so if every key of
+    // the first row is requested, the projection would copy every row unchanged —
+    // return the original array and skip the per-row object rebuilds entirely. This
+    // is the common case for entity_object-widened requests and full-coverage Fields.
+    const probe = rows[0];
+    if (probe && typeof probe === 'object') {
+        let allKept = true;
+        for (const key of Object.keys(probe)) {
+            if (!requestedFieldSet.has(key.toLowerCase())) {
+                allKept = false;
+                break;
+            }
+        }
+        if (allKept) {
+            return rows;
+        }
+    }
+    // Cache lowercase key→keep decisions across rows to avoid repeated allocations
+    const keyCache = new Map();
+    return rows.map((row) => {
+        const source = row;
+        const filtered = {};
+        for (const key of Object.keys(source)) {
+            let keep = keyCache.get(key);
+            if (keep === undefined) {
+                keep = requestedFieldSet.has(key.toLowerCase());
+                keyCache.set(key, keep);
+            }
+            if (keep) {
+                filtered[key] = source[key];
+            }
+        }
+        return filtered;
+    });
+}
 /**
  * Base class for all metadata providers in MemberJunction.
  * Implements common functionality for metadata caching, refresh, and dataset management.
@@ -141,6 +198,7 @@ export class ProviderBase {
          * lingers so that near-sequential identical calls return immediately.
          */
         this._inflightViews = new Map();
+        this._clientFingerprintMemo = new WeakMap();
         this._cachedVisibleExplorerNavigationItems = null;
     }
     // ── Metadata Refresh Check Debounce ────────────────────────────────
@@ -187,6 +245,13 @@ export class ProviderBase {
      * still applies).  Default 5 000 ms.
      */
     static { this.DedupLingerMs = 5000; }
+    /**
+     * Safety cap on the number of linger entries held simultaneously. The linger
+     * window is a latency optimization — under extreme churn (more distinct query
+     * keys than this resolving within one window) new resolutions skip lingering
+     * instead of accumulating result arrays in memory.
+     */
+    static { this.MaxLingerEntries = 500; }
     /******** ABSTRACT SECTION ****************************************************************** */
     /**
      * When true, cached RunView/RunQuery results are returned immediately on a
@@ -327,6 +392,46 @@ export class ProviderBase {
     // ========================================================================
     // PUBLIC API METHODS - Orchestrate Pre → Cache → Internal → Post flow
     // ========================================================================
+    /**
+     * Routes a typed Remote Operation by key to its implementation (see {@link IRemoteOperationProvider}).
+     *
+     * This is the public **power-tool** transport seam. Prefer the typed
+     * `BaseRemotableOperation.Execute()` entry point in application code — `RouteOperation` is the
+     * stringly-typed escape hatch for dynamic dispatch / generic tooling, not for building
+     * significant systems. Server providers override {@link InternalRouteOperation} to execute the
+     * operation in-process; the client (GraphQL) provider overrides it to marshal over the wire.
+     * Only registered, active (and, when AI-authored, approved) operations are routable, and every
+     * call is authorized on the server side.
+     *
+     * @param operationKey - Stable registry key of the operation (e.g. `RecordProcess.RunNow`).
+     * @param input - The operation's typed input payload.
+     * @param options - Optional invocation options (mode, progress callback, user, provider, fingerprint).
+     * @returns The operation result; never throws for logical failures — check `Success`/`ErrorMessage`.
+     */
+    async RouteOperation(operationKey, input, options) {
+        const key = operationKey?.trim();
+        if (!key) {
+            return { Success: false, ResultCode: 'INVALID_OPERATION_KEY', ErrorMessage: 'operationKey is required' };
+        }
+        return this.InternalRouteOperation(key, input, options ?? {});
+    }
+    /**
+     * Provider-specific transport for a Remote Operation. The default implementation reports that
+     * the provider does not support remote operations; concrete providers override it — server
+     * providers resolve and execute the operation in-process, the client provider marshals it over
+     * GraphQL. Kept as an overridable (non-abstract) hook so existing providers remain source-
+     * compatible until they opt in.
+     * @param operationKey - Trimmed, non-empty operation key (validated by {@link RouteOperation}).
+     * @param input - The operation's typed input payload.
+     * @param options - Invocation options (never undefined here; defaulted by {@link RouteOperation}).
+     */
+    InternalRouteOperation(operationKey, _input, _options) {
+        return Promise.resolve({
+            Success: false,
+            ResultCode: 'NOT_SUPPORTED',
+            ErrorMessage: `This provider does not support remote operations (operationKey='${operationKey}')`,
+        });
+    }
     /**
      * Runs a view based on the provided parameters.
      * This method orchestrates the full execution flow: pre-processing, cache check,
@@ -336,6 +441,10 @@ export class ProviderBase {
      * @returns The view results
      */
     async RunView(params, contextUser) {
+        // Shallow-clone so the pipeline's in-place modifications (PlatformSQL resolution,
+        // Fields widening for cache-superset storage) never leak into the CALLER's params
+        // object — reusing a params object across calls must be safe.
+        params = { ...params };
         // Keyset (AfterKey) queries always bypass the server cache: each call uses a
         // different seek key, so a cached entry would never be reusable. Treat them like
         // explicit BypassCache=true requests.
@@ -383,6 +492,10 @@ export class ProviderBase {
      * @returns Array of view results (shallow-copied Results per caller)
      */
     async RunViews(params, contextUser) {
+        // Shallow-clone every param so the pipeline's in-place modifications (PlatformSQL
+        // resolution, Fields widening for cache-superset storage) never leak into the
+        // CALLER's objects — reusing params across calls must be safe.
+        params = params.map(p => ({ ...p }));
         // Bypass dedup for side-effect calls (SaveViewResults creates DB records)
         if (this.ShouldBypassDedup(params)) {
             return this.ExecuteRunViewsPipeline(params, contextUser);
@@ -422,13 +535,16 @@ export class ProviderBase {
         // ── Fresh execution ──
         const promise = this.ExecuteRunViewsPipeline(params, contextUser)
             .then(results => {
-            // Stash resolved results for the linger window
+            // Stash resolved results for the linger window. Safety cap: under
+            // extreme churn (hundreds of distinct keys resolving within one linger
+            // window) skip lingering rather than hold more result arrays in memory —
+            // the linger is a latency optimization, never a correctness requirement.
             const entry = this._inflightViews.get(key);
             if (entry && entry.promise === promise) {
-                entry.resolvedResults = results;
-                entry.resolvedAt = Date.now();
-                // Schedule cleanup after linger expires
-                if (ProviderBase.DedupLingerMs > 0) {
+                if (ProviderBase.DedupLingerMs > 0 && this._inflightViews.size <= ProviderBase.MaxLingerEntries) {
+                    entry.resolvedResults = results;
+                    entry.resolvedAt = Date.now();
+                    // Schedule cleanup after linger expires
                     setTimeout(() => {
                         const current = this._inflightViews.get(key);
                         if (current && current.promise === promise) {
@@ -751,15 +867,67 @@ export class ProviderBase {
     /**
      * Generates a deterministic dedup key for a batch of RunViewParams.
      * Extends the local-cache fingerprint with additional fields that
-     * affect result identity (Fields, UserSearchString, ViewID, ViewName,
-     * contextUser).
-     */
+     * affect result identity (Fields, ResultType, UserSearchString, ViewID,
+     * ViewName, contextUser).
+     *
+     * Unlike the cache fingerprint — which deliberately excludes Fields and
+     * ResultType because the cache stores the full-width superset and projects
+     * / transforms per-read — the dedup layer shares the FINAL pipeline output:
+     * results already projected to one caller's Fields and already transformed
+     * per that caller's ResultType. A linger or in-flight hit hands those rows
+     * to the next caller verbatim (shallow array copy only), so callers with
+     * different Fields or ResultType must NOT share a dedup slot or the second
+     * caller silently receives the first caller's shape.
+     */
+    /**
+     * Single source of truth for whether a RunView call participates in the local
+     * cache (both READ and WRITE). Pre/Post hooks for the singular and batch paths
+     * must all use this predicate — historically each site recomputed it inline and
+     * they drifted (PostRunViews wrote BypassCache results into the cache, poisoning
+     * the Fields-agnostic superset slot with narrow rows).
+     *
+     * Ineligible:
+     * - `BypassCache` — caller explicitly wants true DB state, no cache interaction
+     * - `AfterKey` — keyset pages are single-use AND the fingerprint doesn't include
+     *   the seek key, so caching a page would poison the entity+filter slot
+     * - `ResultType 'count_only'` — returns no rows; caching its empty Results under
+     *   a fingerprint that excludes ResultType would poison row queries
+     * - entities where server caching is disallowed
+     */
+    runViewCacheEligible(param) {
+        return !param.BypassCache &&
+            !param.AfterKey &&
+            param.ResultType !== 'count_only' &&
+            (param.CacheLocal === true || this.TrustLocalCacheCompletely) &&
+            this.IsServerCacheAllowedForEntity(param);
+    }
+    /**
+     * Returns the caller's requested fields (lowercased) unioned with the entity's
+     * primary key field names. Platform contract: when `Fields` is explicitly
+     * specified, results ALWAYS include the primary key(s) — the direct SQL path has
+     * always done this, differential smart-cache merges require it, and entity
+     * linking in UIs depends on it. Applying the same union at every projection site
+     * keeps result shapes identical across cached, non-cached, and smart-cache paths.
+     */
+    static UnionFieldsWithPrimaryKeys(fields, entity) {
+        const result = [...fields];
+        const present = new Set(fields);
+        // Defensive ?? [] — virtual entities can be PK-less, and test doubles may not model PrimaryKeys
+        for (const pk of entity.PrimaryKeys ?? []) {
+            const name = pk.Name.trim().toLowerCase();
+            if (!present.has(name)) {
+                present.add(name);
+                result.push(name);
+            }
+        }
+        return result;
+    }
     GenerateDedupKey(params, contextUser) {
         const parts = params.map(p => {
             const base = LocalCacheManager.Instance.GenerateRunViewFingerprint(p, this.InstanceConnectionString);
-            // Fields is intentionally excluded — cache stores full entity width
-            // and filters on return, so different Fields values are the same query.
             const extras = [
+                ProviderBase.NormalizeFieldsKey(p.Fields),
+                p.ResultType ?? 'simple',
                 p.UserSearchString ?? '',
                 p.ViewID ?? '',
                 p.ViewName ?? '',
@@ -769,6 +937,54 @@ export class ProviderBase {
         });
         return parts.join('||');
     }
+    /**
+     * Normalizes a Fields list into a stable key segment: trimmed, lowercased,
+     * sorted, comma-joined — `'*'` when the caller wants all fields. Matches the
+     * matching semantics of `ProjectRowsToFields` (trim + lowercase) so that
+     * semantically identical requests collapse to the same key. Used by both the
+     * request-dedup key and the client-side cache fingerprint.
+     */
+    static NormalizeFieldsKey(fields) {
+        return fields && fields.length > 0
+            ? fields.map(f => f.trim().toLowerCase()).sort().join(',')
+            : '*';
+    }
+    /**
+     * Client-side cache fingerprint: the shared RunView fingerprint plus a
+     * normalized Fields suffix (`|f:<fields>` or `|f:*`).
+     *
+     * Why the client fingerprint includes Fields when the server's deliberately
+     * does NOT: the server cache widens every cacheable query to ALL entity
+     * fields before the DB hit, stores one full-width superset per entity+filter,
+     * and projects per-read — so a single Fields-agnostic slot can serve any
+     * field subset. The client smart-cache flow does NOT widen (narrow wire
+     * payloads are the point of `Fields` client-side) and does NOT project on
+     * read: rows are stored exactly as the server returned them. Under a
+     * Fields-agnostic fingerprint, a narrow entry would pass the staleness check
+     * for a DIFFERENT field subset of the same entity+filter — `maxUpdatedAt`
+     * and `rowCount` are column-independent — and silently serve rows missing
+     * the newly requested columns. Per-Fields slots make client entries
+     * exact-match only: each field subset stores, validates, and serves its own
+     * shape. (Subset-serving from wider entries was considered and deliberately
+     * rejected: it requires candidate enumeration, per-entry field metadata, and
+     * careful staleness attribution for marginal hit-rate gains.)
+     */
+    clientCacheFingerprint(param) {
+        // Memoized per params object: RunViews shallow-clones params once at entry and
+        // the SAME object references flow through prepare → execute → process, where this
+        // fingerprint was previously recomputed up to 3× per param (string building +
+        // Fields normalization each time). Safe because params are not mutated after the
+        // first computation (entity_object Fields widening happens in
+        // prepareSmartCacheCheckParams BEFORE the first fingerprint call).
+        const memoized = this._clientFingerprintMemo.get(param);
+        if (memoized) {
+            return memoized;
+        }
+        const base = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString);
+        const fingerprint = `${base}|f:${ProviderBase.NormalizeFieldsKey(param.Fields)}`;
+        this._clientFingerprintMemo.set(param, fingerprint);
+        return fingerprint;
+    }
     /**
      * Ranked search over **one** entity's records. See {@link IMetadataProvider.SearchEntity}
      * for the contract and how this differs from {@link EntityByName} /
@@ -1055,6 +1271,80 @@ export class ProviderBase {
      * @returns The query results
      */
     async RunQuery(params, contextUser) {
+        // Shallow-clone for symmetry with RunView — pipeline must never mutate caller objects
+        params = { ...params };
+        // ── CacheLocal: the RunQuery result cache (RunQueryCache category) ──
+        // Engages ONLY on explicit opt-in. Saved queries only (QueryID/QueryName) —
+        // ad-hoc SQL is never cached. Semantics per RunQueryParams JSDoc:
+        //  - cached + unexpired + provider supports RunQueriesWithCacheCheck (client):
+        //    server validates via the Query's CacheValidationSQL → 'current' serves the
+        //    local slot, 'stale'/'no_validation' returns fresh rows and rewrites the slot
+        //  - cached + unexpired + no validation transport (server providers): TTL mode —
+        //    serve directly until expiry
+        //  - miss/expired: run normally, then store with TTL (CacheLocalTTL override,
+        //    else the LocalCacheManager default)
+        const queryCacheEngaged = params.CacheLocal === true
+            && !params.SQL
+            && (!!params.QueryID || !!params.QueryName)
+            && LocalCacheManager.Instance.IsInitialized;
+        let queryFingerprint;
+        if (queryCacheEngaged) {
+            // MaxRows/StartRow shape the result set — they MUST distinguish cache slots,
+            // so fold them into the parameters portion of the fingerprint.
+            const fingerprintParams = {
+                ...(params.Parameters ?? {}),
+                __maxRows: params.MaxRows ?? -1,
+                __startRow: params.StartRow ?? 0
+            };
+            queryFingerprint = LocalCacheManager.Instance.GenerateRunQueryFingerprint(params.QueryID, params.QueryName, fingerprintParams, this.InstanceConnectionString);
+            const cached = await LocalCacheManager.Instance.GetRunQueryResult(queryFingerprint); // TTL-enforced
+            if (cached) {
+                const serveFromSlot = () => ({
+                    QueryID: cached.queryId ?? params.QueryID ?? '',
+                    QueryName: params.QueryName ?? '',
+                    Success: true,
+                    Results: cached.results,
+                    RowCount: cached.results.length,
+                    TotalRowCount: cached.rowCount ?? cached.results.length,
+                    ExecutionTime: 0,
+                    ErrorMessage: '',
+                    CacheHit: true,
+                    CacheKey: queryFingerprint
+                });
+                const checker = this.RunQueriesWithCacheCheck?.bind(this);
+                if (!checker || this.TrustLocalCacheCompletely) {
+                    // TTL mode (server providers / no validation transport)
+                    return serveFromSlot();
+                }
+                // Client smart validation round trip
+                const response = await checker([{
+                        params,
+                        cacheStatus: { maxUpdatedAt: cached.maxUpdatedAt, rowCount: cached.rowCount }
+                    }], contextUser);
+                const check = response.results?.[0];
+                if (response.success && check) {
+                    if (check.status === 'current') {
+                        return serveFromSlot();
+                    }
+                    if ((check.status === 'stale' || check.status === 'no_validation') && check.results) {
+                        const freshRows = check.results;
+                        // Fire-and-forget slot rewrite — same pattern as the RunView client path
+                        LocalCacheManager.Instance.SetRunQueryResult(queryFingerprint, params.QueryName ?? '', freshRows, check.maxUpdatedAt ?? '', check.rowCount, check.queryId, params.CacheLocalTTL).catch(e => LogError(`RunQuery cache rewrite failed: ${e}`));
+                        return {
+                            QueryID: check.queryId ?? params.QueryID ?? '',
+                            QueryName: params.QueryName ?? '',
+                            Success: true,
+                            Results: freshRows,
+                            RowCount: freshRows.length,
+                            TotalRowCount: check.rowCount ?? freshRows.length,
+                            ExecutionTime: 0,
+                            ErrorMessage: ''
+                        };
+                    }
+                }
+                // validation transport failed — fall through to a normal execution
+            }
+        }
         // Pre-processing: telemetry, cache check
         const preResult = await this.PreRunQuery(params, contextUser);
         // Check for cached result - end telemetry with cache hit info
@@ -1070,6 +1360,12 @@ export class ProviderBase {
         const result = await this.InternalRunQuery(params, contextUser);
         // Post-processing: cache storage, telemetry end
         await this.PostRunQuery(result, params, preResult, contextUser);
+        // Store in the RunQuery cache on success (fire-and-forget; TTL per CacheLocalTTL
+        // or the LocalCacheManager default). maxUpdatedAt is unknown for a plain run —
+        // the smart-validation path stamps it when the Query has CacheValidationSQL.
+        if (queryCacheEngaged && queryFingerprint && result.Success) {
+            LocalCacheManager.Instance.SetRunQueryResult(queryFingerprint, result.QueryName, result.Results, '', result.TotalRowCount, result.QueryID, params.CacheLocalTTL).catch(e => LogError(`RunQuery cache write failed: ${e}`));
+        }
         return result;
     }
     /**
@@ -1227,7 +1523,9 @@ export class ProviderBase {
             MaxRows: params.MaxRows,
             StartRow: params.StartRow,
             CacheLocal: params.CacheLocal,
-            _fromEngine: params._fromEngine
+            _fromEngine: params._fromEngine,
+            Exempt: params.Telemetry?.Exempt,
+            ExemptReason: params.Telemetry?.Reason
         }, contextUser?.ID);
         const telemetryTime = performance.now() - telemetryStart;
         // Entity status check
@@ -1238,22 +1536,21 @@ export class ProviderBase {
         // We always fetch ALL fields from the DB so the cache entry is a universal superset
         // that satisfies any future query for the same entity+filter regardless of field subset.
         const entityLookupStart = performance.now();
-        const callerRequestedFields = params.Fields && params.Fields.length > 0
+        let callerRequestedFields = params.Fields && params.Fields.length > 0
             ? params.Fields.map(f => f.trim().toLowerCase())
             : null; // null = caller wants all fields
         // Only override Fields to all entity fields when caching will actually happen
         // for this call. For non-cached calls we respect the caller's narrow Fields
         // end-to-end — there's no cache-coherence concern to preserve.
         const entity = params.EntityName ? this.EntityByName(params.EntityName) : null;
-        const entityCacheAllowed = this.IsServerCacheAllowedForEntity(params);
-        // Keyset (AfterKey) queries are inherently single-use, so we never read from or
-        // write to the cache for them. See RunViewParams.AfterKey JSDoc for rationale.
-        const willCache = !params.BypassCache &&
-            !params.AfterKey &&
-            (params.CacheLocal || this.TrustLocalCacheCompletely) &&
-            entityCacheAllowed;
+        const willCache = this.runViewCacheEligible(params);
         if (entity && willCache) {
             params.Fields = entity.Fields.map(f => f.Name);
+            // Platform contract: explicit Fields always include the primary key(s) —
+            // project back down to requested ∪ PK, matching the direct SQL path.
+            if (callerRequestedFields) {
+                callerRequestedFields = ProviderBase.UnionFieldsWithPrimaryKeys(callerRequestedFields, entity);
+            }
         }
         const entityLookupTime = performance.now() - entityLookupStart;
         // Check local cache if enabled
@@ -1269,23 +1566,7 @@ export class ProviderBase {
                 // Filter cached results to only the caller's requested fields (if specified)
                 let results = cached.results;
                 if (callerRequestedFields && params.ResultType !== 'entity_object') {
-                    // Cache lowercase key→keep decisions across rows to avoid repeated allocations
-                    const requestedFieldSet = new Set(callerRequestedFields);
-                    const keyCache = new Map();
-                    results = results.map((row) => {
-                        const filtered = {};
-                        for (const key of Object.keys(row)) {
-                            let keep = keyCache.get(key);
-                            if (keep === undefined) {
-                                keep = requestedFieldSet.has(key.toLowerCase());
-                                keyCache.set(key, keep);
-                            }
-                            if (keep) {
-                                filtered[key] = row[key];
-                            }
-                        }
-                        return filtered;
-                    });
+                    results = ProjectRowsToFields(results, callerRequestedFields);
                 }
                 // Reconstruct RunViewResult from cached data
                 cachedResult = {
@@ -1316,7 +1597,10 @@ export class ProviderBase {
             telemetryEventId,
             cacheStatus,
             cachedResult,
-            fingerprint
+            fingerprint,
+            // Only non-null when params.Fields was actually widened above — tells
+            // PostRunView to project cache-miss DB results back to the caller's shape
+            callerRequestedFields: (entity && willCache) ? callerRequestedFields : null
         };
     }
     /**
@@ -1340,6 +1624,10 @@ export class ProviderBase {
         const telemetryEventId = TelemetryManager.Instance.StartEvent('RunView', 'ProviderBase.RunViews', {
             BatchSize: params.length,
             Entities: params.map(p => p.EntityName || p.ViewName || p.ViewID).filter(Boolean),
+            // Per-view filter/orderBy parallel to Entities so the telemetry fingerprint can
+            // tell apart two batches over the same entity set but with different filters.
+            Filters: params.map(p => p.ExtraFilter),
+            OrderBys: params.map(p => p.OrderBy),
             _fromEngine: fromEngine
         }, contextUser?.ID);
         // Client-side providers route any CacheLocal params through smart-cache-check:
@@ -1355,6 +1643,8 @@ export class ProviderBase {
         }
         // Traditional caching flow
         const cacheStatusMap = new Map();
+        const callerFieldsMap = new Map();
+        const fingerprintMap = new Map();
         const uncachedParams = [];
         const cachedResults = [];
         let allCached = true;
@@ -1364,21 +1654,23 @@ export class ProviderBase {
             await this.EntityStatusCheck(param, 'PreRunViews');
             // Save caller's original Fields, then always fetch all fields from DB.
             // One cache entry per entity+filter satisfies all field subsets.
-            const callerFields = param.Fields && param.Fields.length > 0
+            let callerFields = param.Fields && param.Fields.length > 0
                 ? param.Fields.map(f => f.trim().toLowerCase())
                 : null;
             // Only override Fields to all entity fields when caching will actually happen
             // for this call. For non-cached calls we respect the caller's narrow Fields
             // end-to-end — there's no cache-coherence concern to preserve.
             const batchEntity = param.EntityName ? this.EntityByName(param.EntityName) : null;
-            const batchEntityCacheAllowed = this.IsServerCacheAllowedForEntity(param);
-            // Keyset (AfterKey) queries are inherently single-use; never use the cache for them.
-            const batchWillCache = !param.BypassCache &&
-                !param.AfterKey &&
-                (param.CacheLocal || this.TrustLocalCacheCompletely) &&
-                batchEntityCacheAllowed;
+            const batchWillCache = this.runViewCacheEligible(param);
             if (batchEntity && batchWillCache) {
                 param.Fields = batchEntity.Fields.map(f => f.Name);
+                // Platform contract: explicit Fields always include the primary key(s)
+                if (callerFields) {
+                    callerFields = ProviderBase.UnionFieldsWithPrimaryKeys(callerFields, batchEntity);
+                    // Remember the caller's original shape so PostRunViews can project
+                    // cache-miss DB results back down to it
+                    callerFieldsMap.set(i, callerFields);
+                }
             }
             // Check local cache if enabled or if server trusts its cache completely
             // BypassCache skips cache entirely — used by maintenance actions querying for
@@ -1386,29 +1678,13 @@ export class ProviderBase {
             if (batchWillCache && LocalCacheManager.Instance.IsInitialized) {
                 const rlsWhereClause = this.ComputeRunViewRLSWhereClause(param, contextUser);
                 const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString, rlsWhereClause);
+                fingerprintMap.set(i, fingerprint);
                 const cached = await LocalCacheManager.Instance.GetRunViewResult(fingerprint);
                 if (cached) {
                     // Filter cached results to caller's requested fields (if specified and not entity_object)
                     let results = cached.results;
                     if (callerFields && param.ResultType !== 'entity_object') {
-                        // ⚡ Bolt: Cache key-to-lowercase string resolutions to eliminate O(n*c) string allocations and array search operations.
-                        // This improves post-cache filtering by ~40-50% for large datasets with many columns.
-                        const requestedFieldSet = new Set(callerFields);
-                        const keyCache = new Map();
-                        results = results.map((row) => {
-                            const filtered = {};
-                            for (const key of Object.keys(row)) {
-                                let keep = keyCache.get(key);
-                                if (keep === undefined) {
-                                    keep = requestedFieldSet.has(key.toLowerCase());
-                                    keyCache.set(key, keep);
-                                }
-                                if (keep) {
-                                    filtered[key] = row[key];
-                                }
-                            }
-                            return filtered;
-                        });
+                        results = ProjectRowsToFields(results, callerFields);
                     }
                     const cachedViewResult = {
                         Success: true,
@@ -1448,7 +1724,9 @@ export class ProviderBase {
                 ? cachedResults.filter(r => r !== null)
                 : (hasCacheHits ? cachedResults : undefined),
             uncachedParams: allCached ? undefined : uncachedParams,
-            cacheStatusMap
+            cacheStatusMap,
+            callerFieldsMap: callerFieldsMap.size > 0 ? callerFieldsMap : undefined,
+            fingerprintMap: fingerprintMap.size > 0 ? fingerprintMap : undefined
         };
     }
     /**
@@ -1472,8 +1750,7 @@ export class ProviderBase {
                 param.Fields = entity.Fields.map(f => f.Name);
             }
             if (param.CacheLocal && LocalCacheManager.Instance.IsInitialized) {
-                const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString);
-                cacheable.push({ paramIndex: i, fingerprint });
+                cacheable.push({ paramIndex: i, fingerprint: this.clientCacheFingerprint(param) });
             }
         }
         // Phase 2 — batched read: one IDB transaction (or one Redis MGET) returns
@@ -1541,7 +1818,7 @@ export class ProviderBase {
         const currentFingerprints = [];
         for (const sr of response.results) {
             if (sr.status === 'current' && params[sr.viewIndex]) {
-                currentFingerprints.push(LocalCacheManager.Instance.GenerateRunViewFingerprint(params[sr.viewIndex], this.InstanceConnectionString));
+                currentFingerprints.push(this.clientCacheFingerprint(params[sr.viewIndex]));
             }
         }
         const preResolvedCache = currentFingerprints.length > 0
@@ -1606,7 +1883,7 @@ export class ProviderBase {
             // Cache is current - use the pre-resolved cache entry from the batched read
             // (executeSmartCacheCheck reads all 'current' fingerprints in one IDB
             // transaction up front, so we don't pay per-param transaction overhead here).
-            const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString);
+            const fingerprint = this.clientCacheFingerprint(param);
             const cached = preResolvedCache.get(fingerprint) ?? null;
             if (cached) {
                 const cachedResult = {
@@ -1642,7 +1919,7 @@ export class ProviderBase {
         }
         else if (checkResult.status === 'differential') {
             // Cache is stale but we have differential data - merge with cached data
-            const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString);
+            const fingerprint = this.clientCacheFingerprint(param);
             // Get entity info for primary key field name
             const entity = this.EntityByName(param.EntityName);
             const primaryKeyFieldName = entity?.FirstPrimaryKey?.Name || 'ID';
@@ -1689,7 +1966,7 @@ export class ProviderBase {
             };
             // Update the local cache with fresh data (don't await - fire and forget for performance)
             if (param.CacheLocal && checkResult.maxUpdatedAt && LocalCacheManager.Instance.IsInitialized) {
-                const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(param, this.InstanceConnectionString);
+                const fingerprint = this.clientCacheFingerprint(param);
                 // Note: We don't await here to avoid blocking the response
                 // Cache update happens in background
                 LocalCacheManager.Instance.SetRunViewResult(fingerprint, param, checkResult.results || [], checkResult.maxUpdatedAt, checkResult.aggregateResults, // Include aggregate results in cache
@@ -1782,8 +2059,10 @@ export class ProviderBase {
         // with circular subscriber references that break JSON.stringify.
         // On cache read, TransformSimpleObjectToEntityObject is called to restore
         // entity objects when ResultType === 'entity_object'.
-        const postEntityCacheAllowed = this.IsServerCacheAllowedForEntity(params);
-        if ((params.CacheLocal || this.TrustLocalCacheCompletely) && postEntityCacheAllowed && result.Success && preResult.fingerprint && LocalCacheManager.Instance.IsInitialized) {
+        // runViewCacheEligible is the same predicate PreRunView used to decide whether to
+        // widen Fields — only widened (superset) results may be written to the cache.
+        // preResult.fingerprint doubles as a guard (only computed when eligible).
+        if (this.runViewCacheEligible(params) && result.Success && preResult.fingerprint && LocalCacheManager.Instance.IsInitialized) {
             const maxUpdatedAt = this.extractMaxUpdatedAt(result.Results);
             await LocalCacheManager.Instance.SetRunViewResult(preResult.fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults, result.TotalRowCount, this);
         }
@@ -1796,6 +2075,15 @@ export class ProviderBase {
             await LocalCacheManager.Instance.SetRunViewResult(fingerprint, params, result.Results, maxUpdatedAt, result.AggregateResults, result.TotalRowCount, this);
             LogStatusEx({ message: `  📦 [Auto-Cache] RunView "${params.EntityName || params.ViewName || 'unknown'}" — ${result.Results.length} rows auto-cached (small + unfiltered)`, verboseOnly: true });
         }
+        // Project cache-miss DB results back down to the caller's originally requested
+        // fields. PreRunView widened params.Fields to ALL entity fields so the cache
+        // entry (written above) is a universal superset — but the caller must receive
+        // the same shape on a miss as they do on a hit (which projects from cache).
+        // Must run AFTER the cache writes (cache keeps the superset) and only for
+        // plain-object results (entity objects need all fields).
+        if (result.Success && preResult.callerRequestedFields && params.ResultType !== 'entity_object') {
+            result.Results = ProjectRowsToFields(result.Results, preResult.callerRequestedFields);
+        }
         // Transform the result set into BaseEntity-derived objects, if needed
         await this.TransformSimpleObjectToEntityObject(params, result, contextUser);
         // Run registered PostRunView hooks (e.g., data masking, audit logging)
@@ -1834,10 +2122,17 @@ export class ProviderBase {
             if (cacheInfo?.status === 'hit') {
                 continue;
             }
-            const rlsWhereClause = this.ComputeRunViewRLSWhereClause(params[i], contextUser);
-            const fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params[i], this.InstanceConnectionString, rlsWhereClause);
-            const batchEntityCacheAllowed = this.IsServerCacheAllowedForEntity(params[i]);
-            if ((params[i].CacheLocal || this.TrustLocalCacheCompletely) && batchEntityCacheAllowed && results[i].Success && LocalCacheManager.Instance.IsInitialized) {
+            // Reuse the fingerprint PreRunViews already computed for this index —
+            // recomputing means rebuilding the RLS where-clause + fingerprint string
+            // per item. Compute lazily only for indexes PreRunViews skipped (cache
+            // was disabled for them but auto-cache/OnDataChanged may still need it).
+            const fingerprint = preResult.fingerprintMap?.get(i)
+                ?? LocalCacheManager.Instance.GenerateRunViewFingerprint(params[i], this.InstanceConnectionString, this.ComputeRunViewRLSWhereClause(params[i], contextUser));
+            // CRITICAL: must be the SAME eligibility predicate PreRunViews used to decide
+            // whether to widen Fields. Writing a non-widened (narrow or keyset-paged)
+            // result here poisons the Fields-agnostic superset slot — this exact gate
+            // previously omitted BypassCache/AfterKey and cached narrow BypassCache rows.
+            if (this.runViewCacheEligible(params[i]) && results[i].Success && LocalCacheManager.Instance.IsInitialized) {
                 const maxUpdatedAt = this.extractMaxUpdatedAt(results[i].Results);
                 cachePromises.push(LocalCacheManager.Instance.SetRunViewResult(fingerprint, params[i], results[i].Results, maxUpdatedAt, results[i].AggregateResults, results[i].TotalRowCount, this));
             }
@@ -1852,6 +2147,24 @@ export class ProviderBase {
             }
         }
         await Promise.all(cachePromises);
+        // Project cache-miss DB results back down to each caller's originally
+        // requested fields. PreRunViews widened those params' Fields to ALL entity
+        // fields so the cache entries (written above) are universal supersets — but
+        // callers must receive the same shape on a miss as on a hit (which projects
+        // from cache). Skip hits (already projected) and entity_object results
+        // (need all fields).
+        if (preResult.callerFieldsMap) {
+            for (let i = 0; i < results.length; i++) {
+                const cacheInfo = preResult.cacheStatusMap?.get(i);
+                if (cacheInfo?.status === 'hit') {
+                    continue;
+                }
+                const callerFields = preResult.callerFieldsMap.get(i);
+                if (callerFields && results[i].Success && params[i].ResultType !== 'entity_object') {
+                    results[i].Results = ProjectRowsToFields(results[i].Results, callerFields);
+                }
+            }
+        }
         // Transform results to entity objects AFTER caching plain objects.
         // Skip results that came from cache hits — they're already entity objects.
         const transformPromises = [];
@@ -2001,16 +2314,18 @@ export class ProviderBase {
     shouldAutoCache(params, result) {
         if (!this.TrustLocalCacheCompletely)
             return false;
-        if (params.BypassCache)
-            return false; // caller explicitly wants no caching
         if (params.CacheLocal)
             return false; // already handled
+        // Same eligibility predicate as the main cache path — covers BypassCache,
+        // AfterKey (keyset pages), count_only, and cache-disallowed entities. An
+        // auto-cached keyset page or count_only result would poison the
+        // entity+filter slot just like the main-path variants of those bugs.
+        if (!this.runViewCacheEligible(params))
+            return false;
         if (!LocalCacheManager.Instance.IsInitialized)
             return false;
         if (!result.Success)
             return false;
-        if (!this.IsServerCacheAllowedForEntity(params))
-            return false;
         if (ProviderBase.ServerAutoCacheMaxRows <= 0)
             return false;
         if ((result.Results?.length ?? 0) > ProviderBase.ServerAutoCacheMaxRows)
@@ -2026,6 +2341,14 @@ export class ProviderBase {
     }
     extractMaxUpdatedAt(results) {
         let maxDate = null;
+        // Early exit: SQL result rows are uniform — if the first row carries neither
+        // timestamp column, none do, and the full O(rows) scan is pointless.
+        if (results.length > 0 && results[0] && typeof results[0] === 'object') {
+            const probe = results[0];
+            if (probe['__mj_UpdatedAt'] === undefined && probe['UpdatedAt'] === undefined) {
+                return '';
+            }
+        }
         for (const item of results) {
             if (item && typeof item === 'object') {
                 const record = item;
@@ -2109,7 +2432,9 @@ export class ProviderBase {
             ResultType: params.ResultType,
             MaxRows: params.MaxRows,
             StartRow: params.StartRow,
-            _fromEngine: params._fromEngine
+            _fromEngine: params._fromEngine,
+            Exempt: params.Telemetry?.Exempt,
+            ExemptReason: params.Telemetry?.Reason
         }, contextUser?.ID);
         // Store on params object for retrieval in PostProcessRunView
         params._telemetryEventId = eventId;
@@ -2150,10 +2475,19 @@ export class ProviderBase {
     async PreProcessRunViews(params, contextUser) {
         // Start telemetry tracking for batch operation
         const fromEngine = params.some(p => p._fromEngine);
+        // A batch is exempt only when EVERY constituent view opts out — a mixed batch should still
+        // be analyzed. Reason is taken from the first view that supplied one.
+        const batchExempt = params.length > 0 && params.every(p => p.Telemetry?.Exempt);
         const eventId = TelemetryManager.Instance.StartEvent('RunView', 'ProviderBase.RunViews', {
             BatchSize: params.length,
             Entities: params.map(p => p.EntityName || p.ViewName || p.ViewID).filter(Boolean),
-            _fromEngine: fromEngine
+            // Per-view filter/orderBy parallel to Entities so the telemetry fingerprint can
+            // tell apart two batches over the same entity set but with different filters.
+            Filters: params.map(p => p.ExtraFilter),
+            OrderBys: params.map(p => p.OrderBy),
+            _fromEngine: fromEngine,
+            Exempt: batchExempt,
+            ExemptReason: params.find(p => p.Telemetry?.Reason)?.Telemetry?.Reason
         }, contextUser?.ID);
         // Store on first param for retrieval in PostProcessRunViews (using a special key to avoid collision)
         if (params.length > 0) {