@memberjunction/core 6.1.0-edge.0 → 6.1.0-edge.2

package/dist/generic/providerBase.js

@@ -17,6 +17,7 @@ import { RunView } from "../views/runView.js";
 import { IsPlatformSQL } from "./platformSQL.js";
 import { GetDataHooks } from "./dataHooks.js";
 import { TransformSimpleObjectToEntityObject } from "./util.js";
+import { LoadRelatedRecordsBatched } from "./relatedRecordBatchLoader.js";
 /**
  * Creates a new instance of AllMetadata from a simple object.
  * Handles deserialization and proper instantiation of all metadata classes.
@@ -143,7 +144,16 @@ export function ProjectRowsToFields(rows, requestedFields) {
             }
         }
         if (allKept) {
-            return rows;
+            // ...but only when handing the input back is safe. A `Fields` request is documented
+            // to yield a per-caller row set the caller may mutate, and full coverage is not a
+            // narrower promise than partial coverage — it just happens to project to the same
+            // shape. Frozen input means `rows` is the cache's shared array, so returning it here
+            // would quietly hand a Fields caller immutable rows and break that contract for the
+            // one field list that covers everything. Fall through to the copy path in that case;
+            // unfrozen input (the DB-miss path) keeps the allocation-free fast path.
+            if (!Object.isFrozen(rows)) {
+                return rows;
+            }
         }
     }
     // Cache lowercase key→keep decisions across rows to avoid repeated allocations
@@ -206,6 +216,21 @@ export class ProviderBase {
         this._clientFingerprintMemo = new WeakMap();
         this._cachedVisibleExplorerNavigationItems = null;
     }
+    /**
+     * Whether this provider can execute a multi-record unit of work atomically, in-process.
+     *
+     * Defaults to `false` — the correct answer for every provider that is not talking directly to a
+     * database, most importantly the client-side `GraphQLDataProvider`. `DatabaseProviderBase`
+     * overrides this to `true` and supplies {@link DatabaseProviderBase.BeginEntityTransaction}.
+     *
+     * `BaseEntity` reads this to decide whether a multi-node save graph runs locally inside a
+     * transaction or is routed to the server as a single unit of work. Defaulting to `false` is the
+     * safe direction: a provider that has not opted in never has non-atomic work mistaken for
+     * atomic work.
+     */
+    get SupportsEntityTransactions() {
+        return false;
+    }
     // ── Metadata Refresh Check Debounce ────────────────────────────────
     /**
      * Minimum interval (ms) between metadata refresh checks to prevent
@@ -520,6 +545,7 @@ export class ProviderBase {
                 // Cache hit — transform and return directly
                 LogStatusEx({ message: `  ✅ [Cache HIT] RunView "${params.EntityName || params.ViewName || 'unknown'}" — ${preResult.cachedResult.Results?.length ?? 0} rows from cache, no DB query`, verboseOnly: true });
                 await this.TransformSimpleObjectToEntityObject(params, preResult.cachedResult, contextUser);
+                await this.ApplyPostRunViewHooksToCacheHit(params, preResult.cachedResult, contextUser);
                 TelemetryManager.Instance.EndEvent(preResult.telemetryEventId, {
                     cacheHit: true,
                     cacheStatus: preResult.cacheStatus,
@@ -533,11 +559,20 @@ export class ProviderBase {
             // Cache miss — execute query, then post-process (stores in cache)
             LogStatusEx({ message: `  🔍 [Cache MISS] RunView "${params.EntityName || params.ViewName || 'unknown'}" — querying database`, verboseOnly: true });
             const result = await this.InternalRunView(params, contextUser);
+            // PostRunView copies any hook-supplied replacement onto `result` in place, so this
+            // reference reflects the hook chain's output.
             await this.PostRunView(result, params, preResult, contextUser);
             return result;
         }
-        // Client-side: delegate to RunViews which uses the smart cache check
-        // (lightweight maxUpdatedAt + rowCount validation against the server)
+        // Delegate to RunViews, which uses the smart cache check (lightweight maxUpdatedAt +
+        // rowCount validation against the server).
+        //
+        // NOT client-only: the guard above also sends SERVER reads down here whenever BypassCache
+        // or AfterKey is set — so every page of a keyset sweep arrives as a size-1 batch. Any
+        // "single vs batch RunView" reasoning must treat this branch as carrying single RunViews
+        // too: in particular, batch telemetry fingerprints must include per-view pagination
+        // cursors, or every page of a sweep collapses onto one fingerprint and falsely fires the
+        // Duplicate analyzer.
         const results = await this.RunViews([params], contextUser);
         return results[0];
     }
@@ -655,6 +690,12 @@ export class ProviderBase {
                 batchSize: params.length,
                 totalResultCount: totalResults
             });
+            // allCached ⇒ every param produced a hit and was pushed in order (PreRunViews only
+            // pushes a null placeholder on the path that clears allCached), so index i of
+            // cachedResults corresponds to params[i].
+            for (let i = 0; i < preResult.cachedResults.length; i++) {
+                await this.ApplyPostRunViewHooksToCacheHit(params[i], preResult.cachedResults[i], contextUser);
+            }
             return preResult.cachedResults;
         }
         // Execute the internal implementation for non-cached items
@@ -1501,6 +1542,9 @@ export class ProviderBase {
                     QueryID: cached.queryId ?? params.QueryID ?? '',
                     QueryName: params.QueryName ?? '',
                     Success: true,
+                    // Transport boundary: `cached.results` is readonly (shared, deep-frozen cache
+                    // rows) while the outbound Results is mutable — the runtime freeze is the
+                    // enforcement. Same cast as the RunView hit paths above.
                     Results: cached.results,
                     RowCount: cached.results.length,
                     TotalRowCount: cached.rowCount ?? cached.results.length,
@@ -1837,7 +1881,9 @@ export class ProviderBase {
             fingerprint = LocalCacheManager.Instance.GenerateRunViewFingerprint(params, this.InstanceConnectionString, rlsWhereClause);
             const cached = await LocalCacheManager.Instance.GetRunViewResult(fingerprint);
             if (cached) {
-                // Filter cached results to only the caller's requested fields (if specified)
+                // These rows are the cache's shared, deep-frozen objects — the runtime freeze is
+                // what stops a consumer from corrupting the cache. Anything that needs to
+                // transform them must map onto copies.
                 let results = cached.results;
                 if (callerRequestedFields && params.ResultType !== 'entity_object') {
                     results = ProjectRowsToFields(results, callerRequestedFields);
@@ -1899,11 +1945,34 @@ export class ProviderBase {
         const fromEngine = params.some(p => p._fromEngine);
         const telemetryEventId = TelemetryManager.Instance.StartEvent('RunView', 'ProviderBase.RunViews', {
             BatchSize: params.length,
-            Entities: params.map(p => p.EntityName || p.ViewName || p.ViewID).filter(Boolean),
+            // '' placeholder (NOT .filter(Boolean)) for a view identified only by ViewEntity:
+            // Entities must stay index-parallel to Filters/OrderBys/StartRows/AfterKeys or the
+            // fingerprint attributes one view's filter/cursor to the next named entity.
+            // generateRunViewFingerprint skips falsy entries without disturbing the indexes.
+            Entities: params.map(p => p.EntityName || p.ViewName || p.ViewID || ''),
             // Per-view filter/orderBy parallel to Entities so the telemetry fingerprint can
             // tell apart two batches over the same entity set but with different filters.
             Filters: params.map(p => p.ExtraFilter),
             OrderBys: params.map(p => p.OrderBy),
+            // Per-view pagination cursors, also parallel to Entities. Every page of a sweep
+            // shares the same entity+filter+orderBy and differs only here, so without these
+            // the pages collapse onto one fingerprint and the Duplicate analyzer fires from
+            // page 2 on. This path carries single RunViews too: RunView() delegates to
+            // RunViews([params]) whenever BypassCache or AfterKey is set (and always on the
+            // client), which is exactly what a keyset sweep does on every page.
+            StartRows: params.map(p => p.StartRow),
+            AfterKeys: params.map(p => p.AfterKey?.ToConcatenatedString()),
+            // Exemption was threaded through the DEPRECATED batch twin but not this one, so
+            // RunViewParams.Telemetry.Exempt was silently ineffective for every batch RunView —
+            // and, because RunView() delegates here whenever BypassCache or AfterKey is set,
+            // for those single reads too. A caller marking an intentional repeat got warned
+            // anyway, with no indication their exemption had been dropped.
+            //
+            // Exempt only when EVERY view in the batch is exempt: a batch is one telemetry
+            // event, so exempting it on the strength of one member would silently suppress
+            // findings about the others.
+            Exempt: params.length > 0 && params.every(p => p.Telemetry?.Exempt),
+            ExemptReason: params.find(p => p.Telemetry?.Reason)?.Telemetry?.Reason,
             _fromEngine: fromEngine
         }, contextUser?.ID);
         // Client-side providers route any CacheLocal params through smart-cache-check:
@@ -1963,7 +2032,7 @@ export class ProviderBase {
                 fingerprintMap.set(i, fingerprint);
                 const cached = await LocalCacheManager.Instance.GetRunViewResult(fingerprint);
                 if (cached) {
-                    // Filter cached results to caller's requested fields (if specified and not entity_object)
+                    // Shared, deep-frozen cache rows — same contract as the single-view hit path.
                     let results = cached.results;
                     if (callerFields && param.ResultType !== 'entity_object') {
                         results = ProjectRowsToFields(results, callerFields);
@@ -2396,8 +2465,19 @@ export class ProviderBase {
         }
         // Transform the result set into BaseEntity-derived objects, if needed
         await this.TransformSimpleObjectToEntityObject(params, result, contextUser);
-        // Run registered PostRunView hooks (e.g., data masking, audit logging)
-        result = await this.RunPostRunViewHooks(params, result, contextUser);
+        // Run registered PostRunView hooks (e.g., data masking, audit logging).
+        //
+        // A hook may RETURN a replacement result rather than mutating the one it was handed —
+        // that is what `PostRunViewHook`'s signature promises, and it is the only option left
+        // now that cached rows are frozen. Reassigning the local `result` would drop it on the
+        // floor, because RunView returns the reference IT holds. Copy the replacement's fields
+        // onto that reference instead, so the caller observes the hook's changes without
+        // PostRunView having to change its return type (which would break external
+        // subclasses that override it).
+        const hooked = await this.RunPostRunViewHooks(params, result, contextUser);
+        if (hooked && hooked !== result) {
+            Object.assign(result, hooked);
+        }
         // Register OnDataChanged callback if provided and we have a fingerprint
         if (params.OnDataChanged && preResult.fingerprint) {
             result.Unsubscribe = LocalCacheManager.Instance.RegisterChangeCallback(preResult.fingerprint, params.OnDataChanged);
@@ -2542,6 +2622,43 @@ export class ProviderBase {
         }
         return result;
     }
+    /**
+     * Applies the PostRunView hook chain to a result that was served from cache, mutating
+     * `result` in place so the caller's reference reflects the chain's output.
+     *
+     * ## Why cache hits must run the hooks
+     * PostRunView is the OUTPUT half of the enforcement seam (data masking / audit). Hooks
+     * receive `contextUser`, so masking is PER-USER, while the cache slot is shared across
+     * users — there is no correct way to apply masking once at write time on behalf of a
+     * reader who has not arrived yet. A hit that skips the chain therefore returns rows the
+     * miss path would have masked.
+     *
+     * This previously appeared to work by accident: PostRunView writes the cache BEFORE
+     * running the hooks, so a hook that masked rows in place was writing through into the
+     * cached objects — which both made later hits look masked and baked one user's masking
+     * decision into a shared slot. Freeze-on-write removes that write-through, which is what
+     * makes running the chain here necessary rather than merely tidier.
+     *
+     * ## Why mutating `result` in place is safe
+     * Cache-hit results are FRESH wrapper objects built per hit by PreRunView/PreRunViews —
+     * only `.Results` points at shared cache state. A hook that returns a replacement (the
+     * required pattern now that rows are frozen) is copied onto that per-hit wrapper, so it
+     * can never write back into the cache.
+     *
+     * ## Why the guard
+     * `GetDataHooks` is a memoized store read (~30ns), but `await`-ing the async chain costs
+     * a microtask (~750ns) — comparable to the entire cache lookup this rides on. The
+     * overwhelmingly common case is zero registered hooks, so check first and skip the await.
+     */
+    async ApplyPostRunViewHooksToCacheHit(params, result, contextUser) {
+        if (GetDataHooks('PostRunView').length === 0) {
+            return;
+        }
+        const hooked = await this.RunPostRunViewHooks(params, result, contextUser);
+        if (hooked && hooked !== result) {
+            Object.assign(result, hooked);
+        }
+    }
     /**
      * Post-processing hook for RunQuery.
      * Handles cache storage and telemetry end.
@@ -2806,11 +2923,17 @@ export class ProviderBase {
         const batchExempt = params.length > 0 && params.every(p => p.Telemetry?.Exempt);
         const eventId = TelemetryManager.Instance.StartEvent('RunView', 'ProviderBase.RunViews', {
             BatchSize: params.length,
-            Entities: params.map(p => p.EntityName || p.ViewName || p.ViewID).filter(Boolean),
+            // '' placeholder (NOT .filter(Boolean)) — see PreRunViews: Entities must stay
+            // index-parallel to Filters/OrderBys/StartRows/AfterKeys.
+            Entities: params.map(p => p.EntityName || p.ViewName || p.ViewID || ''),
             // Per-view filter/orderBy parallel to Entities so the telemetry fingerprint can
             // tell apart two batches over the same entity set but with different filters.
             Filters: params.map(p => p.ExtraFilter),
             OrderBys: params.map(p => p.OrderBy),
+            // Per-view pagination cursors — see PreRunViews: keeps each page of a sweep a
+            // distinct fingerprint instead of a false Duplicate RunView from page 2 onward.
+            StartRows: params.map(p => p.StartRow),
+            AfterKeys: params.map(p => p.AfterKey?.ToConcatenatedString()),
             _fromEngine: fromEngine,
             Exempt: batchExempt,
             ExemptReason: params.find(p => p.Telemetry?.Reason)?.Telemetry?.Reason
@@ -2866,9 +2989,180 @@ export class ProviderBase {
      * @param contextUser - The user context for permissions
      */
     async TransformSimpleObjectToEntityObject(param, result, contextUser) {
+        // Mutually exclusive with the entity branch below: entity objects get real types from
+        // BaseEntity's Get/Set conversion, so normalization applies only to non-entity results.
+        this.NormalizeSimpleRowTypes(param, result);
         if (param.ResultType === 'entity_object' && result && result.Success && result.Results?.length > 0) {
             result.Results = await TransformSimpleObjectToEntityObject(this, param.EntityName, result.Results, contextUser);
+            // Opt-in batched child loading: ONE query per named collection across the whole result
+            // set, not one per row. Companion eager loading is deliberately kept out of
+            // LoadFromData() (which is the per-row path above) precisely so that populating children
+            // for a view cannot degrade into N+1 — see relatedRecordBatchLoader.ts.
+            if (param.IncludeRelatedRecords?.length) {
+                await LoadRelatedRecordsBatched(result.Results, param.IncludeRelatedRecords, this, contextUser);
+            }
+        }
+    }
+    /**
+     * Normalizes non-entity (`'simple'`) result rows so `Date` and numeric columns hold real
+     * `Date`s and `number`s on EVERY tier, matching what the generated entity types declare.
+     *
+     * ## Why this is unconditional
+     *
+     * Before this existed, the value a simple read returned for a `DATETIME` column depended on
+     * where the code happened to run: a fresh server-side query yields real `Date` objects (the
+     * driver parses them and `AdjustDatetimeFields` timezone-adjusts them), a server-side Redis
+     * cache hit yields ISO strings (`JSON.parse` with no reviver), and a browser client over
+     * GraphQL yields ISO strings (rows are `JSON.stringify`'d on the wire). Same call, three
+     * shapes. MJ's contract is a unified programming interface on both sides of the wire, so the
+     * one representation the platform's own generated types declare — `Date` — is enforced here,
+     * at the one choke point every provider's RunView pipeline flows through.
+     *
+     * ## What it does NOT do
+     *
+     * It makes date and number VALUES match the generated types; it does not make a caller's `T`
+     * honest in general. A `Status` column typed as a closed union still holds whatever string the
+     * database held, and plain rows never have entity methods. If you need the type to be fully
+     * true, use `ResultType: 'entity_object'`.
+     *
+     * ## Cost and cache safety
+     *
+     * The field-key lists are computed once per view from `EntityInfo`, not per cell. Rows already
+     * in the right shape — the common server-side case, where the driver returned `Date`s — are
+     * detected and the ORIGINAL array is kept untouched: same array identity, same row objects,
+     * zero copying. A row is shallow-copied only when a cell actually converts, and that copy is
+     * load-bearing: on a cache hit the rows handed back can be the cache's OWN objects (the
+     * in-memory server store holds them by reference), so converting in place would write `Date`s
+     * into the cache entry itself and corrupt it for serialization and for later readers.
+     *
+     * Per-cell rules:
+     * - `Date` instances pass through untouched, so the pass is idempotent on every path.
+     * - `NULL`/`undefined` cells are left alone rather than becoming epoch-1970 dates.
+     * - An unparseable value is left as-is rather than written as `Invalid Date`, which renders
+     *   as that literal string and destroys the evidence of what the database actually held.
+     * - An integer string outside `Number.MAX_SAFE_INTEGER` stays a string: the PostgreSQL
+     *   provider deliberately returns unsafe-range BIGINTs as strings to avoid precision loss,
+     *   and `Number('9007199254740993')` "succeeds" while silently corrupting the value.
+     *
+     * View-based runs (`ViewID`/`ViewName` with neither `EntityName` nor a loaded `ViewEntity`)
+     * skip normalization: resolving the entity would take an async User Views read this late in
+     * the pipeline. Pass `EntityName` alongside the view identifier to get normalized rows.
+     */
+    NormalizeSimpleRowTypes(param, result) {
+        if (param.ResultType === 'entity_object' || param.ResultType === 'count_only') {
+            return;
+        }
+        if (!result?.Success || !result.Results?.length) {
+            return;
+        }
+        const entity = this.resolveEntityForNormalization(param);
+        if (!entity) {
+            // An unresolvable entity name is already a failed query elsewhere; normalization is
+            // not the place to raise it, and guessing field types would be worse than raw rows.
+            return;
+        }
+        // Once per view, not once per cell. Each entry lists the row keys one field can appear
+        // under: the batch transport keys rows by Name, the singular transport adds CodeName.
+        const dateKeys = this.normalizationKeys(entity, EntityFieldTSType.Date);
+        const numberKeys = this.normalizationKeys(entity, EntityFieldTSType.Number);
+        if (!dateKeys.length && !numberKeys.length) {
+            return;
+        }
+        let anyRowChanged = false;
+        const normalized = result.Results.map(row => {
+            const converted = this.normalizeSimpleRow(row, dateKeys, numberKeys);
+            if (converted) {
+                anyRowChanged = true;
+                return converted;
+            }
+            return row;
+        });
+        if (anyRowChanged) {
+            result.Results = normalized;
+        }
+    }
+    /**
+     * Resolves the {@link EntityInfo} normalization should read field types from, using only
+     * synchronously available information on the params.
+     */
+    resolveEntityForNormalization(param) {
+        if (param.EntityName) {
+            return this.EntityByName(param.EntityName);
+        }
+        if (param.ViewEntity) {
+            // Weak typing mirrors RunView.GetEntityNameFromRunViewParams: MJCore cannot import
+            // the core-entities UserView subclass without creating a circular dependency.
+            const entityID = param.ViewEntity.Get('EntityID');
+            return entityID ? this.EntityByID(entityID) : undefined;
+        }
+        return undefined;
+    }
+    /**
+     * The row keys each field of the given TSType can appear under, one entry per field.
+     */
+    normalizationKeys(entity, tsType) {
+        return entity.Fields
+            .filter(f => f.TSType === tsType)
+            .map(f => (f.CodeName && f.CodeName !== f.Name ? [f.Name, f.CodeName] : [f.Name]));
+    }
+    /**
+     * Returns a converted shallow copy of the row, or null when no cell needed converting —
+     * so untouched rows keep their identity and cached rows are never written to.
+     */
+    normalizeSimpleRow(row, dateKeys, numberKeys) {
+        if (!row || typeof row !== 'object') {
+            return null;
+        }
+        let copy = null;
+        for (const keys of dateKeys) {
+            for (const key of keys) {
+                const date = this.parseDateCell((copy ?? row)[key]);
+                if (date) {
+                    copy = copy ?? { ...row };
+                    copy[key] = date;
+                }
+            }
+        }
+        for (const keys of numberKeys) {
+            for (const key of keys) {
+                const num = this.parseNumericCell((copy ?? row)[key]);
+                if (num !== null) {
+                    copy = copy ?? { ...row };
+                    copy[key] = num;
+                }
+            }
+        }
+        return copy;
+    }
+    /**
+     * A real Date for a convertible cell, or null to leave the cell untouched. Existing Date
+     * instances, NULLs, and unparseable values all return null — see the per-cell rules on
+     * {@link NormalizeSimpleRowTypes}.
+     */
+    parseDateCell(value) {
+        if (typeof value !== 'string' && typeof value !== 'number') {
+            return null;
+        }
+        const date = new Date(value);
+        return Number.isNaN(date.getTime()) ? null : date;
+    }
+    /**
+     * A number for a convertible string cell, or null to leave the cell untouched.
+     */
+    parseNumericCell(value) {
+        if (typeof value !== 'string' || value.trim() === '') {
+            return null;
+        }
+        const num = Number(value);
+        if (!Number.isFinite(num)) {
+            return null;
+        }
+        // An integer string beyond the safe range is a deliberate driver choice (PostgreSQL
+        // returns unsafe BIGINTs as strings): converting would silently corrupt the value.
+        if (Number.isInteger(num) && !Number.isSafeInteger(num)) {
+            return null;
         }
+        return num;
     }
     /**
      * Returns the currently loaded local metadata from within the instance