@memberjunction/core 6.2.0-edge.6 → 6.2.0-edge.8

package/dist/generic/localCacheManager.js

@@ -1,4 +1,4 @@
-import { BaseSingleton, MJGlobal, MJEventType } from "@memberjunction/global";
+import { BaseSingleton, MJGlobal, MJEventType, IsValueEncrypted, ENCRYPTED_SENTINEL } from "@memberjunction/global";
 import { IsMaterializedDataSource, NormalizeAlternateBaseViewName } from "../views/runView.js";
 import { GetEffectiveAlternateBaseViewName } from "./alternateBaseViews.js";
 import { ViewPropertyValuesKey } from "../views/viewProperties.js";
@@ -467,6 +467,17 @@ export class LocalCacheManager extends BaseSingleton {
             || this.hasNarrowingSegment(parts)
             || this.hasAggregates(parts);
     }
+    /**
+     * Returns true when a cached `RunView` slot identified by `fingerprint` is eligible for
+     * in-place maintenance across entity saves (unfiltered, unordered, uncapped, no aggregates
+     * or narrowing segments).
+     */
+    IsRunViewFingerprintMaintainableOnSave(fingerprint) {
+        if (!fingerprint || !fingerprint.includes('|')) {
+            return false;
+        }
+        return !this.isSubsetFingerprint(fingerprint) && !this.isFilteredFingerprint(fingerprint);
+    }
     /**
      * Returns true if the slot was cached under an ORDER BY (fingerprint segment [2]).
      *
@@ -870,7 +881,7 @@ export class LocalCacheManager extends BaseSingleton {
      * @param fingerprint - The slot key
      * @param expiresAt - Absolute expiry (ms since epoch), when the slot has one
      */
-    runViewWriteOptions(fingerprint, expiresAt) {
+    runViewWriteOptions(fingerprint, expiresAt, pubSubOverride) {
         const options = {};
         const entity = this.extractEntityFromFingerprint(fingerprint);
         if (entity) {
@@ -879,6 +890,10 @@ export class LocalCacheManager extends BaseSingleton {
         if (expiresAt) {
             options.TTLSeconds = Math.max(1, Math.ceil((expiresAt - Date.now()) / 1000));
         }
+        if (pubSubOverride) {
+            options.pubSubAction = pubSubOverride.pubSubAction;
+            options.pubSubData = pubSubOverride.pubSubData;
+        }
         return options;
     }
     /** Storage write options for any registry entry; only RunView slots are indexed. */
@@ -1018,6 +1033,21 @@ export class LocalCacheManager extends BaseSingleton {
         const baseEntity = entityEvent.baseEntity;
         if (!baseEntity?.EntityInfo?.Name)
             return;
+        if (entityEvent.type === 'delete') {
+            await this.invalidateCascadeDeletedRelatedEntities(baseEntity.EntityInfo);
+        }
+        if ((entityEvent.type === 'save' || entityEvent.type === 'delete') && baseEntity.EntityInfo.IsParentType) {
+            const activeChainLower = new Set();
+            let currChild = baseEntity.ISAChild;
+            while (currChild) {
+                const cName = currChild.EntityInfo?.Name?.trim().toLowerCase();
+                if (cName) {
+                    activeChainLower.add(cName);
+                }
+                currChild = currChild.ISAChild;
+            }
+            await this.invalidateISADescendantEntities(baseEntity.EntityInfo, activeChainLower);
+        }
         const entityName = baseEntity.EntityInfo.Name;
         // Short-circuit: if caching is disabled for this entity, skip the fingerprint scan
         if (!this.IsCachingEnabledForEntity(baseEntity.EntityInfo))
@@ -1318,14 +1348,99 @@ export class LocalCacheManager extends BaseSingleton {
             list.push(change);
             byEntity.set(change.EntityInfo.Name, list);
         }
+        const touchedLower = new Set(Array.from(byEntity.keys(), k => k.trim().toLowerCase()));
         for (const [entityName, changes] of byEntity) {
             await this.applyEntityChanges(entityName, changes);
+            const cascadeDelete = changes.find(c => c.Type === 'delete' && c.EntityInfo?.CascadeDeletes);
+            if (cascadeDelete) {
+                await this.invalidateCascadeDeletedRelatedEntities(cascadeDelete.EntityInfo);
+            }
+            const parentInfo = changes[0]?.EntityInfo;
+            if (parentInfo?.IsParentType) {
+                await this.invalidateISADescendantEntities(parentInfo, touchedLower);
+            }
         }
     }
     /** Invalidates every entity a batch touched — for a batch that failed or was abandoned. */
     async invalidateBatchEntities(batch) {
-        for (const entityName of batch.TouchedEntities.keys()) {
+        for (const [entityName, entityInfo] of batch.TouchedEntities.entries()) {
             await this.InvalidateEntityCaches(entityName);
+            await this.invalidateCascadeDeletedRelatedEntities(entityInfo);
+            await this.invalidateISADescendantEntities(entityInfo);
+        }
+    }
+    /**
+     * Invalidates cached RunView/dataset slots for IS-A child entities whose SQL views join
+     * `entityInfo`'s table when a mutation on `entityInfo` was not accompanied by a row-level
+     * update for those children:
+     * - Overlapping subtypes (`AllowMultipleSubtypes === true`): sibling subtypes can share the
+     *   same parent PK, so any child not in `updatedEntityNamesLower` may hold stale inherited
+     *   parent columns and must be invalidated.
+     * - Disjoint subtypes (`AllowMultipleSubtypes === false`): a parent PK belongs to at most one
+     *   child subtype. If one of `entityInfo.ChildEntities` was already updated in the same
+     *   operation (`updatedEntityNamesLower`), sibling child tables cannot contain that PK and stay
+     *   cached; if no child was part of the operation (e.g. a bare parent save/delete or a parent-only
+     *   remote invalidation), all child subtypes are invalidated.
+     */
+    async invalidateISADescendantEntities(entityInfo, updatedEntityNamesLower) {
+        if (!entityInfo?.IsParentType || !Array.isArray(entityInfo.ChildEntities) || entityInfo.ChildEntities.length === 0) {
+            return;
+        }
+        const directChildren = entityInfo.ChildEntities;
+        if (!entityInfo.AllowMultipleSubtypes && updatedEntityNamesLower && updatedEntityNamesLower.size > 0) {
+            const anyDirectChildUpdated = directChildren.some(c => c?.Name && updatedEntityNamesLower.has(c.Name.trim().toLowerCase()));
+            if (anyDirectChildUpdated) {
+                return;
+            }
+        }
+        const visited = new Set([(entityInfo.Name ?? '').trim().toLowerCase()]);
+        const queue = [...directChildren];
+        while (queue.length > 0) {
+            const child = queue.shift();
+            const childName = child?.Name?.trim();
+            if (!childName) {
+                continue;
+            }
+            const childLower = childName.toLowerCase();
+            if (visited.has(childLower)) {
+                continue;
+            }
+            visited.add(childLower);
+            if (!updatedEntityNamesLower?.has(childLower)) {
+                await this.InvalidateEntityCaches(childName);
+            }
+            if (Array.isArray(child.ChildEntities) && child.ChildEntities.length > 0) {
+                queue.push(...child.ChildEntities);
+            }
+        }
+    }
+    /**
+     * When a parent entity with `CascadeDeletes = true` is deleted (e.g. via its generated
+     * `spDelete*` stored procedure), the database deletes its `One To Many` child records directly
+     * without raising per-child `BaseEntity.Delete()` events. Invalidate cached RunView/dataset
+     * slots for those child entities so cascade-deleted rows do not linger in `LocalCacheManager`.
+     */
+    async invalidateCascadeDeletedRelatedEntities(entityInfo) {
+        if (!entityInfo?.CascadeDeletes || !Array.isArray(entityInfo.RelatedEntities)) {
+            return;
+        }
+        const parentLower = (entityInfo.Name ?? '').trim().toLowerCase();
+        const invalidated = new Set();
+        for (const rel of entityInfo.RelatedEntities) {
+            const relatedName = rel?.RelatedEntity?.trim();
+            if (!relatedName) {
+                continue;
+            }
+            const relType = (rel.Type ?? 'One To Many').trim().toUpperCase();
+            if (relType !== 'ONE TO MANY') {
+                continue;
+            }
+            const relatedLower = relatedName.toLowerCase();
+            if (relatedLower === parentLower || invalidated.has(relatedLower)) {
+                continue;
+            }
+            invalidated.add(relatedLower);
+            await this.InvalidateEntityCaches(relatedName);
         }
     }
     /** Applies one entity's buffered changes to every slot indexed for it, each slot once. */
@@ -1386,6 +1501,12 @@ export class LocalCacheManager extends BaseSingleton {
         // the Metadata helper itself proxies to the global provider with sensible fallbacks.
         const md = entityEvent.provider ?? new Metadata();
         const entityInfo = md.EntityByName(entityName);
+        if (payload?.action === 'delete' && entityInfo) {
+            await this.invalidateCascadeDeletedRelatedEntities(entityInfo);
+        }
+        if ((payload?.action === 'save' || payload?.action === 'delete') && entityInfo?.IsParentType) {
+            await this.invalidateISADescendantEntities(entityInfo);
+        }
         if (entityInfo && !this.IsCachingEnabledForEntity(entityInfo))
             return;
         const fingerprints = await this.resolveFingerprintsForEntity(entityName);
@@ -1457,7 +1578,8 @@ export class LocalCacheManager extends BaseSingleton {
                 if (key.KeyValuePairs.some(kv => kv.Value == null))
                     return;
                 LogStatusVerbose(`LocalCacheManager: remote-invalidate (save) for "${entityName}" PK=${key.ToConcatenatedString()}, updating ${fingerprints.size} cached fingerprint(s)`);
-                const rowComplete = LocalCacheManager.RowHasEveryField(entityInfo, recordData); // per row, not per slot
+                const rowComplete = LocalCacheManager.RowHasEveryField(entityInfo, recordData)
+                    && !LocalCacheManager.RowContainsPlaintextEncryptedField(entityInfo, recordData); // per row, not per slot
                 for (const fingerprint of fingerprintSnapshot) {
                     try {
                         // Subset slot (MaxRows/StartRow): upserting would grow the slot past the
@@ -1581,9 +1703,12 @@ export class LocalCacheManager extends BaseSingleton {
         }
         else if (!this.isFilteredFingerprint(fingerprint)) {
             // Unfiltered cache: update the record in place, unless the saved row is missing columns
+            // or carries decrypted plaintext on an Encrypt=true field (BaseEntity.GetAll() holds
+            // in-memory plaintext after Save(), which must never be persisted to shared cache/Redis).
             const entityData = baseEntity.GetAll();
-            if (!LocalCacheManager.RowHasEveryField(baseEntity.EntityInfo, entityData)) {
-                LogStatusVerbose(`LocalCacheManager: Invalidating "${fingerprint.substring(0, 60)}" — the saved row is missing columns`);
+            if (!LocalCacheManager.RowHasEveryField(baseEntity.EntityInfo, entityData) ||
+                LocalCacheManager.RowContainsPlaintextEncryptedField(baseEntity.EntityInfo, entityData)) {
+                LogStatusVerbose(`LocalCacheManager: Invalidating "${fingerprint.substring(0, 60)}" — the saved row is missing columns or holds decrypted plaintext`);
                 await this.InvalidateRunViewResult(fingerprint);
                 return;
             }
@@ -1631,14 +1756,48 @@ export class LocalCacheManager extends BaseSingleton {
         }
         return true;
     }
-    /** Whether every saved row among `changes` carries every column ({@link RowHasEveryField}). */
+    /**
+     * Returns `true` when `entity` declares one or more encrypted fields (`Encrypt && EncryptionKeyID`)
+     * and `row` contains a non-empty string value for any such field that is neither ciphertext
+     * (`$ENC$...`) nor `ENCRYPTED_SENTINEL`.
+     *
+     * Used as a defense-in-depth gate so decrypted in-memory entity values (e.g. from
+     * `BaseEntity.GetAll()` during `save` events) are never written in plaintext to local or
+     * distributed Redis caches.
+     */
+    static RowContainsPlaintextEncryptedField(entity, row) {
+        if (!entity || !row || typeof row !== 'object') {
+            return false;
+        }
+        const encryptedFields = Array.isArray(entity.EncryptedFields)
+            ? entity.EncryptedFields
+            : Array.isArray(entity.Fields)
+                ? entity.Fields.filter(f => f.Encrypt && Boolean(f.EncryptionKeyID))
+                : [];
+        if (encryptedFields.length === 0) {
+            return false;
+        }
+        const rowMap = row;
+        for (const field of encryptedFields) {
+            if (!field.EncryptionKeyID)
+                continue;
+            const val = rowMap[field.Name];
+            if (typeof val === 'string' && val.length > 0 && val !== ENCRYPTED_SENTINEL && !IsValueEncrypted(val)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    /** Whether every saved row among `changes` carries every column ({@link RowHasEveryField}) and no plaintext encrypted values. */
     static everySavedRowComplete(changes) {
-        return changes.every(c => c.Type !== 'save' || LocalCacheManager.RowHasEveryField(c.EntityInfo, c.Record));
+        return changes.every(c => c.Type !== 'save' ||
+            (LocalCacheManager.RowHasEveryField(c.EntityInfo, c.Record) &&
+                !LocalCacheManager.RowContainsPlaintextEncryptedField(c.EntityInfo, c.Record)));
     }
     /**
      * True when a saved row may be written into `fingerprint`'s slot in place: the slot is neither
      * filtered (including row-security and field-security segments) nor a subset, and the row
-     * carries every column ({@link RowHasEveryField}). Otherwise the slot must be invalidated.
+     * carries every column ({@link RowHasEveryField}) with no plaintext secrets. Otherwise the slot must be invalidated.
      *
      * @param fingerprint - The slot to maintain
      * @param entity - The saved record's entity
@@ -1647,7 +1806,8 @@ export class LocalCacheManager extends BaseSingleton {
     CanUpsertRowIntoSlot(fingerprint, entity, row) {
         return !this.isFilteredFingerprint(fingerprint)
             && !this.isSubsetFingerprint(fingerprint)
-            && LocalCacheManager.RowHasEveryField(entity, row);
+            && LocalCacheManager.RowHasEveryField(entity, row)
+            && !LocalCacheManager.RowContainsPlaintextEncryptedField(entity, row);
     }
     /**
      * Handles a save/delete that could not be applied to a slot this process indexed — in practice
@@ -1746,7 +1906,25 @@ export class LocalCacheManager extends BaseSingleton {
             }
         }
         else {
-            // For set/removed, dispatch only to callbacks for the specific fingerprint
+            this._lastVerifiedAt.delete(event.CacheKey);
+            if (event.Action === 'row_delta' && event.Data) {
+                const existingEntry = this._registry.get(event.CacheKey);
+                if (existingEntry) {
+                    try {
+                        const parsedDelta = JSON.parse(event.Data);
+                        if (typeof parsedDelta.rowCount === 'number') {
+                            existingEntry.rowCount = parsedDelta.rowCount;
+                        }
+                        if (typeof parsedDelta.maxUpdatedAt === 'string') {
+                            existingEntry.maxUpdatedAt = parsedDelta.maxUpdatedAt;
+                        }
+                    }
+                    catch {
+                        // Best-effort local registry metadata update; callbacks handle invalid payloads safely.
+                    }
+                }
+            }
+            // For set/row_delta/removed, dispatch only to callbacks for the specific fingerprint
             const callbacks = this._changeCallbacks.get(event.CacheKey);
             if (callbacks) {
                 for (const cb of callbacks) {
@@ -2408,6 +2586,17 @@ export class LocalCacheManager extends BaseSingleton {
                     LogStatusEx({ message: `[CACHE-WRITE-GATE] Skipping cache write for external entity "${params.EntityName}" — no TTL provided (would never invalidate)`, verboseOnly: true });
                     return;
                 }
+                // Defense-in-depth on server caches (TrustLocalCacheCompletely === true):
+                // never persist decrypted plaintext values for Encrypt=true columns into shared/Redis cache.
+                const cacheProvider = provider;
+                if (entity &&
+                    cacheProvider?.TrustLocalCacheCompletely === true &&
+                    !options?.ProviderInternalScaffolding &&
+                    Array.isArray(results) &&
+                    results.some(r => typeof r === 'object' && r !== null && LocalCacheManager.RowContainsPlaintextEncryptedField(entity, r))) {
+                    LogStatusEx({ message: `[CACHE-WRITE-GATE] Skipping server cache write for "${params.EntityName}" — rows contain decrypted plaintext for an encrypted field`, verboseOnly: true });
+                    return;
+                }
             }
             catch (err) {
                 // fall through and write — fail-open is safer than fail-closed here
@@ -2948,11 +3137,23 @@ export class LocalCacheManager extends BaseSingleton {
                 }
                 const pkFieldNames = changes[0].Key.KeyValuePairs.map(kv => kv.FieldName);
                 const resultMap = this.indexRowsByKey(cached.results, pkFieldNames);
-                if (!this.applyChangesToRowMap(resultMap, changes)) {
+                const deltaChanges = this.applyChangesToRowMap(resultMap, changes);
+                if (!deltaChanges.changed) {
                     return true; // only removals of rows the slot does not hold
                 }
-                return await this.storeCachedResults(fingerprint, Array.from(resultMap.values()), newMaxUpdatedAt, { totalRowCount: cached.totalRowCount, rowCount: cached.results.length, schemaHash: cached.schemaHash,
-                    schemaStamp: cached.schemaStamp, providerInternalScaffolding: cached.providerInternalScaffolding });
+                return await this.storeCachedResults(fingerprint, Array.from(resultMap.values()), newMaxUpdatedAt, {
+                    totalRowCount: cached.totalRowCount,
+                    rowCount: cached.results.length,
+                    schemaHash: cached.schemaHash,
+                    schemaStamp: cached.schemaStamp,
+                    providerInternalScaffolding: cached.providerInternalScaffolding,
+                }, cached.providerInternalScaffolding
+                    ? undefined
+                    : {
+                        upsertedRows: deltaChanges.upsertedRows,
+                        deletedPrimaryKeys: deltaChanges.deletedPrimaryKeys,
+                        primaryKeyFields: pkFieldNames,
+                    });
             }
             catch (e) {
                 LogError(`LocalCacheManager.ApplyRowChanges failed: ${e}`);
@@ -2975,20 +3176,38 @@ export class LocalCacheManager extends BaseSingleton {
         }
         return resultMap;
     }
-    /** Applies `changes` to `rows` in order. @returns True when anything changed. */
+    /** Applies `changes` to `rows` in order and collects the collapsed net row delta for pub/sub. */
     applyChangesToRowMap(rows, changes) {
         let changed = false;
+        const upsertsByKey = new Map();
+        const deletesByKey = new Map();
         for (const change of changes) {
             const keyStr = this.cheapKeyFromCompositeKey(change.Key);
             if (change.Row) {
                 rows.set(keyStr, change.Row);
+                deletesByKey.delete(keyStr);
+                upsertsByKey.set(keyStr, change.Row);
                 changed = true;
             }
             else if (rows.delete(keyStr)) {
+                upsertsByKey.delete(keyStr);
+                const pkRecord = {};
+                for (const kv of change.Key.KeyValuePairs) {
+                    const v = kv.Value;
+                    pkRecord[kv.FieldName] =
+                        typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean' || v === null
+                            ? v
+                            : String(v);
+                }
+                deletesByKey.set(keyStr, pkRecord);
                 changed = true;
             }
         }
-        return changed;
+        return {
+            changed,
+            upsertedRows: Array.from(upsertsByKey.values()),
+            deletedPrimaryKeys: Array.from(deletesByKey.values()),
+        };
     }
     /**
      * Stores updated results array back to the cache and updates the registry.
@@ -3004,7 +3223,7 @@ export class LocalCacheManager extends BaseSingleton {
      * full-dataset slot is unchanged (prior total == prior length) while a subset slot keeps a
      * correct total.
      */
-    async storeCachedResults(fingerprint, updatedResults, newMaxUpdatedAt, prior) {
+    async storeCachedResults(fingerprint, updatedResults, newMaxUpdatedAt, prior, rowDelta) {
         // Stamp the slot the way the fill path does — the newest row's __mj_UpdatedAt — so the
         // stamp means the same thing whichever funnel wrote the slot and matches what the database
         // reports for the same rows (the client smart-cache check compares the two). The caller's
@@ -3058,7 +3277,23 @@ export class LocalCacheManager extends BaseSingleton {
         if (!prior?.providerInternalScaffolding) {
             this.freezeRowDataIfProviderSharesReferences(data);
         }
-        await this._storageProvider.SetItem(fingerprint, data, CacheCategory.RunViewCache, this.runViewWriteOptions(fingerprint, existingEntry?.expiresAt));
+        let pubSubOverride;
+        if (rowDelta) {
+            const deltaPayload = {
+                upsertedRows: rowDelta.upsertedRows,
+                deletedPrimaryKeys: rowDelta.deletedPrimaryKeys,
+                primaryKeyFields: rowDelta.primaryKeyFields,
+                maxUpdatedAt,
+                rowCount: updatedResults.length,
+                totalRowCount: data.totalRowCount ?? updatedResults.length,
+                schemaHash: data.schemaHash,
+            };
+            pubSubOverride = {
+                pubSubAction: 'row_delta',
+                pubSubData: JSON.stringify(deltaPayload),
+            };
+        }
+        await this._storageProvider.SetItem(fingerprint, data, CacheCategory.RunViewCache, this.runViewWriteOptions(fingerprint, existingEntry?.expiresAt, pubSubOverride));
         this._lastVerifiedAt.set(fingerprint, now);
         if (existingEntry) {
             existingEntry.maxUpdatedAt = maxUpdatedAt;