@memberjunction/core 5.44.0 → 5.45.1

package/dist/generic/baseEngine.js

@@ -1,7 +1,7 @@
 import { BaseSingleton, MJEventType, MJGlobal } from "@memberjunction/global";
 import { TelemetryManager } from "./telemetryManager.js";
 import { BehaviorSubject, Subject } from "rxjs";
-import { debounceTime } from "rxjs/operators";
+import { buffer, debounceTime, filter } from "rxjs/operators";
 import { RunView } from "../views/runView.js";
 import { LogError, LogStatus } from "./logging.js";
 import { Metadata } from "./metadata.js";
@@ -107,7 +107,19 @@ export class BaseEngine extends BaseSingleton {
         this._propertySubjects = new Map();
         this._isPermissionConstrained = false;
         this._deniedEntityNames = [];
+        /**
+         * Per-property monotonic full-refresh counter. Guards {@link LoadSingleEntityConfig}
+         * against overlapping full refreshes clobbering each other: when several full-refresh
+         * RunViews for the same property are in flight at once (e.g. a burst of saves each
+         * landing in its own debounce window because the round-trip exceeds the DebounceTime),
+         * only the latest-INITIATED refresh may commit its results. Without this, whichever
+         * RunView happens to RESOLVE last wins — which can be an earlier-initiated request that
+         * read a staler snapshot, leaving the cache "one operation behind" until a full reload
+         * reconciles it. Keyed by config PropertyName. See {@link beginConfigRefresh}.
+         */
+        this._configRefreshGeneration = new Map();
         this._entityEventDebounceTime = 1500; // Default debounce time in milliseconds (1.5 seconds)
+        this._eventRefreshRetryTimers = new Map();
     }
     /**
      * Observable that emits when any data property changes due to a refresh.
@@ -564,10 +576,18 @@ export class BaseEngine extends BaseSingleton {
                 }
                 // Fall through to server fetch if direct delete failed
             }
-            // Fallback: re-fetch from server (missing data or apply failure)
+            // Fallback: re-fetch from server (missing data or apply failure). Bypass the cache
+            // for the same reason as the local-event path: this fetch runs BECAUSE a
+            // (cross-server) write signaled the cache stale, so reading back through it would
+            // re-sync the pre-write snapshot.
             let refreshCount = 0;
             for (const config of matchingConfigs) {
-                await this.LoadSingleConfig(config, this._contextUser);
+                await this.LoadSingleConfig(config, this._contextUser, /*bypassCache*/ true);
+                if (!this.configLoadedSuccessfully(config.PropertyName)) {
+                    // Same one-shot hazard as the debounced path: the invalidation event is
+                    // consumed, so a transient failure here needs a bounded retry too.
+                    this.scheduleEventRefreshRetry(config, 1);
+                }
                 refreshCount++;
             }
             if (refreshCount > 0) {
@@ -663,10 +683,16 @@ export class BaseEngine extends BaseSingleton {
     }
     /**
      * This method handles the debouncing process, by default using the EntityEventDebounceTime property to set the debounce time. Debouncing is
-     * done on a per-entity basis, meaning that if the debounce time passes for a specific entity name, the event will be processed. This is done to
+     * done on a per-entity basis, meaning that if the debounce time passes for a specific entity name, the events will be processed. This is done to
      * prevent multiple events from being processed in quick succession for a single entity which would cause a lot of wasted processing.
      *
-     * Override this method if you want to change how debouncing time such as having variable debounce times per-entity, etc.
+     * ALL events raised during the debounce window are buffered and delivered as one batch to
+     * {@link ProcessEntityEvents} — not just the last one. The refresh-vs-skip decision must be
+     * an OR over every coalesced event: judging only the last event would let an
+     * already-applied write (e.g., an engine method's in-place save of a cached instance)
+     * mask an earlier fresh-instance save the array has never seen.
+     *
+     * Override this method if you want to change how debouncing works, such as having variable debounce times per-entity, etc.
      * @param event
      * @returns
      */
@@ -679,8 +705,11 @@ export class BaseEngine extends BaseSingleton {
                 // Use config-specific debounce time or fall back to default
                 const debounceTimeValue = matchingConfig?.DebounceTime ?? this.EntityEventDebounceTime;
                 const subject = new Subject();
-                subject.pipe(debounceTime(debounceTimeValue)).subscribe(async (e) => {
-                    await this.ProcessEntityEvent(e);
+                subject.pipe(
+                // Collect every event in the window; the debounced stream closes the buffer,
+                // so the batch is emitted once the entity has been quiet for the full window.
+                buffer(subject.pipe(debounceTime(debounceTimeValue))), filter(batch => batch.length > 0)).subscribe(async (batch) => {
+                    await this.ProcessEntityEvents(batch);
                 });
                 this._entityEventSubjects.set(entityName, subject);
             }
@@ -704,55 +733,82 @@ export class BaseEngine extends BaseSingleton {
         return this._entityEventDebounceTime;
     }
     /**
-     * This method does the actual work of processing the entity event. It is not directly called from the event handler because we want to first debounce the events
-     * which also introduces a delay which is usually desirable so that our processing is typically outside of the scope of any transaction processing that would have
-     * originated the event.
-     *
-     * This is the best method to override if you want to change the actual processing of an entity event but do NOT want to modify the debouncing behavior.
+     * Back-compat single-event wrapper around {@link ProcessEntityEvents}. The debounced
+     * pipeline delivers full batches to ProcessEntityEvents — override THAT method to change
+     * event-processing behavior; this wrapper exists for subclasses/tests that process one
+     * event at a time.
      */
     async ProcessEntityEvent(event) {
+        return this.ProcessEntityEvents([event]);
+    }
+    /**
+     * Does the actual work of processing all entity events coalesced into one debounce window.
+     * Not called directly from the event handler because we first debounce the events, which also
+     * introduces a delay that is usually desirable so processing happens outside the scope of any
+     * transaction processing that originated the events.
+     *
+     * Per matching config, the decision is an OR over the whole batch:
+     * - If ANY event's changes are not yet reflected in the config's array, run the refresh
+     *   (or apply each such event via immediate mutation when the config allows it). A single
+     *   full refresh covers every event in the window.
+     * - Else, if any event's changes were already applied (in-place save of a cached instance,
+     *   manual push after create), notify observers once — the refresh is redundant but the
+     *   notification is not.
+     * - Deletes of rows absent from the array stay silent: "already spliced by engine code"
+     *   (that code owns the notification, see {@link notifyAlreadyAppliedMutation}) is
+     *   indistinguishable from "never matched this config's Filter", and notifying would
+     *   assert phantom deletes to filtered configs' observers.
+     *
+     * A transiently-failed refresh schedules a bounded retry via
+     * {@link scheduleEventRefreshRetry} — without it, the consumed debounce event would leave
+     * observers permanently stale until an unrelated event arrived.
+     *
+     * This is the best method to override if you want to change the actual processing of entity
+     * events but do NOT want to modify the debouncing behavior.
+     */
+    async ProcessEntityEvents(events) {
+        if (!events || events.length === 0) {
+            return;
+        }
         try {
-            const entityName = event.baseEntity.EntityInfo.Name.toLowerCase().trim();
+            const entityName = events[0].baseEntity.EntityInfo.Name.toLowerCase().trim();
             let refreshCount = 0;
             for (const config of this.Configs) {
                 if (config.AutoRefresh && config.Type === 'entity' && config.EntityName?.trim().toLowerCase() === entityName) {
-                    // For UPDATE events, check if the exact object is already in our array.
-                    // If so, it's already been mutated in place - no need to refresh.
-                    if (event.type === 'save' && event.saveSubType === 'update') {
-                        if (this.isEntityAlreadyInArray(config, event.baseEntity)) {
-                            // Object already in array and updated in place, skip refresh
-                            // LogStatus(`>>> Skipping refresh for ${config.PropertyName} - object already in array`);
-                            continue;
-                        }
-                    }
-                    // For CREATE events, check if the entity was already added to our array
-                    // (e.g., by engine methods like InstallApplication that manually push).
-                    if (event.type === 'save' && event.saveSubType === 'create') {
-                        if (this.isEntityAlreadyInArray(config, event.baseEntity)) {
-                            // Object already in array (manually added), skip refresh
-                            // LogStatus(`>>> Skipping refresh for ${config.PropertyName} - newly created object already in array`);
-                            continue;
+                    const classified = events.map(e => ({ event: e, disposition: this.classifyEventForConfig(config, e) }));
+                    const needingWork = classified.filter(c => c.disposition.action === 'refresh');
+                    if (needingWork.length > 0) {
+                        // Check if we can use immediate array mutation instead of running a view
+                        if (this.canUseImmediateMutation(config)) {
+                            // Apply, in order, every event the array hasn't seen yet
+                            for (const c of needingWork) {
+                                await this.applyImmediateMutation(config, c.event);
+                            }
                         }
-                    }
-                    // For DELETE events, check if the entity was already removed from our array
-                    // (e.g., by engine methods like UninstallApplication that manually splice).
-                    // Also check by primary key since the object reference may still exist but be removed.
-                    if (event.type === 'delete') {
-                        if (!this.isEntityInArrayByRefOrKey(config, event.baseEntity)) {
-                            // Object not in array (already removed), skip refresh
-                            // LogStatus(`>>> Skipping refresh for ${config.PropertyName} - deleted object not in array`);
-                            continue;
+                        else {
+                            // One full refresh covers every event in the window. Bypass the cache:
+                            // this refresh is triggered BY a save/delete of this very entity, so a
+                            // cached view result is precisely what may be stale. Reading true DB
+                            // state guarantees the engine cache reflects the write that just fired
+                            // the event — otherwise a filtered/ordered config (which can't be
+                            // updated in place, e.g. UserInfoEngine's per-user '_UserApplications')
+                            // re-syncs the PRE-write snapshot and the UI sits "one operation behind"
+                            // until a full page reload repopulates the cache.
+                            await this.LoadSingleConfig(config, this._contextUser, /*bypassCache*/ true);
+                            if (!this.configLoadedSuccessfully(config.PropertyName)) {
+                                this.scheduleEventRefreshRetry(config, 1);
+                            }
+                            refreshCount++;
                         }
                     }
-                    // Check if we can use immediate array mutation instead of running a view
-                    if (this.canUseImmediateMutation(config)) {
-                        // LogStatus(`>>> Immediate mutation for ${config.PropertyName} due to BaseEntity ${event.type} event for: ${event.baseEntity.EntityInfo.Name}`);
-                        await this.applyImmediateMutation(config, event);
-                    }
                     else {
-                        // LogStatus(`>>> Refreshing metadata for ${config.PropertyName} due to BaseEntity ${event.type} event for: ${event.baseEntity.EntityInfo.Name}, pkey: ${event.baseEntity.PrimaryKey.ToString()}`);
-                        await this.LoadSingleConfig(config, this._contextUser);
-                        refreshCount++;
+                        // No event requires a refresh. If any event's changes were already
+                        // applied to the array, notify observers once (latest such event wins);
+                        // otherwise every event was a silent delete-of-absent-row.
+                        const lastNotify = [...classified].reverse().find(c => c.disposition.action === 'notify');
+                        if (lastNotify && lastNotify.disposition.action === 'notify') {
+                            this.notifyAlreadyAppliedMutation(config, lastNotify.disposition.changeType, lastNotify.event.baseEntity);
+                        }
                     }
                 }
             }
@@ -770,6 +826,38 @@ export class BaseEngine extends BaseSingleton {
             LogError(e);
         }
     }
+    /**
+     * Classifies a single entity event against a single config's backing array:
+     * - 'refresh' — the array does not yet reflect this event's changes; a refresh
+     *   (or immediate mutation) is required.
+     * - 'notify' — the array already reflects the change (in-place save of the array's own
+     *   cached instance, or a manually-pushed create); observers still need a notification.
+     * - 'silent' — a delete of a row absent from the array; "already spliced" is
+     *   indistinguishable from "never matched the Filter", so no notification is emitted
+     *   (manual-splice engine code owns that notification).
+     *
+     * For deletes, the by-key membership check uses the event payload's pre-delete OldValues
+     * snapshot — BaseEntity.Delete() calls NewRecord() right after raising the event, which
+     * wipes field values and REGENERATES the primary key, so the live entity's key can never
+     * match the deleted row by the time the debounced handler runs.
+     */
+    classifyEventForConfig(config, event) {
+        if (event.type === 'save') {
+            if (event.saveSubType === 'update' || event.saveSubType === 'create') {
+                if (this.isEntityAlreadyInArray(config, event.baseEntity)) {
+                    return { action: 'notify', changeType: event.saveSubType === 'create' ? 'add' : 'update' };
+                }
+            }
+            return { action: 'refresh' };
+        }
+        if (event.type === 'delete') {
+            const oldValues = event.payload?.OldValues;
+            return this.isEntityInArrayByRefOrKey(config, event.baseEntity, oldValues)
+                ? { action: 'refresh' }
+                : { action: 'silent' };
+        }
+        return { action: 'silent' };
+    }
     /**
      * Checks if the exact entity object reference is already in the config's data array.
      * Used to skip unnecessary refreshes for UPDATE events where the object was mutated in place.
@@ -788,14 +876,19 @@ export class BaseEngine extends BaseSingleton {
     /**
      * Checks if an entity is in the config's data array by object reference OR by primary key match.
      * Used for DELETE events where we need to know if the entity still exists in the array.
-     * The object reference may still exist (entity.Delete() just marks it deleted), but if it was
-     * manually spliced out by engine code, we check by primary key as fallback.
+     *
+     * For deletes, pass `preDeleteValues` (the event payload's OldValues snapshot): by the time
+     * the debounced handler runs, BaseEntity.Delete() has already called NewRecord(), which wipes
+     * the entity's fields and regenerates its primary key — so a by-key check against the live
+     * entity can never match the deleted row. Same hazard (and same OldValues workaround) as
+     * LocalCacheManager.HandleBaseEntityEvent.
      *
      * @param config - The configuration to check
      * @param entity - The entity to look for
+     * @param preDeleteValues - Pre-delete field snapshot (delete event payload's OldValues)
      * @returns true if the entity is in the array (by reference or by primary key)
      */
-    isEntityInArrayByRefOrKey(config, entity) {
+    isEntityInArrayByRefOrKey(config, entity, preDeleteValues) {
         const currentData = this[config.PropertyName];
         if (!currentData) {
             return false;
@@ -804,9 +897,108 @@ export class BaseEngine extends BaseSingleton {
         if (currentData.indexOf(entity) >= 0) {
             return true;
         }
-        // Fallback: check by primary key
+        // Preferred by-key check: build the key from the pre-delete snapshot
+        if (preDeleteValues) {
+            const key = new CompositeKey();
+            key.LoadFromEntityInfoAndRecord(entity.EntityInfo, preDeleteValues);
+            if (key.KeyValuePairs.length > 0 && !key.KeyValuePairs.some(kv => kv.Value == null)) {
+                return currentData.some(e => e.PrimaryKey.Equals(key));
+            }
+        }
+        // Fallback: check by the entity's current primary key
         return this.findEntityIndexByPrimaryKeys(currentData, entity) >= 0;
     }
+    /**
+     * Emits change notifications for a config whose backing array ALREADY reflects the
+     * entity event — e.g., engine code saved the array's own cached instance in place,
+     * or manually pushed a newly created entity after Save. In those cases
+     * {@link ProcessEntityEvent} safely skips the redundant refresh, but the notification
+     * must NOT be skipped: without it, `DataChange$` and `ObserveProperty` subscribers
+     * (and anything derived from them downstream) never learn the array changed and are
+     * stranded on stale state.
+     *
+     * Engine subclasses that manually SPLICE a deleted row out of a config's array must
+     * call this themselves ('delete') right after splicing — the debounced event handler
+     * cannot distinguish "already spliced" from "never matched this config's Filter", so
+     * it stays silent for absent rows.
+     *
+     * Deliberately does not run AdditionalLoading — the skip paths never did, and
+     * engines that maintain their arrays manually own any derived-data updates themselves.
+     */
+    notifyAlreadyAppliedMutation(config, changeType, entity) {
+        // Don't emit for configs that were skipped due to permission denial or never
+        // loaded — an emission with fabricated empty data would invite consumers to
+        // read engine getters that throw PermissionConstrainedError.
+        const mapEntry = this._dataMap.get(config.PropertyName);
+        if (mapEntry?.permissionDenied) {
+            return;
+        }
+        const currentData = this[config.PropertyName];
+        if (!currentData) {
+            return;
+        }
+        this.NotifyDataChange(config, currentData, changeType, entity);
+        this.emitPropertyChange(config.PropertyName);
+    }
+    /**
+     * True when the config's last load attempt left it in a successfully-loaded state.
+     * Reads the same map entry {@link HandleSingleViewResult} writes — a transient failure
+     * (network, server restart) records loadedSuccessfully=false; a permission denial is
+     * recorded as loaded-empty (true) and is deliberately NOT retryable.
+     */
+    configLoadedSuccessfully(propertyName) {
+        return this._dataMap.get(propertyName)?.loadedSuccessfully === true;
+    }
+    /**
+     * Maximum number of retries for an event-triggered config refresh that failed transiently.
+     * Overridable by subclasses that want more or less persistence.
+     */
+    get MaxEventRefreshRetries() {
+        return 2;
+    }
+    /**
+     * Schedules a bounded, backed-off retry of a config refresh that failed transiently during
+     * entity-event processing. Without this, one failed RunView after a save would permanently
+     * strand every observer on stale data — the debounced event is already consumed, so nothing
+     * else re-runs the refresh until an unrelated event for the same entity arrives.
+     *
+     * At most one retry is pending per property at a time; a retry that succeeds notifies
+     * observers through the normal HandleSingleViewResult → NotifyDataChange path. Permission
+     * denials never reach here (HandleSingleViewResult marks them loaded-empty).
+     *
+     * @param config - The config whose refresh failed
+     * @param attempt - 1-based attempt number; delays back off linearly (2s, 4s, ...)
+     */
+    scheduleEventRefreshRetry(config, attempt) {
+        const key = config.PropertyName;
+        if (this._eventRefreshRetryTimers.has(key)) {
+            return; // a retry is already pending for this property
+        }
+        if (attempt > this.MaxEventRefreshRetries) {
+            LogError(`BaseEngine: giving up on event-triggered refresh of ${config.EntityName} → ${key} after ${this.MaxEventRefreshRetries} retries`);
+            return;
+        }
+        const delayMs = 2000 * attempt;
+        const timer = setTimeout(async () => {
+            this._eventRefreshRetryTimers.delete(key);
+            try {
+                // Bypass the cache: this retries an event-triggered refresh, so the cache is
+                // still the stale copy the original write invalidated — reading through it
+                // could "succeed" with stale data and reinstate the one-operation-behind bug.
+                await this.LoadSingleConfig(config, this._contextUser, /*bypassCache*/ true);
+                if (this.configLoadedSuccessfully(key)) {
+                    await this.AdditionalLoading(this._contextUser);
+                }
+                else {
+                    this.scheduleEventRefreshRetry(config, attempt + 1);
+                }
+            }
+            catch (e) {
+                LogError(e);
+            }
+        }, delayMs);
+        this._eventRefreshRetryTimers.set(key, timer);
+    }
     /**
      * Determines if an immediate array mutation can be used instead of running a full view refresh.
      * Immediate mutations are only safe when:
@@ -898,8 +1090,6 @@ export class BaseEngine extends BaseSingleton {
             if (event.saveSubType === 'create') {
                 // For create, first check if the exact object is already in the array
                 const existsByRef = currentData.indexOf(entity) >= 0;
-                // if already in the array, nothing to do, but we keep going
-                // in the method as there is stuff below the outer if block
                 if (!existsByRef) {
                     // Check by composite primary key in case it was added with a different object reference
                     const indexByKey = this.findEntityIndexByPrimaryKeys(currentData, entity);
@@ -916,11 +1106,15 @@ export class BaseEngine extends BaseSingleton {
                         this.NotifyDataChange(config, currentData, 'add', cached);
                     }
                 }
+                else {
+                    // Already in the array by reference (engine code pushed it manually after
+                    // Save) — the array is current, but DataChange$ subscribers still need to
+                    // hear about the change. Pass the in-array instance, not the clone.
+                    this.NotifyDataChange(config, currentData, 'add', entity);
+                }
             }
             else {
                 // Update: first check if the exact object is already in the array
-                // if already in the array, we don't do anything but we keep going
-                // in the method so stuff at end can be done
                 const existsByRef = currentData.indexOf(entity) >= 0;
                 if (!existsByRef) {
                     // Find by composite primary key and replace
@@ -938,6 +1132,11 @@ export class BaseEngine extends BaseSingleton {
                         this.NotifyDataChange(config, currentData, 'add', cached);
                     }
                 }
+                else {
+                    // In-place save of the array's own cached instance — the array is already
+                    // current, but DataChange$ subscribers still need to hear about the change.
+                    this.NotifyDataChange(config, currentData, 'update', entity);
+                }
             }
         }
         else if (event.type === 'delete') {
@@ -1170,6 +1369,25 @@ export class BaseEngine extends BaseSingleton {
         else
             return await this.LoadSingleEntityConfig(config, contextUser, bypassCache);
     }
+    /**
+     * Opens a new full-refresh "generation" for a property and returns its token. Each call
+     * bumps the property's monotonic counter, so a token is the latest iff no later refresh
+     * for that property has begun since. See {@link _configRefreshGeneration}.
+     */
+    beginConfigRefresh(propertyName) {
+        const next = (this._configRefreshGeneration.get(propertyName) ?? 0) + 1;
+        this._configRefreshGeneration.set(propertyName, next);
+        return next;
+    }
+    /**
+     * True when `generation` is still the most recent token handed out by
+     * {@link beginConfigRefresh} for `propertyName` — i.e. no newer full refresh for this
+     * property has started since. A refresh whose token is stale must NOT commit its results:
+     * a newer refresh was initiated afterward and read a more-recent state.
+     */
+    isLatestConfigRefresh(propertyName, generation) {
+        return this._configRefreshGeneration.get(propertyName) === generation;
+    }
     /**
      * Handles the process of loading a single config of type 'entity'.
      * @param config
@@ -1177,6 +1395,12 @@ export class BaseEngine extends BaseSingleton {
      * @param bypassCache - When true, bypasses server-side cache to get fresh data from the database
      */
     async LoadSingleEntityConfig(config, contextUser, bypassCache = false) {
+        // Claim a refresh generation BEFORE the awaited RunView. If another full refresh for
+        // this same property starts while our RunView is in flight, ours becomes stale and must
+        // not commit — otherwise concurrent refreshes (the filtered/OrderBy path, e.g.
+        // UserInfoEngine's per-user '_UserApplications') apply out of initiation-order and the
+        // cache ends up "one operation behind". See _configRefreshGeneration.
+        const generation = this.beginConfigRefresh(config.PropertyName);
         const p = this.RunViewProviderToUse;
         const rv = new RunView(p);
         const result = await rv.RunView({
@@ -1190,6 +1414,14 @@ export class BaseEngine extends BaseSingleton {
             CacheLocalTTL: config.CacheLocalTTL,
             BypassCache: bypassCache
         }, contextUser);
+        // A newer full refresh superseded us while we awaited — drop this (staler) snapshot
+        // rather than clobber the newer one. The newer refresh owns the assignment, the
+        // observer notification, and (on failure) the retry, so bailing here leaves no
+        // observer un-notified. The single-refresh path is unaffected: its generation is
+        // still the latest when its RunView returns.
+        if (!this.isLatestConfigRefresh(config.PropertyName, generation)) {
+            return;
+        }
         this.HandleSingleViewResult(config, result, contextUser);
         this.emitPropertyChange(config.PropertyName);
     }