@memberjunction/core 6.1.0-edge.1 → 6.1.0-edge.3

package/dist/generic/localCacheManager.js

@@ -1,4 +1,5 @@
 import { BaseSingleton, MJGlobal, MJEventType } from "@memberjunction/global";
+import { IsMaterializedDataSource } from "../views/runView.js";
 import { LogError, LogStatusEx } from "./logging.js";
 import { BaseEntity } from "./baseEntity.js";
 import { Metadata } from "./metadata.js";
@@ -7,6 +8,50 @@ import { CompositeKey, KeyValuePair } from "./compositeKey.js";
 function LogStatusVerbose(message) {
     LogStatusEx({ message, verboseOnly: true });
 }
+/**
+ * Recursively freezes a value that is about to enter the cache, so consumers holding the
+ * same reference cannot mutate shared state.
+ *
+ * Only invoked when the storage provider reports
+ * {@link ILocalStorageProvider.SharesReferences} — a serializing backend (IndexedDB,
+ * localStorage, Redis, MMKV) already isolates stored data, and freezing there would
+ * needlessly immobilize the caller's own rows.
+ *
+ * Freezing the ARRAY matters as much as the rows: `results.sort()` / `.push()` on a live
+ * cache array silently reorders or grows the cached slot for every later reader.
+ *
+ * Freezes BEFORE recursing, so the `isFrozen` short-circuit both skips already-frozen
+ * subtrees (cheap re-entry for in-place slot maintenance, which carries existing rows
+ * forward by reference) and terminates cycles.
+ *
+ * Two value kinds are skipped, not frozen — accepted residuals:
+ * - Binary payloads (`Buffer`/TypedArray/`ArrayBuffer`, e.g. `varbinary` columns): the spec
+ *   makes `Object.freeze` THROW on a non-empty view, so attempting it would turn a cache
+ *   write into a crash.
+ * - `Date` internal slots: `Object.freeze` cannot protect them — `setHours` and friends
+ *   still work.
+ */
+function deepFreezeCacheValue(value, visited = new WeakSet()) {
+    if (value === null || typeof value !== 'object'
+        || ArrayBuffer.isView(value) || value instanceof ArrayBuffer) {
+        return value;
+    }
+    // Cycle termination and re-entry short-circuit key off an explicit visited set rather than
+    // `Object.isFrozen`. Those are not the same test: an object frozen SHALLOWLY by someone else
+    // (a caller that ran `Object.freeze(row)` before handing it over) reports frozen while its
+    // nested values are still writable, so keying off isFrozen would skip the whole subtree and
+    // leave shared state mutable. Already-frozen objects are cheap to re-freeze; unvisited
+    // children are the thing that must not be skipped.
+    if (visited.has(value)) {
+        return value;
+    }
+    visited.add(value);
+    Object.freeze(value);
+    for (const nested of Object.values(value)) {
+        deepFreezeCacheValue(nested, visited);
+    }
+    return value;
+}
 // ============================================================================
 // DEFAULT CONFIGURATION
 // ============================================================================
@@ -76,6 +121,12 @@ export class LocalCacheManager extends BaseSingleton {
     constructor() {
         super();
         this._storageProvider = null;
+        /**
+         * Whether the active storage provider hands back live object references, resolved once at
+         * initialization — from the provider's declared {@link ILocalStorageProvider.SharesReferences}
+         * when it states one, otherwise measured empirically. Gates the defensive deep-freeze.
+         */
+        this._sharesReferences = false;
         this._registry = new Map();
         this._initialized = false;
         this._initializePromise = null;
@@ -138,14 +189,64 @@ export class LocalCacheManager extends BaseSingleton {
         this._initializePromise = this.doInitialize(storageProvider, config);
         return this._initializePromise;
     }
+    /**
+     * Decides whether the storage provider hands back live object references — the condition
+     * that makes the defensive deep-freeze necessary.
+     *
+     * Prefers the provider's declared {@link ILocalStorageProvider.SharesReferences}. When a
+     * provider does not state one (any implementation written before that property existed),
+     * MEASURE it rather than guessing: store a sentinel object, read it back, and compare
+     * identity. A reference-sharing store returns the very same object; anything with a
+     * serialization or structured-clone boundary returns a copy. This is what keeps the
+     * property optional — an external provider that never declares it still gets the correct
+     * protection instead of silently losing it to a falsy default.
+     *
+     * Fails closed to `false` if the probe cannot complete (a provider whose backing store is
+     * not ready at init): that matches the pre-freeze behavior rather than immobilizing rows
+     * for a provider we could not classify.
+     */
+    async resolveSharesReferences(storageProvider) {
+        if (typeof storageProvider.SharesReferences === 'boolean') {
+            return storageProvider.SharesReferences;
+        }
+        const probeKey = '__mj_sharesreferences_probe__';
+        const sentinel = { probe: true };
+        try {
+            await storageProvider.SetItem(probeKey, sentinel, CacheCategory.Default);
+            const readBack = await storageProvider.GetItem(probeKey, CacheCategory.Default);
+            const shares = readBack === sentinel;
+            LogStatusVerbose(`[CACHE-INIT] Storage provider "${storageProvider.constructor?.name ?? 'unknown'}" did not declare ` +
+                `SharesReferences; probed it as ${shares} (freeze-on-write ${shares ? 'ENABLED' : 'disabled'}).`);
+            return shares;
+        }
+        catch (e) {
+            LogError(`LocalCacheManager: could not probe SharesReferences on the storage provider; ` +
+                `assuming it isolates (freeze-on-write disabled). Declare SharesReferences to be explicit. ${e}`);
+            return false;
+        }
+        finally {
+            try {
+                await storageProvider.Remove(probeKey, CacheCategory.Default);
+            }
+            catch {
+                /* best-effort cleanup — a leftover probe key is harmless */
+            }
+        }
+    }
     /**
      * Internal initialization logic - only called once by the first caller
      */
     async doInitialize(storageProvider, config) {
-        this._storageProvider = storageProvider;
         if (config) {
             this._config = { ...this._config, ...config };
         }
+        // Resolve the freeze decision BEFORE publishing the provider. The probe awaits I/O, and
+        // `_storageProvider` is what every write path reads to decide whether to freeze — so
+        // assigning it first opens a window where writes see the new provider paired with the
+        // PREVIOUS provider's (or the default `false`) freeze decision. Same ordering bug that
+        // `SetStorageProvider` had; fixed here too rather than left as the one asymmetric path.
+        this._sharesReferences = await this.resolveSharesReferences(storageProvider);
+        this._storageProvider = storageProvider;
         await this.loadRegistry();
         this._initialized = true;
         // Start periodic eviction sweep for TTL-expired entries
@@ -193,12 +294,24 @@ export class LocalCacheManager extends BaseSingleton {
      * @param newProvider - The new storage provider to use
      */
     async SetStorageProvider(newProvider) {
+        // The freeze decision belongs to the ACTIVE provider, not to whichever one happened to be
+        // installed at Initialize. MJAPI initializes on the in-memory provider during engine
+        // loading and swaps to Redis afterward, so these two have OPPOSITE reference semantics on
+        // every Redis deployment — carrying the old answer forward means freezing rows Redis has
+        // already isolated (all of the hazard, none of the protection), or, on the reverse swap,
+        // silently dropping the protection.
+        //
+        // Resolved BEFORE `_storageProvider` is published in both branches: the probe awaits I/O,
+        // and write paths read `_storageProvider` to decide whether to freeze, so publishing first
+        // would pair the new provider with the old provider's decision for the duration of the probe.
         if (!this._initialized) {
             // Not yet initialized — just set the provider and return
+            this._sharesReferences = await this.resolveSharesReferences(newProvider);
             this._storageProvider = newProvider;
             return;
         }
         const oldProvider = this._storageProvider;
+        this._sharesReferences = await this.resolveSharesReferences(newProvider);
         this._storageProvider = newProvider;
         // Migrate existing cached data from old provider to new provider
         const entries = this.GetAllEntries();
@@ -932,7 +1045,8 @@ export class LocalCacheManager extends BaseSingleton {
      *
      * @param name - The dataset name
      * @param itemFilters - Optional filters applied to the dataset
-     * @param dataset - The dataset result to cache
+     * @param dataset - The dataset result to cache. Deep-frozen on reference-sharing storage,
+     *                  like every other cache write funnel — see below.
      * @param keyPrefix - Prefix for the cache key (typically includes connection info)
      */
     async SetDataset(name, itemFilters, dataset, keyPrefix) {
@@ -942,6 +1056,17 @@ export class LocalCacheManager extends BaseSingleton {
         // Estimate size from a string representation (used only for cache eviction
         // accounting; the actual stored value is the native object).
         const sizeBytes = this.estimateSize(JSON.stringify(dataset));
+        // Fourth write funnel, held to the same contract as SetRunViewResult /
+        // SetRunQueryResult / storeCachedResults. `GetDataset` hands this object straight back
+        // out, so on a reference-sharing provider every reader shares it — the same exposure the
+        // other three close. It has no in-repo caller today (GetDatasetByName caches per ITEM via
+        // SetRunViewResult), but it is exported public API, so an external caller would otherwise
+        // get an unprotected slot with no indication that it differs from the documented rule.
+        //
+        // Frozen BEFORE the awaited eviction below, for the same reason as the other funnels: a
+        // yield point between the decision to cache and the freeze is a window in which the
+        // caller can still mutate what is about to become shared state.
+        this.freezeRowDataIfProviderSharesReferences(dataset);
         // Check if we need to evict entries
         await this.evictIfNeeded(sizeBytes);
         try {
@@ -1078,7 +1203,7 @@ export class LocalCacheManager extends BaseSingleton {
      *   pre-RLS format so normal cache sharing is preserved and no existing entries are invalidated.
      * @returns A unique, human-readable fingerprint string
      */
-    GenerateRunViewFingerprint(params, connectionPrefix, rlsWhereClause) {
+    GenerateRunViewFingerprint(params, connectionPrefix, rlsWhereClause, datasetSegment) {
         const entity = params.EntityName?.trim() || 'Unknown';
         const rawFilter = params.ExtraFilter;
         const filter = (typeof rawFilter === 'string' ? rawFilter : rawFilter ? JSON.stringify(rawFilter) : '').trim();
@@ -1128,6 +1253,15 @@ export class LocalCacheManager extends BaseSingleton {
         if (params.IgnoreMaxRows === true) {
             parts.push('imr:1');
         }
+        // DataSource segment. RunViewParams.DataSource:'Materialized' routes the read to the entity's
+        // materialized snapshot view (GetEffectiveBaseView), a DIFFERENT physical source than the default
+        // live base view — so a Live read and a Materialized read of the same entity/filter/orderBy MUST
+        // NOT share a cache slot (else one is silently served the other's source). Appended ONLY for the
+        // non-default 'Materialized' so every existing (Live/default) fingerprint stays byte-for-byte
+        // identical and no existing cache entries are invalidated.
+        if (IsMaterializedDataSource(params.DataSource)) {
+            parts.push('ds:materialized');
+        }
         // Keyset (AfterKey) seek cursor MUST be part of the fingerprint. Each keyset page
         // sends a different AfterKey but otherwise-identical params; without this, sequential
         // pages collide on the same fingerprint and the dedup/linger layer hands page N+1 the
@@ -1158,6 +1292,25 @@ export class LocalCacheManager extends BaseSingleton {
         if (viewKey.length > 0) {
             parts.push(`vw:${viewKey}`);
         }
+        // Dataset namespace. `GetDatasetByName` caches each dataset ITEM's rows through this same
+        // builder, supplying only entity + the item's WhereClause — and every shipped item has a
+        // NULL WhereClause, so without this segment a dataset item emits the identical fingerprint
+        // to a plain unfiltered read of the same entity and the two silently share one slot. That
+        // is not merely a stale-data risk: the MJ_Metadata dataset writes its rows with
+        // `ProviderInternalScaffolding` (deliberately UNFROZEN, because bootstrap rearranges them
+        // in place), so the shared slot hands ordinary callers unprotected rows for the hottest
+        // entities in the process — and, in the other direction, an ordinary read repopulating an
+        // evicted slot stores it FROZEN and the next metadata refresh throws.
+        //
+        // Dataset items may also project columns (`DatasetItem.Columns`), where a RunView slot is
+        // always the full field set — so the two are not interchangeable in shape either.
+        //
+        // Appended ONLY when supplied, so ordinary reads keep their exact pre-existing key and no
+        // existing cache entry is invalidated by this change.
+        const dataset = (datasetSegment ?? '').trim();
+        if (dataset.length > 0) {
+            parts.push(`ds:${dataset}`);
+        }
         // Only include connection if provided
         if (connection) {
             parts.push(connection);
@@ -1260,6 +1413,64 @@ export class LocalCacheManager extends BaseSingleton {
             return undefined;
         }
     }
+    /**
+     * Deep-freezes an about-to-be-stored cache payload when — and only when — the active
+     * storage provider hands out live object references
+     * ({@link ILocalStorageProvider.SharesReferences}).
+     *
+     * This is the cache's structural defense against consumer corruption. Under a
+     * reference-sharing provider the rows a caller receives ARE the cached rows, in both
+     * directions: on a hit the reader gets the stored array, and on a miss the cache stored
+     * the caller's own array. Any in-place mutation therefore edits process-wide state. One
+     * such mutation shipped as a P1 (a resolver renamed `__mj_CreatedAt` to its GraphQL
+     * transport alias in place, so every later read served rows `BaseEntity.SetMany`
+     * rejects). Freezing makes that failure immediate and attributable — a `TypeError` at
+     * the offending line — instead of silent corruption, and costs nothing per cache hit.
+     *
+     * On serializing providers this is a no-op: their stored data is already isolated, and
+     * freezing would only immobilize the caller's own rows for no safety gain.
+     *
+     * @param payload - The envelope being handed to the storage provider. Frozen in place;
+     *                  the same reference is returned for call-site convenience.
+     */
+    /**
+     * Whether the CURRENTLY-INSTALLED provider hands back live references.
+     *
+     * Prefers the active provider's own declaration, read live on each call — a property read,
+     * so it costs nothing on the write path, and it stays correct no matter how many times the
+     * provider is swapped or which code path does the swapping. `_sharesReferences` (resolved at
+     * `Initialize`/`SetStorageProvider`) is the fallback for providers that declare nothing, where
+     * the answer can only come from the async probe and therefore cannot be recomputed here.
+     *
+     * Belt-and-braces with the re-resolution in {@link SetStorageProvider}: that keeps the probed
+     * value correct across swaps, and this keeps DECLARED providers correct even if some future
+     * swap site forgets to re-resolve.
+     */
+    activeProviderSharesReferences() {
+        const declared = this._storageProvider?.SharesReferences;
+        return typeof declared === 'boolean' ? declared : this._sharesReferences;
+    }
+    freezeRowDataIfProviderSharesReferences(payload) {
+        if (!this.activeProviderSharesReferences()) {
+            return payload;
+        }
+        try {
+            return deepFreezeCacheValue(payload);
+        }
+        catch (e) {
+            // The freeze is protective, never load-bearing — an exotic value Object.freeze
+            // rejects (beyond the guarded binary kinds) must degrade to a stored write rather
+            // than taking down the read path it defends.
+            //
+            // The payload is PARTIALLY frozen at this point and cannot be un-frozen: the walk
+            // freezes parent-first, so everything visited before the throw is already immutable.
+            // Say that plainly instead of claiming "unfrozen" — an operator debugging a
+            // downstream TypeError needs to know the entry is a mix, not a clean opt-out.
+            LogError(`LocalCacheManager: freeze-on-write failed partway; storing the entry with ` +
+                `whatever was frozen before the failure (partial protection, not none). ${e}`);
+            return payload;
+        }
+    }
     /**
      * Stores a RunView result in the cache.
      *
@@ -1276,8 +1487,11 @@ export class LocalCacheManager extends BaseSingleton {
      *   AllowCaching gating in multi-provider scenarios (parallel client connections to multiple
      *   servers). Falls back to `Metadata.Provider` (global default) when omitted, which is fine
      *   for single-provider apps but wrong when AllowCaching differs across servers.
+     * @param ttlMs - Optional time-based expiry (required for external-data-source entities).
+     * @param options - See {@link CacheWriteOptions}. Pass `{ ProviderInternalScaffolding: true }`
+     *   only for slots the writing provider is the sole consumer of.
      */
-    async SetRunViewResult(fingerprint, params, results, maxUpdatedAt, aggregateResults, totalRowCount, provider, ttlMs) {
+    async SetRunViewResult(fingerprint, params, results, maxUpdatedAt, aggregateResults, totalRowCount, provider, ttlMs, options) {
         if (!this._storageProvider || !this._config.enabled)
             return;
         // Keyset (AfterKey) queries are inherently single-use — each call uses a different
@@ -1336,6 +1550,10 @@ export class LocalCacheManager extends BaseSingleton {
         }
         // Persist results, maxUpdatedAt, aggregateResults, totalRowCount, and schemaHash
         const data = { results, maxUpdatedAt };
+        if (options?.ProviderInternalScaffolding) {
+            // Persisted so event-driven in-place maintenance carries the exemption forward.
+            data.providerInternalScaffolding = true;
+        }
         if (aggregateResults && aggregateResults.length > 0) {
             data.aggregateResults = aggregateResults;
         }
@@ -1361,10 +1579,23 @@ export class LocalCacheManager extends BaseSingleton {
             LogStatusEx({ message: `[CACHE-WRITE-GATE] Skipping cache write for "${params.EntityName || fingerprint.substring(0, 60)}" — estimated entry size ${sizeBytes} bytes exceeds per-entry cap (${this._config.maxEntryPercentOfCache}% of ${this._config.maxSizeBytes} byte budget)` });
             return;
         }
-        // Per-entity memory limit: evict oldest entries for this entity if over budget
+        // The oversized gate above is the ONLY step that can decline this write, and it is
+        // synchronous — so by here the row set is definitely becoming shared state, and it is
+        // frozen with no intervening yield point. That ordering is load-bearing: the eviction
+        // steps below are awaited, and callers hold this very array while they run (the
+        // smart-cache stale leg does not await this method at all). Freezing after them leaves
+        // a window in which shared rows are handed out still mutable — and `BaseEntity`
+        // samples `Object.isFrozen` once at load, so a freeze arriving mid-construction makes
+        // a later field READ throw. Declining writes still leave the caller's rows mutable,
+        // because the decline happens before this point.
+        if (!options?.ProviderInternalScaffolding) {
+            this.freezeRowDataIfProviderSharesReferences(data);
+        }
+        // Per-entity memory limit: evict oldest entries for this entity if over budget.
+        // Evicts OTHER entries only — it cannot cancel this write.
         const entityName = params.EntityName || 'Unknown';
         await this.enforcePerEntityMemoryLimit(entityName, sizeBytes);
-        // Check if we need to evict entries (global budget)
+        // Check if we need to evict entries (global budget). Also eviction-only.
         await this.evictIfNeeded(sizeBytes);
         try {
             // Native object storage — IDB structured-clones, localStorage / Redis serialize internally.
@@ -1504,6 +1735,8 @@ export class LocalCacheManager extends BaseSingleton {
             totalRowCount: parsed.totalRowCount,
             // Surfaced so in-place maintenance can carry it forward on rewrite (B38).
             schemaHash: parsed.schemaHash,
+            // Same carry-forward rationale: a maintained scaffolding slot must stay exempt.
+            providerInternalScaffolding: parsed.providerInternalScaffolding,
         };
         if (parsed.aggregateResults) {
             result.aggregateResults = parsed.aggregateResults;
@@ -1717,7 +1950,8 @@ export class LocalCacheManager extends BaseSingleton {
                 // Upsert the entity (add or replace)
                 resultMap.set(keyStr, entityData);
                 const updatedResults = Array.from(resultMap.values());
-                return await this.storeCachedResults(fingerprint, updatedResults, newMaxUpdatedAt, { totalRowCount: cached.totalRowCount, rowCount: cached.results.length, schemaHash: cached.schemaHash });
+                return await this.storeCachedResults(fingerprint, updatedResults, newMaxUpdatedAt, { totalRowCount: cached.totalRowCount, rowCount: cached.results.length, schemaHash: cached.schemaHash,
+                    providerInternalScaffolding: cached.providerInternalScaffolding });
             }
             catch (e) {
                 LogError(`LocalCacheManager.UpsertSingleEntity failed: ${e}`);
@@ -1759,7 +1993,8 @@ export class LocalCacheManager extends BaseSingleton {
                 }
                 resultMap.delete(keyStr);
                 const updatedResults = Array.from(resultMap.values());
-                return await this.storeCachedResults(fingerprint, updatedResults, newMaxUpdatedAt, { totalRowCount: cached.totalRowCount, rowCount: cached.results.length, schemaHash: cached.schemaHash });
+                return await this.storeCachedResults(fingerprint, updatedResults, newMaxUpdatedAt, { totalRowCount: cached.totalRowCount, rowCount: cached.results.length, schemaHash: cached.schemaHash,
+                    providerInternalScaffolding: cached.providerInternalScaffolding });
             }
             catch (e) {
                 LogError(`LocalCacheManager.RemoveSingleEntity failed: ${e}`);
@@ -1800,6 +2035,12 @@ export class LocalCacheManager extends BaseSingleton {
         if (prior?.schemaHash) {
             data.schemaHash = prior.schemaHash;
         }
+        // Carry the freeze exemption forward too — a scaffolding slot that gets maintained in
+        // place must not silently become frozen, or the owner that mutates those rows breaks on
+        // the next read (for the metadata dataset that means booting with no metadata).
+        if (prior?.providerInternalScaffolding) {
+            data.providerInternalScaffolding = true;
+        }
         // Aggregates are deliberately NOT carried here — see hasAggregates(): an aggregate-bearing
         // slot is invalidated on mutation rather than maintained, so this path never runs for one.
         if (prior?.totalRowCount != null) {
@@ -1810,6 +2051,13 @@ export class LocalCacheManager extends BaseSingleton {
         // value is the native object. This runs on every save/delete event per matching
         // unfiltered fingerprint, so avoiding a full serialization here matters most.
         const sizeBytes = this.estimateResultsSize(updatedResults);
+        // Second write funnel — this path bypasses SetRunViewResult entirely, so it must
+        // freeze too. Rows carried forward from the prior slot are already frozen (the
+        // isFrozen short-circuit makes re-entry cheap); what this catches is the NEW array
+        // and the freshly upserted row. Scaffolding slots stay exempt (see above).
+        if (!prior?.providerInternalScaffolding) {
+            this.freezeRowDataIfProviderSharesReferences(data);
+        }
         await this._storageProvider.SetItem(fingerprint, data, CacheCategory.RunViewCache);
         const existingEntry = this._registry.get(fingerprint);
         if (existingEntry) {
@@ -1911,7 +2159,14 @@ export class LocalCacheManager extends BaseSingleton {
             LogStatusEx({ message: `[CACHE-WRITE-GATE] Skipping cache write for query "${queryName}" — estimated entry size ${sizeBytes} bytes exceeds per-entry cap (${this._config.maxEntryPercentOfCache}% of ${this._config.maxSizeBytes} byte budget)` });
             return;
         }
-        // Check if we need to evict entries
+        // Same reference-sharing exposure as the RunView path (GetRunQueryResult hands
+        // `results` straight back out) — closed here rather than waiting for a mutator to
+        // find it. Placed immediately after the only declining gate and BEFORE the awaited
+        // eviction below, for the same reason as SetRunViewResult: callers of this method do
+        // not always await it, so any yield point before the freeze is a window in which the
+        // caller can still mutate the rows the cache is about to store.
+        this.freezeRowDataIfProviderSharesReferences(data);
+        // Check if we need to evict entries. Evicts OTHER entries only — cannot cancel this write.
         await this.evictIfNeeded(sizeBytes);
         const now = Date.now();
         const expiresAt = ttlMs ? now + ttlMs : undefined;