@memberjunction/metadata-sync 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/lib/sync-engine.js

@@ -17,6 +17,9 @@ import { GetDialect, IsDateSQLType, IsUuidSQLType } from '@memberjunction/sql-di
 import { JsonPreprocessor } from './json-preprocessor.js';
 import { DescribeMissingEntitySubclass } from './entity-subclass-guard.js';
 import { BatchContextIndex } from './batch-context-index.js';
+import { ParseEnvReference, OptionalEnvMissingError, EnvVariableNotSetError } from './env-reference.js';
+import { EnvDisplayMask, EnvSourceLabel, MIN_MASKED_VALUE_LENGTH } from './env-display.js';
+import { LOOKUP_OPTIONAL_MESSAGE, ParseLookupCreateFields, SplitLookupReference } from './lookup-reference.js';
 import { METADATA_KEYWORDS, IsMetadataKeyword, IsNonKeywordAtSymbol, ExtractKeywordValue } from '../constants/metadata-keywords.js';
 /**
  * Custom error class for lookup failures that can be deferred.
@@ -93,6 +96,10 @@ export class SyncEngine {
      */
     constructor(contextUser) {
         this.syncMetadataEngine = null;
+        /** Names of every environment variable this engine resolved, kept for the engine's lifetime and never cleared. Never holds values. */
+        this.ResolvedEnvVariables = new Set();
+        /** Runs right before a `@lookup:…?create` lookup saves the record it creates, with the env-sourced fields of that record. */
+        this.BeforeLookupCreateSave = null;
         /**
          * Where non-fatal warnings raised inside the engine go (the BaseEntity-fallback notice on the
          * lookup auto-create path, for one). PushService points this at its own warnings list so the
@@ -128,6 +135,7 @@ export class SyncEngine {
      * - `@url:address` - Fetches content from a URL
      * - `@lookup:Entity.Field=Value` - Looks up an entity ID by field value
      * - `@env:VARIABLE` - Reads an environment variable
+     * - `@env:VARIABLE?optional` - Like `@env:`, but a missing variable throws {@link OptionalEnvMissingError} so the caller can leave the field out
      *
      * @param value - The field value to process
      * @param baseDir - Base directory for resolving relative file paths
@@ -170,7 +178,9 @@ export class SyncEngine {
             // Check if it's an array or a plain object (not a Date, etc.)
             if (Array.isArray(value) || value.constructor === Object) {
                 // First recursively process any @lookup, @file, @parent references inside the object
-                const processedValue = await this.processJsonFieldValues(value, baseDir, parentRecord, rootRecord, depth, batchContext, recordProvider, ownerRecord);
+                const envCollector = this.envOnlyCollector(resolutionCollector);
+                const processedValue = await this.processJsonFieldValues(value, baseDir, parentRecord, rootRecord, depth, batchContext, recordProvider, ownerRecord, envCollector, fieldName);
+                this.mergeEnvFields(resolutionCollector, envCollector);
                 // Then convert to pretty-printed JSON string for inline metadata objects
                 // Objects from @file references will be handled by BaseEntity during save
                 return JSON.stringify(processedValue, null, 2);
@@ -196,13 +206,14 @@ export class SyncEngine {
                 const ownerName = ownerRecord.EntityInfo?.Name || 'unknown';
                 throw new Error(`@owner:${ownerFieldName} resolved to undefined on owner ${ownerName}. Field does not exist or has undefined value.`);
             }
+            const sourceVariables = this.recordCopiedEnvValue(resolvedValue, resolutionCollector, fieldName);
             // Track the resolution if collector is provided
             if (resolutionCollector && fieldName) {
                 resolutionCollector.notes.push({
                     type: 'parent',
                     field: `${resolutionCollector.fieldPrefix}.${fieldName}`,
                     expression: value,
-                    resolved: String(resolvedValue)
+                    resolved: sourceVariables.length > 0 ? EnvSourceLabel(sourceVariables) : String(resolvedValue)
                 });
             }
             return resolvedValue;
@@ -214,13 +225,14 @@ export class SyncEngine {
             }
             const parentFieldName = ExtractKeywordValue(value) || '';
             const resolvedValue = parentRecord.Get(parentFieldName);
+            const sourceVariables = this.recordCopiedEnvValue(resolvedValue, resolutionCollector, fieldName);
             // Track the resolution if collector is provided
             if (resolutionCollector && fieldName) {
                 resolutionCollector.notes.push({
                     type: 'parent',
                     field: `${resolutionCollector.fieldPrefix}.${fieldName}`,
                     expression: value,
-                    resolved: String(resolvedValue)
+                    resolved: sourceVariables.length > 0 ? EnvSourceLabel(sourceVariables) : String(resolvedValue)
                 });
             }
             return resolvedValue;
@@ -257,7 +269,9 @@ export class SyncEngine {
                         }
                         // Now recursively process any @file references within the JSON
                         const fileDir = path.dirname(fullPath);
-                        processedJson = await this.processJsonFieldValues(processedJson, fileDir, parentRecord, rootRecord, depth + 1, batchContext, recordProvider);
+                        const envCollector = this.envOnlyCollector(resolutionCollector);
+                        processedJson = await this.processJsonFieldValues(processedJson, fileDir, parentRecord, rootRecord, depth + 1, batchContext, recordProvider, undefined, envCollector, fieldName);
+                        this.mergeEnvFields(resolutionCollector, envCollector);
                         // Return the processed JSON object directly without stringifying
                         // Let BaseEntity handle serialization when saving to database
                         // This ensures proper escaping of embedded code/scripts
@@ -299,25 +313,25 @@ export class SyncEngine {
             const lookupStr = ExtractKeywordValue(value);
             // Parse lookup with optional flags: ?create, ?allowDefer
             // Format: EntityName.Field1=Value1&Field2=Value2?create&allowDefer&OtherField=Value
-            const entityMatch = lookupStr.match(/^([^.]+)\./);
-            if (!entityMatch) {
+            const split = SplitLookupReference(lookupStr);
+            if (!split) {
                 throw new Error(`Invalid lookup format: ${value}`);
             }
-            const entityName = entityMatch[1];
-            const remaining = lookupStr.substring(entityName.length + 1);
-            // Split into lookup part and flags part
-            const questionMarkIndex = remaining.indexOf('?');
-            const lookupPart = questionMarkIndex >= 0 ? remaining.substring(0, questionMarkIndex) : remaining;
-            const flagsPart = questionMarkIndex >= 0 ? remaining.substring(questionMarkIndex + 1) : '';
-            // Parse flags from the query string portion
-            const flags = new Set(flagsPart.split('&').map(f => f.split('=')[0].toLowerCase()));
-            const hasCreate = flags.has('create');
-            const allowDefer = flags.has('allowdefer');
+            if (split.UnknownBareFlags.includes('optional')) {
+                throw new Error(`Invalid @lookup reference: ${value}. ${LOOKUP_OPTIONAL_MESSAGE}`);
+            }
+            const entityName = split.EntityName;
+            const lookupPart = split.CriteriaText;
+            const flagsPart = split.FlagsText;
+            const hasCreate = split.FlagNames.has('create');
+            const allowDefer = split.FlagNames.has('allowdefer');
             // Parse all lookup fields (can be multiple with &)
             const lookupFields = [];
             const lookupPairs = lookupPart.split('&');
             // Collector for nested resolutions within this lookup
             const nestedResolutions = [];
+            // Criteria and create fields whose values came from @env:, for the lookup messages
+            const lookupEnvFields = new Map();
             for (const pair of lookupPairs) {
                 const fieldMatch = pair.match(/^(.+?)=(.+)$/);
                 if (!fieldMatch) {
@@ -326,15 +340,16 @@ export class SyncEngine {
                 const [, lookupFieldName, fieldValue] = fieldMatch;
                 const rawFieldValue = fieldValue.trim();
                 // Recursively process the field value to resolve any nested @ commands
-                // Create a temporary collector to capture nested resolutions
-                const nestedCollector = { notes: [], fieldPrefix: '' };
+                // Create a temporary collector to capture nested resolutions and env-sourced criteria
+                const nestedCollector = { notes: [], fieldPrefix: '', EnvFields: new Map() };
                 const processedValue = await this.ProcessFieldValue(rawFieldValue, baseDir, parentRecord, rootRecord, depth + 1, batchContext, nestedCollector, lookupFieldName, // Pass field name for tracking
                 recordProvider);
+                const criteriaVariables = this.addLookupEnvField(lookupEnvFields, lookupFieldName.trim(), nestedCollector);
                 // If the raw value was a lookup expression that got resolved, track it as nested
                 if (rawFieldValue.startsWith(METADATA_KEYWORDS.LOOKUP) && rawFieldValue !== processedValue) {
                     nestedResolutions.push({
                         expression: rawFieldValue,
-                        resolved: String(processedValue)
+                        resolved: criteriaVariables ? EnvSourceLabel(criteriaVariables) : String(processedValue)
                     });
                 }
                 lookupFields.push({ fieldName: lookupFieldName.trim(), fieldValue: processedValue });
@@ -346,28 +361,22 @@ export class SyncEngine {
             // These are key=value pairs after the flags (e.g., ?create&Description=Some%20Value)
             let createFields = {};
             if (hasCreate && flagsPart) {
-                const flagPairs = flagsPart.split('&');
-                for (const pair of flagPairs) {
-                    // Skip known flags
-                    if (pair.toLowerCase() === 'create' || pair.toLowerCase() === 'allowdefer') {
-                        continue;
-                    }
-                    const [key, val] = pair.split('=');
-                    if (key && val) {
-                        const decodedVal = decodeURIComponent(val);
-                        // Recursively process the field value to resolve any nested @ commands
-                        createFields[key] = await this.ProcessFieldValue(decodedVal, baseDir, parentRecord, rootRecord, depth + 1, batchContext, undefined, undefined, recordProvider);
-                    }
+                for (const createField of ParseLookupCreateFields(flagsPart)) {
+                    // Recursively process the field value to resolve any nested @ commands
+                    const createCollector = { notes: [], fieldPrefix: '', EnvFields: new Map() };
+                    createFields[createField.Name] = await this.ProcessFieldValue(createField.Value, baseDir, parentRecord, rootRecord, depth + 1, batchContext, createCollector, createField.Name, recordProvider);
+                    this.addLookupEnvField(lookupEnvFields, createField.Name, createCollector);
                 }
             }
-            const resolvedValue = await this.ResolveLookup(entityName, lookupFields, hasCreate, createFields, batchContext, allowDefer, value, recordProvider);
+            const resolvedValue = await this.ResolveLookup(entityName, lookupFields, hasCreate, createFields, batchContext, allowDefer, value, recordProvider, lookupEnvFields);
+            const sourceVariables = this.recordCopiedEnvValue(resolvedValue, resolutionCollector, fieldName);
             // Track the resolution if collector is provided
             if (resolutionCollector && fieldName) {
                 const note = {
                     type: 'lookup',
                     field: `${resolutionCollector.fieldPrefix}.${fieldName}`,
                     expression: value,
-                    resolved: String(resolvedValue)
+                    resolved: sourceVariables.length > 0 ? EnvSourceLabel(sourceVariables) : String(resolvedValue)
                 };
                 // Include nested resolutions if any were captured
                 if (nestedResolutions.length > 0) {
@@ -379,10 +388,23 @@ export class SyncEngine {
         }
         // Check for @env: reference
         if (value.startsWith(METADATA_KEYWORDS.ENV)) {
-            const envVar = ExtractKeywordValue(value);
-            const envValue = process.env[envVar];
+            const reference = ParseEnvReference(value);
+            if (!reference) {
+                throw new Error(`Invalid @env reference: ${value}`);
+            }
+            const envValue = process.env[reference.VariableName];
             if (envValue === undefined) {
-                throw new Error(`Environment variable not found: ${envVar}`);
+                if (reference.Optional) {
+                    throw new OptionalEnvMissingError(reference.VariableName);
+                }
+                const field = resolutionCollector?.fieldPrefix && fieldName ? `${resolutionCollector.fieldPrefix}.${fieldName}` : fieldName;
+                throw new EnvVariableNotSetError(reference.VariableName, field);
+            }
+            this.ResolvedEnvVariables.add(reference.VariableName);
+            if (resolutionCollector?.EnvFields && fieldName) {
+                const variables = resolutionCollector.EnvFields.get(fieldName) ?? new Set();
+                variables.add(reference.VariableName);
+                resolutionCollector.EnvFields.set(fieldName, variables);
             }
             return envValue;
         }
@@ -443,7 +465,9 @@ export class SyncEngine {
         });
         return `${encodeURIComponent(entityName.toLowerCase())}|${parts.join('&')}`;
     }
-    async ResolveLookup(entityName, lookupFields, autoCreate = false, createFields = {}, batchContext, allowDefer = false, originalValue, recordProvider) {
+    async ResolveLookup(entityName, lookupFields, autoCreate = false, createFields = {}, batchContext, allowDefer = false, originalValue, recordProvider, envFields) {
+        // Criteria and create fields whose values came from @env: print as labels
+        const envMask = new EnvDisplayMask(envFields);
         const lookupCacheKey = this.buildLookupCacheKey(entityName, lookupFields);
         if (this.syncMetadataEngine) {
             const cachedId = this.syncMetadataEngine.getCachedLookup(lookupCacheKey);
@@ -624,16 +648,18 @@ export class SyncEngine {
                 }
             }
             // Save the new record (new records are always dirty)
-            const filterDesc = lookupFields.map(({ fieldName, fieldValue }) => `${fieldName}='${fieldValue}'`).join(' AND ');
-            console.log(`📝 Auto-creating ${entityName} record where ${filterDesc}`);
+            console.log(`📝 Auto-creating ${entityName} record where ${this.describeLookupFilter(lookupFields, envMask)}`);
+            if (envFields) {
+                this.BeforeLookupCreateSave?.(newEntity, envFields);
+            }
             const saved = await newEntity.Save();
             if (!saved) {
                 const message = newEntity.LatestResult?.Message;
                 if (message) {
-                    throw new Error(`Failed to auto-create ${entityName}: ${message}`);
+                    throw new Error(`Failed to auto-create ${entityName}: ${envMask.MaskText(message)}`);
                 }
                 const errors = newEntity.LatestResult?.Errors?.map(err => typeof err === 'string' ? err : (err?.message || JSON.stringify(err)))?.join(', ') || 'Unknown error';
-                throw new Error(`Failed to auto-create ${entityName}: ${errors}`);
+                throw new Error(`Failed to auto-create ${entityName}: ${envMask.MaskText(errors)}`);
             }
             // Return the new ID
             if (entityInfo.PrimaryKeys.length > 0) {
@@ -647,16 +673,38 @@ export class SyncEngine {
             }
         }
         const filterDesc = lookupFields.map(({ fieldName, fieldValue }) => `${fieldName}='${fieldValue}'`).join(' AND ');
-        const errorMessage = `Lookup failed: No record found in '${entityName}' where ${filterDesc}`;
+        const errorMessage = `Lookup failed: No record found in '${entityName}' where ${this.describeLookupFilter(lookupFields, envMask)}`;
         // If allowDefer is true, throw DeferrableLookupError so caller can queue for retry
         if (allowDefer) {
             throw new DeferrableLookupError(errorMessage, entityName, lookupFields, originalValue || `@lookup:${entityName}.${filterDesc}`);
         }
         throw new Error(errorMessage);
     }
+    /** `Field='value' AND …` for messages; env-sourced criteria print as `Field=<from @env:VAR>`. */
+    describeLookupFilter(lookupFields, envMask) {
+        return lookupFields
+            .map(({ fieldName, fieldValue }) => `${fieldName}=${envMask.NewValue(fieldName, `'${fieldValue}'`)}`)
+            .join(' AND ');
+    }
+    /**
+     * Adds the variable names a lookup criterion or create field recorded in its scratch collector to the
+     * lookup's map. `@env:` values and `@parent:`/`@lookup:` results equal to an env value are recorded.
+     * @returns The field's variable names in the lookup's map, or undefined when it is not env-sourced
+     */
+    addLookupEnvField(lookupEnvFields, lookupFieldName, scratch) {
+        const recorded = new Set();
+        scratch.EnvFields?.forEach(names => names.forEach(name => recorded.add(name)));
+        if (recorded.size === 0) {
+            return undefined;
+        }
+        const merged = lookupEnvFields.get(lookupFieldName) ?? new Set();
+        recorded.forEach(name => merged.add(name));
+        lookupEnvFields.set(lookupFieldName, merged);
+        return merged;
+    }
     /** @deprecated Use {@link ResolveLookup}. */
-    async resolveLookup(entityName, lookupFields, autoCreate = false, createFields = {}, batchContext, allowDefer = false, originalValue, recordProvider) {
-        return this.ResolveLookup(entityName, lookupFields, autoCreate, createFields, batchContext, allowDefer, originalValue, recordProvider);
+    async resolveLookup(entityName, lookupFields, autoCreate = false, createFields = {}, batchContext, allowDefer = false, originalValue, recordProvider, envFields) {
+        return this.ResolveLookup(entityName, lookupFields, autoCreate, createFields, batchContext, allowDefer, originalValue, recordProvider, envFields);
     }
     /**
      * Build cascading defaults for a file path and process field values
@@ -667,17 +715,16 @@ export class SyncEngine {
      *
      * @param filePath - Path to the file being processed
      * @param entityConfig - Entity configuration containing base defaults
+     * @param resolutionCollector - Optional collector; when it has an `EnvFields` map, env-sourced defaults are recorded in it (no notes are added)
+     * @param skippedOptional - Optional map; receives field → variable for each default left out because its `?optional` variable is not set
      * @returns Processed defaults with all references resolved
      * @throws Error if any default value processing fails
      */
-    async BuildDefaults(filePath, entityConfig) {
-        const parts = path.dirname(filePath).split(path.sep);
+    async BuildDefaults(filePath, entityConfig, resolutionCollector, skippedOptional) {
         let defaults = { ...entityConfig.defaults };
-        // Walk up the directory tree building defaults
-        let currentPath = '';
-        for (const part of parts) {
-            currentPath = path.join(currentPath, part);
-            const folderConfig = await this.loadFolderConfig(currentPath);
+        // Folder defaults from the sync root down to the file's folder; deeper folders win
+        for (const dir of await this.folderChainFor(path.dirname(path.resolve(filePath)))) {
+            const folderConfig = await this.loadFolderConfig(dir);
             if (folderConfig?.defaults) {
                 defaults = { ...defaults, ...folderConfig.defaults };
             }
@@ -687,17 +734,126 @@ export class SyncEngine {
         const baseDir = path.dirname(filePath);
         for (const [field, value] of Object.entries(defaults)) {
             try {
-                processedDefaults[field] = await this.ProcessFieldValue(value, baseDir, null, null, 0);
+                const envCollector = this.envOnlyCollector(resolutionCollector);
+                processedDefaults[field] = await this.ProcessFieldValue(value, baseDir, null, null, 0, undefined, envCollector, field);
+                this.mergeEnvFields(resolutionCollector, envCollector);
             }
             catch (error) {
+                // A default whose optional variable is not set is left out
+                if (error instanceof OptionalEnvMissingError) {
+                    skippedOptional?.set(field, error.VariableName);
+                    continue;
+                }
                 throw new Error(`Failed to process default for field '${field}': ${error}`);
             }
         }
         return processedDefaults;
     }
     /** @deprecated Use {@link BuildDefaults}. */
-    async buildDefaults(filePath, entityConfig) {
-        return this.BuildDefaults(filePath, entityConfig);
+    async buildDefaults(filePath, entityConfig, resolutionCollector, skippedOptional) {
+        return this.BuildDefaults(filePath, entityConfig, resolutionCollector, skippedOptional);
+    }
+    /**
+     * The folders from the sync root (the nearest folder at or above `startDir` whose
+     * `.mj-sync.json` has no `entity`) down to `startDir`. Without a sync root, the chain
+     * starts at the file system root.
+     * The walk passes through folders without a `.mj-sync.json`, because record sub-folders have none (unlike `FindSyncRootDir`, which finds the settings root).
+     */
+    async folderChainFor(startDir) {
+        const chain = [];
+        let current = startDir;
+        for (;;) {
+            chain.push(current);
+            if (await this.isSyncRoot(current)) {
+                break;
+            }
+            const parent = path.dirname(current);
+            if (parent === current) {
+                break;
+            }
+            current = parent;
+        }
+        return chain.reverse();
+    }
+    /** True when `dir` has a readable `.mj-sync.json` with no (or an empty) `entity`. */
+    async isSyncRoot(dir) {
+        const configPath = path.join(dir, '.mj-sync.json');
+        if (!(await fs.pathExists(configPath))) {
+            return false;
+        }
+        try {
+            const config = await fs.readJson(configPath);
+            if (typeof config !== 'object' || config === null || Array.isArray(config)) {
+                return false;
+            }
+            return !('entity' in config) || !config.entity;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * A scratch collector that records `@env:` fields into its own map and drops notes, or undefined when the
+     * caller's collector has no map. {@link mergeEnvFields} copies its map into the caller's map on success.
+     */
+    envOnlyCollector(resolutionCollector) {
+        if (!resolutionCollector?.EnvFields) {
+            return undefined;
+        }
+        return { notes: [], fieldPrefix: resolutionCollector.fieldPrefix, EnvFields: new Map() };
+    }
+    /** Adds the variable names of a scratch collector to the caller's `EnvFields` map. */
+    mergeEnvFields(target, scratch) {
+        if (!target?.EnvFields || !scratch?.EnvFields) {
+            return;
+        }
+        for (const [field, variables] of scratch.EnvFields) {
+            const merged = target.EnvFields.get(field) ?? new Set();
+            variables.forEach(variable => merged.add(variable));
+            target.EnvFields.set(field, merged);
+        }
+    }
+    /**
+     * Finds the resolved variables whose current value is in a value copied by `@parent:` or `@owner:`, or returned by `@lookup:`
+     * (trimmed, case-insensitive): the whole value, or for values of 6 or more characters any part of it,
+     * plain or JSON-escaped. Records them under `fieldName` when the collector has an `EnvFields` map.
+     * @returns The matching variable names, sorted
+     */
+    recordCopiedEnvValue(copiedValue, resolutionCollector, fieldName) {
+        if (copiedValue === null || copiedValue === undefined) {
+            return [];
+        }
+        const copied = (typeof copiedValue === 'object' ? JSON.stringify(copiedValue) : String(copiedValue)).trim().toLowerCase();
+        if (copied === '') {
+            return [];
+        }
+        const names = [];
+        for (const name of this.ResolvedEnvVariables) {
+            const envValue = process.env[name];
+            if (envValue === undefined || envValue.trim() === '') {
+                continue;
+            }
+            if (this.holdsCopiedEnvValue(copied, envValue.trim().toLowerCase())) {
+                names.push(name);
+            }
+        }
+        names.sort();
+        if (names.length > 0 && resolutionCollector?.EnvFields && fieldName) {
+            const variables = resolutionCollector.EnvFields.get(fieldName) ?? new Set();
+            names.forEach(name => variables.add(name));
+            resolutionCollector.EnvFields.set(fieldName, variables);
+        }
+        return names;
+    }
+    /** True when the copied text is the variable value, or holds a value of at least `MIN_MASKED_VALUE_LENGTH` characters. */
+    holdsCopiedEnvValue(copied, envValue) {
+        if (copied === envValue) {
+            return true;
+        }
+        if (envValue.length < MIN_MASKED_VALUE_LENGTH) {
+            return false;
+        }
+        return copied.includes(envValue) || copied.includes(JSON.stringify(envValue).slice(1, -1));
     }
     /**
      * Load folder configuration from .mj-folder.json file
@@ -989,8 +1145,24 @@ export class SyncEngine {
                 }
                 return cached;
             }
-            return null;
+            // A cache miss is NOT proof the row is absent. The preload cache learns about rows saved
+            // during this push only through BaseEngine's save-event bus, and inside an open transaction
+            // that mutation is deferred to commit (rollback-safe engine caches) — and the default push is
+            // one transaction. So a record created earlier in this same push (e.g. by another directory)
+            // is missing here until commit. Confirm against the database on the push's own provider,
+            // which shares the transaction and sees its uncommitted rows. Don't cache the hit: a delegated
+            // slot is a donor engine's live array, and an uncommitted row placed there would outlive a
+            // rollback. The deferred event mutation populates the cache once the transaction commits.
+            return this.loadFromDatabase(entityName, entityInfo, primaryKey, recordProvider, false);
         }
+        return this.loadFromDatabase(entityName, entityInfo, primaryKey, recordProvider, true);
+    }
+    /**
+     * Loads a record by primary key straight from the database, using the record's provider when one
+     * is passed so the read joins that provider's transaction. Returns `null` when no row exists.
+     * `cacheResult` controls whether a hit is added to the preload cache.
+     */
+    async loadFromDatabase(entityName, entityInfo, primaryKey, recordProvider, cacheResult) {
         // First, check if the record exists using RunView to avoid "Error in BaseEntity.Load" messages
         // when records don't exist (which is a normal scenario during sync operations).
         // Use the graph provider when one is passed so an update does not replace a
@@ -1026,7 +1198,7 @@ export class SyncEngine {
         compositeKey.LoadFromSimpleObject(primaryKey);
         const loaded = await entity.InnerLoad(compositeKey);
         const loadedEntity = loaded ? entity : null;
-        if (loadedEntity && this.syncMetadataEngine) {
+        if (loadedEntity && cacheResult && this.syncMetadataEngine) {
             this.syncMetadataEngine.addEntityToCache(entityName, loadedEntity);
         }
         return loadedEntity;
@@ -1120,7 +1292,7 @@ export class SyncEngine {
      * @returns Promise resolving to processed JSON object
      * @private
      */
-    async processJsonFieldValues(obj, baseDir, parentRecord, rootRecord, depth = 0, batchContext, recordProvider, ownerRecord) {
+    async processJsonFieldValues(obj, baseDir, parentRecord, rootRecord, depth = 0, batchContext, recordProvider, ownerRecord, envCollector, envFieldName) {
         // Handle null and undefined
         if (obj === null || obj === undefined) {
             return obj;
@@ -1128,13 +1300,13 @@ export class SyncEngine {
         // Handle top-level strings (important for array elements that are strings with @ syntax)
         if (typeof obj === 'string') {
             if (IsMetadataKeyword(obj)) {
-                return this.ProcessFieldValue(obj, baseDir, parentRecord, rootRecord, depth, batchContext, undefined, undefined, recordProvider, ownerRecord);
+                return this.ProcessFieldValue(obj, baseDir, parentRecord, rootRecord, depth, batchContext, envCollector, envFieldName, recordProvider, ownerRecord);
             }
             return obj;
         }
         // Handle arrays
         if (Array.isArray(obj)) {
-            return Promise.all(obj.map(item => this.processJsonFieldValues(item, baseDir, parentRecord, rootRecord, depth, batchContext, recordProvider, ownerRecord)));
+            return Promise.all(obj.map(item => this.processJsonFieldValues(item, baseDir, parentRecord, rootRecord, depth, batchContext, recordProvider, ownerRecord, envCollector, envFieldName)));
         }
         // Handle objects
         if (typeof obj === 'object') {
@@ -1145,7 +1317,7 @@ export class SyncEngine {
                     // Check if this looks like a reference that needs processing
                     // Only process known reference types, ignore other @ strings (like npm packages)
                     if (IsMetadataKeyword(value)) {
-                        result[key] = await this.ProcessFieldValue(value, baseDir, parentRecord, rootRecord, depth, batchContext, undefined, undefined, recordProvider, ownerRecord);
+                        result[key] = await this.ProcessFieldValue(value, baseDir, parentRecord, rootRecord, depth, batchContext, envCollector, envFieldName, recordProvider, ownerRecord);
                     }
                     else {
                         result[key] = value;
@@ -1153,7 +1325,7 @@ export class SyncEngine {
                 }
                 else if (typeof value === 'object') {
                     // Recursively process nested objects
-                    result[key] = await this.processJsonFieldValues(value, baseDir, parentRecord, rootRecord, depth, batchContext, recordProvider, ownerRecord);
+                    result[key] = await this.processJsonFieldValues(value, baseDir, parentRecord, rootRecord, depth, batchContext, recordProvider, ownerRecord, envCollector, envFieldName);
                 }
                 else {
                     // Keep primitive values as-is