@memberjunction/metadata-sync 6.2.0-edge.4 → 6.2.0-edge.5

package/dist/services/ValidationService.js

@@ -1,10 +1,19 @@
 import { Metadata, RunView } from '@memberjunction/core';
 import { UUIDsEqual } from '@memberjunction/global';
 import * as fs from 'fs';
+import fse from 'fs-extra';
 import * as path from 'path';
 import { minimatch } from 'minimatch';
+import fastGlob from 'fast-glob';
+import { RecordFieldNames } from '../lib/entity-field-names.js';
+import { RECORD_FILE_GLOB_OPTIONS } from '../lib/record-file-glob.js';
 import { GetSystemUser } from '../lib/provider-utils.js';
+import { IsIgnoredDirectoryName } from '../lib/ignore-directories.js';
+import { MapJsonStartLines } from '../lib/json-record-lines.js';
 import { ParseMetadataReference } from '../lib/reference-parser.js';
+import { ParseEnvReference } from '../lib/env-reference.js';
+import { LOOKUP_OPTIONAL_MESSAGE } from '../lib/lookup-reference.js';
+import { EvaluatePushWhen, PushWhenError } from '../lib/push-when.js';
 import { METADATA_KEYWORDS, IsMetadataKeyword, ExtractKeywordValue } from '../constants/metadata-keywords.js';
 import { ResolveCollectionRelationship } from '@memberjunction/record-graph';
 export class ValidationService {
@@ -14,6 +23,10 @@ export class ValidationService {
         this.entityDependencies = new Map();
         this.processedEntities = new Set();
         this.claimedPrimaryKeys = new Map();
+        /** 1-based start line of each record object read from a record file. */
+        this.recordStartLines = new WeakMap();
+        /** The entity folder being validated. Push resolves record `@file:` references from it. */
+        this.entityDir = '';
         this.userRoleCache = new Map();
         this.metadata = new Metadata(); // global-provider-ok: metadata sync operates on the configured provider only
         this.options = {
@@ -48,13 +61,26 @@ export class ValidationService {
         if (config.userRoleValidation?.enabled) {
             await this.loadUserRoles();
         }
-        const directories = await this.getDirectoriesInOrder(dir, config);
+        if (!config.entity && config.pushWhen !== undefined) {
+            this.addError({
+                type: 'entity',
+                severity: 'error',
+                file: configPath,
+                message: 'pushWhen is only allowed in an entity folder .mj-sync.json',
+                suggestion: 'Move pushWhen into the .mj-sync.json of each folder it should gate',
+            });
+        }
+        // A config with an entity means `dir` is one entity folder, unless the caller treats it as a root
+        const isEntityFolder = !!config.entity && !this.options.ValidateChildFoldersOnly;
+        const candidates = isEntityFolder ? [''] : await this.getDirectoriesInOrder(dir, config);
+        const directories = candidates.filter((subDir) => this.pushWhenAllows(path.join(dir, subDir)));
         let totalFiles = 0;
         let totalEntities = 0;
         const fileResults = new Map();
         for (const subDir of directories) {
             const subDirPath = path.join(dir, subDir);
             const result = await this.validateEntityDirectory(subDirPath);
+            this.warnNestedPushWhen(subDirPath);
             if (result) {
                 totalFiles += result.files;
                 totalEntities += result.entities;
@@ -63,7 +89,6 @@ export class ValidationService {
                 }
             }
         }
-        // Validate dependency order
         await this.validateDependencyOrder(directories);
         return {
             isValid: this.errors.length === 0,
@@ -78,6 +103,35 @@ export class ValidationService {
             },
         };
     }
+    /** Warns about each `.mj-sync.json` below `dir` that has `pushWhen`; push over a parent folder ignores it. */
+    warnNestedPushWhen(dir) {
+        const configPaths = fastGlob.sync('*/**/.mj-sync.json', {
+            cwd: dir,
+            absolute: true,
+            dot: true,
+            ignore: ['**/node_modules/**'],
+            suppressErrors: true,
+        });
+        for (const configPath of configPaths) {
+            let config;
+            try {
+                config = fse.readJsonSync(configPath);
+            }
+            catch {
+                continue;
+            }
+            if (config === null || typeof config !== 'object' || Array.isArray(config) || !('pushWhen' in config)) {
+                continue;
+            }
+            this.addWarning({
+                type: 'validation',
+                severity: 'warning',
+                file: configPath,
+                message: 'pushWhen in a nested folder has no effect when push runs over a parent folder',
+                suggestion: 'Move the folder directly under the metadata root, or push it with --dir',
+            });
+        }
+    }
     /** @deprecated Use {@link ValidateDirectory}. */
     async validateDirectory(dir) {
         return this.ValidateDirectory(dir);
@@ -86,31 +140,12 @@ export class ValidationService {
      * Validates a single entity directory
      */
     async validateEntityDirectory(dir) {
-        // Check for .mj-folder.json first (new format)
-        let configPath = path.join(dir, '.mj-folder.json');
-        let config;
-        if (fs.existsSync(configPath)) {
-            config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
-            // .mj-folder.json uses entityName field
-            if (!config.entityName) {
-                this.addError({
-                    type: 'validation',
-                    severity: 'error',
-                    file: configPath,
-                    message: 'Missing entityName field in .mj-folder.json',
-                });
-                return null;
-            }
-            config.entity = config.entityName; // Normalize to entity field
-        }
-        else {
-            // Fall back to .mj-sync.json (legacy format)
-            configPath = path.join(dir, '.mj-sync.json');
-            if (!fs.existsSync(configPath)) {
-                return null;
-            }
-            config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+        // The folder's entity comes from .mj-sync.json; .mj-folder.json holds only defaults
+        const configPath = path.join(dir, '.mj-sync.json');
+        if (!fs.existsSync(configPath)) {
+            return null;
         }
+        const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
         // Validate entity name exists
         if (!config.entity || config.entity.trim() === '') {
             this.addError({
@@ -121,7 +156,8 @@ export class ValidationService {
             });
             return null;
         }
-        const entityInfo = this.metadata.EntityByName(config.entity);
+        const entityConfig = { ...config, entity: config.entity };
+        const entityInfo = this.metadata.EntityByName(entityConfig.entity);
         if (!entityInfo) {
             this.addError({
                 type: 'entity',
@@ -145,11 +181,12 @@ export class ValidationService {
                 }
             }
         }
+        this.entityDir = path.resolve(dir);
         const files = await this.getMatchingFiles(dir, config.filePattern);
         let totalEntities = 0;
         const fileResults = new Map();
         for (const file of files) {
-            const result = await this.validateFile(file, entityInfo, config);
+            const result = await this.validateFile(file, entityInfo, entityConfig);
             if (result) {
                 totalEntities += result.entityCount;
                 fileResults.set(file, result);
@@ -161,11 +198,11 @@ export class ValidationService {
      * Validates a single metadata file
      */
     async validateFile(filePath, entityInfo, config) {
-        const fileErrors = [];
         let entityCount = 0;
         try {
-            const content = fs.readFileSync(filePath, 'utf8');
+            const content = fs.readFileSync(filePath, 'utf8').replace(/^\uFEFF/, '');
             const data = JSON.parse(content);
+            this.recordStartLines = MapJsonStartLines(content, data);
             const entities = Array.isArray(data) ? data : [data];
             entityCount = entities.length;
             for (const entityData of entities) {
@@ -173,7 +210,7 @@ export class ValidationService {
             }
         }
         catch (error) {
-            fileErrors.push({
+            this.addError({
                 type: 'entity',
                 severity: 'error',
                 file: filePath,
@@ -222,24 +259,8 @@ export class ValidationService {
                 });
             }
         }
-        // Check for double ownership across metadata files (§6)
-        if (entityData.primaryKey && Object.keys(entityData.primaryKey).length > 0) {
-            const claimKey = `${entityInfo.Name}:${JSON.stringify(entityData.primaryKey)}`;
-            const existingFile = this.claimedPrimaryKeys.get(claimKey);
-            if (existingFile && existingFile !== filePath) {
-                this.addError({
-                    type: 'field',
-                    severity: 'error',
-                    entity: entityInfo.Name,
-                    file: filePath,
-                    message: `Record with primaryKey ${JSON.stringify(entityData.primaryKey)} on entity "${entityInfo.Name}" is already claimed by "${existingFile}". Double ownership is not allowed.`,
-                    suggestion: 'Ensure records are owned by only one sync file or collection',
-                });
-            }
-            else {
-                this.claimedPrimaryKeys.set(claimKey, filePath);
-            }
-        }
+        this.validatePrimaryKeyEnvReferences(entityData.primaryKey, entityInfo.Name, filePath);
+        this.checkRecordClaim(entityData, entityInfo.Name, filePath);
         // Skip validation for deletion records - they don't need field validation or reference checks
         if (entityData.deleteRecord?.delete === true) {
             // Only validate that primaryKey exists for deletion records
@@ -285,6 +306,7 @@ export class ValidationService {
         // block) so the required-field best-practice check can be limited to NEW
         // records — an existing record already holds the value in the DB, so a
         // missing NOT-NULL field in the file is not a problem and would only be noise.
+        this.reportTemplateValues(entityData.fields, entityData, entityInfo.Name, filePath);
         await this.validateFields(entityData.fields, entityInfo, filePath, parentContext, !!entityData.primaryKey, ownerContext);
         // Track dependencies
         this.trackEntityDependencies(entityData, entityInfo.Name, filePath);
@@ -360,9 +382,10 @@ export class ValidationService {
      */
     async validateFields(fields, entityInfo, filePath, parentContext, isExistingRecord = false, ownerContext) {
         const entityFields = entityInfo.Fields;
-        const fieldMap = new Map(entityFields.map((f) => [f.Name, f]));
+        // Field names match the entity's fields without regard to case, as push matches them
+        const fieldMap = new Map(entityFields.map((f) => [f.Name.toLowerCase(), f]));
         for (const [fieldName, fieldValue] of Object.entries(fields)) {
-            const fieldInfo = fieldMap.get(fieldName);
+            const fieldInfo = fieldMap.get(fieldName.toLowerCase());
             if (!fieldInfo) {
                 // Check if this might be a virtual property (getter/setter)
                 try {
@@ -399,8 +422,24 @@ export class ValidationService {
                     continue;
                 }
             }
-            // Check if field is settable (not system field)
-            if (fieldInfo.ReadOnly || fieldName.startsWith('__mj_')) {
+            const isSystemField = fieldInfo.Name.startsWith('__mj_');
+            // A read-only field that push ignores (such as a joined display field that pull writes) is a warning
+            if (fieldInfo.ReadOnly && !fieldInfo.IsPrimaryKey && !isSystemField) {
+                const pullAdvice = fieldInfo.IsVirtual
+                    ? 'set pull.ignoreVirtualFields: true to stop pull writing it'
+                    : 'add it to pull.excludeFields to stop pull writing it';
+                this.addWarning({
+                    type: 'validation',
+                    severity: 'warning',
+                    entity: entityInfo.Name,
+                    field: fieldName,
+                    file: filePath,
+                    message: `Field "${fieldName}" is read-only: push ignores it; ${pullAdvice}`,
+                });
+                continue;
+            }
+            // A system field or the primary key cannot be set in fields
+            if (fieldInfo.ReadOnly || isSystemField) {
                 this.addError({
                     type: 'field',
                     severity: 'error',
@@ -422,9 +461,10 @@ export class ValidationService {
         // pure noise that masks genuine warnings. New records (no primaryKey) still
         // get the full check since their required fields must be supplied to insert.
         if (this.options.checkBestPractices && !isExistingRecord) {
+            const recordFields = new RecordFieldNames(fields);
             for (const field of entityFields) {
                 // Skip if field allows null or has a value already (use 'in' to handle falsy values like 0, false, "")
-                if (field.AllowsNull || field.Name in fields) {
+                if (field.AllowsNull || recordFields.Has(field.Name)) {
                     continue;
                 }
                 // Skip if field has a default value
@@ -577,6 +617,107 @@ export class ValidationService {
             }
         }
     }
+    /** Reports each `@template:` value in a record's fields. Push saves that text unchanged instead of loading a file. */
+    reportTemplateValues(fields, record, entityName, filePath) {
+        const line = this.recordStartLines.get(record);
+        for (const [fieldName, value] of Object.entries(fields)) {
+            if (typeof value === 'string' && value.startsWith(METADATA_KEYWORDS.TEMPLATE)) {
+                this.addTemplateError(value, entityName, fieldName, filePath, line);
+            }
+        }
+    }
+    addTemplateError(value, entityName, fieldName, filePath, line) {
+        const location = line === undefined ? filePath : `${filePath}:${line}`;
+        this.addError({
+            type: 'reference',
+            severity: 'error',
+            entity: entityName,
+            field: fieldName,
+            file: filePath,
+            message: `@template: is not supported by mj sync push; push would save the literal text "${value}" to ${entityName}.${fieldName} ` +
+                `at ${location}. Use @file: instead.`,
+        });
+    }
+    /**
+     * Records an entry for its entity and primary key. Several entries may describe the same record,
+     * but two entries that set the same field to different raw values give an error.
+     * A key that holds an `@parent:`, `@owner:` or `@root:` reference anywhere, or an object value, is not
+     * recorded.
+     */
+    checkRecordClaim(entityData, entityName, filePath) {
+        if (!entityData.primaryKey || Object.keys(entityData.primaryKey).length === 0 || this.hasRelativeKeyValue(entityData.primaryKey)) {
+            return;
+        }
+        const fields = entityData.fields;
+        const claim = {
+            Path: filePath,
+            Line: this.recordStartLines.get(entityData),
+            Fields: fields && typeof fields === 'object' && !Array.isArray(fields) ? fields : {},
+        };
+        const claimKey = `${entityName}:${JSON.stringify(entityData.primaryKey)}`;
+        const claims = this.claimedPrimaryKeys.get(claimKey) ?? [];
+        for (const earlier of claims) {
+            const conflictingFields = Object.keys(claim.Fields).filter((field) => Object.prototype.hasOwnProperty.call(earlier.Fields, field) && !this.jsonValuesEqual(earlier.Fields[field], claim.Fields[field]));
+            if (conflictingFields.length > 0) {
+                this.addClaimConflictError(entityName, entityData.primaryKey, conflictingFields, earlier, claim);
+            }
+        }
+        claims.push(claim);
+        this.claimedPrimaryKeys.set(claimKey, claims);
+    }
+    /**
+     * True when a key value can depend on the parent, owner or root record, so equal key text can name
+     * different records: it holds one of these references (also inside a `@lookup:`), or it is an object.
+     */
+    hasRelativeKeyValue(primaryKey) {
+        const relativePrefixes = [METADATA_KEYWORDS.PARENT, METADATA_KEYWORDS.OWNER, METADATA_KEYWORDS.ROOT];
+        return Object.values(primaryKey).some((value) => (typeof value === 'object' && value !== null) ||
+            (typeof value === 'string' && relativePrefixes.some((prefix) => value.includes(prefix))));
+    }
+    /** Adds the error for two entries of one record that set fields to different values. */
+    addClaimConflictError(entityName, primaryKey, conflictingFields, earlier, later) {
+        const earlierLocation = this.formatClaimLocation(earlier);
+        const laterLocation = this.formatClaimLocation(later);
+        const fieldText = conflictingFields
+            .map((field) => `field "${field}" is ${this.formatClaimValue(earlier.Fields[field])} at ${earlierLocation} and ${this.formatClaimValue(later.Fields[field])} at ${laterLocation}`)
+            .join('; ');
+        this.addError({
+            type: 'field',
+            severity: 'error',
+            entity: entityName,
+            field: conflictingFields.length === 1 ? conflictingFields[0] : undefined,
+            file: later.Path,
+            message: `Conflicting values for ${entityName} record ${JSON.stringify(primaryKey)}: ${fieldText}. ` +
+                'Make both entries set the same value, or set the field in one entry only.',
+        });
+    }
+    formatClaimLocation(claim) {
+        return claim.Line === undefined ? claim.Path : `${claim.Path}:${claim.Line}`;
+    }
+    /** A raw field value as JSON text, cut to 80 characters. */
+    formatClaimValue(value) {
+        const text = JSON.stringify(value) ?? String(value);
+        return text.length > 80 ? `${text.slice(0, 77)}...` : text;
+    }
+    /** Deep equality of two parsed JSON values. Object key order does not matter. */
+    jsonValuesEqual(left, right) {
+        if (left === right) {
+            return true;
+        }
+        if (Array.isArray(left) || Array.isArray(right)) {
+            return (Array.isArray(left) &&
+                Array.isArray(right) &&
+                left.length === right.length &&
+                left.every((item, index) => this.jsonValuesEqual(item, right[index])));
+        }
+        if (left === null || right === null || typeof left !== 'object' || typeof right !== 'object') {
+            return false;
+        }
+        const leftEntries = Object.entries(left);
+        const rightValues = new Map(Object.entries(right));
+        return (leftEntries.length === rightValues.size &&
+            leftEntries.every(([key, value]) => rightValues.has(key) && this.jsonValuesEqual(value, rightValues.get(key))));
+    }
     /**
      * Check if a string is actually a MetadataSync reference (not just any @ string)
      */
@@ -601,13 +742,13 @@ export class ValidationService {
         }
         switch (parsed.type) {
             case METADATA_KEYWORDS.FILE:
-                await this.validateFileReference(parsed.value, filePath, entityInfo.Name, fieldInfo.Name);
+                await this.validateFileReference(parsed.value, filePath, this.entityDir, entityInfo.Name, fieldInfo.Name);
                 break;
             case METADATA_KEYWORDS.LOOKUP:
                 await this.validateLookupReference(parsed, filePath, entityInfo.Name, fieldInfo.Name);
                 break;
             case METADATA_KEYWORDS.TEMPLATE:
-                await this.validateTemplateReference(parsed.value, filePath, entityInfo.Name, fieldInfo.Name);
+                // Reported by reportTemplateValues: push does not load @template: files
                 break;
             case METADATA_KEYWORDS.OWNER:
                 this.validateOwnerReference(parsed.value, ownerContext, filePath, entityInfo.Name, fieldInfo.Name);
@@ -630,11 +771,11 @@ export class ValidationService {
         return ParseMetadataReference(reference);
     }
     /**
-     * Validates @file: references
+     * Validates @file: references. `baseDir` is the folder push resolves the path from: the entity
+     * folder for a record field, or the folder of the JSON file that holds the reference.
      */
-    async validateFileReference(filePath, sourceFile, entityName, fieldName, visitedFiles) {
-        const dir = path.dirname(sourceFile);
-        const resolvedPath = path.resolve(dir, filePath);
+    async validateFileReference(filePath, sourceFile, baseDir, entityName, fieldName, visitedFiles) {
+        const resolvedPath = path.resolve(baseDir, filePath);
         // Initialize visited files set if not provided (for circular reference detection)
         const visited = visitedFiles || new Set();
         // Check for circular references
@@ -707,6 +848,17 @@ export class ValidationService {
      * Validates @lookup: references
      */
     async validateLookupReference(parsed, sourceFile, entityName, fieldName) {
+        if (parsed.UnknownFlags?.includes('optional')) {
+            this.addError({
+                type: 'reference',
+                severity: 'error',
+                entity: entityName,
+                field: fieldName,
+                file: sourceFile,
+                message: LOOKUP_OPTIONAL_MESSAGE,
+                suggestion: 'Remove ?optional. Use @env:VAR?optional only as a whole field value.',
+            });
+        }
         const lookupEntity = this.metadata.EntityByName(parsed.entity);
         if (!lookupEntity) {
             this.addError({
@@ -755,40 +907,6 @@ export class ValidationService {
         // Track dependency
         this.addEntityDependency(entityName, parsed.entity);
     }
-    /**
-     * Validates @template: references
-     */
-    async validateTemplateReference(templatePath, sourceFile, entityName, fieldName) {
-        const dir = path.dirname(sourceFile);
-        const resolvedPath = path.resolve(dir, templatePath);
-        if (!fs.existsSync(resolvedPath)) {
-            this.addError({
-                type: 'reference',
-                severity: 'error',
-                entity: entityName,
-                field: fieldName,
-                file: sourceFile,
-                message: `Template file not found: "${templatePath}"`,
-                suggestion: `Create template at: ${resolvedPath}`,
-            });
-            return;
-        }
-        // Validate template is valid JSON
-        try {
-            JSON.parse(fs.readFileSync(resolvedPath, 'utf8'));
-        }
-        catch (error) {
-            this.addError({
-                type: 'reference',
-                severity: 'error',
-                entity: entityName,
-                field: fieldName,
-                file: sourceFile,
-                message: `Template file is not valid JSON: "${templatePath}"`,
-                details: error instanceof Error ? error.message : String(error),
-            });
-        }
-    }
     /**
      * Validates @parent: references
      */
@@ -936,10 +1054,12 @@ export class ValidationService {
      * Get directories in order based on config
      */
     async getDirectoriesInOrder(rootDir, config) {
+        // The folders push visits: no dot folders, none that match ignoreDirectories
         const allDirs = fs
             .readdirSync(rootDir)
             .filter((f) => fs.statSync(path.join(rootDir, f)).isDirectory())
-            .filter((d) => !d.startsWith('.'));
+            .filter((d) => !d.startsWith('.'))
+            .filter((d) => !IsIgnoredDirectoryName(d, config.ignoreDirectories));
         let orderedDirs;
         if (config.directoryOrder && Array.isArray(config.directoryOrder)) {
             const ordered = config.directoryOrder.filter((d) => allDirs.includes(d));
@@ -952,6 +1072,61 @@ export class ValidationService {
         // Apply include/exclude filters if specified
         return this.applyDirectoryFilters(orderedDirs);
     }
+    /**
+     * Checks a folder's pushWhen and reports a malformed value or an unreadable
+     * .mj-sync.json as an error. Returns false when the config cannot be read, or when
+     * HonorPushWhen is set and push would skip the folder.
+     */
+    pushWhenAllows(dirPath) {
+        const configPath = path.join(dirPath, '.mj-sync.json');
+        if (!fs.existsSync(configPath)) {
+            return true;
+        }
+        let config;
+        try {
+            config = fse.readJsonSync(configPath);
+        }
+        catch (error) {
+            this.addError({
+                type: 'entity',
+                severity: 'error',
+                file: configPath,
+                message: `Cannot read .mj-sync.json: ${this.readErrorReason(error, configPath)}`,
+            });
+            return false;
+        }
+        if (typeof config !== 'object' || config === null || Array.isArray(config)) {
+            this.addError({
+                type: 'entity',
+                severity: 'error',
+                file: configPath,
+                message: '.mj-sync.json must contain a JSON object',
+            });
+            return false;
+        }
+        const pushWhen = 'pushWhen' in config ? config.pushWhen : undefined;
+        if (pushWhen === undefined) {
+            return true;
+        }
+        const problem = PushWhenError(pushWhen);
+        if (problem) {
+            this.addError({
+                type: 'entity',
+                severity: 'error',
+                file: configPath,
+                message: problem,
+                suggestion: 'Use "pushWhen": "@env:VARIABLE"',
+            });
+            return true;
+        }
+        return !this.options.HonorPushWhen || EvaluatePushWhen(pushWhen).Push;
+    }
+    /** The reason from a read error, without the leading `<path>: ` that fs-extra adds. */
+    readErrorReason(error, filePath) {
+        const message = error instanceof Error ? error.message : String(error);
+        const prefix = `${filePath}: `;
+        return message.startsWith(prefix) ? message.slice(prefix.length) : message;
+    }
     /**
      * Apply include/exclude filters to directory list
      */
@@ -971,21 +1146,10 @@ export class ValidationService {
         }
         return filteredDirs;
     }
-    /**
-     * Get files matching pattern
-     */
+    /** The record files of an entity folder, found the way push finds them, as sorted absolute paths. */
     async getMatchingFiles(dir, pattern) {
-        const files = fs.readdirSync(dir).filter((f) => fs.statSync(path.join(dir, f)).isFile());
-        // Strip leading **/ from glob patterns (we only match in current directory)
-        const normalizedPattern = pattern.replace(/^\*\*\//, '');
-        // Simple glob pattern matching
-        if (normalizedPattern === '*.json') {
-            return files.filter((f) => f.endsWith('.json') && !f.startsWith('.mj-'));
-        }
-        else if (normalizedPattern === '.*.json') {
-            return files.filter((f) => f.startsWith('.') && f.endsWith('.json') && !f.startsWith('.mj-'));
-        }
-        return files;
+        const files = await fastGlob(pattern || '*.json', { ...RECORD_FILE_GLOB_OPTIONS, cwd: dir });
+        return files.sort();
     }
     /**
      * Add an error
@@ -1070,6 +1234,7 @@ export class ValidationService {
         this.entityDependencies.clear();
         this.processedEntities.clear();
         this.claimedPrimaryKeys.clear();
+        this.recordStartLines = new WeakMap();
         this.userRoleCache.clear();
     }
     /**
@@ -1302,6 +1467,63 @@ export class ValidationService {
             });
         }
     }
+    /** Checks an `@env:` reference inside a JSON file: its syntax, and whether a required variable is set. */
+    validateNestedEnvReference(value, sourceFile, entityName, key) {
+        const reference = ParseEnvReference(value);
+        if (!reference) {
+            this.addError({
+                type: 'reference',
+                severity: 'error',
+                entity: entityName,
+                field: key,
+                file: sourceFile,
+                message: `Invalid reference format: "${value}"`,
+            });
+            return;
+        }
+        if (!reference.Optional && !process.env[reference.VariableName]) {
+            this.addWarning({
+                type: 'validation',
+                severity: 'warning',
+                entity: entityName,
+                field: key,
+                file: sourceFile,
+                message: `Environment variable "${reference.VariableName}" is not currently set`,
+                suggestion: `Ensure this variable is set before running push operations`,
+            });
+        }
+    }
+    /** Reports a primary key `@env:` reference that is malformed or uses `?optional`; push needs the value to find the record. */
+    validatePrimaryKeyEnvReferences(primaryKey, entityName, filePath) {
+        for (const [field, value] of Object.entries(primaryKey ?? {})) {
+            if (typeof value !== 'string' || !value.startsWith(METADATA_KEYWORDS.ENV)) {
+                continue;
+            }
+            const reference = ParseEnvReference(value);
+            if (!reference) {
+                this.addError({
+                    type: 'reference',
+                    severity: 'error',
+                    entity: entityName,
+                    field,
+                    file: filePath,
+                    message: `Invalid reference format: "${value}"`,
+                    suggestion: 'Use @env:VARIABLE',
+                });
+            }
+            else if (reference.Optional) {
+                this.addError({
+                    type: 'reference',
+                    severity: 'error',
+                    entity: entityName,
+                    field,
+                    file: filePath,
+                    message: `Primary key field "${field}" cannot use ?optional: ${value}`,
+                    suggestion: 'Remove ?optional from the primary key reference',
+                });
+            }
+        }
+    }
     /**
      * Recursively validates all @ references in a JSON structure
      */
@@ -1321,7 +1543,7 @@ export class ValidationService {
                     if (value.startsWith(METADATA_KEYWORDS.FILE)) {
                         const filePath = ExtractKeywordValue(value);
                         // Recursively validate the file reference (with circular detection)
-                        await this.validateFileReference(filePath, sourceFile, entityName, key, visitedFiles);
+                        await this.validateFileReference(filePath, sourceFile, path.dirname(sourceFile), entityName, key, visitedFiles);
                     }
                     else if (value.startsWith(METADATA_KEYWORDS.LOOKUP)) {
                         // Parse and validate lookup reference
@@ -1331,8 +1553,7 @@ export class ValidationService {
                         }
                     }
                     else if (value.startsWith(METADATA_KEYWORDS.TEMPLATE)) {
-                        const templatePath = ExtractKeywordValue(value);
-                        await this.validateTemplateReference(templatePath, sourceFile, entityName, key);
+                        this.addTemplateError(value, entityName, key, sourceFile);
                     }
                     else if (value.startsWith(METADATA_KEYWORDS.PARENT)) {
                         const parsed = this.parseReference(value);
@@ -1353,18 +1574,7 @@ export class ValidationService {
                         }
                     }
                     else if (value.startsWith(METADATA_KEYWORDS.ENV)) {
-                        const envVar = ExtractKeywordValue(value);
-                        if (!process.env[envVar]) {
-                            this.addWarning({
-                                type: 'validation',
-                                severity: 'warning',
-                                entity: entityName,
-                                field: key,
-                                file: sourceFile,
-                                message: `Environment variable "${envVar}" is not currently set`,
-                                suggestion: `Ensure this variable is set before running push operations`,
-                            });
-                        }
+                        this.validateNestedEnvReference(value, sourceFile, entityName, key);
                     }
                 }
                 else if (value && typeof value === 'object') {
@@ -1516,7 +1726,7 @@ export class ValidationService {
                 });
                 return;
             }
-            await this.validateExtensionFields(extObj.fields, parentEntityInfo, childEntityInfo, filePath, config, depth);
+            await this.validateExtensionFields(extObj.fields, extObj, parentEntityInfo, childEntityInfo, filePath, config, depth);
             return;
         }
         // Case 2: Map form where keys are subtype entity names (e.g. { "Event Order Lines": { fields: { ... } } })
@@ -1558,14 +1768,14 @@ export class ValidationService {
             }
             const subRecord = subVal;
             if ('fields' in subRecord && typeof subRecord.fields === 'object' && subRecord.fields !== null) {
-                await this.validateExtensionFields(subRecord.fields, parentEntityInfo, childEntityInfo, filePath, config, depth);
+                await this.validateExtensionFields(subRecord.fields, subRecord, parentEntityInfo, childEntityInfo, filePath, config, depth);
             }
         }
     }
     /**
      * Validates leaf fields within an extension block (§4.1, §4.5, §9)
      */
-    async validateExtensionFields(fields, parentEntityInfo, childEntityInfo, filePath, config, depth) {
+    async validateExtensionFields(fields, extensionRecord, parentEntityInfo, childEntityInfo, filePath, config, depth) {
         for (const fieldName of Object.keys(fields)) {
             const isParentField = parentEntityInfo.Fields.some((f) => f.Name.toLowerCase() === fieldName.toLowerCase());
             if (isParentField) {
@@ -1580,6 +1790,7 @@ export class ValidationService {
                 });
             }
         }
+        this.reportTemplateValues(fields, extensionRecord, childEntityInfo.Name, filePath);
         await this.validateFields(fields, childEntityInfo, filePath, { entity: parentEntityInfo.Name, field: 'extension' }, false, { entity: parentEntityInfo.Name });
     }
     /**