@memberjunction/metadata-sync 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/services/PullService.js

@@ -7,6 +7,24 @@ import { configManager } from '../lib/config-manager.js';
 import { FileWriteBatch } from '../lib/file-write-batch.js';
 import { RecordProcessor } from '../lib/RecordProcessor.js';
 import { createPrimaryKeyLookup, extractPrimaryKeyValues } from '../lib/record-primary-key.js';
+import { DescribeMissingEntitySubclass } from '../lib/entity-subclass-guard.js';
+import { ApplyPullOverrides } from '../lib/pull-option-overrides.js';
+import { ExistingRecordIndex, UnsetPrimaryKeyVariableError } from '../lib/existing-record-index.js';
+/**
+ * Resolves `pull.backupDirectory` against the directory of the file being backed up (default
+ * `.backups`). An absolute path is taken as relative to that directory, as `path.join` always has.
+ *
+ * @throws when the result is outside that directory (`../..`): pull must not write copies of
+ *         metadata files elsewhere on disk.
+ */
+function resolveBackupDirectory(fileDir, backupDirName) {
+    const backupDir = path.join(fileDir, backupDirName || '.backups');
+    const relative = path.relative(fileDir, backupDir);
+    if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+        throw new Error(`pull.backupDirectory '${backupDirName}' points outside ${fileDir}. It must be a directory inside the folder of the files it backs up.`);
+    }
+    return backupDir;
+}
 /** Validates that a string is a well-formed ISO 8601 timestamp. */
 function isValidISOTimestamp(value) {
     return /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:?\d{2})?)?$/.test(value);
@@ -15,6 +33,8 @@ export class PullService {
     constructor(syncEngine, contextUser, stateManager) {
         this.createdBackupFiles = [];
         this.createdBackupDirs = new Set();
+        /** Backup path → the file it copies, so a rollback restores to the right place for any backupDirectory. */
+        this.backupSources = new Map();
         this.syncEngine = syncEngine;
         this.contextUser = contextUser;
         this.fileWriteBatch = new FileWriteBatch();
@@ -38,6 +58,8 @@ export class PullService {
         this.fileWriteBatch.clear();
         this.createdBackupFiles = [];
         this.createdBackupDirs.clear();
+        this.backupSources.clear();
+        this.recordProcessor.WarningSink = (message) => callbacks?.onWarn?.(`⚠️  ${message}`);
         let targetDir;
         let entityConfig;
         // Check if we should use a specific target directory
@@ -75,6 +97,7 @@ export class PullService {
                 throw new Error(`Invalid entity configuration in ${targetDir}`);
             }
         }
+        entityConfig = ApplyPullOverrides(entityConfig, this.pullOverridesFor(options));
         // Show configuration notice only if relevant and in verbose mode
         if (options.verbose && entityConfig?.pull?.appendRecordsToExistingFile && entityConfig?.pull?.newFileName) {
             const targetFile = path.join(targetDir, entityConfig.pull.newFileName.endsWith('.json')
@@ -84,6 +107,12 @@ export class PullService {
                 callbacks?.onLog?.(`\n📝 Configuration: New records will be appended to existing file '${path.basename(targetFile)}'`);
             }
         }
+        // Records load through the ClassFactory; without the entity's own subclass they arrive as a
+        // generic BaseEntity. Pull stays correct (keys are read through Get()), but say so once.
+        const subclassWarning = DescribeMissingEntitySubclass(options.entity, { Operation: 'pull', DryRun: options.dryRun });
+        if (subclassWarning) {
+            callbacks?.onWarn?.(`⚠️  ${subclassWarning}`);
+        }
         // Pull records
         callbacks?.onProgress?.(`Pulling ${options.entity} records`);
         const rv = new RunView();
@@ -132,9 +161,14 @@ export class PullService {
         // Process records with error handling and rollback
         let pullResult;
         try {
-            pullResult = await this.processRecords(result.Results, options, targetDir, entityConfig, callbacks);
+            pullResult = await this.processRecords(result.Results, options, targetDir, entityConfig, callbacks, sinceTimestamp);
             // Write all batched file changes at once
             if (!options.dryRun) {
+                // Back up every existing file this pull is about to rewrite — updated and appended-to
+                // alike — before anything is written, so a failed pull can restore all of them.
+                if (entityConfig.pull?.backupBeforeUpdate) {
+                    await this.backupPendingFiles(entityConfig.pull.backupDirectory, options.verbose, callbacks);
+                }
                 const filesWritten = await this.fileWriteBatch.flush();
                 if (options.verbose && filesWritten > 0) {
                     callbacks?.onSuccess?.(`Wrote ${filesWritten} files with consistent property ordering`);
@@ -142,7 +176,7 @@ export class PullService {
             }
             // Operation succeeded - clean up backup files
             if (!options.dryRun) {
-                await this.CleanupBackupFiles();
+                await this.cleanupBackupFiles();
             }
         }
         catch (error) {
@@ -150,8 +184,10 @@ export class PullService {
             // Attempt to rollback file changes if not in dry run mode
             if (!options.dryRun) {
                 try {
-                    await this.rollbackFileChanges(callbacks);
-                    callbacks?.onWarn?.('File changes have been rolled back due to operation failure');
+                    const restored = await this.rollbackFileChanges(callbacks);
+                    if (restored > 0) {
+                        callbacks?.onWarn?.('File changes have been rolled back due to operation failure');
+                    }
                 }
                 catch (rollbackError) {
                     callbacks?.onError?.(`Rollback failed: ${rollbackError.message || rollbackError}`);
@@ -162,7 +198,8 @@ export class PullService {
         // Incremental pull: detect soft-deleted records and remove their local files.
         // Records deleted via __mj_DeletedAt won't appear in the UpdatedAt query,
         // so we need a separate check.
-        if (sinceTimestamp && !options.dryRun) {
+        // A multi-file pull removes soft-deleted entries from the file itself
+        if (sinceTimestamp && !options.dryRun && !options.multiFile) {
             await this.removeDeletedRecordFiles(options, sinceTimestamp, targetDir, callbacks);
         }
         // Persist incremental state after a successful pull
@@ -175,11 +212,21 @@ export class PullService {
             targetDir
         };
     }
+    /** The CLI overrides of one pull; the update/create options keep their option names. */
+    pullOverridesFor(options) {
+        return {
+            MergeStrategy: options.MergeStrategy,
+            PreserveFields: options.PreserveFields,
+            ExcludeFields: options.ExcludeFields,
+            UpdateExistingRecords: options.updateExistingRecords,
+            CreateNewFileIfNotFound: options.createNewFileIfNotFound,
+        };
+    }
     /** @deprecated Use {@link Pull}. */
     async pull(options, callbacks) {
         return this.Pull(options, callbacks);
     }
-    async processRecords(records, options, targetDir, entityConfig, callbacks) {
+    async processRecords(records, options, targetDir, entityConfig, callbacks, sinceTimestamp) {
         const entityInfo = this.syncEngine.getEntityInfo(options.entity);
         if (!entityInfo) {
             throw new Error(`Entity information not found for: ${options.entity}`);
@@ -193,38 +240,64 @@ export class PullService {
         const batchedRelatedData = await this.prefetchRelatedEntities(records, entityConfig, entityInfo, options.verbose, callbacks);
         // If multi-file flag is set, collect all records
         if (options.multiFile) {
+            const fileName = options.multiFile.endsWith('.json') ? options.multiFile : `${options.multiFile}.json`;
+            const filePath = path.join(targetDir, fileName);
+            const existingRecords = await this.readMultiFile(filePath);
+            const existingIndex = await this.buildMultiFileIndex(filePath, existingRecords, targetDir, entityConfig, entityInfo);
+            existingIndex.ThrowIfUnsetKeyVariables(options.entity);
             const allRecords = [];
+            const matched = new Map();
             const errors = [];
+            const unsetKeyErrors = [];
             // Process records in parallel for multi-file mode
             const recordPromises = records.map(async (record, index) => {
                 try {
                     const primaryKey = extractPrimaryKeyValues(record, entityInfo);
-                    // Process record for multi-file
-                    const recordData = await this.recordProcessor.processRecord(record, primaryKey, targetDir, entityConfig, options.verbose, true, undefined, 0, new Set(), undefined, batchedRelatedData);
-                    return { success: true, recordData, index };
+                    const found = existingIndex.Find(primaryKey);
+                    const existing = found?.Record;
+                    // Only an entry of the target file is replaced in place; a record from another file is added
+                    const targetEntry = found?.FilePath === filePath ? existing : undefined;
+                    const recordData = await this.recordProcessor.processRecord(record, primaryKey, targetDir, entityConfig, options.verbose, !existing, existing, 0, new Set(), undefined, batchedRelatedData);
+                    return { success: true, recordData, existing: targetEntry, index };
                 }
                 catch (error) {
-                    const errorMessage = `Failed to process record ${index + 1}: ${error.message || error}`;
+                    if (this.keepUnsetKeyError(error, index, filePath, options.entity, unsetKeyErrors)) {
+                        return { success: false, recordData: null, existing: undefined, index };
+                    }
+                    const errorMessage = `Failed to process record ${index + 1}: ${error instanceof Error ? error.message : String(error)}`;
                     errors.push(errorMessage);
                     callbacks?.onWarn?.(errorMessage);
-                    return { success: false, recordData: null, index };
+                    return { success: false, recordData: null, existing: undefined, index };
                 }
             });
             const recordResults = await Promise.all(recordPromises);
+            this.throwFirstUnsetKeyError(unsetKeyErrors);
             // Collect successful records
             for (const result of recordResults) {
                 if (result.success && result.recordData) {
                     allRecords.push(result.recordData);
+                    if (result.existing) {
+                        matched.set(result.existing, result.recordData);
+                    }
                     processed++;
                 }
             }
             if (options.verbose) {
                 callbacks?.onProgress?.(`Processed ${processed}/${records.length} records in parallel`);
             }
-            // Queue all records to single file for batched write
-            if (allRecords.length > 0) {
-                const fileName = options.multiFile.endsWith('.json') ? options.multiFile : `${options.multiFile}.json`;
-                const filePath = path.join(targetDir, fileName);
+            // A full pull writes the query result; an incremental pull updates the existing file
+            if (sinceTimestamp) {
+                const deleted = await this.softDeletedMultiFileEntries(options, sinceTimestamp, existingIndex, filePath, entityInfo, callbacks);
+                if (allRecords.length > 0 || deleted.size > 0) {
+                    const kept = existingRecords.filter((existing) => !deleted.has(existing));
+                    this.fileWriteBatch.queueWrite(filePath, this.upsertMultiFileRecords(kept, matched, allRecords));
+                    callbacks?.onSuccess?.(`Queued ${processed} records for ${path.basename(filePath)}`);
+                }
+                if (deleted.size > 0) {
+                    callbacks?.onSuccess?.(`Removed ${deleted.size} entries for soft-deleted records from ${path.basename(filePath)}`);
+                }
+            }
+            else if (allRecords.length > 0) {
                 this.fileWriteBatch.queueWrite(filePath, allRecords);
                 callbacks?.onSuccess?.(`Queued ${processed} records for ${path.basename(filePath)}`);
             }
@@ -252,6 +325,98 @@ export class PullService {
         }
         return { processed, created, updated, skipped };
     }
+    /**
+     * The existing-record index of a multi-file pull: the target file's entries first, then the records
+     * of the folder's other record files (found as a single-file pull finds them), so every `@env:`
+     * reference is kept.
+     * @throws when another record file cannot be read, before any write
+     */
+    async buildMultiFileIndex(filePath, targetRecords, targetDir, entityConfig, entityInfo) {
+        const filePattern = entityConfig.pull?.filePattern || entityConfig.filePattern || '*.json';
+        const otherFiles = (await FindExistingRecordFiles(targetDir, filePattern, entityConfig.ignoreDirectories ?? []))
+            .filter((other) => path.resolve(other) !== path.resolve(filePath));
+        const otherEntries = await this.loadExistingRecords(otherFiles);
+        const targetEntries = targetRecords.map((existing) => ({ Record: existing, FilePath: filePath, InArray: true }));
+        return ExistingRecordIndex.Build([...targetEntries, ...otherEntries], entityInfo);
+    }
+    /**
+     * Records in a multi-file target: none when the file does not exist or holds only whitespace, one for
+     * a file that holds an object.
+     * @throws when the file is not valid JSON, with the parse error as the cause
+     */
+    async readMultiFile(filePath) {
+        if (!(await fs.pathExists(filePath))) {
+            return [];
+        }
+        let content;
+        try {
+            const text = await fs.readFile(filePath, 'utf8');
+            if (text.trim() === '') {
+                return [];
+            }
+            content = JSON.parse(text.replace(/^\uFEFF/, ''));
+        }
+        catch (cause) {
+            throw new Error(`Cannot read ${filePath}. Pull does not overwrite a file it cannot read.`, { cause });
+        }
+        return Array.isArray(content) ? content : [content];
+    }
+    /**
+     * The entries of a multi-file target whose rows were soft-deleted since the timestamp. Empty when
+     * the entity has no soft-delete column or the query fails. Entries of other record files are left out.
+     */
+    async softDeletedMultiFileEntries(options, sinceTimestamp, existingIndex, filePath, entityInfo, callbacks) {
+        const entries = new Set();
+        const rows = await this.findSoftDeletedRows(options, sinceTimestamp, callbacks);
+        for (const row of rows) {
+            const key = {};
+            for (const pk of entityInfo.PrimaryKeys) {
+                key[pk.Name] = row[pk.Name];
+            }
+            const entry = existingIndex.Find(key);
+            if (entry && entry.FilePath === filePath) {
+                entries.add(entry.Record);
+            }
+        }
+        return entries;
+    }
+    /**
+     * Rows soft-deleted since the timestamp (`__mj_DeletedAt`). Empty when the query fails, for example
+     * for an entity with hard deletes; a verbose note says so.
+     */
+    async findSoftDeletedRows(options, sinceTimestamp, callbacks) {
+        try {
+            if (!isValidISOTimestamp(sinceTimestamp)) {
+                throw new Error('Invalid --since timestamp format. Expected ISO 8601 (e.g., 2026-04-07T10:00:00Z)');
+            }
+            const deletedFilter = `[__mj_DeletedAt] > '${sinceTimestamp}'`;
+            const fullFilter = options.filter ? `(${options.filter}) AND ${deletedFilter}` : deletedFilter;
+            const result = await new RunView().RunView({
+                EntityName: options.entity,
+                ExtraFilter: fullFilter,
+                ResultType: 'simple',
+            }, this.contextUser);
+            return result.Success ? result.Results : [];
+        }
+        catch {
+            // The entity may not support soft deletes (__mj_DeletedAt column may not exist)
+            if (options.verbose) {
+                callbacks?.onLog?.(`   Note: could not check for deleted records (entity may use hard deletes)`);
+            }
+            return [];
+        }
+    }
+    /** The existing entries with each matched entry replaced in place, then the pulled records that matched none. */
+    upsertMultiFileRecords(existingRecords, matched, pulled) {
+        const replacements = new Set(matched.values());
+        const result = existingRecords.map((existing) => matched.get(existing) ?? existing);
+        for (const record of pulled) {
+            if (!replacements.has(record)) {
+                result.push(record);
+            }
+        }
+        return result;
+    }
     /**
      * Clean up backup files created during the pull operation
      * Should be called after successful pull operations to remove persistent backup files
@@ -333,40 +498,28 @@ export class PullService {
      * (which filters by __mj_UpdatedAt) would never notice deletions.
      */
     async removeDeletedRecordFiles(options, sinceTimestamp, targetDir, callbacks) {
-        try {
-            if (!isValidISOTimestamp(sinceTimestamp)) {
-                throw new Error('Invalid --since timestamp format. Expected ISO 8601 (e.g., 2026-04-07T10:00:00Z)');
-            }
-            const rv = new RunView();
-            const deletedFilter = `[__mj_DeletedAt] > '${sinceTimestamp}'`;
-            const baseFilter = options.filter || '';
-            const fullFilter = baseFilter
-                ? `(${baseFilter}) AND ${deletedFilter}`
-                : deletedFilter;
-            const result = await rv.RunView({
-                EntityName: options.entity,
-                ExtraFilter: fullFilter,
-                ResultType: 'simple',
-            }, this.contextUser);
-            if (!result.Success || result.Results.length === 0) {
-                return;
-            }
-            const entityInfo = this.syncEngine.getEntityInfo(options.entity);
-            const pkField = entityInfo?.PrimaryKeys?.[0]?.Name ?? 'ID';
-            let removed = 0;
-            for (const record of result.Results) {
-                const id = record[pkField];
-                if (!id)
-                    continue;
-                // Try common file naming patterns.
-                // NOTE: This is a best-effort heuristic. If the entity uses a custom
-                // file naming scheme (e.g., via newFileName config) these patterns may
-                // miss. A single-file-per-entity layout is also not covered here.
-                const candidates = [
-                    path.join(targetDir, `${id}.json`),
-                    path.join(targetDir, `${record.Name ?? id}.json`),
-                    path.join(targetDir, `${(record.Name ?? '').replace(/[^a-zA-Z0-9-_ ]/g, '')}.json`),
-                ];
+        const rows = await this.findSoftDeletedRows(options, sinceTimestamp, callbacks);
+        if (rows.length === 0) {
+            return;
+        }
+        const entityInfo = this.syncEngine.getEntityInfo(options.entity);
+        const pkField = entityInfo?.PrimaryKeys?.[0]?.Name ?? 'ID';
+        let removed = 0;
+        for (const record of rows) {
+            const id = record[pkField];
+            if (!id)
+                continue;
+            const name = typeof record.Name === 'string' ? record.Name : undefined;
+            // Try common file naming patterns.
+            // NOTE: This is a best-effort heuristic. If the entity uses a custom
+            // file naming scheme (e.g., via newFileName config) these patterns may
+            // miss. A single-file-per-entity layout is also not covered here.
+            const candidates = [
+                path.join(targetDir, `${id}.json`),
+                path.join(targetDir, `${name ?? id}.json`),
+                path.join(targetDir, `${(name ?? '').replace(/[^a-zA-Z0-9-_ ]/g, '')}.json`),
+            ];
+            try {
                 for (const filePath of candidates) {
                     if (await fs.pathExists(filePath)) {
                         await fs.remove(filePath);
@@ -378,38 +531,37 @@ export class PullService {
                     }
                 }
             }
-            if (removed > 0) {
-                callbacks?.onSuccess?.(`Removed ${removed} files for soft-deleted records`);
+            catch {
+                // Non-fatal: the file stays and the pull goes on
+                if (options.verbose) {
+                    callbacks?.onLog?.(`   Note: could not remove the file of a deleted record`);
+                }
             }
         }
-        catch (error) {
-            // Non-fatal — log and continue. Entity may not support soft deletes
-            // (__mj_DeletedAt column may not exist), in which case the query fails.
-            if (options.verbose) {
-                callbacks?.onLog?.(`   Note: could not check for deleted records (entity may use hard deletes)`);
-            }
+        if (removed > 0) {
+            callbacks?.onSuccess?.(`Removed ${removed} files for soft-deleted records`);
         }
     }
     /**
      * Rollback file changes by restoring from backup files
      * Called when pull operation fails after files have been modified
+     * @returns the number of files restored
      */
     async rollbackFileChanges(callbacks) {
         if (this.createdBackupFiles.length === 0) {
             callbacks?.onLog?.('No backup files found - no rollback needed');
-            return;
+            return 0;
         }
         callbacks?.onProgress?.(`Rolling back ${this.createdBackupFiles.length} file changes...`);
         const errors = [];
         let restoredCount = 0;
         for (const backupPath of this.createdBackupFiles) {
             try {
-                // Extract original file path from backup path
-                const backupDir = path.dirname(backupPath);
-                const backupFileName = path.basename(backupPath);
-                // Remove timestamp and .backup extension to get original filename
-                const originalFileName = backupFileName.replace(/\.\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z\.backup$/, '.json');
-                const originalFilePath = path.join(path.dirname(backupDir), originalFileName);
+                const originalFilePath = this.backupSources.get(backupPath);
+                if (!originalFilePath) {
+                    errors.push(`No source recorded for backup ${backupPath}`);
+                    continue;
+                }
                 if (await fs.pathExists(backupPath)) {
                     await fs.copy(backupPath, originalFilePath);
                     restoredCount++;
@@ -423,6 +575,7 @@ export class PullService {
             throw new Error(`Rollback completed with ${errors.length} errors (${restoredCount} files restored):\n${errors.join('\n')}`);
         }
         callbacks?.onSuccess?.(`Successfully rolled back ${restoredCount} file changes`);
+        return restoredCount;
     }
     async processIndividualRecords(records, options, targetDir, entityConfig, entityInfo, callbacks, batchedRelatedData) {
         let processed = 0;
@@ -436,28 +589,31 @@ export class PullService {
             callbacks?.onLog?.(`Found ${existingFiles.length} existing files matching pattern '${filePattern}'`);
             existingFiles.forEach(f => callbacks?.onLog?.(`  - ${path.basename(f)}`));
         }
-        // Load existing records and build lookup map
-        const existingRecordsMap = await this.loadExistingRecords(existingFiles, entityInfo);
+        // Load existing records and index them by the resolved primary key
+        const existingEntries = await this.loadExistingRecords(existingFiles);
+        const existingIndex = ExistingRecordIndex.Build(existingEntries, entityInfo);
+        existingIndex.ThrowIfUnsetKeyVariables(options.entity);
         if (options.verbose) {
-            callbacks?.onLog?.(`Loaded ${existingRecordsMap.size} existing records from files`);
+            callbacks?.onLog?.(`Loaded ${existingEntries.length} existing records from files`);
+            if (existingIndex.DuplicateCount > 0) {
+                callbacks?.onLog?.(`${existingIndex.DuplicateCount} existing records repeat a primary key; the first one in file order is used`);
+            }
         }
         // Separate records into new and existing
         const newRecords = [];
         const existingRecordsToUpdate = [];
         for (const record of records) {
             const primaryKey = extractPrimaryKeyValues(record, entityInfo);
-            // Create lookup key
-            const lookupKey = createPrimaryKeyLookup(primaryKey);
-            const existingFileInfo = existingRecordsMap.get(lookupKey);
-            if (existingFileInfo) {
+            const existing = existingIndex.Find(primaryKey);
+            if (existing) {
                 // Record exists locally
                 if (entityConfig.pull?.updateExistingRecords !== false) {
-                    existingRecordsToUpdate.push({ record, primaryKey, filePath: existingFileInfo.filePath });
+                    existingRecordsToUpdate.push({ record, primaryKey, existing });
                 }
                 else {
                     skipped++;
                     if (options.verbose) {
-                        callbacks?.onLog?.(`Skipping existing record: ${lookupKey}`);
+                        callbacks?.onLog?.(`Skipping existing record: ${ExistingRecordIndex.FileLookupOf(existing.Record)}`);
                     }
                 }
             }
@@ -469,45 +625,40 @@ export class PullService {
                 else {
                     skipped++;
                     if (options.verbose) {
-                        callbacks?.onLog?.(`Skipping new record (createNewFileIfNotFound=false): ${lookupKey}`);
+                        callbacks?.onLog?.(`Skipping new record (createNewFileIfNotFound=false): ${createPrimaryKeyLookup(primaryKey)}`);
                     }
                 }
             }
         }
-        // Track which files have been backed up to avoid duplicates
-        const backedUpFiles = new Set();
         const errors = [];
+        const unsetKeyErrors = [];
         // Process existing records updates in parallel
         if (existingRecordsToUpdate.length > 0) {
             callbacks?.onProgress?.(`Updating existing records (parallel processing)`);
-            const updatePromises = existingRecordsToUpdate.map(async ({ record, primaryKey, filePath }, index) => {
-                try {
-                    // Create backup if configured (only once per file)
-                    if (entityConfig.pull?.backupBeforeUpdate && !backedUpFiles.has(filePath)) {
-                        await this.createBackup(filePath, entityConfig.pull?.backupDirectory);
-                        backedUpFiles.add(filePath);
-                    }
-                    // Load existing file data
-                    const existingData = await fs.readJson(filePath);
-                    // Find the specific existing record that matches this primary key
-                    let existingRecordData;
-                    if (Array.isArray(existingData)) {
-                        // Find the matching record in the array
-                        const matchingRecord = existingData.find(r => createPrimaryKeyLookup(r.primaryKey) === createPrimaryKeyLookup(primaryKey));
-                        existingRecordData = matchingRecord || existingData[0]; // Fallback to first if not found
-                    }
-                    else {
-                        existingRecordData = existingData;
+            const mergeStrategy = entityConfig.pull?.mergeStrategy || 'merge';
+            const updatePromises = existingRecordsToUpdate.map(async ({ record, primaryKey, existing }, index) => {
+                const filePath = existing.FilePath ?? '';
+                if (mergeStrategy === 'skip') {
+                    if (options.verbose) {
+                        callbacks?.onLog?.(`Skipped (mergeStrategy=skip): ${filePath}`);
                     }
+                    return { success: true, skipped: true, index };
+                }
+                try {
+                    // Backups are taken for every file the batch will rewrite, just before it flushes.
+                    const existingRecordData = existing.Record;
                     // Process the new record data (isNewRecord = false for updates)
                     const newRecordData = await this.recordProcessor.processRecord(record, primaryKey, targetDir, entityConfig, options.verbose, false, existingRecordData, 0, new Set(), undefined, batchedRelatedData);
                     // Apply merge strategy
-                    const mergedData = await this.mergeRecords(existingRecordData, newRecordData, entityConfig.pull?.mergeStrategy || 'merge', entityConfig.pull?.preserveFields || []);
+                    const mergedData = await this.mergeRecords(existingRecordData, newRecordData, mergeStrategy, entityConfig.pull?.preserveFields || []);
+                    // A field left out because it holds an env value does not come back from the existing file
+                    for (const fieldName of this.recordProcessor.EnvOmittedFieldsOf(newRecordData)) {
+                        delete mergedData.fields[fieldName];
+                    }
                     // Queue updated data for batched write
-                    if (Array.isArray(existingData)) {
-                        // Queue array update - batch will handle merging
-                        const primaryKeyLookup = createPrimaryKeyLookup(primaryKey);
-                        this.fileWriteBatch.queueArrayUpdate(filePath, mergedData, primaryKeyLookup);
+                    if (existing.InArray) {
+                        // The batch finds the entry by the key the file holds
+                        this.fileWriteBatch.queueArrayUpdate(filePath, mergedData, ExistingRecordIndex.FileLookupOf(existingRecordData));
                     }
                     else {
                         // Queue single record update
@@ -516,17 +667,21 @@ export class PullService {
                     if (options.verbose) {
                         callbacks?.onLog?.(`Updated: ${filePath}`);
                     }
-                    return { success: true, index };
+                    return { success: true, skipped: false, index };
                 }
                 catch (error) {
+                    if (this.keepUnsetKeyError(error, index, filePath, options.entity, unsetKeyErrors)) {
+                        return { success: false, skipped: false, index };
+                    }
                     const errorMessage = `Failed to update record ${index + 1}: ${error.message || error}`;
                     errors.push(errorMessage);
                     callbacks?.onWarn?.(errorMessage);
-                    return { success: false, index };
+                    return { success: false, skipped: false, index };
                 }
             });
             const updateResults = await Promise.all(updatePromises);
-            updated = updateResults.filter(r => r.success).length;
+            updated = updateResults.filter(r => r.success && !r.skipped).length;
+            skipped += updateResults.filter(r => r.skipped).length;
             processed += updated;
             if (options.verbose) {
                 callbacks?.onSuccess?.(`Completed ${updated}/${existingRecordsToUpdate.length} record updates`);
@@ -553,6 +708,9 @@ export class PullService {
                         return { success: true, index };
                     }
                     catch (error) {
+                        if (this.keepUnsetKeyError(error, existingRecordsToUpdate.length + index, filePath, options.entity, unsetKeyErrors)) {
+                            return { success: false, index };
+                        }
                         const errorMessage = `Failed to process new record ${index + 1}: ${error.message || error}`;
                         errors.push(errorMessage);
                         callbacks?.onWarn?.(errorMessage);
@@ -574,6 +732,9 @@ export class PullService {
                         return { success: true, index };
                     }
                     catch (error) {
+                        if (this.keepUnsetKeyError(error, existingRecordsToUpdate.length + index, undefined, options.entity, unsetKeyErrors)) {
+                            return { success: false, index };
+                        }
                         const errorMessage = `Failed to process new record ${index + 1}: ${error.message || error}`;
                         errors.push(errorMessage);
                         callbacks?.onWarn?.(errorMessage);
@@ -588,12 +749,30 @@ export class PullService {
                 }
             }
         }
+        this.throwFirstUnsetKeyError(unsetKeyErrors);
         // If there were errors during parallel processing, throw them
         if (errors.length > 0) {
             throw new Error(`Parallel processing completed with ${errors.length} errors:\n${errors.join('\n')}`);
         }
         return { processed, updated, created, skipped };
     }
+    /**
+     * Keeps an unset key variable error, with the record file and the pulled entity in its message,
+     * so the pull rethrows it as is. Returns false for any other error.
+     */
+    keepUnsetKeyError(error, index, filePath, pulledEntityName, into) {
+        if (!(error instanceof UnsetPrimaryKeyVariableError)) {
+            return false;
+        }
+        into.push({ Index: index, Error: error.InRecordFile(filePath, pulledEntityName) });
+        return true;
+    }
+    /** @throws the kept UnsetPrimaryKeyVariableError of the lowest record index, when there is one */
+    throwFirstUnsetKeyError(errors) {
+        if (errors.length > 0) {
+            throw errors.reduce((first, next) => (next.Index < first.Index ? next : first)).Error;
+        }
+    }
     async processRecord(record, primaryKey, targetDir, entityConfig, verbose, batchedRelatedData) {
         const recordData = await this.recordProcessor.processRecord(record, primaryKey, targetDir, entityConfig, verbose, true, undefined, 0, new Set(), undefined, batchedRelatedData);
         // Determine file path
@@ -650,24 +829,28 @@ export class PullService {
         }
         return batchedRelatedData.size > 0 ? batchedRelatedData : undefined;
     }
-    async loadExistingRecords(files, _entityInfo) {
-        const recordsMap = new Map();
+    /**
+     * Reads every record in the files.
+     * @throws when a file cannot be read, before any write: a record in it could otherwise be written again as new
+     */
+    async loadExistingRecords(files) {
+        const entries = [];
         for (const filePath of files) {
+            let fileData;
             try {
-                const fileData = await fs.readJson(filePath);
-                const records = Array.isArray(fileData) ? fileData : [fileData];
-                for (const record of records) {
-                    if (record.primaryKey) {
-                        const lookupKey = createPrimaryKeyLookup(record.primaryKey);
-                        recordsMap.set(lookupKey, { filePath, recordData: record });
-                    }
-                }
+                fileData = await fs.readJson(filePath);
             }
-            catch (error) {
-                // Skip files that can't be parsed
+            catch (cause) {
+                throw new Error(`Cannot read ${filePath}. Pull does not write new records while a record file in the folder cannot be read.`, { cause });
+            }
+            const records = Array.isArray(fileData) ? fileData : [fileData];
+            for (const record of records) {
+                if (record?.primaryKey) {
+                    entries.push({ Record: record, FilePath: filePath, InArray: Array.isArray(fileData) });
+                }
             }
         }
-        return recordsMap;
+        return entries;
     }
     async findEntityDirectories(entityName) {
         const dirs = [];
@@ -757,13 +940,32 @@ export class PullService {
         result.sync = newData.sync;
         return result;
     }
+    /**
+     * Backs up each existing file the write batch is about to rewrite, once per file. Files the
+     * batch will create fresh have nothing to back up.
+     */
+    async backupPendingFiles(backupDirName, verbose, callbacks) {
+        for (const filePath of this.fileWriteBatch.getPendingFiles()) {
+            if (!(await fs.pathExists(filePath))) {
+                continue;
+            }
+            const backupPath = await this.createBackup(filePath, backupDirName);
+            if (verbose) {
+                callbacks?.onLog?.(`Backed up ${path.basename(filePath)} → ${path.relative(path.dirname(filePath), backupPath)}`);
+            }
+        }
+    }
+    /**
+     * Copies `filePath` to a timestamped `.backup` file under `backupDirName` (default `.backups`,
+     * relative to the file's directory). Throws if the copy fails: a pull asked to back up must not
+     * go on to rewrite a file it could not back up.
+     */
     async createBackup(filePath, backupDirName) {
         const dir = path.dirname(filePath);
         const fileName = path.basename(filePath);
-        const backupDir = path.join(dir, backupDirName || '.backups');
+        const backupDir = resolveBackupDirectory(dir, backupDirName);
         // Ensure backup directory exists
         await fs.ensureDir(backupDir);
-        // Track the backup directory for cleanup
         this.createdBackupDirs.add(backupDir);
         const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
         // Remove .json extension, add timestamp, then add .backup extension
@@ -771,12 +973,13 @@ export class PullService {
         const backupPath = path.join(backupDir, backupFileName);
         try {
             await fs.copy(filePath, backupPath);
-            // Track the created backup file for cleanup
-            this.createdBackupFiles.push(backupPath);
         }
         catch (error) {
-            // Log error but don't throw
+            throw new Error(`Failed to back up ${filePath} to ${backupPath}: ${error instanceof Error ? error.message : String(error)}`);
         }
+        this.createdBackupFiles.push(backupPath);
+        this.backupSources.set(backupPath, filePath);
+        return backupPath;
     }
 }
 //# sourceMappingURL=PullService.js.map