@memberjunction/metadata-sync 2.67.0 → 2.68.0

package/dist/lib/RelatedEntityHandler.d.ts

@@ -0,0 +1,75 @@
+import { BaseEntity, UserInfo } from '@memberjunction/core';
+import { SyncEngine, RecordData } from '../lib/sync-engine';
+import { RelatedEntityConfig, EntityConfig } from '../config';
+/**
+ * Handles loading and processing of related entities for records
+ */
+export declare class RelatedEntityHandler {
+    private syncEngine;
+    private contextUser;
+    constructor(syncEngine: SyncEngine, contextUser: UserInfo);
+    /**
+     * Load related entities for a record
+     */
+    loadRelatedEntities(parentRecord: BaseEntity, relationConfig: RelatedEntityConfig, parentEntityConfig: EntityConfig, existingRelatedEntities: RecordData[], processRecordData: (record: BaseEntity, primaryKey: Record<string, any>, targetDir: string, entityConfig: EntityConfig, verbose?: boolean, isNewRecord?: boolean, existingRecordData?: RecordData, currentDepth?: number, ancestryPath?: Set<string>, fieldOverrides?: Record<string, any>) => Promise<RecordData>, currentDepth: number, ancestryPath: Set<string>, verbose?: boolean): Promise<RecordData[]>;
+    /**
+     * Queries the database for related entities
+     */
+    private queryRelatedEntities;
+    /**
+     * Builds the filter for querying related entities
+     */
+    private buildRelatedEntityFilter;
+    /**
+     * Creates entity config for related entity processing
+     */
+    private createRelatedEntityConfig;
+    /**
+     * Processes all related records (both existing and new)
+     */
+    private processRelatedRecords;
+    /**
+     * Builds a map of database records by primary key for efficient lookup
+     */
+    private buildDatabaseRecordMap;
+    /**
+     * Processes existing related entities
+     */
+    private processExistingRelatedEntities;
+    /**
+     * Processes a single existing related entity
+     */
+    private processExistingRelatedEntity;
+    /**
+     * Processes new related entities
+     */
+    private processNewRelatedEntities;
+    /**
+     * Processes a single new related entity
+     */
+    private processNewRelatedEntity;
+    /**
+     * Creates field overrides for @parent:ID replacement
+     */
+    private createFieldOverrides;
+    /**
+     * Builds primary key for a record
+     */
+    private buildPrimaryKeyForRecord;
+    /**
+     * Get the primary key value from a record
+     */
+    private getRecordPrimaryKey;
+    /**
+     * Get a field value from a record, handling both entity objects and plain objects
+     */
+    private getFieldValue;
+    /**
+     * Log warning message if verbose mode is enabled
+     */
+    private logWarning;
+    /**
+     * Log error message if verbose mode is enabled
+     */
+    private logError;
+}

package/dist/lib/RelatedEntityHandler.js

@@ -0,0 +1,273 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RelatedEntityHandler = void 0;
+const core_1 = require("@memberjunction/core");
+/**
+ * Handles loading and processing of related entities for records
+ */
+class RelatedEntityHandler {
+    syncEngine;
+    contextUser;
+    constructor(syncEngine, contextUser) {
+        this.syncEngine = syncEngine;
+        this.contextUser = contextUser;
+    }
+    /**
+     * Load related entities for a record
+     */
+    async loadRelatedEntities(parentRecord, relationConfig, parentEntityConfig, existingRelatedEntities, processRecordData, currentDepth, ancestryPath, verbose) {
+        try {
+            const parentPrimaryKey = this.getRecordPrimaryKey(parentRecord);
+            if (!parentPrimaryKey) {
+                this.logWarning('Unable to determine primary key for parent record', verbose);
+                return [];
+            }
+            const relatedRecords = await this.queryRelatedEntities(parentPrimaryKey, relationConfig, verbose);
+            if (!relatedRecords) {
+                return [];
+            }
+            const relatedEntityConfig = this.createRelatedEntityConfig(relationConfig, parentEntityConfig);
+            return await this.processRelatedRecords(relatedRecords, relationConfig, relatedEntityConfig, existingRelatedEntities, processRecordData, currentDepth, ancestryPath, verbose);
+        }
+        catch (error) {
+            this.logError(`Error loading related entities for ${relationConfig.entity}`, error, verbose);
+            return [];
+        }
+    }
+    /**
+     * Queries the database for related entities
+     */
+    async queryRelatedEntities(parentPrimaryKey, relationConfig, verbose) {
+        const filter = this.buildRelatedEntityFilter(parentPrimaryKey, relationConfig);
+        if (verbose) {
+            console.log(`Loading related entities: ${relationConfig.entity} with filter: ${filter}`);
+        }
+        const rv = new core_1.RunView();
+        const result = await rv.RunView({
+            EntityName: relationConfig.entity,
+            ExtraFilter: filter,
+            ResultType: 'entity_object'
+        }, this.contextUser);
+        if (!result.Success) {
+            this.logWarning(`Failed to load related entities ${relationConfig.entity}: ${result.ErrorMessage}`, verbose);
+            return null;
+        }
+        if (verbose) {
+            console.log(`Found ${result.Results.length} related records for ${relationConfig.entity}`);
+        }
+        return result.Results;
+    }
+    /**
+     * Builds the filter for querying related entities
+     */
+    buildRelatedEntityFilter(parentPrimaryKey, relationConfig) {
+        let filter = `${relationConfig.foreignKey} = '${parentPrimaryKey}'`;
+        if (relationConfig.filter) {
+            filter += ` AND (${relationConfig.filter})`;
+        }
+        return filter;
+    }
+    /**
+     * Creates entity config for related entity processing
+     */
+    createRelatedEntityConfig(relationConfig, parentEntityConfig) {
+        return {
+            entity: relationConfig.entity,
+            pull: {
+                excludeFields: relationConfig.excludeFields || [],
+                lookupFields: relationConfig.lookupFields || {},
+                externalizeFields: relationConfig.externalizeFields || [],
+                relatedEntities: relationConfig.relatedEntities || {},
+                ignoreVirtualFields: parentEntityConfig.pull?.ignoreVirtualFields || false,
+                ignoreNullFields: parentEntityConfig.pull?.ignoreNullFields || false
+            }
+        };
+    }
+    /**
+     * Processes all related records (both existing and new)
+     */
+    async processRelatedRecords(dbRecords, relationConfig, relatedEntityConfig, existingRelatedEntities, processRecordData, currentDepth, ancestryPath, verbose) {
+        const dbRecordMap = this.buildDatabaseRecordMap(dbRecords);
+        const relatedRecords = [];
+        const processedIds = new Set();
+        // Process existing related entities first (preserving order)
+        await this.processExistingRelatedEntities(existingRelatedEntities, dbRecordMap, relationConfig, relatedEntityConfig, processRecordData, currentDepth, ancestryPath, relatedRecords, processedIds, verbose);
+        // Process new related entities (append to end)
+        await this.processNewRelatedEntities(dbRecords, relationConfig, relatedEntityConfig, processRecordData, currentDepth, ancestryPath, relatedRecords, processedIds, verbose);
+        return relatedRecords;
+    }
+    /**
+     * Builds a map of database records by primary key for efficient lookup
+     */
+    buildDatabaseRecordMap(dbRecords) {
+        const dbRecordMap = new Map();
+        for (const relatedRecord of dbRecords) {
+            const relatedPrimaryKey = this.getRecordPrimaryKey(relatedRecord);
+            if (relatedPrimaryKey) {
+                dbRecordMap.set(relatedPrimaryKey, relatedRecord);
+            }
+        }
+        return dbRecordMap;
+    }
+    /**
+     * Processes existing related entities
+     */
+    async processExistingRelatedEntities(existingRelatedEntities, dbRecordMap, relationConfig, relatedEntityConfig, processRecordData, currentDepth, ancestryPath, relatedRecords, processedIds, verbose) {
+        for (const existingRelatedEntity of existingRelatedEntities) {
+            const existingPrimaryKey = existingRelatedEntity.primaryKey?.ID;
+            if (!existingPrimaryKey) {
+                this.logWarning('Existing related entity missing primary key, skipping', verbose);
+                continue;
+            }
+            const dbRecord = dbRecordMap.get(existingPrimaryKey);
+            if (!dbRecord) {
+                if (verbose) {
+                    console.log(`Related entity ${existingPrimaryKey} no longer exists in database, removing from results`);
+                }
+                continue; // Skip deleted records
+            }
+            const recordData = await this.processExistingRelatedEntity(dbRecord, existingPrimaryKey, relationConfig, relatedEntityConfig, processRecordData, existingRelatedEntity, currentDepth, ancestryPath, verbose);
+            if (recordData) {
+                relatedRecords.push(recordData);
+                processedIds.add(existingPrimaryKey);
+            }
+        }
+    }
+    /**
+     * Processes a single existing related entity
+     */
+    async processExistingRelatedEntity(dbRecord, existingPrimaryKey, relationConfig, relatedEntityConfig, processRecordData, existingRelatedEntity, currentDepth, ancestryPath, verbose) {
+        const relatedRecordPrimaryKey = this.buildPrimaryKeyForRecord(existingPrimaryKey, dbRecord, relationConfig.entity);
+        const fieldOverrides = this.createFieldOverrides(dbRecord, relationConfig);
+        if (!fieldOverrides) {
+            return null;
+        }
+        return await processRecordData(dbRecord, relatedRecordPrimaryKey, '', // targetDir not needed for related entities
+        relatedEntityConfig, verbose, false, // isNewRecord = false for existing records
+        existingRelatedEntity, // Pass existing data for change detection
+        currentDepth + 1, ancestryPath, fieldOverrides // Pass the field override for @parent:ID
+        );
+    }
+    /**
+     * Processes new related entities
+     */
+    async processNewRelatedEntities(dbRecords, relationConfig, relatedEntityConfig, processRecordData, currentDepth, ancestryPath, relatedRecords, processedIds, verbose) {
+        for (const relatedRecord of dbRecords) {
+            const relatedPrimaryKey = this.getRecordPrimaryKey(relatedRecord);
+            if (!relatedPrimaryKey || processedIds.has(relatedPrimaryKey)) {
+                continue; // Skip already processed records
+            }
+            const recordData = await this.processNewRelatedEntity(relatedRecord, relatedPrimaryKey, relationConfig, relatedEntityConfig, processRecordData, currentDepth, ancestryPath, verbose);
+            if (recordData) {
+                relatedRecords.push(recordData);
+                processedIds.add(relatedPrimaryKey);
+            }
+        }
+    }
+    /**
+     * Processes a single new related entity
+     */
+    async processNewRelatedEntity(relatedRecord, relatedPrimaryKey, relationConfig, relatedEntityConfig, processRecordData, currentDepth, ancestryPath, verbose) {
+        const relatedRecordPrimaryKey = this.buildPrimaryKeyForRecord(relatedPrimaryKey, relatedRecord, relationConfig.entity);
+        const fieldOverrides = this.createFieldOverrides(relatedRecord, relationConfig);
+        if (!fieldOverrides) {
+            return null;
+        }
+        return await processRecordData(relatedRecord, relatedRecordPrimaryKey, '', // targetDir not needed for related entities
+        relatedEntityConfig, verbose, true, // isNewRecord = true for new records
+        undefined, // No existing data for new records
+        currentDepth + 1, ancestryPath, fieldOverrides // Pass the field override for @parent:ID
+        );
+    }
+    /**
+     * Creates field overrides for @parent:ID replacement
+     */
+    createFieldOverrides(record, relationConfig) {
+        if (typeof record.GetAll === 'function') {
+            const data = record.GetAll();
+            if (data[relationConfig.foreignKey] !== undefined) {
+                return { [relationConfig.foreignKey]: '@parent:ID' };
+            }
+        }
+        else {
+            if (record[relationConfig.foreignKey] !== undefined) {
+                return { [relationConfig.foreignKey]: '@parent:ID' };
+            }
+        }
+        return null;
+    }
+    /**
+     * Builds primary key for a record
+     */
+    buildPrimaryKeyForRecord(primaryKeyValue, record, entityName) {
+        const relatedRecordPrimaryKey = {};
+        const entityInfo = this.syncEngine.getEntityInfo(entityName);
+        for (const pk of entityInfo?.PrimaryKeys || []) {
+            if (pk.Name === 'ID') {
+                relatedRecordPrimaryKey[pk.Name] = primaryKeyValue;
+            }
+            else {
+                // For compound keys, get the value from the related record
+                relatedRecordPrimaryKey[pk.Name] = this.getFieldValue(record, pk.Name);
+            }
+        }
+        return relatedRecordPrimaryKey;
+    }
+    /**
+     * Get the primary key value from a record
+     */
+    getRecordPrimaryKey(record) {
+        if (!record)
+            return null;
+        // Try to get ID directly
+        if (record.ID)
+            return record.ID;
+        // Try to get from GetAll() method if it's an entity object
+        if (typeof record.GetAll === 'function') {
+            const data = record.GetAll();
+            if (data.ID)
+                return data.ID;
+        }
+        // Try common variations
+        if (record.id)
+            return record.id;
+        if (record.Id)
+            return record.Id;
+        return null;
+    }
+    /**
+     * Get a field value from a record, handling both entity objects and plain objects
+     */
+    getFieldValue(record, fieldName) {
+        if (!record)
+            return null;
+        // Try to get field directly using bracket notation with type assertion
+        if (record[fieldName] !== undefined)
+            return record[fieldName];
+        // Try to get from GetAll() method if it's an entity object
+        if (typeof record.GetAll === 'function') {
+            const data = record.GetAll();
+            if (data[fieldName] !== undefined)
+                return data[fieldName];
+        }
+        return null;
+    }
+    /**
+     * Log warning message if verbose mode is enabled
+     */
+    logWarning(message, verbose) {
+        if (verbose) {
+            console.warn(message);
+        }
+    }
+    /**
+     * Log error message if verbose mode is enabled
+     */
+    logError(message, error, verbose) {
+        if (verbose) {
+            console.error(`${message}: ${error}`);
+        }
+    }
+}
+exports.RelatedEntityHandler = RelatedEntityHandler;
+//# sourceMappingURL=RelatedEntityHandler.js.map

package/dist/lib/RelatedEntityHandler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RelatedEntityHandler.js","sourceRoot":"","sources":["../../src/lib/RelatedEntityHandler.ts"],"names":[],"mappings":";;;AAAA,+CAAqE;AAIrE;;GAEG;AACH,MAAa,oBAAoB;IAErB;IACA;IAFV,YACU,UAAsB,EACtB,WAAqB;QADrB,eAAU,GAAV,UAAU,CAAY;QACtB,gBAAW,GAAX,WAAW,CAAU;IAC5B,CAAC;IAEJ;;OAEG;IACH,KAAK,CAAC,mBAAmB,CACvB,YAAwB,EACxB,cAAmC,EACnC,kBAAgC,EAChC,uBAAqC,EACrC,iBAWwB,EACxB,YAAoB,EACpB,YAAyB,EACzB,OAAiB;QAEjB,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;YAChE,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,IAAI,CAAC,UAAU,CAAC,mDAAmD,EAAE,OAAO,CAAC,CAAC;gBAC9E,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,oBAAoB,CACpD,gBAAgB,EAChB,cAAc,EACd,OAAO,CACR,CAAC;YAEF,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,MAAM,mBAAmB,GAAG,IAAI,CAAC,yBAAyB,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;YAE/F,OAAO,MAAM,IAAI,CAAC,qBAAqB,CACrC,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,uBAAuB,EACvB,iBAAiB,EACjB,YAAY,EACZ,YAAY,EACZ,OAAO,CACR,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,QAAQ,CAAC,sCAAsC,cAAc,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAC7F,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAChC,gBAAwB,EACxB,cAAmC,EACnC,OAAiB;QAEjB,MAAM,MAAM,GAAG,IAAI,CAAC,wBAAwB,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAC;QAE/E,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,6BAA6B,cAAc,CAAC,MAAM,iBAAiB,MAAM,EAAE,CAAC,CAAC;QAC3F,CAAC;QAED,MAAM,EAAE,GAAG,IAAI,cAAO,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;YAC9B,UAAU,EAAE,cAAc,CAAC,MAAM;YACjC,WAAW,EAAE,MAAM;YACnB,UAAU,EAAE,eAAe;SAC5B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAErB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,mCAAmC,cAAc,CAAC,MAAM,KAAK,MAAM,CAAC,YAAY,EAAE,EAAE,OAAO,CAAC,CAAC;YAC7G,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,MAAM,wBAAwB,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7F,CAAC;QAED,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,wBAAwB,CAAC,gBAAwB,EAAE,cAAmC;QAC5F,IAAI,MAAM,GAAG,GAAG,cAAc,CAAC,UAAU,OAAO,gBAAgB,GAAG,CAAC;QACpE,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,IAAI,SAAS,cAAc,CAAC,MAAM,GAAG,CAAC;QAC9C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,yBAAyB,CAC/B,cAAmC,EACnC,kBAAgC;QAEhC,OAAO;YACL,MAAM,EAAE,cAAc,CAAC,MAAM;YAC7B,IAAI,EAAE;gBACJ,aAAa,EAAE,cAAc,CAAC,aAAa,IAAI,EAAE;gBACjD,YAAY,EAAE,cAAc,CAAC,YAAY,IAAI,EAAE;gBAC/C,iBAAiB,EAAE,cAAc,CAAC,iBAAiB,IAAI,EAAE;gBACzD,eAAe,EAAE,cAAc,CAAC,eAAe,IAAI,EAAE;gBACrD,mBAAmB,EAAE,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,IAAI,KAAK;gBAC1E,gBAAgB,EAAE,kBAAkB,CAAC,IAAI,EAAE,gBAAgB,IAAI,KAAK;aACrE;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CACjC,SAAuB,EACvB,cAAmC,EACnC,mBAAiC,EACjC,uBAAqC,EACrC,iBAA2B,EAC3B,YAAoB,EACpB,YAAyB,EACzB,OAAiB;QAEjB,MAAM,WAAW,GAAG,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAiB,EAAE,CAAC;QACxC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QAEvC,6DAA6D;QAC7D,MAAM,IAAI,CAAC,8BAA8B,CACvC,uBAAuB,EACvB,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,iBAAiB,EACjB,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,OAAO,CACR,CAAC;QAEF,+CAA+C;QAC/C,MAAM,IAAI,CAAC,yBAAyB,CAClC,SAAS,EACT,cAAc,EACd,mBAAmB,EACnB,iBAAiB,EACjB,YAAY,EACZ,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,OAAO,CACR,CAAC;QAEF,OAAO,cAAc,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,SAAuB;QACpD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;QAElD,KAAK,MAAM,aAAa,IAAI,SAAS,EAAE,CAAC;YACtC,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC;YAClE,IAAI,iBAAiB,EAAE,CAAC;gBACtB,WAAW,CAAC,GAAG,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,8BAA8B,CAC1C,uBAAqC,EACrC,WAAoC,EACpC,cAAmC,EACnC,mBAAiC,EACjC,iBAA2B,EAC3B,YAAoB,EACpB,YAAyB,EACzB,cAA4B,EAC5B,YAAyB,EACzB,OAAiB;QAEjB,KAAK,MAAM,qBAAqB,IAAI,uBAAuB,EAAE,CAAC;YAC5D,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,UAAU,EAAE,EAAE,CAAC;YAChE,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACxB,IAAI,CAAC,UAAU,CAAC,uDAAuD,EAAE,OAAO,CAAC,CAAC;gBAClF,SAAS;YACX,CAAC;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YACrD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,GAAG,CAAC,kBAAkB,kBAAkB,sDAAsD,CAAC,CAAC;gBAC1G,CAAC;gBACD,SAAS,CAAC,uBAAuB;YACnC,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,4BAA4B,CACxD,QAAQ,EACR,kBAAkB,EAClB,cAAc,EACd,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,YAAY,EACZ,YAAY,EACZ,OAAO,CACR,CAAC;YAEF,IAAI,UAAU,EAAE,CAAC;gBACf,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAChC,YAAY,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,4BAA4B,CACxC,QAAoB,EACpB,kBAA0B,EAC1B,cAAmC,EACnC,mBAAiC,EACjC,iBAA2B,EAC3B,qBAAiC,EACjC,YAAoB,EACpB,YAAyB,EACzB,OAAiB;QAEjB,MAAM,uBAAuB,GAAG,IAAI,CAAC,wBAAwB,CAC3D,kBAAkB,EAClB,QAAQ,EACR,cAAc,CAAC,MAAM,CACtB,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QAC3E,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,MAAM,iBAAiB,CAC5B,QAAQ,EACR,uBAAuB,EACvB,EAAE,EAAE,4CAA4C;QAChD,mBAAmB,EACnB,OAAO,EACP,KAAK,EAAE,2CAA2C;QAClD,qBAAqB,EAAE,0CAA0C;QACjE,YAAY,GAAG,CAAC,EAChB,YAAY,EACZ,cAAc,CAAC,yCAAyC;SACzD,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,yBAAyB,CACrC,SAAuB,EACvB,cAAmC,EACnC,mBAAiC,EACjC,iBAA2B,EAC3B,YAAoB,EACpB,YAAyB,EACzB,cAA4B,EAC5B,YAAyB,EACzB,OAAiB;QAEjB,KAAK,MAAM,aAAa,IAAI,SAAS,EAAE,CAAC;YACtC,MAAM,iBAAiB,GAAG,IAAI,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC;YAClE,IAAI,CAAC,iBAAiB,IAAI,YAAY,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC9D,SAAS,CAAC,iCAAiC;YAC7C,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACnD,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACnB,iBAAiB,EACjB,YAAY,EACZ,YAAY,EACZ,OAAO,CACR,CAAC;YAEF,IAAI,UAAU,EAAE,CAAC;gBACf,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAChC,YAAY,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,uBAAuB,CACnC,aAAyB,EACzB,iBAAyB,EACzB,cAAmC,EACnC,mBAAiC,EACjC,iBAA2B,EAC3B,YAAoB,EACpB,YAAyB,EACzB,OAAiB;QAEjB,MAAM,uBAAuB,GAAG,IAAI,CAAC,wBAAwB,CAC3D,iBAAiB,EACjB,aAAa,EACb,cAAc,CAAC,MAAM,CACtB,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QAChF,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,MAAM,iBAAiB,CAC5B,aAAa,EACb,uBAAuB,EACvB,EAAE,EAAE,4CAA4C;QAChD,mBAAmB,EACnB,OAAO,EACP,IAAI,EAAE,qCAAqC;QAC3C,SAAS,EAAE,mCAAmC;QAC9C,YAAY,GAAG,CAAC,EAChB,YAAY,EACZ,cAAc,CAAC,yCAAyC;SACzD,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,MAAkB,EAAE,cAAmC;QAClF,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,KAAK,SAAS,EAAE,CAAC;gBAClD,OAAO,EAAE,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,YAAY,EAAE,CAAC;YACvD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAK,MAAc,CAAC,cAAc,CAAC,UAAU,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC7D,OAAO,EAAE,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,YAAY,EAAE,CAAC;YACvD,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,wBAAwB,CAC9B,eAAuB,EACvB,MAAkB,EAClB,UAAkB;QAElB,MAAM,uBAAuB,GAAwB,EAAE,CAAC;QACxD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAE7D,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,WAAW,IAAI,EAAE,EAAE,CAAC;YAC/C,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gBACrB,uBAAuB,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACN,2DAA2D;gBAC3D,uBAAuB,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;QAED,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,MAAkB;QAC5C,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAEzB,yBAAyB;QACzB,IAAK,MAAc,CAAC,EAAE;YAAE,OAAQ,MAAc,CAAC,EAAE,CAAC;QAElD,2DAA2D;QAC3D,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,EAAE;gBAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAC9B,CAAC;QAED,wBAAwB;QACxB,IAAK,MAAc,CAAC,EAAE;YAAE,OAAQ,MAAc,CAAC,EAAE,CAAC;QAClD,IAAK,MAAc,CAAC,EAAE;YAAE,OAAQ,MAAc,CAAC,EAAE,CAAC;QAElD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,MAAkB,EAAE,SAAiB;QACzD,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAEzB,uEAAuE;QACvE,IAAK,MAAc,CAAC,SAAS,CAAC,KAAK,SAAS;YAAE,OAAQ,MAAc,CAAC,SAAS,CAAC,CAAC;QAEhF,2DAA2D;QAC3D,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,OAAe,EAAE,OAAiB;QACnD,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,QAAQ,CAAC,OAAe,EAAE,KAAU,EAAE,OAAiB;QAC7D,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,KAAK,KAAK,EAAE,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;CACF;AArcD,oDAqcC","sourcesContent":["import { BaseEntity, RunView, UserInfo } from '@memberjunction/core';\nimport { SyncEngine, RecordData } from '../lib/sync-engine';\nimport { RelatedEntityConfig, EntityConfig } from '../config';\n\n/**\n * Handles loading and processing of related entities for records\n */\nexport class RelatedEntityHandler {\n  constructor(\n    private syncEngine: SyncEngine,\n    private contextUser: UserInfo\n  ) {}\n\n  /**\n   * Load related entities for a record\n   */\n  async loadRelatedEntities(\n    parentRecord: BaseEntity,\n    relationConfig: RelatedEntityConfig,\n    parentEntityConfig: EntityConfig,\n    existingRelatedEntities: RecordData[],\n    processRecordData: (\n      record: BaseEntity,\n      primaryKey: Record<string, any>,\n      targetDir: string,\n      entityConfig: EntityConfig,\n      verbose?: boolean,\n      isNewRecord?: boolean,\n      existingRecordData?: RecordData,\n      currentDepth?: number,\n      ancestryPath?: Set<string>,\n      fieldOverrides?: Record<string, any>\n    ) => Promise<RecordData>,\n    currentDepth: number,\n    ancestryPath: Set<string>,\n    verbose?: boolean\n  ): Promise<RecordData[]> {\n    try {\n      const parentPrimaryKey = this.getRecordPrimaryKey(parentRecord);\n      if (!parentPrimaryKey) {\n        this.logWarning('Unable to determine primary key for parent record', verbose);\n        return [];\n      }\n\n      const relatedRecords = await this.queryRelatedEntities(\n        parentPrimaryKey, \n        relationConfig, \n        verbose\n      );\n\n      if (!relatedRecords) {\n        return [];\n      }\n\n      const relatedEntityConfig = this.createRelatedEntityConfig(relationConfig, parentEntityConfig);\n      \n      return await this.processRelatedRecords(\n        relatedRecords,\n        relationConfig,\n        relatedEntityConfig,\n        existingRelatedEntities,\n        processRecordData,\n        currentDepth,\n        ancestryPath,\n        verbose\n      );\n    } catch (error) {\n      this.logError(`Error loading related entities for ${relationConfig.entity}`, error, verbose);\n      return [];\n    }\n  }\n\n  /**\n   * Queries the database for related entities\n   */\n  private async queryRelatedEntities(\n    parentPrimaryKey: string,\n    relationConfig: RelatedEntityConfig,\n    verbose?: boolean\n  ): Promise<BaseEntity[] | null> {\n    const filter = this.buildRelatedEntityFilter(parentPrimaryKey, relationConfig);\n    \n    if (verbose) {\n      console.log(`Loading related entities: ${relationConfig.entity} with filter: ${filter}`);\n    }\n\n    const rv = new RunView();\n    const result = await rv.RunView({\n      EntityName: relationConfig.entity,\n      ExtraFilter: filter,\n      ResultType: 'entity_object'\n    }, this.contextUser);\n\n    if (!result.Success) {\n      this.logWarning(`Failed to load related entities ${relationConfig.entity}: ${result.ErrorMessage}`, verbose);\n      return null;\n    }\n\n    if (verbose) {\n      console.log(`Found ${result.Results.length} related records for ${relationConfig.entity}`);\n    }\n\n    return result.Results;\n  }\n\n  /**\n   * Builds the filter for querying related entities\n   */\n  private buildRelatedEntityFilter(parentPrimaryKey: string, relationConfig: RelatedEntityConfig): string {\n    let filter = `${relationConfig.foreignKey} = '${parentPrimaryKey}'`;\n    if (relationConfig.filter) {\n      filter += ` AND (${relationConfig.filter})`;\n    }\n    return filter;\n  }\n\n  /**\n   * Creates entity config for related entity processing\n   */\n  private createRelatedEntityConfig(\n    relationConfig: RelatedEntityConfig, \n    parentEntityConfig: EntityConfig\n  ): EntityConfig {\n    return {\n      entity: relationConfig.entity,\n      pull: {\n        excludeFields: relationConfig.excludeFields || [],\n        lookupFields: relationConfig.lookupFields || {},\n        externalizeFields: relationConfig.externalizeFields || [],\n        relatedEntities: relationConfig.relatedEntities || {},\n        ignoreVirtualFields: parentEntityConfig.pull?.ignoreVirtualFields || false,\n        ignoreNullFields: parentEntityConfig.pull?.ignoreNullFields || false\n      }\n    };\n  }\n\n  /**\n   * Processes all related records (both existing and new)\n   */\n  private async processRelatedRecords(\n    dbRecords: BaseEntity[],\n    relationConfig: RelatedEntityConfig,\n    relatedEntityConfig: EntityConfig,\n    existingRelatedEntities: RecordData[],\n    processRecordData: Function,\n    currentDepth: number,\n    ancestryPath: Set<string>,\n    verbose?: boolean\n  ): Promise<RecordData[]> {\n    const dbRecordMap = this.buildDatabaseRecordMap(dbRecords);\n    const relatedRecords: RecordData[] = [];\n    const processedIds = new Set<string>();\n\n    // Process existing related entities first (preserving order)\n    await this.processExistingRelatedEntities(\n      existingRelatedEntities,\n      dbRecordMap,\n      relationConfig,\n      relatedEntityConfig,\n      processRecordData,\n      currentDepth,\n      ancestryPath,\n      relatedRecords,\n      processedIds,\n      verbose\n    );\n\n    // Process new related entities (append to end)\n    await this.processNewRelatedEntities(\n      dbRecords,\n      relationConfig,\n      relatedEntityConfig,\n      processRecordData,\n      currentDepth,\n      ancestryPath,\n      relatedRecords,\n      processedIds,\n      verbose\n    );\n\n    return relatedRecords;\n  }\n\n  /**\n   * Builds a map of database records by primary key for efficient lookup\n   */\n  private buildDatabaseRecordMap(dbRecords: BaseEntity[]): Map<string, BaseEntity> {\n    const dbRecordMap = new Map<string, BaseEntity>();\n    \n    for (const relatedRecord of dbRecords) {\n      const relatedPrimaryKey = this.getRecordPrimaryKey(relatedRecord);\n      if (relatedPrimaryKey) {\n        dbRecordMap.set(relatedPrimaryKey, relatedRecord);\n      }\n    }\n    \n    return dbRecordMap;\n  }\n\n  /**\n   * Processes existing related entities\n   */\n  private async processExistingRelatedEntities(\n    existingRelatedEntities: RecordData[],\n    dbRecordMap: Map<string, BaseEntity>,\n    relationConfig: RelatedEntityConfig,\n    relatedEntityConfig: EntityConfig,\n    processRecordData: Function,\n    currentDepth: number,\n    ancestryPath: Set<string>,\n    relatedRecords: RecordData[],\n    processedIds: Set<string>,\n    verbose?: boolean\n  ): Promise<void> {\n    for (const existingRelatedEntity of existingRelatedEntities) {\n      const existingPrimaryKey = existingRelatedEntity.primaryKey?.ID;\n      if (!existingPrimaryKey) {\n        this.logWarning('Existing related entity missing primary key, skipping', verbose);\n        continue;\n      }\n\n      const dbRecord = dbRecordMap.get(existingPrimaryKey);\n      if (!dbRecord) {\n        if (verbose) {\n          console.log(`Related entity ${existingPrimaryKey} no longer exists in database, removing from results`);\n        }\n        continue; // Skip deleted records\n      }\n\n      const recordData = await this.processExistingRelatedEntity(\n        dbRecord,\n        existingPrimaryKey,\n        relationConfig,\n        relatedEntityConfig,\n        processRecordData,\n        existingRelatedEntity,\n        currentDepth,\n        ancestryPath,\n        verbose\n      );\n\n      if (recordData) {\n        relatedRecords.push(recordData);\n        processedIds.add(existingPrimaryKey);\n      }\n    }\n  }\n\n  /**\n   * Processes a single existing related entity\n   */\n  private async processExistingRelatedEntity(\n    dbRecord: BaseEntity,\n    existingPrimaryKey: string,\n    relationConfig: RelatedEntityConfig,\n    relatedEntityConfig: EntityConfig,\n    processRecordData: Function,\n    existingRelatedEntity: RecordData,\n    currentDepth: number,\n    ancestryPath: Set<string>,\n    verbose?: boolean\n  ): Promise<RecordData | null> {\n    const relatedRecordPrimaryKey = this.buildPrimaryKeyForRecord(\n      existingPrimaryKey, \n      dbRecord, \n      relationConfig.entity\n    );\n\n    const fieldOverrides = this.createFieldOverrides(dbRecord, relationConfig);\n    if (!fieldOverrides) {\n      return null;\n    }\n\n    return await processRecordData(\n      dbRecord,\n      relatedRecordPrimaryKey,\n      '', // targetDir not needed for related entities\n      relatedEntityConfig,\n      verbose,\n      false, // isNewRecord = false for existing records\n      existingRelatedEntity, // Pass existing data for change detection\n      currentDepth + 1,\n      ancestryPath,\n      fieldOverrides // Pass the field override for @parent:ID\n    );\n  }\n\n  /**\n   * Processes new related entities\n   */\n  private async processNewRelatedEntities(\n    dbRecords: BaseEntity[],\n    relationConfig: RelatedEntityConfig,\n    relatedEntityConfig: EntityConfig,\n    processRecordData: Function,\n    currentDepth: number,\n    ancestryPath: Set<string>,\n    relatedRecords: RecordData[],\n    processedIds: Set<string>,\n    verbose?: boolean\n  ): Promise<void> {\n    for (const relatedRecord of dbRecords) {\n      const relatedPrimaryKey = this.getRecordPrimaryKey(relatedRecord);\n      if (!relatedPrimaryKey || processedIds.has(relatedPrimaryKey)) {\n        continue; // Skip already processed records\n      }\n\n      const recordData = await this.processNewRelatedEntity(\n        relatedRecord,\n        relatedPrimaryKey,\n        relationConfig,\n        relatedEntityConfig,\n        processRecordData,\n        currentDepth,\n        ancestryPath,\n        verbose\n      );\n\n      if (recordData) {\n        relatedRecords.push(recordData);\n        processedIds.add(relatedPrimaryKey);\n      }\n    }\n  }\n\n  /**\n   * Processes a single new related entity\n   */\n  private async processNewRelatedEntity(\n    relatedRecord: BaseEntity,\n    relatedPrimaryKey: string,\n    relationConfig: RelatedEntityConfig,\n    relatedEntityConfig: EntityConfig,\n    processRecordData: Function,\n    currentDepth: number,\n    ancestryPath: Set<string>,\n    verbose?: boolean\n  ): Promise<RecordData | null> {\n    const relatedRecordPrimaryKey = this.buildPrimaryKeyForRecord(\n      relatedPrimaryKey, \n      relatedRecord, \n      relationConfig.entity\n    );\n\n    const fieldOverrides = this.createFieldOverrides(relatedRecord, relationConfig);\n    if (!fieldOverrides) {\n      return null;\n    }\n\n    return await processRecordData(\n      relatedRecord,\n      relatedRecordPrimaryKey,\n      '', // targetDir not needed for related entities\n      relatedEntityConfig,\n      verbose,\n      true, // isNewRecord = true for new records\n      undefined, // No existing data for new records\n      currentDepth + 1,\n      ancestryPath,\n      fieldOverrides // Pass the field override for @parent:ID\n    );\n  }\n\n  /**\n   * Creates field overrides for @parent:ID replacement\n   */\n  private createFieldOverrides(record: BaseEntity, relationConfig: RelatedEntityConfig): Record<string, any> | null {\n    if (typeof record.GetAll === 'function') {\n      const data = record.GetAll();\n      if (data[relationConfig.foreignKey] !== undefined) {\n        return { [relationConfig.foreignKey]: '@parent:ID' };\n      }\n    } else {\n      if ((record as any)[relationConfig.foreignKey] !== undefined) {\n        return { [relationConfig.foreignKey]: '@parent:ID' };\n      }\n    }\n    return null;\n  }\n\n  /**\n   * Builds primary key for a record\n   */\n  private buildPrimaryKeyForRecord(\n    primaryKeyValue: string, \n    record: BaseEntity, \n    entityName: string\n  ): Record<string, any> {\n    const relatedRecordPrimaryKey: Record<string, any> = {};\n    const entityInfo = this.syncEngine.getEntityInfo(entityName);\n    \n    for (const pk of entityInfo?.PrimaryKeys || []) {\n      if (pk.Name === 'ID') {\n        relatedRecordPrimaryKey[pk.Name] = primaryKeyValue;\n      } else {\n        // For compound keys, get the value from the related record\n        relatedRecordPrimaryKey[pk.Name] = this.getFieldValue(record, pk.Name);\n      }\n    }\n    \n    return relatedRecordPrimaryKey;\n  }\n\n  /**\n   * Get the primary key value from a record\n   */\n  private getRecordPrimaryKey(record: BaseEntity): string | null {\n    if (!record) return null;\n    \n    // Try to get ID directly\n    if ((record as any).ID) return (record as any).ID;\n    \n    // Try to get from GetAll() method if it's an entity object\n    if (typeof record.GetAll === 'function') {\n      const data = record.GetAll();\n      if (data.ID) return data.ID;\n    }\n    \n    // Try common variations\n    if ((record as any).id) return (record as any).id;\n    if ((record as any).Id) return (record as any).Id;\n    \n    return null;\n  }\n\n  /**\n   * Get a field value from a record, handling both entity objects and plain objects\n   */\n  private getFieldValue(record: BaseEntity, fieldName: string): any {\n    if (!record) return null;\n    \n    // Try to get field directly using bracket notation with type assertion\n    if ((record as any)[fieldName] !== undefined) return (record as any)[fieldName];\n    \n    // Try to get from GetAll() method if it's an entity object\n    if (typeof record.GetAll === 'function') {\n      const data = record.GetAll();\n      if (data[fieldName] !== undefined) return data[fieldName];\n    }\n    \n    return null;\n  }\n\n  /**\n   * Log warning message if verbose mode is enabled\n   */\n  private logWarning(message: string, verbose?: boolean): void {\n    if (verbose) {\n      console.warn(message);\n    }\n  }\n\n  /**\n   * Log error message if verbose mode is enabled\n   */\n  private logError(message: string, error: any, verbose?: boolean): void {\n    if (verbose) {\n      console.error(`${message}: ${error}`);\n    }\n  }\n}"]}

package/dist/lib/file-write-batch.d.ts

@@ -0,0 +1,61 @@
+import { RecordData } from './sync-engine';
+/**
+ * Batches file write operations to improve performance and ensure consistent property ordering.
+ * Collects all changes during processing and writes each file only once at the end.
+ */
+export declare class FileWriteBatch {
+    private changes;
+    private fileContents;
+    /**
+     * Queue a complete file write operation
+     * @param filePath - Path to the file
+     * @param data - RecordData or array of RecordData to write
+     */
+    queueWrite(filePath: string, data: RecordData | RecordData[]): void;
+    /**
+     * Queue an array record update operation
+     * @param filePath - Path to the file containing the array
+     * @param updatedRecord - The updated record data
+     * @param primaryKeyLookup - Primary key lookup string to identify the record
+     */
+    queueArrayUpdate(filePath: string, updatedRecord: RecordData, primaryKeyLookup: string): void;
+    /**
+     * Queue a single record update operation
+     * @param filePath - Path to the file
+     * @param updatedRecord - The updated record data
+     */
+    queueSingleUpdate(filePath: string, updatedRecord: RecordData): void;
+    /**
+     * Load and cache file contents if not already loaded
+     */
+    private ensureFileLoaded;
+    /**
+     * Apply all queued changes to in-memory file contents
+     */
+    private applyChanges;
+    /**
+     * Write all batched changes to files using JsonWriteHelper for consistent ordering
+     * @returns Number of files written
+     */
+    flush(): Promise<number>;
+    /**
+     * Clear all batched changes without writing
+     */
+    clear(): void;
+    /**
+     * Get the number of files that will be written
+     */
+    getPendingFileCount(): number;
+    /**
+     * Get all pending file paths
+     */
+    getPendingFiles(): string[];
+    /**
+     * Add a change to the batch
+     */
+    private addChange;
+    /**
+     * Create a primary key lookup string (same logic as PullService)
+     */
+    private createPrimaryKeyLookup;
+}

package/dist/lib/file-write-batch.js

@@ -0,0 +1,180 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileWriteBatch = void 0;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+const json_write_helper_1 = require("./json-write-helper");
+/**
+ * Batches file write operations to improve performance and ensure consistent property ordering.
+ * Collects all changes during processing and writes each file only once at the end.
+ */
+class FileWriteBatch {
+    changes = new Map();
+    fileContents = new Map();
+    /**
+     * Queue a complete file write operation
+     * @param filePath - Path to the file
+     * @param data - RecordData or array of RecordData to write
+     */
+    queueWrite(filePath, data) {
+        const absolutePath = path_1.default.resolve(filePath);
+        this.addChange(absolutePath, {
+            filePath: absolutePath,
+            operation: 'write',
+            data
+        });
+    }
+    /**
+     * Queue an array record update operation
+     * @param filePath - Path to the file containing the array
+     * @param updatedRecord - The updated record data
+     * @param primaryKeyLookup - Primary key lookup string to identify the record
+     */
+    queueArrayUpdate(filePath, updatedRecord, primaryKeyLookup) {
+        const absolutePath = path_1.default.resolve(filePath);
+        this.addChange(absolutePath, {
+            filePath: absolutePath,
+            operation: 'update-array',
+            data: updatedRecord,
+            primaryKeyLookup
+        });
+    }
+    /**
+     * Queue a single record update operation
+     * @param filePath - Path to the file
+     * @param updatedRecord - The updated record data
+     */
+    queueSingleUpdate(filePath, updatedRecord) {
+        const absolutePath = path_1.default.resolve(filePath);
+        this.addChange(absolutePath, {
+            filePath: absolutePath,
+            operation: 'update-single',
+            data: updatedRecord
+        });
+    }
+    /**
+     * Load and cache file contents if not already loaded
+     */
+    async ensureFileLoaded(filePath) {
+        if (this.fileContents.has(filePath)) {
+            return;
+        }
+        try {
+            if (await fs_extra_1.default.pathExists(filePath)) {
+                const content = await fs_extra_1.default.readJson(filePath);
+                this.fileContents.set(filePath, content);
+            }
+        }
+        catch (error) {
+            // If file doesn't exist or can't be read, we'll create it fresh
+            // Don't throw here - let the calling code handle it
+        }
+    }
+    /**
+     * Apply all queued changes to in-memory file contents
+     */
+    async applyChanges() {
+        for (const [filePath, changes] of this.changes) {
+            await this.ensureFileLoaded(filePath);
+            let currentContent = this.fileContents.get(filePath) || [];
+            for (const change of changes) {
+                switch (change.operation) {
+                    case 'write':
+                        // Complete overwrite
+                        currentContent = change.data;
+                        break;
+                    case 'update-array':
+                        // Update a specific record in an array
+                        if (Array.isArray(currentContent) && change.primaryKeyLookup) {
+                            const index = currentContent.findIndex(r => this.createPrimaryKeyLookup(r.primaryKey || {}) === change.primaryKeyLookup);
+                            if (index >= 0) {
+                                currentContent[index] = change.data;
+                            }
+                            else {
+                                // Record not found, append it
+                                currentContent.push(change.data);
+                            }
+                        }
+                        else {
+                            // File doesn't contain an array, make it one
+                            currentContent = [change.data];
+                        }
+                        break;
+                    case 'update-single':
+                        // Replace the entire file content with a single record
+                        currentContent = change.data;
+                        break;
+                }
+            }
+            // Update the in-memory content
+            if (currentContent !== undefined) {
+                this.fileContents.set(filePath, currentContent);
+            }
+        }
+    }
+    /**
+     * Write all batched changes to files using JsonWriteHelper for consistent ordering
+     * @returns Number of files written
+     */
+    async flush() {
+        if (this.changes.size === 0) {
+            return 0;
+        }
+        // Apply all changes to in-memory content first
+        await this.applyChanges();
+        // Write all files using JsonWriteHelper
+        const writePromises = [];
+        for (const [filePath, content] of this.fileContents.entries()) {
+            // Ensure directory exists
+            const dir = path_1.default.dirname(filePath);
+            await fs_extra_1.default.ensureDir(dir);
+            // Write using JsonWriteHelper for consistent property ordering
+            writePromises.push(json_write_helper_1.JsonWriteHelper.writeOrderedRecordData(filePath, content));
+        }
+        await Promise.all(writePromises);
+        const filesWritten = this.fileContents.size;
+        // Clear all batched changes
+        this.clear();
+        return filesWritten;
+    }
+    /**
+     * Clear all batched changes without writing
+     */
+    clear() {
+        this.changes.clear();
+        this.fileContents.clear();
+    }
+    /**
+     * Get the number of files that will be written
+     */
+    getPendingFileCount() {
+        return this.changes.size;
+    }
+    /**
+     * Get all pending file paths
+     */
+    getPendingFiles() {
+        return Array.from(this.changes.keys());
+    }
+    /**
+     * Add a change to the batch
+     */
+    addChange(filePath, change) {
+        if (!this.changes.has(filePath)) {
+            this.changes.set(filePath, []);
+        }
+        this.changes.get(filePath).push(change);
+    }
+    /**
+     * Create a primary key lookup string (same logic as PullService)
+     */
+    createPrimaryKeyLookup(primaryKey) {
+        const keys = Object.keys(primaryKey).sort();
+        return keys.map(k => `${k}:${primaryKey[k]}`).join('|');
+    }
+}
+exports.FileWriteBatch = FileWriteBatch;
+//# sourceMappingURL=file-write-batch.js.map

package/dist/lib/file-write-batch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-write-batch.js","sourceRoot":"","sources":["../../src/lib/file-write-batch.ts"],"names":[],"mappings":";;;;;;AAAA,wDAA0B;AAC1B,gDAAwB;AAExB,2DAAsD;AAkBtD;;;GAGG;AACH,MAAa,cAAc;IACjB,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC1C,YAAY,GAAG,IAAI,GAAG,EAAqC,CAAC;IAEpE;;;;OAIG;IACH,UAAU,CAAC,QAAgB,EAAE,IAA+B;QAC1D,MAAM,YAAY,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;YAC3B,QAAQ,EAAE,YAAY;YACtB,SAAS,EAAE,OAAO;YAClB,IAAI;SACL,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,gBAAgB,CAAC,QAAgB,EAAE,aAAyB,EAAE,gBAAwB;QACpF,MAAM,YAAY,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;YAC3B,QAAQ,EAAE,YAAY;YACtB,SAAS,EAAE,cAAc;YACzB,IAAI,EAAE,aAAa;YACnB,gBAAgB;SACjB,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,QAAgB,EAAE,aAAyB;QAC3D,MAAM,YAAY,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;YAC3B,QAAQ,EAAE,YAAY;YACtB,SAAS,EAAE,eAAe;YAC1B,IAAI,EAAE,aAAa;SACpB,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QAC7C,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,IAAI,MAAM,kBAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBAC5C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gEAAgE;YAChE,oDAAoD;QACtD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAEtC,IAAI,cAAc,GAA8B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC;oBACzB,KAAK,OAAO;wBACV,qBAAqB;wBACrB,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC;wBAC7B,MAAM;oBAER,KAAK,cAAc;wBACjB,uCAAuC;wBACvC,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;4BAC7D,MAAM,KAAK,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CACzC,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,KAAK,MAAM,CAAC,gBAAgB,CAC5E,CAAC;4BAEF,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gCACf,cAAc,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,IAAkB,CAAC;4BACpD,CAAC;iCAAM,CAAC;gCACN,8BAA8B;gCAC9B,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,IAAkB,CAAC,CAAC;4BACjD,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,6CAA6C;4BAC7C,cAAc,GAAG,CAAC,MAAM,CAAC,IAAkB,CAAC,CAAC;wBAC/C,CAAC;wBACD,MAAM;oBAER,KAAK,eAAe;wBAClB,uDAAuD;wBACvD,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC;wBAC7B,MAAM;gBACV,CAAC;YACH,CAAC;YAED,+BAA+B;YAC/B,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;gBACjC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,+CAA+C;QAC/C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAE1B,wCAAwC;QACxC,MAAM,aAAa,GAAoB,EAAE,CAAC;QAE1C,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9D,0BAA0B;YAC1B,MAAM,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnC,MAAM,kBAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAExB,+DAA+D;YAC/D,aAAa,CAAC,IAAI,CAChB,mCAAe,CAAC,sBAAsB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAC1D,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAEjC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;QAE5C,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,mBAAmB;QACjB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,QAAgB,EAAE,MAAkB;QACpD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACjC,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,UAA+B;QAC5D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1D,CAAC;CACF;AA/LD,wCA+LC","sourcesContent":["import fs from 'fs-extra';\nimport path from 'path';\nimport { RecordData } from './sync-engine';\nimport { JsonWriteHelper } from './json-write-helper';\n\n/**\n * Represents a pending change to a file\n */\ninterface FileChange {\n  /** Path to the file */\n  filePath: string;\n  /** Type of change operation */\n  operation: 'write' | 'update-array' | 'update-single';\n  /** The data to write */\n  data: RecordData | RecordData[];\n  /** For array updates: the index of the record to update */\n  arrayIndex?: number;\n  /** For updates: the primary key lookup to identify the record */\n  primaryKeyLookup?: string;\n}\n\n/**\n * Batches file write operations to improve performance and ensure consistent property ordering.\n * Collects all changes during processing and writes each file only once at the end.\n */\nexport class FileWriteBatch {\n  private changes = new Map<string, FileChange[]>();\n  private fileContents = new Map<string, RecordData | RecordData[]>();\n  \n  /**\n   * Queue a complete file write operation\n   * @param filePath - Path to the file\n   * @param data - RecordData or array of RecordData to write\n   */\n  queueWrite(filePath: string, data: RecordData | RecordData[]): void {\n    const absolutePath = path.resolve(filePath);\n    this.addChange(absolutePath, {\n      filePath: absolutePath,\n      operation: 'write',\n      data\n    });\n  }\n  \n  /**\n   * Queue an array record update operation\n   * @param filePath - Path to the file containing the array\n   * @param updatedRecord - The updated record data\n   * @param primaryKeyLookup - Primary key lookup string to identify the record\n   */\n  queueArrayUpdate(filePath: string, updatedRecord: RecordData, primaryKeyLookup: string): void {\n    const absolutePath = path.resolve(filePath);\n    this.addChange(absolutePath, {\n      filePath: absolutePath,\n      operation: 'update-array',\n      data: updatedRecord,\n      primaryKeyLookup\n    });\n  }\n  \n  /**\n   * Queue a single record update operation\n   * @param filePath - Path to the file\n   * @param updatedRecord - The updated record data\n   */\n  queueSingleUpdate(filePath: string, updatedRecord: RecordData): void {\n    const absolutePath = path.resolve(filePath);\n    this.addChange(absolutePath, {\n      filePath: absolutePath,\n      operation: 'update-single',\n      data: updatedRecord\n    });\n  }\n  \n  /**\n   * Load and cache file contents if not already loaded\n   */\n  private async ensureFileLoaded(filePath: string): Promise<void> {\n    if (this.fileContents.has(filePath)) {\n      return;\n    }\n    \n    try {\n      if (await fs.pathExists(filePath)) {\n        const content = await fs.readJson(filePath);\n        this.fileContents.set(filePath, content);\n      }\n    } catch (error) {\n      // If file doesn't exist or can't be read, we'll create it fresh\n      // Don't throw here - let the calling code handle it\n    }\n  }\n  \n  /**\n   * Apply all queued changes to in-memory file contents\n   */\n  private async applyChanges(): Promise<void> {\n    for (const [filePath, changes] of this.changes) {\n      await this.ensureFileLoaded(filePath);\n      \n      let currentContent: RecordData | RecordData[] = this.fileContents.get(filePath) || [];\n      \n      for (const change of changes) {\n        switch (change.operation) {\n          case 'write':\n            // Complete overwrite\n            currentContent = change.data;\n            break;\n            \n          case 'update-array':\n            // Update a specific record in an array\n            if (Array.isArray(currentContent) && change.primaryKeyLookup) {\n              const index = currentContent.findIndex(r => \n                this.createPrimaryKeyLookup(r.primaryKey || {}) === change.primaryKeyLookup\n              );\n              \n              if (index >= 0) {\n                currentContent[index] = change.data as RecordData;\n              } else {\n                // Record not found, append it\n                currentContent.push(change.data as RecordData);\n              }\n            } else {\n              // File doesn't contain an array, make it one\n              currentContent = [change.data as RecordData];\n            }\n            break;\n            \n          case 'update-single':\n            // Replace the entire file content with a single record\n            currentContent = change.data;\n            break;\n        }\n      }\n      \n      // Update the in-memory content\n      if (currentContent !== undefined) {\n        this.fileContents.set(filePath, currentContent);\n      }\n    }\n  }\n  \n  /**\n   * Write all batched changes to files using JsonWriteHelper for consistent ordering\n   * @returns Number of files written\n   */\n  async flush(): Promise<number> {\n    if (this.changes.size === 0) {\n      return 0;\n    }\n    \n    // Apply all changes to in-memory content first\n    await this.applyChanges();\n    \n    // Write all files using JsonWriteHelper\n    const writePromises: Promise<void>[] = [];\n    \n    for (const [filePath, content] of this.fileContents.entries()) {\n      // Ensure directory exists\n      const dir = path.dirname(filePath);\n      await fs.ensureDir(dir);\n      \n      // Write using JsonWriteHelper for consistent property ordering\n      writePromises.push(\n        JsonWriteHelper.writeOrderedRecordData(filePath, content)\n      );\n    }\n    \n    await Promise.all(writePromises);\n    \n    const filesWritten = this.fileContents.size;\n    \n    // Clear all batched changes\n    this.clear();\n    \n    return filesWritten;\n  }\n  \n  /**\n   * Clear all batched changes without writing\n   */\n  clear(): void {\n    this.changes.clear();\n    this.fileContents.clear();\n  }\n  \n  /**\n   * Get the number of files that will be written\n   */\n  getPendingFileCount(): number {\n    return this.changes.size;\n  }\n  \n  /**\n   * Get all pending file paths\n   */\n  getPendingFiles(): string[] {\n    return Array.from(this.changes.keys());\n  }\n  \n  /**\n   * Add a change to the batch\n   */\n  private addChange(filePath: string, change: FileChange): void {\n    if (!this.changes.has(filePath)) {\n      this.changes.set(filePath, []);\n    }\n    this.changes.get(filePath)!.push(change);\n  }\n  \n  /**\n   * Create a primary key lookup string (same logic as PullService)\n   */\n  private createPrimaryKeyLookup(primaryKey: Record<string, any>): string {\n    const keys = Object.keys(primaryKey).sort();\n    return keys.map(k => `${k}:${primaryKey[k]}`).join('|');\n  }\n}"]}