@memberjunction/metadata-sync 2.67.0 → 2.68.0

package/dist/lib/FieldExternalizer.js

@@ -0,0 +1,177 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FieldExternalizer = void 0;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+/**
+ * Handles externalization of field values to separate files with @file: references
+ */
+class FieldExternalizer {
+    /**
+     * Externalize a field value to a separate file and return @file: reference
+     */
+    async externalizeField(fieldName, fieldValue, pattern, recordData, targetDir, existingFileReference, mergeStrategy = 'merge', verbose) {
+        const { finalFilePath, fileReference } = this.determineFilePath(pattern, recordData, targetDir, existingFileReference, mergeStrategy, fieldName, verbose);
+        const shouldWrite = await this.shouldWriteFile(finalFilePath, fieldValue, fieldName);
+        if (shouldWrite) {
+            await this.writeExternalFile(finalFilePath, fieldValue, fieldName, verbose);
+        }
+        else if (verbose) {
+            console.log(`External file ${finalFilePath} unchanged, skipping write`);
+        }
+        return fileReference;
+    }
+    /**
+     * Determines the file path and reference for externalization
+     */
+    determineFilePath(pattern, recordData, targetDir, existingFileReference, mergeStrategy = 'merge', fieldName = '', verbose) {
+        if (this.shouldUseExistingReference(existingFileReference, mergeStrategy)) {
+            return this.useExistingFileReference(existingFileReference, targetDir, verbose);
+        }
+        return this.createNewFileReference(pattern, recordData, targetDir, fieldName, verbose);
+    }
+    /**
+     * Checks if we should use an existing file reference
+     */
+    shouldUseExistingReference(existingFileReference, mergeStrategy = 'merge') {
+        return mergeStrategy === 'merge' &&
+            !!existingFileReference &&
+            typeof existingFileReference === 'string' &&
+            existingFileReference.startsWith('@file:');
+    }
+    /**
+     * Uses an existing file reference
+     */
+    useExistingFileReference(existingFileReference, targetDir, verbose) {
+        const existingPath = existingFileReference.substring(6); // Remove @file: prefix
+        const finalFilePath = path_1.default.resolve(targetDir, existingPath);
+        if (verbose) {
+            console.log(`Using existing external file: ${finalFilePath}`);
+        }
+        return { finalFilePath, fileReference: existingFileReference };
+    }
+    /**
+     * Creates a new file reference using the pattern
+     */
+    createNewFileReference(pattern, recordData, targetDir, fieldName, verbose) {
+        const processedPattern = this.processPattern(pattern, recordData, fieldName);
+        const cleanPattern = this.removeFilePrefix(processedPattern);
+        const finalFilePath = path_1.default.resolve(targetDir, cleanPattern);
+        const fileReference = `@file:${cleanPattern}`;
+        if (verbose) {
+            console.log(`Creating new external file: ${finalFilePath}`);
+        }
+        return { finalFilePath, fileReference };
+    }
+    /**
+     * Processes pattern placeholders with actual values
+     */
+    processPattern(pattern, recordData, fieldName) {
+        let processedPattern = pattern;
+        // Replace common placeholders
+        processedPattern = this.replacePlaceholder(processedPattern, 'Name', recordData.Name);
+        processedPattern = this.replacePlaceholder(processedPattern, 'ID', recordData.ID);
+        processedPattern = this.replacePlaceholder(processedPattern, 'FieldName', fieldName);
+        // Replace any other field placeholders
+        processedPattern = this.replaceFieldPlaceholders(processedPattern, recordData);
+        return processedPattern;
+    }
+    /**
+     * Replaces a single placeholder in the pattern
+     */
+    replacePlaceholder(pattern, placeholder, value) {
+        if (value != null) {
+            const sanitizedValue = this.sanitizeForFilename(String(value));
+            return pattern.replace(new RegExp(`\\{${placeholder}\\}`, 'g'), sanitizedValue);
+        }
+        return pattern;
+    }
+    /**
+     * Replaces field placeholders with values from the record
+     */
+    replaceFieldPlaceholders(pattern, recordData) {
+        let processedPattern = pattern;
+        for (const [key, value] of Object.entries(recordData)) {
+            if (value != null) {
+                const sanitizedValue = this.sanitizeForFilename(String(value));
+                processedPattern = processedPattern.replace(new RegExp(`\\{${key}\\}`, 'g'), sanitizedValue);
+            }
+        }
+        return processedPattern;
+    }
+    /**
+     * Removes @file: prefix if present
+     */
+    removeFilePrefix(pattern) {
+        return pattern.startsWith('@file:') ? pattern.substring(6) : pattern;
+    }
+    /**
+     * Determines if the file should be written based on content comparison
+     */
+    async shouldWriteFile(finalFilePath, fieldValue, fieldName) {
+        if (!(await fs_extra_1.default.pathExists(finalFilePath))) {
+            return true; // File doesn't exist, should write
+        }
+        try {
+            const existingContent = await fs_extra_1.default.readFile(finalFilePath, 'utf8');
+            const contentToWrite = this.prepareContentForWriting(fieldValue, fieldName);
+            return existingContent !== contentToWrite;
+        }
+        catch (error) {
+            return true; // Error reading existing file, should write
+        }
+    }
+    /**
+     * Writes the external file with the field content
+     */
+    async writeExternalFile(finalFilePath, fieldValue, fieldName, verbose) {
+        // Ensure the directory exists
+        await fs_extra_1.default.ensureDir(path_1.default.dirname(finalFilePath));
+        // Write the field value to the file
+        const contentToWrite = this.prepareContentForWriting(fieldValue, fieldName);
+        await fs_extra_1.default.writeFile(finalFilePath, contentToWrite, 'utf8');
+        if (verbose) {
+            console.log(`Wrote externalized field ${fieldName} to ${finalFilePath}`);
+        }
+    }
+    /**
+     * Prepares content for writing, with JSON pretty-printing if applicable
+     */
+    prepareContentForWriting(fieldValue, fieldName) {
+        let contentToWrite = String(fieldValue);
+        // If the value looks like JSON, try to pretty-print it
+        if (this.shouldPrettyPrintAsJson(fieldName)) {
+            try {
+                const parsed = JSON.parse(contentToWrite);
+                contentToWrite = JSON.stringify(parsed, null, 2);
+            }
+            catch {
+                // Not valid JSON, use as-is
+            }
+        }
+        return contentToWrite;
+    }
+    /**
+     * Determines if content should be pretty-printed as JSON
+     */
+    shouldPrettyPrintAsJson(fieldName) {
+        const lowerFieldName = fieldName.toLowerCase();
+        return lowerFieldName.includes('json') || lowerFieldName.includes('example');
+    }
+    /**
+     * Sanitize a string for use in filenames
+     */
+    sanitizeForFilename(input) {
+        return input
+            .toLowerCase()
+            .replace(/\s+/g, '-') // Replace spaces with hyphens
+            .replace(/[^a-z0-9.-]/g, '') // Remove special characters except dots and hyphens
+            .replace(/--+/g, '-') // Replace multiple hyphens with single hyphen
+            .replace(/^-+|-+$/g, ''); // Remove leading/trailing hyphens
+    }
+}
+exports.FieldExternalizer = FieldExternalizer;
+//# sourceMappingURL=FieldExternalizer.js.map

package/dist/lib/FieldExternalizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FieldExternalizer.js","sourceRoot":"","sources":["../../src/lib/FieldExternalizer.ts"],"names":[],"mappings":";;;;;;AAAA,wDAA0B;AAC1B,gDAAwB;AAGxB;;GAEG;AACH,MAAa,iBAAiB;IAC5B;;OAEG;IACH,KAAK,CAAC,gBAAgB,CACpB,SAAiB,EACjB,UAAe,EACf,OAAe,EACf,UAAsB,EACtB,SAAiB,EACjB,qBAA8B,EAC9B,gBAAwB,OAAO,EAC/B,OAAiB;QAEjB,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAC7D,OAAO,EACP,UAAU,EACV,SAAS,EACT,qBAAqB,EACrB,aAAa,EACb,SAAS,EACT,OAAO,CACR,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;QAErF,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC9E,CAAC;aAAM,IAAI,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,iBAAiB,aAAa,4BAA4B,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,iBAAiB,CACvB,OAAe,EACf,UAAsB,EACtB,SAAiB,EACjB,qBAA8B,EAC9B,gBAAwB,OAAO,EAC/B,YAAoB,EAAE,EACtB,OAAiB;QAEjB,IAAI,IAAI,CAAC,0BAA0B,CAAC,qBAAqB,EAAE,aAAa,CAAC,EAAE,CAAC;YAC1E,OAAO,IAAI,CAAC,wBAAwB,CAAC,qBAAsB,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QACnF,CAAC;QAED,OAAO,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACzF,CAAC;IAED;;OAEG;IACK,0BAA0B,CAAC,qBAA8B,EAAE,gBAAwB,OAAO;QAChG,OAAO,aAAa,KAAK,OAAO;YACzB,CAAC,CAAC,qBAAqB;YACvB,OAAO,qBAAqB,KAAK,QAAQ;YACzC,qBAAqB,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACK,wBAAwB,CAC9B,qBAA6B,EAC7B,SAAiB,EACjB,OAAiB;QAEjB,MAAM,YAAY,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB;QAChF,MAAM,aAAa,GAAG,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAE5D,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,iCAAiC,aAAa,EAAE,CAAC,CAAC;QAChE,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,qBAAqB,EAAE,CAAC;IACjE,CAAC;IAED;;OAEG;IACK,sBAAsB,CAC5B,OAAe,EACf,UAAsB,EACtB,SAAiB,EACjB,SAAiB,EACjB,OAAiB;QAEjB,MAAM,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;QAC7E,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;QAC7D,MAAM,aAAa,GAAG,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAC5D,MAAM,aAAa,GAAG,SAAS,YAAY,EAAE,CAAC;QAE9C,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,+BAA+B,aAAa,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,OAAe,EAAE,UAAsB,EAAE,SAAiB;QAC/E,IAAI,gBAAgB,GAAG,OAAO,CAAC;QAE/B,8BAA8B;QAC9B,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,MAAM,EAAG,UAAkB,CAAC,IAAI,CAAC,CAAC;QAC/F,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,IAAI,EAAG,UAAkB,CAAC,EAAE,CAAC,CAAC;QAC3F,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QAErF,uCAAuC;QACvC,gBAAgB,GAAG,IAAI,CAAC,wBAAwB,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;QAE/E,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,OAAe,EAAE,WAAmB,EAAE,KAAU;QACzE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/D,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,KAAK,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC;QAClF,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,wBAAwB,CAAC,OAAe,EAAE,UAAsB;QACtE,IAAI,gBAAgB,GAAG,OAAO,CAAC;QAE/B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAiB,CAAC,EAAE,CAAC;YAC7D,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC/D,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,KAAK,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,OAAe;QACtC,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACvE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAAC,aAAqB,EAAE,UAAe,EAAE,SAAiB;QACrF,IAAI,CAAC,CAAC,MAAM,kBAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;YAC1C,OAAO,IAAI,CAAC,CAAC,mCAAmC;QAClD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YACjE,MAAM,cAAc,GAAG,IAAI,CAAC,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAE5E,OAAO,eAAe,KAAK,cAAc,CAAC;QAC5C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,CAAC,4CAA4C;QAC3D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC7B,aAAqB,EACrB,UAAe,EACf,SAAiB,EACjB,OAAiB;QAEjB,8BAA8B;QAC9B,MAAM,kBAAE,CAAC,SAAS,CAAC,cAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;QAEhD,oCAAoC;QACpC,MAAM,cAAc,GAAG,IAAI,CAAC,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC5E,MAAM,kBAAE,CAAC,SAAS,CAAC,aAAa,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;QAE1D,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,OAAO,aAAa,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED;;OAEG;IACK,wBAAwB,CAAC,UAAe,EAAE,SAAiB;QACjE,IAAI,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QAExC,uDAAuD;QACvD,IAAI,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;gBAC1C,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACnD,CAAC;YAAC,MAAM,CAAC;gBACP,4BAA4B;YAC9B,CAAC;QACH,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,uBAAuB,CAAC,SAAiB;QAC/C,MAAM,cAAc,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;QAC/C,OAAO,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,KAAa;QACvC,OAAO,KAAK;aACT,WAAW,EAAE;aACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,8BAA8B;aACnD,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,oDAAoD;aAChF,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,8CAA8C;aACnE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,kCAAkC;IAChE,CAAC;CACF;AAxOD,8CAwOC","sourcesContent":["import fs from 'fs-extra';\nimport path from 'path';\nimport { BaseEntity } from '@memberjunction/core';\n\n/**\n * Handles externalization of field values to separate files with @file: references\n */\nexport class FieldExternalizer {\n  /**\n   * Externalize a field value to a separate file and return @file: reference\n   */\n  async externalizeField(\n    fieldName: string,\n    fieldValue: any,\n    pattern: string,\n    recordData: BaseEntity,\n    targetDir: string,\n    existingFileReference?: string,\n    mergeStrategy: string = 'merge',\n    verbose?: boolean\n  ): Promise<string> {\n    const { finalFilePath, fileReference } = this.determineFilePath(\n      pattern, \n      recordData, \n      targetDir, \n      existingFileReference, \n      mergeStrategy, \n      fieldName, \n      verbose\n    );\n    \n    const shouldWrite = await this.shouldWriteFile(finalFilePath, fieldValue, fieldName);\n    \n    if (shouldWrite) {\n      await this.writeExternalFile(finalFilePath, fieldValue, fieldName, verbose);\n    } else if (verbose) {\n      console.log(`External file ${finalFilePath} unchanged, skipping write`);\n    }\n    \n    return fileReference;\n  }\n\n  /**\n   * Determines the file path and reference for externalization\n   */\n  private determineFilePath(\n    pattern: string,\n    recordData: BaseEntity,\n    targetDir: string,\n    existingFileReference?: string,\n    mergeStrategy: string = 'merge',\n    fieldName: string = '',\n    verbose?: boolean\n  ): { finalFilePath: string; fileReference: string } {\n    if (this.shouldUseExistingReference(existingFileReference, mergeStrategy)) {\n      return this.useExistingFileReference(existingFileReference!, targetDir, verbose);\n    }\n    \n    return this.createNewFileReference(pattern, recordData, targetDir, fieldName, verbose);\n  }\n\n  /**\n   * Checks if we should use an existing file reference\n   */\n  private shouldUseExistingReference(existingFileReference?: string, mergeStrategy: string = 'merge'): boolean {\n    return mergeStrategy === 'merge' && \n           !!existingFileReference && \n           typeof existingFileReference === 'string' && \n           existingFileReference.startsWith('@file:');\n  }\n\n  /**\n   * Uses an existing file reference\n   */\n  private useExistingFileReference(\n    existingFileReference: string, \n    targetDir: string, \n    verbose?: boolean\n  ): { finalFilePath: string; fileReference: string } {\n    const existingPath = existingFileReference.substring(6); // Remove @file: prefix\n    const finalFilePath = path.resolve(targetDir, existingPath);\n    \n    if (verbose) {\n      console.log(`Using existing external file: ${finalFilePath}`);\n    }\n    \n    return { finalFilePath, fileReference: existingFileReference };\n  }\n\n  /**\n   * Creates a new file reference using the pattern\n   */\n  private createNewFileReference(\n    pattern: string,\n    recordData: BaseEntity,\n    targetDir: string,\n    fieldName: string,\n    verbose?: boolean\n  ): { finalFilePath: string; fileReference: string } {\n    const processedPattern = this.processPattern(pattern, recordData, fieldName);\n    const cleanPattern = this.removeFilePrefix(processedPattern);\n    const finalFilePath = path.resolve(targetDir, cleanPattern);\n    const fileReference = `@file:${cleanPattern}`;\n    \n    if (verbose) {\n      console.log(`Creating new external file: ${finalFilePath}`);\n    }\n    \n    return { finalFilePath, fileReference };\n  }\n\n  /**\n   * Processes pattern placeholders with actual values\n   */\n  private processPattern(pattern: string, recordData: BaseEntity, fieldName: string): string {\n    let processedPattern = pattern;\n    \n    // Replace common placeholders\n    processedPattern = this.replacePlaceholder(processedPattern, 'Name', (recordData as any).Name);\n    processedPattern = this.replacePlaceholder(processedPattern, 'ID', (recordData as any).ID);\n    processedPattern = this.replacePlaceholder(processedPattern, 'FieldName', fieldName);\n    \n    // Replace any other field placeholders\n    processedPattern = this.replaceFieldPlaceholders(processedPattern, recordData);\n    \n    return processedPattern;\n  }\n\n  /**\n   * Replaces a single placeholder in the pattern\n   */\n  private replacePlaceholder(pattern: string, placeholder: string, value: any): string {\n    if (value != null) {\n      const sanitizedValue = this.sanitizeForFilename(String(value));\n      return pattern.replace(new RegExp(`\\\\{${placeholder}\\\\}`, 'g'), sanitizedValue);\n    }\n    return pattern;\n  }\n\n  /**\n   * Replaces field placeholders with values from the record\n   */\n  private replaceFieldPlaceholders(pattern: string, recordData: BaseEntity): string {\n    let processedPattern = pattern;\n    \n    for (const [key, value] of Object.entries(recordData as any)) {\n      if (value != null) {\n        const sanitizedValue = this.sanitizeForFilename(String(value));\n        processedPattern = processedPattern.replace(new RegExp(`\\\\{${key}\\\\}`, 'g'), sanitizedValue);\n      }\n    }\n    \n    return processedPattern;\n  }\n\n  /**\n   * Removes @file: prefix if present\n   */\n  private removeFilePrefix(pattern: string): string {\n    return pattern.startsWith('@file:') ? pattern.substring(6) : pattern;\n  }\n\n  /**\n   * Determines if the file should be written based on content comparison\n   */\n  private async shouldWriteFile(finalFilePath: string, fieldValue: any, fieldName: string): Promise<boolean> {\n    if (!(await fs.pathExists(finalFilePath))) {\n      return true; // File doesn't exist, should write\n    }\n    \n    try {\n      const existingContent = await fs.readFile(finalFilePath, 'utf8');\n      const contentToWrite = this.prepareContentForWriting(fieldValue, fieldName);\n      \n      return existingContent !== contentToWrite;\n    } catch (error) {\n      return true; // Error reading existing file, should write\n    }\n  }\n\n  /**\n   * Writes the external file with the field content\n   */\n  private async writeExternalFile(\n    finalFilePath: string, \n    fieldValue: any, \n    fieldName: string, \n    verbose?: boolean\n  ): Promise<void> {\n    // Ensure the directory exists\n    await fs.ensureDir(path.dirname(finalFilePath));\n    \n    // Write the field value to the file\n    const contentToWrite = this.prepareContentForWriting(fieldValue, fieldName);\n    await fs.writeFile(finalFilePath, contentToWrite, 'utf8');\n    \n    if (verbose) {\n      console.log(`Wrote externalized field ${fieldName} to ${finalFilePath}`);\n    }\n  }\n\n  /**\n   * Prepares content for writing, with JSON pretty-printing if applicable\n   */\n  private prepareContentForWriting(fieldValue: any, fieldName: string): string {\n    let contentToWrite = String(fieldValue);\n    \n    // If the value looks like JSON, try to pretty-print it\n    if (this.shouldPrettyPrintAsJson(fieldName)) {\n      try {\n        const parsed = JSON.parse(contentToWrite);\n        contentToWrite = JSON.stringify(parsed, null, 2);\n      } catch {\n        // Not valid JSON, use as-is\n      }\n    }\n    \n    return contentToWrite;\n  }\n\n  /**\n   * Determines if content should be pretty-printed as JSON\n   */\n  private shouldPrettyPrintAsJson(fieldName: string): boolean {\n    const lowerFieldName = fieldName.toLowerCase();\n    return lowerFieldName.includes('json') || lowerFieldName.includes('example');\n  }\n\n  /**\n   * Sanitize a string for use in filenames\n   */\n  private sanitizeForFilename(input: string): string {\n    return input\n      .toLowerCase()\n      .replace(/\\s+/g, '-') // Replace spaces with hyphens\n      .replace(/[^a-z0-9.-]/g, '') // Remove special characters except dots and hyphens\n      .replace(/--+/g, '-') // Replace multiple hyphens with single hyphen\n      .replace(/^-+|-+$/g, ''); // Remove leading/trailing hyphens\n  }\n}"]}

package/dist/lib/RecordProcessor.d.ts

@@ -0,0 +1,82 @@
+import { BaseEntity, UserInfo } from '@memberjunction/core';
+import { SyncEngine, RecordData } from '../lib/sync-engine';
+import { EntityConfig } from '../config';
+/**
+ * Handles the core processing of individual record data into the sync format
+ */
+export declare class RecordProcessor {
+    private syncEngine;
+    private contextUser;
+    private propertyExtractor;
+    private fieldExternalizer;
+    private relatedEntityHandler;
+    constructor(syncEngine: SyncEngine, contextUser: UserInfo);
+    /**
+     * Processes a record into the standardized RecordData format
+     */
+    processRecord(record: BaseEntity, primaryKey: Record<string, any>, targetDir: string, entityConfig: EntityConfig, verbose?: boolean, isNewRecord?: boolean, existingRecordData?: RecordData, currentDepth?: number, ancestryPath?: Set<string>, fieldOverrides?: Record<string, any>): Promise<RecordData>;
+    /**
+     * Processes entity data into fields and related entities
+     */
+    private processEntityData;
+    /**
+     * Processes individual fields from the entity
+     */
+    private processFields;
+    /**
+     * Determines if a field should be skipped during processing
+     */
+    private shouldSkipField;
+    /**
+     * Checks if a virtual field should be skipped
+     */
+    private shouldSkipVirtualField;
+    /**
+     * Processes a single field value through various transformations
+     */
+    private processFieldValue;
+    /**
+     * Applies lookup field conversion if configured
+     */
+    private applyLookupFieldConversion;
+    /**
+     * Trims string values to remove whitespace
+     */
+    private trimStringValue;
+    /**
+     * Applies field externalization if configured
+     */
+    private applyFieldExternalization;
+    /**
+     * Gets the externalization pattern for a field
+     */
+    private getExternalizationPattern;
+    /**
+     * Gets externalization pattern from array configuration
+     */
+    private getArrayExternalizationPattern;
+    /**
+     * Gets externalization pattern from object configuration
+     */
+    private getObjectExternalizationPattern;
+    /**
+     * Creates a BaseEntity-like object for externalization processing
+     */
+    private createRecordDataForExternalization;
+    /**
+     * Processes related entities for the record
+     */
+    private processRelatedEntities;
+    /**
+     * Calculates sync metadata including checksum and last modified timestamp
+     */
+    private calculateSyncMetadata;
+    /**
+     * Checks if the record has externalized fields
+     */
+    private hasExternalizedFields;
+    /**
+     * Convert a GUID value to @lookup syntax by looking up the human-readable value
+     */
+    private convertGuidToLookup;
+}

package/dist/lib/RecordProcessor.js

@@ -0,0 +1,309 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecordProcessor = void 0;
+const core_1 = require("@memberjunction/core");
+const json_write_helper_1 = require("./json-write-helper");
+const EntityPropertyExtractor_1 = require("./EntityPropertyExtractor");
+const FieldExternalizer_1 = require("./FieldExternalizer");
+const RelatedEntityHandler_1 = require("./RelatedEntityHandler");
+/**
+ * Handles the core processing of individual record data into the sync format
+ */
+class RecordProcessor {
+    syncEngine;
+    contextUser;
+    propertyExtractor;
+    fieldExternalizer;
+    relatedEntityHandler;
+    constructor(syncEngine, contextUser) {
+        this.syncEngine = syncEngine;
+        this.contextUser = contextUser;
+        this.propertyExtractor = new EntityPropertyExtractor_1.EntityPropertyExtractor();
+        this.fieldExternalizer = new FieldExternalizer_1.FieldExternalizer();
+        this.relatedEntityHandler = new RelatedEntityHandler_1.RelatedEntityHandler(syncEngine, contextUser);
+    }
+    /**
+     * Processes a record into the standardized RecordData format
+     */
+    async processRecord(record, primaryKey, targetDir, entityConfig, verbose, isNewRecord = true, existingRecordData, currentDepth = 0, ancestryPath = new Set(), fieldOverrides) {
+        // Extract all properties from the entity
+        const allProperties = this.propertyExtractor.extractAllProperties(record, fieldOverrides);
+        // Process fields and related entities
+        const { fields, relatedEntities } = await this.processEntityData(allProperties, record, primaryKey, targetDir, entityConfig, existingRecordData, currentDepth, ancestryPath, verbose);
+        // Calculate checksum and sync metadata
+        const syncData = await this.calculateSyncMetadata(fields, targetDir, entityConfig, existingRecordData, verbose);
+        // Build the final record data with proper ordering
+        return json_write_helper_1.JsonWriteHelper.createOrderedRecordData(fields, relatedEntities, primaryKey, syncData);
+    }
+    /**
+     * Processes entity data into fields and related entities
+     */
+    async processEntityData(allProperties, record, primaryKey, targetDir, entityConfig, existingRecordData, currentDepth, ancestryPath, verbose) {
+        const fields = {};
+        const relatedEntities = {};
+        // Process individual fields
+        await this.processFields(allProperties, primaryKey, targetDir, entityConfig, existingRecordData, fields, verbose);
+        // Process related entities if configured
+        await this.processRelatedEntities(record, entityConfig, existingRecordData, currentDepth, ancestryPath, relatedEntities, verbose);
+        return { fields, relatedEntities };
+    }
+    /**
+     * Processes individual fields from the entity
+     */
+    async processFields(allProperties, primaryKey, targetDir, entityConfig, existingRecordData, fields, verbose) {
+        const entityInfo = this.syncEngine.getEntityInfo(entityConfig.entity);
+        for (const [fieldName, fieldValue] of Object.entries(allProperties)) {
+            if (this.shouldSkipField(fieldName, fieldValue, primaryKey, entityConfig, entityInfo)) {
+                continue;
+            }
+            let processedValue = await this.processFieldValue(fieldName, fieldValue, allProperties, targetDir, entityConfig, existingRecordData, verbose);
+            fields[fieldName] = processedValue;
+        }
+    }
+    /**
+     * Determines if a field should be skipped during processing
+     */
+    shouldSkipField(fieldName, fieldValue, primaryKey, entityConfig, entityInfo) {
+        // Skip primary key fields
+        if (primaryKey[fieldName] !== undefined) {
+            return true;
+        }
+        // Skip internal fields
+        if (fieldName.startsWith('__mj_')) {
+            return true;
+        }
+        // Skip excluded fields
+        if (entityConfig.pull?.excludeFields?.includes(fieldName)) {
+            return true;
+        }
+        // Skip virtual fields if configured
+        if (this.shouldSkipVirtualField(fieldName, entityConfig, entityInfo)) {
+            return true;
+        }
+        // Skip null fields if configured
+        if (entityConfig.pull?.ignoreNullFields && fieldValue === null) {
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Checks if a virtual field should be skipped
+     */
+    shouldSkipVirtualField(fieldName, entityConfig, entityInfo) {
+        if (!entityConfig.pull?.ignoreVirtualFields || !entityInfo) {
+            return false;
+        }
+        const fieldInfo = entityInfo.Fields.find(f => f.Name === fieldName);
+        return fieldInfo?.IsVirtual === true;
+    }
+    /**
+     * Processes a single field value through various transformations
+     */
+    async processFieldValue(fieldName, fieldValue, allProperties, targetDir, entityConfig, existingRecordData, verbose) {
+        let processedValue = fieldValue;
+        // Apply lookup field conversion if configured
+        processedValue = await this.applyLookupFieldConversion(fieldName, processedValue, entityConfig, verbose);
+        // Trim string values
+        processedValue = this.trimStringValue(processedValue);
+        // Apply field externalization if configured
+        processedValue = await this.applyFieldExternalization(fieldName, processedValue, allProperties, targetDir, entityConfig, existingRecordData, verbose);
+        return processedValue;
+    }
+    /**
+     * Applies lookup field conversion if configured
+     */
+    async applyLookupFieldConversion(fieldName, fieldValue, entityConfig, verbose) {
+        const lookupConfig = entityConfig.pull?.lookupFields?.[fieldName];
+        if (!lookupConfig || fieldValue == null) {
+            return fieldValue;
+        }
+        try {
+            return await this.convertGuidToLookup(String(fieldValue), lookupConfig, verbose);
+        }
+        catch (error) {
+            if (verbose) {
+                console.warn(`Failed to convert ${fieldName} to lookup: ${error}`);
+            }
+            return fieldValue; // Keep original value if lookup fails
+        }
+    }
+    /**
+     * Trims string values to remove whitespace
+     */
+    trimStringValue(value) {
+        return typeof value === 'string' ? value.trim() : value;
+    }
+    /**
+     * Applies field externalization if configured
+     */
+    async applyFieldExternalization(fieldName, fieldValue, allProperties, targetDir, entityConfig, existingRecordData, verbose) {
+        if (!entityConfig.pull?.externalizeFields || fieldValue == null) {
+            return fieldValue;
+        }
+        const externalizePattern = this.getExternalizationPattern(fieldName, entityConfig);
+        if (!externalizePattern) {
+            return fieldValue;
+        }
+        try {
+            const existingFileReference = existingRecordData?.fields?.[fieldName];
+            const recordData = this.createRecordDataForExternalization(allProperties);
+            return await this.fieldExternalizer.externalizeField(fieldName, fieldValue, externalizePattern, recordData, targetDir, existingFileReference, entityConfig.pull?.mergeStrategy || 'merge', verbose);
+        }
+        catch (error) {
+            if (verbose) {
+                console.warn(`Failed to externalize field ${fieldName}: ${error}`);
+            }
+            return fieldValue; // Keep original value if externalization fails
+        }
+    }
+    /**
+     * Gets the externalization pattern for a field
+     */
+    getExternalizationPattern(fieldName, entityConfig) {
+        const externalizeConfig = entityConfig.pull?.externalizeFields;
+        if (!externalizeConfig)
+            return null;
+        if (Array.isArray(externalizeConfig)) {
+            return this.getArrayExternalizationPattern(fieldName, externalizeConfig);
+        }
+        else {
+            return this.getObjectExternalizationPattern(fieldName, externalizeConfig);
+        }
+    }
+    /**
+     * Gets externalization pattern from array configuration
+     */
+    getArrayExternalizationPattern(fieldName, externalizeConfig) {
+        if (externalizeConfig.length > 0 && typeof externalizeConfig[0] === 'string') {
+            // Simple string array format
+            if (externalizeConfig.includes(fieldName)) {
+                return `@file:{Name}.${fieldName.toLowerCase()}.md`;
+            }
+        }
+        else {
+            // Array of objects format
+            const fieldConfig = externalizeConfig
+                .find(config => config.field === fieldName);
+            if (fieldConfig) {
+                return fieldConfig.pattern;
+            }
+        }
+        return null;
+    }
+    /**
+     * Gets externalization pattern from object configuration
+     */
+    getObjectExternalizationPattern(fieldName, externalizeConfig) {
+        const fieldConfig = externalizeConfig[fieldName];
+        if (fieldConfig) {
+            const extension = fieldConfig.extension || '.md';
+            return `@file:{Name}.${fieldName.toLowerCase()}${extension}`;
+        }
+        return null;
+    }
+    /**
+     * Creates a BaseEntity-like object for externalization processing
+     */
+    createRecordDataForExternalization(allProperties) {
+        return allProperties;
+    }
+    /**
+     * Processes related entities for the record
+     */
+    async processRelatedEntities(record, entityConfig, existingRecordData, currentDepth, ancestryPath, relatedEntities, verbose) {
+        if (!entityConfig.pull?.relatedEntities) {
+            return;
+        }
+        for (const [relationKey, relationConfig] of Object.entries(entityConfig.pull.relatedEntities)) {
+            try {
+                const existingRelated = existingRecordData?.relatedEntities?.[relationKey] || [];
+                const relatedRecords = await this.relatedEntityHandler.loadRelatedEntities(record, relationConfig, entityConfig, existingRelated, this.processRecord.bind(this), // Pass bound method reference
+                currentDepth, ancestryPath, verbose);
+                if (relatedRecords.length > 0) {
+                    relatedEntities[relationKey] = relatedRecords;
+                }
+            }
+            catch (error) {
+                if (verbose) {
+                    console.warn(`Failed to load related entities for ${relationKey}: ${error}`);
+                }
+            }
+        }
+    }
+    /**
+     * Calculates sync metadata including checksum and last modified timestamp
+     */
+    async calculateSyncMetadata(fields, targetDir, entityConfig, existingRecordData, verbose) {
+        // Determine if we should include external file content in checksum
+        const hasExternalizedFields = this.hasExternalizedFields(fields, entityConfig);
+        const checksum = hasExternalizedFields
+            ? await this.syncEngine.calculateChecksumWithFileContent(fields, targetDir)
+            : this.syncEngine.calculateChecksum(fields);
+        if (verbose && hasExternalizedFields) {
+            console.log(`Calculated checksum including external file content for record`);
+        }
+        // Compare with existing checksum to determine if data changed
+        if (existingRecordData?.sync?.checksum === checksum) {
+            // No change detected - preserve existing sync metadata
+            if (verbose) {
+                console.log(`No changes detected for record, preserving existing timestamp`);
+            }
+            return {
+                lastModified: existingRecordData.sync.lastModified,
+                checksum: checksum
+            };
+        }
+        else {
+            // Change detected - update timestamp
+            if (verbose && existingRecordData?.sync?.checksum) {
+                console.log(`Changes detected for record, updating timestamp`);
+            }
+            return {
+                lastModified: new Date().toISOString(),
+                checksum: checksum
+            };
+        }
+    }
+    /**
+     * Checks if the record has externalized fields
+     */
+    hasExternalizedFields(fields, entityConfig) {
+        return !!entityConfig.pull?.externalizeFields &&
+            Object.values(fields).some(value => typeof value === 'string' && value.startsWith('@file:'));
+    }
+    /**
+     * Convert a GUID value to @lookup syntax by looking up the human-readable value
+     */
+    async convertGuidToLookup(guidValue, lookupConfig, verbose) {
+        if (!guidValue || typeof guidValue !== 'string') {
+            return guidValue;
+        }
+        try {
+            const rv = new core_1.RunView();
+            const result = await rv.RunView({
+                EntityName: lookupConfig.entity,
+                ExtraFilter: `ID = '${guidValue}'`,
+                ResultType: 'entity_object'
+            }, this.contextUser);
+            if (result.Success && result.Results && result.Results.length > 0) {
+                const targetRecord = result.Results[0];
+                const lookupValue = targetRecord[lookupConfig.field];
+                if (lookupValue != null) {
+                    return `@lookup:${lookupConfig.entity}.${lookupConfig.field}=${lookupValue}`;
+                }
+            }
+            if (verbose) {
+                console.warn(`Lookup failed for ${guidValue} in ${lookupConfig.entity}.${lookupConfig.field}`);
+            }
+            return guidValue; // Return original GUID if lookup fails
+        }
+        catch (error) {
+            if (verbose) {
+                console.warn(`Error during lookup conversion: ${error}`);
+            }
+            return guidValue;
+        }
+    }
+}
+exports.RecordProcessor = RecordProcessor;
+//# sourceMappingURL=RecordProcessor.js.map

package/dist/lib/RecordProcessor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RecordProcessor.js","sourceRoot":"","sources":["../../src/lib/RecordProcessor.ts"],"names":[],"mappings":";;;AAAA,+CAAiF;AAGjF,2DAAsD;AACtD,uEAAoE;AACpE,2DAAwD;AACxD,iEAA8D;AAE9D;;GAEG;AACH,MAAa,eAAe;IAMhB;IACA;IANF,iBAAiB,CAA0B;IAC3C,iBAAiB,CAAoB;IACrC,oBAAoB,CAAuB;IAEnD,YACU,UAAsB,EACtB,WAAqB;QADrB,eAAU,GAAV,UAAU,CAAY;QACtB,gBAAW,GAAX,WAAW,CAAU;QAE7B,IAAI,CAAC,iBAAiB,GAAG,IAAI,iDAAuB,EAAE,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,qCAAiB,EAAE,CAAC;QACjD,IAAI,CAAC,oBAAoB,GAAG,IAAI,2CAAoB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAChF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,MAAkB,EAClB,UAA+B,EAC/B,SAAiB,EACjB,YAA0B,EAC1B,OAAiB,EACjB,cAAuB,IAAI,EAC3B,kBAA+B,EAC/B,eAAuB,CAAC,EACxB,eAA4B,IAAI,GAAG,EAAE,EACrC,cAAoC;QAEpC,yCAAyC;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,oBAAoB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QAE1F,sCAAsC;QACtC,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAC9D,aAAa,EACb,MAAM,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,OAAO,CACR,CAAC;QAEF,uCAAuC;QACvC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAC/C,MAAM,EACN,SAAS,EACT,YAAY,EACZ,kBAAkB,EAClB,OAAO,CACR,CAAC;QAEF,mDAAmD;QACnD,OAAO,mCAAe,CAAC,uBAAuB,CAC5C,MAAM,EACN,eAAe,EACf,UAAU,EACV,QAAQ,CACT,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC7B,aAAkC,EAClC,MAAkB,EAClB,UAA+B,EAC/B,SAAiB,EACjB,YAA0B,EAC1B,kBAA0C,EAC1C,YAAoB,EACpB,YAAyB,EACzB,OAAiB;QAEjB,MAAM,MAAM,GAAwB,EAAE,CAAC;QACvC,MAAM,eAAe,GAAiC,EAAE,CAAC;QAEzD,4BAA4B;QAC5B,MAAM,IAAI,CAAC,aAAa,CACtB,aAAa,EACb,UAAU,EACV,SAAS,EACT,YAAY,EACZ,kBAAkB,EAClB,MAAM,EACN,OAAO,CACR,CAAC;QAEF,yCAAyC;QACzC,MAAM,IAAI,CAAC,sBAAsB,CAC/B,MAAM,EACN,YAAY,EACZ,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACZ,eAAe,EACf,OAAO,CACR,CAAC;QAEF,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,aAAkC,EAClC,UAA+B,EAC/B,SAAiB,EACjB,YAA0B,EAC1B,kBAA0C,EAC1C,MAA2B,EAC3B,OAAiB;QAEjB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAEtE,KAAK,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACpE,IAAI,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,CAAC;gBACtF,SAAS;YACX,CAAC;YAED,IAAI,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAC/C,SAAS,EACT,UAAU,EACV,aAAa,EACb,SAAS,EACT,YAAY,EACZ,kBAAkB,EAClB,OAAO,CACR,CAAC;YAEF,MAAM,CAAC,SAAS,CAAC,GAAG,cAAc,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,eAAe,CACrB,SAAiB,EACjB,UAAe,EACf,UAA+B,EAC/B,YAA0B,EAC1B,UAA6B;QAE7B,0BAA0B;QAC1B,IAAI,UAAU,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,uBAAuB;QACvB,IAAI,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,uBAAuB;QACvB,IAAI,YAAY,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,oCAAoC;QACpC,IAAI,IAAI,CAAC,sBAAsB,CAAC,SAAS,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,iCAAiC;QACjC,IAAI,YAAY,CAAC,IAAI,EAAE,gBAAgB,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,sBAAsB,CAC5B,SAAiB,EACjB,YAA0B,EAC1B,UAA6B;QAE7B,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,mBAAmB,IAAI,CAAC,UAAU,EAAE,CAAC;YAC3D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QACpE,OAAO,SAAS,EAAE,SAAS,KAAK,IAAI,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC7B,SAAiB,EACjB,UAAe,EACf,aAAkC,EAClC,SAAiB,EACjB,YAA0B,EAC1B,kBAA0C,EAC1C,OAAiB;QAEjB,IAAI,cAAc,GAAG,UAAU,CAAC;QAEhC,8CAA8C;QAC9C,cAAc,GAAG,MAAM,IAAI,CAAC,0BAA0B,CACpD,SAAS,EACT,cAAc,EACd,YAAY,EACZ,OAAO,CACR,CAAC;QAEF,qBAAqB;QACrB,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAEtD,4CAA4C;QAC5C,cAAc,GAAG,MAAM,IAAI,CAAC,yBAAyB,CACnD,SAAS,EACT,cAAc,EACd,aAAa,EACb,SAAS,EACT,YAAY,EACZ,kBAAkB,EAClB,OAAO,CACR,CAAC;QAEF,OAAO,cAAc,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,0BAA0B,CACtC,SAAiB,EACjB,UAAe,EACf,YAA0B,EAC1B,OAAiB;QAEjB,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,CAAC,YAAY,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACxC,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QACnF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,IAAI,CAAC,qBAAqB,SAAS,eAAe,KAAK,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,OAAO,UAAU,CAAC,CAAC,sCAAsC;QAC3D,CAAC;IACH,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,KAAU;QAChC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1D,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,yBAAyB,CACrC,SAAiB,EACjB,UAAe,EACf,aAAkC,EAClC,SAAiB,EACjB,YAA0B,EAC1B,kBAA0C,EAC1C,OAAiB;QAEjB,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,iBAAiB,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YAChE,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,MAAM,kBAAkB,GAAG,IAAI,CAAC,yBAAyB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACnF,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,qBAAqB,GAAG,kBAAkB,EAAE,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC;YACtE,MAAM,UAAU,GAAG,IAAI,CAAC,kCAAkC,CAAC,aAAa,CAAC,CAAC;YAE1E,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAClD,SAAS,EACT,UAAU,EACV,kBAAkB,EAClB,UAAU,EACV,SAAS,EACT,qBAAqB,EACrB,YAAY,CAAC,IAAI,EAAE,aAAa,IAAI,OAAO,EAC3C,OAAO,CACR,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,IAAI,CAAC,+BAA+B,SAAS,KAAK,KAAK,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,OAAO,UAAU,CAAC,CAAC,+CAA+C;QACpE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,yBAAyB,CAAC,SAAiB,EAAE,YAA0B;QAC7E,MAAM,iBAAiB,GAAG,YAAY,CAAC,IAAI,EAAE,iBAAiB,CAAC;QAC/D,IAAI,CAAC,iBAAiB;YAAE,OAAO,IAAI,CAAC;QAEpC,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,8BAA8B,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;QAC3E,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,+BAA+B,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED;;OAEG;IACK,8BAA8B,CACpC,SAAiB,EACjB,iBAAwB;QAExB,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,iBAAiB,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC7E,6BAA6B;YAC7B,IAAK,iBAA8B,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxD,OAAO,gBAAgB,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC;YACtD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,0BAA0B;YAC1B,MAAM,WAAW,GAAI,iBAA6D;iBAC/E,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;YAC9C,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,WAAW,CAAC,OAAO,CAAC;YAC7B,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,+BAA+B,CACrC,SAAiB,EACjB,iBAAsC;QAEtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,IAAI,KAAK,CAAC;YACjD,OAAO,gBAAgB,SAAS,CAAC,WAAW,EAAE,GAAG,SAAS,EAAE,CAAC;QAC/D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACK,kCAAkC,CAAC,aAAkC;QAC3E,OAAO,aAAkC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,sBAAsB,CAClC,MAAkB,EAClB,YAA0B,EAC1B,kBAA0C,EAC1C,YAAoB,EACpB,YAAyB,EACzB,eAA6C,EAC7C,OAAiB;QAEjB,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC;YACxC,OAAO;QACT,CAAC;QAED,KAAK,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;YAC9F,IAAI,CAAC;gBACH,MAAM,eAAe,GAAG,kBAAkB,EAAE,eAAe,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;gBAEjF,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,mBAAmB,CACxE,MAAM,EACN,cAAc,EACd,YAAY,EACZ,eAAe,EACf,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,8BAA8B;gBAC7D,YAAY,EACZ,YAAY,EACZ,OAAO,CACR,CAAC;gBAEF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,eAAe,CAAC,WAAW,CAAC,GAAG,cAAc,CAAC;gBAChD,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,CAAC,IAAI,CAAC,uCAAuC,WAAW,KAAK,KAAK,EAAE,CAAC,CAAC;gBAC/E,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CACjC,MAA2B,EAC3B,SAAiB,EACjB,YAA0B,EAC1B,kBAA0C,EAC1C,OAAiB;QAEjB,mEAAmE;QACnE,MAAM,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAE/E,MAAM,QAAQ,GAAG,qBAAqB;YACpC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,gCAAgC,CAAC,MAAM,EAAE,SAAS,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAE9C,IAAI,OAAO,IAAI,qBAAqB,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,gEAAgE,CAAC,CAAC;QAChF,CAAC;QAED,8DAA8D;QAC9D,IAAI,kBAAkB,EAAE,IAAI,EAAE,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACpD,uDAAuD;YACvD,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;YAC/E,CAAC;YACD,OAAO;gBACL,YAAY,EAAE,kBAAkB,CAAC,IAAI,CAAC,YAAY;gBAClD,QAAQ,EAAE,QAAQ;aACnB,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,qCAAqC;YACrC,IAAI,OAAO,IAAI,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;YACjE,CAAC;YACD,OAAO;gBACL,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACtC,QAAQ,EAAE,QAAQ;aACnB,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,MAA2B,EAAE,YAA0B;QACnF,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,iBAAiB;YACtC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CACjC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CACxD,CAAC;IACX,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAC/B,SAAiB,EACjB,YAA+C,EAC/C,OAAiB;QAEjB,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;YAChD,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,IAAI,cAAO,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAC9B,UAAU,EAAE,YAAY,CAAC,MAAM;gBAC/B,WAAW,EAAE,SAAS,SAAS,GAAG;gBAClC,UAAU,EAAE,eAAe;aAC5B,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAErB,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClE,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACvC,MAAM,WAAW,GAAG,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBAErD,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;oBACxB,OAAO,WAAW,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,KAAK,IAAI,WAAW,EAAE,CAAC;gBAC/E,CAAC;YACH,CAAC;YAED,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,IAAI,CAAC,qBAAqB,SAAS,OAAO,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC;YACjG,CAAC;YAED,OAAO,SAAS,CAAC,CAAC,uCAAuC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,IAAI,CAAC,mCAAmC,KAAK,EAAE,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AApfD,0CAofC","sourcesContent":["import { BaseEntity, RunView, UserInfo, EntityInfo } from '@memberjunction/core';\nimport { SyncEngine, RecordData } from '../lib/sync-engine';\nimport { EntityConfig } from '../config';\nimport { JsonWriteHelper } from './json-write-helper';\nimport { EntityPropertyExtractor } from './EntityPropertyExtractor';\nimport { FieldExternalizer } from './FieldExternalizer';\nimport { RelatedEntityHandler } from './RelatedEntityHandler';\n\n/**\n * Handles the core processing of individual record data into the sync format\n */\nexport class RecordProcessor {\n  private propertyExtractor: EntityPropertyExtractor;\n  private fieldExternalizer: FieldExternalizer;\n  private relatedEntityHandler: RelatedEntityHandler;\n\n  constructor(\n    private syncEngine: SyncEngine,\n    private contextUser: UserInfo\n  ) {\n    this.propertyExtractor = new EntityPropertyExtractor();\n    this.fieldExternalizer = new FieldExternalizer();\n    this.relatedEntityHandler = new RelatedEntityHandler(syncEngine, contextUser);\n  }\n\n  /**\n   * Processes a record into the standardized RecordData format\n   */\n  async processRecord(\n    record: BaseEntity, \n    primaryKey: Record<string, any>,\n    targetDir: string, \n    entityConfig: EntityConfig,\n    verbose?: boolean,\n    isNewRecord: boolean = true,\n    existingRecordData?: RecordData,\n    currentDepth: number = 0,\n    ancestryPath: Set<string> = new Set(),\n    fieldOverrides?: Record<string, any>\n  ): Promise<RecordData> {\n    // Extract all properties from the entity\n    const allProperties = this.propertyExtractor.extractAllProperties(record, fieldOverrides);\n    \n    // Process fields and related entities\n    const { fields, relatedEntities } = await this.processEntityData(\n      allProperties,\n      record,\n      primaryKey,\n      targetDir,\n      entityConfig,\n      existingRecordData,\n      currentDepth,\n      ancestryPath,\n      verbose\n    );\n    \n    // Calculate checksum and sync metadata\n    const syncData = await this.calculateSyncMetadata(\n      fields, \n      targetDir, \n      entityConfig, \n      existingRecordData, \n      verbose\n    );\n    \n    // Build the final record data with proper ordering\n    return JsonWriteHelper.createOrderedRecordData(\n      fields,\n      relatedEntities,\n      primaryKey,\n      syncData\n    );\n  }\n\n  /**\n   * Processes entity data into fields and related entities\n   */\n  private async processEntityData(\n    allProperties: Record<string, any>,\n    record: BaseEntity,\n    primaryKey: Record<string, any>,\n    targetDir: string,\n    entityConfig: EntityConfig,\n    existingRecordData: RecordData | undefined,\n    currentDepth: number,\n    ancestryPath: Set<string>,\n    verbose?: boolean\n  ): Promise<{ fields: Record<string, any>; relatedEntities: Record<string, RecordData[]> }> {\n    const fields: Record<string, any> = {};\n    const relatedEntities: Record<string, RecordData[]> = {};\n    \n    // Process individual fields\n    await this.processFields(\n      allProperties, \n      primaryKey, \n      targetDir, \n      entityConfig, \n      existingRecordData, \n      fields, \n      verbose\n    );\n    \n    // Process related entities if configured\n    await this.processRelatedEntities(\n      record, \n      entityConfig, \n      existingRecordData, \n      currentDepth, \n      ancestryPath, \n      relatedEntities, \n      verbose\n    );\n    \n    return { fields, relatedEntities };\n  }\n\n  /**\n   * Processes individual fields from the entity\n   */\n  private async processFields(\n    allProperties: Record<string, any>,\n    primaryKey: Record<string, any>,\n    targetDir: string,\n    entityConfig: EntityConfig,\n    existingRecordData: RecordData | undefined,\n    fields: Record<string, any>,\n    verbose?: boolean\n  ): Promise<void> {\n    const entityInfo = this.syncEngine.getEntityInfo(entityConfig.entity);\n    \n    for (const [fieldName, fieldValue] of Object.entries(allProperties)) {\n      if (this.shouldSkipField(fieldName, fieldValue, primaryKey, entityConfig, entityInfo)) {\n        continue;\n      }\n      \n      let processedValue = await this.processFieldValue(\n        fieldName,\n        fieldValue,\n        allProperties,\n        targetDir,\n        entityConfig,\n        existingRecordData,\n        verbose\n      );\n      \n      fields[fieldName] = processedValue;\n    }\n  }\n\n  /**\n   * Determines if a field should be skipped during processing\n   */\n  private shouldSkipField(\n    fieldName: string,\n    fieldValue: any,\n    primaryKey: Record<string, any>,\n    entityConfig: EntityConfig,\n    entityInfo: EntityInfo | null\n  ): boolean {\n    // Skip primary key fields\n    if (primaryKey[fieldName] !== undefined) {\n      return true;\n    }\n    \n    // Skip internal fields\n    if (fieldName.startsWith('__mj_')) {\n      return true;\n    }\n    \n    // Skip excluded fields\n    if (entityConfig.pull?.excludeFields?.includes(fieldName)) {\n      return true;\n    }\n    \n    // Skip virtual fields if configured\n    if (this.shouldSkipVirtualField(fieldName, entityConfig, entityInfo)) {\n      return true;\n    }\n    \n    // Skip null fields if configured\n    if (entityConfig.pull?.ignoreNullFields && fieldValue === null) {\n      return true;\n    }\n    \n    return false;\n  }\n\n  /**\n   * Checks if a virtual field should be skipped\n   */\n  private shouldSkipVirtualField(\n    fieldName: string,\n    entityConfig: EntityConfig,\n    entityInfo: EntityInfo | null\n  ): boolean {\n    if (!entityConfig.pull?.ignoreVirtualFields || !entityInfo) {\n      return false;\n    }\n    \n    const fieldInfo = entityInfo.Fields.find(f => f.Name === fieldName);\n    return fieldInfo?.IsVirtual === true;\n  }\n\n  /**\n   * Processes a single field value through various transformations\n   */\n  private async processFieldValue(\n    fieldName: string,\n    fieldValue: any,\n    allProperties: Record<string, any>,\n    targetDir: string,\n    entityConfig: EntityConfig,\n    existingRecordData: RecordData | undefined,\n    verbose?: boolean\n  ): Promise<any> {\n    let processedValue = fieldValue;\n    \n    // Apply lookup field conversion if configured\n    processedValue = await this.applyLookupFieldConversion(\n      fieldName, \n      processedValue, \n      entityConfig, \n      verbose\n    );\n    \n    // Trim string values\n    processedValue = this.trimStringValue(processedValue);\n    \n    // Apply field externalization if configured\n    processedValue = await this.applyFieldExternalization(\n      fieldName,\n      processedValue,\n      allProperties,\n      targetDir,\n      entityConfig,\n      existingRecordData,\n      verbose\n    );\n    \n    return processedValue;\n  }\n\n  /**\n   * Applies lookup field conversion if configured\n   */\n  private async applyLookupFieldConversion(\n    fieldName: string,\n    fieldValue: any,\n    entityConfig: EntityConfig,\n    verbose?: boolean\n  ): Promise<any> {\n    const lookupConfig = entityConfig.pull?.lookupFields?.[fieldName];\n    if (!lookupConfig || fieldValue == null) {\n      return fieldValue;\n    }\n    \n    try {\n      return await this.convertGuidToLookup(String(fieldValue), lookupConfig, verbose);\n    } catch (error) {\n      if (verbose) {\n        console.warn(`Failed to convert ${fieldName} to lookup: ${error}`);\n      }\n      return fieldValue; // Keep original value if lookup fails\n    }\n  }\n\n  /**\n   * Trims string values to remove whitespace\n   */\n  private trimStringValue(value: any): any {\n    return typeof value === 'string' ? value.trim() : value;\n  }\n\n  /**\n   * Applies field externalization if configured\n   */\n  private async applyFieldExternalization(\n    fieldName: string,\n    fieldValue: any,\n    allProperties: Record<string, any>,\n    targetDir: string,\n    entityConfig: EntityConfig,\n    existingRecordData: RecordData | undefined,\n    verbose?: boolean\n  ): Promise<any> {\n    if (!entityConfig.pull?.externalizeFields || fieldValue == null) {\n      return fieldValue;\n    }\n    \n    const externalizePattern = this.getExternalizationPattern(fieldName, entityConfig);\n    if (!externalizePattern) {\n      return fieldValue;\n    }\n    \n    try {\n      const existingFileReference = existingRecordData?.fields?.[fieldName];\n      const recordData = this.createRecordDataForExternalization(allProperties);\n      \n      return await this.fieldExternalizer.externalizeField(\n        fieldName,\n        fieldValue,\n        externalizePattern,\n        recordData,\n        targetDir,\n        existingFileReference,\n        entityConfig.pull?.mergeStrategy || 'merge',\n        verbose\n      );\n    } catch (error) {\n      if (verbose) {\n        console.warn(`Failed to externalize field ${fieldName}: ${error}`);\n      }\n      return fieldValue; // Keep original value if externalization fails\n    }\n  }\n\n  /**\n   * Gets the externalization pattern for a field\n   */\n  private getExternalizationPattern(fieldName: string, entityConfig: EntityConfig): string | null {\n    const externalizeConfig = entityConfig.pull?.externalizeFields;\n    if (!externalizeConfig) return null;\n    \n    if (Array.isArray(externalizeConfig)) {\n      return this.getArrayExternalizationPattern(fieldName, externalizeConfig);\n    } else {\n      return this.getObjectExternalizationPattern(fieldName, externalizeConfig);\n    }\n  }\n\n  /**\n   * Gets externalization pattern from array configuration\n   */\n  private getArrayExternalizationPattern(\n    fieldName: string, \n    externalizeConfig: any[]\n  ): string | null {\n    if (externalizeConfig.length > 0 && typeof externalizeConfig[0] === 'string') {\n      // Simple string array format\n      if ((externalizeConfig as string[]).includes(fieldName)) {\n        return `@file:{Name}.${fieldName.toLowerCase()}.md`;\n      }\n    } else {\n      // Array of objects format\n      const fieldConfig = (externalizeConfig as Array<{field: string; pattern: string}>)\n        .find(config => config.field === fieldName);\n      if (fieldConfig) {\n        return fieldConfig.pattern;\n      }\n    }\n    return null;\n  }\n\n  /**\n   * Gets externalization pattern from object configuration\n   */\n  private getObjectExternalizationPattern(\n    fieldName: string, \n    externalizeConfig: Record<string, any>\n  ): string | null {\n    const fieldConfig = externalizeConfig[fieldName];\n    if (fieldConfig) {\n      const extension = fieldConfig.extension || '.md';\n      return `@file:{Name}.${fieldName.toLowerCase()}${extension}`;\n    }\n    return null;\n  }\n\n  /**\n   * Creates a BaseEntity-like object for externalization processing\n   */\n  private createRecordDataForExternalization(allProperties: Record<string, any>): BaseEntity {\n    return allProperties as any as BaseEntity;\n  }\n\n  /**\n   * Processes related entities for the record\n   */\n  private async processRelatedEntities(\n    record: BaseEntity,\n    entityConfig: EntityConfig,\n    existingRecordData: RecordData | undefined,\n    currentDepth: number,\n    ancestryPath: Set<string>,\n    relatedEntities: Record<string, RecordData[]>,\n    verbose?: boolean\n  ): Promise<void> {\n    if (!entityConfig.pull?.relatedEntities) {\n      return;\n    }\n    \n    for (const [relationKey, relationConfig] of Object.entries(entityConfig.pull.relatedEntities)) {\n      try {\n        const existingRelated = existingRecordData?.relatedEntities?.[relationKey] || [];\n        \n        const relatedRecords = await this.relatedEntityHandler.loadRelatedEntities(\n          record,\n          relationConfig,\n          entityConfig,\n          existingRelated,\n          this.processRecord.bind(this), // Pass bound method reference\n          currentDepth,\n          ancestryPath,\n          verbose\n        );\n        \n        if (relatedRecords.length > 0) {\n          relatedEntities[relationKey] = relatedRecords;\n        }\n      } catch (error) {\n        if (verbose) {\n          console.warn(`Failed to load related entities for ${relationKey}: ${error}`);\n        }\n      }\n    }\n  }\n\n  /**\n   * Calculates sync metadata including checksum and last modified timestamp\n   */\n  private async calculateSyncMetadata(\n    fields: Record<string, any>,\n    targetDir: string,\n    entityConfig: EntityConfig,\n    existingRecordData: RecordData | undefined,\n    verbose?: boolean\n  ): Promise<{ lastModified: string; checksum: string }> {\n    // Determine if we should include external file content in checksum\n    const hasExternalizedFields = this.hasExternalizedFields(fields, entityConfig);\n    \n    const checksum = hasExternalizedFields\n      ? await this.syncEngine.calculateChecksumWithFileContent(fields, targetDir)\n      : this.syncEngine.calculateChecksum(fields);\n    \n    if (verbose && hasExternalizedFields) {\n      console.log(`Calculated checksum including external file content for record`);\n    }\n    \n    // Compare with existing checksum to determine if data changed\n    if (existingRecordData?.sync?.checksum === checksum) {\n      // No change detected - preserve existing sync metadata\n      if (verbose) {\n        console.log(`No changes detected for record, preserving existing timestamp`);\n      }\n      return {\n        lastModified: existingRecordData.sync.lastModified,\n        checksum: checksum\n      };\n    } else {\n      // Change detected - update timestamp\n      if (verbose && existingRecordData?.sync?.checksum) {\n        console.log(`Changes detected for record, updating timestamp`);\n      }\n      return {\n        lastModified: new Date().toISOString(),\n        checksum: checksum\n      };\n    }\n  }\n\n  /**\n   * Checks if the record has externalized fields\n   */\n  private hasExternalizedFields(fields: Record<string, any>, entityConfig: EntityConfig): boolean {\n    return !!entityConfig.pull?.externalizeFields && \n           Object.values(fields).some(value => \n             typeof value === 'string' && value.startsWith('@file:')\n           );\n  }\n\n  /**\n   * Convert a GUID value to @lookup syntax by looking up the human-readable value\n   */\n  private async convertGuidToLookup(\n    guidValue: string,\n    lookupConfig: { entity: string; field: string },\n    verbose?: boolean\n  ): Promise<string> {\n    if (!guidValue || typeof guidValue !== 'string') {\n      return guidValue;\n    }\n\n    try {\n      const rv = new RunView();\n      const result = await rv.RunView({\n        EntityName: lookupConfig.entity,\n        ExtraFilter: `ID = '${guidValue}'`,\n        ResultType: 'entity_object'\n      }, this.contextUser);\n\n      if (result.Success && result.Results && result.Results.length > 0) {\n        const targetRecord = result.Results[0];\n        const lookupValue = targetRecord[lookupConfig.field];\n        \n        if (lookupValue != null) {\n          return `@lookup:${lookupConfig.entity}.${lookupConfig.field}=${lookupValue}`;\n        }\n      }\n\n      if (verbose) {\n        console.warn(`Lookup failed for ${guidValue} in ${lookupConfig.entity}.${lookupConfig.field}`);\n      }\n      \n      return guidValue; // Return original GUID if lookup fails\n    } catch (error) {\n      if (verbose) {\n        console.warn(`Error during lookup conversion: ${error}`);\n      }\n      return guidValue;\n    }\n  }\n}"]}