@memberjunction/metadata-sync 2.46.0

package/dist/lib/sync-engine.js

@@ -0,0 +1,254 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SyncEngine = void 0;
+const path_1 = __importDefault(require("path"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const crypto_1 = __importDefault(require("crypto"));
+const axios_1 = __importDefault(require("axios"));
+const core_1 = require("@memberjunction/core");
+class SyncEngine {
+    metadata;
+    contextUser;
+    constructor(contextUser) {
+        this.metadata = new core_1.Metadata();
+        this.contextUser = contextUser;
+    }
+    async initialize() {
+        // Initialize metadata
+        await this.metadata.Refresh();
+    }
+    /**
+     * Process special references in field values
+     */
+    async processFieldValue(value, baseDir, parentRecord, rootRecord) {
+        if (typeof value !== 'string') {
+            return value;
+        }
+        // Check for @parent: reference
+        if (value.startsWith('@parent:')) {
+            if (!parentRecord) {
+                throw new Error(`@parent reference used but no parent record available: ${value}`);
+            }
+            const fieldName = value.substring(8);
+            return parentRecord.Get(fieldName);
+        }
+        // Check for @root: reference
+        if (value.startsWith('@root:')) {
+            if (!rootRecord) {
+                throw new Error(`@root reference used but no root record available: ${value}`);
+            }
+            const fieldName = value.substring(6);
+            return rootRecord.Get(fieldName);
+        }
+        // Check for @file: reference
+        if (value.startsWith('@file:')) {
+            const filePath = value.substring(6);
+            const fullPath = path_1.default.resolve(baseDir, filePath);
+            if (await fs_extra_1.default.pathExists(fullPath)) {
+                return await fs_extra_1.default.readFile(fullPath, 'utf-8');
+            }
+            else {
+                throw new Error(`File not found: ${fullPath}`);
+            }
+        }
+        // Check for @url: reference
+        if (value.startsWith('@url:')) {
+            const url = value.substring(5);
+            try {
+                const response = await axios_1.default.get(url);
+                return response.data;
+            }
+            catch (error) {
+                throw new Error(`Failed to fetch URL: ${url} - ${error}`);
+            }
+        }
+        // Check for @lookup: reference
+        if (value.startsWith('@lookup:')) {
+            const lookupStr = value.substring(8);
+            // Parse lookup with optional create syntax
+            // Format: EntityName.FieldName=Value?create&OtherField=Value
+            const entityMatch = lookupStr.match(/^([^.]+)\./);
+            if (!entityMatch) {
+                throw new Error(`Invalid lookup format: ${value}`);
+            }
+            const entityName = entityMatch[1];
+            const remaining = lookupStr.substring(entityName.length + 1);
+            // Check if this has ?create syntax
+            const hasCreate = remaining.includes('?create');
+            const lookupPart = hasCreate ? remaining.split('?')[0] : remaining;
+            // Parse the main lookup field
+            const fieldMatch = lookupPart.match(/^(.+?)=(.+)$/);
+            if (!fieldMatch) {
+                throw new Error(`Invalid lookup format: ${value}`);
+            }
+            const [, fieldName, fieldValue] = fieldMatch;
+            // Parse additional fields for creation if ?create is present
+            let createFields = {};
+            if (hasCreate && remaining.includes('?create&')) {
+                const createPart = remaining.split('?create&')[1];
+                const pairs = createPart.split('&');
+                for (const pair of pairs) {
+                    const [key, val] = pair.split('=');
+                    if (key && val) {
+                        createFields[key] = decodeURIComponent(val);
+                    }
+                }
+            }
+            return await this.resolveLookup(entityName, fieldName, fieldValue, hasCreate, createFields);
+        }
+        // Check for @env: reference
+        if (value.startsWith('@env:')) {
+            const envVar = value.substring(5);
+            const envValue = process.env[envVar];
+            if (envValue === undefined) {
+                throw new Error(`Environment variable not found: ${envVar}`);
+            }
+            return envValue;
+        }
+        return value;
+    }
+    /**
+     * Resolve a lookup reference to an ID, optionally creating the record if it doesn't exist
+     */
+    async resolveLookup(entityName, fieldName, fieldValue, autoCreate = false, createFields = {}) {
+        // Debug logging handled by caller if needed
+        const rv = new core_1.RunView();
+        const entityInfo = this.metadata.EntityByName(entityName);
+        if (!entityInfo) {
+            throw new Error(`Entity not found: ${entityName}`);
+        }
+        const field = entityInfo.Fields.find(f => f.Name.trim().toLowerCase() === fieldName.trim().toLowerCase());
+        const quotes = field?.NeedsQuotes ? "'" : '';
+        const result = await rv.RunView({
+            EntityName: entityName,
+            ExtraFilter: `${fieldName} = ${quotes}${fieldValue.replace(/'/g, "''")}${quotes}`,
+            MaxRows: 1
+        }, this.contextUser);
+        if (result.Success && result.Results.length > 0) {
+            if (entityInfo.PrimaryKeys.length > 0) {
+                const pkeyField = entityInfo.PrimaryKeys[0].Name;
+                const id = result.Results[0][pkeyField];
+                return id;
+            }
+        }
+        // If not found and auto-create is enabled, create the record
+        if (autoCreate) {
+            const newEntity = await this.metadata.GetEntityObject(entityName, this.contextUser);
+            if (!newEntity) {
+                throw new Error(`Failed to create entity object for: ${entityName}`);
+            }
+            newEntity.NewRecord();
+            // Set the lookup field
+            if (fieldName in newEntity) {
+                newEntity[fieldName] = fieldValue;
+            }
+            // Set any additional fields provided
+            for (const [key, value] of Object.entries(createFields)) {
+                if (key in newEntity) {
+                    newEntity[key] = value;
+                }
+            }
+            // Save the new record
+            const saved = await newEntity.Save();
+            if (!saved) {
+                const errors = newEntity.LatestResult?.Errors?.join(', ') || 'Unknown error';
+                throw new Error(`Failed to auto-create ${entityName}: ${errors}`);
+            }
+            // Return the new ID
+            if (entityInfo.PrimaryKeys.length > 0) {
+                const pkeyField = entityInfo.PrimaryKeys[0].Name;
+                const newId = newEntity.Get(pkeyField);
+                return newId;
+            }
+        }
+        throw new Error(`Lookup failed: No record found in '${entityName}' where ${fieldName}='${fieldValue}'`);
+    }
+    /**
+     * Build cascading defaults for a file path and process field values
+     */
+    async buildDefaults(filePath, entityConfig) {
+        const parts = path_1.default.dirname(filePath).split(path_1.default.sep);
+        let defaults = { ...entityConfig.defaults };
+        // Walk up the directory tree building defaults
+        let currentPath = '';
+        for (const part of parts) {
+            currentPath = path_1.default.join(currentPath, part);
+            const folderConfig = await this.loadFolderConfig(currentPath);
+            if (folderConfig?.defaults) {
+                defaults = { ...defaults, ...folderConfig.defaults };
+            }
+        }
+        // Process all default values (lookups, file references, etc.)
+        const processedDefaults = {};
+        const baseDir = path_1.default.dirname(filePath);
+        for (const [field, value] of Object.entries(defaults)) {
+            try {
+                processedDefaults[field] = await this.processFieldValue(value, baseDir, null, null);
+            }
+            catch (error) {
+                throw new Error(`Failed to process default for field '${field}': ${error}`);
+            }
+        }
+        return processedDefaults;
+    }
+    /**
+     * Load folder configuration
+     */
+    async loadFolderConfig(dir) {
+        const configPath = path_1.default.join(dir, '.mj-folder.json');
+        if (await fs_extra_1.default.pathExists(configPath)) {
+            try {
+                return await fs_extra_1.default.readJson(configPath);
+            }
+            catch (error) {
+                console.error(`Error loading folder config at ${configPath}:`, error);
+                return null;
+            }
+        }
+        return null;
+    }
+    /**
+     * Calculate checksum for data
+     */
+    calculateChecksum(data) {
+        const hash = crypto_1.default.createHash('sha256');
+        hash.update(JSON.stringify(data, null, 2));
+        return hash.digest('hex');
+    }
+    /**
+     * Get entity info by name
+     */
+    getEntityInfo(entityName) {
+        return this.metadata.EntityByName(entityName);
+    }
+    /**
+     * Create a new entity object
+     */
+    async createEntityObject(entityName) {
+        const entity = await this.metadata.GetEntityObject(entityName, this.contextUser);
+        if (!entity) {
+            throw new Error(`Failed to create entity object for: ${entityName}`);
+        }
+        return entity;
+    }
+    /**
+     * Load an entity by primary key
+     */
+    async loadEntity(entityName, primaryKey) {
+        const entity = await this.createEntityObject(entityName);
+        const entityInfo = this.getEntityInfo(entityName);
+        if (!entityInfo) {
+            throw new Error(`Entity not found: ${entityName}`);
+        }
+        // Handle composite keys
+        const compositeKey = new core_1.CompositeKey();
+        compositeKey.LoadFromSimpleObject(primaryKey);
+        const loaded = await entity.InnerLoad(compositeKey);
+        return loaded ? entity : null;
+    }
+}
+exports.SyncEngine = SyncEngine;
+//# sourceMappingURL=sync-engine.js.map

package/dist/lib/sync-engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-engine.js","sourceRoot":"","sources":["../../src/lib/sync-engine.ts"],"names":[],"mappings":";;;;;;AAAA,gDAAwB;AACxB,wDAA0B;AAC1B,oDAA4B;AAC5B,kDAA0B;AAC1B,+CAAyG;AAazG,MAAa,UAAU;IACb,QAAQ,CAAW;IACnB,WAAW,CAAW;IAE9B,YAAY,WAAqB;QAC/B,IAAI,CAAC,QAAQ,GAAG,IAAI,eAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,UAAU;QACd,sBAAsB;QACtB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,iBAAiB,CAAC,KAAU,EAAE,OAAe,EAAE,YAAgC,EAAE,UAA8B;QACnH,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,+BAA+B;QAC/B,IAAI,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,0DAA0D,KAAK,EAAE,CAAC,CAAC;YACrF,CAAC;YACD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACrC,OAAO,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrC,CAAC;QAED,6BAA6B;QAC7B,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,sDAAsD,KAAK,EAAE,CAAC,CAAC;YACjF,CAAC;YACD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACrC,OAAO,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACnC,CAAC;QAED,6BAA6B;QAC7B,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,QAAQ,GAAG,cAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAEjD,IAAI,MAAM,kBAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAClC,OAAO,MAAM,kBAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAE/B,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACtC,OAAO,QAAQ,CAAC,IAAI,CAAC;YACvB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,+BAA+B;QAC/B,IAAI,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAErC,2CAA2C;YAC3C,6DAA6D;YAC7D,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAClD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAE7D,mCAAmC;YACnC,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAChD,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAEnE,8BAA8B;YAC9B,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YACpD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,GAAG,UAAU,CAAC;YAE7C,6DAA6D;YAC7D,IAAI,YAAY,GAAwB,EAAE,CAAC;YAC3C,IAAI,SAAS,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChD,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACnC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;wBACf,YAAY,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;oBAC9C,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;QAC9F,CAAC;QAED,4BAA4B;QAC5B,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAErC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,mCAAmC,MAAM,EAAE,CAAC,CAAC;YAC/D,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CACjB,UAAkB,EAClB,SAAiB,EACjB,UAAkB,EAClB,aAAsB,KAAK,EAC3B,eAAoC,EAAE;QAEtC,4CAA4C;QAE5C,MAAM,EAAE,GAAG,IAAI,cAAO,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QAC1D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1G,MAAM,MAAM,GAAG,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;YAC9B,UAAU,EAAE,UAAU;YACtB,WAAW,EAAE,GAAG,SAAS,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,MAAM,EAAE;YACjF,OAAO,EAAE,CAAC;SACX,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAErB,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,IAAI,UAAU,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACjD,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACxC,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,6DAA6D;QAC7D,IAAI,UAAU,EAAE,CAAC;YAEf,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACpF,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,uCAAuC,UAAU,EAAE,CAAC,CAAC;YACvE,CAAC;YAED,SAAS,CAAC,SAAS,EAAE,CAAC;YAEtB,uBAAuB;YACvB,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;gBAC1B,SAAiB,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC;YAC7C,CAAC;YAED,qCAAqC;YACrC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;gBACxD,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;oBACpB,SAAiB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;gBAClC,CAAC;YACH,CAAC;YAED,sBAAsB;YACtB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,MAAM,GAAG,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,eAAe,CAAC;gBAC7E,MAAM,IAAI,KAAK,CAAC,yBAAyB,UAAU,KAAK,MAAM,EAAE,CAAC,CAAC;YACpE,CAAC;YAED,oBAAoB;YACpB,IAAI,UAAU,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACjD,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACvC,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,UAAU,WAAW,SAAS,KAAK,UAAU,GAAG,CAAC,CAAC;IAC1G,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,QAAgB,EAAE,YAA0B;QAC9D,MAAM,KAAK,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,cAAI,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,QAAQ,GAAwB,EAAE,GAAG,YAAY,CAAC,QAAQ,EAAE,CAAC;QAEjE,+CAA+C;QAC/C,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,WAAW,GAAG,cAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YAC3C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YAE9D,IAAI,YAAY,EAAE,QAAQ,EAAE,CAAC;gBAC3B,QAAQ,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,YAAY,CAAC,QAAQ,EAAE,CAAC;YACvD,CAAC;QACH,CAAC;QAED,8DAA8D;QAC9D,MAAM,iBAAiB,GAAwB,EAAE,CAAC;QAClD,MAAM,OAAO,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAEvC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,iBAAiB,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YACtF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,MAAM,KAAK,EAAE,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAED,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,GAAW;QACxC,MAAM,UAAU,GAAG,cAAI,CAAC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;QAErD,IAAI,MAAM,kBAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,OAAO,MAAM,kBAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,kCAAkC,UAAU,GAAG,EAAE,KAAK,CAAC,CAAC;gBACtE,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,IAAS;QACzB,MAAM,IAAI,GAAG,gBAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,UAAkB;QAC9B,OAAO,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,UAAkB;QACzC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACjF,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,uCAAuC,UAAU,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,UAAkB,EAAE,UAA+B;QAClE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAElD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,wBAAwB;QACxB,MAAM,YAAY,GAAG,IAAI,mBAAY,EAAE,CAAC;QACxC,YAAY,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QAEpD,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAChC,CAAC;CACF;AAnSD,gCAmSC","sourcesContent":["import path from 'path';\nimport fs from 'fs-extra';\nimport crypto from 'crypto';\nimport axios from 'axios';\nimport { EntityInfo, Metadata, RunView, BaseEntity, CompositeKey, UserInfo } from '@memberjunction/core';\nimport { EntityConfig, FolderConfig } from '../config';\n\nexport interface RecordData {\n  primaryKey?: Record<string, any>;\n  fields: Record<string, any>;\n  relatedEntities?: Record<string, RecordData[]>;\n  sync?: {\n    lastModified: string;\n    checksum: string;\n  };\n}\n\nexport class SyncEngine {\n  private metadata: Metadata;\n  private contextUser: UserInfo;\n\n  constructor(contextUser: UserInfo) {\n    this.metadata = new Metadata();\n    this.contextUser = contextUser;\n  }\n  \n  async initialize(): Promise<void> {\n    // Initialize metadata\n    await this.metadata.Refresh();\n  }\n  \n  /**\n   * Process special references in field values\n   */\n  async processFieldValue(value: any, baseDir: string, parentRecord?: BaseEntity | null, rootRecord?: BaseEntity | null): Promise<any> {\n    if (typeof value !== 'string') {\n      return value;\n    }\n    \n    // Check for @parent: reference\n    if (value.startsWith('@parent:')) {\n      if (!parentRecord) {\n        throw new Error(`@parent reference used but no parent record available: ${value}`);\n      }\n      const fieldName = value.substring(8);\n      return parentRecord.Get(fieldName);\n    }\n    \n    // Check for @root: reference\n    if (value.startsWith('@root:')) {\n      if (!rootRecord) {\n        throw new Error(`@root reference used but no root record available: ${value}`);\n      }\n      const fieldName = value.substring(6);\n      return rootRecord.Get(fieldName);\n    }\n    \n    // Check for @file: reference\n    if (value.startsWith('@file:')) {\n      const filePath = value.substring(6);\n      const fullPath = path.resolve(baseDir, filePath);\n      \n      if (await fs.pathExists(fullPath)) {\n        return await fs.readFile(fullPath, 'utf-8');\n      } else {\n        throw new Error(`File not found: ${fullPath}`);\n      }\n    }\n    \n    // Check for @url: reference\n    if (value.startsWith('@url:')) {\n      const url = value.substring(5);\n      \n      try {\n        const response = await axios.get(url);\n        return response.data;\n      } catch (error) {\n        throw new Error(`Failed to fetch URL: ${url} - ${error}`);\n      }\n    }\n    \n    // Check for @lookup: reference\n    if (value.startsWith('@lookup:')) {\n      const lookupStr = value.substring(8);\n      \n      // Parse lookup with optional create syntax\n      // Format: EntityName.FieldName=Value?create&OtherField=Value\n      const entityMatch = lookupStr.match(/^([^.]+)\\./);\n      if (!entityMatch) {\n        throw new Error(`Invalid lookup format: ${value}`);\n      }\n      \n      const entityName = entityMatch[1];\n      const remaining = lookupStr.substring(entityName.length + 1);\n      \n      // Check if this has ?create syntax\n      const hasCreate = remaining.includes('?create');\n      const lookupPart = hasCreate ? remaining.split('?')[0] : remaining;\n      \n      // Parse the main lookup field\n      const fieldMatch = lookupPart.match(/^(.+?)=(.+)$/);\n      if (!fieldMatch) {\n        throw new Error(`Invalid lookup format: ${value}`);\n      }\n      \n      const [, fieldName, fieldValue] = fieldMatch;\n      \n      // Parse additional fields for creation if ?create is present\n      let createFields: Record<string, any> = {};\n      if (hasCreate && remaining.includes('?create&')) {\n        const createPart = remaining.split('?create&')[1];\n        const pairs = createPart.split('&');\n        for (const pair of pairs) {\n          const [key, val] = pair.split('=');\n          if (key && val) {\n            createFields[key] = decodeURIComponent(val);\n          }\n        }\n      }\n      \n      return await this.resolveLookup(entityName, fieldName, fieldValue, hasCreate, createFields);\n    }\n    \n    // Check for @env: reference\n    if (value.startsWith('@env:')) {\n      const envVar = value.substring(5);\n      const envValue = process.env[envVar];\n      \n      if (envValue === undefined) {\n        throw new Error(`Environment variable not found: ${envVar}`);\n      }\n      \n      return envValue;\n    }\n    \n    return value;\n  }\n  \n  /**\n   * Resolve a lookup reference to an ID, optionally creating the record if it doesn't exist\n   */\n  async resolveLookup(\n    entityName: string, \n    fieldName: string, \n    fieldValue: string,\n    autoCreate: boolean = false,\n    createFields: Record<string, any> = {}\n  ): Promise<string> {\n    // Debug logging handled by caller if needed\n    \n    const rv = new RunView();\n    const entityInfo = this.metadata.EntityByName(entityName);\n    if (!entityInfo) {\n      throw new Error(`Entity not found: ${entityName}`);\n    }\n    \n    const field = entityInfo.Fields.find(f => f.Name.trim().toLowerCase() === fieldName.trim().toLowerCase());\n    const quotes = field?.NeedsQuotes ? \"'\" : '';\n    const result = await rv.RunView({\n      EntityName: entityName,\n      ExtraFilter: `${fieldName} = ${quotes}${fieldValue.replace(/'/g, \"''\")}${quotes}`,\n      MaxRows: 1\n    }, this.contextUser);\n    \n    if (result.Success && result.Results.length > 0) {\n      if (entityInfo.PrimaryKeys.length > 0) {\n        const pkeyField = entityInfo.PrimaryKeys[0].Name;\n        const id = result.Results[0][pkeyField];\n        return id;\n      }\n    }\n    \n    // If not found and auto-create is enabled, create the record\n    if (autoCreate) {\n      \n      const newEntity = await this.metadata.GetEntityObject(entityName, this.contextUser);\n      if (!newEntity) {\n        throw new Error(`Failed to create entity object for: ${entityName}`);\n      }\n      \n      newEntity.NewRecord();\n      \n      // Set the lookup field\n      if (fieldName in newEntity) {\n        (newEntity as any)[fieldName] = fieldValue;\n      }\n      \n      // Set any additional fields provided\n      for (const [key, value] of Object.entries(createFields)) {\n        if (key in newEntity) {\n          (newEntity as any)[key] = value;\n        }\n      }\n      \n      // Save the new record\n      const saved = await newEntity.Save();\n      if (!saved) {\n        const errors = newEntity.LatestResult?.Errors?.join(', ') || 'Unknown error';\n        throw new Error(`Failed to auto-create ${entityName}: ${errors}`);\n      }\n      \n      // Return the new ID\n      if (entityInfo.PrimaryKeys.length > 0) {\n        const pkeyField = entityInfo.PrimaryKeys[0].Name;\n        const newId = newEntity.Get(pkeyField);\n        return newId;\n      }\n    }\n    \n    throw new Error(`Lookup failed: No record found in '${entityName}' where ${fieldName}='${fieldValue}'`);\n  }\n  \n  /**\n   * Build cascading defaults for a file path and process field values\n   */\n  async buildDefaults(filePath: string, entityConfig: EntityConfig): Promise<Record<string, any>> {\n    const parts = path.dirname(filePath).split(path.sep);\n    let defaults: Record<string, any> = { ...entityConfig.defaults };\n    \n    // Walk up the directory tree building defaults\n    let currentPath = '';\n    for (const part of parts) {\n      currentPath = path.join(currentPath, part);\n      const folderConfig = await this.loadFolderConfig(currentPath);\n      \n      if (folderConfig?.defaults) {\n        defaults = { ...defaults, ...folderConfig.defaults };\n      }\n    }\n    \n    // Process all default values (lookups, file references, etc.)\n    const processedDefaults: Record<string, any> = {};\n    const baseDir = path.dirname(filePath);\n    \n    for (const [field, value] of Object.entries(defaults)) {\n      try {\n        processedDefaults[field] = await this.processFieldValue(value, baseDir, null, null);\n      } catch (error) {\n        throw new Error(`Failed to process default for field '${field}': ${error}`);\n      }\n    }\n    \n    return processedDefaults;\n  }\n  \n  /**\n   * Load folder configuration\n   */\n  private async loadFolderConfig(dir: string): Promise<FolderConfig | null> {\n    const configPath = path.join(dir, '.mj-folder.json');\n    \n    if (await fs.pathExists(configPath)) {\n      try {\n        return await fs.readJson(configPath);\n      } catch (error) {\n        console.error(`Error loading folder config at ${configPath}:`, error);\n        return null;\n      }\n    }\n    \n    return null;\n  }\n  \n  /**\n   * Calculate checksum for data\n   */\n  calculateChecksum(data: any): string {\n    const hash = crypto.createHash('sha256');\n    hash.update(JSON.stringify(data, null, 2));\n    return hash.digest('hex');\n  }\n  \n  /**\n   * Get entity info by name\n   */\n  getEntityInfo(entityName: string): EntityInfo | null {\n    return this.metadata.EntityByName(entityName);\n  }\n  \n  /**\n   * Create a new entity object\n   */\n  async createEntityObject(entityName: string): Promise<BaseEntity> {\n    const entity = await this.metadata.GetEntityObject(entityName, this.contextUser);\n    if (!entity) {\n      throw new Error(`Failed to create entity object for: ${entityName}`);\n    }\n    return entity;\n  }\n  \n  /**\n   * Load an entity by primary key\n   */\n  async loadEntity(entityName: string, primaryKey: Record<string, any>): Promise<BaseEntity | null> {\n    const entity = await this.createEntityObject(entityName);\n    const entityInfo = this.getEntityInfo(entityName);\n    \n    if (!entityInfo) {\n      throw new Error(`Entity not found: ${entityName}`);\n    }\n    \n    // Handle composite keys\n    const compositeKey = new CompositeKey();\n    compositeKey.LoadFromSimpleObject(primaryKey);\n    const loaded = await entity.InnerLoad(compositeKey);\n    \n    return loaded ? entity : null;\n  }\n}"]}

package/oclif.manifest.json

@@ -0,0 +1,196 @@
+{
+  "commands": {
+    "init": {
+      "aliases": [],
+      "args": {},
+      "description": "Initialize a directory for metadata synchronization",
+      "examples": [
+        "<%= config.bin %> <%= command.id %>"
+      ],
+      "flags": {},
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "init",
+      "pluginAlias": "@memberjunction/metadata-sync",
+      "pluginName": "@memberjunction/metadata-sync",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": false,
+      "relativePath": [
+        "dist",
+        "commands",
+        "init",
+        "index.js"
+      ]
+    },
+    "pull": {
+      "aliases": [],
+      "args": {},
+      "description": "Pull metadata from database to local files",
+      "examples": [
+        "<%= config.bin %> <%= command.id %> --entity=\"AI Prompts\"",
+        "<%= config.bin %> <%= command.id %> --entity=\"AI Prompts\" --filter=\"CategoryID='customer-service-id'\""
+      ],
+      "flags": {
+        "entity": {
+          "description": "Entity name to pull",
+          "name": "entity",
+          "required": true,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "filter": {
+          "description": "Additional filter for pulling specific records",
+          "name": "filter",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "dry-run": {
+          "description": "Show what would be pulled without actually pulling",
+          "name": "dry-run",
+          "allowNo": false,
+          "type": "boolean"
+        }
+      },
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "pull",
+      "pluginAlias": "@memberjunction/metadata-sync",
+      "pluginName": "@memberjunction/metadata-sync",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": false,
+      "relativePath": [
+        "dist",
+        "commands",
+        "pull",
+        "index.js"
+      ]
+    },
+    "push": {
+      "aliases": [],
+      "args": {},
+      "description": "Push local file changes to the database",
+      "examples": [
+        "<%= config.bin %> <%= command.id %>",
+        "<%= config.bin %> <%= command.id %> --dry-run",
+        "<%= config.bin %> <%= command.id %> --dir=\"ai-prompts\"",
+        "<%= config.bin %> <%= command.id %> --ci"
+      ],
+      "flags": {
+        "dir": {
+          "description": "Specific entity directory to push",
+          "name": "dir",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
+        "dry-run": {
+          "description": "Show what would be pushed without actually pushing",
+          "name": "dry-run",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "ci": {
+          "description": "CI mode - no prompts, fail on issues",
+          "name": "ci",
+          "allowNo": false,
+          "type": "boolean"
+        },
+        "verbose": {
+          "char": "v",
+          "description": "Show detailed field-level output",
+          "name": "verbose",
+          "allowNo": false,
+          "type": "boolean"
+        }
+      },
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "push",
+      "pluginAlias": "@memberjunction/metadata-sync",
+      "pluginName": "@memberjunction/metadata-sync",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": false,
+      "relativePath": [
+        "dist",
+        "commands",
+        "push",
+        "index.js"
+      ]
+    },
+    "status": {
+      "aliases": [],
+      "args": {},
+      "description": "Show status of local files vs database",
+      "examples": [
+        "<%= config.bin %> <%= command.id %>",
+        "<%= config.bin %> <%= command.id %> --dir=\"ai-prompts\""
+      ],
+      "flags": {
+        "dir": {
+          "description": "Specific entity directory to check status",
+          "name": "dir",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        }
+      },
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "status",
+      "pluginAlias": "@memberjunction/metadata-sync",
+      "pluginName": "@memberjunction/metadata-sync",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": false,
+      "relativePath": [
+        "dist",
+        "commands",
+        "status",
+        "index.js"
+      ]
+    },
+    "watch": {
+      "aliases": [],
+      "args": {},
+      "description": "Watch for file changes and automatically push to database",
+      "examples": [
+        "<%= config.bin %> <%= command.id %>",
+        "<%= config.bin %> <%= command.id %> --dir=\"ai-prompts\""
+      ],
+      "flags": {
+        "dir": {
+          "description": "Specific entity directory to watch",
+          "name": "dir",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        }
+      },
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "watch",
+      "pluginAlias": "@memberjunction/metadata-sync",
+      "pluginName": "@memberjunction/metadata-sync",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": false,
+      "relativePath": [
+        "dist",
+        "commands",
+        "watch",
+        "index.js"
+      ]
+    }
+  },
+  "version": "2.46.0"
+}

package/package.json

@@ -0,0 +1,85 @@
+{
+  "name": "@memberjunction/metadata-sync",
+  "version": "2.46.0",
+  "description": "MemberJunction metadata synchronization CLI tool",
+  "keywords": [
+    "oclif",
+    "metadata",
+    "sync"
+  ],
+  "homepage": "https://github.com/MemberJunction/MJ",
+  "bugs": {
+    "url": "https://github.com/MemberJunction/MJ/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/MemberJunction/MJ.git"
+  },
+  "license": "ISC",
+  "author": "MemberJunction",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "mj-sync": "bin/run.js"
+  },
+  "files": [
+    "/bin",
+    "/dist",
+    "/oclif.manifest.json"
+  ],
+  "scripts": {
+    "build": "tsc -b",
+    "prepack": "npm run build && oclif manifest && oclif readme",
+    "postpack": "rimraf oclif.manifest.json",
+    "version": "oclif readme && git add README.md"
+  },
+  "oclif": {
+    "bin": "mj-sync",
+    "commands": "./dist/commands",
+    "hooks": {
+      "init": "./dist/hooks/init"
+    },
+    "dirname": "mj-sync",
+    "plugins": [
+      "@oclif/plugin-help",
+      "@oclif/plugin-warn-if-update-available",
+      "@oclif/plugin-version"
+    ],
+    "warn-if-update-available": {
+      "timeoutInDays": 1
+    }
+  },
+  "dependencies": {
+    "@inquirer/prompts": "^5.0.1",
+    "@memberjunction/core": "2.46.0",
+    "@memberjunction/core-entities": "2.46.0",
+    "@memberjunction/core-entities-server": "2.46.0",
+    "@memberjunction/sqlserver-dataprovider": "2.46.0",
+    "@memberjunction/graphql-dataprovider": "2.46.0",
+    "@oclif/core": "^3",
+    "@oclif/plugin-help": "^6",
+    "@oclif/plugin-version": "^2.0.17",
+    "@oclif/plugin-warn-if-update-available": "^3.0.16",
+    "chokidar": "^3.6.0",
+    "cosmiconfig": "9.0.0",
+    "dotenv": "16.4.5",
+    "fast-glob": "^3.3.2",
+    "fs-extra": "^11.2.0",
+    "ora-classic": "^5.4.2",
+    "zod": "^3.23.4",
+    "crypto": "^1.0.1",
+    "axios": "^1.6.8",
+    "typeorm": "^0.3.20"
+  },
+  "devDependencies": {
+    "@oclif/prettier-config": "^0.2.1",
+    "@types/fs-extra": "^11.0.4",
+    "oclif": "^4",
+    "rimraf": "5.0.7",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.5"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  }
+}