@memberjunction/metadata-sync 2.116.0 → 2.118.0

package/README.md

@@ -77,7 +77,18 @@ The Metadata Sync tool bridges the gap between database-stored metadata and file
 - **Push**: Upload local file changes to database
   - Process embedded collections automatically
   - Verbose mode (`-v`) for detailed output
+  - Directory filtering with `--include` and `--exclude` options
 - **Status**: Show what would change without making modifications
+  - Directory filtering with `--include` and `--exclude` options
+
+### Directory Filtering
+- **Selective Processing**: Use `--include` or `--exclude` to filter which entity directories are processed
+- **Pattern Support**: Supports glob patterns like `ai-*`, `*-test`, etc.
+- **Mutually Exclusive**: Cannot use both `--include` and `--exclude` together
+- **Use Cases**:
+  - Speed up development by excluding large/slow directories
+  - Focus on specific entities during debugging
+  - Create specialized sync workflows for different teams
 
 ### Development Workflow Integration
 - Watch mode for automatic syncing during development
@@ -1363,6 +1374,19 @@ mj sync push --dry-run
 # Push with parallel processing
 mj sync push --parallel-batch-size=20  # Process 20 records in parallel (default: 10, max: 50)
 
+# Directory filtering - exclude specific directories
+mj sync push --exclude="actions"                    # Exclude single directory
+mj sync push --exclude="actions,templates"          # Exclude multiple directories (comma-separated)
+mj sync push --exclude="*-test,*-old"              # Exclude using glob patterns
+
+# Directory filtering - include only specific directories
+mj sync push --include="prompts,agent-types"        # Include only these directories
+mj sync push --include="ai-*"                       # Include using glob patterns
+
+# Filtering works with other options
+mj sync push --dir="metadata" --exclude="actions" --dry-run
+mj sync status --exclude="actions,templates"
+
 # Show status of local vs database
 mj sync status
 

package/dist/index.d.ts

@@ -7,6 +7,15 @@ export { SQLLogger } from './lib/sql-logger';
 export { TransactionManager } from './lib/transaction-manager';
 export { JsonWriteHelper } from './lib/json-write-helper';
 export { FileWriteBatch } from './lib/file-write-batch';
+export { RecordDependencyAnalyzer } from './lib/record-dependency-analyzer';
+export type { FlattenedRecord, ReverseDependency, DependencyAnalysisResult } from './lib/record-dependency-analyzer';
+export { EntityForeignKeyHelper } from './lib/entity-foreign-key-helper';
+export type { ReverseFKInfo } from './lib/entity-foreign-key-helper';
+export { DatabaseReferenceScanner } from './lib/database-reference-scanner';
+export type { DatabaseReference } from './lib/database-reference-scanner';
+export { DeletionAuditor } from './lib/deletion-auditor';
+export type { DeletionAudit } from './lib/deletion-auditor';
+export { DeletionReportGenerator } from './lib/deletion-report-generator';
 export { InitService } from './services/InitService';
 export type { InitOptions, InitCallbacks } from './services/InitService';
 export { PullService } from './services/PullService';

package/dist/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDataProvider = exports.findEntityDirectories = exports.getSystemUser = exports.initializeProvider = exports.loadFolderConfig = exports.loadEntityConfig = exports.loadSyncConfig = exports.loadMJConfig = exports.FormattingService = exports.ValidationService = exports.WatchService = exports.FileResetService = exports.StatusService = exports.PushService = exports.PullService = exports.InitService = exports.FileWriteBatch = exports.JsonWriteHelper = exports.TransactionManager = exports.SQLLogger = exports.resetSyncEngine = exports.getSyncEngine = exports.configManager = exports.ConfigManager = exports.SyncEngine = exports.FileBackupManager = void 0;
+exports.getDataProvider = exports.findEntityDirectories = exports.getSystemUser = exports.initializeProvider = exports.loadFolderConfig = exports.loadEntityConfig = exports.loadSyncConfig = exports.loadMJConfig = exports.FormattingService = exports.ValidationService = exports.WatchService = exports.FileResetService = exports.StatusService = exports.PushService = exports.PullService = exports.InitService = exports.DeletionReportGenerator = exports.DeletionAuditor = exports.DatabaseReferenceScanner = exports.EntityForeignKeyHelper = exports.RecordDependencyAnalyzer = exports.FileWriteBatch = exports.JsonWriteHelper = exports.TransactionManager = exports.SQLLogger = exports.resetSyncEngine = exports.getSyncEngine = exports.configManager = exports.ConfigManager = exports.SyncEngine = exports.FileBackupManager = void 0;
 const core_entities_server_1 = require("@memberjunction/core-entities-server");
 // Core library exports
 var file_backup_manager_1 = require("./lib/file-backup-manager");
@@ -21,6 +21,17 @@ var json_write_helper_1 = require("./lib/json-write-helper");
 Object.defineProperty(exports, "JsonWriteHelper", { enumerable: true, get: function () { return json_write_helper_1.JsonWriteHelper; } });
 var file_write_batch_1 = require("./lib/file-write-batch");
 Object.defineProperty(exports, "FileWriteBatch", { enumerable: true, get: function () { return file_write_batch_1.FileWriteBatch; } });
+// Deletion audit exports
+var record_dependency_analyzer_1 = require("./lib/record-dependency-analyzer");
+Object.defineProperty(exports, "RecordDependencyAnalyzer", { enumerable: true, get: function () { return record_dependency_analyzer_1.RecordDependencyAnalyzer; } });
+var entity_foreign_key_helper_1 = require("./lib/entity-foreign-key-helper");
+Object.defineProperty(exports, "EntityForeignKeyHelper", { enumerable: true, get: function () { return entity_foreign_key_helper_1.EntityForeignKeyHelper; } });
+var database_reference_scanner_1 = require("./lib/database-reference-scanner");
+Object.defineProperty(exports, "DatabaseReferenceScanner", { enumerable: true, get: function () { return database_reference_scanner_1.DatabaseReferenceScanner; } });
+var deletion_auditor_1 = require("./lib/deletion-auditor");
+Object.defineProperty(exports, "DeletionAuditor", { enumerable: true, get: function () { return deletion_auditor_1.DeletionAuditor; } });
+var deletion_report_generator_1 = require("./lib/deletion-report-generator");
+Object.defineProperty(exports, "DeletionReportGenerator", { enumerable: true, get: function () { return deletion_report_generator_1.DeletionReportGenerator; } });
 // Service exports
 var InitService_1 = require("./services/InitService");
 Object.defineProperty(exports, "InitService", { enumerable: true, get: function () { return InitService_1.InitService; } });

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,+EAAgG;AAEhG,uBAAuB;AACvB,iEAA8D;AAArD,wHAAA,iBAAiB,OAAA;AAC1B,iDAA+C;AAAtC,yGAAA,UAAU,OAAA;AAEnB,uDAAoE;AAA3D,+GAAA,aAAa,OAAA;AAAE,+GAAA,aAAa,OAAA;AACrC,6DAAyE;AAAhE,kHAAA,aAAa,OAAA;AAAE,oHAAA,eAAe,OAAA;AACvC,+CAA6C;AAApC,uGAAA,SAAS,OAAA;AAClB,iEAA+D;AAAtD,yHAAA,kBAAkB,OAAA;AAC3B,6DAA0D;AAAjD,oHAAA,eAAe,OAAA;AACxB,2DAAwD;AAA/C,kHAAA,cAAc,OAAA;AAEvB,kBAAkB;AAClB,sDAAqD;AAA5C,0GAAA,WAAW,OAAA;AAGpB,sDAAqD;AAA5C,0GAAA,WAAW,OAAA;AAGpB,sDAAqD;AAA5C,0GAAA,WAAW,OAAA;AAGpB,0DAAyD;AAAhD,8GAAA,aAAa,OAAA;AAGtB,gEAA+D;AAAtD,oHAAA,gBAAgB,OAAA;AAGzB,wDAAuD;AAA9C,4GAAA,YAAY,OAAA;AAGrB,kEAAiE;AAAxD,sHAAA,iBAAiB,OAAA;AAC1B,kEAAiE;AAAxD,sHAAA,iBAAiB,OAAA;AAE1B,sBAAsB;AACtB,mCAQkB;AAPhB,sGAAA,YAAY,OAAA;AACZ,wGAAA,cAAc,OAAA;AACd,0GAAA,gBAAgB,OAAA;AAChB,0GAAA,gBAAgB,OAAA;AAMlB,qBAAqB;AACrB,uDAK8B;AAJ5B,oHAAA,kBAAkB,OAAA;AAClB,+GAAA,aAAa,OAAA;AACb,uHAAA,qBAAqB,OAAA;AACrB,iHAAA,eAAe,OAAA;AAejB,IAAA,+DAAwC,GAAE,CAAC","sourcesContent":["import { LoadAIPromptEntityExtendedServerSubClass } from '@memberjunction/core-entities-server';\n\n// Core library exports\nexport { FileBackupManager } from './lib/file-backup-manager';\nexport { SyncEngine } from './lib/sync-engine';\nexport type { RecordData } from './lib/sync-engine';\nexport { ConfigManager, configManager } from './lib/config-manager';\nexport { getSyncEngine, resetSyncEngine } from './lib/singleton-manager';\nexport { SQLLogger } from './lib/sql-logger';\nexport { TransactionManager } from './lib/transaction-manager';\nexport { JsonWriteHelper } from './lib/json-write-helper';\nexport { FileWriteBatch } from './lib/file-write-batch';\n\n// Service exports\nexport { InitService } from './services/InitService';\nexport type { InitOptions, InitCallbacks } from './services/InitService';\n\nexport { PullService } from './services/PullService';\nexport type { PullOptions, PullCallbacks, PullResult } from './services/PullService';\n\nexport { PushService } from './services/PushService';\nexport type { PushOptions, PushCallbacks, PushResult } from './services/PushService';\n\nexport { StatusService } from './services/StatusService';\nexport type { StatusOptions, StatusCallbacks, StatusResult } from './services/StatusService';\n\nexport { FileResetService } from './services/FileResetService';\nexport type { FileResetOptions, FileResetCallbacks, FileResetResult } from './services/FileResetService';\n\nexport { WatchService } from './services/WatchService';\nexport type { WatchOptions, WatchCallbacks, WatchResult } from './services/WatchService';\n\nexport { ValidationService } from './services/ValidationService';\nexport { FormattingService } from './services/FormattingService';\n\n// Configuration types\nexport {\n  loadMJConfig,\n  loadSyncConfig,\n  loadEntityConfig,\n  loadFolderConfig,\n  type EntityConfig,\n  type FolderConfig,\n  type RelatedEntityConfig\n} from './config';\n\n// Provider utilities\nexport {\n  initializeProvider,\n  getSystemUser,\n  findEntityDirectories,\n  getDataProvider\n} from './lib/provider-utils';\n\n// Validation types\nexport type {\n  ValidationResult,\n  ValidationError,\n  ValidationWarning,\n  EntityDependency,\n  FileValidationResult,\n  ValidationOptions,\n  ReferenceType,\n  ParsedReference\n} from './types/validation';\n\nLoadAIPromptEntityExtendedServerSubClass();"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,+EAAgG;AAEhG,uBAAuB;AACvB,iEAA8D;AAArD,wHAAA,iBAAiB,OAAA;AAC1B,iDAA+C;AAAtC,yGAAA,UAAU,OAAA;AAEnB,uDAAoE;AAA3D,+GAAA,aAAa,OAAA;AAAE,+GAAA,aAAa,OAAA;AACrC,6DAAyE;AAAhE,kHAAA,aAAa,OAAA;AAAE,oHAAA,eAAe,OAAA;AACvC,+CAA6C;AAApC,uGAAA,SAAS,OAAA;AAClB,iEAA+D;AAAtD,yHAAA,kBAAkB,OAAA;AAC3B,6DAA0D;AAAjD,oHAAA,eAAe,OAAA;AACxB,2DAAwD;AAA/C,kHAAA,cAAc,OAAA;AAEvB,yBAAyB;AACzB,+EAA4E;AAAnE,sIAAA,wBAAwB,OAAA;AAEjC,6EAAyE;AAAhE,mIAAA,sBAAsB,OAAA;AAE/B,+EAA4E;AAAnE,sIAAA,wBAAwB,OAAA;AAEjC,2DAAyD;AAAhD,mHAAA,eAAe,OAAA;AAExB,6EAA0E;AAAjE,oIAAA,uBAAuB,OAAA;AAEhC,kBAAkB;AAClB,sDAAqD;AAA5C,0GAAA,WAAW,OAAA;AAGpB,sDAAqD;AAA5C,0GAAA,WAAW,OAAA;AAGpB,sDAAqD;AAA5C,0GAAA,WAAW,OAAA;AAGpB,0DAAyD;AAAhD,8GAAA,aAAa,OAAA;AAGtB,gEAA+D;AAAtD,oHAAA,gBAAgB,OAAA;AAGzB,wDAAuD;AAA9C,4GAAA,YAAY,OAAA;AAGrB,kEAAiE;AAAxD,sHAAA,iBAAiB,OAAA;AAC1B,kEAAiE;AAAxD,sHAAA,iBAAiB,OAAA;AAE1B,sBAAsB;AACtB,mCAQkB;AAPhB,sGAAA,YAAY,OAAA;AACZ,wGAAA,cAAc,OAAA;AACd,0GAAA,gBAAgB,OAAA;AAChB,0GAAA,gBAAgB,OAAA;AAMlB,qBAAqB;AACrB,uDAK8B;AAJ5B,oHAAA,kBAAkB,OAAA;AAClB,+GAAA,aAAa,OAAA;AACb,uHAAA,qBAAqB,OAAA;AACrB,iHAAA,eAAe,OAAA;AAejB,IAAA,+DAAwC,GAAE,CAAC","sourcesContent":["import { LoadAIPromptEntityExtendedServerSubClass } from '@memberjunction/core-entities-server';\n\n// Core library exports\nexport { FileBackupManager } from './lib/file-backup-manager';\nexport { SyncEngine } from './lib/sync-engine';\nexport type { RecordData } from './lib/sync-engine';\nexport { ConfigManager, configManager } from './lib/config-manager';\nexport { getSyncEngine, resetSyncEngine } from './lib/singleton-manager';\nexport { SQLLogger } from './lib/sql-logger';\nexport { TransactionManager } from './lib/transaction-manager';\nexport { JsonWriteHelper } from './lib/json-write-helper';\nexport { FileWriteBatch } from './lib/file-write-batch';\n\n// Deletion audit exports\nexport { RecordDependencyAnalyzer } from './lib/record-dependency-analyzer';\nexport type { FlattenedRecord, ReverseDependency, DependencyAnalysisResult } from './lib/record-dependency-analyzer';\nexport { EntityForeignKeyHelper } from './lib/entity-foreign-key-helper';\nexport type { ReverseFKInfo } from './lib/entity-foreign-key-helper';\nexport { DatabaseReferenceScanner } from './lib/database-reference-scanner';\nexport type { DatabaseReference } from './lib/database-reference-scanner';\nexport { DeletionAuditor } from './lib/deletion-auditor';\nexport type { DeletionAudit } from './lib/deletion-auditor';\nexport { DeletionReportGenerator } from './lib/deletion-report-generator';\n\n// Service exports\nexport { InitService } from './services/InitService';\nexport type { InitOptions, InitCallbacks } from './services/InitService';\n\nexport { PullService } from './services/PullService';\nexport type { PullOptions, PullCallbacks, PullResult } from './services/PullService';\n\nexport { PushService } from './services/PushService';\nexport type { PushOptions, PushCallbacks, PushResult } from './services/PushService';\n\nexport { StatusService } from './services/StatusService';\nexport type { StatusOptions, StatusCallbacks, StatusResult } from './services/StatusService';\n\nexport { FileResetService } from './services/FileResetService';\nexport type { FileResetOptions, FileResetCallbacks, FileResetResult } from './services/FileResetService';\n\nexport { WatchService } from './services/WatchService';\nexport type { WatchOptions, WatchCallbacks, WatchResult } from './services/WatchService';\n\nexport { ValidationService } from './services/ValidationService';\nexport { FormattingService } from './services/FormattingService';\n\n// Configuration types\nexport {\n  loadMJConfig,\n  loadSyncConfig,\n  loadEntityConfig,\n  loadFolderConfig,\n  type EntityConfig,\n  type FolderConfig,\n  type RelatedEntityConfig\n} from './config';\n\n// Provider utilities\nexport {\n  initializeProvider,\n  getSystemUser,\n  findEntityDirectories,\n  getDataProvider\n} from './lib/provider-utils';\n\n// Validation types\nexport type {\n  ValidationResult,\n  ValidationError,\n  ValidationWarning,\n  EntityDependency,\n  FileValidationResult,\n  ValidationOptions,\n  ReferenceType,\n  ParsedReference\n} from './types/validation';\n\nLoadAIPromptEntityExtendedServerSubClass();"]}

package/dist/lib/database-reference-scanner.d.ts

@@ -0,0 +1,56 @@
+import { Metadata, UserInfo, CompositeKey } from '@memberjunction/core';
+import { FlattenedRecord } from './record-dependency-analyzer';
+import { ReverseFKInfo } from './entity-foreign-key-helper';
+/**
+ * Represents a database record that references a record being deleted
+ */
+export interface DatabaseReference {
+    entityName: string;
+    primaryKey: CompositeKey;
+    referencingField: string;
+    referencedEntity: string;
+    referencedKey: CompositeKey;
+    existsInMetadata: boolean;
+}
+/**
+ * Scans the database for existing records that reference records marked for deletion
+ * This helps identify:
+ * 1. Database-only records that will prevent deletion
+ * 2. Records that should be included in the deletion plan
+ */
+export declare class DatabaseReferenceScanner {
+    private metadata;
+    private contextUser;
+    constructor(metadata: Metadata, contextUser: UserInfo);
+    /**
+     * Scan database for records that reference records marked for deletion
+     *
+     * @param recordsToDelete Records that will be deleted
+     * @param reverseFKMap Map of entity -> entities that reference it
+     * @param allMetadataRecords All records from metadata (for checking if DB record exists in metadata)
+     * @returns Array of database references found
+     */
+    scanForReferences(recordsToDelete: FlattenedRecord[], reverseFKMap: Map<string, ReverseFKInfo[]>, allMetadataRecords: FlattenedRecord[]): Promise<DatabaseReference[]>;
+    /**
+     * Extract primary key from a flattened record
+     */
+    private extractPrimaryKey;
+    /**
+     * Get primary key from a simple record (plain object from RunView with ResultType: 'simple')
+     */
+    private getPrimaryKeyFromSimpleRecord;
+    /**
+     * Check if a database record exists in metadata files
+     */
+    private checkIfInMetadata;
+    /**
+     * Get orphaned references (database-only records not in metadata)
+     * These will prevent deletion unless handled
+     */
+    getOrphanedReferences(references: DatabaseReference[]): DatabaseReference[];
+    /**
+     * Get metadata references (records in metadata that reference deletion targets)
+     * These should already be marked for deletion if the user set things up correctly
+     */
+    getMetadataReferences(references: DatabaseReference[]): DatabaseReference[];
+}

package/dist/lib/database-reference-scanner.js

@@ -0,0 +1,175 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatabaseReferenceScanner = void 0;
+const core_1 = require("@memberjunction/core");
+/**
+ * Scans the database for existing records that reference records marked for deletion
+ * This helps identify:
+ * 1. Database-only records that will prevent deletion
+ * 2. Records that should be included in the deletion plan
+ */
+class DatabaseReferenceScanner {
+    metadata;
+    contextUser;
+    constructor(metadata, contextUser) {
+        this.metadata = metadata;
+        this.contextUser = contextUser;
+    }
+    /**
+     * Scan database for records that reference records marked for deletion
+     *
+     * @param recordsToDelete Records that will be deleted
+     * @param reverseFKMap Map of entity -> entities that reference it
+     * @param allMetadataRecords All records from metadata (for checking if DB record exists in metadata)
+     * @returns Array of database references found
+     */
+    async scanForReferences(recordsToDelete, reverseFKMap, allMetadataRecords) {
+        const references = [];
+        const rv = new core_1.RunView();
+        for (const record of recordsToDelete) {
+            const entityName = record.entityName;
+            const primaryKey = this.extractPrimaryKey(record);
+            if (!primaryKey) {
+                console.warn(`Cannot scan references for ${entityName}: no primary key found`);
+                continue;
+            }
+            // Find all entities that could reference this one
+            const referencingEntities = reverseFKMap.get(entityName) || [];
+            for (const refInfo of referencingEntities) {
+                try {
+                    // Query database for existing references
+                    // Use the actual value from the primary key, not the concatenated string format
+                    const pkValue = primaryKey.KeyValuePairs.length === 1
+                        ? primaryKey.KeyValuePairs[0].Value
+                        : primaryKey.ToConcatenatedString();
+                    const filter = `${refInfo.fieldName} = '${pkValue}'`;
+                    const result = await rv.RunView({
+                        EntityName: refInfo.entityName,
+                        ExtraFilter: filter,
+                        ResultType: 'simple' // More efficient - we don't need entity objects
+                    }, this.contextUser);
+                    if (result.Success && result.Results && result.Results.length > 0) {
+                        // Found database records that reference this record
+                        for (const dbRecord of result.Results) {
+                            const refPrimaryKey = this.getPrimaryKeyFromSimpleRecord(dbRecord, refInfo.entityName);
+                            if (!refPrimaryKey) {
+                                console.warn(`Cannot get primary key for ${refInfo.entityName} record`);
+                                continue;
+                            }
+                            references.push({
+                                entityName: refInfo.entityName,
+                                primaryKey: refPrimaryKey,
+                                referencingField: refInfo.fieldName,
+                                referencedEntity: entityName,
+                                referencedKey: primaryKey,
+                                existsInMetadata: this.checkIfInMetadata(refInfo.entityName, refPrimaryKey, allMetadataRecords // Check against ALL metadata, not just deletes
+                                )
+                            });
+                        }
+                    }
+                }
+                catch (error) {
+                    console.error(`Error scanning references from ${refInfo.entityName}.${refInfo.fieldName} ` +
+                        `to ${entityName}:`, error);
+                }
+            }
+        }
+        return references;
+    }
+    /**
+     * Extract primary key from a flattened record
+     */
+    extractPrimaryKey(record) {
+        const entityInfo = this.metadata.Entities.find(e => e.Name === record.entityName);
+        if (!entityInfo) {
+            return null;
+        }
+        const primaryKeys = entityInfo.PrimaryKeys;
+        if (primaryKeys.length === 0) {
+            return null;
+        }
+        // Single primary key
+        if (primaryKeys.length === 1) {
+            const pkField = primaryKeys[0].Name;
+            const pkValue = record.record.primaryKey?.[pkField] || record.record.fields?.[pkField];
+            if (!pkValue) {
+                return null;
+            }
+            return core_1.CompositeKey.FromID(pkValue.toString());
+        }
+        // Composite primary key
+        const keyPairs = [];
+        for (const pk of primaryKeys) {
+            const value = record.record.primaryKey?.[pk.Name] || record.record.fields?.[pk.Name];
+            if (!value) {
+                return null;
+            }
+            keyPairs.push({ FieldName: pk.Name, Value: value.toString() });
+        }
+        return core_1.CompositeKey.FromKeyValuePairs(keyPairs);
+    }
+    /**
+     * Get primary key from a simple record (plain object from RunView with ResultType: 'simple')
+     */
+    getPrimaryKeyFromSimpleRecord(record, entityName) {
+        const entityInfo = this.metadata.Entities.find(e => e.Name === entityName);
+        if (!entityInfo) {
+            return null;
+        }
+        const primaryKeys = entityInfo.PrimaryKeys;
+        if (primaryKeys.length === 0) {
+            return null;
+        }
+        // Single primary key
+        if (primaryKeys.length === 1) {
+            const pkField = primaryKeys[0].Name;
+            const pkValue = record[pkField];
+            if (!pkValue) {
+                return null;
+            }
+            return core_1.CompositeKey.FromID(pkValue.toString());
+        }
+        // Composite primary key
+        const keyPairs = [];
+        for (const pk of primaryKeys) {
+            const value = record[pk.Name];
+            if (!value) {
+                return null;
+            }
+            keyPairs.push({ FieldName: pk.Name, Value: value.toString() });
+        }
+        return core_1.CompositeKey.FromKeyValuePairs(keyPairs);
+    }
+    /**
+     * Check if a database record exists in metadata files
+     */
+    checkIfInMetadata(entityName, primaryKey, records) {
+        const keyString = primaryKey.ToConcatenatedString();
+        for (const record of records) {
+            if (record.entityName !== entityName) {
+                continue;
+            }
+            const recordKey = this.extractPrimaryKey(record);
+            if (recordKey && recordKey.ToConcatenatedString() === keyString) {
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Get orphaned references (database-only records not in metadata)
+     * These will prevent deletion unless handled
+     */
+    getOrphanedReferences(references) {
+        return references.filter(ref => !ref.existsInMetadata);
+    }
+    /**
+     * Get metadata references (records in metadata that reference deletion targets)
+     * These should already be marked for deletion if the user set things up correctly
+     */
+    getMetadataReferences(references) {
+        return references.filter(ref => ref.existsInMetadata);
+    }
+}
+exports.DatabaseReferenceScanner = DatabaseReferenceScanner;
+//# sourceMappingURL=database-reference-scanner.js.map

package/dist/lib/database-reference-scanner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"database-reference-scanner.js","sourceRoot":"","sources":["../../src/lib/database-reference-scanner.ts"],"names":[],"mappings":";;;AAAA,+CAAiF;AAgBjF;;;;;GAKG;AACH,MAAa,wBAAwB;IAErB;IACA;IAFZ,YACY,QAAkB,EAClB,WAAqB;QADrB,aAAQ,GAAR,QAAQ,CAAU;QAClB,gBAAW,GAAX,WAAW,CAAU;IAC9B,CAAC;IAEJ;;;;;;;OAOG;IACH,KAAK,CAAC,iBAAiB,CACnB,eAAkC,EAClC,YAA0C,EAC1C,kBAAqC;QAErC,MAAM,UAAU,GAAwB,EAAE,CAAC;QAC3C,MAAM,EAAE,GAAG,IAAI,cAAO,EAAE,CAAC;QAEzB,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YACrC,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,UAAU,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,8BAA8B,UAAU,wBAAwB,CAAC,CAAC;gBAC/E,SAAS;YACb,CAAC;YAED,kDAAkD;YAClD,MAAM,mBAAmB,GAAG,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAE/D,KAAK,MAAM,OAAO,IAAI,mBAAmB,EAAE,CAAC;gBACxC,IAAI,CAAC;oBACD,yCAAyC;oBACzC,gFAAgF;oBAChF,MAAM,OAAO,GAAG,UAAU,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC;wBACjD,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK;wBACnC,CAAC,CAAC,UAAU,CAAC,oBAAoB,EAAE,CAAC;oBAExC,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,OAAO,OAAO,GAAG,CAAC;oBAErD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;wBAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;wBAC9B,WAAW,EAAE,MAAM;wBACnB,UAAU,EAAE,QAAQ,CAAC,gDAAgD;qBACxE,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;oBAErB,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChE,oDAAoD;wBACpD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;4BACpC,MAAM,aAAa,GAAG,IAAI,CAAC,6BAA6B,CACpD,QAAQ,EACR,OAAO,CAAC,UAAU,CACrB,CAAC;4BAEF,IAAI,CAAC,aAAa,EAAE,CAAC;gCACjB,OAAO,CAAC,IAAI,CAAC,8BAA8B,OAAO,CAAC,UAAU,SAAS,CAAC,CAAC;gCACxE,SAAS;4BACb,CAAC;4BAED,UAAU,CAAC,IAAI,CAAC;gCACZ,UAAU,EAAE,OAAO,CAAC,UAAU;gCAC9B,UAAU,EAAE,aAAa;gCACzB,gBAAgB,EAAE,OAAO,CAAC,SAAS;gCACnC,gBAAgB,EAAE,UAAU;gCAC5B,aAAa,EAAE,UAAU;gCACzB,gBAAgB,EAAE,IAAI,CAAC,iBAAiB,CACpC,OAAO,CAAC,UAAU,EAClB,aAAa,EACb,kBAAkB,CAAE,+CAA+C;iCACtE;6BACJ,CAAC,CAAC;wBACP,CAAC;oBACL,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,OAAO,CAAC,KAAK,CACT,kCAAkC,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,SAAS,GAAG;wBAC5E,MAAM,UAAU,GAAG,EACnB,KAAK,CACR,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,MAAuB;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,UAAU,CAAC,CAAC;QAClF,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;QAC3C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,qBAAqB;QACrB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACpC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC;YAEvF,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,OAAO,mBAAY,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,wBAAwB;QACxB,MAAM,QAAQ,GAA2C,EAAE,CAAC;QAC5D,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YACrF,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,mBAAY,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACK,6BAA6B,CAAC,MAA+B,EAAE,UAAkB;QACrF,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QAC3E,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;QAC3C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,qBAAqB;QACrB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACpC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;YAEhC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACX,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,OAAO,mBAAY,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,wBAAwB;QACxB,MAAM,QAAQ,GAA2C,EAAE,CAAC;QAC5D,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,OAAO,IAAI,CAAC;YAChB,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,mBAAY,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACK,iBAAiB,CACrB,UAAkB,EAClB,UAAwB,EACxB,OAA0B;QAE1B,MAAM,SAAS,GAAG,UAAU,CAAC,oBAAoB,EAAE,CAAC;QAEpD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,MAAM,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBACnC,SAAS;YACb,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,SAAS,IAAI,SAAS,CAAC,oBAAoB,EAAE,KAAK,SAAS,EAAE,CAAC;gBAC9D,OAAO,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,UAA+B;QACjD,OAAO,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,qBAAqB,CAAC,UAA+B;QACjD,OAAO,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC1D,CAAC;CACJ;AA/MD,4DA+MC","sourcesContent":["import { Metadata, RunView, UserInfo, CompositeKey } from '@memberjunction/core';\nimport { FlattenedRecord } from './record-dependency-analyzer';\nimport { ReverseFKInfo } from './entity-foreign-key-helper';\n\n/**\n * Represents a database record that references a record being deleted\n */\nexport interface DatabaseReference {\n    entityName: string;         // Entity containing the referencing record\n    primaryKey: CompositeKey;   // Primary key of the referencing record\n    referencingField: string;   // Foreign key field making the reference\n    referencedEntity: string;   // Entity being referenced\n    referencedKey: CompositeKey; // Primary key of referenced record\n    existsInMetadata: boolean;  // Whether this record exists in metadata files\n}\n\n/**\n * Scans the database for existing records that reference records marked for deletion\n * This helps identify:\n * 1. Database-only records that will prevent deletion\n * 2. Records that should be included in the deletion plan\n */\nexport class DatabaseReferenceScanner {\n    constructor(\n        private metadata: Metadata,\n        private contextUser: UserInfo\n    ) {}\n\n    /**\n     * Scan database for records that reference records marked for deletion\n     *\n     * @param recordsToDelete Records that will be deleted\n     * @param reverseFKMap Map of entity -> entities that reference it\n     * @param allMetadataRecords All records from metadata (for checking if DB record exists in metadata)\n     * @returns Array of database references found\n     */\n    async scanForReferences(\n        recordsToDelete: FlattenedRecord[],\n        reverseFKMap: Map<string, ReverseFKInfo[]>,\n        allMetadataRecords: FlattenedRecord[]\n    ): Promise<DatabaseReference[]> {\n        const references: DatabaseReference[] = [];\n        const rv = new RunView();\n\n        for (const record of recordsToDelete) {\n            const entityName = record.entityName;\n            const primaryKey = this.extractPrimaryKey(record);\n\n            if (!primaryKey) {\n                console.warn(`Cannot scan references for ${entityName}: no primary key found`);\n                continue;\n            }\n\n            // Find all entities that could reference this one\n            const referencingEntities = reverseFKMap.get(entityName) || [];\n\n            for (const refInfo of referencingEntities) {\n                try {\n                    // Query database for existing references\n                    // Use the actual value from the primary key, not the concatenated string format\n                    const pkValue = primaryKey.KeyValuePairs.length === 1\n                        ? primaryKey.KeyValuePairs[0].Value\n                        : primaryKey.ToConcatenatedString();\n\n                    const filter = `${refInfo.fieldName} = '${pkValue}'`;\n\n                    const result = await rv.RunView({\n                        EntityName: refInfo.entityName,\n                        ExtraFilter: filter,\n                        ResultType: 'simple' // More efficient - we don't need entity objects\n                    }, this.contextUser);\n\n                    if (result.Success && result.Results && result.Results.length > 0) {\n                        // Found database records that reference this record\n                        for (const dbRecord of result.Results) {\n                            const refPrimaryKey = this.getPrimaryKeyFromSimpleRecord(\n                                dbRecord,\n                                refInfo.entityName\n                            );\n\n                            if (!refPrimaryKey) {\n                                console.warn(`Cannot get primary key for ${refInfo.entityName} record`);\n                                continue;\n                            }\n\n                            references.push({\n                                entityName: refInfo.entityName,\n                                primaryKey: refPrimaryKey,\n                                referencingField: refInfo.fieldName,\n                                referencedEntity: entityName,\n                                referencedKey: primaryKey,\n                                existsInMetadata: this.checkIfInMetadata(\n                                    refInfo.entityName,\n                                    refPrimaryKey,\n                                    allMetadataRecords  // Check against ALL metadata, not just deletes\n                                )\n                            });\n                        }\n                    }\n                } catch (error) {\n                    console.error(\n                        `Error scanning references from ${refInfo.entityName}.${refInfo.fieldName} ` +\n                        `to ${entityName}:`,\n                        error\n                    );\n                }\n            }\n        }\n\n        return references;\n    }\n\n    /**\n     * Extract primary key from a flattened record\n     */\n    private extractPrimaryKey(record: FlattenedRecord): CompositeKey | null {\n        const entityInfo = this.metadata.Entities.find(e => e.Name === record.entityName);\n        if (!entityInfo) {\n            return null;\n        }\n\n        const primaryKeys = entityInfo.PrimaryKeys;\n        if (primaryKeys.length === 0) {\n            return null;\n        }\n\n        // Single primary key\n        if (primaryKeys.length === 1) {\n            const pkField = primaryKeys[0].Name;\n            const pkValue = record.record.primaryKey?.[pkField] || record.record.fields?.[pkField];\n\n            if (!pkValue) {\n                return null;\n            }\n\n            return CompositeKey.FromID(pkValue.toString());\n        }\n\n        // Composite primary key\n        const keyPairs: { FieldName: string; Value: string }[] = [];\n        for (const pk of primaryKeys) {\n            const value = record.record.primaryKey?.[pk.Name] || record.record.fields?.[pk.Name];\n            if (!value) {\n                return null;\n            }\n            keyPairs.push({ FieldName: pk.Name, Value: value.toString() });\n        }\n\n        return CompositeKey.FromKeyValuePairs(keyPairs);\n    }\n\n    /**\n     * Get primary key from a simple record (plain object from RunView with ResultType: 'simple')\n     */\n    private getPrimaryKeyFromSimpleRecord(record: Record<string, unknown>, entityName: string): CompositeKey | null {\n        const entityInfo = this.metadata.Entities.find(e => e.Name === entityName);\n        if (!entityInfo) {\n            return null;\n        }\n\n        const primaryKeys = entityInfo.PrimaryKeys;\n        if (primaryKeys.length === 0) {\n            return null;\n        }\n\n        // Single primary key\n        if (primaryKeys.length === 1) {\n            const pkField = primaryKeys[0].Name;\n            const pkValue = record[pkField];\n\n            if (!pkValue) {\n                return null;\n            }\n\n            return CompositeKey.FromID(pkValue.toString());\n        }\n\n        // Composite primary key\n        const keyPairs: { FieldName: string; Value: string }[] = [];\n        for (const pk of primaryKeys) {\n            const value = record[pk.Name];\n            if (!value) {\n                return null;\n            }\n            keyPairs.push({ FieldName: pk.Name, Value: value.toString() });\n        }\n\n        return CompositeKey.FromKeyValuePairs(keyPairs);\n    }\n\n    /**\n     * Check if a database record exists in metadata files\n     */\n    private checkIfInMetadata(\n        entityName: string,\n        primaryKey: CompositeKey,\n        records: FlattenedRecord[]\n    ): boolean {\n        const keyString = primaryKey.ToConcatenatedString();\n\n        for (const record of records) {\n            if (record.entityName !== entityName) {\n                continue;\n            }\n\n            const recordKey = this.extractPrimaryKey(record);\n            if (recordKey && recordKey.ToConcatenatedString() === keyString) {\n                return true;\n            }\n        }\n\n        return false;\n    }\n\n    /**\n     * Get orphaned references (database-only records not in metadata)\n     * These will prevent deletion unless handled\n     */\n    getOrphanedReferences(references: DatabaseReference[]): DatabaseReference[] {\n        return references.filter(ref => !ref.existsInMetadata);\n    }\n\n    /**\n     * Get metadata references (records in metadata that reference deletion targets)\n     * These should already be marked for deletion if the user set things up correctly\n     */\n    getMetadataReferences(references: DatabaseReference[]): DatabaseReference[] {\n        return references.filter(ref => ref.existsInMetadata);\n    }\n}\n"]}

package/dist/lib/deletion-auditor.d.ts

@@ -0,0 +1,76 @@
+import { Metadata, UserInfo } from '@memberjunction/core';
+import { FlattenedRecord, ReverseDependency } from './record-dependency-analyzer';
+import { DatabaseReference } from './database-reference-scanner';
+/**
+ * Complete audit result for deletion operations
+ */
+export interface DeletionAudit {
+    explicitDeletes: Map<string, FlattenedRecord>;
+    implicitDeletes: Map<string, FlattenedRecord>;
+    alreadyDeleted: Map<string, FlattenedRecord>;
+    databaseOnlyReferences: DatabaseReference[];
+    reverseDependencies: Map<string, ReverseDependency[]>;
+    deletionLevels: FlattenedRecord[][];
+    circularDependencies: string[][];
+    orphanedReferences: ReverseDependency[];
+}
+/**
+ * Performs comprehensive deletion auditing across all metadata files
+ * Identifies all records that need to be deleted, in what order, and potential issues
+ */
+export declare class DeletionAuditor {
+    private metadata;
+    private contextUser;
+    constructor(metadata: Metadata, contextUser: UserInfo);
+    /**
+     * Perform comprehensive deletion audit across all metadata files
+     *
+     * This analyzes:
+     * 1. Which records are explicitly marked for deletion
+     * 2. Which records must be implicitly deleted (due to FK constraints)
+     * 3. Database-only references that will prevent deletion
+     * 4. Correct deletion order (reverse topological sort)
+     * 5. Circular dependencies
+     *
+     * @param allRecords All flattened records from all metadata files
+     * @returns Complete deletion audit
+     */
+    auditDeletions(allRecords: FlattenedRecord[]): Promise<DeletionAudit>;
+    /**
+     * Find all records marked with delete: true
+     */
+    private findExplicitDeletes;
+    /**
+     * Find all records that depend on records being deleted (implicit deletes)
+     * Uses BFS to find transitive dependents
+     */
+    private findImplicitDeletes;
+    /**
+     * Check which records actually exist in the database
+     * Returns records separated into existing (need deletion) and already deleted
+     */
+    private checkRecordExistence;
+    /**
+     * Find orphaned references (database records not in metadata files)
+     * These will prevent deletion unless handled
+     */
+    private findOrphanedReferences;
+    /**
+     * Check for circular dependencies among records to delete
+     * This would prevent safe deletion order
+     */
+    private findCircularDependencies;
+    /**
+     * Get a display name for a record (for error messages)
+     */
+    private getRecordDisplayName;
+    /**
+     * Check if deletion audit has blocking issues
+     */
+    isValid(audit: DeletionAudit): boolean;
+    /**
+     * Check if deletion audit requires user confirmation
+     * (due to implicit deletes)
+     */
+    requiresConfirmation(audit: DeletionAudit): boolean;
+}