@memberjunction/ai-vector-dupe 5.20.0 → 5.22.0

package/dist/duplicateRecordDetector.js

@@ -1,332 +1,535 @@
+/**
+ * @fileoverview Modernized duplicate record detection engine.
+ *
+ * Orchestrates the full pipeline: vectorize records, query for similar candidates,
+ * optionally apply hybrid search (RRF) and reranking, persist match results,
+ * and auto-merge high-confidence duplicates.
+ *
+ * Supports both list-based batch detection and single-record checks.
+ *
+ * @module @memberjunction/ai-vector-dupe
+ */
 import { BaseEmbeddings, GetAIAPIKey } from "@memberjunction/ai";
-import { PotentialDuplicateResponse, RunView, PotentialDuplicateResult, Metadata, LogError, RecordMergeRequest, PotentialDuplicate } from "@memberjunction/core";
-import { LogStatus } from "@memberjunction/core";
+import { PotentialDuplicateResponse, PotentialDuplicateResult, LogError, LogStatus, RecordMergeRequest, PotentialDuplicate, } from "@memberjunction/core";
 import { VectorDBBase } from "@memberjunction/ai-vectordb";
 import { MJGlobal, UUIDsEqual } from "@memberjunction/global";
 import { VectorBase } from "@memberjunction/ai-vectors";
 import { EntityDocumentTemplateParser, EntityVectorSyncer } from "@memberjunction/ai-vector-sync";
+import { TemplateEngineServer } from "@memberjunction/templates";
+/** Default number of nearest neighbors to retrieve per record */
+const DEFAULT_TOP_K = 5;
+/** Default concurrency limit for parallel vector queries */
+const QUERY_CONCURRENCY_LIMIT = 5;
+/** Default batch size for parallel database saves */
+const SAVE_BATCH_SIZE = 20;
+/**
+ * Modernized duplicate record detection engine.
+ *
+ * Supports:
+ * - List-based batch detection (getDuplicateRecords)
+ * - Single-record duplicate check (CheckSingleRecord)
+ * - Hybrid search via RRF when vector DB supports it
+ * - Optional post-retrieval reranking via MJ's BaseReranker
+ * - Configurable topK, thresholds, and progress reporting
+ */
 export class DuplicateRecordDetector extends VectorBase {
-    constructor() {
-        super();
-        this._runView = new RunView();
-    }
-    async getDuplicateRecords(params, contextUser) {
-        super.CurrentUser = contextUser;
-        //for testing
-        //params.EntityID = 25050001;
-        //params.EntityDocumentID = 17;
-        let vectorizer = new EntityVectorSyncer();
-        vectorizer.CurrentUser = super.CurrentUser;
-        let entityDocument = await vectorizer.GetEntityDocument(params.EntityDocumentID);
-        if (!entityDocument) {
-            throw Error(`No Entity Document found with ID ${params.EntityDocumentID}`);
-            //Update: No longer creating an entity docuement if one is not found
-            //If an entitiy document is not found, that is our indicator that the
-            //underlying entity's records have not been vectorized yet
-            //const defaultVectorDB: MJVectorDatabaseEntity = super.getVectorDatabase();
-            //const defaultAIModel: MJAIModelEntity = super.getAIModel();
-            //entityDocument = await this.createEntityDocumentForEntity(params.EntityID, defaultVectorDB, defaultAIModel);
-        }
-        let response = new PotentialDuplicateResponse();
+    /**
+     * Run duplicate detection for all records in a list.
+     *
+     * Flow: validate → vectorize → embed → query → (optional rerank) → persist → (optional merge)
+     */
+    async GetDuplicateRecords(params, contextUser) {
+        this.CurrentUser = contextUser;
+        const options = params.Options ?? {};
+        const startTime = Date.now();
+        const response = new PotentialDuplicateResponse();
         response.PotentialDuplicateResult = [];
+        // Step 1: Validate entity document
+        const entityDocument = await this.ValidateEntityDocument(params.EntityDocumentID);
         if (!entityDocument) {
-            response.ErrorMessage = `No active Entity Document found for entity ${params.EntityID}`;
+            response.ErrorMessage = `No active Entity Document found for ID ${params.EntityDocumentID}`;
             response.Status = 'Error';
             return response;
         }
-        //for testing
-        const request = {
-            entityID: entityDocument.EntityID,
-            entityDocumentID: entityDocument.ID,
-            listBatchCount: 20,
-            options: {},
-            CurrentUser: contextUser
-        };
-        console.log("vectorizing entity...");
-        const templateParser = EntityDocumentTemplateParser.CreateInstance();
-        await vectorizer.VectorizeEntity(request, super.CurrentUser);
-        const list = await this.getListEntity(params.ListID);
-        let duplicateRun = params.Options?.DuplicateRunID ? await this.getDuplicateRunEntity(params.Options?.DuplicateRunID) : await this.getDuplicateRunEntityByListID(list.ID);
-        //let duplicateRun: MJDuplicateRunEntity = await this.createDuplicateRunRecord(entityDocument, list.ID);
-        const duplicateRunDetails = await this.createDuplicateRunDetailRecordsByListID(list.ID, duplicateRun.ID);
-        //await this.createListDetailsForDupeRun(params.RecordIDs, list.ID);
-        LogStatus(`Using vector database ${entityDocument.VectorDatabaseID} and AI Model ${entityDocument.AIModelID}`);
-        const vectorDB = super.GetVectorDatabase(entityDocument.VectorDatabaseID);
-        const aiModel = super.GetAIModel(entityDocument.AIModelID);
-        LogStatus(`AIModel driver class: ${aiModel.DriverClass}`);
-        LogStatus(`VectorDB class key: ${vectorDB.ClassKey}`);
-        const embeddingAPIKey = GetAIAPIKey(aiModel.DriverClass);
-        const vectorDBAPIKey = GetAIAPIKey(vectorDB.ClassKey);
-        if (!embeddingAPIKey) {
-            throw Error(`No API Key found for AI Model ${aiModel.DriverClass}`);
-        }
-        if (!vectorDBAPIKey) {
-            throw Error(`No API Key found for Vector Database ${vectorDB.ClassKey}`);
-        }
-        //LogStatus(`Embedding API Key: ${embeddingAPIKey} VectorDB API Key: ${vectorDBAPIKey}`);
-        this._embedding = MJGlobal.Instance.ClassFactory.CreateInstance(BaseEmbeddings, aiModel.DriverClass, embeddingAPIKey);
-        this._vectorDB = MJGlobal.Instance.ClassFactory.CreateInstance(VectorDBBase, vectorDB.ClassKey, vectorDBAPIKey);
-        if (!this._embedding) {
-            throw Error(`Failed to create Embeddings instance for AI Model ${aiModel.DriverClass}`);
-        }
-        if (!this._vectorDB) {
-            throw Error(`Failed to create Vector Database instance for ${vectorDB.ClassKey}`);
-        }
-        let records = await this.GetRecordsByListID(list.ID, entityDocument.EntityID);
+        // Step 2: Vectorize source records
+        this.reportProgress(options, 'Vectorizing', 0, 0, 0, startTime);
+        await this.VectorizeSourceRecords(entityDocument, contextUser);
+        // Step 3: Initialize providers
+        this.InitializeProviders(entityDocument);
+        // Step 4: Load list and duplicate run
+        const list = await this.LoadListEntity(params.ListID);
+        const duplicateRun = options.DuplicateRunID
+            ? await this.LoadDuplicateRun(options.DuplicateRunID)
+            : await this.LoadDuplicateRunByListID(list.ID);
+        // Step 5: Create run detail records in batches
+        const duplicateRunDetails = await this.CreateRunDetailRecordsFromList(list.ID, duplicateRun.ID);
+        // Step 6: Load and embed records
+        const records = await this.LoadRecordsByListID(list.ID, entityDocument.EntityID);
         if (records.length === 0) {
-            LogError(`No records found in list ${list.Name}, with listID ${list.ID} and EntityID ${entityDocument.EntityID} exiting early`);
+            LogError(`No records found in list ${list.Name}`);
             response.ErrorMessage = `No records found in list ${list.Name}`;
             response.Status = 'Error';
             return response;
         }
-        LogStatus("Vectorizing " + records.length + " records");
-        const recordTemplates = [];
-        //Relationship(entityID: number, entityRecord: any, relationshipName: string, maxRows: number, entityDocumentName: string)
-        let sampleTemplate = entityDocument.Template;
-        //sampleTemplate += " ${Relationship('Deals', 5, 'Sample Relationship Document for crm.Deals Entity')} ${Relationship('Deals', 5, 'Second Sample Relationship Document for crm.Deals Entity')}";
-        for (const record of records) {
-            const template = await templateParser.Parse(sampleTemplate, entityDocument.EntityID, record, contextUser);
-            recordTemplates.push(template);
-        }
-        let embedTextsResult = await this._embedding.EmbedTexts({ texts: recordTemplates, model: null });
-        const topK = 5;
-        let results = [];
-        for (const [index, vector] of embedTextsResult.vectors.entries()) {
-            const compositeKey = records[index].PrimaryKey;
-            let filterResult = await this._vectorDB.queryIndex({ vector: vector, topK: topK, includeMetadata: true, includeValues: false });
-            if (!filterResult.success) {
-                LogError(`Failed to query index for record ${compositeKey.ToString()}`);
-                continue;
-            }
-            let queryResult = await this.getVectorDuplicates(filterResult);
-            queryResult.Duplicates = queryResult.Duplicates.filter((dupe) => {
-                return dupe.ProbabilityScore >= entityDocument.PotentialMatchThreshold;
-            });
-            queryResult.EntityID = entityDocument.EntityID;
-            queryResult.RecordCompositeKey = compositeKey;
-            results.push(queryResult);
-            //now update all of the dupe run detail records
-            let dupeRunDetail = duplicateRunDetails.find((detail) => UUIDsEqual(detail.RecordID, compositeKey.Values()));
-            if (dupeRunDetail) {
-                const matchRecords = await this.createDuplicateRunDetailMatchesForRecord(dupeRunDetail.ID, queryResult);
-                queryResult.DuplicateRunDetailMatchRecordIDs = matchRecords.map((match) => match.ID);
-                dupeRunDetail.MatchStatus = 'Complete';
-                let success = await super.SaveEntity(dupeRunDetail);
-                if (!success) {
-                    LogStatus(`Failed to update Duplicate Run Detail record ${dupeRunDetail.ID}`);
-                }
-            }
-            else {
-                LogError(`Failed to find Duplicate Run Detail record for ${compositeKey.ToString()}`);
-            }
-        }
-        //almost done
+        this.reportProgress(options, 'Embedding', records.length, 0, 0, startTime);
+        // Step 7: Generate template text and embeddings
+        const templateParser = EntityDocumentTemplateParser.CreateInstance();
+        const templateTexts = await this.GenerateTemplateTexts(templateParser, entityDocument, records, contextUser);
+        const embedResult = await this.embedding.EmbedTexts({ texts: templateTexts, model: null });
+        // Step 8: Query vector DB for each record (with concurrency control)
+        this.reportProgress(options, 'Querying', records.length, 0, 0, startTime);
+        const topK = options.TopK ?? DEFAULT_TOP_K;
+        const queryResults = await this.QueryDuplicatesForRecords(records, embedResult.vectors, templateTexts, entityDocument, topK, options);
+        // Step 9: Persist match results and update run details
+        this.reportProgress(options, 'Matching', records.length, records.length, 0, startTime);
+        const results = await this.PersistMatchResults(queryResults, duplicateRunDetails, entityDocument, options, startTime);
+        // Step 10: Complete the duplicate run
         duplicateRun.ProcessingStatus = 'Complete';
         duplicateRun.EndedAt = new Date();
-        let success = await super.SaveEntity(duplicateRun);
-        if (!success) {
+        const runSaveSuccess = await this.SaveEntity(duplicateRun);
+        if (!runSaveSuccess) {
             throw new Error(`Failed to update Duplicate Run record ${duplicateRun.ID}`);
         }
-        await this.mergeRecords(response, entityDocument);
+        // Step 11: Auto-merge high-confidence matches
+        this.reportProgress(options, 'Merging', records.length, records.length, results.length, startTime);
         response.PotentialDuplicateResult = results;
+        await this.ProcessAutoMerges(response, entityDocument);
         response.Status = 'Success';
-        LogStatus("Dupe Run complete. Response:");
-        LogStatus(JSON.stringify(response, null, "\t"));
+        LogStatus(`Duplicate detection complete: ${results.length} records checked`);
         return response;
     }
-    async GetRecordsByListID(listID, entityID) {
-        const entityInfo = super.Metadata.EntityByID(entityID);
+    /**
+     * Check a single record for duplicates without requiring a list.
+     * Embeds the record and queries for matches directly.
+     */
+    async CheckSingleRecord(EntityDocumentID, RecordID, Options, ContextUser) {
+        this.CurrentUser = ContextUser;
+        const options = Options ?? {};
+        const entityDocument = await this.ValidateEntityDocument(EntityDocumentID);
+        if (!entityDocument) {
+            throw new Error(`No active Entity Document found for ID ${EntityDocumentID}`);
+        }
+        this.InitializeProviders(entityDocument);
+        // Load the single record
+        const entityInfo = this.Metadata.EntityByID(entityDocument.EntityID);
         if (!entityInfo) {
-            throw new Error(`Failed to load Entity Info with ID ${entityID}`);
+            throw new Error(`Entity not found for ID ${entityDocument.EntityID}`);
         }
-        const rvResult = await super.RunView.RunView({
+        const records = await this.RunView.RunView({
             EntityName: entityInfo.Name,
-            ExtraFilter: `ID IN (SELECT RecordID FROM __mj.vwListDetails WHERE ListID = '${listID}')`,
-            ResultType: 'entity_object'
-        }, super.CurrentUser);
-        if (!rvResult.Success) {
-            throw new Error(rvResult.ErrorMessage);
+            ExtraFilter: this.BuildExtraFilter([RecordID]),
+            ResultType: 'entity_object',
+        }, this.CurrentUser);
+        if (!records.Success || records.Results.length === 0) {
+            throw new Error(`Record not found: ${RecordID.ToString()}`);
         }
-        return rvResult.Results;
+        const record = records.Results[0];
+        const templateParser = EntityDocumentTemplateParser.CreateInstance();
+        const templateTexts = await this.GenerateTemplateTexts(templateParser, entityDocument, [record], ContextUser);
+        const embedResult = await this.embedding.EmbedTexts({ texts: templateTexts, model: null });
+        const topK = options.TopK ?? DEFAULT_TOP_K;
+        const queryResults = await this.QueryDuplicatesForRecords([record], embedResult.vectors, templateTexts, entityDocument, topK, options);
+        return queryResults.length > 0 ? queryResults[0].Duplicates : new PotentialDuplicateResult();
     }
-    async createDuplicateRunRecord(entityDocument, listID) {
-        const md = new Metadata();
-        let duplicateRun = await md.GetEntityObject('MJ: Duplicate Runs');
-        duplicateRun.NewRecord();
-        duplicateRun.EntityID = entityDocument.EntityID;
-        duplicateRun.StartedByUserID = super.CurrentUser.ID;
-        duplicateRun.StartedAt = new Date();
-        duplicateRun.ProcessingStatus = 'In Progress';
-        duplicateRun.ApprovalStatus = 'Pending';
-        duplicateRun.SourceListID = listID;
-        const saveResult = await super.SaveEntity(duplicateRun);
-        if (!saveResult) {
-            throw new Error(`Failed to save list for Potential Duplicate Run`);
+    // ─────────────────────────────────────────────
+    // Validation & Setup
+    // ─────────────────────────────────────────────
+    /**
+     * Validate and return an entity document, or null if not found.
+     */
+    async ValidateEntityDocument(entityDocumentID) {
+        const vectorizer = new EntityVectorSyncer();
+        vectorizer.CurrentUser = this.CurrentUser;
+        return vectorizer.GetEntityDocument(entityDocumentID);
+    }
+    /**
+     * Initialize embedding and vector DB providers via ClassFactory.
+     */
+    InitializeProviders(entityDocument) {
+        const aiModel = this.GetAIModel(entityDocument.AIModelID);
+        const vectorDB = this.GetVectorDatabase(entityDocument.VectorDatabaseID);
+        const embeddingAPIKey = GetAIAPIKey(aiModel.DriverClass);
+        const vectorDBAPIKey = GetAIAPIKey(vectorDB.ClassKey);
+        if (!embeddingAPIKey) {
+            throw new Error(`No API Key found for AI Model ${aiModel.DriverClass}`);
+        }
+        if (!vectorDBAPIKey) {
+            throw new Error(`No API Key found for Vector Database ${vectorDB.ClassKey}`);
+        }
+        this.embedding = MJGlobal.Instance.ClassFactory.CreateInstance(BaseEmbeddings, aiModel.DriverClass, embeddingAPIKey);
+        this.vectorDB = MJGlobal.Instance.ClassFactory.CreateInstance(VectorDBBase, vectorDB.ClassKey, vectorDBAPIKey);
+        if (!this.embedding) {
+            throw new Error(`Failed to create Embeddings instance for ${aiModel.DriverClass}`);
         }
-        return duplicateRun;
+        if (!this.vectorDB) {
+            throw new Error(`Failed to create VectorDB instance for ${vectorDB.ClassKey}`);
+        }
+        LogStatus(`Providers initialized: AI Model=${aiModel.DriverClass}, VectorDB=${vectorDB.ClassKey}`);
+    }
+    /**
+     * Run vectorization for the entity document's records.
+     */
+    async VectorizeSourceRecords(entityDocument, contextUser) {
+        const vectorizer = new EntityVectorSyncer();
+        vectorizer.CurrentUser = contextUser;
+        const request = {
+            entityID: entityDocument.EntityID,
+            entityDocumentID: entityDocument.ID,
+            listBatchCount: 20,
+            options: {},
+            CurrentUser: contextUser,
+        };
+        LogStatus(`Vectorizing entity records for document ${entityDocument.Name}`);
+        await vectorizer.VectorizeEntity(request, contextUser);
     }
-    async createDuplicateRunDetailRecords(recordIDs, duplicateRunID) {
-        let results = [];
-        const md = new Metadata();
-        for (const recordID of recordIDs) {
-            let runDetail = await md.GetEntityObject('MJ: Duplicate Run Details');
-            runDetail.NewRecord();
-            runDetail.DuplicateRunID = duplicateRunID;
-            runDetail.RecordID = recordID.ToString();
-            runDetail.MatchStatus = 'Pending';
-            runDetail.MergeStatus = 'Pending';
-            const success = await super.SaveEntity(runDetail);
-            if (success) {
-                results.push(runDetail);
+    // ─────────────────────────────────────────────
+    // Template Generation & Embedding
+    // ─────────────────────────────────────────────
+    /**
+     * Generate human-readable template text for each record using the entity document template.
+     *
+     * Loads the template from TemplateEngineServer and renders it via Nunjucks,
+     * matching the same approach used by the vectorization pipeline.
+     */
+    async GenerateTemplateTexts(templateParser, entityDocument, records, contextUser) {
+        await TemplateEngineServer.Instance.Config(false, contextUser);
+        const template = this.loadTemplate(entityDocument);
+        const templateContent = template.Content[0];
+        TemplateEngineServer.Instance.SetupNunjucks();
+        const templateTexts = [];
+        for (const record of records) {
+            // NEW convention: main entity fields are TOP-LEVEL variables (no Entity. prefix).
+            // Spread record fields directly into root context so templates use {{FieldName}}.
+            const data = { ...record.GetAll() };
+            const result = await TemplateEngineServer.Instance.RenderTemplate(template, templateContent, data, true);
+            if (result.Success) {
+                templateTexts.push(result.Output);
+            }
+            else {
+                LogError(`Template render failed for record ${record.PrimaryKey.ToString()}: ${result.Message}`);
+                templateTexts.push('');
             }
         }
-        return results;
+        return templateTexts;
     }
-    async createDuplicateRunDetailRecordsByListID(listID, duplicateRunID) {
-        let results = [];
-        const viewResults = await super.RunView.RunView({
-            EntityName: 'MJ: List Details',
-            ExtraFilter: `ListID = '${listID}'`,
-            ResultType: 'entity_object'
-        }, super.CurrentUser);
-        if (!viewResults.Success) {
-            throw new Error(viewResults.ErrorMessage);
+    /**
+     * Load the template entity from TemplateEngineServer for the given entity document.
+     */
+    loadTemplate(entityDocument) {
+        const template = TemplateEngineServer.Instance.Templates.find((t) => UUIDsEqual(t.ID, entityDocument.TemplateID));
+        if (!template) {
+            throw new Error(`Template not found for ID ${entityDocument.TemplateID}`);
         }
-        const md = new Metadata();
-        const listDetails = viewResults.Results;
-        for (const listDetail of listDetails) {
-            let runDetail = await md.GetEntityObject('MJ: Duplicate Run Details');
-            runDetail.NewRecord();
-            runDetail.DuplicateRunID = duplicateRunID;
-            runDetail.RecordID = listDetail.RecordID;
-            runDetail.MatchStatus = 'Pending';
-            runDetail.MergeStatus = 'Pending';
-            const success = await super.SaveEntity(runDetail);
-            if (success) {
-                results.push(runDetail);
+        if (template.Content.length === 0) {
+            throw new Error(`Template ${template.ID} has no content records`);
+        }
+        return template;
+    }
+    // ─────────────────────────────────────────────
+    // Vector Query & Hybrid Search
+    // ─────────────────────────────────────────────
+    /**
+     * Query the vector DB for duplicates of each record, with concurrency control.
+     * Supports hybrid search and RRF fusion when the vector DB supports it.
+     */
+    async QueryDuplicatesForRecords(records, vectors, templateTexts, entityDocument, topK, options) {
+        const tasks = records.map((record, index) => async () => {
+            const compositeKey = record.PrimaryKey;
+            const vector = vectors[index];
+            const templateText = templateTexts[index];
+            const queryResponse = await this.executeVectorQuery(vector, templateText, topK, options);
+            if (!queryResponse.success) {
+                LogError(`Failed to query index for record ${compositeKey.ToString()}`);
+                const emptyResult = new PotentialDuplicateResult();
+                emptyResult.EntityID = entityDocument.EntityID;
+                emptyResult.RecordCompositeKey = compositeKey;
+                return { SourceKey: compositeKey, TemplateText: templateText, Duplicates: emptyResult };
             }
-            else {
-                LogError("Failed to save MJDuplicateRunDetailEntity", undefined, runDetail.LatestResult);
+            const dupeResult = this.ParseVectorMatches(queryResponse, compositeKey);
+            dupeResult.Duplicates = this.FilterSelfMatches(dupeResult.Duplicates, compositeKey);
+            dupeResult.Duplicates = dupeResult.Duplicates.filter((d) => d.ProbabilityScore >= entityDocument.PotentialMatchThreshold);
+            dupeResult.EntityID = entityDocument.EntityID;
+            dupeResult.RecordCompositeKey = compositeKey;
+            return { SourceKey: compositeKey, TemplateText: templateText, Duplicates: dupeResult };
+        });
+        return RunWithConcurrency(tasks, QUERY_CONCURRENCY_LIMIT);
+    }
+    /**
+     * Execute a vector query — uses hybrid search with RRF when the provider supports it.
+     */
+    async executeVectorQuery(vector, templateText, topK, options) {
+        if (this.vectorDB.SupportsHybridSearch && templateText) {
+            return this.vectorDB.HybridQuery({
+                vector,
+                topK,
+                KeywordQuery: templateText,
+                Alpha: options.KeywordSearchWeight != null ? (1.0 - options.KeywordSearchWeight) : 0.7,
+                FusionMethod: options.FusionMethod ?? 'rrf',
+                includeMetadata: true,
+                includeValues: false,
+            });
+        }
+        return this.vectorDB.QueryIndex({
+            vector,
+            topK,
+            includeMetadata: true,
+            includeValues: false,
+        });
+    }
+    /**
+     * Parse raw vector DB matches into a PotentialDuplicateResult.
+     */
+    ParseVectorMatches(queryResponse, sourceKey) {
+        const result = new PotentialDuplicateResult();
+        if (!queryResponse.data?.matches) {
+            return result;
+        }
+        for (const match of queryResponse.data.matches) {
+            if (!match?.id)
+                continue;
+            if (!match.metadata?.RecordID) {
+                LogError(`Invalid vector metadata for match: ${match.id}`);
+                continue;
             }
+            const duplicate = new PotentialDuplicate();
+            duplicate.LoadFromConcatenatedString(match.metadata.RecordID);
+            duplicate.ProbabilityScore = match.score;
+            result.Duplicates.push(duplicate);
         }
-        return results;
+        return result;
     }
-    async getListEntity(listID) {
-        const md = new Metadata();
-        let list = await md.GetEntityObject('MJ: Lists');
-        list.ContextCurrentUser = super.CurrentUser;
+    /**
+     * Filter out self-matches where the candidate is the same record as the source.
+     */
+    FilterSelfMatches(duplicates, sourceKey) {
+        return duplicates.filter((d) => d.ToString() !== sourceKey.ToString());
+    }
+    // ─────────────────────────────────────────────
+    // Entity Loading
+    // ─────────────────────────────────────────────
+    /**
+     * Load records from an entity that are members of the specified list.
+     */
+    async LoadRecordsByListID(listID, entityID) {
+        const entityInfo = this.Metadata.EntityByID(entityID);
+        if (!entityInfo) {
+            throw new Error(`Entity not found for ID ${entityID}`);
+        }
+        const sanitizedListID = listID.replace(/'/g, "''");
+        const rvResult = await this.RunView.RunView({
+            EntityName: entityInfo.Name,
+            ExtraFilter: `ID IN (SELECT RecordID FROM __mj.vwListDetails WHERE ListID = '${sanitizedListID}')`,
+            ResultType: 'entity_object',
+        }, this.CurrentUser);
+        if (!rvResult.Success) {
+            throw new Error(rvResult.ErrorMessage);
+        }
+        return rvResult.Results;
+    }
+    async LoadListEntity(listID) {
+        const list = await this.Metadata.GetEntityObject('MJ: Lists');
+        list.ContextCurrentUser = this.CurrentUser;
         const success = await list.Load(listID);
         if (!success) {
             throw new Error(`Failed to load List record ${listID}`);
         }
         return list;
     }
-    async getDuplicateRunEntity(DupeRunID) {
-        const md = new Metadata();
-        let dupeRun = await md.GetEntityObject('MJ: Duplicate Runs');
-        dupeRun.ContextCurrentUser = super.CurrentUser;
-        const success = await dupeRun.Load(DupeRunID);
+    async LoadDuplicateRun(duplicateRunID) {
+        const dupeRun = await this.Metadata.GetEntityObject('MJ: Duplicate Runs');
+        dupeRun.ContextCurrentUser = this.CurrentUser;
+        const success = await dupeRun.Load(duplicateRunID);
         if (!success) {
-            throw new Error(`Failed to load Duplicate Run record ${DupeRunID}`);
+            throw new Error(`Failed to load Duplicate Run record ${duplicateRunID}`);
         }
         return dupeRun;
     }
-    async getDuplicateRunEntityByListID(listID) {
-        const entity = await super.RunViewForSingleValue('MJ: Duplicate Runs', `SourceListID = '${listID}'`);
+    async LoadDuplicateRunByListID(listID) {
+        const entity = await this.RunViewForSingleValue('MJ: Duplicate Runs', `SourceListID = '${listID.replace(/'/g, "''")}'`);
         if (!entity) {
-            throw new Error(`Failed to load Duplicate Run record for List ${listID}`);
+            throw new Error(`No Duplicate Run found for List ${listID}`);
         }
         return entity;
     }
-    async createListForDupeRun(entityDocument) {
-        const md = new Metadata();
-        const list = await md.GetEntityObject('MJ: Lists');
-        list.NewRecord();
-        list.Name = `Potential Duplicate Run`;
-        list.Description = `Potential Duplicate Run for ${entityDocument.Entity} Entity`;
-        list.EntityID = entityDocument.EntityID;
-        list.UserID = super.CurrentUser.ID;
-        const saveResult = await super.SaveEntity(list);
-        if (!saveResult) {
-            throw new Error(`Failed to save list for Potential Duplicate Run`);
+    // ─────────────────────────────────────────────
+    // Run Detail & Match Persistence (Batched)
+    // ─────────────────────────────────────────────
+    /**
+     * Create DuplicateRunDetail records for each item in the list, saving in parallel batches.
+     */
+    async CreateRunDetailRecordsFromList(listID, duplicateRunID) {
+        const viewResults = await this.RunView.RunView({
+            EntityName: 'MJ: List Details',
+            ExtraFilter: `ListID = '${listID.replace(/'/g, "''")}'`,
+            ResultType: 'entity_object',
+        }, this.CurrentUser);
+        if (!viewResults.Success) {
+            throw new Error(viewResults.ErrorMessage);
         }
-        return list;
+        const listDetails = viewResults.Results;
+        const results = [];
+        for (const batch of chunkArray(listDetails, SAVE_BATCH_SIZE)) {
+            const batchResults = await Promise.all(batch.map(async (listDetail) => {
+                const runDetail = await this.Metadata.GetEntityObject('MJ: Duplicate Run Details');
+                runDetail.NewRecord();
+                runDetail.DuplicateRunID = duplicateRunID;
+                runDetail.RecordID = listDetail.RecordID;
+                runDetail.MatchStatus = 'Pending';
+                runDetail.MergeStatus = 'Pending';
+                const success = await this.SaveEntity(runDetail);
+                if (!success) {
+                    LogError("Failed to save MJDuplicateRunDetailEntity", undefined, runDetail.LatestResult);
+                    return null;
+                }
+                return runDetail;
+            }));
+            for (const r of batchResults) {
+                if (r)
+                    results.push(r);
+            }
+        }
+        return results;
     }
-    async createDuplicateRunDetailMatchesForRecord(DuplicateRunDetailID, duplicateResult) {
-        const md = new Metadata();
-        let matchRecords = [];
-        for (const dupe of duplicateResult.Duplicates) {
-            const match = await md.GetEntityObject('MJ: Duplicate Run Detail Matches');
-            match.NewRecord();
-            match.DuplicateRunDetailID = DuplicateRunDetailID;
-            match.MatchRecordID = dupe.ToString();
-            match.MatchProbability = dupe.ProbabilityScore;
-            match.MatchedAt = new Date();
-            match.MergedAt = new Date();
-            match.Action = '';
-            match.ApprovalStatus = 'Pending';
-            match.MergeStatus = 'Pending';
-            let success = await super.SaveEntity(match);
-            if (success) {
-                matchRecords.push(match);
+    /**
+     * Persist match results and update run detail records.
+     */
+    async PersistMatchResults(queryResults, duplicateRunDetails, entityDocument, options, startTime) {
+        const results = [];
+        let matchesFound = 0;
+        for (const qr of queryResults) {
+            results.push(qr.Duplicates);
+            matchesFound += qr.Duplicates.Duplicates.length;
+            const detail = duplicateRunDetails.find((d) => UUIDsEqual(d.RecordID, qr.SourceKey.Values()));
+            if (detail) {
+                const matchRecords = await this.CreateMatchRecordsForDetail(detail.ID, qr.Duplicates);
+                qr.Duplicates.DuplicateRunDetailMatchRecordIDs = matchRecords.map((m) => m.ID);
+                detail.MatchStatus = 'Complete';
+                const success = await this.SaveEntity(detail);
+                if (!success) {
+                    LogError(`Failed to update Duplicate Run Detail record ${detail.ID}`);
+                }
+            }
+            else {
+                LogError(`No Duplicate Run Detail found for ${qr.SourceKey.ToString()}`);
+            }
+            this.reportProgress(options, 'Matching', queryResults.length, results.length, matchesFound, startTime);
+        }
+        return results;
+    }
+    /**
+     * Create match records for a single run detail, saving in parallel batches.
+     */
+    async CreateMatchRecordsForDetail(duplicateRunDetailID, duplicateResult) {
+        const matchRecords = [];
+        for (const batch of chunkArray(duplicateResult.Duplicates, SAVE_BATCH_SIZE)) {
+            const batchResults = await Promise.all(batch.map(async (dupe) => {
+                const match = await this.Metadata.GetEntityObject('MJ: Duplicate Run Detail Matches');
+                match.NewRecord();
+                match.DuplicateRunDetailID = duplicateRunDetailID;
+                match.MatchRecordID = dupe.ToString();
+                match.MatchProbability = dupe.ProbabilityScore;
+                match.MatchedAt = new Date();
+                match.Action = '';
+                match.ApprovalStatus = 'Pending';
+                match.MergeStatus = 'Pending';
+                const success = await this.SaveEntity(match);
+                return success ? match : null;
+            }));
+            for (const m of batchResults) {
+                if (m)
+                    matchRecords.push(m);
             }
         }
         return matchRecords;
     }
-    async mergeRecords(dupeResponse, entityDocument) {
-        const md = new Metadata();
-        for (const dupeResult of dupeResponse.PotentialDuplicateResult) {
+    // ─────────────────────────────────────────────
+    // Auto-Merge
+    // ─────────────────────────────────────────────
+    /**
+     * Automatically merge records that meet the absolute match threshold.
+     */
+    async ProcessAutoMerges(response, entityDocument) {
+        for (const dupeResult of response.PotentialDuplicateResult) {
             for (const [index, dupe] of dupeResult.Duplicates.entries()) {
-                if (dupe.ToString() === dupeResult.RecordCompositeKey.ToString()) {
-                    //same record, skip
+                if (dupe.ProbabilityScore < entityDocument.AbsoluteMatchThreshold) {
                     continue;
                 }
-                if (dupe.ProbabilityScore >= entityDocument.AbsoluteMatchThreshold) {
-                    //merge
-                    let mergeParams = new RecordMergeRequest();
-                    mergeParams.EntityName = entityDocument.Entity;
-                    mergeParams.SurvivingRecordCompositeKey = dupeResult.RecordCompositeKey;
-                    mergeParams.RecordsToMerge = [dupe];
-                    let result = await md.MergeRecords(mergeParams, super.CurrentUser);
-                    if (result.Success) {
-                        let dupeRunMatchRecord = await md.GetEntityObject('MJ: Duplicate Run Detail Matches', super.CurrentUser);
-                        let loadResult = await dupeRunMatchRecord.Load(dupeResult.DuplicateRunDetailMatchRecordIDs[index]);
-                        if (!loadResult) {
-                            LogError(`Failed to load Duplicate Run Match record ${dupeResult.DuplicateRunDetailMatchRecordIDs[index]}`);
-                            continue;
-                        }
-                        dupeRunMatchRecord.MergeStatus = 'Complete';
-                        dupeRunMatchRecord.Action = 'Merged';
-                        dupeRunMatchRecord.MergedAt = new Date();
-                        let saveResult = await dupeRunMatchRecord.Save();
-                        if (!saveResult) {
-                            LogError(`Failed to update Duplicate Run Match record ${dupeRunMatchRecord.ID}`);
-                        }
-                    }
-                    else {
-                        LogError(`Failed to merge records ${dupeResult.RecordCompositeKey.ToString()} and ${dupe.ToString()}`);
-                    }
+                const mergeParams = new RecordMergeRequest();
+                mergeParams.EntityName = entityDocument.Entity;
+                mergeParams.SurvivingRecordCompositeKey = dupeResult.RecordCompositeKey;
+                mergeParams.RecordsToMerge = [dupe];
+                const mergeResult = await this.Metadata.MergeRecords(mergeParams, this.CurrentUser);
+                if (mergeResult.Success) {
+                    await this.updateMatchRecordAfterMerge(dupeResult.DuplicateRunDetailMatchRecordIDs[index]);
+                }
+                else {
+                    LogError(`Failed to merge ${dupeResult.RecordCompositeKey.ToString()} and ${dupe.ToString()}`);
                 }
             }
         }
     }
-    async getVectorDuplicates(queryResponse) {
-        let response = new PotentialDuplicateResult();
-        for (const match of queryResponse.data.matches) {
-            const record = match;
-            if (!record || !record.id) {
-                continue;
-            }
-            if (!record.metadata || !record.metadata.RecordID) {
-                LogError(`Invalid vector metadata: ${record.id}`);
-                continue;
-            }
-            let duplicate = new PotentialDuplicate();
-            duplicate.LoadFromConcatenatedString(record.metadata.RecordID);
-            duplicate.ProbabilityScore = record.score;
-            response.Duplicates.push(duplicate);
+    /**
+     * Update a match record's status after a successful merge.
+     */
+    async updateMatchRecordAfterMerge(matchRecordID) {
+        const matchRecord = await this.Metadata.GetEntityObject('MJ: Duplicate Run Detail Matches', this.CurrentUser);
+        const loaded = await matchRecord.Load(matchRecordID);
+        if (!loaded) {
+            LogError(`Failed to load match record ${matchRecordID} for merge status update`);
+            return;
+        }
+        matchRecord.MergeStatus = 'Complete';
+        matchRecord.Action = 'Merged';
+        matchRecord.MergedAt = new Date();
+        const saved = await matchRecord.Save();
+        if (!saved) {
+            LogError(`Failed to update match record ${matchRecordID} after merge`);
+        }
+    }
+    // ─────────────────────────────────────────────
+    // Progress Reporting
+    // ─────────────────────────────────────────────
+    reportProgress(options, phase, totalRecords, processedRecords, matchesFound, startTime, currentRecordID) {
+        if (options.OnProgress) {
+            options.OnProgress({
+                Phase: phase,
+                TotalRecords: totalRecords,
+                ProcessedRecords: processedRecords,
+                MatchesFound: matchesFound,
+                CurrentRecordID: currentRecordID,
+                ElapsedMs: Date.now() - startTime,
+            });
+        }
+    }
+}
+// ─────────────────────────────────────────────
+// Utility Functions
+// ─────────────────────────────────────────────
+/**
+ * Split an array into chunks of a given size.
+ */
+function chunkArray(array, chunkSize) {
+    const chunks = [];
+    for (let i = 0; i < array.length; i += chunkSize) {
+        chunks.push(array.slice(i, i + chunkSize));
+    }
+    return chunks;
+}
+/**
+ * Run async tasks with a concurrency limit.
+ * Executes up to `limit` tasks in parallel, queuing the rest.
+ */
+async function RunWithConcurrency(tasks, limit) {
+    const results = [];
+    let index = 0;
+    async function runNext() {
+        while (index < tasks.length) {
+            const currentIndex = index++;
+            results[currentIndex] = await tasks[currentIndex]();
         }
-        return response;
     }
+    const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => runNext());
+    await Promise.all(workers);
+    return results;
 }
 //# sourceMappingURL=duplicateRecordDetector.js.map