@memberjunction/ai-vector-dupe 5.21.0 → 5.23.0

package/dist/duplicateRecordDetector.js

@@ -1,332 +1,811 @@
+/**
+ * @fileoverview Modernized duplicate record detection engine.
+ *
+ * Orchestrates the full pipeline: vectorize records, query for similar candidates,
+ * optionally apply hybrid search (RRF) and reranking, persist match results,
+ * and auto-merge high-confidence duplicates.
+ *
+ * Supports three record-source modes:
+ * 1. List-based batch detection (ListID provided)
+ * 2. View-based detection (ViewID provided)
+ * 3. Entity-wide detection (no ListID/ViewID — scans all records or applies ExtraFilter)
+ *
+ * Also supports single-record checks via CheckSingleRecord().
+ *
+ * @module @memberjunction/ai-vector-dupe
+ */
 import { BaseEmbeddings, GetAIAPIKey } from "@memberjunction/ai";
-import { PotentialDuplicateResponse, RunView, PotentialDuplicateResult, Metadata, LogError, RecordMergeRequest, PotentialDuplicate } from "@memberjunction/core";
-import { LogStatus } from "@memberjunction/core";
+import { PotentialDuplicateResponse, CompositeKey, PotentialDuplicateResult, LogError, LogStatus, RecordMergeRequest, PotentialDuplicate, RunView, } from "@memberjunction/core";
 import { VectorDBBase } from "@memberjunction/ai-vectordb";
 import { MJGlobal, UUIDsEqual } from "@memberjunction/global";
 import { VectorBase } from "@memberjunction/ai-vectors";
 import { EntityDocumentTemplateParser, EntityVectorSyncer } from "@memberjunction/ai-vector-sync";
+import { TemplateEngineServer } from "@memberjunction/templates";
+/** Default number of nearest neighbors to retrieve per record */
+const DEFAULT_TOP_K = 5;
+/** Default concurrency limit for parallel vector queries */
+const DEFAULT_QUERY_CONCURRENCY = 10;
+/** Default batch size for loading records and parallel database saves */
+const DEFAULT_BATCH_SIZE = 500;
+/** Default batch size for parallel database saves */
+const SAVE_BATCH_SIZE = 20;
+/**
+ * Modernized duplicate record detection engine.
+ *
+ * Supports:
+ * - List-based batch detection (getDuplicateRecords)
+ * - View/filter/full-entity batch detection (vector-first approach)
+ * - Single-record duplicate check (CheckSingleRecord)
+ * - Hybrid search via RRF when vector DB supports it
+ * - Optional post-retrieval reranking via MJ's BaseReranker
+ * - Configurable topK, thresholds, and progress reporting
+ */
 export class DuplicateRecordDetector extends VectorBase {
     constructor() {
-        super();
-        this._runView = new RunView();
-    }
-    async getDuplicateRecords(params, contextUser) {
-        super.CurrentUser = contextUser;
-        //for testing
-        //params.EntityID = 25050001;
-        //params.EntityDocumentID = 17;
-        let vectorizer = new EntityVectorSyncer();
-        vectorizer.CurrentUser = super.CurrentUser;
-        let entityDocument = await vectorizer.GetEntityDocument(params.EntityDocumentID);
-        if (!entityDocument) {
-            throw Error(`No Entity Document found with ID ${params.EntityDocumentID}`);
-            //Update: No longer creating an entity docuement if one is not found
-            //If an entitiy document is not found, that is our indicator that the
-            //underlying entity's records have not been vectorized yet
-            //const defaultVectorDB: MJVectorDatabaseEntity = super.getVectorDatabase();
-            //const defaultAIModel: MJAIModelEntity = super.getAIModel();
-            //entityDocument = await this.createEntityDocumentForEntity(params.EntityID, defaultVectorDB, defaultAIModel);
-        }
-        let response = new PotentialDuplicateResponse();
+        super(...arguments);
+        /**
+         * Tracks already-seen source↔match pairs across the entire run to suppress inverse duplicates.
+         * If A→B is persisted, B→A is skipped. Key format: "smallerID::largerID" for consistent ordering.
+         */
+        this._seenPairs = new Set();
+    }
+    /**
+     * Run duplicate detection for records identified by ListID, ViewID, ExtraFilter,
+     * or all records in the entity (vector-first approach).
+     *
+     * Flow: validate -> vectorize -> init providers -> load/create run ->
+     *       load record IDs -> batch(embed -> query -> persist) -> complete run -> auto-merge
+     */
+    async GetDuplicateRecords(params, contextUser) {
+        this.CurrentUser = contextUser;
+        this._seenPairs.clear(); // Reset for each new run
+        const options = params.Options ?? {};
+        const startTime = Date.now();
+        const response = new PotentialDuplicateResponse();
         response.PotentialDuplicateResult = [];
+        // Step 1: Validate entity document
+        const entityDocument = await this.ValidateEntityDocument(params.EntityDocumentID);
         if (!entityDocument) {
-            response.ErrorMessage = `No active Entity Document found for entity ${params.EntityID}`;
+            response.ErrorMessage = `No active Entity Document found for ID ${params.EntityDocumentID}`;
             response.Status = 'Error';
             return response;
         }
-        //for testing
-        const request = {
-            entityID: entityDocument.EntityID,
-            entityDocumentID: entityDocument.ID,
-            listBatchCount: 20,
-            options: {},
-            CurrentUser: contextUser
-        };
-        console.log("vectorizing entity...");
+        // Step 2: Optionally vectorize source records (default: skip — vectors should already exist from sync)
+        if (options.Revectorize) {
+            this.reportProgress(options, 'Vectorizing', 0, 0, 0, startTime);
+            await this.VectorizeSourceRecords(entityDocument, contextUser);
+        }
+        // Step 3: Initialize providers
+        await this.InitializeProviders(entityDocument);
+        // Step 4: Create or load DuplicateRun
+        const duplicateRun = await this.ResolveOrCreateDuplicateRun(params, entityDocument, options);
+        // Step 5: Load record IDs to check (batch-friendly — only IDs)
+        this.reportProgress(options, 'Loading', 0, 0, 0, startTime);
+        const entityInfo = this.Metadata.EntityByID(entityDocument.EntityID);
+        if (!entityInfo) {
+            response.ErrorMessage = `Entity not found for ID ${entityDocument.EntityID}`;
+            response.Status = 'Error';
+            return response;
+        }
+        const recordIDs = await this.LoadRecordIDsToCheck(params, entityInfo);
+        if (recordIDs.length === 0) {
+            response.ErrorMessage = 'No records found to check for duplicates';
+            response.Status = 'Error';
+            return response;
+        }
+        // Step 6: Process in batches
+        const batchSize = DEFAULT_BATCH_SIZE;
+        const concurrency = this.GetQueryConcurrency(entityDocument);
+        const topK = options.TopK ?? DEFAULT_TOP_K;
+        const templateParser = EntityDocumentTemplateParser.CreateInstance();
+        let totalMatchesFound = 0;
+        for (let offset = 0; offset < recordIDs.length; offset += batchSize) {
+            const batchIDs = recordIDs.slice(offset, offset + batchSize);
+            const batchResults = await this.ProcessBatch(batchIDs, entityInfo, entityDocument, templateParser, duplicateRun.ID, topK, concurrency, options, startTime, recordIDs.length, offset, totalMatchesFound, contextUser);
+            response.PotentialDuplicateResult.push(...batchResults.Results);
+            totalMatchesFound += batchResults.MatchesFound;
+        }
+        // Step 7: Complete the duplicate run
+        duplicateRun.ProcessingStatus = 'Complete';
+        duplicateRun.EndedAt = new Date();
+        const runSaveSuccess = await this.SaveEntity(duplicateRun);
+        if (!runSaveSuccess) {
+            throw new Error(`Failed to update Duplicate Run record ${duplicateRun.ID}`);
+        }
+        // Step 8: Auto-merge high-confidence matches
+        this.reportProgress(options, 'Merging', recordIDs.length, recordIDs.length, totalMatchesFound, startTime);
+        await this.ProcessAutoMerges(response, entityDocument, options);
+        response.Status = 'Success';
+        LogStatus(`Duplicate detection complete: ${recordIDs.length} records checked, ${totalMatchesFound} matches found`);
+        return response;
+    }
+    /**
+     * Check a single record for duplicates without requiring a list.
+     * Embeds the record and queries for matches directly.
+     */
+    async CheckSingleRecord(EntityDocumentID, RecordID, Options, ContextUser) {
+        this.CurrentUser = ContextUser;
+        const options = Options ?? {};
+        const entityDocument = await this.ValidateEntityDocument(EntityDocumentID);
+        if (!entityDocument) {
+            throw new Error(`No active Entity Document found for ID ${EntityDocumentID}`);
+        }
+        await this.InitializeProviders(entityDocument);
+        // Load the single record
+        const entityInfo = this.Metadata.EntityByID(entityDocument.EntityID);
+        if (!entityInfo) {
+            throw new Error(`Entity not found for ID ${entityDocument.EntityID}`);
+        }
+        const records = await this.RunView.RunView({
+            EntityName: entityInfo.Name,
+            ExtraFilter: this.BuildExtraFilter([RecordID]),
+            ResultType: 'entity_object',
+        }, this.CurrentUser);
+        if (!records.Success || records.Results.length === 0) {
+            throw new Error(`Record not found: ${RecordID.ToString()}`);
+        }
+        const record = records.Results[0];
         const templateParser = EntityDocumentTemplateParser.CreateInstance();
-        await vectorizer.VectorizeEntity(request, super.CurrentUser);
-        const list = await this.getListEntity(params.ListID);
-        let duplicateRun = params.Options?.DuplicateRunID ? await this.getDuplicateRunEntity(params.Options?.DuplicateRunID) : await this.getDuplicateRunEntityByListID(list.ID);
-        //let duplicateRun: MJDuplicateRunEntity = await this.createDuplicateRunRecord(entityDocument, list.ID);
-        const duplicateRunDetails = await this.createDuplicateRunDetailRecordsByListID(list.ID, duplicateRun.ID);
-        //await this.createListDetailsForDupeRun(params.RecordIDs, list.ID);
-        LogStatus(`Using vector database ${entityDocument.VectorDatabaseID} and AI Model ${entityDocument.AIModelID}`);
-        const vectorDB = super.GetVectorDatabase(entityDocument.VectorDatabaseID);
-        const aiModel = super.GetAIModel(entityDocument.AIModelID);
-        LogStatus(`AIModel driver class: ${aiModel.DriverClass}`);
-        LogStatus(`VectorDB class key: ${vectorDB.ClassKey}`);
+        const templateTexts = await this.GenerateTemplateTexts(templateParser, entityDocument, [record], ContextUser);
+        const embedResult = await this.embedding.EmbedTexts({ texts: templateTexts, model: null });
+        const topK = options.TopK ?? DEFAULT_TOP_K;
+        const queryResults = await this.QueryDuplicatesForRecords([record], embedResult.vectors, templateTexts, entityDocument, topK, options, this.GetQueryConcurrency(entityDocument));
+        return queryResults.length > 0 ? queryResults[0].Duplicates : new PotentialDuplicateResult();
+    }
+    // ─────────────────────────────────────────────
+    // Batch Processing
+    // ─────────────────────────────────────────────
+    /**
+     * Result from processing a single batch of records.
+     */
+    async ProcessBatch(batchIDs, entityInfo, entityDocument, templateParser, duplicateRunID, topK, concurrency, options, startTime, totalRecords, processedSoFar, matchesSoFar, contextUser) {
+        // 6a: Load full record data for this batch (needed for template rendering)
+        const compositeKeys = batchIDs.map(id => {
+            const ck = new CompositeKey();
+            ck.KeyValuePairs.push({ FieldName: entityInfo.FirstPrimaryKey.Name, Value: id });
+            return ck;
+        });
+        const records = await this.LoadRecordsByKeys(compositeKeys, entityInfo);
+        if (records.length === 0) {
+            return { Results: [], MatchesFound: 0 };
+        }
+        // 6b: Build source record metadata map for rich UI display
+        const sourceMetadataMap = this.buildSourceMetadataMap(records, entityInfo);
+        // 6c: Create DuplicateRunDetail records for this batch
+        const duplicateRunDetails = await this.CreateRunDetailRecords(batchIDs, duplicateRunID, entityInfo, sourceMetadataMap);
+        // 6c: Generate template texts and embed
+        this.reportProgress(options, 'Embedding', totalRecords, processedSoFar, matchesSoFar, startTime);
+        const templateTexts = await this.GenerateTemplateTexts(templateParser, entityDocument, records, contextUser);
+        const embedResult = await this.embedding.EmbedTexts({ texts: templateTexts, model: null });
+        // 6d: Query vector DB for each record with concurrency control
+        this.reportProgress(options, 'Querying', totalRecords, processedSoFar, matchesSoFar, startTime);
+        const queryResults = await this.QueryDuplicatesForRecords(records, embedResult.vectors, templateTexts, entityDocument, topK, options, concurrency);
+        // 6e: Persist match results and update run details
+        this.reportProgress(options, 'Matching', totalRecords, processedSoFar + records.length, matchesSoFar, startTime);
+        const results = await this.PersistMatchResults(queryResults, duplicateRunDetails, entityDocument, options, startTime);
+        const batchMatches = results.reduce((sum, r) => sum + r.Duplicates.length, 0);
+        return { Results: results, MatchesFound: batchMatches };
+    }
+    // ─────────────────────────────────────────────
+    // Record ID Loading (multiple strategies)
+    // ─────────────────────────────────────────────
+    /**
+     * Load the IDs of records to check, using the appropriate strategy based on the request.
+     * Returns an array of primary key value strings.
+     */
+    async LoadRecordIDsToCheck(params, entityInfo) {
+        if (params.ListID) {
+            return this.LoadRecordIDsFromList(params.ListID);
+        }
+        if (params.ViewID) {
+            return this.LoadRecordIDsFromView(params.ViewID, entityInfo);
+        }
+        // ExtraFilter or all records
+        return this.LoadRecordIDsFromEntity(entityInfo, params.ExtraFilter);
+    }
+    /**
+     * Load record IDs from a list's detail records.
+     */
+    async LoadRecordIDsFromList(listID) {
+        const sanitizedListID = listID.replace(/'/g, "''");
+        const viewResults = await this.RunView.RunView({
+            EntityName: 'MJ: List Details',
+            ExtraFilter: `ListID = '${sanitizedListID}'`,
+            Fields: ['RecordID'],
+            ResultType: 'simple',
+        }, this.CurrentUser);
+        if (!viewResults.Success) {
+            throw new Error(`Failed to load list details: ${viewResults.ErrorMessage}`);
+        }
+        return viewResults.Results.map(r => r.RecordID);
+    }
+    /**
+     * Load record IDs by running a saved view.
+     */
+    async LoadRecordIDsFromView(viewID, entityInfo) {
+        const pkField = entityInfo.FirstPrimaryKey.Name;
+        const sanitizedViewID = viewID.replace(/'/g, "''");
+        // Load the view definition to get its filter
+        const viewEntity = await this.RunViewForSingleValue('Views', `ID = '${sanitizedViewID}'`);
+        if (!viewEntity) {
+            throw new Error(`View not found: ${viewID}`);
+        }
+        // Run the entity with the view's filter to get IDs
+        const viewResults = await this.RunView.RunView({
+            ViewID: viewID,
+            Fields: [pkField],
+            ResultType: 'simple',
+        }, this.CurrentUser);
+        if (!viewResults.Success) {
+            throw new Error(`Failed to run view ${viewID}: ${viewResults.ErrorMessage}`);
+        }
+        return viewResults.Results.map(r => r[pkField]);
+    }
+    /**
+     * Load record IDs directly from the entity, optionally filtered.
+     * Uses Fields: ['ID'] and ResultType: 'simple' for efficiency.
+     */
+    async LoadRecordIDsFromEntity(entityInfo, extraFilter) {
+        const pkField = entityInfo.FirstPrimaryKey.Name;
+        const viewResults = await this.RunView.RunView({
+            EntityName: entityInfo.Name,
+            ExtraFilter: extraFilter,
+            Fields: [pkField],
+            ResultType: 'simple',
+        }, this.CurrentUser);
+        if (!viewResults.Success) {
+            throw new Error(`Failed to load record IDs from ${entityInfo.Name}: ${viewResults.ErrorMessage}`);
+        }
+        return viewResults.Results.map(r => r[pkField]);
+    }
+    // ─────────────────────────────────────────────
+    // Validation & Setup
+    // ─────────────────────────────────────────────
+    /**
+     * Validate and return an entity document, or null if not found.
+     */
+    async ValidateEntityDocument(entityDocumentID) {
+        const vectorizer = new EntityVectorSyncer();
+        vectorizer.CurrentUser = this.CurrentUser;
+        return vectorizer.GetEntityDocument(entityDocumentID);
+    }
+    /**
+     * Initialize embedding and vector DB providers via ClassFactory.
+     */
+    async InitializeProviders(entityDocument) {
+        const aiModel = this.GetAIModel(entityDocument.AIModelID);
+        const vectorDB = this.GetVectorDatabase(entityDocument.VectorDatabaseID);
         const embeddingAPIKey = GetAIAPIKey(aiModel.DriverClass);
         const vectorDBAPIKey = GetAIAPIKey(vectorDB.ClassKey);
         if (!embeddingAPIKey) {
-            throw Error(`No API Key found for AI Model ${aiModel.DriverClass}`);
+            throw new Error(`No API Key found for AI Model ${aiModel.DriverClass}`);
         }
         if (!vectorDBAPIKey) {
-            throw Error(`No API Key found for Vector Database ${vectorDB.ClassKey}`);
+            throw new Error(`No API Key found for Vector Database ${vectorDB.ClassKey}`);
         }
-        //LogStatus(`Embedding API Key: ${embeddingAPIKey} VectorDB API Key: ${vectorDBAPIKey}`);
-        this._embedding = MJGlobal.Instance.ClassFactory.CreateInstance(BaseEmbeddings, aiModel.DriverClass, embeddingAPIKey);
-        this._vectorDB = MJGlobal.Instance.ClassFactory.CreateInstance(VectorDBBase, vectorDB.ClassKey, vectorDBAPIKey);
-        if (!this._embedding) {
-            throw Error(`Failed to create Embeddings instance for AI Model ${aiModel.DriverClass}`);
+        this.embedding = MJGlobal.Instance.ClassFactory.CreateInstance(BaseEmbeddings, aiModel.DriverClass, embeddingAPIKey);
+        this.vectorDB = MJGlobal.Instance.ClassFactory.CreateInstance(VectorDBBase, vectorDB.ClassKey, vectorDBAPIKey);
+        if (!this.embedding) {
+            throw new Error(`Failed to create Embeddings instance for ${aiModel.DriverClass}`);
         }
-        if (!this._vectorDB) {
-            throw Error(`Failed to create Vector Database instance for ${vectorDB.ClassKey}`);
+        if (!this.vectorDB) {
+            throw new Error(`Failed to create VectorDB instance for ${vectorDB.ClassKey}`);
         }
-        let records = await this.GetRecordsByListID(list.ID, entityDocument.EntityID);
-        if (records.length === 0) {
-            LogError(`No records found in list ${list.Name}, with listID ${list.ID} and EntityID ${entityDocument.EntityID} exiting early`);
-            response.ErrorMessage = `No records found in list ${list.Name}`;
-            response.Status = 'Error';
-            return response;
-        }
-        LogStatus("Vectorizing " + records.length + " records");
-        const recordTemplates = [];
-        //Relationship(entityID: number, entityRecord: any, relationshipName: string, maxRows: number, entityDocumentName: string)
-        let sampleTemplate = entityDocument.Template;
-        //sampleTemplate += " ${Relationship('Deals', 5, 'Sample Relationship Document for crm.Deals Entity')} ${Relationship('Deals', 5, 'Second Sample Relationship Document for crm.Deals Entity')}";
-        for (const record of records) {
-            const template = await templateParser.Parse(sampleTemplate, entityDocument.EntityID, record, contextUser);
-            recordTemplates.push(template);
-        }
-        let embedTextsResult = await this._embedding.EmbedTexts({ texts: recordTemplates, model: null });
-        const topK = 5;
-        let results = [];
-        for (const [index, vector] of embedTextsResult.vectors.entries()) {
-            const compositeKey = records[index].PrimaryKey;
-            let filterResult = await this._vectorDB.queryIndex({ vector: vector, topK: topK, includeMetadata: true, includeValues: false });
-            if (!filterResult.success) {
-                LogError(`Failed to query index for record ${compositeKey.ToString()}`);
-                continue;
+        // Resolve the vector index name from the entity document's VectorIndexID
+        // This is the actual Pinecone/pgvector/Qdrant index name needed for QueryIndex calls
+        if (entityDocument.VectorIndexID) {
+            const rv = new RunView();
+            const indexResult = await rv.RunView({
+                EntityName: 'MJ: Vector Indexes',
+                ExtraFilter: `ID='${entityDocument.VectorIndexID}'`,
+                Fields: ['Name'],
+                ResultType: 'simple',
+                MaxRows: 1
+            }, this.CurrentUser);
+            if (indexResult.Success && indexResult.Results.length > 0) {
+                this.indexName = indexResult.Results[0].Name;
             }
-            let queryResult = await this.getVectorDuplicates(filterResult);
-            queryResult.Duplicates = queryResult.Duplicates.filter((dupe) => {
-                return dupe.ProbabilityScore >= entityDocument.PotentialMatchThreshold;
-            });
-            queryResult.EntityID = entityDocument.EntityID;
-            queryResult.RecordCompositeKey = compositeKey;
-            results.push(queryResult);
-            //now update all of the dupe run detail records
-            let dupeRunDetail = duplicateRunDetails.find((detail) => UUIDsEqual(detail.RecordID, compositeKey.Values()));
-            if (dupeRunDetail) {
-                const matchRecords = await this.createDuplicateRunDetailMatchesForRecord(dupeRunDetail.ID, queryResult);
-                queryResult.DuplicateRunDetailMatchRecordIDs = matchRecords.map((match) => match.ID);
-                dupeRunDetail.MatchStatus = 'Complete';
-                let success = await super.SaveEntity(dupeRunDetail);
-                if (!success) {
-                    LogStatus(`Failed to update Duplicate Run Detail record ${dupeRunDetail.ID}`);
+        }
+        if (!this.indexName) {
+            throw new Error(`No vector index found for entity document "${entityDocument.Name}". Ensure VectorIndexID is set on the entity document.`);
+        }
+        LogStatus(`Providers initialized: AI Model=${aiModel.DriverClass}, VectorDB=${vectorDB.ClassKey}, Index=${this.indexName}`);
+    }
+    /**
+     * Run vectorization for the entity document's records.
+     */
+    async VectorizeSourceRecords(entityDocument, contextUser) {
+        const vectorizer = new EntityVectorSyncer();
+        vectorizer.CurrentUser = contextUser;
+        const request = {
+            entityID: entityDocument.EntityID,
+            entityDocumentID: entityDocument.ID,
+            listBatchCount: 20,
+            options: {},
+            CurrentUser: contextUser,
+        };
+        LogStatus(`Vectorizing entity records for document ${entityDocument.Name}`);
+        await vectorizer.VectorizeEntity(request, contextUser);
+    }
+    /**
+     * Read the maxConcurrentRequests from the VectorDatabase entity's Configuration column,
+     * falling back to DEFAULT_QUERY_CONCURRENCY if not set.
+     */
+    GetQueryConcurrency(entityDocument) {
+        const vectorDBEntity = this.GetVectorDatabase(entityDocument.VectorDatabaseID);
+        if (vectorDBEntity.Configuration) {
+            try {
+                const config = JSON.parse(vectorDBEntity.Configuration);
+                if (config.throughput?.maxConcurrentRequests != null) {
+                    return config.throughput.maxConcurrentRequests;
                 }
             }
-            else {
-                LogError(`Failed to find Duplicate Run Detail record for ${compositeKey.ToString()}`);
+            catch {
+                // Invalid JSON in Configuration — fall through to default
             }
         }
-        //almost done
-        duplicateRun.ProcessingStatus = 'Complete';
-        duplicateRun.EndedAt = new Date();
-        let success = await super.SaveEntity(duplicateRun);
+        return DEFAULT_QUERY_CONCURRENCY;
+    }
+    // ─────────────────────────────────────────────
+    // DuplicateRun Management
+    // ─────────────────────────────────────────────
+    /**
+     * Resolve an existing DuplicateRun or create a new one.
+     * Supports both list-based and list-free operation.
+     */
+    async ResolveOrCreateDuplicateRun(params, entityDocument, options) {
+        // If a specific run ID was provided, load it
+        if (options.DuplicateRunID) {
+            return this.LoadDuplicateRun(options.DuplicateRunID);
+        }
+        // If a ListID is provided, try to find an existing run for that list
+        if (params.ListID) {
+            const existing = await this.FindDuplicateRunByListID(params.ListID);
+            if (existing) {
+                return existing;
+            }
+        }
+        // Create a new DuplicateRun
+        return this.CreateDuplicateRun(entityDocument, params.ListID);
+    }
+    /**
+     * Create a new DuplicateRun record.
+     */
+    async CreateDuplicateRun(entityDocument, listID) {
+        const dupeRun = await this.Metadata.GetEntityObject('MJ: Duplicate Runs', this.CurrentUser);
+        dupeRun.NewRecord();
+        dupeRun.EntityID = entityDocument.EntityID;
+        dupeRun.StartedByUserID = this.CurrentUser?.ID;
+        dupeRun.StartedAt = new Date();
+        dupeRun.ProcessingStatus = 'In Progress';
+        dupeRun.ApprovalStatus = 'Pending';
+        if (listID) {
+            dupeRun.SourceListID = listID;
+        }
+        const success = await this.SaveEntity(dupeRun);
         if (!success) {
-            throw new Error(`Failed to update Duplicate Run record ${duplicateRun.ID}`);
+            throw new Error('Failed to create Duplicate Run record');
         }
-        await this.mergeRecords(response, entityDocument);
-        response.PotentialDuplicateResult = results;
-        response.Status = 'Success';
-        LogStatus("Dupe Run complete. Response:");
-        LogStatus(JSON.stringify(response, null, "\t"));
-        return response;
+        return dupeRun;
     }
-    async GetRecordsByListID(listID, entityID) {
-        const entityInfo = super.Metadata.EntityByID(entityID);
+    async LoadDuplicateRun(duplicateRunID) {
+        const dupeRun = await this.Metadata.GetEntityObject('MJ: Duplicate Runs', this.CurrentUser);
+        dupeRun.ContextCurrentUser = this.CurrentUser;
+        const success = await dupeRun.Load(duplicateRunID);
+        if (!success) {
+            throw new Error(`Failed to load Duplicate Run record ${duplicateRunID}`);
+        }
+        return dupeRun;
+    }
+    /**
+     * Try to find an existing DuplicateRun for a given ListID. Returns null if none found.
+     */
+    async FindDuplicateRunByListID(listID) {
+        return this.RunViewForSingleValue('MJ: Duplicate Runs', `SourceListID = '${listID.replace(/'/g, "''")}'`);
+    }
+    // ─────────────────────────────────────────────
+    // Entity Loading
+    // ─────────────────────────────────────────────
+    /**
+     * Load full entity objects for a batch of composite keys.
+     */
+    async LoadRecordsByKeys(compositeKeys, entityInfo) {
+        if (compositeKeys.length === 0) {
+            return [];
+        }
+        const rvResult = await this.RunView.RunView({
+            EntityName: entityInfo.Name,
+            ExtraFilter: this.BuildExtraFilter(compositeKeys),
+            ResultType: 'entity_object',
+        }, this.CurrentUser);
+        if (!rvResult.Success) {
+            throw new Error(rvResult.ErrorMessage);
+        }
+        return rvResult.Results;
+    }
+    /**
+     * Load records from an entity that are members of the specified list.
+     * Kept for backward compatibility.
+     */
+    async LoadRecordsByListID(listID, entityID) {
+        const entityInfo = this.Metadata.EntityByID(entityID);
         if (!entityInfo) {
-            throw new Error(`Failed to load Entity Info with ID ${entityID}`);
+            throw new Error(`Entity not found for ID ${entityID}`);
         }
-        const rvResult = await super.RunView.RunView({
+        const sanitizedListID = listID.replace(/'/g, "''");
+        const rvResult = await this.RunView.RunView({
             EntityName: entityInfo.Name,
-            ExtraFilter: `ID IN (SELECT RecordID FROM __mj.vwListDetails WHERE ListID = '${listID}')`,
-            ResultType: 'entity_object'
-        }, super.CurrentUser);
+            ExtraFilter: `ID IN (SELECT RecordID FROM __mj.vwListDetails WHERE ListID = '${sanitizedListID}')`,
+            ResultType: 'entity_object',
+        }, this.CurrentUser);
         if (!rvResult.Success) {
             throw new Error(rvResult.ErrorMessage);
         }
         return rvResult.Results;
     }
-    async createDuplicateRunRecord(entityDocument, listID) {
-        const md = new Metadata();
-        let duplicateRun = await md.GetEntityObject('MJ: Duplicate Runs');
-        duplicateRun.NewRecord();
-        duplicateRun.EntityID = entityDocument.EntityID;
-        duplicateRun.StartedByUserID = super.CurrentUser.ID;
-        duplicateRun.StartedAt = new Date();
-        duplicateRun.ProcessingStatus = 'In Progress';
-        duplicateRun.ApprovalStatus = 'Pending';
-        duplicateRun.SourceListID = listID;
-        const saveResult = await super.SaveEntity(duplicateRun);
-        if (!saveResult) {
-            throw new Error(`Failed to save list for Potential Duplicate Run`);
-        }
-        return duplicateRun;
-    }
-    async createDuplicateRunDetailRecords(recordIDs, duplicateRunID) {
-        let results = [];
-        const md = new Metadata();
-        for (const recordID of recordIDs) {
-            let runDetail = await md.GetEntityObject('MJ: Duplicate Run Details');
-            runDetail.NewRecord();
-            runDetail.DuplicateRunID = duplicateRunID;
-            runDetail.RecordID = recordID.ToString();
-            runDetail.MatchStatus = 'Pending';
-            runDetail.MergeStatus = 'Pending';
-            const success = await super.SaveEntity(runDetail);
-            if (success) {
-                results.push(runDetail);
-            }
+    async LoadListEntity(listID) {
+        const list = await this.Metadata.GetEntityObject('MJ: Lists');
+        list.ContextCurrentUser = this.CurrentUser;
+        const success = await list.Load(listID);
+        if (!success) {
+            throw new Error(`Failed to load List record ${listID}`);
         }
-        return results;
+        return list;
     }
-    async createDuplicateRunDetailRecordsByListID(listID, duplicateRunID) {
-        let results = [];
-        const viewResults = await super.RunView.RunView({
-            EntityName: 'MJ: List Details',
-            ExtraFilter: `ListID = '${listID}'`,
-            ResultType: 'entity_object'
-        }, super.CurrentUser);
-        if (!viewResults.Success) {
-            throw new Error(viewResults.ErrorMessage);
-        }
-        const md = new Metadata();
-        const listDetails = viewResults.Results;
-        for (const listDetail of listDetails) {
-            let runDetail = await md.GetEntityObject('MJ: Duplicate Run Details');
-            runDetail.NewRecord();
-            runDetail.DuplicateRunID = duplicateRunID;
-            runDetail.RecordID = listDetail.RecordID;
-            runDetail.MatchStatus = 'Pending';
-            runDetail.MergeStatus = 'Pending';
-            const success = await super.SaveEntity(runDetail);
-            if (success) {
-                results.push(runDetail);
+    // ─────────────────────────────────────────────
+    // Template Generation & Embedding
+    // ─────────────────────────────────────────────
+    /**
+     * Generate human-readable template text for each record using the entity document template.
+     *
+     * Loads the template from TemplateEngineServer and renders it via Nunjucks,
+     * matching the same approach used by the vectorization pipeline.
+     */
+    async GenerateTemplateTexts(templateParser, entityDocument, records, contextUser) {
+        await TemplateEngineServer.Instance.Config(false, contextUser);
+        const template = this.loadTemplate(entityDocument);
+        const templateContent = template.Content[0];
+        TemplateEngineServer.Instance.SetupNunjucks();
+        const templateTexts = [];
+        for (const record of records) {
+            // NEW convention: main entity fields are TOP-LEVEL variables (no Entity. prefix).
+            // Spread record fields directly into root context so templates use {{FieldName}}.
+            const data = { ...record.GetAll() };
+            const result = await TemplateEngineServer.Instance.RenderTemplate(template, templateContent, data, true);
+            if (result.Success) {
+                templateTexts.push(result.Output);
             }
             else {
-                LogError("Failed to save MJDuplicateRunDetailEntity", undefined, runDetail.LatestResult);
+                LogError(`Template render failed for record ${record.PrimaryKey.ToString()}: ${result.Message}`);
+                templateTexts.push('');
             }
         }
-        return results;
+        return templateTexts;
     }
-    async getListEntity(listID) {
-        const md = new Metadata();
-        let list = await md.GetEntityObject('MJ: Lists');
-        list.ContextCurrentUser = super.CurrentUser;
-        const success = await list.Load(listID);
-        if (!success) {
-            throw new Error(`Failed to load List record ${listID}`);
+    /**
+     * Load the template entity from TemplateEngineServer for the given entity document.
+     */
+    loadTemplate(entityDocument) {
+        const template = TemplateEngineServer.Instance.Templates.find((t) => UUIDsEqual(t.ID, entityDocument.TemplateID));
+        if (!template) {
+            throw new Error(`Template not found for ID ${entityDocument.TemplateID}`);
         }
-        return list;
+        if (template.Content.length === 0) {
+            throw new Error(`Template ${template.ID} has no content records`);
+        }
+        return template;
     }
-    async getDuplicateRunEntity(DupeRunID) {
-        const md = new Metadata();
-        let dupeRun = await md.GetEntityObject('MJ: Duplicate Runs');
-        dupeRun.ContextCurrentUser = super.CurrentUser;
-        const success = await dupeRun.Load(DupeRunID);
-        if (!success) {
-            throw new Error(`Failed to load Duplicate Run record ${DupeRunID}`);
+    // ─────────────────────────────────────────────
+    // Vector Query & Hybrid Search
+    // ─────────────────────────────────────────────
+    /**
+     * Query the vector DB for duplicates of each record, with concurrency control.
+     * Supports hybrid search and RRF fusion when the vector DB supports it.
+     */
+    async QueryDuplicatesForRecords(records, vectors, templateTexts, entityDocument, topK, options, concurrency) {
+        const tasks = records.map((record, index) => async () => {
+            const compositeKey = record.PrimaryKey;
+            const vector = vectors[index];
+            const templateText = templateTexts[index];
+            const queryResponse = await this.executeVectorQuery(vector, templateText, topK, options);
+            if (!queryResponse.success) {
+                LogError(`Failed to query index for record ${compositeKey.ToString()}`);
+                const emptyResult = new PotentialDuplicateResult();
+                emptyResult.EntityID = entityDocument.EntityID;
+                emptyResult.RecordCompositeKey = compositeKey;
+                return { SourceKey: compositeKey, TemplateText: templateText, Duplicates: emptyResult };
+            }
+            const dupeResult = this.ParseVectorMatches(queryResponse, compositeKey);
+            dupeResult.Duplicates = this.FilterSelfMatches(dupeResult.Duplicates, compositeKey);
+            const potentialThreshold = options.PotentialMatchThreshold ?? entityDocument.PotentialMatchThreshold;
+            dupeResult.Duplicates = dupeResult.Duplicates.filter((d) => d.ProbabilityScore >= potentialThreshold);
+            dupeResult.EntityID = entityDocument.EntityID;
+            dupeResult.RecordCompositeKey = compositeKey;
+            return { SourceKey: compositeKey, TemplateText: templateText, Duplicates: dupeResult };
+        });
+        return RunWithConcurrency(tasks, concurrency);
+    }
+    /**
+     * Execute a vector query — uses hybrid search with RRF when the provider supports it.
+     */
+    async executeVectorQuery(vector, templateText, topK, options) {
+        if (this.vectorDB.SupportsHybridSearch && templateText) {
+            return this.vectorDB.HybridQuery({
+                vector,
+                topK,
+                KeywordQuery: templateText,
+                Alpha: options.KeywordSearchWeight != null ? (1.0 - options.KeywordSearchWeight) : 0.7,
+                FusionMethod: options.FusionMethod ?? 'rrf',
+                includeMetadata: true,
+                includeValues: false,
+            });
         }
-        return dupeRun;
+        return this.vectorDB.QueryIndex({
+            id: this.indexName,
+            vector,
+            topK,
+            includeMetadata: true,
+            includeValues: false,
+        });
     }
-    async getDuplicateRunEntityByListID(listID) {
-        const entity = await super.RunViewForSingleValue('MJ: Duplicate Runs', `SourceListID = '${listID}'`);
-        if (!entity) {
-            throw new Error(`Failed to load Duplicate Run record for List ${listID}`);
-        }
-        return entity;
-    }
-    async createListForDupeRun(entityDocument) {
-        const md = new Metadata();
-        const list = await md.GetEntityObject('MJ: Lists');
-        list.NewRecord();
-        list.Name = `Potential Duplicate Run`;
-        list.Description = `Potential Duplicate Run for ${entityDocument.Entity} Entity`;
-        list.EntityID = entityDocument.EntityID;
-        list.UserID = super.CurrentUser.ID;
-        const saveResult = await super.SaveEntity(list);
-        if (!saveResult) {
-            throw new Error(`Failed to save list for Potential Duplicate Run`);
+    /**
+     * Parse raw vector DB matches into a PotentialDuplicateResult.
+     */
+    ParseVectorMatches(queryResponse, sourceKey) {
+        const result = new PotentialDuplicateResult();
+        if (!queryResponse.data?.matches) {
+            return result;
         }
-        return list;
+        for (const match of queryResponse.data.matches) {
+            if (!match?.id)
+                continue;
+            if (!match.metadata?.RecordID) {
+                LogError(`Invalid vector metadata for match: ${match.id}`);
+                continue;
+            }
+            const duplicate = new PotentialDuplicate();
+            duplicate.LoadFromConcatenatedString(match.metadata.RecordID);
+            duplicate.ProbabilityScore = match.score;
+            // Capture the full vector metadata for rich UI display
+            duplicate.VectorMetadata = { ...match.metadata };
+            result.Duplicates.push(duplicate);
+        }
+        return result;
     }
-    async createDuplicateRunDetailMatchesForRecord(DuplicateRunDetailID, duplicateResult) {
-        const md = new Metadata();
-        let matchRecords = [];
-        for (const dupe of duplicateResult.Duplicates) {
-            const match = await md.GetEntityObject('MJ: Duplicate Run Detail Matches');
-            match.NewRecord();
-            match.DuplicateRunDetailID = DuplicateRunDetailID;
-            match.MatchRecordID = dupe.ToString();
-            match.MatchProbability = dupe.ProbabilityScore;
-            match.MatchedAt = new Date();
-            match.MergedAt = new Date();
-            match.Action = '';
-            match.ApprovalStatus = 'Pending';
-            match.MergeStatus = 'Pending';
-            let success = await super.SaveEntity(match);
-            if (success) {
-                matchRecords.push(match);
+    /**
+     * Filter out self-matches where the candidate is the same record as the source.
+     */
+    /**
+     * Build a map of recordID → JSON metadata string from loaded BaseEntity records.
+     * Extracts the entity's name field and a few key display fields for rich UI rendering.
+     */
+    buildSourceMetadataMap(records, entityInfo) {
+        const metadataMap = new Map();
+        const nameField = entityInfo.NameField;
+        // Collect a small set of useful display fields
+        const displayFieldNames = ['Name', 'Title', 'Description', 'Status', 'Type']
+            .filter(fn => entityInfo.Fields.find(f => f.Name === fn));
+        for (const record of records) {
+            const pk = record.PrimaryKey;
+            const id = pk.KeyValuePairs.length === 1 ? String(pk.KeyValuePairs[0].Value) : pk.Values();
+            const meta = {
+                Entity: entityInfo.Name,
+            };
+            if (entityInfo.Icon) {
+                meta['EntityIcon'] = entityInfo.Icon;
+            }
+            if (nameField) {
+                const nameVal = record.Get(nameField.Name);
+                if (nameVal != null)
+                    meta['Name'] = String(nameVal);
+            }
+            for (const fn of displayFieldNames) {
+                if (fn !== nameField?.Name) {
+                    const val = record.Get(fn);
+                    if (val != null) {
+                        const str = String(val);
+                        meta[fn] = str.length > 200 ? str.substring(0, 197) + '...' : str;
+                    }
+                }
+            }
+            metadataMap.set(id, JSON.stringify(meta));
+        }
+        return metadataMap;
+    }
+    FilterSelfMatches(duplicates, sourceKey) {
+        return duplicates.filter((d) => d.ToString() !== sourceKey.ToString());
+    }
+    // ─────────────────────────────────────────────
+    // Run Detail & Match Persistence (Batched)
+    // ─────────────────────────────────────────────
+    /**
+     * Create DuplicateRunDetail records for a batch of record IDs.
+     */
+    async CreateRunDetailRecords(recordIDs, duplicateRunID, entityInfo, metadataMap) {
+        const results = [];
+        const pkFieldName = entityInfo.FirstPrimaryKey.Name;
+        for (const batch of chunkArray(recordIDs, SAVE_BATCH_SIZE)) {
+            const batchResults = await Promise.all(batch.map(async (recordID) => {
+                const runDetail = await this.Metadata.GetEntityObject('MJ: Duplicate Run Details', this.CurrentUser);
+                runDetail.NewRecord();
+                runDetail.DuplicateRunID = duplicateRunID;
+                // Store RecordID in standard MJ URL segment format (e.g., "ID|uuid")
+                runDetail.RecordID = `${pkFieldName}|${recordID}`;
+                runDetail.MatchStatus = 'Pending';
+                runDetail.MergeStatus = 'Pending';
+                runDetail.RecordMetadata = metadataMap?.get(recordID) ?? null;
+                const success = await this.SaveEntity(runDetail);
+                if (!success) {
+                    LogError("Failed to save MJDuplicateRunDetailEntity", undefined, runDetail.LatestResult);
+                    return null;
+                }
+                return runDetail;
+            }));
+            for (const r of batchResults) {
+                if (r)
+                    results.push(r);
+            }
+        }
+        return results;
+    }
+    /**
+     * Persist match results and update run detail records.
+     */
+    async PersistMatchResults(queryResults, duplicateRunDetails, entityDocument, options, startTime) {
+        const results = [];
+        let matchesFound = 0;
+        for (const qr of queryResults) {
+            // Filter out inverse duplicates: if A→B was already persisted, skip B→A
+            const sourceId = qr.SourceKey.Values();
+            qr.Duplicates.Duplicates = qr.Duplicates.Duplicates.filter(dupe => {
+                const matchId = dupe.Values();
+                const pairKey = sourceId < matchId ? `${sourceId}::${matchId}` : `${matchId}::${sourceId}`;
+                if (this._seenPairs.has(pairKey)) {
+                    return false; // Inverse already recorded
+                }
+                this._seenPairs.add(pairKey);
+                return true;
+            });
+            results.push(qr.Duplicates);
+            matchesFound += qr.Duplicates.Duplicates.length;
+            const sourceKey = qr.SourceKey;
+            const detail = duplicateRunDetails.find((d) => {
+                const detailKey = new CompositeKey();
+                detailKey.LoadFromConcatenatedString(d.RecordID);
+                return detailKey.Equals(sourceKey);
+            });
+            if (detail) {
+                const matchRecords = await this.CreateMatchRecordsForDetail(detail.ID, qr.Duplicates);
+                qr.Duplicates.DuplicateRunDetailMatchRecordIDs = matchRecords.map((m) => m.ID);
+                detail.MatchStatus = 'Complete';
+                const success = await this.SaveEntity(detail);
+                if (!success) {
+                    LogError(`Failed to update Duplicate Run Detail record ${detail.ID}`);
+                }
+            }
+            else {
+                LogError(`No Duplicate Run Detail found for ${qr.SourceKey.ToString()}`);
+            }
+            this.reportProgress(options, 'Matching', queryResults.length, results.length, matchesFound, startTime);
+        }
+        return results;
+    }
+    /**
+     * Create match records for a single run detail, saving in parallel batches.
+     */
+    async CreateMatchRecordsForDetail(duplicateRunDetailID, duplicateResult) {
+        const matchRecords = [];
+        for (const batch of chunkArray(duplicateResult.Duplicates, SAVE_BATCH_SIZE)) {
+            const batchResults = await Promise.all(batch.map(async (dupe) => {
+                const match = await this.Metadata.GetEntityObject('MJ: Duplicate Run Detail Matches', this.CurrentUser);
+                match.NewRecord();
+                match.DuplicateRunDetailID = duplicateRunDetailID;
+                match.MatchRecordID = dupe.ToURLSegment();
+                match.MatchProbability = dupe.ProbabilityScore;
+                match.MatchedAt = new Date();
+                match.Action = '';
+                match.ApprovalStatus = 'Pending';
+                match.MergeStatus = 'Pending';
+                match.RecordMetadata = dupe.VectorMetadata ? JSON.stringify(dupe.VectorMetadata) : null;
+                const success = await this.SaveEntity(match);
+                return success ? match : null;
+            }));
+            for (const m of batchResults) {
+                if (m)
+                    matchRecords.push(m);
             }
         }
         return matchRecords;
     }
-    async mergeRecords(dupeResponse, entityDocument) {
-        const md = new Metadata();
-        for (const dupeResult of dupeResponse.PotentialDuplicateResult) {
+    // ─────────────────────────────────────────────
+    // Auto-Merge
+    // ─────────────────────────────────────────────
+    /**
+     * Automatically merge records that meet the absolute match threshold.
+     */
+    async ProcessAutoMerges(response, entityDocument, options = {}) {
+        const absoluteThreshold = options.AbsoluteMatchThreshold ?? entityDocument.AbsoluteMatchThreshold;
+        for (const dupeResult of response.PotentialDuplicateResult) {
             for (const [index, dupe] of dupeResult.Duplicates.entries()) {
-                if (dupe.ToString() === dupeResult.RecordCompositeKey.ToString()) {
-                    //same record, skip
+                if (dupe.ProbabilityScore < absoluteThreshold) {
                     continue;
                 }
-                if (dupe.ProbabilityScore >= entityDocument.AbsoluteMatchThreshold) {
-                    //merge
-                    let mergeParams = new RecordMergeRequest();
-                    mergeParams.EntityName = entityDocument.Entity;
-                    mergeParams.SurvivingRecordCompositeKey = dupeResult.RecordCompositeKey;
-                    mergeParams.RecordsToMerge = [dupe];
-                    let result = await md.MergeRecords(mergeParams, super.CurrentUser);
-                    if (result.Success) {
-                        let dupeRunMatchRecord = await md.GetEntityObject('MJ: Duplicate Run Detail Matches', super.CurrentUser);
-                        let loadResult = await dupeRunMatchRecord.Load(dupeResult.DuplicateRunDetailMatchRecordIDs[index]);
-                        if (!loadResult) {
-                            LogError(`Failed to load Duplicate Run Match record ${dupeResult.DuplicateRunDetailMatchRecordIDs[index]}`);
-                            continue;
-                        }
-                        dupeRunMatchRecord.MergeStatus = 'Complete';
-                        dupeRunMatchRecord.Action = 'Merged';
-                        dupeRunMatchRecord.MergedAt = new Date();
-                        let saveResult = await dupeRunMatchRecord.Save();
-                        if (!saveResult) {
-                            LogError(`Failed to update Duplicate Run Match record ${dupeRunMatchRecord.ID}`);
-                        }
-                    }
-                    else {
-                        LogError(`Failed to merge records ${dupeResult.RecordCompositeKey.ToString()} and ${dupe.ToString()}`);
-                    }
+                const mergeParams = new RecordMergeRequest();
+                mergeParams.EntityName = entityDocument.Entity;
+                mergeParams.SurvivingRecordCompositeKey = dupeResult.RecordCompositeKey;
+                mergeParams.RecordsToMerge = [dupe];
+                const mergeResult = await this.Metadata.MergeRecords(mergeParams, this.CurrentUser);
+                if (mergeResult.Success) {
+                    await this.updateMatchRecordAfterMerge(dupeResult.DuplicateRunDetailMatchRecordIDs[index]);
+                }
+                else {
+                    LogError(`Failed to merge ${dupeResult.RecordCompositeKey.ToString()} and ${dupe.ToString()}`);
                 }
             }
         }
     }
-    async getVectorDuplicates(queryResponse) {
-        let response = new PotentialDuplicateResult();
-        for (const match of queryResponse.data.matches) {
-            const record = match;
-            if (!record || !record.id) {
-                continue;
-            }
-            if (!record.metadata || !record.metadata.RecordID) {
-                LogError(`Invalid vector metadata: ${record.id}`);
-                continue;
-            }
-            let duplicate = new PotentialDuplicate();
-            duplicate.LoadFromConcatenatedString(record.metadata.RecordID);
-            duplicate.ProbabilityScore = record.score;
-            response.Duplicates.push(duplicate);
+    /**
+     * Update a match record's status after a successful merge.
+     */
+    async updateMatchRecordAfterMerge(matchRecordID) {
+        const matchRecord = await this.Metadata.GetEntityObject('MJ: Duplicate Run Detail Matches', this.CurrentUser);
+        const loaded = await matchRecord.Load(matchRecordID);
+        if (!loaded) {
+            LogError(`Failed to load match record ${matchRecordID} for merge status update`);
+            return;
+        }
+        matchRecord.MergeStatus = 'Complete';
+        matchRecord.Action = 'Merged';
+        matchRecord.MergedAt = new Date();
+        const saved = await matchRecord.Save();
+        if (!saved) {
+            LogError(`Failed to update match record ${matchRecordID} after merge`);
+        }
+    }
+    // ─────────────────────────────────────────────
+    // Progress Reporting
+    // ─────────────────────────────────────────────
+    reportProgress(options, phase, totalRecords, processedRecords, matchesFound, startTime, currentRecordID) {
+        if (options.OnProgress) {
+            options.OnProgress({
+                Phase: phase,
+                TotalRecords: totalRecords,
+                ProcessedRecords: processedRecords,
+                MatchesFound: matchesFound,
+                CurrentRecordID: currentRecordID,
+                ElapsedMs: Date.now() - startTime,
+            });
+        }
+    }
+}
+// ─────────────────────────────────────────────
+// Utility Functions
+// ─────────────────────────────────────────────
+/**
+ * Split an array into chunks of a given size.
+ */
+function chunkArray(array, chunkSize) {
+    const chunks = [];
+    for (let i = 0; i < array.length; i += chunkSize) {
+        chunks.push(array.slice(i, i + chunkSize));
+    }
+    return chunks;
+}
+/**
+ * Run async tasks with a concurrency limit.
+ * Executes up to `limit` tasks in parallel, queuing the rest.
+ */
+async function RunWithConcurrency(tasks, limit) {
+    const results = [];
+    let index = 0;
+    async function runNext() {
+        while (index < tasks.length) {
+            const currentIndex = index++;
+            results[currentIndex] = await tasks[currentIndex]();
         }
-        return response;
     }
+    const workers = Array.from({ length: Math.min(limit, tasks.length) }, () => runNext());
+    await Promise.all(workers);
+    return results;
 }
 //# sourceMappingURL=duplicateRecordDetector.js.map