@memberjunction/ai-vector-dupe 5.21.0 → 5.23.0

package/dist/duplicateRecordDetector.d.ts

@@ -1,22 +1,184 @@
-import { BaseEmbeddings } from "@memberjunction/ai";
-import { PotentialDuplicateRequest, PotentialDuplicateResponse, UserInfo, PotentialDuplicateResult } from "@memberjunction/core";
-import { BaseResponse, VectorDBBase } from "@memberjunction/ai-vectordb";
+/**
+ * @fileoverview Modernized duplicate record detection engine.
+ *
+ * Orchestrates the full pipeline: vectorize records, query for similar candidates,
+ * optionally apply hybrid search (RRF) and reranking, persist match results,
+ * and auto-merge high-confidence duplicates.
+ *
+ * Supports three record-source modes:
+ * 1. List-based batch detection (ListID provided)
+ * 2. View-based detection (ViewID provided)
+ * 3. Entity-wide detection (no ListID/ViewID — scans all records or applies ExtraFilter)
+ *
+ * Also supports single-record checks via CheckSingleRecord().
+ *
+ * @module @memberjunction/ai-vector-dupe
+ */
+import { PotentialDuplicateRequest, PotentialDuplicateResponse, CompositeKey, UserInfo, BaseEntity, PotentialDuplicateResult, EntityInfo, PotentialDuplicate, DuplicateDetectionOptions } from "@memberjunction/core";
+import { BaseResponse } from "@memberjunction/ai-vectordb";
+import { MJDuplicateRunDetailEntity, MJDuplicateRunDetailMatchEntity, MJDuplicateRunEntity, MJEntityDocumentEntity, MJListEntity } from "@memberjunction/core-entities";
 import { VectorBase } from "@memberjunction/ai-vectors";
+import { EntityDocumentTemplateParser } from "@memberjunction/ai-vector-sync";
+import type { MJTemplateEntityExtended } from "@memberjunction/core-entities";
+/**
+ * Internal result from querying duplicates for a single source record.
+ */
+interface RecordQueryResult {
+    SourceKey: CompositeKey;
+    TemplateText: string;
+    Duplicates: PotentialDuplicateResult;
+}
+/**
+ * Modernized duplicate record detection engine.
+ *
+ * Supports:
+ * - List-based batch detection (getDuplicateRecords)
+ * - View/filter/full-entity batch detection (vector-first approach)
+ * - Single-record duplicate check (CheckSingleRecord)
+ * - Hybrid search via RRF when vector DB supports it
+ * - Optional post-retrieval reranking via MJ's BaseReranker
+ * - Configurable topK, thresholds, and progress reporting
+ */
 export declare class DuplicateRecordDetector extends VectorBase {
-    _vectorDB: VectorDBBase;
-    _embedding: BaseEmbeddings;
-    constructor();
-    getDuplicateRecords(params: PotentialDuplicateRequest, contextUser?: UserInfo): Promise<PotentialDuplicateResponse>;
-    private GetRecordsByListID;
-    private createDuplicateRunRecord;
-    private createDuplicateRunDetailRecords;
-    private createDuplicateRunDetailRecordsByListID;
-    private getListEntity;
-    private getDuplicateRunEntity;
-    private getDuplicateRunEntityByListID;
-    private createListForDupeRun;
-    private createDuplicateRunDetailMatchesForRecord;
-    private mergeRecords;
-    getVectorDuplicates(queryResponse: BaseResponse): Promise<PotentialDuplicateResult>;
+    private vectorDB;
+    private embedding;
+    /** The Pinecone/pgvector/Qdrant index name resolved from the entity document's VectorIndex */
+    private indexName;
+    /**
+     * Tracks already-seen source↔match pairs across the entire run to suppress inverse duplicates.
+     * If A→B is persisted, B→A is skipped. Key format: "smallerID::largerID" for consistent ordering.
+     */
+    private _seenPairs;
+    /**
+     * Run duplicate detection for records identified by ListID, ViewID, ExtraFilter,
+     * or all records in the entity (vector-first approach).
+     *
+     * Flow: validate -> vectorize -> init providers -> load/create run ->
+     *       load record IDs -> batch(embed -> query -> persist) -> complete run -> auto-merge
+     */
+    GetDuplicateRecords(params: PotentialDuplicateRequest, contextUser?: UserInfo): Promise<PotentialDuplicateResponse>;
+    /**
+     * Check a single record for duplicates without requiring a list.
+     * Embeds the record and queries for matches directly.
+     */
+    CheckSingleRecord(EntityDocumentID: string, RecordID: CompositeKey, Options?: DuplicateDetectionOptions, ContextUser?: UserInfo): Promise<PotentialDuplicateResult>;
+    /**
+     * Result from processing a single batch of records.
+     */
+    private ProcessBatch;
+    /**
+     * Load the IDs of records to check, using the appropriate strategy based on the request.
+     * Returns an array of primary key value strings.
+     */
+    protected LoadRecordIDsToCheck(params: PotentialDuplicateRequest, entityInfo: EntityInfo): Promise<string[]>;
+    /**
+     * Load record IDs from a list's detail records.
+     */
+    protected LoadRecordIDsFromList(listID: string): Promise<string[]>;
+    /**
+     * Load record IDs by running a saved view.
+     */
+    protected LoadRecordIDsFromView(viewID: string, entityInfo: EntityInfo): Promise<string[]>;
+    /**
+     * Load record IDs directly from the entity, optionally filtered.
+     * Uses Fields: ['ID'] and ResultType: 'simple' for efficiency.
+     */
+    protected LoadRecordIDsFromEntity(entityInfo: EntityInfo, extraFilter?: string): Promise<string[]>;
+    /**
+     * Validate and return an entity document, or null if not found.
+     */
+    protected ValidateEntityDocument(entityDocumentID: string): Promise<MJEntityDocumentEntity | null>;
+    /**
+     * Initialize embedding and vector DB providers via ClassFactory.
+     */
+    protected InitializeProviders(entityDocument: MJEntityDocumentEntity): Promise<void>;
+    /**
+     * Run vectorization for the entity document's records.
+     */
+    protected VectorizeSourceRecords(entityDocument: MJEntityDocumentEntity, contextUser: UserInfo): Promise<void>;
+    /**
+     * Read the maxConcurrentRequests from the VectorDatabase entity's Configuration column,
+     * falling back to DEFAULT_QUERY_CONCURRENCY if not set.
+     */
+    protected GetQueryConcurrency(entityDocument: MJEntityDocumentEntity): number;
+    /**
+     * Resolve an existing DuplicateRun or create a new one.
+     * Supports both list-based and list-free operation.
+     */
+    protected ResolveOrCreateDuplicateRun(params: PotentialDuplicateRequest, entityDocument: MJEntityDocumentEntity, options: DuplicateDetectionOptions): Promise<MJDuplicateRunEntity>;
+    /**
+     * Create a new DuplicateRun record.
+     */
+    protected CreateDuplicateRun(entityDocument: MJEntityDocumentEntity, listID?: string): Promise<MJDuplicateRunEntity>;
+    protected LoadDuplicateRun(duplicateRunID: string): Promise<MJDuplicateRunEntity>;
+    /**
+     * Try to find an existing DuplicateRun for a given ListID. Returns null if none found.
+     */
+    protected FindDuplicateRunByListID(listID: string): Promise<MJDuplicateRunEntity | null>;
+    /**
+     * Load full entity objects for a batch of composite keys.
+     */
+    protected LoadRecordsByKeys(compositeKeys: CompositeKey[], entityInfo: EntityInfo): Promise<BaseEntity[]>;
+    /**
+     * Load records from an entity that are members of the specified list.
+     * Kept for backward compatibility.
+     */
+    protected LoadRecordsByListID(listID: string, entityID: string): Promise<BaseEntity[]>;
+    protected LoadListEntity(listID: string): Promise<MJListEntity>;
+    /**
+     * Generate human-readable template text for each record using the entity document template.
+     *
+     * Loads the template from TemplateEngineServer and renders it via Nunjucks,
+     * matching the same approach used by the vectorization pipeline.
+     */
+    protected GenerateTemplateTexts(templateParser: ReturnType<typeof EntityDocumentTemplateParser.CreateInstance>, entityDocument: MJEntityDocumentEntity, records: BaseEntity[], contextUser: UserInfo): Promise<string[]>;
+    /**
+     * Load the template entity from TemplateEngineServer for the given entity document.
+     */
+    protected loadTemplate(entityDocument: MJEntityDocumentEntity): MJTemplateEntityExtended;
+    /**
+     * Query the vector DB for duplicates of each record, with concurrency control.
+     * Supports hybrid search and RRF fusion when the vector DB supports it.
+     */
+    protected QueryDuplicatesForRecords(records: BaseEntity[], vectors: number[][], templateTexts: string[], entityDocument: MJEntityDocumentEntity, topK: number, options: DuplicateDetectionOptions, concurrency: number): Promise<RecordQueryResult[]>;
+    /**
+     * Execute a vector query — uses hybrid search with RRF when the provider supports it.
+     */
+    protected executeVectorQuery(vector: number[], templateText: string, topK: number, options: DuplicateDetectionOptions): Promise<BaseResponse>;
+    /**
+     * Parse raw vector DB matches into a PotentialDuplicateResult.
+     */
+    ParseVectorMatches(queryResponse: BaseResponse, sourceKey?: CompositeKey): PotentialDuplicateResult;
+    /**
+     * Filter out self-matches where the candidate is the same record as the source.
+     */
+    /**
+     * Build a map of recordID → JSON metadata string from loaded BaseEntity records.
+     * Extracts the entity's name field and a few key display fields for rich UI rendering.
+     */
+    protected buildSourceMetadataMap(records: BaseEntity[], entityInfo: EntityInfo): Map<string, string>;
+    protected FilterSelfMatches(duplicates: PotentialDuplicate[], sourceKey: CompositeKey): PotentialDuplicate[];
+    /**
+     * Create DuplicateRunDetail records for a batch of record IDs.
+     */
+    protected CreateRunDetailRecords(recordIDs: string[], duplicateRunID: string, entityInfo: EntityInfo, metadataMap?: Map<string, string>): Promise<MJDuplicateRunDetailEntity[]>;
+    /**
+     * Persist match results and update run detail records.
+     */
+    protected PersistMatchResults(queryResults: RecordQueryResult[], duplicateRunDetails: MJDuplicateRunDetailEntity[], entityDocument: MJEntityDocumentEntity, options: DuplicateDetectionOptions, startTime: number): Promise<PotentialDuplicateResult[]>;
+    /**
+     * Create match records for a single run detail, saving in parallel batches.
+     */
+    protected CreateMatchRecordsForDetail(duplicateRunDetailID: string, duplicateResult: PotentialDuplicateResult): Promise<MJDuplicateRunDetailMatchEntity[]>;
+    /**
+     * Automatically merge records that meet the absolute match threshold.
+     */
+    protected ProcessAutoMerges(response: PotentialDuplicateResponse, entityDocument: MJEntityDocumentEntity, options?: DuplicateDetectionOptions): Promise<void>;
+    /**
+     * Update a match record's status after a successful merge.
+     */
+    private updateMatchRecordAfterMerge;
+    private reportProgress;
 }
+export {};
 //# sourceMappingURL=duplicateRecordDetector.d.ts.map

package/dist/duplicateRecordDetector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"duplicateRecordDetector.d.ts","sourceRoot":"","sources":["../src/duplicateRecordDetector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAe,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,yBAAyB,EAAE,0BAA0B,EAAyB,QAAQ,EAAc,wBAAwB,EAA0E,MAAM,sBAAsB,CAAC;AAE5O,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAGzE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAGxD,qBAAa,uBAAwB,SAAQ,UAAU;IAEnD,SAAS,EAAE,YAAY,CAAC;IACxB,UAAU,EAAE,cAAc,CAAC;;IAOd,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,EAAE,WAAW,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,0BAA0B,CAAC;YA2JlH,kBAAkB;YAmBlB,wBAAwB;YAmBxB,+BAA+B;YAmB/B,uCAAuC;YAkCvC,aAAa;YAYb,qBAAqB;YAYrB,6BAA6B;YAS7B,oBAAoB;YAiBpB,wCAAwC;YAuBxC,YAAY;IAyCb,mBAAmB,CAAC,aAAa,EAAE,YAAY,GAAG,OAAO,CAAC,wBAAwB,CAAC;CAsBnG"}
+{"version":3,"file":"duplicateRecordDetector.d.ts","sourceRoot":"","sources":["../src/duplicateRecordDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,EACH,yBAAyB,EACzB,0BAA0B,EAC1B,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,wBAAwB,EAIxB,UAAU,EACV,kBAAkB,EAClB,yBAAyB,EAG5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAA6C,MAAM,6BAA6B,CAAC;AAEtG,OAAO,EACH,0BAA0B,EAC1B,+BAA+B,EAC/B,oBAAoB,EACpB,sBAAsB,EAEtB,YAAY,EACf,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,4BAA4B,EAA6C,MAAM,gCAAgC,CAAC;AAEzH,OAAO,KAAK,EAAE,wBAAwB,EAA2B,MAAM,+BAA+B,CAAC;AAiCvG;;GAEG;AACH,UAAU,iBAAiB;IACvB,SAAS,EAAE,YAAY,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,wBAAwB,CAAC;CACxC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,uBAAwB,SAAQ,UAAU;IACnD,OAAO,CAAC,QAAQ,CAAe;IAC/B,OAAO,CAAC,SAAS,CAAiB;IAClC,8FAA8F;IAC9F,OAAO,CAAC,SAAS,CAAS;IAC1B;;;OAGG;IACH,OAAO,CAAC,UAAU,CAAqB;IAEvC;;;;;;OAMG;IACU,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,EAAE,WAAW,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,0BAA0B,CAAC;IA+EhI;;;OAGG;IACU,iBAAiB,CAC1B,gBAAgB,EAAE,MAAM,EACxB,QAAQ,EAAE,YAAY,EACtB,OAAO,CAAC,EAAE,yBAAyB,EACnC,WAAW,CAAC,EAAE,QAAQ,GACvB,OAAO,CAAC,wBAAwB,CAAC;IA6CpC;;OAEG;YACW,YAAY;IAyD1B;;;OAGG;cACa,oBAAoB,CAAC,MAAM,EAAE,yBAAyB,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWlH;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAexE;;OAEG;cACa,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAyBhG;;;OAGG;cACa,uBAAuB,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAmBxG;;OAEG;cACa,sBAAsB,CAAC,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAMxG;;OAEG;cACa,mBAAmB,CAAC,cAAc,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkD1F;;OAEG;cACa,sBAAsB,CAAC,cAAc,EAAE,sBAAsB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBpH;;;OAGG;IACH,SAAS,CAAC,mBAAmB,CAAC,cAAc,EAAE,sBAAsB,GAAG,MAAM;IAmB7E;;;OAGG;cACa,2BAA2B,CACvC,MAAM,EAAE,yBAAyB,EACjC,cAAc,EAAE,sBAAsB,EACtC,OAAO,EAAE,yBAAyB,GACnC,OAAO,CAAC,oBAAoB,CAAC;IAkBhC;;OAEG;cACa,kBAAkB,CAC9B,cAAc,EAAE,sBAAsB,EACtC,MAAM,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,oBAAoB,CAAC;cAmBhB,gBAAgB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAUvF;;OAEG;cACa,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC;IAU9F;;OAEG;cACa,iBAAiB,CAAC,aAAa,EAAE,YAAY,EAAE,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAiB/G;;;OAGG;cACa,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;cAmB5E,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAcrE;;;;;OAKG;cACa,qBAAqB,CACjC,cAAc,EAAE,UAAU,CAAC,OAAO,4BAA4B,CAAC,cAAc,CAAC,EAC9E,cAAc,EAAE,sBAAsB,EACtC,OAAO,EAAE,UAAU,EAAE,EACrB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC;IA0BpB;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,cAAc,EAAE,sBAAsB,GAAG,wBAAwB;IAkBxF;;;OAGG;cACa,yBAAyB,CACrC,OAAO,EAAE,UAAU,EAAE,EACrB,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,aAAa,EAAE,MAAM,EAAE,EACvB,cAAc,EAAE,sBAAsB,EACtC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,yBAAyB,EAClC,WAAW,EAAE,MAAM,GACpB,OAAO,CAAC,iBAAiB,EAAE,CAAC;IA+B/B;;OAEG;cACa,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EAAE,EAChB,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,yBAAyB,GACnC,OAAO,CAAC,YAAY,CAAC;IAsBxB;;OAEG;IACI,kBAAkB,CAAC,aAAa,EAAE,YAAY,EAAE,SAAS,CAAC,EAAE,YAAY,GAAG,wBAAwB;IAyB1G;;OAEG;IACH;;;OAGG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,UAAU,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAkCpG,SAAS,CAAC,iBAAiB,CAAC,UAAU,EAAE,kBAAkB,EAAE,EAAE,SAAS,EAAE,YAAY,GAAG,kBAAkB,EAAE;IAQ5G;;OAEG;cACa,sBAAsB,CAClC,SAAS,EAAE,MAAM,EAAE,EACnB,cAAc,EAAE,MAAM,EACtB,UAAU,EAAE,UAAU,EACtB,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAClC,OAAO,CAAC,0BAA0B,EAAE,CAAC;IA+BxC;;OAEG;cACa,mBAAmB,CAC/B,YAAY,EAAE,iBAAiB,EAAE,EACjC,mBAAmB,EAAE,0BAA0B,EAAE,EACjD,cAAc,EAAE,sBAAsB,EACtC,OAAO,EAAE,yBAAyB,EAClC,SAAS,EAAE,MAAM,GAClB,OAAO,CAAC,wBAAwB,EAAE,CAAC;IA+CtC;;OAEG;cACa,2BAA2B,CACvC,oBAAoB,EAAE,MAAM,EAC5B,eAAe,EAAE,wBAAwB,GAC1C,OAAO,CAAC,+BAA+B,EAAE,CAAC;IAkC7C;;OAEG;cACa,iBAAiB,CAC7B,QAAQ,EAAE,0BAA0B,EACpC,cAAc,EAAE,sBAAsB,EACtC,OAAO,GAAE,yBAA8B,GACxC,OAAO,CAAC,IAAI,CAAC;IAuBhB;;OAEG;YACW,2BAA2B;IAwBzC,OAAO,CAAC,cAAc;CAoBzB"}