@memberjunction/ai-vector-dupe 5.21.0 → 5.22.0

package/dist/duplicateRecordDetector.d.ts

@@ -1,22 +1,120 @@
-import { BaseEmbeddings } from "@memberjunction/ai";
-import { PotentialDuplicateRequest, PotentialDuplicateResponse, UserInfo, PotentialDuplicateResult } from "@memberjunction/core";
-import { BaseResponse, VectorDBBase } from "@memberjunction/ai-vectordb";
+/**
+ * @fileoverview Modernized duplicate record detection engine.
+ *
+ * Orchestrates the full pipeline: vectorize records, query for similar candidates,
+ * optionally apply hybrid search (RRF) and reranking, persist match results,
+ * and auto-merge high-confidence duplicates.
+ *
+ * Supports both list-based batch detection and single-record checks.
+ *
+ * @module @memberjunction/ai-vector-dupe
+ */
+import { PotentialDuplicateRequest, PotentialDuplicateResponse, CompositeKey, UserInfo, BaseEntity, PotentialDuplicateResult, PotentialDuplicate, DuplicateDetectionOptions } from "@memberjunction/core";
+import { BaseResponse } from "@memberjunction/ai-vectordb";
+import { MJDuplicateRunDetailEntity, MJDuplicateRunDetailMatchEntity, MJDuplicateRunEntity, MJEntityDocumentEntity, MJListEntity } from "@memberjunction/core-entities";
 import { VectorBase } from "@memberjunction/ai-vectors";
+import { EntityDocumentTemplateParser } from "@memberjunction/ai-vector-sync";
+import type { MJTemplateEntityExtended } from "@memberjunction/core-entities";
+/**
+ * Internal result from querying duplicates for a single source record.
+ */
+interface RecordQueryResult {
+    SourceKey: CompositeKey;
+    TemplateText: string;
+    Duplicates: PotentialDuplicateResult;
+}
+/**
+ * Modernized duplicate record detection engine.
+ *
+ * Supports:
+ * - List-based batch detection (getDuplicateRecords)
+ * - Single-record duplicate check (CheckSingleRecord)
+ * - Hybrid search via RRF when vector DB supports it
+ * - Optional post-retrieval reranking via MJ's BaseReranker
+ * - Configurable topK, thresholds, and progress reporting
+ */
 export declare class DuplicateRecordDetector extends VectorBase {
-    _vectorDB: VectorDBBase;
-    _embedding: BaseEmbeddings;
-    constructor();
-    getDuplicateRecords(params: PotentialDuplicateRequest, contextUser?: UserInfo): Promise<PotentialDuplicateResponse>;
-    private GetRecordsByListID;
-    private createDuplicateRunRecord;
-    private createDuplicateRunDetailRecords;
-    private createDuplicateRunDetailRecordsByListID;
-    private getListEntity;
-    private getDuplicateRunEntity;
-    private getDuplicateRunEntityByListID;
-    private createListForDupeRun;
-    private createDuplicateRunDetailMatchesForRecord;
-    private mergeRecords;
-    getVectorDuplicates(queryResponse: BaseResponse): Promise<PotentialDuplicateResult>;
+    private vectorDB;
+    private embedding;
+    /**
+     * Run duplicate detection for all records in a list.
+     *
+     * Flow: validate → vectorize → embed → query → (optional rerank) → persist → (optional merge)
+     */
+    GetDuplicateRecords(params: PotentialDuplicateRequest, contextUser?: UserInfo): Promise<PotentialDuplicateResponse>;
+    /**
+     * Check a single record for duplicates without requiring a list.
+     * Embeds the record and queries for matches directly.
+     */
+    CheckSingleRecord(EntityDocumentID: string, RecordID: CompositeKey, Options?: DuplicateDetectionOptions, ContextUser?: UserInfo): Promise<PotentialDuplicateResult>;
+    /**
+     * Validate and return an entity document, or null if not found.
+     */
+    protected ValidateEntityDocument(entityDocumentID: string): Promise<MJEntityDocumentEntity | null>;
+    /**
+     * Initialize embedding and vector DB providers via ClassFactory.
+     */
+    protected InitializeProviders(entityDocument: MJEntityDocumentEntity): void;
+    /**
+     * Run vectorization for the entity document's records.
+     */
+    protected VectorizeSourceRecords(entityDocument: MJEntityDocumentEntity, contextUser: UserInfo): Promise<void>;
+    /**
+     * Generate human-readable template text for each record using the entity document template.
+     *
+     * Loads the template from TemplateEngineServer and renders it via Nunjucks,
+     * matching the same approach used by the vectorization pipeline.
+     */
+    protected GenerateTemplateTexts(templateParser: ReturnType<typeof EntityDocumentTemplateParser.CreateInstance>, entityDocument: MJEntityDocumentEntity, records: BaseEntity[], contextUser: UserInfo): Promise<string[]>;
+    /**
+     * Load the template entity from TemplateEngineServer for the given entity document.
+     */
+    protected loadTemplate(entityDocument: MJEntityDocumentEntity): MJTemplateEntityExtended;
+    /**
+     * Query the vector DB for duplicates of each record, with concurrency control.
+     * Supports hybrid search and RRF fusion when the vector DB supports it.
+     */
+    protected QueryDuplicatesForRecords(records: BaseEntity[], vectors: number[][], templateTexts: string[], entityDocument: MJEntityDocumentEntity, topK: number, options: DuplicateDetectionOptions): Promise<RecordQueryResult[]>;
+    /**
+     * Execute a vector query — uses hybrid search with RRF when the provider supports it.
+     */
+    protected executeVectorQuery(vector: number[], templateText: string, topK: number, options: DuplicateDetectionOptions): Promise<BaseResponse>;
+    /**
+     * Parse raw vector DB matches into a PotentialDuplicateResult.
+     */
+    ParseVectorMatches(queryResponse: BaseResponse, sourceKey?: CompositeKey): PotentialDuplicateResult;
+    /**
+     * Filter out self-matches where the candidate is the same record as the source.
+     */
+    protected FilterSelfMatches(duplicates: PotentialDuplicate[], sourceKey: CompositeKey): PotentialDuplicate[];
+    /**
+     * Load records from an entity that are members of the specified list.
+     */
+    protected LoadRecordsByListID(listID: string, entityID: string): Promise<BaseEntity[]>;
+    protected LoadListEntity(listID: string): Promise<MJListEntity>;
+    protected LoadDuplicateRun(duplicateRunID: string): Promise<MJDuplicateRunEntity>;
+    protected LoadDuplicateRunByListID(listID: string): Promise<MJDuplicateRunEntity>;
+    /**
+     * Create DuplicateRunDetail records for each item in the list, saving in parallel batches.
+     */
+    protected CreateRunDetailRecordsFromList(listID: string, duplicateRunID: string): Promise<MJDuplicateRunDetailEntity[]>;
+    /**
+     * Persist match results and update run detail records.
+     */
+    protected PersistMatchResults(queryResults: RecordQueryResult[], duplicateRunDetails: MJDuplicateRunDetailEntity[], entityDocument: MJEntityDocumentEntity, options: DuplicateDetectionOptions, startTime: number): Promise<PotentialDuplicateResult[]>;
+    /**
+     * Create match records for a single run detail, saving in parallel batches.
+     */
+    protected CreateMatchRecordsForDetail(duplicateRunDetailID: string, duplicateResult: PotentialDuplicateResult): Promise<MJDuplicateRunDetailMatchEntity[]>;
+    /**
+     * Automatically merge records that meet the absolute match threshold.
+     */
+    protected ProcessAutoMerges(response: PotentialDuplicateResponse, entityDocument: MJEntityDocumentEntity): Promise<void>;
+    /**
+     * Update a match record's status after a successful merge.
+     */
+    private updateMatchRecordAfterMerge;
+    private reportProgress;
 }
+export {};
 //# sourceMappingURL=duplicateRecordDetector.d.ts.map

package/dist/duplicateRecordDetector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"duplicateRecordDetector.d.ts","sourceRoot":"","sources":["../src/duplicateRecordDetector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAe,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,yBAAyB,EAAE,0BAA0B,EAAyB,QAAQ,EAAc,wBAAwB,EAA0E,MAAM,sBAAsB,CAAC;AAE5O,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAGzE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAGxD,qBAAa,uBAAwB,SAAQ,UAAU;IAEnD,SAAS,EAAE,YAAY,CAAC;IACxB,UAAU,EAAE,cAAc,CAAC;;IAOd,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,EAAE,WAAW,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,0BAA0B,CAAC;YA2JlH,kBAAkB;YAmBlB,wBAAwB;YAmBxB,+BAA+B;YAmB/B,uCAAuC;YAkCvC,aAAa;YAYb,qBAAqB;YAYrB,6BAA6B;YAS7B,oBAAoB;YAiBpB,wCAAwC;YAuBxC,YAAY;IAyCb,mBAAmB,CAAC,aAAa,EAAE,YAAY,GAAG,OAAO,CAAC,wBAAwB,CAAC;CAsBnG"}
+{"version":3,"file":"duplicateRecordDetector.d.ts","sourceRoot":"","sources":["../src/duplicateRecordDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EACH,yBAAyB,EACzB,0BAA0B,EAC1B,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,wBAAwB,EAKxB,kBAAkB,EAClB,yBAAyB,EAE5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,YAAY,EAAgB,MAAM,6BAA6B,CAAC;AAEzE,OAAO,EACH,0BAA0B,EAC1B,+BAA+B,EAC/B,oBAAoB,EACpB,sBAAsB,EAEtB,YAAY,EACf,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,4BAA4B,EAA6C,MAAM,gCAAgC,CAAC;AAEzH,OAAO,KAAK,EAAE,wBAAwB,EAA2B,MAAM,+BAA+B,CAAC;AA8BvG;;GAEG;AACH,UAAU,iBAAiB;IACvB,SAAS,EAAE,YAAY,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,wBAAwB,CAAC;CACxC;AAED;;;;;;;;;GASG;AACH,qBAAa,uBAAwB,SAAQ,UAAU;IACnD,OAAO,CAAC,QAAQ,CAAe;IAC/B,OAAO,CAAC,SAAS,CAAiB;IAElC;;;;OAIG;IACU,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,EAAE,WAAW,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,0BAA0B,CAAC;IA+EhI;;;OAGG;IACU,iBAAiB,CAC1B,gBAAgB,EAAE,MAAM,EACxB,QAAQ,EAAE,YAAY,EACtB,OAAO,CAAC,EAAE,yBAAyB,EACnC,WAAW,CAAC,EAAE,QAAQ,GACvB,OAAO,CAAC,wBAAwB,CAAC;IA4CpC;;OAEG;cACa,sBAAsB,CAAC,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,IAAI,CAAC;IAMxG;;OAEG;IACH,SAAS,CAAC,mBAAmB,CAAC,cAAc,EAAE,sBAAsB,GAAG,IAAI;IA+B3E;;OAEG;cACa,sBAAsB,CAAC,cAAc,EAAE,sBAAsB,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBpH;;;;;OAKG;cACa,qBAAqB,CACjC,cAAc,EAAE,UAAU,CAAC,OAAO,4BAA4B,CAAC,cAAc,CAAC,EAC9E,cAAc,EAAE,sBAAsB,EACtC,OAAO,EAAE,UAAU,EAAE,EACrB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,MAAM,EAAE,CAAC;IA0BpB;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,cAAc,EAAE,sBAAsB,GAAG,wBAAwB;IAkBxF;;;OAGG;cACa,yBAAyB,CACrC,OAAO,EAAE,UAAU,EAAE,EACrB,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,aAAa,EAAE,MAAM,EAAE,EACvB,cAAc,EAAE,sBAAsB,EACtC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,yBAAyB,GACnC,OAAO,CAAC,iBAAiB,EAAE,CAAC;IA8B/B;;OAEG;cACa,kBAAkB,CAC9B,MAAM,EAAE,MAAM,EAAE,EAChB,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,yBAAyB,GACnC,OAAO,CAAC,YAAY,CAAC;IAqBxB;;OAEG;IACI,kBAAkB,CAAC,aAAa,EAAE,YAAY,EAAE,SAAS,CAAC,EAAE,YAAY,GAAG,wBAAwB;IAuB1G;;OAEG;IACH,SAAS,CAAC,iBAAiB,CAAC,UAAU,EAAE,kBAAkB,EAAE,EAAE,SAAS,EAAE,YAAY,GAAG,kBAAkB,EAAE;IAQ5G;;OAEG;cACa,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;cAmB5E,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;cAUrD,gBAAgB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;cAUvE,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAcvF;;OAEG;cACa,8BAA8B,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,0BAA0B,EAAE,CAAC;IAuC7H;;OAEG;cACa,mBAAmB,CAC/B,YAAY,EAAE,iBAAiB,EAAE,EACjC,mBAAmB,EAAE,0BAA0B,EAAE,EACjD,cAAc,EAAE,sBAAsB,EACtC,OAAO,EAAE,yBAAyB,EAClC,SAAS,EAAE,MAAM,GAClB,OAAO,CAAC,wBAAwB,EAAE,CAAC;IA8BtC;;OAEG;cACa,2BAA2B,CACvC,oBAAoB,EAAE,MAAM,EAC5B,eAAe,EAAE,wBAAwB,GAC1C,OAAO,CAAC,+BAA+B,EAAE,CAAC;IAiC7C;;OAEG;cACa,iBAAiB,CAC7B,QAAQ,EAAE,0BAA0B,EACpC,cAAc,EAAE,sBAAsB,GACvC,OAAO,CAAC,IAAI,CAAC;IAsBhB;;OAEG;YACW,2BAA2B;IAwBzC,OAAO,CAAC,cAAc;CAoBzB"}