@memberjunction/ai-vector-dupe 1.2.1 → 1.3.0

package/dist/duplicateRecordDetector.d.ts

@@ -1,19 +1,21 @@
 import { Embeddings } from "@memberjunction/ai";
-import { PotentialDuplicateRequest, PotentialDuplicateResponse, RunView, UserInfo } from "@memberjunction/core";
+import { PotentialDuplicateRequest, PotentialDuplicateResponse, UserInfo } from "@memberjunction/core";
 import { VectorDBBase } from "@memberjunction/ai-vectordb";
-import { AIModelEntity, VectorDatabaseEntity } from "@memberjunction/core-entities";
-export declare class DuplicateRecordDetector {
-    _contextUser: UserInfo;
-    _runView: RunView;
+import { VectorBase } from "@memberjunction/ai-vectors";
+export declare class DuplicateRecordDetector extends VectorBase {
     _vectorDB: VectorDBBase;
     _embedding: Embeddings;
-    protected GetAIModel(): AIModelEntity;
-    protected GetVectorDatabase(): VectorDatabaseEntity;
+    constructor();
     getDuplicateRecords(params: PotentialDuplicateRequest, contextUser?: UserInfo): Promise<PotentialDuplicateResponse>;
-    private getEntityDocumennt;
-    private getEntityRecord;
-    private buildExtraFilter;
-    private processDuplicateRecords;
-    protected parseStringTemplate(str: string, obj: any): string;
+    private GetRecordsByListID;
+    private createDuplicateRunRecord;
+    private createDuplicateRunDetailRecords;
+    private createDuplicateRunDetailRecordsByListID;
+    private getListEntity;
+    private getDuplicateRunEntity;
+    private getDuplicateRunEntityByListID;
+    private createListForDupeRun;
+    private createDuplicateRunDetailMatchesForRecord;
+    private mergeRecords;
 }
 //# sourceMappingURL=duplicateRecordDetector.d.ts.map

package/dist/duplicateRecordDetector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"duplicateRecordDetector.d.ts","sourceRoot":"","sources":["../src/duplicateRecordDetector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAe,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAqC,yBAAyB,EAAE,0BAA0B,EAAkD,OAAO,EAAE,QAAQ,EAAY,MAAM,sBAAsB,CAAC;AAG7M,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAE3D,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AAGpF,qBAAa,uBAAuB;IAEhC,YAAY,EAAE,QAAQ,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,YAAY,CAAC;IACxB,UAAU,EAAE,UAAU,CAAA;IAMtB,SAAS,CAAC,UAAU,IAAI,aAAa;IASrC,SAAS,CAAC,iBAAiB,IAAI,oBAAoB;IAStC,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,EAAE,WAAW,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,0BAA0B,CAAC;YAwDlH,kBAAkB;YAiBlB,eAAe;IAiB7B,OAAO,CAAC,gBAAgB;IAOxB,OAAO,CAAC,uBAAuB;IA0B/B,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,MAAM;CAY/D"}
+{"version":3,"file":"duplicateRecordDetector.d.ts","sourceRoot":"","sources":["../src/duplicateRecordDetector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAe,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,0BAA0B,EAAyB,QAAQ,EAA6F,MAAM,sBAAsB,CAAC;AAEzN,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAG3D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAGxD,qBAAa,uBAAwB,SAAQ,UAAU;IAEnD,SAAS,EAAE,YAAY,CAAC;IACxB,UAAU,EAAE,UAAU,CAAC;;IAOV,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,EAAE,WAAW,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC,0BAA0B,CAAC;YAsJlH,kBAAkB;YAoBlB,wBAAwB;YAmBxB,+BAA+B;YAmB/B,uCAAuC;YA8BvC,aAAa;YAYb,qBAAqB;YAYrB,6BAA6B;YAS7B,oBAAoB;YAiBpB,wCAAwC;YAuBxC,YAAY;CAwC7B"}

package/dist/duplicateRecordDetector.js

@@ -6,118 +6,282 @@ const core_1 = require("@memberjunction/core");
 const core_2 = require("@memberjunction/core");
 const ai_vectordb_1 = require("@memberjunction/ai-vectordb");
 const global_1 = require("@memberjunction/global");
-const aiengine_1 = require("@memberjunction/aiengine");
-class DuplicateRecordDetector {
-    _contextUser;
-    _runView;
+const ai_vectors_1 = require("@memberjunction/ai-vectors");
+const ai_vector_sync_1 = require("@memberjunction/ai-vector-sync");
+class DuplicateRecordDetector extends ai_vectors_1.VectorBase {
     _vectorDB;
     _embedding;
-    GetAIModel() {
-        const model = aiengine_1.AIEngine.Models.find(m => m.AIModelTypeID === 3);
-        return model;
+    constructor() {
+        super();
+        this._runView = new core_1.RunView();
     }
-    GetVectorDatabase() {
-        if (aiengine_1.AIEngine.VectorDatabases.length > 0) {
-            return aiengine_1.AIEngine.VectorDatabases[0];
+    async getDuplicateRecords(params, contextUser) {
+        super.CurrentUser = contextUser;
+        let vectorizer = new ai_vector_sync_1.EntityVectorSyncer();
+        vectorizer.CurrentUser = super.CurrentUser;
+        let entityDocument = null;
+        if (params.EntityDocumentID) {
+            entityDocument = await vectorizer.GetEntityDocument(params.EntityDocumentID);
         }
         else {
-            throw new Error("No Vector Databases found");
-        }
-    }
-    async getDuplicateRecords(params, contextUser) {
-        this._contextUser = contextUser;
-        if (!this._contextUser) {
-            (0, core_1.LogError)("Context User not found, using default");
-            const md = new core_1.Metadata();
-            this._contextUser = md.CurrentUser;
+            entityDocument = await vectorizer.GetFirstActiveEntityDocumentForEntity(params.EntityID);
+            if (!entityDocument) {
+                throw Error(`No active Entity Document found for entity ${params.EntityID}`);
+            }
         }
-        if (!this._runView) {
-            this._runView = new core_1.RunView();
+        let response = new core_1.PotentialDuplicateResponse();
+        response.PotentialDuplicateResult = [];
+        if (!entityDocument) {
+            response.ErrorMessage = `No active Entity Document found for entity ${params.EntityID}`;
+            response.Status = 'Error';
+            return response;
         }
-        const aiModel = this.GetAIModel();
-        const vectorDB = this.GetVectorDatabase();
+        const list = await this.getListEntity(params.ListID);
+        let duplicateRun = params.Options?.DuplicateRunID ? await this.getDuplicateRunEntity(params.Options?.DuplicateRunID) : await this.getDuplicateRunEntityByListID(list.ID);
+        const duplicateRunDetails = await this.createDuplicateRunDetailRecordsByListID(list.ID, duplicateRun.ID);
+        (0, core_2.LogStatus)(`Using vector database ${entityDocument.VectorDatabaseID} and AI Model ${entityDocument.AIModelID}`);
+        const vectorDB = this.getVectorDatabase(entityDocument.VectorDatabaseID);
+        const aiModel = this.getAIModel(entityDocument.AIModelID);
+        (0, core_2.LogStatus)(`AIModel driver class: ${aiModel.DriverClass}`);
+        (0, core_2.LogStatus)(`VectorDB class key: ${vectorDB.ClassKey}`);
         const embeddingAPIKey = (0, ai_1.GetAIAPIKey)(aiModel.DriverClass);
         const vectorDBAPIKey = (0, ai_1.GetAIAPIKey)(vectorDB.ClassKey);
-        (0, core_2.LogStatus)("Embedding API Key: ", embeddingAPIKey);
-        (0, core_2.LogStatus)("Vector Database API Key: ", vectorDBAPIKey);
+        if (!embeddingAPIKey) {
+            throw Error(`No API Key found for AI Model ${aiModel.DriverClass}`);
+        }
+        if (!vectorDBAPIKey) {
+            throw Error(`No API Key found for Vector Database ${vectorDB.ClassKey}`);
+        }
         this._embedding = global_1.MJGlobal.Instance.ClassFactory.CreateInstance(ai_1.Embeddings, aiModel.DriverClass, embeddingAPIKey);
         this._vectorDB = global_1.MJGlobal.Instance.ClassFactory.CreateInstance(ai_vectordb_1.VectorDBBase, vectorDB.ClassKey, vectorDBAPIKey);
-        const entityDocument = await this.getEntityDocumennt(params.EntityDocumentID);
-        (0, core_2.LogStatus)("Entity Document: " + entityDocument);
-        if (!entityDocument) {
-            (0, core_2.LogStatus)("Entity Document not found");
-            return { EntityID: params.EntityDocumentID, Duplicates: [] };
-        }
-        const entityRecord = await this.getEntityRecord(entityDocument.Type, params.PrimaryKeyValues);
-        if (!entityRecord) {
-            (0, core_2.LogStatus)("Entity Record not found");
-            return { EntityID: entityDocument.EntityID, Duplicates: [] };
-        }
-        const template = this.parseStringTemplate(entityDocument.Template, entityRecord);
-        const embedding = await this._embedding.EmbedText({ text: template, model: "mistral-embed" });
-        const vector = embedding.data;
-        const topK = 10;
-        const queryResult = await this._vectorDB.queryIndex({ vector: vector, topK: topK, includeMetadata: true, includeValues: true });
-        return this.processDuplicateRecords(queryResult.data.matches, entityDocument);
-    }
-    async getEntityDocumennt(id) {
-        const rvResult = await this._runView.RunView({
-            EntityName: "Entity Documents",
-            ExtraFilter: `ID = '${id}'`
-        }, this._contextUser);
-        if (!rvResult || rvResult.RowCount === 0) {
-            return null;
+        if (!this._embedding) {
+            throw Error(`Failed to create Embeddings instance for AI Model ${aiModel.DriverClass}`);
         }
-        else {
-            const vdResults = rvResult?.Results;
-            const entityDocument = vdResults[0];
-            return entityDocument;
+        if (!this._vectorDB) {
+            throw Error(`Failed to create Vector Database instance for ${vectorDB.ClassKey}`);
+        }
+        let records = await this.GetRecordsByListID(list.ID, entityDocument.EntityID);
+        (0, core_2.LogStatus)("Vectorizing " + records.length + " records");
+        const templateParser = ai_vector_sync_1.EntityDocumentTemplateParser.CreateInstance();
+        const recordTemplates = [];
+        let sampleTemplate = entityDocument.Template;
+        for (const record of records) {
+            const template = await templateParser.Parse(sampleTemplate, entityDocument.EntityID, record, contextUser);
+            recordTemplates.push(template);
+        }
+        let embedTextsResult = await this._embedding.EmbedTexts({ texts: recordTemplates, model: null });
+        const topK = 5;
+        let results = [];
+        for (const [index, vector] of embedTextsResult.vectors.entries()) {
+            const compositeKey = records[index].CompositeKey;
+            let queryResult = await this._vectorDB.getVectorDuplicates({ vector: vector, topK: topK, includeMetadata: true, includeValues: false });
+            if (queryResult.success) {
+                queryResult.data.Duplicates = queryResult.data.Duplicates.filter((dupe) => {
+                    return dupe.ProbabilityScore >= entityDocument.PotentialMatchThreshold;
+                });
+                let result = queryResult.data;
+                result.EntityID = entityDocument.EntityID;
+                result.RecordCompositeKey = compositeKey;
+                results.push(result);
+                let dupeRunDetail = duplicateRunDetails.find((detail) => detail.RecordID === compositeKey.ToString());
+                if (dupeRunDetail) {
+                    const matchRecords = await this.createDuplicateRunDetailMatchesForRecord(dupeRunDetail.ID, result);
+                    result.DuplicateRunDetailMatchRecordIDs = matchRecords.map((match) => match.ID);
+                    dupeRunDetail.MatchStatus = 'Complete';
+                    let success = await super.SaveEntity(dupeRunDetail);
+                    if (!success) {
+                        (0, core_2.LogStatus)(`Failed to update Duplicate Run Detail record ${dupeRunDetail.ID}`);
+                    }
+                }
+                else {
+                    (0, core_1.LogError)(`Failed to find Duplicate Run Detail record for ${compositeKey.ToString()}`);
+                }
+            }
         }
+        duplicateRun.ProcessingStatus = 'Complete';
+        duplicateRun.EndedAt = new Date();
+        let success = await super.SaveEntity(duplicateRun);
+        if (!success) {
+            throw new Error(`Failed to update Duplicate Run record ${duplicateRun.ID}`);
+        }
+        await this.mergeRecords(response, entityDocument);
+        response.PotentialDuplicateResult = results;
+        response.Status = 'Success';
+        (0, core_2.LogStatus)("Dupe Run complete. Response:");
+        (0, core_2.LogStatus)(JSON.stringify(response, null, "\t"));
+        return response;
     }
-    async getEntityRecord(entityName, keyValues) {
-        const rvResult = await this._runView.RunView({
-            EntityName: entityName,
-            ExtraFilter: this.buildExtraFilter(keyValues)
-        }, this._contextUser);
-        if (!rvResult || rvResult.RowCount === 0) {
-            return null;
+    async GetRecordsByListID(listID, entityID) {
+        const entity = await super.runViewForSingleValue("Entities", `ID = ${entityID}`);
+        if (!entity) {
+            throw new Error(`Failed to load Entity record ${entityID}`);
         }
-        else {
-            const vdResults = rvResult?.Results;
-            const entityDocument = vdResults[0];
-            return entityDocument;
+        const rvResult = await super.RunView.RunView({
+            EntityName: entity.Name,
+            ExtraFilter: `ID IN (SELECT RecordID FROM __mj.vwListDetails WHERE ListID = ${listID})`,
+            ResultType: 'entity_object'
+        }, super.CurrentUser);
+        if (!rvResult.Success) {
+            throw new Error(rvResult.ErrorMessage);
         }
+        const records = rvResult.Results;
+        return records;
     }
-    buildExtraFilter(keyValues) {
-        return keyValues.map((keyValue, index) => {
-            return keyValue.FieldName + " = " + keyValue.Value;
-        }).join(" AND ");
+    async createDuplicateRunRecord(entityDocument, listID) {
+        const md = new core_1.Metadata();
+        let duplicateRun = await md.GetEntityObject('Duplicate Runs');
+        duplicateRun.NewRecord();
+        duplicateRun.EntityID = entityDocument.EntityID;
+        duplicateRun.StartedByUserID = super.CurrentUser.ID;
+        duplicateRun.StartedAt = new Date();
+        duplicateRun.ProcessingStatus = 'In Progress';
+        duplicateRun.ApprovalStatus = 'Pending';
+        duplicateRun.SourceListID = listID;
+        const saveResult = await super.SaveEntity(duplicateRun);
+        if (!saveResult) {
+            throw new Error(`Failed to save list for Potential Duplicate Run`);
+        }
+        return duplicateRun;
     }
-    processDuplicateRecords(queryResult, entityDocument) {
-        let results = {
-            EntityID: entityDocument.EntityID,
-            Duplicates: []
-        };
-        for (let duplicate of queryResult) {
-            if (duplicate.score > 1) {
-                continue;
+    async createDuplicateRunDetailRecords(recordIDs, duplicateRunID) {
+        let results = [];
+        const md = new core_1.Metadata();
+        for (const recordID of recordIDs) {
+            let runDetail = await md.GetEntityObject('Duplicate Run Details');
+            runDetail.NewRecord();
+            runDetail.DuplicateRunID = duplicateRunID;
+            runDetail.RecordID = recordID.ToString();
+            runDetail.MatchStatus = 'Pending';
+            runDetail.MergeStatus = 'Pending';
+            const success = await super.SaveEntity(runDetail);
+            if (success) {
+                results.push(runDetail);
             }
-            let potentialDuplicate = new core_1.PotentialDuplicate();
-            potentialDuplicate.PrimaryKeyValues = [];
-            potentialDuplicate.PrimaryKeyValues.push({
-                FieldName: "ID",
-                Value: duplicate.id
-            });
-            potentialDuplicate.ProbabilityScore = duplicate.score;
-            results.Duplicates.push(potentialDuplicate);
         }
         return results;
     }
-    parseStringTemplate(str, obj) {
-        let parts = str.split(/\$\{(?!\d)[\wæøåÆØÅ]*\}/);
-        let args = str.match(/[^{\}]+(?=})/g) || [];
-        let parameters = args.map(argument => obj[argument] || (obj[argument] === undefined ? "" : obj[argument]));
-        return String.raw({ raw: parts }, ...parameters);
+    async createDuplicateRunDetailRecordsByListID(listID, duplicateRunID) {
+        let results = [];
+        const viewResults = await super.RunView.RunView({
+            EntityName: 'List Details',
+            ExtraFilter: `ListID = ${listID}`,
+            ResultType: 'entity_object'
+        }, super.CurrentUser);
+        if (!viewResults.Success) {
+            throw new Error(viewResults.ErrorMessage);
+        }
+        const md = new core_1.Metadata();
+        const listDetails = viewResults.Results;
+        for (const listDetail of listDetails) {
+            let runDetail = await md.GetEntityObject('Duplicate Run Details');
+            runDetail.NewRecord();
+            runDetail.DuplicateRunID = duplicateRunID;
+            runDetail.RecordID = `ID = ${listDetail.RecordID}`;
+            runDetail.MatchStatus = 'Pending';
+            runDetail.MergeStatus = 'Pending';
+            const success = await super.SaveEntity(runDetail);
+            if (success) {
+                results.push(runDetail);
+            }
+        }
+        return results;
+    }
+    async getListEntity(listID) {
+        const md = new core_1.Metadata();
+        let list = await md.GetEntityObject('Lists');
+        list.ContextCurrentUser = super.CurrentUser;
+        const success = await list.Load(listID);
+        if (!success) {
+            throw new Error(`Failed to load List record ${listID}`);
+        }
+        return list;
+    }
+    async getDuplicateRunEntity(DupeRunID) {
+        const md = new core_1.Metadata();
+        let dupeRun = await md.GetEntityObject('Duplicate Runs');
+        dupeRun.ContextCurrentUser = super.CurrentUser;
+        const success = await dupeRun.Load(DupeRunID);
+        if (!success) {
+            throw new Error(`Failed to load Duplicate Run record ${DupeRunID}`);
+        }
+        return dupeRun;
+    }
+    async getDuplicateRunEntityByListID(listID) {
+        const entity = await super.runViewForSingleValue('Duplicate Runs', `SourceListID = ${listID}`);
+        if (!entity) {
+            throw new Error(`Failed to load Duplicate Run record for List ${listID}`);
+        }
+        return entity;
+    }
+    async createListForDupeRun(entityDocument) {
+        const md = new core_1.Metadata();
+        const list = await md.GetEntityObject('Lists');
+        list.NewRecord();
+        list.Name = `Potential Duplicate Run`;
+        list.Description = `Potential Duplicate Run for ${entityDocument.Entity} Entity`;
+        list.EntityID = entityDocument.EntityID;
+        list.UserID = super.CurrentUser.ID;
+        const saveResult = await super.SaveEntity(list);
+        if (!saveResult) {
+            throw new Error(`Failed to save list for Potential Duplicate Run`);
+        }
+        return list;
+    }
+    async createDuplicateRunDetailMatchesForRecord(DuplicateRunDetailID, duplicateResult) {
+        const md = new core_1.Metadata();
+        let matchRecords = [];
+        for (const dupe of duplicateResult.Duplicates) {
+            const match = await md.GetEntityObject('Duplicate Run Detail Matches');
+            match.NewRecord();
+            match.DuplicateRunDetailID = DuplicateRunDetailID;
+            match.MatchRecordID = dupe.ToString();
+            match.MatchProbability = dupe.ProbabilityScore;
+            match.MatchedAt = new Date();
+            match.MergedAt = new Date();
+            match.Action = '';
+            match.ApprovalStatus = 'Pending';
+            match.MergeStatus = 'Pending';
+            let success = await super.SaveEntity(match);
+            if (success) {
+                matchRecords.push(match);
+            }
+        }
+        return matchRecords;
+    }
+    async mergeRecords(dupeResponse, entityDocument) {
+        const md = new core_1.Metadata();
+        for (const dupeResult of dupeResponse.PotentialDuplicateResult) {
+            for (const [index, dupe] of dupeResult.Duplicates.entries()) {
+                if (dupe.ToString() === dupeResult.RecordCompositeKey.ToString()) {
+                    continue;
+                }
+                if (dupe.ProbabilityScore >= entityDocument.AbsoluteMatchThreshold) {
+                    let mergeParams = new core_1.RecordMergeRequest();
+                    mergeParams.EntityName = entityDocument.Entity;
+                    mergeParams.SurvivingRecordCompositeKey = dupeResult.RecordCompositeKey;
+                    mergeParams.RecordsToMerge = [dupe];
+                    let result = await md.MergeRecords(mergeParams, super.CurrentUser);
+                    if (result.Success) {
+                        let dupeRunMatchRecord = await md.GetEntityObject('Duplicate Run Detail Matches', super.CurrentUser);
+                        let loadResult = await dupeRunMatchRecord.Load(dupeResult.DuplicateRunDetailMatchRecordIDs[index]);
+                        if (!loadResult) {
+                            (0, core_1.LogError)(`Failed to load Duplicate Run Match record ${dupeResult.DuplicateRunDetailMatchRecordIDs[index]}`);
+                            continue;
+                        }
+                        dupeRunMatchRecord.MergeStatus = 'Complete';
+                        dupeRunMatchRecord.Action = 'Merged';
+                        dupeRunMatchRecord.MergedAt = new Date();
+                        let saveResult = await dupeRunMatchRecord.Save();
+                        if (!saveResult) {
+                            (0, core_1.LogError)(`Failed to update Duplicate Run Match record ${dupeRunMatchRecord.ID}`);
+                        }
+                    }
+                    else {
+                        (0, core_1.LogError)(`Failed to merge records ${dupeResult.RecordCompositeKey.ToString()} and ${dupe.ToString()}`);
+                    }
+                }
+            }
+        }
     }
 }
 exports.DuplicateRecordDetector = DuplicateRecordDetector;

package/dist/duplicateRecordDetector.js.map

@@ -1 +1 @@
-{"version":3,"file":"duplicateRecordDetector.js","sourceRoot":"","sources":["../src/duplicateRecordDetector.ts"],"names":[],"mappings":";;;AAAA,2CAA6D;AAC7D,+CAA6M;AAE7M,+CAAiD;AACjD,6DAA2D;AAC3D,mDAAkD;AAElD,uDAAoD;AAEpD,MAAa,uBAAuB;IAEhC,YAAY,CAAW;IACvB,QAAQ,CAAU;IAClB,SAAS,CAAe;IACxB,UAAU,CAAY;IAMZ,UAAU;QAChB,MAAM,KAAK,GAAG,mBAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,CAA8E,CAAA;QAC3I,OAAO,KAAK,CAAC;IACjB,CAAC;IAMS,iBAAiB;QACvB,IAAG,mBAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAC,CAAC;YACpC,OAAO,mBAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;aACG,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACjD,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,MAAiC,EAAE,WAAsB;QAEtF,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAEhC,IAAG,CAAC,IAAI,CAAC,YAAY,EAAC,CAAC;YACnB,IAAA,eAAQ,EAAC,uCAAuC,CAAC,CAAC;YAClD,MAAM,EAAE,GAAG,IAAI,eAAQ,EAAE,CAAC;YAC1B,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,WAAW,CAAC;QACvC,CAAC;QAED,IAAG,CAAC,IAAI,CAAC,QAAQ,EAAC,CAAC;YACf,IAAI,CAAC,QAAQ,GAAG,IAAI,cAAO,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,OAAO,GAAkB,IAAI,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAyB,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAChE,MAAM,eAAe,GAAW,IAAA,gBAAW,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACjE,MAAM,cAAc,GAAW,IAAA,gBAAW,EAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAE9D,IAAA,gBAAS,EAAC,qBAAqB,EAAE,eAAe,CAAC,CAAC;QAClD,IAAA,gBAAS,EAAC,2BAA2B,EAAE,cAAc,CAAC,CAAC;QAEvD,IAAI,CAAC,UAAU,GAAG,iBAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAa,eAAU,EAAE,OAAO,CAAC,WAAW,EAAE,eAAe,CAAC,CAAA;QAC7H,IAAI,CAAC,SAAS,GAAG,iBAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAe,0BAAY,EAAE,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QAI9H,MAAM,cAAc,GAAmB,MAAM,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC9F,IAAA,gBAAS,EAAC,mBAAmB,GAAG,cAAc,CAAC,CAAC;QAEhD,IAAG,CAAC,cAAc,EAAC,CAAC;YAChB,IAAA,gBAAS,EAAC,2BAA2B,CAAC,CAAC;YACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;QACjE,CAAC;QAGD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAE9F,IAAG,CAAC,YAAY,EAAC,CAAC;YACd,IAAA,gBAAS,EAAC,yBAAyB,CAAC,CAAC;YACrC,OAAO,EAAE,QAAQ,EAAE,cAAc,CAAC,QAAQ,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;QACjE,CAAC;QAGD,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,cAAc,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACjF,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;QAE9F,MAAM,MAAM,GAAa,SAAS,CAAC,IAAI,CAAC;QAGxC,MAAM,IAAI,GAAW,EAAE,CAAC;QACxB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/H,OAAO,IAAI,CAAC,uBAAuB,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAClF,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,EAAU;QACvC,MAAM,QAAQ,GAAkB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;YACxD,UAAU,EAAE,kBAAkB;YAC9B,WAAW,EAAE,SAAS,EAAE,GAAG;SAC9B,EAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAErB,IAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAC,CAAC;YAErC,OAAO,IAAI,CAAC;QAChB,CAAC;aACG,CAAC;YACD,MAAM,SAAS,GAAG,QAAQ,EAAE,OAAgB,CAAC;YAC7C,MAAM,cAAc,GAAmB,SAAS,CAAC,CAAC,CAAC,CAAC;YACpD,OAAO,cAAc,CAAC;QAC1B,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,UAAkB,EAAE,SAA4B;QAC1E,MAAM,QAAQ,GAAkB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;YACxD,UAAU,EAAE,UAAU;YACtB,WAAW,EAAE,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC;SAChD,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAEtB,IAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAC,CAAC;YAErC,OAAO,IAAI,CAAC;QAChB,CAAC;aACG,CAAC;YACD,MAAM,SAAS,GAAG,QAAQ,EAAE,OAAgB,CAAC;YAC7C,MAAM,cAAc,GAAmB,SAAS,CAAC,CAAC,CAAC,CAAC;YACpD,OAAO,cAAc,CAAC;QAC1B,CAAC;IACL,CAAC;IAEO,gBAAgB,CAAC,SAA4B;QACjD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;YACrC,OAAO,QAAQ,CAAC,SAAS,GAAG,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;QACvD,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAGO,uBAAuB,CAAC,WAAgB,EAAE,cAA8B;QAC5E,IAAI,OAAO,GAA+B;YACtC,QAAQ,EAAE,cAAc,CAAC,QAAQ;YACjC,UAAU,EAAE,EAAE;SACjB,CAAC;QAEF,KAAI,IAAI,SAAS,IAAI,WAAW,EAAC,CAAC;YAE9B,IAAG,SAAS,CAAC,KAAK,GAAG,CAAC,EAAC,CAAC;gBAEpB,SAAS;YACb,CAAC;YAED,IAAI,kBAAkB,GAAuB,IAAI,yBAAkB,EAAE,CAAC;YACtE,kBAAkB,CAAC,gBAAgB,GAAG,EAAE,CAAC;YACzC,kBAAkB,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACrC,SAAS,EAAE,IAAI;gBACf,KAAK,EAAE,SAAS,CAAC,EAAE;aACtB,CAAC,CAAC;YACH,kBAAkB,CAAC,gBAAgB,GAAG,SAAS,CAAC,KAAK,CAAC;YACtD,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAES,mBAAmB,CAAC,GAAW,EAAE,GAAQ;QAE/C,IAAI,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAGjD,IAAI,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAI5C,IAAI,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC3G,OAAO,MAAM,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,GAAG,UAAU,CAAC,CAAC;IACrD,CAAC;CACJ;AApKD,0DAoKC"}
+{"version":3,"file":"duplicateRecordDetector.js","sourceRoot":"","sources":["../src/duplicateRecordDetector.ts"],"names":[],"mappings":";;;AAAA,2CAA6D;AAC7D,+CAAyN;AACzN,+CAAiD;AACjD,6DAA2D;AAC3D,mDAAkD;AAElD,2DAAwD;AACxD,mEAAuJ;AAEvJ,MAAa,uBAAwB,SAAQ,uBAAU;IAEnD,SAAS,CAAe;IACxB,UAAU,CAAa;IAEvB;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,IAAI,cAAO,EAAE,CAAC;IAClC,CAAC;IAEM,KAAK,CAAC,mBAAmB,CAAC,MAAiC,EAAE,WAAsB;QACtF,KAAK,CAAC,WAAW,GAAG,WAAW,CAAC;QAMhC,IAAI,UAAU,GAAG,IAAI,mCAAkB,EAAE,CAAC;QAC1C,UAAU,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;QAE3C,IAAI,cAAc,GAAgC,IAAI,CAAC;QACvD,IAAG,MAAM,CAAC,gBAAgB,EAAC,CAAC;YACxB,cAAc,GAAG,MAAM,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACjF,CAAC;aACI,CAAC;YACF,cAAc,GAAG,MAAM,UAAU,CAAC,qCAAqC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzF,IAAG,CAAC,cAAc,EAAC,CAAC;gBAChB,MAAM,KAAK,CAAC,8CAA8C,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;YAOjF,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,GAA+B,IAAI,iCAA0B,EAAE,CAAC;QAC5E,QAAQ,CAAC,wBAAwB,GAAG,EAAE,CAAC;QAEvC,IAAG,CAAC,cAAc,EAAC,CAAC;YAChB,QAAQ,CAAC,YAAY,GAAG,8CAA8C,MAAM,CAAC,QAAQ,EAAE,CAAC;YACxF,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC;YAC1B,OAAO,QAAQ,CAAC;QACpB,CAAC;QAcD,MAAM,IAAI,GAAe,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjE,IAAI,YAAY,GAAuB,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAE7L,MAAM,mBAAmB,GAA+B,MAAM,IAAI,CAAC,uCAAuC,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;QAGrI,IAAA,gBAAS,EAAC,yBAAyB,cAAc,CAAC,gBAAgB,iBAAiB,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC;QAE/G,MAAM,QAAQ,GAAyB,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAC/F,MAAM,OAAO,GAAkB,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAEzE,IAAA,gBAAS,EAAC,yBAAyB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1D,IAAA,gBAAS,EAAC,uBAAuB,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEtD,MAAM,eAAe,GAAW,IAAA,gBAAW,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACjE,MAAM,cAAc,GAAW,IAAA,gBAAW,EAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAE9D,IAAG,CAAC,eAAe,EAAC,CAAC;YACjB,MAAM,KAAK,CAAC,iCAAiC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,IAAG,CAAC,cAAc,EAAC,CAAC;YAChB,MAAM,KAAK,CAAC,wCAAwC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC7E,CAAC;QAGD,IAAI,CAAC,UAAU,GAAG,iBAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAa,eAAU,EAAE,OAAO,CAAC,WAAW,EAAE,eAAe,CAAC,CAAA;QAC7H,IAAI,CAAC,SAAS,GAAG,iBAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAe,0BAAY,EAAE,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QAE9H,IAAG,CAAC,IAAI,CAAC,UAAU,EAAC,CAAC;YACjB,MAAM,KAAK,CAAC,qDAAqD,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5F,CAAC;QAED,IAAG,CAAC,IAAI,CAAC,SAAS,EAAC,CAAC;YAChB,MAAM,KAAK,CAAC,iDAAiD,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;QAE9E,IAAA,gBAAS,EAAC,cAAc,GAAG,OAAO,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;QACxD,MAAM,cAAc,GAAqC,6CAA4B,CAAC,cAAc,EAAE,CAAC;QACvG,MAAM,eAAe,GAAa,EAAE,CAAC;QAErC,IAAI,cAAc,GAAW,cAAc,CAAC,QAAQ,CAAC;QAErD,KAAI,MAAM,MAAM,IAAI,OAAO,EAAC,CAAC;YACzB,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,cAAc,EAAE,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;YAC1G,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;QAED,IAAI,gBAAgB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjG,MAAM,IAAI,GAAW,CAAC,CAAC;QACvB,IAAI,OAAO,GAA+B,EAAE,CAAC;QAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,gBAAgB,CAAC,OAAO,CAAC,OAAO,EAAE,EAAC,CAAC;YAC9D,MAAM,YAAY,GAAiB,OAAO,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC;YAC/D,IAAI,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;YACxI,IAAG,WAAW,CAAC,OAAO,EAAC,CAAC;gBACpB,WAAW,CAAC,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;oBACtE,OAAO,IAAI,CAAC,gBAAgB,IAAI,cAAc,CAAC,uBAAuB,CAAC;gBAC3E,CAAC,CAAC,CAAC;gBACH,IAAI,MAAM,GAA6B,WAAW,CAAC,IAAgC,CAAC;gBAEpF,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC;gBAC1C,MAAM,CAAC,kBAAkB,GAAG,YAAY,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAGrB,IAAI,aAAa,GAA6B,mBAAmB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,KAAK,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAChI,IAAG,aAAa,EAAC,CAAC;oBACd,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,wCAAwC,CAAC,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBACnG,MAAM,CAAC,gCAAgC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAChF,aAAa,CAAC,WAAW,GAAG,UAAU,CAAC;oBACvC,IAAI,OAAO,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;oBACpD,IAAG,CAAC,OAAO,EAAC,CAAC;wBACT,IAAA,gBAAS,EAAC,gDAAgD,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;oBAClF,CAAC;gBACL,CAAC;qBACG,CAAC;oBACD,IAAA,eAAQ,EAAC,kDAAkD,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC1F,CAAC;YACL,CAAC;QACL,CAAC;QAGD,YAAY,CAAC,gBAAgB,GAAG,UAAU,CAAC;QAC3C,YAAY,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAClC,IAAI,OAAO,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QACnD,IAAG,CAAC,OAAO,EAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,yCAAyC,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QAElD,QAAQ,CAAC,wBAAwB,GAAG,OAAO,CAAC;QAC5C,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC;QAE5B,IAAA,gBAAS,EAAC,8BAA8B,CAAC,CAAC;QAC1C,IAAA,gBAAS,EAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAEhD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,MAAc,EAAE,QAAgB;QAC7D,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,qBAAqB,CAAe,UAAU,EAAE,QAAQ,QAAQ,EAAE,CAAC,CAAC;QAC/F,IAAG,CAAC,MAAM,EAAC,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,EAAE,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YACzC,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,WAAW,EAAE,iEAAiE,MAAM,GAAG;YACvF,UAAU,EAAE,eAAe;SAC9B,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;QAEtB,IAAG,CAAC,QAAQ,CAAC,OAAO,EAAC,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC3C,CAAC;QAED,MAAM,OAAO,GAAiB,QAAQ,CAAC,OAAuB,CAAC;QAC/D,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,wBAAwB,CAAC,cAAoC,EAAE,MAAc;QACvF,MAAM,EAAE,GAAa,IAAI,eAAQ,EAAE,CAAC;QACpC,IAAI,YAAY,GAAuB,MAAM,EAAE,CAAC,eAAe,CAAqB,gBAAgB,CAAC,CAAC;QACtG,YAAY,CAAC,SAAS,EAAE,CAAC;QACzB,YAAY,CAAC,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC;QAChD,YAAY,CAAC,eAAe,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;QACpD,YAAY,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QACpC,YAAY,CAAC,gBAAgB,GAAG,aAAa,CAAC;QAC9C,YAAY,CAAC,cAAc,GAAG,SAAS,CAAC;QACxC,YAAY,CAAC,YAAY,GAAG,MAAM,CAAC;QAEnC,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QACxD,IAAG,CAAC,UAAU,EAAC,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,YAAY,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,+BAA+B,CAAC,SAAyB,EAAE,cAAsB;QAC3F,IAAI,OAAO,GAA+B,EAAE,CAAC;QAC7C,MAAM,EAAE,GAAa,IAAI,eAAQ,EAAE,CAAC;QACpC,KAAI,MAAM,QAAQ,IAAI,SAAS,EAAC,CAAC;YAC7B,IAAI,SAAS,GAA6B,MAAM,EAAE,CAAC,eAAe,CAA2B,uBAAuB,CAAC,CAAC;YACtH,SAAS,CAAC,SAAS,EAAE,CAAC;YACtB,SAAS,CAAC,cAAc,GAAG,cAAc,CAAC;YAC1C,SAAS,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACzC,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC;YAClC,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAClD,IAAG,OAAO,EAAC,CAAC;gBACR,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,uCAAuC,CAAC,MAAc,EAAE,cAAsB;QACxF,IAAI,OAAO,GAA+B,EAAE,CAAC;QAC7C,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAC5C,UAAU,EAAE,cAAc;YAC1B,WAAW,EAAE,YAAY,MAAM,EAAE;YACjC,UAAU,EAAE,eAAe;SAAC,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;QAErD,IAAG,CAAC,WAAW,CAAC,OAAO,EAAC,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAC9C,CAAC;QAGD,MAAM,EAAE,GAAa,IAAI,eAAQ,EAAE,CAAC;QACpC,MAAM,WAAW,GAAuB,WAAW,CAAC,OAA6B,CAAC;QAClF,KAAI,MAAM,UAAU,IAAI,WAAW,EAAC,CAAC;YACjC,IAAI,SAAS,GAA6B,MAAM,EAAE,CAAC,eAAe,CAA2B,uBAAuB,CAAC,CAAC;YACtH,SAAS,CAAC,SAAS,EAAE,CAAC;YACtB,SAAS,CAAC,cAAc,GAAG,cAAc,CAAC;YAC1C,SAAS,CAAC,QAAQ,GAAG,QAAQ,UAAU,CAAC,QAAQ,EAAE,CAAC;YACnD,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC;YAClC,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;YAClD,IAAG,OAAO,EAAC,CAAC;gBACR,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC5B,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,MAAc;QACtC,MAAM,EAAE,GAAa,IAAI,eAAQ,EAAE,CAAC;QACpC,IAAI,IAAI,GAAe,MAAM,EAAE,CAAC,eAAe,CAAa,OAAO,CAAC,CAAC;QACrE,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,WAAW,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxC,IAAG,CAAC,OAAO,EAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,SAAiB;QACjD,MAAM,EAAE,GAAa,IAAI,eAAQ,EAAE,CAAC;QACpC,IAAI,OAAO,GAAuB,MAAM,EAAE,CAAC,eAAe,CAAqB,gBAAgB,CAAC,CAAC;QACjG,OAAO,CAAC,kBAAkB,GAAG,KAAK,CAAC,WAAW,CAAC;QAC/C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAG,CAAC,OAAO,EAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,uCAAuC,SAAS,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,MAAc;QACtD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,qBAAqB,CAAqB,gBAAgB,EAAE,kBAAkB,MAAM,EAAE,CAAC,CAAC;QACnH,IAAG,CAAC,MAAM,EAAC,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,gDAAgD,MAAM,EAAE,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,cAAoC;QACnE,MAAM,EAAE,GAAa,IAAI,eAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,GAAe,MAAM,EAAE,CAAC,eAAe,CAAa,OAAO,CAAC,CAAC;QACvE,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,+BAA+B,cAAc,CAAC,MAAM,SAAS,CAAC;QACjF,IAAI,CAAC,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC;QACxC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;QAEnC,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAChD,IAAG,CAAC,UAAU,EAAC,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,wCAAwC,CAAC,oBAA4B,EAAE,eAAyC;QAC1H,MAAM,EAAE,GAAa,IAAI,eAAQ,EAAE,CAAC;QACpC,IAAI,YAAY,GAAoC,EAAE,CAAC;QACvD,KAAI,MAAM,IAAI,IAAI,eAAe,CAAC,UAAU,EAAC,CAAC;YAC1C,MAAM,KAAK,GAAkC,MAAM,EAAE,CAAC,eAAe,CAAgC,8BAA8B,CAAC,CAAC;YACrI,KAAK,CAAC,SAAS,EAAE,CAAC;YAClB,KAAK,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;YAClD,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtC,KAAK,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;YAC/C,KAAK,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;YAC7B,KAAK,CAAC,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;YAC5B,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC;YAClB,KAAK,CAAC,cAAc,GAAG,SAAS,CAAC;YACjC,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC;YAC9B,IAAI,OAAO,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC5C,IAAG,OAAO,EAAC,CAAC;gBACR,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACL,CAAC;QAED,OAAO,YAAY,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,YAAwC,EAAE,cAAoC;QACrG,MAAM,EAAE,GAAa,IAAI,eAAQ,EAAE,CAAC;QACpC,KAAI,MAAM,UAAU,IAAI,YAAY,CAAC,wBAAwB,EAAC,CAAC;YAC3D,KAAI,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,OAAO,EAAE,EAAC,CAAC;gBACxD,IAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,UAAU,CAAC,kBAAkB,CAAC,QAAQ,EAAE,EAAC,CAAC;oBAE7D,SAAS;gBACb,CAAC;gBAED,IAAG,IAAI,CAAC,gBAAgB,IAAI,cAAc,CAAC,sBAAsB,EAAC,CAAC;oBAE/D,IAAI,WAAW,GAAuB,IAAI,yBAAkB,EAAE,CAAC;oBAC/D,WAAW,CAAC,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC;oBAC/C,WAAW,CAAC,2BAA2B,GAAG,UAAU,CAAC,kBAAkB,CAAC;oBACxE,WAAW,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,CAAC;oBACpC,IAAI,MAAM,GAAG,MAAM,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;oBACnE,IAAG,MAAM,CAAC,OAAO,EAAC,CAAC;wBACf,IAAI,kBAAkB,GAAkC,MAAM,EAAE,CAAC,eAAe,CAAgC,8BAA8B,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;wBACnK,IAAI,UAAU,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,gCAAgC,CAAC,KAAK,CAAC,CAAC,CAAC;wBACnG,IAAG,CAAC,UAAU,EAAC,CAAC;4BACZ,IAAA,eAAQ,EAAC,6CAA6C,UAAU,CAAC,gCAAgC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;4BAC5G,SAAS;wBACb,CAAC;wBAED,kBAAkB,CAAC,WAAW,GAAG,UAAU,CAAC;wBAC5C,kBAAkB,CAAC,MAAM,GAAG,QAAQ,CAAC;wBACrC,kBAAkB,CAAC,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;wBAEzC,IAAI,UAAU,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,CAAC;wBACjD,IAAG,CAAC,UAAU,EAAC,CAAC;4BACZ,IAAA,eAAQ,EAAC,+CAA+C,kBAAkB,CAAC,EAAE,EAAE,CAAC,CAAC;wBACrF,CAAC;oBACL,CAAC;yBACG,CAAC;wBACD,IAAA,eAAQ,EAAC,2BAA2B,UAAU,CAAC,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAC3G,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;CACJ;AAzWD,0DAyWC"}

package/dist/generic/entity.types.d.ts

@@ -20,5 +20,6 @@ export type EntitiyRecordDocument = {
     VectorID: number;
     VectorJSON: string;
     EntitiyRecordUpdatedAt: Date;
+    EntityRecordDocumentID: number;
 };
 //# sourceMappingURL=entity.types.d.ts.map

package/dist/generic/entity.types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"entity.types.d.ts","sourceRoot":"","sources":["../../src/generic/entity.types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,cAAc,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB,EAAE,IAAI,CAAA;CAC/B,CAAA"}
+{"version":3,"file":"entity.types.d.ts","sourceRoot":"","sources":["../../src/generic/entity.types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,cAAc,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,qBAAqB,GAAG;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,sBAAsB,EAAE,IAAI,CAAC;IAC7B,sBAAsB,EAAE,MAAM,CAAA;CACjC,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memberjunction/ai-vector-dupe",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "description": "MemberJunction: AI Vector/Entity Sync Package - Handles synchronization between Vector DB and MJ CDP Data",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -15,14 +15,14 @@
   "author": "MemberJunction.com",
   "license": "ISC",
   "dependencies": {
-    "@memberjunction/ai": "~1.2.1",
-    "@memberjunction/ai-vectordb": "~1.2.1",
-    "@memberjunction/ai-vectors": "~1.2.1",
-    "@memberjunction/ai-vectors-pinecone": "~1.2.1",
-    "@memberjunction/aiengine": "~1.2.1",
-    "@memberjunction/core": "~1.2.1",
-    "@memberjunction/global": "~1.2.1",
-    "@memberjunction/core-entities": "~1.2.1",
+    "@memberjunction/ai": "~1.3.0",
+    "@memberjunction/ai-vectordb": "~1.3.0",
+    "@memberjunction/ai-vectors": "~1.3.0",
+    "@memberjunction/ai-vector-sync": "~1.3.0",
+    "@memberjunction/aiengine": "~1.3.0",
+    "@memberjunction/core": "~1.3.0",
+    "@memberjunction/global": "~1.3.0",
+    "@memberjunction/core-entities": "~1.3.0",
     "dotenv": "^16.0.3",
     "typeorm": "^0.3.20"
   },