@memberjunction/ai-vector-dupe 6.2.0-edge.0 → 6.2.0-edge.2

package/dist/duplicateRecordDetector.js

@@ -14,17 +14,21 @@
  *
  * @module @memberjunction/ai-vector-dupe
  */
-import { BaseEmbeddings, GetAIAPIKey } from "@memberjunction/ai";
+import { GetAIAPIKey } from "@memberjunction/ai";
+import { AIEmbeddingRunner } from "@memberjunction/ai-prompts";
 import { PotentialDuplicateResponse, CompositeKey, PotentialDuplicateResult, LogError, LogStatus, RecordMergeRequest, PotentialDuplicate, } from "@memberjunction/core";
 import { VectorDBBase } from "@memberjunction/ai-vectordb";
 import { MJGlobal, UUIDsEqual, NormalizeUUID } from "@memberjunction/global";
 import { KnowledgeHubMetadataEngine, } from "@memberjunction/core-entities";
 import { VectorBase } from "@memberjunction/ai-vectors";
 import { EntityVectorSyncer } from "@memberjunction/ai-vector-sync";
+import { AIEngine } from "@memberjunction/aiengine";
 import { EntityDocumentTemplateParser } from "@memberjunction/entity-documents";
 import { TemplateEngineServer } from "@memberjunction/templates";
-import { DuplicateReasoningProvider } from "./reasoning/DuplicateReasoningProvider.js";
+import { DuplicateReasoningProvider, DECISION_REASONING_PROVIDER_KEY, DECISION_THEN_PROMPT_REASONING_PROVIDER_KEY, } from "./reasoning/DuplicateReasoningProvider.js";
 import { MatchedSetDeltaBuilder } from "./reasoning/MatchedSetDeltaBuilder.js";
+import { DUPLICATE_ENTRY_CHECK_MAX_DECISION_FIELDS, DUPLICATE_ENTRY_CHECK_MAX_FIELD_TEXT_LENGTH, DUPLICATE_ENTRY_CHECK_SERVER_BUDGET_MS, } from "./duplicateEntryCheckTypes.js";
+import { EntryCheckDeadline, EntryCheckStoppedError } from "./entryCheckDeadline.js";
 /** Default number of nearest neighbors to retrieve per record */
 const DEFAULT_TOP_K = 5;
 /** Default concurrency limit for parallel vector queries */
@@ -38,6 +42,14 @@ const DEFAULT_BATCH_SIZE = 500;
 const VECTOR_QUERY_BATCH_SIZE = 100;
 /** Default batch size for parallel database saves */
 const SAVE_BATCH_SIZE = 20;
+/**
+ * The entity-document ReasoningModes that turn on the entry-time check. Both carry a typed
+ * decision, which is fast enough to answer while a person types.
+ */
+const ENTRY_CHECK_REASONING_MODES = new Set([
+    DECISION_REASONING_PROVIDER_KEY,
+    DECISION_THEN_PROMPT_REASONING_PROVIDER_KEY,
+]);
 /**
  * Modernized duplicate record detection engine.
  *
@@ -45,6 +57,7 @@ const SAVE_BATCH_SIZE = 20;
  * - List-based batch detection (getDuplicateRecords)
  * - View/filter/full-entity batch detection (vector-first approach)
  * - Single-record duplicate check (CheckSingleRecord)
+ * - Entry-time check of a new record's unsaved values, flag only (CheckRecordValues)
  * - Hybrid search via RRF when vector DB supports it
  * - Optional post-retrieval reranking via MJ's BaseReranker
  * - Configurable topK, thresholds, and progress reporting
@@ -52,10 +65,11 @@ const SAVE_BATCH_SIZE = 20;
 export class DuplicateRecordDetector extends VectorBase {
     constructor() {
         super(...arguments);
+        this._embeddingRunner = null;
+        this.embeddingModelID = null;
         /**
          * The embedding model's API identifier (e.g. 'Xenova/gte-small', 'text-embedding-3-small'),
-         * resolved from the entity document's AIModel. Passed to `EmbedTexts` — required by local
-         * providers (which load the named ONNX pipeline) and honored by cloud providers.
+         * resolved from the entity document's AIModel.
          */
         this.embeddingModelAPIName = null;
         /**
@@ -64,6 +78,18 @@ export class DuplicateRecordDetector extends VectorBase {
          */
         this._seenPairs = new Set();
     }
+    /**
+     * Protected getter/setter for the AIEmbeddingRunner instance used by duplicate detection.
+     */
+    get EmbeddingRunner() {
+        if (!this._embeddingRunner) {
+            this._embeddingRunner = new AIEmbeddingRunner();
+        }
+        return this._embeddingRunner;
+    }
+    set EmbeddingRunner(value) {
+        this._embeddingRunner = value;
+    }
     /**
      * Run duplicate detection for records identified by ListID, ViewID, ExtraFilter,
      * or all records in the entity (vector-first approach).
@@ -154,7 +180,7 @@ export class DuplicateRecordDetector extends VectorBase {
                 return response;
             }
             const batchIDs = recordIDs.slice(offset, offset + batchSize);
-            const batchResults = await this.ProcessBatch(batchIDs, entityInfo, entityDocument, templateParser, duplicateRun.ID, topK, concurrency, options, startTime, recordIDs.length, offset, totalMatchesFound, contextUser);
+            const batchResults = await this.processBatch(batchIDs, entityInfo, entityDocument, templateParser, duplicateRun.ID, topK, concurrency, options, startTime, recordIDs.length, offset, totalMatchesFound, contextUser);
             response.PotentialDuplicateResult.push(...batchResults.Results);
             totalMatchesFound += batchResults.MatchesFound;
             // Update cursor for resume support
@@ -202,25 +228,317 @@ export class DuplicateRecordDetector extends VectorBase {
             throw new Error(`Record not found: ${RecordID.ToString()}`);
         }
         const record = records.Results[0];
-        const templateParser = EntityDocumentTemplateParser.CreateInstance();
-        const templateTexts = await this.GenerateTemplateTexts(templateParser, entityDocument, [record], ContextUser);
-        const embedResult = await this.embedding.EmbedTexts({ texts: templateTexts, model: this.embeddingModelAPIName });
-        const topK = options.TopK ?? DEFAULT_TOP_K;
-        const queryResults = await this.QueryDuplicatesForRecords([record], embedResult.vectors, templateTexts, entityDocument, topK, options, this.GetQueryConcurrency(entityDocument));
-        if (queryResults.length === 0) {
+        const single = await this.QueryCandidatesForRecord(record, entityDocument, options, ContextUser);
+        if (!single) {
             return new PotentialDuplicateResult();
         }
         // Annotate the (non-persisted) single-record result with LLM reasoning when the entity
         // has it enabled and the set clears the gate — same gate as the batch path. There are no
         // match rows to stamp run ids onto here; the verdict rides on the returned result so a
         // real-time "is this a duplicate?" caller gets recommendation + resolved field map too.
-        const single = queryResults[0];
         const reasoning = await this.RunReasoningForSet(single, entityInfo, entityDocument, ContextUser);
         if (reasoning) {
             this.applyReasoningToResult(single.Duplicates, reasoning.Output, reasoning.FieldMap);
         }
         return single.Duplicates;
     }
+    /**
+     * Check values a person is entering for a new record against the entity's existing records, and
+     * flag the plausible duplicates. It only flags: nothing is saved, merged or blocked.
+     *
+     * 1. **The switch.** The check runs only for an entity with an Active entity document that has
+     *    `EnableLLMReasoning` on and whose `ReasoningMode` is `'Decision'` or `'DecisionThenPrompt'`
+     *    (see {@link FindEntryCheckDocument}). Without one the result is `NotConfigured` and nothing
+     *    else runs.
+     * 2. **Candidates.** The values, each text cut to {@link DUPLICATE_ENTRY_CHECK_MAX_FIELD_TEXT_LENGTH},
+     *    become an unsaved entity object, rendered by the document's template, embedded, and queried
+     *    with the same TopK and threshold as {@link CheckSingleRecord}.
+     * 3. **Permission.** The vector index does not know who is asking, so the candidates are narrowed
+     *    to those the context user can read, with one RunView as that user.
+     * 4. **Decision.** One {@link DecisionReasoningProvider.DecideCandidates} call over the readable
+     *    candidates, in either mode: the prompt half of `'DecisionThenPrompt'` is too slow for entry,
+     *    and it can recommend a merge. Its state is bounded by
+     *    {@link DUPLICATE_ENTRY_CHECK_MAX_DECISION_FIELDS} and the field-text limit. A candidate is
+     *    flagged when the provider bands it `Uncertain`, by its own threshold, on the probability
+     *    calibrated for the model that answered. A candidate the decision gave no answer for is
+     *    flagged. A failed decision flags nothing, and neither does a decision from a model with no
+     *    calibration: its probabilities can't be banded, and flagging every candidate would be the
+     *    vector threshold alone. Both give a `Failed` result with the reason.
+     *
+     * **Budget.** The whole check is bounded by `options.TimeoutMS`
+     * ({@link DUPLICATE_ENTRY_CHECK_SERVER_BUDGET_MS} by default) and by `options.CancellationToken`.
+     * No step starts once either has fired, the check stops waiting for a running step, and the
+     * decision call receives both, so the model call is aborted. A stopped check is `Failed`.
+     *
+     * Never throws: any failure is a `Failed` result with the reason.
+     *
+     * @param entityName the entity the new record belongs to
+     * @param values the entered values by field name; fields the entity does not have are ignored
+     * @param contextUser the person entering the record. Required: the candidates are narrowed to the
+     *   records this user can read, and without a user the check fails rather than skip that step.
+     * @param options overrides for TopK and the potential-match threshold, the budget and cancellation
+     * @returns the flagged candidates, most probable first, or why there are none
+     */
+    async CheckRecordValues(entityName, values, contextUser, options) {
+        const startTime = Date.now();
+        if (!contextUser) {
+            return this.entryCheckResult('Failed', startTime, [], 'A context user is required: the check shows only the records that user can read');
+        }
+        this.CurrentUser = contextUser;
+        const deadline = new EntryCheckDeadline(options?.TimeoutMS ?? DUPLICATE_ENTRY_CHECK_SERVER_BUDGET_MS, options?.CancellationToken);
+        try {
+            return await this.runEntryCheck(entityName, values, options ?? {}, contextUser, deadline, startTime);
+        }
+        catch (e) {
+            if (e instanceof EntryCheckStoppedError) {
+                // Running out of the budget, or being cancelled, is how an abandoned check ends: not a fault.
+                return this.entryCheckResult('Failed', startTime, [], e.message);
+            }
+            const message = e instanceof Error ? e.message : String(e);
+            LogError(`Duplicate entry check for ${entityName} failed: ${message}`);
+            return this.entryCheckResult('Failed', startTime, [], message);
+        }
+        finally {
+            deadline.Dispose();
+        }
+    }
+    /**
+     * Render one record with the entity document's template, embed it, and query the vector index
+     * for its candidates. Shared by {@link CheckSingleRecord} (a saved record) and
+     * {@link CheckRecordValues} (an unsaved one).
+     *
+     * @param deadline bounds each step, for the entry-time check; without one the steps are unbounded
+     * @returns the record's query result, or null when the query produced none
+     */
+    async QueryCandidatesForRecord(record, entityDocument, options, contextUser, deadline) {
+        const step = (name, work) => deadline ? deadline.Run(name, work) : work();
+        const templateParser = EntityDocumentTemplateParser.CreateInstance();
+        const templateTexts = await step('rendering the template', () => this.GenerateTemplateTexts(templateParser, entityDocument, [record], contextUser));
+        const embedResult = await step('embedding the record', () => this.EmbeddingRunner.RunEmbedding({
+            Texts: templateTexts,
+            ModelID: this.embeddingModelID ?? undefined,
+            ContextUser: contextUser,
+            Description: `Duplicate detection single record (${entityDocument.Name})`
+        }));
+        if (!embedResult.Success || !embedResult.Vectors || embedResult.Vectors.length === 0) {
+            throw new Error(`Embedding failed for duplicate detection: ${embedResult.ErrorMessage ?? 'Unknown error'}`);
+        }
+        const vectors = embedResult.Vectors;
+        const topK = options.TopK ?? DEFAULT_TOP_K;
+        const queryResults = await step('querying the vector index', () => this.QueryDuplicatesForRecords([record], vectors, templateTexts, entityDocument, topK, options, this.GetQueryConcurrency(entityDocument)));
+        return queryResults[0] ?? null;
+    }
+    // ─────────────────────────────────────────────
+    // Entry-Time Check
+    // ─────────────────────────────────────────────
+    /**
+     * The entity document that turns the entry-time check on for an entity: an Active document with
+     * `EnableLLMReasoning` on whose `ReasoningMode` is `'Decision'` or `'DecisionThenPrompt'`. The
+     * check makes a decision-model call on every pause in typing, so switching reasoning off on the
+     * document turns the check off too, as it turns off reasoning in batch runs
+     * ({@link IsReasoningGateOpen}). When several qualify, the oldest wins (earliest
+     * `__mj_CreatedAt`, then lowest ID), so adding a document never silently changes which one an
+     * entity uses.
+     *
+     * @returns the document, or null when the check is not configured for the entity
+     */
+    async FindEntryCheckDocument(entityName) {
+        await KnowledgeHubMetadataEngine.Instance.Config(false, this.CurrentUser);
+        const documents = KnowledgeHubMetadataEngine.Instance.GetEntityDocumentsForEntity(entityName)
+            .filter(d => d.Status === 'Active' && d.EnableLLMReasoning && ENTRY_CHECK_REASONING_MODES.has(d.ReasoningMode));
+        return documents.sort(compareOldestFirst)[0] ?? null;
+    }
+    /**
+     * Build an unsaved entity object holding the entered values, so the entity document's template
+     * renders it as it renders a saved record. Values for fields the entity does not have are ignored,
+     * and text longer than {@link DUPLICATE_ENTRY_CHECK_MAX_FIELD_TEXT_LENGTH} is cut, so the
+     * template, the embedding and the decision state never see more of a field than that.
+     */
+    async BuildUnsavedRecord(entityInfo, values, contextUser) {
+        const record = await this.Metadata.GetEntityObject(entityInfo.Name, contextUser);
+        record.NewRecord();
+        record.SetMany(boundEnteredValues(values), true);
+        return record;
+    }
+    /**
+     * The display names of the candidates the context user can read, from one RunView over the
+     * candidate keys run as that user, so entity permissions and row-level security apply. A
+     * candidate missing from the map must not be shown. When the RunView fails the map is empty:
+     * a candidate is shown only when the user is known to be able to read it.
+     *
+     * @param contextUser the person the candidates are narrowed for; the RunView runs as this user
+     * @returns display name by normalized compact record id ({@link NormalizeUUID})
+     */
+    async LoadReadableCandidateNames(candidates, entityInfo, contextUser) {
+        const names = new Map();
+        if (candidates.length === 0) {
+            return names;
+        }
+        const nameFields = this.DisplayNameFields(entityInfo);
+        const result = await this.RunView.RunView({
+            EntityName: entityInfo.Name,
+            ExtraFilter: candidates.map(c => `(${c.ToWhereClause()})`).join(' OR '),
+            Fields: [...new Set([...entityInfo.PrimaryKeys.map(pk => pk.Name), ...nameFields.map(f => f.Name)])],
+            ResultType: 'simple',
+            MaxRows: candidates.length,
+        }, contextUser);
+        if (!result.Success) {
+            LogError(`Duplicate entry check: could not confirm which ${entityInfo.Name} candidates are readable: ${result.ErrorMessage}`);
+            return names;
+        }
+        for (const row of result.Results) {
+            const recordID = CompositeKey.FromEntityRecord(entityInfo, row).ToCompactURLSegment();
+            names.set(NormalizeUUID(recordID), this.rowDisplayName(row, nameFields) || recordID);
+        }
+        return names;
+    }
+    /**
+     * The decision provider the entry-time check asks: whatever the class factory holds for the
+     * `'Decision'` mode, which is {@link DecisionReasoningProvider} unless an app registers a
+     * subclass. Null when that registration is not a decision provider.
+     */
+    ResolveEntryDecisionProvider() {
+        const provider = MJGlobal.Instance.ClassFactory.CreateInstance(DuplicateReasoningProvider, DECISION_REASONING_PROVIDER_KEY);
+        return isDecisionReasoningProvider(provider) ? provider : null;
+    }
+    /**
+     * The fields whose values make up a record's display name: every `IsNameField` field in
+     * Sequence order, or the entity's single `NameField` when none is flagged.
+     */
+    DisplayNameFields(entityInfo) {
+        const nameFields = entityInfo.Fields
+            .filter(f => f.IsNameField)
+            .sort((a, b) => (a.Sequence ?? 9999) - (b.Sequence ?? 9999));
+        // Fall back to singular NameField if no IsNameField flags
+        if (nameFields.length === 0 && entityInfo.NameField) {
+            nameFields.push(entityInfo.NameField);
+        }
+        return nameFields;
+    }
+    /**
+     * The steps of {@link CheckRecordValues} after it has recorded the start time, each bounded by
+     * the deadline. May throw, {@link EntryCheckStoppedError} included.
+     */
+    async runEntryCheck(entityName, values, options, contextUser, deadline, startTime) {
+        const entityDocument = await deadline.Run('finding the entity document', () => this.FindEntryCheckDocument(entityName));
+        if (!entityDocument) {
+            return this.entryCheckResult('NotConfigured', startTime);
+        }
+        const entityInfo = this.Metadata.EntityByID(entityDocument.EntityID);
+        if (!entityInfo) {
+            return this.entryCheckResult('Failed', startTime, [], `Entity not found for ID ${entityDocument.EntityID}`);
+        }
+        await deadline.Run('loading the embedding and vector providers', () => this.InitializeProviders(entityDocument));
+        const record = await deadline.Run('building the unsaved record', () => this.BuildUnsavedRecord(entityInfo, values, contextUser));
+        const query = await this.QueryCandidatesForRecord(record, entityDocument, options, contextUser, deadline);
+        if (!query) {
+            return this.entryCheckResult('Checked', startTime);
+        }
+        const displayNames = await this.keepReadableCandidates(query, entityInfo, contextUser, deadline);
+        if (query.Duplicates.Duplicates.length === 0) {
+            return this.entryCheckResult('Checked', startTime);
+        }
+        const decision = await this.decideEntryCandidates({
+            EntityInfo: entityInfo, EntityDocument: entityDocument, Record: record, Query: query,
+            DisplayNames: displayNames, ContextUser: contextUser, Deadline: deadline,
+        });
+        return 'Error' in decision
+            ? this.entryCheckResult('Failed', startTime, [], decision.Error)
+            : this.entryCheckResult('Checked', startTime, decision.Candidates);
+    }
+    /** Narrow a query result's candidates, in place, to those the context user can read. */
+    async keepReadableCandidates(query, entityInfo, contextUser, deadline) {
+        const names = await deadline.Run('checking which candidates the user can read', () => this.LoadReadableCandidateNames(query.Duplicates.Duplicates, entityInfo, contextUser));
+        query.Duplicates.Duplicates = query.Duplicates.Duplicates.filter(d => names.has(NormalizeUUID(d.ToCompactURLSegment())));
+        return names;
+    }
+    /**
+     * Ask the decision about the readable candidates, with the unsaved record's own values as the
+     * source, and flag the ones the provider bands `Uncertain`. The decision call gets the deadline's
+     * signal and remaining time, so the model call is aborted when the check must stop.
+     */
+    async decideEntryCandidates(state) {
+        const provider = this.ResolveEntryDecisionProvider();
+        if (!provider) {
+            return { Error: `No DecisionReasoningProvider is registered for the '${DECISION_REASONING_PROVIDER_KEY}' reasoning mode` };
+        }
+        const input = await state.Deadline.Run('loading the candidates\' field values', () => this.BuildReasoningInput(state.Query, state.EntityInfo, state.EntityDocument, state.ContextUser, state.Record));
+        const decision = await state.Deadline.Run('asking the decision model', () => provider.DecideCandidates(this.BoundEntryDecisionInput(input), {
+            Provider: this._provider,
+            ContextUser: state.ContextUser,
+            CancellationToken: state.Deadline.Signal,
+            TimeoutMS: state.Deadline.RemainingMS,
+        }));
+        if (!decision.Success) {
+            const message = `Decision failed: ${decision.ErrorMessage ?? 'unknown error'}`;
+            LogError(`Duplicate entry check for ${state.EntityInfo.Name}: ${message}. Nothing is flagged.`);
+            return { Error: message };
+        }
+        if (decision.UncalibratedModel) {
+            // Not logged here: the provider logs a missing calibration once per model, not per check.
+            return { Error: `The decision model "${decision.UncalibratedModel}" has no calibration, so nothing is flagged` };
+        }
+        return { Candidates: this.flagEntryCandidates(provider, decision.Candidates, state) };
+    }
+    /**
+     * The entry-time decision's input, bounded: at most {@link DUPLICATE_ENTRY_CHECK_MAX_DECISION_FIELDS}
+     * differing fields, the ones the person entered first, and every value and label cut to
+     * {@link DUPLICATE_ENTRY_CHECK_MAX_FIELD_TEXT_LENGTH}. A candidate's stored values are cut too:
+     * a long Notes field on an existing record would otherwise reach the model in full.
+     */
+    BoundEntryDecisionInput(input) {
+        const sourceID = input.SourceRecord.RecordID;
+        const entered = (delta) => delta.Values.some(v => this.recordIdMatches(v.RecordID, sourceID) && v.Value != null && v.Value.trim().length > 0);
+        const fields = [...input.FieldDeltas.filter(entered), ...input.FieldDeltas.filter(d => !entered(d))]
+            .slice(0, DUPLICATE_ENTRY_CHECK_MAX_DECISION_FIELDS)
+            .map(delta => ({
+            FieldName: delta.FieldName,
+            Values: delta.Values.map(v => ({ RecordID: v.RecordID, Value: v.Value == null ? null : truncateText(v.Value) })),
+        }));
+        return {
+            ...input,
+            SourceRecord: { ...input.SourceRecord, Label: truncateText(input.SourceRecord.Label) },
+            Candidates: input.Candidates.map(c => ({ ...c, Label: truncateText(c.Label) })),
+            FieldDeltas: fields,
+        };
+    }
+    /**
+     * The candidates the provider bands `Uncertain` (they cannot be ruled out), most probable first.
+     * The decision answers in input order, so each answer lines up with the query's candidate at the
+     * same index.
+     */
+    flagEntryCandidates(provider, answers, state) {
+        const flagged = [];
+        answers.forEach((answer, index) => {
+            const duplicate = state.Query.Duplicates.Duplicates[index];
+            if (!duplicate || provider.BandCandidate(answer).Recommendation !== 'Uncertain') {
+                return;
+            }
+            const recordID = duplicate.ToCompactURLSegment();
+            flagged.push({
+                RecordID: recordID,
+                DisplayName: state.DisplayNames.get(NormalizeUUID(recordID)) ?? recordID,
+                VectorScore: duplicate.ProbabilityScore,
+                Probability: answer.Probability,
+            });
+        });
+        return flagged.sort(compareMostProbableFirst);
+    }
+    /** A row's name-field values joined with spaces; empty when it has none the user may read. */
+    rowDisplayName(row, nameFields) {
+        return nameFields
+            .map(f => row[f.Name])
+            .filter(v => v != null && String(v).trim() !== '')
+            .map(v => String(v))
+            .join(' ');
+    }
+    entryCheckResult(status, startTime, candidates = [], errorMessage) {
+        const result = { Status: status, Candidates: candidates, ElapsedMs: Date.now() - startTime };
+        if (errorMessage) {
+            result.ErrorMessage = errorMessage;
+        }
+        return result;
+    }
     // ─────────────────────────────────────────────
     // Batch Processing
     // ─────────────────────────────────────────────
@@ -250,7 +568,7 @@ export class DuplicateRecordDetector extends VectorBase {
      * @param contextUser - The user context for entity operations
      * @returns Combined results and match count for this batch
      */
-    async ProcessBatch(batchIDs, entityInfo, entityDocument, templateParser, duplicateRunID, topK, concurrency, options, startTime, totalRecords, processedSoFar, matchesSoFar, contextUser) {
+    async processBatch(batchIDs, entityInfo, entityDocument, templateParser, duplicateRunID, topK, concurrency, options, startTime, totalRecords, processedSoFar, matchesSoFar, contextUser) {
         // 6a: Load full record data for this batch (needed for template rendering). Each id is a
         //     compact key segment (see LoadRecordIDsToCheck), so this rebuilds the entity's real
         //     key — a single column of any name, or a composite key — rather than assuming one column.
@@ -272,10 +590,18 @@ export class DuplicateRecordDetector extends VectorBase {
             // Embed this sub-batch
             this.reportProgress(options, 'Embedding', totalRecords, processedSoFar, matchesSoFar, startTime);
             const subTemplateTexts = await this.GenerateTemplateTexts(templateParser, entityDocument, subRecords, contextUser);
-            const subEmbedResult = await this.embedding.EmbedTexts({ texts: subTemplateTexts, model: this.embeddingModelAPIName });
+            const subEmbedResult = await this.EmbeddingRunner.RunEmbedding({
+                Texts: subTemplateTexts,
+                ModelID: this.embeddingModelID ?? undefined,
+                ContextUser: contextUser,
+                Description: `Duplicate detection batch (${entityDocument.Name})`
+            });
+            if (!subEmbedResult.Success || !subEmbedResult.Vectors || subEmbedResult.Vectors.length === 0) {
+                throw new Error(`Embedding failed for duplicate detection batch: ${subEmbedResult.ErrorMessage ?? 'Unknown error'}`);
+            }
             // Query vector DB for each record in the sub-batch with concurrency control
             this.reportProgress(options, 'Querying', totalRecords, processedSoFar, matchesSoFar, startTime);
-            const subQueryResults = await this.QueryDuplicatesForRecords(subRecords, subEmbedResult.vectors, subTemplateTexts, entityDocument, topK, options, concurrency);
+            const subQueryResults = await this.QueryDuplicatesForRecords(subRecords, subEmbedResult.Vectors, subTemplateTexts, entityDocument, topK, options, concurrency);
             allQueryResults.push(...subQueryResults);
         }
         // 6d.5: Drop matches that point to records which no longer exist in the source
@@ -387,7 +713,7 @@ export class DuplicateRecordDetector extends VectorBase {
      */
     async InitializeProviders(entityDocument) {
         // Skip re-initialization if providers are already set for this entity document
-        if (this.embedding && this.vectorDB && this.indexName) {
+        if (this._embeddingRunner && this.vectorDB && this.indexName) {
             return;
         }
         if (!entityDocument.AIModelID) {
@@ -397,27 +723,21 @@ export class DuplicateRecordDetector extends VectorBase {
             throw new Error(`Entity Document '${entityDocument.Name}' (${entityDocument.ID}) cannot be used for duplicate detection: VectorDatabaseID is required but is null.`);
         }
         const aiModel = this.GetAIModel(entityDocument.AIModelID);
-        // The embedding model's API name (e.g. 'Xenova/gte-small') — local providers REQUIRE it
-        // to load the right ONNX pipeline; cloud providers fall back to their own default if absent.
+        this.embeddingModelID = entityDocument.AIModelID;
         this.embeddingModelAPIName = aiModel.APIName;
         // Captured for the vector query id when the provider keys by EntityDocumentID (SVS).
         this.entityDocumentID = entityDocument.ID;
         const vectorDB = this.GetVectorDatabase(entityDocument.VectorDatabaseID);
-        // Resolve API keys. Empty/null is legitimate for local providers — local
-        // embeddings (ONNX runtime) and the in-process SimpleVectorServiceProvider need
-        // no remote credential — so we don't pre-throw on a missing key. Whether the vector
-        // DB genuinely needs one is decided AFTER instantiation via VectorDBBase.RequiresAPIKey;
-        // a cloud provider that truly needs a key will otherwise fail at the inference call
-        // with a more actionable provider-level error. Mirrors EntityVectorSyncer.
-        const embeddingAPIKey = GetAIAPIKey(aiModel.DriverClass) || '';
+        // Only the vector DB key is resolved here. AIEmbeddingRunner resolves embedding
+        // credentials on each call and runs a keyless driver (LocalEmbedding) without one.
+        // An empty vector-DB key is legitimate for in-process and colocated providers, so we
+        // don't pre-throw; whether the DB genuinely needs one is decided AFTER instantiation
+        // via VectorDBBase.RequiresAPIKey. Mirrors EntityVectorSyncer.
         const vectorDBAPIKey = GetAIAPIKey(vectorDB.ClassKey) || '';
-        this.embedding = MJGlobal.Instance.ClassFactory.CreateInstance(BaseEmbeddings, aiModel.DriverClass, embeddingAPIKey);
+        this._embeddingRunner = new AIEmbeddingRunner();
         // Sentinel when keyless so the base ctor's non-empty requirement is satisfied for
         // local providers (which authenticate via the host process, not a key).
         this.vectorDB = MJGlobal.Instance.ClassFactory.CreateInstance(VectorDBBase, vectorDB.ClassKey, vectorDBAPIKey || 'colocated');
-        if (!this.embedding) {
-            throw new Error(`Failed to create Embeddings instance for ${aiModel.DriverClass}`);
-        }
         if (!this.vectorDB) {
             throw new Error(`Failed to create VectorDB instance for ${vectorDB.ClassKey}`);
         }
@@ -425,11 +745,11 @@ export class DuplicateRecordDetector extends VectorBase {
             throw new Error(`No API Key found for Vector Database ${vectorDB.ClassKey}`);
         }
         // Resolve the vector index name from the entity document's VectorIndexID
-        // Uses KnowledgeHubMetadataEngine cache instead of a RunView query
+        // Uses the AIEngine vector index cache instead of a RunView query
         if (entityDocument.VectorIndexID) {
-            const vectorIndex = KnowledgeHubMetadataEngine.Instance.GetVectorIndexByID(entityDocument.VectorIndexID);
+            const vectorIndex = AIEngine.Instance.GetVectorIndexByID(entityDocument.VectorIndexID);
             if (vectorIndex) {
-                this.indexName = vectorIndex.Name;
+                this.indexName = AIEngine.Instance.GetProviderIndexName(vectorIndex);
             }
         }
         if (!this.indexName) {
@@ -787,13 +1107,7 @@ export class DuplicateRecordDetector extends VectorBase {
     buildSourceMetadataMap(records, entityInfo) {
         const metadataMap = new Map();
         // Combine all IsNameField fields in Sequence order for the display name
-        const nameFields = entityInfo.Fields
-            .filter(f => f.IsNameField)
-            .sort((a, b) => (a.Sequence ?? 9999) - (b.Sequence ?? 9999));
-        // Fall back to singular NameField if no IsNameField flags
-        if (nameFields.length === 0 && entityInfo.NameField) {
-            nameFields.push(entityInfo.NameField);
-        }
+        const nameFields = this.DisplayNameFields(entityInfo);
         // Use DefaultInView fields for display, plus IsNameField fields
         const internalNames = new Set(['ID', '__mj_CreatedAt', '__mj_UpdatedAt']);
         const displayFields = entityInfo.Fields
@@ -1065,12 +1379,13 @@ export class DuplicateRecordDetector extends VectorBase {
      * independently), so a false-positive candidate reads NotDuplicate even when another candidate
      * in the same set is a confident Merge. Falls back to the set-level summary only when the
      * reasoner returned no per-candidate verdict for this record. The proposed survivor + field
-     * map stay set-level (they describe the merge of the set's true duplicates).
+     * map stay set-level (they describe the merge of the set's true duplicates). The run id is
+     * the set's, unless this candidate's verdict came from a run of its own.
      *
      * @param candidateRecordID this candidate's record id (matches the input candidate RecordID)
      */
     applyReasoningToMatch(match, reasoning, candidateRecordID) {
-        const verdict = reasoning.CandidateVerdicts.find(v => this.recordIdMatches(v.RecordID, candidateRecordID));
+        const verdict = this.findCandidateVerdict(reasoning, candidateRecordID);
         match.LLMRecommendation = verdict ? verdict.Recommendation : reasoning.Recommendation;
         match.LLMConfidence = verdict ? verdict.Confidence : reasoning.Confidence;
         match.LLMReasoning = (verdict?.Reasoning || reasoning.Reasoning) || null;
@@ -1078,9 +1393,25 @@ export class DuplicateRecordDetector extends VectorBase {
         match.LLMProposedFieldMap = reasoning.FieldChoices.length > 0
             ? JSON.stringify(reasoning.FieldChoices)
             : null;
+        this.applyRunIDsToMatch(match, reasoning, verdict);
+    }
+    /**
+     * Point the match row at the run that produced its verdict: the verdict's own prompt run when
+     * it carries one (a candidate `DecisionThenPrompt`'s decision dropped), else the set's run.
+     */
+    applyRunIDsToMatch(match, reasoning, verdict) {
+        if (verdict?.AIPromptRunID) {
+            match.AIPromptRunID = verdict.AIPromptRunID;
+            match.AIAgentRunID = null;
+            return;
+        }
         match.AIPromptRunID = reasoning.AIPromptRunID ?? null;
         match.AIAgentRunID = reasoning.AIAgentRunID ?? null;
     }
+    /** This candidate's own verdict, matched by record id, or undefined when the reasoner returned none for it. */
+    findCandidateVerdict(reasoning, candidateRecordID) {
+        return reasoning.CandidateVerdicts.find(v => this.recordIdMatches(v.RecordID, candidateRecordID));
+    }
     // ─────────────────────────────────────────────
     // LLM Reasoning (gated, per matched set)
     // ─────────────────────────────────────────────
@@ -1198,14 +1529,17 @@ export class DuplicateRecordDetector extends VectorBase {
     /**
      * Assemble the reasoning input for a matched set: source description, candidate
      * descriptions, and the differing-field deltas loaded for the whole set.
+     *
+     * @param unsavedSource the source record when it is not saved yet (the entry-time check): its
+     *   in-memory values stand in for the database row the delta load cannot find
      */
-    async BuildReasoningInput(qr, entityInfo, entityDocument, contextUser) {
+    async BuildReasoningInput(qr, entityInfo, entityDocument, contextUser, unsavedSource) {
         // Load the field deltas first — the same pass yields a record-id → name label map, which
         // gives BOTH the source and the candidates real names (loaded from the records) instead of
         // raw GUIDs. The vector-metadata name is a weaker fallback; the GUID is the last resort.
         const deltaBuilder = new MatchedSetDeltaBuilder(this.RunView);
         const allKeys = [qr.SourceKey, ...qr.Duplicates.Duplicates];
-        const { FieldDeltas: fieldDeltas, Labels: labels } = await deltaBuilder.Build(entityInfo, allKeys, contextUser);
+        const { FieldDeltas: fieldDeltas, Labels: labels } = await deltaBuilder.Build(entityInfo, allKeys, contextUser, unsavedSource);
         const candidates = qr.Duplicates.Duplicates.map(d => ({
             RecordID: d.Values(),
             Label: this.resolveRecordLabel(labels, d.Values(), this.reasoningLabel(d.VectorMetadata)),
@@ -1252,11 +1586,12 @@ export class DuplicateRecordDetector extends VectorBase {
         return null;
     }
     /**
-     * Carry the set-level verdict + resolved survivor field map onto the result so the
-     * auto-merge step (AutoMergeAboveAbsolute) can consult the recommendation and apply the
-     * literal {FieldName, Value} overrides via {@link RecordMergeRequest.FieldMap}. The UI
-     * still reads the persisted per-row {@link MJDuplicateRunDetailMatchEntity.LLMProposedFieldMap}
-     * (the raw choices) and lets the reviewer override before a manual merge.
+     * Carry the set-level verdict + resolved survivor field map onto the result, and each
+     * candidate's own verdict onto its {@link PotentialDuplicate}, so the auto-merge step
+     * (AutoMergeAboveAbsolute) can consult both recommendations and apply the literal
+     * {FieldName, Value} overrides via {@link RecordMergeRequest.FieldMap}. The UI still reads
+     * the persisted per-row {@link MJDuplicateRunDetailMatchEntity.LLMProposedFieldMap} (the raw
+     * choices) and lets the reviewer override before a manual merge.
      */
     applyReasoningToResult(result, output, fieldMap) {
         if (!output.Success) {
@@ -1265,6 +1600,9 @@ export class DuplicateRecordDetector extends VectorBase {
         result.ReasoningRecommendation = output.Recommendation;
         result.ReasoningFieldMap = fieldMap.length > 0 ? fieldMap : undefined;
         result.ReasoningText = output.Reasoning?.trim() ? output.Reasoning : undefined;
+        for (const dupe of result.Duplicates) {
+            dupe.ReasoningRecommendation = this.findCandidateVerdict(output, dupe.Values())?.Recommendation;
+        }
     }
     // ─────────────────────────────────────────────
     // Auto-Merge
@@ -1301,7 +1639,9 @@ export class DuplicateRecordDetector extends VectorBase {
      * Reasoning path (EnableLLMReasoning = true): `AutomationLevel` governs.
      *   - ReviewAll / LLMGated → never auto-merge (everything goes to human review).
      *   - AutoMergeAboveAbsolute → at/above the absolute threshold AND the set's LLM
-     *     recommendation is 'Merge'.
+     *     recommendation is 'Merge' AND this candidate's own verdict is 'Merge'. A set-level
+     *     'Merge' only says that SOME candidate is a duplicate, so it never merges a candidate
+     *     the reasoner judged otherwise, or returned no verdict for.
      */
     IsAutoMergeEligible(dupe, dupeResult, entityDocument, absoluteThreshold) {
         const aboveAbsolute = dupe.ProbabilityScore >= absoluteThreshold;
@@ -1311,7 +1651,9 @@ export class DuplicateRecordDetector extends VectorBase {
         if (entityDocument.AutomationLevel !== 'AutoMergeAboveAbsolute') {
             return false;
         }
-        return aboveAbsolute && dupeResult.ReasoningRecommendation === 'Merge';
+        return aboveAbsolute
+            && dupeResult.ReasoningRecommendation === 'Merge'
+            && dupe.ReasoningRecommendation === 'Merge';
     }
     /**
      * Execute one guarded auto-merge for an eligible candidate, applying the LLM's resolved
@@ -1387,6 +1729,50 @@ function chunkArray(array, chunkSize) {
     }
     return chunks;
 }
+/**
+ * Whether a reasoning provider is a decision provider. Checked by shape, not `instanceof`, so this
+ * module need not load the decision provider and the AI engine behind it; the package index
+ * registers the provider, as it does every other reasoning mode's.
+ */
+function isDecisionReasoningProvider(provider) {
+    return provider != null
+        && 'DecideCandidates' in provider && typeof provider.DecideCandidates === 'function'
+        && 'BandCandidate' in provider && typeof provider.BandCandidate === 'function';
+}
+/**
+ * The entered values with every string cut to {@link DUPLICATE_ENTRY_CHECK_MAX_FIELD_TEXT_LENGTH}.
+ * Other values pass through as they are.
+ */
+function boundEnteredValues(values) {
+    const bounded = {};
+    for (const [name, value] of Object.entries(values)) {
+        bounded[name] = typeof value === 'string' ? truncateText(value) : value;
+    }
+    return bounded;
+}
+/**
+ * Text cut to at most `max` characters; a cut text ends with an ellipsis, so a reader (the decision
+ * model included) can tell it was cut.
+ */
+function truncateText(text, max = DUPLICATE_ENTRY_CHECK_MAX_FIELD_TEXT_LENGTH) {
+    return text.length <= max ? text : `${text.slice(0, max - 1)}\u2026`;
+}
+/**
+ * Orders entity documents oldest first: by `__mj_CreatedAt`, then by ID, so the order is total
+ * and stable across calls.
+ */
+function compareOldestFirst(a, b) {
+    const byAge = (a.__mj_CreatedAt?.getTime() ?? 0) - (b.__mj_CreatedAt?.getTime() ?? 0);
+    return byAge !== 0 ? byAge : NormalizeUUID(a.ID).localeCompare(NormalizeUUID(b.ID));
+}
+/**
+ * Orders flagged candidates most probable first. A candidate the decision gave no answer for sorts
+ * after every answered one; ties fall back to the vector score.
+ */
+function compareMostProbableFirst(a, b) {
+    const byProbability = (b.Probability ?? -1) - (a.Probability ?? -1);
+    return byProbability !== 0 ? byProbability : b.VectorScore - a.VectorScore;
+}
 /**
  * Run async tasks with a concurrency limit.
  * Executes up to `limit` tasks in parallel, queuing the rest.