@memberjunction/ng-dashboards 6.2.0-edge.5 → 6.2.0-edge.7

package/dist/AI/components/duplicates/duplicate-detection-resource.component.js

@@ -19,10 +19,10 @@ import { debounceTime, takeUntil } from 'rxjs/operators';
 import { CompositeKey, LogStatus, RecordMergeRequest, RunView } from '@memberjunction/core';
 import { MJNotificationService } from '@memberjunction/ng-notifications';
 import { KnowledgeHubMetadataEngine } from '@memberjunction/core-entities';
-import { RegisterClass, UUIDsEqual } from '@memberjunction/global';
+import { EscapeSQLString, RegisterClass, UUIDsEqual } from '@memberjunction/global';
 import { BaseResourceComponent, NavigationService, ActivityService } from '@memberjunction/ng-shared';
 import { BuildDuplicateAgentContext, ResolveEntityDoc, ResolveEntityFilter, } from './duplicate-detection-agent-context';
-import { BuildMatchQueriesForDetailIDs, BuildRunScopedDetailsQuery, GroupMatchesByDetail, PickDefaultEntityDocument, SelectCurrentRunForEntity, } from './duplicate-detection-run-scope';
+import { BuildMatchQueriesForDetailIDs, BuildRunScopedDetailsQuery, GroupMatchesByDetail, PickDefaultEntityDocument, SelectCurrentRunForDocument, } from './duplicate-detection-run-scope';
 import { ValidateStringParam } from '../../../shared/agent-tool-validation';
 import * as i0 from "@angular/core";
 import * as i1 from "./compare-value.component";
@@ -1530,6 +1530,11 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
     DetectionStage = '';
     /** Raw stage key from the last progress event — used to detect phase transitions */
     detectionRawStage = '';
+    /** Run IDs that were found unresponsive so reconnectToActiveRun does not loop on them */
+    staleRunIDs = new Set();
+    activeProgressSub = null;
+    idleTimer = null;
+    closeComparisonTimer = null;
     /** Runtime threshold overrides — initialized from entity doc, adjustable via sliders */
     RunPotentialThreshold = 0.70;
     RunAbsoluteThreshold = 0.95;
@@ -1601,9 +1606,7 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
     FilterByEntity(entityName) {
         this.SelectedEntityFilter = entityName;
         this.Filters.EntityName = entityName;
-        this.autoSelectDisplayMode();
-        this.emitAgentContext();
-        this.cdr.detectChanges();
+        this.applyFilters();
     }
     /** Get paged groups for table view */
     GetPagedGroups() {
@@ -1826,6 +1829,16 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
     }
     ngOnDestroy() {
         super.ngOnDestroy();
+        if (this.idleTimer) {
+            clearTimeout(this.idleTimer);
+            this.idleTimer = null;
+        }
+        if (this.closeComparisonTimer) {
+            clearTimeout(this.closeComparisonTimer);
+            this.closeComparisonTimer = null;
+        }
+        this.activeProgressSub?.unsubscribe();
+        this.activeProgressSub = null;
         this.destroy$.next();
         this.destroy$.complete();
     }
@@ -1936,7 +1949,7 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
      * current (see reviewLoadGeneration).
      */
     async fetchReviewRowsForSelectedDocument() {
-        const run = SelectCurrentRunForEntity(this.Runs, this.SelectedDocumentThresholds?.EntityID);
+        const run = SelectCurrentRunForDocument(this.Runs, this.SelectedEntityDocumentID);
         if (!run) {
             return { run: null, details: [], matches: [] };
         }
@@ -2001,17 +2014,31 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
     /**
      * Check if there's an in-progress detection run and reconnect to its
      * progress subscription. This handles the case where the user navigated
-     * away and came back while a run was active.
+     * away and came back while a run was active, while ignoring abandoned/stale runs.
      */
     reconnectToActiveRun() {
         if (this.IsDetecting)
             return; // Already tracking a run
-        const activeRun = this.Runs.find(r => r.ProcessingStatus === 'In Progress');
+        const now = Date.now();
+        const maxStaleAgeMs = 5 * 60 * 1000; // 5 minutes without a DB update
+        const activeRun = this.Runs.find(r => {
+            if (r.ProcessingStatus !== 'In Progress' || r.EndedAt != null || r.CancellationRequested) {
+                return false;
+            }
+            if (this.staleRunIDs.has(r.ID)) {
+                return false;
+            }
+            const lastActivity = r.__mj_UpdatedAt ?? r.StartedAt;
+            const lastTime = lastActivity ? new Date(lastActivity).getTime() : 0;
+            return lastTime > 0 && (now - lastTime) < maxStaleAgeMs;
+        });
         if (!activeRun)
             return;
         LogStatus(`[DuplicateDetection] Reconnecting to in-progress run ${activeRun.ID}`);
         this.IsDetecting = true;
-        this.DetectionProgress = 0;
+        this.DetectionProgress = (activeRun.TotalItemCount && activeRun.TotalItemCount > 0 && activeRun.ProcessedItemCount != null)
+            ? Math.min(99, Math.round((activeRun.ProcessedItemCount / activeRun.TotalItemCount) * 100))
+            : 0;
         this.DetectionStage = 'Reconnecting...';
         this.cdr.detectChanges();
         this.subscribeToPipelineProgress(activeRun.ID);
@@ -2034,10 +2061,8 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
         this.cdr.detectChanges();
         try {
             const md = this.ProviderToUse;
-            const dupeRun = await md.GetEntityObject('MJ: Duplicate Runs');
-            dupeRun.NewRecord();
             // Look up the EntityID from the entity document's entity name
-            const entityInfo = md.Entities.find(e => e.Name === selectedDoc.EntityName);
+            const entityInfo = md.EntityByName(selectedDoc.EntityName);
             if (!entityInfo) {
                 MJNotificationService.Instance.CreateSimpleNotification(`Entity "${selectedDoc.EntityName}" not found in metadata`, 'error', 5000);
                 this.IsDetecting = false;
@@ -2046,16 +2071,43 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
                 this.cdr.detectChanges();
                 return;
             }
+            // Persist threshold slider overrides onto the selected Entity Document so the
+            // server hook (which reads thresholds from MJ: Entity Documents) uses them.
+            if (this.RunPotentialThreshold !== selectedDoc.PotentialMatchThreshold ||
+                this.RunAbsoluteThreshold !== selectedDoc.AbsoluteMatchThreshold) {
+                const ed = await md.GetEntityObject('MJ: Entity Documents', md.CurrentUser);
+                if (await ed.Load(selectedDoc.ID)) {
+                    ed.PotentialMatchThreshold = this.RunPotentialThreshold;
+                    ed.AbsoluteMatchThreshold = this.RunAbsoluteThreshold;
+                    if (await ed.Save()) {
+                        selectedDoc.PotentialMatchThreshold = this.RunPotentialThreshold;
+                        selectedDoc.AbsoluteMatchThreshold = this.RunAbsoluteThreshold;
+                    }
+                }
+            }
             // DD-1: Track whether merging is available for the selected entity
             this.MergeEnabled = entityInfo.AllowRecordMerge;
             this.ShowMergeWarningBanner = !entityInfo.AllowRecordMerge;
+            const dupeRun = await md.GetEntityObject('MJ: Duplicate Runs');
+            dupeRun.NewRecord();
             dupeRun.EntityID = entityInfo.ID;
+            // The run uses the document picked here, not whichever document the entity updated last.
+            dupeRun.EntityDocumentID = selectedDoc.ID;
             dupeRun.StartedByUserID = this.ProviderToUse.CurrentUser.ID;
             dupeRun.StartedAt = new Date();
             dupeRun.ProcessingStatus = 'In Progress';
             dupeRun.ApprovalStatus = 'Pending';
+            this.detectionActivityID = this.activityService.Start('Duplicate detection', {
+                icon: 'fa-solid fa-clone',
+                detail: selectedDoc.EntityName,
+                progress: 0,
+            });
             const saved = await dupeRun.Save();
             if (!saved) {
+                if (this.detectionActivityID) {
+                    this.activityService.Complete(this.detectionActivityID, 'error');
+                    this.detectionActivityID = null;
+                }
                 console.error('Failed to create duplicate run:', dupeRun.LatestResult?.Message || 'unknown error');
                 MJNotificationService.Instance.CreateSimpleNotification(`Failed to start detection: ${dupeRun.LatestResult?.Message || 'unknown error'}`, 'error', 5000);
                 this.IsDetecting = false;
@@ -2064,17 +2116,28 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
                 this.cdr.detectChanges();
                 return;
             }
-            this.detectionActivityID = this.activityService.Start('Duplicate detection', {
-                icon: 'fa-solid fa-clone',
-                detail: selectedDoc.EntityName,
-                progress: 0,
-            });
-            // Subscribe to progress using the run ID as PipelineRunID
+            // Subscribe only now, with the saved ID. A new record's create request carries no
+            // primary key, so the server assigns the run's ID and the ID NewRecord() generated
+            // in the browser never matches it: a subscription opened before Save() listens for a
+            // run that never publishes, and its status check finds no such run. A run that
+            // finishes during Save() publishes before this subscription exists, so
+            // subscribeToPipelineProgress checks the run's status right away to catch it.
             this.subscribeToPipelineProgress(dupeRun.ID);
         }
         catch (error) {
             const msg = error instanceof Error ? error.message : String(error);
             console.error('[DuplicateDetection] Error starting detection:', msg);
+            if (this.idleTimer) {
+                clearTimeout(this.idleTimer);
+                this.idleTimer = null;
+            }
+            this.activeProgressSub?.unsubscribe();
+            this.activeProgressSub = null;
+            if (this.detectionActivityID) {
+                this.activityService.Complete(this.detectionActivityID, 'error');
+                this.detectionActivityID = null;
+            }
+            MJNotificationService.Instance.CreateSimpleNotification(`Failed to start duplicate detection: ${msg}`, 'error', 5000);
             this.IsDetecting = false;
             this.DetectionStage = '';
             this.emitAgentContext();
@@ -2384,11 +2447,21 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
                 }
             }
         `;
-        let idleTimer = null;
-        const finishDetection = (success) => {
-            if (idleTimer)
-                clearTimeout(idleTimer);
-            rxSub?.unsubscribe();
+        if (this.idleTimer) {
+            clearTimeout(this.idleTimer);
+            this.idleTimer = null;
+        }
+        this.activeProgressSub?.unsubscribe();
+        this.activeProgressSub = null;
+        let consecutiveUnchangedPolls = 0;
+        let lastPolledProcessed = -1;
+        const finishDetection = (success, errorMessage) => {
+            if (this.idleTimer) {
+                clearTimeout(this.idleTimer);
+                this.idleTimer = null;
+            }
+            this.activeProgressSub?.unsubscribe();
+            this.activeProgressSub = null;
             Promise.resolve().then(async () => {
                 this.IsDetecting = false;
                 this.DetectionStage = success ? 'Complete' : 'Error';
@@ -2397,6 +2470,9 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
                     this.activityService.Complete(this.detectionActivityID, success ? 'success' : 'error');
                     this.detectionActivityID = null;
                 }
+                if (!success && errorMessage) {
+                    MJNotificationService.Instance.CreateSimpleNotification(errorMessage, 'error', 6000);
+                }
                 if (success) {
                     await this.LoadData();
                 }
@@ -2404,16 +2480,73 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
                 this.cdr.detectChanges();
             });
         };
-        const resetIdleTimer = () => {
-            if (idleTimer)
-                clearTimeout(idleTimer);
-            idleTimer = setTimeout(() => {
-                if (this.IsDetecting) {
-                    finishDetection(true);
+        const checkRunStatusOnIdle = async () => {
+            if (!this.IsDetecting)
+                return;
+            try {
+                const rv = RunView.FromMetadataProvider(this.ProviderToUse);
+                const res = await rv.RunView({
+                    EntityName: 'MJ: Duplicate Runs',
+                    ExtraFilter: `ID='${EscapeSQLString(pipelineRunID)}'`,
+                    MaxRows: 1,
+                    SkipTotalRowCount: true,
+                    ResultType: 'entity_object'
+                });
+                if (!this.IsDetecting)
+                    return;
+                if (res.Success && res.Results.length > 0) {
+                    const run = res.Results[0];
+                    if (run.ProcessingStatus === 'Complete') {
+                        finishDetection(true);
+                        return;
+                    }
+                    if (run.ProcessingStatus === 'Failed') {
+                        finishDetection(false, run.ProcessingErrorMessage || 'Duplicate detection run failed');
+                        return;
+                    }
+                    if (run.EndedAt != null) {
+                        finishDetection(true);
+                        return;
+                    }
+                    if (run.TotalItemCount && run.TotalItemCount > 0 && run.ProcessedItemCount != null) {
+                        const dbPct = Math.min(99, Math.round((run.ProcessedItemCount / run.TotalItemCount) * 100));
+                        if (dbPct > this.DetectionProgress) {
+                            this.DetectionProgress = dbPct;
+                            this.cdr.detectChanges();
+                        }
+                    }
+                    const currentProcessed = run.ProcessedItemCount ?? 0;
+                    if (currentProcessed === lastPolledProcessed) {
+                        consecutiveUnchangedPolls++;
+                    }
+                    else {
+                        lastPolledProcessed = currentProcessed;
+                        consecutiveUnchangedPolls = 0;
+                    }
+                }
+                else {
+                    consecutiveUnchangedPolls++;
+                }
+                if (consecutiveUnchangedPolls >= 8) {
+                    // 8 consecutive 15s checks (2 minutes) with zero DB or WS progress
+                    this.staleRunIDs.add(pipelineRunID);
+                    finishDetection(false, 'Duplicate detection timed out with no progress from the server.');
+                    return;
                 }
-            }, 60000); // 60s timeout for duplicate detection (can be very long)
+                resetIdleTimer();
+            }
+            catch (err) {
+                console.error('[DuplicateDetection] Error polling run status:', err);
+                resetIdleTimer();
+            }
+        };
+        const resetIdleTimer = () => {
+            if (this.idleTimer)
+                clearTimeout(this.idleTimer);
+            this.idleTimer = setTimeout(() => {
+                void checkRunStatusOnIdle();
+            }, 15000); // Check DB status after 15s of subscription silence
         };
-        resetIdleTimer();
         // Reset phase tracking for this new subscription
         this.detectionRawStage = '';
         const sub = provider.subscribe(subscriptionQuery, { pipelineRunID });
@@ -2425,37 +2558,42 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
                 const stage = progress['Stage'];
                 const pct = Math.max(0, Math.min(100, progress['PercentComplete']));
                 const currentItem = progress['CurrentItem'];
-                // Detect phase transitions vs. within-phase updates
-                const isNewPhase = stage !== this.detectionRawStage;
-                if (isNewPhase) {
-                    // New phase: reset progress and update display
-                    this.detectionRawStage = stage;
+                // Only reset progress when transitioning out of 'vectorize' into the detection pass;
+                // within the detection pass ('extract' and 'autotag' share the global record counter),
+                // keep progress strictly monotonic so the bar never jumps backward.
+                const isLeavingVectorize = this.detectionRawStage === 'vectorize' && stage !== 'vectorize';
+                this.detectionRawStage = stage;
+                this.DetectionStage = this.formatDetectionStage(stage);
+                if (isLeavingVectorize) {
                     this.DetectionProgress = pct;
-                    this.DetectionStage = this.formatDetectionStage(stage);
                 }
-                else {
-                    // Same phase: only move forward (never backward)
-                    if (pct >= this.DetectionProgress) {
-                        this.DetectionProgress = pct;
-                    }
+                else if (pct >= this.DetectionProgress) {
+                    this.DetectionProgress = pct;
                 }
                 this.DetectionCurrentItem = currentItem ?? '';
+                consecutiveUnchangedPolls = 0;
                 this.cdr.detectChanges();
                 if (stage === 'complete') {
                     finishDetection(true);
                 }
                 else if (stage === 'error') {
-                    finishDetection(false);
+                    finishDetection(false, currentItem || 'Duplicate detection failed');
                 }
                 else {
                     resetIdleTimer();
                 }
             },
             error: (err) => {
+                const msg = err instanceof Error ? err.message : String(err);
                 console.error('[DuplicateDetection] Pipeline subscription error:', err);
-                finishDetection(false);
+                finishDetection(false, `Progress stream error: ${msg}`);
             }
         });
+        this.activeProgressSub = rxSub;
+        // Read the run's status now rather than after the first 15s of silence: a run that
+        // completed or failed before this subscription connected publishes nothing more, and
+        // this check is what finishes it. The check re-arms the idle timer itself.
+        void checkRunStatusOnIdle();
     }
     /** Format detection stage names for display */
     formatDetectionStage(stage) {
@@ -2546,6 +2684,10 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
     // ════════════════════════════════════════════
     /** Open the comparison slide-in panel for a group — loads real entity records */
     async OpenComparison(group) {
+        if (this.closeComparisonTimer) {
+            clearTimeout(this.closeComparisonTimer);
+            this.closeComparisonTimer = null;
+        }
         this.ComparisonGroup = group;
         this.ComparisonShowAllFields = true;
         this.SurvivorColumnIndex = 0;
@@ -2577,7 +2719,11 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
         this.ShowMergeConfirm = false;
         this.ComparisonClosing = true;
         this.cdr.detectChanges();
-        setTimeout(() => {
+        if (this.closeComparisonTimer) {
+            clearTimeout(this.closeComparisonTimer);
+        }
+        this.closeComparisonTimer = setTimeout(() => {
+            this.closeComparisonTimer = null;
             this.ComparisonGroup = null;
             this.ComparisonClosing = false;
             this.ComparisonFields = [];
@@ -3132,26 +3278,33 @@ let DuplicateDetectionResourceComponent = class DuplicateDetectionResourceCompon
     }
     /** Approve an individual match */
     async ApproveIndividualMatch(matchInfo) {
-        this.IsSaving = true;
-        matchInfo.Match.ApprovalStatus = 'Approved';
-        await matchInfo.Match.Save();
-        this.IsSaving = false;
-        this.cdr.detectChanges();
+        await this.updateIndividualMatchStatus(matchInfo, 'Approved');
     }
     /** Reject an individual match (skip it from merge) */
     async RejectIndividualMatch(matchInfo) {
-        this.IsSaving = true;
-        matchInfo.Match.ApprovalStatus = 'Rejected';
-        await matchInfo.Match.Save();
-        this.IsSaving = false;
-        this.cdr.detectChanges();
+        await this.updateIndividualMatchStatus(matchInfo, 'Rejected');
     }
     /** Undo a rejected individual match (restore to Pending) */
     async UndoRejectIndividualMatch(matchInfo) {
+        await this.updateIndividualMatchStatus(matchInfo, 'Pending');
+    }
+    async updateIndividualMatchStatus(matchInfo, status) {
+        const previousStatus = matchInfo.Match.ApprovalStatus;
         this.IsSaving = true;
-        matchInfo.Match.ApprovalStatus = 'Pending';
-        await matchInfo.Match.Save();
+        matchInfo.Match.ApprovalStatus = status;
+        const saved = await matchInfo.Match.Save();
         this.IsSaving = false;
+        if (!saved) {
+            matchInfo.Match.ApprovalStatus = previousStatus;
+            MJNotificationService.Instance.CreateSimpleNotification(`Failed to update match status to ${status}`, 'error', 4000);
+            this.cdr.detectChanges();
+            return;
+        }
+        if (this.ComparisonGroup) {
+            this.ComparisonGroup.ApprovalStatus = this.computeDominantApprovalStatus(this.ComparisonGroup.Matches);
+            this.applyFilters();
+            this.emitAgentContext();
+        }
         this.cdr.detectChanges();
     }
     /**