@memberjunction/content-autotagging 5.22.0 → 5.24.0

package/dist/Engine/generic/AutotagBaseEngine.js

@@ -4,9 +4,11 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
     else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
+var AutotagBaseEngine_1;
 import { BaseEngine, Metadata, RunView, LogError, LogStatus } from '@memberjunction/core';
-import { MJGlobal, UUIDsEqual, RegisterClass } from '@memberjunction/global';
+import { MJGlobal, UUIDsEqual, NormalizeUUID, RegisterClass } from '@memberjunction/global';
 import { ContentSourceTypeParams } from './content.types.js';
+import { RateLimiter } from './RateLimiter.js';
 import pdfParse from 'pdf-parse';
 import officeparser from 'officeparser';
 import * as fs from 'fs';
@@ -15,9 +17,16 @@ import { toZonedTime } from 'date-fns-tz';
 import axios from 'axios';
 import * as cheerio from 'cheerio';
 import crypto from 'crypto';
-import { BaseLLM, GetAIAPIKey } from '@memberjunction/ai';
+import { BaseEmbeddings, GetAIAPIKey } from '@memberjunction/ai';
 import { AIEngine } from '@memberjunction/aiengine';
+import { AIPromptRunner, AIModelRunner } from '@memberjunction/ai-prompts';
+import { AIPromptParams } from '@memberjunction/ai-core-plus';
 import { TextChunker } from '@memberjunction/ai-vectors';
+import { VectorDBBase } from '@memberjunction/ai-vectordb';
+import { TagEngine } from '@memberjunction/tag-engine';
+import { KnowledgeHubMetadataEngine } from '@memberjunction/core-entities';
+/** Default batch size for vectorization processing */
+const DEFAULT_VECTORIZE_BATCH_SIZE = 20;
 /**
  * Core engine for content autotagging. Extends BaseEngine to cache content metadata
  * (types, source types, file types, attributes) at startup. Uses AIEngine via composition
@@ -26,43 +35,58 @@ import { TextChunker } from '@memberjunction/ai-vectors';
 let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
     constructor() {
         super(...arguments);
-        // Cached metadata — loaded by BaseEngine.Config() via property configs
-        this._ContentTypes = [];
-        this._ContentSourceTypes = [];
-        this._ContentFileTypes = [];
+        // Cached metadata unique to this engine — loaded by BaseEngine.Config()
         this._ContentTypeAttributes = [];
         this._ContentSourceTypeParams = [];
+        /**
+         * Optional taxonomy JSON string to inject into the autotagging prompt.
+         * Set by the caller (e.g., AutotagEntity) before calling ExtractTextAndProcessWithLLM.
+         * When set, the prompt template receives an `existingTaxonomy` variable containing
+         * the JSON tree of existing tags so the LLM can prefer existing tags.
+         */
+        this.TaxonomyContext = null;
+        /**
+         * When true, skip checksum comparison and reprocess all content items
+         * even if their content hasn't changed. Useful when changing embedding models,
+         * LLM models, or vector databases.
+         */
+        this.ForceReprocess = false;
+        /** Rate limiter for LLM (tagging) API calls */
+        this.LLMRateLimiter = new RateLimiter({ RequestsPerMinute: 60, TokensPerMinute: 100000, Name: 'LLM' });
+        /** Rate limiter for embedding API calls */
+        this.EmbeddingRateLimiter = new RateLimiter({ RequestsPerMinute: 300, TokensPerMinute: 500000, Name: 'Embedding' });
+        /** Rate limiter for vector DB API calls */
+        this.VectorDBRateLimiter = new RateLimiter({ RequestsPerMinute: 200, Name: 'VectorDB' });
+        /**
+         * Optional callback invoked after each ContentItemTag is saved, enabling the
+         * tag taxonomy bridge (ContentItemTag → Tag + TaggedItem). Set by providers
+         * like AutotagEntity that want to link free-text tags to formal taxonomy entries.
+         *
+         * Parameters: (contentItemTag: MJContentItemTagEntity, parentTag: string | null, contextUser: UserInfo)
+         */
+        this.OnContentItemTagSaved = null;
     }
+    static { AutotagBaseEngine_1 = this; }
     static get Instance() {
         return super.getInstance();
     }
-    /** All content types, cached at startup */
-    get ContentTypes() { return this._ContentTypes; }
-    /** All content source types, cached at startup */
-    get ContentSourceTypes() { return this._ContentSourceTypes; }
-    /** All content file types, cached at startup */
-    get ContentFileTypes() { return this._ContentFileTypes; }
+    /** Shortcut to KnowledgeHubMetadataEngine */
+    get khEngine() { return KnowledgeHubMetadataEngine.Instance; }
+    /** All content types — delegated to KnowledgeHubMetadataEngine */
+    get ContentTypes() { return this.khEngine.ContentTypes; }
+    /** All content source types — delegated to KnowledgeHubMetadataEngine */
+    get ContentSourceTypes() { return this.khEngine.ContentSourceTypes; }
+    /** All content file types — delegated to KnowledgeHubMetadataEngine */
+    get ContentFileTypes() { return this.khEngine.ContentFileTypes; }
     /** All content type attributes, cached at startup */
     get ContentTypeAttributes() { return this._ContentTypeAttributes; }
     /** All content source type params, cached at startup */
     get ContentSourceTypeParams() { return this._ContentSourceTypeParams; }
     async Config(forceRefresh, contextUser, provider) {
+        // Content Types, Content Source Types, and Content File Types are delegated to
+        // KnowledgeHubMetadataEngine (avoid redundant loading). Only cache entities unique to this engine.
+        await KnowledgeHubMetadataEngine.Instance.Config(forceRefresh, contextUser);
         const configs = [
-            {
-                Type: 'entity',
-                EntityName: 'MJ: Content Types',
-                PropertyName: '_ContentTypes',
-            },
-            {
-                Type: 'entity',
-                EntityName: 'MJ: Content Source Types',
-                PropertyName: '_ContentSourceTypes',
-            },
-            {
-                Type: 'entity',
-                EntityName: 'MJ: Content File Types',
-                PropertyName: '_ContentFileTypes',
-            },
             {
                 Type: 'entity',
                 EntityName: 'MJ: Content Type Attributes',
@@ -79,27 +103,98 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
     }
     /**
      * Given a list of content items, extract the text from each and process with LLM for tagging.
+     * Items are processed in configurable batches with controlled concurrency within each batch.
+     */
+    /**
+     * Process content items through the LLM tagging pipeline with production-grade
+     * batch management: cursor-based resume, pause/cancel support, rate limiting,
+     * and circuit breaker. Each batch checkpoints progress so interrupted runs
+     * can be resumed from where they left off.
+     *
+     * @param contentItems - items to process
+     * @param contextUser - current user for permissions/audit
+     * @param processRun - optional ContentProcessRun entity for checkpoint tracking
+     * @param config - optional pipeline configuration for rate limits, thresholds
+     * @param onProgress - optional callback for UI progress updates
      */
-    async ExtractTextAndProcessWithLLM(contentItems, contextUser) {
+    async ExtractTextAndProcessWithLLM(contentItems, contextUser, processRun, config, onProgress) {
         if (!contentItems || contentItems.length === 0) {
-            LogStatus('No content items to process');
+            LogStatus('[Autotag] No content items to process');
             return;
         }
-        const processRunParams = new ProcessRunParams();
-        processRunParams.sourceID = contentItems[0].ContentSourceID;
-        processRunParams.startTime = new Date();
-        processRunParams.numItemsProcessed = contentItems.length;
-        for (const contentItem of contentItems) {
-            try {
-                const processingParams = await this.buildProcessingParams(contentItem, contextUser);
-                await this.ProcessContentItemText(processingParams, contextUser);
+        const batchSize = config?.Pipeline?.BatchSize ?? DEFAULT_VECTORIZE_BATCH_SIZE;
+        const errorThreshold = config?.Pipeline?.ErrorThresholdPercent ?? 20;
+        const delayMs = config?.Pipeline?.DelayBetweenBatchesMs ?? 0;
+        // Resume from cursor if available
+        const resumeOffset = processRun?.LastProcessedOffset ?? 0;
+        const itemsToProcess = resumeOffset > 0
+            ? contentItems.slice(resumeOffset)
+            : contentItems;
+        if (resumeOffset > 0) {
+            LogStatus(`[Autotag] Resuming from offset ${resumeOffset} (${itemsToProcess.length} remaining of ${contentItems.length})`);
+        }
+        LogStatus(`[Autotag] Processing ${itemsToProcess.length} items in batches of ${batchSize}`);
+        let totalSuccesses = 0;
+        let totalFailures = 0;
+        let totalProcessed = resumeOffset;
+        for (let i = 0; i < itemsToProcess.length; i += batchSize) {
+            const batch = itemsToProcess.slice(i, i + batchSize);
+            const batchNum = Math.floor(i / batchSize) + 1;
+            let batchOk = 0;
+            let batchFail = 0;
+            // Rate limit before each batch of parallel LLM calls
+            await this.LLMRateLimiter.Acquire();
+            const batchPromises = batch.map(async (contentItem) => {
+                try {
+                    const processingParams = await this.buildProcessingParams(contentItem, contextUser);
+                    await this.ProcessContentItemText(processingParams, contextUser);
+                    batchOk++;
+                }
+                catch (e) {
+                    LogError(`[Autotag] Failed to process item ${contentItem.ID}: ${e instanceof Error ? e.message : String(e)}`);
+                    batchFail++;
+                }
+            });
+            await Promise.all(batchPromises);
+            totalSuccesses += batchOk;
+            totalFailures += batchFail;
+            totalProcessed += batch.length;
+            onProgress?.(totalProcessed, contentItems.length);
+            LogStatus(`[Autotag] Batch ${batchNum}: ${batchOk}/${batch.length} ok (${totalProcessed}/${contentItems.length} total, ${totalFailures} errors)`);
+            // Checkpoint: update cursor and check for cancellation
+            if (processRun) {
+                const shouldContinue = await this.UpdateBatchCursor(processRun, totalProcessed, totalFailures);
+                if (!shouldContinue) {
+                    LogStatus(`[Autotag] Pipeline paused/cancelled at offset ${totalProcessed}`);
+                    return;
+                }
             }
-            catch (e) {
-                LogError(`Failed to process content item: ${contentItem.ID}`, undefined, e);
-                throw e;
+            // Circuit breaker: halt if error rate exceeds threshold
+            if (totalProcessed > 0 && totalFailures > 0) {
+                const errorRate = (totalFailures / totalProcessed) * 100;
+                if (errorRate > errorThreshold) {
+                    LogError(`[Autotag] Circuit breaker triggered: error rate ${errorRate.toFixed(1)}% exceeds threshold ${errorThreshold}%`);
+                    if (processRun) {
+                        processRun.ErrorMessage = `Auto-paused: error rate ${errorRate.toFixed(1)}% exceeded ${errorThreshold}% threshold`;
+                        await this.CompleteBatchedProcessRun(processRun, 'Failed', processRun.ErrorMessage);
+                    }
+                    return;
+                }
+            }
+            // Optional delay between batches (throttling)
+            if (delayMs > 0 && i + batchSize < itemsToProcess.length) {
+                await new Promise(resolve => setTimeout(resolve, delayMs));
             }
         }
+        LogStatus(`[Autotag] LLM tagging complete: ${totalSuccesses} succeeded, ${totalFailures} failed of ${contentItems.length}`);
+        // Post-pipeline hook: recompute tag co-occurrence if TagCoOccurrenceEngine is available
+        await this.recomputeCoOccurrenceIfAvailable(contextUser);
+        // Legacy process run tracking (for backward compatibility)
+        const processRunParams = new ProcessRunParams();
+        processRunParams.sourceID = contentItems[0].ContentSourceID;
+        processRunParams.startTime = processRun?.StartTime ?? new Date();
         processRunParams.endTime = new Date();
+        processRunParams.numItemsProcessed = contentItems.length;
         await this.saveProcessRun(processRunParams, contextUser);
     }
     /**
@@ -116,101 +211,362 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
         processingParams.minTags = minTags;
         processingParams.maxTags = maxTags;
         processingParams.contentItemID = contentItem.ID;
+        LogStatus(`[Autotag] Built params for "${contentItem.Name}" — text length: ${processingParams.text?.length ?? 0}, modelID: ${modelID || 'default'}, tags: ${minTags}-${maxTags}`);
         return processingParams;
     }
     /**
      * Process a content item's text with the LLM and save results.
      */
     async ProcessContentItemText(params, contextUser) {
-        const LLMResults = await this.promptAndRetrieveResultsFromLLM(params, contextUser);
-        await this.saveLLMResults(LLMResults, contextUser);
+        // A8: Update tagging status to Processing
+        await this.updateContentItemTaggingStatus(params.contentItemID, 'Processing', contextUser);
+        try {
+            const LLMResults = await this.promptAndRetrieveResultsFromLLM(params, contextUser);
+            await this.saveLLMResults(LLMResults, contextUser);
+            // A8: Update tagging status to Complete
+            await this.updateContentItemTaggingStatus(params.contentItemID, 'Complete', contextUser);
+        }
+        catch (e) {
+            await this.updateContentItemTaggingStatus(params.contentItemID, 'Failed', contextUser);
+            throw e;
+        }
     }
-    async promptAndRetrieveResultsFromLLM(params, contextUser) {
-        const model = AIEngine.Instance.Models.find(m => UUIDsEqual(m.ID, params.modelID));
-        if (!model) {
-            throw new Error(`AI Model with ID ${params.modelID} not found`);
+    /** Update embedding status for a batch of content items */
+    async updateEmbeddingStatusBatch(items, status, contextUser, embeddingModelID) {
+        for (const item of items) {
+            try {
+                item.EmbeddingStatus = status;
+                if (status === 'Complete') {
+                    item.LastEmbeddedAt = new Date();
+                    if (embeddingModelID)
+                        item.EmbeddingModelID = embeddingModelID;
+                }
+                await item.Save();
+            }
+            catch {
+                // Non-critical
+            }
+        }
+    }
+    /** Update a content item's TaggingStatus and LastTaggedAt */
+    async updateContentItemTaggingStatus(contentItemID, status, contextUser) {
+        try {
+            const md = new Metadata();
+            const item = await md.GetEntityObject('MJ: Content Items', contextUser);
+            await item.Load(contentItemID);
+            item.TaggingStatus = status;
+            if (status === 'Complete') {
+                item.LastTaggedAt = new Date();
+            }
+            await item.Save();
         }
-        const llm = MJGlobal.Instance.ClassFactory.CreateInstance(BaseLLM, model.DriverClass, GetAIAPIKey(model.DriverClass));
-        if (!llm) {
-            throw new Error(`Failed to create LLM instance for driver ${model.DriverClass}`);
+        catch {
+            // Non-critical — don't fail the pipeline for a status update
+        }
+    }
+    /**
+     * Resolves the "Content Autotagging" prompt from the AIEngine cache.
+     * Throws if the prompt is not found or not active.
+     */
+    getAutotagPrompt() {
+        const prompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Content Autotagging');
+        if (!prompt) {
+            throw new Error('AI Prompt "Content Autotagging" not found. Ensure the prompt metadata has been synced to the database.');
+        }
+        if (prompt.Status !== 'Active') {
+            throw new Error(`AI Prompt "Content Autotagging" is not active (Status: ${prompt.Status})`);
+        }
+        return prompt;
+    }
+    /**
+     * Initialize the taxonomy bridge so ALL content source types (RSS, Entity, Website, etc.)
+     * automatically create formal Tag + TaggedItem records from LLM-generated ContentItemTags.
+     *
+     * This sets up:
+     * 1. TagEngine with semantic embeddings for tag matching
+     * 2. TaxonomyContext for prompt injection (tells LLM about existing tags)
+     * 3. OnContentItemTagSaved callback that bridges ContentItemTag → Tag + TaggedItem
+     *
+     * Call this ONCE before running any providers. The bridge stays active until
+     * CleanupTaxonomyBridge() is called.
+     */
+    async InitializeTaxonomyBridge(contextUser) {
+        try {
+            // TagEngine internally ensures AIEngine is loaded before building embeddings.
+            await TagEngine.Instance.Config(false, contextUser);
+            LogStatus(`[TaxonomyBridge] TagEngine initialized with ${TagEngine.Instance.Tags.length} existing tags`);
+            // Inject taxonomy into prompt context as markdown hierarchy
+            // Format: "# RootTag\n## ChildTag\n### GrandChild"
+            // LLM returns paths like "RootTag / ChildTag / GrandChild" for unambiguous matching
+            if (TagEngine.Instance.Tags.length > 0) {
+                const tree = TagEngine.Instance.GetTaxonomyTree();
+                this.TaxonomyContext = this.buildTaxonomyMarkdown(tree);
+                LogStatus(`[TaxonomyBridge] Taxonomy context injected as markdown (${tree.length} root nodes, ${TagEngine.Instance.Tags.length} total tags)`);
+            }
+            // Set up the bridge callback — fires after each ContentItemTag is saved
+            this.OnContentItemTagSaved = async (contentItemTag, parentTagName, ctxUser) => {
+                await this.BridgeContentItemTagToTaxonomy(contentItemTag, parentTagName, ctxUser);
+            };
+            LogStatus(`[TaxonomyBridge] Bridge callback installed`);
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            LogError(`[TaxonomyBridge] Initialization failed, taxonomy features disabled: ${msg}`);
+        }
+    }
+    /**
+     * Clean up the taxonomy bridge after all providers have finished.
+     */
+    CleanupTaxonomyBridge() {
+        this.TaxonomyContext = null;
+        this.OnContentItemTagSaved = null;
+    }
+    /**
+     * Build a markdown-formatted taxonomy for LLM prompt injection.
+     * Uses heading levels for hierarchy depth (# for root, ## for child, etc.)
+     * so the LLM can return tag paths like "Root / Child / Grandchild".
+     */
+    buildTaxonomyMarkdown(tree) {
+        const lines = [];
+        const renderNode = (node, depth, path) => {
+            const prefix = '#'.repeat(Math.min(depth + 1, 6)); // Max 6 heading levels
+            const fullPath = path ? `${path} / ${node.Name}` : node.Name;
+            lines.push(`${prefix} ${node.Name}`);
+            if (node.Children && node.Children.length > 0) {
+                for (const child of node.Children) {
+                    renderNode(child, depth + 1, fullPath);
+                }
+            }
+        };
+        for (const root of tree) {
+            renderNode(root, 0, '');
+        }
+        return lines.join('\n');
+    }
+    /**
+     * Bridge a ContentItemTag to the formal MJ Tag taxonomy.
+     * Uses TagEngine.ResolveTag() in auto-grow mode by default.
+     *
+     * After resolving/creating the formal Tag, also creates a TaggedItem record:
+     * - For Entity sources: tags the original entity record (e.g., Products row)
+     * - For non-Entity sources (RSS, Website, etc.): tags the ContentItem itself
+     */
+    async BridgeContentItemTagToTaxonomy(contentItemTag, parentTagName, contextUser) {
+        try {
+            // If parent tag is suggested by LLM, resolve it through the mutex too
+            // to prevent duplicate parent tags from concurrent batch processing
+            if (parentTagName) {
+                await TagEngine.Instance.ResolveTag(parentTagName, 0, 'auto-grow', null, 0.80, contextUser);
+            }
+            // Resolve the tag using auto-grow mode (create if no match)
+            const formalTag = await TagEngine.Instance.ResolveTag(contentItemTag.Tag, contentItemTag.Weight, 'auto-grow', null, // no root constraint
+            0.80, // similarity threshold — lower to catch plurals/variants like "AI Agent" vs "AI Agents"
+            contextUser);
+            if (formalTag) {
+                // Link ContentItemTag to formal Tag
+                contentItemTag.TagID = formalTag.ID;
+                await contentItemTag.Save();
+                // Create TaggedItem linking the formal Tag to the appropriate entity record
+                await this.createTaggedItemFromContentItemTag(contentItemTag, formalTag.ID, contextUser);
+            }
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            LogError(`[TaxonomyBridge] Failed for tag "${contentItemTag.Tag}": ${msg}`);
+        }
+    }
+    /**
+     * Creates a TaggedItem record linking a formal Tag to the appropriate entity record.
+     *
+     * For Entity-sourced content items: resolves the EntityRecordDocument to find the
+     * original entity (e.g., Products) and record ID, then tags that entity record.
+     *
+     * For non-Entity-sourced content items (RSS, Website, Cloud Storage): tags the
+     * ContentItem itself (EntityID = "MJ: Content Items" entity, RecordID = content item ID).
+     */
+    async createTaggedItemFromContentItemTag(contentItemTag, tagID, contextUser) {
+        try {
+            const md = new Metadata();
+            let entityID;
+            let recordID;
+            // Load the content item to determine source type
+            const rv = new RunView();
+            const ciResult = await rv.RunView({
+                EntityName: 'MJ: Content Items',
+                ExtraFilter: `ID='${contentItemTag.ItemID}'`,
+                ResultType: 'simple',
+                Fields: ['ID', 'EntityRecordDocumentID', 'ContentSourceID'],
+                MaxRows: 1,
+            }, contextUser);
+            if (!ciResult.Success || ciResult.Results.length === 0)
+                return;
+            const ci = ciResult.Results[0];
+            if (ci.EntityRecordDocumentID) {
+                // Entity source — resolve to the original entity record
+                const erdResult = await rv.RunView({
+                    EntityName: 'MJ: Entity Record Documents',
+                    ExtraFilter: `ID='${ci.EntityRecordDocumentID}'`,
+                    ResultType: 'simple',
+                    Fields: ['EntityID', 'RecordID'],
+                    MaxRows: 1,
+                }, contextUser);
+                if (!erdResult.Success || erdResult.Results.length === 0)
+                    return;
+                entityID = erdResult.Results[0].EntityID;
+                recordID = erdResult.Results[0].RecordID;
+            }
+            else {
+                // Non-entity source — tag the ContentItem itself
+                const contentItemsEntity = md.Entities.find(e => e.Name === 'MJ: Content Items');
+                if (!contentItemsEntity)
+                    return;
+                entityID = contentItemsEntity.ID;
+                recordID = contentItemTag.ItemID;
+            }
+            // Check if this TaggedItem already exists (avoid duplicates)
+            const existingResult = await rv.RunView({
+                EntityName: 'MJ: Tagged Items',
+                ExtraFilter: `TagID='${tagID}' AND EntityID='${entityID}' AND RecordID='${recordID}'`,
+                ResultType: 'simple',
+                Fields: ['ID'],
+                MaxRows: 1,
+            }, contextUser);
+            if (existingResult.Success && existingResult.Results.length > 0)
+                return; // Already exists
+            // Create the TaggedItem
+            const taggedItem = await md.GetEntityObject('MJ: Tagged Items', contextUser);
+            taggedItem.NewRecord();
+            taggedItem.TagID = tagID;
+            taggedItem.EntityID = entityID;
+            taggedItem.RecordID = recordID;
+            taggedItem.Weight = contentItemTag.Weight;
+            await taggedItem.Save();
+        }
+        catch (e) {
+            // Non-critical — the ContentItemTag is already saved, TaggedItem is supplemental
+            const msg = e instanceof Error ? e.message : String(e);
+            LogError(`[TaxonomyBridge] Failed to create TaggedItem for tag "${contentItemTag.Tag}": ${msg}`);
+        }
+    }
+    /**
+     * Builds template data for the autotagging prompt from processing params and chunk context.
+     */
+    buildPromptData(params, chunk, previousResults) {
+        const contentSourceType = this.GetContentSourceTypeName(params.contentSourceTypeID);
+        const additionalAttributePrompts = this.GetAdditionalContentTypePrompt(params.contentTypeID);
+        const hasPreviousResults = Object.keys(previousResults).length > 0;
+        // Check if this source type requires content type validation in the prompt
+        const sourceType = this.ContentSourceTypes.find(st => UUIDsEqual(st.ID, params.contentSourceTypeID));
+        const sourceConfig = sourceType?.ConfigurationObject;
+        const requiresContentType = sourceConfig?.RequiresContentType !== false;
+        return {
+            contentType: requiresContentType ? this.GetContentTypeName(params.contentTypeID) : undefined,
+            contentSourceType,
+            minTags: params.minTags,
+            maxTags: params.maxTags,
+            additionalAttributePrompts,
+            existingTaxonomy: this.TaxonomyContext ?? undefined,
+            contentText: chunk,
+            previousResults: hasPreviousResults ? JSON.stringify(previousResults) : undefined,
+        };
+    }
+    async promptAndRetrieveResultsFromLLM(params, contextUser) {
+        await AIEngine.Instance.Config(false, contextUser);
+        const prompt = this.getAutotagPrompt();
+        const tokenLimit = this.resolveTokenLimit(params.modelID);
+        const chunks = this.chunkExtractedText(params.text, tokenLimit);
+        if (chunks.length === 0 || (chunks.length === 1 && (!chunks[0] || chunks[0].trim().length === 0))) {
+            LogError(`[Autotag] No text to process for item ${params.contentItemID}`);
+            return {};
         }
-        const chunks = this.chunkExtractedText(params.text, model.InputTokenLimit);
         let LLMResults = {};
         const startTime = new Date();
-        for (const chunk of chunks) {
-            const { systemPrompt, userPrompt } = await this.getLLMPrompts(params, chunk, LLMResults, contextUser);
-            LLMResults = await this.processChunkWithLLM(llm, systemPrompt, userPrompt, LLMResults, model.APIName);
+        for (let ci = 0; ci < chunks.length; ci++) {
+            try {
+                LLMResults = await this.processChunkWithPromptRunner(prompt, params, chunks[ci], LLMResults, contextUser);
+            }
+            catch (chunkError) {
+                LogError(`[Autotag] Chunk ${ci + 1}/${chunks.length} failed for item ${params.contentItemID}: ${chunkError instanceof Error ? chunkError.message : String(chunkError)}`);
+            }
         }
         LLMResults.processStartTime = startTime;
         LLMResults.processEndTime = new Date();
         LLMResults.contentItemID = params.contentItemID;
         return LLMResults;
     }
-    async processChunkWithLLM(llm, systemPrompt, userPrompt, LLMResults, modelAPIName) {
-        const response = await llm.ChatCompletion({
-            messages: [
-                { role: 'system', content: systemPrompt },
-                { role: 'user', content: userPrompt }
-            ],
-            model: modelAPIName,
-            temperature: 0.0,
-        });
-        const queryResponse = response.data.choices[0]?.message?.content?.trim() || '';
-        let JSONQueryResponse;
-        try {
-            JSONQueryResponse = JSON.parse(queryResponse);
+    /**
+     * Resolves the input token limit for chunking. Uses the model specified by modelID if available,
+     * otherwise falls back to a conservative default.
+     */
+    resolveTokenLimit(modelID) {
+        const DEFAULT_TOKEN_LIMIT = 100000;
+        if (modelID) {
+            const model = AIEngine.Instance.Models.find(m => UUIDsEqual(m.ID, modelID));
+            if (model) {
+                return model.InputTokenLimit;
+            }
+        }
+        return DEFAULT_TOKEN_LIMIT;
+    }
+    /**
+     * Processes a single text chunk using AIPromptRunner and merges results.
+     * Uses the prompt's configured model by default. If ContentType.AIModelID is set,
+     * it is passed as a runtime model override via AIPromptParams.override.
+     */
+    async processChunkWithPromptRunner(prompt, params, chunk, LLMResults, contextUser) {
+        const promptParams = new AIPromptParams();
+        promptParams.prompt = prompt;
+        promptParams.contextUser = contextUser;
+        promptParams.data = this.buildPromptData(params, chunk, LLMResults);
+        promptParams.skipValidation = false;
+        promptParams.attemptJSONRepair = true;
+        promptParams.additionalParameters = { temperature: 0.0 };
+        // If the ContentType specifies a preferred AI model, use it as a runtime override
+        if (params.modelID) {
+            promptParams.override = { modelId: params.modelID };
         }
-        catch (parseError) {
-            LogError('Failed to parse LLM response as JSON', undefined, queryResponse);
+        const runner = new AIPromptRunner();
+        const result = await runner.ExecutePrompt(promptParams);
+        if (!result.success) {
+            LogError(`[Autotag] LLM failed for item ${params.contentItemID}: ${result.errorMessage ?? 'unknown error'}`);
             return LLMResults;
         }
-        for (const key in JSONQueryResponse) {
-            const value = JSONQueryResponse[key];
-            if (value !== null) {
-                LLMResults[key] = value;
+        // Parse the result — AIPromptRunner may return a raw JSON string or a parsed object
+        let chunkResult = null;
+        if (typeof result.result === 'string') {
+            try {
+                chunkResult = JSON.parse(result.result);
+            }
+            catch {
+                LogError(`Failed to parse LLM result as JSON for item ${params.contentItemID}: ${String(result.result).substring(0, 200)}`);
+                return LLMResults;
+            }
+        }
+        else {
+            chunkResult = result.result;
+        }
+        // Merge results from this chunk into the accumulated results
+        if (chunkResult) {
+            for (const key in chunkResult) {
+                const value = chunkResult[key];
+                if (value !== null) {
+                    LLMResults[key] = value;
+                }
             }
         }
         return LLMResults;
     }
-    async getLLMPrompts(params, chunk, LLMResults, contextUser) {
-        const contentType = this.GetContentTypeName(params.contentTypeID);
-        const contentSourceType = this.GetContentSourceTypeName(params.contentSourceTypeID);
-        const additionalContentTypePrompts = this.GetAdditionalContentTypePrompt(params.contentTypeID);
-        const systemPrompt = `You are a highly skilled text analysis assistant. You have decades of experience and pride yourself on your attention to detail and ability to capture both accurate information, as well as tone and subtext.
-        Your task is to accurately extract key information from a provided piece of text based on a series of prompts. You are provided with text that should be a ${contentType}, that has been extracted from a ${contentSourceType}.
-        The text MUST be of the type ${contentType} for the subsequent processing.`;
-        const userPrompt = `
-        If the provided text does not actually appear to be of the type ${contentType}, please disregard everything in the instructions after this and return this exact JSON response: { isValidContent: false (as a boolean) }.
-        Assuming the type of the text is in fact from a ${contentType}, please extract the title of the provided text, a short summary of the provided documents, as well as between ${params.minTags} and ${params.maxTags} topical key words that are most relevant to the text.
-        If there is no title explicitly provided in the text, please provide a title that you think best represents the text.
-        Please provide the keywords in a list format.
-        Make sure the response is just the json file without and formatting or code blocks, and strictly following the format below. Please don't include a greeting in the response, only output the json file:
-
-        {
-            "title": (title here),
-            "description": (description here),
-            "keywords": (list keywords here),
-            "isValidContent": true (as a boolean)
-        }
-
-        ${additionalContentTypePrompts}
-
-        Please make sure the response in is valid JSON format.
-
-        You are also provided with the results so far as additional context, please use them to formulate the best results given the provided text: ${JSON.stringify(LLMResults)}
-        The supplied text is: ${chunk}
-        `;
-        return { systemPrompt, userPrompt };
-    }
     async saveLLMResults(LLMResults, contextUser) {
         if (LLMResults.isValidContent === true) {
             await this.saveResultsToContentItemAttribute(LLMResults, contextUser);
             await this.saveContentItemTags(LLMResults.contentItemID, LLMResults, contextUser);
-            LogStatus(`Results for content item ${LLMResults.contentItemID} saved successfully`);
         }
-        else {
+        else if (LLMResults.isValidContent === false) {
             await this.deleteInvalidContentItem(LLMResults.contentItemID, contextUser);
         }
+        else {
+            LogError(`[Autotag] Unexpected LLM format for item ${LLMResults.contentItemID} — isValidContent missing. Keys: ${Object.keys(LLMResults).join(', ')}`);
+        }
     }
     async deleteInvalidContentItem(contentItemID, contextUser) {
         const md = new Metadata();
@@ -264,21 +620,49 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
     /**
      * Saves keyword tags from LLM results as Content Item Tags.
      * Uses batched saves for better performance.
+     * After each tag is saved, invokes the OnContentItemTagSaved callback (if set)
+     * for taxonomy bridge processing.
      */
     async saveContentItemTags(contentItemID, LLMResults, contextUser) {
         const md = new Metadata();
         const keywords = LLMResults.keywords;
         if (!keywords || !Array.isArray(keywords))
             return;
+        // Normalize keywords — support both formats:
+        //   Old: ["keyword1", "keyword2"]
+        //   New: [{ tag: "keyword1", weight: 0.95 }, { tag: "keyword2", weight: 0.7 }]
+        //   New with parentTag: [{ tag: "keyword1", weight: 0.95, parentTag: "parent" }]
+        const normalizedTags = keywords.map((kw) => {
+            if (typeof kw === 'string') {
+                return { tag: kw, weight: 1.0, parentTag: null };
+            }
+            const obj = kw;
+            return {
+                tag: obj.tag || obj.keyword || String(kw),
+                weight: typeof obj.weight === 'number' ? Math.max(0, Math.min(1, obj.weight)) : 0.5,
+                parentTag: obj.parentTag ?? null,
+            };
+        });
         const BATCH_SIZE = 10;
-        for (let i = 0; i < keywords.length; i += BATCH_SIZE) {
-            const batch = keywords.slice(i, i + BATCH_SIZE);
-            await Promise.all(batch.map(async (keyword) => {
+        for (let i = 0; i < normalizedTags.length; i += BATCH_SIZE) {
+            const batch = normalizedTags.slice(i, i + BATCH_SIZE);
+            await Promise.all(batch.map(async (item) => {
                 const contentItemTag = await md.GetEntityObject('MJ: Content Item Tags', contextUser);
                 contentItemTag.NewRecord();
                 contentItemTag.ItemID = contentItemID;
-                contentItemTag.Tag = keyword;
-                await contentItemTag.Save();
+                contentItemTag.Tag = item.tag;
+                contentItemTag.Set('Weight', item.weight);
+                const saved = await contentItemTag.Save();
+                // Invoke taxonomy bridge callback if set
+                if (saved && this.OnContentItemTagSaved) {
+                    try {
+                        await this.OnContentItemTagSaved(contentItemTag, item.parentTag, contextUser);
+                    }
+                    catch (bridgeError) {
+                        const msg = bridgeError instanceof Error ? bridgeError.message : String(bridgeError);
+                        LogError(`Tag taxonomy bridge failed for tag "${item.tag}": ${msg}`);
+                    }
+                }
             }));
         }
     }
@@ -290,12 +674,17 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
         const md = new Metadata();
         const contentItemID = LLMResults.contentItemID;
         const skipKeys = new Set(['keywords', 'processStartTime', 'processEndTime', 'contentItemID', 'isValidContent']);
-        // Update title and description on the content item
+        // Update title and description on the content item.
+        // For entity-sourced items (EntityRecordDocumentID is set), preserve the
+        // original entity record name — it's more meaningful to users than the
+        // AI-generated title. Only update description.
         if (LLMResults.title || LLMResults.description) {
             const contentItem = await md.GetEntityObject('MJ: Content Items', contextUser);
             await contentItem.Load(contentItemID);
-            if (LLMResults.title)
+            const isEntitySourced = contentItem.EntityRecordDocumentID != null;
+            if (LLMResults.title && !isEntitySourced) {
                 contentItem.Name = LLMResults.title;
+            }
             if (LLMResults.description)
                 contentItem.Description = LLMResults.description;
             await contentItem.Save();
@@ -317,21 +706,24 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
     }
     /**
      * Retrieves all content sources for a given content source type.
+     * Throws if no sources are found.
      */
     async getAllContentSources(contextUser, contentSourceTypeID) {
-        const rv = new RunView();
-        const result = await rv.RunView({
-            EntityName: 'MJ: Content Sources',
-            ResultType: 'entity_object',
-            ExtraFilter: `ContentSourceTypeID='${contentSourceTypeID}'`
-        }, contextUser);
-        if (result.Success && result.Results.length) {
-            return result.Results;
+        const sources = await this.GetAllContentSourcesSafe(contextUser, contentSourceTypeID);
+        if (sources.length === 0) {
+            throw new Error(`No content sources found for content source type with ID '${contentSourceTypeID}'`);
         }
-        throw new Error(`No content sources found for content source type with ID '${contentSourceTypeID}'`);
+        return sources;
+    }
+    /**
+     * Retrieves all content sources for a given content source type.
+     * Returns an empty array (instead of throwing) when no sources are configured.
+     */
+    async GetAllContentSourcesSafe(_contextUser, contentSourceTypeID) {
+        return this.khEngine.ContentSources.filter(s => UUIDsEqual(s.ContentSourceTypeID, contentSourceTypeID));
     }
     SetSubclassContentSourceType(subclass) {
-        const sourceType = this._ContentSourceTypes.find(st => st.Name === subclass);
+        const sourceType = this.ContentSourceTypes.find(st => st.Name === subclass);
         if (!sourceType) {
             throw new Error(`Content Source Type with name '${subclass}' not found in cached metadata`);
         }
@@ -421,7 +813,7 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
         throw new Error(`Failed to retrieve last run date for content source with ID ${contentSourceID}`);
     }
     GetContentItemParams(contentTypeID) {
-        const contentType = this._ContentTypes.find(ct => UUIDsEqual(ct.ID, contentTypeID));
+        const contentType = this.ContentTypes.find(ct => UUIDsEqual(ct.ID, contentTypeID));
         if (!contentType) {
             throw new Error(`Content Type with ID ${contentTypeID} not found in cached metadata`);
         }
@@ -432,21 +824,21 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
         };
     }
     GetContentSourceTypeName(contentSourceTypeID) {
-        const sourceType = this._ContentSourceTypes.find(st => UUIDsEqual(st.ID, contentSourceTypeID));
+        const sourceType = this.ContentSourceTypes.find(st => UUIDsEqual(st.ID, contentSourceTypeID));
         if (!sourceType) {
             throw new Error(`Content Source Type with ID ${contentSourceTypeID} not found in cached metadata`);
         }
         return sourceType.Name;
     }
     GetContentTypeName(contentTypeID) {
-        const contentType = this._ContentTypes.find(ct => UUIDsEqual(ct.ID, contentTypeID));
+        const contentType = this.ContentTypes.find(ct => UUIDsEqual(ct.ID, contentTypeID));
         if (!contentType) {
             throw new Error(`Content Type with ID ${contentTypeID} not found in cached metadata`);
         }
         return contentType.Name;
     }
     GetContentFileTypeName(contentFileTypeID) {
-        const fileType = this._ContentFileTypes.find(ft => UUIDsEqual(ft.ID, contentFileTypeID));
+        const fileType = this.ContentFileTypes.find(ft => UUIDsEqual(ft.ID, contentFileTypeID));
         if (!fileType) {
             throw new Error(`Content File Type with ID ${contentFileTypeID} not found in cached metadata`);
         }
@@ -486,7 +878,7 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
         throw new Error(`Content item with URL ${url} not found`);
     }
     /**
-     * Saves process run metadata to the database.
+     * Saves process run metadata to the database (backward-compatible simple version).
      */
     async saveProcessRun(processRunParams, contextUser) {
         const md = new Metadata();
@@ -495,10 +887,92 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
         processRun.SourceID = processRunParams.sourceID;
         processRun.StartTime = processRunParams.startTime;
         processRun.EndTime = processRunParams.endTime;
-        processRun.Status = 'Complete';
+        processRun.Status = 'Completed';
         processRun.ProcessedItems = processRunParams.numItemsProcessed;
+        processRun.StartedByUserID = contextUser.ID;
         await processRun.Save();
     }
+    /**
+     * Create a new ContentProcessRun record for batched pipeline execution.
+     * Returns the entity so the caller can update cursor/status as batches complete.
+     * Uses the JSONType ConfigurationObject for strongly-typed configuration.
+     */
+    async CreateBatchedProcessRun(sourceID, totalItemCount, batchSize, contextUser, config) {
+        const md = new Metadata();
+        const processRun = await md.GetEntityObject('MJ: Content Process Runs', contextUser);
+        processRun.NewRecord();
+        processRun.SourceID = sourceID;
+        processRun.StartTime = new Date();
+        processRun.Status = 'Running';
+        processRun.ProcessedItems = 0;
+        processRun.TotalItemCount = totalItemCount;
+        processRun.BatchSize = batchSize;
+        processRun.LastProcessedOffset = 0;
+        processRun.ErrorCount = 0;
+        processRun.CancellationRequested = false;
+        processRun.StartedByUserID = contextUser.ID;
+        if (config) {
+            processRun.ConfigurationObject = config;
+        }
+        const saved = await processRun.Save();
+        if (!saved) {
+            throw new Error('Failed to create ContentProcessRun record');
+        }
+        return processRun;
+    }
+    /**
+     * Update a batched process run's cursor position after a batch completes.
+     * Checks CancellationRequested to support pause/cancel.
+     * @returns true if processing should continue, false if cancelled/paused
+     */
+    async UpdateBatchCursor(processRun, processedCount, errorCount) {
+        processRun.ProcessedItems = processedCount;
+        processRun.LastProcessedOffset = processedCount;
+        processRun.ErrorCount = errorCount;
+        await processRun.Save();
+        // Reload to check if cancellation was requested externally
+        await processRun.Load(processRun.ID);
+        if (processRun.CancellationRequested) {
+            processRun.Status = 'Paused';
+            processRun.EndTime = new Date();
+            await processRun.Save();
+            LogStatus(`[Pipeline] Cancellation requested — pausing at offset ${processedCount}`);
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Complete a batched process run (success or failure).
+     */
+    async CompleteBatchedProcessRun(processRun, status, errorMessage) {
+        processRun.Status = status;
+        processRun.EndTime = new Date();
+        if (errorMessage) {
+            processRun.ErrorMessage = errorMessage;
+        }
+        await processRun.Save();
+    }
+    /**
+     * Create rate limiters from the pipeline configuration.
+     */
+    CreateRateLimiters(config) {
+        return {
+            llm: new RateLimiter({
+                RequestsPerMinute: config?.RateLimits?.LLM?.RequestsPerMinute ?? 60,
+                TokensPerMinute: config?.RateLimits?.LLM?.TokensPerMinute ?? 1000000,
+                Name: 'LLM',
+            }),
+            embedding: new RateLimiter({
+                RequestsPerMinute: config?.RateLimits?.Embedding?.RequestsPerMinute ?? 300,
+                TokensPerMinute: config?.RateLimits?.Embedding?.TokensPerMinute ?? 1000000,
+                Name: 'Embedding',
+            }),
+            vectorDB: new RateLimiter({
+                RequestsPerMinute: config?.RateLimits?.VectorDB?.RequestsPerMinute ?? 200,
+                Name: 'VectorDB',
+            }),
+        };
+    }
     async parsePDF(dataBuffer) {
         const dataPDF = await pdfParse(dataBuffer);
         return dataPDF.text;
@@ -530,8 +1004,768 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
                 throw new Error(`File type '${fileExtension}' not supported`);
         }
     }
+    // ---- Direct Vectorization ----
+    /**
+     * Embeds content items and upserts them to the appropriate vector index.
+     * Items are grouped by their resolved (embeddingModel + vectorIndex) pair — derived
+     * from per-ContentSource overrides, per-ContentType defaults, or the global fallback
+     * (first active VectorIndex). Each group is processed in configurable batches with
+     * parallel upserts within each batch.
+     *
+     * Uses AIModelRunner to create AIPromptRun records for each embedding batch,
+     * enabling token/cost tracking and linking to ContentProcessRunDetail records.
+     *
+     * @param items - content items to vectorize
+     * @param contextUser - current user for permissions/audit
+     * @param onProgress - optional callback for progress updates
+     * @param batchSize - number of items per embedding batch
+     * @returns counts of vectorized/skipped items and collected AIPromptRun IDs
+     */
+    async VectorizeContentItems(items, contextUser, onProgress, batchSize = DEFAULT_VECTORIZE_BATCH_SIZE) {
+        const eligible = items.filter(i => i.Text && i.Text.trim().length > 0);
+        if (eligible.length === 0) {
+            LogStatus('VectorizeContentItems: no items with text to vectorize');
+            return { vectorized: 0, skipped: items.length, promptRunIDs: [] };
+        }
+        // Ensure AIEngine is loaded so we can resolve the embedding model
+        await AIEngine.Instance.Config(false, contextUser);
+        // Load content sources + types for per-item infrastructure resolution
+        const { sourceMap, typeMap } = await this.loadContentSourceAndTypeMaps(eligible, contextUser);
+        // Group items by their resolved (embeddingModelID + vectorIndexID) pair
+        const groups = this.groupItemsByInfrastructure(eligible, sourceMap, typeMap);
+        // Load tags for all items in one query
+        const tagMap = await this.loadTagsForItems(eligible, contextUser);
+        let vectorized = 0;
+        let processed = 0;
+        const allPromptRunIDs = [];
+        for (const [groupKey, groupItems] of groups) {
+            const infra = await this.resolveGroupInfrastructure(groupKey, contextUser);
+            const groupResult = await this.vectorizeGroup(groupItems, infra, tagMap, batchSize, contextUser, (batchProcessed) => {
+                processed += batchProcessed;
+                onProgress?.(Math.min(processed, eligible.length), eligible.length);
+            });
+            vectorized += groupResult.vectorized;
+            allPromptRunIDs.push(...groupResult.promptRunIDs);
+        }
+        LogStatus(`VectorizeContentItems: ${vectorized} vectorized, ${items.length - eligible.length} skipped (empty text), ${allPromptRunIDs.length} prompt runs created`);
+        return { vectorized, skipped: items.length - eligible.length, promptRunIDs: allPromptRunIDs };
+    }
+    /**
+     * Process a single infrastructure group: embed texts in batches and upsert to vector DB.
+     * Uses AIModelRunner for each embedding batch to create AIPromptRun records with
+     * token/cost tracking. Upserts within each batch run in parallel for throughput.
+     *
+     * @param items - content items in this infrastructure group
+     * @param infra - resolved embedding + vector DB infrastructure
+     * @param tagMap - pre-loaded tags for metadata enrichment
+     * @param batchSize - number of items per embedding batch
+     * @param contextUser - current user for AIModelRunner tracking
+     * @param onBatchComplete - callback invoked after each batch with item count
+     * @returns count of vectorized items and collected AIPromptRun IDs
+     */
+    async vectorizeGroup(items, infra, tagMap, batchSize, contextUser, onBatchComplete) {
+        let vectorized = 0;
+        const promptRunIDs = [];
+        const modelRunner = new AIModelRunner();
+        // Resolve the "Content Embedding" prompt ID for tracking
+        const embeddingPromptID = this.resolveEmbeddingPromptID();
+        for (let i = 0; i < items.length; i += batchSize) {
+            const batch = items.slice(i, i + batchSize);
+            // Build chunks for each item — items with long text produce multiple chunks
+            const allChunks = this.buildChunksForBatch(batch);
+            const texts = allChunks.map(c => c.text);
+            // Rate limit embedding API call
+            await this.EmbeddingRateLimiter.Acquire(texts.reduce((sum, t) => sum + Math.ceil(t.length / 4), 0));
+            // Use AIModelRunner to embed texts with AIPromptRun tracking
+            const runResult = await modelRunner.RunEmbedding({
+                Texts: texts,
+                ModelID: infra.embeddingModelID,
+                PromptID: embeddingPromptID,
+                ContextUser: contextUser,
+                Description: `Content vectorization batch: ${batch.length} items, ${allChunks.length} chunks`,
+            });
+            if (!runResult.Success || runResult.Vectors.length !== allChunks.length) {
+                LogError(`VectorizeContentItems: embedding returned ${runResult.Vectors.length} vectors for ${allChunks.length} texts — ${runResult.ErrorMessage ?? 'unknown error'}`);
+                onBatchComplete(batch.length);
+                continue;
+            }
+            // Track the AIPromptRun ID for junction table linking
+            if (runResult.PromptRunID) {
+                promptRunIDs.push(runResult.PromptRunID);
+            }
+            const records = this.buildVectorRecords(allChunks, runResult.Vectors, tagMap);
+            const batchSuccess = await this.upsertVectorRecords(records, infra);
+            if (batchSuccess) {
+                vectorized += batch.length;
+            }
+            onBatchComplete(batch.length);
+        }
+        return { vectorized, promptRunIDs };
+    }
+    /**
+     * Resolve the "Content Embedding" prompt ID from AIEngine for AIModelRunner tracking.
+     * Returns undefined if the prompt is not found (AIModelRunner will fall back to
+     * the first active Embedding-type prompt).
+     */
+    resolveEmbeddingPromptID() {
+        const prompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Content Embedding' && p.Status === 'Active');
+        if (prompt) {
+            return prompt.ID;
+        }
+        // Fall back: let AIModelRunner find the first active Embedding prompt
+        LogStatus('[Autotag] "Content Embedding" prompt not found — AIModelRunner will use default embedding prompt');
+        return undefined;
+    }
+    /**
+     * Build text chunks for a batch of content items. Items with long text
+     * produce multiple chunks via TextChunker.
+     */
+    buildChunksForBatch(batch) {
+        const allChunks = [];
+        for (const item of batch) {
+            const chunks = this.buildEmbeddingChunks(item);
+            for (let ci = 0; ci < chunks.length; ci++) {
+                allChunks.push({ item, chunkIndex: ci, text: chunks[ci] });
+            }
+        }
+        return allChunks;
+    }
+    /**
+     * Build VectorRecord objects from embedding chunks and their corresponding vectors.
+     */
+    buildVectorRecords(allChunks, vectors, tagMap) {
+        return allChunks.map((chunk, idx) => ({
+            id: chunk.chunkIndex === 0
+                ? this.contentItemVectorId(chunk.item.ID)
+                : this.contentItemVectorId(chunk.item.ID) + `_chunk${chunk.chunkIndex}`,
+            values: vectors[idx],
+            metadata: this.buildVectorMetadata(chunk.item, tagMap.get(chunk.item.ID))
+        }));
+    }
+    /**
+     * Upsert vector records to the vector database in sub-batches with rate limiting.
+     * Returns true if all sub-batches succeeded.
+     */
+    async upsertVectorRecords(records, infra) {
+        const UPSERT_CHUNK = 50;
+        const upsertPromises = [];
+        for (let j = 0; j < records.length; j += UPSERT_CHUNK) {
+            const chunk = records.slice(j, j + UPSERT_CHUNK);
+            await this.VectorDBRateLimiter.Acquire();
+            upsertPromises.push(Promise.resolve(infra.vectorDB.CreateRecords(chunk, infra.indexName)));
+        }
+        const responses = await Promise.all(upsertPromises);
+        let allSuccess = true;
+        for (const response of responses) {
+            if (!response.success) {
+                LogError(`VectorizeContentItems: upsert failed: ${response.message}`);
+                allSuccess = false;
+            }
+        }
+        return allSuccess;
+    }
+    /**
+     * Load content source and content type records for all unique source/type IDs
+     * referenced by the given items. Returns maps keyed by normalized ID.
+     */
+    async loadContentSourceAndTypeMaps(items, _contextUser) {
+        const sourceIdSet = new Set(items.map(i => NormalizeUUID(i.ContentSourceID)));
+        const typeIdSet = new Set(items.map(i => NormalizeUUID(i.ContentTypeID)));
+        // Use KH engine cached data instead of RunView calls
+        const sourceMap = new Map();
+        for (const src of this.khEngine.ContentSources) {
+            if (sourceIdSet.has(NormalizeUUID(src.ID))) {
+                sourceMap.set(NormalizeUUID(src.ID), src.GetAll());
+            }
+        }
+        const typeMap = new Map();
+        for (const ct of this.ContentTypes) {
+            if (typeIdSet.has(NormalizeUUID(ct.ID))) {
+                typeMap.set(NormalizeUUID(ct.ID), ct.GetAll());
+            }
+        }
+        return { sourceMap, typeMap };
+    }
+    /**
+     * Resolve the (embeddingModelID, vectorIndexID) pair for a content item using
+     * the cascade: ContentSource override -> ContentType default -> null (global fallback).
+     */
+    resolveItemInfrastructureIds(item, sourceMap, typeMap) {
+        const source = sourceMap.get(NormalizeUUID(item.ContentSourceID));
+        if (source) {
+            const srcEmbedding = source['EmbeddingModelID'];
+            const srcVector = source['VectorIndexID'];
+            if (srcEmbedding && srcVector) {
+                return { embeddingModelID: srcEmbedding, vectorIndexID: srcVector };
+            }
+        }
+        const contentType = typeMap.get(NormalizeUUID(item.ContentTypeID));
+        if (contentType) {
+            const typeEmbedding = contentType['EmbeddingModelID'];
+            const typeVector = contentType['VectorIndexID'];
+            if (typeEmbedding && typeVector) {
+                return { embeddingModelID: typeEmbedding, vectorIndexID: typeVector };
+            }
+        }
+        // Global fallback — will be resolved in resolveGroupInfrastructure
+        return { embeddingModelID: null, vectorIndexID: null };
+    }
+    /**
+     * Group items by their resolved (embeddingModelID + vectorIndexID) key.
+     * Items with the same pair share infrastructure and can be batched together.
+     */
+    groupItemsByInfrastructure(items, sourceMap, typeMap) {
+        const groups = new Map();
+        for (const item of items) {
+            const { embeddingModelID, vectorIndexID } = this.resolveItemInfrastructureIds(item, sourceMap, typeMap);
+            const key = this.infraGroupKey(embeddingModelID, vectorIndexID);
+            const group = groups.get(key) ?? [];
+            group.push(item);
+            groups.set(key, group);
+        }
+        return groups;
+    }
+    /** Create a stable cache key for an (embeddingModelID, vectorIndexID) pair */
+    infraGroupKey(embeddingModelID, vectorIndexID) {
+        const e = embeddingModelID ? NormalizeUUID(embeddingModelID) : 'default';
+        const v = vectorIndexID ? NormalizeUUID(vectorIndexID) : 'default';
+        return `${e}|${v}`;
+    }
+    /**
+     * Resolve a group key into concrete infrastructure instances. For the 'default|default'
+     * key, falls back to the first active VectorIndex (original behavior).
+     */
+    async resolveGroupInfrastructure(groupKey, contextUser) {
+        const [embeddingPart, vectorPart] = groupKey.split('|');
+        const isDefault = embeddingPart === 'default' || vectorPart === 'default';
+        if (isDefault) {
+            return this.getDefaultVectorInfrastructure(contextUser);
+        }
+        return this.buildVectorInfrastructure(embeddingPart, vectorPart, contextUser);
+    }
+    /**
+     * Build infrastructure from explicit embeddingModelID and vectorIndexID.
+     * Looks up the vector index by ID and the embedding model from AIEngine.
+     */
+    async buildVectorInfrastructure(embeddingModelID, vectorIndexID, _contextUser) {
+        const vectorIndex = this.khEngine.GetVectorIndexById(vectorIndexID);
+        if (!vectorIndex) {
+            throw new Error(`Vector index ${vectorIndexID} not found in KnowledgeHubMetadataEngine cache`);
+        }
+        return this.createInfrastructureFromIndex(vectorIndex.Name, vectorIndex.VectorDatabaseID, embeddingModelID);
+    }
+    /**
+     * Fallback: resolve infrastructure from the first available VectorIndex (original behavior).
+     */
+    async getDefaultVectorInfrastructure(_contextUser) {
+        const vectorIndexes = this.khEngine.VectorIndexes;
+        if (vectorIndexes.length === 0) {
+            throw new Error('No vector indexes found — create one in the Configuration tab first');
+        }
+        const vectorIndex = vectorIndexes[0];
+        return this.createInfrastructureFromIndex(vectorIndex.Name, vectorIndex.VectorDatabaseID, vectorIndex.EmbeddingModelID);
+    }
+    /**
+     * Shared helper: given vector index details and embedding model ID, resolve all
+     * driver instances needed for embedding + upsert. Uses AIEngine for Vector Databases.
+     */
+    async createInfrastructureFromIndex(indexName, vectorDatabaseID, embeddingModelID) {
+        const vectorDBEntity = AIEngine.Instance.VectorDatabases.find(db => UUIDsEqual(db.ID, vectorDatabaseID));
+        if (!vectorDBEntity || !vectorDBEntity.ClassKey) {
+            throw new Error(`Vector database ${vectorDatabaseID} not found in AIEngine cache`);
+        }
+        const vectorDBClassKey = vectorDBEntity.ClassKey;
+        const aiModel = this.findEmbeddingModel(embeddingModelID);
+        const driverClass = aiModel.DriverClass;
+        const embeddingModelName = aiModel.APIName ?? aiModel.Name;
+        LogStatus(`VectorizeContentItems: USING embedding model "${aiModel.Name}" (${driverClass}), vector DB "${vectorDBClassKey}", index "${indexName}"`);
+        const embedding = this.createEmbeddingInstance(driverClass);
+        const vectorDB = this.createVectorDBInstance(vectorDBClassKey);
+        return { embedding, vectorDB, indexName, embeddingModelName, embeddingModelID };
+    }
+    /** Find an embedding model by ID in AIEngine, with helpful error reporting */
+    findEmbeddingModel(embeddingModelID) {
+        const aiModel = AIEngine.Instance.Models.find(m => UUIDsEqual(m.ID, embeddingModelID));
+        if (!aiModel) {
+            const embModels = AIEngine.Instance.Models.filter(m => m.DriverClass?.includes('Embed') || m.Name?.includes('embed'));
+            LogError(`VectorizeContentItems: embeddingModelID ${embeddingModelID} NOT FOUND. Available: ${JSON.stringify(embModels.map(m => ({ id: m.ID, name: m.Name, driver: m.DriverClass })))}`);
+            throw new Error(`Embedding model ${embeddingModelID} not found in AIEngine — ensure AIEngine is configured`);
+        }
+        return aiModel;
+    }
+    /** Create a BaseEmbeddings instance for a given driver class */
+    createEmbeddingInstance(driverClass) {
+        const apiKey = GetAIAPIKey(driverClass);
+        if (!apiKey) {
+            throw new Error(`No API key found for embedding driver ${driverClass} — set AI_VENDOR_API_KEY__${driverClass} in .env`);
+        }
+        const instance = MJGlobal.Instance.ClassFactory.CreateInstance(BaseEmbeddings, driverClass, apiKey);
+        if (!instance)
+            throw new Error(`Failed to create embedding instance for ${driverClass}`);
+        return instance;
+    }
+    /** Create a VectorDBBase instance for a given class key */
+    createVectorDBInstance(classKey) {
+        const apiKey = GetAIAPIKey(classKey);
+        if (!apiKey) {
+            throw new Error(`No API key found for vector DB ${classKey} — set AI_VENDOR_API_KEY__${classKey} in .env`);
+        }
+        const instance = MJGlobal.Instance.ClassFactory.CreateInstance(VectorDBBase, classKey, apiKey);
+        if (!instance)
+            throw new Error(`Failed to create vector DB instance for ${classKey}`);
+        return instance;
+    }
+    /** SHA-1 deterministic vector ID for a content item */
+    contentItemVectorId(contentItemId) {
+        return crypto.createHash('sha1').update(`content-item_${contentItemId}`).digest('hex');
+    }
+    /** Build the text that gets embedded: Title + Description + full Text */
+    /**
+     * Max tokens per embedding chunk. text-embedding-3-small supports 8,191 tokens.
+     * We use a conservative limit to avoid hitting the boundary.
+     */
+    static { this.MAX_EMBEDDING_TOKENS = 7500; }
+    /**
+     * Build the text to embed for a content item, and chunk it if it exceeds
+     * the embedding model's token limit. Returns one or more text chunks.
+     */
+    buildEmbeddingChunks(item) {
+        const parts = [];
+        if (item.Name)
+            parts.push(item.Name);
+        if (item.Description)
+            parts.push(item.Description);
+        if (item.Text)
+            parts.push(item.Text);
+        const full = parts.join('\n');
+        // Rough char estimate: 1 token ≈ 4 chars
+        const charLimit = AutotagBaseEngine_1.MAX_EMBEDDING_TOKENS * 4;
+        if (full.length <= charLimit) {
+            return [full];
+        }
+        // Chunk using TextChunker for token-aware splitting
+        LogStatus(`[Autotag] Chunking embedding text for "${item.Name}" (${full.length} chars, ~${Math.ceil(full.length / 4)} tokens)`);
+        try {
+            const chunkParams = {
+                Text: full,
+                MaxChunkTokens: AutotagBaseEngine_1.MAX_EMBEDDING_TOKENS,
+                OverlapTokens: 100,
+            };
+            const chunks = TextChunker.ChunkText(chunkParams);
+            LogStatus(`[Autotag] Split into ${chunks.length} chunks for embedding`);
+            return chunks.map(c => c.Text);
+        }
+        catch {
+            // Fallback: simple character-based splitting
+            const result = [];
+            for (let i = 0; i < full.length; i += charLimit) {
+                result.push(full.substring(i, i + charLimit));
+            }
+            return result;
+        }
+    }
+    /** Build metadata stored alongside the vector — truncate large text fields */
+    buildVectorMetadata(item, tags) {
+        const META_TEXT_LIMIT = 1000;
+        const meta = {
+            RecordID: item.ID,
+            Entity: 'MJ: Content Items',
+            ContentSourceID: item.ContentSourceID,
+            ContentSourceTypeID: item.ContentSourceTypeID,
+        };
+        if (item.Name)
+            meta['Title'] = item.Name.substring(0, META_TEXT_LIMIT);
+        if (item.Description)
+            meta['Description'] = item.Description.substring(0, META_TEXT_LIMIT);
+        if (item.URL)
+            meta['URL'] = item.URL;
+        if (tags && tags.length > 0)
+            meta['Tags'] = tags;
+        return meta;
+    }
+    /** Load all tags for the given items in a single RunView call */
+    async loadTagsForItems(items, contextUser) {
+        const tagMap = new Map();
+        const rv = new RunView();
+        const ids = items.map(i => `'${i.ID}'`).join(',');
+        const result = await rv.RunView({
+            EntityName: 'MJ: Content Item Tags',
+            ExtraFilter: `ItemID IN (${ids})`,
+            ResultType: 'entity_object'
+        }, contextUser);
+        if (result.Success) {
+            for (const tag of result.Results) {
+                const existing = tagMap.get(tag.ItemID) ?? [];
+                existing.push(tag.Tag);
+                tagMap.set(tag.ItemID, existing);
+            }
+        }
+        return tagMap;
+    }
+    // ---- Content Deduplication ----
+    /**
+     * Attempts to recompute tag co-occurrence data after the LLM tagging pipeline completes.
+     * Uses dynamic import to avoid a hard dependency on the tag-engine package.
+     * If TagCoOccurrenceEngine is not available or fails, it logs a warning and continues.
+     */
+    async recomputeCoOccurrenceIfAvailable(contextUser) {
+        try {
+            // Dynamic check: TagCoOccurrenceEngine is registered via class factory
+            const { TagCoOccurrenceEngine } = await import('@memberjunction/tag-engine');
+            const engine = TagCoOccurrenceEngine.Instance;
+            if (engine && typeof engine.RecomputeCoOccurrence === 'function') {
+                LogStatus('[Autotag] Recomputing tag co-occurrence after pipeline completion...');
+                const result = await engine.RecomputeCoOccurrence(contextUser);
+                LogStatus(`[Autotag] Co-occurrence recompute complete: ${result.PairsUpdated} pairs updated, ${result.PairsDeleted} deleted`);
+            }
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            LogStatus(`[Autotag] Co-occurrence recompute skipped (not available): ${msg}`);
+        }
+    }
+    /**
+     * Detects duplicate content items by matching the given item's checksum against
+     * other content items from **different** content sources. When an exact checksum
+     * match is found, a {@link MJContentItemDuplicateEntity} record is created with
+     * `DetectionMethod = 'Checksum'` and `SimilarityScore = 1.0`.
+     *
+     * Duplicate pairs are stored in canonical order (lower ID = ContentItemAID) to
+     * prevent mirror duplicates. If a duplicate pair already exists for the same
+     * detection method, no new record is created.
+     *
+     * @param contentItem - The content item whose checksum should be checked for duplicates.
+     *                      Must already be saved (i.e., have a valid ID and Checksum).
+     * @param contextUser - The authenticated user context for data access and audit.
+     * @returns A promise that resolves when detection is complete. Does not throw on
+     *          failure — errors are logged and swallowed to avoid disrupting the pipeline.
+     */
+    async DetectChecksumDuplicates(contentItem, contextUser) {
+        if (!contentItem.Checksum) {
+            return; // No checksum to compare
+        }
+        try {
+            const matches = await this.findItemsByChecksum(contentItem.Checksum, contentItem.ContentSourceID, contentItem.ID, contextUser);
+            for (const match of matches) {
+                await this.createDuplicateRecordIfNotExists(contentItem.ID, match.ID, 1.0, 'Checksum', contextUser);
+            }
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            LogError(`[Dedup] Checksum detection failed for item ${contentItem.ID}: ${msg}`);
+        }
+    }
+    /**
+     * Detects duplicate content items by matching the given item's title (Name field)
+     * against other content items from **different** content sources. When an exact
+     * title match is found, a {@link MJContentItemDuplicateEntity} record is created
+     * with `DetectionMethod = 'Title'` and `SimilarityScore = 1.0`.
+     *
+     * Duplicate pairs are stored in canonical order (lower ID = ContentItemAID) to
+     * prevent mirror duplicates. If a duplicate pair already exists for the same
+     * detection method, no new record is created.
+     *
+     * @param contentItem - The content item whose title should be checked for duplicates.
+     *                      Must already be saved (i.e., have a valid ID and Name).
+     * @param contextUser - The authenticated user context for data access and audit.
+     * @returns A promise that resolves when detection is complete. Does not throw on
+     *          failure — errors are logged and swallowed to avoid disrupting the pipeline.
+     */
+    async DetectTitleDuplicates(contentItem, contextUser) {
+        if (!contentItem.Name || contentItem.Name.trim().length === 0) {
+            return; // No title to compare
+        }
+        try {
+            const matches = await this.findItemsByTitle(contentItem.Name, contentItem.ContentSourceID, contentItem.ID, contextUser);
+            for (const match of matches) {
+                await this.createDuplicateRecordIfNotExists(contentItem.ID, match.ID, 1.0, 'Title', contextUser);
+            }
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            LogError(`[Dedup] Title detection failed for item ${contentItem.ID}: ${msg}`);
+        }
+    }
+    /**
+     * Runs all non-vector deduplication checks (checksum and title) for a content item.
+     * This is a convenience method intended to be called after saving/updating a content item.
+     *
+     * @param contentItem - The saved content item to check for duplicates.
+     * @param contextUser - The authenticated user context for data access and audit.
+     */
+    async DetectDuplicates(contentItem, contextUser) {
+        await Promise.all([
+            this.DetectChecksumDuplicates(contentItem, contextUser),
+            this.DetectTitleDuplicates(contentItem, contextUser),
+        ]);
+    }
+    /**
+     * Detects near-duplicate content items by querying the vector index for items
+     * with high cosine similarity (> 0.95 threshold). Only creates duplicate records
+     * for matches from DIFFERENT content sources to avoid self-matches.
+     *
+     * This is expensive so it only checks the top 3 most similar results.
+     * Controlled by the `enableVectorDedup` flag.
+     *
+     * @param contentItem - The content item to check (must have text and be vectorized).
+     * @param contextUser - The authenticated user context for data access and audit.
+     * @param enableVectorDedup - Whether to run vector-based dedup (default false).
+     */
+    async DetectVectorDuplicates(contentItem, contextUser, enableVectorDedup = false) {
+        if (!enableVectorDedup)
+            return;
+        if (!contentItem.Text || contentItem.Text.trim().length === 0)
+            return;
+        try {
+            await this.performVectorDedupCheck(contentItem, contextUser);
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            LogError(`[Dedup] Vector detection failed for item ${contentItem.ID}: ${msg}`);
+        }
+    }
+    /**
+     * Internal implementation of vector-based dedup. Resolves the vector infrastructure
+     * for the item, embeds its text, queries for similar vectors, and creates duplicate
+     * records for high-similarity matches from different sources.
+     */
+    async performVectorDedupCheck(contentItem, contextUser) {
+        // Need AIEngine loaded to resolve embedding model
+        await AIEngine.Instance.Config(false, contextUser);
+        // Load the content source + type maps for this single item
+        const { sourceMap, typeMap } = await this.loadContentSourceAndTypeMaps([contentItem], contextUser);
+        // Resolve infrastructure for this item
+        const groups = this.groupItemsByInfrastructure([contentItem], sourceMap, typeMap);
+        if (groups.size === 0) {
+            LogStatus(`[Dedup] No vector infrastructure found for item ${contentItem.ID}, skipping vector dedup`);
+            return;
+        }
+        const [groupKey] = groups.entries().next().value;
+        const infra = await this.resolveGroupInfrastructure(groupKey, contextUser);
+        // Embed the item's text
+        const text = contentItem.Text.trim();
+        const truncated = text.length > 8000 ? text.substring(0, 8000) : text;
+        await this.EmbeddingRateLimiter.Acquire(Math.ceil(truncated.length / 4));
+        const modelRunner = new AIModelRunner();
+        const embeddingPromptID = this.resolveEmbeddingPromptID();
+        const runResult = await modelRunner.RunEmbedding({
+            ModelID: infra.embeddingModelID,
+            Texts: [truncated],
+            PromptID: embeddingPromptID ?? undefined,
+            ContextUser: contextUser,
+        });
+        if (!runResult?.Vectors || runResult.Vectors.length === 0) {
+            LogStatus(`[Dedup] Embedding failed for item ${contentItem.ID}, skipping vector dedup`);
+            return;
+        }
+        const queryVector = runResult.Vectors[0];
+        // Query vector DB for top 4 most similar (top 3 useful + 1 for self-match)
+        const queryResponse = await infra.vectorDB.QueryIndex({
+            vector: queryVector,
+            topK: 4,
+            includeMetadata: true,
+        });
+        const responseData = queryResponse;
+        if (!responseData.success || !responseData.data)
+            return;
+        // The data property contains matches (QueryResponse shape)
+        const matches = responseData.data.matches;
+        if (!matches || matches.length === 0)
+            return;
+        // Filter: different source, similarity > 0.95, not self
+        const VECTOR_DEDUP_THRESHOLD = 0.95;
+        let matchCount = 0;
+        for (const match of matches) {
+            if (matchCount >= 3)
+                break; // Only check top 3
+            const matchScore = match.score ?? 0;
+            if (matchScore < VECTOR_DEDUP_THRESHOLD)
+                continue;
+            // Extract content item ID from vector metadata
+            const matchItemID = match.metadata?.['contentItemID'];
+            if (!matchItemID || UUIDsEqual(matchItemID, contentItem.ID))
+                continue;
+            // Check if the match is from a different source
+            const isDifferentSource = await this.isFromDifferentSource(matchItemID, contentItem.ContentSourceID, contextUser);
+            if (!isDifferentSource)
+                continue;
+            await this.createDuplicateRecordIfNotExists(contentItem.ID, matchItemID, matchScore, 'Vector', contextUser);
+            matchCount++;
+        }
+        if (matchCount > 0) {
+            LogStatus(`[Dedup] Vector dedup found ${matchCount} near-duplicate(s) for item ${contentItem.ID}`);
+        }
+    }
+    /**
+     * Check if a content item belongs to a different source than the given sourceID.
+     */
+    async isFromDifferentSource(itemID, excludeSourceID, contextUser) {
+        const rv = new RunView();
+        const result = await rv.RunView({
+            EntityName: 'MJ: Content Items',
+            Fields: ['ContentSourceID'],
+            ExtraFilter: `ID = '${itemID}'`,
+            ResultType: 'simple',
+            MaxRows: 1,
+        }, contextUser);
+        if (!result.Success || result.Results.length === 0)
+            return false;
+        return result.Results[0].ContentSourceID !== excludeSourceID;
+    }
+    /**
+     * Resolves a duplicate record by updating its Status and Resolution fields.
+     *
+     * @param duplicateID - The ID of the ContentItemDuplicate record.
+     * @param resolution - The resolution choice: 'KeepA', 'KeepB', 'NotDuplicate'.
+     * @param contextUser - The authenticated user context.
+     */
+    async ResolveContentDuplicate(duplicateID, resolution, contextUser) {
+        try {
+            const md = new Metadata();
+            const duplicate = await md.GetEntityObject('MJ: Content Item Duplicates', contextUser);
+            const loaded = await duplicate.Load(duplicateID);
+            if (!loaded) {
+                LogError(`[Dedup] Could not load duplicate record ${duplicateID} for resolution`);
+                return false;
+            }
+            this.applyDuplicateResolution(duplicate, resolution);
+            const saved = await duplicate.Save();
+            if (!saved) {
+                LogError(`[Dedup] Failed to save resolution for duplicate ${duplicateID}: ${duplicate.LatestResult?.Message ?? 'Unknown error'}`);
+                return false;
+            }
+            LogStatus(`[Dedup] Resolved duplicate ${duplicateID}: ${resolution}`);
+            return true;
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            LogError(`[Dedup] Error resolving duplicate ${duplicateID}: ${msg}`);
+            return false;
+        }
+    }
+    /**
+     * Applies the resolution to a duplicate record by setting the Status and Resolution fields.
+     */
+    applyDuplicateResolution(duplicate, resolution) {
+        if (resolution === 'NotDuplicate') {
+            duplicate.Status = 'Dismissed';
+            duplicate.Resolution = 'NotDuplicate';
+        }
+        else {
+            // KeepA or KeepB — mark as Merged
+            duplicate.Status = 'Merged';
+            duplicate.Resolution = resolution;
+        }
+    }
+    /**
+     * Finds content items with the same checksum from different content sources.
+     *
+     * @param checksum - The SHA-256 checksum to search for.
+     * @param excludeSourceID - The content source ID to exclude (the item's own source).
+     * @param excludeItemID - The content item ID to exclude (the item itself).
+     * @param contextUser - The authenticated user context.
+     * @returns An array of matching content items (simple objects with ID field).
+     */
+    async findItemsByChecksum(checksum, excludeSourceID, excludeItemID, contextUser) {
+        const rv = new RunView();
+        const result = await rv.RunView({
+            EntityName: 'MJ: Content Items',
+            Fields: ['ID'],
+            ExtraFilter: `Checksum = '${checksum.replace(/'/g, "''")}' AND ContentSourceID <> '${excludeSourceID}' AND ID <> '${excludeItemID}'`,
+            ResultType: 'simple'
+        }, contextUser);
+        if (!result.Success) {
+            LogError(`[Dedup] RunView failed for checksum lookup: ${result.ErrorMessage}`);
+            return [];
+        }
+        return result.Results;
+    }
+    /**
+     * Finds content items with the same title (Name) from different content sources.
+     *
+     * @param title - The title to search for (exact match).
+     * @param excludeSourceID - The content source ID to exclude (the item's own source).
+     * @param excludeItemID - The content item ID to exclude (the item itself).
+     * @param contextUser - The authenticated user context.
+     * @returns An array of matching content items (simple objects with ID field).
+     */
+    async findItemsByTitle(title, excludeSourceID, excludeItemID, contextUser) {
+        const rv = new RunView();
+        const escapedTitle = title.replace(/'/g, "''");
+        const result = await rv.RunView({
+            EntityName: 'MJ: Content Items',
+            Fields: ['ID'],
+            ExtraFilter: `Name = '${escapedTitle}' AND ContentSourceID <> '${excludeSourceID}' AND ID <> '${excludeItemID}'`,
+            ResultType: 'simple'
+        }, contextUser);
+        if (!result.Success) {
+            LogError(`[Dedup] RunView failed for title lookup: ${result.ErrorMessage}`);
+            return [];
+        }
+        return result.Results;
+    }
+    /**
+     * Creates a {@link MJContentItemDuplicateEntity} record for a detected duplicate pair,
+     * but only if one does not already exist for the same pair and detection method.
+     *
+     * IDs are stored in canonical order: the lexicographically smaller ID is always
+     * ContentItemAID to prevent mirror duplicates (A,B) vs (B,A).
+     *
+     * @param itemAID - One of the duplicate item IDs.
+     * @param itemBID - The other duplicate item ID.
+     * @param similarityScore - The similarity score (0.0 to 1.0).
+     * @param detectionMethod - How the duplicate was detected.
+     * @param contextUser - The authenticated user context.
+     */
+    async createDuplicateRecordIfNotExists(itemAID, itemBID, similarityScore, detectionMethod, contextUser) {
+        // Canonical ordering: lower normalized ID = A
+        const normalizedA = NormalizeUUID(itemAID);
+        const normalizedB = NormalizeUUID(itemBID);
+        const [canonicalAID, canonicalBID] = normalizedA < normalizedB
+            ? [itemAID, itemBID]
+            : [itemBID, itemAID];
+        // Check if this pair already exists for the same detection method
+        const exists = await this.duplicatePairExists(canonicalAID, canonicalBID, detectionMethod, contextUser);
+        if (exists) {
+            return;
+        }
+        const md = new Metadata();
+        const duplicate = await md.GetEntityObject('MJ: Content Item Duplicates', contextUser);
+        duplicate.NewRecord();
+        duplicate.ContentItemAID = canonicalAID;
+        duplicate.ContentItemBID = canonicalBID;
+        duplicate.SimilarityScore = similarityScore;
+        duplicate.DetectionMethod = detectionMethod;
+        duplicate.Status = 'Pending';
+        const saved = await duplicate.Save();
+        if (!saved) {
+            LogError(`[Dedup] Failed to save duplicate record for pair (${canonicalAID}, ${canonicalBID}) method=${detectionMethod}`);
+        }
+        else {
+            LogStatus(`[Dedup] Detected ${detectionMethod} duplicate: (${canonicalAID}, ${canonicalBID}) score=${similarityScore}`);
+        }
+    }
+    /**
+     * Checks whether a duplicate record already exists for the given pair and detection method.
+     *
+     * @param canonicalAID - The canonical (ordered) ContentItemAID.
+     * @param canonicalBID - The canonical (ordered) ContentItemBID.
+     * @param detectionMethod - The detection method to check.
+     * @param contextUser - The authenticated user context.
+     * @returns True if a record already exists.
+     */
+    async duplicatePairExists(canonicalAID, canonicalBID, detectionMethod, contextUser) {
+        const rv = new RunView();
+        const result = await rv.RunView({
+            EntityName: 'MJ: Content Item Duplicates',
+            Fields: ['ID'],
+            ExtraFilter: `ContentItemAID = '${canonicalAID}' AND ContentItemBID = '${canonicalBID}' AND DetectionMethod = '${detectionMethod}'`,
+            ResultType: 'simple'
+        }, contextUser);
+        return result.Success && result.Results.length > 0;
+    }
 };
-AutotagBaseEngine = __decorate([
+AutotagBaseEngine = AutotagBaseEngine_1 = __decorate([
     RegisterClass(BaseEngine, 'AutotagBaseEngine')
 ], AutotagBaseEngine);
 export { AutotagBaseEngine };