@memberjunction/content-autotagging 5.22.0 → 5.23.0

package/dist/Engine/generic/AutotagBaseEngine.js

@@ -5,7 +5,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
     return c > 3 && r && Object.defineProperty(target, key, r), r;
 };
 import { BaseEngine, Metadata, RunView, LogError, LogStatus } from '@memberjunction/core';
-import { MJGlobal, UUIDsEqual, RegisterClass } from '@memberjunction/global';
+import { MJGlobal, UUIDsEqual, NormalizeUUID, RegisterClass } from '@memberjunction/global';
 import { ContentSourceTypeParams } from './content.types.js';
 import pdfParse from 'pdf-parse';
 import officeparser from 'officeparser';
@@ -15,9 +15,14 @@ import { toZonedTime } from 'date-fns-tz';
 import axios from 'axios';
 import * as cheerio from 'cheerio';
 import crypto from 'crypto';
-import { BaseLLM, GetAIAPIKey } from '@memberjunction/ai';
+import { BaseEmbeddings, GetAIAPIKey } from '@memberjunction/ai';
 import { AIEngine } from '@memberjunction/aiengine';
+import { AIPromptRunner } from '@memberjunction/ai-prompts';
+import { AIPromptParams } from '@memberjunction/ai-core-plus';
 import { TextChunker } from '@memberjunction/ai-vectors';
+import { VectorDBBase } from '@memberjunction/ai-vectordb';
+/** Default batch size for vectorization processing */
+const DEFAULT_VECTORIZE_BATCH_SIZE = 20;
 /**
  * Core engine for content autotagging. Extends BaseEngine to cache content metadata
  * (types, source types, file types, attributes) at startup. Uses AIEngine via composition
@@ -79,8 +84,9 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
     }
     /**
      * Given a list of content items, extract the text from each and process with LLM for tagging.
+     * Items are processed in configurable batches with controlled concurrency within each batch.
      */
-    async ExtractTextAndProcessWithLLM(contentItems, contextUser) {
+    async ExtractTextAndProcessWithLLM(contentItems, contextUser, batchSize = DEFAULT_VECTORIZE_BATCH_SIZE, onProgress) {
         if (!contentItems || contentItems.length === 0) {
             LogStatus('No content items to process');
             return;
@@ -89,16 +95,35 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
         processRunParams.sourceID = contentItems[0].ContentSourceID;
         processRunParams.startTime = new Date();
         processRunParams.numItemsProcessed = contentItems.length;
-        for (const contentItem of contentItems) {
-            try {
-                const processingParams = await this.buildProcessingParams(contentItem, contextUser);
-                await this.ProcessContentItemText(processingParams, contextUser);
-            }
-            catch (e) {
-                LogError(`Failed to process content item: ${contentItem.ID}`, undefined, e);
-                throw e;
-            }
+        let totalProcessed = 0;
+        LogStatus(`ExtractTextAndProcessWithLLM: processing ${contentItems.length} items in batches of ${batchSize}`);
+        let batchSuccesses = 0;
+        let batchFailures = 0;
+        for (let i = 0; i < contentItems.length; i += batchSize) {
+            const batch = contentItems.slice(i, i + batchSize);
+            const batchNum = Math.floor(i / batchSize) + 1;
+            let batchOk = 0, batchFail = 0;
+            const batchPromises = batch.map(async (contentItem) => {
+                try {
+                    const processingParams = await this.buildProcessingParams(contentItem, contextUser);
+                    await this.ProcessContentItemText(processingParams, contextUser);
+                    totalProcessed++;
+                    batchOk++;
+                    onProgress?.(totalProcessed, contentItems.length, contentItem.Name);
+                }
+                catch (e) {
+                    LogError(`Failed to process content item: ${contentItem.ID}`, undefined, e);
+                    totalProcessed++;
+                    batchFail++;
+                    onProgress?.(totalProcessed, contentItems.length);
+                }
+            });
+            await Promise.all(batchPromises);
+            batchSuccesses += batchOk;
+            batchFailures += batchFail;
+            LogStatus(`Batch ${batchNum}: ${batchOk} succeeded, ${batchFail} failed (${totalProcessed}/${contentItems.length} total)`);
         }
+        LogStatus(`ExtractTextAndProcessWithLLM complete: ${batchSuccesses} succeeded, ${batchFailures} failed out of ${contentItems.length}`);
         processRunParams.endTime = new Date();
         await this.saveProcessRun(processRunParams, contextUser);
     }
@@ -125,88 +150,121 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
         const LLMResults = await this.promptAndRetrieveResultsFromLLM(params, contextUser);
         await this.saveLLMResults(LLMResults, contextUser);
     }
-    async promptAndRetrieveResultsFromLLM(params, contextUser) {
-        const model = AIEngine.Instance.Models.find(m => UUIDsEqual(m.ID, params.modelID));
-        if (!model) {
-            throw new Error(`AI Model with ID ${params.modelID} not found`);
+    /**
+     * Resolves the "Content Autotagging" prompt from the AIEngine cache.
+     * Throws if the prompt is not found or not active.
+     */
+    getAutotagPrompt() {
+        const prompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Content Autotagging');
+        if (!prompt) {
+            throw new Error('AI Prompt "Content Autotagging" not found. Ensure the prompt metadata has been synced to the database.');
         }
-        const llm = MJGlobal.Instance.ClassFactory.CreateInstance(BaseLLM, model.DriverClass, GetAIAPIKey(model.DriverClass));
-        if (!llm) {
-            throw new Error(`Failed to create LLM instance for driver ${model.DriverClass}`);
+        if (prompt.Status !== 'Active') {
+            throw new Error(`AI Prompt "Content Autotagging" is not active (Status: ${prompt.Status})`);
         }
-        const chunks = this.chunkExtractedText(params.text, model.InputTokenLimit);
+        return prompt;
+    }
+    /**
+     * Builds template data for the autotagging prompt from processing params and chunk context.
+     */
+    buildPromptData(params, chunk, previousResults) {
+        const contentType = this.GetContentTypeName(params.contentTypeID);
+        const contentSourceType = this.GetContentSourceTypeName(params.contentSourceTypeID);
+        const additionalAttributePrompts = this.GetAdditionalContentTypePrompt(params.contentTypeID);
+        const hasPreviousResults = Object.keys(previousResults).length > 0;
+        return {
+            contentType,
+            contentSourceType,
+            minTags: params.minTags,
+            maxTags: params.maxTags,
+            additionalAttributePrompts,
+            contentText: chunk,
+            previousResults: hasPreviousResults ? JSON.stringify(previousResults) : undefined,
+        };
+    }
+    async promptAndRetrieveResultsFromLLM(params, contextUser) {
+        await AIEngine.Instance.Config(false, contextUser);
+        const prompt = this.getAutotagPrompt();
+        // Determine token limit for chunking: use override model if set, else first prompt-model, else a default
+        const tokenLimit = this.resolveTokenLimit(params.modelID);
+        const chunks = this.chunkExtractedText(params.text, tokenLimit);
         let LLMResults = {};
         const startTime = new Date();
         for (const chunk of chunks) {
-            const { systemPrompt, userPrompt } = await this.getLLMPrompts(params, chunk, LLMResults, contextUser);
-            LLMResults = await this.processChunkWithLLM(llm, systemPrompt, userPrompt, LLMResults, model.APIName);
+            LLMResults = await this.processChunkWithPromptRunner(prompt, params, chunk, LLMResults, contextUser);
         }
         LLMResults.processStartTime = startTime;
         LLMResults.processEndTime = new Date();
         LLMResults.contentItemID = params.contentItemID;
         return LLMResults;
     }
-    async processChunkWithLLM(llm, systemPrompt, userPrompt, LLMResults, modelAPIName) {
-        const response = await llm.ChatCompletion({
-            messages: [
-                { role: 'system', content: systemPrompt },
-                { role: 'user', content: userPrompt }
-            ],
-            model: modelAPIName,
-            temperature: 0.0,
-        });
-        const queryResponse = response.data.choices[0]?.message?.content?.trim() || '';
-        let JSONQueryResponse;
-        try {
-            JSONQueryResponse = JSON.parse(queryResponse);
+    /**
+     * Resolves the input token limit for chunking. Uses the model specified by modelID if available,
+     * otherwise falls back to a conservative default.
+     */
+    resolveTokenLimit(modelID) {
+        const DEFAULT_TOKEN_LIMIT = 100000;
+        if (modelID) {
+            const model = AIEngine.Instance.Models.find(m => UUIDsEqual(m.ID, modelID));
+            if (model) {
+                return model.InputTokenLimit;
+            }
         }
-        catch (parseError) {
-            LogError('Failed to parse LLM response as JSON', undefined, queryResponse);
+        return DEFAULT_TOKEN_LIMIT;
+    }
+    /**
+     * Processes a single text chunk using AIPromptRunner and merges results.
+     * Uses the prompt's configured model by default. If ContentType.AIModelID is set,
+     * it is passed as a runtime model override via AIPromptParams.override.
+     */
+    async processChunkWithPromptRunner(prompt, params, chunk, LLMResults, contextUser) {
+        const promptParams = new AIPromptParams();
+        promptParams.prompt = prompt;
+        promptParams.contextUser = contextUser;
+        promptParams.data = this.buildPromptData(params, chunk, LLMResults);
+        promptParams.skipValidation = false;
+        promptParams.attemptJSONRepair = true;
+        promptParams.additionalParameters = { temperature: 0.0 };
+        // If the ContentType specifies a preferred AI model, use it as a runtime override
+        if (params.modelID) {
+            promptParams.override = { modelId: params.modelID };
+        }
+        const runner = new AIPromptRunner();
+        // Per-item logging removed for cleanliness — batch-level logging in ExtractTextAndProcessWithLLM
+        const result = await runner.ExecutePrompt(promptParams);
+        if (!result.success) {
+            LogError(`AIPromptRunner FAILED for content item ${params.contentItemID}: ${result.errorMessage ?? 'no error message'}`, undefined, result);
             return LLMResults;
         }
-        for (const key in JSONQueryResponse) {
-            const value = JSONQueryResponse[key];
-            if (value !== null) {
-                LLMResults[key] = value;
+        // Parse the result — AIPromptRunner may return a raw JSON string or a parsed object
+        let chunkResult = null;
+        if (typeof result.result === 'string') {
+            try {
+                chunkResult = JSON.parse(result.result);
+            }
+            catch {
+                LogError(`Failed to parse LLM result as JSON for item ${params.contentItemID}: ${String(result.result).substring(0, 200)}`);
+                return LLMResults;
+            }
+        }
+        else {
+            chunkResult = result.result;
+        }
+        // Merge results from this chunk into the accumulated results
+        if (chunkResult) {
+            for (const key in chunkResult) {
+                const value = chunkResult[key];
+                if (value !== null) {
+                    LLMResults[key] = value;
+                }
             }
         }
         return LLMResults;
     }
-    async getLLMPrompts(params, chunk, LLMResults, contextUser) {
-        const contentType = this.GetContentTypeName(params.contentTypeID);
-        const contentSourceType = this.GetContentSourceTypeName(params.contentSourceTypeID);
-        const additionalContentTypePrompts = this.GetAdditionalContentTypePrompt(params.contentTypeID);
-        const systemPrompt = `You are a highly skilled text analysis assistant. You have decades of experience and pride yourself on your attention to detail and ability to capture both accurate information, as well as tone and subtext.
-        Your task is to accurately extract key information from a provided piece of text based on a series of prompts. You are provided with text that should be a ${contentType}, that has been extracted from a ${contentSourceType}.
-        The text MUST be of the type ${contentType} for the subsequent processing.`;
-        const userPrompt = `
-        If the provided text does not actually appear to be of the type ${contentType}, please disregard everything in the instructions after this and return this exact JSON response: { isValidContent: false (as a boolean) }.
-        Assuming the type of the text is in fact from a ${contentType}, please extract the title of the provided text, a short summary of the provided documents, as well as between ${params.minTags} and ${params.maxTags} topical key words that are most relevant to the text.
-        If there is no title explicitly provided in the text, please provide a title that you think best represents the text.
-        Please provide the keywords in a list format.
-        Make sure the response is just the json file without and formatting or code blocks, and strictly following the format below. Please don't include a greeting in the response, only output the json file:
-
-        {
-            "title": (title here),
-            "description": (description here),
-            "keywords": (list keywords here),
-            "isValidContent": true (as a boolean)
-        }
-
-        ${additionalContentTypePrompts}
-
-        Please make sure the response in is valid JSON format.
-
-        You are also provided with the results so far as additional context, please use them to formulate the best results given the provided text: ${JSON.stringify(LLMResults)}
-        The supplied text is: ${chunk}
-        `;
-        return { systemPrompt, userPrompt };
-    }
     async saveLLMResults(LLMResults, contextUser) {
         if (LLMResults.isValidContent === true) {
             await this.saveResultsToContentItemAttribute(LLMResults, contextUser);
             await this.saveContentItemTags(LLMResults.contentItemID, LLMResults, contextUser);
-            LogStatus(`Results for content item ${LLMResults.contentItemID} saved successfully`);
         }
         else {
             await this.deleteInvalidContentItem(LLMResults.contentItemID, contextUser);
@@ -270,14 +328,28 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
         const keywords = LLMResults.keywords;
         if (!keywords || !Array.isArray(keywords))
             return;
+        // Normalize keywords — support both formats:
+        //   Old: ["keyword1", "keyword2"]
+        //   New: [{ tag: "keyword1", weight: 0.95 }, { tag: "keyword2", weight: 0.7 }]
+        const normalizedTags = keywords.map((kw) => {
+            if (typeof kw === 'string') {
+                return { tag: kw, weight: 1.0 };
+            }
+            const obj = kw;
+            return {
+                tag: obj.tag || obj.keyword || String(kw),
+                weight: typeof obj.weight === 'number' ? Math.max(0, Math.min(1, obj.weight)) : 0.5,
+            };
+        });
         const BATCH_SIZE = 10;
-        for (let i = 0; i < keywords.length; i += BATCH_SIZE) {
-            const batch = keywords.slice(i, i + BATCH_SIZE);
-            await Promise.all(batch.map(async (keyword) => {
+        for (let i = 0; i < normalizedTags.length; i += BATCH_SIZE) {
+            const batch = normalizedTags.slice(i, i + BATCH_SIZE);
+            await Promise.all(batch.map(async (item) => {
                 const contentItemTag = await md.GetEntityObject('MJ: Content Item Tags', contextUser);
                 contentItemTag.NewRecord();
                 contentItemTag.ItemID = contentItemID;
-                contentItemTag.Tag = keyword;
+                contentItemTag.Tag = item.tag;
+                contentItemTag.Set('Weight', item.weight);
                 await contentItemTag.Save();
             }));
         }
@@ -530,6 +602,320 @@ let AutotagBaseEngine = class AutotagBaseEngine extends BaseEngine {
                 throw new Error(`File type '${fileExtension}' not supported`);
         }
     }
+    // ---- Direct Vectorization ----
+    /**
+     * Embeds content items and upserts them to the appropriate vector index.
+     * Items are grouped by their resolved (embeddingModel + vectorIndex) pair — derived
+     * from per-ContentSource overrides, per-ContentType defaults, or the global fallback
+     * (first active VectorIndex). Each group is processed in configurable batches with
+     * parallel upserts within each batch.
+     */
+    async VectorizeContentItems(items, contextUser, onProgress, batchSize = DEFAULT_VECTORIZE_BATCH_SIZE) {
+        const eligible = items.filter(i => i.Text && i.Text.trim().length > 0);
+        if (eligible.length === 0) {
+            LogStatus('VectorizeContentItems: no items with text to vectorize');
+            return { vectorized: 0, skipped: items.length };
+        }
+        // Ensure AIEngine is loaded so we can resolve the embedding model
+        await AIEngine.Instance.Config(false, contextUser);
+        // Load content sources + types for per-item infrastructure resolution
+        const { sourceMap, typeMap } = await this.loadContentSourceAndTypeMaps(eligible, contextUser);
+        // Group items by their resolved (embeddingModelID + vectorIndexID) pair
+        const groups = this.groupItemsByInfrastructure(eligible, sourceMap, typeMap);
+        // Load tags for all items in one query
+        const tagMap = await this.loadTagsForItems(eligible, contextUser);
+        let vectorized = 0;
+        let processed = 0;
+        for (const [groupKey, groupItems] of groups) {
+            const infra = await this.resolveGroupInfrastructure(groupKey, contextUser);
+            const groupVectorized = await this.vectorizeGroup(groupItems, infra, tagMap, batchSize, (batchProcessed) => {
+                processed += batchProcessed;
+                onProgress?.(Math.min(processed, eligible.length), eligible.length);
+            });
+            vectorized += groupVectorized;
+        }
+        LogStatus(`VectorizeContentItems: ${vectorized} vectorized, ${items.length - eligible.length} skipped (empty text)`);
+        return { vectorized, skipped: items.length - eligible.length };
+    }
+    /**
+     * Process a single infrastructure group: embed texts in batches and upsert to vector DB.
+     * Upserts within each batch run in parallel for throughput.
+     */
+    async vectorizeGroup(items, infra, tagMap, batchSize, onBatchComplete) {
+        let vectorized = 0;
+        for (let i = 0; i < items.length; i += batchSize) {
+            const batch = items.slice(i, i + batchSize);
+            const texts = batch.map(item => this.buildEmbeddingText(item));
+            const embedResult = await infra.embedding.EmbedTexts({ texts, model: infra.embeddingModelName });
+            if (!embedResult.vectors || embedResult.vectors.length !== batch.length) {
+                LogError(`VectorizeContentItems: embedding returned ${embedResult.vectors?.length ?? 0} vectors for ${batch.length} texts`);
+                onBatchComplete(batch.length);
+                continue;
+            }
+            const records = batch.map((item, idx) => ({
+                id: this.contentItemVectorId(item.ID),
+                values: embedResult.vectors[idx],
+                metadata: this.buildVectorMetadata(item, tagMap.get(item.ID))
+            }));
+            // Upsert records in parallel sub-batches for throughput
+            const UPSERT_CHUNK = 50;
+            const upsertPromises = [];
+            for (let j = 0; j < records.length; j += UPSERT_CHUNK) {
+                const chunk = records.slice(j, j + UPSERT_CHUNK);
+                upsertPromises.push(Promise.resolve(infra.vectorDB.CreateRecords(chunk, infra.indexName)));
+            }
+            const responses = await Promise.all(upsertPromises);
+            let batchSuccess = true;
+            for (const response of responses) {
+                if (!response.success) {
+                    LogError(`VectorizeContentItems: upsert failed: ${response.message}`);
+                    batchSuccess = false;
+                }
+            }
+            if (batchSuccess) {
+                vectorized += batch.length;
+            }
+            onBatchComplete(batch.length);
+        }
+        return vectorized;
+    }
+    /**
+     * Load content source and content type records for all unique source/type IDs
+     * referenced by the given items. Returns maps keyed by normalized ID.
+     */
+    async loadContentSourceAndTypeMaps(items, contextUser) {
+        const sourceIds = [...new Set(items.map(i => i.ContentSourceID))];
+        const typeIds = [...new Set(items.map(i => i.ContentTypeID))];
+        const rv = new RunView();
+        const [sourceResult, typeResult] = await rv.RunViews([
+            {
+                EntityName: 'MJ: Content Sources',
+                ExtraFilter: `ID IN (${sourceIds.map(id => `'${id}'`).join(',')})`,
+                ResultType: 'simple'
+            },
+            {
+                EntityName: 'MJ: Content Types',
+                ExtraFilter: `ID IN (${typeIds.map(id => `'${id}'`).join(',')})`,
+                ResultType: 'simple'
+            }
+        ], contextUser);
+        const sourceMap = new Map();
+        if (sourceResult.Success) {
+            for (const row of sourceResult.Results) {
+                const rec = row;
+                sourceMap.set(NormalizeUUID(rec['ID']), rec);
+            }
+        }
+        const typeMap = new Map();
+        if (typeResult.Success) {
+            for (const row of typeResult.Results) {
+                const rec = row;
+                typeMap.set(NormalizeUUID(rec['ID']), rec);
+            }
+        }
+        return { sourceMap, typeMap };
+    }
+    /**
+     * Resolve the (embeddingModelID, vectorIndexID) pair for a content item using
+     * the cascade: ContentSource override -> ContentType default -> null (global fallback).
+     */
+    resolveItemInfrastructureIds(item, sourceMap, typeMap) {
+        const source = sourceMap.get(NormalizeUUID(item.ContentSourceID));
+        if (source) {
+            const srcEmbedding = source['EmbeddingModelID'];
+            const srcVector = source['VectorIndexID'];
+            if (srcEmbedding && srcVector) {
+                return { embeddingModelID: srcEmbedding, vectorIndexID: srcVector };
+            }
+        }
+        const contentType = typeMap.get(NormalizeUUID(item.ContentTypeID));
+        if (contentType) {
+            const typeEmbedding = contentType['EmbeddingModelID'];
+            const typeVector = contentType['VectorIndexID'];
+            if (typeEmbedding && typeVector) {
+                return { embeddingModelID: typeEmbedding, vectorIndexID: typeVector };
+            }
+        }
+        // Global fallback — will be resolved in resolveGroupInfrastructure
+        return { embeddingModelID: null, vectorIndexID: null };
+    }
+    /**
+     * Group items by their resolved (embeddingModelID + vectorIndexID) key.
+     * Items with the same pair share infrastructure and can be batched together.
+     */
+    groupItemsByInfrastructure(items, sourceMap, typeMap) {
+        const groups = new Map();
+        for (const item of items) {
+            const { embeddingModelID, vectorIndexID } = this.resolveItemInfrastructureIds(item, sourceMap, typeMap);
+            const key = this.infraGroupKey(embeddingModelID, vectorIndexID);
+            const group = groups.get(key) ?? [];
+            group.push(item);
+            groups.set(key, group);
+        }
+        return groups;
+    }
+    /** Create a stable cache key for an (embeddingModelID, vectorIndexID) pair */
+    infraGroupKey(embeddingModelID, vectorIndexID) {
+        const e = embeddingModelID ? NormalizeUUID(embeddingModelID) : 'default';
+        const v = vectorIndexID ? NormalizeUUID(vectorIndexID) : 'default';
+        return `${e}|${v}`;
+    }
+    /**
+     * Resolve a group key into concrete infrastructure instances. For the 'default|default'
+     * key, falls back to the first active VectorIndex (original behavior).
+     */
+    async resolveGroupInfrastructure(groupKey, contextUser) {
+        const [embeddingPart, vectorPart] = groupKey.split('|');
+        const isDefault = embeddingPart === 'default' || vectorPart === 'default';
+        if (isDefault) {
+            return this.getDefaultVectorInfrastructure(contextUser);
+        }
+        return this.buildVectorInfrastructure(embeddingPart, vectorPart, contextUser);
+    }
+    /**
+     * Build infrastructure from explicit embeddingModelID and vectorIndexID.
+     * Looks up the vector index by ID and the embedding model from AIEngine.
+     */
+    async buildVectorInfrastructure(embeddingModelID, vectorIndexID, contextUser) {
+        const rv = new RunView();
+        const indexResult = await rv.RunView({
+            EntityName: 'MJ: Vector Indexes',
+            ExtraFilter: `ID='${vectorIndexID}'`,
+            ResultType: 'simple',
+            MaxRows: 1
+        }, contextUser);
+        if (!indexResult.Success || indexResult.Results.length === 0) {
+            throw new Error(`Vector index ${vectorIndexID} not found`);
+        }
+        const vectorIndex = indexResult.Results[0];
+        return this.createInfrastructureFromIndex(vectorIndex, embeddingModelID, contextUser);
+    }
+    /**
+     * Fallback: resolve infrastructure from the first active VectorIndex (original behavior).
+     */
+    async getDefaultVectorInfrastructure(contextUser) {
+        const rv = new RunView();
+        const indexResult = await rv.RunView({
+            EntityName: 'MJ: Vector Indexes',
+            ResultType: 'simple',
+            MaxRows: 1
+        }, contextUser);
+        if (!indexResult.Success || indexResult.Results.length === 0) {
+            throw new Error('No vector indexes found — create one in the Configuration tab first');
+        }
+        const vectorIndex = indexResult.Results[0];
+        const embeddingModelID = vectorIndex['EmbeddingModelID'];
+        return this.createInfrastructureFromIndex(vectorIndex, embeddingModelID, contextUser);
+    }
+    /**
+     * Shared helper: given a vector index record and embedding model ID, resolve all
+     * driver instances needed for embedding + upsert.
+     */
+    async createInfrastructureFromIndex(vectorIndex, embeddingModelID, contextUser) {
+        const indexName = vectorIndex['Name'];
+        const vectorDatabaseID = vectorIndex['VectorDatabaseID'];
+        const rv = new RunView();
+        const dbResult = await rv.RunView({
+            EntityName: 'MJ: Vector Databases',
+            ExtraFilter: `ID='${vectorDatabaseID}'`,
+            ResultType: 'simple',
+            MaxRows: 1
+        }, contextUser);
+        if (!dbResult.Success || dbResult.Results.length === 0) {
+            throw new Error(`Vector database ${vectorDatabaseID} not found`);
+        }
+        const vectorDBClassKey = dbResult.Results[0]['ClassKey'];
+        const aiModel = this.findEmbeddingModel(embeddingModelID);
+        const driverClass = aiModel.DriverClass;
+        const embeddingModelName = aiModel.APIName ?? aiModel.Name;
+        LogStatus(`VectorizeContentItems: USING embedding model "${aiModel.Name}" (${driverClass}), vector DB "${vectorDBClassKey}", index "${indexName}"`);
+        const embedding = this.createEmbeddingInstance(driverClass);
+        const vectorDB = this.createVectorDBInstance(vectorDBClassKey);
+        return { embedding, vectorDB, indexName, embeddingModelName };
+    }
+    /** Find an embedding model by ID in AIEngine, with helpful error reporting */
+    findEmbeddingModel(embeddingModelID) {
+        const aiModel = AIEngine.Instance.Models.find(m => UUIDsEqual(m.ID, embeddingModelID));
+        if (!aiModel) {
+            const embModels = AIEngine.Instance.Models.filter(m => m.DriverClass?.includes('Embed') || m.Name?.includes('embed'));
+            LogError(`VectorizeContentItems: embeddingModelID ${embeddingModelID} NOT FOUND. Available: ${JSON.stringify(embModels.map(m => ({ id: m.ID, name: m.Name, driver: m.DriverClass })))}`);
+            throw new Error(`Embedding model ${embeddingModelID} not found in AIEngine — ensure AIEngine is configured`);
+        }
+        return aiModel;
+    }
+    /** Create a BaseEmbeddings instance for a given driver class */
+    createEmbeddingInstance(driverClass) {
+        const apiKey = GetAIAPIKey(driverClass);
+        if (!apiKey) {
+            throw new Error(`No API key found for embedding driver ${driverClass} — set AI_VENDOR_API_KEY__${driverClass} in .env`);
+        }
+        const instance = MJGlobal.Instance.ClassFactory.CreateInstance(BaseEmbeddings, driverClass, apiKey);
+        if (!instance)
+            throw new Error(`Failed to create embedding instance for ${driverClass}`);
+        return instance;
+    }
+    /** Create a VectorDBBase instance for a given class key */
+    createVectorDBInstance(classKey) {
+        const apiKey = GetAIAPIKey(classKey);
+        if (!apiKey) {
+            throw new Error(`No API key found for vector DB ${classKey} — set AI_VENDOR_API_KEY__${classKey} in .env`);
+        }
+        const instance = MJGlobal.Instance.ClassFactory.CreateInstance(VectorDBBase, classKey, apiKey);
+        if (!instance)
+            throw new Error(`Failed to create vector DB instance for ${classKey}`);
+        return instance;
+    }
+    /** SHA-1 deterministic vector ID for a content item */
+    contentItemVectorId(contentItemId) {
+        return crypto.createHash('sha1').update(`content-item_${contentItemId}`).digest('hex');
+    }
+    /** Build the text that gets embedded: Title + Description + full Text */
+    buildEmbeddingText(item) {
+        const parts = [];
+        if (item.Name)
+            parts.push(item.Name);
+        if (item.Description)
+            parts.push(item.Description);
+        if (item.Text)
+            parts.push(item.Text);
+        return parts.join('\n');
+    }
+    /** Build metadata stored alongside the vector — truncate large text fields */
+    buildVectorMetadata(item, tags) {
+        const META_TEXT_LIMIT = 1000;
+        const meta = {
+            RecordID: item.ID,
+            Entity: 'MJ: Content Items',
+        };
+        if (item.Name)
+            meta['Title'] = item.Name.substring(0, META_TEXT_LIMIT);
+        if (item.Description)
+            meta['Description'] = item.Description.substring(0, META_TEXT_LIMIT);
+        if (item.URL)
+            meta['URL'] = item.URL;
+        if (tags && tags.length > 0)
+            meta['Tags'] = tags;
+        return meta;
+    }
+    /** Load all tags for the given items in a single RunView call */
+    async loadTagsForItems(items, contextUser) {
+        const tagMap = new Map();
+        const rv = new RunView();
+        const ids = items.map(i => `'${i.ID}'`).join(',');
+        const result = await rv.RunView({
+            EntityName: 'MJ: Content Item Tags',
+            ExtraFilter: `ItemID IN (${ids})`,
+            ResultType: 'entity_object'
+        }, contextUser);
+        if (result.Success) {
+            for (const tag of result.Results) {
+                const existing = tagMap.get(tag.ItemID) ?? [];
+                existing.push(tag.Tag);
+                tagMap.set(tag.ItemID, existing);
+            }
+        }
+        return tagMap;
+    }
 };
 AutotagBaseEngine = __decorate([
     RegisterClass(BaseEngine, 'AutotagBaseEngine')