@memberjunction/ai-vector-sync 5.21.0 → 5.22.0

package/dist/models/entityVectorSync.js

@@ -1,16 +1,13 @@
+import { createHash } from 'node:crypto';
 import { BaseEmbeddings, GetAIAPIKey } from '@memberjunction/ai';
 import { VectorDBBase } from '@memberjunction/ai-vectordb';
 import { VectorBase } from '@memberjunction/ai-vectors';
-import { LogError, LogStatus, Metadata, RunView } from '@memberjunction/core';
+import { LogError, LogStatus, Metadata, RunView, ValidationResult } from '@memberjunction/core';
 import { MJGlobal, UUIDsEqual } from '@memberjunction/global';
 import { pipeline } from 'node:stream/promises';
-import { BatchWorker } from './BatchWorker.js';
 import { EntityDocumentCache } from './EntityDocumentCache.js';
 import { PagedRecords } from './PagedRecords.js';
-import { resolve, dirname } from 'node:path';
-import { fileURLToPath } from 'node:url';
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
+import { AsyncBatchTransform } from './AsyncBatchTransform.js';
 import { PassThrough, Transform } from 'node:stream';
 import { AIEngine } from '@memberjunction/aiengine';
 import { TemplateEngineServer } from '@memberjunction/templates';
@@ -36,13 +33,16 @@ export class EntityVectorSyncer extends VectorBase {
         if (!contextUser) {
             throw new Error('ContextUser is required to vectorize the entity');
         }
-        const delayTimeMS = 250;
         const startTime = new Date().getTime();
         super.CurrentUser = contextUser;
         await TemplateEngineServer.Instance.Config(false, contextUser);
         const entityDocument = await this.GetEntityDocument(params.entityDocumentID);
         const vectorIndexEntity = await this.GetOrCreateVectorIndex(entityDocument);
         const obj = await this.GetVectorDatabaseAndEmbeddingClassByEntityDocumentID(params.entityDocumentID);
+        // Parse configuration for pipeline tuning
+        const docConfig = this.parseDocumentConfig(entityDocument);
+        const pipelineConfig = docConfig.pipeline;
+        const delayTimeMS = pipelineConfig?.delayBetweenCallsMs ?? 250;
         const md = new Metadata();
         const entity = md.Entities.find((e) => UUIDsEqual(e.ID, params.entityID));
         if (!entity) {
@@ -59,52 +59,310 @@ export class EntityVectorSyncer extends VectorBase {
         if (template.Content.length > 1) {
             throw new Error('Templates used by Entity Documents should only have one associated Template Content record.');
         }
-        //If this function doesnt throw an error, then the template is valid
-        const validTemplate = this.ValidateTemplateContextParamAlignment(template);
-        const pageSize = params.listBatchCount || 50;
+        this.ValidateTemplateContextParamAlignment(template);
+        // Pipeline tuning: explicit params override config, config overrides defaults
+        const pageSize = params.listBatchCount || pipelineConfig?.fetchBatchSize || 50;
         const dataStream = new PagedRecords();
-        let templateObj = {
-            ...template.GetAll(),
-            Params: template.Params.map((p) => p.To())
+        const templateContent = template.Content[0];
+        const vectorCreator = this.createVectorCreator(template, templateContent, obj.embedding, delayTimeMS, params.VectorizeBatchCount || pipelineConfig?.vectorizeBatchSize, pipelineConfig?.maxConcurrentEmbeddings);
+        const vectorUpserter = this.createVectorUpserter(entityDocument, templateContent, obj.vectorDB, vectorIndexEntity.Name, delayTimeMS, params.UpsertBatchCount || pipelineConfig?.upsertBatchSize);
+        const erdUpserter = new AsyncBatchTransform({
+            batchSize: 10,
+            concurrencyLimit: 2,
+            processBatch: async (batch) => {
+                await Promise.all(batch.map(item => this.UpsertEntityRecordDocumentRecords(item, super.CurrentUser)));
+                return batch;
+            },
+        });
+        // Track progress: count records fed into the paging stream and records that exit the pipeline
+        let totalRecordsFed = 0;
+        let processedRecords = 0;
+        const onProgress = params.OnProgress;
+        // Wrap addPage to track total records fed into the stream
+        const originalAddPage = dataStream.addPage.bind(dataStream);
+        dataStream.addPage = (items) => {
+            totalRecordsFed += items.length;
+            originalAddPage(items);
         };
-        const workerContext = {
-            executionId: Date.now(),
-            entity,
-            entityDocument: entityDocument.To(),
-            template: templateObj,
-            templateContent: template.Content[0].To(),
-            vectorDBClassKey: obj.vectorDBClassKey,
-            vectorDBAPIKey: obj.vectorDBAPIKey,
-            embeddingDriverClass: obj.embeddingDriverClass,
-            embeddingAPIKey: obj.embeddingAPIKey,
-            delayTimeMS
+        // Progress tracker sits at the end of the pipeline and emits updates.
+        // Throttled to emit only when the percentage changes to avoid flooding PubSub.
+        let lastEmittedPct = -1;
+        let dataStreamEnded = false;
+        const progressTracker = new Transform({
+            objectMode: true,
+            transform(chunk, _encoding, callback) {
+                processedRecords++;
+                if (onProgress) {
+                    const elapsed = new Date().getTime() - startTime;
+                    const pct = totalRecordsFed > 0 ? Math.min(Math.round((processedRecords / totalRecordsFed) * 100), 99) : 0;
+                    // Detect when all records have been processed and data stream is done
+                    if (dataStreamEnded && processedRecords >= totalRecordsFed) {
+                        onProgress({
+                            TotalRecords: totalRecordsFed,
+                            ProcessedRecords: processedRecords,
+                            Stage: 'complete',
+                            PercentComplete: 100,
+                            ElapsedMs: elapsed,
+                        });
+                    }
+                    else {
+                        // Throttle: only emit on 5% boundaries to avoid flooding PubSub/WebSocket
+                        const bucket = Math.floor(pct / 5) * 5;
+                        if (bucket !== lastEmittedPct) {
+                            lastEmittedPct = bucket;
+                            onProgress({
+                                TotalRecords: totalRecordsFed,
+                                ProcessedRecords: processedRecords,
+                                Stage: 'upserting',
+                                PercentComplete: pct,
+                                ElapsedMs: elapsed,
+                            });
+                        }
+                    }
+                }
+                callback(null, chunk);
+            }
+        });
+        // Track when the data stream finishes feeding records
+        const originalEndStream = dataStream.endStream.bind(dataStream);
+        dataStream.endStream = () => {
+            dataStreamEnded = true;
+            originalEndStream();
         };
-        //Handles vectorizing the records 
-        const VectorCreator = new BatchWorker({
-            workerFile: resolve(__dirname, 'workers/VectorizeTemplates.js'),
-            batchSize: params.VectorizeBatchCount || 50,
-            concurrencyLimit: 2,
-            contextUser: super.CurrentUser,
-            workerContext,
+        this.startDataPaging(dataStream, params, md, entity, template, vectorIndexEntity, entityDocument, pageSize);
+        LogStatus('Starting pipeline');
+        await pipeline(dataStream, vectorCreator, vectorUpserter, erdUpserter, progressTracker, new PassThrough({ objectMode: true }));
+        const elapsedMs = new Date().getTime() - startTime;
+        const elapsedSeconds = elapsedMs / 1000;
+        LogStatus(`Finished vectorizing ${entityDocument.Entity} entity in ${elapsedSeconds} seconds (${(elapsedSeconds / 60).toFixed(1)} minutes)`);
+        // Emit final 100% completion
+        if (onProgress) {
+            onProgress({
+                TotalRecords: totalRecordsFed,
+                ProcessedRecords: processedRecords,
+                Stage: 'complete',
+                PercentComplete: 100,
+                ElapsedMs: elapsedMs,
+            });
+        }
+        return { success: true, status: 'Complete', errorMessage: '' };
+    }
+    /**
+     * Creates an AsyncBatchTransform that renders templates and generates embeddings
+     * in the main thread. This replaces the worker_threads-based VectorizeTemplates
+     * worker, which couldn't access ClassFactory registrations in its isolated V8 context.
+     */
+    createVectorCreator(template, templateContent, embedding, delayTimeMS, batchSize, concurrencyLimit) {
+        return new AsyncBatchTransform({
+            batchSize: batchSize || 50,
+            concurrencyLimit: concurrencyLimit ?? 2,
+            processBatch: (batch) => this.renderAndEmbedBatch(batch, template, templateContent, embedding, delayTimeMS),
         });
-        VectorCreator.on('error', (err) => {
-            LogError('Error in VectorCreator worker', null, err);
+    }
+    /**
+     * Creates an AsyncBatchTransform that upserts vectors to the vector database
+     * in the main thread. This replaces the worker_threads-based UpsertVectors
+     * worker for the same ClassFactory reasons.
+     */
+    createVectorUpserter(entityDocument, templateContent, vectorDB, indexName, delayTimeMS, batchSize) {
+        return new AsyncBatchTransform({
+            batchSize: batchSize || 50,
+            concurrencyLimit: 2,
+            processBatch: (batch) => this.upsertBatchToVectorDB(batch, entityDocument, templateContent, vectorDB, indexName, delayTimeMS),
         });
-        //Handles upserting the vectors into the vector database
-        const VectorUpserter = new BatchWorker({
-            workerFile: resolve(__dirname, 'workers/UpsertVectors.js'),
-            batchSize: params.UpsertBatchCount || 50,
-            contextUser: super.CurrentUser,
-            workerContext
+    }
+    /**
+     * Renders templates for a batch of entity records and generates embeddings for the rendered text.
+     */
+    async renderAndEmbedBatch(batch, template, templateContent, embedding, delayTimeMS) {
+        TemplateEngineServer.Instance.SetupNunjucks();
+        const validEntries = [];
+        for (const entityData of batch) {
+            const validationResult = this.validateTemplateInput(template, entityData);
+            if (!validationResult.Success) {
+                LogError(`Validation error for record`, undefined, validationResult.Errors.map(e => e.Message).join('\n'));
+                continue;
+            }
+            const result = await TemplateEngineServer.Instance.RenderTemplate(template, templateContent, entityData, true);
+            if (result.Success) {
+                validEntries.push({ text: result.Output, record: entityData });
+            }
+            else {
+                LogError(`Error rendering template`, undefined, result.Message);
+            }
+        }
+        if (validEntries.length === 0) {
+            return [];
+        }
+        const embeddings = await embedding.EmbedTexts({ texts: validEntries.map(e => e.text), model: null });
+        await new Promise((resolve) => setTimeout(resolve, delayTimeMS));
+        return embeddings.vectors.map((vector, index) => ({
+            ID: index,
+            Vector: vector,
+            EntityData: validEntries[index].record,
+            __mj_recordID: String(validEntries[index].record.__mj_recordID),
+            __mj_compositeKey: String(validEntries[index].record.__mj_compositeKey ?? ''),
+            EntityDocument: validEntries[index].record.__mj_entityDocument,
+            VectorID: String(validEntries[index].record.VectorID ?? ''),
+            VectorIndexID: String(validEntries[index].record.VectorIndexID ?? ''),
+            TemplateContent: templateContent.TemplateText,
+        }));
+    }
+    /**
+     * Upserts a batch of embedding data as vector records into the vector database.
+     */
+    /** Default max chars for large text fields (nvarchar(MAX) or MaxLength > 5000) in vector metadata */
+    static { this.DEFAULT_LARGE_FIELD_TRUNCATION = 1000; }
+    /**
+     * Parse the EntityDocumentConfiguration JSON from an entity document.
+     * Returns an empty object if the Configuration column is null or invalid JSON.
+     */
+    parseDocumentConfig(entityDocument) {
+        const raw = entityDocument.Configuration;
+        if (!raw)
+            return {};
+        try {
+            return JSON.parse(raw);
+        }
+        catch {
+            LogError(`Invalid JSON in EntityDocument.Configuration for "${entityDocument.Name}", using defaults`);
+            return {};
+        }
+    }
+    /**
+     * Get fields to include in vector metadata for display in search results.
+     * Respects EntityDocumentConfiguration.metadata.fieldStrategy and per-field overrides.
+     *
+     * Default behavior (no config or fieldStrategy = "all"):
+     *   Include all fields except PKs, binary types, and system (__mj_*) fields.
+     *
+     * "include" strategy: only fields explicitly listed in config.metadata.fields with included=true.
+     * "exclude" strategy: all eligible fields except those listed with included=false.
+     */
+    getDisplayFields(entityInfo, metadataConfig) {
+        if (!entityInfo)
+            return [];
+        const skipTypes = new Set(['uniqueidentifier', 'varbinary', 'image', 'binary', 'timestamp', 'rowversion']);
+        const allEligible = entityInfo.Fields.filter(f => !f.IsPrimaryKey &&
+            !f.Name.startsWith('__mj_') &&
+            !skipTypes.has(f.Type.toLowerCase()));
+        const strategy = metadataConfig?.fieldStrategy ?? 'all';
+        const fieldOverrides = metadataConfig?.fields ?? {};
+        switch (strategy) {
+            case 'include':
+                // Only include fields explicitly marked as included
+                return allEligible.filter(f => fieldOverrides[f.Name]?.included === true);
+            case 'exclude':
+                // Include all except those explicitly excluded
+                return allEligible.filter(f => fieldOverrides[f.Name]?.included !== false);
+            case 'all':
+            default:
+                // Include everything, but respect individual field exclusions
+                return allEligible.filter(f => fieldOverrides[f.Name]?.included !== false);
+        }
+    }
+    /**
+     * Get the truncation limit for a field based on its MaxLength and
+     * optional per-field or global overrides from EntityDocumentConfiguration.
+     */
+    getFieldTruncationLimit(field, metadataConfig) {
+        // Check for per-field override first
+        const fieldConfig = metadataConfig?.fields?.[field.Name];
+        if (fieldConfig?.truncationLimit != null && fieldConfig.truncationLimit > 0) {
+            return fieldConfig.truncationLimit;
+        }
+        // For small fields, use the field's own MaxLength
+        if (field.MaxLength && field.MaxLength > 0 && field.MaxLength <= 5000) {
+            return field.MaxLength;
+        }
+        // Large field — use global override or default
+        return metadataConfig?.defaultTruncationLimit ?? EntityVectorSyncer.DEFAULT_LARGE_FIELD_TRUNCATION;
+    }
+    async upsertBatchToVectorDB(batch, entityDocument, templateContent, vectorDB, indexName, delayTimeMS) {
+        // Parse entity document configuration for metadata enrichment settings
+        const docConfig = this.parseDocumentConfig(entityDocument);
+        const metadataConfig = docConfig.metadata;
+        // Get entity metadata for enriching vector metadata with display fields
+        const md = new Metadata();
+        const entityInfo = md.Entities.find(e => UUIDsEqual(e.ID, entityDocument.EntityID));
+        const displayFields = this.getDisplayFields(entityInfo, metadataConfig);
+        const vectorRecords = batch.map((embeddingItem) => {
+            // Deterministic vector ID: SHA-1 hash of entityDocumentID + compositeKey
+            // ensures re-syncing upserts in place (no duplicates) and stays under
+            // Pinecone's 512-byte ID limit (hash is 40 chars)
+            const raw = `${entityDocument.ID}_${embeddingItem.__mj_compositeKey}`;
+            const hash = createHash('sha1').update(raw).digest('hex');
+            const vectorId = hash;
+            embeddingItem.VectorID = vectorId;
+            // Build enriched metadata with display fields from the record
+            const metadata = {
+                RecordID: String(embeddingItem.__mj_compositeKey ?? ''),
+                Entity: entityDocument.Entity,
+                TemplateID: templateContent.ID,
+            };
+            // Add entity icon if available (respects includeEntityIcon config, default true)
+            if (entityInfo?.Icon && (metadataConfig?.includeEntityIcon !== false)) {
+                metadata['EntityIcon'] = entityInfo.Icon;
+            }
+            // Add __mj_UpdatedAt for recency sorting (respects includeUpdatedAt config, default true)
+            const record = embeddingItem.EntityData;
+            if (record['__mj_UpdatedAt'] && (metadataConfig?.includeUpdatedAt !== false)) {
+                metadata['__mj_UpdatedAt'] = String(record['__mj_UpdatedAt']);
+            }
+            // Add display fields with appropriate truncation (respects config overrides)
+            for (const field of displayFields) {
+                const val = record[field.Name];
+                if (val != null) {
+                    const strVal = String(val);
+                    const limit = this.getFieldTruncationLimit(field, metadataConfig);
+                    metadata[field.Name] = strVal.length > limit ? strVal.substring(0, limit) : strVal;
+                }
+            }
+            return {
+                id: vectorId,
+                values: embeddingItem.Vector,
+                metadata
+            };
         });
-        VectorUpserter.on('error', (err) => {
-            LogError('Error in VectorUpserter worker', null, err);
+        const response = await vectorDB.CreateRecords(vectorRecords, indexName);
+        if (!response.success) {
+            LogError('Unable to save records to vector database', undefined, response.message);
+        }
+        await new Promise((resolve) => setTimeout(resolve, delayTimeMS));
+        return batch;
+    }
+    /**
+     * Validates template input data against template parameter definitions
+     */
+    validateTemplateInput(template, data) {
+        const result = new ValidationResult();
+        const params = template.Params;
+        if (!params) {
+            result.Errors.push({ Source: '', Message: 'Params property not found on the template.', Value: '', Type: 'Failure' });
+        }
+        params?.forEach((p) => {
+            if (p.IsRequired) {
+                // For Record type params, fields are spread to root level (flat convention)
+                // so check that data has any keys, not a specific key matching the param name
+                if (p.Type === 'Record') {
+                    if (Object.keys(data).length === 0) {
+                        result.Errors.push({ Source: p.Name, Message: `Parameter ${p.Name} is required.`, Value: undefined, Type: 'Failure' });
+                    }
+                    return;
+                }
+                const val = data[p.Name];
+                if (val === undefined || val === null || (typeof val === 'string' && val.trim() === '')) {
+                    result.Errors.push({ Source: p.Name, Message: `Parameter ${p.Name} is required.`, Value: val, Type: 'Failure' });
+                }
+            }
         });
-        //short for entity record document upserter
-        // handles upserting entity record document records
-        const ERCUpserter = new Transform({ objectMode: true, transform: (chunk, encoding, callback) => {
-                this.UpsertEntityRecordDocumentRecords(chunk, super.CurrentUser).then(() => callback(null)).catch(callback);
-            } });
+        result.Success = !result.Errors.some(e => e.Type === 'Failure');
+        return result;
+    }
+    /**
+     * Starts the async data paging loop that feeds records into the stream pipeline.
+     */
+    startDataPaging(dataStream, params, md, entity, template, vectorIndexEntity, entityDocument, pageSize) {
         const getData = async () => {
             let pageNumber = 0;
             if (params.StartingOffset) {
@@ -113,7 +371,7 @@ export class EntityVectorSyncer extends VectorBase {
             }
             let hasMore = true;
             while (hasMore) {
-                let pageRecordRequest = {
+                const pageRecordRequest = {
                     EntityID: params.entityID,
                     PageNumber: pageNumber,
                     PageSize: pageSize,
@@ -121,20 +379,20 @@ export class EntityVectorSyncer extends VectorBase {
                 };
                 if (params.listID) {
                     const coreSchema = md.ConfigData.MJCoreSchemaName;
-                    pageRecordRequest.Filter = this.buildListFilter(entity, coreSchema, params.listID);
+                    pageRecordRequest.Filter = this.BuildListFilter(entity, coreSchema, params.listID);
                 }
                 const recordsPage = await super.PageRecordsByEntityID(pageRecordRequest);
                 const relatedData = await this.GetRelatedTemplateDataForBatch(entity, recordsPage, template);
                 const items = [];
                 LogStatus(`Fetched page ${pageNumber + 1} with ${recordsPage.length} records to process`);
                 for (const record of recordsPage) {
-                    const templateData = await this.GetTemplateData(entity, record, template, relatedData);
-                    //we need a reference to this record's ID for the upsert worker
-                    templateData.__mj_recordID = record[entity.FirstPrimaryKey.Name];
-                    templateData.__mj_compositeKey = entity.PrimaryKeys.map((key) => `${key.Name}|${record[key.Name]}`).join("||");
-                    //we also need a reference to the vector index's ID
+                    const typedRecord = record;
+                    const templateData = await this.GetTemplateData(entity, typedRecord, template, relatedData);
+                    templateData.__mj_recordID = typedRecord[entity.FirstPrimaryKey.Name];
+                    templateData.__mj_compositeKey = entity.PrimaryKeys.map((key) => `${key.Name}|${typedRecord[key.Name]}`).join('||');
                     templateData.VectorIndexID = vectorIndexEntity.ID;
                     templateData.TemplateContent = template.Content[0].TemplateText;
+                    templateData.__mj_entityDocument = { ID: entityDocument.ID, EntityID: entityDocument.EntityID, Name: entityDocument.Name };
                     items.push(templateData);
                 }
                 dataStream.addPage(items);
@@ -146,16 +404,10 @@ export class EntityVectorSyncer extends VectorBase {
             }
             dataStream.endStream();
         };
-        // page data asynchrounously and add to the data stream
-        getData();
-        LogStatus('Starting pipeline');
-        await pipeline(dataStream, VectorCreator, VectorUpserter, ERCUpserter, new PassThrough({ objectMode: true }));
-        const endTime = new Date().getTime();
-        //convert ms to seconds
-        const elapsedSeconds = (endTime - startTime) / 1000;
-        const elapsedMinutes = elapsedSeconds / 60;
-        LogStatus(`Finished vectorizing ${entityDocument.Entity} entity in ${elapsedSeconds} seconds (${elapsedMinutes} minutes)`);
-        return null;
+        getData().catch((error) => {
+            LogError('Error during data paging', undefined, error);
+            dataStream.endStream();
+        });
     }
     /**
      * This method will create a default Entity Document for the given entityID, vectorDatabase, and AIModel
@@ -170,7 +422,7 @@ export class EntityVectorSyncer extends VectorBase {
         if (!entity)
             throw new Error(`Entity with ID ${EntityID} not found.`);
         const EDTemplate = entity.Fields.map((ef) => {
-            return `${ef.Name}: \$\{${ef.Name}\}`;
+            return `${ef.Name}: {{${ef.Name}}}`;
         }).join(' ');
         const rv = new RunView();
         const rvResult = await rv.RunView({
@@ -306,9 +558,6 @@ export class EntityVectorSyncer extends VectorBase {
             vectorIndexEntity.VectorDatabaseID = entityDocument.VectorDatabaseID;
             vectorIndexEntity.EmbeddingModelID = entityDocument.AIModelID;
             vectorIndexEntity.Name = `Vector Index for entityDocument ${entityDocument.EntityID}`;
-            vectorIndexEntity.Set('EntityRecordUpdatedAt', new Date());
-            vectorIndexEntity.Set('EntityDocumentID', entityDocument.ID);
-            //not a very descriptive description, but the view has the name of the vectorDB and embedding model used
             vectorIndexEntity.Description = `Vector Index that uses the Vector database ${entityDocument.VectorDatabaseID} and ${entityDocument.AIModelID} as the embedding model`;
             const saveResult = await super.SaveEntity(vectorIndexEntity);
             if (saveResult) {
@@ -387,7 +636,7 @@ export class EntityVectorSyncer extends VectorBase {
     }
     BuildTemplateContent(entityFields) {
         return entityFields.map((field) => {
-            return `{{Entity.${field.Name}}}`;
+            return `{{${field.Name}}}`;
         }).join(' ');
     }
     async GetEntityFieldsForSimilaritySearch(entityID) {
@@ -407,29 +656,35 @@ export class EntityVectorSyncer extends VectorBase {
     async GetTemplateData(entity, record, template, relatedData) {
         const templateData = {};
         for (const param of template.Params) {
-            if (templateData[param.Name]) {
-                continue;
-            }
             switch (param.Type) {
                 case 'Record':
-                    // this one is simple, we create a property by the provided name, and set the value to the record we are currently processing
-                    templateData[param.Name] = record;
+                    // NEW convention: main entity fields are TOP-LEVEL variables (no Entity. prefix).
+                    // Spread record fields directly into the root context so templates use {{FieldName}}.
+                    Object.assign(templateData, record);
                     break;
-                case 'Entity':
-                    // here we need to grab the related data from another entity and filter it down for the record we are current processing so it only shows the related data
-                    // the metadata in the param tells us what we need to know
+                case 'Entity': {
+                    if (templateData[param.Name]) {
+                        continue;
+                    }
                     const paramData = relatedData.find((rd) => rd.ParamName === param.Name);
                     if (!paramData) {
                         LogError(`No related data found for param ${param.Name} in template ${template.ID}`);
                         break;
                     }
-                    // now filter down the data in d to just this record and set the value of the context data to the filtered data
-                    templateData[param.Name] = paramData.Data.filter((rdfr) => rdfr[param.LinkedParameterField] === record[entity.FirstPrimaryKey.Name]);
+                    // Related entities use their relationship name as prefix: {{RelationshipName.FieldName}}
+                    const pkValue = record[entity.FirstPrimaryKey.Name];
+                    templateData[param.Name] = paramData.Data.filter((rdfr) => {
+                        const typedRdfr = rdfr;
+                        return typedRdfr[param.LinkedParameterField] === pkValue;
+                    });
+                    break;
+                }
+                case 'Scalar':
+                    // Flat convention: entity fields are top-level, so pull directly from record
+                    templateData[param.Name] = record[param.Name] ?? '';
                     break;
-                case "Array":
-                case "Scalar":
-                case "Object":
-                    // do nothing here, as we don't directly support these param types
+                case 'Array':
+                case 'Object':
                     LogError(`Unsupported parameter type ${param.Type} for parameter ${param.Name} in template ${template.ID}`);
                     break;
             }
@@ -444,9 +699,12 @@ export class EntityVectorSyncer extends VectorBase {
             }
             const relatedEntity = templateParam.Entity;
             const relatedField = templateParam.LinkedParameterField;
-            // construct a filter for the related field so that we constrain the results to just the set of records linked to our recipients
-            const quotes = entity.FirstPrimaryKey.NeedsQuotes ? "'" : "";
-            const filter = `${relatedField} in (${records.map((record) => `${quotes}${record[entity.FirstPrimaryKey.Name]}${quotes}`).join(',')})`;
+            const quotes = entity.FirstPrimaryKey.NeedsQuotes ? "'" : '';
+            const pkName = entity.FirstPrimaryKey.Name;
+            const filter = `${relatedField} in (${records.map((record) => {
+                const typedRecord = record;
+                return `${quotes}${typedRecord[pkName]}${quotes}`;
+            }).join(',')})`;
             const finalFilter = templateParam.ExtraFilter ? `(${filter}) AND (${templateParam.ExtraFilter})` : filter;
             const result = await super.RunView.RunView({
                 EntityName: relatedEntity,
@@ -454,10 +712,7 @@ export class EntityVectorSyncer extends VectorBase {
                 ResultType: 'simple'
             }, super.CurrentUser);
             if (result && result.Success) {
-                const data = { ParamName: '', Data: [] };
-                data.ParamName = templateParam.Name,
-                    data.Data = result.Results;
-                relatedData.push(data);
+                relatedData.push({ ParamName: templateParam.Name, Data: result.Results });
             }
             else {
                 LogError(`Error getting related data for entity ${relatedEntity} with filter ${finalFilter}`, undefined, result.ErrorMessage);
@@ -465,52 +720,55 @@ export class EntityVectorSyncer extends VectorBase {
         }
         return relatedData;
     }
-    // Rather than having this logic inside the EntityRecordDocumentWorker class, it's placed here instead
-    //so that the logic behind vectorizing entities isnt spread out across so many files
+    /**
+     * Creates or updates an Entity Record Document record linking a source record to its vector embedding.
+     */
     async UpsertEntityRecordDocumentRecords(embeddingData, contextUser) {
-        let md = super.Metadata;
-        let rv = super.RunView;
-        let vectorIndex = await md.GetEntityObject('MJ: Vector Indexes', contextUser);
-        let vectorIndexID = embeddingData.VectorIndexID.toString();
-        let loadResult = await vectorIndex.Load(vectorIndexID);
+        const md = super.Metadata;
+        const rv = super.RunView;
+        const vectorIndexID = String(embeddingData.VectorIndexID);
+        const vectorIndex = await md.GetEntityObject('MJ: Vector Indexes', contextUser);
+        const loadResult = await vectorIndex.Load(vectorIndexID);
         if (!loadResult) {
-            LogError(`Vector Index with ID ${embeddingData.VectorIndexID} not found`);
+            LogError(`Vector Index with ID ${vectorIndexID} not found`);
             return;
         }
-        let entityDocument = embeddingData.EntityDocument;
-        let entityID = entityDocument.EntityID;
-        let existingRecords = [];
+        const entityDocument = embeddingData.EntityDocument;
+        const entityID = String(entityDocument.EntityID);
+        const recordID = String(embeddingData.__mj_recordID);
+        const entityDocumentID = String(entityDocument.ID);
         const runViewResult = await rv.RunView({
             EntityName: 'MJ: Entity Record Documents',
-            ExtraFilter: `EntityID = '${entityID}' AND EntityDocumentID = '${entityDocument.ID}' AND RecordID in ('${embeddingData.__mj_recordID}')`,
+            ExtraFilter: `EntityID = '${entityID}' AND EntityDocumentID = '${entityDocumentID}' AND RecordID in ('${recordID}')`,
             ResultType: 'entity_object'
         }, contextUser);
+        let existingRecords = [];
         if (runViewResult.Success) {
             existingRecords = runViewResult.Results;
         }
         else {
-            LogError(`Error getting existing Entity Record Documents`, undefined, runViewResult.ErrorMessage);
+            LogError('Error getting existing Entity Record Documents', undefined, runViewResult.ErrorMessage);
         }
-        let erdEntity = existingRecords.find((er) => er.Get("RecordID").toString() === embeddingData.__mj_recordID.toString());
+        let erdEntity = existingRecords.find((er) => er.RecordID === recordID);
         if (!erdEntity) {
             erdEntity = await md.GetEntityObject('MJ: Entity Record Documents', contextUser);
             erdEntity.NewRecord();
         }
-        erdEntity.Set("EntityID", entityID.toString());
-        erdEntity.Set("RecordID", embeddingData.__mj_recordID.toString());
-        erdEntity.Set("DocumentText", embeddingData.TemplateContent);
-        erdEntity.Set("VectorID", embeddingData.VectorID.toString());
-        erdEntity.Set("VectorJSON", JSON.stringify(embeddingData.Vector));
-        erdEntity.Set("VectorIndexID", embeddingData.VectorIndexID.toString());
-        erdEntity.Set("EntityRecordUpdatedAt", new Date());
-        erdEntity.Set("EntityDocumentID", embeddingData.EntityDocument.ID.toString());
+        erdEntity.EntityID = entityID;
+        erdEntity.RecordID = recordID;
+        erdEntity.DocumentText = embeddingData.TemplateContent ?? null;
+        erdEntity.VectorID = embeddingData.VectorID != null ? String(embeddingData.VectorID) : null;
+        erdEntity.VectorJSON = JSON.stringify(embeddingData.Vector);
+        erdEntity.VectorIndexID = vectorIndexID;
+        erdEntity.EntityRecordUpdatedAt = new Date();
+        erdEntity.EntityDocumentID = entityDocumentID;
         erdEntity.ContextCurrentUser = contextUser;
-        let erdEntitySaveResult = await erdEntity.Save();
+        const erdEntitySaveResult = await erdEntity.Save();
         if (!erdEntitySaveResult) {
             LogError('Error saving Entity Record Document Entity', undefined, erdEntity.LatestResult);
         }
         else {
-            LogStatus("Upserting Entity Record Documents: Complete");
+            LogStatus('Upserting Entity Record Documents: Complete');
         }
     }
     /**
@@ -539,7 +797,7 @@ export class EntityVectorSyncer extends VectorBase {
      * For single PK entities, uses a simple IN clause.
      * For composite PK entities, uses an EXISTS clause that concatenates PK columns to match the RecordID format.
      */
-    buildListFilter(entity, listDetailsSchema, listId) {
+    BuildListFilter(entity, listDetailsSchema, listId) {
         const primaryKeys = entity.PrimaryKeys;
         if (primaryKeys.length === 1) {
             // Simple case: single primary key