@memberjunction/ai-vector-sync 3.3.0 → 4.0.0

package/dist/models/workers/UpsertVectors.js

@@ -1,22 +1,18 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.UpsertVectorRecords = void 0;
-const node_worker_threads_1 = require("node:worker_threads");
-const ai_vectordb_1 = require("@memberjunction/ai-vectordb");
-const global_1 = require("@memberjunction/global");
-const ai_vectors_pinecone_1 = require("@memberjunction/ai-vectors-pinecone");
-const core_1 = require("@memberjunction/core");
-(0, ai_vectors_pinecone_1.LoadPineconeVectorDB)();
+import { parentPort, threadId, workerData } from 'node:worker_threads';
+import { VectorDBBase } from '@memberjunction/ai-vectordb';
+import { MJGlobal } from '@memberjunction/global';
+import { LogError } from '@memberjunction/core';
 const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-async function UpsertVectorRecords() {
-    const { batch, context } = node_worker_threads_1.workerData;
+export async function UpsertVectorRecords() {
+    const { batch, context } = workerData;
     const embeddingsBatch = batch;
     if (!embeddingsBatch) {
         throw new Error('Embeddings are required for the ArchiveWorker');
     }
     const entityDocument = context.entityDocument;
-    const vectorDB = global_1.MJGlobal.Instance.ClassFactory.CreateInstance(ai_vectordb_1.VectorDBBase, context.vectorDBClassKey, context.vectorDBAPIKey);
+    const vectorDB = MJGlobal.Instance.ClassFactory.CreateInstance(VectorDBBase, context.vectorDBClassKey, context.vectorDBAPIKey);
     const startTime = Date.now();
+    //console.log('\t##### Archiver started #####', { threadId, now: Date.now() % 10_000, elapsed: Date.now() - context.executionId });
     const vectorRecords = embeddingsBatch.map((embedding) => {
         const guid = uuidv4();
         embedding.VectorID = guid;
@@ -32,15 +28,14 @@ async function UpsertVectorRecords() {
     });
     const response = await vectorDB.createRecords(vectorRecords);
     if (!response.success) {
-        (0, core_1.LogError)('Unable to successfully save records to vector database', undefined, response.message);
+        LogError('Unable to successfully save records to vector database', undefined, response.message);
     }
     const runTime = Date.now() - startTime;
     const elapsed = Date.now() - context.executionId;
-    console.log('\t##### Upserting records to vector database: Complete #####', { threadId: node_worker_threads_1.threadId, now: Date.now() % 100_000, runTime, elapsed });
-    await delay(context.delayTimeMS);
-    node_worker_threads_1.parentPort.postMessage({ ...node_worker_threads_1.workerData, ...batch });
+    console.log('\t##### Upserting records to vector database: Complete #####', { threadId, now: Date.now() % 100_000, runTime, elapsed });
+    await delay(context.delayTimeMS); //short deplay to avoid getting rate limited by the embedding model's api 
+    parentPort.postMessage({ ...workerData, ...batch });
 }
-exports.UpsertVectorRecords = UpsertVectorRecords;
 function uuidv4() {
     return "10000000-1000-4000-8000-100000000000".replace(/[018]/g, c => (+c ^ crypto.getRandomValues(new Uint8Array(1))[0] & 15 >> +c / 4).toString(16));
 }

package/dist/models/workers/UpsertVectors.js.map

@@ -1 +1 @@
-{"version":3,"file":"UpsertVectors.js","sourceRoot":"","sources":["../../../src/models/workers/UpsertVectors.ts"],"names":[],"mappings":";;;AAAA,6DAAuE;AAEvE,6DAAuF;AACvF,mDAAkD;AAClD,6EAA2E;AAC3E,+CAAgD;AAGhD,IAAA,0CAAoB,GAAE,CAAC;AAEvB,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEzE,KAAK,UAAU,mBAAmB;IACvC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,gCAA8C,CAAC;IAE1E,MAAM,eAAe,GAAoB,KAAY,CAAC;IACtD,IAAG,CAAC,eAAe,EAAC,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;IAC9C,MAAM,QAAQ,GAAiB,iBAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAe,0BAAY,EAAE,OAAO,CAAC,gBAAgB,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;IAE3J,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAG7B,MAAM,aAAa,GAAmB,eAAe,CAAC,GAAG,CAAC,CAAC,SAAwB,EAAE,EAAE;QACrF,MAAM,IAAI,GAAW,MAAM,EAAE,CAAC;QAC9B,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC;QAC1B,OAAO;YACL,EAAE,EAAE,IAAI;YACR,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,QAAQ,EAAE;gBACR,QAAQ,EAAE,SAAS,CAAC,iBAAiB,CAAC,QAAQ,EAAE;gBAChD,MAAM,EAAE,cAAc,CAAC,MAAM;gBAC7B,UAAU,EAAE,OAAO,CAAC,eAAe,CAAC,EAAE;aACvC;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAiB,MAAM,QAAQ,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;IAC3E,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtB,IAAA,eAAQ,EAAC,wDAAwD,EAAE,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAClG,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,8DAA8D,EAAE,EAAE,QAAQ,EAAR,8BAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACvI,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACjC,gCAAU,CAAC,WAAW,CAAC,EAAE,GAAG,gCAAU,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AACtD,CAAC;AAtCD,kDAsCC;AAED,SAAS,MAAM;IACb,OAAO,sCAAsC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAClE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAChF,CAAC;AACJ,CAAC;AAED,mBAAmB,EAAE,CAAC"}
+{"version":3,"file":"UpsertVectors.js","sourceRoot":"","sources":["../../../src/models/workers/UpsertVectors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAgB,YAAY,EAAgB,MAAM,6BAA6B,CAAC;AACvF,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,KAAK,UAAU,mBAAmB;IACvC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,UAA8C,CAAC;IAE1E,MAAM,eAAe,GAAoB,KAAY,CAAC;IACtD,IAAG,CAAC,eAAe,EAAC,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;IAC9C,MAAM,QAAQ,GAAiB,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAe,YAAY,EAAE,OAAO,CAAC,gBAAgB,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;IAE3J,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,mIAAmI;IAEnI,MAAM,aAAa,GAAmB,eAAe,CAAC,GAAG,CAAC,CAAC,SAAwB,EAAE,EAAE;QACrF,MAAM,IAAI,GAAW,MAAM,EAAE,CAAC;QAC9B,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC;QAC1B,OAAO;YACL,EAAE,EAAE,IAAI;YACR,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,QAAQ,EAAE;gBACR,QAAQ,EAAE,SAAS,CAAC,iBAAiB,CAAC,QAAQ,EAAE;gBAChD,MAAM,EAAE,cAAc,CAAC,MAAM;gBAC7B,UAAU,EAAE,OAAO,CAAC,eAAe,CAAC,EAAE;aACvC;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAiB,MAAM,QAAQ,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;IAC3E,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtB,QAAQ,CAAC,wDAAwD,EAAE,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;IAClG,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,8DAA8D,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACvI,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,0EAA0E;IAC5G,UAAU,CAAC,WAAW,CAAC,EAAE,GAAG,UAAU,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,MAAM;IACb,OAAO,sCAAsC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAClE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAChF,CAAC;AACJ,CAAC;AAED,mBAAmB,EAAE,CAAC"}

package/dist/models/workers/VectorizeTemplates.js

@@ -1,35 +1,34 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.VectorizeEntity = void 0;
-const node_worker_threads_1 = require("node:worker_threads");
-const templates_1 = require("@memberjunction/templates");
-const core_1 = require("@memberjunction/core");
-const global_1 = require("@memberjunction/global");
-const ai_1 = require("@memberjunction/ai");
+import { parentPort, threadId, workerData } from 'node:worker_threads';
+import { TemplateEngineServer } from '@memberjunction/templates';
+import { LogError, ValidationResult } from '@memberjunction/core';
+import { MJGlobal } from '@memberjunction/global';
+import { BaseEmbeddings } from '@memberjunction/ai';
+// AI provider loading now handled by @memberjunction/aiengine
 const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-async function VectorizeEntity() {
-    const { batch, context } = node_worker_threads_1.workerData;
+export async function VectorizeEntity() {
+    const { batch, context } = workerData;
     if (!batch) {
         throw new Error('batch is required for the AnnotationWorker');
     }
     const template = context.template;
     const templateContent = context.templateContent;
-    templates_1.TemplateEngineServer.Instance.SetupNunjucks();
+    TemplateEngineServer.Instance.SetupNunjucks();
     const startTime = Date.now();
-    const embedding = global_1.MJGlobal.Instance.ClassFactory.CreateInstance(ai_1.BaseEmbeddings, context.embeddingDriverClass, context.embeddingAPIKey);
+    //console.log('\t##### Annotator started #####', { threadId, now: Date.now() % 10_000, elapsed: Date.now() - context.executionId });
+    const embedding = MJGlobal.Instance.ClassFactory.CreateInstance(BaseEmbeddings, context.embeddingDriverClass, context.embeddingAPIKey);
     const processedBatch = [];
     for (const entityData of batch) {
         const validationResult = ValidateTemplateInput(template, entityData);
         if (!validationResult.Success) {
-            (0, core_1.LogError)(`Validation error for record ${entityData.ID}`, undefined, validationResult.Errors.map(e => e.Message).join('\n'));
+            LogError(`Validation error for record ${entityData.ID}`, undefined, validationResult.Errors.map(e => e.Message).join('\n'));
             continue;
         }
-        let result = await templates_1.TemplateEngineServer.Instance.RenderTemplate(template, templateContent, entityData, true);
+        let result = await TemplateEngineServer.Instance.RenderTemplate(template, templateContent, entityData, true);
         if (result.Success) {
             processedBatch.push(result.Output);
         }
         else {
-            (0, core_1.LogError)(`Error rendering template for record ${entityData.ID}`, undefined, result.Message);
+            LogError(`Error rendering template for record ${entityData.ID}`, undefined, result.Message);
         }
     }
     const embeddings = await embedding.EmbedTexts({ texts: processedBatch, model: null });
@@ -48,13 +47,16 @@ async function VectorizeEntity() {
     });
     const runTime = Date.now() - startTime;
     const elapsed = Date.now() - context.executionId;
-    console.log('\t##### Generating Vectors: Complete #####', { threadId: node_worker_threads_1.threadId, now: Date.now() % 100_000, runTime, elapsed });
-    await delay(context.delayTimeMS);
-    node_worker_threads_1.parentPort.postMessage({ ...node_worker_threads_1.workerData, batch: embeddingBatch });
+    console.log('\t##### Generating Vectors: Complete #####', { threadId, now: Date.now() % 100_000, runTime, elapsed });
+    await delay(context.delayTimeMS); //short deplay to avoid getting rate limited by the embedding model's api 
+    parentPort.postMessage({ ...workerData, batch: embeddingBatch });
 }
-exports.VectorizeEntity = VectorizeEntity;
+/**
+ * This method is different from the Validate() method which validates the state of the Template itself. This method validates the data object provided meets the requirements for the template's parameter definitions.
+ * @param data - the data object to validate against the template's parameter definitions
+ */
 function ValidateTemplateInput(template, data) {
-    const result = new core_1.ValidationResult();
+    const result = new ValidationResult();
     let params = template.Params;
     if (!params) {
         result.Errors.push({
@@ -77,6 +79,7 @@ function ValidateTemplateInput(template, data) {
             }
         }
     });
+    // now set result's top level success falg based on the existence of ANY failure record within the errors collection
     result.Success = result.Errors.some(e => e.Type === 'Failure') ? false : true;
     return result;
 }

package/dist/models/workers/VectorizeTemplates.js.map

@@ -1 +1 @@
-{"version":3,"file":"VectorizeTemplates.js","sourceRoot":"","sources":["../../../src/models/workers/VectorizeTemplates.ts"],"names":[],"mappings":";;;AAAA,6DAAuE;AAEvE,yDAAiE;AAGjE,+CAAkE;AAClE,mDAAkD;AAClD,2CAAsE;AAItE,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEzE,KAAK,UAAU,eAAe;IACnC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,gCAA+C,CAAC;IAE3E,IAAG,CAAC,KAAK,EAAC,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,QAAQ,GAA2B,OAAO,CAAC,QAAQ,CAAC;IAC1D,MAAM,eAAe,GAA0B,OAAO,CAAC,eAAe,CAAC;IACvE,gCAAoB,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAG7B,MAAM,SAAS,GAAmB,iBAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAiB,mBAAc,EAAE,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IACvK,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;QAC/B,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAA,eAAQ,EAAC,+BAA+B,UAAU,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5H,SAAS;QACX,CAAC;QAED,IAAI,MAAM,GAAyB,MAAM,gCAAoB,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QACnI,IAAG,MAAM,CAAC,OAAO,EAAC,CAAC;YACjB,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;aACG,CAAC;YACH,IAAA,eAAQ,EAAC,uCAAuC,UAAU,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAqB,MAAM,SAAS,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACxG,MAAM,cAAc,GAAoB,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAgB,EAAE,KAAa,EAAE,EAAE;QACjG,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC;YACxB,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,aAAa;YACzC,iBAAiB,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,iBAAiB;YACjD,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ;YAC/B,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,aAAa;YACzC,eAAe,EAAE,eAAe,CAAC,YAAY;SAC9C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,4CAA4C,EAAE,EAAE,QAAQ,EAAR,8BAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACrH,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACjC,gCAAU,CAAC,WAAW,CAAC,EAAE,GAAG,gCAAU,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;AACnE,CAAC;AAnDD,0CAmDC;AAMD,SAAS,qBAAqB,CAAC,QAAgC,EAAE,IAAS;IACxE,MAAM,MAAM,GAAG,IAAI,uBAAgB,EAAE,CAAC;IACtC,IAAI,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAE7B,IAAG,CAAC,MAAM,EAAC,CAAC;QACV,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACjB,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,4CAA4C;YACrD,KAAK,EAAE,EAAE;YACT,IAAI,EAAE,SAAS;SAChB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;QACpB,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI;gBAC5D,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAC,CAAC;gBAC3E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACjB,MAAM,EAAE,CAAC,CAAC,IAAI;oBACd,OAAO,EAAE,aAAa,CAAC,CAAC,IAAI,eAAe;oBAC3C,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,SAAS;iBAChB,CAAC,CAAC;YACL,CAAC;QACP,CAAC;IACL,CAAC,CAAC,CAAC;IAGH,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,eAAe,EAAE,CAAC"}
+{"version":3,"file":"VectorizeTemplates.js","sourceRoot":"","sources":["../../../src/models/workers/VectorizeTemplates.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAGjE,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAoB,MAAM,oBAAoB,CAAC;AAEtE,8DAA8D;AAE9D,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,UAA+C,CAAC;IAE3E,IAAG,CAAC,KAAK,EAAC,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,QAAQ,GAA2B,OAAO,CAAC,QAAQ,CAAC;IAC1D,MAAM,eAAe,GAA0B,OAAO,CAAC,eAAe,CAAC;IACvE,oBAAoB,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,oIAAoI;IAEpI,MAAM,SAAS,GAAmB,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAiB,cAAc,EAAE,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IACvK,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;QAC/B,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YAC9B,QAAQ,CAAC,+BAA+B,UAAU,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5H,SAAS;QACX,CAAC;QAED,IAAI,MAAM,GAAyB,MAAM,oBAAoB,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QACnI,IAAG,MAAM,CAAC,OAAO,EAAC,CAAC;YACjB,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;aACG,CAAC;YACH,QAAQ,CAAC,uCAAuC,UAAU,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAqB,MAAM,SAAS,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACxG,MAAM,cAAc,GAAoB,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAgB,EAAE,KAAa,EAAE,EAAE;QACjG,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC;YACxB,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,aAAa;YACzC,iBAAiB,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,iBAAiB;YACjD,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ;YAC/B,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,aAAa;YACzC,eAAe,EAAE,eAAe,CAAC,YAAY;SAC9C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,4CAA4C,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACrH,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,0EAA0E;IAC5G,UAAU,CAAC,WAAW,CAAC,EAAE,GAAG,UAAU,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;AACnE,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,QAAgC,EAAE,IAAS;IACxE,MAAM,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACtC,IAAI,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAE7B,IAAG,CAAC,MAAM,EAAC,CAAC;QACV,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACjB,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,4CAA4C;YACrD,KAAK,EAAE,EAAE;YACT,IAAI,EAAE,SAAS;SAChB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;QACpB,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI;gBAC5D,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAC,CAAC;gBAC3E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACjB,MAAM,EAAE,CAAC,CAAC,IAAI;oBACd,OAAO,EAAE,aAAa,CAAC,CAAC,IAAI,eAAe;oBAC3C,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,SAAS;iBAChB,CAAC,CAAC;YACL,CAAC;QACP,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,oHAAoH;IACpH,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,eAAe,EAAE,CAAC"}

package/dist/models/workers/upserter.js

@@ -1,11 +1,8 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PostMessagePostUpsert = void 0;
-const node_worker_threads_1 = require("node:worker_threads");
-function PostMessagePostUpsert() {
-    const { batch, context } = node_worker_threads_1.workerData;
-    node_worker_threads_1.parentPort.postMessage({ ...node_worker_threads_1.workerData, ...batch });
+import { parentPort, workerData } from 'node:worker_threads';
+export function PostMessagePostUpsert() {
+    //just pass the data back to the main thread
+    const { batch, context } = workerData;
+    parentPort.postMessage({ ...workerData, ...batch });
 }
-exports.PostMessagePostUpsert = PostMessagePostUpsert;
 PostMessagePostUpsert();
 //# sourceMappingURL=upserter.js.map

package/dist/models/workers/upserter.js.map

@@ -1 +1 @@
-{"version":3,"file":"upserter.js","sourceRoot":"","sources":["../../../src/models/workers/upserter.ts"],"names":[],"mappings":";;;AAAA,6DAA6D;AAI7D,SAAgB,qBAAqB;IAEnC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,gCAA8C,CAAC;IAC1E,gCAAU,CAAC,WAAW,CAAC,EAAE,GAAG,gCAAU,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AACtD,CAAC;AAJD,sDAIC;AAED,qBAAqB,EAAE,CAAC"}
+{"version":3,"file":"upserter.js","sourceRoot":"","sources":["../../../src/models/workers/upserter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAI7D,MAAM,UAAU,qBAAqB;IACjC,4CAA4C;IAC9C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,UAA8C,CAAC;IAC1E,UAAU,CAAC,WAAW,CAAC,EAAE,GAAG,UAAU,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AACtD,CAAC;AAED,qBAAqB,EAAE,CAAC"}

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@memberjunction/ai-vector-sync",
-  "version": "3.3.0",
+  "type": "module",
+  "version": "4.0.0",
   "description": "MemberJunction: AI Vector/Entity Sync Package - handles synchronization between MemberJunction entities and vector databases",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -8,31 +9,31 @@
     "/dist"
   ],
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && tsc-alias -f",
     "buld:pgk": "pkgroll --sourcemap --minify",
     "watch": "pkgroll --sourcemap --watch"
   },
   "author": "MemberJunction.com",
   "license": "ISC",
   "dependencies": {
-    "@memberjunction/ai": "3.3.0",
-    "@memberjunction/ai-vectordb": "3.3.0",
-    "@memberjunction/ai-vectors": "3.3.0",
-    "@memberjunction/ai-vectors-pinecone": "3.3.0",
-    "@memberjunction/aiengine": "3.3.0",
-    "@memberjunction/core": "3.3.0",
-    "@memberjunction/core-entities": "3.3.0",
-    "@memberjunction/global": "3.3.0",
-    "@memberjunction/templates": "3.3.0",
-    "@memberjunction/templates-base-types": "3.3.0",
-    "dotenv": "^16.4.1",
-    "mssql": "^11.0.1"
+    "@memberjunction/ai": "4.0.0",
+    "@memberjunction/ai-vectordb": "4.0.0",
+    "@memberjunction/ai-vectors": "4.0.0",
+    "@memberjunction/ai-vectors-pinecone": "4.0.0",
+    "@memberjunction/aiengine": "4.0.0",
+    "@memberjunction/core": "4.0.0",
+    "@memberjunction/core-entities": "4.0.0",
+    "@memberjunction/global": "4.0.0",
+    "@memberjunction/templates": "4.0.0",
+    "@memberjunction/templates-base-types": "4.0.0",
+    "dotenv": "^17.2.4",
+    "mssql": "^12.2.0"
   },
   "devDependencies": {
-    "@types/mssql": "^9.1.5",
-    "@types/node": "20.14.2",
-    "pkgroll": "2.1.1",
-    "typescript": "^5.4.5"
+    "@types/mssql": "^9.1.9",
+    "@types/node": "24.10.11",
+    "pkgroll": "2.23.0",
+    "typescript": "^5.9.3"
   },
   "repository": {
     "type": "git",