@memberjunction/ai-vector-sync 2.4.1 → 2.5.0

package/dist/models/workers/VectorizeTemplates.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VectorizeEntity = void 0;
+const node_worker_threads_1 = require("node:worker_threads");
+const templates_1 = require("@memberjunction/templates");
+const core_1 = require("@memberjunction/core");
+const global_1 = require("@memberjunction/global");
+const ai_1 = require("@memberjunction/ai");
+const ai_openai_1 = require("@memberjunction/ai-openai");
+const ai_mistral_1 = require("@memberjunction/ai-mistral");
+(0, ai_openai_1.LoadOpenAILLM)();
+(0, ai_mistral_1.LoadMistralEmbedding)();
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+async function VectorizeEntity() {
+    const { batch, context } = node_worker_threads_1.workerData;
+    if (!batch) {
+        throw new Error('batch is required for the AnnotationWorker');
+    }
+    const template = context.template;
+    const templateContent = context.templateContent;
+    templates_1.TemplateEngineServer.Instance.SetupNunjucks();
+    const startTime = Date.now();
+    const embedding = global_1.MJGlobal.Instance.ClassFactory.CreateInstance(ai_1.Embeddings, context.embeddingDriverClass, context.embeddingAPIKey);
+    const processedBatch = [];
+    for (const entityData of batch) {
+        const validationResult = ValidateTemplateInput(template, entityData);
+        if (!validationResult.Success) {
+            (0, core_1.LogError)(`Validation error for record ${entityData.ID}`, undefined, validationResult.Errors.map(e => e.Message).join('\n'));
+            continue;
+        }
+        let result = await templates_1.TemplateEngineServer.Instance.RenderTemplate(template, templateContent, entityData, true);
+        if (result.Success) {
+            processedBatch.push(result.Output);
+        }
+        else {
+            (0, core_1.LogError)(`Error rendering template for record ${entityData.ID}`, undefined, result.Message);
+        }
+    }
+    const embeddings = await embedding.EmbedTexts({ texts: processedBatch, model: null });
+    const embeddingBatch = embeddings.vectors.map((vector, index) => {
+        return {
+            ID: index,
+            Vector: vector,
+            EntityData: batch[index],
+            __mj_recordID: batch[index].__mj_recordID,
+            __mj_compositeKey: batch[index].__mj_compositeKey,
+            EntityDocument: context.entityDocument,
+            VectorID: batch[index].VectorID,
+            VectorIndexID: batch[index].VectorIndexID,
+            TemplateContent: templateContent.TemplateText
+        };
+    });
+    const runTime = Date.now() - startTime;
+    const elapsed = Date.now() - context.executionId;
+    console.log('\t##### Generating Vectors: Complate #####', { threadId: node_worker_threads_1.threadId, now: Date.now() % 100_000, runTime, elapsed });
+    await delay(context.delayTimeMS);
+    node_worker_threads_1.parentPort.postMessage({ ...node_worker_threads_1.workerData, batch: embeddingBatch });
+}
+exports.VectorizeEntity = VectorizeEntity;
+function ValidateTemplateInput(template, data) {
+    const result = new core_1.ValidationResult();
+    let params = template.Params;
+    if (!params) {
+        result.Errors.push({
+            Source: "",
+            Message: "Params property not found on the template.",
+            Value: "",
+            Type: 'Failure'
+        });
+    }
+    params?.forEach((p) => {
+        if (p.IsRequired) {
+            if (!data || data[p.Name] === undefined || data[p.Name] === null ||
+                (typeof data[p.Name] === 'string' && data[p.Name].toString().trim() === '')) {
+                result.Errors.push({
+                    Source: p.Name,
+                    Message: `Parameter ${p.Name} is required.`,
+                    Value: data[p.Name],
+                    Type: 'Failure'
+                });
+            }
+        }
+    });
+    result.Success = result.Errors.some(e => e.Type === 'Failure') ? false : true;
+    return result;
+}
+VectorizeEntity();
+//# sourceMappingURL=VectorizeTemplates.js.map

package/dist/models/workers/VectorizeTemplates.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VectorizeTemplates.js","sourceRoot":"","sources":["../../../src/models/workers/VectorizeTemplates.ts"],"names":[],"mappings":";;;AAAA,6DAAuE;AAGvE,yDAAiE;AAEjE,+CAAkE;AAClE,mDAAkD;AAClD,2CAAkE;AAElE,yDAA0D;AAC1D,2DAAkE;AAGlE,IAAA,yBAAa,GAAE,CAAC;AAChB,IAAA,iCAAoB,GAAE,CAAC;AAEvB,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEzE,KAAK,UAAU,eAAe;IACnC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,gCAA+C,CAAC;IAE3E,IAAG,CAAC,KAAK,EAAC,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,QAAQ,GAA2B,OAAO,CAAC,QAAQ,CAAC;IAC1D,MAAM,eAAe,GAA0B,OAAO,CAAC,eAAe,CAAC;IACvE,gCAAoB,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;IAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAG7B,MAAM,SAAS,GAAe,iBAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,cAAc,CAAa,eAAU,EAAE,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IAC3J,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;QAC/B,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAA,eAAQ,EAAC,+BAA+B,UAAU,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5H,SAAS;QACX,CAAC;QAED,IAAI,MAAM,GAAyB,MAAM,gCAAoB,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QACnI,IAAG,MAAM,CAAC,OAAO,EAAC,CAAC;YACjB,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;aACG,CAAC;YACH,IAAA,eAAQ,EAAC,uCAAuC,UAAU,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAqB,MAAM,SAAS,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACxG,MAAM,cAAc,GAAoB,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAgB,EAAE,KAAa,EAAE,EAAE;QACjG,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC;YACxB,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,aAAa;YACzC,iBAAiB,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,iBAAiB;YACjD,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ;YAC/B,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,aAAa;YACzC,eAAe,EAAE,eAAe,CAAC,YAAY;SAC9C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,4CAA4C,EAAE,EAAE,QAAQ,EAAR,8BAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACrH,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACjC,gCAAU,CAAC,WAAW,CAAC,EAAE,GAAG,gCAAU,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;AACnE,CAAC;AAnDD,0CAmDC;AAMD,SAAS,qBAAqB,CAAC,QAAgC,EAAE,IAAS;IACxE,MAAM,MAAM,GAAG,IAAI,uBAAgB,EAAE,CAAC;IACtC,IAAI,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAE7B,IAAG,CAAC,MAAM,EAAC,CAAC;QACV,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACjB,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,4CAA4C;YACrD,KAAK,EAAE,EAAE;YACT,IAAI,EAAE,SAAS;SAChB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;QACpB,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI;gBAC5D,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAC,CAAC;gBAC3E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACjB,MAAM,EAAE,CAAC,CAAC,IAAI;oBACd,OAAO,EAAE,aAAa,CAAC,CAAC,IAAI,eAAe;oBAC3C,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;oBACnB,IAAI,EAAE,SAAS;iBAChB,CAAC,CAAC;YACL,CAAC;QACP,CAAC;IACL,CAAC,CAAC,CAAC;IAGH,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,eAAe,EAAE,CAAC"}

package/dist/models/workers/upserter.d.ts

@@ -0,0 +1,2 @@
+export declare function PostMessagePostUpsert(): void;
+//# sourceMappingURL=upserter.d.ts.map

package/dist/models/workers/upserter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upserter.d.ts","sourceRoot":"","sources":["../../../src/models/workers/upserter.ts"],"names":[],"mappings":"AAIA,wBAAgB,qBAAqB,IAAI,IAAI,CAI5C"}

package/dist/models/workers/upserter.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostMessagePostUpsert = void 0;
+const node_worker_threads_1 = require("node:worker_threads");
+function PostMessagePostUpsert() {
+    const { batch, context } = node_worker_threads_1.workerData;
+    node_worker_threads_1.parentPort.postMessage({ ...node_worker_threads_1.workerData, ...batch });
+}
+exports.PostMessagePostUpsert = PostMessagePostUpsert;
+PostMessagePostUpsert();
+//# sourceMappingURL=upserter.js.map

package/dist/models/workers/upserter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"upserter.js","sourceRoot":"","sources":["../../../src/models/workers/upserter.ts"],"names":[],"mappings":";;;AAAA,6DAA6D;AAI7D,SAAgB,qBAAqB;IAEnC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,gCAA8C,CAAC;IAC1E,gCAAU,CAAC,WAAW,CAAC,EAAE,GAAG,gCAAU,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;AACtD,CAAC;AAJD,sDAIC;AAED,qBAAqB,EAAE,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memberjunction/ai-vector-sync",
-  "version": "2.4.1",
+  "version": "2.5.0",
   "description": "MemberJunction: AI Vector/Entity Sync Package - handles synchronization between MemberJunction entities and vector databases",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -8,24 +8,24 @@
     "/dist"
   ],
   "scripts": {
-    "build": "pkgroll --sourcemap --minify",
-    "build:tsc": "tsc",
+    "build": "tsc",
+    "buld:pgk": "pkgroll --sourcemap --minify",
     "watch": "pkgroll --sourcemap --watch"
   },
   "author": "MemberJunction.com",
   "license": "ISC",
   "dependencies": {
-    "@memberjunction/ai": "2.4.1",
-    "@memberjunction/ai-vectordb": "2.4.1",
-    "@memberjunction/ai-vectors": "2.4.1",
-    "@memberjunction/ai-vectors-pinecone": "2.4.1",
-    "@memberjunction/ai-mistral": "2.4.1",
-    "@memberjunction/aiengine": "2.4.1",
-    "@memberjunction/core": "2.4.1",
-    "@memberjunction/global": "2.4.1",
-    "@memberjunction/templates": "2.4.1",
-    "@memberjunction/templates-base-types": "2.4.1",
-    "@memberjunction/ai-openai": "2.4.1",
+    "@memberjunction/ai": "2.5.0",
+    "@memberjunction/ai-vectordb": "2.5.0",
+    "@memberjunction/ai-vectors": "2.5.0",
+    "@memberjunction/ai-vectors-pinecone": "2.5.0",
+    "@memberjunction/ai-mistral": "2.5.0",
+    "@memberjunction/aiengine": "2.5.0",
+    "@memberjunction/core": "2.5.0",
+    "@memberjunction/global": "2.5.0",
+    "@memberjunction/templates": "2.5.0",
+    "@memberjunction/templates-base-types": "2.5.0",
+    "@memberjunction/ai-openai": "2.5.0",
     "dotenv": "^16.4.1",
     "typeorm": "^0.3.20"
   },