@vertesia/workflow 0.81.0 → 0.81.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vertesia/workflow",
-  "version": "0.81.0",
+  "version": "0.81.1",
   "type": "module",
   "description": "Vertesia workflow DSL",
   "main": "./lib/esm/index.js",
@@ -44,11 +44,11 @@
     "tmp": "^0.2.4",
     "tmp-promise": "^3.0.3",
     "yaml": "^2.6.0",
+    "@vertesia/memory": "0.81.1",
+    "@vertesia/common": "0.81.1",
+    "@vertesia/client": "0.81.1",
     "@llumiverse/common": "0.24.0",
-    "@vertesia/common": "0.81.0",
-    "@vertesia/client": "0.81.0",
-    "@vertesia/api-fetch-client": "0.81.0",
-    "@vertesia/memory": "0.81.0"
+    "@vertesia/api-fetch-client": "0.81.1"
   },
   "ts_dual_module": {
     "outDir": "lib",

package/src/activities/generateEmbeddings.ts

@@ -11,8 +11,8 @@ import {
 } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { DocumentNotFoundError } from "../errors.js";
-import { fetchBlobAsBase64, md5 } from "../utils/blobs.js";
-import { DocPart, getContentParts } from "../utils/chunks.js";
+import { fetchBlobAsBase64 } from "../utils/blobs.js";
+import { DocPart } from "../utils/chunks.js";
 import { countTokens } from "../utils/tokens.js";
 
 export interface GenerateEmbeddingsParams {
@@ -155,8 +155,7 @@ interface ExecuteGenerateEmbeddingsParams {
 }
 
 async function generateTextEmbeddings(
-    { document, client, type, config }: ExecuteGenerateEmbeddingsParams,
-    parts?: DocPart[],
+    { document, client, type, config }: ExecuteGenerateEmbeddingsParams
 ) {
 
     if (!document) {
@@ -185,161 +184,32 @@ async function generateTextEmbeddings(
         };
     }
 
-    const { environment, model } = config;
+    const { environment } = config;
 
-    const partDefinitions = parts ?? [];
-
-    // Count tokens if not already done
-    if (!document.tokens?.count && type === SupportedEmbeddingTypes.text) {
-        log.debug("Updating token count for document: " + document.id);
-        const tokensData = countTokens(document.text!);
-        await client.objects.update(document.id, {
-            tokens: {
-                ...tokensData,
-                etag: document.text_etag ?? md5(document.text!),
-            },
-        });
-        document.tokens = {
-            ...tokensData,
-            etag: document.text_etag ?? md5(document.text!),
-        };
+    // Count tokens if needed, do not rely on existing token count
+    let tokenCount : number | undefined = undefined;
+    if (type === SupportedEmbeddingTypes.text && document.text) {
+        tokenCount = countTokens(document.text).count;
     }
 
     const maxTokens = config.max_tokens ?? 8000;
 
     //generate embeddings for the main doc if document isn't too large
-    //if too large, we'll just generate embeddings for the parts
-    //then we can generate embeddings for the main document by averaging the tensors
     log.info(`Generating ${type} embeddings for document ${document.id}`);
     if (
         type === SupportedEmbeddingTypes.text &&
-        document.tokens?.count &&
-        document.tokens?.count > maxTokens
+        tokenCount !== undefined &&
+        tokenCount > maxTokens
     ) {
-        log.info("Document too large, generating embeddings for parts");
-
-        if (!document.text) {
-            return {
-                id: document.id,
-                status: "failed",
-                message: "no text found",
-            };
-        }
-
-        if (!partDefinitions || partDefinitions.length === 0) {
-            log.info(
-                "No parts found for document, skipping embeddings generation",
-            );
-            return {
-                id: document.id,
-                status: "failed",
-                message: "no parts found",
-            };
-        }
-
-        log.info("Generating embeddings for parts", {
-            parts: partDefinitions,
-            max_tokens: maxTokens,
-        });
-        const docParts = getContentParts(document.text, partDefinitions);
-
-        log.info(`Retrieved ${docParts.length} parts`);
-        const start = new Date().getTime();
-        const generatePartEmbeddings = async (
-            partContent: string,
-            i: number,
-        ) => {
-            const localStart = new Date().getTime();
-            try {
-                log.info(`Generating embeddings for part ${i}`, {
-                    text_len: partContent.length,
-                });
-                if (!partContent) {
-                    return {
-                        id: i,
-                        number: i,
-                        result: null,
-                        status: "skipped",
-                        message: "no text found",
-                    };
-                }
-
-                const e = await generateEmbeddingsFromStudio(
-                    partContent,
-                    environment,
-                    client,
-                    model,
-                ).catch((e) => {
-                    log.error("Error generating embeddings for part " + i, {
-                        text_length: partContent.length,
-                        error: e,
-                    });
-                    return null;
-                });
-
-                if (!e || !e.values) {
-                    return {
-                        id: i,
-                        number: i,
-                        result: null,
-                        message: "no embeddings generated",
-                    };
-                }
-
-                if (e.values.length === 0) {
-                    return {
-                        id: i,
-                        number: i,
-                        result: null,
-                        message: "no embeddings generated",
-                    };
-                }
-                log.info(`Generated embeddings for part ${i}`, {
-                    len: e.values.length,
-                    duration: new Date().getTime() - localStart,
-                });
-
-                return { number: i, result: e };
-            } catch (err: any) {
-                log.info(
-                    `Error generating ${type} embeddings for part ${i} of ${document.id}`,
-                    { error: err },
-                );
-                return {
-                    number: i,
-                    result: null,
-                    message: "error generating embeddings",
-                    error: err.message,
-                };
-            }
-        };
-
-        const partEmbeddings = await Promise.all(
-            docParts.map((part, i) => generatePartEmbeddings(part, i)),
-        );
-        const validPartEmbeddings = partEmbeddings
-            .filter((e) => e.result !== null)
-            .map((e) => e.result);
-        const averagedEmbedding = computeAttentionEmbedding(
-            validPartEmbeddings.map((e) => e.values),
-        );
-        log.info(
-            `Averaged embeddings for document ${document.id} in ${(new Date().getTime() - start) / 1000} seconds`,
-            {
-                len: averagedEmbedding.length,
-                count: validPartEmbeddings.length,
-                max_tokens: maxTokens,
-            },
+        //TODO: Review strategy for large documents
+        log.warn(
+            `Document too large for ${type} embeddings generation, skipping (${tokenCount} tokens)`,
         );
-        await client.objects.setEmbedding(document.id, type, {
-            values: averagedEmbedding,
-            model: validPartEmbeddings[0].model,
-            etag: document.text_etag,
-        });
-        log.info(`Object ${document.id} embedding set`, {
-            type,
-            len: averagedEmbedding.length,
-        });
+        return {
+            id: document.id,
+            status: "skipped",
+            message: `${type} embeddings generation, skipped for large document (${tokenCount} tokens)`,
+        }
     } else {
         log.info(`Generating ${type} embeddings for document`);
 
@@ -480,40 +350,3 @@ async function generateEmbeddingsFromStudio(
             throw e;
         });
 }
-
-//Simplified attention mechanism
-// This is a naive implementation and should be replaced with a more sophisticated
-// using tensorflow in a specific package
-function computeAttentionEmbedding(chunkEmbeddings: number[][]): number[] {
-    if (chunkEmbeddings.length === 0) return [];
-
-    const start = new Date().getTime();
-
-    // Generate random attention weights
-    const attentionWeights = chunkEmbeddings.map(() => Math.random());
-
-    // Apply softmax to get attention scores
-    const expWeights = attentionWeights.map((w) => Math.exp(w));
-    const sumExpWeights = expWeights.reduce((sum, val) => sum + val, 0);
-    const attentionScores = expWeights.map((w) => w / sumExpWeights);
-
-    // Get embedding dimension
-    const embeddingDim = chunkEmbeddings[0].length;
-
-    // Initialize document embedding
-    const documentEmbedding = new Array(embeddingDim).fill(0);
-
-    // Weighted sum of embeddings
-    for (let i = 0; i < chunkEmbeddings.length; i++) {
-        for (let j = 0; j < embeddingDim; j++) {
-            documentEmbedding[j] += chunkEmbeddings[i][j] * attentionScores[i];
-        }
-    }
-
-    const duration = new Date().getTime() - start;
-    console.log(
-        `Computed document embedding in ${duration}ms for ${chunkEmbeddings.length} chunks`,
-    );
-
-    return documentEmbedding;
-}