@vertesia/workflow 0.80.0 → 0.81.1

package/lib/cjs/activities/generateEmbeddings.js

@@ -6,7 +6,6 @@ const common_1 = require("@vertesia/common");
 const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
 const errors_js_1 = require("../errors.js");
 const blobs_js_1 = require("../utils/blobs.js");
-const chunks_js_1 = require("../utils/chunks.js");
 const tokens_js_1 = require("../utils/tokens.js");
 async function generateEmbeddings(payload) {
     const { params, client, objectId, fetchProject } = await (0, ActivityContext_js_1.setupActivity)(payload);
@@ -81,7 +80,7 @@ async function generateEmbeddings(payload) {
     }
     return res;
 }
-async function generateTextEmbeddings({ document, client, type, config }, parts) {
+async function generateTextEmbeddings({ document, client, type, config }) {
     if (!document) {
         return { status: "error", message: "document is null or undefined" };
     }
@@ -103,127 +102,25 @@ async function generateTextEmbeddings({ document, client, type, config }, parts)
             message: "no properties found",
         };
     }
-    const { environment, model } = config;
-    const partDefinitions = parts ?? [];
-    // Count tokens if not already done
-    if (!document.tokens?.count && type === common_1.SupportedEmbeddingTypes.text) {
-        activity_1.log.debug("Updating token count for document: " + document.id);
-        const tokensData = (0, tokens_js_1.countTokens)(document.text);
-        await client.objects.update(document.id, {
-            tokens: {
-                ...tokensData,
-                etag: document.text_etag ?? (0, blobs_js_1.md5)(document.text),
-            },
-        });
-        document.tokens = {
-            ...tokensData,
-            etag: document.text_etag ?? (0, blobs_js_1.md5)(document.text),
-        };
+    const { environment } = config;
+    // Count tokens if needed, do not rely on existing token count
+    let tokenCount = undefined;
+    if (type === common_1.SupportedEmbeddingTypes.text && document.text) {
+        tokenCount = (0, tokens_js_1.countTokens)(document.text).count;
     }
     const maxTokens = config.max_tokens ?? 8000;
     //generate embeddings for the main doc if document isn't too large
-    //if too large, we'll just generate embeddings for the parts
-    //then we can generate embeddings for the main document by averaging the tensors
     activity_1.log.info(`Generating ${type} embeddings for document ${document.id}`);
     if (type === common_1.SupportedEmbeddingTypes.text &&
-        document.tokens?.count &&
-        document.tokens?.count > maxTokens) {
-        activity_1.log.info("Document too large, generating embeddings for parts");
-        if (!document.text) {
-            return {
-                id: document.id,
-                status: "failed",
-                message: "no text found",
-            };
-        }
-        if (!partDefinitions || partDefinitions.length === 0) {
-            activity_1.log.info("No parts found for document, skipping embeddings generation");
-            return {
-                id: document.id,
-                status: "failed",
-                message: "no parts found",
-            };
-        }
-        activity_1.log.info("Generating embeddings for parts", {
-            parts: partDefinitions,
-            max_tokens: maxTokens,
-        });
-        const docParts = (0, chunks_js_1.getContentParts)(document.text, partDefinitions);
-        activity_1.log.info(`Retrieved ${docParts.length} parts`);
-        const start = new Date().getTime();
-        const generatePartEmbeddings = async (partContent, i) => {
-            const localStart = new Date().getTime();
-            try {
-                activity_1.log.info(`Generating embeddings for part ${i}`, {
-                    text_len: partContent.length,
-                });
-                if (!partContent) {
-                    return {
-                        id: i,
-                        number: i,
-                        result: null,
-                        status: "skipped",
-                        message: "no text found",
-                    };
-                }
-                const e = await generateEmbeddingsFromStudio(partContent, environment, client, model).catch((e) => {
-                    activity_1.log.error("Error generating embeddings for part " + i, {
-                        text_length: partContent.length,
-                        error: e,
-                    });
-                    return null;
-                });
-                if (!e || !e.values) {
-                    return {
-                        id: i,
-                        number: i,
-                        result: null,
-                        message: "no embeddings generated",
-                    };
-                }
-                if (e.values.length === 0) {
-                    return {
-                        id: i,
-                        number: i,
-                        result: null,
-                        message: "no embeddings generated",
-                    };
-                }
-                activity_1.log.info(`Generated embeddings for part ${i}`, {
-                    len: e.values.length,
-                    duration: new Date().getTime() - localStart,
-                });
-                return { number: i, result: e };
-            }
-            catch (err) {
-                activity_1.log.info(`Error generating ${type} embeddings for part ${i} of ${document.id}`, { error: err });
-                return {
-                    number: i,
-                    result: null,
-                    message: "error generating embeddings",
-                    error: err.message,
-                };
-            }
+        tokenCount !== undefined &&
+        tokenCount > maxTokens) {
+        //TODO: Review strategy for large documents
+        activity_1.log.warn(`Document too large for ${type} embeddings generation, skipping (${tokenCount} tokens)`);
+        return {
+            id: document.id,
+            status: "skipped",
+            message: `${type} embeddings generation, skipped for large document (${tokenCount} tokens)`,
         };
-        const partEmbeddings = await Promise.all(docParts.map((part, i) => generatePartEmbeddings(part, i)));
-        const validPartEmbeddings = partEmbeddings
-            .filter((e) => e.result !== null)
-            .map((e) => e.result);
-        const averagedEmbedding = computeAttentionEmbedding(validPartEmbeddings.map((e) => e.values));
-        activity_1.log.info(`Averaged embeddings for document ${document.id} in ${(new Date().getTime() - start) / 1000} seconds`, {
-            len: averagedEmbedding.length,
-            count: validPartEmbeddings.length,
-            max_tokens: maxTokens,
-        });
-        await client.objects.setEmbedding(document.id, type, {
-            values: averagedEmbedding,
-            model: validPartEmbeddings[0].model,
-            etag: document.text_etag,
-        });
-        activity_1.log.info(`Object ${document.id} embedding set`, {
-            type,
-            len: averagedEmbedding.length,
-        });
     }
     else {
         activity_1.log.info(`Generating ${type} embeddings for document`);
@@ -328,31 +225,4 @@ async function generateEmbeddingsFromStudio(text, env, client, model) {
         throw e;
     });
 }
-//Simplified attention mechanism
-// This is a naive implementation and should be replaced with a more sophisticated
-// using tensorflow in a specific package
-function computeAttentionEmbedding(chunkEmbeddings) {
-    if (chunkEmbeddings.length === 0)
-        return [];
-    const start = new Date().getTime();
-    // Generate random attention weights
-    const attentionWeights = chunkEmbeddings.map(() => Math.random());
-    // Apply softmax to get attention scores
-    const expWeights = attentionWeights.map((w) => Math.exp(w));
-    const sumExpWeights = expWeights.reduce((sum, val) => sum + val, 0);
-    const attentionScores = expWeights.map((w) => w / sumExpWeights);
-    // Get embedding dimension
-    const embeddingDim = chunkEmbeddings[0].length;
-    // Initialize document embedding
-    const documentEmbedding = new Array(embeddingDim).fill(0);
-    // Weighted sum of embeddings
-    for (let i = 0; i < chunkEmbeddings.length; i++) {
-        for (let j = 0; j < embeddingDim; j++) {
-            documentEmbedding[j] += chunkEmbeddings[i][j] * attentionScores[i];
-        }
-    }
-    const duration = new Date().getTime() - start;
-    console.log(`Computed document embedding in ${duration}ms for ${chunkEmbeddings.length} chunks`);
-    return documentEmbedding;
-}
 //# sourceMappingURL=generateEmbeddings.js.map

package/lib/cjs/activities/generateEmbeddings.js.map

@@ -1 +1 @@
-{"version":3,"file":"generateEmbeddings.js","sourceRoot":"","sources":["../../../src/activities/generateEmbeddings.ts"],"names":[],"mappings":";;AAmDA,gDA8FC;AAhJD,mDAA2C;AAE3C,6CAO0B;AAC1B,wEAAgE;AAChE,4CAAqD;AACrD,gDAA2D;AAC3D,kDAA8D;AAC9D,kDAAiD;AAoC1C,KAAK,UAAU,kBAAkB,CACpC,OAA8D;IAE9D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAC5C,MAAM,IAAA,kCAAa,EAA2B,OAAO,CAAC,CAAC;IAC3D,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAE/B,MAAM,WAAW,GAAG,MAAM,YAAY,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,MAAM,IAAI,iCAAqB,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,iCAAqB,CAAC,oCAAoC,EAAE;YAClE,QAAQ;SACX,CAAC,CAAC;IACP,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,MAAM,IAAI,iCAAqB,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;QACxD,cAAG,CAAC,IAAI,CACJ,2CAA2C,IAAI,gBAAgB,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,SAAS,GAAG,EAC5G,EAAE,MAAM,EAAE,CACb,CAAC;QACF,OAAO;YACH,EAAE,EAAE,QAAQ;YACZ,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,2CAA2C,IAAI,EAAE;SAC7D,CAAC;IACN,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,GAAG,IAAI,6CAA6C,QAAQ,EAAE,EAAE;QACrE,KAAK;QACL,MAAM;KACT,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACX,iHAAiH,CACpH,CAAC;IACN,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAC1C,QAAQ,EACR,8CAA8C,CACjD,CAAC;IAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,iCAAqB,CAAC,oBAAoB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,iCAAqB,CAAC,4BAA4B,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,GAAG,CAAC;IAER,QAAQ,IAAI,EAAE,CAAC;QACX,KAAK,gCAAuB,CAAC,IAAI;YAC7B,GAAG,GAAG,MAAM,sBAAsB,CAAC;gBAC/B,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,IAAI;aACP,CAAC,CAAC;YACH,MAAM;QACV,KAAK,gCAAuB,CAAC,UAAU;YACnC,GAAG,GAAG,MAAM,sBAAsB,CAAC;gBAC/B,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,IAAI;aACP,CAAC,CAAC;YACH,MAAM;QACV,KAAK,gCAAuB,CAAC,KAAK;YAC9B,GAAG,GAAG,MAAM,uBAAuB,CAAC;gBAChC,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,IAAI;aACP,CAAC,CAAC;YACH,MAAM;QACV;YACI,GAAG,GAAG;gBACF,EAAE,EAAE,QAAQ;gBACZ,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,+BAA+B,IAAI,EAAE;aACjD,CAAC;IACV,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC;AAWD,KAAK,UAAU,sBAAsB,CACjC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAmC,EACnE,KAAiB;IAGjB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,+BAA+B,EAAE,CAAC;IACzE,CAAC;IAED,IACI,IAAI,KAAK,gCAAuB,CAAC,IAAI;QACrC,IAAI,KAAK,gCAAuB,CAAC,UAAU,EAC7C,CAAC;QACC,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,+BAA+B,IAAI,EAAE;SACjD,CAAC;IACN,CAAC;IAED,IAAI,IAAI,KAAK,gCAAuB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC1D,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3E,CAAC;IACD,IAAI,IAAI,KAAK,gCAAuB,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;QACvE,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,qBAAqB;SACjC,CAAC;IACN,CAAC;IAED,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAEtC,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE,CAAC;IAEpC,mCAAmC;IACnC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,KAAK,gCAAuB,CAAC,IAAI,EAAE,CAAC;QACnE,cAAG,CAAC,KAAK,CAAC,qCAAqC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,UAAU,GAAG,IAAA,uBAAW,EAAC,QAAQ,CAAC,IAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE;YACrC,MAAM,EAAE;gBACJ,GAAG,UAAU;gBACb,IAAI,EAAE,QAAQ,CAAC,SAAS,IAAI,IAAA,cAAG,EAAC,QAAQ,CAAC,IAAK,CAAC;aAClD;SACJ,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,GAAG;YACd,GAAG,UAAU;YACb,IAAI,EAAE,QAAQ,CAAC,SAAS,IAAI,IAAA,cAAG,EAAC,QAAQ,CAAC,IAAK,CAAC;SAClD,CAAC;IACN,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC;IAE5C,kEAAkE;IAClE,4DAA4D;IAC5D,gFAAgF;IAChF,cAAG,CAAC,IAAI,CAAC,cAAc,IAAI,4BAA4B,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IACtE,IACI,IAAI,KAAK,gCAAuB,CAAC,IAAI;QACrC,QAAQ,CAAC,MAAM,EAAE,KAAK;QACtB,QAAQ,CAAC,MAAM,EAAE,KAAK,GAAG,SAAS,EACpC,CAAC;QACC,cAAG,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QAEhE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO;gBACH,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,eAAe;aAC3B,CAAC;QACN,CAAC;QAED,IAAI,CAAC,eAAe,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnD,cAAG,CAAC,IAAI,CACJ,6DAA6D,CAChE,CAAC;YACF,OAAO;gBACH,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,gBAAgB;aAC5B,CAAC;QACN,CAAC;QAED,cAAG,CAAC,IAAI,CAAC,iCAAiC,EAAE;YACxC,KAAK,EAAE,eAAe;YACtB,UAAU,EAAE,SAAS;SACxB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAA,2BAAe,EAAC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;QAEjE,cAAG,CAAC,IAAI,CAAC,aAAa,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,sBAAsB,GAAG,KAAK,EAChC,WAAmB,EACnB,CAAS,EACX,EAAE;YACA,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,CAAC;gBACD,cAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,EAAE,EAAE;oBAC5C,QAAQ,EAAE,WAAW,CAAC,MAAM;iBAC/B,CAAC,CAAC;gBACH,IAAI,CAAC,WAAW,EAAE,CAAC;oBACf,OAAO;wBACH,EAAE,EAAE,CAAC;wBACL,MAAM,EAAE,CAAC;wBACT,MAAM,EAAE,IAAI;wBACZ,MAAM,EAAE,SAAS;wBACjB,OAAO,EAAE,eAAe;qBAC3B,CAAC;gBACN,CAAC;gBAED,MAAM,CAAC,GAAG,MAAM,4BAA4B,CACxC,WAAW,EACX,WAAW,EACX,MAAM,EACN,KAAK,CACR,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;oBACV,cAAG,CAAC,KAAK,CAAC,uCAAuC,GAAG,CAAC,EAAE;wBACnD,WAAW,EAAE,WAAW,CAAC,MAAM;wBAC/B,KAAK,EAAE,CAAC;qBACX,CAAC,CAAC;oBACH,OAAO,IAAI,CAAC;gBAChB,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;oBAClB,OAAO;wBACH,EAAE,EAAE,CAAC;wBACL,MAAM,EAAE,CAAC;wBACT,MAAM,EAAE,IAAI;wBACZ,OAAO,EAAE,yBAAyB;qBACrC,CAAC;gBACN,CAAC;gBAED,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACxB,OAAO;wBACH,EAAE,EAAE,CAAC;wBACL,MAAM,EAAE,CAAC;wBACT,MAAM,EAAE,IAAI;wBACZ,OAAO,EAAE,yBAAyB;qBACrC,CAAC;gBACN,CAAC;gBACD,cAAG,CAAC,IAAI,CAAC,iCAAiC,CAAC,EAAE,EAAE;oBAC3C,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM;oBACpB,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,UAAU;iBAC9C,CAAC,CAAC;gBAEH,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YACpC,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,cAAG,CAAC,IAAI,CACJ,oBAAoB,IAAI,wBAAwB,CAAC,OAAO,QAAQ,CAAC,EAAE,EAAE,EACrE,EAAE,KAAK,EAAE,GAAG,EAAE,CACjB,CAAC;gBACF,OAAO;oBACH,MAAM,EAAE,CAAC;oBACT,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE,6BAA6B;oBACtC,KAAK,EAAE,GAAG,CAAC,OAAO;iBACrB,CAAC;YACN,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAC7D,CAAC;QACF,MAAM,mBAAmB,GAAG,cAAc;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;aAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAC1B,MAAM,iBAAiB,GAAG,yBAAyB,CAC/C,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAC3C,CAAC;QACF,cAAG,CAAC,IAAI,CACJ,oCAAoC,QAAQ,CAAC,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,UAAU,EACrG;YACI,GAAG,EAAE,iBAAiB,CAAC,MAAM;YAC7B,KAAK,EAAE,mBAAmB,CAAC,MAAM;YACjC,UAAU,EAAE,SAAS;SACxB,CACJ,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE;YACjD,MAAM,EAAE,iBAAiB;YACzB,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC,KAAK;YACnC,IAAI,EAAE,QAAQ,CAAC,SAAS;SAC3B,CAAC,CAAC;QACH,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,CAAC,EAAE,gBAAgB,EAAE;YAC5C,IAAI;YACJ,GAAG,EAAE,iBAAiB,CAAC,MAAM;SAChC,CAAC,CAAC;IACP,CAAC;SAAM,CAAC;QACJ,cAAG,CAAC,IAAI,CAAC,cAAc,IAAI,0BAA0B,CAAC,CAAC;QAEvD,MAAM,GAAG,GAAG,MAAM,4BAA4B,CAC1C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAC9B,WAAW,EACX,MAAM,CACT,CAAC;QACF,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACtB,OAAO;gBACH,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,yBAAyB;aACrC,CAAC;QACN,CAAC;QAED,cAAG,CAAC,IAAI,CAAC,GAAG,IAAI,sCAAsC,QAAQ,CAAC,EAAE,EAAE,EAAE;YACjE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;SACzB,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE;YACjD,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,IAAI,EAAE,QAAQ,CAAC,SAAS;SAC3B,CAAC,CAAC;QAEH,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI;YACJ,MAAM,EAAE,WAAW;YACnB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;SACzB,CAAC;IACN,CAAC;AACL,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,EACnC,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,MAAM,GACwB;IAC9B,cAAG,CAAC,IAAI,CAAC,2CAA2C,GAAG,QAAQ,CAAC,EAAE,EAAE;QAChE,OAAO,EAAE,QAAQ,CAAC,OAAO;KAC5B,CAAC,CAAC;IACH,IACI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;QAC7C,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,EAC1C,CAAC;QACC,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI;YACJ,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,yBAAyB;SACrC,CAAC;IACN,CAAC;IACD,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAEtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,EAAE;QAChE,MAAM,EAAE,6BAAoB,CAAC,IAAI;QACjC,mBAAmB,EAAE,IAAI;QACzB,QAAQ,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IACjE,CAAC;SAAM,IACH,MAAM,CAAC,MAAM,KAAK,QAAQ;QAC1B,CAAC,MAAM,CAAC,UAAU;QAClB,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,EAC3B,CAAC;QACC,MAAM,IAAI,iCAAqB,CAAC,4BAA4B,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACrC,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACtB,MAAM,IAAI,iCAAqB,CAAC,8BAA8B,EAAE;YAC5D,QAAQ,CAAC,EAAE;SACd,CAAC,CAAC;IACP,CAAC;IAED,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAEzD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY;SAChC,UAAU,CAAC,WAAW,EAAE;QACrB,KAAK;QACL,KAAK;KACR,CAAC;SACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;SAClB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,cAAG,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;IAEP,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,yBAAyB;SACrC,CAAC;IACN,CAAC;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAC7B,QAAQ,CAAC,EAAE,EACX,gCAAuB,CAAC,KAAK,EAC7B;QACI,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,IAAI,EAAE,QAAQ,CAAC,SAAS;KAC3B,CACJ,CAAC;IAEF,OAAO;QACH,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,IAAI;QACJ,MAAM,EAAE,WAAW;QACnB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;KACzB,CAAC;AACN,CAAC;AAED,KAAK,UAAU,4BAA4B,CACvC,IAAY,EACZ,GAAW,EACX,MAAsB,EACtB,KAAc;IAEd,cAAG,CAAC,IAAI,CACJ,qCAAqC,IAAI,CAAC,MAAM,2BAA2B,GAAG,EAAE,CACnF,CAAC;IAEF,OAAO,MAAM,CAAC,YAAY;SACrB,UAAU,CAAC,GAAG,EAAE;QACb,IAAI;QACJ,KAAK;KACR,CAAC;SACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;SAClB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,cAAG,CAAC,KAAK,CAAC,sCAAsC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;AACX,CAAC;AAED,gCAAgC;AAChC,kFAAkF;AAClF,yCAAyC;AACzC,SAAS,yBAAyB,CAAC,eAA2B;IAC1D,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE5C,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;IAEnC,oCAAoC;IACpC,MAAM,gBAAgB,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAElE,wCAAwC;IACxC,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC;IAEjE,0BAA0B;IAC1B,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAE/C,gCAAgC;IAChC,MAAM,iBAAiB,GAAG,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE1D,6BAA6B;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,iBAAiB,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC;IAC9C,OAAO,CAAC,GAAG,CACP,kCAAkC,QAAQ,UAAU,eAAe,CAAC,MAAM,SAAS,CACtF,CAAC;IAEF,OAAO,iBAAiB,CAAC;AAC7B,CAAC"}
+{"version":3,"file":"generateEmbeddings.js","sourceRoot":"","sources":["../../../src/activities/generateEmbeddings.ts"],"names":[],"mappings":";;AAmDA,gDA8FC;AAhJD,mDAA2C;AAE3C,6CAO0B;AAC1B,wEAAgE;AAChE,4CAAqD;AACrD,gDAAsD;AAEtD,kDAAiD;AAoC1C,KAAK,UAAU,kBAAkB,CACpC,OAA8D;IAE9D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,GAC5C,MAAM,IAAA,kCAAa,EAA2B,OAAO,CAAC,CAAC;IAC3D,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAE/B,MAAM,WAAW,GAAG,MAAM,YAAY,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC3D,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,MAAM,IAAI,iCAAqB,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACV,MAAM,IAAI,iCAAqB,CAAC,oCAAoC,EAAE;YAClE,QAAQ;SACX,CAAC,CAAC;IACP,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,MAAM,IAAI,iCAAqB,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;QACxD,cAAG,CAAC,IAAI,CACJ,2CAA2C,IAAI,gBAAgB,WAAW,CAAC,IAAI,KAAK,WAAW,CAAC,SAAS,GAAG,EAC5G,EAAE,MAAM,EAAE,CACb,CAAC;QACF,OAAO;YACH,EAAE,EAAE,QAAQ;YACZ,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,2CAA2C,IAAI,EAAE;SAC7D,CAAC;IACN,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,GAAG,IAAI,6CAA6C,QAAQ,EAAE,EAAE;QACrE,KAAK;QACL,MAAM;KACT,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACX,iHAAiH,CACpH,CAAC;IACN,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAC1C,QAAQ,EACR,8CAA8C,CACjD,CAAC;IAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,iCAAqB,CAAC,oBAAoB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,iCAAqB,CAAC,4BAA4B,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,GAAG,CAAC;IAER,QAAQ,IAAI,EAAE,CAAC;QACX,KAAK,gCAAuB,CAAC,IAAI;YAC7B,GAAG,GAAG,MAAM,sBAAsB,CAAC;gBAC/B,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,IAAI;aACP,CAAC,CAAC;YACH,MAAM;QACV,KAAK,gCAAuB,CAAC,UAAU;YACnC,GAAG,GAAG,MAAM,sBAAsB,CAAC;gBAC/B,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,IAAI;aACP,CAAC,CAAC;YACH,MAAM;QACV,KAAK,gCAAuB,CAAC,KAAK;YAC9B,GAAG,GAAG,MAAM,uBAAuB,CAAC;gBAChC,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,IAAI;aACP,CAAC,CAAC;YACH,MAAM;QACV;YACI,GAAG,GAAG;gBACF,EAAE,EAAE,QAAQ;gBACZ,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,+BAA+B,IAAI,EAAE;aACjD,CAAC;IACV,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC;AAWD,KAAK,UAAU,sBAAsB,CACjC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAmC;IAGnE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,+BAA+B,EAAE,CAAC;IACzE,CAAC;IAED,IACI,IAAI,KAAK,gCAAuB,CAAC,IAAI;QACrC,IAAI,KAAK,gCAAuB,CAAC,UAAU,EAC7C,CAAC;QACC,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,+BAA+B,IAAI,EAAE;SACjD,CAAC;IACN,CAAC;IAED,IAAI,IAAI,KAAK,gCAAuB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC1D,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3E,CAAC;IACD,IAAI,IAAI,KAAK,gCAAuB,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;QACvE,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,qBAAqB;SACjC,CAAC;IACN,CAAC;IAED,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAE/B,8DAA8D;IAC9D,IAAI,UAAU,GAAwB,SAAS,CAAC;IAChD,IAAI,IAAI,KAAK,gCAAuB,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QACzD,UAAU,GAAG,IAAA,uBAAW,EAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;IAClD,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC;IAE5C,kEAAkE;IAClE,cAAG,CAAC,IAAI,CAAC,cAAc,IAAI,4BAA4B,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IACtE,IACI,IAAI,KAAK,gCAAuB,CAAC,IAAI;QACrC,UAAU,KAAK,SAAS;QACxB,UAAU,GAAG,SAAS,EACxB,CAAC;QACC,2CAA2C;QAC3C,cAAG,CAAC,IAAI,CACJ,0BAA0B,IAAI,qCAAqC,UAAU,UAAU,CAC1F,CAAC;QACF,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,GAAG,IAAI,uDAAuD,UAAU,UAAU;SAC9F,CAAA;IACL,CAAC;SAAM,CAAC;QACJ,cAAG,CAAC,IAAI,CAAC,cAAc,IAAI,0BAA0B,CAAC,CAAC;QAEvD,MAAM,GAAG,GAAG,MAAM,4BAA4B,CAC1C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAC9B,WAAW,EACX,MAAM,CACT,CAAC;QACF,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACtB,OAAO;gBACH,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,MAAM,EAAE,QAAQ;gBAChB,OAAO,EAAE,yBAAyB;aACrC,CAAC;QACN,CAAC;QAED,cAAG,CAAC,IAAI,CAAC,GAAG,IAAI,sCAAsC,QAAQ,CAAC,EAAE,EAAE,EAAE;YACjE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;SACzB,CAAC,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE;YACjD,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,IAAI,EAAE,QAAQ,CAAC,SAAS;SAC3B,CAAC,CAAC;QAEH,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI;YACJ,MAAM,EAAE,WAAW;YACnB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;SACzB,CAAC;IACN,CAAC;AACL,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,EACnC,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,MAAM,GACwB;IAC9B,cAAG,CAAC,IAAI,CAAC,2CAA2C,GAAG,QAAQ,CAAC,EAAE,EAAE;QAChE,OAAO,EAAE,QAAQ,CAAC,OAAO;KAC5B,CAAC,CAAC;IACH,IACI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC;QAC7C,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,EAC1C,CAAC;QACC,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI;YACJ,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,yBAAyB;SACrC,CAAC;IACN,CAAC;IACD,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAEtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,EAAE;QAChE,MAAM,EAAE,6BAAoB,CAAC,IAAI;QACjC,mBAAmB,EAAE,IAAI;QACzB,QAAQ,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IACjE,CAAC;SAAM,IACH,MAAM,CAAC,MAAM,KAAK,QAAQ;QAC1B,CAAC,MAAM,CAAC,UAAU;QAClB,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,EAC3B,CAAC;QACC,MAAM,IAAI,iCAAqB,CAAC,4BAA4B,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACrC,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QACtB,MAAM,IAAI,iCAAqB,CAAC,8BAA8B,EAAE;YAC5D,QAAQ,CAAC,EAAE;SACd,CAAC,CAAC;IACP,CAAC;IAED,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAEzD,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY;SAChC,UAAU,CAAC,WAAW,EAAE;QACrB,KAAK;QACL,KAAK;KACR,CAAC;SACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;SAClB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,cAAG,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;IAEP,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QACtB,OAAO;YACH,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ;YAChB,OAAO,EAAE,yBAAyB;SACrC,CAAC;IACN,CAAC;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAC7B,QAAQ,CAAC,EAAE,EACX,gCAAuB,CAAC,KAAK,EAC7B;QACI,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,IAAI,EAAE,QAAQ,CAAC,SAAS;KAC3B,CACJ,CAAC;IAEF,OAAO;QACH,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,IAAI;QACJ,MAAM,EAAE,WAAW;QACnB,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;KACzB,CAAC;AACN,CAAC;AAED,KAAK,UAAU,4BAA4B,CACvC,IAAY,EACZ,GAAW,EACX,MAAsB,EACtB,KAAc;IAEd,cAAG,CAAC,IAAI,CACJ,qCAAqC,IAAI,CAAC,MAAM,2BAA2B,GAAG,EAAE,CACnF,CAAC;IAEF,OAAO,MAAM,CAAC,YAAY;SACrB,UAAU,CAAC,GAAG,EAAE;QACb,IAAI;QACJ,KAAK;KACR,CAAC;SACD,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;SAClB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,cAAG,CAAC,KAAK,CAAC,sCAAsC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;AACX,CAAC"}

package/lib/cjs/activities/index-dsl.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.checkRateLimit = exports.setDocumentStatus = exports.notifyWebhook = exports.prepareVideo = exports.transcribeMedia = exports.convertPdfToStructuredText = exports.handleDslError = exports.getObjectFromStore = exports.generateOrAssignContentType = exports.generateVideoRendition = exports.generateImageRendition = exports.generateEmbeddings = exports.generateDocumentProperties = exports.extractDocumentText = exports.executeInteraction = exports.createPdfDocumentFromSource = exports.chunkDocument = exports.updateDocumentFromInteractionRun = exports.createOrUpdateDocumentFromInteractionRun = exports.createDocumentTypeFromInteractionRun = void 0;
+exports.setDocumentStatus = exports.generateVideoRendition = exports.generateImageRendition = exports.checkRateLimit = exports.notifyWebhook = exports.transcribeMedia = exports.saveGladiaTranscription = exports.convertPdfToStructuredText = exports.prepareVideo = exports.handleDslError = exports.getObjectFromStore = exports.generateOrAssignContentType = exports.generateEmbeddings = exports.generateDocumentProperties = exports.extractDocumentText = exports.executeInteraction = exports.createPdfDocumentFromSource = exports.chunkDocument = exports.updateDocumentFromInteractionRun = exports.createOrUpdateDocumentFromInteractionRun = exports.createDocumentTypeFromInteractionRun = void 0;
 // Export here DSL activities
 var createDocumentTypeFromInteractionRun_js_1 = require("./advanced/createDocumentTypeFromInteractionRun.js");
 Object.defineProperty(exports, "createDocumentTypeFromInteractionRun", { enumerable: true, get: function () { return createDocumentTypeFromInteractionRun_js_1.createDocumentTypeFromInteractionRun; } });
@@ -20,26 +20,28 @@ var generateDocumentProperties_js_1 = require("./generateDocumentProperties.js")
 Object.defineProperty(exports, "generateDocumentProperties", { enumerable: true, get: function () { return generateDocumentProperties_js_1.generateDocumentProperties; } });
 var generateEmbeddings_js_1 = require("./generateEmbeddings.js");
 Object.defineProperty(exports, "generateEmbeddings", { enumerable: true, get: function () { return generateEmbeddings_js_1.generateEmbeddings; } });
-var generateImageRendition_js_1 = require("./renditions/generateImageRendition.js");
-Object.defineProperty(exports, "generateImageRendition", { enumerable: true, get: function () { return generateImageRendition_js_1.generateImageRendition; } });
-var generateVideoRendition_js_1 = require("./renditions/generateVideoRendition.js");
-Object.defineProperty(exports, "generateVideoRendition", { enumerable: true, get: function () { return generateVideoRendition_js_1.generateVideoRendition; } });
 var generateOrAssignContentType_js_1 = require("./generateOrAssignContentType.js");
 Object.defineProperty(exports, "generateOrAssignContentType", { enumerable: true, get: function () { return generateOrAssignContentType_js_1.generateOrAssignContentType; } });
 var getObjectFromStore_js_1 = require("./getObjectFromStore.js");
 Object.defineProperty(exports, "getObjectFromStore", { enumerable: true, get: function () { return getObjectFromStore_js_1.getObjectFromStore; } });
 var handleError_js_1 = require("./handleError.js");
 Object.defineProperty(exports, "handleDslError", { enumerable: true, get: function () { return handleError_js_1.handleDslError; } });
+var prepareVideo_js_1 = require("./media/prepareVideo.js");
+Object.defineProperty(exports, "prepareVideo", { enumerable: true, get: function () { return prepareVideo_js_1.prepareVideo; } });
 var processPdfWithTextract_js_1 = require("./media/processPdfWithTextract.js");
 Object.defineProperty(exports, "convertPdfToStructuredText", { enumerable: true, get: function () { return processPdfWithTextract_js_1.convertPdfToStructuredText; } });
+var saveGladiaTranscription_js_1 = require("./media/saveGladiaTranscription.js");
+Object.defineProperty(exports, "saveGladiaTranscription", { enumerable: true, get: function () { return saveGladiaTranscription_js_1.saveGladiaTranscription; } });
 var transcribeMediaWithGladia_js_1 = require("./media/transcribeMediaWithGladia.js");
 Object.defineProperty(exports, "transcribeMedia", { enumerable: true, get: function () { return transcribeMediaWithGladia_js_1.transcribeMedia; } });
-var prepareVideo_js_1 = require("./media/prepareVideo.js");
-Object.defineProperty(exports, "prepareVideo", { enumerable: true, get: function () { return prepareVideo_js_1.prepareVideo; } });
 var notifyWebhook_js_1 = require("./notifyWebhook.js");
 Object.defineProperty(exports, "notifyWebhook", { enumerable: true, get: function () { return notifyWebhook_js_1.notifyWebhook; } });
-var setDocumentStatus_js_1 = require("./setDocumentStatus.js");
-Object.defineProperty(exports, "setDocumentStatus", { enumerable: true, get: function () { return setDocumentStatus_js_1.setDocumentStatus; } });
 var rateLimiter_js_1 = require("./rateLimiter.js");
 Object.defineProperty(exports, "checkRateLimit", { enumerable: true, get: function () { return rateLimiter_js_1.checkRateLimit; } });
+var generateImageRendition_js_1 = require("./renditions/generateImageRendition.js");
+Object.defineProperty(exports, "generateImageRendition", { enumerable: true, get: function () { return generateImageRendition_js_1.generateImageRendition; } });
+var generateVideoRendition_js_1 = require("./renditions/generateVideoRendition.js");
+Object.defineProperty(exports, "generateVideoRendition", { enumerable: true, get: function () { return generateVideoRendition_js_1.generateVideoRendition; } });
+var setDocumentStatus_js_1 = require("./setDocumentStatus.js");
+Object.defineProperty(exports, "setDocumentStatus", { enumerable: true, get: function () { return setDocumentStatus_js_1.setDocumentStatus; } });
 //# sourceMappingURL=index-dsl.js.map

package/lib/cjs/activities/index-dsl.js.map

@@ -1 +1 @@
-{"version":3,"file":"index-dsl.js","sourceRoot":"","sources":["../../../src/activities/index-dsl.ts"],"names":[],"mappings":";;;AAAA,6BAA6B;AAC7B,8GAA0G;AAAjG,+JAAA,oCAAoC,OAAA;AAC7C,sHAAkH;AAAzG,uKAAA,wCAAwC,OAAA;AACjD,sGAAkG;AAAzF,uJAAA,gCAAgC,OAAA;AACzC,uDAAmD;AAA1C,iHAAA,aAAa,OAAA;AACtB,2EAA2E;AAAlE,yIAAA,2BAA2B,OAAA;AACpC,iEAA6D;AAApD,2HAAA,kBAAkB,OAAA;AAC3B,mEAA+D;AAAtD,6HAAA,mBAAmB,OAAA;AAC5B,iFAA6E;AAApE,2IAAA,0BAA0B,OAAA;AACnC,iEAA6D;AAApD,2HAAA,kBAAkB,OAAA;AAC3B,oFAAgF;AAAvE,mIAAA,sBAAsB,OAAA;AAC/B,oFAAgF;AAAvE,mIAAA,sBAAsB,OAAA;AAC/B,mFAA+E;AAAtE,6IAAA,2BAA2B,OAAA;AACpC,iEAA6D;AAApD,2HAAA,kBAAkB,OAAA;AAC3B,mDAAkD;AAAzC,gHAAA,cAAc,OAAA;AACvB,+EAA+E;AAAtE,uIAAA,0BAA0B,OAAA;AACnC,qFAAuE;AAA9D,+HAAA,eAAe,OAAA;AACxB,2DAAuD;AAA9C,+GAAA,YAAY,OAAA;AACrB,uDAAmD;AAA1C,iHAAA,aAAa,OAAA;AACtB,+DAA2D;AAAlD,yHAAA,iBAAiB,OAAA;AAC1B,mDAAkD;AAAzC,gHAAA,cAAc,OAAA"}
+{"version":3,"file":"index-dsl.js","sourceRoot":"","sources":["../../../src/activities/index-dsl.ts"],"names":[],"mappings":";;;AAAA,6BAA6B;AAC7B,8GAA0G;AAAjG,+JAAA,oCAAoC,OAAA;AAC7C,sHAAkH;AAAzG,uKAAA,wCAAwC,OAAA;AACjD,sGAAkG;AAAzF,uJAAA,gCAAgC,OAAA;AACzC,uDAAmD;AAA1C,iHAAA,aAAa,OAAA;AACtB,2EAA2E;AAAlE,yIAAA,2BAA2B,OAAA;AACpC,iEAA6D;AAApD,2HAAA,kBAAkB,OAAA;AAC3B,mEAA+D;AAAtD,6HAAA,mBAAmB,OAAA;AAC5B,iFAA6E;AAApE,2IAAA,0BAA0B,OAAA;AACnC,iEAA6D;AAApD,2HAAA,kBAAkB,OAAA;AAC3B,mFAA+E;AAAtE,6IAAA,2BAA2B,OAAA;AACpC,iEAA6D;AAApD,2HAAA,kBAAkB,OAAA;AAC3B,mDAAkD;AAAzC,gHAAA,cAAc,OAAA;AACvB,2DAAuD;AAA9C,+GAAA,YAAY,OAAA;AACrB,+EAA+E;AAAtE,uIAAA,0BAA0B,OAAA;AACnC,iFAA6E;AAApE,qIAAA,uBAAuB,OAAA;AAChC,qFAAuE;AAA9D,+HAAA,eAAe,OAAA;AAExB,uDAAmD;AAA1C,iHAAA,aAAa,OAAA;AACtB,mDAAkD;AAAzC,gHAAA,cAAc,OAAA;AACvB,oFAAgF;AAAvE,mIAAA,sBAAsB,OAAA;AAC/B,oFAAgF;AAAvE,mIAAA,sBAAsB,OAAA;AAC/B,+DAA2D;AAAlD,yHAAA,iBAAiB,OAAA"}

package/lib/cjs/activities/media/saveGladiaTranscription.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.saveGladiaTranscription = saveGladiaTranscription;
+const activity_1 = require("@temporalio/activity");
+const api_fetch_client_1 = require("@vertesia/api-fetch-client");
+const common_1 = require("@vertesia/common");
+const ActivityContext_js_1 = require("../../dsl/setup/ActivityContext.js");
+const result_types_js_1 = require("../../result-types.js");
+const GLADIA_URL = "https://api.gladia.io/v2";
+/**
+ * Fetches transcription results from Gladia and saves them to the content object.
+ * This activity is called after transcribeMedia completes via webhook callback.
+ */
+async function saveGladiaTranscription(payload) {
+    const { params, client, objectId } = await (0, ActivityContext_js_1.setupActivity)(payload);
+    const gladiaConfig = await client.projects.integrations.retrieve(payload.project_id, common_1.SupportedIntegrations.gladia);
+    if (!gladiaConfig || !gladiaConfig.enabled) {
+        return {
+            hasText: false,
+            objectId,
+            status: result_types_js_1.TextExtractionStatus.error,
+            error: "Gladia integration not enabled",
+        };
+    }
+    const gladiaClient = new api_fetch_client_1.FetchClient(gladiaConfig.url ?? GLADIA_URL);
+    gladiaClient.withHeaders({ "x-gladia-key": gladiaConfig.api_key });
+    activity_1.log.info(`Fetching transcription result from Gladia`, { objectId, transcriptionId: params.gladiaTranscriptionId });
+    const transcriptionResult = await gladiaClient.get(`/transcription/${params.gladiaTranscriptionId}`);
+    if (transcriptionResult.status === 'error') {
+        activity_1.log.error(`Gladia transcription failed`, { objectId, error: transcriptionResult });
+        return {
+            hasText: false,
+            objectId,
+            status: result_types_js_1.TextExtractionStatus.error,
+            error: "Gladia transcription failed",
+        };
+    }
+    if (transcriptionResult.status !== 'done') {
+        activity_1.log.warn(`Gladia transcription not ready`, { objectId, status: transcriptionResult.status });
+        return {
+            hasText: false,
+            objectId,
+            status: result_types_js_1.TextExtractionStatus.error,
+            error: `Gladia transcription not ready: ${transcriptionResult.status}`,
+        };
+    }
+    const object = await client.objects.retrieve(objectId, "+text");
+    const segments = processUtterances(transcriptionResult.result.transcription.utterances);
+    const fullText = transcriptionResult.result.transcription.full_transcript;
+    await client.objects.update(objectId, {
+        text: fullText,
+        text_etag: object.content?.etag,
+        transcript: {
+            segments,
+            etag: object.content?.etag
+        },
+        metadata: {
+            ...object.metadata,
+            duration: transcriptionResult.result.metadata.audio_duration,
+            languages: transcriptionResult.result.transcription.languages
+        }
+    });
+    activity_1.log.info(`Saved transcription for object`, { objectId, textLength: fullText?.length, segmentCount: segments.length });
+    return {
+        hasText: (fullText?.length ?? 0) > 0,
+        objectId,
+        status: result_types_js_1.TextExtractionStatus.success,
+        message: `Transcription saved with ${segments.length} segments`
+    };
+}
+function processUtterances(utterances) {
+    return utterances.map(u => ({
+        start: u.start,
+        end: u.end,
+        text: u.text,
+        speaker: u.speaker,
+        confidence: u.confidence,
+        language: u.language
+    }));
+}
+//# sourceMappingURL=saveGladiaTranscription.js.map

package/lib/cjs/activities/media/saveGladiaTranscription.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"saveGladiaTranscription.js","sourceRoot":"","sources":["../../../../src/activities/media/saveGladiaTranscription.ts"],"names":[],"mappings":";;AAoBA,0DAmEC;AAvFD,mDAA2C;AAC3C,iEAAyD;AACzD,6CAA6K;AAC7K,2EAAmE;AACnE,2DAAmF;AAUnF,MAAM,UAAU,GAAG,0BAA0B,CAAC;AAE9C;;;GAGG;AACI,KAAK,UAAU,uBAAuB,CAAC,OAAmE;IAC7G,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAgC,OAAO,CAAC,CAAC;IAEjG,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,8BAAqB,CAAC,MAAM,CAAoC,CAAC;IACtJ,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QACzC,OAAO;YACH,OAAO,EAAE,KAAK;YACd,QAAQ;YACR,MAAM,EAAE,sCAAoB,CAAC,KAAK;YAClC,KAAK,EAAE,gCAAgC;SAC1C,CAAC;IACN,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,8BAAW,CAAC,YAAY,CAAC,GAAG,IAAI,UAAU,CAAC,CAAC;IACrE,YAAY,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;IAEnE,cAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,CAAC,qBAAqB,EAAE,CAAC,CAAC;IAEnH,MAAM,mBAAmB,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,qBAAqB,EAAE,CAA8B,CAAC;IAElI,IAAI,mBAAmB,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACzC,cAAG,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,mBAAmB,EAAE,CAAC,CAAC;QACnF,OAAO;YACH,OAAO,EAAE,KAAK;YACd,QAAQ;YACR,MAAM,EAAE,sCAAoB,CAAC,KAAK;YAClC,KAAK,EAAE,6BAA6B;SACvC,CAAC;IACN,CAAC;IAED,IAAI,mBAAmB,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QACxC,cAAG,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7F,OAAO;YACH,OAAO,EAAE,KAAK;YACd,QAAQ;YACR,MAAM,EAAE,sCAAoB,CAAC,KAAK;YAClC,KAAK,EAAE,mCAAmC,mBAAmB,CAAC,MAAM,EAAE;SACzE,CAAC;IACN,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEhE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,mBAAmB,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IACxF,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC;IAE1E,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;QAClC,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI;QAC/B,UAAU,EAAE;YACR,QAAQ;YACR,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI;SAC7B;QACD,QAAQ,EAAE;YACN,GAAG,MAAM,CAAC,QAAQ;YAClB,QAAQ,EAAE,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc;YAC5D,SAAS,EAAE,mBAAmB,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS;SAC/B;KACrC,CAAC,CAAC;IAEH,cAAG,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAEtH,OAAO;QACH,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;QACpC,QAAQ;QACR,MAAM,EAAE,sCAAoB,CAAC,OAAO;QACpC,OAAO,EAAE,4BAA4B,QAAQ,CAAC,MAAM,WAAW;KAClE,CAAC;AACN,CAAC;AAED,SAAS,iBAAiB,CAAC,UAA6B;IACpD,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACvB,CAAC,CAAC,CAAC;AACR,CAAC"}

package/lib/cjs/activities/media/transcribeMediaWithGladia.js

@@ -1,9 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.transcribeMedia = transcribeMedia;
-const common_1 = require("@vertesia/common");
 const activity_1 = require("@temporalio/activity");
 const api_fetch_client_1 = require("@vertesia/api-fetch-client");
+const common_1 = require("@vertesia/common");
 const ActivityContext_js_1 = require("../../dsl/setup/ActivityContext.js");
 const errors_js_1 = require("../../errors.js");
 const index_js_1 = require("../../index.js");
@@ -45,7 +45,7 @@ async function transcribeMedia(payload) {
     }
     activity_1.log.info(`Using media URL for transcription`, { objectId, mediaUrl: mediaSource });
     const taskToken = Buffer.from((0, activity_1.activityInfo)().taskToken).toString('base64url');
-    const callbackUrl = generateCallbackUrlForGladia(client.store.baseUrl, payload.auth_token, taskToken, objectId);
+    const callbackUrl = generateCallbackUrlForGladia(client.store.baseUrl, taskToken, objectId);
     activity_1.log.info(`Transcribing media ${mediaUrl} with Gladia`, { objectId, callbackUrl });
     try {
         const res = await gladiaClient.post("/transcription", {
@@ -61,7 +61,6 @@ async function transcribeMedia(payload) {
             }
         });
         activity_1.log.info(`Transcription request sent to Gladia`, { objectId, res });
-        throw new activity_1.CompleteAsyncError();
     }
     catch (error) {
         if (error instanceof api_fetch_client_1.RequestError && error.status === 422) {
@@ -72,13 +71,12 @@ async function transcribeMedia(payload) {
                 error: `Gladia transcription error: ${error.message}`,
             };
         }
-        else {
-            activity_1.log.error(`Error sending transcription request to Gladia for object ${objectId}`, { error });
-            throw error;
-        }
+        activity_1.log.error(`Error sending transcription request to Gladia for object ${objectId}`, { error });
+        throw error;
     }
+    throw new activity_1.CompleteAsyncError();
 }
-function generateCallbackUrlForGladia(baseUrl, authToken, taskToken, objectId) {
-    return `${baseUrl}/api/v1/webhooks/gladia/${objectId}?access_token=${authToken}&task_token=${taskToken}`;
+function generateCallbackUrlForGladia(baseUrl, taskToken, objectId) {
+    return `${baseUrl}/webhooks/gladia/${objectId}?task_token=${taskToken}`;
 }
 //# sourceMappingURL=transcribeMediaWithGladia.js.map

package/lib/cjs/activities/media/transcribeMediaWithGladia.js.map

@@ -1 +1 @@
-{"version":3,"file":"transcribeMediaWithGladia.js","sourceRoot":"","sources":["../../../../src/activities/media/transcribeMediaWithGladia.ts"],"names":[],"mappings":";;AAuBA,0CAiFC;AAxGD,6CAAgL;AAChL,mDAA6E;AAC7E,iEAAuE;AACvE,2EAAmE;AACnE,+CAAwD;AACxD,6CAA4E;AAgB5E,MAAM,UAAU,GAAG,0BAA0B,CAAC;AAEvC,KAAK,UAAU,eAAe,CAAC,OAA2D;IAE7F,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAwB,OAAO,CAAC,CAAC;IAEzF,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,8BAAqB,CAAC,MAAM,CAAoC,CAAC;IACtJ,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QACzC,OAAO;YACH,OAAO,EAAE,KAAK;YACd,QAAQ;YACR,MAAM,EAAE,+BAAoB,CAAC,KAAK;YAClC,KAAK,EAAE,gCAAgC;SAC1C,CAAA;IACL,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,IAAI,8BAAW,CAAC,YAAY,CAAC,GAAG,IAAI,UAAU,CAAC,CAAC;IACrE,YAAY,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;IAEnE,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,+BAAoB,CAAC,OAAO,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAA;IACnI,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,iCAAqB,CAAC,8BAA8B,QAAQ,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,qEAAqE;IACrE,IAAI,WAAW,GAAW,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;IAChD,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,KAAK,sBAAa,CAAC,KAAK,EAAE,CAAC;QAChD,MAAM,aAAa,GAAG,MAAM,CAAC,QAAyB,CAAC;QACvD,MAAM,cAAc,GAAG,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,6BAAoB,CAAC,CAAC;QAC5F,IAAI,cAAc,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;YAClC,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC;YAC5C,cAAG,CAAC,IAAI,CAAC,0CAA0C,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QACpF,CAAC;IACL,CAAC;IAED,wCAAwC;IACxC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAEzE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,iCAAqB,CAAC,gCAAgC,WAAW,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,IAAA,uBAAY,GAAE,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC9E,MAAM,WAAW,GAAG,4BAA4B,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEhH,cAAG,CAAC,IAAI,CAAC,sBAAsB,QAAQ,cAAc,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;IAElF,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,gBAAgB,EAAE;YAClD,OAAO,EAAE;gBACL,SAAS,EAAE,QAAQ;gBACnB,YAAY,EAAE,WAAW;gBACzB,oBAAoB,EAAE,IAAI;gBAC1B,qBAAqB,EAAE,IAAI;gBAC3B,SAAS,EAAE,IAAI;gBACf,gBAAgB,EAAE;oBACd,OAAO,EAAE,CAAC,KAAK,CAAC;iBACnB;aACJ;SACJ,CAAoC,CAAC;QAEtC,cAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,IAAI,6BAAkB,EAAE,CAAC;IAEnC,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QAClB,IAAI,KAAK,YAAY,+BAAY,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACxD,OAAO;gBACH,OAAO,EAAE,KAAK;gBACd,QAAQ;gBACR,MAAM,EAAE,+BAAoB,CAAC,KAAK;gBAClC,KAAK,EAAE,+BAA+B,KAAK,CAAC,OAAO,EAAE;aACxD,CAAA;QACL,CAAC;aAAM,CAAC;YACJ,cAAG,CAAC,KAAK,CAAC,4DAA4D,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7F,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;AACL,CAAC;AAED,SAAS,4BAA4B,CAAC,OAAe,EAAE,SAAiB,EAAE,SAAiB,EAAE,QAAgB;IACzG,OAAO,GAAG,OAAO,2BAA2B,QAAQ,iBAAiB,SAAS,eAAe,SAAS,EAAE,CAAC;AAC7G,CAAC"}
+{"version":3,"file":"transcribeMediaWithGladia.js","sourceRoot":"","sources":["../../../../src/activities/media/transcribeMediaWithGladia.ts"],"names":[],"mappings":";;AA4BA,0CA+EC;AA3GD,mDAA6E;AAC7E,iEAAuE;AACvE,6CAAgL;AAChL,2EAAmE;AACnE,+CAAwD;AACxD,6CAA4E;AAqB5E,MAAM,UAAU,GAAG,0BAA0B,CAAC;AAEvC,KAAK,UAAU,eAAe,CAAC,OAA2D;IAE7F,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAwB,OAAO,CAAC,CAAC;IAEzF,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,8BAAqB,CAAC,MAAM,CAAoC,CAAC;IACtJ,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QACzC,OAAO;YACH,OAAO,EAAE,KAAK;YACd,QAAQ;YACR,MAAM,EAAE,+BAAoB,CAAC,KAAK;YAClC,KAAK,EAAE,gCAAgC;SAC1C,CAAA;IACL,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,IAAI,8BAAW,CAAC,YAAY,CAAC,GAAG,IAAI,UAAU,CAAC,CAAC;IACrE,YAAY,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;IAEnE,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,+BAAoB,CAAC,OAAO,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAA;IACnI,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,iCAAqB,CAAC,8BAA8B,QAAQ,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,qEAAqE;IACrE,IAAI,WAAW,GAAW,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;IAChD,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,KAAK,sBAAa,CAAC,KAAK,EAAE,CAAC;QAChD,MAAM,aAAa,GAAG,MAAM,CAAC,QAAyB,CAAC;QACvD,MAAM,cAAc,GAAG,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,6BAAoB,CAAC,CAAC;QAC5F,IAAI,cAAc,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;YAClC,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC;YAC5C,cAAG,CAAC,IAAI,CAAC,0CAA0C,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;QACpF,CAAC;IACL,CAAC;IAED,wCAAwC;IACxC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAEzE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,iCAAqB,CAAC,gCAAgC,WAAW,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;IAEnF,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,IAAA,uBAAY,GAAE,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC9E,MAAM,WAAW,GAAG,4BAA4B,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAE5F,cAAG,CAAC,IAAI,CAAC,sBAAsB,QAAQ,cAAc,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;IAElF,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,gBAAgB,EAAE;YAClD,OAAO,EAAE;gBACL,SAAS,EAAE,QAAQ;gBACnB,YAAY,EAAE,WAAW;gBACzB,oBAAoB,EAAE,IAAI;gBAC1B,qBAAqB,EAAE,IAAI;gBAC3B,SAAS,EAAE,IAAI;gBACf,gBAAgB,EAAE;oBACd,OAAO,EAAE,CAAC,KAAK,CAAC;iBACnB;aACJ;SACJ,CAAoC,CAAC;QACtC,cAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;IACxE,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QAClB,IAAI,KAAK,YAAY,+BAAY,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACxD,OAAO;gBACH,OAAO,EAAE,KAAK;gBACd,QAAQ;gBACR,MAAM,EAAE,+BAAoB,CAAC,KAAK;gBAClC,KAAK,EAAE,+BAA+B,KAAK,CAAC,OAAO,EAAE;aACxD,CAAA;QACL,CAAC;QACD,cAAG,CAAC,KAAK,CAAC,4DAA4D,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7F,MAAM,KAAK,CAAC;IAChB,CAAC;IAED,MAAM,IAAI,6BAAkB,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,4BAA4B,CAAC,OAAe,EAAE,SAAiB,EAAE,QAAgB;IACtF,OAAO,GAAG,OAAO,oBAAoB,QAAQ,eAAe,SAAS,EAAE,CAAC;AAC5E,CAAC"}

package/lib/esm/activities/generateEmbeddings.js

@@ -2,8 +2,7 @@ import { log } from "@temporalio/activity";
 import { ImageRenditionFormat, SupportedEmbeddingTypes, } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { DocumentNotFoundError } from "../errors.js";
-import { fetchBlobAsBase64, md5 } from "../utils/blobs.js";
-import { getContentParts } from "../utils/chunks.js";
+import { fetchBlobAsBase64 } from "../utils/blobs.js";
 import { countTokens } from "../utils/tokens.js";
 export async function generateEmbeddings(payload) {
     const { params, client, objectId, fetchProject } = await setupActivity(payload);
@@ -78,7 +77,7 @@ export async function generateEmbeddings(payload) {
     }
     return res;
 }
-async function generateTextEmbeddings({ document, client, type, config }, parts) {
+async function generateTextEmbeddings({ document, client, type, config }) {
     if (!document) {
         return { status: "error", message: "document is null or undefined" };
     }
@@ -100,127 +99,25 @@ async function generateTextEmbeddings({ document, client, type, config }, parts)
             message: "no properties found",
         };
     }
-    const { environment, model } = config;
-    const partDefinitions = parts ?? [];
-    // Count tokens if not already done
-    if (!document.tokens?.count && type === SupportedEmbeddingTypes.text) {
-        log.debug("Updating token count for document: " + document.id);
-        const tokensData = countTokens(document.text);
-        await client.objects.update(document.id, {
-            tokens: {
-                ...tokensData,
-                etag: document.text_etag ?? md5(document.text),
-            },
-        });
-        document.tokens = {
-            ...tokensData,
-            etag: document.text_etag ?? md5(document.text),
-        };
+    const { environment } = config;
+    // Count tokens if needed, do not rely on existing token count
+    let tokenCount = undefined;
+    if (type === SupportedEmbeddingTypes.text && document.text) {
+        tokenCount = countTokens(document.text).count;
     }
     const maxTokens = config.max_tokens ?? 8000;
     //generate embeddings for the main doc if document isn't too large
-    //if too large, we'll just generate embeddings for the parts
-    //then we can generate embeddings for the main document by averaging the tensors
     log.info(`Generating ${type} embeddings for document ${document.id}`);
     if (type === SupportedEmbeddingTypes.text &&
-        document.tokens?.count &&
-        document.tokens?.count > maxTokens) {
-        log.info("Document too large, generating embeddings for parts");
-        if (!document.text) {
-            return {
-                id: document.id,
-                status: "failed",
-                message: "no text found",
-            };
-        }
-        if (!partDefinitions || partDefinitions.length === 0) {
-            log.info("No parts found for document, skipping embeddings generation");
-            return {
-                id: document.id,
-                status: "failed",
-                message: "no parts found",
-            };
-        }
-        log.info("Generating embeddings for parts", {
-            parts: partDefinitions,
-            max_tokens: maxTokens,
-        });
-        const docParts = getContentParts(document.text, partDefinitions);
-        log.info(`Retrieved ${docParts.length} parts`);
-        const start = new Date().getTime();
-        const generatePartEmbeddings = async (partContent, i) => {
-            const localStart = new Date().getTime();
-            try {
-                log.info(`Generating embeddings for part ${i}`, {
-                    text_len: partContent.length,
-                });
-                if (!partContent) {
-                    return {
-                        id: i,
-                        number: i,
-                        result: null,
-                        status: "skipped",
-                        message: "no text found",
-                    };
-                }
-                const e = await generateEmbeddingsFromStudio(partContent, environment, client, model).catch((e) => {
-                    log.error("Error generating embeddings for part " + i, {
-                        text_length: partContent.length,
-                        error: e,
-                    });
-                    return null;
-                });
-                if (!e || !e.values) {
-                    return {
-                        id: i,
-                        number: i,
-                        result: null,
-                        message: "no embeddings generated",
-                    };
-                }
-                if (e.values.length === 0) {
-                    return {
-                        id: i,
-                        number: i,
-                        result: null,
-                        message: "no embeddings generated",
-                    };
-                }
-                log.info(`Generated embeddings for part ${i}`, {
-                    len: e.values.length,
-                    duration: new Date().getTime() - localStart,
-                });
-                return { number: i, result: e };
-            }
-            catch (err) {
-                log.info(`Error generating ${type} embeddings for part ${i} of ${document.id}`, { error: err });
-                return {
-                    number: i,
-                    result: null,
-                    message: "error generating embeddings",
-                    error: err.message,
-                };
-            }
+        tokenCount !== undefined &&
+        tokenCount > maxTokens) {
+        //TODO: Review strategy for large documents
+        log.warn(`Document too large for ${type} embeddings generation, skipping (${tokenCount} tokens)`);
+        return {
+            id: document.id,
+            status: "skipped",
+            message: `${type} embeddings generation, skipped for large document (${tokenCount} tokens)`,
         };
-        const partEmbeddings = await Promise.all(docParts.map((part, i) => generatePartEmbeddings(part, i)));
-        const validPartEmbeddings = partEmbeddings
-            .filter((e) => e.result !== null)
-            .map((e) => e.result);
-        const averagedEmbedding = computeAttentionEmbedding(validPartEmbeddings.map((e) => e.values));
-        log.info(`Averaged embeddings for document ${document.id} in ${(new Date().getTime() - start) / 1000} seconds`, {
-            len: averagedEmbedding.length,
-            count: validPartEmbeddings.length,
-            max_tokens: maxTokens,
-        });
-        await client.objects.setEmbedding(document.id, type, {
-            values: averagedEmbedding,
-            model: validPartEmbeddings[0].model,
-            etag: document.text_etag,
-        });
-        log.info(`Object ${document.id} embedding set`, {
-            type,
-            len: averagedEmbedding.length,
-        });
     }
     else {
         log.info(`Generating ${type} embeddings for document`);
@@ -325,31 +222,4 @@ async function generateEmbeddingsFromStudio(text, env, client, model) {
         throw e;
     });
 }
-//Simplified attention mechanism
-// This is a naive implementation and should be replaced with a more sophisticated
-// using tensorflow in a specific package
-function computeAttentionEmbedding(chunkEmbeddings) {
-    if (chunkEmbeddings.length === 0)
-        return [];
-    const start = new Date().getTime();
-    // Generate random attention weights
-    const attentionWeights = chunkEmbeddings.map(() => Math.random());
-    // Apply softmax to get attention scores
-    const expWeights = attentionWeights.map((w) => Math.exp(w));
-    const sumExpWeights = expWeights.reduce((sum, val) => sum + val, 0);
-    const attentionScores = expWeights.map((w) => w / sumExpWeights);
-    // Get embedding dimension
-    const embeddingDim = chunkEmbeddings[0].length;
-    // Initialize document embedding
-    const documentEmbedding = new Array(embeddingDim).fill(0);
-    // Weighted sum of embeddings
-    for (let i = 0; i < chunkEmbeddings.length; i++) {
-        for (let j = 0; j < embeddingDim; j++) {
-            documentEmbedding[j] += chunkEmbeddings[i][j] * attentionScores[i];
-        }
-    }
-    const duration = new Date().getTime() - start;
-    console.log(`Computed document embedding in ${duration}ms for ${chunkEmbeddings.length} chunks`);
-    return documentEmbedding;
-}
 //# sourceMappingURL=generateEmbeddings.js.map