@vertesia/workflow 0.51.0 → 0.54.0

package/src/activities/extractDocumentText.ts

@@ -1,33 +1,41 @@
 import { log } from "@temporalio/activity";
-import { ContentObject, CreateContentObjectPayload, DSLActivityExecutionPayload, DSLActivitySpec } from '@vertesia/common';
-import { mutoolPdfToText } from '../conversion/mutool.js';
-import { manyToMarkdown } from '../conversion/pandoc.js';
+import {
+    ContentObject,
+    CreateContentObjectPayload,
+    DSLActivityExecutionPayload,
+    DSLActivitySpec,
+} from "@vertesia/common";
+import { mutoolPdfToText } from "../conversion/mutool.js";
+import { markdownWithPandoc } from "../conversion/pandoc.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
-import { NoDocumentFound } from '../errors.js';
-import { TextExtractionResult, TextExtractionStatus } from '../result-types.js';
-import { fetchBlobAsBuffer, md5 } from '../utils/blobs.js';
-import { countTokens } from '../utils/tokens.js';
+import { NoDocumentFound } from "../errors.js";
+import { TextExtractionResult, TextExtractionStatus } from "../result-types.js";
+import { fetchBlobAsBuffer, md5 } from "../utils/blobs.js";
+import { countTokens } from "../utils/tokens.js";
+import { markdownWithMarkitdown } from "../conversion/markitdown.js";
 
 //@ts-ignore
 const JSON: DSLActivitySpec = {
-    name: 'extractDocumentText',
-}
+    name: "extractDocumentText",
+};
 
 // doesn't have any own param
-export interface ExtractDocumentTextParams { };
+export interface ExtractDocumentTextParams {}
 export interface ExtractDocumentText extends DSLActivitySpec<ExtractDocumentTextParams> {
-    name: 'extractDocumentText';
+    name: "extractDocumentText";
     projection?: never;
 }
 
-export async function extractDocumentText(payload: DSLActivityExecutionPayload<ExtractDocumentTextParams>): Promise<TextExtractionResult> {
+export async function extractDocumentText(
+    payload: DSLActivityExecutionPayload<ExtractDocumentTextParams>,
+): Promise<TextExtractionResult> {
     const { client, objectId } = await setupActivity(payload);
 
     const r = await client.objects.find({
         query: { _id: objectId },
         limit: 1,
-        select: "+text"
-    })
+        select: "+text",
+    });
     const doc = r[0] as ContentObject;
     if (!doc) {
         log.error(`Document ${objectId} not found`);
@@ -36,7 +44,6 @@ export async function extractDocumentText(payload: DSLActivityExecutionPayload<E
 
     log.info(`Extracting text for object ${doc.id}`);
 
-
     if (!doc.content?.type || !doc.content?.source) {
         if (doc.text) {
             return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text present and no source or type");
@@ -58,74 +65,80 @@ export async function extractDocumentText(payload: DSLActivityExecutionPayload<E
         return createResponse(doc, "", TextExtractionStatus.error, e.message);
     }
 
-
     let txt: string;
 
     switch (doc.content.type) {
-
-        case 'application/pdf':
-            //if pdf is more than 2MB, use mutool
+        case "application/pdf":
             txt = await mutoolPdfToText(fileBuffer);
             break;
 
-        case 'text/plain':
-            txt = fileBuffer.toString('utf8')
+        case "text/plain":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //docx
-        case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
-            txt = await manyToMarkdown(fileBuffer, 'docx');
+        case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+            txt = await markdownWithMarkitdown(fileBuffer, "docx");
+            break;
+
+        //pptx
+        case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
+            txt = await markdownWithMarkitdown(fileBuffer, "pptx");
             break;
 
         //html
-        case 'text/html':
-            txt = await manyToMarkdown(fileBuffer, 'html');
+        case "text/html":
+            txt = await markdownWithPandoc(fileBuffer, "html");
             break;
 
         //opendocument
-        case 'application/vnd.oasis.opendocument.text':
-            txt = await manyToMarkdown(fileBuffer, 'odt');
+        case "application/vnd.oasis.opendocument.text":
+            txt = await markdownWithPandoc(fileBuffer, "odt");
             break;
 
         //rtf
-        case 'application/rtf':
-            txt = await manyToMarkdown(fileBuffer, 'rtf');
+        case "application/rtf":
+            txt = await markdownWithPandoc(fileBuffer, "rtf");
             break;
 
         //markdown
-        case 'text/markdown':
-            txt = fileBuffer.toString('utf8');
+        case "text/markdown":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //csv
-        case 'text/csv':
-            txt = fileBuffer.toString('utf8');
+        case "text/csv":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //typescript
-        case 'application/typescript':
-            txt = fileBuffer.toString('utf8');
+        case "application/typescript":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //javascript
-        case 'application/javascript':
-            txt = fileBuffer.toString('utf8');
+        case "application/javascript":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //json
-        case 'application/json':
-            txt = fileBuffer.toString('utf8');
+        case "application/json":
+            txt = fileBuffer.toString("utf8");
             break;
 
         default:
             if (sniffIfText(fileBuffer)) {
-                txt = fileBuffer.toString('utf8'); //TODO: add charset detection
+                txt = fileBuffer.toString("utf8"); //TODO: add charset detection
                 break;
             }
-            return createResponse(doc, doc.text ?? '', TextExtractionStatus.skipped, `Unsupported mime type: ${doc.content.type}`);
+            return createResponse(
+                doc,
+                doc.text ?? "",
+                TextExtractionStatus.skipped,
+                `Unsupported mime type: ${doc.content.type}`,
+            );
     }
 
-
     const tokensData = countTokens(txt);
     const etag = doc.content.etag ?? md5(txt);
 
@@ -135,15 +148,20 @@ export async function extractDocumentText(payload: DSLActivityExecutionPayload<E
         tokens: {
             ...tokensData,
             etag: etag,
-        }
-    }
+        },
+    };
 
     await client.objects.update(doc.id, updateData);
 
     return createResponse(doc, txt, TextExtractionStatus.success);
 }
 
-function createResponse(doc: ContentObject, text: string, status: TextExtractionStatus, message?: string): TextExtractionResult {
+function createResponse(
+    doc: ContentObject,
+    text: string,
+    status: TextExtractionStatus,
+    message?: string,
+): TextExtractionResult {
     return {
         status,
         message,
@@ -151,18 +169,37 @@ function createResponse(doc: ContentObject, text: string, status: TextExtraction
         len: text.length,
         objectId: doc.id,
         hasText: !!text,
-    }
-
+    };
 }
 
-
-//if file is less than 100KB, check if it looks like text
 function sniffIfText(buf: Buffer) {
-    if (buf.length < 100 * 1024) {
-        const s = buf.toString('utf8');
-        if (s.length > 0) {
-            return true;
+    // If file is too large, don't even try
+    if (buf.length > 500 * 1024) {
+        return false;
+    }
+
+    // Count binary/control characters
+    let binaryCount = 0;
+    const sampleSize = Math.min(buf.length, 1000); // Check first 1000 bytes
+
+    for (let i = 0; i < sampleSize; i++) {
+        // Count control characters (except common whitespace)
+        const byte = buf[i];
+        if ((byte < 32 && ![9, 10, 13].includes(byte)) || byte === 0) {
+            binaryCount++;
         }
     }
-    return false;
+
+    // If more than 10% binary/control chars, probably not text
+    if (binaryCount / sampleSize > 0.1) {
+        return false;
+    }
+
+    // Additional check for valid UTF-8 encoding
+    try {
+        const s = buf.toString("utf8");
+        return s.length > 0 && !s.includes("\uFFFD"); // Replacement character
+    } catch (e) {
+        return false;
+    }
 }

package/src/activities/generateDocumentProperties.ts

@@ -1,10 +1,10 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { TruncateSpec } from "../utils/tokens.js";
 import { InteractionExecutionParams, executeInteractionFromActivity } from "./executeInteraction.js";
 
-const INT_EXTRACT_INFORMATION = "sys:ExtractInformation"
+const INT_EXTRACT_INFORMATION = "sys:ExtractInformation";
 export interface GenerateDocumentPropertiesParams extends InteractionExecutionParams {
     typesHint?: string[];
     /**
@@ -17,10 +17,12 @@ export interface GenerateDocumentPropertiesParams extends InteractionExecutionPa
     use_vision?: boolean;
 }
 export interface GenerateDocumentProperties extends DSLActivitySpec<GenerateDocumentPropertiesParams> {
-    name: 'generateDocumentProperties';
+    name: "generateDocumentProperties";
 }
 
-export async function generateDocumentProperties(payload: DSLActivityExecutionPayload<GenerateDocumentPropertiesParams>) {
+export async function generateDocumentProperties(
+    payload: DSLActivityExecutionPayload<GenerateDocumentPropertiesParams>,
+) {
     const context = await setupActivity<GenerateDocumentPropertiesParams>(payload);
     const { params, client, objectId } = context;
     const interactionName = params.interactionName ?? INT_EXTRACT_INFORMATION;
@@ -32,7 +34,7 @@ export async function generateDocumentProperties(payload: DSLActivityExecutionPa
 
     if (!doc?.text && !params.use_vision && !doc?.content?.type?.startsWith("image/")) {
         log.warn(`Object ${objectId} not found or text is empty`);
-        return { status: "failed", error: "no-text" }
+        return { status: "failed", error: "no-text" };
     }
 
     if (!type || !type.object_schema) {
@@ -50,16 +52,19 @@ export async function generateDocumentProperties(payload: DSLActivityExecutionPa
         }
 
         log.info(`Object ${objectId} is not an image or pdf`);
-        return undefined
-    }
+        return undefined;
+    };
 
     const promptData = {
         content: doc.text ?? undefined,
         image: getImageRef() ?? undefined,
         human_context: project?.configuration?.human_context ?? undefined,
-    }
+    };
 
-    log.info(` Extracting information from object ${objectId} with type ${type.name}`, payload.debug_mode ? { params, } : undefined);
+    log.info(
+        ` Extracting information from object ${objectId} with type ${type.name}`,
+        payload.debug_mode ? { params } : undefined,
+    );
 
     const infoRes = await executeInteractionFromActivity(
         client,
@@ -70,24 +75,40 @@ export async function generateDocumentProperties(payload: DSLActivityExecutionPa
             result_schema: type.object_schema,
         },
         promptData,
-        payload.debug_mode ?? false
+        payload.debug_mode ?? false,
     );
 
+    const getText = () => {
+        if (doc.text) {
+            return undefined;
+        }
+        let text = "";
+        if (infoRes.result.title) {
+            text += infoRes.result.title + "\n";
+        }
+        if (infoRes.result.description) {
+            text += infoRes.result.description;
+        }
+        if (text) {
+            return text;
+        } else {
+            return undefined;
+        }
+    };
+
     log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
     await client.objects.update(doc.id, {
         properties: {
             ...infoRes.result,
-            etag: doc.text_etag
+            etag: doc.text_etag,
         },
-        text: infoRes.result.description ?? undefined,
+        text: getText(),
         generation_run_info: {
             id: infoRes.id,
             date: new Date().toISOString(),
             model: infoRes.modelId,
-        }
+        },
     });
 
-
     return { status: "completed" };
-
-}
+}

package/src/activities/generateEmbeddings.ts

@@ -1,19 +1,42 @@
-import { VertesiaClient } from "@vertesia/client";
-import { ContentObject, DSLActivityExecutionPayload, DSLActivitySpec, ProjectConfigurationEmbeddings, SupportedEmbeddingTypes } from "@vertesia/common";
 import { EmbeddingsResult } from "@llumiverse/core";
 import { log } from "@temporalio/activity";
-import * as tf from '@tensorflow/tfjs-node';
+import { VertesiaClient } from "@vertesia/client";
+import { ContentObject, DSLActivityExecutionPayload, DSLActivitySpec, ProjectConfigurationEmbeddings, SupportedEmbeddingTypes } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { NoDocumentFound } from '../errors.js';
 import { fetchBlobAsBase64, md5 } from "../utils/blobs.js";
+import { DocPart, getContentParts } from "../utils/chunks.js";
 import { countTokens } from "../utils/tokens.js";
 
 
 export interface GenerateEmbeddingsParams {
+
+    /**
+     * The model to use for embedding generation
+     * If not set, the default model for the project will be used
+     */
     model?: string;
+
+    /**
+     * The environment to use for embedding generation
+     * If not set, the default environment for the project will be used
+     */
     environment?: string;
+
+    /**
+     * If true, force embedding generation even if the document already has embeddings
+     */
     force?: boolean;
+
+    /**
+     * The embedding type to generate
+     */
     type: SupportedEmbeddingTypes;
+
+    /**
+     * The DocParts to use for long documents
+     */
+    parts?: DocPart[];
 }
 
 export interface GenerateEmbeddings extends DSLActivitySpec<GenerateEmbeddingsParams> {
@@ -103,7 +126,7 @@ interface ExecuteGenerateEmbeddingsParams {
     force?: boolean;
 }
 
-async function generateTextEmbeddings({ document, client, type, config }: ExecuteGenerateEmbeddingsParams) {
+async function generateTextEmbeddings({ document, client, type, config }: ExecuteGenerateEmbeddingsParams, parts?: DocPart[],) {
     // if (!force && document.embeddings[type]?.etag === (document.text_etag ?? md5(document.text))) {
     //     return { id: objectId, status: "skipped", message: "embeddings already generated" }
     // }
@@ -125,6 +148,8 @@ async function generateTextEmbeddings({ document, client, type, config }: Execut
 
     const { environment, model } = config;
 
+    const partDefinitions = parts ?? [];
+
     // Count tokens if not already done
     if (!document.tokens?.count && type === SupportedEmbeddingTypes.text) {
         log.debug('Updating token count for document: ' + document.id);
@@ -150,79 +175,64 @@ async function generateTextEmbeddings({ document, client, type, config }: Execut
     if (type === SupportedEmbeddingTypes.text && document.tokens?.count && document.tokens?.count > maxTokens) {
         log.info('Document too large, generating embeddings for parts');
 
-        if (!document.parts || document.parts.length === 0) {
-            return { id: document.id, status: "skipped", message: "no parts found" }
+
+        if (!document.text) {
+            return { id: document.id, status: "failed", message: "no text found" }
         }
 
-        const docParts = await Promise.all(document.parts?.map(async (partId) => client.objects.retrieve(partId, "+text +embeddings +properties +tokens")));
-        log.info(`Retrieved ${docParts.length} parts`)
+        if (!partDefinitions || partDefinitions.length === 0) {
+            log.info('No parts found for document, skipping embeddings generation');
+            return { id: document.id, status: "failed", message: "no parts found" }
+        }
 
-        const generatePartEmbeddings = async (part: ContentObject<any>, i: number) => {
-            try {
-                log.info(`Generating embeddings for part ${part.id}`, { text_len: part.text?.length })
-                if (!part.text) {
-                    return { id: part.id, number: i, result: null, status: "skipped", message: "no text found" }
-                }
 
-                if (part.tokens?.count && part.tokens.count > maxTokens) {
-                    log.info('Part too large, skipping embeddings generation for part', { part: part.id, tokens: part.tokens.count });
-                    return { id: part.id, number: i, result: null, message: "part too large" }
+        log.info('Generating embeddings for parts', { parts: partDefinitions, max_tokens: maxTokens });
+        const docParts = getContentParts(document.text, partDefinitions);
+
+
+        log.info(`Retrieved ${docParts.length} parts`)
+        const start = new Date().getTime();
+        const generatePartEmbeddings = async (partContent: string, i: number) => {
+            const localStart = new Date().getTime();
+            try {
+                log.info(`Generating embeddings for part ${i}`, { text_len: partContent.length })
+                if (!partContent) {
+                    return { id: i, number: i, result: null, status: "skipped", message: "no text found" }
                 }
 
-                const e = await generateEmbeddingsFromStudio(part.text, environment, client, model).catch(e => {
-                    log.error('Error generating embeddings for part', { part: part.id, tokens: part.tokens, text_length: part.text?.length, error: e });
+                const e = await generateEmbeddingsFromStudio(partContent, environment, client, model).catch(e => {
+                    log.error('Error generating embeddings for part ' + i, { text_length: partContent.length, error: e });
                     return null;
                 });
 
                 if (!e || !e.values) {
-                    return { id: part.id, number: i, result: null, message: "no embeddings generated" }
+                    return { id: i, number: i, result: null, message: "no embeddings generated" }
                 }
 
-                log.info(`Embeddings generated for part ${part.id}, updating object in the store.`)
-                await client.objects.setEmbedding(part.id, SupportedEmbeddingTypes.text,
-                    {
-                        values: e.values,
-                        model: e.model,
-                        etag: part.text_etag
-                    }).catch(err => {
-                        log.info(`Error updating embeddings on part ${part.id}`);
-                        return { id: part.id, number: i, result: null, message: "error setting embeddings on part", error: err.message }
-                    })
-
-                log.info('Generated embeddings for part: ' + part.id);
-                return { id: part.id, number: i, result: e }
+                if (e.values.length === 0) {
+                    return { id: i, number: i, result: null, message: "no embeddings generated" }
+                }
+                log.info(`Generated embeddings for part ${i}`, { len: e.values.length, duration: new Date().getTime() - localStart });
+
+                return { number: i, result: e }
             } catch (err: any) {
-                log.info(`Error generating ${type} embeddings for part ${part.id} of ${document.id}`, { error: err });
-                return { id: part.id, number: i, result: null, message: "error generating embeddings", error: err.message }
+                log.info(`Error generating ${type} embeddings for part ${i} of ${document.id}`, { error: err });
+                return { number: i, result: null, message: "error generating embeddings", error: err.message }
             }
         }
 
-        const promises = docParts.map((p, i) => generatePartEmbeddings(p, i))
-        const res = await Promise.all(promises);
-        // let i = 0;
-        // for (const p of docParts) {
-        //     log.info(`Processing part ${p.id}`)
-        //     const r = await generatePartEmbeddings(p, i++);
-        //     res.push(r)
-        // }
-
-
-        // Filter out parts without embeddings
-        const validEmbeddings = res.filter(item => item.result !== null) as { id: string, number: number, result: EmbeddingsResult }[];
-
-        // Compute the document-level embedding using TensorFlow for attention mechanism
-        log.info('Computing document-level embedding using TF');
-        const documentEmbedding = computeAttentionEmbedding(validEmbeddings.map(item => item.result.values));
-
-        // Save the document-level embedding
+        const partEmbeddings = await Promise.all(docParts.map((part, i) => generatePartEmbeddings(part, i)));
+        const validPartEmbeddings = partEmbeddings.filter(e => e.result !== null).map(e => e.result);
+        const averagedEmbedding = computeAttentionEmbedding(validPartEmbeddings.map(e => e.values));
+        log.info(`Averaged embeddings for document ${document.id} in ${(new Date().getTime() - start) / 1000} seconds`, { len: averagedEmbedding.length, count: validPartEmbeddings.length, max_tokens: maxTokens });
         await client.objects.setEmbedding(document.id, type,
             {
-                values: documentEmbedding,
-                model: "attention",
+                values: averagedEmbedding,
+                model: validPartEmbeddings[0].model,
                 etag: document.text_etag
             }
         );
-        return { id: document.id, status: "completed", parts: docParts.map(i => i.id), len: documentEmbedding.length, part_embeddings: res.map(r => { return { id: r.id, status: r.status, error: r.error, message: r.message } }) }
+        log.info(`Object ${document.id} embedding set`, { type, len: averagedEmbedding.length });
 
     } else {
         log.info(`Generating ${type} embeddings for document`);
@@ -311,35 +321,37 @@ async function generateEmbeddingsFromStudio(text: string, env: string, client: V
 
 }
 
-function computeAttentionEmbedding(embeddingsArray: number[][], axis: number = 0) {
-    if (embeddingsArray.length === 0) return [];
-    log.info('Computing attention embedding for', { embeddingsArrays: embeddingsArray.map(a => a.length) });
+//Simplified attention mechanism
+// This is a naive implementation and should be replaced with a more sophisticated 
+// using tensorflow in a specific package
+function computeAttentionEmbedding(chunkEmbeddings: number[][]): number[] {
+    if (chunkEmbeddings.length === 0) return [];
+
     const start = new Date().getTime();
 
-    // Convert embeddings array to TensorFlow tensor
-    const embeddingsTensor = tf.tensor(embeddingsArray);
+    // Generate random attention weights
+    const attentionWeights = chunkEmbeddings.map(() => Math.random());
 
-    // Initialize trainable attention weights
-    const attentionWeights = tf.variable(tf.randomNormal([embeddingsArray.length]), true);
+    // Apply softmax to get attention scores
+    const expWeights = attentionWeights.map(w => Math.exp(w));
+    const sumExpWeights = expWeights.reduce((sum, val) => sum + val, 0);
+    const attentionScores = expWeights.map(w => w / sumExpWeights);
 
-    // Compute attention scoresje sui
-    const attentionScores = tf.softmax(attentionWeights);
+    // Get embedding dimension
+    const embeddingDim = chunkEmbeddings[0].length;
 
-    // Compute weighted sum of embeddings
-    const weightedEmbeddings = tf.mul(embeddingsTensor.transpose(), attentionScores).transpose();
-    const documentEmbeddingTensor = tf.sum(weightedEmbeddings, axis);
+    // Initialize document embedding
+    const documentEmbedding = new Array(embeddingDim).fill(0);
 
-    // Convert the result back to a JavaScript array
-    const documentEmbedding = documentEmbeddingTensor.arraySync() as number[];
-    const duration = (new Date().getTime() - start);
-    log.info(`Computed attention embeddings in ${duration}ms - array size: ${documentEmbedding.length}`, { length: documentEmbedding.length });
+    // Weighted sum of embeddings
+    for (let i = 0; i < chunkEmbeddings.length; i++) {
+        for (let j = 0; j < embeddingDim; j++) {
+            documentEmbedding[j] += chunkEmbeddings[i][j] * attentionScores[i];
+        }
+    }
 
-    // Clean up tensors
-    embeddingsTensor.dispose();
-    attentionWeights.dispose();
-    attentionScores.dispose();
-    weightedEmbeddings.dispose();
-    documentEmbeddingTensor.dispose();
+    const duration = new Date().getTime() - start;
+    console.log(`Computed document embedding in ${duration}ms for ${chunkEmbeddings.length} chunks`);
 
     return documentEmbedding;
-}
+}