@vertesia/workflow 0.51.0 → 0.52.0

package/src/activities/generateEmbeddings.ts

@@ -1,19 +1,42 @@
-import { VertesiaClient } from "@vertesia/client";
-import { ContentObject, DSLActivityExecutionPayload, DSLActivitySpec, ProjectConfigurationEmbeddings, SupportedEmbeddingTypes } from "@vertesia/common";
 import { EmbeddingsResult } from "@llumiverse/core";
 import { log } from "@temporalio/activity";
-import * as tf from '@tensorflow/tfjs-node';
+import { VertesiaClient } from "@vertesia/client";
+import { ContentObject, DSLActivityExecutionPayload, DSLActivitySpec, ProjectConfigurationEmbeddings, SupportedEmbeddingTypes } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { NoDocumentFound } from '../errors.js';
 import { fetchBlobAsBase64, md5 } from "../utils/blobs.js";
+import { DocPart, getContentParts } from "../utils/chunks.js";
 import { countTokens } from "../utils/tokens.js";
 
 
 export interface GenerateEmbeddingsParams {
+
+    /**
+     * The model to use for embedding generation
+     * If not set, the default model for the project will be used
+     */
     model?: string;
+
+    /**
+     * The environment to use for embedding generation
+     * If not set, the default environment for the project will be used
+     */
     environment?: string;
+
+    /**
+     * If true, force embedding generation even if the document already has embeddings
+     */
     force?: boolean;
+
+    /**
+     * The embedding type to generate
+     */
     type: SupportedEmbeddingTypes;
+
+    /**
+     * The DocParts to use for long documents
+     */
+    parts?: DocPart[];
 }
 
 export interface GenerateEmbeddings extends DSLActivitySpec<GenerateEmbeddingsParams> {
@@ -103,7 +126,7 @@ interface ExecuteGenerateEmbeddingsParams {
     force?: boolean;
 }
 
-async function generateTextEmbeddings({ document, client, type, config }: ExecuteGenerateEmbeddingsParams) {
+async function generateTextEmbeddings({ document, client, type, config }: ExecuteGenerateEmbeddingsParams, parts?: DocPart[],) {
     // if (!force && document.embeddings[type]?.etag === (document.text_etag ?? md5(document.text))) {
     //     return { id: objectId, status: "skipped", message: "embeddings already generated" }
     // }
@@ -125,6 +148,8 @@ async function generateTextEmbeddings({ document, client, type, config }: Execut
 
     const { environment, model } = config;
 
+    const partDefinitions = parts ?? [];
+
     // Count tokens if not already done
     if (!document.tokens?.count && type === SupportedEmbeddingTypes.text) {
         log.debug('Updating token count for document: ' + document.id);
@@ -150,79 +175,64 @@ async function generateTextEmbeddings({ document, client, type, config }: Execut
     if (type === SupportedEmbeddingTypes.text && document.tokens?.count && document.tokens?.count > maxTokens) {
         log.info('Document too large, generating embeddings for parts');
 
-        if (!document.parts || document.parts.length === 0) {
-            return { id: document.id, status: "skipped", message: "no parts found" }
+
+        if (!document.text) {
+            return { id: document.id, status: "failed", message: "no text found" }
         }
 
-        const docParts = await Promise.all(document.parts?.map(async (partId) => client.objects.retrieve(partId, "+text +embeddings +properties +tokens")));
-        log.info(`Retrieved ${docParts.length} parts`)
+        if (!partDefinitions || partDefinitions.length === 0) {
+            log.info('No parts found for document, skipping embeddings generation');
+            return { id: document.id, status: "failed", message: "no parts found" }
+        }
 
-        const generatePartEmbeddings = async (part: ContentObject<any>, i: number) => {
-            try {
-                log.info(`Generating embeddings for part ${part.id}`, { text_len: part.text?.length })
-                if (!part.text) {
-                    return { id: part.id, number: i, result: null, status: "skipped", message: "no text found" }
-                }
 
-                if (part.tokens?.count && part.tokens.count > maxTokens) {
-                    log.info('Part too large, skipping embeddings generation for part', { part: part.id, tokens: part.tokens.count });
-                    return { id: part.id, number: i, result: null, message: "part too large" }
+        log.info('Generating embeddings for parts', { parts: partDefinitions, max_tokens: maxTokens });
+        const docParts = getContentParts(document.text, partDefinitions);
+
+
+        log.info(`Retrieved ${docParts.length} parts`)
+        const start = new Date().getTime();
+        const generatePartEmbeddings = async (partContent: string, i: number) => {
+            const localStart = new Date().getTime();
+            try {
+                log.info(`Generating embeddings for part ${i}`, { text_len: partContent.length })
+                if (!partContent) {
+                    return { id: i, number: i, result: null, status: "skipped", message: "no text found" }
                 }
 
-                const e = await generateEmbeddingsFromStudio(part.text, environment, client, model).catch(e => {
-                    log.error('Error generating embeddings for part', { part: part.id, tokens: part.tokens, text_length: part.text?.length, error: e });
+                const e = await generateEmbeddingsFromStudio(partContent, environment, client, model).catch(e => {
+                    log.error('Error generating embeddings for part ' + i, { text_length: partContent.length, error: e });
                     return null;
                 });
 
                 if (!e || !e.values) {
-                    return { id: part.id, number: i, result: null, message: "no embeddings generated" }
+                    return { id: i, number: i, result: null, message: "no embeddings generated" }
                 }
 
-                log.info(`Embeddings generated for part ${part.id}, updating object in the store.`)
-                await client.objects.setEmbedding(part.id, SupportedEmbeddingTypes.text,
-                    {
-                        values: e.values,
-                        model: e.model,
-                        etag: part.text_etag
-                    }).catch(err => {
-                        log.info(`Error updating embeddings on part ${part.id}`);
-                        return { id: part.id, number: i, result: null, message: "error setting embeddings on part", error: err.message }
-                    })
-
-                log.info('Generated embeddings for part: ' + part.id);
-                return { id: part.id, number: i, result: e }
+                if (e.values.length === 0) {
+                    return { id: i, number: i, result: null, message: "no embeddings generated" }
+                }
+                log.info(`Generated embeddings for part ${i}`, { len: e.values.length, duration: new Date().getTime() - localStart });
+
+                return { inumber: i, result: e }
             } catch (err: any) {
-                log.info(`Error generating ${type} embeddings for part ${part.id} of ${document.id}`, { error: err });
-                return { id: part.id, number: i, result: null, message: "error generating embeddings", error: err.message }
+                log.info(`Error generating ${type} embeddings for part ${i} of ${document.id}`, { error: err });
+                return { number: i, result: null, message: "error generating embeddings", error: err.message }
             }
         }
 
-        const promises = docParts.map((p, i) => generatePartEmbeddings(p, i))
-        const res = await Promise.all(promises);
-        // let i = 0;
-        // for (const p of docParts) {
-        //     log.info(`Processing part ${p.id}`)
-        //     const r = await generatePartEmbeddings(p, i++);
-        //     res.push(r)
-        // }
-
-
-        // Filter out parts without embeddings
-        const validEmbeddings = res.filter(item => item.result !== null) as { id: string, number: number, result: EmbeddingsResult }[];
-
-        // Compute the document-level embedding using TensorFlow for attention mechanism
-        log.info('Computing document-level embedding using TF');
-        const documentEmbedding = computeAttentionEmbedding(validEmbeddings.map(item => item.result.values));
-
-        // Save the document-level embedding
+        const partEmbeddings = await Promise.all(docParts.map((part, i) => generatePartEmbeddings(part, i)));
+        const validPartEmbeddings = partEmbeddings.filter(e => e.result !== null).map(e => e.result);
+        const averagedEmbedding = computeAttentionEmbedding(validPartEmbeddings.map(e => e.values));
+        log.info(`Averaged embeddings for document ${document.id} in ${(new Date().getTime() - start) / 1000} seconds`, { len: averagedEmbedding.length, count: validPartEmbeddings.length, max_tokens: maxTokens });
         await client.objects.setEmbedding(document.id, type,
             {
-                values: documentEmbedding,
-                model: "attention",
+                values: averagedEmbedding,
+                model: validPartEmbeddings[0].model,
                 etag: document.text_etag
             }
         );
-        return { id: document.id, status: "completed", parts: docParts.map(i => i.id), len: documentEmbedding.length, part_embeddings: res.map(r => { return { id: r.id, status: r.status, error: r.error, message: r.message } }) }
+        log.info(`Object ${document.id} embedding set`, { type, len: averagedEmbedding.length });
 
     } else {
         log.info(`Generating ${type} embeddings for document`);
@@ -311,35 +321,37 @@ async function generateEmbeddingsFromStudio(text: string, env: string, client: V
 
 }
 
-function computeAttentionEmbedding(embeddingsArray: number[][], axis: number = 0) {
-    if (embeddingsArray.length === 0) return [];
-    log.info('Computing attention embedding for', { embeddingsArrays: embeddingsArray.map(a => a.length) });
+//Simplified attention mechanism
+// This is a naive implementation and should be replaced with a more sophisticated 
+// using tensorflow in a specific package
+function computeAttentionEmbedding(chunkEmbeddings: number[][]): number[] {
+    if (chunkEmbeddings.length === 0) return [];
+
     const start = new Date().getTime();
 
-    // Convert embeddings array to TensorFlow tensor
-    const embeddingsTensor = tf.tensor(embeddingsArray);
+    // Generate random attention weights
+    const attentionWeights = chunkEmbeddings.map(() => Math.random());
 
-    // Initialize trainable attention weights
-    const attentionWeights = tf.variable(tf.randomNormal([embeddingsArray.length]), true);
+    // Apply softmax to get attention scores
+    const expWeights = attentionWeights.map(w => Math.exp(w));
+    const sumExpWeights = expWeights.reduce((sum, val) => sum + val, 0);
+    const attentionScores = expWeights.map(w => w / sumExpWeights);
 
-    // Compute attention scoresje sui
-    const attentionScores = tf.softmax(attentionWeights);
+    // Get embedding dimension
+    const embeddingDim = chunkEmbeddings[0].length;
 
-    // Compute weighted sum of embeddings
-    const weightedEmbeddings = tf.mul(embeddingsTensor.transpose(), attentionScores).transpose();
-    const documentEmbeddingTensor = tf.sum(weightedEmbeddings, axis);
+    // Initialize document embedding
+    const documentEmbedding = new Array(embeddingDim).fill(0);
 
-    // Convert the result back to a JavaScript array
-    const documentEmbedding = documentEmbeddingTensor.arraySync() as number[];
-    const duration = (new Date().getTime() - start);
-    log.info(`Computed attention embeddings in ${duration}ms - array size: ${documentEmbedding.length}`, { length: documentEmbedding.length });
+    // Weighted sum of embeddings
+    for (let i = 0; i < chunkEmbeddings.length; i++) {
+        for (let j = 0; j < embeddingDim; j++) {
+            documentEmbedding[j] += chunkEmbeddings[i][j] * attentionScores[i];
+        }
+    }
 
-    // Clean up tensors
-    embeddingsTensor.dispose();
-    attentionWeights.dispose();
-    attentionScores.dispose();
-    weightedEmbeddings.dispose();
-    documentEmbeddingTensor.dispose();
+    const duration = new Date().getTime() - start;
+    console.log(`Computed document embedding in ${duration}ms for ${chunkEmbeddings.length} chunks`);
 
     return documentEmbedding;
-}
+}

package/src/activities/generateImageRendition.ts

@@ -1,31 +1,27 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec, RenditionProperties } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { NodeStreamSource } from "@vertesia/client/node";
+import { DSLActivityExecutionPayload, DSLActivitySpec, RenditionProperties } from "@vertesia/common";
 import fs from 'fs';
-import sharp, { FormatEnum } from "sharp";
+import ffmpeg from 'fluent-ffmpeg';
+import path from 'path';
+import os from 'os';
 import { imageResizer } from "../conversion/image.js";
-import { pdfToImages } from "../conversion/mutool.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { NoDocumentFound, WorkflowParamNotFound } from "../errors.js";
-import { fetchBlobAsBuffer, saveBlobToTempFile } from "../utils/blobs.js";
-import { NodeStreamSource } from "../utils/memory.js";
+import { saveBlobToTempFile } from "../utils/blobs.js";
+
 interface GenerateImageRenditionParams {
     max_hw: number; //maximum size of the longuest side of the image
-    format: keyof FormatEnum; //format of the output image
-    multi_page?: boolean; //if true, generate a multi-page rendition
+    format: string; //format of the output image
 }
 
-
 export interface GenerateImageRendition extends DSLActivitySpec<GenerateImageRenditionParams> {
-
-    name: 'generateImageRendition';
-
+    name: "generateImageRendition";
 }
 
-
 export async function generateImageRendition(payload: DSLActivityExecutionPayload<GenerateImageRenditionParams>) {
     const { client, objectId, params } = await setupActivity<GenerateImageRenditionParams>(payload);
 
-    const supportedNonImageInputTypes = ['application/pdf']
     const inputObject = await client.objects.retrieve(objectId).catch((err) => {
         log.error(`Failed to retrieve document ${objectId}`, err);
         if (err.response?.status === 404) {
@@ -33,7 +29,7 @@ export async function generateImageRendition(payload: DSLActivityExecutionPayloa
         }
         throw err;
     });
-    const renditionType = await client.types.getTypeByName('Rendition');
+    const renditionType = await client.types.getTypeByName("Rendition");
 
     if (!params.format) {
         log.error(`Format not found`);
@@ -50,85 +46,136 @@ export async function generateImageRendition(payload: DSLActivityExecutionPayloa
         throw new NoDocumentFound(`Document ${objectId} has no source`, [objectId]);
     }
 
-    if (!inputObject.content.type || (!inputObject.content.type?.startsWith('image/') && !supportedNonImageInputTypes.includes(inputObject.content.type))) {
-        log.error(`Document ${objectId} is not an image`);
-        throw new NoDocumentFound(`Document ${objectId} is not an image or pdf: ${inputObject.content.type}`, [objectId]);
+    if (!inputObject.content.type || (!inputObject.content.type?.startsWith("image/") && !inputObject.content.type?.startsWith("video/"))) {
+        log.error(`Document ${objectId} is not an image or a video: ${inputObject.content.type}`);
+        throw new NoDocumentFound(`Document ${objectId} is not an image or a video: ${inputObject.content.type}`, [objectId]);
     }
 
     //array of rendition files to upload
     let renditionPages: string[] = [];
 
-    //if PDF, convert to pages
-    if (inputObject.content.type === 'application/pdf') {
-        const pdfBuffer = await fetchBlobAsBuffer(client, inputObject.content.source);
-        const pages = await pdfToImages(pdfBuffer);
-        if (!pages.length) {
-            log.error(`Failed to convert pdf to image`);
-            throw new Error(`Failed to convert pdf to image`);
+    if (inputObject.content.type.startsWith('image/')) {
+        const imageFile = await saveBlobToTempFile(client, inputObject.content.source);
+        log.info(`Image ${objectId} copied to ${imageFile}`);
+        renditionPages.push(imageFile);
+    } else if (inputObject.content.type.startsWith('video/')) {
+        const videoFile = await saveBlobToTempFile(client, inputObject.content.source);
+        const tempOutputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'video-rendition-'));
+        const thumbnailPath = path.join(tempOutputDir, 'thumbnail.png');
+
+        try {
+            // Extract a frame at 10% of the video duration
+            await new Promise<void>((resolve, reject) => {
+                ffmpeg.ffprobe(videoFile, (err, metadata) => {
+                    if (err) {
+                        log.error(`Failed to probe video metadata: ${err.message}`);
+                        return reject(err);
+                    }
+
+                    const duration = metadata.format.duration || 0;
+                    const timestamp = Math.max(0.1 * duration, 1);
+
+                    ffmpeg(videoFile)
+                        .screenshots({
+                            timestamps: [timestamp],
+                            filename: 'thumbnail.png',
+                            folder: tempOutputDir,
+                            size: `${params.max_hw}x?`
+                        })
+                        .on('end', () => {
+                            log.info(`Video frame extraction complete for ${objectId}`);
+                            resolve();
+                        })
+                        .on('error', (err) => {
+                            log.error(`Error extracting frame from video: ${err.message}`);
+                            reject(err);
+                        });
+                });
+            });
+
+            if (fs.existsSync(thumbnailPath)) {
+                renditionPages.push(thumbnailPath);
+            } else {
+                throw new Error(`Failed to generate thumbnail for video ${objectId}`);
+            }
+        } catch (error) {
+            log.error(`Error generating image rendition for video: ${error instanceof Error ? error.message : 'Unknown error'}`);
+            throw new Error(`Failed to generate image rendition for video: ${objectId}`);
         }
-        renditionPages = [...pages];
-    } else if (inputObject.content.type.startsWith('image/')) {
-        const tmpFile = await saveBlobToTempFile(client, inputObject.content.source);
-        const filestats = fs.statSync(tmpFile);
-        log.info(`Image ${objectId} copied to ${tmpFile}`, { filestats });
-        renditionPages.push(tmpFile);
     }
 
     //generate rendition name, pass an index for multi parts
     const getRenditionName = (index: number = 0) => {
         const name = `renditions/${objectId}/${params.max_hw}/${index}.${params.format}`;
         return name;
-    }
+    };
 
     if (!renditionPages || !renditionPages.length) {
         log.error(`Failed to generate rendition for ${objectId}`);
         throw new Error(`Failed to generate rendition for ${objectId}`);
     }
 
-    log.info(`Uploading rendition for ${objectId} with ${renditionPages.length} pages (max_hw: ${params.max_hw}, format: ${params.format})`, { renditionPages });
+    log.info(
+        `Uploading rendition for ${objectId} with ${renditionPages.length} pages (max_hw: ${params.max_hw}, format: ${params.format})`,
+        { renditionPages },
+    );
     const uploads = renditionPages.map(async (page, i) => {
         const pageId = getRenditionName(i);
-        const resized = sharp(page).pipe(imageResizer(params.max_hw, params.format));
-
-        const source = new NodeStreamSource(
-            resized,
-            pageId.replace('renditions/', '').replace('/', '_'),
-            'image/' + params.format,
-            pageId,
-        )
-
-        log.info(`Uploading rendition for ${objectId} page ${i} with max_hw: ${params.max_hw} and format: ${params.format}`);
-        return client.objects.upload(source).catch((err) => {
-            log.error(`Failed to upload rendition for ${objectId} page ${i}`, err);
+        let resizedImagePath = null;
+
+        try {
+            // Resize the image using ImageMagick
+            resizedImagePath = await imageResizer(page, params.max_hw, params.format);
+
+            // Create a read stream from the resized image file
+            const fileStream = fs.createReadStream(resizedImagePath);
+
+            const source = new NodeStreamSource(
+                fileStream,
+                pageId.split("/").pop() ?? "0." + params.format,
+                "image/" + params.format,
+                pageId,
+            );
+
+            log.info(
+                `Uploading rendition for ${objectId} page ${i} with max_hw: ${params.max_hw} and format: ${params.format}`,
+            );
+
+            const result = await client.objects.upload(source).catch((err) => {
+                log.error(`Failed to upload rendition for ${objectId} page ${i}`, { error: err });
+                return Promise.resolve(null);
+            });
+
+            return result;
+        } catch (error) {
+            log.error(`Failed to process rendition for ${objectId} page ${i}`, { error });
             return Promise.resolve(null);
-        });
+        }
     });
 
     const uploaded = await Promise.all(uploads);
     if (!uploaded || !uploaded.length || !uploaded[0]) {
         log.error(`Failed to upload rendition for ${objectId}`);
-        throw new Error(`Failed to upload rendition for ${objectId}`);
+        throw new Error(`Failed to upload rendition for ${objectId} - upload object is empty`);
     }
 
-
-    log.info(`Creating rendition for ${objectId} with max_hw: ${params.max_hw} and format: ${params.format}`, { uploaded });
+    log.info(`Creating rendition for ${objectId} with max_hw: ${params.max_hw} and format: ${params.format}`, {
+        uploaded,
+    });
     const rendition = await client.objects.create({
         name: inputObject.name + ` [Rendition ${params.max_hw}]`,
         type: renditionType.id,
         parent: inputObject.id,
         content: uploaded[0],
         properties: {
-            mime_type: 'image/' + params.format,
+            mime_type: "image/" + params.format,
             source_etag: inputObject.content.source,
             height: params.max_hw,
             width: params.max_hw,
-            multipart: uploaded.length > 1,
-            total_parts: uploaded.length
-        } satisfies RenditionProperties
+        } satisfies RenditionProperties,
     });
 
     log.info(`Rendition ${rendition.id} created for ${objectId}`, { rendition });
 
     return { id: rendition.id, format: params.format, status: "success" };
-
 }

package/src/activities/generateOrAssignContentType.ts

@@ -1,5 +1,5 @@
 import { log } from "@temporalio/activity";
-import { CreateContentObjectTypePayload, DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { ContentObjectTypeItem, CreateContentObjectTypePayload, DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { ActivityContext, setupActivity } from "../dsl/setup/ActivityContext.js";
 import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
 import { InteractionExecutionParams, executeInteractionFromActivity } from "./executeInteraction.js";
@@ -56,12 +56,7 @@ export async function generateOrAssignContentType(payload: DSLActivityExecutionP
     const types = await client.types.list();
 
     //make a list of all existing types, and add hints if any
-    const existing_types = types.map(t => t.name).filter(n => !["DocumentPart", "Rendition"].includes(n));
-    if (params.typesHint) {
-        const newHints = params.typesHint.filter((t: string) => !existing_types.includes(t));
-        existing_types.push(...newHints);
-    }
-
+    const existing_types = types.filter(t => !["DocumentPart", "Rendition"].includes(t.name));
     const content = object.text ? truncByMaxTokens(object.text, params.truncate || 4000) : undefined;
 
     const getImage = async () => {
@@ -82,7 +77,7 @@ export async function generateOrAssignContentType(payload: DSLActivityExecutionP
 
     const fileRef = await getImage();
 
-    log.info("Execute SelectDocumentType interaction on content with \nexisting types: " + existing_types.join(","));
+    log.info("Execute SelectDocumentType interaction on content with \nexisting types: " + existing_types.map(t => t.name).join(","));
 
     const res = await executeInteractionFromActivity(client, interactionName, params, {
         existing_types, content, image: fileRef
@@ -98,7 +93,6 @@ export async function generateOrAssignContentType(payload: DSLActivityExecutionP
     if (!selectedType) {
         log.warn("Document type not idenfified: starting type generation");
         const newType = await generateNewType(context, existing_types, content, fileRef);
-
         selectedType = { id: newType.id, name: newType.name };
     }
 
@@ -119,14 +113,14 @@ export async function generateOrAssignContentType(payload: DSLActivityExecutionP
     };
 }
 
-async function generateNewType(context: ActivityContext<GenerateOrAssignContentTypeParams>, existing_types: string[], content?: string, fileRef?: string) {
+async function generateNewType(context: ActivityContext<GenerateOrAssignContentTypeParams>, existing_types: ContentObjectTypeItem[], content?: string, fileRef?: string) {
     const { client, params } = context;
 
     const project = await context.fetchProject();
     const interactionName = params.interactionNames?.generateMetadataModel ?? INT_GENERATE_METADATA_MODEL;
 
     const genTypeRes = await executeInteractionFromActivity(client, interactionName, params, {
-        existing_types: existing_types,
+        existing_types: existing_types.map(t => t.name),
         content: content,
         human_context: project?.configuration?.human_context ?? undefined,
         image: fileRef ? fileRef : undefined

package/src/activities/notifyWebhook.ts

@@ -1,9 +1,9 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { WorkflowParamNotFound } from "../errors.js";
 
-interface NotifyWebhookParams {
+export interface NotifyWebhookParams {
     target_url: string; //URL to send the notification to
     method: 'GET' | 'POST'; //HTTP method to use
     payload: Record<string, any>; //payload to send (if POST then as JSON body, if GET then as query string)