@vertesia/workflow 0.52.0 → 0.55.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vertesia/workflow",
-  "version": "0.52.0",
+  "version": "0.55.0",
   "type": "module",
   "description": "Composable prompts workflow dsl",
   "main": "./lib/esm/index.js",
@@ -19,6 +19,7 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@smithy/types": "^3.7.2",
+    "@temporalio/proto": "^1.11.5",
     "@temporalio/testing": "^1.11.5",
     "@temporalio/worker": "^1.11.5",
     "@types/fluent-ffmpeg": "^2.1.27",
@@ -50,9 +51,9 @@
     "tmp": "^0.2.3",
     "tmp-promise": "^3.0.3",
     "yaml": "^2.6.0",
-    "@vertesia/common": "0.52.0",
-    "@vertesia/client": "0.53.0",
-    "@llumiverse/core": "0.17.0"
+    "@llumiverse/core": "0.17.0",
+    "@vertesia/common": "0.55.0",
+    "@vertesia/client": "0.55.0"
   },
   "ts_dual_module": {
     "outDir": "lib",

package/src/activities/createDocumentFromOther.ts

@@ -24,7 +24,7 @@ export interface CreatePdfDocumentFromSource extends DSLActivitySpec<CreatePdfDo
 
 
 /**
- * Create a new PDF by extrracting pages from a source PDF
+ * Create a new PDF by extracting pages from a source PDF
  * @returns
  */
 export async function createPdfDocumentFromSource(payload: DSLActivityExecutionPayload<CreatePdfDocumentFromSourceParams>) {

package/src/activities/executeInteraction.ts

@@ -1,16 +1,23 @@
 import { ModelOptions } from "@llumiverse/core";
 import { activityInfo, log } from "@temporalio/activity";
 import { VertesiaClient } from "@vertesia/client";
-import { DSLActivityExecutionPayload, DSLActivitySpec, ExecutionRun, ExecutionRunStatus, InteractionExecutionConfiguration, RunSearchPayload } from "@vertesia/common";
+import {
+    DSLActivityExecutionPayload,
+    DSLActivitySpec,
+    ExecutionRun,
+    ExecutionRunStatus,
+    InteractionExecutionConfiguration,
+    RunSearchPayload,
+} from "@vertesia/common";
 import { projectResult } from "../dsl/projections.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
-import { ActivityParamNotFound } from "../errors.js";
+import { ActivityParamInvalid, ActivityParamNotFound } from "../errors.js";
 import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
 
 //Example:
 //@ts-ignore
 const JSON: DSLActivitySpec = {
-    name: 'executeInteraction',
+    name: "executeInteraction",
     import: ["defaultModel", "guidlineId", "docTypeId"],
     params: {
         defaultModel: "${model}",
@@ -23,8 +30,8 @@ const JSON: DSLActivitySpec = {
         result_schema: "${docType.object_schema}",
         prompt_data: {
             documents: "${documents}",
-            guidline: "${guidline.text}"
-        }
+            guidline: "${guidline.text}",
+        },
     },
     fetch: {
         documents: {
@@ -41,7 +48,7 @@ const JSON: DSLActivitySpec = {
                 id: "${guidlineId}",
             },
             select: "+text",
-            on_not_found: "throw"
+            on_not_found: "throw",
         },
         docType: {
             type: "document_type",
@@ -50,19 +57,19 @@ const JSON: DSLActivitySpec = {
                 id: "${docTypeId}",
             },
             select: "+object_schema",
-        }
-    }
-}
+        },
+    },
+};
 export interface InteractionExecutionParams {
     /**
      * The environment to use. If not specified the project default environment will be used.
-     * If the latter is not specified an exeption will be thrown.
+     * If the latter is not specified an exception will be thrown.
      */
     environment?: string;
     /**
      * The model to use. If not specified the project default model will be used.
      * If the latter is not specified the default model of the environment will be used.
-     * If the latter is not specified an exeption will be thrown.
+     * If the latter is not specified an exception will be thrown.
      */
     model?: string;
 
@@ -90,7 +97,7 @@ export interface InteractionExecutionParams {
 /**
  * TODO: must be kept in sync with InteractionAsyncExecutionPayload form @vertesia/common
  * Also see the executeInteractionAsync endpoint on the server for how the client payload is sent to the workflow.
- * (interaction is translsted to interactionName)
+ * (interaction is translated to interactionName)
  */
 export interface ExecuteInteractionParams extends InteractionExecutionParams {
     //TODO rename to interaction as in InteractionAsyncExecutionPayload
@@ -106,13 +113,11 @@ export interface ExecuteInteractionParams extends InteractionExecutionParams {
 }
 
 export interface ExecuteInteraction extends DSLActivitySpec<ExecuteInteractionParams> {
-    name: 'executeInteraction';
+    name: "executeInteraction";
 }
 
 export async function executeInteraction(payload: DSLActivityExecutionPayload<ExecuteInteractionParams>) {
-    const {
-        client, params
-    } = await setupActivity<ExecuteInteractionParams>(payload);
+    const { client, params } = await setupActivity<ExecuteInteractionParams>(payload);
 
     const { interactionName, prompt_data, static_prompt_data: wf_prompt_data } = params;
     if (wf_prompt_data) {
@@ -131,17 +136,37 @@ export async function executeInteraction(payload: DSLActivityExecutionPayload<Ex
         }
     }
 
-    const res = await executeInteractionFromActivity(client, interactionName, params, prompt_data, payload.debug_mode);
-
-    return projectResult(payload, params, res, {
-        runId: res.id,
-        status: res.status,
-        result: res.result,
-    });
-
+    try {
+        const res = await executeInteractionFromActivity(
+            client,
+            interactionName,
+            params,
+            prompt_data,
+            payload.debug_mode,
+        );
+        return projectResult(payload, params, res, {
+            runId: res.id,
+            status: res.status,
+            result: res.result,
+        });
+    } catch (error: any) {
+        log.error("Failed to execute interaction", { error });
+        if (error.message.includes("Failed to validate merged prompt schema")) {
+            //issue with the input data, don't retry
+            throw new ActivityParamInvalid("prompt_data", payload.activity, error.message);
+        } else {
+            throw error;
+        }
+    }
 }
 
-export async function executeInteractionFromActivity(client: VertesiaClient, interactionName: string, params: InteractionExecutionParams, prompt_data: any, debug?: boolean) {
+export async function executeInteractionFromActivity(
+    client: VertesiaClient,
+    interactionName: string,
+    params: InteractionExecutionParams,
+    prompt_data: any,
+    debug?: boolean,
+) {
     const userTags = params.tags;
     const info = activityInfo();
     const runId = info.workflowExecution.runId;
@@ -161,7 +186,7 @@ export async function executeInteractionFromActivity(client: VertesiaClient, int
             };
             const previousRun = await client.runs.search(payload).then((res) => {
                 log.info("Search results", { results: res });
-                return res ? res[0] ?? undefined : undefined
+                return res ? (res[0] ?? undefined) : undefined;
             });
 
             if (previousRun) {
@@ -178,26 +203,28 @@ export async function executeInteractionFromActivity(client: VertesiaClient, int
         environment: params.environment,
         model: params.model,
         model_options: params.model_options,
-    }
+    };
     const data = {
         ...prompt_data,
         previous_error: previousStudioExecutionRun?.error,
-    }
+    };
 
     const result_schema = params.result_schema;
 
     log.debug(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
 
-    const res = await client.interactions.executeByName(interactionName, {
-        config,
-        data,
-        result_schema,
-        tags,
-        stream: false,
-    }).catch((err) => {
-        log.error(`Error executing interaction ${interactionName}`, { err });
-        throw new Error(`Interaction Execution failed ${interactionName}: ${err.message}`);
-    });
+    const res = await client.interactions
+        .executeByName(interactionName, {
+            config,
+            data,
+            result_schema,
+            tags,
+            stream: false,
+        })
+        .catch((err) => {
+            log.error(`Error executing interaction ${interactionName}`, { err });
+            throw new Error(`Interaction Execution failed ${interactionName}: ${err.message}`);
+        });
 
     if (debug) {
         log.info(`Interaction executed ${interactionName}`, res);
@@ -209,4 +236,4 @@ export async function executeInteractionFromActivity(client: VertesiaClient, int
     }
 
     return res;
-}
+}

package/src/activities/extractDocumentText.ts

@@ -1,33 +1,41 @@
 import { log } from "@temporalio/activity";
-import { ContentObject, CreateContentObjectPayload, DSLActivityExecutionPayload, DSLActivitySpec } from '@vertesia/common';
-import { mutoolPdfToText } from '../conversion/mutool.js';
-import { manyToMarkdown } from '../conversion/pandoc.js';
+import {
+    ContentObject,
+    CreateContentObjectPayload,
+    DSLActivityExecutionPayload,
+    DSLActivitySpec,
+} from "@vertesia/common";
+import { mutoolPdfToText } from "../conversion/mutool.js";
+import { markdownWithPandoc } from "../conversion/pandoc.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
-import { NoDocumentFound } from '../errors.js';
-import { TextExtractionResult, TextExtractionStatus } from '../result-types.js';
-import { fetchBlobAsBuffer, md5 } from '../utils/blobs.js';
-import { countTokens } from '../utils/tokens.js';
+import { NoDocumentFound } from "../errors.js";
+import { TextExtractionResult, TextExtractionStatus } from "../result-types.js";
+import { fetchBlobAsBuffer, md5 } from "../utils/blobs.js";
+import { countTokens } from "../utils/tokens.js";
+import { markdownWithMarkitdown } from "../conversion/markitdown.js";
 
 //@ts-ignore
 const JSON: DSLActivitySpec = {
-    name: 'extractDocumentText',
-}
+    name: "extractDocumentText",
+};
 
 // doesn't have any own param
-export interface ExtractDocumentTextParams { };
+export interface ExtractDocumentTextParams {}
 export interface ExtractDocumentText extends DSLActivitySpec<ExtractDocumentTextParams> {
-    name: 'extractDocumentText';
+    name: "extractDocumentText";
     projection?: never;
 }
 
-export async function extractDocumentText(payload: DSLActivityExecutionPayload<ExtractDocumentTextParams>): Promise<TextExtractionResult> {
+export async function extractDocumentText(
+    payload: DSLActivityExecutionPayload<ExtractDocumentTextParams>,
+): Promise<TextExtractionResult> {
     const { client, objectId } = await setupActivity(payload);
 
     const r = await client.objects.find({
         query: { _id: objectId },
         limit: 1,
-        select: "+text"
-    })
+        select: "+text",
+    });
     const doc = r[0] as ContentObject;
     if (!doc) {
         log.error(`Document ${objectId} not found`);
@@ -36,7 +44,6 @@ export async function extractDocumentText(payload: DSLActivityExecutionPayload<E
 
     log.info(`Extracting text for object ${doc.id}`);
 
-
     if (!doc.content?.type || !doc.content?.source) {
         if (doc.text) {
             return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text present and no source or type");
@@ -58,74 +65,80 @@ export async function extractDocumentText(payload: DSLActivityExecutionPayload<E
         return createResponse(doc, "", TextExtractionStatus.error, e.message);
     }
 
-
     let txt: string;
 
     switch (doc.content.type) {
-
-        case 'application/pdf':
-            //if pdf is more than 2MB, use mutool
+        case "application/pdf":
             txt = await mutoolPdfToText(fileBuffer);
             break;
 
-        case 'text/plain':
-            txt = fileBuffer.toString('utf8')
+        case "text/plain":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //docx
-        case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
-            txt = await manyToMarkdown(fileBuffer, 'docx');
+        case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+            txt = await markdownWithMarkitdown(fileBuffer, "docx");
+            break;
+
+        //pptx
+        case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
+            txt = await markdownWithMarkitdown(fileBuffer, "pptx");
             break;
 
         //html
-        case 'text/html':
-            txt = await manyToMarkdown(fileBuffer, 'html');
+        case "text/html":
+            txt = await markdownWithPandoc(fileBuffer, "html");
             break;
 
         //opendocument
-        case 'application/vnd.oasis.opendocument.text':
-            txt = await manyToMarkdown(fileBuffer, 'odt');
+        case "application/vnd.oasis.opendocument.text":
+            txt = await markdownWithPandoc(fileBuffer, "odt");
             break;
 
         //rtf
-        case 'application/rtf':
-            txt = await manyToMarkdown(fileBuffer, 'rtf');
+        case "application/rtf":
+            txt = await markdownWithPandoc(fileBuffer, "rtf");
             break;
 
         //markdown
-        case 'text/markdown':
-            txt = fileBuffer.toString('utf8');
+        case "text/markdown":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //csv
-        case 'text/csv':
-            txt = fileBuffer.toString('utf8');
+        case "text/csv":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //typescript
-        case 'application/typescript':
-            txt = fileBuffer.toString('utf8');
+        case "application/typescript":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //javascript
-        case 'application/javascript':
-            txt = fileBuffer.toString('utf8');
+        case "application/javascript":
+            txt = fileBuffer.toString("utf8");
             break;
 
         //json
-        case 'application/json':
-            txt = fileBuffer.toString('utf8');
+        case "application/json":
+            txt = fileBuffer.toString("utf8");
             break;
 
         default:
             if (sniffIfText(fileBuffer)) {
-                txt = fileBuffer.toString('utf8'); //TODO: add charset detection
+                txt = fileBuffer.toString("utf8"); //TODO: add charset detection
                 break;
             }
-            return createResponse(doc, doc.text ?? '', TextExtractionStatus.skipped, `Unsupported mime type: ${doc.content.type}`);
+            return createResponse(
+                doc,
+                doc.text ?? "",
+                TextExtractionStatus.skipped,
+                `Unsupported mime type: ${doc.content.type}`,
+            );
     }
 
-
     const tokensData = countTokens(txt);
     const etag = doc.content.etag ?? md5(txt);
 
@@ -135,15 +148,20 @@ export async function extractDocumentText(payload: DSLActivityExecutionPayload<E
         tokens: {
             ...tokensData,
             etag: etag,
-        }
-    }
+        },
+    };
 
     await client.objects.update(doc.id, updateData);
 
     return createResponse(doc, txt, TextExtractionStatus.success);
 }
 
-function createResponse(doc: ContentObject, text: string, status: TextExtractionStatus, message?: string): TextExtractionResult {
+function createResponse(
+    doc: ContentObject,
+    text: string,
+    status: TextExtractionStatus,
+    message?: string,
+): TextExtractionResult {
     return {
         status,
         message,
@@ -151,11 +169,9 @@ function createResponse(doc: ContentObject, text: string, status: TextExtraction
         len: text.length,
         objectId: doc.id,
         hasText: !!text,
-    }
-
+    };
 }
 
-
 function sniffIfText(buf: Buffer) {
     // If file is too large, don't even try
     if (buf.length > 500 * 1024) {
@@ -181,9 +197,9 @@ function sniffIfText(buf: Buffer) {
 
     // Additional check for valid UTF-8 encoding
     try {
-        const s = buf.toString('utf8');
-        return s.length > 0 && !s.includes('\uFFFD'); // Replacement character
+        const s = buf.toString("utf8");
+        return s.length > 0 && !s.includes("\uFFFD"); // Replacement character
     } catch (e) {
         return false;
     }
-}
+}

package/src/activities/generateEmbeddings.ts

@@ -214,7 +214,7 @@ async function generateTextEmbeddings({ document, client, type, config }: Execut
                 }
                 log.info(`Generated embeddings for part ${i}`, { len: e.values.length, duration: new Date().getTime() - localStart });
 
-                return { inumber: i, result: e }
+                return { number: i, result: e }
             } catch (err: any) {
                 log.info(`Error generating ${type} embeddings for part ${i} of ${document.id}`, { error: err });
                 return { number: i, result: null, message: "error generating embeddings", error: err.message }

package/src/activities/generateImageRendition.ts

@@ -1,17 +1,17 @@
 import { log } from "@temporalio/activity";
 import { NodeStreamSource } from "@vertesia/client/node";
 import { DSLActivityExecutionPayload, DSLActivitySpec, RenditionProperties } from "@vertesia/common";
-import fs from 'fs';
 import ffmpeg from 'fluent-ffmpeg';
-import path from 'path';
+import fs from 'fs';
 import os from 'os';
+import path from 'path';
 import { imageResizer } from "../conversion/image.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { NoDocumentFound, WorkflowParamNotFound } from "../errors.js";
 import { saveBlobToTempFile } from "../utils/blobs.js";
 
 interface GenerateImageRenditionParams {
-    max_hw: number; //maximum size of the longuest side of the image
+    max_hw: number; //maximum size of the longest side of the image
     format: string; //format of the output image
 }
 
@@ -20,11 +20,20 @@ export interface GenerateImageRendition extends DSLActivitySpec<GenerateImageRen
 }
 
 export async function generateImageRendition(payload: DSLActivityExecutionPayload<GenerateImageRenditionParams>) {
-    const { client, objectId, params } = await setupActivity<GenerateImageRenditionParams>(payload);
+    const { client, objectId, params: originParams } = await setupActivity<GenerateImageRenditionParams>(payload);
+
+    // Fix: Use maxHeightWidth if max_hw is not provided
+    const params = {
+        ...originParams,
+        max_hw: originParams.max_hw || (originParams as any).maxHeightWidth || 1024, // Default to 1024 if both are missing
+        format: originParams.format || (originParams as any).format_output || 'png'  // Default to png if format is missing
+    };
+
+    log.info(`Generating image rendition for ${objectId}`, { originParams, params });
 
     const inputObject = await client.objects.retrieve(objectId).catch((err) => {
-        log.error(`Failed to retrieve document ${objectId}`, err);
-        if (err.response?.status === 404) {
+        log.error(`Failed to retrieve document ${objectId}`, { err });
+        if (err.message.includes("not found")) {
             throw new NoDocumentFound(`Document ${objectId} not found`, [objectId]);
         }
         throw err;
@@ -124,38 +133,50 @@ export async function generateImageRendition(payload: DSLActivityExecutionPayloa
         let resizedImagePath = null;
 
         try {
+            log.info(`Resizing image for ${objectId} page ${i}`, { page, params });
             // Resize the image using ImageMagick
             resizedImagePath = await imageResizer(page, params.max_hw, params.format);
 
             // Create a read stream from the resized image file
             const fileStream = fs.createReadStream(resizedImagePath);
-
+            const format = "image/" + params.format;
+            const fileId = pageId.split("/").pop() ?? "0." + params.format;
             const source = new NodeStreamSource(
                 fileStream,
-                pageId.split("/").pop() ?? "0." + params.format,
-                "image/" + params.format,
+                fileId,
+                format,
                 pageId,
             );
 
             log.info(
-                `Uploading rendition for ${objectId} page ${i} with max_hw: ${params.max_hw} and format: ${params.format}`,
+                `Uploading rendition for ${objectId} page ${i} with max_hw: ${params.max_hw} and format: ${params.format}`, {
+                resizedImagePath,
+                fileId,
+                format,
+                pageId,
+            }
             );
 
             const result = await client.objects.upload(source).catch((err) => {
-                log.error(`Failed to upload rendition for ${objectId} page ${i}`, { error: err });
-                return Promise.resolve(null);
+                log.error(`Failed to upload rendition for ${objectId} page ${i}`, {
+                    error: err,
+                    errorMessage: err.message,
+                    stack: err.stack
+                });
+                return Promise.reject(`Upload failed: ${err.message}`);
             });
+            log.info(`Rendition uploaded for ${objectId} page ${i}`, { result });
 
             return result;
         } catch (error) {
             log.error(`Failed to process rendition for ${objectId} page ${i}`, { error });
-            return Promise.resolve(null);
+            return Promise.reject(error instanceof Error ? error.message : null);
         }
     });
 
     const uploaded = await Promise.all(uploads);
     if (!uploaded || !uploaded.length || !uploaded[0]) {
-        log.error(`Failed to upload rendition for ${objectId}`);
+        log.error(`Failed to upload rendition for ${objectId}`, { uploaded });
         throw new Error(`Failed to upload rendition for ${objectId} - upload object is empty`);
     }