@vertesia/workflow 0.51.0 → 0.54.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vertesia/workflow",
-  "version": "0.51.0",
+  "version": "0.54.0",
   "type": "module",
   "description": "Composable prompts workflow dsl",
   "main": "./lib/esm/index.js",
@@ -19,14 +19,16 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@smithy/types": "^3.7.2",
+    "@temporalio/proto": "^1.11.5",
     "@temporalio/testing": "^1.11.5",
     "@temporalio/worker": "^1.11.5",
+    "@types/fluent-ffmpeg": "^2.1.27",
     "@types/jsonwebtoken": "^9.0.7",
     "@types/node": "^22.5.1",
     "@types/papaparse": "^5.3.15",
     "@types/tmp": "^0.2.6",
     "ts-dual-module": "^0.6.3",
-    "vitest": "^2.1.9"
+    "vitest": "^3.0.9"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.693.0",
@@ -34,11 +36,11 @@
     "@aws-sdk/credential-providers": "^3.693.0",
     "@temporalio/activity": "^1.11.5",
     "@temporalio/workflow": "^1.11.5",
-    "@tensorflow/tfjs-node": "^4.19.0",
     "@types/json-schema": "^7.0.15",
     "@vertesia/memory": "^0.43.0",
     "api-fetch-client": "^0.13.0",
     "fast-deep-equal": "^3.1.3",
+    "fluent-ffmpeg": "^2.1.3",
     "jsonwebtoken": "^9.0.2",
     "ms": "3.0.0-canary.1",
     "node-web-stream-adapters": "^0.2.1",
@@ -47,10 +49,11 @@
     "sharp": "^0.33.4",
     "tiktoken": "^1.0.15",
     "tmp": "^0.2.3",
+    "tmp-promise": "^3.0.3",
     "yaml": "^2.6.0",
-    "@llumiverse/core": "0.15.0",
-    "@vertesia/client": "0.51.0",
-    "@vertesia/common": "0.51.0"
+    "@llumiverse/core": "0.17.0",
+    "@vertesia/client": "0.54.0",
+    "@vertesia/common": "0.54.0"
   },
   "ts_dual_module": {
     "outDir": "lib",

package/src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts

@@ -1,5 +1,5 @@
-import { ContentObjectStatus, DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { ContentObjectStatus, DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { setupActivity } from "../../dsl/setup/ActivityContext.js";
 import { ActivityParamNotFound, NoDocumentFound } from "../../errors.js";
 interface CreateOrUpdateObjectFromInteractionRunParams {
@@ -19,9 +19,21 @@ interface CreateOrUpdateObjectFromInteractionRunParams {
      */
     update_existing_id?: string,
 
+    /**
+     * The name of the object to use. If not provided, the name will be generated from the interaction result
+     */
     fallback_name?: string, // a name to use if no one was generated by the interaction
 
+    /**
+     * The name of the parent object to use. If not provided, the document will be created at the root level
+     */
     parent?: string, // the parent object id
+
+    /**
+     * The name of the property to use for the text. If not provided, the text will be set to the result of the interaction
+     */
+    update_text_from_property?: string,
+
 }
 
 export interface CreateOrUpdateObjectFromInteractionRun extends DSLActivitySpec<CreateOrUpdateObjectFromInteractionRunParams> {
@@ -80,6 +92,13 @@ export async function createOrUpdateDocumentFromInteractionRun(payload: DSLActiv
         }
     };
 
+    if (params.update_text_from_property) {
+        const text = docPayload.properties[params.update_text_from_property];
+        if (text) {
+            docPayload.text = text;
+        }
+    }
+
     //create or update the document
     let newDoc: boolean = false;
     let doc = undefined;

package/src/activities/chunkDocument.ts

@@ -1,18 +1,12 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { DocPart } from "../utils/chunks.js";
 import { InteractionExecutionParams, executeInteractionFromActivity } from "./executeInteraction.js";
 
 const INT_CHUNK_DOCUMENT = "sys:ChunkDocument"
 
-interface DocPart {
 
-    line_number_start: number
-    line_number_end: number
-    name: string
-    type: string
-
-}
 
 export interface ChunkDocumentResult {
     id: string
@@ -22,9 +16,28 @@ export interface ChunkDocumentResult {
 }
 
 export interface ChunkDocumentParams extends InteractionExecutionParams {
+
+    /**
+     * If true, force chunking even if the document is already chunked
+     */
     force?: boolean;
+
+    /**
+     * The interaction name to use for chunking
+     * If not set, the default interaction will be used
+     */
     interactionName?: string;
+
+    /**
+     * The object type to use for the document parts
+     * If not set, the type of the document will be used
+     */
     docPartType?: string;
+
+    /**
+     * If true, create parts as document objects
+     */
+    createParts?: boolean;
 }
 
 export interface ChunkDocument extends DSLActivitySpec<ChunkDocumentParams> {
@@ -75,48 +88,55 @@ export async function chunkDocument(payload: DSLActivityExecutionPayload<ChunkDo
         return { id: objectId, status: "failed", parts: [], message: "no parts found" }
     }
 
-    const partDocs = await Promise.all(parts.map(async (part, i) => {
 
-        const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
+    /**
+     * Only create parts as document if the flag is set
+     */
+    if (params.createParts) {
+
+        const partDocs = await Promise.all(parts.map(async (part, i) => {
 
-        const location = () => {
-            let location = document.location;
-            if (location.endsWith('/')) {
-                location += document.name + "/" + part.type
+            const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
+
+            const location = () => {
+                let location = document.location;
+                if (location.endsWith('/')) {
+                    location += document.name + "/" + part.type
+                }
+                location += '/' + document.name + "/" + part.type;
+                return location;
             }
-            location += '/' + document.name + "/" + part.type;
-            return location;
+
+            const docPart = await client.objects.create({
+                name: part.name,
+                parent: objectId,
+                text: text,
+                location: location(),
+                properties: {
+                    part_number: i + 1,
+                    etag: document.text_etag,
+                    source_line_start: part.line_number_start,
+                    source_line_end: part.line_number_end,
+                    title: part.name
+                }
+            });
+            return docPart;
+        }));
+
+        //delete previous parts
+        if (document.parts && document.parts.length > 0) {
+            log.info('Deleting previous parts for object ID: ' + objectId, { parts: document.parts });
+            await Promise.all(document.parts.map(async (partId) => {
+                await client.objects.delete(partId);
+            }));
         }
 
-        const docPart = await client.objects.create({
-            name: part.name,
-            parent: objectId,
-            text: text,
-            location: location(),
-            properties: {
-                part_number: i + 1,
-                etag: document.text_etag,
-                source_line_start: part.line_number_start,
-                source_line_end: part.line_number_end,
-                title: part.name
-            }
+        await client.objects.update(objectId, {
+            parts: partDocs.map(p => p.id),
+            parts_etag: document.text_etag
         });
-        return docPart;
-    }));
-
-    //delete previous parts
-    if (document.parts && document.parts.length > 0) {
-        log.info('Deleting previous parts for object ID: ' + objectId, { parts: document.parts });
-        await Promise.all(document.parts.map(async (partId) => {
-            await client.objects.delete(partId);
-        }));
     }
 
-    await client.objects.update(objectId, {
-        parts: partDocs.map(p => p.id),
-        parts_etag: document.text_etag
-    });
-
     log.info(`Object ${objectId} chunking completed`, { parts: document.parts });
 
     return { id: objectId, status: "completed", parts: document.parts }

package/src/activities/createDocumentFromOther.ts

@@ -1,11 +1,11 @@
 import { log } from "@temporalio/activity";
+import { NodeStreamSource } from "@vertesia/client/node";
 import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import fs from 'fs';
 import { pdfExtractPages } from "../conversion/mutool.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { NoDocumentFound } from "../errors.js";
 import { saveBlobToTempFile } from "../utils/blobs.js";
-import { NodeStreamSource } from "../utils/memory.js";
 
 interface CreatePdfDocumentFromSourceParams {
 
@@ -24,7 +24,7 @@ export interface CreatePdfDocumentFromSource extends DSLActivitySpec<CreatePdfDo
 
 
 /**
- * Create a new PDF by extrracting pages from a source PDF
+ * Create a new PDF by extracting pages from a source PDF
  * @returns
  */
 export async function createPdfDocumentFromSource(payload: DSLActivityExecutionPayload<CreatePdfDocumentFromSourceParams>) {

package/src/activities/executeInteraction.ts

@@ -1,15 +1,23 @@
-import { VertesiaClient } from "@vertesia/client";
-import { DSLActivityExecutionPayload, DSLActivitySpec, ExecutionRun, ExecutionRunStatus, InteractionExecutionConfiguration, RunSearchPayload } from "@vertesia/common";
+import { ModelOptions } from "@llumiverse/core";
 import { activityInfo, log } from "@temporalio/activity";
+import { VertesiaClient } from "@vertesia/client";
+import {
+    DSLActivityExecutionPayload,
+    DSLActivitySpec,
+    ExecutionRun,
+    ExecutionRunStatus,
+    InteractionExecutionConfiguration,
+    RunSearchPayload,
+} from "@vertesia/common";
 import { projectResult } from "../dsl/projections.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { ActivityParamInvalid, ActivityParamNotFound } from "../errors.js";
 import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
-import { ModelOptions } from "@llumiverse/core";
 
 //Example:
 //@ts-ignore
 const JSON: DSLActivitySpec = {
-    name: 'executeInteraction',
+    name: "executeInteraction",
     import: ["defaultModel", "guidlineId", "docTypeId"],
     params: {
         defaultModel: "${model}",
@@ -22,8 +30,8 @@ const JSON: DSLActivitySpec = {
         result_schema: "${docType.object_schema}",
         prompt_data: {
             documents: "${documents}",
-            guidline: "${guidline.text}"
-        }
+            guidline: "${guidline.text}",
+        },
     },
     fetch: {
         documents: {
@@ -40,7 +48,7 @@ const JSON: DSLActivitySpec = {
                 id: "${guidlineId}",
             },
             select: "+text",
-            on_not_found: "throw"
+            on_not_found: "throw",
         },
         docType: {
             type: "document_type",
@@ -49,19 +57,19 @@ const JSON: DSLActivitySpec = {
                 id: "${docTypeId}",
             },
             select: "+object_schema",
-        }
-    }
-}
+        },
+    },
+};
 export interface InteractionExecutionParams {
     /**
      * The environment to use. If not specified the project default environment will be used.
-     * If the latter is not specified an exeption will be thrown.
+     * If the latter is not specified an exception will be thrown.
      */
     environment?: string;
     /**
      * The model to use. If not specified the project default model will be used.
      * If the latter is not specified the default model of the environment will be used.
-     * If the latter is not specified an exeption will be thrown.
+     * If the latter is not specified an exception will be thrown.
      */
     model?: string;
 
@@ -83,26 +91,43 @@ export interface InteractionExecutionParams {
     /**
      * Options to control generation
      */
-    model_options?: ModelOptions; 
+    model_options?: ModelOptions;
 }
 
-
+/**
+ * TODO: must be kept in sync with InteractionAsyncExecutionPayload form @vertesia/common
+ * Also see the executeInteractionAsync endpoint on the server for how the client payload is sent to the workflow.
+ * (interaction is translated to interactionName)
+ */
 export interface ExecuteInteractionParams extends InteractionExecutionParams {
+    //TODO rename to interaction as in InteractionAsyncExecutionPayload
     interactionName: string;
     prompt_data: Record<string, any>;
-    truncate?: Record<string, TruncateSpec>
+    /**
+     * Additional prompt data passed by the workflow configuration. This will be merged with prompt_data if any.
+     * You should use `import: ["static_prompt_data"]` to import the workflow prompt data as static_prompt_data param.
+     * Otherwise the workflow prompt data will be ignored.
+     */
+    static_prompt_data?: Record<string, any>;
+    truncate?: Record<string, TruncateSpec>;
 }
 
 export interface ExecuteInteraction extends DSLActivitySpec<ExecuteInteractionParams> {
-    name: 'executeInteraction';
+    name: "executeInteraction";
 }
 
 export async function executeInteraction(payload: DSLActivityExecutionPayload<ExecuteInteractionParams>) {
-    const {
-        client, params
-    } = await setupActivity<ExecuteInteractionParams>(payload);
+    const { client, params } = await setupActivity<ExecuteInteractionParams>(payload);
 
-    const { interactionName, prompt_data } = params;
+    const { interactionName, prompt_data, static_prompt_data: wf_prompt_data } = params;
+    if (wf_prompt_data) {
+        Object.assign(prompt_data, wf_prompt_data);
+    }
+
+    if (!interactionName) {
+        log.error("Missing interactionName", { params });
+        throw new ActivityParamNotFound("interactionName", payload.activity);
+    }
 
     if (params.truncate) {
         const truncate = params.truncate;
@@ -111,17 +136,37 @@ export async function executeInteraction(payload: DSLActivityExecutionPayload<Ex
         }
     }
 
-    const res = await executeInteractionFromActivity(client, interactionName, params, prompt_data, payload.debug_mode);
-
-    return projectResult(payload, params, res, {
-        runId: res.id,
-        status: res.status,
-        result: res.result,
-    });
-
+    try {
+        const res = await executeInteractionFromActivity(
+            client,
+            interactionName,
+            params,
+            prompt_data,
+            payload.debug_mode,
+        );
+        return projectResult(payload, params, res, {
+            runId: res.id,
+            status: res.status,
+            result: res.result,
+        });
+    } catch (error: any) {
+        log.error("Failed to execute interaction", { error });
+        if (error.message.includes("Failed to validate merged prompt schema")) {
+            //issue with the input data, don't retry
+            throw new ActivityParamInvalid("prompt_data", payload.activity, error.message);
+        } else {
+            throw error;
+        }
+    }
 }
 
-export async function executeInteractionFromActivity(client: VertesiaClient, interactionName: string, params: InteractionExecutionParams, prompt_data: any, debug?: boolean) {
+export async function executeInteractionFromActivity(
+    client: VertesiaClient,
+    interactionName: string,
+    params: InteractionExecutionParams,
+    prompt_data: any,
+    debug?: boolean,
+) {
     const userTags = params.tags;
     const info = activityInfo();
     const runId = info.workflowExecution.runId;
@@ -141,7 +186,7 @@ export async function executeInteractionFromActivity(client: VertesiaClient, int
             };
             const previousRun = await client.runs.search(payload).then((res) => {
                 log.info("Search results", { results: res });
-                return res ? res[0] ?? undefined : undefined
+                return res ? (res[0] ?? undefined) : undefined;
             });
 
             if (previousRun) {
@@ -158,28 +203,28 @@ export async function executeInteractionFromActivity(client: VertesiaClient, int
         environment: params.environment,
         model: params.model,
         model_options: params.model_options,
-    }
+    };
     const data = {
         ...prompt_data,
         previous_error: previousStudioExecutionRun?.error,
-    }
+    };
 
     const result_schema = params.result_schema;
 
-    if (debug) {
-        log.info(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
-    }
-
-    const res = await client.interactions.executeByName(interactionName, {
-        config,
-        data,
-        result_schema,
-        tags,
-        stream: false,
-    }).catch((err) => {
-        log.error(`Error executing interaction ${interactionName}`, { err });
-        throw new Error(`Interaction Execution failed ${interactionName}: ${err.message}`);
-    });
+    log.debug(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
+
+    const res = await client.interactions
+        .executeByName(interactionName, {
+            config,
+            data,
+            result_schema,
+            tags,
+            stream: false,
+        })
+        .catch((err) => {
+            log.error(`Error executing interaction ${interactionName}`, { err });
+            throw new Error(`Interaction Execution failed ${interactionName}: ${err.message}`);
+        });
 
     if (debug) {
         log.info(`Interaction executed ${interactionName}`, res);
@@ -191,4 +236,4 @@ export async function executeInteractionFromActivity(client: VertesiaClient, int
     }
 
     return res;
-}
+}