@vertesia/workflow 0.78.0 → 0.79.1

package/package.json

@@ -1,128 +1,136 @@
 {
-  "name": "@vertesia/workflow",
-  "version": "0.78.0",
-  "type": "module",
-  "description": "Vertesia workflow DSL",
-  "main": "./lib/esm/index.js",
-  "files": [
-    "lib",
-    "src"
-  ],
-  "bin": {
-    "bundle-workflows": "./bin/bundle-workflows.mjs"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/vertesia/composableai.git",
-    "directory": "zeno/workflow"
-  },
-  "license": "Apache-2.0",
-  "devDependencies": {
-    "@smithy/types": "^3.7.2",
-    "@temporalio/proto": "^1.11.5",
-    "@temporalio/testing": "^1.11.5",
-    "@temporalio/worker": "^1.11.5",
-    "@types/jsonwebtoken": "^9.0.7",
-    "@types/node": "^22.5.1",
-    "@types/papaparse": "^5.3.15",
-    "@types/tmp": "^0.2.6",
-    "ts-dual-module": "^0.6.3",
-    "vitest": "^3.0.9"
-  },
-  "dependencies": {
-    "@aws-sdk/client-s3": "^3.693.0",
-    "@aws-sdk/client-textract": "^3.693.0",
-    "@aws-sdk/credential-providers": "^3.693.0",
-    "@temporalio/activity": "^1.11.5",
-    "@temporalio/workflow": "^1.11.5",
-    "@types/json-schema": "^7.0.15",
-    "fast-deep-equal": "^3.1.3",
-    "jsonwebtoken": "^9.0.2",
-    "mime": "^4.0.0",
-    "ms": "3.0.0-canary.1",
-    "node-web-stream-adapters": "^0.2.1",
-    "p-limit": "^6.2.0",
-    "papaparse": "^5.4.1",
-    "seedrandom": "^3.0.5",
-    "sharp": "^0.33.4",
-    "tiktoken": "^1.0.15",
-    "tmp": "^0.2.3",
-    "tmp-promise": "^3.0.3",
-    "yaml": "^2.6.0",
-    "@vertesia/client": "0.78.0",
-    "@llumiverse/common": "0.22.0",
-    "@vertesia/api-fetch-client": "0.78.0",
-    "@vertesia/common": "0.78.0",
-    "@vertesia/memory": "0.49.0"
-  },
-  "ts_dual_module": {
-    "outDir": "lib",
-    "exports": {
-      "activities": "./activities/index.js",
-      "dsl-activities": "./activities/index-dsl.js",
-      "workflows": "./workflows",
-      "workflows-bundle": "./workflows-bundle.js",
-      "vars": "./vars"
-    }
-  },
-  "exports": {
-    ".": {
-      "types": "./lib/types/index.d.ts",
-      "import": "./lib/esm/index.js",
-      "require": "./lib/cjs/index.js"
+    "name": "@vertesia/workflow",
+    "version": "0.79.1",
+    "type": "module",
+    "description": "Vertesia workflow DSL",
+    "main": "./lib/esm/index.js",
+    "files": [
+        "lib",
+        "src"
+    ],
+    "bin": {
+        "bundle-workflows": "./bin/bundle-workflows.mjs"
     },
-    "./vars": {
-      "types": "./lib/types/vars.d.ts",
-      "import": "./lib/esm/vars.js",
-      "require": "./lib/cjs/vars.js"
+    "scripts": {
+        "test": "vitest run",
+        "build": "pnpm exec tsmod build && node ./bin/bundle-workflows.mjs lib/esm/workflows.js lib/workflows-bundle.js",
+        "clean": "rm -rf ./lib tsconfig.tsbuildinfo"
     },
-    "./activities": {
-      "types": "./lib/types/activities/index.d.ts",
-      "import": "./lib/esm/activities/index.js",
-      "require": "./lib/cjs/activities/index.js"
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/vertesia/composableai.git",
+        "directory": "zeno/workflow"
     },
-    "./dsl-activities": {
-      "types": "./lib/types/activities/index-dsl.d.ts",
-      "import": "./lib/esm/activities/index-dsl.js",
-      "require": "./lib/cjs/activities/index-dsl.js"
+    "license": "Apache-2.0",
+    "devDependencies": {
+        "@smithy/types": "^3.7.2",
+        "@temporalio/proto": "^1.11.5",
+        "@temporalio/testing": "^1.11.5",
+        "@temporalio/worker": "^1.11.5",
+        "@types/jsonwebtoken": "^9.0.7",
+        "@types/node": "^22.5.1",
+        "@types/papaparse": "^5.3.15",
+        "@types/tmp": "^0.2.6",
+        "ts-dual-module": "^0.6.3",
+        "vitest": "^3.0.9"
     },
-    "./workflows": {
-      "types": "./lib/types/workflows.d.ts",
-      "import": "./lib/esm/workflows.js",
-      "require": "./lib/cjs/workflows.js"
+    "dependencies": {
+        "@aws-sdk/client-s3": "^3.693.0",
+        "@aws-sdk/client-textract": "^3.693.0",
+        "@aws-sdk/credential-providers": "^3.693.0",
+        "@llumiverse/common": "workspace:*",
+        "@temporalio/activity": "^1.11.5",
+        "@temporalio/workflow": "^1.11.5",
+        "@types/json-schema": "^7.0.15",
+        "@vertesia/api-fetch-client": "workspace:*",
+        "@vertesia/client": "workspace:*",
+        "@vertesia/common": "workspace:*",
+        "@vertesia/memory": "workspace:*",
+        "fast-deep-equal": "^3.1.3",
+        "jsonwebtoken": "^9.0.2",
+        "mime": "^4.0.0",
+        "ms": "3.0.0-canary.1",
+        "node-web-stream-adapters": "^0.2.1",
+        "p-limit": "^6.2.0",
+        "papaparse": "^5.4.1",
+        "seedrandom": "^3.0.5",
+        "sharp": "^0.33.4",
+        "tiktoken": "^1.0.15",
+        "tmp": "^0.2.3",
+        "tmp-promise": "^3.0.3",
+        "yaml": "^2.6.0"
     },
-    "./dsl": {
-      "types": "./lib/types/dsl.d.ts",
-      "import": "./lib/esm/dsl.js",
-      "require": "./lib/cjs/dsl.js"
+    "ts_dual_module": {
+        "outDir": "lib",
+        "exports": {
+            "activities": "./activities/index.js",
+            "dsl-activities": "./activities/index-dsl.js",
+            "workflows": "./workflows",
+            "workflows-bundle": "./workflows-bundle.js",
+            "vars": "./vars"
+        }
     },
-    "./workflows-bundle": {
-      "import": "./lib/workflows-bundle.js"
-    }
-  },
-  "types": "./lib/types/index.d.ts",
-  "typesVersions": {
-    "*": {
-      "activities": [
-        "./lib/types/activities/index.d.ts"
-      ],
-      "dsl-activities": [
-        "./lib/types/activities/index-dsl.d.ts"
-      ],
-      "workflows": [
-        "./lib/types/workflows.d.ts"
-      ],
-      "dsl": [
-        "./lib/types/dsl.d.ts"
-      ],
-      "vars": [
-        "./lib/types/dsl/vars.d.ts"
-      ]
+    "exports": {
+        ".": {
+            "types": "./lib/types/index.d.ts",
+            "import": "./lib/esm/index.js",
+            "require": "./lib/cjs/index.js"
+        },
+        "./vars": {
+            "types": "./lib/types/vars.d.ts",
+            "import": "./lib/esm/vars.js",
+            "require": "./lib/cjs/vars.js"
+        },
+        "./activities": {
+            "types": "./lib/types/activities/index.d.ts",
+            "import": "./lib/esm/activities/index.js",
+            "require": "./lib/cjs/activities/index.js"
+        },
+        "./dsl-activities": {
+            "types": "./lib/types/activities/index-dsl.d.ts",
+            "import": "./lib/esm/activities/index-dsl.js",
+            "require": "./lib/cjs/activities/index-dsl.js"
+        },
+        "./workflows": {
+            "types": "./lib/types/workflows.d.ts",
+            "import": "./lib/esm/workflows.js",
+            "require": "./lib/cjs/workflows.js"
+        },
+        "./dsl": {
+            "types": "./lib/types/dsl.d.ts",
+            "import": "./lib/esm/dsl.js",
+            "require": "./lib/cjs/dsl.js"
+        },
+        "./workflows-bundle": {
+            "import": "./lib/workflows-bundle.js"
+        },
+        "./errors": {
+            "types": "./lib/types/errors.d.ts",
+            "import": "./lib/esm/errors.js",
+            "require": "./lib/cjs/errors.js"
+        }
+    },
+    "types": "./lib/types/index.d.ts",
+    "typesVersions": {
+        "*": {
+            "activities": [
+                "./lib/types/activities/index.d.ts"
+            ],
+            "dsl-activities": [
+                "./lib/types/activities/index-dsl.d.ts"
+            ],
+            "workflows": [
+                "./lib/types/workflows.d.ts"
+            ],
+            "dsl": [
+                "./lib/types/dsl.d.ts"
+            ],
+            "vars": [
+                "./lib/types/dsl/vars.d.ts"
+            ],
+            "errors": [
+                "./lib/types/errors.d.ts"
+            ]
+        }
     }
-  },
-  "scripts": {
-    "test": "vitest run",
-    "build": "pnpm exec tsmod build && node ./bin/bundle-workflows.mjs lib/esm/workflows.js lib/workflows-bundle.js",
-    "clean": "rm -rf ./lib tsconfig.tsbuildinfo"
-  }
-}
+}

package/src/activities/advanced/createDocumentTypeFromInteractionRun.ts

@@ -1,5 +1,6 @@
-import { CreateContentObjectTypePayload, DSLActivityExecutionPayload, DSLActivitySpec, ExecutionRun } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { InteractionOutput } from "@vertesia/client";
+import { CreateContentObjectTypePayload, DSLActivityExecutionPayload, DSLActivitySpec, ExecutionRun } from "@vertesia/common";
 import { projectResult } from "../../dsl/projections.js";
 import { setupActivity } from "../../dsl/setup/ActivityContext.js";
 import { ActivityParamNotFoundError } from "../../errors.js";
@@ -28,18 +29,18 @@ export async function createDocumentTypeFromInteractionRun(payload: DSLActivityE
         throw new ActivityParamNotFoundError("run", payload.activity);
     }
 
-    const genTypeRes = params.run.result;
+    const jsonResult = InteractionOutput.from(params.run.result).object();
 
-    if (!genTypeRes.document_type) {
-        log.error("No name generated for type: " + JSON.stringify(genTypeRes), genTypeRes);
+    if (!jsonResult.document_type) {
+        log.error("No name generated for type: " + JSON.stringify(jsonResult), jsonResult);
         throw new Error("No name generated for type");
     }
 
-    log.info("Generated schema for type", genTypeRes.metadata_schema);
+    log.info("Generated schema for type", jsonResult.metadata_schema);
     const typeData: CreateContentObjectTypePayload = {
-        name: genTypeRes.document_type,
-        object_schema: genTypeRes.metadata_schema,
-        is_chunkable: !!genTypeRes.is_chunkable,
+        name: jsonResult.document_type,
+        object_schema: jsonResult.metadata_schema,
+        is_chunkable: !!jsonResult.is_chunkable,
     }
 
     const type = await client.types.create(typeData);

package/src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts

@@ -68,28 +68,30 @@ export async function createOrUpdateDocumentFromInteractionRun(payload: DSLActiv
 
 
     const result = run.result;
-    const resultIsObject = typeof result === 'object';
     const inputData = run.parameters;
 
-    let name: string;
-    if (resultIsObject) {
-        name = result['name'] || result["title"] || inputData['name'] || params.fallback_name || 'Untitled';
-    } else {
-        name = inputData['name'] || params.fallback_name || 'Untitled';
+    // Try to parse result as JSON, fallback to text if not valid JSON
+    let jsonResult: any = null;
+    try {
+        jsonResult = result.object();
+    } catch (e) {
+        log.info("Result is not valid JSON, will use text content instead", { error: e instanceof Error ? e.message : String(e) });
     }
 
+    const name = jsonResult?.['name'] || jsonResult?.["title"] || inputData['name'] || params.fallback_name || undefined;
+
     const docPayload = {
         name,
         parent: params.parent ?? undefined,
-        properties: resultIsObject ? result : {},
-        text: !resultIsObject ? result : undefined,
+        properties: jsonResult ? jsonResult : {},
+        text: !jsonResult ? result.text() : undefined,
         type: type?.id,
         status: ContentObjectStatus.completed,
         generation_run_info: {
             id: run.id,
             date: new Date().toISOString(),
             model: run.modelId,
-            target: resultIsObject ? 'properties' : 'text'
+            target: jsonResult ? 'properties' : 'text'
         }
     };
 

package/src/activities/advanced/updateDocumentFromInteractionRun.ts

@@ -1,3 +1,4 @@
+import { InteractionOutput } from "@vertesia/client";
 import { DSLActivityExecutionPayload, DSLActivitySpec, ExecutionRun } from "@vertesia/common";
 import { setupActivity } from "../../dsl/setup/ActivityContext.js";
 import { ActivityParamNotFoundError } from "../../errors.js";
@@ -22,7 +23,7 @@ export async function updateDocumentFromInteractionRun(payload: DSLActivityExecu
         throw new ActivityParamNotFoundError("run", payload.activity);
     }
 
-    const docProps = params.run.result;
+    const docProps = InteractionOutput.from(params.run.result).object();
 
     if (!docProps) {
         return { status: "failed", error: "no-props" };

package/src/activities/chunkDocument.ts

@@ -82,7 +82,9 @@ export async function chunkDocument(payload: DSLActivityExecutionPayload<ChunkDo
         content: instrumented
     });
 
-    const parts = res.result.parts as DocPart[];
+    const jsonResult = res.result.object();
+
+    const parts = jsonResult.parts as DocPart[];
     if (!parts || parts.length === 0) {
         log.warn('No parts found for object ID: ' + objectId, res);
         return { id: objectId, status: "failed", parts: [], message: "no parts found" }

package/src/activities/executeInteraction.ts

@@ -1,4 +1,4 @@
-import { Modalities, ModelOptions } from "@llumiverse/common";
+import { CompletionResult, Modalities, ModelOptions } from "@llumiverse/common";
 import { activityInfo, log } from "@temporalio/activity";
 import { VertesiaClient } from "@vertesia/client";
 import { NodeStreamSource } from "@vertesia/client/node";
@@ -13,7 +13,7 @@ import {
 } from "@vertesia/common";
 import { projectResult } from "../dsl/projections.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
-import { ActivityParamInvalidError, ActivityParamNotFoundError } from "../errors.js";
+import { ActivityParamInvalidError, ActivityParamNotFoundError, ResourceExhaustedError } from "../errors.js";
 import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
 import { Readable } from "stream";
 
@@ -98,6 +98,11 @@ export interface InteractionExecutionParams {
      * Options to control generation
      */
     model_options?: ModelOptions;
+
+    /**
+     * activity won't be retried if it fails due to resource exhaustion (429)
+     */
+    exit_on_resource_exhaustion?: boolean;
 }
 
 /**
@@ -150,52 +155,56 @@ export async function executeInteraction(payload: DSLActivityExecutionPayload<Ex
             prompt_data,
             payload.debug_mode,
         );
-        
+
+        let completionResult: CompletionResult[] = res.result;
+
         // Handle image uploads if the result contains base64 images
         if (res.output_modality === Modalities.image) {
-            const images = res.result.images;
+            const images = res.result.images();
             const uploadedImages = await Promise.all(
                 images.map((image: string, index: number) => {
                     // Extract base64 data and create buffer
                     const base64Data = image.replace(/^data:image\/[a-z]+;base64,/, "");
                     const buffer = Buffer.from(base64Data, 'base64');
-                    
+
                     // Generate filename
                     const { runId } = activityInfo().workflowExecution;
                     const { activityId } = activityInfo();
                     const filename = `generated-image-${runId}-${activityId}-${index}.png`;
-                
+
                     // Create a readable stream from the buffer
                     const stream = Readable.from(buffer);
-                    
+
                     const source = new NodeStreamSource(
                         stream,
                         filename,
                         "image/png",
                     );
-                    
+
                     return client.files.uploadFile(source);
                 })
             );
-            res.result.images = uploadedImages;
+            completionResult = uploadedImages.map(file => ({ type: "image", value: file }));
         }
 
         return projectResult(payload, params, res, {
             runId: res.id,
             status: res.status,
-            result: res.result,
+            result: completionResult,
         });
 
     } catch (error: any) {
-        log.error("Failed to execute interaction", { error });
-        if (error.message.includes("Failed to validate merged prompt schema")) {
+        log.error(`Failed to execute interaction ${interactionName}`, { error });
+        if (error.statusCode === 429 && params.exit_on_resource_exhaustion) {
+            throw new ResourceExhaustedError(error.statusCode, "Resource exhausted - rate limit exceeded");
+        } else if (error.message.includes("Failed to validate merged prompt schema")) {
             //issue with the input data, don't retry
             throw new ActivityParamInvalidError("prompt_data", payload.activity, error.message);
         } else if (error.message.includes("modelId: Path `modelId` is required")) {
             //issue with the input data, don't retry
             throw new ActivityParamInvalidError("model", payload.activity, error.message);
         } else {
-            throw error;
+            throw new Error(`Interaction Execution failed ${interactionName}: ${error.message}`);
         }
     }
 }
@@ -270,7 +279,7 @@ export async function executeInteractionFromActivity(
         })
         .catch((err) => {
             log.error(`Error executing interaction ${interactionName}`, { err });
-            throw new Error(`Interaction Execution failed ${interactionName}: ${err.message}`);
+            throw err;
         });
 
     if (debug) {

package/src/activities/generateDocumentProperties.ts

@@ -1,7 +1,7 @@
 import { log } from "@temporalio/activity";
 import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
-import { TruncateSpec } from "../utils/tokens.js";
+import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
 import { InteractionExecutionParams, executeInteractionFromActivity } from "./executeInteraction.js";
 
 const INT_EXTRACT_INFORMATION = "sys:ExtractInformation";
@@ -55,8 +55,12 @@ export async function generateDocumentProperties(
         return undefined;
     };
 
+    const content = doc.text
+        ? truncByMaxTokens(doc.text, params.truncate || 30000)
+        : undefined;
+
     const promptData = {
-        content: doc.text ?? undefined,
+        content: content,
         image: getImageRef() ?? undefined,
         human_context: project?.configuration?.human_context ?? undefined,
     };
@@ -84,11 +88,12 @@ export async function generateDocumentProperties(
             return undefined;
         }
         let text = "";
-        if (infoRes.result.title) {
-            text += infoRes.result.title + "\n";
+        const jsonResult = infoRes.result.object();
+        if (jsonResult.title) {
+            text += jsonResult.title + "\n";
         }
-        if (infoRes.result.description) {
-            text += infoRes.result.description;
+        if (jsonResult.description) {
+            text += jsonResult.description;
         }
         if (text) {
             return text;
@@ -100,7 +105,7 @@ export async function generateDocumentProperties(
     log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
     await client.objects.update(doc.id, {
         properties: {
-            ...infoRes.result,
+            ...infoRes.result.object(),
             etag: doc.text_etag,
         },
         text: getText(),

package/src/activities/generateOrAssignContentType.ts

@@ -119,7 +119,7 @@ export async function generateOrAssignContentType(
 
   log.info(
     "Execute SelectDocumentType interaction on content with \nexisting types - passing full types: " +
-      existing_types.filter((t) => !t.tags?.includes("system")),
+    existing_types.filter((t) => !t.tags?.includes("system")),
   );
 
   const res = await executeInteractionFromActivity(
@@ -133,12 +133,15 @@ export async function generateOrAssignContentType(
     },
   );
 
-  log.info("Selected Content Type Result: " + JSON.stringify(res.result));
+  const jsonResult = res.result.object();
+
+  log.info("Selected Content Type Result: " + JSON.stringify(jsonResult));
+
 
   //if type is not identified or not present in the database, generate a new type
   let selectedType: { id: string; name: string } | undefined = undefined;
 
-  selectedType = types.find((t) => t.name === res.result.document_type);
+  selectedType = types.find((t) => t.name === jsonResult.document_type);
 
   if (!selectedType) {
     log.warn("Document type not identified: starting type generation");
@@ -153,7 +156,7 @@ export async function generateOrAssignContentType(
 
   if (!selectedType) {
     log.error("Type not found: ", res.result);
-    throw new Error("Type not found: " + res.result.document_type);
+    throw new Error("Type not found: " + jsonResult.document_type);
   }
 
   //update object with selected type
@@ -193,18 +196,20 @@ async function generateNewType(
     },
   );
 
-  if (!genTypeRes.result.document_type) {
+  const jsonResult = genTypeRes.result.object();
+
+  if (!jsonResult.document_type) {
     log.error("No name generated for type", genTypeRes);
     throw new Error("No name generated for type");
   }
 
-  log.info("Generated schema for type", genTypeRes.result.metadata_schema);
+  log.info("Generated schema for type", jsonResult.metadata_schema);
   const typeData: CreateContentObjectTypePayload = {
-    name: genTypeRes.result.document_type,
-    description: genTypeRes.result.document_type_description,
-    object_schema: genTypeRes.result.metadata_schema,
-    is_chunkable: genTypeRes.result.is_chunkable,
-    table_layout: genTypeRes.result.table_layout,
+    name: jsonResult.document_type,
+    description: jsonResult.document_type_description,
+    object_schema: jsonResult.metadata_schema,
+    is_chunkable: jsonResult.is_chunkable,
+    table_layout: jsonResult.table_layout,
   };
 
   const type = await client.types.create(typeData);

package/src/activities/index-dsl.ts

@@ -15,6 +15,7 @@ export { getObjectFromStore } from "./getObjectFromStore.js";
 export { handleDslError } from "./handleError.js";
 export { convertPdfToStructuredText } from "./media/processPdfWithTextract.js";
 export { transcribeMedia } from "./media/transcribeMediaWithGladia.js";
+export { prepareVideo } from "./media/prepareVideo.js";
 export { notifyWebhook } from "./notifyWebhook.js";
 export { setDocumentStatus } from "./setDocumentStatus.js";
-export { identifyTextSections } from "./identifyTextSections.js";
+export { checkRateLimit } from "./rateLimiter.js";