@vertesia/workflow 0.51.0 → 0.52.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vertesia/workflow",
-  "version": "0.51.0",
+  "version": "0.52.0",
   "type": "module",
   "description": "Composable prompts workflow dsl",
   "main": "./lib/esm/index.js",
@@ -21,12 +21,13 @@
     "@smithy/types": "^3.7.2",
     "@temporalio/testing": "^1.11.5",
     "@temporalio/worker": "^1.11.5",
+    "@types/fluent-ffmpeg": "^2.1.27",
     "@types/jsonwebtoken": "^9.0.7",
     "@types/node": "^22.5.1",
     "@types/papaparse": "^5.3.15",
     "@types/tmp": "^0.2.6",
     "ts-dual-module": "^0.6.3",
-    "vitest": "^2.1.9"
+    "vitest": "^3.0.9"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.693.0",
@@ -34,11 +35,11 @@
     "@aws-sdk/credential-providers": "^3.693.0",
     "@temporalio/activity": "^1.11.5",
     "@temporalio/workflow": "^1.11.5",
-    "@tensorflow/tfjs-node": "^4.19.0",
     "@types/json-schema": "^7.0.15",
     "@vertesia/memory": "^0.43.0",
     "api-fetch-client": "^0.13.0",
     "fast-deep-equal": "^3.1.3",
+    "fluent-ffmpeg": "^2.1.3",
     "jsonwebtoken": "^9.0.2",
     "ms": "3.0.0-canary.1",
     "node-web-stream-adapters": "^0.2.1",
@@ -47,10 +48,11 @@
     "sharp": "^0.33.4",
     "tiktoken": "^1.0.15",
     "tmp": "^0.2.3",
+    "tmp-promise": "^3.0.3",
     "yaml": "^2.6.0",
-    "@llumiverse/core": "0.15.0",
-    "@vertesia/client": "0.51.0",
-    "@vertesia/common": "0.51.0"
+    "@vertesia/common": "0.52.0",
+    "@vertesia/client": "0.53.0",
+    "@llumiverse/core": "0.17.0"
   },
   "ts_dual_module": {
     "outDir": "lib",

package/src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts

@@ -1,5 +1,5 @@
-import { ContentObjectStatus, DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { ContentObjectStatus, DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { setupActivity } from "../../dsl/setup/ActivityContext.js";
 import { ActivityParamNotFound, NoDocumentFound } from "../../errors.js";
 interface CreateOrUpdateObjectFromInteractionRunParams {
@@ -19,9 +19,21 @@ interface CreateOrUpdateObjectFromInteractionRunParams {
      */
     update_existing_id?: string,
 
+    /**
+     * The name of the object to use. If not provided, the name will be generated from the interaction result
+     */
     fallback_name?: string, // a name to use if no one was generated by the interaction
 
+    /**
+     * The name of the parent object to use. If not provided, the document will be created at the root level
+     */
     parent?: string, // the parent object id
+
+    /**
+     * The name of the property to use for the text. If not provided, the text will be set to the result of the interaction
+     */
+    update_text_from_property?: string,
+
 }
 
 export interface CreateOrUpdateObjectFromInteractionRun extends DSLActivitySpec<CreateOrUpdateObjectFromInteractionRunParams> {
@@ -80,6 +92,13 @@ export async function createOrUpdateDocumentFromInteractionRun(payload: DSLActiv
         }
     };
 
+    if (params.update_text_from_property) {
+        const text = docPayload.properties[params.update_text_from_property];
+        if (text) {
+            docPayload.text = text;
+        }
+    }
+
     //create or update the document
     let newDoc: boolean = false;
     let doc = undefined;

package/src/activities/chunkDocument.ts

@@ -1,18 +1,12 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { DocPart } from "../utils/chunks.js";
 import { InteractionExecutionParams, executeInteractionFromActivity } from "./executeInteraction.js";
 
 const INT_CHUNK_DOCUMENT = "sys:ChunkDocument"
 
-interface DocPart {
 
-    line_number_start: number
-    line_number_end: number
-    name: string
-    type: string
-
-}
 
 export interface ChunkDocumentResult {
     id: string
@@ -22,9 +16,28 @@ export interface ChunkDocumentResult {
 }
 
 export interface ChunkDocumentParams extends InteractionExecutionParams {
+
+    /**
+     * If true, force chunking even if the document is already chunked
+     */
     force?: boolean;
+
+    /**
+     * The interaction name to use for chunking
+     * If not set, the default interaction will be used
+     */
     interactionName?: string;
+
+    /**
+     * The object type to use for the document parts
+     * If not set, the type of the document will be used
+     */
     docPartType?: string;
+
+    /**
+     * If true, create parts as document objects
+     */
+    createParts?: boolean;
 }
 
 export interface ChunkDocument extends DSLActivitySpec<ChunkDocumentParams> {
@@ -75,48 +88,55 @@ export async function chunkDocument(payload: DSLActivityExecutionPayload<ChunkDo
         return { id: objectId, status: "failed", parts: [], message: "no parts found" }
     }
 
-    const partDocs = await Promise.all(parts.map(async (part, i) => {
 
-        const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
+    /**
+     * Only create parts as document if the flag is set
+     */
+    if (params.createParts) {
+
+        const partDocs = await Promise.all(parts.map(async (part, i) => {
 
-        const location = () => {
-            let location = document.location;
-            if (location.endsWith('/')) {
-                location += document.name + "/" + part.type
+            const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
+
+            const location = () => {
+                let location = document.location;
+                if (location.endsWith('/')) {
+                    location += document.name + "/" + part.type
+                }
+                location += '/' + document.name + "/" + part.type;
+                return location;
             }
-            location += '/' + document.name + "/" + part.type;
-            return location;
+
+            const docPart = await client.objects.create({
+                name: part.name,
+                parent: objectId,
+                text: text,
+                location: location(),
+                properties: {
+                    part_number: i + 1,
+                    etag: document.text_etag,
+                    source_line_start: part.line_number_start,
+                    source_line_end: part.line_number_end,
+                    title: part.name
+                }
+            });
+            return docPart;
+        }));
+
+        //delete previous parts
+        if (document.parts && document.parts.length > 0) {
+            log.info('Deleting previous parts for object ID: ' + objectId, { parts: document.parts });
+            await Promise.all(document.parts.map(async (partId) => {
+                await client.objects.delete(partId);
+            }));
         }
 
-        const docPart = await client.objects.create({
-            name: part.name,
-            parent: objectId,
-            text: text,
-            location: location(),
-            properties: {
-                part_number: i + 1,
-                etag: document.text_etag,
-                source_line_start: part.line_number_start,
-                source_line_end: part.line_number_end,
-                title: part.name
-            }
+        await client.objects.update(objectId, {
+            parts: partDocs.map(p => p.id),
+            parts_etag: document.text_etag
         });
-        return docPart;
-    }));
-
-    //delete previous parts
-    if (document.parts && document.parts.length > 0) {
-        log.info('Deleting previous parts for object ID: ' + objectId, { parts: document.parts });
-        await Promise.all(document.parts.map(async (partId) => {
-            await client.objects.delete(partId);
-        }));
     }
 
-    await client.objects.update(objectId, {
-        parts: partDocs.map(p => p.id),
-        parts_etag: document.text_etag
-    });
-
     log.info(`Object ${objectId} chunking completed`, { parts: document.parts });
 
     return { id: objectId, status: "completed", parts: document.parts }

package/src/activities/createDocumentFromOther.ts

@@ -1,11 +1,11 @@
 import { log } from "@temporalio/activity";
+import { NodeStreamSource } from "@vertesia/client/node";
 import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import fs from 'fs';
 import { pdfExtractPages } from "../conversion/mutool.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { NoDocumentFound } from "../errors.js";
 import { saveBlobToTempFile } from "../utils/blobs.js";
-import { NodeStreamSource } from "../utils/memory.js";
 
 interface CreatePdfDocumentFromSourceParams {
 

package/src/activities/executeInteraction.ts

@@ -1,10 +1,11 @@
+import { ModelOptions } from "@llumiverse/core";
+import { activityInfo, log } from "@temporalio/activity";
 import { VertesiaClient } from "@vertesia/client";
 import { DSLActivityExecutionPayload, DSLActivitySpec, ExecutionRun, ExecutionRunStatus, InteractionExecutionConfiguration, RunSearchPayload } from "@vertesia/common";
-import { activityInfo, log } from "@temporalio/activity";
 import { projectResult } from "../dsl/projections.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { ActivityParamNotFound } from "../errors.js";
 import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
-import { ModelOptions } from "@llumiverse/core";
 
 //Example:
 //@ts-ignore
@@ -83,14 +84,25 @@ export interface InteractionExecutionParams {
     /**
      * Options to control generation
      */
-    model_options?: ModelOptions; 
+    model_options?: ModelOptions;
 }
 
-
+/**
+ * TODO: must be kept in sync with InteractionAsyncExecutionPayload form @vertesia/common
+ * Also see the executeInteractionAsync endpoint on the server for how the client payload is sent to the workflow.
+ * (interaction is translsted to interactionName)
+ */
 export interface ExecuteInteractionParams extends InteractionExecutionParams {
+    //TODO rename to interaction as in InteractionAsyncExecutionPayload
     interactionName: string;
     prompt_data: Record<string, any>;
-    truncate?: Record<string, TruncateSpec>
+    /**
+     * Additional prompt data passed by the workflow configuration. This will be merged with prompt_data if any.
+     * You should use `import: ["static_prompt_data"]` to import the workflow prompt data as static_prompt_data param.
+     * Otherwise the workflow prompt data will be ignored.
+     */
+    static_prompt_data?: Record<string, any>;
+    truncate?: Record<string, TruncateSpec>;
 }
 
 export interface ExecuteInteraction extends DSLActivitySpec<ExecuteInteractionParams> {
@@ -102,7 +114,15 @@ export async function executeInteraction(payload: DSLActivityExecutionPayload<Ex
         client, params
     } = await setupActivity<ExecuteInteractionParams>(payload);
 
-    const { interactionName, prompt_data } = params;
+    const { interactionName, prompt_data, static_prompt_data: wf_prompt_data } = params;
+    if (wf_prompt_data) {
+        Object.assign(prompt_data, wf_prompt_data);
+    }
+
+    if (!interactionName) {
+        log.error("Missing interactionName", { params });
+        throw new ActivityParamNotFound("interactionName", payload.activity);
+    }
 
     if (params.truncate) {
         const truncate = params.truncate;
@@ -166,9 +186,7 @@ export async function executeInteractionFromActivity(client: VertesiaClient, int
 
     const result_schema = params.result_schema;
 
-    if (debug) {
-        log.info(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
-    }
+    log.debug(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
 
     const res = await client.interactions.executeByName(interactionName, {
         config,

package/src/activities/extractDocumentText.ts

@@ -156,13 +156,34 @@ function createResponse(doc: ContentObject, text: string, status: TextExtraction
 }
 
 
-//if file is less than 100KB, check if it looks like text
 function sniffIfText(buf: Buffer) {
-    if (buf.length < 100 * 1024) {
-        const s = buf.toString('utf8');
-        if (s.length > 0) {
-            return true;
+    // If file is too large, don't even try
+    if (buf.length > 500 * 1024) {
+        return false;
+    }
+
+    // Count binary/control characters
+    let binaryCount = 0;
+    const sampleSize = Math.min(buf.length, 1000); // Check first 1000 bytes
+
+    for (let i = 0; i < sampleSize; i++) {
+        // Count control characters (except common whitespace)
+        const byte = buf[i];
+        if ((byte < 32 && ![9, 10, 13].includes(byte)) || byte === 0) {
+            binaryCount++;
         }
     }
-    return false;
-}
+
+    // If more than 10% binary/control chars, probably not text
+    if (binaryCount / sampleSize > 0.1) {
+        return false;
+    }
+
+    // Additional check for valid UTF-8 encoding
+    try {
+        const s = buf.toString('utf8');
+        return s.length > 0 && !s.includes('\uFFFD'); // Replacement character
+    } catch (e) {
+        return false;
+    }
+}

package/src/activities/generateDocumentProperties.ts

@@ -1,10 +1,10 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { TruncateSpec } from "../utils/tokens.js";
 import { InteractionExecutionParams, executeInteractionFromActivity } from "./executeInteraction.js";
 
-const INT_EXTRACT_INFORMATION = "sys:ExtractInformation"
+const INT_EXTRACT_INFORMATION = "sys:ExtractInformation";
 export interface GenerateDocumentPropertiesParams extends InteractionExecutionParams {
     typesHint?: string[];
     /**
@@ -17,10 +17,12 @@ export interface GenerateDocumentPropertiesParams extends InteractionExecutionPa
     use_vision?: boolean;
 }
 export interface GenerateDocumentProperties extends DSLActivitySpec<GenerateDocumentPropertiesParams> {
-    name: 'generateDocumentProperties';
+    name: "generateDocumentProperties";
 }
 
-export async function generateDocumentProperties(payload: DSLActivityExecutionPayload<GenerateDocumentPropertiesParams>) {
+export async function generateDocumentProperties(
+    payload: DSLActivityExecutionPayload<GenerateDocumentPropertiesParams>,
+) {
     const context = await setupActivity<GenerateDocumentPropertiesParams>(payload);
     const { params, client, objectId } = context;
     const interactionName = params.interactionName ?? INT_EXTRACT_INFORMATION;
@@ -32,7 +34,7 @@ export async function generateDocumentProperties(payload: DSLActivityExecutionPa
 
     if (!doc?.text && !params.use_vision && !doc?.content?.type?.startsWith("image/")) {
         log.warn(`Object ${objectId} not found or text is empty`);
-        return { status: "failed", error: "no-text" }
+        return { status: "failed", error: "no-text" };
     }
 
     if (!type || !type.object_schema) {
@@ -50,16 +52,19 @@ export async function generateDocumentProperties(payload: DSLActivityExecutionPa
         }
 
         log.info(`Object ${objectId} is not an image or pdf`);
-        return undefined
-    }
+        return undefined;
+    };
 
     const promptData = {
         content: doc.text ?? undefined,
         image: getImageRef() ?? undefined,
         human_context: project?.configuration?.human_context ?? undefined,
-    }
+    };
 
-    log.info(` Extracting information from object ${objectId} with type ${type.name}`, payload.debug_mode ? { params, } : undefined);
+    log.info(
+        ` Extracting information from object ${objectId} with type ${type.name}`,
+        payload.debug_mode ? { params } : undefined,
+    );
 
     const infoRes = await executeInteractionFromActivity(
         client,
@@ -70,24 +75,40 @@ export async function generateDocumentProperties(payload: DSLActivityExecutionPa
             result_schema: type.object_schema,
         },
         promptData,
-        payload.debug_mode ?? false
+        payload.debug_mode ?? false,
     );
 
+    const getText = () => {
+        if (doc.text) {
+            return undefined;
+        }
+        let text = "";
+        if (infoRes.result.title) {
+            text += infoRes.result.title + "\n";
+        }
+        if (infoRes.result.description) {
+            text += infoRes.result.description;
+        }
+        if (text) {
+            return text;
+        } else {
+            return undefined;
+        }
+    };
+
     log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
     await client.objects.update(doc.id, {
         properties: {
             ...infoRes.result,
-            etag: doc.text_etag
+            etag: doc.text_etag,
         },
-        text: infoRes.result.description ?? undefined,
+        text: getText(),
         generation_run_info: {
             id: infoRes.id,
             date: new Date().toISOString(),
             model: infoRes.modelId,
-        }
+        },
     });
 
-
     return { status: "completed" };
-
-}
+}