@vertesia/workflow 0.24.0-dev.202601221707

package/src/activities/createDocumentFromOther.ts

@@ -0,0 +1,92 @@
+import { log } from "@temporalio/activity";
+import { NodeStreamSource } from "@vertesia/client/node";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import fs from 'fs';
+import { pdfExtractPages } from "../conversion/mutool.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { DocumentNotFoundError } from "../errors.js";
+import { saveBlobToTempFile } from "../utils/blobs.js";
+
+interface CreatePdfDocumentFromSourceParams {
+
+    target_object_type: string; //type of the object to create
+    title: string; //title of the object to create
+    filename?: string; //filename of the object to create
+    pages: number[]; //pages to extract into the new document
+    parent?: string; //set the new document as child of the source document
+
+}
+
+
+export interface CreatePdfDocumentFromSource extends DSLActivitySpec<CreatePdfDocumentFromSourceParams> {
+    name: 'createPdfDocumentFromSource';
+}
+
+
+/**
+ * Create a new PDF by extracting pages from a source PDF
+ * @returns
+ */
+export async function createPdfDocumentFromSource(payload: DSLActivityExecutionPayload<CreatePdfDocumentFromSourceParams>) {
+    const { client, objectId, params } = await setupActivity<CreatePdfDocumentFromSourceParams>(payload);
+    const inputObject = await client.objects.retrieve(objectId);
+
+    const { pages, filename, title } = params;
+    log.info(`Creating PDF from source`, { objectId, pages, filename, title });
+
+    if (!pages || pages.length === 0) {
+        log.error(`No pages provided`);
+        throw new Error(`No pages provided`);
+    }
+
+    if (!inputObject) {
+        log.error(`Document ${objectId} not found`);
+        throw new DocumentNotFoundError(`Document ${objectId} not found`, [objectId]);
+    }
+
+    if (!inputObject.content?.source) {
+        log.error(`Document ${objectId} has no source`);
+        throw new DocumentNotFoundError(`Document ${objectId} has no source`, [objectId]);
+    }
+
+    if (!inputObject.content.type || (!inputObject.content.type?.startsWith('application/pdf'))) {
+        log.error(`Document ${objectId} is not an image`);
+        throw new DocumentNotFoundError(`Document ${objectId} is not an image or pdf: ${inputObject.content.type}`, [objectId]);
+    }
+
+    const targetType = await client.types.getTypeByName(params.target_object_type);
+    if (!targetType) {
+        log.error(`Type ${params.target_object_type} not found`);
+        throw new DocumentNotFoundError(`Type ${params.target_object_type} not found`);
+    }
+
+    const tmpFile = await saveBlobToTempFile(client, inputObject.content.source, ".pdf");
+    const newPdf = await pdfExtractPages(tmpFile, pages);
+    log.info(`PDF created from pages ${pages.join(', ')} `, { newPdf });
+    const name = `pages-${pages.join('-')}.pdf`;
+
+    const sourceToUpload = new NodeStreamSource(
+        fs.createReadStream(newPdf),
+        name,
+        "application/pdf"
+    )
+
+    log.info(`Uploading file ${newPdf} `);
+    const upload = await client.objects.upload(sourceToUpload);
+    log.info(`File uploaded ${upload.source} `);
+
+    const newObject = await client.objects.create({
+        type: targetType.id,
+        name: title || targetType.name,
+        parent: objectId,
+        content: {
+            source: upload.source,
+            name: upload.name,
+            type: 'application/pdf'
+        }
+    });
+
+    return { newObjectId: newObject.id, uploadedFile: upload.name };
+
+
+}

package/src/activities/executeInteraction.ts

@@ -0,0 +1,300 @@
+import { CompletionResult, ModelOptions } from "@llumiverse/common";
+import { activityInfo, log } from "@temporalio/activity";
+import { VertesiaClient } from "@vertesia/client";
+import { NodeStreamSource } from "@vertesia/client/node";
+import {
+    DSLActivityExecutionPayload,
+    DSLActivitySpec,
+    ExecutionRun,
+    ExecutionRunStatus,
+    ExecutionRunWorkflow,
+    InteractionExecutionConfiguration,
+    RunSearchPayload,
+} from "@vertesia/common";
+import { projectResult } from "../dsl/projections.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { ActivityParamInvalidError, ActivityParamNotFoundError, ResourceExhaustedError } from "../errors.js";
+import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
+import { Readable } from "stream";
+
+//Example:
+//@ts-ignore
+const JSON: DSLActivitySpec = {
+    name: "executeInteraction",
+    import: ["defaultModel", "guidlineId", "docTypeId"],
+    params: {
+        defaultModel: "${model}",
+        interactionName: "GenerateSummary",
+        model: "${defaultModel ?? 'gpt4'}",
+        environment: "13456",
+        max_tokens: 100,
+        temperature: 0.5,
+        tags: ["test"],
+        result_schema: "${docType.object_schema}",
+        prompt_data: {
+            documents: "${documents}",
+            guidline: "${guidline.text}",
+        },
+    },
+    fetch: {
+        documents: {
+            type: "document",
+            query: {
+                id: { $in: "${objectIds}" },
+            },
+            select: "+text",
+        },
+        guidline: {
+            type: "document",
+            limit: 1,
+            query: {
+                id: "${guidlineId}",
+            },
+            select: "+text",
+            on_not_found: "throw",
+        },
+        docType: {
+            type: "document_type",
+            limit: 1,
+            query: {
+                id: "${docTypeId}",
+            },
+            select: "+object_schema",
+        },
+    },
+};
+export interface InteractionExecutionParams {
+    /**
+     * The environment to use. If not specified the project default environment will be used.
+     * If the latter is not specified an exception will be thrown.
+     */
+    environment?: string;
+    /**
+     * The model to use. If not specified the project default model will be used.
+     * If the latter is not specified the default model of the environment will be used.
+     * If the latter is not specified an exception will be thrown.
+     */
+    model?: string;
+
+    /**
+     * Request a JSON schema for the result
+     */
+    result_schema?: any;
+
+    /** Wether to validate the result against the schema */
+    validate_result?: boolean;
+
+    /**
+     * Tags to add to the execution run
+     */
+    tags?: string[];
+
+    /**
+     * Wether or not to include the previous error in the interaction prompt data
+     */
+    include_previous_error?: boolean;
+
+    /**
+     * Options to control generation
+     */
+    model_options?: ModelOptions;
+
+    /**
+     * activity won't be retried if it fails due to resource exhaustion (429)
+     */
+    exit_on_resource_exhaustion?: boolean;
+}
+
+/**
+ * TODO: must be kept in sync with InteractionAsyncExecutionPayload form @vertesia/common
+ * Also see the executeInteractionAsync endpoint on the server for how the client payload is sent to the workflow.
+ * (interaction is translated to interactionName)
+ */
+export interface ExecuteInteractionParams extends InteractionExecutionParams {
+    //TODO rename to interaction as in InteractionAsyncExecutionPayload
+    interactionName: string;
+    prompt_data: Record<string, any>;
+    /**
+     * Additional prompt data passed by the workflow configuration. This will be merged with prompt_data if any.
+     * You should use `import: ["static_prompt_data"]` to import the workflow prompt data as static_prompt_data param.
+     * Otherwise the workflow prompt data will be ignored.
+     */
+    static_prompt_data?: Record<string, any>;
+    truncate?: Record<string, TruncateSpec>;
+}
+
+export interface ExecuteInteraction extends DSLActivitySpec<ExecuteInteractionParams> {
+    name: "executeInteraction";
+}
+
+export async function executeInteraction(payload: DSLActivityExecutionPayload<ExecuteInteractionParams>) {
+    const { client, params } = await setupActivity<ExecuteInteractionParams>(payload);
+
+    const { interactionName, prompt_data, static_prompt_data: wf_prompt_data } = params;
+    if (wf_prompt_data) {
+        Object.assign(prompt_data, wf_prompt_data);
+    }
+
+    if (!interactionName) {
+        log.error("Missing interactionName", { params });
+        throw new ActivityParamNotFoundError("interactionName", payload.activity);
+    }
+
+    if (params.truncate) {
+        const truncate = params.truncate;
+        for (const [key, value] of Object.entries(truncate)) {
+            prompt_data[key] = truncByMaxTokens(prompt_data[key], value);
+        }
+    }
+
+    try {
+        const res = await executeInteractionFromActivity(
+            client,
+            interactionName,
+            params,
+            prompt_data,
+            payload.debug_mode,
+        );
+
+        let completionResult: CompletionResult[] = res.result;
+
+        // Handle image uploads if the result contains base64 images
+        const imageResults = completionResult.filter(r => r.type === "image");
+        if (imageResults.length > 0) {
+            const uploadedImages = await Promise.all(
+                completionResult.map(async (item, index) => {
+                    if (item.type === "image") {
+                        const image = item.value;
+                        // Extract base64 data and create buffer
+                        const base64Data = image.replace(/^data:image\/[a-z]+;base64,/, "");
+                        const buffer = Buffer.from(base64Data, 'base64');
+
+                        // Generate filename
+                        const { runId } = activityInfo().workflowExecution;
+                        const { activityId } = activityInfo();
+                        const filename = `generated-image-${runId}-${activityId}-${index}.png`;
+
+                        // Create a readable stream from the buffer
+                        const stream = Readable.from(buffer);
+
+                        const source = new NodeStreamSource(
+                            stream,
+                            filename,
+                            "image/png",
+                        );
+
+                        const file = await client.files.uploadFile(source);
+                        return { type: "image", value: file } as CompletionResult;
+                    }
+                    return item;
+                })
+            );
+            completionResult = uploadedImages;
+        }
+
+        return projectResult(payload, params, res, {
+            runId: res.id,
+            status: res.status,
+            result: completionResult,
+        });
+
+    } catch (error: any) {
+        log.error(`Failed to execute interaction ${interactionName}`, { error });
+        if (error.statusCode === 429 && params.exit_on_resource_exhaustion) {
+            throw new ResourceExhaustedError(error.statusCode, "Resource exhausted - rate limit exceeded");
+        } else if (error.message.includes("Failed to validate merged prompt schema")) {
+            //issue with the input data, don't retry
+            throw new ActivityParamInvalidError("prompt_data", payload.activity, error.message);
+        } else if (error.message.includes("modelId: Path `modelId` is required")) {
+            //issue with the input data, don't retry
+            throw new ActivityParamInvalidError("model", payload.activity, error.message);
+        } else {
+            throw new Error(`Interaction Execution failed ${interactionName}: ${error.message}`);
+        }
+    }
+}
+
+export async function executeInteractionFromActivity(
+    client: VertesiaClient,
+    interactionName: string,
+    params: InteractionExecutionParams,
+    prompt_data: any,
+    debug?: boolean,
+) {
+    const userTags = params.tags;
+    const info = activityInfo();
+    const runId = info.workflowExecution.runId;
+    let tags = ["workflow"];
+    if (userTags) {
+        tags = tags.concat(userTags);
+    }
+    const workflow: ExecutionRunWorkflow = {
+        run_id: info.workflowExecution.runId,
+        workflow_id: info.workflowExecution.workflowId,
+        activity_type: info.activityType,
+    };
+
+    let previousStudioExecutionRun: ExecutionRun | undefined = undefined;
+    if (params.include_previous_error) {
+        //retrieve last failed run if any
+        if (info.attempt > 1) {
+            log.info("Retrying, searching for previous run", { prev_run_id: runId });
+            const payload: RunSearchPayload = {
+                query: { workflow_run_ids: [runId] },
+                limit: 1,
+            };
+            const previousRun = await client.runs.search(payload).then((res) => {
+                log.info("Search results", { results: res });
+                return res ? (res[0] ?? undefined) : undefined;
+            });
+
+            if (previousRun) {
+                log.info("Found previous run", { previousRun });
+                previousStudioExecutionRun = await client.runs.retrieve(previousRun.id);
+            }
+        }
+    }
+    if (debug && previousStudioExecutionRun?.error) {
+        log.info(`Found  previous run error`, { error: previousStudioExecutionRun?.error });
+    }
+
+    const config: InteractionExecutionConfiguration = {
+        environment: params.environment,
+        model: params.model,
+        model_options: params.model_options,
+        do_validate: params.validate_result,
+    };
+    const data = {
+        ...prompt_data,
+        previous_error: previousStudioExecutionRun?.error,
+    };
+
+    const result_schema = params.result_schema;
+
+    log.debug(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags, workflow });
+
+    const res = await client.interactions
+        .executeByName(interactionName, {
+            config,
+            data,
+            result_schema,
+            tags,
+            stream: false,
+            workflow,
+        })
+        .catch((err) => {
+            log.error(`Error executing interaction ${interactionName}`, { err });
+            throw err;
+        });
+
+    if (debug) {
+        log.info(`Interaction executed ${interactionName}`, res);
+    }
+
+    if (res.error || res.status === ExecutionRunStatus.failed) {
+        log.error(`Error executing interaction ${interactionName}`, { error: res.error });
+        throw new Error(`Interaction Execution failed ${interactionName}: ${res.error}`);
+    }
+
+    return res;
+}

package/src/activities/extractDocumentText.ts

@@ -0,0 +1,205 @@
+import { log } from "@temporalio/activity";
+import {
+    ContentObject,
+    CreateContentObjectPayload,
+    DSLActivityExecutionPayload,
+    DSLActivitySpec,
+} from "@vertesia/common";
+import { markdownWithMarkitdown } from "../conversion/markitdown.js";
+import { mutoolPdfToText } from "../conversion/mutool.js";
+import { markdownWithPandoc } from "../conversion/pandoc.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { DocumentNotFoundError } from "../errors.js";
+import { TextExtractionResult, TextExtractionStatus } from "../result-types.js";
+import { fetchBlobAsBuffer, md5 } from "../utils/blobs.js";
+import { countTokens } from "../utils/tokens.js";
+
+//@ts-ignore
+const JSON: DSLActivitySpec = {
+    name: "extractDocumentText",
+};
+
+// doesn't have any own param
+export interface ExtractDocumentTextParams {}
+export interface ExtractDocumentText extends DSLActivitySpec<ExtractDocumentTextParams> {
+    name: "extractDocumentText";
+    projection?: never;
+}
+
+export async function extractDocumentText(
+    payload: DSLActivityExecutionPayload<ExtractDocumentTextParams>,
+): Promise<TextExtractionResult> {
+    const { client, objectId } = await setupActivity(payload);
+
+    const r = await client.objects.find({
+        query: { _id: objectId },
+        limit: 1,
+        select: "+text",
+    });
+    const doc = r[0] as ContentObject;
+    if (!doc) {
+        log.error(`Document ${objectId} not found`);
+        throw new DocumentNotFoundError(`Document ${objectId} not found`, payload.objectIds);
+    }
+
+    log.info(`Extracting text for object ${doc.id}`);
+
+    if (!doc.content?.type || !doc.content?.source) {
+        if (doc.text) {
+            return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text present and no source or type");
+        } else {
+            return createResponse(doc, "", TextExtractionStatus.error, "No source or type found");
+        }
+    }
+
+    //skip if text already extracted and proper etag
+    if (doc.text && doc.text.length > 0 && doc.text_etag === doc.content.etag) {
+        return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text already extracted");
+    }
+
+    let fileBuffer: Buffer;
+    try {
+        fileBuffer = await fetchBlobAsBuffer(client, doc.content.source);
+    } catch (e: any) {
+        log.error(`Error reading file: ${e}`);
+        return createResponse(doc, "", TextExtractionStatus.error, e.message);
+    }
+
+    let txt: string;
+
+    switch (doc.content.type) {
+        case "application/pdf":
+            txt = await mutoolPdfToText(fileBuffer);
+            break;
+
+        case "text/plain":
+            txt = fileBuffer.toString("utf8");
+            break;
+
+        //docx
+        case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+            txt = await markdownWithMarkitdown(fileBuffer, "docx");
+            break;
+
+        //pptx
+        case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
+            txt = await markdownWithMarkitdown(fileBuffer, "pptx");
+            break;
+
+        //html
+        case "text/html":
+            txt = await markdownWithPandoc(fileBuffer, "html");
+            break;
+
+        //opendocument
+        case "application/vnd.oasis.opendocument.text":
+            txt = await markdownWithPandoc(fileBuffer, "odt");
+            break;
+
+        //rtf
+        case "application/rtf":
+            txt = await markdownWithPandoc(fileBuffer, "rtf");
+            break;
+
+        //markdown
+        case "text/markdown":
+            txt = fileBuffer.toString("utf8");
+            break;
+
+        //csv
+        case "text/csv":
+            txt = fileBuffer.toString("utf8");
+            break;
+
+        //typescript
+        case "application/typescript":
+            txt = fileBuffer.toString("utf8");
+            break;
+
+        //javascript
+        case "application/javascript":
+            txt = fileBuffer.toString("utf8");
+            break;
+
+        //json
+        case "application/json":
+            txt = fileBuffer.toString("utf8");
+            break;
+
+        default:
+            if (sniffIfText(fileBuffer)) {
+                txt = fileBuffer.toString("utf8"); //TODO: add charset detection
+                break;
+            }
+            return createResponse(
+                doc,
+                doc.text ?? "",
+                TextExtractionStatus.skipped,
+                `Unsupported mime type: ${doc.content.type}`,
+            );
+    }
+
+    const tokensData = countTokens(txt);
+    const etag = doc.content.etag ?? md5(txt);
+
+    const updateData: CreateContentObjectPayload = {
+        text: txt,
+        text_etag: etag,
+        tokens: {
+            ...tokensData,
+            etag: etag,
+        },
+    };
+
+    await client.objects.update(doc.id, updateData);
+
+    return createResponse(doc, txt, TextExtractionStatus.success);
+}
+
+function createResponse(
+    doc: ContentObject,
+    text: string,
+    status: TextExtractionStatus,
+    message?: string,
+): TextExtractionResult {
+    return {
+        status,
+        message,
+        tokens: doc.tokens,
+        len: text.length,
+        objectId: doc.id,
+        hasText: !!text,
+    };
+}
+
+function sniffIfText(buf: Buffer) {
+    // If file is too large, don't even try
+    if (buf.length > 500 * 1024) {
+        return false;
+    }
+
+    // Count binary/control characters
+    let binaryCount = 0;
+    const sampleSize = Math.min(buf.length, 1000); // Check first 1000 bytes
+
+    for (let i = 0; i < sampleSize; i++) {
+        // Count control characters (except common whitespace)
+        const byte = buf[i];
+        if ((byte < 32 && ![9, 10, 13].includes(byte)) || byte === 0) {
+            binaryCount++;
+        }
+    }
+
+    // If more than 10% binary/control chars, probably not text
+    if (binaryCount / sampleSize > 0.1) {
+        return false;
+    }
+
+    // Additional check for valid UTF-8 encoding
+    try {
+        const s = buf.toString("utf8");
+        return s.length > 0 && !s.includes("\uFFFD"); // Replacement character
+    } catch (e) {
+        return false;
+    }
+}