@vertesia/workflow 0.42.2

package/src/activities/executeInteraction.ts

@@ -0,0 +1,191 @@
+import { ComposableClient } from "@vertesia/client";
+import { DSLActivityExecutionPayload, DSLActivitySpec, ExecutionRun, ExecutionRunStatus, InteractionExecutionConfiguration, RunSearchPayload } from "@vertesia/common";
+import { activityInfo, log } from "@temporalio/activity";
+import { projectResult } from "../dsl/projections.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
+import { ModelOptions } from "@llumiverse/core";
+
+//Example:
+//@ts-ignore
+const JSON: DSLActivitySpec = {
+    name: 'executeInteraction',
+    import: ["defaultModel", "guidlineId", "docTypeId"],
+    params: {
+        defaultModel: "${model}",
+        interactionName: "GenerateSummary",
+        model: "${defaultModel ?? 'gpt4'}",
+        environment: "13456",
+        max_tokens: 100,
+        temperature: 0.5,
+        tags: ["test"],
+        result_schema: "${docType.object_schema}",
+        prompt_data: {
+            documents: "${documents}",
+            guidline: "${guidline.text}"
+        }
+    },
+    fetch: {
+        documents: {
+            type: "document",
+            query: {
+                id: { $in: "${objectIds}" },
+            },
+            select: "+text",
+        },
+        guidline: {
+            type: "document",
+            limit: 1,
+            query: {
+                id: "${guidlineId}",
+            },
+            select: "+text",
+            on_not_found: "throw"
+        },
+        docType: {
+            type: "document_type",
+            limit: 1,
+            query: {
+                id: "${docTypeId}",
+            },
+            select: "+object_schema",
+        }
+    }
+}
+
+export interface InteractionExecutionParams extends ModelOptions{
+    /**
+     * The environment to use. If not specified the project default environment will be used.
+     * If the latter is not specified an exeption will be thrown.
+     */
+    environment?: string;
+    /**
+     * The model to use. If not specified the project default model will be used.
+     * If the latter is not specified the default model of the environment will be used.
+     * If the latter is not specified an exeption will be thrown.
+     */
+    model?: string;
+
+    /**
+     * Force a JSON schema for the result
+     */
+    result_schema?: any;
+
+    /**
+     * Tags to add to the execution run
+     */
+    tags?: string[];
+
+    /**
+     * Wether or not to include the previous error in the interaction prompt data
+     */
+    include_previous_error?: boolean;
+}
+
+
+export interface ExecuteInteractionParams extends InteractionExecutionParams {
+    interactionName: string;
+    prompt_data: Record<string, any>;
+    truncate?: Record<string, TruncateSpec>
+}
+
+export interface ExecuteInteraction extends DSLActivitySpec<ExecuteInteractionParams> {
+    name: 'executeInteraction';
+}
+
+export async function executeInteraction(payload: DSLActivityExecutionPayload) {
+    const {
+        client, params
+    } = await setupActivity<ExecuteInteractionParams>(payload);
+
+    const { interactionName, prompt_data } = params;
+
+    if (params.truncate) {
+        const truncate = params.truncate;
+        for (const [key, value] of Object.entries(truncate)) {
+            prompt_data[key] = truncByMaxTokens(prompt_data[key], value);
+        }
+    }
+
+    const res = await executeInteractionFromActivity(client, interactionName, params, prompt_data, payload.debug_mode);
+
+    return projectResult(payload, params, res, {
+        runId: res.id,
+        status: res.status,
+        result: res.result,
+    });
+
+}
+
+export async function executeInteractionFromActivity(client: ComposableClient, interactionName: string, params: InteractionExecutionParams, prompt_data: any, debug?: boolean) {
+    const userTags = params.tags;
+    const info = activityInfo();
+    const runId = info.workflowExecution.runId;
+    let tags = ["workflow", `tmpRunId:${runId}`]; //TODO use wf:wfName
+    if (userTags) {
+        tags = tags.concat(userTags);
+    }
+
+    let previousStudioExecutionRun: ExecutionRun | undefined = undefined;
+    if (params.include_previous_error) {
+        //retrieve last failed run if any
+        if (info.attempt > 1) {
+            log.info("Retrying, searching for previous run", { tags: ["tmpRunId:" + runId] });
+            const payload: RunSearchPayload = {
+                query: { tags: ["tmpRunId:" + info.workflowExecution.runId] },
+                limit: 1,
+            };
+            const previousRun = await client.runs.search(payload).then((res) => {
+                log.info("Search results", { results: res });
+                return res ? res[0] ?? undefined : undefined
+            });
+
+            if (previousRun) {
+                log.info("Found previous run", { previousRun });
+                previousStudioExecutionRun = await client.runs.retrieve(previousRun.id);
+            }
+        }
+    }
+    if (debug && previousStudioExecutionRun?.error) {
+        log.info(`Found  previous run error`, { error: previousStudioExecutionRun?.error });
+    }
+
+    const config: InteractionExecutionConfiguration = {
+        environment: params.environment,
+        model: params.model,
+        max_tokens: params.max_tokens,
+        temperature: params.temperature
+    }
+    const data = {
+        ...prompt_data,
+        previous_error: previousStudioExecutionRun?.error,
+    }
+
+    const result_schema = params.result_schema;
+
+    if (debug) {
+        log.info(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
+    }
+
+    const res = await client.interactions.executeByName(interactionName, {
+        config,
+        data,
+        result_schema,
+        tags,
+        stream: false,
+    }).catch((err) => {
+        log.error(`Error executing interaction ${interactionName}`, { err });
+        throw new Error(`Interaction Execution failed ${interactionName}: ${err.message}`);
+    });
+
+    if (debug) {
+        log.info(`Interaction executed ${interactionName}`, res);
+    }
+
+    if (res.error || res.status === ExecutionRunStatus.failed) {
+        log.error(`Error executing interaction ${interactionName}`, { error: res.error });
+        throw new Error(`Interaction Execution failed ${interactionName}: ${res.error}`);
+    }
+
+    return res;
+}

package/src/activities/extractDocumentText.ts

@@ -0,0 +1,174 @@
+import { ContentObject, CreateContentObjectPayload, DSLActivityExecutionPayload, DSLActivitySpec } from '@vertesia/common';
+import { log } from "@temporalio/activity";
+import { mutoolPdfToText } from '../conversion/mutool.js';
+import { manyToMarkdown } from '../conversion/pandoc.js';
+import { trasformPdfToMarkdown } from '../conversion/pdf.js';
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { NoDocumentFound } from '../errors.js';
+import { TextExtractionResult, TextExtractionStatus } from '../result-types.js';
+import { fetchBlobAsBuffer, md5 } from '../utils/blobs.js';
+import { countTokens } from '../utils/tokens.js';
+
+//@ts-ignore
+const JSON: DSLActivitySpec = {
+    name: 'extractDocumentText',
+}
+
+// doesn't have any own param
+export type ExtractDocumentTextParams = never;
+
+export interface ExtractDocumentText extends DSLActivitySpec<ExtractDocumentTextParams> {
+    name: 'extractDocumentText';
+    projection?: never;
+}
+
+export async function extractDocumentText(payload: DSLActivityExecutionPayload): Promise<TextExtractionResult> {
+    const { client, objectId } = await setupActivity(payload);
+
+    const r = await client.objects.find({
+        query: { _id: objectId },
+        limit: 1,
+        select: "+text"
+    })
+    const doc = r[0] as ContentObject;
+    if (!doc) {
+        log.error(`Document ${objectId} not found`);
+        throw new NoDocumentFound(`Document ${objectId} not found`, payload.objectIds);
+    }
+
+    log.info(`Extracting text for object ${doc.id}`);
+
+
+    if (!doc.content?.type || !doc.content?.source) {
+        if (doc.text) {
+            return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text present and no source or type");
+        } else {
+            return createResponse(doc, "", TextExtractionStatus.error, "No source or type found");
+        }
+    }
+
+    //skip if text already extracted and proper etag
+    if (doc.text && doc.text.length > 0 && doc.text_etag === doc.content.etag) {
+        return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text already extracted");
+    }
+
+    let fileBuffer: Buffer;
+    try {
+        fileBuffer = await fetchBlobAsBuffer(client, doc.content.source);
+    } catch (e: any) {
+        log.error(`Error reading file: ${e}`);
+        return createResponse(doc, "", TextExtractionStatus.error, e.message);
+    }
+
+
+    let txt: string;
+
+    switch (doc.content.type) {
+
+        case 'application/pdf':
+            //if pdf is more than 2MB, use mutool
+            if (fileBuffer.length > 2 * 1024 * 1024) {
+                txt = await mutoolPdfToText(fileBuffer);
+            } else {
+                txt = await trasformPdfToMarkdown(fileBuffer);
+            }
+            break;
+
+        case 'text/plain':
+            txt = fileBuffer.toString('utf8')
+            break;
+
+        //docx
+        case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+            txt = await manyToMarkdown(fileBuffer, 'docx');
+            break;
+
+        //html
+        case 'text/html':
+            txt = await manyToMarkdown(fileBuffer, 'html');
+            break;
+
+        //opendocument
+        case 'application/vnd.oasis.opendocument.text':
+            txt = await manyToMarkdown(fileBuffer, 'odt');
+            break;
+
+        //rtf
+        case 'application/rtf':
+            txt = await manyToMarkdown(fileBuffer, 'rtf');
+            break;
+
+        //markdown
+        case 'text/markdown':
+            txt = fileBuffer.toString('utf8');
+            break;
+
+        //csv
+        case 'text/csv':
+            txt = fileBuffer.toString('utf8');
+            break;
+
+        //typescript
+        case 'application/typescript':
+            txt = fileBuffer.toString('utf8');
+            break;
+
+        //javascript
+        case 'application/javascript':
+            txt = fileBuffer.toString('utf8');
+            break;
+
+        //json
+        case 'application/json':
+            txt = fileBuffer.toString('utf8');
+            break;
+
+        default:
+            if (sniffIfText(fileBuffer)) {
+                txt = fileBuffer.toString('utf8'); //TODO: add charset detection
+                break;
+            }
+            return createResponse(doc, doc.text ?? '', TextExtractionStatus.skipped, `Unsupported mime type: ${doc.content.type}`);
+    }
+
+
+    const tokensData = countTokens(txt);
+    const etag = doc.content.etag ?? md5(txt);
+
+    const updateData: CreateContentObjectPayload = {
+        text: txt,
+        text_etag: etag,
+        tokens: {
+            ...tokensData,
+            etag: etag,
+        }
+    }
+
+    await client.objects.update(doc.id, updateData);
+
+    return createResponse(doc, txt, TextExtractionStatus.success);
+}
+
+function createResponse(doc: ContentObject, text: string, status: TextExtractionStatus, message?: string): TextExtractionResult {
+    return {
+        status,
+        message,
+        tokens: doc.tokens,
+        len: text.length,
+        objectId: doc.id,
+        hasText: !!text,
+    }
+
+}
+
+
+//if file is less than 100KB, check if it looks like text
+function sniffIfText(buf: Buffer) {
+    if (buf.length < 100 * 1024) {
+        const s = buf.toString('utf8');
+        if (s.length > 0) {
+            return true;
+        }
+    }
+    return false;
+}

package/src/activities/generateDocumentProperties.ts

@@ -0,0 +1,93 @@
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { log } from "@temporalio/activity";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { TruncateSpec } from "../utils/tokens.js";
+import { InteractionExecutionParams, executeInteractionFromActivity } from "./executeInteraction.js";
+
+const INT_EXTRACT_INFORMATION = "sys:ExtractInformation"
+export interface GenerateDocumentPropertiesParams extends InteractionExecutionParams {
+    typesHint?: string[];
+    /**
+     * truncate the input doc text to the specified max_tokens
+     */
+    truncate?: TruncateSpec;
+
+    interactionName?: string;
+
+    use_vision?: boolean;
+}
+export interface GenerateDocumentProperties extends DSLActivitySpec<GenerateDocumentPropertiesParams> {
+    name: 'generateDocumentProperties';
+}
+
+export async function generateDocumentProperties(payload: DSLActivityExecutionPayload) {
+    const context = await setupActivity<GenerateDocumentPropertiesParams>(payload);
+    const { params, client, objectId } = context;
+    const interactionName = params.interactionName ?? INT_EXTRACT_INFORMATION;
+
+    const project = await context.fetchProject();
+
+    const doc = await client.objects.retrieve(objectId, "+text");
+    const type = doc.type ? await client.types.retrieve(doc.type.id) : undefined;
+
+    if (!doc?.text && !params.use_vision && !doc?.content?.type?.startsWith("image/")) {
+        log.warn(`Object ${objectId} not found or text is empty`);
+        return { status: "failed", error: "no-text" }
+    }
+
+    if (!type || !type.object_schema) {
+        log.info(`Object ${objectId} has no schema`);
+        return { document: objectId, status: "skipped", message: "no schema defined on type" };
+    }
+
+    const getImageRef = () => {
+        if (doc.content?.type?.startsWith("image/")) {
+            return "store:" + doc.id;
+        }
+
+        if (params.use_vision && doc.content?.type?.startsWith("application/pdf")) {
+            return "store:" + doc.id;
+        }
+
+        log.info(`Object ${objectId} is not an image or pdf`);
+        return undefined
+    }
+
+    const promptData = {
+        content: doc.text ?? undefined,
+        image: getImageRef() ?? undefined,
+        human_context: project?.configuration?.human_context ?? undefined,
+    }
+
+    log.info(` Extracting information from object ${objectId} with type ${type.name}`, payload.debug_mode ? { params, } : undefined);
+
+    const infoRes = await executeInteractionFromActivity(
+        client,
+        interactionName,
+        {
+            ...params,
+            include_previous_error: true,
+            result_schema: type.object_schema,
+        },
+        promptData,
+        payload.debug_mode ?? false
+    );
+
+    log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
+    await client.objects.update(doc.id, {
+        properties: {
+            ...infoRes.result,
+            etag: doc.text_etag
+        },
+        text: infoRes.result.description ?? undefined,
+        generation_run_info: {
+            id: infoRes.id,
+            date: new Date().toISOString(),
+            model: infoRes.modelId,
+        }
+    });
+
+
+    return { status: "completed" };
+
+}