@vertesia/workflow 0.54.0 → 0.55.0

package/lib/esm/activities/executeInteraction.js

@@ -0,0 +1,150 @@
+import { activityInfo, log } from "@temporalio/activity";
+import { ExecutionRunStatus, } from "@vertesia/common";
+import { projectResult } from "../dsl/projections.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { ActivityParamInvalid, ActivityParamNotFound } from "../errors.js";
+import { truncByMaxTokens } from "../utils/tokens.js";
+//Example:
+//@ts-ignore
+const JSON = {
+    name: "executeInteraction",
+    import: ["defaultModel", "guidlineId", "docTypeId"],
+    params: {
+        defaultModel: "${model}",
+        interactionName: "GenerateSummary",
+        model: "${defaultModel ?? 'gpt4'}",
+        environment: "13456",
+        max_tokens: 100,
+        temperature: 0.5,
+        tags: ["test"],
+        result_schema: "${docType.object_schema}",
+        prompt_data: {
+            documents: "${documents}",
+            guidline: "${guidline.text}",
+        },
+    },
+    fetch: {
+        documents: {
+            type: "document",
+            query: {
+                id: { $in: "${objectIds}" },
+            },
+            select: "+text",
+        },
+        guidline: {
+            type: "document",
+            limit: 1,
+            query: {
+                id: "${guidlineId}",
+            },
+            select: "+text",
+            on_not_found: "throw",
+        },
+        docType: {
+            type: "document_type",
+            limit: 1,
+            query: {
+                id: "${docTypeId}",
+            },
+            select: "+object_schema",
+        },
+    },
+};
+export async function executeInteraction(payload) {
+    const { client, params } = await setupActivity(payload);
+    const { interactionName, prompt_data, static_prompt_data: wf_prompt_data } = params;
+    if (wf_prompt_data) {
+        Object.assign(prompt_data, wf_prompt_data);
+    }
+    if (!interactionName) {
+        log.error("Missing interactionName", { params });
+        throw new ActivityParamNotFound("interactionName", payload.activity);
+    }
+    if (params.truncate) {
+        const truncate = params.truncate;
+        for (const [key, value] of Object.entries(truncate)) {
+            prompt_data[key] = truncByMaxTokens(prompt_data[key], value);
+        }
+    }
+    try {
+        const res = await executeInteractionFromActivity(client, interactionName, params, prompt_data, payload.debug_mode);
+        return projectResult(payload, params, res, {
+            runId: res.id,
+            status: res.status,
+            result: res.result,
+        });
+    }
+    catch (error) {
+        log.error("Failed to execute interaction", { error });
+        if (error.message.includes("Failed to validate merged prompt schema")) {
+            //issue with the input data, don't retry
+            throw new ActivityParamInvalid("prompt_data", payload.activity, error.message);
+        }
+        else {
+            throw error;
+        }
+    }
+}
+export async function executeInteractionFromActivity(client, interactionName, params, prompt_data, debug) {
+    const userTags = params.tags;
+    const info = activityInfo();
+    const runId = info.workflowExecution.runId;
+    let tags = ["workflow", `tmpRunId:${runId}`]; //TODO use wf:wfName
+    if (userTags) {
+        tags = tags.concat(userTags);
+    }
+    let previousStudioExecutionRun = undefined;
+    if (params.include_previous_error) {
+        //retrieve last failed run if any
+        if (info.attempt > 1) {
+            log.info("Retrying, searching for previous run", { tags: ["tmpRunId:" + runId] });
+            const payload = {
+                query: { tags: ["tmpRunId:" + info.workflowExecution.runId] },
+                limit: 1,
+            };
+            const previousRun = await client.runs.search(payload).then((res) => {
+                log.info("Search results", { results: res });
+                return res ? (res[0] ?? undefined) : undefined;
+            });
+            if (previousRun) {
+                log.info("Found previous run", { previousRun });
+                previousStudioExecutionRun = await client.runs.retrieve(previousRun.id);
+            }
+        }
+    }
+    if (debug && previousStudioExecutionRun?.error) {
+        log.info(`Found  previous run error`, { error: previousStudioExecutionRun?.error });
+    }
+    const config = {
+        environment: params.environment,
+        model: params.model,
+        model_options: params.model_options,
+    };
+    const data = {
+        ...prompt_data,
+        previous_error: previousStudioExecutionRun?.error,
+    };
+    const result_schema = params.result_schema;
+    log.debug(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
+    const res = await client.interactions
+        .executeByName(interactionName, {
+        config,
+        data,
+        result_schema,
+        tags,
+        stream: false,
+    })
+        .catch((err) => {
+        log.error(`Error executing interaction ${interactionName}`, { err });
+        throw new Error(`Interaction Execution failed ${interactionName}: ${err.message}`);
+    });
+    if (debug) {
+        log.info(`Interaction executed ${interactionName}`, res);
+    }
+    if (res.error || res.status === ExecutionRunStatus.failed) {
+        log.error(`Error executing interaction ${interactionName}`, { error: res.error });
+        throw new Error(`Interaction Execution failed ${interactionName}: ${res.error}`);
+    }
+    return res;
+}
+//# sourceMappingURL=executeInteraction.js.map

package/lib/esm/activities/executeInteraction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"executeInteraction.js","sourceRoot":"","sources":["../../../src/activities/executeInteraction.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAIH,kBAAkB,GAGrB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC3E,OAAO,EAAgB,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEpE,UAAU;AACV,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,oBAAoB;IAC1B,MAAM,EAAE,CAAC,cAAc,EAAE,YAAY,EAAE,WAAW,CAAC;IACnD,MAAM,EAAE;QACJ,YAAY,EAAE,UAAU;QACxB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,2BAA2B;QAClC,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,GAAG;QAChB,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,aAAa,EAAE,0BAA0B;QACzC,WAAW,EAAE;YACT,SAAS,EAAE,cAAc;YACzB,QAAQ,EAAE,kBAAkB;SAC/B;KACJ;IACD,KAAK,EAAE;QACH,SAAS,EAAE;YACP,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE;gBACH,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aAC9B;YACD,MAAM,EAAE,OAAO;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,eAAe;aACtB;YACD,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,OAAO;SACxB;QACD,OAAO,EAAE;YACL,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,cAAc;aACrB;YACD,MAAM,EAAE,gBAAgB;SAC3B;KACJ;CACJ,CAAC;AAyDF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAA8D;IACnG,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAA2B,OAAO,CAAC,CAAC;IAElF,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,kBAAkB,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACpF,IAAI,cAAc,EAAE,CAAC;QACjB,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACnB,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACjD,MAAM,IAAI,qBAAqB,CAAC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAC5C,MAAM,EACN,eAAe,EACf,MAAM,EACN,WAAW,EACX,OAAO,CAAC,UAAU,CACrB,CAAC;QACF,OAAO,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE;YACvC,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;SACrB,CAAC,CAAC;IACP,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QAClB,GAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACtD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAAE,CAAC;YACpE,wCAAwC;YACxC,MAAM,IAAI,oBAAoB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACnF,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAChD,MAAsB,EACtB,eAAuB,EACvB,MAAkC,EAClC,WAAgB,EAChB,KAAe;IAEf,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,YAAY,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,YAAY,KAAK,EAAE,CAAC,CAAC,CAAC,oBAAoB;IAClE,IAAI,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,0BAA0B,GAA6B,SAAS,CAAC;IACrE,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAChC,iCAAiC;QACjC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;YAClF,MAAM,OAAO,GAAqB;gBAC9B,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE;gBAC7D,KAAK,EAAE,CAAC;aACX,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/D,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBACd,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChD,0BAA0B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,KAAK,IAAI,0BAA0B,EAAE,KAAK,EAAE,CAAC;QAC7C,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,MAAM,GAAsC;QAC9C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,aAAa,EAAE,MAAM,CAAC,aAAa;KACtC,CAAC;IACF,MAAM,IAAI,GAAG;QACT,GAAG,WAAW;QACd,cAAc,EAAE,0BAA0B,EAAE,KAAK;KACpD,CAAC;IAEF,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAE3C,GAAG,CAAC,KAAK,CAAC,gCAAgC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAEpG,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY;SAChC,aAAa,CAAC,eAAe,EAAE;QAC5B,MAAM;QACN,IAAI;QACJ,aAAa;QACb,IAAI;QACJ,MAAM,EAAE,KAAK;KAChB,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,GAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEP,IAAI,KAAK,EAAE,CAAC;QACR,GAAG,CAAC,IAAI,CAAC,wBAAwB,eAAe,EAAE,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,kBAAkB,CAAC,MAAM,EAAE,CAAC;QACxD,GAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}

package/lib/esm/activities/extractDocumentText.js

@@ -0,0 +1,153 @@
+import { log } from "@temporalio/activity";
+import { mutoolPdfToText } from "../conversion/mutool.js";
+import { markdownWithPandoc } from "../conversion/pandoc.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { NoDocumentFound } from "../errors.js";
+import { TextExtractionStatus } from "../result-types.js";
+import { fetchBlobAsBuffer, md5 } from "../utils/blobs.js";
+import { countTokens } from "../utils/tokens.js";
+import { markdownWithMarkitdown } from "../conversion/markitdown.js";
+//@ts-ignore
+const JSON = {
+    name: "extractDocumentText",
+};
+export async function extractDocumentText(payload) {
+    const { client, objectId } = await setupActivity(payload);
+    const r = await client.objects.find({
+        query: { _id: objectId },
+        limit: 1,
+        select: "+text",
+    });
+    const doc = r[0];
+    if (!doc) {
+        log.error(`Document ${objectId} not found`);
+        throw new NoDocumentFound(`Document ${objectId} not found`, payload.objectIds);
+    }
+    log.info(`Extracting text for object ${doc.id}`);
+    if (!doc.content?.type || !doc.content?.source) {
+        if (doc.text) {
+            return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text present and no source or type");
+        }
+        else {
+            return createResponse(doc, "", TextExtractionStatus.error, "No source or type found");
+        }
+    }
+    //skip if text already extracted and proper etag
+    if (doc.text && doc.text.length > 0 && doc.text_etag === doc.content.etag) {
+        return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text already extracted");
+    }
+    let fileBuffer;
+    try {
+        fileBuffer = await fetchBlobAsBuffer(client, doc.content.source);
+    }
+    catch (e) {
+        log.error(`Error reading file: ${e}`);
+        return createResponse(doc, "", TextExtractionStatus.error, e.message);
+    }
+    let txt;
+    switch (doc.content.type) {
+        case "application/pdf":
+            txt = await mutoolPdfToText(fileBuffer);
+            break;
+        case "text/plain":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //docx
+        case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+            txt = await markdownWithMarkitdown(fileBuffer, "docx");
+            break;
+        //pptx
+        case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
+            txt = await markdownWithMarkitdown(fileBuffer, "pptx");
+            break;
+        //html
+        case "text/html":
+            txt = await markdownWithPandoc(fileBuffer, "html");
+            break;
+        //opendocument
+        case "application/vnd.oasis.opendocument.text":
+            txt = await markdownWithPandoc(fileBuffer, "odt");
+            break;
+        //rtf
+        case "application/rtf":
+            txt = await markdownWithPandoc(fileBuffer, "rtf");
+            break;
+        //markdown
+        case "text/markdown":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //csv
+        case "text/csv":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //typescript
+        case "application/typescript":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //javascript
+        case "application/javascript":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //json
+        case "application/json":
+            txt = fileBuffer.toString("utf8");
+            break;
+        default:
+            if (sniffIfText(fileBuffer)) {
+                txt = fileBuffer.toString("utf8"); //TODO: add charset detection
+                break;
+            }
+            return createResponse(doc, doc.text ?? "", TextExtractionStatus.skipped, `Unsupported mime type: ${doc.content.type}`);
+    }
+    const tokensData = countTokens(txt);
+    const etag = doc.content.etag ?? md5(txt);
+    const updateData = {
+        text: txt,
+        text_etag: etag,
+        tokens: {
+            ...tokensData,
+            etag: etag,
+        },
+    };
+    await client.objects.update(doc.id, updateData);
+    return createResponse(doc, txt, TextExtractionStatus.success);
+}
+function createResponse(doc, text, status, message) {
+    return {
+        status,
+        message,
+        tokens: doc.tokens,
+        len: text.length,
+        objectId: doc.id,
+        hasText: !!text,
+    };
+}
+function sniffIfText(buf) {
+    // If file is too large, don't even try
+    if (buf.length > 500 * 1024) {
+        return false;
+    }
+    // Count binary/control characters
+    let binaryCount = 0;
+    const sampleSize = Math.min(buf.length, 1000); // Check first 1000 bytes
+    for (let i = 0; i < sampleSize; i++) {
+        // Count control characters (except common whitespace)
+        const byte = buf[i];
+        if ((byte < 32 && ![9, 10, 13].includes(byte)) || byte === 0) {
+            binaryCount++;
+        }
+    }
+    // If more than 10% binary/control chars, probably not text
+    if (binaryCount / sampleSize > 0.1) {
+        return false;
+    }
+    // Additional check for valid UTF-8 encoding
+    try {
+        const s = buf.toString("utf8");
+        return s.length > 0 && !s.includes("\uFFFD"); // Replacement character
+    }
+    catch (e) {
+        return false;
+    }
+}
+//# sourceMappingURL=extractDocumentText.js.map

package/lib/esm/activities/extractDocumentText.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractDocumentText.js","sourceRoot":"","sources":["../../../src/activities/extractDocumentText.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AAO3C,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAwB,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,iBAAiB,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAErE,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,qBAAqB;CAC9B,CAAC;AASF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA+D;IAE/D,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;IAE1D,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,KAAK,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE;QACxB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,OAAO;KAClB,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAkB,CAAC;IAClC,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,eAAe,CAAC,YAAY,QAAQ,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IACnF,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,8BAA8B,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC7C,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,oBAAoB,CAAC,OAAO,EAAE,oCAAoC,CAAC,CAAC;QAC7G,CAAC;aAAM,CAAC;YACJ,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,oBAAoB,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;QAC1F,CAAC;IACL,CAAC;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACxE,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,oBAAoB,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACD,UAAU,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;QACtC,OAAO,cAAc,CAAC,GAAG,EAAE,EAAE,EAAE,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,GAAW,CAAC;IAEhB,QAAQ,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,iBAAiB;YAClB,GAAG,GAAG,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM;QAEV,KAAK,YAAY;YACb,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,MAAM;QACN,KAAK,yEAAyE;YAC1E,GAAG,GAAG,MAAM,sBAAsB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM;QAEV,MAAM;QACN,KAAK,2EAA2E;YAC5E,GAAG,GAAG,MAAM,sBAAsB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACvD,MAAM;QAEV,MAAM;QACN,KAAK,WAAW;YACZ,GAAG,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACnD,MAAM;QAEV,cAAc;QACd,KAAK,yCAAyC;YAC1C,GAAG,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAClD,MAAM;QAEV,KAAK;QACL,KAAK,iBAAiB;YAClB,GAAG,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAClD,MAAM;QAEV,UAAU;QACV,KAAK,eAAe;YAChB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,KAAK;QACL,KAAK,UAAU;YACX,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,YAAY;QACZ,KAAK,wBAAwB;YACzB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV,MAAM;QACN,KAAK,kBAAkB;YACnB,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM;QAEV;YACI,IAAI,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1B,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,6BAA6B;gBAChE,MAAM;YACV,CAAC;YACD,OAAO,cAAc,CACjB,GAAG,EACH,GAAG,CAAC,IAAI,IAAI,EAAE,EACd,oBAAoB,CAAC,OAAO,EAC5B,0BAA0B,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAC/C,CAAC;IACV,CAAC;IAED,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAE1C,MAAM,UAAU,GAA+B;QAC3C,IAAI,EAAE,GAAG;QACT,SAAS,EAAE,IAAI;QACf,MAAM,EAAE;YACJ,GAAG,UAAU;YACb,IAAI,EAAE,IAAI;SACb;KACJ,CAAC;IAEF,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAEhD,OAAO,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,cAAc,CACnB,GAAkB,EAClB,IAAY,EACZ,MAA4B,EAC5B,OAAgB;IAEhB,OAAO;QACH,MAAM;QACN,OAAO;QACP,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,GAAG,EAAE,IAAI,CAAC,MAAM;QAChB,QAAQ,EAAE,GAAG,CAAC,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,IAAI;KAClB,CAAC;AACN,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC5B,uCAAuC;IACvC,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,GAAG,IAAI,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,kCAAkC;IAClC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,yBAAyB;IAExE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,sDAAsD;QACtD,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YAC3D,WAAW,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAED,2DAA2D;IAC3D,IAAI,WAAW,GAAG,UAAU,GAAG,GAAG,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,4CAA4C;IAC5C,IAAI,CAAC;QACD,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB;IAC1E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC"}

package/lib/esm/activities/generateDocumentProperties.js

@@ -0,0 +1,74 @@
+import { log } from "@temporalio/activity";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { executeInteractionFromActivity } from "./executeInteraction.js";
+const INT_EXTRACT_INFORMATION = "sys:ExtractInformation";
+export async function generateDocumentProperties(payload) {
+    const context = await setupActivity(payload);
+    const { params, client, objectId } = context;
+    const interactionName = params.interactionName ?? INT_EXTRACT_INFORMATION;
+    const project = await context.fetchProject();
+    const doc = await client.objects.retrieve(objectId, "+text");
+    const type = doc.type ? await client.types.retrieve(doc.type.id) : undefined;
+    if (!doc?.text && !params.use_vision && !doc?.content?.type?.startsWith("image/")) {
+        log.warn(`Object ${objectId} not found or text is empty`);
+        return { status: "failed", error: "no-text" };
+    }
+    if (!type || !type.object_schema) {
+        log.info(`Object ${objectId} has no schema`);
+        return { document: objectId, status: "skipped", message: "no schema defined on type" };
+    }
+    const getImageRef = () => {
+        if (doc.content?.type?.startsWith("image/")) {
+            return "store:" + doc.id;
+        }
+        if (params.use_vision && doc.content?.type?.startsWith("application/pdf")) {
+            return "store:" + doc.id;
+        }
+        log.info(`Object ${objectId} is not an image or pdf`);
+        return undefined;
+    };
+    const promptData = {
+        content: doc.text ?? undefined,
+        image: getImageRef() ?? undefined,
+        human_context: project?.configuration?.human_context ?? undefined,
+    };
+    log.info(` Extracting information from object ${objectId} with type ${type.name}`, payload.debug_mode ? { params } : undefined);
+    const infoRes = await executeInteractionFromActivity(client, interactionName, {
+        ...params,
+        include_previous_error: true,
+        result_schema: type.object_schema,
+    }, promptData, payload.debug_mode ?? false);
+    const getText = () => {
+        if (doc.text) {
+            return undefined;
+        }
+        let text = "";
+        if (infoRes.result.title) {
+            text += infoRes.result.title + "\n";
+        }
+        if (infoRes.result.description) {
+            text += infoRes.result.description;
+        }
+        if (text) {
+            return text;
+        }
+        else {
+            return undefined;
+        }
+    };
+    log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
+    await client.objects.update(doc.id, {
+        properties: {
+            ...infoRes.result,
+            etag: doc.text_etag,
+        },
+        text: getText(),
+        generation_run_info: {
+            id: infoRes.id,
+            date: new Date().toISOString(),
+            model: infoRes.modelId,
+        },
+    });
+    return { status: "completed" };
+}
+//# sourceMappingURL=generateDocumentProperties.js.map

package/lib/esm/activities/generateDocumentProperties.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generateDocumentProperties.js","sourceRoot":"","sources":["../../../src/activities/generateDocumentProperties.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAA8B,8BAA8B,EAAE,MAAM,yBAAyB,CAAC;AAErG,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;AAgBzD,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC5C,OAAsE;IAEtE,MAAM,OAAO,GAAG,MAAM,aAAa,CAAmC,OAAO,CAAC,CAAC;IAC/E,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,uBAAuB,CAAC;IAE1E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAE7C,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE7E,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChF,GAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,6BAA6B,CAAC,CAAC;QAC1D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAClD,CAAC;IAED,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,gBAAgB,CAAC,CAAC;QAC7C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,2BAA2B,EAAE,CAAC;IAC3F,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,EAAE;QACrB,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACxE,OAAO,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,yBAAyB,CAAC,CAAC;QACtD,OAAO,SAAS,CAAC;IACrB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG;QACf,OAAO,EAAE,GAAG,CAAC,IAAI,IAAI,SAAS;QAC9B,KAAK,EAAE,WAAW,EAAE,IAAI,SAAS;QACjC,aAAa,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,IAAI,SAAS;KACpE,CAAC;IAEF,GAAG,CAAC,IAAI,CACJ,uCAAuC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EACxE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAC9C,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,8BAA8B,CAChD,MAAM,EACN,eAAe,EACf;QACI,GAAG,MAAM;QACT,sBAAsB,EAAE,IAAI;QAC5B,aAAa,EAAE,IAAI,CAAC,aAAa;KACpC,EACD,UAAU,EACV,OAAO,CAAC,UAAU,IAAI,KAAK,CAC9B,CAAC;IAEF,MAAM,OAAO,GAAG,GAAG,EAAE;QACjB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;QACxC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC;QACvC,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACP,OAAO,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACJ,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC,CAAC;IAEF,GAAG,CAAC,IAAI,CAAC,qCAAqC,QAAQ,cAAc,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;IACxG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE;QAChC,UAAU,EAAE;YACR,GAAG,OAAO,CAAC,MAAM;YACjB,IAAI,EAAE,GAAG,CAAC,SAAS;SACtB;QACD,IAAI,EAAE,OAAO,EAAE;QACf,mBAAmB,EAAE;YACjB,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC9B,KAAK,EAAE,OAAO,CAAC,OAAO;SACzB;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACnC,CAAC"}

package/lib/esm/activities/generateEmbeddings.js

@@ -0,0 +1,245 @@
+import { log } from "@temporalio/activity";
+import { SupportedEmbeddingTypes } from "@vertesia/common";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { NoDocumentFound } from '../errors.js';
+import { fetchBlobAsBase64, md5 } from "../utils/blobs.js";
+import { getContentParts } from "../utils/chunks.js";
+import { countTokens } from "../utils/tokens.js";
+export async function generateEmbeddings(payload) {
+    const { params, client, objectId, fetchProject } = await setupActivity(payload);
+    const { force, type } = params;
+    const projectData = await fetchProject();
+    const config = projectData?.configuration.embeddings[type];
+    if (!projectData) {
+        throw new NoDocumentFound('Project not found', [payload.project_id]);
+    }
+    if (!config) {
+        throw new NoDocumentFound('Embeddings configuration not found', [objectId]);
+    }
+    if (!projectData) {
+        throw new NoDocumentFound('Project not found', [payload.project_id]);
+    }
+    if (!projectData?.configuration.embeddings[type]?.enabled) {
+        log.info(`Embeddings generation disabled for type ${type} on project: ${projectData.name} (${projectData.namespace})`, { config });
+        return { id: objectId, status: "skipped", message: `Embeddings generation disabled for type ${type}` };
+    }
+    log.info(`${type} embedding generation starting for object ${objectId}`, { force, config });
+    if (!config.environment) {
+        throw new Error('No environment found in project configuration. Set environment in project configuration to generate embeddings.');
+    }
+    const document = await client.objects.retrieve(objectId, "+text +parts +embeddings +tokens +properties");
+    if (!document) {
+        throw new NoDocumentFound('Document not found', [objectId]);
+    }
+    if (!document.content) {
+        throw new NoDocumentFound('Document content not found', [objectId]);
+    }
+    let res;
+    switch (type) {
+        case SupportedEmbeddingTypes.text:
+            res = await generateTextEmbeddings({
+                client,
+                config,
+                document,
+                type
+            });
+            break;
+        case SupportedEmbeddingTypes.properties:
+            res = await generateTextEmbeddings({
+                client,
+                config,
+                document,
+                type,
+            });
+            break;
+        case SupportedEmbeddingTypes.image:
+            res = await generateImageEmbeddings({
+                client,
+                config,
+                document,
+                type
+            });
+            break;
+        default:
+            res = { id: objectId, status: "failed", message: `unsupported embedding type: ${type}` };
+    }
+    return res;
+}
+async function generateTextEmbeddings({ document, client, type, config }, parts) {
+    // if (!force && document.embeddings[type]?.etag === (document.text_etag ?? md5(document.text))) {
+    //     return { id: objectId, status: "skipped", message: "embeddings already generated" }
+    // }
+    if (!document) {
+        return { status: "error", message: "document is null or undefined" };
+    }
+    if (type !== SupportedEmbeddingTypes.text && type !== SupportedEmbeddingTypes.properties) {
+        return { id: document.id, status: "failed", message: `unsupported embedding type: ${type}` };
+    }
+    if (type === SupportedEmbeddingTypes.text && !document.text) {
+        return { id: document.id, status: "failed", message: "no text found" };
+    }
+    if (type === SupportedEmbeddingTypes.properties && !document?.properties) {
+        return { id: document.id, status: "failed", message: "no properties found" };
+    }
+    const { environment, model } = config;
+    const partDefinitions = parts ?? [];
+    // Count tokens if not already done
+    if (!document.tokens?.count && type === SupportedEmbeddingTypes.text) {
+        log.debug('Updating token count for document: ' + document.id);
+        const tokensData = countTokens(document.text);
+        await client.objects.update(document.id, {
+            tokens: {
+                ...tokensData,
+                etag: document.text_etag ?? md5(document.text)
+            }
+        });
+        document.tokens = {
+            ...tokensData,
+            etag: document.text_etag ?? md5(document.text)
+        };
+    }
+    const maxTokens = config.max_tokens ?? 8000;
+    //generate embeddings for the main doc if document isn't too large
+    //if too large, we'll just generate embeddings for the parts
+    //then we can generate embeddings for the main document by averaging the tensors
+    log.info(`Generating ${type} embeddings for document ${document.id}`);
+    if (type === SupportedEmbeddingTypes.text && document.tokens?.count && document.tokens?.count > maxTokens) {
+        log.info('Document too large, generating embeddings for parts');
+        if (!document.text) {
+            return { id: document.id, status: "failed", message: "no text found" };
+        }
+        if (!partDefinitions || partDefinitions.length === 0) {
+            log.info('No parts found for document, skipping embeddings generation');
+            return { id: document.id, status: "failed", message: "no parts found" };
+        }
+        log.info('Generating embeddings for parts', { parts: partDefinitions, max_tokens: maxTokens });
+        const docParts = getContentParts(document.text, partDefinitions);
+        log.info(`Retrieved ${docParts.length} parts`);
+        const start = new Date().getTime();
+        const generatePartEmbeddings = async (partContent, i) => {
+            const localStart = new Date().getTime();
+            try {
+                log.info(`Generating embeddings for part ${i}`, { text_len: partContent.length });
+                if (!partContent) {
+                    return { id: i, number: i, result: null, status: "skipped", message: "no text found" };
+                }
+                const e = await generateEmbeddingsFromStudio(partContent, environment, client, model).catch(e => {
+                    log.error('Error generating embeddings for part ' + i, { text_length: partContent.length, error: e });
+                    return null;
+                });
+                if (!e || !e.values) {
+                    return { id: i, number: i, result: null, message: "no embeddings generated" };
+                }
+                if (e.values.length === 0) {
+                    return { id: i, number: i, result: null, message: "no embeddings generated" };
+                }
+                log.info(`Generated embeddings for part ${i}`, { len: e.values.length, duration: new Date().getTime() - localStart });
+                return { number: i, result: e };
+            }
+            catch (err) {
+                log.info(`Error generating ${type} embeddings for part ${i} of ${document.id}`, { error: err });
+                return { number: i, result: null, message: "error generating embeddings", error: err.message };
+            }
+        };
+        const partEmbeddings = await Promise.all(docParts.map((part, i) => generatePartEmbeddings(part, i)));
+        const validPartEmbeddings = partEmbeddings.filter(e => e.result !== null).map(e => e.result);
+        const averagedEmbedding = computeAttentionEmbedding(validPartEmbeddings.map(e => e.values));
+        log.info(`Averaged embeddings for document ${document.id} in ${(new Date().getTime() - start) / 1000} seconds`, { len: averagedEmbedding.length, count: validPartEmbeddings.length, max_tokens: maxTokens });
+        await client.objects.setEmbedding(document.id, type, {
+            values: averagedEmbedding,
+            model: validPartEmbeddings[0].model,
+            etag: document.text_etag
+        });
+        log.info(`Object ${document.id} embedding set`, { type, len: averagedEmbedding.length });
+    }
+    else {
+        log.info(`Generating ${type} embeddings for document`);
+        const res = await generateEmbeddingsFromStudio(JSON.stringify(document[type]), environment, client);
+        if (!res || !res.values) {
+            return { id: document.id, status: "failed", message: "no embeddings generated" };
+        }
+        log.info(`${type} embeddings generated for document ${document.id}`, { len: res.values.length });
+        await client.objects.setEmbedding(document.id, type, {
+            values: res.values,
+            model: res.model,
+            etag: document.text_etag
+        });
+        return { id: document.id, type, status: "completed", len: res.values.length };
+    }
+}
+async function generateImageEmbeddings({ document, client, type, config }) {
+    log.info('Generating image embeddings for document ' + document.id, { content: document.content });
+    if (!document.content?.type?.startsWith('image/') && !document.content?.type?.includes('pdf')) {
+        return { id: document.id, type, status: "failed", message: "content is not an image" };
+    }
+    const { environment, model } = config;
+    const resRnd = await client.store.objects.getRendition(document.id, {
+        format: "image/png",
+        max_hw: 1024,
+        generate_if_missing: true
+    });
+    if (resRnd.status === 'generating') {
+        throw new Error("Rendition is generating, will retry later");
+    }
+    else if (resRnd.status === "failed" || !resRnd.rendition) {
+        throw new NoDocumentFound("Rendition retrieval failed", [document.id]);
+    }
+    if (!resRnd.rendition.content.source) {
+        throw new NoDocumentFound("No source found in rendition", [document.id]);
+    }
+    const image = await fetchBlobAsBase64(client, resRnd.rendition.content.source);
+    const res = await client.environments.embeddings(environment, {
+        image,
+        model
+    }).then(res => res).catch(e => {
+        log.error('Error generating embeddings for image', { error: e });
+        throw e;
+    });
+    if (!res || !res.values) {
+        return { id: document.id, status: "failed", message: "no embeddings generated" };
+    }
+    await client.objects.setEmbedding(document.id, SupportedEmbeddingTypes.image, {
+        values: res.values,
+        model: res.model,
+        etag: document.text_etag
+    });
+    return { id: document.id, type, status: "completed", len: res.values.length };
+}
+async function generateEmbeddingsFromStudio(text, env, client, model) {
+    log.info(`Generating embeddings for text of ${text.length} chars with environment ${env}`);
+    return client.environments.embeddings(env, {
+        text,
+        model
+    }).then(res => res).catch(e => {
+        log.error('Error generating embeddings for text', { error: e });
+        throw e;
+    });
+}
+//Simplified attention mechanism
+// This is a naive implementation and should be replaced with a more sophisticated 
+// using tensorflow in a specific package
+function computeAttentionEmbedding(chunkEmbeddings) {
+    if (chunkEmbeddings.length === 0)
+        return [];
+    const start = new Date().getTime();
+    // Generate random attention weights
+    const attentionWeights = chunkEmbeddings.map(() => Math.random());
+    // Apply softmax to get attention scores
+    const expWeights = attentionWeights.map(w => Math.exp(w));
+    const sumExpWeights = expWeights.reduce((sum, val) => sum + val, 0);
+    const attentionScores = expWeights.map(w => w / sumExpWeights);
+    // Get embedding dimension
+    const embeddingDim = chunkEmbeddings[0].length;
+    // Initialize document embedding
+    const documentEmbedding = new Array(embeddingDim).fill(0);
+    // Weighted sum of embeddings
+    for (let i = 0; i < chunkEmbeddings.length; i++) {
+        for (let j = 0; j < embeddingDim; j++) {
+            documentEmbedding[j] += chunkEmbeddings[i][j] * attentionScores[i];
+        }
+    }
+    const duration = new Date().getTime() - start;
+    console.log(`Computed document embedding in ${duration}ms for ${chunkEmbeddings.length} chunks`);
+    return documentEmbedding;
+}
+//# sourceMappingURL=generateEmbeddings.js.map