@vertesia/workflow 0.42.2

package/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2024 Composable Prompts
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

package/README.md

@@ -0,0 +1,24 @@
+# Zeno Workflows
+
+This package implements the workflow tasks shared by all content object types.
+
+There are 3 main sequential tasks:
+
+1. Build Content
+2. Build Properties
+3. Build Tree
+
+## Build Content
+
+This task is generating the text content from the object content source if any.
+
+If no content source was provided, e.g. the object text field is already filled by the client or the object type doesn't support generating text content, this task will do nothing
+
+## Build Properties
+This task is generating metadata properties or the `object` field.
+
+It will do nothing for object types that doesn't need to generate extra properties.
+
+## Build Tree
+This task is generating the sub-objects if any. 
+

package/bin/bundle-workflows.mjs

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+
+import { bundleWorkflowCode } from '@temporalio/worker';
+import { writeFile } from 'fs/promises';
+import path from 'path';
+
+async function bundle(wsPath, bundlePath) {
+    const { code } = await bundleWorkflowCode({
+        workflowsPath: path.resolve(wsPath),
+    });
+    const codePath = path.resolve(bundlePath);
+    await writeFile(codePath, code);
+    console.log(`Bundle written to ${codePath}`);
+}
+
+const wsPath = process.argv[2];
+const bundlePath = process.argv[3];
+if (!wsPath || !bundlePath) {
+    console.error('Usage: build-workflows <workflows-path> <bundle-path>');
+    process.exit(1);
+}
+
+bundle(wsPath, bundlePath).catch((err) => {
+    console.error(err);
+    process.exit(1);
+});

package/lib/cjs/activities/advanced/createDocumentTypeFromInteractionRun.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDocumentTypeFromInteractionRun = createDocumentTypeFromInteractionRun;
+const activity_1 = require("@temporalio/activity");
+const projections_js_1 = require("../../dsl/projections.js");
+const ActivityContext_js_1 = require("../../dsl/setup/ActivityContext.js");
+const errors_js_1 = require("../../errors.js");
+async function createDocumentTypeFromInteractionRun(payload) {
+    const { params, client } = await (0, ActivityContext_js_1.setupActivity)(payload);
+    if (!params.run) {
+        throw new errors_js_1.ActivityParamNotFound("run", payload.activity);
+    }
+    const genTypeRes = params.run.result;
+    if (!genTypeRes.document_type) {
+        activity_1.log.error("No name generated for type: " + JSON.stringify(genTypeRes), genTypeRes);
+        throw new Error("No name generated for type");
+    }
+    activity_1.log.info("Generated schema for type", genTypeRes.metadata_schema);
+    const typeData = {
+        name: genTypeRes.document_type,
+        object_schema: genTypeRes.metadata_schema,
+        is_chunkable: !!genTypeRes.is_chunkable,
+    };
+    const type = await client.types.create(typeData);
+    if (params.updateObjectId) {
+        await client.objects.update(params.updateObjectId, {
+            type: type.id,
+        });
+    }
+    return (0, projections_js_1.projectResult)(payload, params, type, { id: type.id, name: type.name });
+}
+//# sourceMappingURL=createDocumentTypeFromInteractionRun.js.map

package/lib/cjs/activities/advanced/createDocumentTypeFromInteractionRun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createDocumentTypeFromInteractionRun.js","sourceRoot":"","sources":["../../../../src/activities/advanced/createDocumentTypeFromInteractionRun.ts"],"names":[],"mappings":";;AAuBA,oFA8BC;AApDD,mDAA2C;AAC3C,6DAAyD;AACzD,2EAAmE;AACnE,+CAAwD;AAmBjD,KAAK,UAAU,oCAAoC,CAAC,OAAoC;IAC3F,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAA6C,OAAO,CAAC,CAAC;IAEpG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,iCAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;IAErC,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QAC5B,cAAG,CAAC,KAAK,CAAC,8BAA8B,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,CAAC;QACnF,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAClD,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,UAAU,CAAC,eAAe,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAmC;QAC7C,IAAI,EAAE,UAAU,CAAC,aAAa;QAC9B,aAAa,EAAE,UAAU,CAAC,eAAe;QACzC,YAAY,EAAE,CAAC,CAAC,UAAU,CAAC,YAAY;KAC1C,CAAA;IAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEjD,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,EAAE;SAChB,CAAC,CAAC;IACP,CAAC;IAED,OAAO,IAAA,8BAAa,EAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAClF,CAAC"}

package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createOrUpdateDocumentFromInteractionRun = createOrUpdateDocumentFromInteractionRun;
+const common_1 = require("@vertesia/common");
+const activity_1 = require("@temporalio/activity");
+const ActivityContext_js_1 = require("../../dsl/setup/ActivityContext.js");
+const errors_js_1 = require("../../errors.js");
+async function createOrUpdateDocumentFromInteractionRun(payload) {
+    const { params, client } = await (0, ActivityContext_js_1.setupActivity)(payload);
+    const runId = params.run_id;
+    const objectTypeName = params.object_type;
+    if (!runId) {
+        throw new errors_js_1.ActivityParamNotFound("run_id", payload.activity);
+    }
+    if (!objectTypeName && !params.update_existing_id) {
+        throw new errors_js_1.ActivityParamNotFound("object_type", payload.activity);
+    }
+    activity_1.log.info("Creating document from interaction result", { runId, objectTypeName });
+    const run = await client.runs.retrieve(runId).catch((e) => {
+        throw new errors_js_1.NoDocumentFound(`Error fetching run ${runId}: ${e.message}`);
+    });
+    const type = objectTypeName ?
+        await client.types.getTypeByName(objectTypeName).catch((e) => {
+            throw new errors_js_1.NoDocumentFound(`Error fetching type ${objectTypeName}: ${e.message}`);
+        })
+        : undefined;
+    const result = run.result;
+    const resultIsObject = typeof result === 'object';
+    const inputData = run.parameters;
+    let name;
+    if (resultIsObject) {
+        name = result['name'] || result["title"] || inputData['name'] || params.fallback_name || 'Untitled';
+    }
+    else {
+        name = inputData['name'] || params.fallback_name || 'Untitled';
+    }
+    const docPayload = {
+        name,
+        parent: params.parent ?? undefined,
+        properties: resultIsObject ? result : {},
+        text: !resultIsObject ? result : undefined,
+        type: type?.id,
+        status: common_1.ContentObjectStatus.completed,
+        generation_run_info: {
+            id: run.id,
+            date: new Date().toISOString(),
+            model: run.modelId,
+            target: resultIsObject ? 'properties' : 'text'
+        }
+    };
+    //create or update the document
+    let newDoc = false;
+    let doc = undefined;
+    if (params.update_existing_id) {
+        activity_1.log.info(`Updating existing document ${params.update_existing_id}`);
+        doc = await client.objects.update(params.update_existing_id, docPayload);
+    }
+    else {
+        activity_1.log.info(`Creating new document of type ${objectTypeName}`);
+        doc = await client.objects.create(docPayload);
+        newDoc = true;
+    }
+    activity_1.log.info(`Document ${objectTypeName + ' '}${doc.id}(${doc.name}) ${newDoc ? 'created' : 'updated'}`);
+    return { id: doc.id, isNew: newDoc, type: name };
+}
+//# sourceMappingURL=createOrUpdateDocumentFromInteractionRun.js.map

package/lib/cjs/activities/advanced/createOrUpdateDocumentFromInteractionRun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createOrUpdateDocumentFromInteractionRun.js","sourceRoot":"","sources":["../../../../src/activities/advanced/createOrUpdateDocumentFromInteractionRun.ts"],"names":[],"mappings":";;AA8BA,4FAkEC;AAhGD,6CAAqG;AACrG,mDAA2C;AAC3C,2EAAmE;AACnE,+CAAyE;AA2BlE,KAAK,UAAU,wCAAwC,CAAC,OAAoC;IAC/F,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAA+C,OAAO,CAAC,CAAC;IAEtG,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC;IAE1C,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,MAAM,IAAI,iCAAqB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,CAAC,cAAc,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAChD,MAAM,IAAI,iCAAqB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACrE,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;IAEjF,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACtD,MAAM,IAAI,2BAAe,CAAC,sBAAsB,KAAK,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC;QACzB,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzD,MAAM,IAAI,2BAAe,CAAC,uBAAuB,cAAc,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,CAAC,CAAC;QACF,CAAC,CAAC,SAAS,CAAC;IAGhB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC1B,MAAM,cAAc,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC;IAClD,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC;IAEjC,IAAI,IAAY,CAAC;IACjB,IAAI,cAAc,EAAE,CAAC;QACjB,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;IACxG,CAAC;SAAM,CAAC;QACJ,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;IACnE,CAAC;IAED,MAAM,UAAU,GAAG;QACf,IAAI;QACJ,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;QAClC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QACxC,IAAI,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QAC1C,IAAI,EAAE,IAAI,EAAE,EAAE;QACd,MAAM,EAAE,4BAAmB,CAAC,SAAS;QACrC,mBAAmB,EAAE;YACjB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC9B,KAAK,EAAE,GAAG,CAAC,OAAO;YAClB,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM;SACjD;KACJ,CAAC;IAEF,+BAA+B;IAC/B,IAAI,MAAM,GAAY,KAAK,CAAC;IAC5B,IAAI,GAAG,GAAG,SAAS,CAAC;IACpB,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC5B,cAAG,CAAC,IAAI,CAAC,8BAA8B,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC;QACpE,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAC;IAC7E,CAAC;SAAM,CAAC;QACJ,cAAG,CAAC,IAAI,CAAC,iCAAiC,cAAc,EAAE,CAAC,CAAC;QAC5D,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,GAAG,IAAI,CAAC;IAClB,CAAC;IAED,cAAG,CAAC,IAAI,CAAC,YAAY,cAAc,GAAG,GAAG,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IACrG,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;AACpD,CAAC"}

package/lib/cjs/activities/advanced/updateDocumentFromInteractionRun.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.updateDocumentFromInteractionRun = updateDocumentFromInteractionRun;
+const ActivityContext_js_1 = require("../../dsl/setup/ActivityContext.js");
+const errors_js_1 = require("../../errors.js");
+async function updateDocumentFromInteractionRun(payload) {
+    const { params, client, objectId } = await (0, ActivityContext_js_1.setupActivity)(payload);
+    if (!params.run) {
+        throw new errors_js_1.ActivityParamNotFound("run", payload.activity);
+    }
+    const docProps = params.run.result;
+    if (!docProps) {
+        return { status: "failed", error: "no-props" };
+    }
+    await client.objects.update(objectId, docProps);
+    return { status: "success" };
+}
+//# sourceMappingURL=updateDocumentFromInteractionRun.js.map

package/lib/cjs/activities/advanced/updateDocumentFromInteractionRun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"updateDocumentFromInteractionRun.js","sourceRoot":"","sources":["../../../../src/activities/advanced/updateDocumentFromInteractionRun.ts"],"names":[],"mappings":";;AAiBA,4EAgBC;AAhCD,2EAAmE;AACnE,+CAAwD;AAejD,KAAK,UAAU,gCAAgC,CAAC,OAAoC;IACvF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAyC,OAAO,CAAC,CAAC;IAE1G,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,iCAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;IAEnC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAEhD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC"}

package/lib/cjs/activities/chunkDocument.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chunkDocument = chunkDocument;
+const activity_1 = require("@temporalio/activity");
+const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
+const executeInteraction_js_1 = require("./executeInteraction.js");
+const INT_CHUNK_DOCUMENT = "sys:ChunkDocument";
+async function chunkDocument(payload) {
+    const { params, client, objectId } = await (0, ActivityContext_js_1.setupActivity)(payload);
+    const { force } = params;
+    const interactionName = params.interactionName ?? INT_CHUNK_DOCUMENT;
+    activity_1.log.info(`Object ${objectId} chunking started`);
+    const document = await client.objects.retrieve(objectId, "+text");
+    const type = document.type ? await client.types.retrieve(document.type.id) : undefined;
+    if (!type?.is_chunkable) {
+        activity_1.log.warn('Type is not chunkable for object ID: ' + objectId);
+        return { id: objectId, status: "skipped", message: "type not chunkable" };
+    }
+    //check if text is present
+    if (!document.text) {
+        activity_1.log.warn('No text found for object ID: ' + objectId);
+        return { id: objectId, status: "failed", message: "no text found" };
+    }
+    if (!force && document.parts && document.parts.length > 0 && document.parts_etag === document.text_etag) {
+        activity_1.log.info('Document already chunked for object ID: ' + objectId);
+        return { id: objectId, status: "skipped", message: "document already chunked with correct etag" };
+    }
+    //instrument the text with line numbers
+    const lines = document.text.split('\n');
+    const instrumented = lines.map((l, i) => `{%${i}%}${l}`).join('\n');
+    const res = await (0, executeInteraction_js_1.executeInteractionFromActivity)(client, interactionName, params, {
+        objectId: objectId,
+        content: instrumented
+    });
+    const parts = res.result.parts;
+    if (!parts || parts.length === 0) {
+        activity_1.log.warn('No parts found for object ID: ' + objectId, res);
+        return { id: objectId, status: "failed", parts: [], message: "no parts found" };
+    }
+    const partDocs = await Promise.all(parts.map(async (part, i) => {
+        const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
+        const location = () => {
+            let location = document.location;
+            if (location.endsWith('/')) {
+                location += document.name + "/" + part.type;
+            }
+            location += '/' + document.name + "/" + part.type;
+            return location;
+        };
+        const docPart = await client.objects.create({
+            name: part.name,
+            parent: objectId,
+            text: text,
+            location: location(),
+            properties: {
+                part_number: i + 1,
+                etag: document.text_etag,
+                source_line_start: part.line_number_start,
+                source_line_end: part.line_number_end,
+                title: part.name
+            }
+        });
+        return docPart;
+    }));
+    //delete previous parts
+    if (document.parts && document.parts.length > 0) {
+        activity_1.log.info('Deleting previous parts for object ID: ' + objectId, { parts: document.parts });
+        await Promise.all(document.parts.map(async (partId) => {
+            await client.objects.delete(partId);
+        }));
+    }
+    await client.objects.update(objectId, {
+        parts: partDocs.map(p => p.id),
+        parts_etag: document.text_etag
+    });
+    activity_1.log.info(`Object ${objectId} chunking completed`, { parts: document.parts });
+    return { id: objectId, status: "completed", parts: document.parts };
+}
+//# sourceMappingURL=chunkDocument.js.map

package/lib/cjs/activities/chunkDocument.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkDocument.js","sourceRoot":"","sources":["../../../src/activities/chunkDocument.ts"],"names":[],"mappings":";;AAkCA,sCAyFC;AA1HD,mDAA2C;AAC3C,wEAAgE;AAChE,mEAAqG;AAErG,MAAM,kBAAkB,GAAG,mBAAmB,CAAA;AA6BvC,KAAK,UAAU,aAAa,CAAC,OAAoC;IACpE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAsB,OAAO,CAAC,CAAC;IAEvF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,kBAAkB,CAAC;IAErE,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,mBAAmB,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC;QACtB,cAAG,CAAC,IAAI,CAAC,uCAAuC,GAAG,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC7E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjB,cAAG,CAAC,IAAI,CAAC,+BAA+B,GAAG,QAAQ,CAAC,CAAC;QACrD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,CAAA;IACvE,CAAC;IAED,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,SAAS,EAAE,CAAC;QACtG,cAAG,CAAC,IAAI,CAAC,0CAA0C,GAAG,QAAQ,CAAC,CAAC;QAChE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAA;IACrG,CAAC;IAED,uCAAuC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEnE,MAAM,GAAG,GAAG,MAAM,IAAA,sDAA8B,EAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE;QAC9E,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,YAAY;KACxB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAkB,CAAC;IAC5C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,cAAG,CAAC,IAAI,CAAC,gCAAgC,GAAG,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC3D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAA;IACnF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAE3D,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE1G,MAAM,QAAQ,GAAG,GAAG,EAAE;YAClB,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,QAAQ,IAAI,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;YAC/C,CAAC;YACD,QAAQ,IAAI,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;YAClD,OAAO,QAAQ,CAAC;QACpB,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;YACxC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,QAAQ;YAChB,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,QAAQ,EAAE;YACpB,UAAU,EAAE;gBACR,WAAW,EAAE,CAAC,GAAG,CAAC;gBAClB,IAAI,EAAE,QAAQ,CAAC,SAAS;gBACxB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;gBACzC,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,IAAI,CAAC,IAAI;aACnB;SACJ,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACnB,CAAC,CAAC,CAAC,CAAC;IAEJ,uBAAuB;IACvB,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,cAAG,CAAC,IAAI,CAAC,yCAAyC,GAAG,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QAC1F,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAClD,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC,CAAC;IACR,CAAC;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;QAClC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,UAAU,EAAE,QAAQ,CAAC,SAAS;KACjC,CAAC,CAAC;IAEH,cAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,qBAAqB,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAA;AAEvE,CAAC"}

package/lib/cjs/activities/createDocumentFromOther.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createPdfDocumentFromSource = createPdfDocumentFromSource;
+const activity_1 = require("@temporalio/activity");
+const fs_1 = __importDefault(require("fs"));
+const mutool_js_1 = require("../conversion/mutool.js");
+const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
+const errors_js_1 = require("../errors.js");
+const blobs_js_1 = require("../utils/blobs.js");
+const memory_js_1 = require("../utils/memory.js");
+/**
+ * Create a new PDF by extrracting pages from a source PDF
+ * @returns
+ */
+async function createPdfDocumentFromSource(payload) {
+    const { client, objectId, params } = await (0, ActivityContext_js_1.setupActivity)(payload);
+    const inputObject = await client.objects.retrieve(objectId);
+    const { pages, filename, title } = params;
+    activity_1.log.info(`Creating PDF from source`, { objectId, pages, filename, title });
+    if (!pages || pages.length === 0) {
+        activity_1.log.error(`No pages provided`);
+        throw new Error(`No pages provided`);
+    }
+    if (!inputObject) {
+        activity_1.log.error(`Document ${objectId} not found`);
+        throw new errors_js_1.NoDocumentFound(`Document ${objectId} not found`, [objectId]);
+    }
+    if (!inputObject.content?.source) {
+        activity_1.log.error(`Document ${objectId} has no source`);
+        throw new errors_js_1.NoDocumentFound(`Document ${objectId} has no source`, [objectId]);
+    }
+    if (!inputObject.content.type || (!inputObject.content.type?.startsWith('application/pdf'))) {
+        activity_1.log.error(`Document ${objectId} is not an image`);
+        throw new errors_js_1.NoDocumentFound(`Document ${objectId} is not an image or pdf: ${inputObject.content.type}`, [objectId]);
+    }
+    const targetType = await client.types.getTypeByName(params.target_object_type);
+    if (!targetType) {
+        activity_1.log.error(`Type ${params.target_object_type} not found`);
+        throw new errors_js_1.NoDocumentFound(`Type ${params.target_object_type} not found`);
+    }
+    const tmpFile = await (0, blobs_js_1.saveBlobToTempFile)(client, inputObject.content.source, ".pdf");
+    const newPdf = await (0, mutool_js_1.pdfExtractPages)(tmpFile, pages);
+    activity_1.log.info(`PDF created from pages ${pages.join(', ')} `, { newPdf });
+    const name = `pages-${pages.join('-')}.pdf`;
+    const sourceToUpload = new memory_js_1.NodeStreamSource(fs_1.default.createReadStream(newPdf), name, "application/pdf");
+    activity_1.log.info(`Uploading file ${newPdf} `);
+    const upload = await client.objects.upload(sourceToUpload);
+    activity_1.log.info(`File uploaded ${upload.source} `);
+    const newObject = await client.objects.create({
+        type: targetType.id,
+        name: title || targetType.name,
+        parent: objectId,
+        content: {
+            source: upload.source,
+            name: upload.name,
+            type: 'application/pdf'
+        }
+    });
+    return { newObjectId: newObject.id, uploadedFile: upload.name };
+}
+//# sourceMappingURL=createDocumentFromOther.js.map

package/lib/cjs/activities/createDocumentFromOther.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createDocumentFromOther.js","sourceRoot":"","sources":["../../../src/activities/createDocumentFromOther.ts"],"names":[],"mappings":";;;;;AA6BA,kEA8DC;AA1FD,mDAA2C;AAC3C,4CAAoB;AACpB,uDAA0D;AAC1D,wEAAgE;AAChE,4CAA+C;AAC/C,gDAAuD;AACvD,kDAAsD;AAkBtD;;;GAGG;AACI,KAAK,UAAU,2BAA2B,CAAC,OAAoC;IAClF,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,kCAAa,EAAoC,OAAO,CAAC,CAAC;IACrG,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAE5D,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAC1C,cAAG,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,cAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,YAAY,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC/B,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,gBAAgB,CAAC,CAAC;QAChD,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,gBAAgB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC;QAC1F,cAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,kBAAkB,CAAC,CAAC;QAClD,MAAM,IAAI,2BAAe,CAAC,YAAY,QAAQ,4BAA4B,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACtH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC/E,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,cAAG,CAAC,KAAK,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;QACzD,MAAM,IAAI,2BAAe,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,IAAA,6BAAkB,EAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACrD,cAAG,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;IAE5C,MAAM,cAAc,GAAG,IAAI,4BAAgB,CACvC,YAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAC3B,IAAI,EACJ,iBAAiB,CACpB,CAAA;IAED,cAAG,CAAC,IAAI,CAAC,kBAAkB,MAAM,GAAG,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3D,cAAG,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAE5C,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1C,IAAI,EAAE,UAAU,CAAC,EAAE;QACnB,IAAI,EAAE,KAAK,IAAI,UAAU,CAAC,IAAI;QAC9B,MAAM,EAAE,QAAQ;QAChB,OAAO,EAAE;YACL,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,iBAAiB;SAC1B;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AAGpE,CAAC"}

package/lib/cjs/activities/executeInteraction.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeInteraction = executeInteraction;
+exports.executeInteractionFromActivity = executeInteractionFromActivity;
+const common_1 = require("@vertesia/common");
+const activity_1 = require("@temporalio/activity");
+const projections_js_1 = require("../dsl/projections.js");
+const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
+const tokens_js_1 = require("../utils/tokens.js");
+//Example:
+//@ts-ignore
+const JSON = {
+    name: 'executeInteraction',
+    import: ["defaultModel", "guidlineId", "docTypeId"],
+    params: {
+        defaultModel: "${model}",
+        interactionName: "GenerateSummary",
+        model: "${defaultModel ?? 'gpt4'}",
+        environment: "13456",
+        max_tokens: 100,
+        temperature: 0.5,
+        tags: ["test"],
+        result_schema: "${docType.object_schema}",
+        prompt_data: {
+            documents: "${documents}",
+            guidline: "${guidline.text}"
+        }
+    },
+    fetch: {
+        documents: {
+            type: "document",
+            query: {
+                id: { $in: "${objectIds}" },
+            },
+            select: "+text",
+        },
+        guidline: {
+            type: "document",
+            limit: 1,
+            query: {
+                id: "${guidlineId}",
+            },
+            select: "+text",
+            on_not_found: "throw"
+        },
+        docType: {
+            type: "document_type",
+            limit: 1,
+            query: {
+                id: "${docTypeId}",
+            },
+            select: "+object_schema",
+        }
+    }
+};
+async function executeInteraction(payload) {
+    const { client, params } = await (0, ActivityContext_js_1.setupActivity)(payload);
+    const { interactionName, prompt_data } = params;
+    if (params.truncate) {
+        const truncate = params.truncate;
+        for (const [key, value] of Object.entries(truncate)) {
+            prompt_data[key] = (0, tokens_js_1.truncByMaxTokens)(prompt_data[key], value);
+        }
+    }
+    const res = await executeInteractionFromActivity(client, interactionName, params, prompt_data, payload.debug_mode);
+    return (0, projections_js_1.projectResult)(payload, params, res, {
+        runId: res.id,
+        status: res.status,
+        result: res.result,
+    });
+}
+async function executeInteractionFromActivity(client, interactionName, params, prompt_data, debug) {
+    const userTags = params.tags;
+    const info = (0, activity_1.activityInfo)();
+    const runId = info.workflowExecution.runId;
+    let tags = ["workflow", `tmpRunId:${runId}`]; //TODO use wf:wfName
+    if (userTags) {
+        tags = tags.concat(userTags);
+    }
+    let previousStudioExecutionRun = undefined;
+    if (params.include_previous_error) {
+        //retrieve last failed run if any
+        if (info.attempt > 1) {
+            activity_1.log.info("Retrying, searching for previous run", { tags: ["tmpRunId:" + runId] });
+            const payload = {
+                query: { tags: ["tmpRunId:" + info.workflowExecution.runId] },
+                limit: 1,
+            };
+            const previousRun = await client.runs.search(payload).then((res) => {
+                activity_1.log.info("Search results", { results: res });
+                return res ? res[0] ?? undefined : undefined;
+            });
+            if (previousRun) {
+                activity_1.log.info("Found previous run", { previousRun });
+                previousStudioExecutionRun = await client.runs.retrieve(previousRun.id);
+            }
+        }
+    }
+    if (debug && previousStudioExecutionRun?.error) {
+        activity_1.log.info(`Found  previous run error`, { error: previousStudioExecutionRun?.error });
+    }
+    const config = {
+        environment: params.environment,
+        model: params.model,
+        max_tokens: params.max_tokens,
+        temperature: params.temperature
+    };
+    const data = {
+        ...prompt_data,
+        previous_error: previousStudioExecutionRun?.error,
+    };
+    const result_schema = params.result_schema;
+    if (debug) {
+        activity_1.log.info(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags });
+    }
+    const res = await client.interactions.executeByName(interactionName, {
+        config,
+        data,
+        result_schema,
+        tags,
+        stream: false,
+    }).catch((err) => {
+        activity_1.log.error(`Error executing interaction ${interactionName}`, { err });
+        throw new Error(`Interaction Execution failed ${interactionName}: ${err.message}`);
+    });
+    if (debug) {
+        activity_1.log.info(`Interaction executed ${interactionName}`, res);
+    }
+    if (res.error || res.status === common_1.ExecutionRunStatus.failed) {
+        activity_1.log.error(`Error executing interaction ${interactionName}`, { error: res.error });
+        throw new Error(`Interaction Execution failed ${interactionName}: ${res.error}`);
+    }
+    return res;
+}
+//# sourceMappingURL=executeInteraction.js.map

package/lib/cjs/activities/executeInteraction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"executeInteraction.js","sourceRoot":"","sources":["../../../src/activities/executeInteraction.ts"],"names":[],"mappings":";;AA+FA,gDAsBC;AAED,wEAuEC;AA7LD,6CAAuK;AACvK,mDAAyD;AACzD,0DAAsD;AACtD,wEAAgE;AAChE,kDAAoE;AAGpE,UAAU;AACV,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,oBAAoB;IAC1B,MAAM,EAAE,CAAC,cAAc,EAAE,YAAY,EAAE,WAAW,CAAC;IACnD,MAAM,EAAE;QACJ,YAAY,EAAE,UAAU;QACxB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,2BAA2B;QAClC,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,GAAG;QAChB,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,aAAa,EAAE,0BAA0B;QACzC,WAAW,EAAE;YACT,SAAS,EAAE,cAAc;YACzB,QAAQ,EAAE,kBAAkB;SAC/B;KACJ;IACD,KAAK,EAAE;QACH,SAAS,EAAE;YACP,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE;gBACH,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aAC9B;YACD,MAAM,EAAE,OAAO;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,eAAe;aACtB;YACD,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,OAAO;SACxB;QACD,OAAO,EAAE;YACL,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,cAAc;aACrB;YACD,MAAM,EAAE,gBAAgB;SAC3B;KACJ;CACJ,CAAA;AA0CM,KAAK,UAAU,kBAAkB,CAAC,OAAoC;IACzE,MAAM,EACF,MAAM,EAAE,MAAM,EACjB,GAAG,MAAM,IAAA,kCAAa,EAA2B,OAAO,CAAC,CAAC;IAE3D,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEhD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,4BAAgB,EAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAEnH,OAAO,IAAA,8BAAa,EAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE;QACvC,KAAK,EAAE,GAAG,CAAC,EAAE;QACb,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;KACrB,CAAC,CAAC;AAEP,CAAC;AAEM,KAAK,UAAU,8BAA8B,CAAC,MAAwB,EAAE,eAAuB,EAAE,MAAkC,EAAE,WAAgB,EAAE,KAAe;IACzK,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAA,uBAAY,GAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,YAAY,KAAK,EAAE,CAAC,CAAC,CAAC,oBAAoB;IAClE,IAAI,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,0BAA0B,GAA6B,SAAS,CAAC;IACrE,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAChC,iCAAiC;QACjC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,cAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;YAClF,MAAM,OAAO,GAAqB;gBAC9B,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE;gBAC7D,KAAK,EAAE,CAAC;aACX,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/D,cAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAA;YAChD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBACd,cAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChD,0BAA0B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,KAAK,IAAI,0BAA0B,EAAE,KAAK,EAAE,CAAC;QAC7C,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,MAAM,GAAsC;QAC9C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;KAClC,CAAA;IACD,MAAM,IAAI,GAAG;QACT,GAAG,WAAW;QACd,cAAc,EAAE,0BAA0B,EAAE,KAAK;KACpD,CAAA;IAED,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAE3C,IAAI,KAAK,EAAE,CAAC;QACR,cAAG,CAAC,IAAI,CAAC,gCAAgC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACvG,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,eAAe,EAAE;QACjE,MAAM;QACN,IAAI;QACJ,aAAa;QACb,IAAI;QACJ,MAAM,EAAE,KAAK;KAChB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,CAAC;QACR,cAAG,CAAC,IAAI,CAAC,wBAAwB,eAAe,EAAE,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,2BAAkB,CAAC,MAAM,EAAE,CAAC;QACxD,cAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}

package/lib/cjs/activities/extractDocumentText.js

@@ -0,0 +1,140 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractDocumentText = extractDocumentText;
+const activity_1 = require("@temporalio/activity");
+const mutool_js_1 = require("../conversion/mutool.js");
+const pandoc_js_1 = require("../conversion/pandoc.js");
+const pdf_js_1 = require("../conversion/pdf.js");
+const ActivityContext_js_1 = require("../dsl/setup/ActivityContext.js");
+const errors_js_1 = require("../errors.js");
+const result_types_js_1 = require("../result-types.js");
+const blobs_js_1 = require("../utils/blobs.js");
+const tokens_js_1 = require("../utils/tokens.js");
+//@ts-ignore
+const JSON = {
+    name: 'extractDocumentText',
+};
+async function extractDocumentText(payload) {
+    const { client, objectId } = await (0, ActivityContext_js_1.setupActivity)(payload);
+    const r = await client.objects.find({
+        query: { _id: objectId },
+        limit: 1,
+        select: "+text"
+    });
+    const doc = r[0];
+    if (!doc) {
+        activity_1.log.error(`Document ${objectId} not found`);
+        throw new errors_js_1.NoDocumentFound(`Document ${objectId} not found`, payload.objectIds);
+    }
+    activity_1.log.info(`Extracting text for object ${doc.id}`);
+    if (!doc.content?.type || !doc.content?.source) {
+        if (doc.text) {
+            return createResponse(doc, doc.text, result_types_js_1.TextExtractionStatus.skipped, "Text present and no source or type");
+        }
+        else {
+            return createResponse(doc, "", result_types_js_1.TextExtractionStatus.error, "No source or type found");
+        }
+    }
+    //skip if text already extracted and proper etag
+    if (doc.text && doc.text.length > 0 && doc.text_etag === doc.content.etag) {
+        return createResponse(doc, doc.text, result_types_js_1.TextExtractionStatus.skipped, "Text already extracted");
+    }
+    let fileBuffer;
+    try {
+        fileBuffer = await (0, blobs_js_1.fetchBlobAsBuffer)(client, doc.content.source);
+    }
+    catch (e) {
+        activity_1.log.error(`Error reading file: ${e}`);
+        return createResponse(doc, "", result_types_js_1.TextExtractionStatus.error, e.message);
+    }
+    let txt;
+    switch (doc.content.type) {
+        case 'application/pdf':
+            //if pdf is more than 2MB, use mutool
+            if (fileBuffer.length > 2 * 1024 * 1024) {
+                txt = await (0, mutool_js_1.mutoolPdfToText)(fileBuffer);
+            }
+            else {
+                txt = await (0, pdf_js_1.trasformPdfToMarkdown)(fileBuffer);
+            }
+            break;
+        case 'text/plain':
+            txt = fileBuffer.toString('utf8');
+            break;
+        //docx
+        case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+            txt = await (0, pandoc_js_1.manyToMarkdown)(fileBuffer, 'docx');
+            break;
+        //html
+        case 'text/html':
+            txt = await (0, pandoc_js_1.manyToMarkdown)(fileBuffer, 'html');
+            break;
+        //opendocument
+        case 'application/vnd.oasis.opendocument.text':
+            txt = await (0, pandoc_js_1.manyToMarkdown)(fileBuffer, 'odt');
+            break;
+        //rtf
+        case 'application/rtf':
+            txt = await (0, pandoc_js_1.manyToMarkdown)(fileBuffer, 'rtf');
+            break;
+        //markdown
+        case 'text/markdown':
+            txt = fileBuffer.toString('utf8');
+            break;
+        //csv
+        case 'text/csv':
+            txt = fileBuffer.toString('utf8');
+            break;
+        //typescript
+        case 'application/typescript':
+            txt = fileBuffer.toString('utf8');
+            break;
+        //javascript
+        case 'application/javascript':
+            txt = fileBuffer.toString('utf8');
+            break;
+        //json
+        case 'application/json':
+            txt = fileBuffer.toString('utf8');
+            break;
+        default:
+            if (sniffIfText(fileBuffer)) {
+                txt = fileBuffer.toString('utf8'); //TODO: add charset detection
+                break;
+            }
+            return createResponse(doc, doc.text ?? '', result_types_js_1.TextExtractionStatus.skipped, `Unsupported mime type: ${doc.content.type}`);
+    }
+    const tokensData = (0, tokens_js_1.countTokens)(txt);
+    const etag = doc.content.etag ?? (0, blobs_js_1.md5)(txt);
+    const updateData = {
+        text: txt,
+        text_etag: etag,
+        tokens: {
+            ...tokensData,
+            etag: etag,
+        }
+    };
+    await client.objects.update(doc.id, updateData);
+    return createResponse(doc, txt, result_types_js_1.TextExtractionStatus.success);
+}
+function createResponse(doc, text, status, message) {
+    return {
+        status,
+        message,
+        tokens: doc.tokens,
+        len: text.length,
+        objectId: doc.id,
+        hasText: !!text,
+    };
+}
+//if file is less than 100KB, check if it looks like text
+function sniffIfText(buf) {
+    if (buf.length < 100 * 1024) {
+        const s = buf.toString('utf8');
+        if (s.length > 0) {
+            return true;
+        }
+    }
+    return false;
+}
+//# sourceMappingURL=extractDocumentText.js.map