@vertesia/workflow 0.24.0-dev.202601221707

package/lib/esm/activities/advanced/updateDocumentFromInteractionRun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"updateDocumentFromInteractionRun.js","sourceRoot":"","sources":["../../../../src/activities/advanced/updateDocumentFromInteractionRun.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AACnE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAe7D,MAAM,CAAC,KAAK,UAAU,gCAAgC,CAAC,OAA4E;IAC/H,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,aAAa,CAAyC,OAAO,CAAC,CAAC;IAE1G,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,0BAA0B,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC;IAEpE,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7E,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC"}

package/lib/esm/activities/chunkDocument.js

@@ -0,0 +1,82 @@
+import { log } from "@temporalio/activity";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { executeInteractionFromActivity } from "./executeInteraction.js";
+const INT_CHUNK_DOCUMENT = "sys:ChunkDocument";
+export async function chunkDocument(payload) {
+    const { params, client, objectId } = await setupActivity(payload);
+    const { force } = params;
+    const interactionName = params.interactionName ?? INT_CHUNK_DOCUMENT;
+    log.info(`Object ${objectId} chunking started`);
+    const document = await client.objects.retrieve(objectId, "+text");
+    const type = document.type ? await client.types.retrieve(document.type.id) : undefined;
+    if (!type?.is_chunkable) {
+        log.warn('Type is not chunkable for object ID: ' + objectId);
+        return { id: objectId, status: "skipped", message: "type not chunkable" };
+    }
+    //check if text is present
+    if (!document.text) {
+        log.warn('No text found for object ID: ' + objectId);
+        return { id: objectId, status: "failed", message: "no text found" };
+    }
+    if (!force && document.parts && document.parts.length > 0 && document.parts_etag === document.text_etag) {
+        log.info('Document already chunked for object ID: ' + objectId);
+        return { id: objectId, status: "skipped", message: "document already chunked with correct etag" };
+    }
+    //instrument the text with line numbers
+    const lines = document.text.split('\n');
+    const instrumented = lines.map((l, i) => `{%${i}%}${l}`).join('\n');
+    const res = await executeInteractionFromActivity(client, interactionName, params, {
+        objectId: objectId,
+        content: instrumented
+    });
+    const jsonResult = res.result.object();
+    const parts = jsonResult.parts;
+    if (!parts || parts.length === 0) {
+        log.warn('No parts found for object ID: ' + objectId, res);
+        return { id: objectId, status: "failed", parts: [], message: "no parts found" };
+    }
+    /**
+     * Only create parts as document if the flag is set
+     */
+    if (params.createParts) {
+        const partDocs = await Promise.all(parts.map(async (part, i) => {
+            const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
+            const location = () => {
+                let location = document.location;
+                if (location.endsWith('/')) {
+                    location += document.name + "/" + part.type;
+                }
+                location += '/' + document.name + "/" + part.type;
+                return location;
+            };
+            const docPart = await client.objects.create({
+                name: part.name,
+                parent: objectId,
+                text: text,
+                location: location(),
+                properties: {
+                    part_number: i + 1,
+                    etag: document.text_etag,
+                    source_line_start: part.line_number_start,
+                    source_line_end: part.line_number_end,
+                    title: part.name
+                }
+            });
+            return docPart;
+        }));
+        //delete previous parts
+        if (document.parts && document.parts.length > 0) {
+            log.info('Deleting previous parts for object ID: ' + objectId, { parts: document.parts });
+            await Promise.all(document.parts.map(async (partId) => {
+                await client.objects.delete(partId);
+            }));
+        }
+        await client.objects.update(objectId, {
+            parts: partDocs.map(p => p.id),
+            parts_etag: document.text_etag
+        });
+    }
+    log.info(`Object ${objectId} chunking completed`, { parts: document.parts });
+    return { id: objectId, status: "completed", parts: document.parts };
+}
+//# sourceMappingURL=chunkDocument.js.map

package/lib/esm/activities/chunkDocument.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunkDocument.js","sourceRoot":"","sources":["../../../src/activities/chunkDocument.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAA8B,8BAA8B,EAAE,MAAM,yBAAyB,CAAC;AAErG,MAAM,kBAAkB,GAAG,mBAAmB,CAAA;AAyC9C,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAyD;IACzF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,aAAa,CAAsB,OAAO,CAAC,CAAC;IAEvF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,kBAAkB,CAAC;IAErE,GAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,mBAAmB,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC;QACtB,GAAG,CAAC,IAAI,CAAC,uCAAuC,GAAG,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC7E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjB,GAAG,CAAC,IAAI,CAAC,+BAA+B,GAAG,QAAQ,CAAC,CAAC;QACrD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,CAAA;IACvE,CAAC;IAED,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,SAAS,EAAE,CAAC;QACtG,GAAG,CAAC,IAAI,CAAC,0CAA0C,GAAG,QAAQ,CAAC,CAAC;QAChE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,4CAA4C,EAAE,CAAA;IACrG,CAAC;IAED,uCAAuC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEnE,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE;QAC9E,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,YAAY;KACxB,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;IAEvC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAkB,CAAC;IAC5C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC,gCAAgC,GAAG,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC3D,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAA;IACnF,CAAC;IAGD;;OAEG;IACH,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAErB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;YAE3D,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE1G,MAAM,QAAQ,GAAG,GAAG,EAAE;gBAClB,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;gBACjC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,QAAQ,IAAI,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAA;gBAC/C,CAAC;gBACD,QAAQ,IAAI,GAAG,GAAG,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;gBAClD,OAAO,QAAQ,CAAC;YACpB,CAAC,CAAA;YAED,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;gBACxC,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,QAAQ;gBAChB,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,QAAQ,EAAE;gBACpB,UAAU,EAAE;oBACR,WAAW,EAAE,CAAC,GAAG,CAAC;oBAClB,IAAI,EAAE,QAAQ,CAAC,SAAS;oBACxB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;oBACzC,eAAe,EAAE,IAAI,CAAC,eAAe;oBACrC,KAAK,EAAE,IAAI,CAAC,IAAI;iBACnB;aACJ,CAAC,CAAC;YACH,OAAO,OAAO,CAAC;QACnB,CAAC,CAAC,CAAC,CAAC;QAEJ,uBAAuB;QACvB,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,yCAAyC,GAAG,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YAC1F,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBAClD,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC,CAAC;QACR,CAAC;QAED,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE;YAClC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9B,UAAU,EAAE,QAAQ,CAAC,SAAS;SACjC,CAAC,CAAC;IACP,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,UAAU,QAAQ,qBAAqB,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAA;AAEvE,CAAC"}

package/lib/esm/activities/copyParentArtifacts.js

@@ -0,0 +1,124 @@
+import { log, activityInfo } from "@temporalio/activity";
+import { NodeStreamSource } from "@vertesia/client/node";
+import { Readable } from "stream";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+/**
+ * Directories in the agent workspace to copy from parent to child.
+ */
+const WORKSPACE_DIRECTORIES = ['scripts', 'files', 'skills', 'docs', 'out'];
+/**
+ * Files that should never be copied (child has its own).
+ */
+const EXCLUDED_FILES = ['conversation.json', 'tools.json'];
+/**
+ * Copy workspace artifacts from parent workflow's agent space to current workflow's agent space.
+ *
+ * Copies: scripts/, files/, skills/, docs/, out/
+ * Excludes: conversation.json, tools.json
+ */
+export async function copyParentArtifacts(payload) {
+    const { client, params } = await setupActivity(payload);
+    const childRunId = activityInfo().workflowExecution.runId;
+    const parentRunId = params.parent_run_id;
+    log.info(`Copying artifacts from parent ${parentRunId} to child ${childRunId}`);
+    // List all files in parent's agent space
+    const parentFiles = await client.files.listArtifacts(parentRunId);
+    if (parentFiles.length === 0) {
+        log.info("No artifacts in parent agent space");
+        return { copied: 0, files: [] };
+    }
+    const copiedFiles = [];
+    for (const fullPath of parentFiles) {
+        const relativePath = extractRelativePath(fullPath, parentRunId);
+        if (!relativePath)
+            continue;
+        // Only copy workspace directories
+        const dir = relativePath.split('/')[0];
+        if (!WORKSPACE_DIRECTORIES.includes(dir))
+            continue;
+        // Skip excluded files
+        const fileName = relativePath.split('/').pop() || '';
+        if (EXCLUDED_FILES.includes(fileName))
+            continue;
+        try {
+            await copyArtifact(client, parentRunId, childRunId, relativePath);
+            copiedFiles.push(relativePath);
+        }
+        catch (err) {
+            log.warn(`Failed to copy ${relativePath}: ${err.message}`);
+        }
+    }
+    log.info(`Copied ${copiedFiles.length} artifacts from parent`);
+    return { copied: copiedFiles.length, files: copiedFiles };
+}
+/**
+ * Extract relative path from full artifact storage path.
+ * Storage paths can be: "store_xxx/agents/{runId}/..." or "agents/{runId}/..."
+ */
+function extractRelativePath(fullPath, runId) {
+    // Look for the pattern: agents/{runId}/
+    const marker = `/agents/${runId}/`;
+    const idx = fullPath.lastIndexOf(marker);
+    if (idx >= 0) {
+        return fullPath.slice(idx + marker.length);
+    }
+    // Also try without leading slash
+    const legacyPrefix = `agents/${runId}/`;
+    if (fullPath.startsWith(legacyPrefix)) {
+        return fullPath.slice(legacyPrefix.length);
+    }
+    // If path contains the runId, try to extract after it
+    const runIdIdx = fullPath.indexOf(runId);
+    if (runIdIdx >= 0) {
+        const afterRunId = fullPath.slice(runIdIdx + runId.length);
+        if (afterRunId.startsWith('/')) {
+            return afterRunId.slice(1);
+        }
+    }
+    return null;
+}
+/**
+ * Copy a single artifact from parent to child agent space
+ */
+async function copyArtifact(client, parentRunId, childRunId, relativePath) {
+    // Download from parent's agent space
+    const stream = await client.files.downloadArtifact(parentRunId, relativePath);
+    // Convert web stream to node stream for upload
+    const nodeStream = Readable.fromWeb(stream);
+    // Determine mime type from extension
+    const mimeType = getMimeType(relativePath);
+    const fileName = relativePath.split('/').pop() || relativePath;
+    // Upload to child's agent space at the same relative path
+    const source = new NodeStreamSource(nodeStream, fileName, mimeType);
+    await client.files.uploadArtifact(childRunId, relativePath, source);
+}
+/**
+ * Get mime type from file extension
+ */
+function getMimeType(filePath) {
+    const ext = filePath.split('.').pop()?.toLowerCase();
+    const mimeTypes = {
+        'json': 'application/json',
+        'txt': 'text/plain',
+        'md': 'text/markdown',
+        'html': 'text/html',
+        'csv': 'text/csv',
+        'png': 'image/png',
+        'jpg': 'image/jpeg',
+        'jpeg': 'image/jpeg',
+        'gif': 'image/gif',
+        'svg': 'image/svg+xml',
+        'pdf': 'application/pdf',
+        'xml': 'application/xml',
+        'zip': 'application/zip',
+        'js': 'application/javascript',
+        'ts': 'application/typescript',
+        'py': 'text/x-python',
+        'sh': 'text/x-shellscript',
+        'bash': 'text/x-shellscript',
+        'yaml': 'text/yaml',
+        'yml': 'text/yaml',
+    };
+    return mimeTypes[ext || ''] || 'application/octet-stream';
+}
+//# sourceMappingURL=copyParentArtifacts.js.map

package/lib/esm/activities/copyParentArtifacts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"copyParentArtifacts.js","sourceRoot":"","sources":["../../../src/activities/copyParentArtifacts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAEhE;;GAEG;AACH,MAAM,qBAAqB,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;AAE5E;;GAEG;AACH,MAAM,cAAc,GAAG,CAAC,mBAAmB,EAAE,YAAY,CAAC,CAAC;AAW3D;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA+D;IAE/D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAA4B,OAAO,CAAC,CAAC;IACnF,MAAM,UAAU,GAAG,YAAY,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC;IAEzC,GAAG,CAAC,IAAI,CAAC,iCAAiC,WAAW,aAAa,UAAU,EAAE,CAAC,CAAC;IAEhF,yCAAyC;IACzC,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;IAElE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,GAAG,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QAC/C,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACpC,CAAC;IAED,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,QAAQ,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,YAAY,GAAG,mBAAmB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAChE,IAAI,CAAC,YAAY;YAAE,SAAS;QAE5B,kCAAkC;QAClC,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAEnD,sBAAsB;QACtB,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QACrD,IAAI,cAAc,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,SAAS;QAEhD,IAAI,CAAC;YACD,MAAM,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;YAClE,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,GAAG,CAAC,IAAI,CAAC,kBAAkB,YAAY,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,MAAM,wBAAwB,CAAC,CAAC;IAC/D,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;AAC9D,CAAC;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAAC,QAAgB,EAAE,KAAa;IACxD,wCAAwC;IACxC,MAAM,MAAM,GAAG,WAAW,KAAK,GAAG,CAAC;IACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QACX,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,iCAAiC;IACjC,MAAM,YAAY,GAAG,UAAU,KAAK,GAAG,CAAC;IACxC,IAAI,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACpC,OAAO,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,sDAAsD;IACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3D,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CACvB,MAAW,EACX,WAAmB,EACnB,UAAkB,EAClB,YAAoB;IAEpB,qCAAqC;IACrC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IAE9E,+CAA+C;IAC/C,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAa,CAAC,CAAC;IAEnD,qCAAqC;IACrC,MAAM,QAAQ,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,YAAY,CAAC;IAE/D,0DAA0D;IAC1D,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAC/B,UAAU,EACV,QAAQ,EACR,QAAQ,CACX,CAAC;IAEF,MAAM,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;AACxE,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,QAAgB;IACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;IACrD,MAAM,SAAS,GAA2B;QACtC,MAAM,EAAE,kBAAkB;QAC1B,KAAK,EAAE,YAAY;QACnB,IAAI,EAAE,eAAe;QACrB,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,UAAU;QACjB,KAAK,EAAE,WAAW;QAClB,KAAK,EAAE,YAAY;QACnB,MAAM,EAAE,YAAY;QACpB,KAAK,EAAE,WAAW;QAClB,KAAK,EAAE,eAAe;QACtB,KAAK,EAAE,iBAAiB;QACxB,KAAK,EAAE,iBAAiB;QACxB,KAAK,EAAE,iBAAiB;QACxB,IAAI,EAAE,wBAAwB;QAC9B,IAAI,EAAE,wBAAwB;QAC9B,IAAI,EAAE,eAAe;QACrB,IAAI,EAAE,oBAAoB;QAC1B,MAAM,EAAE,oBAAoB;QAC5B,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,WAAW;KACrB,CAAC;IACF,OAAO,SAAS,CAAC,GAAG,IAAI,EAAE,CAAC,IAAI,0BAA0B,CAAC;AAC9D,CAAC"}

package/lib/esm/activities/createDocumentFromOther.js

@@ -0,0 +1,58 @@
+import { log } from "@temporalio/activity";
+import { NodeStreamSource } from "@vertesia/client/node";
+import fs from 'fs';
+import { pdfExtractPages } from "../conversion/mutool.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { DocumentNotFoundError } from "../errors.js";
+import { saveBlobToTempFile } from "../utils/blobs.js";
+/**
+ * Create a new PDF by extracting pages from a source PDF
+ * @returns
+ */
+export async function createPdfDocumentFromSource(payload) {
+    const { client, objectId, params } = await setupActivity(payload);
+    const inputObject = await client.objects.retrieve(objectId);
+    const { pages, filename, title } = params;
+    log.info(`Creating PDF from source`, { objectId, pages, filename, title });
+    if (!pages || pages.length === 0) {
+        log.error(`No pages provided`);
+        throw new Error(`No pages provided`);
+    }
+    if (!inputObject) {
+        log.error(`Document ${objectId} not found`);
+        throw new DocumentNotFoundError(`Document ${objectId} not found`, [objectId]);
+    }
+    if (!inputObject.content?.source) {
+        log.error(`Document ${objectId} has no source`);
+        throw new DocumentNotFoundError(`Document ${objectId} has no source`, [objectId]);
+    }
+    if (!inputObject.content.type || (!inputObject.content.type?.startsWith('application/pdf'))) {
+        log.error(`Document ${objectId} is not an image`);
+        throw new DocumentNotFoundError(`Document ${objectId} is not an image or pdf: ${inputObject.content.type}`, [objectId]);
+    }
+    const targetType = await client.types.getTypeByName(params.target_object_type);
+    if (!targetType) {
+        log.error(`Type ${params.target_object_type} not found`);
+        throw new DocumentNotFoundError(`Type ${params.target_object_type} not found`);
+    }
+    const tmpFile = await saveBlobToTempFile(client, inputObject.content.source, ".pdf");
+    const newPdf = await pdfExtractPages(tmpFile, pages);
+    log.info(`PDF created from pages ${pages.join(', ')} `, { newPdf });
+    const name = `pages-${pages.join('-')}.pdf`;
+    const sourceToUpload = new NodeStreamSource(fs.createReadStream(newPdf), name, "application/pdf");
+    log.info(`Uploading file ${newPdf} `);
+    const upload = await client.objects.upload(sourceToUpload);
+    log.info(`File uploaded ${upload.source} `);
+    const newObject = await client.objects.create({
+        type: targetType.id,
+        name: title || targetType.name,
+        parent: objectId,
+        content: {
+            source: upload.source,
+            name: upload.name,
+            type: 'application/pdf'
+        }
+    });
+    return { newObjectId: newObject.id, uploadedFile: upload.name };
+}
+//# sourceMappingURL=createDocumentFromOther.js.map

package/lib/esm/activities/createDocumentFromOther.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createDocumentFromOther.js","sourceRoot":"","sources":["../../../src/activities/createDocumentFromOther.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAkBvD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,OAAuE;IACrH,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAoC,OAAO,CAAC,CAAC;IACrG,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAE5D,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAC1C,GAAG,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,GAAG,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,GAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,qBAAqB,CAAC,YAAY,QAAQ,YAAY,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClF,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC/B,GAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,gBAAgB,CAAC,CAAC;QAChD,MAAM,IAAI,qBAAqB,CAAC,YAAY,QAAQ,gBAAgB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACtF,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC;QAC1F,GAAG,CAAC,KAAK,CAAC,YAAY,QAAQ,kBAAkB,CAAC,CAAC;QAClD,MAAM,IAAI,qBAAqB,CAAC,YAAY,QAAQ,4BAA4B,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5H,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC/E,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,GAAG,CAAC,KAAK,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;QACzD,MAAM,IAAI,qBAAqB,CAAC,QAAQ,MAAM,CAAC,kBAAkB,YAAY,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACrD,GAAG,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,SAAS,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;IAE5C,MAAM,cAAc,GAAG,IAAI,gBAAgB,CACvC,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAC3B,IAAI,EACJ,iBAAiB,CACpB,CAAA;IAED,GAAG,CAAC,IAAI,CAAC,kBAAkB,MAAM,GAAG,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC3D,GAAG,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAE5C,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1C,IAAI,EAAE,UAAU,CAAC,EAAE;QACnB,IAAI,EAAE,KAAK,IAAI,UAAU,CAAC,IAAI;QAC9B,MAAM,EAAE,QAAQ;QAChB,OAAO,EAAE;YACL,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,IAAI,EAAE,iBAAiB;SAC1B;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AAGpE,CAAC"}

package/lib/esm/activities/executeInteraction.js

@@ -0,0 +1,190 @@
+import { activityInfo, log } from "@temporalio/activity";
+import { NodeStreamSource } from "@vertesia/client/node";
+import { ExecutionRunStatus, } from "@vertesia/common";
+import { projectResult } from "../dsl/projections.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { ActivityParamInvalidError, ActivityParamNotFoundError, ResourceExhaustedError } from "../errors.js";
+import { truncByMaxTokens } from "../utils/tokens.js";
+import { Readable } from "stream";
+//Example:
+//@ts-ignore
+const JSON = {
+    name: "executeInteraction",
+    import: ["defaultModel", "guidlineId", "docTypeId"],
+    params: {
+        defaultModel: "${model}",
+        interactionName: "GenerateSummary",
+        model: "${defaultModel ?? 'gpt4'}",
+        environment: "13456",
+        max_tokens: 100,
+        temperature: 0.5,
+        tags: ["test"],
+        result_schema: "${docType.object_schema}",
+        prompt_data: {
+            documents: "${documents}",
+            guidline: "${guidline.text}",
+        },
+    },
+    fetch: {
+        documents: {
+            type: "document",
+            query: {
+                id: { $in: "${objectIds}" },
+            },
+            select: "+text",
+        },
+        guidline: {
+            type: "document",
+            limit: 1,
+            query: {
+                id: "${guidlineId}",
+            },
+            select: "+text",
+            on_not_found: "throw",
+        },
+        docType: {
+            type: "document_type",
+            limit: 1,
+            query: {
+                id: "${docTypeId}",
+            },
+            select: "+object_schema",
+        },
+    },
+};
+export async function executeInteraction(payload) {
+    const { client, params } = await setupActivity(payload);
+    const { interactionName, prompt_data, static_prompt_data: wf_prompt_data } = params;
+    if (wf_prompt_data) {
+        Object.assign(prompt_data, wf_prompt_data);
+    }
+    if (!interactionName) {
+        log.error("Missing interactionName", { params });
+        throw new ActivityParamNotFoundError("interactionName", payload.activity);
+    }
+    if (params.truncate) {
+        const truncate = params.truncate;
+        for (const [key, value] of Object.entries(truncate)) {
+            prompt_data[key] = truncByMaxTokens(prompt_data[key], value);
+        }
+    }
+    try {
+        const res = await executeInteractionFromActivity(client, interactionName, params, prompt_data, payload.debug_mode);
+        let completionResult = res.result;
+        // Handle image uploads if the result contains base64 images
+        const imageResults = completionResult.filter(r => r.type === "image");
+        if (imageResults.length > 0) {
+            const uploadedImages = await Promise.all(completionResult.map(async (item, index) => {
+                if (item.type === "image") {
+                    const image = item.value;
+                    // Extract base64 data and create buffer
+                    const base64Data = image.replace(/^data:image\/[a-z]+;base64,/, "");
+                    const buffer = Buffer.from(base64Data, 'base64');
+                    // Generate filename
+                    const { runId } = activityInfo().workflowExecution;
+                    const { activityId } = activityInfo();
+                    const filename = `generated-image-${runId}-${activityId}-${index}.png`;
+                    // Create a readable stream from the buffer
+                    const stream = Readable.from(buffer);
+                    const source = new NodeStreamSource(stream, filename, "image/png");
+                    const file = await client.files.uploadFile(source);
+                    return { type: "image", value: file };
+                }
+                return item;
+            }));
+            completionResult = uploadedImages;
+        }
+        return projectResult(payload, params, res, {
+            runId: res.id,
+            status: res.status,
+            result: completionResult,
+        });
+    }
+    catch (error) {
+        log.error(`Failed to execute interaction ${interactionName}`, { error });
+        if (error.statusCode === 429 && params.exit_on_resource_exhaustion) {
+            throw new ResourceExhaustedError(error.statusCode, "Resource exhausted - rate limit exceeded");
+        }
+        else if (error.message.includes("Failed to validate merged prompt schema")) {
+            //issue with the input data, don't retry
+            throw new ActivityParamInvalidError("prompt_data", payload.activity, error.message);
+        }
+        else if (error.message.includes("modelId: Path `modelId` is required")) {
+            //issue with the input data, don't retry
+            throw new ActivityParamInvalidError("model", payload.activity, error.message);
+        }
+        else {
+            throw new Error(`Interaction Execution failed ${interactionName}: ${error.message}`);
+        }
+    }
+}
+export async function executeInteractionFromActivity(client, interactionName, params, prompt_data, debug) {
+    const userTags = params.tags;
+    const info = activityInfo();
+    const runId = info.workflowExecution.runId;
+    let tags = ["workflow"];
+    if (userTags) {
+        tags = tags.concat(userTags);
+    }
+    const workflow = {
+        run_id: info.workflowExecution.runId,
+        workflow_id: info.workflowExecution.workflowId,
+        activity_type: info.activityType,
+    };
+    let previousStudioExecutionRun = undefined;
+    if (params.include_previous_error) {
+        //retrieve last failed run if any
+        if (info.attempt > 1) {
+            log.info("Retrying, searching for previous run", { prev_run_id: runId });
+            const payload = {
+                query: { workflow_run_ids: [runId] },
+                limit: 1,
+            };
+            const previousRun = await client.runs.search(payload).then((res) => {
+                log.info("Search results", { results: res });
+                return res ? (res[0] ?? undefined) : undefined;
+            });
+            if (previousRun) {
+                log.info("Found previous run", { previousRun });
+                previousStudioExecutionRun = await client.runs.retrieve(previousRun.id);
+            }
+        }
+    }
+    if (debug && previousStudioExecutionRun?.error) {
+        log.info(`Found  previous run error`, { error: previousStudioExecutionRun?.error });
+    }
+    const config = {
+        environment: params.environment,
+        model: params.model,
+        model_options: params.model_options,
+        do_validate: params.validate_result,
+    };
+    const data = {
+        ...prompt_data,
+        previous_error: previousStudioExecutionRun?.error,
+    };
+    const result_schema = params.result_schema;
+    log.debug(`About to execute interaction ${interactionName}`, { config, data, result_schema, tags, workflow });
+    const res = await client.interactions
+        .executeByName(interactionName, {
+        config,
+        data,
+        result_schema,
+        tags,
+        stream: false,
+        workflow,
+    })
+        .catch((err) => {
+        log.error(`Error executing interaction ${interactionName}`, { err });
+        throw err;
+    });
+    if (debug) {
+        log.info(`Interaction executed ${interactionName}`, res);
+    }
+    if (res.error || res.status === ExecutionRunStatus.failed) {
+        log.error(`Error executing interaction ${interactionName}`, { error: res.error });
+        throw new Error(`Interaction Execution failed ${interactionName}: ${res.error}`);
+    }
+    return res;
+}
+//# sourceMappingURL=executeInteraction.js.map

package/lib/esm/activities/executeInteraction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"executeInteraction.js","sourceRoot":"","sources":["../../../src/activities/executeInteraction.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAIH,kBAAkB,GAIrB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAC7G,OAAO,EAAgB,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,UAAU;AACV,YAAY;AACZ,MAAM,IAAI,GAAoB;IAC1B,IAAI,EAAE,oBAAoB;IAC1B,MAAM,EAAE,CAAC,cAAc,EAAE,YAAY,EAAE,WAAW,CAAC;IACnD,MAAM,EAAE;QACJ,YAAY,EAAE,UAAU;QACxB,eAAe,EAAE,iBAAiB;QAClC,KAAK,EAAE,2BAA2B;QAClC,WAAW,EAAE,OAAO;QACpB,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,GAAG;QAChB,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,aAAa,EAAE,0BAA0B;QACzC,WAAW,EAAE;YACT,SAAS,EAAE,cAAc;YACzB,QAAQ,EAAE,kBAAkB;SAC/B;KACJ;IACD,KAAK,EAAE;QACH,SAAS,EAAE;YACP,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE;gBACH,EAAE,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE;aAC9B;YACD,MAAM,EAAE,OAAO;SAClB;QACD,QAAQ,EAAE;YACN,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,eAAe;aACtB;YACD,MAAM,EAAE,OAAO;YACf,YAAY,EAAE,OAAO;SACxB;QACD,OAAO,EAAE;YACL,IAAI,EAAE,eAAe;YACrB,KAAK,EAAE,CAAC;YACR,KAAK,EAAE;gBACH,EAAE,EAAE,cAAc;aACrB;YACD,MAAM,EAAE,gBAAgB;SAC3B;KACJ;CACJ,CAAC;AAiEF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAA8D;IACnG,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAA2B,OAAO,CAAC,CAAC;IAElF,MAAM,EAAE,eAAe,EAAE,WAAW,EAAE,kBAAkB,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACpF,IAAI,cAAc,EAAE,CAAC;QACjB,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACnB,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACjD,MAAM,IAAI,0BAA0B,CAAC,iBAAiB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,GAAG,CAAC,GAAG,gBAAgB,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;IAED,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,8BAA8B,CAC5C,MAAM,EACN,eAAe,EACf,MAAM,EACN,WAAW,EACX,OAAO,CAAC,UAAU,CACrB,CAAC;QAEF,IAAI,gBAAgB,GAAuB,GAAG,CAAC,MAAM,CAAC;QAEtD,4DAA4D;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;QACtE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,gBAAgB,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;gBACvC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;oBACzB,wCAAwC;oBACxC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC;oBACpE,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;oBAEjD,oBAAoB;oBACpB,MAAM,EAAE,KAAK,EAAE,GAAG,YAAY,EAAE,CAAC,iBAAiB,CAAC;oBACnD,MAAM,EAAE,UAAU,EAAE,GAAG,YAAY,EAAE,CAAC;oBACtC,MAAM,QAAQ,GAAG,mBAAmB,KAAK,IAAI,UAAU,IAAI,KAAK,MAAM,CAAC;oBAEvE,2CAA2C;oBAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAErC,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAC/B,MAAM,EACN,QAAQ,EACR,WAAW,CACd,CAAC;oBAEF,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;oBACnD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAsB,CAAC;gBAC9D,CAAC;gBACD,OAAO,IAAI,CAAC;YAChB,CAAC,CAAC,CACL,CAAC;YACF,gBAAgB,GAAG,cAAc,CAAC;QACtC,CAAC;QAED,OAAO,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE;YACvC,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,MAAM,EAAE,gBAAgB;SAC3B,CAAC,CAAC;IAEP,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QAClB,GAAG,CAAC,KAAK,CAAC,iCAAiC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACzE,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,IAAI,MAAM,CAAC,2BAA2B,EAAE,CAAC;YACjE,MAAM,IAAI,sBAAsB,CAAC,KAAK,CAAC,UAAU,EAAE,0CAA0C,CAAC,CAAC;QACnG,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,yCAAyC,CAAC,EAAE,CAAC;YAC3E,wCAAwC;YACxC,MAAM,IAAI,yBAAyB,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACxF,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,qCAAqC,CAAC,EAAE,CAAC;YACvE,wCAAwC;YACxC,MAAM,IAAI,yBAAyB,CAAC,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAClF,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzF,CAAC;IACL,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAChD,MAAsB,EACtB,eAAuB,EACvB,MAAkC,EAClC,WAAgB,EAChB,KAAe;IAEf,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;IAC7B,MAAM,IAAI,GAAG,YAAY,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACxB,IAAI,QAAQ,EAAE,CAAC;QACX,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IACD,MAAM,QAAQ,GAAyB;QACnC,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,KAAK;QACpC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,UAAU;QAC9C,aAAa,EAAE,IAAI,CAAC,YAAY;KACnC,CAAC;IAEF,IAAI,0BAA0B,GAA6B,SAAS,CAAC;IACrE,IAAI,MAAM,CAAC,sBAAsB,EAAE,CAAC;QAChC,iCAAiC;QACjC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACzE,MAAM,OAAO,GAAqB;gBAC9B,KAAK,EAAE,EAAE,gBAAgB,EAAE,CAAC,KAAK,CAAC,EAAE;gBACpC,KAAK,EAAE,CAAC;aACX,CAAC;YACF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/D,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBACd,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBAChD,0BAA0B,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IACD,IAAI,KAAK,IAAI,0BAA0B,EAAE,KAAK,EAAE,CAAC;QAC7C,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,MAAM,GAAsC;QAC9C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,WAAW,EAAE,MAAM,CAAC,eAAe;KACtC,CAAC;IACF,MAAM,IAAI,GAAG;QACT,GAAG,WAAW;QACd,cAAc,EAAE,0BAA0B,EAAE,KAAK;KACpD,CAAC;IAEF,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;IAE3C,GAAG,CAAC,KAAK,CAAC,gCAAgC,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE9G,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,YAAY;SAChC,aAAa,CAAC,eAAe,EAAE;QAC5B,MAAM;QACN,IAAI;QACJ,aAAa;QACb,IAAI;QACJ,MAAM,EAAE,KAAK;QACb,QAAQ;KACX,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,GAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrE,MAAM,GAAG,CAAC;IACd,CAAC,CAAC,CAAC;IAEP,IAAI,KAAK,EAAE,CAAC;QACR,GAAG,CAAC,IAAI,CAAC,wBAAwB,eAAe,EAAE,EAAE,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,kBAAkB,CAAC,MAAM,EAAE,CAAC;QACxD,GAAG,CAAC,KAAK,CAAC,+BAA+B,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,gCAAgC,eAAe,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}

package/lib/esm/activities/extractDocumentText.js

@@ -0,0 +1,153 @@
+import { log } from "@temporalio/activity";
+import { markdownWithMarkitdown } from "../conversion/markitdown.js";
+import { mutoolPdfToText } from "../conversion/mutool.js";
+import { markdownWithPandoc } from "../conversion/pandoc.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { DocumentNotFoundError } from "../errors.js";
+import { TextExtractionStatus } from "../result-types.js";
+import { fetchBlobAsBuffer, md5 } from "../utils/blobs.js";
+import { countTokens } from "../utils/tokens.js";
+//@ts-ignore
+const JSON = {
+    name: "extractDocumentText",
+};
+export async function extractDocumentText(payload) {
+    const { client, objectId } = await setupActivity(payload);
+    const r = await client.objects.find({
+        query: { _id: objectId },
+        limit: 1,
+        select: "+text",
+    });
+    const doc = r[0];
+    if (!doc) {
+        log.error(`Document ${objectId} not found`);
+        throw new DocumentNotFoundError(`Document ${objectId} not found`, payload.objectIds);
+    }
+    log.info(`Extracting text for object ${doc.id}`);
+    if (!doc.content?.type || !doc.content?.source) {
+        if (doc.text) {
+            return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text present and no source or type");
+        }
+        else {
+            return createResponse(doc, "", TextExtractionStatus.error, "No source or type found");
+        }
+    }
+    //skip if text already extracted and proper etag
+    if (doc.text && doc.text.length > 0 && doc.text_etag === doc.content.etag) {
+        return createResponse(doc, doc.text, TextExtractionStatus.skipped, "Text already extracted");
+    }
+    let fileBuffer;
+    try {
+        fileBuffer = await fetchBlobAsBuffer(client, doc.content.source);
+    }
+    catch (e) {
+        log.error(`Error reading file: ${e}`);
+        return createResponse(doc, "", TextExtractionStatus.error, e.message);
+    }
+    let txt;
+    switch (doc.content.type) {
+        case "application/pdf":
+            txt = await mutoolPdfToText(fileBuffer);
+            break;
+        case "text/plain":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //docx
+        case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+            txt = await markdownWithMarkitdown(fileBuffer, "docx");
+            break;
+        //pptx
+        case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
+            txt = await markdownWithMarkitdown(fileBuffer, "pptx");
+            break;
+        //html
+        case "text/html":
+            txt = await markdownWithPandoc(fileBuffer, "html");
+            break;
+        //opendocument
+        case "application/vnd.oasis.opendocument.text":
+            txt = await markdownWithPandoc(fileBuffer, "odt");
+            break;
+        //rtf
+        case "application/rtf":
+            txt = await markdownWithPandoc(fileBuffer, "rtf");
+            break;
+        //markdown
+        case "text/markdown":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //csv
+        case "text/csv":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //typescript
+        case "application/typescript":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //javascript
+        case "application/javascript":
+            txt = fileBuffer.toString("utf8");
+            break;
+        //json
+        case "application/json":
+            txt = fileBuffer.toString("utf8");
+            break;
+        default:
+            if (sniffIfText(fileBuffer)) {
+                txt = fileBuffer.toString("utf8"); //TODO: add charset detection
+                break;
+            }
+            return createResponse(doc, doc.text ?? "", TextExtractionStatus.skipped, `Unsupported mime type: ${doc.content.type}`);
+    }
+    const tokensData = countTokens(txt);
+    const etag = doc.content.etag ?? md5(txt);
+    const updateData = {
+        text: txt,
+        text_etag: etag,
+        tokens: {
+            ...tokensData,
+            etag: etag,
+        },
+    };
+    await client.objects.update(doc.id, updateData);
+    return createResponse(doc, txt, TextExtractionStatus.success);
+}
+function createResponse(doc, text, status, message) {
+    return {
+        status,
+        message,
+        tokens: doc.tokens,
+        len: text.length,
+        objectId: doc.id,
+        hasText: !!text,
+    };
+}
+function sniffIfText(buf) {
+    // If file is too large, don't even try
+    if (buf.length > 500 * 1024) {
+        return false;
+    }
+    // Count binary/control characters
+    let binaryCount = 0;
+    const sampleSize = Math.min(buf.length, 1000); // Check first 1000 bytes
+    for (let i = 0; i < sampleSize; i++) {
+        // Count control characters (except common whitespace)
+        const byte = buf[i];
+        if ((byte < 32 && ![9, 10, 13].includes(byte)) || byte === 0) {
+            binaryCount++;
+        }
+    }
+    // If more than 10% binary/control chars, probably not text
+    if (binaryCount / sampleSize > 0.1) {
+        return false;
+    }
+    // Additional check for valid UTF-8 encoding
+    try {
+        const s = buf.toString("utf8");
+        return s.length > 0 && !s.includes("\uFFFD"); // Replacement character
+    }
+    catch (e) {
+        return false;
+    }
+}
+//# sourceMappingURL=extractDocumentText.js.map