@vertesia/workflow 0.50.1 → 0.52.0

package/src/system/notifyWebhookWorkflow.ts

@@ -1,8 +1,9 @@
 
 import { log } from "@temporalio/workflow";
 import { ContentEventName, WorkflowExecutionPayload } from "@vertesia/common";
-import * as activities from "../activities/index-dsl.js";
+import * as activities from "../activities/notifyWebhook.js";
 import { dslProxyActivities } from "../dsl/dslProxyActivities.js";
+import { WF_NON_RETRYABLE_ERRORS } from "../errors.js";
 
 const {
     notifyWebhook
@@ -13,7 +14,7 @@ const {
         backoffCoefficient: 2,
         maximumAttempts: 5,
         maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
+        nonRetryableErrorTypes: WF_NON_RETRYABLE_ERRORS,
     },
 });
 

package/src/system/recalculateEmbeddingsWorkflow.ts

@@ -2,7 +2,7 @@
 import { SupportedEmbeddingTypes, WorkflowExecutionPayload } from "@vertesia/common";
 import * as activities from "../activities/index-dsl.js";
 import { dslProxyActivities } from "../dsl/dslProxyActivities.js";
-import { NoDocumentFound } from "../errors.js";
+import { WF_NON_RETRYABLE_ERRORS } from "../errors.js";
 
 const {
     generateEmbeddings,
@@ -13,7 +13,7 @@ const {
         backoffCoefficient: 2,
         maximumAttempts: 10,
         maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [NoDocumentFound.name],
+        nonRetryableErrorTypes: WF_NON_RETRYABLE_ERRORS,
     },
 });
 

package/src/utils/blobs.ts

@@ -10,7 +10,12 @@ export async function fetchBlobAsStream(client: VertesiaClient, blobUri: string)
     try {
         return await client.files.downloadFile(blobUri);
     } catch (err: any) {
-        throw new NoDocumentFound(`Failed to download blob ${blobUri}: ${err.message}`, []);
+        if (err.message.includes("not found")) {
+            //TODO improve error handling with a fetch fail error class in the client
+            throw new NoDocumentFound(`Failed to download blob ${blobUri}: ${err.message}`, []);
+        } else {
+            throw new Error(`Failed to download blob ${blobUri}: ${err.message}`);
+        }
     }
 }
 export async function fetchBlobAsBuffer(client: VertesiaClient, blobUri: string): Promise<Buffer> {
@@ -24,7 +29,7 @@ export async function fetchBlobAsBuffer(client: VertesiaClient, blobUri: string)
 
 export async function fetchBlobAsBase64(client: VertesiaClient, blobUri: string): Promise<string> {
     const buffer = await fetchBlobAsBuffer(client, blobUri);
-    return buffer.toString('base64');
+    return buffer.toString("base64");
 }
 
 export async function saveBlobToFile(client: VertesiaClient, blobUri: string, toFile: string): Promise<void> {
@@ -35,9 +40,9 @@ export async function saveBlobToFile(client: VertesiaClient, blobUri: string, to
 
 export async function saveBlobToTempFile(client: VertesiaClient, blobUri: string, fileExt?: string): Promise<string> {
     const tmpFile = tmp.fileSync({
-        prefix: "composable-activity-",
-        postfix: fileExt,
-        discardDescriptor: true
+        prefix: "vertesia-activity-",
+        postfix: fileExt ? "." + fileExt : "",
+        discardDescriptor: true,
     });
     await saveBlobToFile(client, blobUri, tmpFile.name);
     return tmpFile.name;
@@ -47,12 +52,12 @@ async function writeChunksToStream(chunks: AsyncIterable<Uint8Array>, out: NodeJ
     for await (const chunk of chunks) {
         if (!out.write(chunk)) {
             // If the internal buffer is full, wait until it's drained
-            await new Promise(resolve => out.once('drain', resolve));
+            await new Promise((resolve) => out.once("drain", resolve));
         }
     }
     out.end(); // Close the stream when done
 }
 
 export function md5(contents: string) {
-    return crypto.createHash('md5').update(contents).digest("hex");
+    return crypto.createHash("md5").update(contents).digest("hex");
 }

package/src/utils/chunks.ts

@@ -0,0 +1,17 @@
+
+export interface DocPart {
+    line_number_start: number
+    line_number_end: number
+    name: string
+    type: string
+}
+
+export const getContentPart = (content: string, part: DocPart): string => {
+    const lines = content.split('\n');
+    const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
+    return text;
+}
+
+export const getContentParts = (content: string, parts: DocPart[]): string[] => {
+    return parts.map(part => getContentPart(content, part));
+}

package/src/utils/client.ts

@@ -3,21 +3,22 @@
  */
 
 import { VertesiaClient } from "@vertesia/client";
-import { WorkflowExecutionPayload } from "@vertesia/common";
+import { WorkflowExecutionBaseParams } from "@vertesia/common";
+import { WorkflowParamNotFound } from "../errors.js";
 
 
-export function getClient(payload: WorkflowExecutionPayload) {
+export function getVertesiaClient(payload: WorkflowExecutionBaseParams) {
 
     if (!payload.auth_token) {
-        throw new Error("Authentication Token is missing from WorkflowExecutionPayload.authToken");
+        throw new WorkflowParamNotFound("Authentication Token is missing from WorkflowExecutionPayload.authToken");
     }
 
     if (!payload.config?.studio_url) {
-        throw new Error("Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl");
+        throw new WorkflowParamNotFound("Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl");
     }
 
     if (!payload.config?.store_url) {
-        throw new Error("Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl");
+        throw new WorkflowParamNotFound("Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl");
     }
 
     const client = new VertesiaClient({

package/src/utils/memory.ts

@@ -1,9 +1,9 @@
-import { VertesiaClient, StreamSource } from "@vertesia/client";
+import { VertesiaClient } from "@vertesia/client";
+import { NodeStreamSource } from "@vertesia/client/node";
 import { Commands, MemoryPack, buildMemoryPack as _buildMemoryPack, loadMemoryPack as _loadMemoryPack } from "@vertesia/memory";
 import { createReadStream, createWriteStream } from "fs";
 import { rm } from "fs/promises";
-import { readableToWebStream, webStreamToReadable } from "node-web-stream-adapters";
-import { Readable } from "stream";
+import { webStreamToReadable } from "node-web-stream-adapters";
 import { pipeline } from "stream/promises";
 
 import tmp from "tmp";
@@ -11,11 +11,6 @@ import zlib from "zlib";
 
 tmp.setGracefulCleanup();
 
-export class NodeStreamSource extends StreamSource {
-    constructor(stream: Readable, name: string, type?: string, id?: string) {
-        super(readableToWebStream(stream), name, type, id);
-    }
-}
 
 export async function publishMemoryPack(client: VertesiaClient, file: string, name: string): Promise<void> {
     const stream = createReadStream(file);

package/src/workflows.ts

@@ -3,7 +3,5 @@
  */
 export { dslWorkflow } from "./dsl/dsl-workflow.js";
 export { iterativeGenerationWorkflow } from "./iterative-generation/iterativeGenerationWorkflow.js";
-export { generateObjectText } from "./system/generateObjectText.js";
 export { notifyWebhookWorkflow } from "./system/notifyWebhookWorkflow.js";
 export { recalculateEmbeddingsWorkflow } from "./system/recalculateEmbeddingsWorkflow.js";
-

package/lib/cjs/conversion/pdf.js

@@ -1,13 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.trasformPdfToMarkdown = trasformPdfToMarkdown;
-const pdf2md_1 = __importDefault(require("@opendocsg/pdf2md"));
-const pdf2mdFn = pdf2md_1.default;
-function trasformPdfToMarkdown(buffer) {
-    const arr = new Uint8Array(buffer);
-    return pdf2mdFn(arr);
-}
-//# sourceMappingURL=pdf.js.map

package/lib/cjs/conversion/pdf.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../src/conversion/pdf.ts"],"names":[],"mappings":";;;;;AAIA,sDAGC;AAPD,+DAAuC;AAEvC,MAAM,QAAQ,GAAG,gBAA4D,CAAC;AAE9E,SAAgB,qBAAqB,CAAC,MAAc;IAChD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}

package/lib/cjs/system/generateObjectText.js

@@ -1,76 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateObjectText = generateObjectText;
-const workflow_1 = require("@temporalio/workflow");
-const dslProxyActivities_js_1 = require("../dsl/dslProxyActivities.js");
-const errors_js_1 = require("../errors.js");
-const { getObjectFromStore, extractDocumentText } = (0, dslProxyActivities_js_1.dslProxyActivities)("generateTextWorkflow", {
-    startToCloseTimeout: "5 minute",
-    retry: {
-        initialInterval: '5s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-const { transcribeMedia, convertPdfToStructuredText } = (0, dslProxyActivities_js_1.dslProxyActivities)("generateTextWorkflow", {
-    startToCloseTimeout: "30 minute",
-    retry: {
-        initialInterval: '30s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-async function generateObjectText(payload) {
-    const { objectIds } = payload;
-    const objectId = objectIds[0];
-    const object = await getObjectFromStore(payload, {});
-    if (!object.content?.source) {
-        throw new errors_js_1.NoDocumentFound(`No source or mimetype found for object ${objectId}`, objectIds);
-    }
-    const mimetype = object.content.type;
-    if (!mimetype) {
-        throw new errors_js_1.NoDocumentFound(`No mimetype found for object ${objectId}`, objectIds);
-    }
-    const converter = ConverterActivity.find(({ type }) => type.test(mimetype));
-    if (!converter) {
-        throw new errors_js_1.NoDocumentFound(`No converter found for mimetype ${mimetype}`, objectIds);
-    }
-    workflow_1.log.info(`Converting file type ${mimetype} to text with ${converter.name}`);
-    const res = await converter.activity(payload)(payload, converter.params);
-    workflow_1.log.info("Generated text for object", { res, objectId });
-    return res;
-}
-const ConverterActivity = [
-    {
-        type: /application\/pdf/,
-        activity: (payload) => {
-            const useTextractForPDF = payload.vars?.useTextractForPdf ?? false;
-            return useTextractForPDF ? convertPdfToStructuredText : extractDocumentText;
-        },
-        name: "ConvertPdfToStructuredText",
-        params: {},
-    },
-    {
-        type: /audio\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /video\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /.+/,
-        activity: () => extractDocumentText,
-        name: "extractText",
-        params: {},
-    }
-];
-//# sourceMappingURL=generateObjectText.js.map

package/lib/cjs/system/generateObjectText.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"generateObjectText.js","sourceRoot":"","sources":["../../../src/system/generateObjectText.ts"],"names":[],"mappings":";;AAqCA,gDA0BC;AA9DD,mDAA2C;AAG3C,wEAAkE;AAClE,4CAA+C;AAG/C,MAAM,EACF,kBAAkB,EAClB,mBAAmB,EACtB,GAAG,IAAA,0CAAkB,EAAoB,sBAAsB,EAAE;IAC9D,mBAAmB,EAAE,UAAU;IAC/B,KAAK,EAAE;QACH,eAAe,EAAE,IAAI;QACrB,kBAAkB,EAAE,CAAC;QACrB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,IAAI;QACtC,sBAAsB,EAAE,EAAE;KAC7B;CACJ,CAAC,CAAC;AAEH,MAAM,EACF,eAAe,EACf,0BAA0B,EAC7B,GAAG,IAAA,0CAAkB,EAAoB,sBAAsB,EAAE;IAC9D,mBAAmB,EAAE,WAAW;IAChC,KAAK,EAAE;QACH,eAAe,EAAE,KAAK;QACtB,kBAAkB,EAAE,CAAC;QACrB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,IAAI;QACtC,sBAAsB,EAAE,EAAE;KAC7B;CACJ,CAAC,CAAC;AAGI,KAAK,UAAU,kBAAkB,CAAC,OAAiC;IAEtE,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAErD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,2BAAe,CAAC,0CAA0C,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;IACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,2BAAe,CAAC,gCAAgC,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,MAAM,IAAI,2BAAe,CAAC,mCAAmC,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IACxF,CAAC;IACD,cAAG,CAAC,IAAI,CAAC,wBAAwB,QAAQ,iBAAiB,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IAE5E,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAEzE,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IACzD,OAAO,GAAG,CAAC;AAEf,CAAC;AAGD,MAAM,iBAAiB,GAAG;IACtB;QACI,IAAI,EAAE,kBAAkB;QACxB,QAAQ,EAAE,CAAC,OAAiC,EAAE,EAAE;YAC5C,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,EAAE,iBAAiB,IAAI,KAAK,CAAC;YACnE,OAAO,iBAAiB,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,mBAAmB,CAAC;QAChF,CAAC;QACD,IAAI,EAAE,4BAA4B;QAClC,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,GAAG,EAAE,CAAC,eAAe;QAC/B,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,GAAG,EAAE,CAAC,eAAe;QAC/B,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,IAAI;QACV,QAAQ,EAAE,GAAG,EAAE,CAAC,mBAAmB;QACnC,IAAI,EAAE,aAAa;QACnB,MAAM,EAAE,EAAE;KACb;CACJ,CAAA"}

package/lib/esm/conversion/pdf.js

@@ -1,7 +0,0 @@
-import pdf2md from "@opendocsg/pdf2md";
-const pdf2mdFn = pdf2md;
-export function trasformPdfToMarkdown(buffer) {
-    const arr = new Uint8Array(buffer);
-    return pdf2mdFn(arr);
-}
-//# sourceMappingURL=pdf.js.map

package/lib/esm/conversion/pdf.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../src/conversion/pdf.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,mBAAmB,CAAC;AAEvC,MAAM,QAAQ,GAAG,MAA4D,CAAC;AAE9E,MAAM,UAAU,qBAAqB,CAAC,MAAc;IAChD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}

package/lib/esm/system/generateObjectText.js

@@ -1,73 +0,0 @@
-import { log } from "@temporalio/workflow";
-import { dslProxyActivities } from "../dsl/dslProxyActivities.js";
-import { NoDocumentFound } from "../errors.js";
-const { getObjectFromStore, extractDocumentText } = dslProxyActivities("generateTextWorkflow", {
-    startToCloseTimeout: "5 minute",
-    retry: {
-        initialInterval: '5s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-const { transcribeMedia, convertPdfToStructuredText } = dslProxyActivities("generateTextWorkflow", {
-    startToCloseTimeout: "30 minute",
-    retry: {
-        initialInterval: '30s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-export async function generateObjectText(payload) {
-    const { objectIds } = payload;
-    const objectId = objectIds[0];
-    const object = await getObjectFromStore(payload, {});
-    if (!object.content?.source) {
-        throw new NoDocumentFound(`No source or mimetype found for object ${objectId}`, objectIds);
-    }
-    const mimetype = object.content.type;
-    if (!mimetype) {
-        throw new NoDocumentFound(`No mimetype found for object ${objectId}`, objectIds);
-    }
-    const converter = ConverterActivity.find(({ type }) => type.test(mimetype));
-    if (!converter) {
-        throw new NoDocumentFound(`No converter found for mimetype ${mimetype}`, objectIds);
-    }
-    log.info(`Converting file type ${mimetype} to text with ${converter.name}`);
-    const res = await converter.activity(payload)(payload, converter.params);
-    log.info("Generated text for object", { res, objectId });
-    return res;
-}
-const ConverterActivity = [
-    {
-        type: /application\/pdf/,
-        activity: (payload) => {
-            const useTextractForPDF = payload.vars?.useTextractForPdf ?? false;
-            return useTextractForPDF ? convertPdfToStructuredText : extractDocumentText;
-        },
-        name: "ConvertPdfToStructuredText",
-        params: {},
-    },
-    {
-        type: /audio\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /video\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /.+/,
-        activity: () => extractDocumentText,
-        name: "extractText",
-        params: {},
-    }
-];
-//# sourceMappingURL=generateObjectText.js.map

package/lib/esm/system/generateObjectText.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"generateObjectText.js","sourceRoot":"","sources":["../../../src/system/generateObjectText.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AAG3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAG/C,MAAM,EACF,kBAAkB,EAClB,mBAAmB,EACtB,GAAG,kBAAkB,CAAoB,sBAAsB,EAAE;IAC9D,mBAAmB,EAAE,UAAU;IAC/B,KAAK,EAAE;QACH,eAAe,EAAE,IAAI;QACrB,kBAAkB,EAAE,CAAC;QACrB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,IAAI;QACtC,sBAAsB,EAAE,EAAE;KAC7B;CACJ,CAAC,CAAC;AAEH,MAAM,EACF,eAAe,EACf,0BAA0B,EAC7B,GAAG,kBAAkB,CAAoB,sBAAsB,EAAE;IAC9D,mBAAmB,EAAE,WAAW;IAChC,KAAK,EAAE;QACH,eAAe,EAAE,KAAK;QACtB,kBAAkB,EAAE,CAAC;QACrB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,IAAI;QACtC,sBAAsB,EAAE,EAAE;KAC7B;CACJ,CAAC,CAAC;AAGH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAiC;IAEtE,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAErD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,eAAe,CAAC,0CAA0C,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;IACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,eAAe,CAAC,gCAAgC,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,MAAM,IAAI,eAAe,CAAC,mCAAmC,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IACxF,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,wBAAwB,QAAQ,iBAAiB,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IAE5E,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAEzE,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IACzD,OAAO,GAAG,CAAC;AAEf,CAAC;AAGD,MAAM,iBAAiB,GAAG;IACtB;QACI,IAAI,EAAE,kBAAkB;QACxB,QAAQ,EAAE,CAAC,OAAiC,EAAE,EAAE;YAC5C,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,EAAE,iBAAiB,IAAI,KAAK,CAAC;YACnE,OAAO,iBAAiB,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,mBAAmB,CAAC;QAChF,CAAC;QACD,IAAI,EAAE,4BAA4B;QAClC,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,GAAG,EAAE,CAAC,eAAe;QAC/B,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,GAAG,EAAE,CAAC,eAAe;QAC/B,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,IAAI;QACV,QAAQ,EAAE,GAAG,EAAE,CAAC,mBAAmB;QACnC,IAAI,EAAE,aAAa;QACnB,MAAM,EAAE,EAAE;KACb;CACJ,CAAA"}

package/lib/types/conversion/pdf.d.ts

@@ -1,2 +0,0 @@
-export declare function trasformPdfToMarkdown(buffer: Buffer): Promise<string>;
-//# sourceMappingURL=pdf.d.ts.map

package/lib/types/conversion/pdf.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../../src/conversion/pdf.ts"],"names":[],"mappings":"AAIA,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,mBAGnD"}

package/lib/types/system/generateObjectText.d.ts

@@ -1,4 +0,0 @@
-import { WorkflowExecutionPayload } from "@vertesia/common";
-import { TextExtractionResult } from "../index.js";
-export declare function generateObjectText(payload: WorkflowExecutionPayload): Promise<TextExtractionResult>;
-//# sourceMappingURL=generateObjectText.d.ts.map

package/lib/types/system/generateObjectText.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"generateObjectText.d.ts","sourceRoot":"","sources":["../../../src/system/generateObjectText.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AAI5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AA+BnD,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA0BzG"}

package/src/conversion/pdf.test.ts

@@ -1,35 +0,0 @@
-import { MockActivityEnvironment, TestWorkflowEnvironment } from '@temporalio/testing';
-import fs from 'fs';
-import path from 'path';
-import { beforeAll, expect, test } from 'vitest';
-import { trasformPdfToMarkdown } from '../conversion/pdf';
-
-
-let testEnv: TestWorkflowEnvironment;
-let activityContext: MockActivityEnvironment;
-
-beforeAll(async () => {
-  //testEnv = await TestWorkflowEnvironment.createLocal();
-  activityContext = new MockActivityEnvironment();
-});
-
-
-test('Converts a PDF to markdown', async () => {
-  const pdfPath = path.resolve(__dirname, '../../fixtures', 'test-pdf1.pdf');
-  const pdfBuffer = fs.readFileSync(pdfPath);
-  const result = await activityContext.run(trasformPdfToMarkdown, pdfBuffer);
-
-  expect(result).toContain('America');
-});
-
-// https://github.com/becomposable/studio/issues/432 Skip tests
-// The file "test-pdf2.pdf" could not be opened. It may be damanged or use a file format that
-// Preview doesn't recognize.
-test.skip('Converts another PDF to markdown', async () => {
-    const pdfPath = path.resolve(__dirname, '../../fixtures', 'test-pdf2.pdf');
-    const pdfBuffer = fs.readFileSync(pdfPath);
-    const result = await activityContext.run(trasformPdfToMarkdown, pdfBuffer);
-
-    expect(result).toContain('America');    
-
-  });

package/src/conversion/pdf.ts

@@ -1,8 +0,0 @@
-import pdf2md from "@opendocsg/pdf2md";
-
-const pdf2mdFn = pdf2md as unknown as (buffer: Uint8Array) => Promise<string>;
-
-export function trasformPdfToMarkdown(buffer: Buffer) {
-    const arr = new Uint8Array(buffer);
-    return pdf2mdFn(arr);
-}

package/src/system/generateObjectText.ts

@@ -1,95 +0,0 @@
-
-import { log } from "@temporalio/workflow";
-import { WorkflowExecutionPayload } from "@vertesia/common";
-import * as activities from "../activities/index-dsl.js";
-import { dslProxyActivities } from "../dsl/dslProxyActivities.js";
-import { NoDocumentFound } from "../errors.js";
-import { TextExtractionResult } from "../index.js";
-
-const {
-    getObjectFromStore,
-    extractDocumentText
-} = dslProxyActivities<typeof activities>("generateTextWorkflow", {
-    startToCloseTimeout: "5 minute",
-    retry: {
-        initialInterval: '5s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-
-const {
-    transcribeMedia,
-    convertPdfToStructuredText
-} = dslProxyActivities<typeof activities>("generateTextWorkflow", {
-    startToCloseTimeout: "30 minute",
-    retry: {
-        initialInterval: '30s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-
-
-export async function generateObjectText(payload: WorkflowExecutionPayload): Promise<TextExtractionResult> {
-
-    const { objectIds } = payload;
-    const objectId = objectIds[0];
-
-    const object = await getObjectFromStore(payload, {});
-
-    if (!object.content?.source) {
-        throw new NoDocumentFound(`No source or mimetype found for object ${objectId}`, objectIds);
-    }
-    const mimetype = object.content.type;
-    if (!mimetype) {
-        throw new NoDocumentFound(`No mimetype found for object ${objectId}`, objectIds);
-    }
-
-    const converter = ConverterActivity.find(({ type }) => type.test(mimetype));
-    if (!converter) {
-        throw new NoDocumentFound(`No converter found for mimetype ${mimetype}`, objectIds);
-    }
-    log.info(`Converting file type ${mimetype} to text with ${converter.name}`);
-
-    const res = await converter.activity(payload)(payload, converter.params);
-
-    log.info("Generated text for object", { res, objectId });
-    return res;
-
-}
-
-
-const ConverterActivity = [
-    {
-        type: /application\/pdf/,
-        activity: (payload: WorkflowExecutionPayload) => {
-            const useTextractForPDF = payload.vars?.useTextractForPdf ?? false;
-            return useTextractForPDF ? convertPdfToStructuredText : extractDocumentText;
-        },
-        name: "ConvertPdfToStructuredText",
-        params: {},
-    },
-    {
-        type: /audio\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /video\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /.+/,
-        activity: () => extractDocumentText,
-        name: "extractText",
-        params: {},
-    }
-]