@vertesia/workflow 0.50.1 → 0.51.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vertesia/workflow",
-  "version": "0.50.1",
+  "version": "0.51.0",
   "type": "module",
   "description": "Composable prompts workflow dsl",
   "main": "./lib/esm/index.js",
@@ -18,6 +18,7 @@
   },
   "license": "Apache-2.0",
   "devDependencies": {
+    "@smithy/types": "^3.7.2",
     "@temporalio/testing": "^1.11.5",
     "@temporalio/worker": "^1.11.5",
     "@types/jsonwebtoken": "^9.0.7",
@@ -25,19 +26,17 @@
     "@types/papaparse": "^5.3.15",
     "@types/tmp": "^0.2.6",
     "ts-dual-module": "^0.6.3",
-    "vitest": "^2.1.8",
-    "@smithy/types": "^3.7.2"
+    "vitest": "^2.1.9"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.693.0",
     "@aws-sdk/client-textract": "^3.693.0",
     "@aws-sdk/credential-providers": "^3.693.0",
-    "@vertesia/memory": "^0.43.0",
-    "@opendocsg/pdf2md": "0.2.0",
     "@temporalio/activity": "^1.11.5",
     "@temporalio/workflow": "^1.11.5",
     "@tensorflow/tfjs-node": "^4.19.0",
     "@types/json-schema": "^7.0.15",
+    "@vertesia/memory": "^0.43.0",
     "api-fetch-client": "^0.13.0",
     "fast-deep-equal": "^3.1.3",
     "jsonwebtoken": "^9.0.2",
@@ -50,13 +49,14 @@
     "tmp": "^0.2.3",
     "yaml": "^2.6.0",
     "@llumiverse/core": "0.15.0",
-    "@vertesia/client": "0.50.1",
-    "@vertesia/common": "0.50.1"
+    "@vertesia/client": "0.51.0",
+    "@vertesia/common": "0.51.0"
   },
   "ts_dual_module": {
     "outDir": "lib",
     "exports": {
-      "activities": "./activities",
+      "activities": "./activities/index.js",
+      "dsl-activities": "./activities/index-dsl.js",
       "workflows": "./workflows",
       "workflows-bundle": "./workflows-bundle.js",
       "vars": "./vars"
@@ -78,11 +78,21 @@
       "import": "./lib/esm/activities/index.js",
       "require": "./lib/cjs/activities/index.js"
     },
+    "./dsl-activities": {
+      "types": "./lib/types/activities/index-dsl.d.ts",
+      "import": "./lib/esm/activities/index-dsl.js",
+      "require": "./lib/cjs/activities/index-dsl.js"
+    },
     "./workflows": {
       "types": "./lib/types/workflows.d.ts",
       "import": "./lib/esm/workflows.js",
       "require": "./lib/cjs/workflows.js"
     },
+    "./dsl": {
+      "types": "./lib/types/dsl.d.ts",
+      "import": "./lib/esm/dsl.js",
+      "require": "./lib/cjs/dsl.js"
+    },
     "./workflows-bundle": {
       "import": "./lib/workflows-bundle.js"
     }
@@ -93,9 +103,15 @@
       "activities": [
         "./lib/types/activities/index.d.ts"
       ],
+      "dsl-activities": [
+        "./lib/types/activities/index-dsl.d.ts"
+      ],
       "workflows": [
         "./lib/types/workflows.d.ts"
       ],
+      "dsl": [
+        "./lib/types/dsl.d.ts"
+      ],
       "vars": [
         "./lib/types/dsl/vars.d.ts"
       ]

package/src/activities/createDocumentFromOther.ts

@@ -1,5 +1,5 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
 import fs from 'fs';
 import { pdfExtractPages } from "../conversion/mutool.js";
 import { setupActivity } from "../dsl/setup/ActivityContext.js";

package/src/activities/executeInteraction.ts

@@ -52,8 +52,7 @@ const JSON: DSLActivitySpec = {
         }
     }
 }
-
-export interface InteractionExecutionParams extends ModelOptions {
+export interface InteractionExecutionParams {
     /**
      * The environment to use. If not specified the project default environment will be used.
      * If the latter is not specified an exeption will be thrown.
@@ -80,6 +79,11 @@ export interface InteractionExecutionParams extends ModelOptions {
      * Wether or not to include the previous error in the interaction prompt data
      */
     include_previous_error?: boolean;
+
+    /**
+     * Options to control generation
+     */
+    model_options?: ModelOptions; 
 }
 
 
@@ -153,8 +157,7 @@ export async function executeInteractionFromActivity(client: VertesiaClient, int
     const config: InteractionExecutionConfiguration = {
         environment: params.environment,
         model: params.model,
-        max_tokens: params.max_tokens,
-        temperature: params.temperature
+        model_options: params.model_options,
     }
     const data = {
         ...prompt_data,

package/src/activities/extractDocumentText.ts

@@ -1,8 +1,7 @@
-import { ContentObject, CreateContentObjectPayload, DSLActivityExecutionPayload, DSLActivitySpec } from '@vertesia/common';
 import { log } from "@temporalio/activity";
+import { ContentObject, CreateContentObjectPayload, DSLActivityExecutionPayload, DSLActivitySpec } from '@vertesia/common';
 import { mutoolPdfToText } from '../conversion/mutool.js';
 import { manyToMarkdown } from '../conversion/pandoc.js';
-import { trasformPdfToMarkdown } from '../conversion/pdf.js';
 import { setupActivity } from "../dsl/setup/ActivityContext.js";
 import { NoDocumentFound } from '../errors.js';
 import { TextExtractionResult, TextExtractionStatus } from '../result-types.js';
@@ -66,11 +65,7 @@ export async function extractDocumentText(payload: DSLActivityExecutionPayload<E
 
         case 'application/pdf':
             //if pdf is more than 2MB, use mutool
-            if (fileBuffer.length > 2 * 1024 * 1024) {
-                txt = await mutoolPdfToText(fileBuffer);
-            } else {
-                txt = await trasformPdfToMarkdown(fileBuffer);
-            }
+            txt = await mutoolPdfToText(fileBuffer);
             break;
 
         case 'text/plain':

package/src/dsl/dslProxyActivities.ts

@@ -1,12 +1,12 @@
 import { ActivityOptions, proxyActivities } from "@temporalio/workflow";
-import { DSLActivityExecutionPayload, WorkflowExecutionPayload } from "@vertesia/common";
+import { DSLActivityExecutionPayload, WorkflowExecutionBaseParams, WorkflowExecutionPayload } from "@vertesia/common";
 
 export interface DslActivityFunction<ParamsT extends Record<string, any> = any, ReturnT = any> {
     (payload: DSLActivityExecutionPayload<ParamsT>): Promise<ReturnT>;
 }
 
 export interface DslSimplifiedActivityFunction<ParamsT = any, ReturnT = any> {
-    (payload: WorkflowExecutionPayload, params: ParamsT): Promise<ReturnT>;
+    (payload: WorkflowExecutionBaseParams, params: ParamsT): Promise<ReturnT>;
 }
 
 export function dslProxyActivities<

package/src/dsl/setup/ActivityContext.ts

@@ -1,12 +1,12 @@
+import { log } from "@temporalio/activity";
 import { VertesiaClient } from "@vertesia/client";
 import { DSLActivityExecutionPayload, DSLWorkflowExecutionPayload, Project, WorkflowExecutionPayload } from "@vertesia/common";
-import { log } from "@temporalio/activity";
 import { NoDocumentFound, WorkflowParamNotFound } from "../../errors.js";
-import { getClient } from "../../utils/client.js";
+import { getProjectFromToken } from "../../utils/auth.js";
+import { getVertesiaClient } from "../../utils/client.js";
 import { Vars } from "../vars.js";
 import { getFetchProvider, registerFetchProviderFactory } from "./fetch/index.js";
 import { DocumentProvider, DocumentTypeProvider, InteractionRunProvider } from "./fetch/providers.js";
-import { getProjectFromToken } from "../../utils/auth.js";
 
 
 registerFetchProviderFactory(DocumentProvider.ID, DocumentProvider.factory);
@@ -59,7 +59,7 @@ export async function setupActivity<ParamsT extends Record<string, any>>(payload
         log.info(`Setting up activity ${payload.activity.name}`, { config: payload.config, activity: payload.activity, params: payload.params, vars });
     }
 
-    const client = getClient(payload);
+    const client = getVertesiaClient(payload);
     const fetchSpecs = payload.activity.fetch;
     if (fetchSpecs) {
 

package/src/dsl.ts

@@ -0,0 +1,3 @@
+export * from "./dsl/dslProxyActivities.js";
+export * from "./result-types.js";
+export * from "./errors.js";

package/src/index.ts

@@ -20,9 +20,16 @@ export * from "./activities/generateDocumentProperties.js";
 export * from "./activities/generateEmbeddings.js";
 export * from "./activities/generateImageRendition.js";
 export * from "./activities/generateOrAssignContentType.js";
-export * from "./activities/setDocumentStatus.js";
 export * from "./activities/notifyWebhook.js";
+export * from "./activities/setDocumentStatus.js";
 export * from "./iterative-generation/activities/index.js";
 
+export * from "./dsl/setup/ActivityContext.js";
+export * from "./errors.js";
 export * from "./result-types.js";
-export * from "./dsl/dslProxyActivities.js";
+export * from "./utils/blobs.js";
+export * from "./utils/client.js";
+export * from "./utils/memory.js";
+export * from "./utils/tokens.js";
+
+export * from "./conversion/image.js";

package/src/iterative-generation/activities/extractToc.ts

@@ -1,10 +1,10 @@
+import { log } from "@temporalio/activity";
 import { WorkflowExecutionPayload } from "@vertesia/common";
 import { parse as parseYaml } from "yaml";
-import { getClient } from "../../utils/client.js";
+import { getVertesiaClient } from "../../utils/client.js";
 import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
 import { IterativeGenerationPayload, OutputMemoryMeta, Toc, TocIndex } from "../types.js";
 import { tocIndex } from "../utils.js";
-import { log } from "@temporalio/activity";
 
 /**
  * This activity is called if the toc was provided in the payload. Otherwise
@@ -15,7 +15,7 @@ import { log } from "@temporalio/activity";
 export async function it_gen_extractToc(payload: WorkflowExecutionPayload): Promise<TocIndex | null> {
     const vars = payload.vars as IterativeGenerationPayload;
     const memory = vars.memory;
-    const client = getClient(payload);
+    const client = getVertesiaClient(payload);
 
     const inMemory = await loadMemoryPack(client, `${memory}/input`);
     let tocJson: string | null = null;

package/src/iterative-generation/activities/finalizeOutput.ts

@@ -1,6 +1,6 @@
-import { WorkflowExecutionPayload } from "@vertesia/common";
 import { log } from "@temporalio/activity";
-import { getClient } from "../../utils/client.js";
+import { WorkflowExecutionPayload } from "@vertesia/common";
+import { getVertesiaClient } from "../../utils/client.js";
 import { expandVars } from "../../utils/expand-vars.js";
 import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
 import { IterativeGenerationPayload, Section, SECTION_ID_PLACEHOLDER, TocSection } from "../types.js";
@@ -9,7 +9,7 @@ export async function it_gen_finalizeOutput(payload: WorkflowExecutionPayload):
     const vars = payload.vars as IterativeGenerationPayload;
 
     const memory = vars.memory;
-    const client = getClient(payload);
+    const client = getVertesiaClient(payload);
     const inMemory = await loadMemoryPack(client, `${memory}/input`);
     const outMemory = await loadMemoryPack(client, `${memory}/output`);
 

package/src/iterative-generation/activities/generatePart.ts

@@ -1,14 +1,14 @@
+import { ApplicationFailure } from "@temporalio/workflow";
 import { WorkflowExecutionPayload } from "@vertesia/common";
 import { MemoryPack } from "@vertesia/memory";
-import { ApplicationFailure } from "@temporalio/workflow";
-import { getClient } from "../../utils/client.js";
+import { getVertesiaClient } from "../../utils/client.js";
 import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
 import { IterativeGenerationPayload, OutputMemoryMeta, Section, TocPart, TocSection } from "../types.js";
 import { executeWithVars, expectMemoryIsConsistent } from "../utils.js";
 
 export async function it_gen_generatePart(payload: WorkflowExecutionPayload, path: number[]) {
     const vars = payload.vars as IterativeGenerationPayload;
-    const client = getClient(payload);
+    const client = getVertesiaClient(payload);
     const memory = vars.memory;
 
     const [sectionIndex, partIndex] = path;

package/src/iterative-generation/activities/generateToc.ts

@@ -1,5 +1,5 @@
 import { WorkflowExecutionPayload } from "@vertesia/common";
-import { getClient } from "../../utils/client.js";
+import { getVertesiaClient } from "../../utils/client.js";
 import { buildAndPublishMemoryPack } from "../../utils/memory.js";
 import { IterativeGenerationPayload, OutputMemoryMeta, Toc, TocIndex } from "../types.js";
 import { executeWithVars, tocIndex } from "../utils.js";
@@ -80,7 +80,7 @@ export async function it_gen_generateToc(payload: WorkflowExecutionPayload): Pro
 
     const schema = vars.toc_schema || defaultTocSchema;
 
-    const client = getClient(payload);
+    const client = getVertesiaClient(payload);
 
     const run = await executeWithVars(client, vars.interaction, vars, undefined, schema);
 

package/src/iterative-generation/utils.ts

@@ -2,15 +2,16 @@ import { VertesiaClient } from "@vertesia/client";
 import { ExecutionRun } from "@vertesia/common";
 import { ApplicationFailure } from "@temporalio/workflow";
 import { OutputMemoryMeta, PartIndex, Toc, TocIndex, TocSection } from "./types.js";
+import { ModelOptions, TextFallbackOptions } from "@llumiverse/core";
 
+//TODO: For whole file, support for options beyond max_tokens and temperature and multiple modalities.
 export interface ExecuteOptions {
     interaction: string;
     memory: string;
     memory_mapping?: Record<string, any>;
     environment?: string;
     model?: string;
-    max_tokens?: number;
-    temperature?: number;
+    model_options?: ModelOptions;
     result_schema?: Record<string, any>;
 }
 
@@ -24,8 +25,7 @@ export async function execute<T = any>(client: VertesiaClient, options: ExecuteO
         config: {
             environment: options.environment,
             model: options.model,
-            max_tokens: options.max_tokens,
-            temperature: options.temperature,
+            model_options: options.model_options,
         }
     });
 }
@@ -36,14 +36,18 @@ export function executeWithVars<T = any>(client: VertesiaClient, interaction: st
     } else {
         mapping = vars.input_mapping;
     }
+    const model_options: TextFallbackOptions = {
+        _option_id: "text-fallback",
+        max_tokens: vars.max_tokens,
+        temperature: vars.temperature
+    }
     return execute(client, {
         interaction: interaction,
         memory: `${vars.memory}/input`,
         memory_mapping: mapping,
         environment: vars.environment,
         model: vars.model,
-        max_tokens: vars.max_tokens,
-        temperature: vars.temperature,
+        model_options: model_options,
         result_schema: result_schema
     });
 }

package/src/system/notifyWebhookWorkflow.ts

@@ -1,7 +1,7 @@
 
 import { log } from "@temporalio/workflow";
 import { ContentEventName, WorkflowExecutionPayload } from "@vertesia/common";
-import * as activities from "../activities/index-dsl.js";
+import * as activities from "../activities/notifyWebhook.js";
 import { dslProxyActivities } from "../dsl/dslProxyActivities.js";
 
 const {

package/src/utils/blobs.ts

@@ -35,8 +35,8 @@ export async function saveBlobToFile(client: VertesiaClient, blobUri: string, to
 
 export async function saveBlobToTempFile(client: VertesiaClient, blobUri: string, fileExt?: string): Promise<string> {
     const tmpFile = tmp.fileSync({
-        prefix: "composable-activity-",
-        postfix: fileExt,
+        prefix: "vertesia-activity-",
+        postfix: "." + fileExt,
         discardDescriptor: true
     });
     await saveBlobToFile(client, blobUri, tmpFile.name);

package/src/utils/client.ts

@@ -3,10 +3,10 @@
  */
 
 import { VertesiaClient } from "@vertesia/client";
-import { WorkflowExecutionPayload } from "@vertesia/common";
+import { WorkflowExecutionBaseParams } from "@vertesia/common";
 
 
-export function getClient(payload: WorkflowExecutionPayload) {
+export function getVertesiaClient(payload: WorkflowExecutionBaseParams) {
 
     if (!payload.auth_token) {
         throw new Error("Authentication Token is missing from WorkflowExecutionPayload.authToken");

package/src/workflows.ts

@@ -3,7 +3,5 @@
  */
 export { dslWorkflow } from "./dsl/dsl-workflow.js";
 export { iterativeGenerationWorkflow } from "./iterative-generation/iterativeGenerationWorkflow.js";
-export { generateObjectText } from "./system/generateObjectText.js";
 export { notifyWebhookWorkflow } from "./system/notifyWebhookWorkflow.js";
 export { recalculateEmbeddingsWorkflow } from "./system/recalculateEmbeddingsWorkflow.js";
-

package/lib/cjs/conversion/pdf.js

@@ -1,13 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.trasformPdfToMarkdown = trasformPdfToMarkdown;
-const pdf2md_1 = __importDefault(require("@opendocsg/pdf2md"));
-const pdf2mdFn = pdf2md_1.default;
-function trasformPdfToMarkdown(buffer) {
-    const arr = new Uint8Array(buffer);
-    return pdf2mdFn(arr);
-}
-//# sourceMappingURL=pdf.js.map

package/lib/cjs/conversion/pdf.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../src/conversion/pdf.ts"],"names":[],"mappings":";;;;;AAIA,sDAGC;AAPD,+DAAuC;AAEvC,MAAM,QAAQ,GAAG,gBAA4D,CAAC;AAE9E,SAAgB,qBAAqB,CAAC,MAAc;IAChD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}

package/lib/cjs/system/generateObjectText.js

@@ -1,76 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateObjectText = generateObjectText;
-const workflow_1 = require("@temporalio/workflow");
-const dslProxyActivities_js_1 = require("../dsl/dslProxyActivities.js");
-const errors_js_1 = require("../errors.js");
-const { getObjectFromStore, extractDocumentText } = (0, dslProxyActivities_js_1.dslProxyActivities)("generateTextWorkflow", {
-    startToCloseTimeout: "5 minute",
-    retry: {
-        initialInterval: '5s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-const { transcribeMedia, convertPdfToStructuredText } = (0, dslProxyActivities_js_1.dslProxyActivities)("generateTextWorkflow", {
-    startToCloseTimeout: "30 minute",
-    retry: {
-        initialInterval: '30s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-async function generateObjectText(payload) {
-    const { objectIds } = payload;
-    const objectId = objectIds[0];
-    const object = await getObjectFromStore(payload, {});
-    if (!object.content?.source) {
-        throw new errors_js_1.NoDocumentFound(`No source or mimetype found for object ${objectId}`, objectIds);
-    }
-    const mimetype = object.content.type;
-    if (!mimetype) {
-        throw new errors_js_1.NoDocumentFound(`No mimetype found for object ${objectId}`, objectIds);
-    }
-    const converter = ConverterActivity.find(({ type }) => type.test(mimetype));
-    if (!converter) {
-        throw new errors_js_1.NoDocumentFound(`No converter found for mimetype ${mimetype}`, objectIds);
-    }
-    workflow_1.log.info(`Converting file type ${mimetype} to text with ${converter.name}`);
-    const res = await converter.activity(payload)(payload, converter.params);
-    workflow_1.log.info("Generated text for object", { res, objectId });
-    return res;
-}
-const ConverterActivity = [
-    {
-        type: /application\/pdf/,
-        activity: (payload) => {
-            const useTextractForPDF = payload.vars?.useTextractForPdf ?? false;
-            return useTextractForPDF ? convertPdfToStructuredText : extractDocumentText;
-        },
-        name: "ConvertPdfToStructuredText",
-        params: {},
-    },
-    {
-        type: /audio\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /video\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /.+/,
-        activity: () => extractDocumentText,
-        name: "extractText",
-        params: {},
-    }
-];
-//# sourceMappingURL=generateObjectText.js.map

package/lib/cjs/system/generateObjectText.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"generateObjectText.js","sourceRoot":"","sources":["../../../src/system/generateObjectText.ts"],"names":[],"mappings":";;AAqCA,gDA0BC;AA9DD,mDAA2C;AAG3C,wEAAkE;AAClE,4CAA+C;AAG/C,MAAM,EACF,kBAAkB,EAClB,mBAAmB,EACtB,GAAG,IAAA,0CAAkB,EAAoB,sBAAsB,EAAE;IAC9D,mBAAmB,EAAE,UAAU;IAC/B,KAAK,EAAE;QACH,eAAe,EAAE,IAAI;QACrB,kBAAkB,EAAE,CAAC;QACrB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,IAAI;QACtC,sBAAsB,EAAE,EAAE;KAC7B;CACJ,CAAC,CAAC;AAEH,MAAM,EACF,eAAe,EACf,0BAA0B,EAC7B,GAAG,IAAA,0CAAkB,EAAoB,sBAAsB,EAAE;IAC9D,mBAAmB,EAAE,WAAW;IAChC,KAAK,EAAE;QACH,eAAe,EAAE,KAAK;QACtB,kBAAkB,EAAE,CAAC;QACrB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,IAAI;QACtC,sBAAsB,EAAE,EAAE;KAC7B;CACJ,CAAC,CAAC;AAGI,KAAK,UAAU,kBAAkB,CAAC,OAAiC;IAEtE,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAErD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,2BAAe,CAAC,0CAA0C,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;IACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,2BAAe,CAAC,gCAAgC,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,MAAM,IAAI,2BAAe,CAAC,mCAAmC,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IACxF,CAAC;IACD,cAAG,CAAC,IAAI,CAAC,wBAAwB,QAAQ,iBAAiB,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IAE5E,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAEzE,cAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IACzD,OAAO,GAAG,CAAC;AAEf,CAAC;AAGD,MAAM,iBAAiB,GAAG;IACtB;QACI,IAAI,EAAE,kBAAkB;QACxB,QAAQ,EAAE,CAAC,OAAiC,EAAE,EAAE;YAC5C,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,EAAE,iBAAiB,IAAI,KAAK,CAAC;YACnE,OAAO,iBAAiB,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,mBAAmB,CAAC;QAChF,CAAC;QACD,IAAI,EAAE,4BAA4B;QAClC,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,GAAG,EAAE,CAAC,eAAe;QAC/B,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,GAAG,EAAE,CAAC,eAAe;QAC/B,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,IAAI;QACV,QAAQ,EAAE,GAAG,EAAE,CAAC,mBAAmB;QACnC,IAAI,EAAE,aAAa;QACnB,MAAM,EAAE,EAAE;KACb;CACJ,CAAA"}

package/lib/esm/conversion/pdf.js

@@ -1,7 +0,0 @@
-import pdf2md from "@opendocsg/pdf2md";
-const pdf2mdFn = pdf2md;
-export function trasformPdfToMarkdown(buffer) {
-    const arr = new Uint8Array(buffer);
-    return pdf2mdFn(arr);
-}
-//# sourceMappingURL=pdf.js.map

package/lib/esm/conversion/pdf.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../src/conversion/pdf.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,mBAAmB,CAAC;AAEvC,MAAM,QAAQ,GAAG,MAA4D,CAAC;AAE9E,MAAM,UAAU,qBAAqB,CAAC,MAAc;IAChD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}

package/lib/esm/system/generateObjectText.js

@@ -1,73 +0,0 @@
-import { log } from "@temporalio/workflow";
-import { dslProxyActivities } from "../dsl/dslProxyActivities.js";
-import { NoDocumentFound } from "../errors.js";
-const { getObjectFromStore, extractDocumentText } = dslProxyActivities("generateTextWorkflow", {
-    startToCloseTimeout: "5 minute",
-    retry: {
-        initialInterval: '5s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-const { transcribeMedia, convertPdfToStructuredText } = dslProxyActivities("generateTextWorkflow", {
-    startToCloseTimeout: "30 minute",
-    retry: {
-        initialInterval: '30s',
-        backoffCoefficient: 2,
-        maximumAttempts: 5,
-        maximumInterval: 100 * 30 * 1000, //ms
-        nonRetryableErrorTypes: [],
-    },
-});
-export async function generateObjectText(payload) {
-    const { objectIds } = payload;
-    const objectId = objectIds[0];
-    const object = await getObjectFromStore(payload, {});
-    if (!object.content?.source) {
-        throw new NoDocumentFound(`No source or mimetype found for object ${objectId}`, objectIds);
-    }
-    const mimetype = object.content.type;
-    if (!mimetype) {
-        throw new NoDocumentFound(`No mimetype found for object ${objectId}`, objectIds);
-    }
-    const converter = ConverterActivity.find(({ type }) => type.test(mimetype));
-    if (!converter) {
-        throw new NoDocumentFound(`No converter found for mimetype ${mimetype}`, objectIds);
-    }
-    log.info(`Converting file type ${mimetype} to text with ${converter.name}`);
-    const res = await converter.activity(payload)(payload, converter.params);
-    log.info("Generated text for object", { res, objectId });
-    return res;
-}
-const ConverterActivity = [
-    {
-        type: /application\/pdf/,
-        activity: (payload) => {
-            const useTextractForPDF = payload.vars?.useTextractForPdf ?? false;
-            return useTextractForPDF ? convertPdfToStructuredText : extractDocumentText;
-        },
-        name: "ConvertPdfToStructuredText",
-        params: {},
-    },
-    {
-        type: /audio\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /video\/.+/,
-        activity: () => transcribeMedia,
-        name: "TranscribeMedia",
-        params: {},
-    },
-    {
-        type: /.+/,
-        activity: () => extractDocumentText,
-        name: "extractText",
-        params: {},
-    }
-];
-//# sourceMappingURL=generateObjectText.js.map

package/lib/esm/system/generateObjectText.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"generateObjectText.js","sourceRoot":"","sources":["../../../src/system/generateObjectText.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AAG3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAG/C,MAAM,EACF,kBAAkB,EAClB,mBAAmB,EACtB,GAAG,kBAAkB,CAAoB,sBAAsB,EAAE;IAC9D,mBAAmB,EAAE,UAAU;IAC/B,KAAK,EAAE;QACH,eAAe,EAAE,IAAI;QACrB,kBAAkB,EAAE,CAAC;QACrB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,IAAI;QACtC,sBAAsB,EAAE,EAAE;KAC7B;CACJ,CAAC,CAAC;AAEH,MAAM,EACF,eAAe,EACf,0BAA0B,EAC7B,GAAG,kBAAkB,CAAoB,sBAAsB,EAAE;IAC9D,mBAAmB,EAAE,WAAW;IAChC,KAAK,EAAE;QACH,eAAe,EAAE,KAAK;QACtB,kBAAkB,EAAE,CAAC;QACrB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,IAAI;QACtC,sBAAsB,EAAE,EAAE;KAC7B;CACJ,CAAC,CAAC;AAGH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAiC;IAEtE,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAErD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,eAAe,CAAC,0CAA0C,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;IACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,MAAM,IAAI,eAAe,CAAC,gCAAgC,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,MAAM,IAAI,eAAe,CAAC,mCAAmC,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IACxF,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,wBAAwB,QAAQ,iBAAiB,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IAE5E,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAEzE,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IACzD,OAAO,GAAG,CAAC;AAEf,CAAC;AAGD,MAAM,iBAAiB,GAAG;IACtB;QACI,IAAI,EAAE,kBAAkB;QACxB,QAAQ,EAAE,CAAC,OAAiC,EAAE,EAAE;YAC5C,MAAM,iBAAiB,GAAG,OAAO,CAAC,IAAI,EAAE,iBAAiB,IAAI,KAAK,CAAC;YACnE,OAAO,iBAAiB,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,mBAAmB,CAAC;QAChF,CAAC;QACD,IAAI,EAAE,4BAA4B;QAClC,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,GAAG,EAAE,CAAC,eAAe;QAC/B,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,GAAG,EAAE,CAAC,eAAe;QAC/B,IAAI,EAAE,iBAAiB;QACvB,MAAM,EAAE,EAAE;KACb;IACD;QACI,IAAI,EAAE,IAAI;QACV,QAAQ,EAAE,GAAG,EAAE,CAAC,mBAAmB;QACnC,IAAI,EAAE,aAAa;QACnB,MAAM,EAAE,EAAE;KACb;CACJ,CAAA"}

package/lib/types/conversion/pdf.d.ts

@@ -1,2 +0,0 @@
-export declare function trasformPdfToMarkdown(buffer: Buffer): Promise<string>;
-//# sourceMappingURL=pdf.d.ts.map

package/lib/types/conversion/pdf.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../../src/conversion/pdf.ts"],"names":[],"mappings":"AAIA,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,mBAGnD"}

package/lib/types/system/generateObjectText.d.ts

@@ -1,4 +0,0 @@
-import { WorkflowExecutionPayload } from "@vertesia/common";
-import { TextExtractionResult } from "../index.js";
-export declare function generateObjectText(payload: WorkflowExecutionPayload): Promise<TextExtractionResult>;
-//# sourceMappingURL=generateObjectText.d.ts.map

package/lib/types/system/generateObjectText.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"generateObjectText.d.ts","sourceRoot":"","sources":["../../../src/system/generateObjectText.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AAI5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AA+BnD,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA0BzG"}

package/src/conversion/pdf.test.ts

@@ -1,35 +0,0 @@
-import { MockActivityEnvironment, TestWorkflowEnvironment } from '@temporalio/testing';
-import fs from 'fs';
-import path from 'path';
-import { beforeAll, expect, test } from 'vitest';
-import { trasformPdfToMarkdown } from '../conversion/pdf';
-
-
-let testEnv: TestWorkflowEnvironment;
-let activityContext: MockActivityEnvironment;
-
-beforeAll(async () => {
-  //testEnv = await TestWorkflowEnvironment.createLocal();
-  activityContext = new MockActivityEnvironment();
-});
-
-
-test('Converts a PDF to markdown', async () => {
-  const pdfPath = path.resolve(__dirname, '../../fixtures', 'test-pdf1.pdf');
-  const pdfBuffer = fs.readFileSync(pdfPath);
-  const result = await activityContext.run(trasformPdfToMarkdown, pdfBuffer);
-
-  expect(result).toContain('America');
-});
-
-// https://github.com/becomposable/studio/issues/432 Skip tests
-// The file "test-pdf2.pdf" could not be opened. It may be damanged or use a file format that
-// Preview doesn't recognize.
-test.skip('Converts another PDF to markdown', async () => {
-    const pdfPath = path.resolve(__dirname, '../../fixtures', 'test-pdf2.pdf');
-    const pdfBuffer = fs.readFileSync(pdfPath);
-    const result = await activityContext.run(trasformPdfToMarkdown, pdfBuffer);
-
-    expect(result).toContain('America');    
-
-  });