@vertesia/workflow 0.42.2

package/src/conversion/mutool.ts

@@ -0,0 +1,180 @@
+import { log } from '@temporalio/activity';
+import { spawn } from 'child_process';
+import fs from 'fs';
+import tmp from 'tmp';
+
+
+
+/**
+ * Convert a pdf file to text
+ * TODO: pass file reference instead of Buffer
+ */
+
+export function mutoolPdfToText(buffer: Buffer): Promise<string> {
+
+    const inputFile = tmp.fileSync({ postfix: '.pdf' });
+    const targetFileName = tmp.tmpNameSync({ postfix: '.txt' });
+
+    fs.writeSync(inputFile.fd, buffer);
+
+    return new Promise((resolve, reject) => {
+
+
+        log.info("Converting pdf to text", { inputFile: inputFile.name, targetFileName });
+
+        const command = spawn("mutool", ["convert", "-o", targetFileName, inputFile.name]);
+
+        command.on('exit', function (code) {
+            if (code) {
+                reject(new Error(`mutool exited with code ${code}`));
+            }
+        });
+
+
+        command.on('close', function (code) {
+            if (code) {
+                reject(new Error(`mutool exited with code ${code}`));
+            } else {
+                return fs.readFile(targetFileName, 'utf8', (err, data) => {
+                    if (err) {
+                        reject(err);
+                    }
+                    return resolve(data);
+                });
+            };
+        });
+
+        command.on('error', (err) => {
+            reject(err);
+        });
+
+    });
+
+}
+
+/**
+ * 
+ * Convert a pdf files to images (one image per page), as PNG format
+ * 
+ * @param file 
+ * @param pages
+ * @returns  
+ */
+export async function pdfToImages(file: Buffer | string, pages?: number[]): Promise<string[]> {
+
+    const workDir = tmp.dirSync();
+    log.info(`Converting pdf to images`, { workDir: workDir.name, input_type: typeof file, pages });
+
+    if (file instanceof Buffer) {
+        fs.writeFileSync(`${workDir.name}/input.pdf`, file);
+    }
+    const filename = typeof file === 'string' ? file : `${workDir.name}/input.pdf`;
+
+    const args = [
+        "draw", 
+        "-o", `${workDir.name}/page-%d.png`,
+        filename,
+    ];
+
+    if (pages) {
+        args.push(pages.join(','));
+    }
+
+    return new Promise((resolve, reject) => {
+
+        const command = spawn("mutool", args);
+        log.info(`Executing mutool command`, { workDir: workDir.name, filename, command: command.spawnargs });
+
+        let errors = '';
+
+        command.stderr.on('data', (data) => {
+            errors += data;
+        });
+
+        command.on('exit', function (code) {
+
+            if (code) {
+                log.error(`mutool exited with code ${code}`, { errors });
+                reject(new Error(`mutool exited with code ${code}`));
+            }
+        });
+
+        command.on('close', function (code) {
+            if (code) {
+                reject(new Error(`mutool finished with code ${code}`));
+            } else {
+                const files = fs.readdirSync(workDir.name);
+                const images = files.filter(f => f.endsWith('.png')).map(f => `${workDir.name}/${f}`);
+                log.info(`Converted pdf to ${images.length} images`, files);
+                return resolve(images);
+            };
+        });
+
+        command.on('error', (err) => {
+            reject(err);
+        });
+
+    });
+
+}
+
+
+/**
+ * Get somes pages from a PDF to create a new one
+ */
+
+export async function pdfExtractPages(file: Buffer | string, pages: number[]): Promise<string> {
+
+    const workDir = tmp.dirSync();
+    log.info(`Getting pages from pdf`, { workDir: workDir.name, input_type: typeof file, pages });
+
+    if (file instanceof Buffer) {
+        fs.writeFileSync(`${workDir.name}/input.pdf`, file);
+    }
+    const filename = typeof file === 'string' ? file : `${workDir.name}/input.pdf`;
+
+    const args = [
+        "merge",
+        "-o", `${workDir.name}/output.pdf`,
+        "-O", "garbage=compact,sanitize",
+        filename,
+        pages.join(','),
+    ];
+
+    return new Promise((resolve, reject) => {
+
+        const command = spawn("mutool", args);
+        log.info(`Executing mutool command`, { workDir: workDir.name, filename, command: command.spawnargs });
+
+        let errors = '';
+
+        command.stderr.on('data', (data) => {
+            errors += data;
+        });
+
+        command.on('exit', function (code) {
+
+            if (code) {
+                log.error(`mutool exited with code ${code}`, { errors });
+                reject(new Error(`mutool exited with code ${code}`));
+            }
+        });
+
+        command.on('close', function (code) {
+            if (code) {
+                reject(new Error(`mutool finished with code ${code}`));
+            } else {
+                const file = `${workDir.name}/output.pdf`;
+                log.info(`Extracted pages from pdf`, { pages, file });
+                return resolve(file);
+            };
+        });
+
+        command.on('error', (err) => {
+            reject(err);
+        });
+
+    });
+
+
+}

package/src/conversion/pandoc.test.ts

@@ -0,0 +1,22 @@
+import { MockActivityEnvironment, TestWorkflowEnvironment } from '@temporalio/testing';
+import fs from 'fs';
+import path from 'path';
+import { beforeAll, expect, test } from 'vitest';
+import { manyToMarkdown } from '../conversion/pandoc';
+
+
+let testEnv: TestWorkflowEnvironment;
+let activityContext: MockActivityEnvironment;
+
+beforeAll(async () => {
+  testEnv = await TestWorkflowEnvironment.createLocal();
+  activityContext = new MockActivityEnvironment();
+});
+
+
+// Add more test cases for other file types (ODT, DOCX) if needed
+test('should convert docx to markdown', async () => {
+  const docx = fs.readFileSync(path.join(__dirname, '../../fixtures', 'us-ciia.docx'));
+  const result = await activityContext.run(manyToMarkdown, Buffer.from(docx), 'docx');
+  expect(result).to.include('confidential');
+});

package/src/conversion/pandoc.ts

@@ -0,0 +1,44 @@
+import { log } from '@temporalio/activity';
+import { spawn } from 'child_process';
+import { PassThrough } from 'stream';
+
+
+export function manyToMarkdown(buffer: Buffer, fromFormat: string): Promise<string> {
+
+  const fromType = undefined;
+
+  return new Promise((resolve, reject) => {
+    log.info(`Converting ${fromType} to markdown`);
+    const input = new PassThrough();
+    input.end(buffer);
+
+    let result: string[] = [];
+
+    const command = spawn("pandoc", ["-t", "markdown", '-f', fromFormat], {
+      stdio: 'pipe',
+    });
+    input.pipe(command.stdin);
+
+    command.stdout.on('data', function (data: string) {
+      result.push(data.toString());
+    });
+    command.on('exit', function (code) {
+      if (code) {
+        reject(new Error(`pandoc exited with code ${code}`));
+      }
+    });
+    command.on('close', function (code) {
+      if (code) {
+        reject(new Error(`pandoc exited with code ${code}`));
+      } else {
+        resolve(result.join(''))
+      }
+    });
+
+    command.on('error', (err) => {
+      reject(err);
+    });
+
+  });
+
+}

package/src/conversion/pdf.test.ts

@@ -0,0 +1,35 @@
+import { MockActivityEnvironment, TestWorkflowEnvironment } from '@temporalio/testing';
+import fs from 'fs';
+import path from 'path';
+import { beforeAll, expect, test } from 'vitest';
+import { trasformPdfToMarkdown } from '../conversion/pdf';
+
+
+let testEnv: TestWorkflowEnvironment;
+let activityContext: MockActivityEnvironment;
+
+beforeAll(async () => {
+  //testEnv = await TestWorkflowEnvironment.createLocal();
+  activityContext = new MockActivityEnvironment();
+});
+
+
+test('Converts a PDF to markdown', async () => {
+  const pdfPath = path.resolve(__dirname, '../../fixtures', 'test-pdf1.pdf');
+  const pdfBuffer = fs.readFileSync(pdfPath);
+  const result = await activityContext.run(trasformPdfToMarkdown, pdfBuffer);
+
+  expect(result).toContain('America');
+});
+
+// https://github.com/becomposable/studio/issues/432 Skip tests
+// The file "test-pdf2.pdf" could not be opened. It may be damanged or use a file format that
+// Preview doesn't recognize.
+test.skip('Converts another PDF to markdown', async () => {
+    const pdfPath = path.resolve(__dirname, '../../fixtures', 'test-pdf2.pdf');
+    const pdfBuffer = fs.readFileSync(pdfPath);
+    const result = await activityContext.run(trasformPdfToMarkdown, pdfBuffer);
+
+    expect(result).toContain('America');    
+
+  });

package/src/conversion/pdf.ts

@@ -0,0 +1,8 @@
+import pdf2md from "@opendocsg/pdf2md";
+
+const pdf2mdFn = pdf2md as unknown as (buffer: Uint8Array) => Promise<string>;
+
+export function trasformPdfToMarkdown(buffer: Buffer) {
+    const arr = new Uint8Array(buffer);
+    return pdf2mdFn(arr);
+}

package/src/dsl/conditions.ts

@@ -0,0 +1,76 @@
+import equal from 'fast-deep-equal';
+
+function $exists(value: any, arg: boolean) {
+    return (value !== undefined) === arg;
+}
+function $null(value: any, arg: boolean) {
+    return (value == null) === arg;
+}
+
+function $eq(value: any, arg: any) {
+    if (Array.isArray(arg)) {
+        return equal(value, arg);
+    } else if (typeof arg === 'object') {
+        return equal(value, arg);
+    } else {
+        return value === arg;
+    }
+}
+function $ne(value: any, arg: any) {
+    return !$eq(value, arg);
+}
+function $or(value: any, arg: any[]) {
+    return arg.some(c => matchCondition(value, c));
+}
+function $in(value: any, arg: any[]) {
+    return arg.includes(value);
+}
+function $nin(value: any, arg: any[]) {
+    return !$in(value, arg);
+}
+function $regexp(value: string, arg: string) {
+    return new RegExp(arg).test(value);
+}
+function $endsWith(value: string, arg: string) {
+    return value.endsWith(arg);
+}
+function $startsWith(value: string, arg: string) {
+    return value.startsWith(arg);
+}
+function $contains(value: string, arg: string) {
+    return value.includes(arg);
+}
+function $lt(value: number, arg: number) {
+    return value < arg;
+}
+function $gt(value: number, arg: number) {
+    return value > arg;
+}
+function $lte(value: number, arg: number) {
+    return value <= arg;
+}
+function $gte(value: number, arg: number) {
+    return value >= arg;
+}
+
+const conditionFns: Record<string, any> = {
+    $exists, $null,
+    $eq, $ne,
+    $in, $nin,
+    $regexp, $startsWith, $endsWith, $contains,
+    $lt, $gt, $lte, $gte,
+    $or,
+}
+
+export function matchCondition(value: any, conditions: Record<string, any>) {
+    for (const key of Object.keys(conditions)) {
+        const cond = conditionFns[key];
+        if (!cond) {
+            throw new Error(`Unknown condition: ${key}`);
+        }
+        if (!cond(value, conditions[key])) {
+            return false;
+        }
+    }
+    return true;
+}

package/src/dsl/dsl-workflow.test.ts

@@ -0,0 +1,58 @@
+import { describe, test, expect } from "vitest";
+import { computeActivityOptions } from "./dsl-workflow.ts";
+
+describe('Workflow DSL', () => {
+    test('compute activity options without custom options', () => {
+        expect(computeActivityOptions({}, {
+            startToCloseTimeout: 1000,
+            scheduleToCloseTimeout: 2000,
+            scheduleToStartTimeout: 3000,
+            retry: {
+                initialInterval: 4000,
+                maximumInterval: 5000,
+                maximumAttempts: 6,
+                backoffCoefficient: 7,
+                nonRetryableErrorTypes: ['error']
+            }
+        })).toEqual({
+            startToCloseTimeout: 1000,
+            scheduleToCloseTimeout: 2000,
+            scheduleToStartTimeout: 3000,
+            retry: {
+                initialInterval: 4000,
+                maximumInterval: 5000,
+                maximumAttempts: 6,
+                backoffCoefficient: 7,
+                nonRetryableErrorTypes: ['error']
+            }
+        })
+    })
+
+    test('compute activity options with some custom options', () => {
+        expect(computeActivityOptions({
+            startToCloseTimeout: 100,
+        }, {
+            startToCloseTimeout: 1000,
+            scheduleToCloseTimeout: 2000,
+            scheduleToStartTimeout: 3000,
+            retry: {
+                initialInterval: 4000,
+                maximumInterval: 5000,
+                maximumAttempts: 6,
+                backoffCoefficient: 7,
+                nonRetryableErrorTypes: ['error']
+            }
+        })).toEqual({
+            startToCloseTimeout: `100ms`, // custom value
+            scheduleToCloseTimeout: 2000,
+            scheduleToStartTimeout: 3000,
+            retry: {
+                initialInterval: 4000,
+                maximumInterval: 5000,
+                maximumAttempts: 6,
+                backoffCoefficient: 7,
+                nonRetryableErrorTypes: ['error']
+            }
+        })
+    })
+});

package/src/dsl/dsl-workflow.ts

@@ -0,0 +1,235 @@
+import {
+    DSLActivityExecutionPayload,
+    DSLActivityOptions,
+    DSLActivitySpec,
+    DSLChildWorkflowStep,
+    DSLWorkflowExecutionPayload,
+    WorkflowExecutionPayload
+} from "@vertesia/common";
+import { ActivityInterfaceFor, ActivityOptions, executeChild, log, proxyActivities, startChild, UntypedActivities } from "@temporalio/workflow";
+import ms, { StringValue } from 'ms';
+import { ActivityParamNotFound, NoDocumentFound, WorkflowParamNotFound } from "../errors.js";
+import { Vars } from "./vars.js";
+
+interface BaseActivityPayload extends WorkflowExecutionPayload {
+    workflow_name: string;
+    debug_mode?: boolean;
+}
+
+function dslActivityPayload(basePayload: BaseActivityPayload, activity: DSLActivitySpec, params: Record<string, any> = {}) {
+    return {
+        ...basePayload,
+        activity,
+        params,
+    } as DSLActivityExecutionPayload
+}
+
+export async function dslWorkflow(payload: DSLWorkflowExecutionPayload) {
+
+    const definition = payload.workflow;
+    if (!definition) {
+        throw new WorkflowParamNotFound("workflow");
+    }
+    // the base payload wiull be used to create the activities payload
+    const basePayload: BaseActivityPayload = {
+        ...payload,
+        workflow_name: definition.name,
+        debug_mode: !!definition.debug_mode,
+    }
+    delete (basePayload as any).workflow;
+
+    const defaultOptions: ActivityOptions = {
+        ...convertDSLActivityOptions(definition.options),
+        startToCloseTimeout: "5 minute",
+        retry: {
+            initialInterval: '30s',
+            backoffCoefficient: 2,
+            maximumAttempts: 20,
+            maximumInterval: 100 * 30 * 1000, //ms
+            nonRetryableErrorTypes: [
+                NoDocumentFound.name,
+                ActivityParamNotFound.name,
+                WorkflowParamNotFound.name,
+            ],
+        },
+    };
+    log.debug("Global activity options", {
+        activityOptions: defaultOptions,
+    });
+    const defaultProxy = proxyActivities(defaultOptions);
+    log.debug("Default activity proxy is ready");
+    // merge default vars with the payload vars and add objectIds and obejctId
+    const vars = new Vars({
+        ...definition.vars,
+        ...payload.vars,
+        objectIds: payload.objectIds || [],
+        objectId: payload.objectIds ? payload.objectIds[0] : undefined
+    });
+
+    log.info("Executing workflow", { payload });
+
+    if (definition.steps) {
+        for (const step of definition.steps) {
+            const stepType = step.type;
+            if (stepType === 'workflow') {
+                const childWorkflowStep = step as DSLChildWorkflowStep;
+                if (childWorkflowStep.async) {
+                    await startChildWorkflow(childWorkflowStep, payload, vars, basePayload.debug_mode);
+                } else {
+                    await executeChildWorkflow(childWorkflowStep, payload, vars, basePayload.debug_mode);
+                }
+            } else { // activity
+                await runActivity(step as DSLActivitySpec, basePayload, vars, defaultProxy, defaultOptions);
+            }
+        }
+    } else if (definition.activities) { // legacy support
+        for (const activity of definition.activities) {
+            await runActivity(activity, basePayload, vars, defaultProxy, defaultOptions);
+        }
+    } else {
+        throw new Error("No steps or activities found in the workflow definition");
+    }
+    return vars.getValue(definition.result || 'result');
+}
+
+async function startChildWorkflow(step: DSLChildWorkflowStep, payload: DSLWorkflowExecutionPayload, vars: Vars, debug_mode?: boolean) {
+    const resolvedVars = vars.resolve();
+    if (step.vars) {
+        // copy user vars (from step definition) to the resolved vars
+        Object.assign(resolvedVars, step.vars);
+    }
+    if (debug_mode) {
+        log.debug(`Workflow vars before starting child workflow ${step.name}`, { vars: resolvedVars });
+    }
+    //@ts-ignore
+    const handle = await startChild(step.name, {
+        ...step.options,
+        args: [{
+            ...payload,
+            workflow: step.spec,
+            vars: resolvedVars
+        }]
+    });
+    if (step.output) {
+        vars.setValue(step.output, handle.workflowId);
+    }
+}
+
+async function executeChildWorkflow(step: DSLChildWorkflowStep, payload: DSLWorkflowExecutionPayload, vars: Vars, debug_mode?: boolean) {
+    const resolvedVars = vars.resolve();
+    if (step.vars) {
+        // copy user vars (from step definition) to the resolved vars
+        Object.assign(resolvedVars, step.vars);
+    }
+    if (debug_mode) {
+        log.debug(`Workflow vars before excuting child workflow ${step.name}`, { vars: resolvedVars });
+    }
+    //@ts-ignore
+    const result = await executeChild(step.name, {
+        ...step.options,
+        args: [{
+            ...payload,
+            workflow: step.spec,
+            vars: resolvedVars,
+        }]
+    });
+
+    if (step.output) {
+        vars.setValue(step.output, result);
+        if (debug_mode) {
+            log.debug(`Workflow vars after executing child workflow ${step.name}`, { vars: vars.resolve() });
+        }
+    } else if (debug_mode) {
+        log.debug(`Workflow vars after executing child workflow ${step.name}`, { vars: resolvedVars });
+    }
+}
+
+async function runActivity(activity: DSLActivitySpec, basePayload: BaseActivityPayload, vars: Vars, defaultProxy: ActivityInterfaceFor<UntypedActivities>, defaultOptions: ActivityOptions) {
+    if (basePayload.debug_mode) {
+        log.debug(`Workflow vars before executing activity ${activity.name}`, { vars: vars.resolve() });
+    }
+    if (activity.condition && !vars.match(activity.condition)) {
+        log.info("Activity skiped: condition not satisfied", activity.condition);
+        return;
+    }
+    const importParams = vars.createImportVars(activity.import);
+    const executionPayload = dslActivityPayload(basePayload, activity, importParams);
+    log.info("Executing activity: " + activity.name, { payload: executionPayload });
+
+    let proxy = defaultProxy;
+    if (activity.options) {
+        const options = computeActivityOptions(activity.options, defaultOptions);
+        log.debug("Use custom activity options", {
+            activityName: activity.name,
+            activityOptions: options,
+        });
+        proxy = proxyActivities(options)
+    } else {
+        log.debug("Use default activity options", {
+            activityName: activity.name,
+            activityOptions: defaultOptions,
+        });
+    }
+
+    const fn = proxy[activity.name];
+    if (activity.parallel) {
+        //TODO execute in parallel
+        log.info("Parallel execution not yet implemented");
+    } else {
+        log.info("Executing activity: " + activity.name, { importParams });
+        const result = await fn(executionPayload);
+        if (activity.output) {
+            vars.setValue(activity.output, result);
+        }
+    }
+    if (basePayload.debug_mode) {
+        log.debug(`Workflow vars after executing activity ${activity.name}`, { vars: vars.resolve() });
+    }
+}
+
+export function computeActivityOptions(customOptions: DSLActivityOptions, defaultOptions: ActivityOptions): ActivityOptions {
+    const options = convertDSLActivityOptions(customOptions);
+    return {
+        ...defaultOptions,
+        ...options,
+        retry: {
+            ...defaultOptions.retry,
+            ...options.retry,
+        }
+    }
+}
+
+function convertDSLActivityOptions(options?: DSLActivityOptions): ActivityOptions {
+    if (!options) {
+        return {};
+    }
+    let result: ActivityOptions = {};
+    if (options.startToCloseTimeout) {
+        result.startToCloseTimeout = ms(options.startToCloseTimeout as StringValue);
+    }
+    if (options.scheduleToCloseTimeout) {
+        result.scheduleToCloseTimeout = ms(options.scheduleToCloseTimeout as StringValue);
+    }
+    if (options.scheduleToStartTimeout) {
+        result.scheduleToStartTimeout = ms(options.scheduleToStartTimeout as StringValue);
+    }
+    if (options.retry) {
+        result.retry = {};
+        if (options.retry.initialInterval) {
+            result.retry.initialInterval = ms(options.retry.initialInterval as StringValue);
+        }
+        if (options.retry.maximumInterval) {
+            result.retry.maximumInterval = ms(options.retry.maximumInterval as StringValue);
+        }
+        if (options.retry.maximumAttempts) {
+            result.retry.maximumAttempts = options.retry.maximumAttempts;
+        }
+        if (options.retry.backoffCoefficient) {
+            result.retry.backoffCoefficient = options.retry.backoffCoefficient;
+        }
+        if (options.retry.nonRetryableErrorTypes) {
+            result.retry.nonRetryableErrorTypes = options.retry.nonRetryableErrorTypes;
+        }
+    }
+    return result;
+}

package/src/dsl/ms.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Types for the `ms` package (version 3.0.0-canary.1)
+ */
+declare module "ms" {
+    declare type Unit = 'Years' | 'Year' | 'Yrs' | 'Yr' | 'Y' | 'Weeks' | 'Week' | 'W' | 'Days' | 'Day' | 'D' | 'Hours' | 'Hour' | 'Hrs' | 'Hr' | 'H' | 'Minutes' | 'Minute' | 'Mins' | 'Min' | 'M' | 'Seconds' | 'Second' | 'Secs' | 'Sec' | 's' | 'Milliseconds' | 'Millisecond' | 'Msecs' | 'Msec' | 'Ms';
+    declare type UnitAnyCase = Unit | Uppercase<Unit> | Lowercase<Unit>;
+    export declare type StringValue = `${number}` | `${number}${UnitAnyCase}` | `${number} ${UnitAnyCase}`;
+    declare function ms(value: StringValue, options?: Options): number;
+    declare function ms(value: number, options?: Options): string;
+    export default ms;
+}