@vertesia/workflow 0.24.0-dev.202601221707

package/src/dsl/dsl-workflow.ts

@@ -0,0 +1,397 @@
+import {
+    ActivityInterfaceFor,
+    ActivityOptions,
+    CancellationScope,
+    executeChild,
+    isCancellation,
+    log,
+    patched,
+    proxyActivities,
+    sleep,
+    startChild,
+    UntypedActivities,
+} from "@temporalio/workflow";
+import {
+    DSLActivityExecutionPayload,
+    DSLActivityOptions,
+    DSLActivitySpec,
+    DSLChildWorkflowStep,
+    DSLWorkflowExecutionPayload,
+    DSLWorkflowSpec,
+    getDocumentIds,
+    getTenantId,
+    WorkflowExecutionPayload
+} from "@vertesia/common";
+import ms, { StringValue } from 'ms';
+import { HandleDslErrorParams } from "../activities/handleError.js";
+import * as activities from "../activities/index.js";
+import { WF_NON_RETRYABLE_ERRORS, WorkflowParamNotFoundError } from "../errors.js";
+import { Vars } from "./vars.js";
+import { RateLimitParams } from "../activities/rateLimiter.js";
+
+
+interface BaseActivityPayload extends WorkflowExecutionPayload {
+    workflow_name: string;
+    debug_mode?: boolean;
+}
+
+function dslActivityPayload<ParamsT extends Record<string, any>>(basePayload: BaseActivityPayload, activity: DSLActivitySpec, params: ParamsT) {
+    return {
+        ...basePayload,
+        activity,
+        params: params || {},
+    } as DSLActivityExecutionPayload<ParamsT>
+}
+
+export async function dslWorkflow(payload: DSLWorkflowExecutionPayload) {
+
+    const definition = payload.workflow;
+    if (!definition) {
+        throw new WorkflowParamNotFoundError("workflow");
+    }
+    // the base payload will be used to create the activities payload
+    const basePayload: BaseActivityPayload = {
+        ...payload,
+        workflow_name: definition.name,
+        debug_mode: !!definition.debug_mode,
+    }
+    delete (basePayload as any).workflow;
+
+    const defaultOptions: ActivityOptions = {
+        ...convertDSLActivityOptions(definition.options),
+        startToCloseTimeout: "5 minute",
+        retry: {
+            initialInterval: '10s',
+            backoffCoefficient: 2,
+            maximumAttempts: 10,
+            maximumInterval: 100 * 30 * 1000, //ms
+            nonRetryableErrorTypes: WF_NON_RETRYABLE_ERRORS,
+        },
+    };
+    log.debug("Global activity options", {
+        activityOptions: defaultOptions,
+    });
+    const defaultProxy = proxyActivities(defaultOptions);
+    log.debug("Default activity proxy is ready");
+    // merge default vars with the payload vars and add objectIds and objectId
+    const vars = new Vars({
+        ...definition.vars,
+        ...payload.vars,
+        objectIds: payload.objectIds || [],
+        objectId: payload.objectIds ? payload.objectIds[0] : undefined
+    });
+
+    log.info("Executing workflow", { payload });
+
+    // TODO(mhuang): remove patch when all workflows are migrated to v2
+    //   It avoids breaking the ongoing workflow execution running in v1 and also allows us to
+    //   deploy the new error handler in production.
+    //   See https://docs.temporal.io/develop/typescript/versioning
+    if (patched('dsl-workflow-error-handling')) {
+        // v2: new version with error handler
+        try {
+            await executeSteps(definition, payload, basePayload, vars, defaultProxy, defaultOptions);
+        } catch (e) {
+            await handleError(e, basePayload, defaultOptions);
+        }
+    } else {
+        // v1: old version without error handler, deprecated since v0.52.0
+        await executeSteps(definition, payload, basePayload, vars, defaultProxy, defaultOptions);
+    }
+
+    return vars.getValue(definition.result || 'result');
+}
+
+async function executeSteps(definition: DSLWorkflowSpec, payload: DSLWorkflowExecutionPayload, basePayload: BaseActivityPayload, vars: Vars, defaultProxy: ActivityInterfaceFor<UntypedActivities>, defaultOptions: ActivityOptions) {
+    if (definition.steps) {
+        for (const step of definition.steps) {
+            const stepType = step.type;
+            if (stepType === 'workflow') {
+                const childWorkflowStep = step as DSLChildWorkflowStep;
+                if (childWorkflowStep.async) {
+                    await startChildWorkflow(childWorkflowStep, payload, vars, basePayload.debug_mode);
+                } else {
+                    await executeChildWorkflow(childWorkflowStep, payload, vars, basePayload.debug_mode);
+                }
+            } else { // activity
+                await runActivity(step as DSLActivitySpec, basePayload, vars, defaultProxy, defaultOptions);
+            }
+        }
+    } else if (definition.activities) { // legacy support
+        for (const activity of definition.activities) {
+            await runActivity(activity, basePayload, vars, defaultProxy, defaultOptions);
+        }
+    } else {
+        throw new Error("No steps or activities found in the workflow definition");
+    }
+}
+
+async function handleError(originalError: any, basePayload: BaseActivityPayload, defaultOptions: ActivityOptions) {
+    const { handleDslError } = proxyActivities<typeof activities>(defaultOptions);
+
+    const payload = dslActivityPayload(
+        basePayload,
+        {
+            name: "handleDslError",
+            params: { errorMessage: originalError.message },
+        } as DSLActivitySpec,
+        { errorMessage: originalError.message } satisfies HandleDslErrorParams,
+    )
+
+    if (isCancellation(originalError)) {
+        log.warn(`Workflow execution cancelled, executing error handler to update document status`, { error: originalError });
+        // Cleanup logic must be in a nonCancellable scope
+        // If we'd run cleanup outside of a nonCancellable scope it would've been cancelled
+        // before being started because the Workflow's root scope is cancelled.
+        // see https://docs.temporal.io/develop/typescript/cancellation
+        await CancellationScope.nonCancellable(() => handleDslError(payload));
+    } else {
+        log.warn(`Workflow execution failed, executing error handler to update document status`, { error: originalError });
+        await handleDslError(payload);
+    }
+    throw originalError;
+}
+
+async function startChildWorkflow(step: DSLChildWorkflowStep, payload: DSLWorkflowExecutionPayload, vars: Vars, debug_mode?: boolean) {
+    const resolvedVars = vars.resolve();
+    if (step.vars) {
+        // copy user vars (from step definition) to the resolved vars, resolving any expressions
+        const resolvedStepVars = vars.resolveParams(step.vars);
+        Object.assign(resolvedVars, resolvedStepVars);
+    }
+    if (debug_mode) {
+        log.debug(`Workflow vars before starting child workflow ${step.name}`, { vars: resolvedVars });
+    }
+
+    const handle = await startChild(step.name, {
+        ...step.options,
+        args: [{
+            ...payload,
+            workflow: step.spec,
+            vars: resolvedVars,
+        }],
+        memo: {
+            InitiatedBy: payload.initiated_by,
+        },
+        searchAttributes: {
+            AccountId: [payload.account_id],
+            DocumentId: getDocumentIds(payload),
+            ProjectId: [payload.project_id],
+            TenantId: [getTenantId(payload.account_id, payload.project_id)],
+            InitiatedBy: payload.initiated_by ? [payload.initiated_by] : [],
+        },
+    });
+    if (step.output) {
+        vars.setValue(step.output, handle.workflowId);
+    }
+}
+
+async function executeChildWorkflow(step: DSLChildWorkflowStep, payload: DSLWorkflowExecutionPayload, vars: Vars, debug_mode?: boolean) {
+    const resolvedVars = vars.resolve();
+    if (step.vars) {
+        // copy user vars (from step definition) to the resolved vars, resolving any expressions
+        const resolvedStepVars = vars.resolveParams(step.vars);
+        Object.assign(resolvedVars, resolvedStepVars);
+    }
+    if (debug_mode) {
+        log.debug(`Workflow vars before executing child workflow ${step.name}`, { vars: resolvedVars });
+    }
+
+    const result = await executeChild(step.name, {
+        ...step.options,
+        args: [{
+            ...payload,
+            workflow: step.spec,
+            vars: resolvedVars,
+        }],
+        memo: {
+            InitiatedBy: payload.initiated_by,
+        },
+        searchAttributes: {
+            AccountId: [payload.account_id],
+            DocumentId: getDocumentIds(payload),
+            ProjectId: [payload.project_id],
+            TenantId: [getTenantId(payload.account_id, payload.project_id)],
+            InitiatedBy: payload.initiated_by ? [payload.initiated_by] : [],
+        },
+    });
+
+    if (step.output) {
+        vars.setValue(step.output, result);
+        if (debug_mode) {
+            log.debug(`Workflow vars after executing child workflow ${step.name}`, { vars: vars.resolve() });
+        }
+    } else if (debug_mode) {
+        log.debug(`Workflow vars after executing child workflow ${step.name}`, { vars: resolvedVars });
+    }
+}
+
+function buildRateLimitParams(activity: DSLActivitySpec, executionPayload: DSLActivityExecutionPayload<any>): RateLimitParams {
+    // resolve payload params
+    const vars = new Vars({
+        ...executionPayload.params, // imported params (doesn't contain expressions)
+        ...executionPayload.activity.params, // activity params (may contain expressions)
+    });
+    const params = vars.resolve();
+
+    let interactionId: string;
+
+    switch (activity.name) {
+        case "executeInteraction":
+            interactionId = params.interactionName;
+            break;
+
+        case "generateDocumentProperties":
+            interactionId = params.interactionName || "sys:ExtractInformation";
+            break;
+
+        case "identifyTextSections":
+            interactionId = params.interactionName || "sys:IdentifyTextSections";
+            break;
+
+        case "generateOrAssignContentType":
+            interactionId = params.interactionNames?.selectDocumentType || "sys:SelectDocumentType";
+            break;
+
+        case "chunkDocument":
+            interactionId = params.interactionName || "sys:ChunkDocument";
+            break;
+
+        default:
+            // For any other rate-limited activities, try to extract what we can
+            interactionId = params.interactionName;
+            break;
+    }
+
+    if (!interactionId) {
+        throw new Error(`No interaction ID could be determined for activity ${activity.name}`);
+    }
+
+    return {
+        interactionIdOrEndpoint: interactionId,
+        environmentId: params.environment,
+        modelId: params.model,
+    };
+}
+
+async function runActivity(activity: DSLActivitySpec, basePayload: BaseActivityPayload, vars: Vars, defaultProxy: ActivityInterfaceFor<UntypedActivities>, defaultOptions: ActivityOptions) {
+    if (basePayload.debug_mode) {
+        log.debug(`Workflow vars before executing activity ${activity.name}`, { vars: vars.resolve() });
+    }
+    if (activity.condition && !vars.match(activity.condition)) {
+        log.info("Activity skipped: condition not satisfied", activity.condition);
+        return;
+    }
+    const importParams = vars.createImportVars(activity.import);
+    const executionPayload = dslActivityPayload(basePayload, activity, importParams);
+    log.info("Executing activity: " + activity.name, { payload: executionPayload });
+
+    let proxy = defaultProxy;
+    if (activity.options) {
+        const options = computeActivityOptions(activity.options, defaultOptions);
+        log.debug("Use custom activity options", {
+            activityName: activity.name,
+            activityOptions: options,
+        });
+        proxy = proxyActivities(options)
+    } else {
+        log.debug("Use default activity options", {
+            activityName: activity.name,
+            activityOptions: defaultOptions,
+        });
+    }
+
+    if (patched('system-activity-taskqueue')) {
+        // hack: do nothing, remove later
+        // https://github.com/vertesia/composableai/pull/544/files
+    }
+
+    // call rate limiter depending on the activity type
+    const rateLimitedActivities = [
+        "chunkDocument",
+        "executeInteraction",
+        "generateDocumentProperties",
+        "generateOrAssignContentType",
+        "identifyTextSections",
+    ];
+
+    if (activity.name && rateLimitedActivities.includes(activity.name)) {
+        log.info(`Applying rate limit for activity ${activity.name}`);
+        // Apply rate limiting logic here
+         // Check rate limit first - loop until no delay
+        const rateLimitParams = buildRateLimitParams(activity, executionPayload);
+
+        const rateLimitPayload = dslActivityPayload(basePayload, activity, rateLimitParams);
+        let rateLimitResult = await proxy.checkRateLimit(rateLimitPayload);
+    
+        while (rateLimitResult.delayMs > 0) {
+            log.info(`Rate limit delay applied: ${rateLimitResult.delayMs}ms`);
+            await sleep(rateLimitResult.delayMs);
+
+            // Check again after sleeping
+            rateLimitResult = await proxy.checkRateLimit(rateLimitPayload);
+        }
+    }
+
+    const fn = proxy[activity.name];
+    if (activity.parallel) {
+        //TODO execute in parallel
+        log.info("Parallel execution not yet implemented");
+    } else {
+        log.info("Executing activity: " + activity.name, { importParams });
+        const result = await fn(executionPayload);
+        if (activity.output) {
+            vars.setValue(activity.output, result);
+        }
+    }
+    if (basePayload.debug_mode) {
+        log.debug(`Workflow vars after executing activity ${activity.name}`, { vars: vars.resolve() });
+    }
+}
+
+export function computeActivityOptions(customOptions: DSLActivityOptions, defaultOptions: ActivityOptions): ActivityOptions {
+    const options = convertDSLActivityOptions(customOptions);
+    return {
+        ...defaultOptions,
+        ...options,
+        retry: {
+            ...defaultOptions.retry,
+            ...options.retry,
+        }
+    }
+}
+
+function convertDSLActivityOptions(options?: DSLActivityOptions): ActivityOptions {
+    if (!options) {
+        return {};
+    }
+    let result: ActivityOptions = {};
+    if (options.startToCloseTimeout) {
+        result.startToCloseTimeout = ms(options.startToCloseTimeout as StringValue);
+    }
+    if (options.scheduleToCloseTimeout) {
+        result.scheduleToCloseTimeout = ms(options.scheduleToCloseTimeout as StringValue);
+    }
+    if (options.scheduleToStartTimeout) {
+        result.scheduleToStartTimeout = ms(options.scheduleToStartTimeout as StringValue);
+    }
+    if (options.retry) {
+        result.retry = {};
+        if (options.retry.initialInterval) {
+            result.retry.initialInterval = ms(options.retry.initialInterval as StringValue);
+        }
+        if (options.retry.maximumInterval) {
+            result.retry.maximumInterval = ms(options.retry.maximumInterval as StringValue);
+        }
+        if (options.retry.maximumAttempts) {
+            result.retry.maximumAttempts = options.retry.maximumAttempts;
+        }
+        if (options.retry.backoffCoefficient) {
+            result.retry.backoffCoefficient = options.retry.backoffCoefficient;
+        }
+        if (options.retry.nonRetryableErrorTypes) {
+            result.retry.nonRetryableErrorTypes = options.retry.nonRetryableErrorTypes;
+        }
+    }
+    return result;
+}

package/src/dsl/dslProxyActivities.ts

@@ -0,0 +1,38 @@
+import { ActivityOptions, proxyActivities } from "@temporalio/workflow";
+import { DSLActivityExecutionPayload, WorkflowExecutionBaseParams, WorkflowExecutionPayload } from "@vertesia/common";
+
+export interface DslActivityFunction<ParamsT extends Record<string, any> = any, ReturnT = any> {
+    (payload: DSLActivityExecutionPayload<ParamsT>): Promise<ReturnT>;
+}
+
+export interface DslSimplifiedActivityFunction<ParamsT = any, ReturnT = any> {
+    (payload: WorkflowExecutionBaseParams, params: ParamsT): Promise<ReturnT>;
+}
+
+export function dslProxyActivities<
+    ActivitiesT extends Record<string, DslActivityFunction<any, any>>
+>(workflowName: string, options: ActivityOptions = {}) {
+    type DslActivities = {
+        [K in keyof ActivitiesT]: ActivitiesT[K] extends DslActivityFunction<infer ParamsT, infer ReturnT>
+        ? DslSimplifiedActivityFunction<ParamsT, ReturnT>
+        : never;
+    };
+
+    const activities = proxyActivities<ActivitiesT>(options) as ActivitiesT;
+
+    return new Proxy({}, {
+        get(_target, prop) {
+            const activityFn = activities[prop as keyof ActivitiesT] as DslActivityFunction;
+            return (payload: WorkflowExecutionPayload, params: any) => {
+                return activityFn({
+                    ...payload,
+                    activity: {
+                        name: prop as string,
+                    },
+                    workflow_name: workflowName,
+                    params,
+                });
+            };
+        },
+    }) as unknown as DslActivities;
+}

package/src/dsl/ms.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Types for the `ms` package (version 3.0.0-canary.1)
+ */
+declare module "ms" {
+    declare type Unit = 'Years' | 'Year' | 'Yrs' | 'Yr' | 'Y' | 'Weeks' | 'Week' | 'W' | 'Days' | 'Day' | 'D' | 'Hours' | 'Hour' | 'Hrs' | 'Hr' | 'H' | 'Minutes' | 'Minute' | 'Mins' | 'Min' | 'M' | 'Seconds' | 'Second' | 'Secs' | 'Sec' | 's' | 'Milliseconds' | 'Millisecond' | 'Msecs' | 'Msec' | 'Ms';
+    declare type UnitAnyCase = Unit | Uppercase<Unit> | Lowercase<Unit>;
+    export declare type StringValue = `${number}` | `${number}${UnitAnyCase}` | `${number} ${UnitAnyCase}`;
+    declare function ms(value: StringValue, options?: Options): number;
+    declare function ms(value: number, options?: Options): string;
+    export default ms;
+}

package/src/dsl/projections.test.ts

@@ -0,0 +1,159 @@
+import { describe, test, expect } from "vitest";
+import { Vars } from "./vars.ts";
+import { makeProjection } from "./projections.ts";
+
+describe('Result Projections', () => {
+
+    test('simple projection', () => {
+        const params: Record<string, any> = {
+            docTypes: [{ id: 1, name: "type1" }, { id: 2, name: "type2" }],
+            objectId: "123"
+        }
+        const result = {
+            result: {
+                document_type: "C",
+            }
+        }
+        const spec = {
+            docType: "${docTypes[0].name}",
+            objectId: "${objectId}",
+            resultType: "${#.result.document_type}",
+            timestamp: 123456
+        }
+        const out = makeProjection(spec, params, result);
+
+        expect(Object.keys(out).length).toBe(4);
+        expect(out.docType).toBe("type1");
+        expect(out.objectId).toBe("123");
+        expect(out.resultType).toBe("C");
+        expect(out.timestamp).toBe(123456);
+        expect(out.foo).toBeUndefined();
+    })
+
+    test('$element match', () => {
+        const params: Record<string, any> = {
+            docTypes: [{ id: 1, name: "type1" }, { id: 2, name: "type2" }],
+            objectId: "123"
+        }
+        const result = {
+            result: {
+                document_type: "type2",
+            }
+        }
+        const spec = {
+            docType: {
+                $element: {
+                    from: "${docTypes}",
+                    where: { name: { $eq: "${#.result.document_type}" } },
+                    else: { id: 0, name: "unknown" }
+                }
+            },
+            isNewType: {
+                $eval: {
+                    "#.result.document_type": { $nin: "${docTypes.name}" }
+                }
+            }
+        }
+
+        const out = makeProjection(spec, params, result);
+
+
+        expect(Object.keys(out).length).toBe(2);
+        expect(out.docType).toStrictEqual({ id: 2, name: "type2" });
+        expect(out.isNewType).toBe(false);
+    })
+
+    test('$element does not match', () => {
+        const params: Record<string, any> = {
+            docTypes: [{ id: 1, name: "type1" }, { id: 2, name: "type2" }],
+            objectId: "123"
+        }
+        const result = {
+            result: {
+                document_type: "a_new_type",
+            }
+        }
+        const projection = {
+            docType: {
+                $element: {
+                    from: "${docTypes}",
+                    where: { name: { $eq: "${#.result.document_type}" } },
+                    else: { id: 0, name: "${#.result.document_type}" }
+                }
+            },
+            isNewType: {
+                $eval: {
+                    "#.result.document_type": { $nin: "${docTypes.name}" }
+                }
+            }
+        }
+
+        const out = makeProjection(projection, params, result);
+
+
+        expect(Object.keys(out).length).toBe(2);
+        expect(out.docType).toStrictEqual({ id: 0, name: "a_new_type" });
+        expect(out.isNewType).toBe(true);
+    })
+
+    test('$element with field', () => {
+        const params: Record<string, any> = {
+            docTypes: [{ id: 1, name: "type1" }, { id: 2, name: "type2" }],
+            objectId: "123"
+        }
+        const result = {
+            result: {
+                document_type: "type2",
+            }
+        }
+        const projection = {
+            other: null,
+            name: "${#.result.document_type}",
+            id: {
+                $element: {
+                    field: "id",
+                    from: "${docTypes}",
+                    where: { name: { $eq: "${#.result.document_type}" } },
+                    else: null
+                }
+            },
+        }
+
+        const out = makeProjection(projection, params, result);
+
+
+        expect(Object.keys(out).length).toBe(3);
+        expect(out).toStrictEqual({ id: 2, name: "type2", other: null });
+    })
+
+    test('$element with field - no match', () => {
+        const params: Record<string, any> = {
+            docTypes: [{ id: 1, name: "type1" }, { id: 2, name: "type2" }],
+            objectId: "123"
+        }
+        const result = {
+            result: {
+                document_type: "a_new_type",
+            }
+        }
+        const projection = {
+            other: null,
+            name: "${#.result.document_type}",
+            id: {
+                $element: {
+                    field: "id",
+                    from: "${docTypes}",
+                    where: { name: { $eq: "${#.result.document_type}" } },
+                    else: null
+                }
+            },
+        }
+
+        const out = makeProjection(projection, params, result);
+
+
+        expect(Object.keys(out).length).toBe(3);
+        expect(out).toStrictEqual({ id: null, name: "a_new_type", other: null });
+    })
+
+});

package/src/dsl/projections.ts

@@ -0,0 +1,72 @@
+import { DSLActivityExecutionPayload } from "@vertesia/common";
+import { matchCondition } from "./conditions.js";
+import { Vars } from "./vars.js";
+
+
+interface ProjectOperation {
+    (arg: any, vars: Vars): any
+}
+
+interface ElementOperation {
+    field?: string,
+    from: any[],
+    where: Record<string, any>,
+    else: any
+}
+const operations: Record<string, ProjectOperation> = {
+    $element(arg: ElementOperation, _vars: Vars) {
+        const where = arg.where;
+        const whereKeys = Object.keys(where);
+        const r = arg.from.find((elem: any) => {
+            for (const key of whereKeys) {
+                const value = key === '_' ? elem : elem[key];
+                if (matchCondition(value, where[key])) {
+                    return true;
+                }
+            }
+            return false;
+        })
+        if (arg.field) {
+            return r ? r[arg.field] : arg.else;
+        } else {
+            return r || arg.else;
+        }
+    },
+    $eval(arg: any, vars: Vars) {
+        return vars.match(arg);
+    }
+}
+
+function runProjection(obj: any, vars: Vars) {
+    if (obj && !Array.isArray(obj) && typeof obj === "object") {
+        const keys = Object.keys(obj)
+        if (keys.length === 1) {
+            const key = keys[0];
+            const fn = operations[key];
+            if (fn) {
+                return fn(obj[key], vars);
+            }
+        }
+    }
+    return obj; // return the value as is
+}
+
+export function projectResult<TParams extends Record<string, any>>(payload: DSLActivityExecutionPayload<TParams>, params: Record<string, any>, result: any, fallback: any) {
+    return payload.activity.projection ? makeProjection(payload.activity.projection, params, result) : fallback;
+}
+
+export function makeProjection(spec: Record<string, any>, params: Record<string, any>, result: any) {
+    const vars = new Vars({
+        ...params,
+        '#': result,
+    });
+
+    const projection = vars.resolveParams(spec);
+
+    const out: any = {}
+    for (const [key, value] of Object.entries(projection)) {
+        out[key] = runProjection(value, vars);
+    }
+
+    return out;
+}