@vertesia/workflow 0.24.0-dev.202601221707

package/src/dsl/setup/ActivityContext.ts

@@ -0,0 +1,178 @@
+import { log, activityInfo } from "@temporalio/activity";
+import { VertesiaClient } from "@vertesia/client";
+import {
+    DSLActivityExecutionPayload,
+    DSLWorkflowExecutionPayload,
+    Project,
+    WorkflowExecutionPayload,
+} from "@vertesia/common";
+import {
+    DocumentNotFoundError,
+    WorkflowParamNotFoundError,
+} from "../../errors.js";
+import { getProjectFromToken } from "../../utils/auth.js";
+import { getVertesiaClient } from "../../utils/client.js";
+import { Vars } from "../vars.js";
+import {
+    getFetchProvider,
+    registerFetchProviderFactory,
+} from "./fetch/index.js";
+import {
+    DocumentProvider,
+    DocumentTypeProvider,
+    InteractionRunProvider,
+} from "./fetch/providers.js";
+
+registerFetchProviderFactory(DocumentProvider.ID, DocumentProvider.factory);
+registerFetchProviderFactory(
+    DocumentTypeProvider.ID,
+    DocumentTypeProvider.factory,
+);
+registerFetchProviderFactory(
+    InteractionRunProvider.ID,
+    InteractionRunProvider.factory,
+);
+
+export class ActivityContext<ParamsT extends Record<string, any>> {
+    client: VertesiaClient;
+    _project?: Promise<Project | undefined>;
+
+    constructor(
+        public payload: DSLActivityExecutionPayload<ParamsT>,
+        client: VertesiaClient,
+        public params: ParamsT,
+    ) {
+        this.client = client;
+        this.fetchProject = this.fetchProject.bind(this);
+    }
+
+    get objectIds() {
+        return this.payload.objectIds;
+    }
+
+    get objectId() {
+        const objectId = this.payload.objectIds && this.payload.objectIds[0];
+        if (!objectId) {
+            log.error("No objectId found in payload");
+            throw new WorkflowParamNotFoundError(
+                "objectIds[0]",
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
+            );
+        }
+        return objectId;
+    }
+
+    get activityInfo() {
+        return activityInfo();
+    }
+
+    get runId() {
+        const runId = activityInfo().workflowExecution.runId;
+        if (!runId) {
+            log.error("No runId found in activityInfo");
+            throw new WorkflowParamNotFoundError(
+                "runId",
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
+            );
+        }
+        return runId;
+    }
+
+    get workflowId() {
+        const workflowId = activityInfo().workflowExecution.workflowId;
+        if (!workflowId) {
+            log.error("No workflowId found in activityInfo");
+            throw new WorkflowParamNotFoundError(
+                "workflowId",
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
+            );
+        }
+        return workflowId;
+    }
+
+    fetchProject() {
+        if (!this._project) {
+            this._project = _fetchProject(this.client, this.payload);
+        }
+        return this._project;
+    }
+}
+
+export async function setupActivity<ParamsT extends Record<string, any>>(
+    payload: DSLActivityExecutionPayload<ParamsT>,
+) {
+    const isDebugMode = !!payload.debug_mode;
+
+    const vars = new Vars({
+        ...payload.params, // imported params (doesn't contain expressions)
+        ...payload.activity.params, // activity params (may contain expressions)
+    });
+
+    //}
+    if (isDebugMode) {
+        log.info(`Setting up activity ${payload.activity.name}`, {
+            config: payload.config,
+            activity: payload.activity,
+            params: payload.params,
+            vars,
+        });
+    }
+
+    const client = await getVertesiaClient(payload);
+    const fetchSpecs = payload.activity.fetch;
+    if (fetchSpecs) {
+        const keys = Object.keys(fetchSpecs);
+        if (keys.length > 0) {
+            // create a new Vars instance to store the fetched data
+            for (const key of keys) {
+                let fetchSpec = fetchSpecs[key];
+                let query = fetchSpec.query;
+                if (query) {
+                    query = vars.resolveParams(query);
+                    fetchSpec = { ...fetchSpec, query };
+                }
+
+                const provider = getFetchProvider(client, fetchSpec);
+
+                log.info(
+                    `Fetching data for ${key} with provider ${provider.name}`,
+                    { fetchSpec },
+                );
+                const result = await provider.fetch(fetchSpec);
+                if (result && result.length > 0) {
+                    if (fetchSpec.limit === 1) {
+                        vars.setValue(key, result[0]);
+                    } else {
+                        vars.setValue(key, result);
+                    }
+                } else if (fetchSpec.on_not_found === "throw") {
+                    throw new DocumentNotFoundError(
+                        "No documents found for: " + JSON.stringify(fetchSpec),
+                    );
+                } else {
+                    vars.setValue(key, null);
+                }
+            }
+        }
+    }
+
+    const params = vars.resolve() as ParamsT;
+    return new ActivityContext<ParamsT>(payload, client, params);
+}
+
+async function _fetchProject(
+    client: VertesiaClient,
+    payload: WorkflowExecutionPayload,
+) {
+    const project = await getProjectFromToken(payload.auth_token);
+    return project ? await client.projects.retrieve(project.id) : undefined;
+}

package/src/dsl/setup/fetch/DataProvider.ts

@@ -0,0 +1,45 @@
+import { FindPayload } from "@vertesia/common";
+
+function parseSelector(selector: string) {
+    const parts = selector.split(/\s+/);
+    const result: Record<string, number> = {};
+    for (const part of parts) {
+        if (part.startsWith("-")) {
+            result[part.substring(1)] = 0;
+        } else {
+            result[part] = 1;
+        }
+    }
+    return result;
+}
+
+function applyProjection(result: Record<string, any>, select: string) {
+    if (!result) return result;
+    let selectorObj: Record<string, number | boolean>;
+    if (typeof select === 'string') {
+        selectorObj = parseSelector(select);
+    } else {
+        selectorObj = select;
+    }
+
+    const out: Record<string, any> = {};
+    for (const key of Object.keys(result)) {
+        if (selectorObj[key]) {
+            out[key] = result[key];
+        }
+    }
+    return out;
+}
+
+export abstract class DataProvider {
+    constructor(public name: string, public isProjectionSupported = false) {
+    }
+    async fetch(payload: FindPayload) {
+        let results = await this.doFetch(payload);
+        if (payload.select && !this.isProjectionSupported) {
+            results = results.map((result: Record<string, any>) => applyProjection(result, payload.select!));
+        }
+        return results;
+    }
+    abstract doFetch(payload: FindPayload): Promise<Record<string, any>[]>;
+}

package/src/dsl/setup/fetch/index.ts

@@ -0,0 +1,19 @@
+import { ActivityFetchSpec } from "@vertesia/common";
+import { VertesiaClient } from "@vertesia/client";
+import { DataProvider } from "./DataProvider.js";
+
+
+const factories: Record<string, ((client: VertesiaClient, source?: string) => DataProvider)> = {};
+
+
+export function registerFetchProviderFactory(name: string, factory: ((client: VertesiaClient) => DataProvider)) {
+    factories[name] = factory;
+}
+
+export function getFetchProvider(client: VertesiaClient, fetchSpec: ActivityFetchSpec) {
+    const factory = factories[fetchSpec.type];
+    if (!factory) {
+        throw new Error("Unknown data provider: " + fetchSpec.source);
+    }
+    return factory(client, fetchSpec.source);
+}

package/src/dsl/setup/fetch/providers.ts

@@ -0,0 +1,67 @@
+import { FindPayload } from "@vertesia/common";
+import { VertesiaClient } from "@vertesia/client";
+import { DataProvider } from "./DataProvider.js";
+
+function useMongoId(query: Record<string, any>) {
+    if (query.id) {
+        const result = { ...query, _id: query.id }
+        delete (result as any).id;
+        return result;
+    }
+    return query;
+}
+
+export class DocumentProvider extends DataProvider {
+    static ID = "document";
+    constructor(public client: VertesiaClient) {
+        super(DocumentProvider.ID, true);
+    }
+
+    doFetch(payload: FindPayload): Promise<Record<string, any>[]> {
+        const query = useMongoId(payload.query);
+        return this.client.objects.find({
+            query, select: payload.select, limit: payload.limit
+        });
+    }
+
+    static factory(client: VertesiaClient) {
+        return new DocumentProvider(client);
+    }
+}
+
+export class DocumentTypeProvider extends DataProvider {
+    static ID = "document_type";
+    constructor(public client: VertesiaClient) {
+        super(DocumentTypeProvider.ID, true);
+    }
+
+    doFetch(payload: FindPayload): Promise<Record<string, any>[]> {
+        const query = useMongoId(payload.query);
+        return this.client.types.find({
+            query, select: payload.select, limit: payload.limit
+        });
+    }
+
+    static factory(client: VertesiaClient) {
+        return new DocumentTypeProvider(client);
+    }
+}
+
+export class InteractionRunProvider extends DataProvider {
+    static ID = "interaction_run";
+    constructor(public client: VertesiaClient) {
+        super(DocumentProvider.ID, true);
+    }
+
+    doFetch(payload: FindPayload): Promise<Record<string, any>[]> {
+        const query = useMongoId(payload.query);
+        return this.client.runs.find({
+            query, select: payload.select, limit: payload.limit
+        });
+    }
+
+    static factory(client: VertesiaClient) {
+        return new InteractionRunProvider(client);
+    }
+
+}

package/src/dsl/test/test-child-workflow.ts

@@ -0,0 +1,6 @@
+import { WorkflowExecutionPayload } from "@vertesia/common";
+export { dslWorkflow } from "../dsl-workflow.js";
+
+export async function testChildWorkflow(payload: WorkflowExecutionPayload) {
+    return `Child: Hello, ${payload.vars.name}!`;
+}

package/src/dsl/validation.test.ts

@@ -0,0 +1,257 @@
+import { describe, expect, test } from "vitest";
+import { validateWorkflow } from "./validation.ts";
+
+describe('workflow validation', () => {
+
+    test('empty object is not a valid workflow', () => {
+        const workflow: any = {
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(2);
+    })
+
+    test('activities is required', () => {
+        const workflow: any = {
+            name: "test",
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(1);
+    })
+
+    test('activities should be an array', () => {
+        const workflow: any = {
+            name: "test",
+            activities: {}
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(1);
+    })
+
+    test('activities array should have at least one item', () => {
+        const workflow: any = {
+            name: "test",
+            activities: []
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(1);
+    })
+
+    test('activity should have a name', () => {
+        const workflow: any = {
+            name: "test",
+            activities: [{}]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(1);
+    })
+
+    test('allow empty activity', () => {
+        const workflow: any = {
+            name: "test",
+            activities: [{ name: "test" }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(0);
+    })
+
+    test('import undeclared var', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true },
+            activities: [{
+                name: "test",
+                import: ["foo", "bar"]
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(1);
+    })
+
+
+    test('import declared vars', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true, "bar": true },
+            activities: [{
+                name: "test",
+                import: ["foo", "bar"]
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(0);
+    })
+
+    test('import unknown imported var through expressions', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true },
+            activities: [{
+                name: "test",
+                import: [{ "foo": "foo", "barLen": "bar.length" }]
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(1);
+    })
+
+    test('import declared vars through expressions', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true, "bar": "true" },
+            activities: [{
+                name: "test",
+                import: [{ "foo": "foo", "barLen": "bar.length" }]
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(0);
+    })
+
+    test('detect self references', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "object_type": "thetype" },
+            activities: [{
+                name: "test",
+                import: ["object_type"],
+                params: {
+                    "object_type": "${object_type}"
+                }
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(1);
+        expect(errors[0].includes("Self referencing parameter")).toBe(true);
+    })
+
+    test('reference known vars in fetch', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true, "bar": "true" },
+            activities: [{
+                name: "test",
+                import: [{ "foo": "foo", "barLen": "bar.length" }],
+                fetch: {
+                    doc: {
+                        query: {
+                            foo: "${foo}",
+                            barLen: "${barLen}"
+                        }
+                    }
+                }
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        console.log('##############errors', errors);
+
+        expect(errors.length).toBe(0);
+    })
+
+    test('reference unknown vars in fetch', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true, "bar": "true" },
+            activities: [{
+                name: "test",
+                import: [],
+                fetch: {
+                    doc: {
+                        query: {
+                            foo: "${foo}",
+                            barLen: "${barLen}"
+                        }
+                    }
+                }
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(2);
+    })
+
+    test('reference one unknown and one known var in fetch', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true, "bar": "true" },
+            activities: [{
+                name: "test",
+                import: ["foo"],
+                fetch: {
+                    doc: {
+                        query: {
+                            foo: "${foo}",
+                            barLen: "${barLen}"
+                        }
+                    }
+                }
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(1);
+    })
+
+    test('reference 2 unknown vars in params', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true, "bar": "true" },
+            activities: [{
+                name: "test",
+                import: ["foo"],
+                params: {
+                    barLength: "${barLen}",
+                    doc: "${doc.text}"
+                }
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(2);
+    })
+
+    test('reference 1 unknown vars in params', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true, "bar": "true" },
+            activities: [{
+                name: "test",
+                import: ["foo", { "barLen": "bar.length" }],
+                params: {
+                    barLength: "${barLen}",
+                    doc: "${doc.text}"
+                }
+            }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(1);
+    })
+
+    test('reference known vars in params', () => {
+        const workflow: any = {
+            name: "test",
+            vars: { "foo": true, "bar": "true" },
+            activities: [
+                {
+                    name: "test0",
+                    output: "previousResult"
+                }, {
+                    name: "test",
+                    import: ["foo", { "barLen": "bar.length" }, "previousResult"],
+                    fetch: {
+                        doc: {
+                            query: {
+                                foo: "${foo}",
+                                barLen: "${barLen}"
+                            }
+                        }
+                    },
+                    params: {
+                        fooParam: "${foo}",
+                        barLenParam: "${barLen}",
+                        doc: "${doc.text}",
+                        prev: "${previousResult}"
+                    }
+                }]
+        }
+        const errors = validateWorkflow(workflow);
+        expect(errors.length).toBe(0);
+    })
+
+})

package/src/dsl/validation.ts

@@ -0,0 +1,125 @@
+import { DSLActivitySpec, DSLWorkflowSpec, DSLWorkflowStep } from "@vertesia/common";
+import { Vars, splitPath } from "./vars.js";
+
+export function validateWorkflow(workflow: DSLWorkflowSpec, vars: string[] = []) {
+    const errors: string[] = [];
+    const workflowVars = new Set(vars);
+    workflowVars.add("objectId");
+    workflowVars.add("objectIds");
+    if (workflow.vars) {
+        for (const v of Object.keys(workflow.vars)) {
+            workflowVars.add(v);
+        }
+    }
+    if (!workflow.name) {
+        errors.push("Workflow 'name' property is required");
+    }
+    if (workflow.steps && workflow.activities) {
+        errors.push("You must use either 'steps' or 'activities'. You cannot use both. Prefer using steps.");
+        return errors;
+    }
+    if (!workflow.steps && !workflow.activities) {
+        errors.push("The workflow requires one of 'steps' or 'activities' properties. Neither is present.");
+        return errors;
+    }
+    const stepsPropName = workflow.steps ? "steps" : "activities";
+    const steps = workflow.steps || workflow.activities;
+    if (!steps || !Array.isArray(steps)) {
+        errors.push(`Workflow '${stepsPropName}' property is required`);
+        return errors;
+    }
+    if (!steps || !Array.isArray(steps)) {
+        errors.push(`Workflow '${stepsPropName}' should be an array`);
+        return errors;
+    }
+    if (!steps.length) {
+        errors.push("Workflow should have at least one step or activity");
+    }
+
+    const activities: DSLActivitySpec[] = stepsPropName === "steps" ? (steps as DSLWorkflowStep[]).filter(s => s.type === "activity") : steps;
+    for (const activity of activities) {
+        const activityErrors = validateActivity(activity, workflowVars);
+        if (activityErrors.length > 0) {
+            for (const err of activityErrors) {
+                errors.push(`Activity "${activity.name}": ${err}`);
+            }
+        }
+        if (activity.output) {
+            workflowVars.add(activity.output);
+        }
+    }
+
+    return errors;
+}
+
+export function validateActivity(activity: DSLActivitySpec, workflowVars: Set<string>): string[] {
+    const errors: string[] = [];
+    if (!activity.name) {
+        errors.push("Activity name is required");
+    }
+    const importedVars = activity.import;
+    const localVars: Record<string, boolean> = {};
+    if (importedVars && importedVars.length > 0) {
+        for (const entry of importedVars) {
+            if (typeof entry === 'string') {
+                pushLocalVar(entry, localVars, errors);
+                if (!workflowVars.has(entry)) {
+                    errors.push(`Unknown workflow variable "${entry}" in import declaration`);
+                }
+            } else {
+                for (const key of Object.keys(entry)) {
+                    pushLocalVar(key, localVars, errors);
+                    const wfExpr = entry[key];
+                    const wfVar = splitPath(wfExpr)[0];
+                    if (!workflowVars.has(wfVar)) {
+                        errors.push(`Unknown workflow variable "${wfExpr}" in import declaration`);
+                    }
+                }
+            }
+        }
+    }
+
+    // collect fetch vars and check expressions in fetch declarations
+    const fetch = activity.fetch;
+    if (fetch) {
+        const keys = Object.keys(fetch);
+        for (const key of keys) {
+            const query = fetch[key].query;
+            // check expressions in query
+            validateExpressions(query, localVars, errors);
+        }
+        for (const key of keys) {
+            pushLocalVar(key, localVars, errors);
+        }
+    }
+    // check expressions in activity params
+    if (activity.params) {
+        validateExpressions(activity.params, localVars, errors, true);
+    }
+    return errors;
+}
+
+
+function validateExpressions(target: Record<string, any>, localVars: Record<string, boolean>, errors: string[], checkSelfReference = false) {
+    const vars = new Vars(localVars);
+    const refs = vars.getUnknownReferences(target);
+    for (const ref of refs) {
+        errors.push(`Unknown variable "${ref.name}" in expression "${ref.expression}"`)
+    }
+    if (checkSelfReference) {
+        // check for self references like `"object_type": "${object_type}"`.
+        for (const key of Object.keys(target)) {
+            const value = target[key];
+            if (typeof value === 'string' && value.includes("${" + key + "}")) {
+                errors.push(`Self referencing parameter "${key}" in expression "${value}"`);
+            }
+        }
+    }
+}
+
+function pushLocalVar(v: string, localVars: Record<string, boolean>, errors: string[]) {
+    if (localVars[v] === true) {
+        errors.push(`Duplicate variable "${v}"`);
+    }
+    localVars[v] = true;
+}