@vertesia/workflow 0.42.2

package/src/dsl/vars.ts

@@ -0,0 +1,340 @@
+import { ImportSpec } from "@vertesia/common";
+import { matchCondition } from "./conditions.js";
+import { ObjectKey, ObjectVisitor, ObjectWalker } from "./walk.js";
+
+const FALLBACK_VALUE_SEP = "??";
+
+function decodeLiteralValue(value: string) {
+    if (value.startsWith("'") && value.endsWith("'")) {
+        value = '"' + value.slice(1, -1).replace(/(?<!\\)"/g, '\\"') + '"'
+    }
+    return JSON.parse(value);
+}
+
+export function splitPath(path: string) {
+    if (path.indexOf('[') > -1) { // support array like paths
+        path = path.replace(/\[(\d+)\]/g, '.$1');
+    }
+    return path.split('.');
+}
+
+/**
+ * Get the property named by "name" of the given object
+ * If an array is idnexed using a string key then a map is done and an array with the content of the properties with that name are returned
+ * Ex: docs.text => will return an array of text properties of the docs array
+ * @param object the obejct
+ * @param name the name of the property.
+ * @returns the property value
+ */
+function _prop(object: any, name: string) {
+    if (object === undefined) {
+        return undefined;
+    }
+    object = _valueOf(object); // resolve Value objects
+    if (Array.isArray(object)) {
+        const index = +name;
+        if (isNaN(index)) {
+            // map array to property
+            return object.map(item => item[name]);
+        } else {
+            return _valueOf(object[index]);
+        }
+    } else {
+        return _valueOf(object[name]);
+    }
+
+}
+
+function _valueOf(value: any) {
+    return value instanceof Value ? value.value : value;
+}
+
+export function resolveField(object: any, path: string[]) {
+    let p = object as any;
+    if (!p) return p;
+    if (!path.length) return _valueOf(p);
+    const last = path.length - 1;
+    for (let i = 0; i < last; i++) {
+        p = _prop(p, path[i])
+        if (!p) {
+            return undefined;
+        }
+    }
+    return _prop(p, path[last]);
+}
+
+abstract class Value<T = any> {
+    abstract value: T;
+    abstract stringify(): string;
+}
+class LiteralValue<T = any> extends Value<T> {
+    constructor(public value: T) {
+        super();
+    }
+    stringify() {
+        return String(this.value);
+    }
+}
+
+class RefValue extends Value {
+    constructor(public vars: Vars, public path: string[], public defaultValue?: any) {
+        super();
+    }
+    get value() {
+        const v = this.vars.getValueFromPath(this.path);
+        if (v === undefined) {
+            if (this.defaultValue !== undefined) {
+                return this.defaultValue;
+            } else {
+                return this.vars.preserveRefs ? this.stringify() : undefined;
+            }
+        } else {
+            return v;
+        }
+    }
+
+    stringify() {
+        return "${" + this.path.join('.') + "}";
+    }
+}
+
+class ExprValue extends Value {
+    constructor(public vars: Vars, public parts: Value[]) {
+        super();
+    }
+    get value() {
+        const out: string[] = [];
+        for (const seg of this.parts) {
+            const v = seg.value;
+            if (v !== undefined) {
+                out.push(String(seg.value));
+            }
+        }
+        return out.join('');
+    }
+
+    stringify() {
+        const out = [];
+        for (const seg of this.parts) {
+            out.push(seg.stringify());
+        }
+        return "${" + out.join('') + "}";
+    }
+
+}
+
+
+export class Vars {
+    map: Record<string, any>;
+    /**
+     * This property is used when resolving params. It contains the list of references that should not be resolved to their value
+     * but instead they need to return the string representation of the expression to eb able to regenrate another Vars instance with the same expression
+     */
+    //TODO this feature is no more used - it was replaced by importVars so we can now delete `preserveRefs`
+    preserveRefs: Set<string> | undefined;
+
+    constructor(vars?: Record<string, any>) {
+        this.map = vars ? this.parse(vars) : {};
+    }
+
+    parse(vars: Record<string, any>): Record<string, any> {
+        return new ObjectWalker().map(vars, (_key, value) => {
+            if (typeof value === 'string') {
+                return this.createValue(this, value);
+            } else {
+                return value;
+            }
+        });
+    }
+
+    load(vars: Record<string, any>) {
+        const toAppend = this.parse(vars);
+        this.map = Object.assign(this.map, toAppend);
+        return this;
+    }
+
+    /**
+     * Set a literal value (canboot set a ref)
+     * To add refs use `append()`
+     * @param name
+     * @param value
+     */
+    setValue(name: string, value: any) {
+        this.map[name] = value;
+    }
+
+    getValue(path: string) {
+        return resolveField(this.map, splitPath(path));
+    }
+
+    getValueFromPath(path: string[]) {
+        return resolveField(this.map, path);
+    }
+
+    has(name: string) {
+        return this.map[name] !== undefined;
+    }
+
+    match(match: Record<string, any>) {
+        for (const name of Object.keys(match)) {
+            const value = this.getValue(name);
+            if (!matchCondition(value, match[name])) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    resolveParams(params: Record<string, any>, preserveRefs?: Set<string>) {
+        this.preserveRefs = preserveRefs;
+        try {
+            return new ObjectWalker().map(params, (_key, value) => {
+                if (typeof value === 'string') {
+                    const v = this.createValue(this, value)
+                    return v instanceof Value ? v.value : v;
+                } else {
+                    return value;
+                }
+            });
+        } finally {
+            this.preserveRefs = undefined;
+        }
+    }
+
+    resolve(preserveRefs?: Set<string>): Record<string, any> {
+        function map(_key: ObjectKey, value: any) {
+            if (value instanceof Value) {
+                const v = value.value;
+                if (v && typeof v === 'object') {
+                    if (Array.isArray(v) || v.constructor === Object) {
+                        // an array or plain object - recurse into the
+                        // value to find other nested Ref values if any...
+                        return new ObjectWalker().map(v, map);
+                    }
+                } else {
+                    return v;
+                }
+            } else {
+                return value
+            }
+        }
+        try {
+            this.preserveRefs = preserveRefs;
+            return new ObjectWalker().map(this.map, map);
+        } finally {
+            this.preserveRefs = undefined;
+        }
+    }
+
+    createRefValue(ref: string) {
+        const index = ref.indexOf(FALLBACK_VALUE_SEP);
+        let defaultValue: any;
+        if (index > -1) {
+            defaultValue = decodeLiteralValue(ref.substring(index + FALLBACK_VALUE_SEP.length).trim());
+            ref = ref.substring(0, index).trim();
+        }
+        if (ref === '.' || ref.indexOf('..') > -1) {
+            throw new Error("Invalid variable reference: " + ref)
+        }
+        return new RefValue(this, splitPath(ref), defaultValue);
+    }
+
+    createValue(vars: Vars, obj: any) {
+        if (!obj) {
+            return obj;
+        }
+        if (typeof obj === 'string') {
+            if (obj.indexOf('${') > -1) {
+                const segments = obj.split(/(\${[^}]+})/);
+                if (segments.length === 1) {
+                    const seg = segments[0];
+                    return this.createRefValue(seg.substring(2, seg.length - 1));
+                }
+                const parts: Value[] = [];
+                for (const seg of segments) {
+                    if (seg === '') continue;
+                    if (seg.startsWith('${') && seg.endsWith('}')) {
+                        parts.push(this.createRefValue(seg.substring(2, seg.length - 1)));
+                    } else {
+                        parts.push(new LiteralValue(seg));
+                    }
+                }
+                if (parts.length === 1) {
+                    return parts[0];
+                } else {
+                    return new ExprValue(vars, parts);
+                }
+            } else {
+                return obj;
+            }
+        } else {
+            return obj;
+        }
+    }
+
+    createImportVars(importSpec: ImportSpec | undefined) {
+        if (!importSpec || importSpec.length === 0) {
+            return {};
+        }
+        const result: Record<string, any> = {};
+
+        for (const importVar of importSpec) {
+            if (typeof importVar === "string") {
+                addImportVar(importVar, undefined, this, result);
+            } else {
+                for (const key of Object.keys(importVar)) {
+                    addImportVar(importVar[key], key, this, result);
+                }
+            }
+        }
+        return result;
+    }
+
+    getUnknownReferences(obj: any) {
+        const visitor = new UnknownRefrencesVisitor(this);
+        new ObjectWalker().walk(obj, visitor);
+        return visitor.result;
+    }
+}
+
+function addImportVar(varPath: string, asName: string | undefined, vars: Vars, result: Record<string, any>) {
+    let isRequired = false;
+    if (varPath.endsWith("!")) {
+        isRequired = true;
+        varPath = varPath.slice(0, -1);
+    }
+    let value = vars.getValue(varPath);
+    if (value === undefined && isRequired) {
+        throw new Error(`Import variable ${varPath} is required but not found`);
+    }
+    result[asName || varPath] = value;
+}
+
+
+class UnknownRefrencesVisitor implements ObjectVisitor {
+
+    result: { name: string, expression: string }[] = [];
+
+    constructor(public vars: Vars) {
+    }
+
+    onValue(_key: ObjectKey, value: any) {
+        const vars = this.vars;
+        if (typeof value === "string") {
+            const v = vars.createValue(vars, value);
+            if (v instanceof ExprValue) {
+                for (const p of v.parts) {
+                    if (p instanceof RefValue) {
+                        if (!vars.has(p.path[0])) {
+                            this.result.push({ name: p.path.join('.'), expression: p.stringify() });
+                        }
+                    }
+                }
+            } else if (v instanceof RefValue) {
+                if (!vars.has(v.path[0])) {
+                    this.result.push({ name: v.path.join('.'), expression: v.stringify() });
+                }
+            }
+        }
+    }
+}

package/src/dsl/walk.test.ts

@@ -0,0 +1,81 @@
+import { describe, expect, test } from "vitest";
+import { ObjectWalker } from "./walk.ts";
+
+describe('walk object', () => {
+
+    test('find string values', () => {
+        const obj = {
+            name: "foo",
+            age: 42,
+            children: [{
+                name: "bar",
+                age: 12,
+            },
+            {
+                name: "baz",
+                age: 15,
+            }],
+            folder: {
+                subfolder: {
+                    name: "file",
+                }
+            }
+        }
+        const values = ["foo", "bar", "baz", "file"].sort().join(',');
+        const found: string[] = [];
+        new ObjectWalker().walk(obj, {
+            onValue(key, value) {
+                if (typeof value === "string") {
+                    found.push(value);
+                }
+            }
+        })
+        expect(found.sort().join(',')).toBe(values);
+    })
+
+    test('map numbers to string values', () => {
+        const obj = {
+            name: "foo",
+            age: 42,
+            children: [{
+                name: "bar",
+                age: 12,
+            },
+            {
+                name: "baz",
+                age: 15,
+            }],
+            folder: {
+                subfolder: {
+                    name: 123,
+                }
+            }
+        }
+        const r = new ObjectWalker().map(obj, (_key, value) => {
+            if (typeof value === "number") {
+                return String(value)
+            }
+            return value;
+        });
+        expect(r.age).toBe("42");
+        expect(r.children[0].age).toBe("12");
+        expect(r.children[1].age).toBe("15");
+        expect(r.folder.subfolder.name).toBe("123");
+    })
+
+    test('map numbers in an array to string values', () => {
+        const obj = [123, { x: 1 }, { y: 2 }, { z: 3 }]
+        const r = new ObjectWalker().map(obj, (_key, value) => {
+            if (typeof value === "number") {
+                return String(value)
+            }
+            return value;
+        });
+        expect(r.length).toBe(4);
+        expect(r[0]).toBe("123");
+        expect(r[1].x).toBe("1");
+        expect(r[2].y).toBe("2");
+        expect(r[3].z).toBe("3");
+    })
+
+})

package/src/dsl/walk.ts

@@ -0,0 +1,103 @@
+
+export type ObjectKey = string | number | undefined;
+export interface ObjectVisitor {
+    onStartObject?: (key: ObjectKey, value: any) => void;
+    onEndObject?: (key: ObjectKey, value: any) => void;
+    onStartIteration?: (key: ObjectKey, value: Iterable<any>) => void;
+    onEndIteration?: (key: ObjectKey, value: Iterable<any>) => void;
+    onValue?: (key: ObjectKey, value: any) => void;
+}
+
+export class ObjectWalker {
+    supportIterators = false; // only array are supported by default
+    constructor(supportIterators = false) {
+        this.supportIterators = supportIterators;
+    }
+    walk(obj: any, visitor: ObjectVisitor) {
+        this._walk(undefined, obj, visitor);
+    }
+    _walk(key: ObjectKey, obj: any, visitor: ObjectVisitor) {
+        const type = typeof obj;
+        if (!obj || type !== 'object' || obj instanceof Date) {
+            visitor.onValue && visitor.onValue(key, obj);
+        } else if (Array.isArray(obj)) {
+            this._walkIterable(key, obj, visitor);
+        } else if (this.supportIterators && obj[Symbol.iterator] === 'function') {
+            this._walkIterable(key, obj, visitor);
+        } else if (obj.constructor === Object) { // a plain object
+            this._walkObject(key, obj, visitor);
+        } else { // a random object - we treat it as a value
+            visitor.onValue && visitor.onValue(key, obj);
+        }
+    }
+
+    _walkIterable(key: ObjectKey, obj: any, visitor: ObjectVisitor) {
+        visitor.onStartIteration && visitor.onStartIteration(key, obj);
+        let i = 0;
+        for (const value of obj) {
+            this._walk(i++, value, visitor);
+        }
+        visitor.onEndIteration && visitor.onEndIteration(key, obj);
+    }
+
+    _walkObject(key: ObjectKey, obj: any, visitor: ObjectVisitor) {
+        visitor.onStartObject && visitor.onStartObject(key, obj);
+        for (const k of Object.keys(obj)) {
+            this._walk(k, obj[k], visitor);
+        }
+        visitor.onEndObject && visitor.onEndObject(key, obj);
+    }
+
+    map(obj: any, mapFn: (key: ObjectKey, value: any) => any) {
+        const visitor = new MapVisitor(mapFn);
+        this.walk(obj, visitor);
+        return visitor.result;
+    }
+}
+
+class MapVisitor implements ObjectVisitor {
+    result: any;
+    current: any;
+    stack: any[] = [];
+    constructor(private mapFn: (key: ObjectKey, value: any) => any) { }
+
+    onStartObject(key: ObjectKey) {
+        if (key === undefined) {
+            this.result = {};
+            this.current = this.result;
+        } else {
+            this.stack.push(this.current);
+            const obj = {};
+            this.current[key] = obj;
+            this.current = obj;
+        }
+    }
+    onEndObject() {
+        this.current = this.stack.pop();
+    }
+
+    onStartIteration(key: ObjectKey) {
+        if (key === undefined) {
+            this.result = [];
+            this.current = this.result;
+        } else {
+            this.stack.push(this.current);
+            const ar: any[] = [];
+            this.current[key] = ar;
+            this.current = ar;
+        }
+    }
+
+    onEndIteration() {
+        this.current = this.stack.pop();
+    }
+
+    onValue(key: ObjectKey, value: any) {
+        const r = this.mapFn(key, value);
+        if (key === undefined) {
+            this.result = r;
+        } else if (r !== undefined) {
+            this.current[key] = r;
+        }
+    }
+}

package/src/dsl/workflow-exec-child.test.ts

@@ -0,0 +1,182 @@
+import { ContentEventName, DSLActivityExecutionPayload, DSLWorkflowExecutionPayload, DSLWorkflowStep } from '@vertesia/common';
+import { TestWorkflowEnvironment } from '@temporalio/testing';
+import { Worker } from '@temporalio/worker';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+import { dslWorkflow } from './dsl-workflow.js';
+import { setupActivity } from './setup/ActivityContext.js';
+
+async function sayHelloFromParent(payload: DSLActivityExecutionPayload) {
+    const { params } = await setupActivity(payload);
+    return `Parent: Hello, ${params.name}!`;
+}
+
+async function sayHelloFromDSLChild(payload: DSLActivityExecutionPayload) {
+    const { params } = await setupActivity(payload);
+    return `DSL Child: Hello, ${params.name}!`;
+}
+
+async function prepareResult(payload: DSLActivityExecutionPayload) {
+    const { params } = await setupActivity(payload);
+    return [params.parent, params.child]
+}
+
+const steps1: DSLWorkflowStep[] = [
+    {
+        type: 'activity',
+        name: 'sayHelloFromParent',
+        output: 'parent',
+        import: ["name"],
+    },
+    {
+        type: 'workflow',
+        name: 'testChildWorkflow',
+        output: 'child',
+    },
+    {
+        type: 'activity',
+        name: 'prepareResult',
+        import: ["parent", "child"],
+        output: 'result',
+    }
+]
+
+const childSteps: DSLWorkflowStep[] = [
+    {
+        type: 'activity',
+        name: 'sayHelloFromDSLChild',
+        output: 'result',
+        import: ["name"],
+    },
+]
+
+const steps2: DSLWorkflowStep[] = [
+    {
+        type: 'activity',
+        name: 'sayHelloFromParent',
+        output: 'parent',
+        import: ["name"],
+    },
+    {
+        type: 'workflow',
+        name: 'dslWorkflow',
+        output: 'child',
+        spec: {
+            name: 'testChildWorkflow',
+            steps: childSteps,
+            vars: {}
+        }
+    },
+    {
+        type: 'activity',
+        name: 'prepareResult',
+        import: ["parent", "child"],
+        output: 'result',
+    }
+]
+
+
+// ========== test env setup ==========
+
+
+describe('DSL Workflow with chld workflows', () => {
+
+    let testEnv: TestWorkflowEnvironment;
+
+    beforeAll(async () => {
+        testEnv = await TestWorkflowEnvironment.createLocal();
+    });
+
+    afterAll(async () => {
+        await testEnv?.teardown();
+    });
+
+    test('execute child workflow', async () => {
+        const { client, nativeConnection } = testEnv;
+        const taskQueue = 'test';
+
+        const name = 'Foo';
+
+        const worker = await Worker.create({
+            connection: nativeConnection,
+            taskQueue,
+            workflowsPath: new URL("./test/test-child-workflow.ts", import.meta.url).pathname,
+            activities: { sayHelloFromParent, prepareResult },
+        });
+
+        const payload: DSLWorkflowExecutionPayload = {
+            event: ContentEventName.create,
+            objectIds: ['123'],
+            vars: {},
+            account_id: '123',
+            project_id: '123',
+            timestamp: Date.now(),
+            wf_rule_name: 'test',
+            auth_token: 'test',
+            config: {
+                studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
+                store_url: process.env.CP_STODRE_URL || "http://localhost:8082",
+            },
+            workflow: {
+                steps: steps1,
+                vars: {
+                    name,
+                },
+                name: 'test',
+            }
+        }
+
+        let result = await worker.runUntil(client.workflow.execute(dslWorkflow, {
+            args: [payload],
+            workflowId: 'test',
+            taskQueue,
+        }));
+
+        expect(result).toEqual([`Parent: Hello, ${name}!`, `Child: Hello, ${name}!`]);
+
+    });
+
+    test('execute DSL child workflow', async () => {
+        const { client, nativeConnection } = testEnv;
+        const taskQueue = 'test';
+
+        const name = 'Bar';
+
+        const worker = await Worker.create({
+            connection: nativeConnection,
+            taskQueue,
+            workflowsPath: new URL("./test/test-child-workflow.ts", import.meta.url).pathname,
+            activities: { sayHelloFromParent, prepareResult, sayHelloFromDSLChild },
+        });
+
+        const payload: DSLWorkflowExecutionPayload = {
+            event: ContentEventName.create,
+            objectIds: ['123'],
+            vars: {},
+            account_id: '123',
+            project_id: '123',
+            timestamp: Date.now(),
+            wf_rule_name: 'test',
+            auth_token: 'test',
+            config: {
+                studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
+                store_url: process.env.CP_STODRE_URL || "http://localhost:8082",
+            },
+            workflow: {
+                steps: steps2,
+                vars: {
+                    name,
+                },
+                name: 'test',
+            }
+        }
+
+        let result = await worker.runUntil(client.workflow.execute(dslWorkflow, {
+            args: [payload],
+            workflowId: 'test',
+            taskQueue,
+        }));
+
+        expect(result).toEqual([`Parent: Hello, ${name}!`, `DSL Child: Hello, ${name}!`]);
+
+    });
+});